diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..14f295a
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,31 @@
+name: CI
+
+on:
+ pull_request:
+ push:
+ branches: [main]
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ - name: Format
+ run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
+ - name: Vet
+ run: go vet ./...
+ - name: Test (recorded fixtures, no network)
+ run: go test -race ./...
+ - name: Fuzz the scope normaliser briefly
+ run: go test ./internal/model -run '^$' -fuzz '^FuzzNormalizeTarget$' -fuzztime 20s
+ - name: Known vulnerabilities
+ run: go run golang.org/x/vuln/cmd/govulncheck@latest ./...
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 0000000..4d9c56f
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,135 @@
+name: Publish bundle
+
+# Daily: build the next program bundle from the previous release and the
+# sources, sign it, verify it and publish it as a GitHub Release.
+#
+# Two jobs on purpose. "build" talks to the sources and never sees the
+# signing key. "publish" runs in the protected "publish" environment (only
+# the default branch can deploy to it), holds the signing key and nothing
+# else, and only signs, verifies and uploads what "build" produced.
+
+on:
+ schedule:
+ - cron: "41 3 * * *"
+ workflow_dispatch:
+ inputs:
+ allow_mass_change:
+ description: "Accept a source that lost more than half of its programs (only after checking the source)"
+ type: boolean
+ default: false
+
+permissions:
+ contents: read
+
+concurrency:
+ group: publish
+ cancel-in-progress: false
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ outputs:
+ ready: ${{ steps.keys.outputs.ready }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Key ceremony done?
+ id: keys
+ run: |
+ if [ -s keys/root-keyid.txt ] && [ -s keys/keyset.dsse.json ]; then
+ echo "ready=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "::warning::No keys yet (keys/root-keyid.txt, keys/keyset.dsse.json): nothing is published until the key ceremony in keys/README.md is done"
+ echo "ready=false" >> "$GITHUB_OUTPUT"
+ fi
+ - if: steps.keys.outputs.ready == 'true'
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ cache: false
+ - if: steps.keys.outputs.ready == 'true'
+ name: Test
+ run: go test ./...
+ - if: steps.keys.outputs.ready == 'true'
+ name: Build the CLI
+ run: go build -trimpath -ldflags "-X main.version=${GITHUB_SHA::12}" -o programfeed ./cmd/programfeed
+ - if: steps.keys.outputs.ready == 'true'
+ name: Restore the HTTP cache (conditional requests only; every body is re-validated)
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: httpcache
+ key: httpcache-${{ github.run_id }}
+ restore-keys: httpcache-
+ - if: steps.keys.outputs.ready == 'true'
+ name: Download the previous release
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ mkdir -p prev
+ if gh release view --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+ gh release download --repo "$GITHUB_REPOSITORY" --dir prev --pattern 'snapshot*' --pattern 'latest.dsse.json' --pattern 'keyset.dsse.json'
+ echo "PREV=--prev=prev" >> "$GITHUB_ENV"
+ fi
+ - if: steps.keys.outputs.ready == 'true'
+ name: Build the bundle
+ env:
+ MASS: ${{ inputs.allow_mass_change && '--allow-mass-change' || '' }}
+ run: ./programfeed build --out out --cache httpcache --root-key-id "$(cat keys/root-keyid.txt)" $PREV $MASS
+ - if: steps.keys.outputs.ready == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: bundle
+ path: out/
+ retention-days: 3
+ if-no-files-found: error
+
+ publish:
+ needs: build
+ if: needs.build.outputs.ready == 'true'
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ environment: publish
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ cache: false
+ - name: Build the CLI
+ run: go build -trimpath -o programfeed ./cmd/programfeed
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: bundle
+ path: out
+ - name: Sign
+ env:
+ PROGRAMFEED_SIGNING_KEY: ${{ secrets.PROGRAMFEED_SIGNING_KEY }}
+ run: ./programfeed sign --dir out --keyset keys/keyset.dsse.json --root-key-id "$(cat keys/root-keyid.txt)"
+ - name: Verify as the platform does
+ run: ./programfeed verify --dir out --root-key-id "$(cat keys/root-keyid.txt)"
+ - name: Publish the release
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ tag="$(python3 -c 'import json,base64,sys; e=json.load(open("out/latest.dsse.json")); print(json.loads(base64.b64decode(e["payload"]))["tag"])')"
+ cat > notes.md <<'NOTES'
+ Public program bundle for OpenCTEM: a signed snapshot of public bug-bounty and vulnerability-disclosure programs, and the delta from the previous release.
+
+ Targets marked "inferred" are suggestions, not a program's published scope. See NOTICE for the terms of every source.
+ NOTES
+ gh release create "$tag" --repo "$GITHUB_REPOSITORY" --title "$tag" --notes-file notes.md out/*
+ - name: Keep the last 60 releases
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 --json tagName,createdAt \
+ --jq 'sort_by(.createdAt) | reverse | .[60:] | .[].tagName' |
+ while read -r tag; do
+ gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+ done
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..58de1a3
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+/programfeed
+/out/
+/prev/
+/httpcache/
+*.key
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..eea8991
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,83 @@
+# Contributing
+
+Thanks for helping. Two kinds of contributions are common: adding seed
+domains for the security.txt adapter, and adding a new source adapter.
+
+## Adding seed domains
+
+Add lines to `seeds/securitytxt.txt` (`domain [organisation name]`). Only add
+domains whose owner publishes a vulnerability disclosure policy for public
+use. The domain must already serve `/.well-known/security.txt` with a
+`Contact` and a `Policy` field, or the collector skips it.
+
+## Adding a source adapter
+
+### 1. Check that the source may be collected
+
+Before writing code, open an issue with:
+
+- the official API or machine-readable listing (documentation link);
+- whether it needs authentication (the feed never logs in, uses cookies or
+ tokens, runs JavaScript or bypasses bot protection: a source that needs
+ any of that is not accepted);
+- its terms of service and robots.txt, quoted: they must allow automated
+ collection **and redistribution**, since the feed is published openly
+ (or the publisher must have given written permission, linked in the
+ issue);
+- the licence of the data and the attribution it requires;
+- its rate limits.
+
+### 2. Write the adapter
+
+An adapter is one file in `internal/source/` implementing `source.Source`:
+
+```go
+type Source interface {
+ Meta() Meta // name, URL, terms, licence, attribution
+ Collect(ctx context.Context, c *fetch.Client) (*Result, error)
+}
+```
+
+Rules:
+
+- Use only the `*fetch.Client` it is given (it handles the User-Agent,
+ robots.txt, rate limits, conditional requests, size caps and the
+ public-address check). Pass a size cap that fits the source.
+- Program ids are `:`; use `source.UniqueSlugs` for names.
+ Ids must be stable across runs. The collector refuses programs outside the
+ adapter's own prefix.
+- Fill `Provenance` (source, source URL, fetched-at time). Leave
+ `first_seen`, `last_seen`, `last_changed` and `closed_at` to the collector.
+- Put the program's own scope in `InScope`/`OutOfScope` with
+ `Confidence: published` and set `ScopePublished`. Anything the adapter
+ derives itself is `Confidence: inferred`, with a note saying how.
+- Do not filter scope yourself: the collector normalises every target and
+ records refused ones with the reason.
+- Return an error only when the source as a whole could not be read (the
+ collector then keeps every previous record). Put ids whose state is
+ unknown this run (a host that did not answer) in `Result.Unknown`.
+- Hostile input is expected: decode defensively, cap counts, never trust a
+ field's type.
+
+Register it in `cmd/programfeed/main.go` (`sources`), and add its row to the
+sources table in `README.md` and its terms to `NOTICE`.
+
+### 3. Test it with recorded fixtures
+
+Tests never touch the network. Record a small, representative response
+under `internal/source/testdata/` (trimmed, no personal data beyond what the
+source publishes), serve it with `httptest`, and point the fetch client at
+it (`fetch.Options{AllowPrivate: true, DialOverride: ...}`, see
+`source_test.go`). Include poisoned entries: malformed records, wrong field
+types, control characters, public-suffix or private-range scope.
+
+### 4. Check before opening the pull request
+
+```sh
+gofmt -l .
+go vet ./...
+go test -race ./...
+go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+```
+
+Pull requests target `main`.
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..4ee04be
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,25 @@
+programfeed
+Copyright 2026 The OpenCTEM Authors
+
+This software is licensed under the Apache License, Version 2.0 (LICENSE).
+
+The bundles this software publishes contain data from third-party sources,
+under their own terms:
+
+disclose.io diodb (https://github.com/disclose/diodb)
+ Program directory data, dedicated to the public domain under CC0 1.0
+ Universal: https://github.com/disclose/diodb/blob/master/LICENSE
+
+CISA dotgov-data (https://github.com/cisagov/dotgov-data)
+ The list of US federal .gov domains used as the security.txt seed list,
+ dedicated to the public domain under CC0 1.0 Universal.
+
+security.txt files (RFC 9116)
+ Each program record from the securitytxt adapter is derived from the
+ security.txt file its domain publishes for automated retrieval; the
+ record's provenance names the file. The policies those files link to
+ belong to their publishers and are not copied into the bundles.
+
+Program names, policies and terms remain the property of the organisations
+that publish them. Listing a program in the feed does not imply that the
+organisation endorses OpenCTEM, nor that any testing is authorised.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9f11206
--- /dev/null
+++ b/README.md
@@ -0,0 +1,148 @@
+# programfeed
+
+The public program collector of [OpenCTEM](https://github.com/openctemio/openctem).
+Once a day it reads open, machine-readable sources of **public bug-bounty and
+vulnerability-disclosure programs**, normalises them into one record format,
+validates every record and every scope entry, signs the result and publishes
+it as a GitHub Release. OpenCTEM platforms never call the sources
+themselves: they download (or, air-gapped, upload) a bundle, verify it and
+import it into their public program monitor.
+
+The feed says **who runs a program, where its policy is, whether it pays,
+what scope it publishes and how that changed**. It is not an authorisation to
+test anything: a subscriber reads and accepts a program's terms before any
+active testing, and targets marked `inferred` are suggestions only.
+
+## Sources
+
+| Source | Adapter | What it gives | Licence / terms | Status |
+|---|---|---|---|---|
+| [disclose.io diodb](https://github.com/disclose/diodb) `program-list.json` | `diodb` | ~2,400 programs: name, policy URL, contact, bounty yes/no, safe harbour, disclosure timeline, languages. No structured scope; the host of a program's own security.txt becomes an `inferred` target. | CC0-1.0 | supported |
+| [security.txt](https://www.rfc-editor.org/rfc/rfc9116) (RFC 9116) over a seed list | `securitytxt` | A domain whose `/.well-known/security.txt` has a `Contact` and a `Policy` is a self-hosted disclosure program; the domain is an `inferred` target, the file's SHA-256 is the terms hash. Expired files give a `paused` program. | files published for automated retrieval | supported |
+| Seed: [CISA dotgov-data](https://github.com/cisagov/dotgov-data) `current-federal.csv` | `securitytxt` seed | US federal .gov domains and their organisations (vulnerability disclosure policies under BOD 20-01). | CC0-1.0 | supported |
+| Seed: `seeds/securitytxt.txt` | `securitytxt` seed | Extra domains added by pull request. | — | supported |
+
+Not collected, and why (pull requests that scrape these will be declined):
+
+| Source | Reason |
+|---|---|
+| HackerOne, Intigriti | program APIs require an account token; the feed never logs in |
+| Bugcrowd, YesWeHack | website terms forbid redistributing their data |
+| Immunefi | terms of use forbid automated access to the website without written consent |
+| Open Bug Bounty | listing is behind bot protection |
+| HackenProof, WhiteHub, BugBase and other regional platforms | no official machine-readable listing |
+| third-party dumps of platform scopes | the data comes from platforms whose terms forbid redistribution |
+
+A source moves to "supported" when it offers an official machine-readable
+listing whose terms allow collection and redistribution (or gives written
+permission).
+
+### How the collector behaves on the network
+
+- User-Agent `openctem-programfeed/ (+https://github.com/openctemio/programfeed; security@openctem.io)`
+ and a `From: security@openctem.io` header;
+- `robots.txt` honoured for every origin (product token `openctem-programfeed`;
+ an unreachable robots.txt means "do not fetch", RFC 9309);
+- at most one request per second per host; conditional requests
+ (`If-None-Match`, `If-Modified-Since`) from a cache kept between runs;
+- response size caps (32 KiB for a security.txt, 32 MiB for a list);
+- connections only to public unicast addresses (a seed list or redirect
+ cannot point it at internal services); https only, at most 3 redirects;
+- no login, no cookies, no JavaScript, no bot-protection bypass.
+
+## What a release contains
+
+| Asset | Content |
+|---|---|
+| `latest.dsse.json` | signed pointer: sequence, tag, snapshot and delta manifests, expiry |
+| `snapshot.manifest.dsse.json` | signed manifest of the full snapshot (files, SHA-256, sizes, record counts, sources with licence and as-of time, stats) |
+| `snapshot-programs.jsonl.gz` | every program |
+| `snapshot-changes.jsonl.gz` | the change log of this run |
+| `delta.manifest.dsse.json`, `delta-programs.jsonl.gz`, `delta-changes.jsonl.gz` | the programs that changed since the previous sequence (complete new records) and the same change log |
+| `keyset.dsse.json` | the key set the manifests are signed under |
+
+A platform reads
+`https://github.com/openctemio/programfeed/releases/latest/download/latest.dsse.json`
+and the files of the release it names.
+
+## Record format
+
+One JSON object per line. The JSON Schemas are in [`schema/`](schema/):
+[`program.schema.json`](schema/program.schema.json) and
+[`change.schema.json`](schema/change.schema.json).
+
+```json
+{"id":"securitytxt:agency.gov","source":"securitytxt","platform":"self-hosted","name":"Agency (agency.gov)",
+ "url":"https://agency.gov/vulnerability-disclosure-policy","type":"vdp","status":"open","offers_bounty":false,
+ "scope_published":false,
+ "in_scope":[{"type":"domain","value":"agency.gov","confidence":"inferred","notes":"Domain whose security.txt names this policy; not a published scope."}],
+ "out_of_scope":[],"rejected":[],
+ "rules":{"testing_restrictions":[],"required_headers":[],"safe_harbour":"unknown","languages":["en"]},
+ "terms":{"url":"https://agency.gov/.well-known/security.txt","sha256":"…","fetched_at":"2026-10-10T03:41:00Z"},
+ "contact":{"email":"security@agency.gov"},
+ "provenance":{"source":"securitytxt","source_url":"https://agency.gov/.well-known/security.txt","fetched_at":"2026-10-10T03:41:00Z"},
+ "first_seen":"2026-10-10T03:41:00Z","last_seen":"2026-10-10T03:41:00Z","last_changed":"2026-10-10T03:41:00Z"}
+{"sequence":2,"program":"securitytxt:agency.gov","kind":"target_added","scope":"in","target":{"type":"domain","value":"agency.gov","confidence":"inferred"}}
+```
+
+Key rules:
+
+- `id` is `:`, stable, owned by one adapter.
+- `type` is `bounty` or `vdp`; `status` is `open`, `paused` (listed but not
+ accepting: dead policy link, expired security.txt) or `closed` (the source
+ no longer lists it). A closed program stays 180 days, then a
+ `program_dropped` change removes it.
+- Scope entries are typed (`domain`, `wildcard`, `ip`, `cidr`, `url`,
+ `mobile_app`, `source_repo`, `other`), canonical, sorted and deduplicated;
+ out of scope wins over in scope.
+- `confidence`: `published` (the program's own scope) or `inferred` (derived
+ by the collector). `scope_published` says whether the source carries the
+ program's own structured scope at all.
+- `terms.sha256` is the SHA-256 of the exact terms document the collector
+ read, when it read one.
+- `rejected` lists what a program listed that the scope rules refused, with
+ the reason.
+
+### Scope rules
+
+A scope entry is refused (moved to `rejected`) when it is: a public suffix or
+a wildcard over one (`com`, `*.co.uk`, `*.github.io`); a reserved or
+non-public name (`localhost`, `.local`, `.internal`, `.test`, `example.com`
+…); a private, loopback, link-local, CGNAT, documentation, multicast or
+otherwise reserved address or range; a CIDR wider than /16 (IPv4) or /48
+(IPv6); a URL that is not http(s), has credentials, or whose host fails the
+rules above; malformed, over 512 bytes, or carrying control or bidi
+characters. Each list is capped at 2,000 entries.
+
+## Verifying a bundle yourself
+
+```sh
+go build -o programfeed ./cmd/programfeed
+gh release download --repo openctemio/programfeed --dir bundle
+./programfeed verify --dir bundle --root-key-id "$(cat keys/root-keyid.txt)"
+```
+
+The envelopes, key set, sequence, expiry and caps follow the same
+conventions as the OpenCTEM vulnerability feed; only the payload types
+(`application/vnd.openctem.programfeed.*`) and record files differ, so a
+key set or manifest of one feed is never accepted by the other.
+
+## Building locally
+
+```sh
+go test ./... # recorded fixtures, no network
+go build -o programfeed ./cmd/programfeed
+./programfeed build --out out --cache httpcache # every source
+./programfeed build --out out --sources diodb # one source
+```
+
+A key is needed only to sign; see `keys/README.md`.
+
+## Security
+
+See [SECURITY.md](SECURITY.md) for the trust model and how to report a
+problem. Adding a source: [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Licence
+
+Apache-2.0 (the code). The data keeps its sources' terms; see `NOTICE`.
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..60b1fd5
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,80 @@
+# Security
+
+## Reporting
+
+Please report vulnerabilities privately to **security@openctem.io** or through
+GitHub's private vulnerability reporting on this repository. Do not open a
+public issue.
+
+If you run a program listed in the feed and want it corrected or removed,
+write to the same address.
+
+## Threat model
+
+A bundle tells every OpenCTEM platform which public programs exist and what
+they cover. A forged, stale or poisoned bundle could point subscribers'
+monitoring at organisations that never invited it, hide a program's closure
+or its new out-of-scope entries, or carry hostile strings into the platform.
+The collector itself talks to many third-party hosts and must not become a
+way to reach internal services.
+
+### Integrity and freshness (same scheme as the OpenCTEM vulnerability feed)
+
+- **Offline root.** An Ed25519 root key, kept offline by the maintainer,
+ only signs *key sets*. Platforms pin its key id (`keys/root-keyid.txt`).
+ The root is not shared with any other feed, and the key set payload type
+ is specific to this feed, so a key set or manifest of another feed is
+ refused.
+- **Key set.** `keys/keyset.dsse.json` lists the online signing keys, is
+ valid for at most 180 days and has a version that only goes up. A
+ platform refuses a key set whose version is below one it has accepted.
+- **Online key.** The signing key is a secret of the `publish` GitHub
+ environment, which only the default branch can deploy to. Only the
+ publish job runs there; it signs, verifies and uploads what the build job
+ produced. The build job, which talks to the sources, never sees the key.
+- **Rollback.** Every bundle has a sequence number. A platform refuses a
+ sequence it has already applied, and a delta whose base is not the
+ sequence it has.
+- **Freeze.** Every manifest and pointer expires 7 days after it was made.
+- **Integrity.** The signed manifest holds the SHA-256, size and record
+ count of every file; a mismatch refuses the whole bundle.
+- **Size.** Manifests are capped at 1 MiB, files at 128 MiB compressed and
+ 1 GiB decompressed, records at 4 MiB, and a bundle at 100,000 programs.
+
+### Poisoned data
+
+- Every record is validated before signing and again by the importer:
+ strict JSON (unknown fields refused), id grammar, http(s) URLs without
+ credentials, bounded strings without control or bidi characters, enums,
+ timestamps in order.
+- Every scope entry is normalised; public suffixes and wildcards over them,
+ reserved names, private/reserved/documentation addresses and ranges,
+ CIDRs wider than /16 (IPv4) or /48 (IPv6), and malformed entries are
+ moved to the program's `rejected` list with the reason. A bundle with a
+ scope entry the rules refuse is refused whole by the verifier.
+- Out of scope wins over in scope.
+- Targets the collector derived (`confidence: inferred`) are marked as such;
+ the feed never presents a guess as a program's published scope.
+- An adapter can only write programs under its own id prefix.
+- **Outage guard.** A source that suddenly reports fewer than half of the
+ programs it had (from at least 20) is treated as broken for that run and
+ its previous records are kept; a maintainer can override after checking
+ the source. A host that does not answer keeps its previous record instead
+ of being closed. A closed program stays visible for 180 days before it is
+ dropped.
+
+### The collector on the network
+
+- Connections only to public unicast addresses, checked at connect time
+ (after DNS), so neither a seed list entry nor a redirect can reach
+ loopback, private ranges or cloud metadata addresses.
+- https only, at most 3 redirects, each re-checked against robots.txt.
+- Size caps on every response; timeouts on every request.
+- No credentials, cookies or tokens are ever sent to a source.
+- The HTTP cache between runs only feeds conditional requests; every body
+ served from it goes through the same parsing and validation.
+
+## Key ceremony (maintainers)
+
+See `keys/README.md`. Never commit a private key; the repository holds only
+public material (`root-keyid.txt`, `keyset.dsse.json`).
diff --git a/cmd/programfeed/main.go b/cmd/programfeed/main.go
new file mode 100644
index 0000000..a635c46
--- /dev/null
+++ b/cmd/programfeed/main.go
@@ -0,0 +1,250 @@
+// Command programfeed builds, signs and verifies OpenCTEM program bundles
+// (public bug-bounty and vulnerability-disclosure programs), and manages
+// the signing keys.
+//
+// programfeed build --out DIR [--prev DIR] [--root-key-id ID] [--cache DIR] [--sources a,b] [--seed-file FILE] [--allow-mass-change]
+// programfeed sign --dir DIR --keyset FILE --root-key-id ID (key in $PROGRAMFEED_SIGNING_KEY)
+// programfeed verify --dir DIR --root-key-id ID [--applied N] [--allow-expired]
+// programfeed keys root-init --out FILE
+// programfeed keys gen --out FILE
+// programfeed keys sign-keyset --root FILE --key PUBLIC_KEY [--key ...] --version N [--days 180] --out FILE
+package main
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/base64"
+ "errors"
+ "flag"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/bundle"
+ "github.com/openctemio/programfeed/internal/collect"
+ "github.com/openctemio/programfeed/internal/dsse"
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/source"
+)
+
+// version is set at build time (-ldflags "-X main.version=...").
+var version = "dev"
+
+func main() {
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+ defer stop()
+ if err := run(ctx, os.Args[1:]); err != nil {
+ fmt.Fprintln(os.Stderr, "programfeed:", err)
+ os.Exit(1)
+ }
+}
+
+func run(ctx context.Context, args []string) error {
+ if len(args) == 0 {
+ return errors.New("usage: programfeed build|sign|verify|keys ...")
+ }
+ switch args[0] {
+ case "build":
+ return cmdBuild(ctx, args[1:])
+ case "sign":
+ return cmdSign(args[1:])
+ case "verify":
+ return cmdVerify(args[1:])
+ case "keys":
+ return cmdKeys(args[1:])
+ }
+ return fmt.Errorf("unknown command %q", args[0])
+}
+
+func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }
+
+// sources returns the configured adapters, in order.
+func sources(names, seedFile string) ([]source.Source, error) {
+ var out []source.Source
+ for _, n := range strings.Split(names, ",") {
+ switch strings.TrimSpace(n) {
+ case "diodb":
+ out = append(out, source.DioDB{})
+ case "securitytxt":
+ st := source.SecurityTxt{Progress: func(done, total int) { logf("securitytxt: %d/%d domains", done, total) }}
+ if seedFile != "" {
+ b, err := os.ReadFile(seedFile)
+ if err != nil {
+ return nil, err
+ }
+ st.Extra = source.ParseSeedFile(b)
+ }
+ out = append(out, st)
+ case "":
+ default:
+ return nil, fmt.Errorf("unknown source %q", n)
+ }
+ }
+ if len(out) == 0 {
+ return nil, errors.New("no source selected")
+ }
+ return out, nil
+}
+
+func cmdBuild(ctx context.Context, args []string) error {
+ fs := flag.NewFlagSet("build", flag.ContinueOnError)
+ out := fs.String("out", "", "output directory")
+ prevDir := fs.String("prev", "", "previous release (verified before use)")
+ root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id")
+ mass := fs.Bool("allow-mass-change", false, "accept a source that suddenly lost more than half of its programs")
+ cache := fs.String("cache", "", "HTTP cache directory for conditional requests")
+ names := fs.String("sources", "diodb,securitytxt", "comma-separated adapters")
+ seedFile := fs.String("seed-file", "seeds/securitytxt.txt", "extra security.txt seed domains (empty to skip)")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ if *out == "" {
+ return errors.New("build: --out is required")
+ }
+ srcs, err := sources(*names, *seedFile)
+ if err != nil {
+ return err
+ }
+ now := time.Now().UTC()
+ var prev *bundle.Snapshot
+ if *prevDir != "" {
+ v, err := bundle.VerifyDir(*prevDir, bundle.VerifyOptions{PinnedRoot: *root, Now: now, AllowExpired: true, SkipRecordChk: true})
+ if err != nil {
+ return fmt.Errorf("previous release: %w", err)
+ }
+ if prev, err = bundle.ReadRecords(*prevDir, v.Snapshot); err != nil {
+ return fmt.Errorf("previous release: %w", err)
+ }
+ logf("previous release: sequence %d, %d programs", prev.Sequence, len(prev.Programs))
+ }
+ client := fetch.New(fetch.Options{UserAgent: fetch.UserAgent(version), CacheDir: *cache, Timeout: 20 * time.Second})
+ next, err := collect.Build(ctx, prev, srcs, client, collect.Options{Now: now, AllowMassChange: *mass, Log: logf})
+ if err != nil {
+ return err
+ }
+ snap, delta, _, err := bundle.Write(*out, prev, next, bundle.WriteOptions{Now: now,
+ Collector: bundle.Collector{Version: version, Commit: os.Getenv("GITHUB_SHA")}})
+ if err != nil {
+ return err
+ }
+ st := snap.Stats
+ logf("built sequence %d: %d programs (%d open, %d bounty), %d targets, %d rejected targets, %d changes",
+ snap.Sequence, st.Programs, st.Open, st.Bounty, st.Targets, st.Rejected, st.Changes)
+ if delta != nil {
+ logf("delta from %d: %d programs (+%d added, %d closed, %d dropped)", delta.BaseSequence, delta.Stats.Programs,
+ delta.Stats.Added, delta.Stats.Closed, delta.Stats.Dropped)
+ }
+ return nil
+}
+
+func cmdSign(args []string) error {
+ fs := flag.NewFlagSet("sign", flag.ContinueOnError)
+ dir := fs.String("dir", "", "bundle directory")
+ keyset := fs.String("keyset", "keys/keyset.dsse.json", "key set envelope")
+ root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(os.Getenv("PROGRAMFEED_SIGNING_KEY")))
+ if err != nil || len(seed) != ed25519.SeedSize {
+ return errors.New("sign: PROGRAMFEED_SIGNING_KEY must be a base64 32-byte Ed25519 seed")
+ }
+ ks, err := os.ReadFile(*keyset)
+ if err != nil {
+ return err
+ }
+ return bundle.Sign(*dir, ed25519.NewKeyFromSeed(seed), ks, *root, time.Now())
+}
+
+func cmdVerify(args []string) error {
+ fs := flag.NewFlagSet("verify", flag.ContinueOnError)
+ dir := fs.String("dir", "", "bundle directory")
+ root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id")
+ applied := fs.Uint64("applied", 0, "last applied sequence")
+ expired := fs.Bool("allow-expired", false, "accept an expired bundle")
+ if err := fs.Parse(args); err != nil {
+ return err
+ }
+ v, err := bundle.VerifyDir(*dir, bundle.VerifyOptions{PinnedRoot: *root, AppliedSeq: *applied, Now: time.Now(), AllowExpired: *expired})
+ if err != nil {
+ return err
+ }
+ logf("verified sequence %d (snapshot %d programs, %d targets; key set v%d)", v.Latest.Sequence,
+ v.Snapshot.Stats.Programs, v.Snapshot.Stats.Targets, v.KeySet.Version)
+ return nil
+}
+
+type multi []string
+
+func (m *multi) String() string { return strings.Join(*m, ",") }
+func (m *multi) Set(v string) error { *m = append(*m, v); return nil }
+
+func cmdKeys(args []string) error {
+ if len(args) == 0 {
+ return errors.New("keys: root-init|gen|sign-keyset")
+ }
+ switch args[0] {
+ case "root-init", "gen":
+ fs := flag.NewFlagSet(args[0], flag.ContinueOnError)
+ out := fs.String("out", "", "file to write the private seed to (mode 0600; keep a root offline)")
+ if err := fs.Parse(args[1:]); err != nil {
+ return err
+ }
+ if *out == "" {
+ return errors.New("keys: --out is required")
+ }
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+ f, err := os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return err
+ }
+ if _, err := f.WriteString(base64.StdEncoding.EncodeToString(priv.Seed()) + "\n"); err != nil {
+ f.Close()
+ return err
+ }
+ if err := f.Close(); err != nil {
+ return err
+ }
+ fmt.Printf("key id: %s\npublic key: %s\nseed file: %s\n", dsse.KeyID(pub), base64.StdEncoding.EncodeToString(pub), *out)
+ return nil
+ case "sign-keyset":
+ fs := flag.NewFlagSet("sign-keyset", flag.ContinueOnError)
+ rootFile := fs.String("root", "", "root seed file")
+ ver := fs.Uint64("version", 0, "key set version (higher than the current one)")
+ days := fs.Int("days", 180, "validity in days (at most 180)")
+ out := fs.String("out", "keys/keyset.dsse.json", "output")
+ var keys multi
+ fs.Var(&keys, "key", "base64 public key of an online signing key (repeatable)")
+ if err := fs.Parse(args[1:]); err != nil {
+ return err
+ }
+ raw, err := os.ReadFile(*rootFile)
+ if err != nil {
+ return err
+ }
+ seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw)))
+ if err != nil || len(seed) != ed25519.SeedSize {
+ return errors.New("keys: the root file is not a base64 32-byte seed")
+ }
+ pubs := make([]ed25519.PublicKey, 0, len(keys))
+ for _, k := range keys {
+ b, err := base64.StdEncoding.DecodeString(k)
+ if err != nil || len(b) != ed25519.PublicKeySize {
+ return fmt.Errorf("keys: %q is not a base64 Ed25519 public key", k)
+ }
+ pubs = append(pubs, b)
+ }
+ env, err := dsse.SignKeySet(ed25519.NewKeyFromSeed(seed), *ver, time.Now(), time.Duration(*days)*24*time.Hour, pubs)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(*out, env, 0o644)
+ }
+ return fmt.Errorf("keys: unknown command %q", args[0])
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..fa474c2
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,9 @@
+module github.com/openctemio/programfeed
+
+go 1.26.0
+
+toolchain go1.26.9
+
+require golang.org/x/net v0.60.0
+
+require golang.org/x/text v0.42.0 // indirect
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..1ed037a
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,4 @@
+golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
+golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go
new file mode 100644
index 0000000..cbe237a
--- /dev/null
+++ b/internal/bundle/bundle.go
@@ -0,0 +1,407 @@
+// Package bundle builds, signs, reads and verifies program bundles: JSON
+// Lines files per record type, a signed manifest per snapshot and delta,
+// and a signed latest pointer. The layout, envelopes, key set and checks
+// are the ones of the OpenCTEM vulnerability feed, so one verifier serves
+// both (only the payload types and record files differ).
+package bundle
+
+import (
+ "bufio"
+ "compress/gzip"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "sort"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/dsse"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// Schemas, payload types and file names.
+const (
+ ManifestSchema = "openctem.programfeed/v1"
+ LatestSchema = "openctem.programfeed.latest/v1"
+ ManifestPayloadType = "application/vnd.openctem.programfeed.manifest+json"
+ LatestPayloadType = "application/vnd.openctem.programfeed.latest+json"
+
+ KindSnapshot = "snapshot"
+ KindDelta = "delta"
+
+ LatestFile = "latest.dsse.json"
+ KeySetFile = "keyset.dsse.json"
+ ManifestFile = "%s.manifest.dsse.json" // snapshot | delta
+
+ Validity = 7 * 24 * time.Hour
+)
+
+// Caps (the importer applies the same).
+const (
+ MaxManifestBytes = 1 << 20
+ MaxFileBytes = 128 << 20
+ MaxDecompressedBytes = 1 << 30
+ MaxRecordBytes = 4 << 20
+)
+
+// Record files of a bundle, in import order. A snapshot holds every
+// program and this run's change log; a delta holds the programs that
+// changed (complete new records) and the same change log.
+var recordFiles = []string{"programs", "changes"}
+
+// FileName is the name of a record file of a kind.
+func FileName(kind, records string) string { return kind + "-" + records + ".jsonl.gz" }
+
+// Source names one upstream source, its terms and how much it gave.
+type Source struct {
+ Name string `json:"name"`
+ URL string `json:"url"`
+ AsOf time.Time `json:"as_of"` // last successful read
+ Terms string `json:"terms"`
+ Licence string `json:"licence"`
+ Attribution string `json:"attribution"`
+ Programs int `json:"programs"`
+}
+
+// File is one record file of a manifest.
+type File struct {
+ Name string `json:"name"`
+ SHA256 string `json:"sha256"`
+ Size int64 `json:"size"`
+ Records int `json:"records"`
+}
+
+// Stats summarises a bundle.
+type Stats struct {
+ Programs int `json:"programs"`
+ Open int `json:"open"`
+ Bounty int `json:"bounty"`
+ Targets int `json:"targets"`
+ Rejected int `json:"rejected"`
+ Changes int `json:"changes"`
+ Added int `json:"added"`
+ Closed int `json:"closed"`
+ Dropped int `json:"dropped"`
+}
+
+// Collector identifies the build.
+type Collector struct {
+ Version string `json:"version"`
+ Commit string `json:"commit"`
+}
+
+// Manifest is the signed description of a snapshot or a delta.
+type Manifest struct {
+ Schema string `json:"schema"`
+ Sequence uint64 `json:"sequence"`
+ Kind string `json:"kind"`
+ BaseSequence uint64 `json:"base_sequence,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
+ ExpiresAt time.Time `json:"expires_at"`
+ Sources []Source `json:"sources"`
+ Files []File `json:"files"`
+ Stats Stats `json:"stats"`
+ Collector Collector `json:"collector"`
+}
+
+// Latest is the signed pointer to the newest bundle.
+type Latest struct {
+ Schema string `json:"schema"`
+ Sequence uint64 `json:"sequence"`
+ Tag string `json:"tag"`
+ Snapshot string `json:"snapshot"`
+ Delta string `json:"delta,omitempty"`
+ BaseSequence uint64 `json:"base_sequence,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
+ ExpiresAt time.Time `json:"expires_at"`
+}
+
+// Tag is the release tag of a sequence.
+func Tag(sequence uint64) string { return fmt.Sprintf("v1-%d", sequence) }
+
+// Snapshot is the complete program set plus the change log of the run
+// that produced it.
+type Snapshot struct {
+ Sequence uint64
+ Sources []Source
+ Programs map[string]model.Program
+ Changes []model.Change
+}
+
+// NewSnapshot returns an empty snapshot.
+func NewSnapshot() *Snapshot { return &Snapshot{Programs: map[string]model.Program{}} }
+
+// Source returns a source by name.
+func (s *Snapshot) Source(name string) (Source, bool) {
+ for _, src := range s.Sources {
+ if src.Name == name {
+ return src, true
+ }
+ }
+ return Source{}, false
+}
+
+// SortedIDs returns the program ids in order.
+func (s *Snapshot) SortedIDs() []string {
+ ids := make([]string, 0, len(s.Programs))
+ for id := range s.Programs {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ return ids
+}
+
+// ChangedIDs returns the ids of programs present in the snapshot that the
+// change log touches, in order.
+func (s *Snapshot) ChangedIDs() []string {
+ seen := map[string]bool{}
+ var ids []string
+ for _, c := range s.Changes {
+ if _, ok := s.Programs[c.Program]; ok && !seen[c.Program] {
+ seen[c.Program] = true
+ ids = append(ids, c.Program)
+ }
+ }
+ sort.Strings(ids)
+ return ids
+}
+
+// Validate checks every record of the snapshot.
+func (s *Snapshot) Validate() error {
+ if len(s.Programs) > model.MaxPrograms {
+ return fmt.Errorf("%d programs, over the cap of %d", len(s.Programs), model.MaxPrograms)
+ }
+ for _, id := range s.SortedIDs() {
+ p := s.Programs[id]
+ if p.ID != id {
+ return fmt.Errorf("program %s filed under %s", p.ID, id)
+ }
+ if err := p.Validate(); err != nil {
+ return err
+ }
+ }
+ for i := range s.Changes {
+ c := &s.Changes[i]
+ if err := c.Validate(); err != nil {
+ return err
+ }
+ if c.Sequence != s.Sequence {
+ return fmt.Errorf("change of %s has sequence %d, the snapshot is %d", c.Program, c.Sequence, s.Sequence)
+ }
+ if _, ok := s.Programs[c.Program]; ok == (c.Kind == model.ChangeProgramDropped) {
+ return fmt.Errorf("change %s of %s: a dropped program must be absent, any other present", c.Kind, c.Program)
+ }
+ }
+ return nil
+}
+
+func (s *Snapshot) stats(ids []string) Stats {
+ st := Stats{Programs: len(ids), Changes: len(s.Changes)}
+ for _, id := range ids {
+ p := s.Programs[id]
+ if p.Status == model.StatusOpen {
+ st.Open++
+ }
+ if p.OffersBounty {
+ st.Bounty++
+ }
+ st.Targets += len(p.InScope) + len(p.OutOfScope)
+ st.Rejected += len(p.Rejected)
+ }
+ for _, c := range s.Changes {
+ switch c.Kind {
+ case model.ChangeProgramAdded:
+ st.Added++
+ case model.ChangeProgramClosed:
+ st.Closed++
+ case model.ChangeProgramDropped:
+ st.Dropped++
+ }
+ }
+ return st
+}
+
+// WriteOptions describe a build.
+type WriteOptions struct {
+ Now time.Time
+ Collector Collector
+}
+
+// Write writes the snapshot of next and, when prev is not nil, the delta
+// from prev, with unsigned manifests (payloads) and the latest pointer
+// payload, into dir.
+func Write(dir string, prev, next *Snapshot, opt WriteOptions) (snap, delta *Manifest, latest *Latest, err error) {
+ if err := next.Validate(); err != nil {
+ return nil, nil, nil, fmt.Errorf("refusing to publish an invalid bundle: %w", err)
+ }
+ if prev != nil && next.Sequence <= prev.Sequence {
+ return nil, nil, nil, fmt.Errorf("sequence %d is not after %d", next.Sequence, prev.Sequence)
+ }
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return nil, nil, nil, err
+ }
+ now := opt.Now.UTC().Truncate(time.Second)
+ all := next.SortedIDs()
+ snap = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindSnapshot, CreatedAt: now,
+ ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(all)}
+ if snap.Files, err = writeFiles(dir, KindSnapshot, next, all); err != nil {
+ return nil, nil, nil, err
+ }
+ latest = &Latest{Schema: LatestSchema, Sequence: next.Sequence, Tag: Tag(next.Sequence),
+ Snapshot: fmt.Sprintf(ManifestFile, KindSnapshot), CreatedAt: now, ExpiresAt: now.Add(Validity)}
+ if prev != nil {
+ ids := next.ChangedIDs()
+ delta = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindDelta, BaseSequence: prev.Sequence,
+ CreatedAt: now, ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(ids)}
+ if delta.Files, err = writeFiles(dir, KindDelta, next, ids); err != nil {
+ return nil, nil, nil, err
+ }
+ latest.Delta, latest.BaseSequence = fmt.Sprintf(ManifestFile, KindDelta), prev.Sequence
+ }
+ if err := writeJSON(filepath.Join(dir, "snapshot.manifest.json"), snap); err != nil {
+ return nil, nil, nil, err
+ }
+ if delta != nil {
+ if err := writeJSON(filepath.Join(dir, "delta.manifest.json"), delta); err != nil {
+ return nil, nil, nil, err
+ }
+ }
+ if err := writeJSON(filepath.Join(dir, "latest.json"), latest); err != nil {
+ return nil, nil, nil, err
+ }
+ return snap, delta, latest, nil
+}
+
+func writeFiles(dir, kind string, s *Snapshot, ids []string) ([]File, error) {
+ var files []File
+ for _, rf := range recordFiles {
+ name := FileName(kind, rf)
+ var n int
+ f, err := writeGz(filepath.Join(dir, name), func(enc *json.Encoder) error {
+ switch rf {
+ case "programs":
+ for _, id := range ids {
+ if err := enc.Encode(s.Programs[id]); err != nil {
+ return err
+ }
+ n++
+ }
+ case "changes":
+ for _, c := range s.Changes {
+ if err := enc.Encode(c); err != nil {
+ return err
+ }
+ n++
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ f.Name, f.Records = name, n
+ files = append(files, f)
+ }
+ return files, nil
+}
+
+func writeGz(path string, fill func(*json.Encoder) error) (File, error) {
+ out, err := os.Create(path)
+ if err != nil {
+ return File{}, err
+ }
+ h := sha256.New()
+ counter := &countWriter{}
+ gz, _ := gzip.NewWriterLevel(io.MultiWriter(out, h, counter), gzip.BestCompression)
+ gz.ModTime = time.Unix(0, 0)
+ bw := bufio.NewWriterSize(gz, 1<<20)
+ enc := json.NewEncoder(bw)
+ enc.SetEscapeHTML(false)
+ if err := fill(enc); err != nil {
+ out.Close()
+ return File{}, err
+ }
+ if err := bw.Flush(); err != nil {
+ out.Close()
+ return File{}, err
+ }
+ if err := gz.Close(); err != nil {
+ out.Close()
+ return File{}, err
+ }
+ if err := out.Close(); err != nil {
+ return File{}, err
+ }
+ if counter.n > MaxFileBytes {
+ return File{}, fmt.Errorf("%s is %d bytes, over the %d cap", filepath.Base(path), counter.n, MaxFileBytes)
+ }
+ return File{SHA256: hex.EncodeToString(h.Sum(nil)), Size: counter.n}, nil
+}
+
+type countWriter struct{ n int64 }
+
+func (c *countWriter) Write(p []byte) (int, error) {
+ c.n += int64(len(p))
+ return len(p), nil
+}
+
+func writeJSON(path string, v any) error {
+ b, err := json.MarshalIndent(v, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(path, b, 0o644)
+}
+
+// Sign wraps the unsigned manifests and latest pointer in dir in DSSE
+// envelopes signed by priv, and copies the key set next to them. The
+// signing key must be listed in the key set.
+func Sign(dir string, priv ed25519.PrivateKey, keyset []byte, pinnedRoot string, now time.Time) error {
+ ks, err := dsse.VerifyKeySet(keyset, pinnedRoot, 0, now)
+ if err != nil {
+ return err
+ }
+ pub, _ := priv.Public().(ed25519.PublicKey)
+ listed := false
+ for _, k := range ks.Keys {
+ if k.KeyID == dsse.KeyID(pub) {
+ listed = true
+ }
+ }
+ if !listed {
+ return fmt.Errorf("the signing key %s is not in the key set", dsse.KeyID(pub))
+ }
+ pairs := []struct{ in, out, typ string }{
+ {"snapshot.manifest.json", fmt.Sprintf(ManifestFile, KindSnapshot), ManifestPayloadType},
+ {"delta.manifest.json", fmt.Sprintf(ManifestFile, KindDelta), ManifestPayloadType},
+ {"latest.json", LatestFile, LatestPayloadType},
+ }
+ for _, p := range pairs {
+ payload, err := os.ReadFile(filepath.Join(dir, p.in))
+ if errors.Is(err, os.ErrNotExist) && p.in == "delta.manifest.json" {
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ env, err := dsse.Sign(priv, p.typ, payload)
+ if err != nil {
+ return err
+ }
+ if err := os.WriteFile(filepath.Join(dir, p.out), env, 0o644); err != nil {
+ return err
+ }
+ if err := os.Remove(filepath.Join(dir, p.in)); err != nil {
+ return err
+ }
+ }
+ return os.WriteFile(filepath.Join(dir, KeySetFile), keyset, 0o644)
+}
+
+// Stats summarises the whole snapshot.
+func (s *Snapshot) Stats() Stats { return s.stats(s.SortedIDs()) }
diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go
new file mode 100644
index 0000000..a2c4762
--- /dev/null
+++ b/internal/bundle/bundle_test.go
@@ -0,0 +1,274 @@
+package bundle
+
+import (
+ "compress/gzip"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/dsse"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+type keys struct {
+ rootID string
+ priv ed25519.PrivateKey
+ keyset []byte
+}
+
+func newKeys(t *testing.T) keys {
+ t.Helper()
+ rootPub, root, _ := ed25519.GenerateKey(nil)
+ pub, priv, _ := ed25519.GenerateKey(nil)
+ ks, err := dsse.SignKeySet(root, 1, time.Now().Add(-time.Hour), 90*24*time.Hour, []ed25519.PublicKey{pub})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return keys{rootID: dsse.KeyID(rootPub), priv: priv, keyset: ks}
+}
+
+var t0 = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+
+func program(id, target string) model.Program {
+ p := model.Program{
+ ID: id, Source: "disclose", Platform: "self-hosted", Name: id, URL: "https://openctem.io/security",
+ Type: model.TypeVDP, Status: model.StatusOpen,
+ InScope: []model.Target{{Type: model.TargetDomain, Value: target, Confidence: model.ConfidenceInferred}},
+ OutOfScope: []model.Target{},
+ Terms: model.Terms{URL: "https://openctem.io/security"},
+ Provenance: model.Provenance{Source: "disclose", SourceURL: "https://openctem.io/list.json", FetchedAt: t0},
+ FirstSeen: t0, LastSeen: t0, LastChanged: t0,
+ }
+ model.Normalize(&p)
+ return p
+}
+
+func snapshot(seq uint64, target string) *Snapshot {
+ s := NewSnapshot()
+ s.Sequence = seq
+ s.Sources = []Source{{Name: "disclose", URL: "https://openctem.io/list.json", AsOf: t0, Licence: "CC0-1.0"}}
+ a, b := program("disclose:a", "a.openctem.io"), program("disclose:b", target)
+ s.Programs[a.ID], s.Programs[b.ID] = a, b
+ tg := b.InScope[0]
+ s.Changes = []model.Change{{Sequence: seq, Program: b.ID, Kind: model.ChangeTargetAdded, Scope: "in", Target: &tg}}
+ return s
+}
+
+func build(t *testing.T, k keys, prev, next *Snapshot) string {
+ t.Helper()
+ dir := t.TempDir()
+ if _, _, _, err := Write(dir, prev, next, WriteOptions{Now: time.Now()}); err != nil {
+ t.Fatal(err)
+ }
+ if err := Sign(dir, k.priv, k.keyset, k.rootID, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+func TestRoundTrip(t *testing.T) {
+ k := newKeys(t)
+ prev, next := snapshot(1, "b.openctem.io"), snapshot(2, "c.openctem.io")
+ dir := build(t, k, prev, next)
+ v, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 1, Now: time.Now()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if v.Latest.Sequence != 2 || v.Latest.Tag != "v1-2" || v.Delta == nil || v.Delta.BaseSequence != 1 {
+ t.Fatalf("%+v", v.Latest)
+ }
+ if v.Snapshot.Stats.Programs != 2 || v.Delta.Stats.Programs != 1 || v.Delta.Stats.Changes != 1 {
+ t.Fatalf("stats %+v / %+v", v.Snapshot.Stats, v.Delta.Stats)
+ }
+ got, err := ReadRecords(dir, v.Snapshot)
+ if err != nil || len(got.Programs) != 2 || got.Programs["disclose:b"].InScope[0].Value != "c.openctem.io" {
+ t.Fatalf("snapshot %v %+v", err, got)
+ }
+ d, err := ReadRecords(dir, *v.Delta)
+ if err != nil || len(d.Programs) != 1 || len(d.Changes) != 1 {
+ t.Fatalf("delta %v %+v", err, d)
+ }
+ dir2 := build(t, k, prev, next)
+ for _, f := range v.Snapshot.Files {
+ a, _ := os.ReadFile(filepath.Join(dir, f.Name))
+ b, _ := os.ReadFile(filepath.Join(dir2, f.Name))
+ if string(a) != string(b) {
+ t.Errorf("%s differs between builds", f.Name)
+ }
+ }
+}
+
+func TestVerifyRefuses(t *testing.T) {
+ k := newKeys(t)
+ other := newKeys(t)
+ now := time.Now()
+ cases := map[string]struct {
+ mut func(t *testing.T, dir string)
+ opt VerifyOptions
+ }{
+ "rollback": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 5, Now: now}},
+ "replay same": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 3, Now: now}},
+ "expired": {nil, VerifyOptions{PinnedRoot: k.rootID, Now: now.Add(8 * 24 * time.Hour)}},
+ "other root": {nil, VerifyOptions{PinnedRoot: other.rootID, Now: now}},
+ "no pinned root": {nil, VerifyOptions{Now: now}},
+ "key set rolled back": {nil, VerifyOptions{PinnedRoot: k.rootID, MinKeySetVer: 2, Now: now}},
+ "tampered file": {func(t *testing.T, dir string) {
+ p := filepath.Join(dir, FileName(KindSnapshot, "programs"))
+ b, _ := os.ReadFile(p)
+ b[len(b)/2] ^= 0xff
+ _ = os.WriteFile(p, b, 0o644)
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "swapped key set": {func(t *testing.T, dir string) {
+ _ = os.WriteFile(filepath.Join(dir, KeySetFile), other.keyset, 0o644)
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "vulnerability feed key set": {func(t *testing.T, dir string) {
+ // A key set of another feed (other payload type) is refused even
+ // under the same root.
+ raw, _ := os.ReadFile(filepath.Join(dir, KeySetFile))
+ var env dsse.Envelope
+ _ = json.Unmarshal(raw, &env)
+ env.PayloadType = "application/vnd.openctem.vulnfeed.keyset+json"
+ b, _ := json.Marshal(env)
+ _ = os.WriteFile(filepath.Join(dir, KeySetFile), b, 0o644)
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "pointer signed by a stranger": {func(t *testing.T, dir string) {
+ resignJSON(t, dir, LatestFile, LatestPayloadType, other.priv, func(m map[string]any) {})
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "pointer names another sequence": {func(t *testing.T, dir string) {
+ resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["sequence"] = 9; m["tag"] = "v1-9" })
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "manifest valid for a year": {func(t *testing.T, dir string) {
+ resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) {
+ m["expires_at"] = now.Add(365 * 24 * time.Hour).UTC().Format(time.RFC3339)
+ })
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ "unknown field": {func(t *testing.T, dir string) {
+ resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["extra"] = 1 })
+ }, VerifyOptions{PinnedRoot: k.rootID, Now: now}},
+ }
+ for name, c := range cases {
+ dir := build(t, k, snapshot(2, "b.openctem.io"), snapshot(3, "c.openctem.io"))
+ if c.mut != nil {
+ c.mut(t, dir)
+ }
+ if _, err := VerifyDir(dir, c.opt); err == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+}
+
+// Poisoned records under a valid signature and correct hashes are refused:
+// the importer re-validates every record.
+func TestPoisonedRecordsRefused(t *testing.T) {
+ k := newKeys(t)
+ good := program("disclose:x", "x.openctem.io")
+ line := func(mut func(m map[string]any)) string {
+ b, _ := json.Marshal(good)
+ var m map[string]any
+ _ = json.Unmarshal(b, &m)
+ mut(m)
+ out, _ := json.Marshal(m)
+ return string(out)
+ }
+ target := func(typ, value string) func(m map[string]any) {
+ return func(m map[string]any) {
+ m["in_scope"] = []any{map[string]any{"type": typ, "value": value, "confidence": "published"}}
+ }
+ }
+ cases := map[string]string{
+ "public suffix wildcard": line(target("wildcard", "*.com")),
+ "private range": line(target("cidr", "10.0.0.0/8")),
+ "huge range": line(target("cidr", "1.0.0.0/8")),
+ "metadata address": line(target("ip", "169.254.169.254")),
+ "bare TLD": line(target("domain", "com")),
+ "javascript url": line(func(m map[string]any) { m["url"] = "javascript:alert(1)" }),
+ "closed without time": line(func(m map[string]any) { m["status"] = "closed" }),
+ "extra field": line(func(m map[string]any) { m["sql"] = "x" }),
+ "duplicate": line(func(map[string]any) {}) + "\n" + line(func(map[string]any) {}),
+ }
+ for name, l := range cases {
+ dir := build(t, k, nil, snapshot(1, "b.openctem.io"))
+ replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), l+"\n", strings.Count(l, "\n")+1)
+ if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+ // A change naming a program that is not in the bundle is refused.
+ dir := build(t, k, nil, snapshot(1, "b.openctem.io"))
+ replaceFile(t, dir, k, FileName(KindSnapshot, "changes"), `{"sequence":1,"program":"disclose:ghost","kind":"program_added"}`+"\n", 1)
+ if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil {
+ t.Error("change of an absent program accepted")
+ }
+ // A wrong record count is refused too.
+ dir = build(t, k, nil, snapshot(1, "b.openctem.io"))
+ replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), line(func(map[string]any) {})+"\n", 5)
+ if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil || !strings.Contains(err.Error(), "records") {
+ t.Errorf("record count: %v", err)
+ }
+}
+
+func TestWriteRefusesInvalidSnapshot(t *testing.T) {
+ s := snapshot(1, "b.openctem.io")
+ p := s.Programs["disclose:a"]
+ p.InScope = []model.Target{{Type: model.TargetWildcard, Value: "*.com", Confidence: model.ConfidencePublished}}
+ s.Programs["disclose:a"] = p
+ if _, _, _, err := Write(t.TempDir(), nil, s, WriteOptions{Now: time.Now()}); err == nil {
+ t.Fatal("invalid snapshot written")
+ }
+}
+
+func TestSignRefusesKeyOutsideKeySet(t *testing.T) {
+ k := newKeys(t)
+ _, stranger, _ := ed25519.GenerateKey(nil)
+ dir := t.TempDir()
+ if _, _, _, err := Write(dir, nil, snapshot(1, "b.openctem.io"), WriteOptions{Now: time.Now()}); err != nil {
+ t.Fatal(err)
+ }
+ if err := Sign(dir, stranger, k.keyset, k.rootID, time.Now()); err == nil {
+ t.Fatal("signed with a key outside the key set")
+ }
+}
+
+// replaceFile rewrites one record file, fixes its hash/size/count in the
+// manifest and re-signs the manifest: only record validation can catch it.
+func replaceFile(t *testing.T, dir string, k keys, name, content string, records int) {
+ t.Helper()
+ p := filepath.Join(dir, name)
+ f, _ := os.Create(p)
+ gz := gzip.NewWriter(f)
+ _, _ = gz.Write([]byte(content))
+ _ = gz.Close()
+ _ = f.Close()
+ b, _ := os.ReadFile(p)
+ sum := sha256.Sum256(b)
+ resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) {
+ for _, fi := range m["files"].([]any) {
+ fm := fi.(map[string]any)
+ if fm["name"] == name {
+ fm["sha256"], fm["size"], fm["records"] = hex.EncodeToString(sum[:]), len(b), records
+ }
+ }
+ })
+}
+
+func resignJSON(t *testing.T, dir, file, payloadType string, priv ed25519.PrivateKey, mut func(map[string]any)) {
+ t.Helper()
+ raw, err := os.ReadFile(filepath.Join(dir, file))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var env dsse.Envelope
+ _ = json.Unmarshal(raw, &env)
+ var m map[string]any
+ _ = json.Unmarshal(env.Payload, &m)
+ mut(m)
+ payload, _ := json.Marshal(m)
+ out, _ := dsse.Sign(priv, payloadType, payload)
+ _ = os.WriteFile(filepath.Join(dir, file), out, 0o644)
+}
diff --git a/internal/bundle/verify.go b/internal/bundle/verify.go
new file mode 100644
index 0000000..bd7e4dd
--- /dev/null
+++ b/internal/bundle/verify.go
@@ -0,0 +1,277 @@
+package bundle
+
+import (
+ "bufio"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/dsse"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// Verified is a bundle directory whose signatures, freshness and file
+// hashes checked out.
+type Verified struct {
+ KeySet *dsse.KeySet
+ Latest Latest
+ Snapshot Manifest
+ Delta *Manifest
+}
+
+// VerifyOptions are the consumer's state.
+type VerifyOptions struct {
+ PinnedRoot string
+ MinKeySetVer uint64
+ AppliedSeq uint64
+ Now time.Time
+ AllowExpired bool // the previous release, read while building
+ SkipRecordChk bool
+}
+
+// VerifyDir checks a bundle the way the platform importer does: key set
+// (pinned root, version), latest pointer and manifests (key set, schema,
+// sequence, expiry), every file's size and hash, and every record.
+func VerifyDir(dir string, opt VerifyOptions) (*Verified, error) {
+ ksRaw, err := readCapped(filepath.Join(dir, KeySetFile), dsse.MaxKeySetBytes)
+ if err != nil {
+ return nil, err
+ }
+ ks, err := dsse.VerifyKeySet(ksRaw, opt.PinnedRoot, opt.MinKeySetVer, opt.Now)
+ if err != nil {
+ return nil, err
+ }
+ v := &Verified{KeySet: ks}
+ if err := verifyJSON(ks, filepath.Join(dir, LatestFile), LatestPayloadType, &v.Latest); err != nil {
+ return nil, err
+ }
+ l := v.Latest
+ switch {
+ case l.Schema != LatestSchema:
+ return nil, fmt.Errorf("latest: schema %q", l.Schema)
+ case l.Sequence <= opt.AppliedSeq:
+ return nil, fmt.Errorf("latest: sequence %d is not after the applied %d (rollback refused)", l.Sequence, opt.AppliedSeq)
+ case !opt.AllowExpired && !l.ExpiresAt.After(opt.Now):
+ return nil, fmt.Errorf("latest: expired at %s", l.ExpiresAt.Format(time.RFC3339))
+ case l.Tag != Tag(l.Sequence):
+ return nil, fmt.Errorf("latest: tag %q does not name sequence %d", l.Tag, l.Sequence)
+ }
+ if err := v.loadManifest(dir, l.Snapshot, KindSnapshot, &v.Snapshot, opt); err != nil {
+ return nil, err
+ }
+ if l.Delta != "" {
+ var d Manifest
+ if err := v.loadManifest(dir, l.Delta, KindDelta, &d, opt); err != nil {
+ return nil, err
+ }
+ if d.BaseSequence != l.BaseSequence || d.BaseSequence >= d.Sequence {
+ return nil, fmt.Errorf("delta: base sequence %d does not match the pointer", d.BaseSequence)
+ }
+ v.Delta = &d
+ }
+ return v, nil
+}
+
+func (v *Verified) loadManifest(dir, name, kind string, m *Manifest, opt VerifyOptions) error {
+ if name != fmt.Sprintf(ManifestFile, kind) {
+ return fmt.Errorf("%s manifest name %q", kind, name)
+ }
+ if err := verifyJSON(v.KeySet, filepath.Join(dir, name), ManifestPayloadType, m); err != nil {
+ return err
+ }
+ switch {
+ case m.Schema != ManifestSchema:
+ return fmt.Errorf("%s: schema %q", kind, m.Schema)
+ case m.Kind != kind:
+ return fmt.Errorf("%s: kind %q", kind, m.Kind)
+ case m.Sequence != v.Latest.Sequence:
+ return fmt.Errorf("%s: sequence %d, the pointer says %d", kind, m.Sequence, v.Latest.Sequence)
+ case !opt.AllowExpired && !m.ExpiresAt.After(opt.Now):
+ return fmt.Errorf("%s: expired", kind)
+ case m.ExpiresAt.Sub(m.CreatedAt) > Validity:
+ return fmt.Errorf("%s: valid for more than %s", kind, Validity)
+ case len(m.Files) != len(recordFiles):
+ return fmt.Errorf("%s: %d files", kind, len(m.Files))
+ }
+ for i, f := range m.Files {
+ if f.Name != FileName(kind, recordFiles[i]) {
+ return fmt.Errorf("%s: unexpected file %q", kind, f.Name)
+ }
+ if err := checkFile(filepath.Join(dir, f.Name), f); err != nil {
+ return err
+ }
+ }
+ if opt.SkipRecordChk {
+ return nil
+ }
+ _, err := ReadRecords(dir, *m)
+ return err
+}
+
+func verifyJSON(ks *dsse.KeySet, path, payloadType string, v any) error {
+ raw, err := readCapped(path, MaxManifestBytes)
+ if err != nil {
+ return err
+ }
+ payload, err := dsse.VerifyWith(ks, raw, payloadType, MaxManifestBytes)
+ if err != nil {
+ return fmt.Errorf("%s: %w", filepath.Base(path), err)
+ }
+ if err := dsse.DecodeStrict(payload, v); err != nil {
+ return fmt.Errorf("%s: %w", filepath.Base(path), err)
+ }
+ return nil
+}
+
+func readCapped(path string, max int64) ([]byte, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ b, err := io.ReadAll(io.LimitReader(f, max+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(b)) > max {
+ return nil, fmt.Errorf("%s is over %d bytes", filepath.Base(path), max)
+ }
+ return b, nil
+}
+
+func checkFile(path string, f File) error {
+ if f.Size > MaxFileBytes || f.Size < 0 {
+ return fmt.Errorf("%s: size %d over the cap", f.Name, f.Size)
+ }
+ fh, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer fh.Close()
+ h := sha256.New()
+ n, err := io.Copy(h, io.LimitReader(fh, MaxFileBytes+1))
+ if err != nil {
+ return err
+ }
+ if n != f.Size || hex.EncodeToString(h.Sum(nil)) != f.SHA256 {
+ return fmt.Errorf("%s: size or SHA-256 does not match the manifest", f.Name)
+ }
+ return nil
+}
+
+// ReadRecords reads and validates the record files of a verified manifest
+// into a snapshot (for a delta: only the changed programs, plus the change
+// log).
+func ReadRecords(dir string, m Manifest) (*Snapshot, error) {
+ s := NewSnapshot()
+ s.Sequence, s.Sources = m.Sequence, m.Sources
+ counts := map[string]int{}
+ for _, f := range m.Files {
+ var kind string
+ for _, rf := range recordFiles {
+ if f.Name == FileName(m.Kind, rf) {
+ kind = rf
+ }
+ }
+ err := eachLine(filepath.Join(dir, f.Name), func(line []byte) error {
+ counts[kind]++
+ switch kind {
+ case "programs":
+ var p model.Program
+ if err := strictLine(line, &p); err != nil {
+ return err
+ }
+ if err := p.Validate(); err != nil {
+ return err
+ }
+ if _, dup := s.Programs[p.ID]; dup {
+ return fmt.Errorf("program %s listed twice", p.ID)
+ }
+ if len(s.Programs) >= model.MaxPrograms {
+ return fmt.Errorf("over %d programs", model.MaxPrograms)
+ }
+ s.Programs[p.ID] = p
+ case "changes":
+ var c model.Change
+ if err := strictLine(line, &c); err != nil {
+ return err
+ }
+ if err := c.Validate(); err != nil {
+ return err
+ }
+ if len(s.Changes) >= 10*model.MaxPrograms {
+ return errors.New("too many changes")
+ }
+ s.Changes = append(s.Changes, c)
+ default:
+ return errors.New("unexpected record file")
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("%s: %w", f.Name, err)
+ }
+ if counts[kind] != f.Records {
+ return nil, fmt.Errorf("%s: %d records, the manifest says %d", f.Name, counts[kind], f.Records)
+ }
+ }
+ if err := s.Validate(); err != nil {
+ return nil, err
+ }
+ if m.Kind == KindDelta {
+ if ids := s.ChangedIDs(); len(ids) != len(s.Programs) {
+ return nil, errors.New("delta: a program that no change names")
+ }
+ }
+ return s, nil
+}
+
+func strictLine(line []byte, v any) error { return dsse.DecodeStrict(line, v) }
+
+func eachLine(path string, fn func([]byte) error) error {
+ f, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ gz, err := gzip.NewReader(io.LimitReader(f, MaxFileBytes))
+ if err != nil {
+ return err
+ }
+ defer gz.Close()
+ sc := bufio.NewScanner(&limited{r: gz, left: MaxDecompressedBytes})
+ sc.Buffer(make([]byte, 64<<10), MaxRecordBytes)
+ for sc.Scan() {
+ if len(sc.Bytes()) == 0 {
+ continue
+ }
+ if err := fn(sc.Bytes()); err != nil {
+ return err
+ }
+ }
+ return sc.Err()
+}
+
+// limited fails once more than left bytes are read (a decompression bomb).
+type limited struct {
+ r io.Reader
+ left int64
+}
+
+func (l *limited) Read(p []byte) (int, error) {
+ if l.left <= 0 {
+ return 0, errors.New("decompressed data over the cap")
+ }
+ if int64(len(p)) > l.left {
+ p = p[:l.left]
+ }
+ n, err := l.r.Read(p)
+ l.left -= int64(n)
+ return n, err
+}
diff --git a/internal/collect/collect.go b/internal/collect/collect.go
new file mode 100644
index 0000000..03c8e00
--- /dev/null
+++ b/internal/collect/collect.go
@@ -0,0 +1,334 @@
+// Package collect builds the next snapshot: the previous one merged with
+// what each source says now, and the change log between them.
+package collect
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/bundle"
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+ "github.com/openctemio/programfeed/internal/source"
+)
+
+// DropAfter is how long a closed program stays in the feed before it is
+// dropped (so subscribers see the closure first).
+const DropAfter = 180 * 24 * time.Hour
+
+// Mass-change guard: a source that suddenly reports fewer than half of the
+// programs it had (and had at least massMinimum) is treated as broken for
+// this run, and its previous records are kept.
+const (
+ massShare = 0.5
+ massMinimum = 20
+)
+
+// Options of a build.
+type Options struct {
+ Now time.Time
+ // AllowMassChange lets a source drop more than the guard allows (a
+ // person checked the source).
+ AllowMassChange bool
+ Log func(format string, args ...any)
+}
+
+// Build returns the next snapshot. prev may be nil (first build). It fails
+// only when no source could be read.
+func Build(ctx context.Context, prev *bundle.Snapshot, sources []source.Source, client *fetch.Client, opt Options) (*bundle.Snapshot, error) {
+ if prev == nil {
+ prev = bundle.NewSnapshot()
+ }
+ log := opt.Log
+ if log == nil {
+ log = func(string, ...any) {}
+ }
+ now := opt.Now.UTC().Truncate(time.Second)
+ next := bundle.NewSnapshot()
+ next.Sequence = prev.Sequence + 1
+ ok := 0
+ var failures []string
+ seenSource := map[string]bool{}
+ for _, src := range sources {
+ meta := src.Meta()
+ if seenSource[meta.Name] {
+ return nil, fmt.Errorf("source %q configured twice", meta.Name)
+ }
+ seenSource[meta.Name] = true
+ old := programsOf(prev, meta.Name)
+ res, err := src.Collect(ctx, client)
+ if err == nil {
+ err = guard(res, old, opt.AllowMassChange)
+ }
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ log("%s: %v; keeping its %d previous records", meta.Name, err, len(old))
+ failures = append(failures, meta.Name+": "+err.Error())
+ for _, p := range old {
+ next.Programs[p.ID] = p
+ }
+ if s, had := prev.Source(meta.Name); had {
+ s.Programs = len(old)
+ next.Sources = append(next.Sources, s)
+ }
+ continue
+ }
+ ok++
+ n := merge(prev, next, meta.Name, old, res, now, log)
+ next.Sources = append(next.Sources, bundle.Source{Name: meta.Name, URL: meta.URL, AsOf: now, Terms: meta.Terms,
+ Licence: meta.Licence, Attribution: meta.Attribution, Programs: n})
+ log("%s: %d programs (%d skipped records, %d unreadable kept)", meta.Name, n, res.Skipped, len(res.Unknown))
+ }
+ // Programs of sources no longer configured are kept as they are; a
+ // source is removed from the feed on purpose, in code review.
+ for id, p := range prev.Programs {
+ if !seenSource[p.Source] {
+ next.Programs[id] = p
+ }
+ }
+ for _, s := range prev.Sources {
+ if !seenSource[s.Name] {
+ next.Sources = append(next.Sources, s)
+ }
+ }
+ if ok == 0 {
+ return nil, fmt.Errorf("no source could be read (%s)", strings.Join(failures, "; "))
+ }
+ sort.Slice(next.Sources, func(i, j int) bool { return next.Sources[i].Name < next.Sources[j].Name })
+ sortChanges(next.Changes)
+ return next, next.Validate()
+}
+
+func programsOf(s *bundle.Snapshot, name string) map[string]model.Program {
+ out := map[string]model.Program{}
+ for id, p := range s.Programs {
+ if p.Source == name {
+ out[id] = p
+ }
+ }
+ return out
+}
+
+func guard(res *source.Result, old map[string]model.Program, allow bool) error {
+ open := 0
+ for _, p := range old {
+ if p.Status != model.StatusClosed {
+ open++
+ }
+ }
+ got := len(res.Programs) + len(res.Unknown)
+ if !allow && open >= massMinimum && float64(got) < float64(open)*massShare {
+ return fmt.Errorf("refusing: %d programs now, %d before (rerun with --allow-mass-change after checking the source)", got, open)
+ }
+ return nil
+}
+
+// merge folds one source's result into next and returns how many programs
+// of the source next holds.
+func merge(prev, next *bundle.Snapshot, name string, old map[string]model.Program, res *source.Result, now time.Time, log func(string, ...any)) int {
+ seq := next.Sequence
+ add := func(c model.Change) {
+ c.Sequence = seq
+ next.Changes = append(next.Changes, c)
+ }
+ seen := map[string]bool{}
+ for _, id := range res.Unknown {
+ if p, had := old[id]; had {
+ next.Programs[id] = p
+ seen[id] = true
+ }
+ }
+ for _, p := range res.Programs {
+ if seen[p.ID] {
+ continue
+ }
+ if p.Source != name {
+ log("%s: program %s claims source %q; skipped", name, p.ID, p.Source)
+ continue
+ }
+ model.Normalize(&p)
+ was, had := old[p.ID]
+ p.LastSeen = now
+ if had {
+ p.FirstSeen = was.FirstSeen
+ changes := diff(was, p)
+ if len(changes) > 0 {
+ p.LastChanged = now
+ } else {
+ p.LastChanged = was.LastChanged
+ }
+ if err := p.Validate(); err != nil {
+ log("%s: %v; keeping the previous record", name, err)
+ next.Programs[p.ID] = was
+ seen[p.ID] = true
+ continue
+ }
+ for _, c := range changes {
+ add(c)
+ }
+ } else {
+ p.FirstSeen, p.LastChanged = now, now
+ if err := p.Validate(); err != nil {
+ log("%s: %v; skipped", name, err)
+ continue
+ }
+ add(model.Change{Program: p.ID, Kind: model.ChangeProgramAdded})
+ }
+ next.Programs[p.ID] = p
+ seen[p.ID] = true
+ }
+ ids := make([]string, 0, len(old))
+ for id := range old {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ for _, id := range ids {
+ if seen[id] {
+ continue
+ }
+ p := old[id]
+ switch {
+ case p.Status != model.StatusClosed:
+ at := now
+ p.Status, p.ClosedAt, p.LastChanged = model.StatusClosed, &at, now
+ next.Programs[id] = p
+ add(model.Change{Program: id, Kind: model.ChangeProgramClosed})
+ case now.Sub(*p.ClosedAt) > DropAfter:
+ add(model.Change{Program: id, Kind: model.ChangeProgramDropped})
+ default:
+ next.Programs[id] = p
+ }
+ seen[id] = true
+ }
+ n := 0
+ for _, p := range next.Programs {
+ if p.Source == name {
+ n++
+ }
+ }
+ return n
+}
+
+// diff lists what changed between two records of one program.
+func diff(a, b model.Program) []model.Change {
+ var out []model.Change
+ id := b.ID
+ switch {
+ case a.Status == model.StatusClosed && b.Status != model.StatusClosed:
+ out = append(out, model.Change{Program: id, Kind: model.ChangeProgramReopened})
+ case a.Status != b.Status:
+ out = append(out, model.Change{Program: id, Kind: model.ChangeStatus, Fields: []string{"status"}})
+ }
+ out = append(out, diffTargets(id, "in", a.InScope, b.InScope)...)
+ out = append(out, diffTargets(id, "out", a.OutOfScope, b.OutOfScope)...)
+ if !reflect.DeepEqual(a.Rules, b.Rules) {
+ out = append(out, model.Change{Program: id, Kind: model.ChangeRules, Fields: diffFields(a.Rules, b.Rules)})
+ }
+ if a.Terms.URL != b.Terms.URL || a.Terms.SHA256 != b.Terms.SHA256 {
+ out = append(out, model.Change{Program: id, Kind: model.ChangeTerms})
+ }
+ var fields []string
+ for name, eq := range map[string]bool{
+ "name": a.Name == b.Name, "url": a.URL == b.URL, "platform": a.Platform == b.Platform, "type": a.Type == b.Type,
+ "scope_published": a.ScopePublished == b.ScopePublished, "contact": a.Contact == b.Contact,
+ "rejected": reflect.DeepEqual(a.Rejected, b.Rejected),
+ } {
+ if !eq {
+ fields = append(fields, name)
+ }
+ }
+ if len(fields) > 0 {
+ sort.Strings(fields)
+ out = append(out, model.Change{Program: id, Kind: model.ChangeProgramModified, Fields: fields})
+ }
+ return out
+}
+
+func diffTargets(id, scope string, a, b []model.Target) []model.Change {
+ key := func(t model.Target) string { return t.Type + "\x00" + t.Value }
+ before := map[string]model.Target{}
+ for _, t := range a {
+ before[key(t)] = t
+ }
+ var out []model.Change
+ after := map[string]bool{}
+ for _, t := range b {
+ t := t
+ after[key(t)] = true
+ was, had := before[key(t)]
+ switch {
+ case !had:
+ out = append(out, model.Change{Program: id, Kind: model.ChangeTargetAdded, Scope: scope, Target: &t})
+ case !reflect.DeepEqual(was, t):
+ out = append(out, model.Change{Program: id, Kind: model.ChangeTargetModified, Scope: scope, Target: &t, Fields: diffFields(was, t)})
+ }
+ }
+ for _, t := range a {
+ t := t
+ if !after[key(t)] {
+ out = append(out, model.Change{Program: id, Kind: model.ChangeTargetRemoved, Scope: scope, Target: &t})
+ }
+ }
+ return out
+}
+
+// diffFields names the JSON fields that differ between two structs.
+func diffFields(a, b any) []string {
+ var ma, mb map[string]json.RawMessage
+ ja, _ := json.Marshal(a)
+ jb, _ := json.Marshal(b)
+ _ = json.Unmarshal(ja, &ma)
+ _ = json.Unmarshal(jb, &mb)
+ var out []string
+ for k, v := range ma {
+ if string(mb[k]) != string(v) {
+ out = append(out, k)
+ }
+ }
+ for k := range mb {
+ if _, ok := ma[k]; !ok {
+ out = append(out, k)
+ }
+ }
+ sort.Strings(out)
+ if len(out) > model.MaxChangeFields {
+ out = out[:model.MaxChangeFields]
+ }
+ return out
+}
+
+var kindOrder = map[string]int{
+ model.ChangeProgramAdded: 0, model.ChangeProgramReopened: 1, model.ChangeStatus: 2, model.ChangeProgramModified: 3,
+ model.ChangeRules: 4, model.ChangeTerms: 5, model.ChangeTargetAdded: 6, model.ChangeTargetModified: 7,
+ model.ChangeTargetRemoved: 8, model.ChangeProgramClosed: 9, model.ChangeProgramDropped: 10,
+}
+
+func sortChanges(cs []model.Change) {
+ sort.SliceStable(cs, func(i, j int) bool {
+ a, b := cs[i], cs[j]
+ if a.Program != b.Program {
+ return a.Program < b.Program
+ }
+ if kindOrder[a.Kind] != kindOrder[b.Kind] {
+ return kindOrder[a.Kind] < kindOrder[b.Kind]
+ }
+ if a.Scope != b.Scope {
+ return a.Scope < b.Scope
+ }
+ var ta, tb string
+ if a.Target != nil {
+ ta = a.Target.Type + "\x00" + a.Target.Value
+ }
+ if b.Target != nil {
+ tb = b.Target.Type + "\x00" + b.Target.Value
+ }
+ return ta < tb
+ })
+}
diff --git a/internal/collect/collect_test.go b/internal/collect/collect_test.go
new file mode 100644
index 0000000..c92fc4c
--- /dev/null
+++ b/internal/collect/collect_test.go
@@ -0,0 +1,196 @@
+package collect
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/bundle"
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+ "github.com/openctemio/programfeed/internal/source"
+)
+
+type fake struct {
+ name string
+ res *source.Result
+ err error
+}
+
+func (f fake) Meta() source.Meta {
+ return source.Meta{Name: f.name, URL: "https://openctem.io/" + f.name, Terms: "https://openctem.io/terms", Licence: "CC0-1.0"}
+}
+
+func (f fake) Collect(context.Context, *fetch.Client) (*source.Result, error) {
+ if f.err != nil {
+ return nil, f.err
+ }
+ // Return copies: the collector normalises in place.
+ out := *f.res
+ out.Programs = append([]model.Program(nil), f.res.Programs...)
+ for i := range out.Programs {
+ p := &out.Programs[i]
+ p.InScope = append([]model.Target(nil), p.InScope...)
+ }
+ return &out, nil
+}
+
+var day0 = time.Date(2026, 10, 1, 2, 0, 0, 0, time.UTC)
+
+func prog(src, slug string, targets ...string) model.Program {
+ p := model.Program{
+ ID: src + ":" + slug, Source: src, Platform: "self-hosted", Name: slug, URL: "https://" + slug + ".org/security",
+ Type: model.TypeVDP, Status: model.StatusOpen, InScope: []model.Target{}, OutOfScope: []model.Target{},
+ Terms: model.Terms{URL: "https://" + slug + ".org/security"},
+ Provenance: model.Provenance{Source: src, SourceURL: "https://openctem.io/" + src, FetchedAt: day0},
+ }
+ for _, t := range targets {
+ typ := model.TargetDomain
+ if strings.HasPrefix(t, "*.") {
+ typ = model.TargetWildcard
+ }
+ p.InScope = append(p.InScope, model.Target{Type: typ, Value: t})
+ }
+ return p
+}
+
+func run(t *testing.T, prev *bundle.Snapshot, at time.Time, allowMass bool, srcs ...source.Source) *bundle.Snapshot {
+ t.Helper()
+ next, err := Build(context.Background(), prev, srcs, nil, Options{Now: at, AllowMassChange: allowMass})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return next
+}
+
+func kinds(s *bundle.Snapshot) string {
+ var out []string
+ for _, c := range s.Changes {
+ k := c.Program + " " + c.Kind
+ if c.Target != nil {
+ k += " " + c.Scope + ":" + c.Target.Value
+ }
+ out = append(out, k)
+ }
+ return strings.Join(out, "\n")
+}
+
+func TestBuildLifecycle(t *testing.T) {
+ s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: []model.Program{
+ prog("d", "acme", "acme.org", "*.com"),
+ prog("d", "beta", "beta.org"),
+ prog("d", "acme", "acme.org"), // duplicate id: first wins
+ }}})
+ if s1.Sequence != 1 || len(s1.Programs) != 2 || kinds(s1) != "d:acme program_added\nd:beta program_added" {
+ t.Fatalf("first build:\n%s", kinds(s1))
+ }
+ acme := s1.Programs["d:acme"]
+ if len(acme.InScope) != 1 || len(acme.Rejected) != 1 || !acme.FirstSeen.Equal(day0) {
+ t.Fatalf("acme: %+v", acme)
+ }
+
+ // Day 1: acme gains a target and loses none, beta disappears.
+ day1 := day0.Add(24 * time.Hour)
+ s2 := run(t, s1, day1, false, fake{name: "d", res: &source.Result{Programs: []model.Program{
+ prog("d", "acme", "acme.org", "*.com", "api.acme.org"),
+ }}})
+ want := "d:acme target_added in:api.acme.org\nd:beta program_closed"
+ if s2.Sequence != 2 || kinds(s2) != want {
+ t.Fatalf("day 1:\n%s\nwant\n%s", kinds(s2), want)
+ }
+ if b := s2.Programs["d:beta"]; b.Status != model.StatusClosed || b.ClosedAt == nil || !b.LastSeen.Equal(day0) {
+ t.Errorf("beta: %+v", b)
+ }
+ if a := s2.Programs["d:acme"]; !a.FirstSeen.Equal(day0) || !a.LastChanged.Equal(day1) || !a.LastSeen.Equal(day1) {
+ t.Errorf("acme timestamps: %+v", a)
+ }
+
+ // Day 2: nothing changes: no change entries, last_changed stays.
+ day2 := day1.Add(24 * time.Hour)
+ s3 := run(t, s2, day2, false, fake{name: "d", res: &source.Result{Programs: []model.Program{
+ prog("d", "acme", "acme.org", "*.com", "api.acme.org"),
+ }}})
+ if len(s3.Changes) != 0 || !s3.Programs["d:acme"].LastChanged.Equal(day1) {
+ t.Fatalf("day 2:\n%s", kinds(s3))
+ }
+
+ // Beta comes back with a different target set; acme loses a target.
+ s4 := run(t, s3, day2.Add(time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{
+ prog("d", "acme", "acme.org"),
+ prog("d", "beta", "beta.org", "www.beta.org"),
+ }}})
+ want = "d:acme program_modified\nd:acme target_removed in:api.acme.org\nd:beta program_reopened\nd:beta target_added in:www.beta.org"
+ if kinds(s4) != want {
+ t.Fatalf("reopen:\n%s\nwant\n%s", kinds(s4), want)
+ }
+
+ // A program closed for longer than DropAfter is dropped.
+ s5 := run(t, s4, day2.Add(2*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}})
+ s6 := run(t, s5, day2.Add(DropAfter+3*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}})
+ if _, still := s6.Programs["d:beta"]; still || kinds(s6) != "d:beta program_dropped" {
+ t.Fatalf("drop:\n%s", kinds(s6))
+ }
+}
+
+func TestBuildKeepsRecordsOfFailingSources(t *testing.T) {
+ s1 := run(t, nil, day0, false,
+ fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "one", "one.org"), prog("a", "two", "two.org")}}},
+ fake{name: "b", res: &source.Result{Programs: []model.Program{prog("b", "three", "three.org")}}})
+ day1 := day0.Add(24 * time.Hour)
+ // Source a is down, source b reports "two"'s host unknown... and b's
+ // program unknown: both keep their records unchanged.
+ s2 := run(t, s1, day1, false,
+ fake{name: "a", err: errors.New("HTTP 503")},
+ fake{name: "b", res: &source.Result{Unknown: []string{"b:three"}}})
+ if len(s2.Programs) != 3 || len(s2.Changes) != 0 {
+ t.Fatalf("kept %d, changes:\n%s", len(s2.Programs), kinds(s2))
+ }
+ for _, p := range s2.Programs {
+ if p.Status != model.StatusOpen || !p.LastSeen.Equal(day0) {
+ t.Errorf("%s: %+v", p.ID, p)
+ }
+ }
+ if src, _ := s2.Source("a"); !src.AsOf.Equal(day0) {
+ t.Errorf("a failing source keeps its old as_of: %+v", src)
+ }
+ // Every source failing fails the build.
+ if _, err := Build(context.Background(), s1, []source.Source{fake{name: "a", err: errors.New("down")}}, nil, Options{Now: day1}); err == nil {
+ t.Error("a build with no readable source succeeded")
+ }
+}
+
+func TestMassChangeGuard(t *testing.T) {
+ var many []model.Program
+ for i := 0; i < 40; i++ {
+ many = append(many, prog("d", fmt.Sprintf("p%02d", i), fmt.Sprintf("p%02d.org", i)))
+ }
+ s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: many}})
+ few := &source.Result{Programs: many[:5]}
+ other := fake{name: "o", res: &source.Result{Programs: []model.Program{prog("o", "x", "x.org")}}}
+ // With another readable source, the guarded source keeps its records.
+ s2 := run(t, s1, day0.Add(time.Hour), false, fake{name: "d", res: few}, other)
+ if len(s2.Programs) != 41 || s2.Stats().Closed != 0 {
+ t.Fatalf("guard did not hold: %+v", s2.Stats())
+ }
+ // Alone, it fails the build and says why.
+ if _, err := Build(context.Background(), s1, []source.Source{fake{name: "d", res: few}}, nil, Options{Now: day0.Add(time.Hour)}); err == nil || !strings.Contains(err.Error(), "allow-mass-change") {
+ t.Fatalf("guard alone: %v", err)
+ }
+ s3 := run(t, s1, day0.Add(time.Hour), true, fake{name: "d", res: few})
+ if s3.Stats().Closed != 35 {
+ t.Fatalf("override: %+v", s3.Stats())
+ }
+}
+
+func TestSourceCannotWriteAnotherSourcesPrograms(t *testing.T) {
+ evil := prog("a", "victim", "victim.org") // claims source a, served by b
+ s := run(t, nil, day0, false,
+ fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "real", "real.org")}}},
+ fake{name: "b", res: &source.Result{Programs: []model.Program{evil, prog("b", "ok", "ok.org")}}})
+ if _, ok := s.Programs["a:victim"]; ok || len(s.Programs) != 2 {
+ t.Fatalf("programs: %v", s.SortedIDs())
+ }
+}
diff --git a/internal/dsse/dsse.go b/internal/dsse/dsse.go
new file mode 100644
index 0000000..ac32416
--- /dev/null
+++ b/internal/dsse/dsse.go
@@ -0,0 +1,251 @@
+// Package dsse signs and verifies DSSE v1 envelopes with Ed25519, and the
+// key sets that name the keys allowed to sign bundles.
+//
+// The envelope, pre-authentication encoding and key ids are the ones the
+// OpenCTEM platform already verifies for signed jobs: JSON
+// {"payloadType", "payload" (base64), "signatures": [{"keyid", "sig" (base64)}]},
+// PAE "DSSEv1 ", key id
+// "SHA256:" + hex(sha256(raw public key)).
+package dsse
+
+import (
+ "bytes"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "strconv"
+ "time"
+)
+
+// Envelope is a DSSE v1 envelope.
+type Envelope struct {
+ PayloadType string `json:"payloadType"`
+ Payload []byte `json:"payload"`
+ Signatures []Signature `json:"signatures"`
+}
+
+// Signature is one signature of an envelope.
+type Signature struct {
+ KeyID string `json:"keyid"`
+ Sig []byte `json:"sig"`
+}
+
+// PAE is the DSSE v1 pre-authentication encoding.
+func PAE(payloadType string, payload []byte) []byte {
+ var b bytes.Buffer
+ b.WriteString("DSSEv1 ")
+ b.WriteString(strconv.Itoa(len(payloadType)))
+ b.WriteByte(' ')
+ b.WriteString(payloadType)
+ b.WriteByte(' ')
+ b.WriteString(strconv.Itoa(len(payload)))
+ b.WriteByte(' ')
+ b.Write(payload)
+ return b.Bytes()
+}
+
+// KeyID names a public key.
+func KeyID(pub ed25519.PublicKey) string {
+ sum := sha256.Sum256(pub)
+ return "SHA256:" + hex.EncodeToString(sum[:])
+}
+
+// Sign wraps payload in an envelope signed by priv.
+func Sign(priv ed25519.PrivateKey, payloadType string, payload []byte) ([]byte, error) {
+ if len(priv) != ed25519.PrivateKeySize {
+ return nil, errors.New("dsse: an Ed25519 private key is required")
+ }
+ pub, _ := priv.Public().(ed25519.PublicKey)
+ return json.Marshal(Envelope{
+ PayloadType: payloadType,
+ Payload: payload,
+ Signatures: []Signature{{KeyID: KeyID(pub), Sig: ed25519.Sign(priv, PAE(payloadType, payload))}},
+ })
+}
+
+// Open parses an envelope of the expected type (at most maxBytes) without
+// checking signatures.
+func Open(envelope []byte, payloadType string, maxBytes int) (*Envelope, error) {
+ if len(envelope) == 0 || len(envelope) > maxBytes {
+ return nil, fmt.Errorf("dsse: envelope is empty or over %d bytes", maxBytes)
+ }
+ var env Envelope
+ if err := json.Unmarshal(envelope, &env); err != nil {
+ return nil, fmt.Errorf("dsse: envelope: %w", err)
+ }
+ if env.PayloadType != payloadType {
+ return nil, fmt.Errorf("dsse: payload type %q, want %q", env.PayloadType, payloadType)
+ }
+ return &env, nil
+}
+
+// VerifiedBy reports whether env carries a valid signature of pub.
+func (env *Envelope) VerifiedBy(pub ed25519.PublicKey) bool {
+ id := KeyID(pub)
+ pae := PAE(env.PayloadType, env.Payload)
+ for _, s := range env.Signatures {
+ if s.KeyID == id && ed25519.Verify(pub, pae, s.Sig) {
+ return true
+ }
+ }
+ return false
+}
+
+// DecodeStrict decodes JSON into v, refusing unknown fields and trailing data.
+func DecodeStrict(data []byte, v any) error {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ dec.DisallowUnknownFields()
+ if err := dec.Decode(v); err != nil {
+ return err
+ }
+ if _, err := dec.Token(); !errors.Is(err, io.EOF) {
+ return errors.New("trailing data")
+ }
+ return nil
+}
+
+// Key set (offline root → online keys).
+const (
+ KeySetPayloadType = "application/vnd.openctem.programfeed.keyset+json"
+ KeySetKind = "openctem.programfeed.keyset/v1"
+ MaxKeySetValidity = 180 * 24 * time.Hour
+ MaxKeySetBytes = 64 << 10
+ MaxKeySetKeys = 8
+ maxClockSkew = 5 * time.Minute
+)
+
+// PublicKey describes one online key.
+type PublicKey struct {
+ KeyID string `json:"keyid"`
+ Algorithm string `json:"algorithm"`
+ PublicKey string `json:"public_key"`
+}
+
+// NewPublicKey describes pub.
+func NewPublicKey(pub ed25519.PublicKey) PublicKey {
+ return PublicKey{KeyID: KeyID(pub), Algorithm: "ed25519", PublicKey: base64.StdEncoding.EncodeToString(pub)}
+}
+
+// Decode returns the raw key after checking the id matches.
+func (k PublicKey) Decode() (ed25519.PublicKey, error) {
+ if k.Algorithm != "ed25519" {
+ return nil, fmt.Errorf("key %s: unsupported algorithm %q", k.KeyID, k.Algorithm)
+ }
+ raw, err := base64.StdEncoding.DecodeString(k.PublicKey)
+ if err != nil || len(raw) != ed25519.PublicKeySize || KeyID(raw) != k.KeyID {
+ return nil, fmt.Errorf("key %s: not the 32-byte Ed25519 key of its id", k.KeyID)
+ }
+ return raw, nil
+}
+
+// KeySet is the list of online keys an offline root allows to sign.
+type KeySet struct {
+ Kind string `json:"kind"`
+ Version uint64 `json:"version"`
+ IssuedAt time.Time `json:"issued_at"`
+ NotAfter time.Time `json:"not_after"`
+ Keys []PublicKey `json:"keys"`
+ RootKeyID string `json:"root_keyid"`
+ RootPublicKey string `json:"root_public_key"`
+}
+
+func (ks *KeySet) check() error {
+ switch {
+ case ks.Kind != KeySetKind:
+ return fmt.Errorf("key set: kind %q", ks.Kind)
+ case ks.Version == 0:
+ return errors.New("key set: version must be 1 or more")
+ case ks.IssuedAt.IsZero() || !ks.NotAfter.After(ks.IssuedAt) || ks.NotAfter.Sub(ks.IssuedAt) > MaxKeySetValidity:
+ return fmt.Errorf("key set: validity must be positive and at most %s", MaxKeySetValidity)
+ case len(ks.Keys) == 0 || len(ks.Keys) > MaxKeySetKeys:
+ return fmt.Errorf("key set: 1 to %d keys", MaxKeySetKeys)
+ }
+ root, err := base64.StdEncoding.DecodeString(ks.RootPublicKey)
+ if err != nil || len(root) != ed25519.PublicKeySize || KeyID(root) != ks.RootKeyID {
+ return errors.New("key set: root_public_key is not the key of root_keyid")
+ }
+ seen := map[string]bool{}
+ for _, k := range ks.Keys {
+ if _, err := k.Decode(); err != nil {
+ return fmt.Errorf("key set: %w", err)
+ }
+ if seen[k.KeyID] || k.KeyID == ks.RootKeyID {
+ return fmt.Errorf("key set: key %s listed twice or is the root", k.KeyID)
+ }
+ seen[k.KeyID] = true
+ }
+ return nil
+}
+
+// SignKeySet signs a key set with the offline root.
+func SignKeySet(root ed25519.PrivateKey, version uint64, issuedAt time.Time, validity time.Duration, keys []ed25519.PublicKey) ([]byte, error) {
+ if len(root) != ed25519.PrivateKeySize {
+ return nil, errors.New("key set: an Ed25519 root key is required")
+ }
+ rootPub, _ := root.Public().(ed25519.PublicKey)
+ issuedAt = issuedAt.UTC().Truncate(time.Second)
+ ks := KeySet{Kind: KeySetKind, Version: version, IssuedAt: issuedAt, NotAfter: issuedAt.Add(validity),
+ RootKeyID: KeyID(rootPub), RootPublicKey: base64.StdEncoding.EncodeToString(rootPub)}
+ for _, k := range keys {
+ ks.Keys = append(ks.Keys, NewPublicKey(k))
+ }
+ if err := ks.check(); err != nil {
+ return nil, err
+ }
+ payload, err := json.Marshal(ks)
+ if err != nil {
+ return nil, err
+ }
+ return Sign(root, KeySetPayloadType, payload)
+}
+
+// VerifyKeySet checks a key set: signed by its root, that root is the pinned
+// one, valid now, and its version is not below minVersion.
+func VerifyKeySet(envelope []byte, pinnedRoot string, minVersion uint64, now time.Time) (*KeySet, error) {
+ env, err := Open(envelope, KeySetPayloadType, MaxKeySetBytes)
+ if err != nil {
+ return nil, err
+ }
+ var ks KeySet
+ if err := DecodeStrict(env.Payload, &ks); err != nil {
+ return nil, fmt.Errorf("key set: %w", err)
+ }
+ if err := ks.check(); err != nil {
+ return nil, err
+ }
+ if pinnedRoot == "" || ks.RootKeyID != pinnedRoot {
+ return nil, fmt.Errorf("key set: signed by root %s, the pinned root is %q", ks.RootKeyID, pinnedRoot)
+ }
+ root, _ := base64.StdEncoding.DecodeString(ks.RootPublicKey)
+ if !env.VerifiedBy(root) {
+ return nil, errors.New("key set: bad root signature")
+ }
+ if ks.Version < minVersion {
+ return nil, fmt.Errorf("key set: version %d is below %d (rolled back)", ks.Version, minVersion)
+ }
+ if ks.IssuedAt.After(now.Add(maxClockSkew)) || !ks.NotAfter.After(now.Add(-maxClockSkew)) {
+ return nil, fmt.Errorf("key set: not valid at %s (valid %s to %s)", now.UTC().Format(time.RFC3339),
+ ks.IssuedAt.Format(time.RFC3339), ks.NotAfter.Format(time.RFC3339))
+ }
+ return &ks, nil
+}
+
+// VerifyWith checks envelope against the key set and returns its payload.
+func VerifyWith(ks *KeySet, envelope []byte, payloadType string, maxBytes int) ([]byte, error) {
+ env, err := Open(envelope, payloadType, maxBytes)
+ if err != nil {
+ return nil, err
+ }
+ for _, k := range ks.Keys {
+ pub, err := k.Decode()
+ if err == nil && env.VerifiedBy(pub) {
+ return env.Payload, nil
+ }
+ }
+ return nil, errors.New("dsse: not signed by a key of the key set")
+}
diff --git a/internal/dsse/dsse_test.go b/internal/dsse/dsse_test.go
new file mode 100644
index 0000000..7df7c93
--- /dev/null
+++ b/internal/dsse/dsse_test.go
@@ -0,0 +1,124 @@
+package dsse
+
+import (
+ "crypto/ed25519"
+ "encoding/json"
+ "strings"
+ "testing"
+ "time"
+)
+
+func key(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) {
+ t.Helper()
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return pub, priv
+}
+
+func TestPAEAndEnvelopeFormat(t *testing.T) {
+ if got := string(PAE("t/x", []byte("hello"))); got != "DSSEv1 3 t/x 5 hello" {
+ t.Fatalf("PAE = %q", got)
+ }
+ _, priv := key(t)
+ env, err := Sign(priv, "t/x", []byte(`{"a":1}`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var raw map[string]any
+ if err := json.Unmarshal(env, &raw); err != nil {
+ t.Fatal(err)
+ }
+ sigs, _ := raw["signatures"].([]any)
+ sig, _ := sigs[0].(map[string]any)
+ if raw["payloadType"] != "t/x" || raw["payload"] != "eyJhIjoxfQ==" || !strings.HasPrefix(sig["keyid"].(string), "SHA256:") || sig["sig"] == nil {
+ t.Fatalf("envelope %s", env)
+ }
+}
+
+func TestKeySetAndVerify(t *testing.T) {
+ rootPub, root := key(t)
+ pub, priv := key(t)
+ now := time.Now()
+ ksEnv, err := SignKeySet(root, 3, now, 90*24*time.Hour, []ed25519.PublicKey{pub})
+ if err != nil {
+ t.Fatal(err)
+ }
+ ks, err := VerifyKeySet(ksEnv, KeyID(rootPub), 2, now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ env, _ := Sign(priv, "t/x", []byte("payload"))
+ if got, err := VerifyWith(ks, env, "t/x", 1<<10); err != nil || string(got) != "payload" {
+ t.Fatalf("VerifyWith: %q %v", got, err)
+ }
+
+ otherPub, other := key(t)
+ _ = otherPub
+ forged, _ := Sign(other, "t/x", []byte("payload"))
+ cases := map[string]func() error{
+ "another root pinned": func() error { _, err := VerifyKeySet(ksEnv, KeyID(otherPub), 0, now); return err },
+ "no root pinned": func() error { _, err := VerifyKeySet(ksEnv, "", 0, now); return err },
+ "rolled back": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 4, now); return err },
+ "expired": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(91*24*time.Hour)); return err },
+ "not yet valid": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(-time.Hour)); return err },
+ "unknown signer": func() error { _, err := VerifyWith(ks, forged, "t/x", 1<<10); return err },
+ "wrong type": func() error { _, err := VerifyWith(ks, env, "t/y", 1<<10); return err },
+ "too big": func() error { _, err := VerifyWith(ks, env, "t/x", 10); return err },
+ "tampered payload": func() error {
+ var e Envelope
+ _ = json.Unmarshal(env, &e)
+ e.Payload = []byte("payloae")
+ b, _ := json.Marshal(e)
+ _, err := VerifyWith(ks, b, "t/x", 1<<10)
+ return err
+ },
+ "key set signed by an online key": func() error {
+ bad, _ := SignKeySet(priv, 1, now, time.Hour, []ed25519.PublicKey{otherPub})
+ _, err := VerifyKeySet(bad, KeyID(rootPub), 0, now)
+ return err
+ },
+ }
+ for name, fn := range cases {
+ if fn() == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+}
+
+func TestSignKeySetRefuses(t *testing.T) {
+ rootPub, root := key(t)
+ pub, _ := key(t)
+ now := time.Now()
+ for name, fn := range map[string]func() error{
+ "too long": func() error {
+ _, err := SignKeySet(root, 1, now, 181*24*time.Hour, []ed25519.PublicKey{pub})
+ return err
+ },
+ "version 0": func() error { _, err := SignKeySet(root, 0, now, time.Hour, []ed25519.PublicKey{pub}); return err },
+ "no keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, nil); return err },
+ "root as online": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{rootPub}); return err },
+ "duplicate keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{pub, pub}); return err },
+ "not a root key": func() error {
+ _, err := SignKeySet(ed25519.PrivateKey{1}, 1, now, time.Hour, []ed25519.PublicKey{pub})
+ return err
+ },
+ } {
+ if fn() == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+}
+
+func TestDecodeStrict(t *testing.T) {
+ var v struct{ A int }
+ if DecodeStrict([]byte(`{"A":1}`), &v) != nil || v.A != 1 {
+ t.Fatal("valid")
+ }
+ for _, s := range []string{`{"A":1,"B":2}`, `{"A":1} {}`, `[`} {
+ if DecodeStrict([]byte(s), &v) == nil {
+ t.Errorf("%s accepted", s)
+ }
+ }
+}
diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go
new file mode 100644
index 0000000..bdede14
--- /dev/null
+++ b/internal/fetch/fetch.go
@@ -0,0 +1,295 @@
+// Package fetch is the collector's only way to the network: a polite HTTP
+// client that names itself and its contact, honours robots.txt, spaces
+// requests to one host, makes conditional requests from an on-disk cache,
+// caps response sizes and refuses to connect to non-public addresses (a
+// seed list or a redirect must not point the collector at internal
+// services).
+package fetch
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/netip"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// Product is the robots.txt product token of the collector.
+const Product = "openctem-programfeed"
+
+// UserAgent names the collector, where it lives and who to contact.
+func UserAgent(version string) string {
+ return Product + "/" + version + " (+https://github.com/openctemio/programfeed; security@openctem.io)"
+}
+
+// Errors callers act on.
+var (
+ ErrDisallowed = errors.New("robots.txt disallows this URL")
+ ErrNotFound = errors.New("not found")
+ ErrTooLarge = errors.New("response over the size cap")
+)
+
+const maxRobotsBytes = 512 << 10
+
+// Options configure a client.
+type Options struct {
+ UserAgent string
+ CacheDir string // conditional requests; empty disables the cache
+ MinInterval time.Duration // between two requests to one host
+ Timeout time.Duration // per request
+ // AllowPrivate lets tests talk to httptest servers on loopback. Never
+ // set in the collector.
+ AllowPrivate bool
+ // DialOverride sends every connection to this address (tests only,
+ // with AllowPrivate).
+ DialOverride string
+}
+
+// Client is safe for concurrent use.
+type Client struct {
+ opt Options
+ http *http.Client
+
+ mu sync.Mutex
+ next map[string]time.Time // host -> earliest next request
+ robots map[string]*robotsRules
+}
+
+// Response is a fetched body.
+type Response struct {
+ URL string // final URL after redirects
+ Body []byte
+ FetchedAt time.Time
+ Cached bool // 304: the body is the cached one
+}
+
+// New returns a client.
+func New(opt Options) *Client {
+ if opt.MinInterval == 0 {
+ opt.MinInterval = time.Second
+ }
+ if opt.Timeout == 0 {
+ opt.Timeout = 30 * time.Second
+ }
+ c := &Client{opt: opt, next: map[string]time.Time{}, robots: map[string]*robotsRules{}}
+ dialer := &net.Dialer{Timeout: 10 * time.Second, Control: func(_, address string, _ syscall.RawConn) error {
+ if opt.AllowPrivate {
+ return nil
+ }
+ ap, err := netip.ParseAddrPort(address)
+ if err != nil || !model.PublicAddr(ap.Addr()) {
+ return fmt.Errorf("refusing to connect to non-public address %s", address)
+ }
+ return nil
+ }}
+ dial := dialer.DialContext
+ if opt.DialOverride != "" && opt.AllowPrivate {
+ dial = func(ctx context.Context, network, _ string) (net.Conn, error) {
+ return dialer.DialContext(ctx, network, opt.DialOverride)
+ }
+ }
+ transport := &http.Transport{
+ Proxy: nil,
+ DialContext: dial,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ResponseHeaderTimeout: opt.Timeout,
+ MaxIdleConnsPerHost: 2,
+ IdleConnTimeout: 30 * time.Second,
+ }
+ c.http = &http.Client{Transport: transport, Timeout: opt.Timeout,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= 3 {
+ return errors.New("more than 3 redirects")
+ }
+ if req.URL.Scheme != "https" && !(opt.AllowPrivate && req.URL.Scheme == "http") {
+ return errors.New("redirect to a non-https URL")
+ }
+ // A robots.txt fetch is not itself subject to robots.txt (a
+ // site that redirects everything would otherwise recurse).
+ if req.Context().Value(robotsFetch{}) == nil {
+ if ok, err := c.allowed(req.Context(), req.URL); err != nil || !ok {
+ return ErrDisallowed
+ }
+ }
+ return c.wait(req.Context(), req.URL.Host)
+ }}
+ return c
+}
+
+// Get fetches rawURL (at most maxBytes) after checking robots.txt.
+func (c *Client) Get(ctx context.Context, rawURL string, maxBytes int64) (*Response, error) {
+ u, err := url.Parse(rawURL)
+ if err != nil || (u.Scheme != "https" && !(c.opt.AllowPrivate && u.Scheme == "http")) || u.Host == "" || u.User != nil {
+ return nil, fmt.Errorf("fetch: refusing URL %q", rawURL)
+ }
+ ok, err := c.allowed(ctx, u)
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, fmt.Errorf("%s: %w", rawURL, ErrDisallowed)
+ }
+ return c.get(ctx, u, maxBytes, true)
+}
+
+type cacheEntry struct {
+ URL string `json:"url"`
+ ETag string `json:"etag,omitempty"`
+ LastModified string `json:"last_modified,omitempty"`
+ Body []byte `json:"body"`
+}
+
+func (c *Client) cachePath(u string) string {
+ if c.opt.CacheDir == "" {
+ return ""
+ }
+ sum := sha256.Sum256([]byte(u))
+ return filepath.Join(c.opt.CacheDir, hex.EncodeToString(sum[:])+".json")
+}
+
+func (c *Client) get(ctx context.Context, u *url.URL, maxBytes int64, useCache bool) (*Response, error) {
+ if err := c.wait(ctx, u.Host); err != nil {
+ return nil, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", c.opt.UserAgent)
+ req.Header.Set("From", "security@openctem.io")
+ var cached *cacheEntry
+ path := ""
+ if useCache {
+ path = c.cachePath(u.String())
+ }
+ if path != "" {
+ if b, err := os.ReadFile(path); err == nil {
+ var e cacheEntry
+ if json.Unmarshal(b, &e) == nil && e.URL == u.String() && int64(len(e.Body)) <= maxBytes {
+ cached = &e
+ if e.ETag != "" {
+ req.Header.Set("If-None-Match", e.ETag)
+ }
+ if e.LastModified != "" {
+ req.Header.Set("If-Modified-Since", e.LastModified)
+ }
+ }
+ }
+ }
+ resp, err := c.http.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ now := time.Now().UTC().Truncate(time.Second)
+ switch {
+ case resp.StatusCode == http.StatusNotModified && cached != nil:
+ return &Response{URL: u.String(), Body: cached.Body, FetchedAt: now, Cached: true}, nil
+ case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone:
+ return nil, fmt.Errorf("%s: %w", u, ErrNotFound)
+ case resp.StatusCode != http.StatusOK:
+ return nil, &StatusError{URL: u.String(), Code: resp.StatusCode}
+ }
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(body)) > maxBytes {
+ return nil, fmt.Errorf("%s: %w (%d bytes)", u, ErrTooLarge, maxBytes)
+ }
+ if path != "" && (resp.Header.Get("ETag") != "" || resp.Header.Get("Last-Modified") != "") {
+ e := cacheEntry{URL: u.String(), ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), Body: body}
+ if b, err := json.Marshal(e); err == nil {
+ _ = os.MkdirAll(c.opt.CacheDir, 0o755)
+ _ = os.WriteFile(path, b, 0o644)
+ }
+ }
+ return &Response{URL: resp.Request.URL.String(), Body: body, FetchedAt: now}, nil
+}
+
+// wait blocks until a request to host is polite.
+func (c *Client) wait(ctx context.Context, host string) error {
+ c.mu.Lock()
+ now := time.Now()
+ at := c.next[host]
+ if at.Before(now) {
+ at = now
+ }
+ c.next[host] = at.Add(c.opt.MinInterval)
+ c.mu.Unlock()
+ if d := time.Until(at); d > 0 {
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-t.C:
+ }
+ }
+ return nil
+}
+
+// allowed checks robots.txt of u's origin (fetched once per run). A missing
+// robots.txt (4xx) allows everything; an unreachable one (network error,
+// 5xx) disallows everything, as RFC 9309 asks.
+func (c *Client) allowed(ctx context.Context, u *url.URL) (bool, error) {
+ origin := u.Scheme + "://" + u.Host
+ c.mu.Lock()
+ r, ok := c.robots[origin]
+ c.mu.Unlock()
+ if !ok {
+ ru, _ := url.Parse(origin + "/robots.txt")
+ resp, err := c.get(context.WithValue(ctx, robotsFetch{}, true), ru, maxRobotsBytes, false)
+ switch {
+ case err == nil:
+ r = parseRobots(string(resp.Body), Product)
+ case errors.Is(err, ErrNotFound) || isStatus(err, 400, 499):
+ r = &robotsRules{}
+ case errors.Is(err, ErrTooLarge):
+ r = &robotsRules{}
+ case ctx.Err() != nil:
+ return false, ctx.Err()
+ default:
+ r = &robotsRules{disallowAll: true}
+ }
+ c.mu.Lock()
+ c.robots[origin] = r
+ c.mu.Unlock()
+ }
+ path := u.EscapedPath()
+ if path == "" {
+ path = "/"
+ }
+ if u.RawQuery != "" {
+ path += "?" + u.RawQuery
+ }
+ return r.allows(path), nil
+}
+
+type robotsFetch struct{}
+
+// StatusError is an unexpected HTTP status.
+type StatusError struct {
+ URL string
+ Code int
+}
+
+func (e *StatusError) Error() string { return fmt.Sprintf("%s: HTTP %d", e.URL, e.Code) }
+
+func isStatus(err error, lo, hi int) bool {
+ var se *StatusError
+ return errors.As(err, &se) && se.Code >= lo && se.Code <= hi
+}
diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go
new file mode 100644
index 0000000..e575f47
--- /dev/null
+++ b/internal/fetch/fetch_test.go
@@ -0,0 +1,154 @@
+package fetch
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestRobots(t *testing.T) {
+ body := `
+User-agent: *
+Disallow: /private
+Allow: /private/ok
+
+User-agent: openctem-programfeed
+User-agent: other
+Disallow: /mine/*.json$
+Allow: /
+
+User-agent: badbot
+Disallow: /
+`
+ r := parseRobots(body, Product)
+ cases := map[string]bool{
+ "/": true,
+ "/private": true, // our group applies, not "*"
+ "/mine/a.json": false,
+ "/mine/a.json?x": true,
+ "/mine/dir/b.txt": true,
+ }
+ for p, want := range cases {
+ if got := r.allows(p); got != want {
+ t.Errorf("%s: %v, want %v", p, got, want)
+ }
+ }
+ star := parseRobots(body, "someone-else")
+ if star.allows("/private/x") || !star.allows("/private/ok/1") || !star.allows("/public") {
+ t.Error("star group rules")
+ }
+ if !parseRobots("", Product).allows("/anything") {
+ t.Error("empty robots.txt must allow")
+ }
+}
+
+func newClient(cacheDir string) *Client {
+ return New(Options{UserAgent: UserAgent("test"), CacheDir: cacheDir, MinInterval: time.Millisecond, AllowPrivate: true})
+}
+
+func TestGetPoliteAndConditional(t *testing.T) {
+ var hits, notModified atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/robots.txt":
+ _, _ = w.Write([]byte("User-agent: *\nDisallow: /secret\n"))
+ case "/data.json":
+ hits.Add(1)
+ if !strings.Contains(r.UserAgent(), "security@openctem.io") || r.Header.Get("From") != "security@openctem.io" {
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ if r.Header.Get("If-None-Match") == `"v1"` {
+ notModified.Add(1)
+ w.WriteHeader(http.StatusNotModified)
+ return
+ }
+ w.Header().Set("ETag", `"v1"`)
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ case "/big":
+ _, _ = w.Write([]byte(strings.Repeat("x", 2048)))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer srv.Close()
+ ctx := context.Background()
+ c := newClient(t.TempDir())
+ for i := 0; i < 2; i++ {
+ r, err := c.Get(ctx, srv.URL+"/data.json", 1024)
+ if err != nil || string(r.Body) != `{"ok":true}` || r.Cached != (i == 1) {
+ t.Fatalf("get %d: %v %+v", i, err, r)
+ }
+ }
+ if hits.Load() != 2 || notModified.Load() != 1 {
+ t.Errorf("hits %d, 304s %d", hits.Load(), notModified.Load())
+ }
+ if _, err := c.Get(ctx, srv.URL+"/secret/x", 1024); !errors.Is(err, ErrDisallowed) {
+ t.Errorf("robots: %v", err)
+ }
+ if _, err := c.Get(ctx, srv.URL+"/big", 1024); !errors.Is(err, ErrTooLarge) {
+ t.Errorf("cap: %v", err)
+ }
+ if _, err := c.Get(ctx, srv.URL+"/nope", 1024); !errors.Is(err, ErrNotFound) {
+ t.Errorf("404: %v", err)
+ }
+}
+
+func TestRefusesPrivateAddresses(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("hi")) }))
+ defer srv.Close()
+ // The collector's client (AllowPrivate false) must not reach loopback,
+ // whatever the URL says; it also refuses plain http.
+ c := New(Options{UserAgent: UserAgent("test"), MinInterval: time.Millisecond})
+ if _, err := c.Get(context.Background(), srv.URL+"/", 1024); err == nil {
+ t.Fatal("fetched a loopback URL")
+ }
+ httpsURL := strings.Replace(srv.URL, "http://", "https://", 1)
+ if _, err := c.Get(context.Background(), httpsURL+"/", 1024); err == nil {
+ t.Fatal("connected to a loopback address")
+ }
+ if _, err := c.Get(context.Background(), "https://user:pw@openctem.io/", 1024); err == nil {
+ t.Fatal("accepted credentials in URL")
+ }
+}
+
+func TestUnreachableRobotsDisallows(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/robots.txt" {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+ _, _ = w.Write([]byte("data"))
+ }))
+ defer srv.Close()
+ if _, err := newClient("").Get(context.Background(), srv.URL+"/x", 1024); !errors.Is(err, ErrDisallowed) {
+ t.Fatalf("5xx robots.txt: %v", err)
+ }
+}
+
+// A site that redirects every path (robots.txt included) to its home page
+// must not make the robots.txt check recurse.
+func TestRedirectEverythingTerminates(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/home" {
+ _, _ = w.Write([]byte("home"))
+ return
+ }
+ http.Redirect(w, r, "/home", http.StatusFound)
+ }))
+ defer srv.Close()
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ r, err := newClient("").Get(ctx, srv.URL+"/.well-known/security.txt", 1024)
+ if ctx.Err() != nil {
+ t.Fatal("robots.txt check recursed")
+ }
+ if err != nil || r.URL != srv.URL+"/home" {
+ t.Fatalf("%v %+v", err, r)
+ }
+}
diff --git a/internal/fetch/robots.go b/internal/fetch/robots.go
new file mode 100644
index 0000000..e7ee102
--- /dev/null
+++ b/internal/fetch/robots.go
@@ -0,0 +1,116 @@
+package fetch
+
+import (
+ "strings"
+)
+
+// robotsRules is the group of a robots.txt that applies to the collector
+// (RFC 9309): the group naming our product token, else the "*" group.
+type robotsRules struct {
+ disallowAll bool
+ rules []robotsRule
+}
+
+type robotsRule struct {
+ allow bool
+ pattern string
+}
+
+func parseRobots(body, product string) *robotsRules {
+ product = strings.ToLower(product)
+ type group struct {
+ agents []string
+ rules []robotsRule
+ }
+ var groups []*group
+ var cur *group
+ lastWasAgent := false
+ for _, line := range strings.Split(body, "\n") {
+ if i := strings.IndexByte(line, '#'); i >= 0 {
+ line = line[:i]
+ }
+ k, v, ok := strings.Cut(line, ":")
+ if !ok {
+ continue
+ }
+ k, v = strings.ToLower(strings.TrimSpace(k)), strings.TrimSpace(v)
+ switch k {
+ case "user-agent":
+ if cur == nil || !lastWasAgent {
+ cur = &group{}
+ groups = append(groups, cur)
+ }
+ cur.agents = append(cur.agents, strings.ToLower(v))
+ lastWasAgent = true
+ case "allow", "disallow":
+ lastWasAgent = false
+ if cur == nil || v == "" {
+ continue
+ }
+ cur.rules = append(cur.rules, robotsRule{allow: k == "allow", pattern: v})
+ default:
+ lastWasAgent = false
+ }
+ }
+ var mine, star []robotsRule
+ matched := false
+ for _, g := range groups {
+ for _, a := range g.agents {
+ switch {
+ case a == product:
+ mine = append(mine, g.rules...)
+ matched = true
+ case a == "*":
+ star = append(star, g.rules...)
+ }
+ }
+ }
+ if matched {
+ return &robotsRules{rules: mine}
+ }
+ return &robotsRules{rules: star}
+}
+
+// allows applies the longest matching rule; on a tie, allow wins.
+func (r *robotsRules) allows(path string) bool {
+ if r.disallowAll {
+ return false
+ }
+ best, allow := -1, true
+ for _, rule := range r.rules {
+ if !match(rule.pattern, path) {
+ continue
+ }
+ n := len(rule.pattern)
+ if n > best || (n == best && rule.allow) {
+ best, allow = n, rule.allow
+ }
+ }
+ return allow
+}
+
+// match implements robots.txt patterns: "*" matches any run, a trailing
+// "$" anchors the end, otherwise a prefix match.
+func match(pattern, path string) bool {
+ anchored := strings.HasSuffix(pattern, "$")
+ pattern = strings.TrimSuffix(pattern, "$")
+ parts := strings.Split(pattern, "*")
+ if !strings.HasPrefix(path, parts[0]) {
+ return false
+ }
+ pos := len(parts[0])
+ for i, part := range parts[1:] {
+ if i == len(parts)-2 && anchored {
+ return len(path)-pos >= len(part) && strings.HasSuffix(path, part)
+ }
+ j := strings.Index(path[pos:], part)
+ if j < 0 {
+ return false
+ }
+ pos += j + len(part)
+ }
+ if anchored {
+ return pos == len(path)
+ }
+ return true
+}
diff --git a/internal/model/model.go b/internal/model/model.go
new file mode 100644
index 0000000..23c41e9
--- /dev/null
+++ b/internal/model/model.go
@@ -0,0 +1,409 @@
+// Package model is the feed's record model and its validation.
+//
+// The rules are the ones the OpenCTEM importer applies: a bundle with one
+// record the importer would refuse is refused whole, so the collector
+// refuses to publish it first. Targets are different: a target the scope
+// rules refuse (a public suffix, a private range, a huge CIDR …) does not
+// fail the bundle; it is moved to the program's rejected list with the
+// reason, so subscribers can see what a program claimed and why the feed
+// did not pass it on.
+package model
+
+import (
+ "errors"
+ "fmt"
+ "net/mail"
+ "net/url"
+ "regexp"
+ "sort"
+ "strings"
+ "time"
+ "unicode"
+ "unicode/utf8"
+)
+
+// Limits (the importer applies the same).
+const (
+ MaxNameLen = 200
+ MaxTextLen = 4000
+ MaxURLLen = 2048
+ MaxValueLen = 512
+ MaxTargets = 2000 // per list (in scope, out of scope, rejected)
+ MaxListItems = 32 // testing restrictions, required headers, languages
+ MaxPrograms = 100000
+ MaxChangeFields = 16
+)
+
+// Program types and statuses.
+const (
+ TypeBounty = "bounty"
+ TypeVDP = "vdp"
+
+ StatusOpen = "open"
+ StatusPaused = "paused"
+ StatusClosed = "closed"
+)
+
+// Safe-harbour values.
+const (
+ SafeHarbourFull = "full"
+ SafeHarbourPartial = "partial"
+ SafeHarbourNone = "none"
+ SafeHarbourUnknown = "unknown"
+)
+
+// Target types.
+const (
+ TargetDomain = "domain"
+ TargetWildcard = "wildcard"
+ TargetIP = "ip"
+ TargetCIDR = "cidr"
+ TargetURL = "url"
+ TargetMobileApp = "mobile_app"
+ TargetSourceRepo = "source_repo"
+ TargetOther = "other"
+)
+
+// Target confidence: published = the program's own structured scope;
+// inferred = derived by the collector (for example the host a program's
+// security.txt was served from). An inferred target is a suggestion, never
+// an authorisation to test.
+const (
+ ConfidencePublished = "published"
+ ConfidenceInferred = "inferred"
+)
+
+var (
+ idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$`)
+ sha256RE = regexp.MustCompile(`^[0-9a-f]{64}$`)
+ severities = map[string]bool{"": true, "none": true, "low": true, "medium": true, "high": true, "critical": true}
+ types = map[string]bool{TypeBounty: true, TypeVDP: true}
+ statuses = map[string]bool{StatusOpen: true, StatusPaused: true, StatusClosed: true}
+ harbours = map[string]bool{SafeHarbourFull: true, SafeHarbourPartial: true, SafeHarbourNone: true, SafeHarbourUnknown: true}
+ targetType = map[string]bool{TargetDomain: true, TargetWildcard: true, TargetIP: true, TargetCIDR: true, TargetURL: true,
+ TargetMobileApp: true, TargetSourceRepo: true, TargetOther: true}
+ confidences = map[string]bool{ConfidencePublished: true, ConfidenceInferred: true}
+)
+
+// Program is one public bug-bounty or vulnerability-disclosure program.
+type Program struct {
+ ID string `json:"id"` // ":", stable, owned by one source
+ Source string `json:"source"` // the adapter that produced it
+ Platform string `json:"platform"` // where the program is hosted ("self-hosted" or a platform name)
+ Name string `json:"name"`
+ URL string `json:"url"` // the program's policy page
+ Type string `json:"type"`
+ Status string `json:"status"`
+ OffersBounty bool `json:"offers_bounty"`
+ ScopePublished bool `json:"scope_published"` // the source carries the program's own structured scope
+ InScope []Target `json:"in_scope"`
+ OutOfScope []Target `json:"out_of_scope"`
+ Rejected []Rejected `json:"rejected"`
+ Rules Rules `json:"rules"`
+ Terms Terms `json:"terms"`
+ Contact Contact `json:"contact"`
+ Provenance Provenance `json:"provenance"`
+ FirstSeen time.Time `json:"first_seen"`
+ LastSeen time.Time `json:"last_seen"`
+ LastChanged time.Time `json:"last_changed"`
+ ClosedAt *time.Time `json:"closed_at,omitempty"`
+}
+
+// Target is one scope entry.
+type Target struct {
+ Type string `json:"type"`
+ Value string `json:"value"`
+ Confidence string `json:"confidence"`
+ Eligible *bool `json:"eligible_for_bounty,omitempty"`
+ MaxSev string `json:"max_severity,omitempty"`
+ Notes string `json:"notes,omitempty"`
+}
+
+// Rejected is a target the scope rules refused, with the reason.
+type Rejected struct {
+ Scope string `json:"scope"` // in | out
+ Type string `json:"type"`
+ Value string `json:"value"`
+ Reason string `json:"reason"`
+}
+
+// Rules summarises how a program may be tested.
+type Rules struct {
+ Summary string `json:"summary,omitempty"`
+ TestingRestrictions []string `json:"testing_restrictions"`
+ RateLimit string `json:"rate_limit,omitempty"`
+ RequiredHeaders []string `json:"required_headers"`
+ SafeHarbour string `json:"safe_harbour"`
+ DisclosureDays int `json:"disclosure_days,omitempty"`
+ Languages []string `json:"languages"`
+}
+
+// Terms names the program's terms and, when the collector read them, the
+// SHA-256 of the exact bytes it read.
+type Terms struct {
+ URL string `json:"url"`
+ SHA256 string `json:"sha256,omitempty"`
+ FetchedAt *time.Time `json:"fetched_at,omitempty"`
+}
+
+// Contact is the program's reporting channel.
+type Contact struct {
+ Email string `json:"email,omitempty"`
+ URL string `json:"url,omitempty"`
+}
+
+// Provenance records where and when a record was read.
+type Provenance struct {
+ Source string `json:"source"`
+ SourceURL string `json:"source_url"`
+ FetchedAt time.Time `json:"fetched_at"`
+}
+
+// Validate checks a normalised program.
+func (p *Program) Validate() error {
+ fail := func(format string, args ...any) error {
+ return fmt.Errorf("program %q: %s", p.ID, fmt.Sprintf(format, args...))
+ }
+ switch {
+ case !idRE.MatchString(p.ID):
+ return fail("id must be : in lower case")
+ case !strings.HasPrefix(p.ID, p.Source+":"):
+ return fail("id is not under its source %q", p.Source)
+ case p.Provenance.Source != p.Source:
+ return fail("provenance source %q", p.Provenance.Source)
+ case !safeText(p.Name, MaxNameLen) || strings.TrimSpace(p.Name) == "":
+ return fail("name is empty, too long or has control characters")
+ case !safeText(p.Platform, 64) || p.Platform == "":
+ return fail("platform")
+ case !types[p.Type]:
+ return fail("type %q", p.Type)
+ case !statuses[p.Status]:
+ return fail("status %q", p.Status)
+ case (p.Status == StatusClosed) != (p.ClosedAt != nil):
+ return fail("closed_at must be set exactly when the status is closed")
+ case p.OffersBounty != (p.Type == TypeBounty):
+ return fail("offers_bounty must match the type")
+ case p.FirstSeen.IsZero() || p.LastSeen.Before(p.FirstSeen) || p.LastChanged.Before(p.FirstSeen):
+ return fail("first_seen must not be after last_seen or last_changed")
+ case p.Provenance.FetchedAt.IsZero():
+ return fail("provenance fetched_at")
+ }
+ if err := checkURL(p.URL, true); err != nil {
+ return fail("url: %v", err)
+ }
+ if err := checkURL(p.Provenance.SourceURL, true); err != nil {
+ return fail("provenance source_url: %v", err)
+ }
+ if err := checkURL(p.Terms.URL, true); err != nil {
+ return fail("terms url: %v", err)
+ }
+ if p.Terms.SHA256 != "" && !sha256RE.MatchString(p.Terms.SHA256) {
+ return fail("terms sha256")
+ }
+ if (p.Terms.SHA256 == "") != (p.Terms.FetchedAt == nil) {
+ return fail("terms sha256 and fetched_at go together")
+ }
+ if p.Contact.Email != "" {
+ if a, err := mail.ParseAddress(p.Contact.Email); err != nil || a.Address != p.Contact.Email || len(p.Contact.Email) > 254 {
+ return fail("contact email")
+ }
+ }
+ if p.Contact.URL != "" {
+ if err := checkURL(p.Contact.URL, false); err != nil {
+ return fail("contact url: %v", err)
+ }
+ }
+ if err := p.Rules.validate(); err != nil {
+ return fail("%v", err)
+ }
+ for _, list := range []struct {
+ name string
+ ts []Target
+ }{{"in_scope", p.InScope}, {"out_of_scope", p.OutOfScope}} {
+ if list.ts == nil {
+ return fail("%s must be a list", list.name)
+ }
+ if len(list.ts) > MaxTargets {
+ return fail("%s has more than %d targets", list.name, MaxTargets)
+ }
+ for i, t := range list.ts {
+ if err := t.Validate(); err != nil {
+ return fail("%s: %v", list.name, err)
+ }
+ if i > 0 && !targetLess(list.ts[i-1], t) {
+ return fail("%s is not sorted or has duplicates", list.name)
+ }
+ }
+ }
+ if p.Rejected == nil || len(p.Rejected) > MaxTargets {
+ return fail("rejected must be a list of at most %d", MaxTargets)
+ }
+ for _, r := range p.Rejected {
+ if (r.Scope != "in" && r.Scope != "out") || !safeText(r.Type, 32) || !safeText(r.Value, MaxValueLen) || !safeText(r.Reason, 200) || r.Reason == "" {
+ return fail("rejected entry %q", r.Value)
+ }
+ }
+ return nil
+}
+
+func (r Rules) validate() error {
+ if !safeText(r.Summary, MaxTextLen) || !safeText(r.RateLimit, 200) {
+ return errors.New("rules: summary or rate_limit too long or has control characters")
+ }
+ if !harbours[r.SafeHarbour] {
+ return fmt.Errorf("rules: safe_harbour %q", r.SafeHarbour)
+ }
+ if r.DisclosureDays < 0 || r.DisclosureDays > 3650 {
+ return errors.New("rules: disclosure_days")
+ }
+ for name, l := range map[string][]string{"testing_restrictions": r.TestingRestrictions, "required_headers": r.RequiredHeaders, "languages": r.Languages} {
+ if l == nil || len(l) > MaxListItems {
+ return fmt.Errorf("rules: %s must be a list of at most %d", name, MaxListItems)
+ }
+ for _, s := range l {
+ if s == "" || !safeText(s, 500) {
+ return fmt.Errorf("rules: %s entry", name)
+ }
+ }
+ }
+ return nil
+}
+
+// Validate checks a normalised target (Normalize produces only valid ones).
+func (t Target) Validate() error {
+ if !targetType[t.Type] || !confidences[t.Confidence] || !severities[t.MaxSev] || !safeText(t.Notes, 500) {
+ return fmt.Errorf("target %q: type, confidence, max_severity or notes", t.Value)
+ }
+ n, reason := NormalizeTarget(t.Type, t.Value)
+ if reason != "" || n != t.Value {
+ return fmt.Errorf("target %q is not a normalised %s (%s)", t.Value, t.Type, reason)
+ }
+ return nil
+}
+
+func targetLess(a, b Target) bool {
+ if a.Type != b.Type {
+ return a.Type < b.Type
+ }
+ return a.Value < b.Value
+}
+
+// SortTargets orders targets canonically and drops exact duplicates
+// (same type and value; the first wins).
+func SortTargets(ts []Target) []Target {
+ sort.SliceStable(ts, func(i, j int) bool { return targetLess(ts[i], ts[j]) })
+ out := ts[:0]
+ for i, t := range ts {
+ if i > 0 && t.Type == ts[i-1].Type && t.Value == ts[i-1].Value {
+ continue
+ }
+ out = append(out, t)
+ }
+ return out
+}
+
+// safeText: valid UTF-8, at most max bytes, no control characters other
+// than newline and tab, no bidi overrides.
+func safeText(s string, max int) bool {
+ if len(s) > max || !utf8.ValidString(s) {
+ return false
+ }
+ for _, r := range s {
+ if (unicode.IsControl(r) && r != '\n' && r != '\t') || (r >= 0x202A && r <= 0x202E) || (r >= 0x2066 && r <= 0x2069) {
+ return false
+ }
+ }
+ return true
+}
+
+// checkURL accepts absolute http(s) URLs without credentials; mailto is
+// allowed where links may be mail links.
+func checkURL(s string, httpOnly bool) error {
+ if s == "" || len(s) > MaxURLLen || !safeText(s, MaxURLLen) || strings.ContainsAny(s, " \n\t") {
+ return errors.New("empty, too long or has spaces/control characters")
+ }
+ u, err := url.Parse(s)
+ if err != nil {
+ return err
+ }
+ switch {
+ case u.Scheme == "mailto" && !httpOnly:
+ return nil
+ case u.Scheme != "https" && u.Scheme != "http":
+ return fmt.Errorf("scheme %q", u.Scheme)
+ case u.User != nil:
+ return errors.New("credentials in URL")
+ case u.Host == "":
+ return errors.New("no host")
+ }
+ return nil
+}
+
+// CleanText trims a source string, folds whitespace runs and cuts it to max
+// bytes on a rune boundary; it returns "" for text with control
+// characters.
+func CleanText(s string, max int) string {
+ s = strings.Join(strings.Fields(s), " ")
+ if !safeText(s, len(s)) {
+ return ""
+ }
+ if len(s) <= max {
+ return s
+ }
+ cut := max
+ for cut > 0 && !utf8.RuneStart(s[cut]) {
+ cut--
+ }
+ return s[:cut]
+}
+
+// Change is one entry of a run's change log.
+type Change struct {
+ Sequence uint64 `json:"sequence"`
+ Program string `json:"program"`
+ Kind string `json:"kind"`
+ Scope string `json:"scope,omitempty"` // in | out, for target changes
+ Target *Target `json:"target,omitempty"` // the target after (before, for removed)
+ Fields []string `json:"fields,omitempty"` // for program_modified / target_modified
+}
+
+// Change kinds.
+const (
+ ChangeProgramAdded = "program_added"
+ ChangeProgramClosed = "program_closed"
+ ChangeProgramReopened = "program_reopened"
+ ChangeProgramDropped = "program_dropped"
+ ChangeProgramModified = "program_modified"
+ ChangeStatus = "status_changed"
+ ChangeRules = "rules_changed"
+ ChangeTerms = "terms_changed"
+ ChangeTargetAdded = "target_added"
+ ChangeTargetRemoved = "target_removed"
+ ChangeTargetModified = "target_modified"
+)
+
+var changeKinds = map[string]bool{ChangeProgramAdded: true, ChangeProgramClosed: true, ChangeProgramReopened: true,
+ ChangeProgramDropped: true, ChangeProgramModified: true, ChangeStatus: true, ChangeRules: true, ChangeTerms: true,
+ ChangeTargetAdded: true, ChangeTargetRemoved: true, ChangeTargetModified: true}
+
+// Validate checks a change entry.
+func (c *Change) Validate() error {
+ if !idRE.MatchString(c.Program) || !changeKinds[c.Kind] || c.Sequence == 0 || len(c.Fields) > MaxChangeFields {
+ return fmt.Errorf("change %q %q: program, kind, sequence or fields", c.Program, c.Kind)
+ }
+ isTarget := strings.HasPrefix(c.Kind, "target_")
+ if isTarget != (c.Target != nil) || isTarget != (c.Scope == "in" || c.Scope == "out") || (!isTarget && c.Scope != "") {
+ return fmt.Errorf("change %q %q: target and scope go with target changes only", c.Program, c.Kind)
+ }
+ if c.Target != nil {
+ if err := c.Target.Validate(); err != nil {
+ return err
+ }
+ }
+ for _, f := range c.Fields {
+ if !safeText(f, 64) || f == "" {
+ return fmt.Errorf("change %q: field name", c.Program)
+ }
+ }
+ return nil
+}
diff --git a/internal/model/model_test.go b/internal/model/model_test.go
new file mode 100644
index 0000000..614cf2c
--- /dev/null
+++ b/internal/model/model_test.go
@@ -0,0 +1,216 @@
+package model
+
+import (
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestNormalizeTarget(t *testing.T) {
+ ok := []struct{ typ, in, want string }{
+ {TargetDomain, "OpenCTEM.ORG.", "openctem.org"},
+ {TargetDomain, "api.openctem.io", "api.openctem.io"},
+ {TargetDomain, "bücher.de", "xn--bcher-kva.de"},
+ {TargetDomain, "foo.example.co.uk", "foo.example.co.uk"},
+ {TargetWildcard, "*.OpenCTEM.io", "*.openctem.io"},
+ {TargetWildcard, "*.example.co.uk", "*.example.co.uk"},
+ {TargetIP, "8.8.8.8", "8.8.8.8"},
+ {TargetIP, "::ffff:8.8.4.4", "8.8.4.4"},
+ {TargetIP, "2606:4700::1111", "2606:4700::1111"},
+ {TargetCIDR, "8.8.8.1/24", "8.8.8.0/24"},
+ {TargetCIDR, "203.1.0.0/16", "203.1.0.0/16"},
+ {TargetCIDR, "2606:4700::/48", "2606:4700::/48"},
+ {TargetURL, "HTTPS://App.OpenCTEM.io/login#x", "https://app.openctem.io/login"},
+ {TargetURL, "https://openctem.io:8443", "https://openctem.io:8443/"},
+ {TargetMobileApp, " com.example.app ", "com.example.app"},
+ {TargetSourceRepo, "https://github.com/openctemio/openctem", "https://github.com/openctemio/openctem"},
+ }
+ for _, c := range ok {
+ got, reason := NormalizeTarget(c.typ, c.in)
+ if reason != "" || got != c.want {
+ t.Errorf("%s %q = %q (%s), want %q", c.typ, c.in, got, reason, c.want)
+ }
+ }
+ bad := []struct{ typ, in string }{
+ {TargetDomain, "com"},
+ {TargetDomain, "co.uk"},
+ {TargetDomain, "github.io"}, // private public suffix
+ {TargetDomain, "localhost"},
+ {TargetDomain, "printer.local"},
+ {TargetDomain, "corp.internal"},
+ {TargetDomain, "example.com"}, // documentation
+ {TargetDomain, "10.0.0.1"},
+ {TargetDomain, "a..b.com"},
+ {TargetDomain, "-bad.com"},
+ {TargetDomain, "under_score.com"},
+ {TargetDomain, "x.123"},
+ {TargetDomain, "foo.*.com"},
+ {TargetDomain, strings.Repeat("a", 64) + ".com"},
+ {TargetDomain, "bad.com"},
+ {TargetWildcard, "*.com"},
+ {TargetWildcard, "*.co.uk"},
+ {TargetWildcard, "*.github.io"},
+ {TargetWildcard, "*"},
+ {TargetWildcard, "*.*.example.org"},
+ {TargetWildcard, "example.org"},
+ {TargetIP, "10.1.2.3"},
+ {TargetIP, "127.0.0.1"},
+ {TargetIP, "169.254.169.254"},
+ {TargetIP, "100.64.0.1"},
+ {TargetIP, "192.0.2.10"},
+ {TargetIP, "::1"},
+ {TargetIP, "fd00::1"},
+ {TargetIP, "2001:db8::1"},
+ {TargetIP, "fe80::1%eth0"},
+ {TargetIP, "224.0.0.1"},
+ {TargetIP, "not-an-ip"},
+ {TargetCIDR, "0.0.0.0/0"},
+ {TargetCIDR, "8.0.0.0/8"},
+ {TargetCIDR, "8.8.0.0/15"},
+ {TargetCIDR, "2606::/16"},
+ {TargetCIDR, "10.0.0.0/24"},
+ {TargetCIDR, "172.16.5.0/24"},
+ {TargetCIDR, "::ffff:8.8.8.0/120"},
+ {TargetURL, "javascript:alert(1)"},
+ {TargetURL, "ftp://openctem.io/"},
+ {TargetURL, "https://user:pw@openctem.io/"},
+ {TargetURL, "https://127.0.0.1/"},
+ {TargetURL, "https://[::1]/"},
+ {TargetURL, "https://*.openctem.io/"},
+ {TargetURL, "https://com/"},
+ {TargetURL, "https://openctem.io:99999/"},
+ {TargetOther, ""},
+ {TargetOther, "a\x00b"},
+ {"subnet", "1.2.3.0/24"},
+ }
+ for _, c := range bad {
+ if got, reason := NormalizeTarget(c.typ, c.in); reason == "" {
+ t.Errorf("%s %q accepted as %q", c.typ, c.in, got)
+ }
+ }
+}
+
+func FuzzNormalizeTarget(f *testing.F) {
+ for _, s := range []string{"example.org", "*.a.b.co.uk", "8.8.8.0/24", "https://a.io/x?y#z", "::ffff:1.2.3.4", "xn--bcher-kva.de"} {
+ f.Add(s)
+ }
+ types := []string{TargetDomain, TargetWildcard, TargetIP, TargetCIDR, TargetURL, TargetOther}
+ f.Fuzz(func(t *testing.T, s string) {
+ for _, typ := range types {
+ n, reason := NormalizeTarget(typ, s)
+ if reason != "" {
+ continue
+ }
+ // Normalising is idempotent, so every accepted target validates.
+ if n2, r2 := NormalizeTarget(typ, n); r2 != "" || n2 != n {
+ t.Fatalf("%s %q -> %q -> %q (%s)", typ, s, n, n2, r2)
+ }
+ }
+ })
+}
+
+func validProgram() Program {
+ now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC)
+ return Program{
+ ID: "disclose:openctem", Source: "disclose", Platform: "self-hosted", Name: "OpenCTEM",
+ URL: "https://openctem.io/security", Type: TypeVDP, Status: StatusOpen,
+ InScope: []Target{{Type: TargetWildcard, Value: "*.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}, {Type: TargetIP, Value: "8.8.8.8"}},
+ OutOfScope: []Target{{Type: TargetDomain, Value: "status.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}},
+ Rules: Rules{TestingRestrictions: []string{"no DoS", "no DoS"}},
+ Terms: Terms{URL: "https://openctem.io/security"},
+ Provenance: Provenance{Source: "disclose", SourceURL: "https://example.org/list.json", FetchedAt: now},
+ FirstSeen: now, LastSeen: now, LastChanged: now,
+ }
+}
+
+func TestNormalizeProgram(t *testing.T) {
+ p := validProgram()
+ p.InScope = append(p.InScope, Target{Type: TargetWildcard, Value: "*.com"}, Target{Type: TargetCIDR, Value: "10.0.0.0/8"},
+ Target{Type: TargetDomain, Value: "OPENCTEM.io"})
+ Normalize(&p)
+ if err := p.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ var in []string
+ for _, x := range p.InScope {
+ in = append(in, x.Type+":"+x.Value)
+ if x.Confidence != ConfidencePublished {
+ t.Errorf("confidence %q", x.Confidence)
+ }
+ }
+ // Sorted, deduplicated, and openctem.io dropped because it is also out of scope.
+ if got := strings.Join(in, " "); got != "ip:8.8.8.8 wildcard:*.openctem.io" {
+ t.Errorf("in scope = %s", got)
+ }
+ reasons := map[string]string{}
+ for _, r := range p.Rejected {
+ reasons[r.Value] = r.Reason
+ }
+ if reasons["*.com"] == "" || reasons["10.0.0.0/8"] == "" || reasons["openctem.io"] != "also listed out of scope" {
+ t.Errorf("rejected = %+v", p.Rejected)
+ }
+ if len(p.Rules.TestingRestrictions) != 1 || p.Rules.SafeHarbour != SafeHarbourUnknown {
+ t.Errorf("rules = %+v", p.Rules)
+ }
+}
+
+func TestProgramValidateRefuses(t *testing.T) {
+ cases := map[string]func(p *Program){
+ "bad id": func(p *Program) { p.ID = "Disclose:X" },
+ "id of other source": func(p *Program) { p.ID = "other:x" },
+ "control in name": func(p *Program) { p.Name = "a\x07b" },
+ "empty name": func(p *Program) { p.Name = " " },
+ "bad type": func(p *Program) { p.Type = "paid" },
+ "bounty mismatch": func(p *Program) { p.OffersBounty = true },
+ "closed without time": func(p *Program) { p.Status = StatusClosed },
+ "javascript url": func(p *Program) { p.URL = "javascript:alert(1)" },
+ "terms hash no time": func(p *Program) { p.Terms.SHA256 = strings.Repeat("a", 64) },
+ "bad email": func(p *Program) { p.Contact.Email = "x " },
+ "unsorted targets": func(p *Program) {
+ p.InScope = []Target{{Type: TargetDomain, Value: "b.org", Confidence: ConfidencePublished}, {Type: TargetDomain, Value: "a.org", Confidence: ConfidencePublished}}
+ },
+ "unnormalised target": func(p *Program) {
+ p.InScope = []Target{{Type: TargetDomain, Value: "A.org", Confidence: ConfidencePublished}}
+ },
+ "public suffix target": func(p *Program) {
+ p.InScope = []Target{{Type: TargetWildcard, Value: "*.com", Confidence: ConfidencePublished}}
+ },
+ "nil list": func(p *Program) { p.OutOfScope = nil },
+ "first seen": func(p *Program) { p.FirstSeen = p.LastSeen.Add(time.Hour) },
+ "bad harbour": func(p *Program) { p.Rules.SafeHarbour = "maybe" },
+ "no provenance": func(p *Program) { p.Provenance.FetchedAt = time.Time{} },
+ }
+ for name, mut := range cases {
+ p := validProgram()
+ Normalize(&p)
+ mut(&p)
+ if err := p.Validate(); err == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+}
+
+func TestChangeValidate(t *testing.T) {
+ tgt := Target{Type: TargetDomain, Value: "openctem.io", Confidence: ConfidenceInferred}
+ good := []Change{
+ {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded},
+ {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &tgt},
+ }
+ for _, c := range good {
+ if err := c.Validate(); err != nil {
+ t.Error(err)
+ }
+ }
+ bad := []Change{
+ {Sequence: 0, Program: "disclose:x", Kind: ChangeProgramAdded},
+ {Sequence: 1, Program: "disclose:x", Kind: "deleted"},
+ {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded},
+ {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded, Scope: "in", Target: &tgt},
+ {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &Target{Type: TargetWildcard, Value: "*.com", Confidence: ConfidenceInferred}},
+ }
+ for i, c := range bad {
+ if err := c.Validate(); err == nil {
+ t.Errorf("bad change %d accepted", i)
+ }
+ }
+}
diff --git a/internal/model/normalize.go b/internal/model/normalize.go
new file mode 100644
index 0000000..28585b1
--- /dev/null
+++ b/internal/model/normalize.go
@@ -0,0 +1,103 @@
+package model
+
+import (
+ "sort"
+ "strings"
+)
+
+// Normalize canonicalises the targets of a program an adapter produced:
+// every target is normalised or moved to Rejected with the reason, lists are
+// deduplicated and sorted, an in-scope target that is also out of scope is
+// dropped from in scope (out of scope wins), and lists are capped. Text
+// fields are cleaned. It does not set the timestamps; the collector does.
+func Normalize(p *Program) {
+ p.Name = CleanText(p.Name, MaxNameLen)
+ p.Platform = CleanText(p.Platform, 64)
+ p.Rules.Summary = CleanText(p.Rules.Summary, MaxTextLen)
+ p.Rules.RateLimit = CleanText(p.Rules.RateLimit, 200)
+ p.Rules.TestingRestrictions = cleanList(p.Rules.TestingRestrictions)
+ p.Rules.RequiredHeaders = cleanList(p.Rules.RequiredHeaders)
+ p.Rules.Languages = cleanList(p.Rules.Languages)
+ if p.Rules.SafeHarbour == "" {
+ p.Rules.SafeHarbour = SafeHarbourUnknown
+ }
+ p.OffersBounty = p.Type == TypeBounty
+ var rejected []Rejected
+ norm := func(scope string, in []Target) []Target {
+ out := make([]Target, 0, len(in))
+ for _, t := range in {
+ v, reason := NormalizeTarget(t.Type, t.Value)
+ if reason != "" {
+ rejected = append(rejected, Rejected{Scope: scope, Type: CleanText(t.Type, 32), Value: CleanText(t.Value, MaxValueLen), Reason: reason})
+ continue
+ }
+ t.Value = v
+ if t.Confidence == "" {
+ t.Confidence = ConfidencePublished
+ }
+ if !severities[t.MaxSev] {
+ t.MaxSev = ""
+ }
+ t.Notes = CleanText(t.Notes, 500)
+ out = append(out, t)
+ }
+ out = SortTargets(out)
+ if len(out) > MaxTargets {
+ for _, t := range out[MaxTargets:] {
+ rejected = append(rejected, Rejected{Scope: scope, Type: t.Type, Value: t.Value, Reason: "over the per-program target cap"})
+ }
+ out = out[:MaxTargets]
+ }
+ return out
+ }
+ p.OutOfScope = norm("out", p.OutOfScope)
+ in := norm("in", p.InScope)
+ excluded := map[string]bool{}
+ for _, t := range p.OutOfScope {
+ excluded[t.Type+"\x00"+t.Value] = true
+ }
+ p.InScope = in[:0]
+ for _, t := range in {
+ if excluded[t.Type+"\x00"+t.Value] {
+ rejected = append(rejected, Rejected{Scope: "in", Type: t.Type, Value: t.Value, Reason: "also listed out of scope"})
+ continue
+ }
+ p.InScope = append(p.InScope, t)
+ }
+ for i := range rejected {
+ if rejected[i].Value == "" {
+ rejected[i].Value = "(unprintable)"
+ }
+ if rejected[i].Type == "" {
+ rejected[i].Type = "unknown"
+ }
+ }
+ sort.SliceStable(rejected, func(i, j int) bool {
+ a, b := rejected[i], rejected[j]
+ return strings.Join([]string{a.Scope, a.Type, a.Value, a.Reason}, "\x00") < strings.Join([]string{b.Scope, b.Type, b.Value, b.Reason}, "\x00")
+ })
+ if len(rejected) > MaxTargets {
+ rejected = rejected[:MaxTargets]
+ }
+ if rejected == nil {
+ rejected = []Rejected{}
+ }
+ p.Rejected = rejected
+}
+
+func cleanList(in []string) []string {
+ out := []string{}
+ seen := map[string]bool{}
+ for _, s := range in {
+ s = CleanText(s, 500)
+ if s == "" || seen[s] {
+ continue
+ }
+ seen[s] = true
+ out = append(out, s)
+ if len(out) == MaxListItems {
+ break
+ }
+ }
+ return out
+}
diff --git a/internal/model/schema_test.go b/internal/model/schema_test.go
new file mode 100644
index 0000000..d4e7c5f
--- /dev/null
+++ b/internal/model/schema_test.go
@@ -0,0 +1,60 @@
+package model
+
+import (
+ "encoding/json"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "testing"
+)
+
+// The published JSON Schema documents every field the feed writes, and
+// nothing else.
+func TestSchemaMatchesModel(t *testing.T) {
+ type obj struct {
+ Properties map[string]json.RawMessage `json:"properties"`
+ Defs map[string]obj `json:"$defs"`
+ }
+ read := func(name string) obj {
+ b, err := os.ReadFile("../../schema/" + name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var o obj
+ if err := json.Unmarshal(b, &o); err != nil {
+ t.Fatal(err)
+ }
+ return o
+ }
+ prog, change := read("program.schema.json"), read("change.schema.json")
+ cases := []struct {
+ name string
+ schema obj
+ v any
+ }{
+ {"program", prog, Program{}},
+ {"target", prog.Defs["target"], Target{}},
+ {"rejected", prog.Defs["rejected"], Rejected{}},
+ {"rules", prog.Defs["rules"], Rules{}},
+ {"terms", prog.Defs["terms"], Terms{}},
+ {"contact", prog.Defs["contact"], Contact{}},
+ {"provenance", prog.Defs["provenance"], Provenance{}},
+ {"change", change, Change{}},
+ }
+ for _, c := range cases {
+ var want, got []string
+ rt := reflect.TypeOf(c.v)
+ for i := 0; i < rt.NumField(); i++ {
+ want = append(want, strings.Split(rt.Field(i).Tag.Get("json"), ",")[0])
+ }
+ for k := range c.schema.Properties {
+ got = append(got, k)
+ }
+ sort.Strings(want)
+ sort.Strings(got)
+ if strings.Join(want, ",") != strings.Join(got, ",") {
+ t.Errorf("%s: schema has %v, model has %v", c.name, got, want)
+ }
+ }
+}
diff --git a/internal/model/scope.go b/internal/model/scope.go
new file mode 100644
index 0000000..cece441
--- /dev/null
+++ b/internal/model/scope.go
@@ -0,0 +1,206 @@
+package model
+
+import (
+ "net/netip"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "golang.org/x/net/idna"
+ "golang.org/x/net/publicsuffix"
+)
+
+// Scope caps: a CIDR wider than these is refused (a program does not own
+// a /8, and a wide range in a feed would point scanners at strangers).
+const (
+ MinIPv4Prefix = 16
+ MinIPv6Prefix = 48
+)
+
+// Ranges no program can own on the public internet (RFC 6890 and friends).
+var reserved = func() []netip.Prefix {
+ var out []netip.Prefix
+ for _, s := range []string{
+ "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12",
+ "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24",
+ "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", "255.255.255.255/32",
+ "::/128", "::1/128", "::ffff:0:0/96", "64:ff9b::/96", "64:ff9b:1::/48", "100::/64", "2001::/23",
+ "2001:db8::/32", "2002::/16", "3fff::/20", "5f00::/16", "fc00::/7", "fe80::/10", "ff00::/8",
+ } {
+ out = append(out, netip.MustParsePrefix(s))
+ }
+ return out
+}()
+
+// Names that never resolve on the public internet, or are reserved for
+// documentation.
+var reservedSuffixes = []string{"local", "localhost", "test", "example", "invalid", "internal", "onion", "arpa",
+ "home.arpa", "lan", "corp", "intranet", "private", "example.com", "example.net", "example.org"}
+
+var lookup = idna.New(idna.MapForLookup(), idna.Transitional(false), idna.StrictDomainName(true), idna.BidiRule())
+
+// NormalizeTarget returns the canonical form of a target value of the given
+// type, or a non-empty reason it is refused.
+func NormalizeTarget(typ, value string) (string, string) {
+ v := strings.TrimSpace(value)
+ if v == "" || len(v) > MaxValueLen || !safeText(v, MaxValueLen) {
+ return "", "empty, too long or has control characters"
+ }
+ switch typ {
+ case TargetDomain:
+ return normDomain(v)
+ case TargetWildcard:
+ if !strings.HasPrefix(v, "*.") {
+ return "", "a wildcard must start with *."
+ }
+ d, reason := normDomain(v[2:])
+ if reason != "" {
+ return "", reason
+ }
+ return "*." + d, ""
+ case TargetIP:
+ a, err := netip.ParseAddr(v)
+ if err != nil || a.Zone() != "" {
+ return "", "not an IP address"
+ }
+ a = a.Unmap()
+ if r := reservedAddr(a); r != "" {
+ return "", r
+ }
+ return a.String(), ""
+ case TargetCIDR:
+ p, err := netip.ParsePrefix(v)
+ if err != nil {
+ return "", "not a CIDR"
+ }
+ if p.Addr().Is4In6() {
+ return "", "IPv4-mapped CIDR"
+ }
+ p = p.Masked()
+ if (p.Addr().Is4() && p.Bits() < MinIPv4Prefix) || (p.Addr().Is6() && p.Bits() < MinIPv6Prefix) {
+ return "", "CIDR wider than /" + strconv.Itoa(MinIPv4Prefix) + " (IPv4) or /" + strconv.Itoa(MinIPv6Prefix) + " (IPv6)"
+ }
+ for _, r := range reserved {
+ if r.Overlaps(p) {
+ return "", "private or reserved range"
+ }
+ }
+ return p.String(), ""
+ case TargetURL:
+ return normURL(v)
+ case TargetMobileApp, TargetSourceRepo, TargetOther:
+ s := CleanText(v, MaxValueLen)
+ if s == "" {
+ return "", "empty or has control characters"
+ }
+ return s, ""
+ }
+ return "", "unknown target type"
+}
+
+func reservedAddr(a netip.Addr) string {
+ if !a.IsGlobalUnicast() || a.IsPrivate() {
+ return "private or reserved address"
+ }
+ for _, r := range reserved {
+ if r.Contains(a) {
+ return "private or reserved address"
+ }
+ }
+ return ""
+}
+
+func normDomain(v string) (string, string) {
+ d := strings.TrimSuffix(strings.ToLower(v), ".")
+ if strings.Contains(d, "*") {
+ return "", "wildcard inside a domain"
+ }
+ if _, err := netip.ParseAddr(d); err == nil {
+ return "", "an IP address, not a domain"
+ }
+ ascii, err := lookup.ToASCII(d)
+ if err != nil {
+ return "", "not a valid domain name"
+ }
+ if len(ascii) > 253 || !strings.Contains(ascii, ".") {
+ return "", "not a fully qualified domain name"
+ }
+ labels := strings.Split(ascii, ".")
+ for _, l := range labels {
+ if l == "" || len(l) > 63 || l[0] == '-' || l[len(l)-1] == '-' {
+ return "", "not a valid domain name"
+ }
+ for _, c := range l {
+ if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' {
+ return "", "not a valid domain name"
+ }
+ }
+ }
+ if tld := labels[len(labels)-1]; strings.Trim(tld, "0123456789") == "" {
+ return "", "numeric top-level label"
+ }
+ for _, s := range reservedSuffixes {
+ if ascii == s || strings.HasSuffix(ascii, "."+s) {
+ return "", "reserved or non-public name"
+ }
+ }
+ if ps, _ := publicsuffix.PublicSuffix(ascii); ps == ascii {
+ return "", "a public suffix"
+ }
+ if _, err := publicsuffix.EffectiveTLDPlusOne(ascii); err != nil {
+ return "", "a public suffix"
+ }
+ return ascii, ""
+}
+
+func normURL(v string) (string, string) {
+ u, err := url.Parse(v)
+ if err != nil {
+ return "", "not a URL"
+ }
+ u.Scheme = strings.ToLower(u.Scheme)
+ if u.Scheme != "https" && u.Scheme != "http" {
+ return "", "not an http(s) URL"
+ }
+ if u.User != nil {
+ return "", "credentials in URL"
+ }
+ host := u.Hostname()
+ var h string
+ if a, err := netip.ParseAddr(host); err == nil {
+ a = a.Unmap()
+ if r := reservedAddr(a); r != "" || a.Zone() != "" {
+ return "", "private or reserved address"
+ }
+ h = a.String()
+ if a.Is6() {
+ h = "[" + h + "]"
+ }
+ } else {
+ d, reason := normDomain(host)
+ if reason != "" {
+ return "", reason
+ }
+ h = d
+ }
+ if p := u.Port(); p != "" {
+ n, err := strconv.Atoi(p)
+ if err != nil || n < 1 || n > 65535 {
+ return "", "bad port"
+ }
+ h += ":" + strconv.Itoa(n)
+ }
+ u.Host, u.Fragment, u.RawFragment = h, "", ""
+ if u.Path == "" {
+ u.Path = "/"
+ }
+ s := u.String()
+ if len(s) > MaxValueLen {
+ return "", "too long"
+ }
+ return s, ""
+}
+
+// PublicAddr reports whether a is a public unicast address a program could
+// own (the fetcher also refuses to connect anywhere else).
+func PublicAddr(a netip.Addr) bool { return reservedAddr(a.Unmap()) == "" }
diff --git a/internal/source/diodb.go b/internal/source/diodb.go
new file mode 100644
index 0000000..e9720d7
--- /dev/null
+++ b/internal/source/diodb.go
@@ -0,0 +1,225 @@
+package source
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/mail"
+ "net/url"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// DioDBURL is the disclose.io program list (CC0-1.0).
+const DioDBURL = "https://raw.githubusercontent.com/disclose/diodb/master/program-list.json"
+
+const (
+ maxDioDBBytes = 32 << 20
+ maxDioDBRecords = 50000
+)
+
+// DioDB reads the disclose.io program list: a public directory of
+// self-hosted and platform-hosted programs with their policy URL, contact,
+// bounty and safe-harbour flags. It carries no structured scope; the only
+// target it yields is the host of a program's own security.txt, as an
+// inferred target.
+type DioDB struct {
+ URL string // DioDBURL unless a test overrides it
+}
+
+// Meta implements Source.
+func (d DioDB) Meta() Meta {
+ return Meta{
+ Name: "disclose",
+ URL: d.url(),
+ Terms: "https://github.com/disclose/diodb/blob/master/LICENSE",
+ Licence: "CC0-1.0",
+ Attribution: "Program directory data from the disclose.io diodb project (CC0-1.0).",
+ }
+}
+
+func (d DioDB) url() string {
+ if d.URL != "" {
+ return d.URL
+ }
+ return DioDBURL
+}
+
+type dioRecord struct {
+ ProgramName string `json:"program_name"`
+ PolicyURL string `json:"policy_url"`
+ PolicyURLStatus string `json:"policy_url_status"`
+ ContactEmail string `json:"contact_email"`
+ ContactURL string `json:"contact_url"`
+ OffersBounty json.RawMessage `json:"offers_bounty"`
+ SafeHarbor json.RawMessage `json:"safe_harbor"`
+ SecurityTxtURL json.RawMessage `json:"securitytxt_url"`
+ PreferredLanguage json.RawMessage `json:"preferred_languages"`
+ DisclosureDays json.RawMessage `json:"disclosure_timeline_days"`
+}
+
+// Hosts of platforms that host programs on their own site: a policy URL on
+// one of them names the platform, never the program's own domain.
+var platformHosts = map[string]string{
+ "hackerone.com": "hackerone", "bugcrowd.com": "bugcrowd", "intigriti.com": "intigriti",
+ "yeswehack.com": "yeswehack", "federacy.com": "federacy", "openbugbounty.org": "openbugbounty",
+ "hackenproof.com": "hackenproof", "immunefi.com": "immunefi", "huntr.com": "huntr", "huntr.dev": "huntr",
+ "whitehub.net": "whitehub", "bugbase.ai": "bugbase", "zerocopter.com": "zerocopter", "hackrate.com": "hackrate",
+}
+
+// Collect implements Source.
+func (d DioDB) Collect(ctx context.Context, c *fetch.Client) (*Result, error) {
+ resp, err := c.Get(ctx, d.url(), maxDioDBBytes)
+ if err != nil {
+ return nil, err
+ }
+ var raw []json.RawMessage
+ if err := json.Unmarshal(resp.Body, &raw); err != nil {
+ return nil, fmt.Errorf("diodb: %w", err)
+ }
+ if len(raw) > maxDioDBRecords {
+ return nil, fmt.Errorf("diodb: %d records, over the cap of %d", len(raw), maxDioDBRecords)
+ }
+ res := &Result{}
+ recs := make([]dioRecord, 0, len(raw))
+ names := make([]string, 0, len(raw))
+ for _, r := range raw {
+ var rec dioRecord
+ if json.Unmarshal(r, &rec) != nil || strings.TrimSpace(rec.ProgramName) == "" || !httpURL(rec.PolicyURL) {
+ res.Skipped++
+ continue
+ }
+ recs = append(recs, rec)
+ names = append(names, rec.ProgramName)
+ }
+ slugs := UniqueSlugs(names)
+ meta := d.Meta()
+ for _, rec := range recs {
+ p := model.Program{
+ ID: meta.Name + ":" + slugs[rec.ProgramName],
+ Source: meta.Name,
+ Platform: platformOf(rec.PolicyURL),
+ Name: rec.ProgramName,
+ URL: strings.TrimSpace(rec.PolicyURL),
+ Type: model.TypeVDP,
+ Status: model.StatusOpen,
+ InScope: []model.Target{},
+ OutOfScope: []model.Target{},
+ Rules: model.Rules{
+ SafeHarbour: harbour(str(rec.SafeHarbor)),
+ TestingRestrictions: []string{},
+ RequiredHeaders: []string{},
+ Languages: splitList(str(rec.PreferredLanguage)),
+ },
+ Terms: model.Terms{URL: strings.TrimSpace(rec.PolicyURL)},
+ Provenance: model.Provenance{Source: meta.Name, SourceURL: meta.URL, FetchedAt: resp.FetchedAt},
+ }
+ switch strings.ToLower(str(rec.OffersBounty)) {
+ case "yes", "true", "partial":
+ p.Type = model.TypeBounty
+ }
+ if strings.EqualFold(rec.PolicyURLStatus, "dead") {
+ p.Status = model.StatusPaused
+ p.Rules.Summary = "The policy URL did not answer when the directory last checked it."
+ }
+ if n, err := strconv.Atoi(str(rec.DisclosureDays)); err == nil && n > 0 && n <= 3650 {
+ p.Rules.DisclosureDays = n
+ }
+ if a, err := mail.ParseAddress(strings.TrimSpace(rec.ContactEmail)); err == nil && len(a.Address) <= 254 {
+ p.Contact.Email = a.Address
+ }
+ if u := strings.TrimSpace(rec.ContactURL); httpURL(u) || strings.HasPrefix(u, "mailto:") {
+ p.Contact.URL = u
+ }
+ if host := securityTxtHost(str(rec.SecurityTxtURL)); host != "" {
+ p.InScope = append(p.InScope, model.Target{Type: model.TargetDomain, Value: host, Confidence: model.ConfidenceInferred,
+ Notes: "Host that serves the program's security.txt; not a published scope."})
+ }
+ res.Programs = append(res.Programs, p)
+ }
+ sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID })
+ return res, nil
+}
+
+// str reads a JSON value that the directory stores as a string, a bool, a
+// number or null.
+func str(raw json.RawMessage) string {
+ var s string
+ if json.Unmarshal(raw, &s) == nil {
+ return strings.TrimSpace(s)
+ }
+ var b bool
+ if json.Unmarshal(raw, &b) == nil {
+ return strconv.FormatBool(b)
+ }
+ var n json.Number
+ if json.Unmarshal(raw, &n) == nil {
+ return n.String()
+ }
+ return ""
+}
+
+func httpURL(s string) bool {
+ u, err := url.Parse(strings.TrimSpace(s))
+ return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" && u.User == nil
+}
+
+func hostOf(s string) string {
+ u, err := url.Parse(strings.TrimSpace(s))
+ if err != nil {
+ return ""
+ }
+ return strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
+}
+
+func platformOf(policyURL string) string {
+ h := hostOf(policyURL)
+ for host, name := range platformHosts {
+ if h == host || strings.HasSuffix(h, "."+host) {
+ return name
+ }
+ }
+ return "self-hosted"
+}
+
+// securityTxtHost returns the host of a first-party security.txt URL, or
+// "" when the URL is not one (wrong path, or hosted on a platform).
+func securityTxtHost(s string) string {
+ u, err := url.Parse(strings.TrimSpace(s))
+ if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.User != nil || u.Port() != "" {
+ return ""
+ }
+ if u.Path != "/.well-known/security.txt" && u.Path != "/security.txt" {
+ return ""
+ }
+ if platformOf(s) != "self-hosted" {
+ return ""
+ }
+ return strings.ToLower(u.Hostname())
+}
+
+func harbour(s string) string {
+ switch strings.ToLower(s) {
+ case "full", "yes":
+ return model.SafeHarbourFull
+ case "partial":
+ return model.SafeHarbourPartial
+ case "none", "no":
+ return model.SafeHarbourNone
+ }
+ return model.SafeHarbourUnknown
+}
+
+func splitList(s string) []string {
+ out := []string{}
+ for _, p := range strings.Split(s, ",") {
+ if p = strings.TrimSpace(p); p != "" {
+ out = append(out, p)
+ }
+ }
+ return out
+}
diff --git a/internal/source/securitytxt.go b/internal/source/securitytxt.go
new file mode 100644
index 0000000..299c793
--- /dev/null
+++ b/internal/source/securitytxt.go
@@ -0,0 +1,411 @@
+package source
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/csv"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/url"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+ "unicode/utf8"
+
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// DotGovFederalURL is CISA's list of US federal .gov domains (CC0-1.0).
+const DotGovFederalURL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-federal.csv"
+
+const (
+ maxSeedBytes = 8 << 20
+ maxSeedDomains = 20000
+ maxSecurityTxtBytes = 32 << 10
+)
+
+// Seed is one domain to look for a security.txt on, with the name of the
+// organisation that holds it.
+type Seed struct {
+ Domain string
+ Org string
+}
+
+// SecurityTxt reads RFC 9116 security.txt files from a seed list of
+// domains. A file with a Contact and a Policy field is a self-hosted
+// disclosure program; the domain it was served from is an inferred target
+// (security.txt does not publish a scope).
+type SecurityTxt struct {
+ // SeedCSV is a CSV whose first column is a domain and third column the
+ // organisation (the dotgov-data layout); DotGovFederalURL by default.
+ SeedCSV string
+ // Extra are additional seeds (from the repository's seed file).
+ Extra []Seed
+ // Workers is the number of domains read at once (16 by default; each
+ // host still gets at most one request per MinInterval).
+ Workers int
+ // Progress, when set, is called after every 100 domains.
+ Progress func(done, total int)
+ // Scheme is "https" except in tests.
+ Scheme string
+ // Now is the time used to judge Expires (time.Now by default).
+ Now func() time.Time
+}
+
+// Meta implements Source.
+func (s SecurityTxt) Meta() Meta {
+ return Meta{
+ Name: "securitytxt",
+ URL: s.seedURL(),
+ Terms: "https://www.rfc-editor.org/rfc/rfc9116",
+ Licence: "Seed list: CC0-1.0 (cisagov/dotgov-data). security.txt files: published by each domain for automated retrieval (RFC 9116).",
+ Attribution: "Domain seed list from CISA's dotgov-data (CC0-1.0); program data from each domain's own security.txt.",
+ }
+}
+
+func (s SecurityTxt) seedURL() string {
+ if s.SeedCSV != "" {
+ return s.SeedCSV
+ }
+ return DotGovFederalURL
+}
+
+// Collect implements Source.
+func (s SecurityTxt) Collect(ctx context.Context, c *fetch.Client) (*Result, error) {
+ resp, err := c.Get(ctx, s.seedURL(), maxSeedBytes)
+ if err != nil {
+ return nil, fmt.Errorf("seed list: %w", err)
+ }
+ seeds, err := ParseSeedCSV(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ seeds = append(seeds, s.Extra...)
+ byDomain := map[string]Seed{}
+ for _, sd := range seeds {
+ d, reason := model.NormalizeTarget(model.TargetDomain, sd.Domain)
+ if reason != "" || len(d) > 120 {
+ continue
+ }
+ if _, dup := byDomain[d]; !dup {
+ byDomain[d] = Seed{Domain: d, Org: sd.Org}
+ }
+ }
+ if len(byDomain) > maxSeedDomains {
+ return nil, fmt.Errorf("seed list: %d domains, over the cap of %d", len(byDomain), maxSeedDomains)
+ }
+ domains := make([]string, 0, len(byDomain))
+ for d := range byDomain {
+ domains = append(domains, d)
+ }
+ sort.Strings(domains)
+
+ workers := s.Workers
+ if workers <= 0 {
+ workers = 16
+ }
+ now := time.Now
+ if s.Now != nil {
+ now = s.Now
+ }
+ meta := s.Meta()
+ res := &Result{}
+ var mu sync.Mutex
+ done := 0
+ jobs := make(chan string)
+ var wg sync.WaitGroup
+ for i := 0; i < workers; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for d := range jobs {
+ p, known, err := s.one(ctx, c, meta, byDomain[d], now())
+ mu.Lock()
+ done++
+ if s.Progress != nil && done%100 == 0 {
+ s.Progress(done, len(domains))
+ }
+ switch {
+ case p != nil:
+ res.Programs = append(res.Programs, *p)
+ case err != nil && !known:
+ res.Unknown = append(res.Unknown, meta.Name+":"+d)
+ default:
+ res.Skipped++
+ }
+ mu.Unlock()
+ }
+ }()
+ }
+ for _, d := range domains {
+ select {
+ case jobs <- d:
+ case <-ctx.Done():
+ }
+ }
+ close(jobs)
+ wg.Wait()
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID })
+ sort.Strings(res.Unknown)
+ return res, nil
+}
+
+// one reads one domain. known reports whether the answer is definite (a
+// file, or a definite "no file"); an indefinite failure keeps the
+// previous record.
+func (s SecurityTxt) one(ctx context.Context, c *fetch.Client, meta Meta, sd Seed, now time.Time) (*model.Program, bool, error) {
+ scheme := s.Scheme
+ if scheme == "" {
+ scheme = "https"
+ }
+ u := scheme + "://" + sd.Domain + "/.well-known/security.txt"
+ resp, err := c.Get(ctx, u, maxSecurityTxtBytes)
+ if err != nil {
+ var se *fetch.StatusError
+ definite := errors.Is(err, fetch.ErrNotFound) || errors.Is(err, fetch.ErrDisallowed) ||
+ errors.Is(err, fetch.ErrTooLarge) || (errors.As(err, &se) && se.Code >= 400 && se.Code < 500)
+ return nil, definite, err
+ }
+ f, err := ParseSecurityTxt(resp.Body)
+ if err != nil {
+ return nil, true, err
+ }
+ if !ownedBy(f, resp.URL, sd.Domain) {
+ return nil, true, errors.New("security.txt is served for another host")
+ }
+ policy := firstHTTP(f.Policy)
+ if policy == "" {
+ return nil, true, errors.New("no Policy field")
+ }
+ sum := sha256.Sum256(resp.Body)
+ fetched := resp.FetchedAt
+ name := sd.Org
+ if name == "" {
+ name = sd.Domain
+ } else {
+ name += " (" + sd.Domain + ")"
+ }
+ p := &model.Program{
+ ID: meta.Name + ":" + sd.Domain,
+ Source: meta.Name,
+ Platform: "self-hosted",
+ Name: name,
+ URL: policy,
+ Type: model.TypeVDP,
+ Status: model.StatusOpen,
+ InScope: []model.Target{{Type: model.TargetDomain, Value: sd.Domain, Confidence: model.ConfidenceInferred,
+ Notes: "Domain whose security.txt names this policy; not a published scope."}},
+ OutOfScope: []model.Target{},
+ Rules: model.Rules{SafeHarbour: model.SafeHarbourUnknown, TestingRestrictions: []string{},
+ RequiredHeaders: []string{}, Languages: splitList(strings.Join(f.Languages, ","))},
+ Terms: model.Terms{URL: u, SHA256: hex.EncodeToString(sum[:]), FetchedAt: &fetched},
+ Provenance: model.Provenance{Source: meta.Name, SourceURL: u, FetchedAt: resp.FetchedAt},
+ }
+ if !f.Expires.IsZero() && f.Expires.Before(now) {
+ p.Status = model.StatusPaused
+ p.Rules.Summary = "The security.txt expired on " + f.Expires.UTC().Format("2006-01-02") + "."
+ }
+ for _, ct := range f.Contact {
+ switch {
+ case strings.HasPrefix(ct, "mailto:") && p.Contact.Email == "":
+ if addr := strings.TrimPrefix(ct, "mailto:"); validEmail(addr) {
+ p.Contact.Email = addr
+ }
+ case strings.HasPrefix(ct, "https://") && p.Contact.URL == "":
+ p.Contact.URL = ct
+ }
+ }
+ return p, true, nil
+}
+
+// SecurityTxtFile holds the fields of a security.txt the collector uses.
+type SecurityTxtFile struct {
+ Contact []string
+ Policy []string
+ Canonical []string
+ Languages []string
+ Expires time.Time
+ CVDPolicy []string // draft-behring-cvd-policy; recorded, not yet parsed
+}
+
+// ParseSecurityTxt parses an RFC 9116 file (a cleartext-signed one too; the
+// signature is not checked). It refuses HTML and files without a Contact.
+func ParseSecurityTxt(body []byte) (*SecurityTxtFile, error) {
+ if !utf8.Valid(body) {
+ return nil, errors.New("security.txt: not UTF-8")
+ }
+ text := string(body)
+ if l := strings.ToLower(strings.TrimSpace(text)); strings.HasPrefix(l, "<") || strings.Contains(l, " 2048 {
+ continue
+ }
+ switch strings.ToLower(strings.TrimSpace(k)) {
+ case "contact":
+ f.Contact = appendCapped(f.Contact, v)
+ case "policy":
+ f.Policy = appendCapped(f.Policy, v)
+ case "canonical":
+ f.Canonical = appendCapped(f.Canonical, v)
+ case "preferred-languages":
+ f.Languages = appendCapped(f.Languages, v)
+ case "cvd-policy":
+ f.CVDPolicy = appendCapped(f.CVDPolicy, v)
+ case "expires":
+ if t, err := time.Parse(time.RFC3339, v); err == nil && f.Expires.IsZero() {
+ f.Expires = t
+ }
+ }
+ }
+ if err := sc.Err(); err != nil {
+ return nil, fmt.Errorf("security.txt: %w", err)
+ }
+ if len(f.Contact) == 0 {
+ return nil, errors.New("security.txt: no Contact field")
+ }
+ return f, nil
+}
+
+func appendCapped(l []string, v string) []string {
+ if len(l) >= 16 {
+ return l
+ }
+ return append(l, v)
+}
+
+func stripClearsign(text string) string {
+ _, rest, ok := strings.Cut(text, "\n\n")
+ if !ok {
+ _, rest, _ = strings.Cut(text, "\r\n\r\n")
+ }
+ if i := strings.Index(rest, "-----BEGIN PGP SIGNATURE-----"); i >= 0 {
+ rest = rest[:i]
+ }
+ var b strings.Builder
+ for _, line := range strings.Split(rest, "\n") {
+ b.WriteString(strings.TrimPrefix(line, "- "))
+ b.WriteByte('\n')
+ }
+ return b.String()
+}
+
+// ownedBy: the file was served from the seed domain, or one of its
+// Canonical URLs is on the seed domain. A file reached through a redirect
+// to another organisation's host, with no Canonical naming the seed domain,
+// is that other host's file.
+func ownedBy(f *SecurityTxtFile, finalURL, domain string) bool {
+ if hostIs(finalURL, domain) {
+ return true
+ }
+ for _, c := range f.Canonical {
+ if hostIs(c, domain) {
+ return true
+ }
+ }
+ return false
+}
+
+func hostIs(raw, domain string) bool {
+ u, err := url.Parse(raw)
+ return err == nil && strings.EqualFold(strings.TrimSuffix(u.Hostname(), "."), domain)
+}
+
+func firstHTTP(l []string) string {
+ for _, v := range l {
+ if u, err := url.Parse(v); err == nil && u.Scheme == "https" && u.Host != "" && u.User == nil && len(v) <= model.MaxURLLen && !strings.ContainsAny(v, " \t") {
+ return v
+ }
+ }
+ return ""
+}
+
+func validEmail(s string) bool {
+ if len(s) > 254 || strings.ContainsAny(s, " <>\"") {
+ return false
+ }
+ at := strings.LastIndexByte(s, '@')
+ return at > 0 && at < len(s)-3 && strings.Contains(s[at:], ".")
+}
+
+// ParseSeedCSV reads a seed CSV: header row, domain in column 1, the
+// organisation in column 3 (and a sub-organisation in column 4, if any).
+func ParseSeedCSV(body []byte) ([]Seed, error) {
+ r := csv.NewReader(bytes.NewReader(body))
+ r.FieldsPerRecord = -1
+ r.ReuseRecord = true
+ var out []Seed
+ first := true
+ for {
+ rec, err := r.Read()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return nil, fmt.Errorf("seed list: %w", err)
+ }
+ if first {
+ first = false
+ if len(rec) > 0 && strings.Contains(strings.ToLower(rec[0]), "domain") {
+ continue
+ }
+ }
+ if len(rec) == 0 || strings.TrimSpace(rec[0]) == "" {
+ continue
+ }
+ sd := Seed{Domain: strings.TrimSpace(rec[0])}
+ if len(rec) > 2 {
+ sd.Org = strings.TrimSpace(rec[2])
+ }
+ if len(rec) > 3 && strings.TrimSpace(rec[3]) != "" {
+ sd.Org += " / " + strings.TrimSpace(rec[3])
+ }
+ out = append(out, sd)
+ if len(out) > maxSeedDomains {
+ return nil, fmt.Errorf("seed list: over %d domains", maxSeedDomains)
+ }
+ }
+ return out, nil
+}
+
+// ParseSeedFile reads the repository seed file: one domain per line,
+// optionally followed by whitespace and an organisation name; # comments.
+func ParseSeedFile(body []byte) []Seed {
+ var out []Seed
+ for _, line := range strings.Split(string(body), "\n") {
+ if i := strings.IndexByte(line, '#'); i >= 0 {
+ line = line[:i]
+ }
+ fields := strings.Fields(line)
+ if len(fields) == 0 {
+ continue
+ }
+ out = append(out, Seed{Domain: fields[0], Org: strings.Join(fields[1:], " ")})
+ }
+ return out
+}
diff --git a/internal/source/source.go b/internal/source/source.go
new file mode 100644
index 0000000..696a1bf
--- /dev/null
+++ b/internal/source/source.go
@@ -0,0 +1,78 @@
+// Package source holds the adapters, one per upstream source. An adapter
+// reads its source through the polite fetch client only, and turns what it
+// read into programs; it never validates scope itself (the collector
+// normalises every program the same way) and never sets timestamps other
+// than provenance.
+package source
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "regexp"
+ "strings"
+
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// Meta describes a source and its terms; it is carried in every manifest.
+type Meta struct {
+ Name string // id prefix of its programs, [a-z0-9-]
+ URL string
+ Terms string
+ Licence string
+ Attribution string
+}
+
+// Result is what one run of an adapter produced.
+type Result struct {
+ Programs []model.Program
+ // Unknown lists ids whose state could not be read this run (a host
+ // that did not answer, for example). The collector keeps their
+ // previous record instead of closing them.
+ Unknown []string
+ // Skipped counts source records that were not programs (no policy,
+ // malformed record …), for the log.
+ Skipped int
+}
+
+// Source is one adapter.
+type Source interface {
+ Meta() Meta
+ // Collect reads the whole source. An error means the source as a
+ // whole could not be read; the collector then keeps every previous
+ // record of the source unchanged.
+ Collect(ctx context.Context, c *fetch.Client) (*Result, error)
+}
+
+var slugRE = regexp.MustCompile(`[^a-z0-9]+`)
+
+// Slug turns a name into an id slug ([a-z0-9-], at most 100 characters).
+func Slug(name string) string {
+ s := strings.Trim(slugRE.ReplaceAllString(strings.ToLower(name), "-"), "-")
+ if len(s) > 100 {
+ s = strings.TrimRight(s[:100], "-")
+ }
+ return s
+}
+
+// UniqueSlugs gives every name a stable slug; names whose slugs collide
+// (or are empty) all get a short hash of the name appended, so the result
+// does not depend on the order of the source.
+func UniqueSlugs(names []string) map[string]string {
+ count := map[string]int{}
+ for _, n := range names {
+ count[Slug(n)]++
+ }
+ out := make(map[string]string, len(names))
+ for _, n := range names {
+ s := Slug(n)
+ if count[s] > 1 || s == "" {
+ sum := sha256.Sum256([]byte(n))
+ s = strings.TrimLeft(s+"-"+hex.EncodeToString(sum[:4]), "-")
+ }
+ out[n] = s
+ }
+ return out
+}
diff --git a/internal/source/source_test.go b/internal/source/source_test.go
new file mode 100644
index 0000000..6557a3c
--- /dev/null
+++ b/internal/source/source_test.go
@@ -0,0 +1,200 @@
+package source
+
+import (
+ "context"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/openctemio/programfeed/internal/fetch"
+ "github.com/openctemio/programfeed/internal/model"
+)
+
+// testClient sends every request, whatever its host, to srv.
+func testClient(srv *httptest.Server) *fetch.Client {
+ return fetch.New(fetch.Options{UserAgent: fetch.UserAgent("test"), MinInterval: time.Millisecond, AllowPrivate: true,
+ DialOverride: srv.Listener.Addr().String(), Timeout: 5 * time.Second})
+}
+
+func byID(ps []model.Program) map[string]model.Program {
+ out := map[string]model.Program{}
+ for _, p := range ps {
+ out[p.ID] = p
+ }
+ return out
+}
+
+func TestDioDB(t *testing.T) {
+ body, err := os.ReadFile("testdata/diodb.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/list.json" {
+ _, _ = w.Write(body)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer srv.Close()
+ res, err := DioDB{URL: srv.URL + "/list.json"}.Collect(context.Background(), testClient(srv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Programs) != 12 || res.Skipped != 5 {
+ t.Fatalf("%d programs, %d skipped", len(res.Programs), res.Skipped)
+ }
+ ps := byID(res.Programs)
+ ada := ps["disclose:ada-health"]
+ if len(ada.InScope) != 1 || ada.InScope[0].Value != "ada.com" || ada.InScope[0].Confidence != model.ConfidenceInferred || ada.ScopePublished {
+ t.Errorf("ada: %+v", ada.InScope)
+ }
+ yt := ps["disclose:youtube"]
+ if yt.Type != model.TypeBounty || yt.Platform != "self-hosted" || len(yt.InScope) != 0 || yt.Rules.SafeHarbour != model.SafeHarbourPartial {
+ t.Errorf("youtube: %+v", yt)
+ }
+ ph := ps["disclose:platform-hosted"]
+ if ph.Platform != "hackerone" || ph.Type != model.TypeBounty || len(ph.InScope) != 0 {
+ t.Errorf("a security.txt on a platform host must not become a target: %+v", ph)
+ }
+ if len(ps["disclose:odd-path-txt"].InScope) != 0 {
+ t.Error("a non-standard security.txt path must not become a target")
+ }
+ dead := ps["disclose:dead-link"]
+ if dead.Status != model.StatusPaused || dead.Type != model.TypeBounty || dead.Contact != (model.Contact{}) ||
+ dead.Rules.DisclosureDays != 90 || strings.Join(dead.Rules.Languages, ",") != "en,de" {
+ t.Errorf("dead link: %+v", dead)
+ }
+ dups := 0
+ for id := range ps {
+ if strings.HasPrefix(id, "disclose:duplicate-") {
+ dups++
+ }
+ }
+ if dups != 2 {
+ t.Errorf("colliding slugs must both get a hash suffix (%d)", dups)
+ }
+ // The public-suffix target is passed on; the collector's Normalize
+ // rejects it.
+ pst := ps["disclose:public-suffix-txt"]
+ model.Normalize(&pst)
+ if len(pst.InScope) != 0 || len(pst.Rejected) != 1 {
+ t.Errorf("public suffix: %+v / %+v", pst.InScope, pst.Rejected)
+ }
+}
+
+const goodTxt = `# Our policy
+Contact: mailto:security@agency.gov
+Contact: https://agency.gov/report
+Expires: 2099-01-01T00:00:00Z
+Policy: https://agency.gov/vulnerability-disclosure-policy
+Preferred-Languages: en, es
+Canonical: https://agency.gov/.well-known/security.txt
+`
+
+func TestParseSecurityTxt(t *testing.T) {
+ f, err := ParseSecurityTxt([]byte(goodTxt))
+ if err != nil || len(f.Contact) != 2 || f.Policy[0] != "https://agency.gov/vulnerability-disclosure-policy" || f.Expires.Year() != 2099 {
+ t.Fatalf("%v %+v", err, f)
+ }
+ signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + strings.ReplaceAll(goodTxt, "# Our", "- # Our") +
+ "-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n"
+ if f, err := ParseSecurityTxt([]byte(signed)); err != nil || len(f.Policy) != 1 {
+ t.Errorf("signed: %v", err)
+ }
+ for name, body := range map[string]string{
+ "html": "Contact: x",
+ "no contact": "Policy: https://a.gov/p\n",
+ "binary": "\xff\xfe\x00",
+ } {
+ if _, err := ParseSecurityTxt([]byte(body)); err == nil {
+ t.Errorf("%s: accepted", name)
+ }
+ }
+}
+
+func TestSecurityTxt(t *testing.T) {
+ seed := "Domain name,Domain type,Organization name,Suborganization name\n" +
+ "agency.gov,Federal,Agency,Office\nnopolicy.gov,Federal,NoPolicy,\nhtml.gov,Federal,Html,\nmissing.gov,Federal,Missing,\n" +
+ "down.gov,Federal,Down,\nredirect.gov,Federal,Redirect,\nexpired.gov,Federal,Expired,\nrobots.gov,Federal,Robots,\n" +
+ "localhost,Federal,Bad,\n10.0.0.1,Federal,Bad,\n"
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ host, _, _ := net.SplitHostPort(r.Host)
+ if host == "" {
+ host = r.Host
+ }
+ if r.URL.Path == "/robots.txt" {
+ if host == "robots.gov" {
+ _, _ = w.Write([]byte("User-agent: *\nDisallow: /.well-known/\n"))
+ return
+ }
+ http.NotFound(w, r)
+ return
+ }
+ if r.URL.Path == "/seed.csv" {
+ _, _ = w.Write([]byte(seed))
+ return
+ }
+ switch host {
+ case "agency.gov", "robots.gov":
+ _, _ = w.Write([]byte(goodTxt))
+ case "nopolicy.gov":
+ _, _ = w.Write([]byte("Contact: mailto:a@nopolicy.gov\n"))
+ case "html.gov":
+ _, _ = w.Write([]byte("not found"))
+ case "down.gov":
+ w.WriteHeader(http.StatusBadGateway)
+ case "redirect.gov":
+ http.Redirect(w, r, "http://other.gov/.well-known/security.txt", http.StatusFound)
+ case "other.gov":
+ _, _ = w.Write([]byte("Contact: mailto:a@other.gov\nPolicy: https://other.gov/p\n"))
+ case "expired.gov":
+ _, _ = w.Write([]byte("Contact: mailto:a@expired.gov\nExpires: 2020-01-01T00:00:00Z\nPolicy: https://expired.gov/p\n"))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer srv.Close()
+ st := SecurityTxt{SeedCSV: "http://seed.example.net/seed.csv", Scheme: "http", Workers: 3}
+ res, err := st.Collect(context.Background(), testClient(srv))
+ if err != nil {
+ t.Fatal(err)
+ }
+ ps := byID(res.Programs)
+ if len(ps) != 2 {
+ t.Fatalf("programs: %v", keys(ps))
+ }
+ a := ps["securitytxt:agency.gov"]
+ if a.Name != "Agency / Office (agency.gov)" || a.URL != "https://agency.gov/vulnerability-disclosure-policy" ||
+ a.Contact.Email != "security@agency.gov" || a.Contact.URL != "https://agency.gov/report" || a.Terms.SHA256 == "" ||
+ len(a.InScope) != 1 || a.InScope[0].Value != "agency.gov" || a.InScope[0].Confidence != model.ConfidenceInferred || a.Status != model.StatusOpen {
+ t.Errorf("agency: %+v", a)
+ }
+ if ps["securitytxt:expired.gov"].Status != model.StatusPaused {
+ t.Error("an expired security.txt must give a paused program")
+ }
+ // A host that did not answer is unknown (its previous record is kept);
+ // a definite "no file" is not.
+ if strings.Join(res.Unknown, ",") != "securitytxt:down.gov" {
+ t.Errorf("unknown = %v", res.Unknown)
+ }
+}
+
+func keys(m map[string]model.Program) []string {
+ var out []string
+ for k := range m {
+ out = append(out, k)
+ }
+ return out
+}
+
+func TestSlugs(t *testing.T) {
+ s := UniqueSlugs([]string{"Acme Corp", "acme-corp", "Zeta", "!!!"})
+ if s["Zeta"] != "zeta" || s["Acme Corp"] == s["acme-corp"] || !strings.HasPrefix(s["Acme Corp"], "acme-corp-") || s["!!!"] == "" {
+ t.Errorf("%v", s)
+ }
+}
diff --git a/internal/source/testdata/diodb.json b/internal/source/testdata/diodb.json
new file mode 100644
index 0000000..68d716c
--- /dev/null
+++ b/internal/source/testdata/diodb.json
@@ -0,0 +1,138 @@
+[
+ {
+ "program_name": ".nz Registry",
+ "policy_url": "https://registry.internetnz.nz/about/vulnerability-disclosure-policy/",
+ "launch_date": "",
+ "offers_bounty": "no",
+ "offers_swag": false,
+ "hall_of_fame": "",
+ "safe_harbor": "partial",
+ "public_disclosure": "",
+ "pgp_key": "",
+ "hiring": "",
+ "securitytxt_url": "",
+ "preferred_languages": "",
+ "policy_url_status": "alive",
+ "contact_email": "security@internetnz.net.nz"
+ },
+ {
+ "program_name": "Ashby",
+ "policy_url": "https://www.ashbyhq.com/resources/vulnerability-disclosure",
+ "contact_email": "security@ashbyhq.com",
+ "offers_bounty": "no",
+ "offers_swag": false,
+ "pgp_key": "",
+ "hall_of_fame": "",
+ "hiring": "",
+ "launch_date": "",
+ "policy_url_status": "alive",
+ "preferred_languages": "",
+ "public_disclosure": "",
+ "safe_harbor": "",
+ "securitytxt_url": ""
+ },
+ {
+ "program_name": "Firebase",
+ "policy_url": "https://g.co/vrp",
+ "contact_url": "https://g.co/vulnz",
+ "launch_date": "",
+ "offers_bounty": "no",
+ "offers_swag": true,
+ "hall_of_fame": "",
+ "safe_harbor": "none",
+ "public_disclosure": "",
+ "pgp_key": "",
+ "hiring": "",
+ "securitytxt_url": "",
+ "preferred_languages": "",
+ "policy_url_status": "alive"
+ },
+ {
+ "program_name": "YouTube",
+ "policy_url": "https://g.co/vrp",
+ "contact_url": "https://g.co/vulnz",
+ "launch_date": "",
+ "offers_bounty": "yes",
+ "offers_swag": "",
+ "hall_of_fame": "",
+ "safe_harbor": "partial",
+ "public_disclosure": "",
+ "pgp_key": "",
+ "hiring": "",
+ "securitytxt_url": "",
+ "preferred_languages": "",
+ "policy_url_status": "alive"
+ },
+ {
+ "program_name": "Ada Health",
+ "policy_url": "https://ada.com/security-vulnerability-disclosure-policy/",
+ "contact_url": "https://ada.com/security-vulnerability-disclosure-policy/",
+ "launch_date": "",
+ "offers_bounty": "no",
+ "offers_swag": false,
+ "hall_of_fame": "",
+ "safe_harbor": "partial",
+ "public_disclosure": "",
+ "pgp_key": "https://ada.com/pgp-key.txt",
+ "hiring": "https:/ada.com/careers/",
+ "securitytxt_url": "https://ada.com/.well-known/security.txt",
+ "preferred_languages": "en, de",
+ "policy_url_status": "alive",
+ "contact_email": "security@ada.com"
+ },
+ {
+ "program_name": "",
+ "policy_url": "https://nameless.org/security"
+ },
+ {
+ "program_name": "Script Policy",
+ "policy_url": "javascript:alert(1)"
+ },
+ {
+ "program_name": "No Policy"
+ },
+ {
+ "program_name": "Platform Hosted",
+ "policy_url": "https://hackerone.com/platform-hosted",
+ "securitytxt_url": "https://hackerone.com/.well-known/security.txt",
+ "offers_bounty": true,
+ "safe_harbor": "full"
+ },
+ {
+ "program_name": "Public Suffix Txt",
+ "policy_url": "https://blog.github.io/policy",
+ "securitytxt_url": "https://github.io/.well-known/security.txt"
+ },
+ {
+ "program_name": "Odd Path Txt",
+ "policy_url": "https://oddpath.org/policy",
+ "securitytxt_url": "https://oddpath.org/files/security.txt"
+ },
+ {
+ "program_name": "Dead Link",
+ "policy_url": "https://dead.org/policy",
+ "policy_url_status": "dead",
+ "offers_bounty": "partial",
+ "contact_email": "not an email",
+ "contact_url": "javascript:x",
+ "disclosure_timeline_days": "90",
+ "preferred_languages": "en, de ,,"
+ },
+ {
+ "program_name": "Duplicate",
+ "policy_url": "https://dup-one.org/policy"
+ },
+ {
+ "program_name": "duplicate!",
+ "policy_url": "https://dup-two.org/policy"
+ },
+ {
+ "program_name": "Bell\u0007Name",
+ "policy_url": "https://bell.org/policy"
+ },
+ "not an object",
+ {
+ "program_name": 42,
+ "policy_url": "https://number.org/policy"
+ }
+]
diff --git a/keys/README.md b/keys/README.md
new file mode 100644
index 0000000..6be20a1
--- /dev/null
+++ b/keys/README.md
@@ -0,0 +1,49 @@
+# Keys
+
+This directory holds only public material:
+
+| File | What |
+|---|---|
+| `root-keyid.txt` | key id of the offline root (`SHA256:…`); platforms pin it |
+| `keyset.dsse.json` | the current key set, signed by the root |
+
+## First setup (maintainer, on an offline machine)
+
+```sh
+go build -o programfeed ./cmd/programfeed
+
+# 1. The offline root. Keep root.key offline (hardware-backed storage or an
+# encrypted offline medium, plus a backup). It never goes to CI.
+./programfeed keys root-init --out root.key # prints the root key id
+
+# 2. The online signing key.
+./programfeed keys gen --out signing.key # prints its public key
+
+# 3. The key set: the root allows the online key for up to 180 days.
+./programfeed keys sign-keyset --root root.key --key --version 1 --days 180 --out keys/keyset.dsse.json
+```
+
+Then:
+
+- commit `keys/root-keyid.txt` (the root key id) and `keys/keyset.dsse.json`;
+- create the GitHub environment **`publish`**: deployment branches limited
+ to the default branch; add the secret **`PROGRAMFEED_SIGNING_KEY`** with the
+ content of `signing.key`; delete `signing.key` from the machine;
+- give OpenCTEM platforms the root key id (`PROGRAMFEED_ROOT_KEY_ID`).
+
+Use a root distinct from the vulnerability feed's root: one compromised root
+then cannot sign for both feeds.
+
+Until `keys/root-keyid.txt` and `keys/keyset.dsse.json` are committed, the
+scheduled publish workflow stops at its first step with an error saying so.
+
+## Rotation and revocation
+
+Before the key set expires, or at once if the online key may have leaked:
+generate a new online key, sign a key set with a **higher version** listing
+only the keys still trusted, commit it and replace the environment secret.
+Platforms that have seen the new version refuse the older key set from then
+on.
+
+If the root itself is compromised, generate a new root and have every
+platform pin its new key id.
diff --git a/schema/change.schema.json b/schema/change.schema.json
new file mode 100644
index 0000000..3a11ebe
--- /dev/null
+++ b/schema/change.schema.json
@@ -0,0 +1,19 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://github.com/openctemio/programfeed/schema/change.schema.json",
+ "title": "Change",
+ "description": "One entry of a run's change log: one line of changes.jsonl.gz. Entries are sorted by program, then kind, then scope and target.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["sequence", "program", "kind"],
+ "properties": {
+ "sequence": {"type": "integer", "minimum": 1, "description": "The bundle sequence that made the change."},
+ "program": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$"},
+ "kind": {"enum": ["program_added", "program_reopened", "status_changed", "program_modified", "rules_changed", "terms_changed",
+ "target_added", "target_modified", "target_removed", "program_closed", "program_dropped"],
+ "description": "program_dropped: the program is no longer in the feed (closed for more than 180 days); every other kind names a program present in the same bundle."},
+ "scope": {"enum": ["in", "out"], "description": "Target changes only."},
+ "target": {"$ref": "program.schema.json#/$defs/target", "description": "Target changes only: the target after the change (before, for target_removed)."},
+ "fields": {"type": "array", "maxItems": 16, "items": {"type": "string"}, "description": "For program_modified, rules_changed, status_changed and target_modified: the JSON fields that changed."}
+ }
+}
diff --git a/schema/program.schema.json b/schema/program.schema.json
new file mode 100644
index 0000000..b45c898
--- /dev/null
+++ b/schema/program.schema.json
@@ -0,0 +1,104 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://github.com/openctemio/programfeed/schema/program.schema.json",
+ "title": "Program",
+ "description": "One public bug-bounty or vulnerability-disclosure program: one line of programs.jsonl.gz.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["id", "source", "platform", "name", "url", "type", "status", "offers_bounty", "scope_published",
+ "in_scope", "out_of_scope", "rejected", "rules", "terms", "contact", "provenance", "first_seen", "last_seen", "last_changed"],
+ "properties": {
+ "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$",
+ "description": ":. Stable; owned by exactly one source adapter."},
+ "source": {"type": "string", "description": "The adapter that produced the record (the id prefix)."},
+ "platform": {"type": "string", "maxLength": 64, "description": "Where the program is hosted: \"self-hosted\" or the name of the hosting platform."},
+ "name": {"type": "string", "minLength": 1, "maxLength": 200},
+ "url": {"type": "string", "format": "uri", "maxLength": 2048, "description": "The program's policy page (http or https)."},
+ "type": {"enum": ["bounty", "vdp"]},
+ "status": {"enum": ["open", "paused", "closed"],
+ "description": "closed: the source no longer lists the program (kept 180 days, then dropped). paused: listed but not currently accepting (dead policy link, expired security.txt)."},
+ "offers_bounty": {"type": "boolean", "description": "Always equal to type == bounty."},
+ "scope_published": {"type": "boolean", "description": "True when the source carries the program's own structured scope. When false, in_scope holds at most inferred suggestions."},
+ "in_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}, "description": "Sorted by (type, value), no duplicates. Out of scope wins: a value listed in both appears only in out_of_scope."},
+ "out_of_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}},
+ "rejected": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/rejected"}, "description": "Targets the program listed that the feed's scope rules refused, with the reason."},
+ "rules": {"$ref": "#/$defs/rules"},
+ "terms": {"$ref": "#/$defs/terms"},
+ "contact": {"$ref": "#/$defs/contact"},
+ "provenance": {"$ref": "#/$defs/provenance"},
+ "first_seen": {"type": "string", "format": "date-time"},
+ "last_seen": {"type": "string", "format": "date-time", "description": "Last run in which the source listed the program."},
+ "last_changed": {"type": "string", "format": "date-time", "description": "Last run in which any content (not timestamps) changed."},
+ "closed_at": {"type": "string", "format": "date-time", "description": "Present exactly when status is closed."}
+ },
+ "$defs": {
+ "target": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["type", "value", "confidence"],
+ "properties": {
+ "type": {"enum": ["domain", "wildcard", "ip", "cidr", "url", "mobile_app", "source_repo", "other"]},
+ "value": {"type": "string", "maxLength": 512,
+ "description": "Canonical form: domain = lower-case A-label, not a public suffix, not a reserved name; wildcard = \"*.\" + such a domain; ip = public unicast address; cidr = masked public prefix of at least /16 (IPv4) or /48 (IPv6); url = http(s) without credentials or fragment, host checked as above."},
+ "confidence": {"enum": ["published", "inferred"],
+ "description": "published: the program's own scope. inferred: derived by the collector (for example the host serving the program's security.txt); a suggestion, never an authorisation to test."},
+ "eligible_for_bounty": {"type": "boolean"},
+ "max_severity": {"enum": ["none", "low", "medium", "high", "critical"]},
+ "notes": {"type": "string", "maxLength": 500}
+ }
+ },
+ "rejected": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["scope", "type", "value", "reason"],
+ "properties": {
+ "scope": {"enum": ["in", "out"]},
+ "type": {"type": "string", "maxLength": 32},
+ "value": {"type": "string", "maxLength": 512},
+ "reason": {"type": "string", "minLength": 1, "maxLength": 200}
+ }
+ },
+ "rules": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["testing_restrictions", "required_headers", "safe_harbour", "languages"],
+ "properties": {
+ "summary": {"type": "string", "maxLength": 4000},
+ "testing_restrictions": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}},
+ "rate_limit": {"type": "string", "maxLength": 200},
+ "required_headers": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}},
+ "safe_harbour": {"enum": ["full", "partial", "none", "unknown"]},
+ "disclosure_days": {"type": "integer", "minimum": 0, "maximum": 3650},
+ "languages": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}}
+ }
+ },
+ "terms": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["url"],
+ "properties": {
+ "url": {"type": "string", "format": "uri", "maxLength": 2048},
+ "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "SHA-256 of the exact bytes of the terms document the collector read at fetched_at. Absent when the collector did not read the terms."},
+ "fetched_at": {"type": "string", "format": "date-time", "description": "Present exactly when sha256 is."}
+ }
+ },
+ "contact": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "email": {"type": "string", "maxLength": 254},
+ "url": {"type": "string", "maxLength": 2048, "description": "http(s) or mailto."}
+ }
+ },
+ "provenance": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["source", "source_url", "fetched_at"],
+ "properties": {
+ "source": {"type": "string"},
+ "source_url": {"type": "string", "format": "uri", "maxLength": 2048},
+ "fetched_at": {"type": "string", "format": "date-time"}
+ }
+ }
+ }
+}
diff --git a/seeds/securitytxt.txt b/seeds/securitytxt.txt
new file mode 100644
index 0000000..d5c0417
--- /dev/null
+++ b/seeds/securitytxt.txt
@@ -0,0 +1,3 @@
+# Extra domains whose security.txt the securitytxt adapter reads, one per line,
+# optionally followed by the organisation name. Only add domains whose owner
+# publishes a vulnerability disclosure policy for public use.