diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..14f295a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Format + run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } + - name: Vet + run: go vet ./... + - name: Test (recorded fixtures, no network) + run: go test -race ./... + - name: Fuzz the scope normaliser briefly + run: go test ./internal/model -run '^$' -fuzz '^FuzzNormalizeTarget$' -fuzztime 20s + - name: Known vulnerabilities + run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..4d9c56f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,135 @@ +name: Publish bundle + +# Daily: build the next program bundle from the previous release and the +# sources, sign it, verify it and publish it as a GitHub Release. +# +# Two jobs on purpose. "build" talks to the sources and never sees the +# signing key. "publish" runs in the protected "publish" environment (only +# the default branch can deploy to it), holds the signing key and nothing +# else, and only signs, verifies and uploads what "build" produced. + +on: + schedule: + - cron: "41 3 * * *" + workflow_dispatch: + inputs: + allow_mass_change: + description: "Accept a source that lost more than half of its programs (only after checking the source)" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: publish + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + ready: ${{ steps.keys.outputs.ready }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Key ceremony done? + id: keys + run: | + if [ -s keys/root-keyid.txt ] && [ -s keys/keyset.dsse.json ]; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::No keys yet (keys/root-keyid.txt, keys/keyset.dsse.json): nothing is published until the key ceremony in keys/README.md is done" + echo "ready=false" >> "$GITHUB_OUTPUT" + fi + - if: steps.keys.outputs.ready == 'true' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - if: steps.keys.outputs.ready == 'true' + name: Test + run: go test ./... + - if: steps.keys.outputs.ready == 'true' + name: Build the CLI + run: go build -trimpath -ldflags "-X main.version=${GITHUB_SHA::12}" -o programfeed ./cmd/programfeed + - if: steps.keys.outputs.ready == 'true' + name: Restore the HTTP cache (conditional requests only; every body is re-validated) + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: httpcache + key: httpcache-${{ github.run_id }} + restore-keys: httpcache- + - if: steps.keys.outputs.ready == 'true' + name: Download the previous release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p prev + if gh release view --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + gh release download --repo "$GITHUB_REPOSITORY" --dir prev --pattern 'snapshot*' --pattern 'latest.dsse.json' --pattern 'keyset.dsse.json' + echo "PREV=--prev=prev" >> "$GITHUB_ENV" + fi + - if: steps.keys.outputs.ready == 'true' + name: Build the bundle + env: + MASS: ${{ inputs.allow_mass_change && '--allow-mass-change' || '' }} + run: ./programfeed build --out out --cache httpcache --root-key-id "$(cat keys/root-keyid.txt)" $PREV $MASS + - if: steps.keys.outputs.ready == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bundle + path: out/ + retention-days: 3 + if-no-files-found: error + + publish: + needs: build + if: needs.build.outputs.ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: publish + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - name: Build the CLI + run: go build -trimpath -o programfeed ./cmd/programfeed + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bundle + path: out + - name: Sign + env: + PROGRAMFEED_SIGNING_KEY: ${{ secrets.PROGRAMFEED_SIGNING_KEY }} + run: ./programfeed sign --dir out --keyset keys/keyset.dsse.json --root-key-id "$(cat keys/root-keyid.txt)" + - name: Verify as the platform does + run: ./programfeed verify --dir out --root-key-id "$(cat keys/root-keyid.txt)" + - name: Publish the release + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(python3 -c 'import json,base64,sys; e=json.load(open("out/latest.dsse.json")); print(json.loads(base64.b64decode(e["payload"]))["tag"])')" + cat > notes.md <<'NOTES' + Public program bundle for OpenCTEM: a signed snapshot of public bug-bounty and vulnerability-disclosure programs, and the delta from the previous release. + + Targets marked "inferred" are suggestions, not a program's published scope. See NOTICE for the terms of every source. + NOTES + gh release create "$tag" --repo "$GITHUB_REPOSITORY" --title "$tag" --notes-file notes.md out/* + - name: Keep the last 60 releases + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 --json tagName,createdAt \ + --jq 'sort_by(.createdAt) | reverse | .[60:] | .[].tagName' | + while read -r tag; do + gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58de1a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/programfeed +/out/ +/prev/ +/httpcache/ +*.key diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..eea8991 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing + +Thanks for helping. Two kinds of contributions are common: adding seed +domains for the security.txt adapter, and adding a new source adapter. + +## Adding seed domains + +Add lines to `seeds/securitytxt.txt` (`domain [organisation name]`). Only add +domains whose owner publishes a vulnerability disclosure policy for public +use. The domain must already serve `/.well-known/security.txt` with a +`Contact` and a `Policy` field, or the collector skips it. + +## Adding a source adapter + +### 1. Check that the source may be collected + +Before writing code, open an issue with: + +- the official API or machine-readable listing (documentation link); +- whether it needs authentication (the feed never logs in, uses cookies or + tokens, runs JavaScript or bypasses bot protection: a source that needs + any of that is not accepted); +- its terms of service and robots.txt, quoted: they must allow automated + collection **and redistribution**, since the feed is published openly + (or the publisher must have given written permission, linked in the + issue); +- the licence of the data and the attribution it requires; +- its rate limits. + +### 2. Write the adapter + +An adapter is one file in `internal/source/` implementing `source.Source`: + +```go +type Source interface { + Meta() Meta // name, URL, terms, licence, attribution + Collect(ctx context.Context, c *fetch.Client) (*Result, error) +} +``` + +Rules: + +- Use only the `*fetch.Client` it is given (it handles the User-Agent, + robots.txt, rate limits, conditional requests, size caps and the + public-address check). Pass a size cap that fits the source. +- Program ids are `:`; use `source.UniqueSlugs` for names. + Ids must be stable across runs. The collector refuses programs outside the + adapter's own prefix. +- Fill `Provenance` (source, source URL, fetched-at time). Leave + `first_seen`, `last_seen`, `last_changed` and `closed_at` to the collector. +- Put the program's own scope in `InScope`/`OutOfScope` with + `Confidence: published` and set `ScopePublished`. Anything the adapter + derives itself is `Confidence: inferred`, with a note saying how. +- Do not filter scope yourself: the collector normalises every target and + records refused ones with the reason. +- Return an error only when the source as a whole could not be read (the + collector then keeps every previous record). Put ids whose state is + unknown this run (a host that did not answer) in `Result.Unknown`. +- Hostile input is expected: decode defensively, cap counts, never trust a + field's type. + +Register it in `cmd/programfeed/main.go` (`sources`), and add its row to the +sources table in `README.md` and its terms to `NOTICE`. + +### 3. Test it with recorded fixtures + +Tests never touch the network. Record a small, representative response +under `internal/source/testdata/` (trimmed, no personal data beyond what the +source publishes), serve it with `httptest`, and point the fetch client at +it (`fetch.Options{AllowPrivate: true, DialOverride: ...}`, see +`source_test.go`). Include poisoned entries: malformed records, wrong field +types, control characters, public-suffix or private-range scope. + +### 4. Check before opening the pull request + +```sh +gofmt -l . +go vet ./... +go test -race ./... +go run golang.org/x/vuln/cmd/govulncheck@latest ./... +``` + +Pull requests target `main`. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..4ee04be --- /dev/null +++ b/NOTICE @@ -0,0 +1,25 @@ +programfeed +Copyright 2026 The OpenCTEM Authors + +This software is licensed under the Apache License, Version 2.0 (LICENSE). + +The bundles this software publishes contain data from third-party sources, +under their own terms: + +disclose.io diodb (https://github.com/disclose/diodb) + Program directory data, dedicated to the public domain under CC0 1.0 + Universal: https://github.com/disclose/diodb/blob/master/LICENSE + +CISA dotgov-data (https://github.com/cisagov/dotgov-data) + The list of US federal .gov domains used as the security.txt seed list, + dedicated to the public domain under CC0 1.0 Universal. + +security.txt files (RFC 9116) + Each program record from the securitytxt adapter is derived from the + security.txt file its domain publishes for automated retrieval; the + record's provenance names the file. The policies those files link to + belong to their publishers and are not copied into the bundles. + +Program names, policies and terms remain the property of the organisations +that publish them. Listing a program in the feed does not imply that the +organisation endorses OpenCTEM, nor that any testing is authorised. diff --git a/README.md b/README.md new file mode 100644 index 0000000..9f11206 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +# programfeed + +The public program collector of [OpenCTEM](https://github.com/openctemio/openctem). +Once a day it reads open, machine-readable sources of **public bug-bounty and +vulnerability-disclosure programs**, normalises them into one record format, +validates every record and every scope entry, signs the result and publishes +it as a GitHub Release. OpenCTEM platforms never call the sources +themselves: they download (or, air-gapped, upload) a bundle, verify it and +import it into their public program monitor. + +The feed says **who runs a program, where its policy is, whether it pays, +what scope it publishes and how that changed**. It is not an authorisation to +test anything: a subscriber reads and accepts a program's terms before any +active testing, and targets marked `inferred` are suggestions only. + +## Sources + +| Source | Adapter | What it gives | Licence / terms | Status | +|---|---|---|---|---| +| [disclose.io diodb](https://github.com/disclose/diodb) `program-list.json` | `diodb` | ~2,400 programs: name, policy URL, contact, bounty yes/no, safe harbour, disclosure timeline, languages. No structured scope; the host of a program's own security.txt becomes an `inferred` target. | CC0-1.0 | supported | +| [security.txt](https://www.rfc-editor.org/rfc/rfc9116) (RFC 9116) over a seed list | `securitytxt` | A domain whose `/.well-known/security.txt` has a `Contact` and a `Policy` is a self-hosted disclosure program; the domain is an `inferred` target, the file's SHA-256 is the terms hash. Expired files give a `paused` program. | files published for automated retrieval | supported | +| Seed: [CISA dotgov-data](https://github.com/cisagov/dotgov-data) `current-federal.csv` | `securitytxt` seed | US federal .gov domains and their organisations (vulnerability disclosure policies under BOD 20-01). | CC0-1.0 | supported | +| Seed: `seeds/securitytxt.txt` | `securitytxt` seed | Extra domains added by pull request. | — | supported | + +Not collected, and why (pull requests that scrape these will be declined): + +| Source | Reason | +|---|---| +| HackerOne, Intigriti | program APIs require an account token; the feed never logs in | +| Bugcrowd, YesWeHack | website terms forbid redistributing their data | +| Immunefi | terms of use forbid automated access to the website without written consent | +| Open Bug Bounty | listing is behind bot protection | +| HackenProof, WhiteHub, BugBase and other regional platforms | no official machine-readable listing | +| third-party dumps of platform scopes | the data comes from platforms whose terms forbid redistribution | + +A source moves to "supported" when it offers an official machine-readable +listing whose terms allow collection and redistribution (or gives written +permission). + +### How the collector behaves on the network + +- User-Agent `openctem-programfeed/ (+https://github.com/openctemio/programfeed; security@openctem.io)` + and a `From: security@openctem.io` header; +- `robots.txt` honoured for every origin (product token `openctem-programfeed`; + an unreachable robots.txt means "do not fetch", RFC 9309); +- at most one request per second per host; conditional requests + (`If-None-Match`, `If-Modified-Since`) from a cache kept between runs; +- response size caps (32 KiB for a security.txt, 32 MiB for a list); +- connections only to public unicast addresses (a seed list or redirect + cannot point it at internal services); https only, at most 3 redirects; +- no login, no cookies, no JavaScript, no bot-protection bypass. + +## What a release contains + +| Asset | Content | +|---|---| +| `latest.dsse.json` | signed pointer: sequence, tag, snapshot and delta manifests, expiry | +| `snapshot.manifest.dsse.json` | signed manifest of the full snapshot (files, SHA-256, sizes, record counts, sources with licence and as-of time, stats) | +| `snapshot-programs.jsonl.gz` | every program | +| `snapshot-changes.jsonl.gz` | the change log of this run | +| `delta.manifest.dsse.json`, `delta-programs.jsonl.gz`, `delta-changes.jsonl.gz` | the programs that changed since the previous sequence (complete new records) and the same change log | +| `keyset.dsse.json` | the key set the manifests are signed under | + +A platform reads +`https://github.com/openctemio/programfeed/releases/latest/download/latest.dsse.json` +and the files of the release it names. + +## Record format + +One JSON object per line. The JSON Schemas are in [`schema/`](schema/): +[`program.schema.json`](schema/program.schema.json) and +[`change.schema.json`](schema/change.schema.json). + +```json +{"id":"securitytxt:agency.gov","source":"securitytxt","platform":"self-hosted","name":"Agency (agency.gov)", + "url":"https://agency.gov/vulnerability-disclosure-policy","type":"vdp","status":"open","offers_bounty":false, + "scope_published":false, + "in_scope":[{"type":"domain","value":"agency.gov","confidence":"inferred","notes":"Domain whose security.txt names this policy; not a published scope."}], + "out_of_scope":[],"rejected":[], + "rules":{"testing_restrictions":[],"required_headers":[],"safe_harbour":"unknown","languages":["en"]}, + "terms":{"url":"https://agency.gov/.well-known/security.txt","sha256":"…","fetched_at":"2026-10-10T03:41:00Z"}, + "contact":{"email":"security@agency.gov"}, + "provenance":{"source":"securitytxt","source_url":"https://agency.gov/.well-known/security.txt","fetched_at":"2026-10-10T03:41:00Z"}, + "first_seen":"2026-10-10T03:41:00Z","last_seen":"2026-10-10T03:41:00Z","last_changed":"2026-10-10T03:41:00Z"} +{"sequence":2,"program":"securitytxt:agency.gov","kind":"target_added","scope":"in","target":{"type":"domain","value":"agency.gov","confidence":"inferred"}} +``` + +Key rules: + +- `id` is `:`, stable, owned by one adapter. +- `type` is `bounty` or `vdp`; `status` is `open`, `paused` (listed but not + accepting: dead policy link, expired security.txt) or `closed` (the source + no longer lists it). A closed program stays 180 days, then a + `program_dropped` change removes it. +- Scope entries are typed (`domain`, `wildcard`, `ip`, `cidr`, `url`, + `mobile_app`, `source_repo`, `other`), canonical, sorted and deduplicated; + out of scope wins over in scope. +- `confidence`: `published` (the program's own scope) or `inferred` (derived + by the collector). `scope_published` says whether the source carries the + program's own structured scope at all. +- `terms.sha256` is the SHA-256 of the exact terms document the collector + read, when it read one. +- `rejected` lists what a program listed that the scope rules refused, with + the reason. + +### Scope rules + +A scope entry is refused (moved to `rejected`) when it is: a public suffix or +a wildcard over one (`com`, `*.co.uk`, `*.github.io`); a reserved or +non-public name (`localhost`, `.local`, `.internal`, `.test`, `example.com` +…); a private, loopback, link-local, CGNAT, documentation, multicast or +otherwise reserved address or range; a CIDR wider than /16 (IPv4) or /48 +(IPv6); a URL that is not http(s), has credentials, or whose host fails the +rules above; malformed, over 512 bytes, or carrying control or bidi +characters. Each list is capped at 2,000 entries. + +## Verifying a bundle yourself + +```sh +go build -o programfeed ./cmd/programfeed +gh release download --repo openctemio/programfeed --dir bundle +./programfeed verify --dir bundle --root-key-id "$(cat keys/root-keyid.txt)" +``` + +The envelopes, key set, sequence, expiry and caps follow the same +conventions as the OpenCTEM vulnerability feed; only the payload types +(`application/vnd.openctem.programfeed.*`) and record files differ, so a +key set or manifest of one feed is never accepted by the other. + +## Building locally + +```sh +go test ./... # recorded fixtures, no network +go build -o programfeed ./cmd/programfeed +./programfeed build --out out --cache httpcache # every source +./programfeed build --out out --sources diodb # one source +``` + +A key is needed only to sign; see `keys/README.md`. + +## Security + +See [SECURITY.md](SECURITY.md) for the trust model and how to report a +problem. Adding a source: [CONTRIBUTING.md](CONTRIBUTING.md). + +## Licence + +Apache-2.0 (the code). The data keeps its sources' terms; see `NOTICE`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..60b1fd5 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,80 @@ +# Security + +## Reporting + +Please report vulnerabilities privately to **security@openctem.io** or through +GitHub's private vulnerability reporting on this repository. Do not open a +public issue. + +If you run a program listed in the feed and want it corrected or removed, +write to the same address. + +## Threat model + +A bundle tells every OpenCTEM platform which public programs exist and what +they cover. A forged, stale or poisoned bundle could point subscribers' +monitoring at organisations that never invited it, hide a program's closure +or its new out-of-scope entries, or carry hostile strings into the platform. +The collector itself talks to many third-party hosts and must not become a +way to reach internal services. + +### Integrity and freshness (same scheme as the OpenCTEM vulnerability feed) + +- **Offline root.** An Ed25519 root key, kept offline by the maintainer, + only signs *key sets*. Platforms pin its key id (`keys/root-keyid.txt`). + The root is not shared with any other feed, and the key set payload type + is specific to this feed, so a key set or manifest of another feed is + refused. +- **Key set.** `keys/keyset.dsse.json` lists the online signing keys, is + valid for at most 180 days and has a version that only goes up. A + platform refuses a key set whose version is below one it has accepted. +- **Online key.** The signing key is a secret of the `publish` GitHub + environment, which only the default branch can deploy to. Only the + publish job runs there; it signs, verifies and uploads what the build job + produced. The build job, which talks to the sources, never sees the key. +- **Rollback.** Every bundle has a sequence number. A platform refuses a + sequence it has already applied, and a delta whose base is not the + sequence it has. +- **Freeze.** Every manifest and pointer expires 7 days after it was made. +- **Integrity.** The signed manifest holds the SHA-256, size and record + count of every file; a mismatch refuses the whole bundle. +- **Size.** Manifests are capped at 1 MiB, files at 128 MiB compressed and + 1 GiB decompressed, records at 4 MiB, and a bundle at 100,000 programs. + +### Poisoned data + +- Every record is validated before signing and again by the importer: + strict JSON (unknown fields refused), id grammar, http(s) URLs without + credentials, bounded strings without control or bidi characters, enums, + timestamps in order. +- Every scope entry is normalised; public suffixes and wildcards over them, + reserved names, private/reserved/documentation addresses and ranges, + CIDRs wider than /16 (IPv4) or /48 (IPv6), and malformed entries are + moved to the program's `rejected` list with the reason. A bundle with a + scope entry the rules refuse is refused whole by the verifier. +- Out of scope wins over in scope. +- Targets the collector derived (`confidence: inferred`) are marked as such; + the feed never presents a guess as a program's published scope. +- An adapter can only write programs under its own id prefix. +- **Outage guard.** A source that suddenly reports fewer than half of the + programs it had (from at least 20) is treated as broken for that run and + its previous records are kept; a maintainer can override after checking + the source. A host that does not answer keeps its previous record instead + of being closed. A closed program stays visible for 180 days before it is + dropped. + +### The collector on the network + +- Connections only to public unicast addresses, checked at connect time + (after DNS), so neither a seed list entry nor a redirect can reach + loopback, private ranges or cloud metadata addresses. +- https only, at most 3 redirects, each re-checked against robots.txt. +- Size caps on every response; timeouts on every request. +- No credentials, cookies or tokens are ever sent to a source. +- The HTTP cache between runs only feeds conditional requests; every body + served from it goes through the same parsing and validation. + +## Key ceremony (maintainers) + +See `keys/README.md`. Never commit a private key; the repository holds only +public material (`root-keyid.txt`, `keyset.dsse.json`). diff --git a/cmd/programfeed/main.go b/cmd/programfeed/main.go new file mode 100644 index 0000000..a635c46 --- /dev/null +++ b/cmd/programfeed/main.go @@ -0,0 +1,250 @@ +// Command programfeed builds, signs and verifies OpenCTEM program bundles +// (public bug-bounty and vulnerability-disclosure programs), and manages +// the signing keys. +// +// programfeed build --out DIR [--prev DIR] [--root-key-id ID] [--cache DIR] [--sources a,b] [--seed-file FILE] [--allow-mass-change] +// programfeed sign --dir DIR --keyset FILE --root-key-id ID (key in $PROGRAMFEED_SIGNING_KEY) +// programfeed verify --dir DIR --root-key-id ID [--applied N] [--allow-expired] +// programfeed keys root-init --out FILE +// programfeed keys gen --out FILE +// programfeed keys sign-keyset --root FILE --key PUBLIC_KEY [--key ...] --version N [--days 180] --out FILE +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "strings" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/collect" + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/source" +) + +// version is set at build time (-ldflags "-X main.version=..."). +var version = "dev" + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + if err := run(ctx, os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "programfeed:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("usage: programfeed build|sign|verify|keys ...") + } + switch args[0] { + case "build": + return cmdBuild(ctx, args[1:]) + case "sign": + return cmdSign(args[1:]) + case "verify": + return cmdVerify(args[1:]) + case "keys": + return cmdKeys(args[1:]) + } + return fmt.Errorf("unknown command %q", args[0]) +} + +func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) } + +// sources returns the configured adapters, in order. +func sources(names, seedFile string) ([]source.Source, error) { + var out []source.Source + for _, n := range strings.Split(names, ",") { + switch strings.TrimSpace(n) { + case "diodb": + out = append(out, source.DioDB{}) + case "securitytxt": + st := source.SecurityTxt{Progress: func(done, total int) { logf("securitytxt: %d/%d domains", done, total) }} + if seedFile != "" { + b, err := os.ReadFile(seedFile) + if err != nil { + return nil, err + } + st.Extra = source.ParseSeedFile(b) + } + out = append(out, st) + case "": + default: + return nil, fmt.Errorf("unknown source %q", n) + } + } + if len(out) == 0 { + return nil, errors.New("no source selected") + } + return out, nil +} + +func cmdBuild(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("build", flag.ContinueOnError) + out := fs.String("out", "", "output directory") + prevDir := fs.String("prev", "", "previous release (verified before use)") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + mass := fs.Bool("allow-mass-change", false, "accept a source that suddenly lost more than half of its programs") + cache := fs.String("cache", "", "HTTP cache directory for conditional requests") + names := fs.String("sources", "diodb,securitytxt", "comma-separated adapters") + seedFile := fs.String("seed-file", "seeds/securitytxt.txt", "extra security.txt seed domains (empty to skip)") + if err := fs.Parse(args); err != nil { + return err + } + if *out == "" { + return errors.New("build: --out is required") + } + srcs, err := sources(*names, *seedFile) + if err != nil { + return err + } + now := time.Now().UTC() + var prev *bundle.Snapshot + if *prevDir != "" { + v, err := bundle.VerifyDir(*prevDir, bundle.VerifyOptions{PinnedRoot: *root, Now: now, AllowExpired: true, SkipRecordChk: true}) + if err != nil { + return fmt.Errorf("previous release: %w", err) + } + if prev, err = bundle.ReadRecords(*prevDir, v.Snapshot); err != nil { + return fmt.Errorf("previous release: %w", err) + } + logf("previous release: sequence %d, %d programs", prev.Sequence, len(prev.Programs)) + } + client := fetch.New(fetch.Options{UserAgent: fetch.UserAgent(version), CacheDir: *cache, Timeout: 20 * time.Second}) + next, err := collect.Build(ctx, prev, srcs, client, collect.Options{Now: now, AllowMassChange: *mass, Log: logf}) + if err != nil { + return err + } + snap, delta, _, err := bundle.Write(*out, prev, next, bundle.WriteOptions{Now: now, + Collector: bundle.Collector{Version: version, Commit: os.Getenv("GITHUB_SHA")}}) + if err != nil { + return err + } + st := snap.Stats + logf("built sequence %d: %d programs (%d open, %d bounty), %d targets, %d rejected targets, %d changes", + snap.Sequence, st.Programs, st.Open, st.Bounty, st.Targets, st.Rejected, st.Changes) + if delta != nil { + logf("delta from %d: %d programs (+%d added, %d closed, %d dropped)", delta.BaseSequence, delta.Stats.Programs, + delta.Stats.Added, delta.Stats.Closed, delta.Stats.Dropped) + } + return nil +} + +func cmdSign(args []string) error { + fs := flag.NewFlagSet("sign", flag.ContinueOnError) + dir := fs.String("dir", "", "bundle directory") + keyset := fs.String("keyset", "keys/keyset.dsse.json", "key set envelope") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + if err := fs.Parse(args); err != nil { + return err + } + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(os.Getenv("PROGRAMFEED_SIGNING_KEY"))) + if err != nil || len(seed) != ed25519.SeedSize { + return errors.New("sign: PROGRAMFEED_SIGNING_KEY must be a base64 32-byte Ed25519 seed") + } + ks, err := os.ReadFile(*keyset) + if err != nil { + return err + } + return bundle.Sign(*dir, ed25519.NewKeyFromSeed(seed), ks, *root, time.Now()) +} + +func cmdVerify(args []string) error { + fs := flag.NewFlagSet("verify", flag.ContinueOnError) + dir := fs.String("dir", "", "bundle directory") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + applied := fs.Uint64("applied", 0, "last applied sequence") + expired := fs.Bool("allow-expired", false, "accept an expired bundle") + if err := fs.Parse(args); err != nil { + return err + } + v, err := bundle.VerifyDir(*dir, bundle.VerifyOptions{PinnedRoot: *root, AppliedSeq: *applied, Now: time.Now(), AllowExpired: *expired}) + if err != nil { + return err + } + logf("verified sequence %d (snapshot %d programs, %d targets; key set v%d)", v.Latest.Sequence, + v.Snapshot.Stats.Programs, v.Snapshot.Stats.Targets, v.KeySet.Version) + return nil +} + +type multi []string + +func (m *multi) String() string { return strings.Join(*m, ",") } +func (m *multi) Set(v string) error { *m = append(*m, v); return nil } + +func cmdKeys(args []string) error { + if len(args) == 0 { + return errors.New("keys: root-init|gen|sign-keyset") + } + switch args[0] { + case "root-init", "gen": + fs := flag.NewFlagSet(args[0], flag.ContinueOnError) + out := fs.String("out", "", "file to write the private seed to (mode 0600; keep a root offline)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + if *out == "" { + return errors.New("keys: --out is required") + } + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + f, err := os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := f.WriteString(base64.StdEncoding.EncodeToString(priv.Seed()) + "\n"); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + fmt.Printf("key id: %s\npublic key: %s\nseed file: %s\n", dsse.KeyID(pub), base64.StdEncoding.EncodeToString(pub), *out) + return nil + case "sign-keyset": + fs := flag.NewFlagSet("sign-keyset", flag.ContinueOnError) + rootFile := fs.String("root", "", "root seed file") + ver := fs.Uint64("version", 0, "key set version (higher than the current one)") + days := fs.Int("days", 180, "validity in days (at most 180)") + out := fs.String("out", "keys/keyset.dsse.json", "output") + var keys multi + fs.Var(&keys, "key", "base64 public key of an online signing key (repeatable)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + raw, err := os.ReadFile(*rootFile) + if err != nil { + return err + } + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))) + if err != nil || len(seed) != ed25519.SeedSize { + return errors.New("keys: the root file is not a base64 32-byte seed") + } + pubs := make([]ed25519.PublicKey, 0, len(keys)) + for _, k := range keys { + b, err := base64.StdEncoding.DecodeString(k) + if err != nil || len(b) != ed25519.PublicKeySize { + return fmt.Errorf("keys: %q is not a base64 Ed25519 public key", k) + } + pubs = append(pubs, b) + } + env, err := dsse.SignKeySet(ed25519.NewKeyFromSeed(seed), *ver, time.Now(), time.Duration(*days)*24*time.Hour, pubs) + if err != nil { + return err + } + return os.WriteFile(*out, env, 0o644) + } + return fmt.Errorf("keys: unknown command %q", args[0]) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fa474c2 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/openctemio/programfeed + +go 1.26.0 + +toolchain go1.26.9 + +require golang.org/x/net v0.60.0 + +require golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1ed037a --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go new file mode 100644 index 0000000..cbe237a --- /dev/null +++ b/internal/bundle/bundle.go @@ -0,0 +1,407 @@ +// Package bundle builds, signs, reads and verifies program bundles: JSON +// Lines files per record type, a signed manifest per snapshot and delta, +// and a signed latest pointer. The layout, envelopes, key set and checks +// are the ones of the OpenCTEM vulnerability feed, so one verifier serves +// both (only the payload types and record files differ). +package bundle + +import ( + "bufio" + "compress/gzip" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +// Schemas, payload types and file names. +const ( + ManifestSchema = "openctem.programfeed/v1" + LatestSchema = "openctem.programfeed.latest/v1" + ManifestPayloadType = "application/vnd.openctem.programfeed.manifest+json" + LatestPayloadType = "application/vnd.openctem.programfeed.latest+json" + + KindSnapshot = "snapshot" + KindDelta = "delta" + + LatestFile = "latest.dsse.json" + KeySetFile = "keyset.dsse.json" + ManifestFile = "%s.manifest.dsse.json" // snapshot | delta + + Validity = 7 * 24 * time.Hour +) + +// Caps (the importer applies the same). +const ( + MaxManifestBytes = 1 << 20 + MaxFileBytes = 128 << 20 + MaxDecompressedBytes = 1 << 30 + MaxRecordBytes = 4 << 20 +) + +// Record files of a bundle, in import order. A snapshot holds every +// program and this run's change log; a delta holds the programs that +// changed (complete new records) and the same change log. +var recordFiles = []string{"programs", "changes"} + +// FileName is the name of a record file of a kind. +func FileName(kind, records string) string { return kind + "-" + records + ".jsonl.gz" } + +// Source names one upstream source, its terms and how much it gave. +type Source struct { + Name string `json:"name"` + URL string `json:"url"` + AsOf time.Time `json:"as_of"` // last successful read + Terms string `json:"terms"` + Licence string `json:"licence"` + Attribution string `json:"attribution"` + Programs int `json:"programs"` +} + +// File is one record file of a manifest. +type File struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Records int `json:"records"` +} + +// Stats summarises a bundle. +type Stats struct { + Programs int `json:"programs"` + Open int `json:"open"` + Bounty int `json:"bounty"` + Targets int `json:"targets"` + Rejected int `json:"rejected"` + Changes int `json:"changes"` + Added int `json:"added"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` +} + +// Collector identifies the build. +type Collector struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +// Manifest is the signed description of a snapshot or a delta. +type Manifest struct { + Schema string `json:"schema"` + Sequence uint64 `json:"sequence"` + Kind string `json:"kind"` + BaseSequence uint64 `json:"base_sequence,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + Sources []Source `json:"sources"` + Files []File `json:"files"` + Stats Stats `json:"stats"` + Collector Collector `json:"collector"` +} + +// Latest is the signed pointer to the newest bundle. +type Latest struct { + Schema string `json:"schema"` + Sequence uint64 `json:"sequence"` + Tag string `json:"tag"` + Snapshot string `json:"snapshot"` + Delta string `json:"delta,omitempty"` + BaseSequence uint64 `json:"base_sequence,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` +} + +// Tag is the release tag of a sequence. +func Tag(sequence uint64) string { return fmt.Sprintf("v1-%d", sequence) } + +// Snapshot is the complete program set plus the change log of the run +// that produced it. +type Snapshot struct { + Sequence uint64 + Sources []Source + Programs map[string]model.Program + Changes []model.Change +} + +// NewSnapshot returns an empty snapshot. +func NewSnapshot() *Snapshot { return &Snapshot{Programs: map[string]model.Program{}} } + +// Source returns a source by name. +func (s *Snapshot) Source(name string) (Source, bool) { + for _, src := range s.Sources { + if src.Name == name { + return src, true + } + } + return Source{}, false +} + +// SortedIDs returns the program ids in order. +func (s *Snapshot) SortedIDs() []string { + ids := make([]string, 0, len(s.Programs)) + for id := range s.Programs { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// ChangedIDs returns the ids of programs present in the snapshot that the +// change log touches, in order. +func (s *Snapshot) ChangedIDs() []string { + seen := map[string]bool{} + var ids []string + for _, c := range s.Changes { + if _, ok := s.Programs[c.Program]; ok && !seen[c.Program] { + seen[c.Program] = true + ids = append(ids, c.Program) + } + } + sort.Strings(ids) + return ids +} + +// Validate checks every record of the snapshot. +func (s *Snapshot) Validate() error { + if len(s.Programs) > model.MaxPrograms { + return fmt.Errorf("%d programs, over the cap of %d", len(s.Programs), model.MaxPrograms) + } + for _, id := range s.SortedIDs() { + p := s.Programs[id] + if p.ID != id { + return fmt.Errorf("program %s filed under %s", p.ID, id) + } + if err := p.Validate(); err != nil { + return err + } + } + for i := range s.Changes { + c := &s.Changes[i] + if err := c.Validate(); err != nil { + return err + } + if c.Sequence != s.Sequence { + return fmt.Errorf("change of %s has sequence %d, the snapshot is %d", c.Program, c.Sequence, s.Sequence) + } + if _, ok := s.Programs[c.Program]; ok == (c.Kind == model.ChangeProgramDropped) { + return fmt.Errorf("change %s of %s: a dropped program must be absent, any other present", c.Kind, c.Program) + } + } + return nil +} + +func (s *Snapshot) stats(ids []string) Stats { + st := Stats{Programs: len(ids), Changes: len(s.Changes)} + for _, id := range ids { + p := s.Programs[id] + if p.Status == model.StatusOpen { + st.Open++ + } + if p.OffersBounty { + st.Bounty++ + } + st.Targets += len(p.InScope) + len(p.OutOfScope) + st.Rejected += len(p.Rejected) + } + for _, c := range s.Changes { + switch c.Kind { + case model.ChangeProgramAdded: + st.Added++ + case model.ChangeProgramClosed: + st.Closed++ + case model.ChangeProgramDropped: + st.Dropped++ + } + } + return st +} + +// WriteOptions describe a build. +type WriteOptions struct { + Now time.Time + Collector Collector +} + +// Write writes the snapshot of next and, when prev is not nil, the delta +// from prev, with unsigned manifests (payloads) and the latest pointer +// payload, into dir. +func Write(dir string, prev, next *Snapshot, opt WriteOptions) (snap, delta *Manifest, latest *Latest, err error) { + if err := next.Validate(); err != nil { + return nil, nil, nil, fmt.Errorf("refusing to publish an invalid bundle: %w", err) + } + if prev != nil && next.Sequence <= prev.Sequence { + return nil, nil, nil, fmt.Errorf("sequence %d is not after %d", next.Sequence, prev.Sequence) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, nil, nil, err + } + now := opt.Now.UTC().Truncate(time.Second) + all := next.SortedIDs() + snap = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindSnapshot, CreatedAt: now, + ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(all)} + if snap.Files, err = writeFiles(dir, KindSnapshot, next, all); err != nil { + return nil, nil, nil, err + } + latest = &Latest{Schema: LatestSchema, Sequence: next.Sequence, Tag: Tag(next.Sequence), + Snapshot: fmt.Sprintf(ManifestFile, KindSnapshot), CreatedAt: now, ExpiresAt: now.Add(Validity)} + if prev != nil { + ids := next.ChangedIDs() + delta = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindDelta, BaseSequence: prev.Sequence, + CreatedAt: now, ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(ids)} + if delta.Files, err = writeFiles(dir, KindDelta, next, ids); err != nil { + return nil, nil, nil, err + } + latest.Delta, latest.BaseSequence = fmt.Sprintf(ManifestFile, KindDelta), prev.Sequence + } + if err := writeJSON(filepath.Join(dir, "snapshot.manifest.json"), snap); err != nil { + return nil, nil, nil, err + } + if delta != nil { + if err := writeJSON(filepath.Join(dir, "delta.manifest.json"), delta); err != nil { + return nil, nil, nil, err + } + } + if err := writeJSON(filepath.Join(dir, "latest.json"), latest); err != nil { + return nil, nil, nil, err + } + return snap, delta, latest, nil +} + +func writeFiles(dir, kind string, s *Snapshot, ids []string) ([]File, error) { + var files []File + for _, rf := range recordFiles { + name := FileName(kind, rf) + var n int + f, err := writeGz(filepath.Join(dir, name), func(enc *json.Encoder) error { + switch rf { + case "programs": + for _, id := range ids { + if err := enc.Encode(s.Programs[id]); err != nil { + return err + } + n++ + } + case "changes": + for _, c := range s.Changes { + if err := enc.Encode(c); err != nil { + return err + } + n++ + } + } + return nil + }) + if err != nil { + return nil, err + } + f.Name, f.Records = name, n + files = append(files, f) + } + return files, nil +} + +func writeGz(path string, fill func(*json.Encoder) error) (File, error) { + out, err := os.Create(path) + if err != nil { + return File{}, err + } + h := sha256.New() + counter := &countWriter{} + gz, _ := gzip.NewWriterLevel(io.MultiWriter(out, h, counter), gzip.BestCompression) + gz.ModTime = time.Unix(0, 0) + bw := bufio.NewWriterSize(gz, 1<<20) + enc := json.NewEncoder(bw) + enc.SetEscapeHTML(false) + if err := fill(enc); err != nil { + out.Close() + return File{}, err + } + if err := bw.Flush(); err != nil { + out.Close() + return File{}, err + } + if err := gz.Close(); err != nil { + out.Close() + return File{}, err + } + if err := out.Close(); err != nil { + return File{}, err + } + if counter.n > MaxFileBytes { + return File{}, fmt.Errorf("%s is %d bytes, over the %d cap", filepath.Base(path), counter.n, MaxFileBytes) + } + return File{SHA256: hex.EncodeToString(h.Sum(nil)), Size: counter.n}, nil +} + +type countWriter struct{ n int64 } + +func (c *countWriter) Write(p []byte) (int, error) { + c.n += int64(len(p)) + return len(p), nil +} + +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, b, 0o644) +} + +// Sign wraps the unsigned manifests and latest pointer in dir in DSSE +// envelopes signed by priv, and copies the key set next to them. The +// signing key must be listed in the key set. +func Sign(dir string, priv ed25519.PrivateKey, keyset []byte, pinnedRoot string, now time.Time) error { + ks, err := dsse.VerifyKeySet(keyset, pinnedRoot, 0, now) + if err != nil { + return err + } + pub, _ := priv.Public().(ed25519.PublicKey) + listed := false + for _, k := range ks.Keys { + if k.KeyID == dsse.KeyID(pub) { + listed = true + } + } + if !listed { + return fmt.Errorf("the signing key %s is not in the key set", dsse.KeyID(pub)) + } + pairs := []struct{ in, out, typ string }{ + {"snapshot.manifest.json", fmt.Sprintf(ManifestFile, KindSnapshot), ManifestPayloadType}, + {"delta.manifest.json", fmt.Sprintf(ManifestFile, KindDelta), ManifestPayloadType}, + {"latest.json", LatestFile, LatestPayloadType}, + } + for _, p := range pairs { + payload, err := os.ReadFile(filepath.Join(dir, p.in)) + if errors.Is(err, os.ErrNotExist) && p.in == "delta.manifest.json" { + continue + } + if err != nil { + return err + } + env, err := dsse.Sign(priv, p.typ, payload) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, p.out), env, 0o644); err != nil { + return err + } + if err := os.Remove(filepath.Join(dir, p.in)); err != nil { + return err + } + } + return os.WriteFile(filepath.Join(dir, KeySetFile), keyset, 0o644) +} + +// Stats summarises the whole snapshot. +func (s *Snapshot) Stats() Stats { return s.stats(s.SortedIDs()) } diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go new file mode 100644 index 0000000..a2c4762 --- /dev/null +++ b/internal/bundle/bundle_test.go @@ -0,0 +1,274 @@ +package bundle + +import ( + "compress/gzip" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +type keys struct { + rootID string + priv ed25519.PrivateKey + keyset []byte +} + +func newKeys(t *testing.T) keys { + t.Helper() + rootPub, root, _ := ed25519.GenerateKey(nil) + pub, priv, _ := ed25519.GenerateKey(nil) + ks, err := dsse.SignKeySet(root, 1, time.Now().Add(-time.Hour), 90*24*time.Hour, []ed25519.PublicKey{pub}) + if err != nil { + t.Fatal(err) + } + return keys{rootID: dsse.KeyID(rootPub), priv: priv, keyset: ks} +} + +var t0 = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + +func program(id, target string) model.Program { + p := model.Program{ + ID: id, Source: "disclose", Platform: "self-hosted", Name: id, URL: "https://openctem.io/security", + Type: model.TypeVDP, Status: model.StatusOpen, + InScope: []model.Target{{Type: model.TargetDomain, Value: target, Confidence: model.ConfidenceInferred}}, + OutOfScope: []model.Target{}, + Terms: model.Terms{URL: "https://openctem.io/security"}, + Provenance: model.Provenance{Source: "disclose", SourceURL: "https://openctem.io/list.json", FetchedAt: t0}, + FirstSeen: t0, LastSeen: t0, LastChanged: t0, + } + model.Normalize(&p) + return p +} + +func snapshot(seq uint64, target string) *Snapshot { + s := NewSnapshot() + s.Sequence = seq + s.Sources = []Source{{Name: "disclose", URL: "https://openctem.io/list.json", AsOf: t0, Licence: "CC0-1.0"}} + a, b := program("disclose:a", "a.openctem.io"), program("disclose:b", target) + s.Programs[a.ID], s.Programs[b.ID] = a, b + tg := b.InScope[0] + s.Changes = []model.Change{{Sequence: seq, Program: b.ID, Kind: model.ChangeTargetAdded, Scope: "in", Target: &tg}} + return s +} + +func build(t *testing.T, k keys, prev, next *Snapshot) string { + t.Helper() + dir := t.TempDir() + if _, _, _, err := Write(dir, prev, next, WriteOptions{Now: time.Now()}); err != nil { + t.Fatal(err) + } + if err := Sign(dir, k.priv, k.keyset, k.rootID, time.Now()); err != nil { + t.Fatal(err) + } + return dir +} + +func TestRoundTrip(t *testing.T) { + k := newKeys(t) + prev, next := snapshot(1, "b.openctem.io"), snapshot(2, "c.openctem.io") + dir := build(t, k, prev, next) + v, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 1, Now: time.Now()}) + if err != nil { + t.Fatal(err) + } + if v.Latest.Sequence != 2 || v.Latest.Tag != "v1-2" || v.Delta == nil || v.Delta.BaseSequence != 1 { + t.Fatalf("%+v", v.Latest) + } + if v.Snapshot.Stats.Programs != 2 || v.Delta.Stats.Programs != 1 || v.Delta.Stats.Changes != 1 { + t.Fatalf("stats %+v / %+v", v.Snapshot.Stats, v.Delta.Stats) + } + got, err := ReadRecords(dir, v.Snapshot) + if err != nil || len(got.Programs) != 2 || got.Programs["disclose:b"].InScope[0].Value != "c.openctem.io" { + t.Fatalf("snapshot %v %+v", err, got) + } + d, err := ReadRecords(dir, *v.Delta) + if err != nil || len(d.Programs) != 1 || len(d.Changes) != 1 { + t.Fatalf("delta %v %+v", err, d) + } + dir2 := build(t, k, prev, next) + for _, f := range v.Snapshot.Files { + a, _ := os.ReadFile(filepath.Join(dir, f.Name)) + b, _ := os.ReadFile(filepath.Join(dir2, f.Name)) + if string(a) != string(b) { + t.Errorf("%s differs between builds", f.Name) + } + } +} + +func TestVerifyRefuses(t *testing.T) { + k := newKeys(t) + other := newKeys(t) + now := time.Now() + cases := map[string]struct { + mut func(t *testing.T, dir string) + opt VerifyOptions + }{ + "rollback": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 5, Now: now}}, + "replay same": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 3, Now: now}}, + "expired": {nil, VerifyOptions{PinnedRoot: k.rootID, Now: now.Add(8 * 24 * time.Hour)}}, + "other root": {nil, VerifyOptions{PinnedRoot: other.rootID, Now: now}}, + "no pinned root": {nil, VerifyOptions{Now: now}}, + "key set rolled back": {nil, VerifyOptions{PinnedRoot: k.rootID, MinKeySetVer: 2, Now: now}}, + "tampered file": {func(t *testing.T, dir string) { + p := filepath.Join(dir, FileName(KindSnapshot, "programs")) + b, _ := os.ReadFile(p) + b[len(b)/2] ^= 0xff + _ = os.WriteFile(p, b, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "swapped key set": {func(t *testing.T, dir string) { + _ = os.WriteFile(filepath.Join(dir, KeySetFile), other.keyset, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "vulnerability feed key set": {func(t *testing.T, dir string) { + // A key set of another feed (other payload type) is refused even + // under the same root. + raw, _ := os.ReadFile(filepath.Join(dir, KeySetFile)) + var env dsse.Envelope + _ = json.Unmarshal(raw, &env) + env.PayloadType = "application/vnd.openctem.vulnfeed.keyset+json" + b, _ := json.Marshal(env) + _ = os.WriteFile(filepath.Join(dir, KeySetFile), b, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "pointer signed by a stranger": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, other.priv, func(m map[string]any) {}) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "pointer names another sequence": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["sequence"] = 9; m["tag"] = "v1-9" }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "manifest valid for a year": {func(t *testing.T, dir string) { + resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) { + m["expires_at"] = now.Add(365 * 24 * time.Hour).UTC().Format(time.RFC3339) + }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "unknown field": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["extra"] = 1 }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + } + for name, c := range cases { + dir := build(t, k, snapshot(2, "b.openctem.io"), snapshot(3, "c.openctem.io")) + if c.mut != nil { + c.mut(t, dir) + } + if _, err := VerifyDir(dir, c.opt); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +// Poisoned records under a valid signature and correct hashes are refused: +// the importer re-validates every record. +func TestPoisonedRecordsRefused(t *testing.T) { + k := newKeys(t) + good := program("disclose:x", "x.openctem.io") + line := func(mut func(m map[string]any)) string { + b, _ := json.Marshal(good) + var m map[string]any + _ = json.Unmarshal(b, &m) + mut(m) + out, _ := json.Marshal(m) + return string(out) + } + target := func(typ, value string) func(m map[string]any) { + return func(m map[string]any) { + m["in_scope"] = []any{map[string]any{"type": typ, "value": value, "confidence": "published"}} + } + } + cases := map[string]string{ + "public suffix wildcard": line(target("wildcard", "*.com")), + "private range": line(target("cidr", "10.0.0.0/8")), + "huge range": line(target("cidr", "1.0.0.0/8")), + "metadata address": line(target("ip", "169.254.169.254")), + "bare TLD": line(target("domain", "com")), + "javascript url": line(func(m map[string]any) { m["url"] = "javascript:alert(1)" }), + "closed without time": line(func(m map[string]any) { m["status"] = "closed" }), + "extra field": line(func(m map[string]any) { m["sql"] = "x" }), + "duplicate": line(func(map[string]any) {}) + "\n" + line(func(map[string]any) {}), + } + for name, l := range cases { + dir := build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), l+"\n", strings.Count(l, "\n")+1) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil { + t.Errorf("%s: accepted", name) + } + } + // A change naming a program that is not in the bundle is refused. + dir := build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "changes"), `{"sequence":1,"program":"disclose:ghost","kind":"program_added"}`+"\n", 1) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil { + t.Error("change of an absent program accepted") + } + // A wrong record count is refused too. + dir = build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), line(func(map[string]any) {})+"\n", 5) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil || !strings.Contains(err.Error(), "records") { + t.Errorf("record count: %v", err) + } +} + +func TestWriteRefusesInvalidSnapshot(t *testing.T) { + s := snapshot(1, "b.openctem.io") + p := s.Programs["disclose:a"] + p.InScope = []model.Target{{Type: model.TargetWildcard, Value: "*.com", Confidence: model.ConfidencePublished}} + s.Programs["disclose:a"] = p + if _, _, _, err := Write(t.TempDir(), nil, s, WriteOptions{Now: time.Now()}); err == nil { + t.Fatal("invalid snapshot written") + } +} + +func TestSignRefusesKeyOutsideKeySet(t *testing.T) { + k := newKeys(t) + _, stranger, _ := ed25519.GenerateKey(nil) + dir := t.TempDir() + if _, _, _, err := Write(dir, nil, snapshot(1, "b.openctem.io"), WriteOptions{Now: time.Now()}); err != nil { + t.Fatal(err) + } + if err := Sign(dir, stranger, k.keyset, k.rootID, time.Now()); err == nil { + t.Fatal("signed with a key outside the key set") + } +} + +// replaceFile rewrites one record file, fixes its hash/size/count in the +// manifest and re-signs the manifest: only record validation can catch it. +func replaceFile(t *testing.T, dir string, k keys, name, content string, records int) { + t.Helper() + p := filepath.Join(dir, name) + f, _ := os.Create(p) + gz := gzip.NewWriter(f) + _, _ = gz.Write([]byte(content)) + _ = gz.Close() + _ = f.Close() + b, _ := os.ReadFile(p) + sum := sha256.Sum256(b) + resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) { + for _, fi := range m["files"].([]any) { + fm := fi.(map[string]any) + if fm["name"] == name { + fm["sha256"], fm["size"], fm["records"] = hex.EncodeToString(sum[:]), len(b), records + } + } + }) +} + +func resignJSON(t *testing.T, dir, file, payloadType string, priv ed25519.PrivateKey, mut func(map[string]any)) { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + t.Fatal(err) + } + var env dsse.Envelope + _ = json.Unmarshal(raw, &env) + var m map[string]any + _ = json.Unmarshal(env.Payload, &m) + mut(m) + payload, _ := json.Marshal(m) + out, _ := dsse.Sign(priv, payloadType, payload) + _ = os.WriteFile(filepath.Join(dir, file), out, 0o644) +} diff --git a/internal/bundle/verify.go b/internal/bundle/verify.go new file mode 100644 index 0000000..bd7e4dd --- /dev/null +++ b/internal/bundle/verify.go @@ -0,0 +1,277 @@ +package bundle + +import ( + "bufio" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +// Verified is a bundle directory whose signatures, freshness and file +// hashes checked out. +type Verified struct { + KeySet *dsse.KeySet + Latest Latest + Snapshot Manifest + Delta *Manifest +} + +// VerifyOptions are the consumer's state. +type VerifyOptions struct { + PinnedRoot string + MinKeySetVer uint64 + AppliedSeq uint64 + Now time.Time + AllowExpired bool // the previous release, read while building + SkipRecordChk bool +} + +// VerifyDir checks a bundle the way the platform importer does: key set +// (pinned root, version), latest pointer and manifests (key set, schema, +// sequence, expiry), every file's size and hash, and every record. +func VerifyDir(dir string, opt VerifyOptions) (*Verified, error) { + ksRaw, err := readCapped(filepath.Join(dir, KeySetFile), dsse.MaxKeySetBytes) + if err != nil { + return nil, err + } + ks, err := dsse.VerifyKeySet(ksRaw, opt.PinnedRoot, opt.MinKeySetVer, opt.Now) + if err != nil { + return nil, err + } + v := &Verified{KeySet: ks} + if err := verifyJSON(ks, filepath.Join(dir, LatestFile), LatestPayloadType, &v.Latest); err != nil { + return nil, err + } + l := v.Latest + switch { + case l.Schema != LatestSchema: + return nil, fmt.Errorf("latest: schema %q", l.Schema) + case l.Sequence <= opt.AppliedSeq: + return nil, fmt.Errorf("latest: sequence %d is not after the applied %d (rollback refused)", l.Sequence, opt.AppliedSeq) + case !opt.AllowExpired && !l.ExpiresAt.After(opt.Now): + return nil, fmt.Errorf("latest: expired at %s", l.ExpiresAt.Format(time.RFC3339)) + case l.Tag != Tag(l.Sequence): + return nil, fmt.Errorf("latest: tag %q does not name sequence %d", l.Tag, l.Sequence) + } + if err := v.loadManifest(dir, l.Snapshot, KindSnapshot, &v.Snapshot, opt); err != nil { + return nil, err + } + if l.Delta != "" { + var d Manifest + if err := v.loadManifest(dir, l.Delta, KindDelta, &d, opt); err != nil { + return nil, err + } + if d.BaseSequence != l.BaseSequence || d.BaseSequence >= d.Sequence { + return nil, fmt.Errorf("delta: base sequence %d does not match the pointer", d.BaseSequence) + } + v.Delta = &d + } + return v, nil +} + +func (v *Verified) loadManifest(dir, name, kind string, m *Manifest, opt VerifyOptions) error { + if name != fmt.Sprintf(ManifestFile, kind) { + return fmt.Errorf("%s manifest name %q", kind, name) + } + if err := verifyJSON(v.KeySet, filepath.Join(dir, name), ManifestPayloadType, m); err != nil { + return err + } + switch { + case m.Schema != ManifestSchema: + return fmt.Errorf("%s: schema %q", kind, m.Schema) + case m.Kind != kind: + return fmt.Errorf("%s: kind %q", kind, m.Kind) + case m.Sequence != v.Latest.Sequence: + return fmt.Errorf("%s: sequence %d, the pointer says %d", kind, m.Sequence, v.Latest.Sequence) + case !opt.AllowExpired && !m.ExpiresAt.After(opt.Now): + return fmt.Errorf("%s: expired", kind) + case m.ExpiresAt.Sub(m.CreatedAt) > Validity: + return fmt.Errorf("%s: valid for more than %s", kind, Validity) + case len(m.Files) != len(recordFiles): + return fmt.Errorf("%s: %d files", kind, len(m.Files)) + } + for i, f := range m.Files { + if f.Name != FileName(kind, recordFiles[i]) { + return fmt.Errorf("%s: unexpected file %q", kind, f.Name) + } + if err := checkFile(filepath.Join(dir, f.Name), f); err != nil { + return err + } + } + if opt.SkipRecordChk { + return nil + } + _, err := ReadRecords(dir, *m) + return err +} + +func verifyJSON(ks *dsse.KeySet, path, payloadType string, v any) error { + raw, err := readCapped(path, MaxManifestBytes) + if err != nil { + return err + } + payload, err := dsse.VerifyWith(ks, raw, payloadType, MaxManifestBytes) + if err != nil { + return fmt.Errorf("%s: %w", filepath.Base(path), err) + } + if err := dsse.DecodeStrict(payload, v); err != nil { + return fmt.Errorf("%s: %w", filepath.Base(path), err) + } + return nil +} + +func readCapped(path string, max int64) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, max+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > max { + return nil, fmt.Errorf("%s is over %d bytes", filepath.Base(path), max) + } + return b, nil +} + +func checkFile(path string, f File) error { + if f.Size > MaxFileBytes || f.Size < 0 { + return fmt.Errorf("%s: size %d over the cap", f.Name, f.Size) + } + fh, err := os.Open(path) + if err != nil { + return err + } + defer fh.Close() + h := sha256.New() + n, err := io.Copy(h, io.LimitReader(fh, MaxFileBytes+1)) + if err != nil { + return err + } + if n != f.Size || hex.EncodeToString(h.Sum(nil)) != f.SHA256 { + return fmt.Errorf("%s: size or SHA-256 does not match the manifest", f.Name) + } + return nil +} + +// ReadRecords reads and validates the record files of a verified manifest +// into a snapshot (for a delta: only the changed programs, plus the change +// log). +func ReadRecords(dir string, m Manifest) (*Snapshot, error) { + s := NewSnapshot() + s.Sequence, s.Sources = m.Sequence, m.Sources + counts := map[string]int{} + for _, f := range m.Files { + var kind string + for _, rf := range recordFiles { + if f.Name == FileName(m.Kind, rf) { + kind = rf + } + } + err := eachLine(filepath.Join(dir, f.Name), func(line []byte) error { + counts[kind]++ + switch kind { + case "programs": + var p model.Program + if err := strictLine(line, &p); err != nil { + return err + } + if err := p.Validate(); err != nil { + return err + } + if _, dup := s.Programs[p.ID]; dup { + return fmt.Errorf("program %s listed twice", p.ID) + } + if len(s.Programs) >= model.MaxPrograms { + return fmt.Errorf("over %d programs", model.MaxPrograms) + } + s.Programs[p.ID] = p + case "changes": + var c model.Change + if err := strictLine(line, &c); err != nil { + return err + } + if err := c.Validate(); err != nil { + return err + } + if len(s.Changes) >= 10*model.MaxPrograms { + return errors.New("too many changes") + } + s.Changes = append(s.Changes, c) + default: + return errors.New("unexpected record file") + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + if counts[kind] != f.Records { + return nil, fmt.Errorf("%s: %d records, the manifest says %d", f.Name, counts[kind], f.Records) + } + } + if err := s.Validate(); err != nil { + return nil, err + } + if m.Kind == KindDelta { + if ids := s.ChangedIDs(); len(ids) != len(s.Programs) { + return nil, errors.New("delta: a program that no change names") + } + } + return s, nil +} + +func strictLine(line []byte, v any) error { return dsse.DecodeStrict(line, v) } + +func eachLine(path string, fn func([]byte) error) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(io.LimitReader(f, MaxFileBytes)) + if err != nil { + return err + } + defer gz.Close() + sc := bufio.NewScanner(&limited{r: gz, left: MaxDecompressedBytes}) + sc.Buffer(make([]byte, 64<<10), MaxRecordBytes) + for sc.Scan() { + if len(sc.Bytes()) == 0 { + continue + } + if err := fn(sc.Bytes()); err != nil { + return err + } + } + return sc.Err() +} + +// limited fails once more than left bytes are read (a decompression bomb). +type limited struct { + r io.Reader + left int64 +} + +func (l *limited) Read(p []byte) (int, error) { + if l.left <= 0 { + return 0, errors.New("decompressed data over the cap") + } + if int64(len(p)) > l.left { + p = p[:l.left] + } + n, err := l.r.Read(p) + l.left -= int64(n) + return n, err +} diff --git a/internal/collect/collect.go b/internal/collect/collect.go new file mode 100644 index 0000000..03c8e00 --- /dev/null +++ b/internal/collect/collect.go @@ -0,0 +1,334 @@ +// Package collect builds the next snapshot: the previous one merged with +// what each source says now, and the change log between them. +package collect + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" + "github.com/openctemio/programfeed/internal/source" +) + +// DropAfter is how long a closed program stays in the feed before it is +// dropped (so subscribers see the closure first). +const DropAfter = 180 * 24 * time.Hour + +// Mass-change guard: a source that suddenly reports fewer than half of the +// programs it had (and had at least massMinimum) is treated as broken for +// this run, and its previous records are kept. +const ( + massShare = 0.5 + massMinimum = 20 +) + +// Options of a build. +type Options struct { + Now time.Time + // AllowMassChange lets a source drop more than the guard allows (a + // person checked the source). + AllowMassChange bool + Log func(format string, args ...any) +} + +// Build returns the next snapshot. prev may be nil (first build). It fails +// only when no source could be read. +func Build(ctx context.Context, prev *bundle.Snapshot, sources []source.Source, client *fetch.Client, opt Options) (*bundle.Snapshot, error) { + if prev == nil { + prev = bundle.NewSnapshot() + } + log := opt.Log + if log == nil { + log = func(string, ...any) {} + } + now := opt.Now.UTC().Truncate(time.Second) + next := bundle.NewSnapshot() + next.Sequence = prev.Sequence + 1 + ok := 0 + var failures []string + seenSource := map[string]bool{} + for _, src := range sources { + meta := src.Meta() + if seenSource[meta.Name] { + return nil, fmt.Errorf("source %q configured twice", meta.Name) + } + seenSource[meta.Name] = true + old := programsOf(prev, meta.Name) + res, err := src.Collect(ctx, client) + if err == nil { + err = guard(res, old, opt.AllowMassChange) + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + log("%s: %v; keeping its %d previous records", meta.Name, err, len(old)) + failures = append(failures, meta.Name+": "+err.Error()) + for _, p := range old { + next.Programs[p.ID] = p + } + if s, had := prev.Source(meta.Name); had { + s.Programs = len(old) + next.Sources = append(next.Sources, s) + } + continue + } + ok++ + n := merge(prev, next, meta.Name, old, res, now, log) + next.Sources = append(next.Sources, bundle.Source{Name: meta.Name, URL: meta.URL, AsOf: now, Terms: meta.Terms, + Licence: meta.Licence, Attribution: meta.Attribution, Programs: n}) + log("%s: %d programs (%d skipped records, %d unreadable kept)", meta.Name, n, res.Skipped, len(res.Unknown)) + } + // Programs of sources no longer configured are kept as they are; a + // source is removed from the feed on purpose, in code review. + for id, p := range prev.Programs { + if !seenSource[p.Source] { + next.Programs[id] = p + } + } + for _, s := range prev.Sources { + if !seenSource[s.Name] { + next.Sources = append(next.Sources, s) + } + } + if ok == 0 { + return nil, fmt.Errorf("no source could be read (%s)", strings.Join(failures, "; ")) + } + sort.Slice(next.Sources, func(i, j int) bool { return next.Sources[i].Name < next.Sources[j].Name }) + sortChanges(next.Changes) + return next, next.Validate() +} + +func programsOf(s *bundle.Snapshot, name string) map[string]model.Program { + out := map[string]model.Program{} + for id, p := range s.Programs { + if p.Source == name { + out[id] = p + } + } + return out +} + +func guard(res *source.Result, old map[string]model.Program, allow bool) error { + open := 0 + for _, p := range old { + if p.Status != model.StatusClosed { + open++ + } + } + got := len(res.Programs) + len(res.Unknown) + if !allow && open >= massMinimum && float64(got) < float64(open)*massShare { + return fmt.Errorf("refusing: %d programs now, %d before (rerun with --allow-mass-change after checking the source)", got, open) + } + return nil +} + +// merge folds one source's result into next and returns how many programs +// of the source next holds. +func merge(prev, next *bundle.Snapshot, name string, old map[string]model.Program, res *source.Result, now time.Time, log func(string, ...any)) int { + seq := next.Sequence + add := func(c model.Change) { + c.Sequence = seq + next.Changes = append(next.Changes, c) + } + seen := map[string]bool{} + for _, id := range res.Unknown { + if p, had := old[id]; had { + next.Programs[id] = p + seen[id] = true + } + } + for _, p := range res.Programs { + if seen[p.ID] { + continue + } + if p.Source != name { + log("%s: program %s claims source %q; skipped", name, p.ID, p.Source) + continue + } + model.Normalize(&p) + was, had := old[p.ID] + p.LastSeen = now + if had { + p.FirstSeen = was.FirstSeen + changes := diff(was, p) + if len(changes) > 0 { + p.LastChanged = now + } else { + p.LastChanged = was.LastChanged + } + if err := p.Validate(); err != nil { + log("%s: %v; keeping the previous record", name, err) + next.Programs[p.ID] = was + seen[p.ID] = true + continue + } + for _, c := range changes { + add(c) + } + } else { + p.FirstSeen, p.LastChanged = now, now + if err := p.Validate(); err != nil { + log("%s: %v; skipped", name, err) + continue + } + add(model.Change{Program: p.ID, Kind: model.ChangeProgramAdded}) + } + next.Programs[p.ID] = p + seen[p.ID] = true + } + ids := make([]string, 0, len(old)) + for id := range old { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + if seen[id] { + continue + } + p := old[id] + switch { + case p.Status != model.StatusClosed: + at := now + p.Status, p.ClosedAt, p.LastChanged = model.StatusClosed, &at, now + next.Programs[id] = p + add(model.Change{Program: id, Kind: model.ChangeProgramClosed}) + case now.Sub(*p.ClosedAt) > DropAfter: + add(model.Change{Program: id, Kind: model.ChangeProgramDropped}) + default: + next.Programs[id] = p + } + seen[id] = true + } + n := 0 + for _, p := range next.Programs { + if p.Source == name { + n++ + } + } + return n +} + +// diff lists what changed between two records of one program. +func diff(a, b model.Program) []model.Change { + var out []model.Change + id := b.ID + switch { + case a.Status == model.StatusClosed && b.Status != model.StatusClosed: + out = append(out, model.Change{Program: id, Kind: model.ChangeProgramReopened}) + case a.Status != b.Status: + out = append(out, model.Change{Program: id, Kind: model.ChangeStatus, Fields: []string{"status"}}) + } + out = append(out, diffTargets(id, "in", a.InScope, b.InScope)...) + out = append(out, diffTargets(id, "out", a.OutOfScope, b.OutOfScope)...) + if !reflect.DeepEqual(a.Rules, b.Rules) { + out = append(out, model.Change{Program: id, Kind: model.ChangeRules, Fields: diffFields(a.Rules, b.Rules)}) + } + if a.Terms.URL != b.Terms.URL || a.Terms.SHA256 != b.Terms.SHA256 { + out = append(out, model.Change{Program: id, Kind: model.ChangeTerms}) + } + var fields []string + for name, eq := range map[string]bool{ + "name": a.Name == b.Name, "url": a.URL == b.URL, "platform": a.Platform == b.Platform, "type": a.Type == b.Type, + "scope_published": a.ScopePublished == b.ScopePublished, "contact": a.Contact == b.Contact, + "rejected": reflect.DeepEqual(a.Rejected, b.Rejected), + } { + if !eq { + fields = append(fields, name) + } + } + if len(fields) > 0 { + sort.Strings(fields) + out = append(out, model.Change{Program: id, Kind: model.ChangeProgramModified, Fields: fields}) + } + return out +} + +func diffTargets(id, scope string, a, b []model.Target) []model.Change { + key := func(t model.Target) string { return t.Type + "\x00" + t.Value } + before := map[string]model.Target{} + for _, t := range a { + before[key(t)] = t + } + var out []model.Change + after := map[string]bool{} + for _, t := range b { + t := t + after[key(t)] = true + was, had := before[key(t)] + switch { + case !had: + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetAdded, Scope: scope, Target: &t}) + case !reflect.DeepEqual(was, t): + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetModified, Scope: scope, Target: &t, Fields: diffFields(was, t)}) + } + } + for _, t := range a { + t := t + if !after[key(t)] { + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetRemoved, Scope: scope, Target: &t}) + } + } + return out +} + +// diffFields names the JSON fields that differ between two structs. +func diffFields(a, b any) []string { + var ma, mb map[string]json.RawMessage + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + _ = json.Unmarshal(ja, &ma) + _ = json.Unmarshal(jb, &mb) + var out []string + for k, v := range ma { + if string(mb[k]) != string(v) { + out = append(out, k) + } + } + for k := range mb { + if _, ok := ma[k]; !ok { + out = append(out, k) + } + } + sort.Strings(out) + if len(out) > model.MaxChangeFields { + out = out[:model.MaxChangeFields] + } + return out +} + +var kindOrder = map[string]int{ + model.ChangeProgramAdded: 0, model.ChangeProgramReopened: 1, model.ChangeStatus: 2, model.ChangeProgramModified: 3, + model.ChangeRules: 4, model.ChangeTerms: 5, model.ChangeTargetAdded: 6, model.ChangeTargetModified: 7, + model.ChangeTargetRemoved: 8, model.ChangeProgramClosed: 9, model.ChangeProgramDropped: 10, +} + +func sortChanges(cs []model.Change) { + sort.SliceStable(cs, func(i, j int) bool { + a, b := cs[i], cs[j] + if a.Program != b.Program { + return a.Program < b.Program + } + if kindOrder[a.Kind] != kindOrder[b.Kind] { + return kindOrder[a.Kind] < kindOrder[b.Kind] + } + if a.Scope != b.Scope { + return a.Scope < b.Scope + } + var ta, tb string + if a.Target != nil { + ta = a.Target.Type + "\x00" + a.Target.Value + } + if b.Target != nil { + tb = b.Target.Type + "\x00" + b.Target.Value + } + return ta < tb + }) +} diff --git a/internal/collect/collect_test.go b/internal/collect/collect_test.go new file mode 100644 index 0000000..c92fc4c --- /dev/null +++ b/internal/collect/collect_test.go @@ -0,0 +1,196 @@ +package collect + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" + "github.com/openctemio/programfeed/internal/source" +) + +type fake struct { + name string + res *source.Result + err error +} + +func (f fake) Meta() source.Meta { + return source.Meta{Name: f.name, URL: "https://openctem.io/" + f.name, Terms: "https://openctem.io/terms", Licence: "CC0-1.0"} +} + +func (f fake) Collect(context.Context, *fetch.Client) (*source.Result, error) { + if f.err != nil { + return nil, f.err + } + // Return copies: the collector normalises in place. + out := *f.res + out.Programs = append([]model.Program(nil), f.res.Programs...) + for i := range out.Programs { + p := &out.Programs[i] + p.InScope = append([]model.Target(nil), p.InScope...) + } + return &out, nil +} + +var day0 = time.Date(2026, 10, 1, 2, 0, 0, 0, time.UTC) + +func prog(src, slug string, targets ...string) model.Program { + p := model.Program{ + ID: src + ":" + slug, Source: src, Platform: "self-hosted", Name: slug, URL: "https://" + slug + ".org/security", + Type: model.TypeVDP, Status: model.StatusOpen, InScope: []model.Target{}, OutOfScope: []model.Target{}, + Terms: model.Terms{URL: "https://" + slug + ".org/security"}, + Provenance: model.Provenance{Source: src, SourceURL: "https://openctem.io/" + src, FetchedAt: day0}, + } + for _, t := range targets { + typ := model.TargetDomain + if strings.HasPrefix(t, "*.") { + typ = model.TargetWildcard + } + p.InScope = append(p.InScope, model.Target{Type: typ, Value: t}) + } + return p +} + +func run(t *testing.T, prev *bundle.Snapshot, at time.Time, allowMass bool, srcs ...source.Source) *bundle.Snapshot { + t.Helper() + next, err := Build(context.Background(), prev, srcs, nil, Options{Now: at, AllowMassChange: allowMass}) + if err != nil { + t.Fatal(err) + } + return next +} + +func kinds(s *bundle.Snapshot) string { + var out []string + for _, c := range s.Changes { + k := c.Program + " " + c.Kind + if c.Target != nil { + k += " " + c.Scope + ":" + c.Target.Value + } + out = append(out, k) + } + return strings.Join(out, "\n") +} + +func TestBuildLifecycle(t *testing.T) { + s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com"), + prog("d", "beta", "beta.org"), + prog("d", "acme", "acme.org"), // duplicate id: first wins + }}}) + if s1.Sequence != 1 || len(s1.Programs) != 2 || kinds(s1) != "d:acme program_added\nd:beta program_added" { + t.Fatalf("first build:\n%s", kinds(s1)) + } + acme := s1.Programs["d:acme"] + if len(acme.InScope) != 1 || len(acme.Rejected) != 1 || !acme.FirstSeen.Equal(day0) { + t.Fatalf("acme: %+v", acme) + } + + // Day 1: acme gains a target and loses none, beta disappears. + day1 := day0.Add(24 * time.Hour) + s2 := run(t, s1, day1, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com", "api.acme.org"), + }}}) + want := "d:acme target_added in:api.acme.org\nd:beta program_closed" + if s2.Sequence != 2 || kinds(s2) != want { + t.Fatalf("day 1:\n%s\nwant\n%s", kinds(s2), want) + } + if b := s2.Programs["d:beta"]; b.Status != model.StatusClosed || b.ClosedAt == nil || !b.LastSeen.Equal(day0) { + t.Errorf("beta: %+v", b) + } + if a := s2.Programs["d:acme"]; !a.FirstSeen.Equal(day0) || !a.LastChanged.Equal(day1) || !a.LastSeen.Equal(day1) { + t.Errorf("acme timestamps: %+v", a) + } + + // Day 2: nothing changes: no change entries, last_changed stays. + day2 := day1.Add(24 * time.Hour) + s3 := run(t, s2, day2, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com", "api.acme.org"), + }}}) + if len(s3.Changes) != 0 || !s3.Programs["d:acme"].LastChanged.Equal(day1) { + t.Fatalf("day 2:\n%s", kinds(s3)) + } + + // Beta comes back with a different target set; acme loses a target. + s4 := run(t, s3, day2.Add(time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org"), + prog("d", "beta", "beta.org", "www.beta.org"), + }}}) + want = "d:acme program_modified\nd:acme target_removed in:api.acme.org\nd:beta program_reopened\nd:beta target_added in:www.beta.org" + if kinds(s4) != want { + t.Fatalf("reopen:\n%s\nwant\n%s", kinds(s4), want) + } + + // A program closed for longer than DropAfter is dropped. + s5 := run(t, s4, day2.Add(2*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}}) + s6 := run(t, s5, day2.Add(DropAfter+3*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}}) + if _, still := s6.Programs["d:beta"]; still || kinds(s6) != "d:beta program_dropped" { + t.Fatalf("drop:\n%s", kinds(s6)) + } +} + +func TestBuildKeepsRecordsOfFailingSources(t *testing.T) { + s1 := run(t, nil, day0, false, + fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "one", "one.org"), prog("a", "two", "two.org")}}}, + fake{name: "b", res: &source.Result{Programs: []model.Program{prog("b", "three", "three.org")}}}) + day1 := day0.Add(24 * time.Hour) + // Source a is down, source b reports "two"'s host unknown... and b's + // program unknown: both keep their records unchanged. + s2 := run(t, s1, day1, false, + fake{name: "a", err: errors.New("HTTP 503")}, + fake{name: "b", res: &source.Result{Unknown: []string{"b:three"}}}) + if len(s2.Programs) != 3 || len(s2.Changes) != 0 { + t.Fatalf("kept %d, changes:\n%s", len(s2.Programs), kinds(s2)) + } + for _, p := range s2.Programs { + if p.Status != model.StatusOpen || !p.LastSeen.Equal(day0) { + t.Errorf("%s: %+v", p.ID, p) + } + } + if src, _ := s2.Source("a"); !src.AsOf.Equal(day0) { + t.Errorf("a failing source keeps its old as_of: %+v", src) + } + // Every source failing fails the build. + if _, err := Build(context.Background(), s1, []source.Source{fake{name: "a", err: errors.New("down")}}, nil, Options{Now: day1}); err == nil { + t.Error("a build with no readable source succeeded") + } +} + +func TestMassChangeGuard(t *testing.T) { + var many []model.Program + for i := 0; i < 40; i++ { + many = append(many, prog("d", fmt.Sprintf("p%02d", i), fmt.Sprintf("p%02d.org", i))) + } + s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: many}}) + few := &source.Result{Programs: many[:5]} + other := fake{name: "o", res: &source.Result{Programs: []model.Program{prog("o", "x", "x.org")}}} + // With another readable source, the guarded source keeps its records. + s2 := run(t, s1, day0.Add(time.Hour), false, fake{name: "d", res: few}, other) + if len(s2.Programs) != 41 || s2.Stats().Closed != 0 { + t.Fatalf("guard did not hold: %+v", s2.Stats()) + } + // Alone, it fails the build and says why. + if _, err := Build(context.Background(), s1, []source.Source{fake{name: "d", res: few}}, nil, Options{Now: day0.Add(time.Hour)}); err == nil || !strings.Contains(err.Error(), "allow-mass-change") { + t.Fatalf("guard alone: %v", err) + } + s3 := run(t, s1, day0.Add(time.Hour), true, fake{name: "d", res: few}) + if s3.Stats().Closed != 35 { + t.Fatalf("override: %+v", s3.Stats()) + } +} + +func TestSourceCannotWriteAnotherSourcesPrograms(t *testing.T) { + evil := prog("a", "victim", "victim.org") // claims source a, served by b + s := run(t, nil, day0, false, + fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "real", "real.org")}}}, + fake{name: "b", res: &source.Result{Programs: []model.Program{evil, prog("b", "ok", "ok.org")}}}) + if _, ok := s.Programs["a:victim"]; ok || len(s.Programs) != 2 { + t.Fatalf("programs: %v", s.SortedIDs()) + } +} diff --git a/internal/dsse/dsse.go b/internal/dsse/dsse.go new file mode 100644 index 0000000..ac32416 --- /dev/null +++ b/internal/dsse/dsse.go @@ -0,0 +1,251 @@ +// Package dsse signs and verifies DSSE v1 envelopes with Ed25519, and the +// key sets that name the keys allowed to sign bundles. +// +// The envelope, pre-authentication encoding and key ids are the ones the +// OpenCTEM platform already verifies for signed jobs: JSON +// {"payloadType", "payload" (base64), "signatures": [{"keyid", "sig" (base64)}]}, +// PAE "DSSEv1 ", key id +// "SHA256:" + hex(sha256(raw public key)). +package dsse + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "time" +) + +// Envelope is a DSSE v1 envelope. +type Envelope struct { + PayloadType string `json:"payloadType"` + Payload []byte `json:"payload"` + Signatures []Signature `json:"signatures"` +} + +// Signature is one signature of an envelope. +type Signature struct { + KeyID string `json:"keyid"` + Sig []byte `json:"sig"` +} + +// PAE is the DSSE v1 pre-authentication encoding. +func PAE(payloadType string, payload []byte) []byte { + var b bytes.Buffer + b.WriteString("DSSEv1 ") + b.WriteString(strconv.Itoa(len(payloadType))) + b.WriteByte(' ') + b.WriteString(payloadType) + b.WriteByte(' ') + b.WriteString(strconv.Itoa(len(payload))) + b.WriteByte(' ') + b.Write(payload) + return b.Bytes() +} + +// KeyID names a public key. +func KeyID(pub ed25519.PublicKey) string { + sum := sha256.Sum256(pub) + return "SHA256:" + hex.EncodeToString(sum[:]) +} + +// Sign wraps payload in an envelope signed by priv. +func Sign(priv ed25519.PrivateKey, payloadType string, payload []byte) ([]byte, error) { + if len(priv) != ed25519.PrivateKeySize { + return nil, errors.New("dsse: an Ed25519 private key is required") + } + pub, _ := priv.Public().(ed25519.PublicKey) + return json.Marshal(Envelope{ + PayloadType: payloadType, + Payload: payload, + Signatures: []Signature{{KeyID: KeyID(pub), Sig: ed25519.Sign(priv, PAE(payloadType, payload))}}, + }) +} + +// Open parses an envelope of the expected type (at most maxBytes) without +// checking signatures. +func Open(envelope []byte, payloadType string, maxBytes int) (*Envelope, error) { + if len(envelope) == 0 || len(envelope) > maxBytes { + return nil, fmt.Errorf("dsse: envelope is empty or over %d bytes", maxBytes) + } + var env Envelope + if err := json.Unmarshal(envelope, &env); err != nil { + return nil, fmt.Errorf("dsse: envelope: %w", err) + } + if env.PayloadType != payloadType { + return nil, fmt.Errorf("dsse: payload type %q, want %q", env.PayloadType, payloadType) + } + return &env, nil +} + +// VerifiedBy reports whether env carries a valid signature of pub. +func (env *Envelope) VerifiedBy(pub ed25519.PublicKey) bool { + id := KeyID(pub) + pae := PAE(env.PayloadType, env.Payload) + for _, s := range env.Signatures { + if s.KeyID == id && ed25519.Verify(pub, pae, s.Sig) { + return true + } + } + return false +} + +// DecodeStrict decodes JSON into v, refusing unknown fields and trailing data. +func DecodeStrict(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return errors.New("trailing data") + } + return nil +} + +// Key set (offline root → online keys). +const ( + KeySetPayloadType = "application/vnd.openctem.programfeed.keyset+json" + KeySetKind = "openctem.programfeed.keyset/v1" + MaxKeySetValidity = 180 * 24 * time.Hour + MaxKeySetBytes = 64 << 10 + MaxKeySetKeys = 8 + maxClockSkew = 5 * time.Minute +) + +// PublicKey describes one online key. +type PublicKey struct { + KeyID string `json:"keyid"` + Algorithm string `json:"algorithm"` + PublicKey string `json:"public_key"` +} + +// NewPublicKey describes pub. +func NewPublicKey(pub ed25519.PublicKey) PublicKey { + return PublicKey{KeyID: KeyID(pub), Algorithm: "ed25519", PublicKey: base64.StdEncoding.EncodeToString(pub)} +} + +// Decode returns the raw key after checking the id matches. +func (k PublicKey) Decode() (ed25519.PublicKey, error) { + if k.Algorithm != "ed25519" { + return nil, fmt.Errorf("key %s: unsupported algorithm %q", k.KeyID, k.Algorithm) + } + raw, err := base64.StdEncoding.DecodeString(k.PublicKey) + if err != nil || len(raw) != ed25519.PublicKeySize || KeyID(raw) != k.KeyID { + return nil, fmt.Errorf("key %s: not the 32-byte Ed25519 key of its id", k.KeyID) + } + return raw, nil +} + +// KeySet is the list of online keys an offline root allows to sign. +type KeySet struct { + Kind string `json:"kind"` + Version uint64 `json:"version"` + IssuedAt time.Time `json:"issued_at"` + NotAfter time.Time `json:"not_after"` + Keys []PublicKey `json:"keys"` + RootKeyID string `json:"root_keyid"` + RootPublicKey string `json:"root_public_key"` +} + +func (ks *KeySet) check() error { + switch { + case ks.Kind != KeySetKind: + return fmt.Errorf("key set: kind %q", ks.Kind) + case ks.Version == 0: + return errors.New("key set: version must be 1 or more") + case ks.IssuedAt.IsZero() || !ks.NotAfter.After(ks.IssuedAt) || ks.NotAfter.Sub(ks.IssuedAt) > MaxKeySetValidity: + return fmt.Errorf("key set: validity must be positive and at most %s", MaxKeySetValidity) + case len(ks.Keys) == 0 || len(ks.Keys) > MaxKeySetKeys: + return fmt.Errorf("key set: 1 to %d keys", MaxKeySetKeys) + } + root, err := base64.StdEncoding.DecodeString(ks.RootPublicKey) + if err != nil || len(root) != ed25519.PublicKeySize || KeyID(root) != ks.RootKeyID { + return errors.New("key set: root_public_key is not the key of root_keyid") + } + seen := map[string]bool{} + for _, k := range ks.Keys { + if _, err := k.Decode(); err != nil { + return fmt.Errorf("key set: %w", err) + } + if seen[k.KeyID] || k.KeyID == ks.RootKeyID { + return fmt.Errorf("key set: key %s listed twice or is the root", k.KeyID) + } + seen[k.KeyID] = true + } + return nil +} + +// SignKeySet signs a key set with the offline root. +func SignKeySet(root ed25519.PrivateKey, version uint64, issuedAt time.Time, validity time.Duration, keys []ed25519.PublicKey) ([]byte, error) { + if len(root) != ed25519.PrivateKeySize { + return nil, errors.New("key set: an Ed25519 root key is required") + } + rootPub, _ := root.Public().(ed25519.PublicKey) + issuedAt = issuedAt.UTC().Truncate(time.Second) + ks := KeySet{Kind: KeySetKind, Version: version, IssuedAt: issuedAt, NotAfter: issuedAt.Add(validity), + RootKeyID: KeyID(rootPub), RootPublicKey: base64.StdEncoding.EncodeToString(rootPub)} + for _, k := range keys { + ks.Keys = append(ks.Keys, NewPublicKey(k)) + } + if err := ks.check(); err != nil { + return nil, err + } + payload, err := json.Marshal(ks) + if err != nil { + return nil, err + } + return Sign(root, KeySetPayloadType, payload) +} + +// VerifyKeySet checks a key set: signed by its root, that root is the pinned +// one, valid now, and its version is not below minVersion. +func VerifyKeySet(envelope []byte, pinnedRoot string, minVersion uint64, now time.Time) (*KeySet, error) { + env, err := Open(envelope, KeySetPayloadType, MaxKeySetBytes) + if err != nil { + return nil, err + } + var ks KeySet + if err := DecodeStrict(env.Payload, &ks); err != nil { + return nil, fmt.Errorf("key set: %w", err) + } + if err := ks.check(); err != nil { + return nil, err + } + if pinnedRoot == "" || ks.RootKeyID != pinnedRoot { + return nil, fmt.Errorf("key set: signed by root %s, the pinned root is %q", ks.RootKeyID, pinnedRoot) + } + root, _ := base64.StdEncoding.DecodeString(ks.RootPublicKey) + if !env.VerifiedBy(root) { + return nil, errors.New("key set: bad root signature") + } + if ks.Version < minVersion { + return nil, fmt.Errorf("key set: version %d is below %d (rolled back)", ks.Version, minVersion) + } + if ks.IssuedAt.After(now.Add(maxClockSkew)) || !ks.NotAfter.After(now.Add(-maxClockSkew)) { + return nil, fmt.Errorf("key set: not valid at %s (valid %s to %s)", now.UTC().Format(time.RFC3339), + ks.IssuedAt.Format(time.RFC3339), ks.NotAfter.Format(time.RFC3339)) + } + return &ks, nil +} + +// VerifyWith checks envelope against the key set and returns its payload. +func VerifyWith(ks *KeySet, envelope []byte, payloadType string, maxBytes int) ([]byte, error) { + env, err := Open(envelope, payloadType, maxBytes) + if err != nil { + return nil, err + } + for _, k := range ks.Keys { + pub, err := k.Decode() + if err == nil && env.VerifiedBy(pub) { + return env.Payload, nil + } + } + return nil, errors.New("dsse: not signed by a key of the key set") +} diff --git a/internal/dsse/dsse_test.go b/internal/dsse/dsse_test.go new file mode 100644 index 0000000..7df7c93 --- /dev/null +++ b/internal/dsse/dsse_test.go @@ -0,0 +1,124 @@ +package dsse + +import ( + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" +) + +func key(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return pub, priv +} + +func TestPAEAndEnvelopeFormat(t *testing.T) { + if got := string(PAE("t/x", []byte("hello"))); got != "DSSEv1 3 t/x 5 hello" { + t.Fatalf("PAE = %q", got) + } + _, priv := key(t) + env, err := Sign(priv, "t/x", []byte(`{"a":1}`)) + if err != nil { + t.Fatal(err) + } + var raw map[string]any + if err := json.Unmarshal(env, &raw); err != nil { + t.Fatal(err) + } + sigs, _ := raw["signatures"].([]any) + sig, _ := sigs[0].(map[string]any) + if raw["payloadType"] != "t/x" || raw["payload"] != "eyJhIjoxfQ==" || !strings.HasPrefix(sig["keyid"].(string), "SHA256:") || sig["sig"] == nil { + t.Fatalf("envelope %s", env) + } +} + +func TestKeySetAndVerify(t *testing.T) { + rootPub, root := key(t) + pub, priv := key(t) + now := time.Now() + ksEnv, err := SignKeySet(root, 3, now, 90*24*time.Hour, []ed25519.PublicKey{pub}) + if err != nil { + t.Fatal(err) + } + ks, err := VerifyKeySet(ksEnv, KeyID(rootPub), 2, now) + if err != nil { + t.Fatal(err) + } + env, _ := Sign(priv, "t/x", []byte("payload")) + if got, err := VerifyWith(ks, env, "t/x", 1<<10); err != nil || string(got) != "payload" { + t.Fatalf("VerifyWith: %q %v", got, err) + } + + otherPub, other := key(t) + _ = otherPub + forged, _ := Sign(other, "t/x", []byte("payload")) + cases := map[string]func() error{ + "another root pinned": func() error { _, err := VerifyKeySet(ksEnv, KeyID(otherPub), 0, now); return err }, + "no root pinned": func() error { _, err := VerifyKeySet(ksEnv, "", 0, now); return err }, + "rolled back": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 4, now); return err }, + "expired": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(91*24*time.Hour)); return err }, + "not yet valid": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(-time.Hour)); return err }, + "unknown signer": func() error { _, err := VerifyWith(ks, forged, "t/x", 1<<10); return err }, + "wrong type": func() error { _, err := VerifyWith(ks, env, "t/y", 1<<10); return err }, + "too big": func() error { _, err := VerifyWith(ks, env, "t/x", 10); return err }, + "tampered payload": func() error { + var e Envelope + _ = json.Unmarshal(env, &e) + e.Payload = []byte("payloae") + b, _ := json.Marshal(e) + _, err := VerifyWith(ks, b, "t/x", 1<<10) + return err + }, + "key set signed by an online key": func() error { + bad, _ := SignKeySet(priv, 1, now, time.Hour, []ed25519.PublicKey{otherPub}) + _, err := VerifyKeySet(bad, KeyID(rootPub), 0, now) + return err + }, + } + for name, fn := range cases { + if fn() == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestSignKeySetRefuses(t *testing.T) { + rootPub, root := key(t) + pub, _ := key(t) + now := time.Now() + for name, fn := range map[string]func() error{ + "too long": func() error { + _, err := SignKeySet(root, 1, now, 181*24*time.Hour, []ed25519.PublicKey{pub}) + return err + }, + "version 0": func() error { _, err := SignKeySet(root, 0, now, time.Hour, []ed25519.PublicKey{pub}); return err }, + "no keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, nil); return err }, + "root as online": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{rootPub}); return err }, + "duplicate keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{pub, pub}); return err }, + "not a root key": func() error { + _, err := SignKeySet(ed25519.PrivateKey{1}, 1, now, time.Hour, []ed25519.PublicKey{pub}) + return err + }, + } { + if fn() == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestDecodeStrict(t *testing.T) { + var v struct{ A int } + if DecodeStrict([]byte(`{"A":1}`), &v) != nil || v.A != 1 { + t.Fatal("valid") + } + for _, s := range []string{`{"A":1,"B":2}`, `{"A":1} {}`, `[`} { + if DecodeStrict([]byte(s), &v) == nil { + t.Errorf("%s accepted", s) + } + } +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 0000000..bdede14 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,295 @@ +// Package fetch is the collector's only way to the network: a polite HTTP +// client that names itself and its contact, honours robots.txt, spaces +// requests to one host, makes conditional requests from an on-disk cache, +// caps response sizes and refuses to connect to non-public addresses (a +// seed list or a redirect must not point the collector at internal +// services). +package fetch + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/openctemio/programfeed/internal/model" +) + +// Product is the robots.txt product token of the collector. +const Product = "openctem-programfeed" + +// UserAgent names the collector, where it lives and who to contact. +func UserAgent(version string) string { + return Product + "/" + version + " (+https://github.com/openctemio/programfeed; security@openctem.io)" +} + +// Errors callers act on. +var ( + ErrDisallowed = errors.New("robots.txt disallows this URL") + ErrNotFound = errors.New("not found") + ErrTooLarge = errors.New("response over the size cap") +) + +const maxRobotsBytes = 512 << 10 + +// Options configure a client. +type Options struct { + UserAgent string + CacheDir string // conditional requests; empty disables the cache + MinInterval time.Duration // between two requests to one host + Timeout time.Duration // per request + // AllowPrivate lets tests talk to httptest servers on loopback. Never + // set in the collector. + AllowPrivate bool + // DialOverride sends every connection to this address (tests only, + // with AllowPrivate). + DialOverride string +} + +// Client is safe for concurrent use. +type Client struct { + opt Options + http *http.Client + + mu sync.Mutex + next map[string]time.Time // host -> earliest next request + robots map[string]*robotsRules +} + +// Response is a fetched body. +type Response struct { + URL string // final URL after redirects + Body []byte + FetchedAt time.Time + Cached bool // 304: the body is the cached one +} + +// New returns a client. +func New(opt Options) *Client { + if opt.MinInterval == 0 { + opt.MinInterval = time.Second + } + if opt.Timeout == 0 { + opt.Timeout = 30 * time.Second + } + c := &Client{opt: opt, next: map[string]time.Time{}, robots: map[string]*robotsRules{}} + dialer := &net.Dialer{Timeout: 10 * time.Second, Control: func(_, address string, _ syscall.RawConn) error { + if opt.AllowPrivate { + return nil + } + ap, err := netip.ParseAddrPort(address) + if err != nil || !model.PublicAddr(ap.Addr()) { + return fmt.Errorf("refusing to connect to non-public address %s", address) + } + return nil + }} + dial := dialer.DialContext + if opt.DialOverride != "" && opt.AllowPrivate { + dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, opt.DialOverride) + } + } + transport := &http.Transport{ + Proxy: nil, + DialContext: dial, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: opt.Timeout, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 30 * time.Second, + } + c.http = &http.Client{Transport: transport, Timeout: opt.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return errors.New("more than 3 redirects") + } + if req.URL.Scheme != "https" && !(opt.AllowPrivate && req.URL.Scheme == "http") { + return errors.New("redirect to a non-https URL") + } + // A robots.txt fetch is not itself subject to robots.txt (a + // site that redirects everything would otherwise recurse). + if req.Context().Value(robotsFetch{}) == nil { + if ok, err := c.allowed(req.Context(), req.URL); err != nil || !ok { + return ErrDisallowed + } + } + return c.wait(req.Context(), req.URL.Host) + }} + return c +} + +// Get fetches rawURL (at most maxBytes) after checking robots.txt. +func (c *Client) Get(ctx context.Context, rawURL string, maxBytes int64) (*Response, error) { + u, err := url.Parse(rawURL) + if err != nil || (u.Scheme != "https" && !(c.opt.AllowPrivate && u.Scheme == "http")) || u.Host == "" || u.User != nil { + return nil, fmt.Errorf("fetch: refusing URL %q", rawURL) + } + ok, err := c.allowed(ctx, u) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%s: %w", rawURL, ErrDisallowed) + } + return c.get(ctx, u, maxBytes, true) +} + +type cacheEntry struct { + URL string `json:"url"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Body []byte `json:"body"` +} + +func (c *Client) cachePath(u string) string { + if c.opt.CacheDir == "" { + return "" + } + sum := sha256.Sum256([]byte(u)) + return filepath.Join(c.opt.CacheDir, hex.EncodeToString(sum[:])+".json") +} + +func (c *Client) get(ctx context.Context, u *url.URL, maxBytes int64, useCache bool) (*Response, error) { + if err := c.wait(ctx, u.Host); err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.opt.UserAgent) + req.Header.Set("From", "security@openctem.io") + var cached *cacheEntry + path := "" + if useCache { + path = c.cachePath(u.String()) + } + if path != "" { + if b, err := os.ReadFile(path); err == nil { + var e cacheEntry + if json.Unmarshal(b, &e) == nil && e.URL == u.String() && int64(len(e.Body)) <= maxBytes { + cached = &e + if e.ETag != "" { + req.Header.Set("If-None-Match", e.ETag) + } + if e.LastModified != "" { + req.Header.Set("If-Modified-Since", e.LastModified) + } + } + } + } + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + now := time.Now().UTC().Truncate(time.Second) + switch { + case resp.StatusCode == http.StatusNotModified && cached != nil: + return &Response{URL: u.String(), Body: cached.Body, FetchedAt: now, Cached: true}, nil + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return nil, fmt.Errorf("%s: %w", u, ErrNotFound) + case resp.StatusCode != http.StatusOK: + return nil, &StatusError{URL: u.String(), Code: resp.StatusCode} + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > maxBytes { + return nil, fmt.Errorf("%s: %w (%d bytes)", u, ErrTooLarge, maxBytes) + } + if path != "" && (resp.Header.Get("ETag") != "" || resp.Header.Get("Last-Modified") != "") { + e := cacheEntry{URL: u.String(), ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), Body: body} + if b, err := json.Marshal(e); err == nil { + _ = os.MkdirAll(c.opt.CacheDir, 0o755) + _ = os.WriteFile(path, b, 0o644) + } + } + return &Response{URL: resp.Request.URL.String(), Body: body, FetchedAt: now}, nil +} + +// wait blocks until a request to host is polite. +func (c *Client) wait(ctx context.Context, host string) error { + c.mu.Lock() + now := time.Now() + at := c.next[host] + if at.Before(now) { + at = now + } + c.next[host] = at.Add(c.opt.MinInterval) + c.mu.Unlock() + if d := time.Until(at); d > 0 { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } + return nil +} + +// allowed checks robots.txt of u's origin (fetched once per run). A missing +// robots.txt (4xx) allows everything; an unreachable one (network error, +// 5xx) disallows everything, as RFC 9309 asks. +func (c *Client) allowed(ctx context.Context, u *url.URL) (bool, error) { + origin := u.Scheme + "://" + u.Host + c.mu.Lock() + r, ok := c.robots[origin] + c.mu.Unlock() + if !ok { + ru, _ := url.Parse(origin + "/robots.txt") + resp, err := c.get(context.WithValue(ctx, robotsFetch{}, true), ru, maxRobotsBytes, false) + switch { + case err == nil: + r = parseRobots(string(resp.Body), Product) + case errors.Is(err, ErrNotFound) || isStatus(err, 400, 499): + r = &robotsRules{} + case errors.Is(err, ErrTooLarge): + r = &robotsRules{} + case ctx.Err() != nil: + return false, ctx.Err() + default: + r = &robotsRules{disallowAll: true} + } + c.mu.Lock() + c.robots[origin] = r + c.mu.Unlock() + } + path := u.EscapedPath() + if path == "" { + path = "/" + } + if u.RawQuery != "" { + path += "?" + u.RawQuery + } + return r.allows(path), nil +} + +type robotsFetch struct{} + +// StatusError is an unexpected HTTP status. +type StatusError struct { + URL string + Code int +} + +func (e *StatusError) Error() string { return fmt.Sprintf("%s: HTTP %d", e.URL, e.Code) } + +func isStatus(err error, lo, hi int) bool { + var se *StatusError + return errors.As(err, &se) && se.Code >= lo && se.Code <= hi +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 0000000..e575f47 --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,154 @@ +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestRobots(t *testing.T) { + body := ` +User-agent: * +Disallow: /private +Allow: /private/ok + +User-agent: openctem-programfeed +User-agent: other +Disallow: /mine/*.json$ +Allow: / + +User-agent: badbot +Disallow: / +` + r := parseRobots(body, Product) + cases := map[string]bool{ + "/": true, + "/private": true, // our group applies, not "*" + "/mine/a.json": false, + "/mine/a.json?x": true, + "/mine/dir/b.txt": true, + } + for p, want := range cases { + if got := r.allows(p); got != want { + t.Errorf("%s: %v, want %v", p, got, want) + } + } + star := parseRobots(body, "someone-else") + if star.allows("/private/x") || !star.allows("/private/ok/1") || !star.allows("/public") { + t.Error("star group rules") + } + if !parseRobots("", Product).allows("/anything") { + t.Error("empty robots.txt must allow") + } +} + +func newClient(cacheDir string) *Client { + return New(Options{UserAgent: UserAgent("test"), CacheDir: cacheDir, MinInterval: time.Millisecond, AllowPrivate: true}) +} + +func TestGetPoliteAndConditional(t *testing.T) { + var hits, notModified atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/robots.txt": + _, _ = w.Write([]byte("User-agent: *\nDisallow: /secret\n")) + case "/data.json": + hits.Add(1) + if !strings.Contains(r.UserAgent(), "security@openctem.io") || r.Header.Get("From") != "security@openctem.io" { + w.WriteHeader(http.StatusForbidden) + return + } + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte(`{"ok":true}`)) + case "/big": + _, _ = w.Write([]byte(strings.Repeat("x", 2048))) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + ctx := context.Background() + c := newClient(t.TempDir()) + for i := 0; i < 2; i++ { + r, err := c.Get(ctx, srv.URL+"/data.json", 1024) + if err != nil || string(r.Body) != `{"ok":true}` || r.Cached != (i == 1) { + t.Fatalf("get %d: %v %+v", i, err, r) + } + } + if hits.Load() != 2 || notModified.Load() != 1 { + t.Errorf("hits %d, 304s %d", hits.Load(), notModified.Load()) + } + if _, err := c.Get(ctx, srv.URL+"/secret/x", 1024); !errors.Is(err, ErrDisallowed) { + t.Errorf("robots: %v", err) + } + if _, err := c.Get(ctx, srv.URL+"/big", 1024); !errors.Is(err, ErrTooLarge) { + t.Errorf("cap: %v", err) + } + if _, err := c.Get(ctx, srv.URL+"/nope", 1024); !errors.Is(err, ErrNotFound) { + t.Errorf("404: %v", err) + } +} + +func TestRefusesPrivateAddresses(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("hi")) })) + defer srv.Close() + // The collector's client (AllowPrivate false) must not reach loopback, + // whatever the URL says; it also refuses plain http. + c := New(Options{UserAgent: UserAgent("test"), MinInterval: time.Millisecond}) + if _, err := c.Get(context.Background(), srv.URL+"/", 1024); err == nil { + t.Fatal("fetched a loopback URL") + } + httpsURL := strings.Replace(srv.URL, "http://", "https://", 1) + if _, err := c.Get(context.Background(), httpsURL+"/", 1024); err == nil { + t.Fatal("connected to a loopback address") + } + if _, err := c.Get(context.Background(), "https://user:pw@openctem.io/", 1024); err == nil { + t.Fatal("accepted credentials in URL") + } +} + +func TestUnreachableRobotsDisallows(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/robots.txt" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("data")) + })) + defer srv.Close() + if _, err := newClient("").Get(context.Background(), srv.URL+"/x", 1024); !errors.Is(err, ErrDisallowed) { + t.Fatalf("5xx robots.txt: %v", err) + } +} + +// A site that redirects every path (robots.txt included) to its home page +// must not make the robots.txt check recurse. +func TestRedirectEverythingTerminates(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/home" { + _, _ = w.Write([]byte("home")) + return + } + http.Redirect(w, r, "/home", http.StatusFound) + })) + defer srv.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r, err := newClient("").Get(ctx, srv.URL+"/.well-known/security.txt", 1024) + if ctx.Err() != nil { + t.Fatal("robots.txt check recursed") + } + if err != nil || r.URL != srv.URL+"/home" { + t.Fatalf("%v %+v", err, r) + } +} diff --git a/internal/fetch/robots.go b/internal/fetch/robots.go new file mode 100644 index 0000000..e7ee102 --- /dev/null +++ b/internal/fetch/robots.go @@ -0,0 +1,116 @@ +package fetch + +import ( + "strings" +) + +// robotsRules is the group of a robots.txt that applies to the collector +// (RFC 9309): the group naming our product token, else the "*" group. +type robotsRules struct { + disallowAll bool + rules []robotsRule +} + +type robotsRule struct { + allow bool + pattern string +} + +func parseRobots(body, product string) *robotsRules { + product = strings.ToLower(product) + type group struct { + agents []string + rules []robotsRule + } + var groups []*group + var cur *group + lastWasAgent := false + for _, line := range strings.Split(body, "\n") { + if i := strings.IndexByte(line, '#'); i >= 0 { + line = line[:i] + } + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + k, v = strings.ToLower(strings.TrimSpace(k)), strings.TrimSpace(v) + switch k { + case "user-agent": + if cur == nil || !lastWasAgent { + cur = &group{} + groups = append(groups, cur) + } + cur.agents = append(cur.agents, strings.ToLower(v)) + lastWasAgent = true + case "allow", "disallow": + lastWasAgent = false + if cur == nil || v == "" { + continue + } + cur.rules = append(cur.rules, robotsRule{allow: k == "allow", pattern: v}) + default: + lastWasAgent = false + } + } + var mine, star []robotsRule + matched := false + for _, g := range groups { + for _, a := range g.agents { + switch { + case a == product: + mine = append(mine, g.rules...) + matched = true + case a == "*": + star = append(star, g.rules...) + } + } + } + if matched { + return &robotsRules{rules: mine} + } + return &robotsRules{rules: star} +} + +// allows applies the longest matching rule; on a tie, allow wins. +func (r *robotsRules) allows(path string) bool { + if r.disallowAll { + return false + } + best, allow := -1, true + for _, rule := range r.rules { + if !match(rule.pattern, path) { + continue + } + n := len(rule.pattern) + if n > best || (n == best && rule.allow) { + best, allow = n, rule.allow + } + } + return allow +} + +// match implements robots.txt patterns: "*" matches any run, a trailing +// "$" anchors the end, otherwise a prefix match. +func match(pattern, path string) bool { + anchored := strings.HasSuffix(pattern, "$") + pattern = strings.TrimSuffix(pattern, "$") + parts := strings.Split(pattern, "*") + if !strings.HasPrefix(path, parts[0]) { + return false + } + pos := len(parts[0]) + for i, part := range parts[1:] { + if i == len(parts)-2 && anchored { + return len(path)-pos >= len(part) && strings.HasSuffix(path, part) + } + j := strings.Index(path[pos:], part) + if j < 0 { + return false + } + pos += j + len(part) + } + if anchored { + return pos == len(path) + } + return true +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..23c41e9 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,409 @@ +// Package model is the feed's record model and its validation. +// +// The rules are the ones the OpenCTEM importer applies: a bundle with one +// record the importer would refuse is refused whole, so the collector +// refuses to publish it first. Targets are different: a target the scope +// rules refuse (a public suffix, a private range, a huge CIDR …) does not +// fail the bundle; it is moved to the program's rejected list with the +// reason, so subscribers can see what a program claimed and why the feed +// did not pass it on. +package model + +import ( + "errors" + "fmt" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +// Limits (the importer applies the same). +const ( + MaxNameLen = 200 + MaxTextLen = 4000 + MaxURLLen = 2048 + MaxValueLen = 512 + MaxTargets = 2000 // per list (in scope, out of scope, rejected) + MaxListItems = 32 // testing restrictions, required headers, languages + MaxPrograms = 100000 + MaxChangeFields = 16 +) + +// Program types and statuses. +const ( + TypeBounty = "bounty" + TypeVDP = "vdp" + + StatusOpen = "open" + StatusPaused = "paused" + StatusClosed = "closed" +) + +// Safe-harbour values. +const ( + SafeHarbourFull = "full" + SafeHarbourPartial = "partial" + SafeHarbourNone = "none" + SafeHarbourUnknown = "unknown" +) + +// Target types. +const ( + TargetDomain = "domain" + TargetWildcard = "wildcard" + TargetIP = "ip" + TargetCIDR = "cidr" + TargetURL = "url" + TargetMobileApp = "mobile_app" + TargetSourceRepo = "source_repo" + TargetOther = "other" +) + +// Target confidence: published = the program's own structured scope; +// inferred = derived by the collector (for example the host a program's +// security.txt was served from). An inferred target is a suggestion, never +// an authorisation to test. +const ( + ConfidencePublished = "published" + ConfidenceInferred = "inferred" +) + +var ( + idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$`) + sha256RE = regexp.MustCompile(`^[0-9a-f]{64}$`) + severities = map[string]bool{"": true, "none": true, "low": true, "medium": true, "high": true, "critical": true} + types = map[string]bool{TypeBounty: true, TypeVDP: true} + statuses = map[string]bool{StatusOpen: true, StatusPaused: true, StatusClosed: true} + harbours = map[string]bool{SafeHarbourFull: true, SafeHarbourPartial: true, SafeHarbourNone: true, SafeHarbourUnknown: true} + targetType = map[string]bool{TargetDomain: true, TargetWildcard: true, TargetIP: true, TargetCIDR: true, TargetURL: true, + TargetMobileApp: true, TargetSourceRepo: true, TargetOther: true} + confidences = map[string]bool{ConfidencePublished: true, ConfidenceInferred: true} +) + +// Program is one public bug-bounty or vulnerability-disclosure program. +type Program struct { + ID string `json:"id"` // ":", stable, owned by one source + Source string `json:"source"` // the adapter that produced it + Platform string `json:"platform"` // where the program is hosted ("self-hosted" or a platform name) + Name string `json:"name"` + URL string `json:"url"` // the program's policy page + Type string `json:"type"` + Status string `json:"status"` + OffersBounty bool `json:"offers_bounty"` + ScopePublished bool `json:"scope_published"` // the source carries the program's own structured scope + InScope []Target `json:"in_scope"` + OutOfScope []Target `json:"out_of_scope"` + Rejected []Rejected `json:"rejected"` + Rules Rules `json:"rules"` + Terms Terms `json:"terms"` + Contact Contact `json:"contact"` + Provenance Provenance `json:"provenance"` + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + LastChanged time.Time `json:"last_changed"` + ClosedAt *time.Time `json:"closed_at,omitempty"` +} + +// Target is one scope entry. +type Target struct { + Type string `json:"type"` + Value string `json:"value"` + Confidence string `json:"confidence"` + Eligible *bool `json:"eligible_for_bounty,omitempty"` + MaxSev string `json:"max_severity,omitempty"` + Notes string `json:"notes,omitempty"` +} + +// Rejected is a target the scope rules refused, with the reason. +type Rejected struct { + Scope string `json:"scope"` // in | out + Type string `json:"type"` + Value string `json:"value"` + Reason string `json:"reason"` +} + +// Rules summarises how a program may be tested. +type Rules struct { + Summary string `json:"summary,omitempty"` + TestingRestrictions []string `json:"testing_restrictions"` + RateLimit string `json:"rate_limit,omitempty"` + RequiredHeaders []string `json:"required_headers"` + SafeHarbour string `json:"safe_harbour"` + DisclosureDays int `json:"disclosure_days,omitempty"` + Languages []string `json:"languages"` +} + +// Terms names the program's terms and, when the collector read them, the +// SHA-256 of the exact bytes it read. +type Terms struct { + URL string `json:"url"` + SHA256 string `json:"sha256,omitempty"` + FetchedAt *time.Time `json:"fetched_at,omitempty"` +} + +// Contact is the program's reporting channel. +type Contact struct { + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// Provenance records where and when a record was read. +type Provenance struct { + Source string `json:"source"` + SourceURL string `json:"source_url"` + FetchedAt time.Time `json:"fetched_at"` +} + +// Validate checks a normalised program. +func (p *Program) Validate() error { + fail := func(format string, args ...any) error { + return fmt.Errorf("program %q: %s", p.ID, fmt.Sprintf(format, args...)) + } + switch { + case !idRE.MatchString(p.ID): + return fail("id must be : in lower case") + case !strings.HasPrefix(p.ID, p.Source+":"): + return fail("id is not under its source %q", p.Source) + case p.Provenance.Source != p.Source: + return fail("provenance source %q", p.Provenance.Source) + case !safeText(p.Name, MaxNameLen) || strings.TrimSpace(p.Name) == "": + return fail("name is empty, too long or has control characters") + case !safeText(p.Platform, 64) || p.Platform == "": + return fail("platform") + case !types[p.Type]: + return fail("type %q", p.Type) + case !statuses[p.Status]: + return fail("status %q", p.Status) + case (p.Status == StatusClosed) != (p.ClosedAt != nil): + return fail("closed_at must be set exactly when the status is closed") + case p.OffersBounty != (p.Type == TypeBounty): + return fail("offers_bounty must match the type") + case p.FirstSeen.IsZero() || p.LastSeen.Before(p.FirstSeen) || p.LastChanged.Before(p.FirstSeen): + return fail("first_seen must not be after last_seen or last_changed") + case p.Provenance.FetchedAt.IsZero(): + return fail("provenance fetched_at") + } + if err := checkURL(p.URL, true); err != nil { + return fail("url: %v", err) + } + if err := checkURL(p.Provenance.SourceURL, true); err != nil { + return fail("provenance source_url: %v", err) + } + if err := checkURL(p.Terms.URL, true); err != nil { + return fail("terms url: %v", err) + } + if p.Terms.SHA256 != "" && !sha256RE.MatchString(p.Terms.SHA256) { + return fail("terms sha256") + } + if (p.Terms.SHA256 == "") != (p.Terms.FetchedAt == nil) { + return fail("terms sha256 and fetched_at go together") + } + if p.Contact.Email != "" { + if a, err := mail.ParseAddress(p.Contact.Email); err != nil || a.Address != p.Contact.Email || len(p.Contact.Email) > 254 { + return fail("contact email") + } + } + if p.Contact.URL != "" { + if err := checkURL(p.Contact.URL, false); err != nil { + return fail("contact url: %v", err) + } + } + if err := p.Rules.validate(); err != nil { + return fail("%v", err) + } + for _, list := range []struct { + name string + ts []Target + }{{"in_scope", p.InScope}, {"out_of_scope", p.OutOfScope}} { + if list.ts == nil { + return fail("%s must be a list", list.name) + } + if len(list.ts) > MaxTargets { + return fail("%s has more than %d targets", list.name, MaxTargets) + } + for i, t := range list.ts { + if err := t.Validate(); err != nil { + return fail("%s: %v", list.name, err) + } + if i > 0 && !targetLess(list.ts[i-1], t) { + return fail("%s is not sorted or has duplicates", list.name) + } + } + } + if p.Rejected == nil || len(p.Rejected) > MaxTargets { + return fail("rejected must be a list of at most %d", MaxTargets) + } + for _, r := range p.Rejected { + if (r.Scope != "in" && r.Scope != "out") || !safeText(r.Type, 32) || !safeText(r.Value, MaxValueLen) || !safeText(r.Reason, 200) || r.Reason == "" { + return fail("rejected entry %q", r.Value) + } + } + return nil +} + +func (r Rules) validate() error { + if !safeText(r.Summary, MaxTextLen) || !safeText(r.RateLimit, 200) { + return errors.New("rules: summary or rate_limit too long or has control characters") + } + if !harbours[r.SafeHarbour] { + return fmt.Errorf("rules: safe_harbour %q", r.SafeHarbour) + } + if r.DisclosureDays < 0 || r.DisclosureDays > 3650 { + return errors.New("rules: disclosure_days") + } + for name, l := range map[string][]string{"testing_restrictions": r.TestingRestrictions, "required_headers": r.RequiredHeaders, "languages": r.Languages} { + if l == nil || len(l) > MaxListItems { + return fmt.Errorf("rules: %s must be a list of at most %d", name, MaxListItems) + } + for _, s := range l { + if s == "" || !safeText(s, 500) { + return fmt.Errorf("rules: %s entry", name) + } + } + } + return nil +} + +// Validate checks a normalised target (Normalize produces only valid ones). +func (t Target) Validate() error { + if !targetType[t.Type] || !confidences[t.Confidence] || !severities[t.MaxSev] || !safeText(t.Notes, 500) { + return fmt.Errorf("target %q: type, confidence, max_severity or notes", t.Value) + } + n, reason := NormalizeTarget(t.Type, t.Value) + if reason != "" || n != t.Value { + return fmt.Errorf("target %q is not a normalised %s (%s)", t.Value, t.Type, reason) + } + return nil +} + +func targetLess(a, b Target) bool { + if a.Type != b.Type { + return a.Type < b.Type + } + return a.Value < b.Value +} + +// SortTargets orders targets canonically and drops exact duplicates +// (same type and value; the first wins). +func SortTargets(ts []Target) []Target { + sort.SliceStable(ts, func(i, j int) bool { return targetLess(ts[i], ts[j]) }) + out := ts[:0] + for i, t := range ts { + if i > 0 && t.Type == ts[i-1].Type && t.Value == ts[i-1].Value { + continue + } + out = append(out, t) + } + return out +} + +// safeText: valid UTF-8, at most max bytes, no control characters other +// than newline and tab, no bidi overrides. +func safeText(s string, max int) bool { + if len(s) > max || !utf8.ValidString(s) { + return false + } + for _, r := range s { + if (unicode.IsControl(r) && r != '\n' && r != '\t') || (r >= 0x202A && r <= 0x202E) || (r >= 0x2066 && r <= 0x2069) { + return false + } + } + return true +} + +// checkURL accepts absolute http(s) URLs without credentials; mailto is +// allowed where links may be mail links. +func checkURL(s string, httpOnly bool) error { + if s == "" || len(s) > MaxURLLen || !safeText(s, MaxURLLen) || strings.ContainsAny(s, " \n\t") { + return errors.New("empty, too long or has spaces/control characters") + } + u, err := url.Parse(s) + if err != nil { + return err + } + switch { + case u.Scheme == "mailto" && !httpOnly: + return nil + case u.Scheme != "https" && u.Scheme != "http": + return fmt.Errorf("scheme %q", u.Scheme) + case u.User != nil: + return errors.New("credentials in URL") + case u.Host == "": + return errors.New("no host") + } + return nil +} + +// CleanText trims a source string, folds whitespace runs and cuts it to max +// bytes on a rune boundary; it returns "" for text with control +// characters. +func CleanText(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + if !safeText(s, len(s)) { + return "" + } + if len(s) <= max { + return s + } + cut := max + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] +} + +// Change is one entry of a run's change log. +type Change struct { + Sequence uint64 `json:"sequence"` + Program string `json:"program"` + Kind string `json:"kind"` + Scope string `json:"scope,omitempty"` // in | out, for target changes + Target *Target `json:"target,omitempty"` // the target after (before, for removed) + Fields []string `json:"fields,omitempty"` // for program_modified / target_modified +} + +// Change kinds. +const ( + ChangeProgramAdded = "program_added" + ChangeProgramClosed = "program_closed" + ChangeProgramReopened = "program_reopened" + ChangeProgramDropped = "program_dropped" + ChangeProgramModified = "program_modified" + ChangeStatus = "status_changed" + ChangeRules = "rules_changed" + ChangeTerms = "terms_changed" + ChangeTargetAdded = "target_added" + ChangeTargetRemoved = "target_removed" + ChangeTargetModified = "target_modified" +) + +var changeKinds = map[string]bool{ChangeProgramAdded: true, ChangeProgramClosed: true, ChangeProgramReopened: true, + ChangeProgramDropped: true, ChangeProgramModified: true, ChangeStatus: true, ChangeRules: true, ChangeTerms: true, + ChangeTargetAdded: true, ChangeTargetRemoved: true, ChangeTargetModified: true} + +// Validate checks a change entry. +func (c *Change) Validate() error { + if !idRE.MatchString(c.Program) || !changeKinds[c.Kind] || c.Sequence == 0 || len(c.Fields) > MaxChangeFields { + return fmt.Errorf("change %q %q: program, kind, sequence or fields", c.Program, c.Kind) + } + isTarget := strings.HasPrefix(c.Kind, "target_") + if isTarget != (c.Target != nil) || isTarget != (c.Scope == "in" || c.Scope == "out") || (!isTarget && c.Scope != "") { + return fmt.Errorf("change %q %q: target and scope go with target changes only", c.Program, c.Kind) + } + if c.Target != nil { + if err := c.Target.Validate(); err != nil { + return err + } + } + for _, f := range c.Fields { + if !safeText(f, 64) || f == "" { + return fmt.Errorf("change %q: field name", c.Program) + } + } + return nil +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..614cf2c --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,216 @@ +package model + +import ( + "strings" + "testing" + "time" +) + +func TestNormalizeTarget(t *testing.T) { + ok := []struct{ typ, in, want string }{ + {TargetDomain, "OpenCTEM.ORG.", "openctem.org"}, + {TargetDomain, "api.openctem.io", "api.openctem.io"}, + {TargetDomain, "bücher.de", "xn--bcher-kva.de"}, + {TargetDomain, "foo.example.co.uk", "foo.example.co.uk"}, + {TargetWildcard, "*.OpenCTEM.io", "*.openctem.io"}, + {TargetWildcard, "*.example.co.uk", "*.example.co.uk"}, + {TargetIP, "8.8.8.8", "8.8.8.8"}, + {TargetIP, "::ffff:8.8.4.4", "8.8.4.4"}, + {TargetIP, "2606:4700::1111", "2606:4700::1111"}, + {TargetCIDR, "8.8.8.1/24", "8.8.8.0/24"}, + {TargetCIDR, "203.1.0.0/16", "203.1.0.0/16"}, + {TargetCIDR, "2606:4700::/48", "2606:4700::/48"}, + {TargetURL, "HTTPS://App.OpenCTEM.io/login#x", "https://app.openctem.io/login"}, + {TargetURL, "https://openctem.io:8443", "https://openctem.io:8443/"}, + {TargetMobileApp, " com.example.app ", "com.example.app"}, + {TargetSourceRepo, "https://github.com/openctemio/openctem", "https://github.com/openctemio/openctem"}, + } + for _, c := range ok { + got, reason := NormalizeTarget(c.typ, c.in) + if reason != "" || got != c.want { + t.Errorf("%s %q = %q (%s), want %q", c.typ, c.in, got, reason, c.want) + } + } + bad := []struct{ typ, in string }{ + {TargetDomain, "com"}, + {TargetDomain, "co.uk"}, + {TargetDomain, "github.io"}, // private public suffix + {TargetDomain, "localhost"}, + {TargetDomain, "printer.local"}, + {TargetDomain, "corp.internal"}, + {TargetDomain, "example.com"}, // documentation + {TargetDomain, "10.0.0.1"}, + {TargetDomain, "a..b.com"}, + {TargetDomain, "-bad.com"}, + {TargetDomain, "under_score.com"}, + {TargetDomain, "x.123"}, + {TargetDomain, "foo.*.com"}, + {TargetDomain, strings.Repeat("a", 64) + ".com"}, + {TargetDomain, "bad‮.com"}, + {TargetWildcard, "*.com"}, + {TargetWildcard, "*.co.uk"}, + {TargetWildcard, "*.github.io"}, + {TargetWildcard, "*"}, + {TargetWildcard, "*.*.example.org"}, + {TargetWildcard, "example.org"}, + {TargetIP, "10.1.2.3"}, + {TargetIP, "127.0.0.1"}, + {TargetIP, "169.254.169.254"}, + {TargetIP, "100.64.0.1"}, + {TargetIP, "192.0.2.10"}, + {TargetIP, "::1"}, + {TargetIP, "fd00::1"}, + {TargetIP, "2001:db8::1"}, + {TargetIP, "fe80::1%eth0"}, + {TargetIP, "224.0.0.1"}, + {TargetIP, "not-an-ip"}, + {TargetCIDR, "0.0.0.0/0"}, + {TargetCIDR, "8.0.0.0/8"}, + {TargetCIDR, "8.8.0.0/15"}, + {TargetCIDR, "2606::/16"}, + {TargetCIDR, "10.0.0.0/24"}, + {TargetCIDR, "172.16.5.0/24"}, + {TargetCIDR, "::ffff:8.8.8.0/120"}, + {TargetURL, "javascript:alert(1)"}, + {TargetURL, "ftp://openctem.io/"}, + {TargetURL, "https://user:pw@openctem.io/"}, + {TargetURL, "https://127.0.0.1/"}, + {TargetURL, "https://[::1]/"}, + {TargetURL, "https://*.openctem.io/"}, + {TargetURL, "https://com/"}, + {TargetURL, "https://openctem.io:99999/"}, + {TargetOther, ""}, + {TargetOther, "a\x00b"}, + {"subnet", "1.2.3.0/24"}, + } + for _, c := range bad { + if got, reason := NormalizeTarget(c.typ, c.in); reason == "" { + t.Errorf("%s %q accepted as %q", c.typ, c.in, got) + } + } +} + +func FuzzNormalizeTarget(f *testing.F) { + for _, s := range []string{"example.org", "*.a.b.co.uk", "8.8.8.0/24", "https://a.io/x?y#z", "::ffff:1.2.3.4", "xn--bcher-kva.de"} { + f.Add(s) + } + types := []string{TargetDomain, TargetWildcard, TargetIP, TargetCIDR, TargetURL, TargetOther} + f.Fuzz(func(t *testing.T, s string) { + for _, typ := range types { + n, reason := NormalizeTarget(typ, s) + if reason != "" { + continue + } + // Normalising is idempotent, so every accepted target validates. + if n2, r2 := NormalizeTarget(typ, n); r2 != "" || n2 != n { + t.Fatalf("%s %q -> %q -> %q (%s)", typ, s, n, n2, r2) + } + } + }) +} + +func validProgram() Program { + now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) + return Program{ + ID: "disclose:openctem", Source: "disclose", Platform: "self-hosted", Name: "OpenCTEM", + URL: "https://openctem.io/security", Type: TypeVDP, Status: StatusOpen, + InScope: []Target{{Type: TargetWildcard, Value: "*.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}, {Type: TargetIP, Value: "8.8.8.8"}}, + OutOfScope: []Target{{Type: TargetDomain, Value: "status.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}}, + Rules: Rules{TestingRestrictions: []string{"no DoS", "no DoS"}}, + Terms: Terms{URL: "https://openctem.io/security"}, + Provenance: Provenance{Source: "disclose", SourceURL: "https://example.org/list.json", FetchedAt: now}, + FirstSeen: now, LastSeen: now, LastChanged: now, + } +} + +func TestNormalizeProgram(t *testing.T) { + p := validProgram() + p.InScope = append(p.InScope, Target{Type: TargetWildcard, Value: "*.com"}, Target{Type: TargetCIDR, Value: "10.0.0.0/8"}, + Target{Type: TargetDomain, Value: "OPENCTEM.io"}) + Normalize(&p) + if err := p.Validate(); err != nil { + t.Fatal(err) + } + var in []string + for _, x := range p.InScope { + in = append(in, x.Type+":"+x.Value) + if x.Confidence != ConfidencePublished { + t.Errorf("confidence %q", x.Confidence) + } + } + // Sorted, deduplicated, and openctem.io dropped because it is also out of scope. + if got := strings.Join(in, " "); got != "ip:8.8.8.8 wildcard:*.openctem.io" { + t.Errorf("in scope = %s", got) + } + reasons := map[string]string{} + for _, r := range p.Rejected { + reasons[r.Value] = r.Reason + } + if reasons["*.com"] == "" || reasons["10.0.0.0/8"] == "" || reasons["openctem.io"] != "also listed out of scope" { + t.Errorf("rejected = %+v", p.Rejected) + } + if len(p.Rules.TestingRestrictions) != 1 || p.Rules.SafeHarbour != SafeHarbourUnknown { + t.Errorf("rules = %+v", p.Rules) + } +} + +func TestProgramValidateRefuses(t *testing.T) { + cases := map[string]func(p *Program){ + "bad id": func(p *Program) { p.ID = "Disclose:X" }, + "id of other source": func(p *Program) { p.ID = "other:x" }, + "control in name": func(p *Program) { p.Name = "a\x07b" }, + "empty name": func(p *Program) { p.Name = " " }, + "bad type": func(p *Program) { p.Type = "paid" }, + "bounty mismatch": func(p *Program) { p.OffersBounty = true }, + "closed without time": func(p *Program) { p.Status = StatusClosed }, + "javascript url": func(p *Program) { p.URL = "javascript:alert(1)" }, + "terms hash no time": func(p *Program) { p.Terms.SHA256 = strings.Repeat("a", 64) }, + "bad email": func(p *Program) { p.Contact.Email = "x " }, + "unsorted targets": func(p *Program) { + p.InScope = []Target{{Type: TargetDomain, Value: "b.org", Confidence: ConfidencePublished}, {Type: TargetDomain, Value: "a.org", Confidence: ConfidencePublished}} + }, + "unnormalised target": func(p *Program) { + p.InScope = []Target{{Type: TargetDomain, Value: "A.org", Confidence: ConfidencePublished}} + }, + "public suffix target": func(p *Program) { + p.InScope = []Target{{Type: TargetWildcard, Value: "*.com", Confidence: ConfidencePublished}} + }, + "nil list": func(p *Program) { p.OutOfScope = nil }, + "first seen": func(p *Program) { p.FirstSeen = p.LastSeen.Add(time.Hour) }, + "bad harbour": func(p *Program) { p.Rules.SafeHarbour = "maybe" }, + "no provenance": func(p *Program) { p.Provenance.FetchedAt = time.Time{} }, + } + for name, mut := range cases { + p := validProgram() + Normalize(&p) + mut(&p) + if err := p.Validate(); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestChangeValidate(t *testing.T) { + tgt := Target{Type: TargetDomain, Value: "openctem.io", Confidence: ConfidenceInferred} + good := []Change{ + {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &tgt}, + } + for _, c := range good { + if err := c.Validate(); err != nil { + t.Error(err) + } + } + bad := []Change{ + {Sequence: 0, Program: "disclose:x", Kind: ChangeProgramAdded}, + {Sequence: 1, Program: "disclose:x", Kind: "deleted"}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded, Scope: "in", Target: &tgt}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &Target{Type: TargetWildcard, Value: "*.com", Confidence: ConfidenceInferred}}, + } + for i, c := range bad { + if err := c.Validate(); err == nil { + t.Errorf("bad change %d accepted", i) + } + } +} diff --git a/internal/model/normalize.go b/internal/model/normalize.go new file mode 100644 index 0000000..28585b1 --- /dev/null +++ b/internal/model/normalize.go @@ -0,0 +1,103 @@ +package model + +import ( + "sort" + "strings" +) + +// Normalize canonicalises the targets of a program an adapter produced: +// every target is normalised or moved to Rejected with the reason, lists are +// deduplicated and sorted, an in-scope target that is also out of scope is +// dropped from in scope (out of scope wins), and lists are capped. Text +// fields are cleaned. It does not set the timestamps; the collector does. +func Normalize(p *Program) { + p.Name = CleanText(p.Name, MaxNameLen) + p.Platform = CleanText(p.Platform, 64) + p.Rules.Summary = CleanText(p.Rules.Summary, MaxTextLen) + p.Rules.RateLimit = CleanText(p.Rules.RateLimit, 200) + p.Rules.TestingRestrictions = cleanList(p.Rules.TestingRestrictions) + p.Rules.RequiredHeaders = cleanList(p.Rules.RequiredHeaders) + p.Rules.Languages = cleanList(p.Rules.Languages) + if p.Rules.SafeHarbour == "" { + p.Rules.SafeHarbour = SafeHarbourUnknown + } + p.OffersBounty = p.Type == TypeBounty + var rejected []Rejected + norm := func(scope string, in []Target) []Target { + out := make([]Target, 0, len(in)) + for _, t := range in { + v, reason := NormalizeTarget(t.Type, t.Value) + if reason != "" { + rejected = append(rejected, Rejected{Scope: scope, Type: CleanText(t.Type, 32), Value: CleanText(t.Value, MaxValueLen), Reason: reason}) + continue + } + t.Value = v + if t.Confidence == "" { + t.Confidence = ConfidencePublished + } + if !severities[t.MaxSev] { + t.MaxSev = "" + } + t.Notes = CleanText(t.Notes, 500) + out = append(out, t) + } + out = SortTargets(out) + if len(out) > MaxTargets { + for _, t := range out[MaxTargets:] { + rejected = append(rejected, Rejected{Scope: scope, Type: t.Type, Value: t.Value, Reason: "over the per-program target cap"}) + } + out = out[:MaxTargets] + } + return out + } + p.OutOfScope = norm("out", p.OutOfScope) + in := norm("in", p.InScope) + excluded := map[string]bool{} + for _, t := range p.OutOfScope { + excluded[t.Type+"\x00"+t.Value] = true + } + p.InScope = in[:0] + for _, t := range in { + if excluded[t.Type+"\x00"+t.Value] { + rejected = append(rejected, Rejected{Scope: "in", Type: t.Type, Value: t.Value, Reason: "also listed out of scope"}) + continue + } + p.InScope = append(p.InScope, t) + } + for i := range rejected { + if rejected[i].Value == "" { + rejected[i].Value = "(unprintable)" + } + if rejected[i].Type == "" { + rejected[i].Type = "unknown" + } + } + sort.SliceStable(rejected, func(i, j int) bool { + a, b := rejected[i], rejected[j] + return strings.Join([]string{a.Scope, a.Type, a.Value, a.Reason}, "\x00") < strings.Join([]string{b.Scope, b.Type, b.Value, b.Reason}, "\x00") + }) + if len(rejected) > MaxTargets { + rejected = rejected[:MaxTargets] + } + if rejected == nil { + rejected = []Rejected{} + } + p.Rejected = rejected +} + +func cleanList(in []string) []string { + out := []string{} + seen := map[string]bool{} + for _, s := range in { + s = CleanText(s, 500) + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + if len(out) == MaxListItems { + break + } + } + return out +} diff --git a/internal/model/schema_test.go b/internal/model/schema_test.go new file mode 100644 index 0000000..d4e7c5f --- /dev/null +++ b/internal/model/schema_test.go @@ -0,0 +1,60 @@ +package model + +import ( + "encoding/json" + "os" + "reflect" + "sort" + "strings" + "testing" +) + +// The published JSON Schema documents every field the feed writes, and +// nothing else. +func TestSchemaMatchesModel(t *testing.T) { + type obj struct { + Properties map[string]json.RawMessage `json:"properties"` + Defs map[string]obj `json:"$defs"` + } + read := func(name string) obj { + b, err := os.ReadFile("../../schema/" + name) + if err != nil { + t.Fatal(err) + } + var o obj + if err := json.Unmarshal(b, &o); err != nil { + t.Fatal(err) + } + return o + } + prog, change := read("program.schema.json"), read("change.schema.json") + cases := []struct { + name string + schema obj + v any + }{ + {"program", prog, Program{}}, + {"target", prog.Defs["target"], Target{}}, + {"rejected", prog.Defs["rejected"], Rejected{}}, + {"rules", prog.Defs["rules"], Rules{}}, + {"terms", prog.Defs["terms"], Terms{}}, + {"contact", prog.Defs["contact"], Contact{}}, + {"provenance", prog.Defs["provenance"], Provenance{}}, + {"change", change, Change{}}, + } + for _, c := range cases { + var want, got []string + rt := reflect.TypeOf(c.v) + for i := 0; i < rt.NumField(); i++ { + want = append(want, strings.Split(rt.Field(i).Tag.Get("json"), ",")[0]) + } + for k := range c.schema.Properties { + got = append(got, k) + } + sort.Strings(want) + sort.Strings(got) + if strings.Join(want, ",") != strings.Join(got, ",") { + t.Errorf("%s: schema has %v, model has %v", c.name, got, want) + } + } +} diff --git a/internal/model/scope.go b/internal/model/scope.go new file mode 100644 index 0000000..cece441 --- /dev/null +++ b/internal/model/scope.go @@ -0,0 +1,206 @@ +package model + +import ( + "net/netip" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/idna" + "golang.org/x/net/publicsuffix" +) + +// Scope caps: a CIDR wider than these is refused (a program does not own +// a /8, and a wide range in a feed would point scanners at strangers). +const ( + MinIPv4Prefix = 16 + MinIPv6Prefix = 48 +) + +// Ranges no program can own on the public internet (RFC 6890 and friends). +var reserved = func() []netip.Prefix { + var out []netip.Prefix + for _, s := range []string{ + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", + "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", + "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", "255.255.255.255/32", + "::/128", "::1/128", "::ffff:0:0/96", "64:ff9b::/96", "64:ff9b:1::/48", "100::/64", "2001::/23", + "2001:db8::/32", "2002::/16", "3fff::/20", "5f00::/16", "fc00::/7", "fe80::/10", "ff00::/8", + } { + out = append(out, netip.MustParsePrefix(s)) + } + return out +}() + +// Names that never resolve on the public internet, or are reserved for +// documentation. +var reservedSuffixes = []string{"local", "localhost", "test", "example", "invalid", "internal", "onion", "arpa", + "home.arpa", "lan", "corp", "intranet", "private", "example.com", "example.net", "example.org"} + +var lookup = idna.New(idna.MapForLookup(), idna.Transitional(false), idna.StrictDomainName(true), idna.BidiRule()) + +// NormalizeTarget returns the canonical form of a target value of the given +// type, or a non-empty reason it is refused. +func NormalizeTarget(typ, value string) (string, string) { + v := strings.TrimSpace(value) + if v == "" || len(v) > MaxValueLen || !safeText(v, MaxValueLen) { + return "", "empty, too long or has control characters" + } + switch typ { + case TargetDomain: + return normDomain(v) + case TargetWildcard: + if !strings.HasPrefix(v, "*.") { + return "", "a wildcard must start with *." + } + d, reason := normDomain(v[2:]) + if reason != "" { + return "", reason + } + return "*." + d, "" + case TargetIP: + a, err := netip.ParseAddr(v) + if err != nil || a.Zone() != "" { + return "", "not an IP address" + } + a = a.Unmap() + if r := reservedAddr(a); r != "" { + return "", r + } + return a.String(), "" + case TargetCIDR: + p, err := netip.ParsePrefix(v) + if err != nil { + return "", "not a CIDR" + } + if p.Addr().Is4In6() { + return "", "IPv4-mapped CIDR" + } + p = p.Masked() + if (p.Addr().Is4() && p.Bits() < MinIPv4Prefix) || (p.Addr().Is6() && p.Bits() < MinIPv6Prefix) { + return "", "CIDR wider than /" + strconv.Itoa(MinIPv4Prefix) + " (IPv4) or /" + strconv.Itoa(MinIPv6Prefix) + " (IPv6)" + } + for _, r := range reserved { + if r.Overlaps(p) { + return "", "private or reserved range" + } + } + return p.String(), "" + case TargetURL: + return normURL(v) + case TargetMobileApp, TargetSourceRepo, TargetOther: + s := CleanText(v, MaxValueLen) + if s == "" { + return "", "empty or has control characters" + } + return s, "" + } + return "", "unknown target type" +} + +func reservedAddr(a netip.Addr) string { + if !a.IsGlobalUnicast() || a.IsPrivate() { + return "private or reserved address" + } + for _, r := range reserved { + if r.Contains(a) { + return "private or reserved address" + } + } + return "" +} + +func normDomain(v string) (string, string) { + d := strings.TrimSuffix(strings.ToLower(v), ".") + if strings.Contains(d, "*") { + return "", "wildcard inside a domain" + } + if _, err := netip.ParseAddr(d); err == nil { + return "", "an IP address, not a domain" + } + ascii, err := lookup.ToASCII(d) + if err != nil { + return "", "not a valid domain name" + } + if len(ascii) > 253 || !strings.Contains(ascii, ".") { + return "", "not a fully qualified domain name" + } + labels := strings.Split(ascii, ".") + for _, l := range labels { + if l == "" || len(l) > 63 || l[0] == '-' || l[len(l)-1] == '-' { + return "", "not a valid domain name" + } + for _, c := range l { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' { + return "", "not a valid domain name" + } + } + } + if tld := labels[len(labels)-1]; strings.Trim(tld, "0123456789") == "" { + return "", "numeric top-level label" + } + for _, s := range reservedSuffixes { + if ascii == s || strings.HasSuffix(ascii, "."+s) { + return "", "reserved or non-public name" + } + } + if ps, _ := publicsuffix.PublicSuffix(ascii); ps == ascii { + return "", "a public suffix" + } + if _, err := publicsuffix.EffectiveTLDPlusOne(ascii); err != nil { + return "", "a public suffix" + } + return ascii, "" +} + +func normURL(v string) (string, string) { + u, err := url.Parse(v) + if err != nil { + return "", "not a URL" + } + u.Scheme = strings.ToLower(u.Scheme) + if u.Scheme != "https" && u.Scheme != "http" { + return "", "not an http(s) URL" + } + if u.User != nil { + return "", "credentials in URL" + } + host := u.Hostname() + var h string + if a, err := netip.ParseAddr(host); err == nil { + a = a.Unmap() + if r := reservedAddr(a); r != "" || a.Zone() != "" { + return "", "private or reserved address" + } + h = a.String() + if a.Is6() { + h = "[" + h + "]" + } + } else { + d, reason := normDomain(host) + if reason != "" { + return "", reason + } + h = d + } + if p := u.Port(); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n < 1 || n > 65535 { + return "", "bad port" + } + h += ":" + strconv.Itoa(n) + } + u.Host, u.Fragment, u.RawFragment = h, "", "" + if u.Path == "" { + u.Path = "/" + } + s := u.String() + if len(s) > MaxValueLen { + return "", "too long" + } + return s, "" +} + +// PublicAddr reports whether a is a public unicast address a program could +// own (the fetcher also refuses to connect anywhere else). +func PublicAddr(a netip.Addr) bool { return reservedAddr(a.Unmap()) == "" } diff --git a/internal/source/diodb.go b/internal/source/diodb.go new file mode 100644 index 0000000..e9720d7 --- /dev/null +++ b/internal/source/diodb.go @@ -0,0 +1,225 @@ +package source + +import ( + "context" + "encoding/json" + "fmt" + "net/mail" + "net/url" + "sort" + "strconv" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// DioDBURL is the disclose.io program list (CC0-1.0). +const DioDBURL = "https://raw.githubusercontent.com/disclose/diodb/master/program-list.json" + +const ( + maxDioDBBytes = 32 << 20 + maxDioDBRecords = 50000 +) + +// DioDB reads the disclose.io program list: a public directory of +// self-hosted and platform-hosted programs with their policy URL, contact, +// bounty and safe-harbour flags. It carries no structured scope; the only +// target it yields is the host of a program's own security.txt, as an +// inferred target. +type DioDB struct { + URL string // DioDBURL unless a test overrides it +} + +// Meta implements Source. +func (d DioDB) Meta() Meta { + return Meta{ + Name: "disclose", + URL: d.url(), + Terms: "https://github.com/disclose/diodb/blob/master/LICENSE", + Licence: "CC0-1.0", + Attribution: "Program directory data from the disclose.io diodb project (CC0-1.0).", + } +} + +func (d DioDB) url() string { + if d.URL != "" { + return d.URL + } + return DioDBURL +} + +type dioRecord struct { + ProgramName string `json:"program_name"` + PolicyURL string `json:"policy_url"` + PolicyURLStatus string `json:"policy_url_status"` + ContactEmail string `json:"contact_email"` + ContactURL string `json:"contact_url"` + OffersBounty json.RawMessage `json:"offers_bounty"` + SafeHarbor json.RawMessage `json:"safe_harbor"` + SecurityTxtURL json.RawMessage `json:"securitytxt_url"` + PreferredLanguage json.RawMessage `json:"preferred_languages"` + DisclosureDays json.RawMessage `json:"disclosure_timeline_days"` +} + +// Hosts of platforms that host programs on their own site: a policy URL on +// one of them names the platform, never the program's own domain. +var platformHosts = map[string]string{ + "hackerone.com": "hackerone", "bugcrowd.com": "bugcrowd", "intigriti.com": "intigriti", + "yeswehack.com": "yeswehack", "federacy.com": "federacy", "openbugbounty.org": "openbugbounty", + "hackenproof.com": "hackenproof", "immunefi.com": "immunefi", "huntr.com": "huntr", "huntr.dev": "huntr", + "whitehub.net": "whitehub", "bugbase.ai": "bugbase", "zerocopter.com": "zerocopter", "hackrate.com": "hackrate", +} + +// Collect implements Source. +func (d DioDB) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + resp, err := c.Get(ctx, d.url(), maxDioDBBytes) + if err != nil { + return nil, err + } + var raw []json.RawMessage + if err := json.Unmarshal(resp.Body, &raw); err != nil { + return nil, fmt.Errorf("diodb: %w", err) + } + if len(raw) > maxDioDBRecords { + return nil, fmt.Errorf("diodb: %d records, over the cap of %d", len(raw), maxDioDBRecords) + } + res := &Result{} + recs := make([]dioRecord, 0, len(raw)) + names := make([]string, 0, len(raw)) + for _, r := range raw { + var rec dioRecord + if json.Unmarshal(r, &rec) != nil || strings.TrimSpace(rec.ProgramName) == "" || !httpURL(rec.PolicyURL) { + res.Skipped++ + continue + } + recs = append(recs, rec) + names = append(names, rec.ProgramName) + } + slugs := UniqueSlugs(names) + meta := d.Meta() + for _, rec := range recs { + p := model.Program{ + ID: meta.Name + ":" + slugs[rec.ProgramName], + Source: meta.Name, + Platform: platformOf(rec.PolicyURL), + Name: rec.ProgramName, + URL: strings.TrimSpace(rec.PolicyURL), + Type: model.TypeVDP, + Status: model.StatusOpen, + InScope: []model.Target{}, + OutOfScope: []model.Target{}, + Rules: model.Rules{ + SafeHarbour: harbour(str(rec.SafeHarbor)), + TestingRestrictions: []string{}, + RequiredHeaders: []string{}, + Languages: splitList(str(rec.PreferredLanguage)), + }, + Terms: model.Terms{URL: strings.TrimSpace(rec.PolicyURL)}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: meta.URL, FetchedAt: resp.FetchedAt}, + } + switch strings.ToLower(str(rec.OffersBounty)) { + case "yes", "true", "partial": + p.Type = model.TypeBounty + } + if strings.EqualFold(rec.PolicyURLStatus, "dead") { + p.Status = model.StatusPaused + p.Rules.Summary = "The policy URL did not answer when the directory last checked it." + } + if n, err := strconv.Atoi(str(rec.DisclosureDays)); err == nil && n > 0 && n <= 3650 { + p.Rules.DisclosureDays = n + } + if a, err := mail.ParseAddress(strings.TrimSpace(rec.ContactEmail)); err == nil && len(a.Address) <= 254 { + p.Contact.Email = a.Address + } + if u := strings.TrimSpace(rec.ContactURL); httpURL(u) || strings.HasPrefix(u, "mailto:") { + p.Contact.URL = u + } + if host := securityTxtHost(str(rec.SecurityTxtURL)); host != "" { + p.InScope = append(p.InScope, model.Target{Type: model.TargetDomain, Value: host, Confidence: model.ConfidenceInferred, + Notes: "Host that serves the program's security.txt; not a published scope."}) + } + res.Programs = append(res.Programs, p) + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + return res, nil +} + +// str reads a JSON value that the directory stores as a string, a bool, a +// number or null. +func str(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return strings.TrimSpace(s) + } + var b bool + if json.Unmarshal(raw, &b) == nil { + return strconv.FormatBool(b) + } + var n json.Number + if json.Unmarshal(raw, &n) == nil { + return n.String() + } + return "" +} + +func httpURL(s string) bool { + u, err := url.Parse(strings.TrimSpace(s)) + return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" && u.User == nil +} + +func hostOf(s string) string { + u, err := url.Parse(strings.TrimSpace(s)) + if err != nil { + return "" + } + return strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.") +} + +func platformOf(policyURL string) string { + h := hostOf(policyURL) + for host, name := range platformHosts { + if h == host || strings.HasSuffix(h, "."+host) { + return name + } + } + return "self-hosted" +} + +// securityTxtHost returns the host of a first-party security.txt URL, or +// "" when the URL is not one (wrong path, or hosted on a platform). +func securityTxtHost(s string) string { + u, err := url.Parse(strings.TrimSpace(s)) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.User != nil || u.Port() != "" { + return "" + } + if u.Path != "/.well-known/security.txt" && u.Path != "/security.txt" { + return "" + } + if platformOf(s) != "self-hosted" { + return "" + } + return strings.ToLower(u.Hostname()) +} + +func harbour(s string) string { + switch strings.ToLower(s) { + case "full", "yes": + return model.SafeHarbourFull + case "partial": + return model.SafeHarbourPartial + case "none", "no": + return model.SafeHarbourNone + } + return model.SafeHarbourUnknown +} + +func splitList(s string) []string { + out := []string{} + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} diff --git a/internal/source/securitytxt.go b/internal/source/securitytxt.go new file mode 100644 index 0000000..299c793 --- /dev/null +++ b/internal/source/securitytxt.go @@ -0,0 +1,411 @@ +package source + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// DotGovFederalURL is CISA's list of US federal .gov domains (CC0-1.0). +const DotGovFederalURL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-federal.csv" + +const ( + maxSeedBytes = 8 << 20 + maxSeedDomains = 20000 + maxSecurityTxtBytes = 32 << 10 +) + +// Seed is one domain to look for a security.txt on, with the name of the +// organisation that holds it. +type Seed struct { + Domain string + Org string +} + +// SecurityTxt reads RFC 9116 security.txt files from a seed list of +// domains. A file with a Contact and a Policy field is a self-hosted +// disclosure program; the domain it was served from is an inferred target +// (security.txt does not publish a scope). +type SecurityTxt struct { + // SeedCSV is a CSV whose first column is a domain and third column the + // organisation (the dotgov-data layout); DotGovFederalURL by default. + SeedCSV string + // Extra are additional seeds (from the repository's seed file). + Extra []Seed + // Workers is the number of domains read at once (16 by default; each + // host still gets at most one request per MinInterval). + Workers int + // Progress, when set, is called after every 100 domains. + Progress func(done, total int) + // Scheme is "https" except in tests. + Scheme string + // Now is the time used to judge Expires (time.Now by default). + Now func() time.Time +} + +// Meta implements Source. +func (s SecurityTxt) Meta() Meta { + return Meta{ + Name: "securitytxt", + URL: s.seedURL(), + Terms: "https://www.rfc-editor.org/rfc/rfc9116", + Licence: "Seed list: CC0-1.0 (cisagov/dotgov-data). security.txt files: published by each domain for automated retrieval (RFC 9116).", + Attribution: "Domain seed list from CISA's dotgov-data (CC0-1.0); program data from each domain's own security.txt.", + } +} + +func (s SecurityTxt) seedURL() string { + if s.SeedCSV != "" { + return s.SeedCSV + } + return DotGovFederalURL +} + +// Collect implements Source. +func (s SecurityTxt) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + resp, err := c.Get(ctx, s.seedURL(), maxSeedBytes) + if err != nil { + return nil, fmt.Errorf("seed list: %w", err) + } + seeds, err := ParseSeedCSV(resp.Body) + if err != nil { + return nil, err + } + seeds = append(seeds, s.Extra...) + byDomain := map[string]Seed{} + for _, sd := range seeds { + d, reason := model.NormalizeTarget(model.TargetDomain, sd.Domain) + if reason != "" || len(d) > 120 { + continue + } + if _, dup := byDomain[d]; !dup { + byDomain[d] = Seed{Domain: d, Org: sd.Org} + } + } + if len(byDomain) > maxSeedDomains { + return nil, fmt.Errorf("seed list: %d domains, over the cap of %d", len(byDomain), maxSeedDomains) + } + domains := make([]string, 0, len(byDomain)) + for d := range byDomain { + domains = append(domains, d) + } + sort.Strings(domains) + + workers := s.Workers + if workers <= 0 { + workers = 16 + } + now := time.Now + if s.Now != nil { + now = s.Now + } + meta := s.Meta() + res := &Result{} + var mu sync.Mutex + done := 0 + jobs := make(chan string) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for d := range jobs { + p, known, err := s.one(ctx, c, meta, byDomain[d], now()) + mu.Lock() + done++ + if s.Progress != nil && done%100 == 0 { + s.Progress(done, len(domains)) + } + switch { + case p != nil: + res.Programs = append(res.Programs, *p) + case err != nil && !known: + res.Unknown = append(res.Unknown, meta.Name+":"+d) + default: + res.Skipped++ + } + mu.Unlock() + } + }() + } + for _, d := range domains { + select { + case jobs <- d: + case <-ctx.Done(): + } + } + close(jobs) + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + sort.Strings(res.Unknown) + return res, nil +} + +// one reads one domain. known reports whether the answer is definite (a +// file, or a definite "no file"); an indefinite failure keeps the +// previous record. +func (s SecurityTxt) one(ctx context.Context, c *fetch.Client, meta Meta, sd Seed, now time.Time) (*model.Program, bool, error) { + scheme := s.Scheme + if scheme == "" { + scheme = "https" + } + u := scheme + "://" + sd.Domain + "/.well-known/security.txt" + resp, err := c.Get(ctx, u, maxSecurityTxtBytes) + if err != nil { + var se *fetch.StatusError + definite := errors.Is(err, fetch.ErrNotFound) || errors.Is(err, fetch.ErrDisallowed) || + errors.Is(err, fetch.ErrTooLarge) || (errors.As(err, &se) && se.Code >= 400 && se.Code < 500) + return nil, definite, err + } + f, err := ParseSecurityTxt(resp.Body) + if err != nil { + return nil, true, err + } + if !ownedBy(f, resp.URL, sd.Domain) { + return nil, true, errors.New("security.txt is served for another host") + } + policy := firstHTTP(f.Policy) + if policy == "" { + return nil, true, errors.New("no Policy field") + } + sum := sha256.Sum256(resp.Body) + fetched := resp.FetchedAt + name := sd.Org + if name == "" { + name = sd.Domain + } else { + name += " (" + sd.Domain + ")" + } + p := &model.Program{ + ID: meta.Name + ":" + sd.Domain, + Source: meta.Name, + Platform: "self-hosted", + Name: name, + URL: policy, + Type: model.TypeVDP, + Status: model.StatusOpen, + InScope: []model.Target{{Type: model.TargetDomain, Value: sd.Domain, Confidence: model.ConfidenceInferred, + Notes: "Domain whose security.txt names this policy; not a published scope."}}, + OutOfScope: []model.Target{}, + Rules: model.Rules{SafeHarbour: model.SafeHarbourUnknown, TestingRestrictions: []string{}, + RequiredHeaders: []string{}, Languages: splitList(strings.Join(f.Languages, ","))}, + Terms: model.Terms{URL: u, SHA256: hex.EncodeToString(sum[:]), FetchedAt: &fetched}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: u, FetchedAt: resp.FetchedAt}, + } + if !f.Expires.IsZero() && f.Expires.Before(now) { + p.Status = model.StatusPaused + p.Rules.Summary = "The security.txt expired on " + f.Expires.UTC().Format("2006-01-02") + "." + } + for _, ct := range f.Contact { + switch { + case strings.HasPrefix(ct, "mailto:") && p.Contact.Email == "": + if addr := strings.TrimPrefix(ct, "mailto:"); validEmail(addr) { + p.Contact.Email = addr + } + case strings.HasPrefix(ct, "https://") && p.Contact.URL == "": + p.Contact.URL = ct + } + } + return p, true, nil +} + +// SecurityTxtFile holds the fields of a security.txt the collector uses. +type SecurityTxtFile struct { + Contact []string + Policy []string + Canonical []string + Languages []string + Expires time.Time + CVDPolicy []string // draft-behring-cvd-policy; recorded, not yet parsed +} + +// ParseSecurityTxt parses an RFC 9116 file (a cleartext-signed one too; the +// signature is not checked). It refuses HTML and files without a Contact. +func ParseSecurityTxt(body []byte) (*SecurityTxtFile, error) { + if !utf8.Valid(body) { + return nil, errors.New("security.txt: not UTF-8") + } + text := string(body) + if l := strings.ToLower(strings.TrimSpace(text)); strings.HasPrefix(l, "<") || strings.Contains(l, " 2048 { + continue + } + switch strings.ToLower(strings.TrimSpace(k)) { + case "contact": + f.Contact = appendCapped(f.Contact, v) + case "policy": + f.Policy = appendCapped(f.Policy, v) + case "canonical": + f.Canonical = appendCapped(f.Canonical, v) + case "preferred-languages": + f.Languages = appendCapped(f.Languages, v) + case "cvd-policy": + f.CVDPolicy = appendCapped(f.CVDPolicy, v) + case "expires": + if t, err := time.Parse(time.RFC3339, v); err == nil && f.Expires.IsZero() { + f.Expires = t + } + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("security.txt: %w", err) + } + if len(f.Contact) == 0 { + return nil, errors.New("security.txt: no Contact field") + } + return f, nil +} + +func appendCapped(l []string, v string) []string { + if len(l) >= 16 { + return l + } + return append(l, v) +} + +func stripClearsign(text string) string { + _, rest, ok := strings.Cut(text, "\n\n") + if !ok { + _, rest, _ = strings.Cut(text, "\r\n\r\n") + } + if i := strings.Index(rest, "-----BEGIN PGP SIGNATURE-----"); i >= 0 { + rest = rest[:i] + } + var b strings.Builder + for _, line := range strings.Split(rest, "\n") { + b.WriteString(strings.TrimPrefix(line, "- ")) + b.WriteByte('\n') + } + return b.String() +} + +// ownedBy: the file was served from the seed domain, or one of its +// Canonical URLs is on the seed domain. A file reached through a redirect +// to another organisation's host, with no Canonical naming the seed domain, +// is that other host's file. +func ownedBy(f *SecurityTxtFile, finalURL, domain string) bool { + if hostIs(finalURL, domain) { + return true + } + for _, c := range f.Canonical { + if hostIs(c, domain) { + return true + } + } + return false +} + +func hostIs(raw, domain string) bool { + u, err := url.Parse(raw) + return err == nil && strings.EqualFold(strings.TrimSuffix(u.Hostname(), "."), domain) +} + +func firstHTTP(l []string) string { + for _, v := range l { + if u, err := url.Parse(v); err == nil && u.Scheme == "https" && u.Host != "" && u.User == nil && len(v) <= model.MaxURLLen && !strings.ContainsAny(v, " \t") { + return v + } + } + return "" +} + +func validEmail(s string) bool { + if len(s) > 254 || strings.ContainsAny(s, " <>\"") { + return false + } + at := strings.LastIndexByte(s, '@') + return at > 0 && at < len(s)-3 && strings.Contains(s[at:], ".") +} + +// ParseSeedCSV reads a seed CSV: header row, domain in column 1, the +// organisation in column 3 (and a sub-organisation in column 4, if any). +func ParseSeedCSV(body []byte) ([]Seed, error) { + r := csv.NewReader(bytes.NewReader(body)) + r.FieldsPerRecord = -1 + r.ReuseRecord = true + var out []Seed + first := true + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("seed list: %w", err) + } + if first { + first = false + if len(rec) > 0 && strings.Contains(strings.ToLower(rec[0]), "domain") { + continue + } + } + if len(rec) == 0 || strings.TrimSpace(rec[0]) == "" { + continue + } + sd := Seed{Domain: strings.TrimSpace(rec[0])} + if len(rec) > 2 { + sd.Org = strings.TrimSpace(rec[2]) + } + if len(rec) > 3 && strings.TrimSpace(rec[3]) != "" { + sd.Org += " / " + strings.TrimSpace(rec[3]) + } + out = append(out, sd) + if len(out) > maxSeedDomains { + return nil, fmt.Errorf("seed list: over %d domains", maxSeedDomains) + } + } + return out, nil +} + +// ParseSeedFile reads the repository seed file: one domain per line, +// optionally followed by whitespace and an organisation name; # comments. +func ParseSeedFile(body []byte) []Seed { + var out []Seed + for _, line := range strings.Split(string(body), "\n") { + if i := strings.IndexByte(line, '#'); i >= 0 { + line = line[:i] + } + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + out = append(out, Seed{Domain: fields[0], Org: strings.Join(fields[1:], " ")}) + } + return out +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000..696a1bf --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,78 @@ +// Package source holds the adapters, one per upstream source. An adapter +// reads its source through the polite fetch client only, and turns what it +// read into programs; it never validates scope itself (the collector +// normalises every program the same way) and never sets timestamps other +// than provenance. +package source + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// Meta describes a source and its terms; it is carried in every manifest. +type Meta struct { + Name string // id prefix of its programs, [a-z0-9-] + URL string + Terms string + Licence string + Attribution string +} + +// Result is what one run of an adapter produced. +type Result struct { + Programs []model.Program + // Unknown lists ids whose state could not be read this run (a host + // that did not answer, for example). The collector keeps their + // previous record instead of closing them. + Unknown []string + // Skipped counts source records that were not programs (no policy, + // malformed record …), for the log. + Skipped int +} + +// Source is one adapter. +type Source interface { + Meta() Meta + // Collect reads the whole source. An error means the source as a + // whole could not be read; the collector then keeps every previous + // record of the source unchanged. + Collect(ctx context.Context, c *fetch.Client) (*Result, error) +} + +var slugRE = regexp.MustCompile(`[^a-z0-9]+`) + +// Slug turns a name into an id slug ([a-z0-9-], at most 100 characters). +func Slug(name string) string { + s := strings.Trim(slugRE.ReplaceAllString(strings.ToLower(name), "-"), "-") + if len(s) > 100 { + s = strings.TrimRight(s[:100], "-") + } + return s +} + +// UniqueSlugs gives every name a stable slug; names whose slugs collide +// (or are empty) all get a short hash of the name appended, so the result +// does not depend on the order of the source. +func UniqueSlugs(names []string) map[string]string { + count := map[string]int{} + for _, n := range names { + count[Slug(n)]++ + } + out := make(map[string]string, len(names)) + for _, n := range names { + s := Slug(n) + if count[s] > 1 || s == "" { + sum := sha256.Sum256([]byte(n)) + s = strings.TrimLeft(s+"-"+hex.EncodeToString(sum[:4]), "-") + } + out[n] = s + } + return out +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go new file mode 100644 index 0000000..6557a3c --- /dev/null +++ b/internal/source/source_test.go @@ -0,0 +1,200 @@ +package source + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// testClient sends every request, whatever its host, to srv. +func testClient(srv *httptest.Server) *fetch.Client { + return fetch.New(fetch.Options{UserAgent: fetch.UserAgent("test"), MinInterval: time.Millisecond, AllowPrivate: true, + DialOverride: srv.Listener.Addr().String(), Timeout: 5 * time.Second}) +} + +func byID(ps []model.Program) map[string]model.Program { + out := map[string]model.Program{} + for _, p := range ps { + out[p.ID] = p + } + return out +} + +func TestDioDB(t *testing.T) { + body, err := os.ReadFile("testdata/diodb.json") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/list.json" { + _, _ = w.Write(body) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + res, err := DioDB{URL: srv.URL + "/list.json"}.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + if len(res.Programs) != 12 || res.Skipped != 5 { + t.Fatalf("%d programs, %d skipped", len(res.Programs), res.Skipped) + } + ps := byID(res.Programs) + ada := ps["disclose:ada-health"] + if len(ada.InScope) != 1 || ada.InScope[0].Value != "ada.com" || ada.InScope[0].Confidence != model.ConfidenceInferred || ada.ScopePublished { + t.Errorf("ada: %+v", ada.InScope) + } + yt := ps["disclose:youtube"] + if yt.Type != model.TypeBounty || yt.Platform != "self-hosted" || len(yt.InScope) != 0 || yt.Rules.SafeHarbour != model.SafeHarbourPartial { + t.Errorf("youtube: %+v", yt) + } + ph := ps["disclose:platform-hosted"] + if ph.Platform != "hackerone" || ph.Type != model.TypeBounty || len(ph.InScope) != 0 { + t.Errorf("a security.txt on a platform host must not become a target: %+v", ph) + } + if len(ps["disclose:odd-path-txt"].InScope) != 0 { + t.Error("a non-standard security.txt path must not become a target") + } + dead := ps["disclose:dead-link"] + if dead.Status != model.StatusPaused || dead.Type != model.TypeBounty || dead.Contact != (model.Contact{}) || + dead.Rules.DisclosureDays != 90 || strings.Join(dead.Rules.Languages, ",") != "en,de" { + t.Errorf("dead link: %+v", dead) + } + dups := 0 + for id := range ps { + if strings.HasPrefix(id, "disclose:duplicate-") { + dups++ + } + } + if dups != 2 { + t.Errorf("colliding slugs must both get a hash suffix (%d)", dups) + } + // The public-suffix target is passed on; the collector's Normalize + // rejects it. + pst := ps["disclose:public-suffix-txt"] + model.Normalize(&pst) + if len(pst.InScope) != 0 || len(pst.Rejected) != 1 { + t.Errorf("public suffix: %+v / %+v", pst.InScope, pst.Rejected) + } +} + +const goodTxt = `# Our policy +Contact: mailto:security@agency.gov +Contact: https://agency.gov/report +Expires: 2099-01-01T00:00:00Z +Policy: https://agency.gov/vulnerability-disclosure-policy +Preferred-Languages: en, es +Canonical: https://agency.gov/.well-known/security.txt +` + +func TestParseSecurityTxt(t *testing.T) { + f, err := ParseSecurityTxt([]byte(goodTxt)) + if err != nil || len(f.Contact) != 2 || f.Policy[0] != "https://agency.gov/vulnerability-disclosure-policy" || f.Expires.Year() != 2099 { + t.Fatalf("%v %+v", err, f) + } + signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + strings.ReplaceAll(goodTxt, "# Our", "- # Our") + + "-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n" + if f, err := ParseSecurityTxt([]byte(signed)); err != nil || len(f.Policy) != 1 { + t.Errorf("signed: %v", err) + } + for name, body := range map[string]string{ + "html": "Contact: x", + "no contact": "Policy: https://a.gov/p\n", + "binary": "\xff\xfe\x00", + } { + if _, err := ParseSecurityTxt([]byte(body)); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestSecurityTxt(t *testing.T) { + seed := "Domain name,Domain type,Organization name,Suborganization name\n" + + "agency.gov,Federal,Agency,Office\nnopolicy.gov,Federal,NoPolicy,\nhtml.gov,Federal,Html,\nmissing.gov,Federal,Missing,\n" + + "down.gov,Federal,Down,\nredirect.gov,Federal,Redirect,\nexpired.gov,Federal,Expired,\nrobots.gov,Federal,Robots,\n" + + "localhost,Federal,Bad,\n10.0.0.1,Federal,Bad,\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host, _, _ := net.SplitHostPort(r.Host) + if host == "" { + host = r.Host + } + if r.URL.Path == "/robots.txt" { + if host == "robots.gov" { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /.well-known/\n")) + return + } + http.NotFound(w, r) + return + } + if r.URL.Path == "/seed.csv" { + _, _ = w.Write([]byte(seed)) + return + } + switch host { + case "agency.gov", "robots.gov": + _, _ = w.Write([]byte(goodTxt)) + case "nopolicy.gov": + _, _ = w.Write([]byte("Contact: mailto:a@nopolicy.gov\n")) + case "html.gov": + _, _ = w.Write([]byte("not found")) + case "down.gov": + w.WriteHeader(http.StatusBadGateway) + case "redirect.gov": + http.Redirect(w, r, "http://other.gov/.well-known/security.txt", http.StatusFound) + case "other.gov": + _, _ = w.Write([]byte("Contact: mailto:a@other.gov\nPolicy: https://other.gov/p\n")) + case "expired.gov": + _, _ = w.Write([]byte("Contact: mailto:a@expired.gov\nExpires: 2020-01-01T00:00:00Z\nPolicy: https://expired.gov/p\n")) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + st := SecurityTxt{SeedCSV: "http://seed.example.net/seed.csv", Scheme: "http", Workers: 3} + res, err := st.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + ps := byID(res.Programs) + if len(ps) != 2 { + t.Fatalf("programs: %v", keys(ps)) + } + a := ps["securitytxt:agency.gov"] + if a.Name != "Agency / Office (agency.gov)" || a.URL != "https://agency.gov/vulnerability-disclosure-policy" || + a.Contact.Email != "security@agency.gov" || a.Contact.URL != "https://agency.gov/report" || a.Terms.SHA256 == "" || + len(a.InScope) != 1 || a.InScope[0].Value != "agency.gov" || a.InScope[0].Confidence != model.ConfidenceInferred || a.Status != model.StatusOpen { + t.Errorf("agency: %+v", a) + } + if ps["securitytxt:expired.gov"].Status != model.StatusPaused { + t.Error("an expired security.txt must give a paused program") + } + // A host that did not answer is unknown (its previous record is kept); + // a definite "no file" is not. + if strings.Join(res.Unknown, ",") != "securitytxt:down.gov" { + t.Errorf("unknown = %v", res.Unknown) + } +} + +func keys(m map[string]model.Program) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} + +func TestSlugs(t *testing.T) { + s := UniqueSlugs([]string{"Acme Corp", "acme-corp", "Zeta", "!!!"}) + if s["Zeta"] != "zeta" || s["Acme Corp"] == s["acme-corp"] || !strings.HasPrefix(s["Acme Corp"], "acme-corp-") || s["!!!"] == "" { + t.Errorf("%v", s) + } +} diff --git a/internal/source/testdata/diodb.json b/internal/source/testdata/diodb.json new file mode 100644 index 0000000..68d716c --- /dev/null +++ b/internal/source/testdata/diodb.json @@ -0,0 +1,138 @@ +[ + { + "program_name": ".nz Registry", + "policy_url": "https://registry.internetnz.nz/about/vulnerability-disclosure-policy/", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": false, + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive", + "contact_email": "security@internetnz.net.nz" + }, + { + "program_name": "Ashby", + "policy_url": "https://www.ashbyhq.com/resources/vulnerability-disclosure", + "contact_email": "security@ashbyhq.com", + "offers_bounty": "no", + "offers_swag": false, + "pgp_key": "", + "hall_of_fame": "", + "hiring": "", + "launch_date": "", + "policy_url_status": "alive", + "preferred_languages": "", + "public_disclosure": "", + "safe_harbor": "", + "securitytxt_url": "" + }, + { + "program_name": "Firebase", + "policy_url": "https://g.co/vrp", + "contact_url": "https://g.co/vulnz", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": true, + "hall_of_fame": "", + "safe_harbor": "none", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive" + }, + { + "program_name": "YouTube", + "policy_url": "https://g.co/vrp", + "contact_url": "https://g.co/vulnz", + "launch_date": "", + "offers_bounty": "yes", + "offers_swag": "", + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive" + }, + { + "program_name": "Ada Health", + "policy_url": "https://ada.com/security-vulnerability-disclosure-policy/", + "contact_url": "https://ada.com/security-vulnerability-disclosure-policy/", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": false, + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "https://ada.com/pgp-key.txt", + "hiring": "https:/ada.com/careers/", + "securitytxt_url": "https://ada.com/.well-known/security.txt", + "preferred_languages": "en, de", + "policy_url_status": "alive", + "contact_email": "security@ada.com" + }, + { + "program_name": "", + "policy_url": "https://nameless.org/security" + }, + { + "program_name": "Script Policy", + "policy_url": "javascript:alert(1)" + }, + { + "program_name": "No Policy" + }, + { + "program_name": "Platform Hosted", + "policy_url": "https://hackerone.com/platform-hosted", + "securitytxt_url": "https://hackerone.com/.well-known/security.txt", + "offers_bounty": true, + "safe_harbor": "full" + }, + { + "program_name": "Public Suffix Txt", + "policy_url": "https://blog.github.io/policy", + "securitytxt_url": "https://github.io/.well-known/security.txt" + }, + { + "program_name": "Odd Path Txt", + "policy_url": "https://oddpath.org/policy", + "securitytxt_url": "https://oddpath.org/files/security.txt" + }, + { + "program_name": "Dead Link", + "policy_url": "https://dead.org/policy", + "policy_url_status": "dead", + "offers_bounty": "partial", + "contact_email": "not an email", + "contact_url": "javascript:x", + "disclosure_timeline_days": "90", + "preferred_languages": "en, de ,," + }, + { + "program_name": "Duplicate", + "policy_url": "https://dup-one.org/policy" + }, + { + "program_name": "duplicate!", + "policy_url": "https://dup-two.org/policy" + }, + { + "program_name": "Bell\u0007Name", + "policy_url": "https://bell.org/policy" + }, + "not an object", + { + "program_name": 42, + "policy_url": "https://number.org/policy" + } +] diff --git a/keys/README.md b/keys/README.md new file mode 100644 index 0000000..6be20a1 --- /dev/null +++ b/keys/README.md @@ -0,0 +1,49 @@ +# Keys + +This directory holds only public material: + +| File | What | +|---|---| +| `root-keyid.txt` | key id of the offline root (`SHA256:…`); platforms pin it | +| `keyset.dsse.json` | the current key set, signed by the root | + +## First setup (maintainer, on an offline machine) + +```sh +go build -o programfeed ./cmd/programfeed + +# 1. The offline root. Keep root.key offline (hardware-backed storage or an +# encrypted offline medium, plus a backup). It never goes to CI. +./programfeed keys root-init --out root.key # prints the root key id + +# 2. The online signing key. +./programfeed keys gen --out signing.key # prints its public key + +# 3. The key set: the root allows the online key for up to 180 days. +./programfeed keys sign-keyset --root root.key --key --version 1 --days 180 --out keys/keyset.dsse.json +``` + +Then: + +- commit `keys/root-keyid.txt` (the root key id) and `keys/keyset.dsse.json`; +- create the GitHub environment **`publish`**: deployment branches limited + to the default branch; add the secret **`PROGRAMFEED_SIGNING_KEY`** with the + content of `signing.key`; delete `signing.key` from the machine; +- give OpenCTEM platforms the root key id (`PROGRAMFEED_ROOT_KEY_ID`). + +Use a root distinct from the vulnerability feed's root: one compromised root +then cannot sign for both feeds. + +Until `keys/root-keyid.txt` and `keys/keyset.dsse.json` are committed, the +scheduled publish workflow stops at its first step with an error saying so. + +## Rotation and revocation + +Before the key set expires, or at once if the online key may have leaked: +generate a new online key, sign a key set with a **higher version** listing +only the keys still trusted, commit it and replace the environment secret. +Platforms that have seen the new version refuse the older key set from then +on. + +If the root itself is compromised, generate a new root and have every +platform pin its new key id. diff --git a/schema/change.schema.json b/schema/change.schema.json new file mode 100644 index 0000000..3a11ebe --- /dev/null +++ b/schema/change.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openctemio/programfeed/schema/change.schema.json", + "title": "Change", + "description": "One entry of a run's change log: one line of changes.jsonl.gz. Entries are sorted by program, then kind, then scope and target.", + "type": "object", + "additionalProperties": false, + "required": ["sequence", "program", "kind"], + "properties": { + "sequence": {"type": "integer", "minimum": 1, "description": "The bundle sequence that made the change."}, + "program": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$"}, + "kind": {"enum": ["program_added", "program_reopened", "status_changed", "program_modified", "rules_changed", "terms_changed", + "target_added", "target_modified", "target_removed", "program_closed", "program_dropped"], + "description": "program_dropped: the program is no longer in the feed (closed for more than 180 days); every other kind names a program present in the same bundle."}, + "scope": {"enum": ["in", "out"], "description": "Target changes only."}, + "target": {"$ref": "program.schema.json#/$defs/target", "description": "Target changes only: the target after the change (before, for target_removed)."}, + "fields": {"type": "array", "maxItems": 16, "items": {"type": "string"}, "description": "For program_modified, rules_changed, status_changed and target_modified: the JSON fields that changed."} + } +} diff --git a/schema/program.schema.json b/schema/program.schema.json new file mode 100644 index 0000000..b45c898 --- /dev/null +++ b/schema/program.schema.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openctemio/programfeed/schema/program.schema.json", + "title": "Program", + "description": "One public bug-bounty or vulnerability-disclosure program: one line of programs.jsonl.gz.", + "type": "object", + "additionalProperties": false, + "required": ["id", "source", "platform", "name", "url", "type", "status", "offers_bounty", "scope_published", + "in_scope", "out_of_scope", "rejected", "rules", "terms", "contact", "provenance", "first_seen", "last_seen", "last_changed"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$", + "description": ":. Stable; owned by exactly one source adapter."}, + "source": {"type": "string", "description": "The adapter that produced the record (the id prefix)."}, + "platform": {"type": "string", "maxLength": 64, "description": "Where the program is hosted: \"self-hosted\" or the name of the hosting platform."}, + "name": {"type": "string", "minLength": 1, "maxLength": 200}, + "url": {"type": "string", "format": "uri", "maxLength": 2048, "description": "The program's policy page (http or https)."}, + "type": {"enum": ["bounty", "vdp"]}, + "status": {"enum": ["open", "paused", "closed"], + "description": "closed: the source no longer lists the program (kept 180 days, then dropped). paused: listed but not currently accepting (dead policy link, expired security.txt)."}, + "offers_bounty": {"type": "boolean", "description": "Always equal to type == bounty."}, + "scope_published": {"type": "boolean", "description": "True when the source carries the program's own structured scope. When false, in_scope holds at most inferred suggestions."}, + "in_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}, "description": "Sorted by (type, value), no duplicates. Out of scope wins: a value listed in both appears only in out_of_scope."}, + "out_of_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}}, + "rejected": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/rejected"}, "description": "Targets the program listed that the feed's scope rules refused, with the reason."}, + "rules": {"$ref": "#/$defs/rules"}, + "terms": {"$ref": "#/$defs/terms"}, + "contact": {"$ref": "#/$defs/contact"}, + "provenance": {"$ref": "#/$defs/provenance"}, + "first_seen": {"type": "string", "format": "date-time"}, + "last_seen": {"type": "string", "format": "date-time", "description": "Last run in which the source listed the program."}, + "last_changed": {"type": "string", "format": "date-time", "description": "Last run in which any content (not timestamps) changed."}, + "closed_at": {"type": "string", "format": "date-time", "description": "Present exactly when status is closed."} + }, + "$defs": { + "target": { + "type": "object", + "additionalProperties": false, + "required": ["type", "value", "confidence"], + "properties": { + "type": {"enum": ["domain", "wildcard", "ip", "cidr", "url", "mobile_app", "source_repo", "other"]}, + "value": {"type": "string", "maxLength": 512, + "description": "Canonical form: domain = lower-case A-label, not a public suffix, not a reserved name; wildcard = \"*.\" + such a domain; ip = public unicast address; cidr = masked public prefix of at least /16 (IPv4) or /48 (IPv6); url = http(s) without credentials or fragment, host checked as above."}, + "confidence": {"enum": ["published", "inferred"], + "description": "published: the program's own scope. inferred: derived by the collector (for example the host serving the program's security.txt); a suggestion, never an authorisation to test."}, + "eligible_for_bounty": {"type": "boolean"}, + "max_severity": {"enum": ["none", "low", "medium", "high", "critical"]}, + "notes": {"type": "string", "maxLength": 500} + } + }, + "rejected": { + "type": "object", + "additionalProperties": false, + "required": ["scope", "type", "value", "reason"], + "properties": { + "scope": {"enum": ["in", "out"]}, + "type": {"type": "string", "maxLength": 32}, + "value": {"type": "string", "maxLength": 512}, + "reason": {"type": "string", "minLength": 1, "maxLength": 200} + } + }, + "rules": { + "type": "object", + "additionalProperties": false, + "required": ["testing_restrictions", "required_headers", "safe_harbour", "languages"], + "properties": { + "summary": {"type": "string", "maxLength": 4000}, + "testing_restrictions": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}}, + "rate_limit": {"type": "string", "maxLength": 200}, + "required_headers": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}}, + "safe_harbour": {"enum": ["full", "partial", "none", "unknown"]}, + "disclosure_days": {"type": "integer", "minimum": 0, "maximum": 3650}, + "languages": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}} + } + }, + "terms": { + "type": "object", + "additionalProperties": false, + "required": ["url"], + "properties": { + "url": {"type": "string", "format": "uri", "maxLength": 2048}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "SHA-256 of the exact bytes of the terms document the collector read at fetched_at. Absent when the collector did not read the terms."}, + "fetched_at": {"type": "string", "format": "date-time", "description": "Present exactly when sha256 is."} + } + }, + "contact": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": {"type": "string", "maxLength": 254}, + "url": {"type": "string", "maxLength": 2048, "description": "http(s) or mailto."} + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["source", "source_url", "fetched_at"], + "properties": { + "source": {"type": "string"}, + "source_url": {"type": "string", "format": "uri", "maxLength": 2048}, + "fetched_at": {"type": "string", "format": "date-time"} + } + } + } +} diff --git a/seeds/securitytxt.txt b/seeds/securitytxt.txt new file mode 100644 index 0000000..d5c0417 --- /dev/null +++ b/seeds/securitytxt.txt @@ -0,0 +1,3 @@ +# Extra domains whose security.txt the securitytxt adapter reads, one per line, +# optionally followed by the organisation name. Only add domains whose owner +# publishes a vulnerability disclosure policy for public use.