diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..14f295a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Format + run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } + - name: Vet + run: go vet ./... + - name: Test (recorded fixtures, no network) + run: go test -race ./... + - name: Fuzz the scope normaliser briefly + run: go test ./internal/model -run '^$' -fuzz '^FuzzNormalizeTarget$' -fuzztime 20s + - name: Known vulnerabilities + run: go run golang.org/x/vuln/cmd/govulncheck@latest ./... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..4d9c56f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,135 @@ +name: Publish bundle + +# Daily: build the next program bundle from the previous release and the +# sources, sign it, verify it and publish it as a GitHub Release. +# +# Two jobs on purpose. "build" talks to the sources and never sees the +# signing key. "publish" runs in the protected "publish" environment (only +# the default branch can deploy to it), holds the signing key and nothing +# else, and only signs, verifies and uploads what "build" produced. + +on: + schedule: + - cron: "41 3 * * *" + workflow_dispatch: + inputs: + allow_mass_change: + description: "Accept a source that lost more than half of its programs (only after checking the source)" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: publish + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + ready: ${{ steps.keys.outputs.ready }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Key ceremony done? + id: keys + run: | + if [ -s keys/root-keyid.txt ] && [ -s keys/keyset.dsse.json ]; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::No keys yet (keys/root-keyid.txt, keys/keyset.dsse.json): nothing is published until the key ceremony in keys/README.md is done" + echo "ready=false" >> "$GITHUB_OUTPUT" + fi + - if: steps.keys.outputs.ready == 'true' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - if: steps.keys.outputs.ready == 'true' + name: Test + run: go test ./... + - if: steps.keys.outputs.ready == 'true' + name: Build the CLI + run: go build -trimpath -ldflags "-X main.version=${GITHUB_SHA::12}" -o programfeed ./cmd/programfeed + - if: steps.keys.outputs.ready == 'true' + name: Restore the HTTP cache (conditional requests only; every body is re-validated) + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: httpcache + key: httpcache-${{ github.run_id }} + restore-keys: httpcache- + - if: steps.keys.outputs.ready == 'true' + name: Download the previous release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p prev + if gh release view --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + gh release download --repo "$GITHUB_REPOSITORY" --dir prev --pattern 'snapshot*' --pattern 'latest.dsse.json' --pattern 'keyset.dsse.json' + echo "PREV=--prev=prev" >> "$GITHUB_ENV" + fi + - if: steps.keys.outputs.ready == 'true' + name: Build the bundle + env: + MASS: ${{ inputs.allow_mass_change && '--allow-mass-change' || '' }} + run: ./programfeed build --out out --cache httpcache --root-key-id "$(cat keys/root-keyid.txt)" $PREV $MASS + - if: steps.keys.outputs.ready == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bundle + path: out/ + retention-days: 3 + if-no-files-found: error + + publish: + needs: build + if: needs.build.outputs.ready == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: publish + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - name: Build the CLI + run: go build -trimpath -o programfeed ./cmd/programfeed + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bundle + path: out + - name: Sign + env: + PROGRAMFEED_SIGNING_KEY: ${{ secrets.PROGRAMFEED_SIGNING_KEY }} + run: ./programfeed sign --dir out --keyset keys/keyset.dsse.json --root-key-id "$(cat keys/root-keyid.txt)" + - name: Verify as the platform does + run: ./programfeed verify --dir out --root-key-id "$(cat keys/root-keyid.txt)" + - name: Publish the release + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(python3 -c 'import json,base64,sys; e=json.load(open("out/latest.dsse.json")); print(json.loads(base64.b64decode(e["payload"]))["tag"])')" + cat > notes.md <<'NOTES' + Public program bundle for OpenCTEM: a signed snapshot of public bug-bounty and vulnerability-disclosure programs, and the delta from the previous release. + + Targets marked "inferred" are suggestions, not a program's published scope. See NOTICE for the terms of every source. + NOTES + gh release create "$tag" --repo "$GITHUB_REPOSITORY" --title "$tag" --notes-file notes.md out/* + - name: Keep the last 60 releases + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 --json tagName,createdAt \ + --jq 'sort_by(.createdAt) | reverse | .[60:] | .[].tagName' | + while read -r tag; do + gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58de1a3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/programfeed +/out/ +/prev/ +/httpcache/ +*.key diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..eea8991 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing + +Thanks for helping. Two kinds of contributions are common: adding seed +domains for the security.txt adapter, and adding a new source adapter. + +## Adding seed domains + +Add lines to `seeds/securitytxt.txt` (`domain [organisation name]`). Only add +domains whose owner publishes a vulnerability disclosure policy for public +use. The domain must already serve `/.well-known/security.txt` with a +`Contact` and a `Policy` field, or the collector skips it. + +## Adding a source adapter + +### 1. Check that the source may be collected + +Before writing code, open an issue with: + +- the official API or machine-readable listing (documentation link); +- whether it needs authentication (the feed never logs in, uses cookies or + tokens, runs JavaScript or bypasses bot protection: a source that needs + any of that is not accepted); +- its terms of service and robots.txt, quoted: they must allow automated + collection **and redistribution**, since the feed is published openly + (or the publisher must have given written permission, linked in the + issue); +- the licence of the data and the attribution it requires; +- its rate limits. + +### 2. Write the adapter + +An adapter is one file in `internal/source/` implementing `source.Source`: + +```go +type Source interface { + Meta() Meta // name, URL, terms, licence, attribution + Collect(ctx context.Context, c *fetch.Client) (*Result, error) +} +``` + +Rules: + +- Use only the `*fetch.Client` it is given (it handles the User-Agent, + robots.txt, rate limits, conditional requests, size caps and the + public-address check). Pass a size cap that fits the source. +- Program ids are `:`; use `source.UniqueSlugs` for names. + Ids must be stable across runs. The collector refuses programs outside the + adapter's own prefix. +- Fill `Provenance` (source, source URL, fetched-at time). Leave + `first_seen`, `last_seen`, `last_changed` and `closed_at` to the collector. +- Put the program's own scope in `InScope`/`OutOfScope` with + `Confidence: published` and set `ScopePublished`. Anything the adapter + derives itself is `Confidence: inferred`, with a note saying how. +- Do not filter scope yourself: the collector normalises every target and + records refused ones with the reason. +- Return an error only when the source as a whole could not be read (the + collector then keeps every previous record). Put ids whose state is + unknown this run (a host that did not answer) in `Result.Unknown`. +- Hostile input is expected: decode defensively, cap counts, never trust a + field's type. + +Register it in `cmd/programfeed/main.go` (`sources`), and add its row to the +sources table in `README.md` and its terms to `NOTICE`. + +### 3. Test it with recorded fixtures + +Tests never touch the network. Record a small, representative response +under `internal/source/testdata/` (trimmed, no personal data beyond what the +source publishes), serve it with `httptest`, and point the fetch client at +it (`fetch.Options{AllowPrivate: true, DialOverride: ...}`, see +`source_test.go`). Include poisoned entries: malformed records, wrong field +types, control characters, public-suffix or private-range scope. + +### 4. Check before opening the pull request + +```sh +gofmt -l . +go vet ./... +go test -race ./... +go run golang.org/x/vuln/cmd/govulncheck@latest ./... +``` + +Pull requests target `main`. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..f18b868 --- /dev/null +++ b/NOTICE @@ -0,0 +1,33 @@ +programfeed +Copyright 2026 The OpenCTEM Authors + +This software is licensed under the Apache License, Version 2.0 (LICENSE). + +The bundles this software publishes contain data from third-party sources, +under their own terms: + +disclose.io diodb (https://github.com/disclose/diodb) + Program directory data, dedicated to the public domain under CC0 1.0 + Universal: https://github.com/disclose/diodb/blob/master/LICENSE + +CISA dotgov-data (https://github.com/cisagov/dotgov-data) + The list of US federal .gov domains used as the security.txt seed list, + dedicated to the public domain under CC0 1.0 Universal. + +security.txt files (RFC 9116) + Each program record from the securitytxt adapter is derived from the + security.txt file its domain publishes for automated retrieval; the + record's provenance names the file. The policies those files link to + belong to their publishers and are not copied into the bundles. + +arkadiyt/bounty-targets-data (https://github.com/arkadiyt/bounty-targets-data) +projectdiscovery/public-bugbounty-programs (https://github.com/projectdiscovery/public-bugbounty-programs) + Public aggregate datasets of bug-bounty program listings, both MIT licensed + as repositories. The listings come from the platforms they name and remain + subject to those platforms' terms: records of a platform are published in + the signed feed only when the publish policy (policy/platforms.json) allows + it, with the attribution text recorded there. Removal requests: TAKEDOWN.md. + +Program names, policies and terms remain the property of the organisations +that publish them. Listing a program in the feed does not imply that the +organisation endorses OpenCTEM, nor that any testing is authorised. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3afc529 --- /dev/null +++ b/README.md @@ -0,0 +1,210 @@ +# programfeed + +The public program collector of [OpenCTEM](https://github.com/openctemio/openctem). +Once a day it reads open, machine-readable sources of **public bug-bounty and +vulnerability-disclosure programs**, normalises them into one record format, +validates every record and every scope entry, signs the result and publishes +it as a GitHub Release. OpenCTEM platforms never call the sources +themselves: they download (or, air-gapped, upload) a bundle, verify it and +import it into their public program monitor. + +The feed says **who runs a program, where its policy is, whether it pays, +what scope it publishes and how that changed**. It is not an authorisation to +test anything: a subscriber reads and accepts a program's terms before any +active testing, and targets marked `inferred` are suggestions only. + +## Sources + +Every source has a **status**: *ingest* (the collector reads it) and +*publish* (its records go into the signed public feed). Publishing is +decided per original platform by the [publish policy](#publish-policy), not +per source. + +| Source | Adapter | What it gives | Licence / terms | Ingest | Publish | +|---|---|---|---|---|---| +| [disclose.io diodb](https://github.com/disclose/diodb) `program-list.json` | `diodb` | ~2,400 programs: name, policy URL, contact, bounty yes/no, safe harbour, disclosure timeline, languages. No structured scope; the host of a program's own security.txt becomes an `inferred` target. | CC0-1.0 | yes | self-hosted programs only; programs hosted on a platform follow that platform's policy | +| [security.txt](https://www.rfc-editor.org/rfc/rfc9116) (RFC 9116) over a seed list | `securitytxt` | A domain whose `/.well-known/security.txt` has a `Contact` and a `Policy` is a self-hosted disclosure program; the domain is an `inferred` target, the file's SHA-256 is the terms hash. Expired files give a `paused` program. | files published for automated retrieval | yes | yes (self-hosted) | +| Seed: [CISA dotgov-data](https://github.com/cisagov/dotgov-data) `current-federal.csv` | `securitytxt` seed | US federal .gov domains and their organisations (vulnerability disclosure policies under BOD 20-01). | CC0-1.0 | yes | yes (self-hosted) | +| Seed: `seeds/securitytxt.txt` | `securitytxt` seed | Extra domains added by pull request. | n/a | yes | yes (self-hosted) | +| [bounty-targets-data](https://github.com/arkadiyt/bounty-targets-data) `data/*_data.json` | `bounty-targets` | Public program listings of HackerOne, Bugcrowd, Intigriti and YesWeHack with their published scope (targets marked `published_by_platform`). Read at one resolved commit; each record names the dataset, the commit, the original platform and the original program page. | repository MIT; the records stay subject to each platform's terms | with `--local-only` | **no**: every platform is `publish=false` until it gives written permission | +| [public-bugbounty-programs](https://github.com/projectdiscovery/public-bugbounty-programs) `dist/data.json` | `pd-bounty` | A community-curated list of program pages and the domains each covers (domains are `inferred` targets). | MIT (list) | yes | self-hosted programs only; with `--local-only` also platform-hosted ones | + +Platforms and the reason each is not published (the machine-readable +version is [`policy/platforms.json`](policy/platforms.json)): + +| Platform | Publish | Reason | +|---|---|---| +| self-hosted | yes | the policy page is the program owner's own | +| HackerOne | no | general terms allow use only as permitted; no redistribution grant. Permission letter drafted. | +| Bugcrowd | no | website terms forbid redistributing any portion of the website. Permission letter drafted. | +| Intigriti | no | no redistribution grant; the official API needs an account token. Permission letter drafted. | +| YesWeHack | no | legal notices forbid redistributing the platform's data. Permission letter drafted. | +| Immunefi | no | terms forbid automated access without written consent. Permission letter drafted. | +| HackenProof | no | no official machine-readable listing. Permission letter drafted. | +| Federacy, Open Bug Bounty, huntr, WhiteHub, BugBase, Zerocopter, Hackrate | no | no redistribution grant found | +| any platform not in the policy | no | unlisted means not published | + +The drafts to the platforms are in [`docs/outreach/`](docs/outreach/); nothing +has been sent. A platform is switched to `publish: true` only after written +permission, with the attribution text it asks for. + +Not collected at all (pull requests that scrape these will be declined): + +| Source | Reason | +|---|---| +| Open Bug Bounty | listing is behind bot protection; the feed does not bypass it | +| Platform websites and undocumented endpoints | the collector reads official files and public datasets, never a platform's site | +| Account-token APIs | the feed never logs in | + +A source moves to "publish" when it offers an official machine-readable +listing whose terms allow collection and redistribution (or gives written +permission). + +### Publish policy + +`policy/platforms.json` lists every original platform with: + +```json +"hackerone": {"publish": false, "attribution": "...", "takedown_contact": "security@openctem.io", "reason": "..."} +``` + +- `publish`: only `true` puts the platform's records in the signed feed. A + platform that is not listed is not published. `publish: true` requires an + `attribution` text; it is added to the source entry of the signed manifest + together with the takedown contact. +- A signed build does not even fetch dataset platforms that are not + published, applies the policy to the finished snapshot (a record is + published only if both its platform and its original platform are), and + applies it to the previous release too: switching a platform to + `publish: false` removes its programs on the next run with a + `program_dropped` change. Tests assert that restricted platforms never + reach the signed output. +- Removal requests: see [TAKEDOWN.md](TAKEDOWN.md). + +### Local-only bundle (self-hosters) + +```sh +./programfeed build --local-only --out local-bundle --cache httpcache +``` + +`--local-only` ignores the publish policy and reads every source, including the +platform datasets, and writes an **unsigned** bundle marked +`collector.local_only` in its manifest. `programfeed sign` refuses such a +bundle, so the project's signing key never signs unfiltered data. It is meant +for an organisation that points its **own** OpenCTEM platform at the bundle +for its own use: you are responsible for the terms of the platforms whose data +it contains, and the bundle must not be redistributed. A local bundle is +rebuilt from scratch every time (`--prev` is refused). + +### How the collector behaves on the network + +- User-Agent `openctem-programfeed/ (+https://github.com/openctemio/programfeed; security@openctem.io)` + and a `From: security@openctem.io` header; +- `robots.txt` honoured for every origin (product token `openctem-programfeed`; + an unreachable robots.txt means "do not fetch", RFC 9309); +- at most one request per second per host; conditional requests + (`If-None-Match`, `If-Modified-Since`) from a cache kept between runs; +- response size caps (32 KiB for a security.txt, 32 MiB for a list); +- connections only to public unicast addresses (a seed list or redirect + cannot point it at internal services); https only, at most 3 redirects; +- no login, no cookies, no JavaScript, no bot-protection bypass. + +## What a release contains + +| Asset | Content | +|---|---| +| `latest.dsse.json` | signed pointer: sequence, tag, snapshot and delta manifests, expiry | +| `snapshot.manifest.dsse.json` | signed manifest of the full snapshot (files, SHA-256, sizes, record counts, sources with licence and as-of time, stats) | +| `snapshot-programs.jsonl.gz` | every program | +| `snapshot-changes.jsonl.gz` | the change log of this run | +| `delta.manifest.dsse.json`, `delta-programs.jsonl.gz`, `delta-changes.jsonl.gz` | the programs that changed since the previous sequence (complete new records) and the same change log | +| `keyset.dsse.json` | the key set the manifests are signed under | + +A platform reads +`https://github.com/openctemio/programfeed/releases/latest/download/latest.dsse.json` +and the files of the release it names. + +## Record format + +One JSON object per line. The JSON Schemas are in [`schema/`](schema/): +[`program.schema.json`](schema/program.schema.json) and +[`change.schema.json`](schema/change.schema.json). + +```json +{"id":"securitytxt:agency.gov","source":"securitytxt","platform":"self-hosted","name":"Agency (agency.gov)", + "url":"https://agency.gov/vulnerability-disclosure-policy","type":"vdp","status":"open","offers_bounty":false, + "scope_published":false, + "in_scope":[{"type":"domain","value":"agency.gov","confidence":"inferred","notes":"Domain whose security.txt names this policy; not a published scope."}], + "out_of_scope":[],"rejected":[], + "rules":{"testing_restrictions":[],"required_headers":[],"safe_harbour":"unknown","languages":["en"]}, + "terms":{"url":"https://agency.gov/.well-known/security.txt","sha256":"…","fetched_at":"2026-10-10T03:41:00Z"}, + "contact":{"email":"security@agency.gov"}, + "provenance":{"source":"securitytxt","source_url":"https://agency.gov/.well-known/security.txt","fetched_at":"2026-10-10T03:41:00Z"}, + "first_seen":"2026-10-10T03:41:00Z","last_seen":"2026-10-10T03:41:00Z","last_changed":"2026-10-10T03:41:00Z"} +{"sequence":2,"program":"securitytxt:agency.gov","kind":"target_added","scope":"in","target":{"type":"domain","value":"agency.gov","confidence":"inferred"}} +``` + +Key rules: + +- `id` is `:`, stable, owned by one adapter. +- `type` is `bounty` or `vdp`; `status` is `open`, `paused` (listed but not + accepting: dead policy link, expired security.txt) or `closed` (the source + no longer lists it). A closed program stays 180 days, then a + `program_dropped` change removes it. +- Scope entries are typed (`domain`, `wildcard`, `ip`, `cidr`, `url`, + `mobile_app`, `source_repo`, `other`), canonical, sorted and deduplicated; + out of scope wins over in scope. +- `confidence`: `published` (the program's own scope), `published_by_platform` + (scope the hosting platform published, read through a public dataset; the + record's `provenance` names the dataset, commit, original platform and + original program page) or `inferred` (derived by the collector). `scope_published` says whether the source carries the + program's own structured scope at all. +- `terms.sha256` is the SHA-256 of the exact terms document the collector + read, when it read one. +- `rejected` lists what a program listed that the scope rules refused, with + the reason. + +### Scope rules + +A scope entry is refused (moved to `rejected`) when it is: a public suffix or +a wildcard over one (`com`, `*.co.uk`, `*.github.io`); a reserved or +non-public name (`localhost`, `.local`, `.internal`, `.test`, `example.com` +…); a private, loopback, link-local, CGNAT, documentation, multicast or +otherwise reserved address or range; a CIDR wider than /16 (IPv4) or /48 +(IPv6); a URL that is not http(s), has credentials, or whose host fails the +rules above; malformed, over 512 bytes, or carrying control or bidi +characters. Each list is capped at 2,000 entries. + +## Verifying a bundle yourself + +```sh +go build -o programfeed ./cmd/programfeed +gh release download --repo openctemio/programfeed --dir bundle +./programfeed verify --dir bundle --root-key-id "$(cat keys/root-keyid.txt)" +``` + +The envelopes, key set, sequence, expiry and caps follow the same +conventions as the OpenCTEM vulnerability feed; only the payload types +(`application/vnd.openctem.programfeed.*`) and record files differ, so a +key set or manifest of one feed is never accepted by the other. + +## Building locally + +```sh +go test ./... # recorded fixtures, no network +go build -o programfeed ./cmd/programfeed +./programfeed build --out out --cache httpcache # every source +./programfeed build --out out --sources diodb # one source +./programfeed build --out out --policy my-policy.json # your own publish policy +``` + +A key is needed only to sign; see `keys/README.md`. + +## Security + +Removal requests: [TAKEDOWN.md](TAKEDOWN.md). See [SECURITY.md](SECURITY.md) for the trust model and how to report a +problem. Adding a source: [CONTRIBUTING.md](CONTRIBUTING.md). + +## Licence + +Apache-2.0 (the code). The data keeps its sources' terms; see `NOTICE`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..60b1fd5 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,80 @@ +# Security + +## Reporting + +Please report vulnerabilities privately to **security@openctem.io** or through +GitHub's private vulnerability reporting on this repository. Do not open a +public issue. + +If you run a program listed in the feed and want it corrected or removed, +write to the same address. + +## Threat model + +A bundle tells every OpenCTEM platform which public programs exist and what +they cover. A forged, stale or poisoned bundle could point subscribers' +monitoring at organisations that never invited it, hide a program's closure +or its new out-of-scope entries, or carry hostile strings into the platform. +The collector itself talks to many third-party hosts and must not become a +way to reach internal services. + +### Integrity and freshness (same scheme as the OpenCTEM vulnerability feed) + +- **Offline root.** An Ed25519 root key, kept offline by the maintainer, + only signs *key sets*. Platforms pin its key id (`keys/root-keyid.txt`). + The root is not shared with any other feed, and the key set payload type + is specific to this feed, so a key set or manifest of another feed is + refused. +- **Key set.** `keys/keyset.dsse.json` lists the online signing keys, is + valid for at most 180 days and has a version that only goes up. A + platform refuses a key set whose version is below one it has accepted. +- **Online key.** The signing key is a secret of the `publish` GitHub + environment, which only the default branch can deploy to. Only the + publish job runs there; it signs, verifies and uploads what the build job + produced. The build job, which talks to the sources, never sees the key. +- **Rollback.** Every bundle has a sequence number. A platform refuses a + sequence it has already applied, and a delta whose base is not the + sequence it has. +- **Freeze.** Every manifest and pointer expires 7 days after it was made. +- **Integrity.** The signed manifest holds the SHA-256, size and record + count of every file; a mismatch refuses the whole bundle. +- **Size.** Manifests are capped at 1 MiB, files at 128 MiB compressed and + 1 GiB decompressed, records at 4 MiB, and a bundle at 100,000 programs. + +### Poisoned data + +- Every record is validated before signing and again by the importer: + strict JSON (unknown fields refused), id grammar, http(s) URLs without + credentials, bounded strings without control or bidi characters, enums, + timestamps in order. +- Every scope entry is normalised; public suffixes and wildcards over them, + reserved names, private/reserved/documentation addresses and ranges, + CIDRs wider than /16 (IPv4) or /48 (IPv6), and malformed entries are + moved to the program's `rejected` list with the reason. A bundle with a + scope entry the rules refuse is refused whole by the verifier. +- Out of scope wins over in scope. +- Targets the collector derived (`confidence: inferred`) are marked as such; + the feed never presents a guess as a program's published scope. +- An adapter can only write programs under its own id prefix. +- **Outage guard.** A source that suddenly reports fewer than half of the + programs it had (from at least 20) is treated as broken for that run and + its previous records are kept; a maintainer can override after checking + the source. A host that does not answer keeps its previous record instead + of being closed. A closed program stays visible for 180 days before it is + dropped. + +### The collector on the network + +- Connections only to public unicast addresses, checked at connect time + (after DNS), so neither a seed list entry nor a redirect can reach + loopback, private ranges or cloud metadata addresses. +- https only, at most 3 redirects, each re-checked against robots.txt. +- Size caps on every response; timeouts on every request. +- No credentials, cookies or tokens are ever sent to a source. +- The HTTP cache between runs only feeds conditional requests; every body + served from it goes through the same parsing and validation. + +## Key ceremony (maintainers) + +See `keys/README.md`. Never commit a private key; the repository holds only +public material (`root-keyid.txt`, `keyset.dsse.json`). diff --git a/TAKEDOWN.md b/TAKEDOWN.md new file mode 100644 index 0000000..37e879c --- /dev/null +++ b/TAKEDOWN.md @@ -0,0 +1,71 @@ +# Takedown and removal requests + +The programfeed collector reads public program information and, for the +platforms that allow it, redistributes it in a signed feed. If you run a +platform or a program and want your data out of the feed, or never in it, +this is how. + +## Who can ask + +- an authorised representative of a **platform** (to stop all records of the + platform), or +- the owner or security contact of a **program** (to remove that program). + +## How to ask + +Email **security@openctem.io** with: + +1. what to remove: the platform, or the program ids / program page URLs + (the `id` field of the record, for example `bounty-targets:hackerone-example`); +2. who you are: write from an address on the platform's or the program's + own domain, or from the security contact the program lists. We may ask + you to confirm through a channel we can verify independently; +3. optionally, why, and any conditions under which you would allow + inclusion later. + +You do not need to cite a legal basis. We act on the request first and ask +questions second. + +## What we commit to + +| Step | Time | +|---|---| +| Acknowledge receipt | within 2 business days | +| Verify the request and stop publishing the records (merge the policy change, run the publish workflow) | within 72 hours of verification | +| Remove earlier published copies from the project's release assets | within 7 days | +| Confirm to you in writing | when done | + +Subscribers pick the removal up through the feed itself: the next bundle +carries a `program_dropped` change for every removed program. + +## How a platform is switched off + +1. In `policy/platforms.json` set the platform's `publish` to `false` (add + the platform with `publish: false` if it is not listed; an unlisted + platform is never published). +2. The next build removes all of that platform's programs. It does this even + for records already in the previous release: they are dropped with a + `program_dropped` change instead of being kept as unchanged. A signed + build also stops fetching that platform's data from aggregate datasets. +3. Run the publish workflow; delete the earlier release assets that contain + the platform's records. + +## How a single program is removed + +Add its id to `excluded_programs` in `policy/platforms.json`. The program is +then never published, whichever source lists it, and is dropped from the +feed on the next build. For a self-hosted program that stays listed because +its security.txt exists, the exclusion keeps it out even though the file is +still there. + +## Local copies + +The collector can also build a local-only bundle (see the README). Those +bundles are made by whoever runs the tool, for their own use, and are not +published by this project. We cannot delete bundles that others built. + +## Permission instead of removal + +A platform that wants to be included, with specific attribution text or +conditions, tells us at **info@openctem.io**. We switch it to +`publish: true` only with the permission in writing. diff --git a/cmd/programfeed/main.go b/cmd/programfeed/main.go new file mode 100644 index 0000000..473f674 --- /dev/null +++ b/cmd/programfeed/main.go @@ -0,0 +1,308 @@ +// Command programfeed builds, signs and verifies OpenCTEM program bundles +// (public bug-bounty and vulnerability-disclosure programs), and manages +// the signing keys. +// +// programfeed build --out DIR [--prev DIR] [--root-key-id ID] [--cache DIR] [--sources a,b] [--policy FILE] [--local-only] [--seed-file FILE] [--allow-mass-change] +// programfeed sign --dir DIR --keyset FILE --root-key-id ID (key in $PROGRAMFEED_SIGNING_KEY) +// programfeed verify --dir DIR --root-key-id ID [--applied N] [--allow-expired] +// programfeed keys root-init --out FILE +// programfeed keys gen --out FILE +// programfeed keys sign-keyset --root FILE --key PUBLIC_KEY [--key ...] --version N [--days 180] --out FILE +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "strings" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/collect" + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/source" + "github.com/openctemio/programfeed/policy" +) + +// version is set at build time (-ldflags "-X main.version=..."). +var version = "dev" + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + if err := run(ctx, os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "programfeed:", err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) error { + if len(args) == 0 { + return errors.New("usage: programfeed build|sign|verify|keys ...") + } + switch args[0] { + case "build": + return cmdBuild(ctx, args[1:]) + case "sign": + return cmdSign(args[1:]) + case "verify": + return cmdVerify(args[1:]) + case "keys": + return cmdKeys(args[1:]) + } + return fmt.Errorf("unknown command %q", args[0]) +} + +func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) } + +// sources returns the configured adapters, in order. +// +// only limits the dataset adapters to platforms the publish policy allows +// (nil = everything, for a local-only build); a dataset adapter with +// nothing left to read is left out. +func sources(names, seedFile string, only []string, localOnly bool) ([]source.Source, error) { + if localOnly { + only = nil + } + var out []source.Source + for _, n := range strings.Split(names, ",") { + switch strings.TrimSpace(n) { + case "diodb": + out = append(out, source.DioDB{}) + case "securitytxt": + st := source.SecurityTxt{Progress: func(done, total int) { logf("securitytxt: %d/%d domains", done, total) }} + if seedFile != "" { + b, err := os.ReadFile(seedFile) + if err != nil { + return nil, err + } + st.Extra = source.ParseSeedFile(b) + } + out = append(out, st) + case "bounty-targets": + if b := source.NewBountyTargets(only); localOnly || len(intersect(only, source.BountyTargetsPlatforms())) > 0 { + out = append(out, b) + } else { + logf("bounty-targets: no platform may be published, not fetched") + } + case "pd-bounty": + if len(only) > 0 || localOnly { + out = append(out, source.NewPDBounty(only)) + } else { + logf("pd-bounty: no platform may be published, not fetched") + } + case "": + default: + return nil, fmt.Errorf("unknown source %q", n) + } + } + if len(out) == 0 { + return nil, errors.New("no source selected") + } + return out, nil +} + +func intersect(a, b []string) []string { + var out []string + for _, x := range a { + for _, y := range b { + if x == y { + out = append(out, x) + } + } + } + return out +} + +func cmdBuild(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("build", flag.ContinueOnError) + out := fs.String("out", "", "output directory") + prevDir := fs.String("prev", "", "previous release (verified before use)") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + mass := fs.Bool("allow-mass-change", false, "accept a source that suddenly lost more than half of its programs") + cache := fs.String("cache", "", "HTTP cache directory for conditional requests") + names := fs.String("sources", "diodb,securitytxt,bounty-targets,pd-bounty", "comma-separated adapters") + policyFile := fs.String("policy", "", "publish policy file (default: the compiled-in policy/platforms.json)") + localOnly := fs.Bool("local-only", false, "ignore the publish policy and write an UNSIGNED local bundle that must not be published") + seedFile := fs.String("seed-file", "seeds/securitytxt.txt", "extra security.txt seed domains (empty to skip)") + if err := fs.Parse(args); err != nil { + return err + } + if *out == "" { + return errors.New("build: --out is required") + } + pol := policy.Default() + if *policyFile != "" { + b, err := os.ReadFile(*policyFile) + if err != nil { + return err + } + if pol, err = policy.Parse(b); err != nil { + return fmt.Errorf("policy: %w", err) + } + } + if *localOnly && *prevDir != "" { + return errors.New("build: a local-only bundle is rebuilt from scratch; --prev is not allowed with --local-only") + } + srcs, err := sources(*names, *seedFile, pol.PublishablePlatforms(), *localOnly) + if err != nil { + return err + } + now := time.Now().UTC() + var prev *bundle.Snapshot + if *prevDir != "" { + v, err := bundle.VerifyDir(*prevDir, bundle.VerifyOptions{PinnedRoot: *root, Now: now, AllowExpired: true, SkipRecordChk: true}) + if err != nil { + return fmt.Errorf("previous release: %w", err) + } + if prev, err = bundle.ReadRecords(*prevDir, v.Snapshot); err != nil { + return fmt.Errorf("previous release: %w", err) + } + logf("previous release: sequence %d, %d programs", prev.Sequence, len(prev.Programs)) + } + var removed []string + if !*localOnly { + prev, removed = pol.PrunePrev(prev) + } + client := fetch.New(fetch.Options{UserAgent: fetch.UserAgent(version), CacheDir: *cache, Timeout: 20 * time.Second}) + next, err := collect.Build(ctx, prev, srcs, client, collect.Options{Now: now, AllowMassChange: *mass, Log: logf}) + if err != nil { + return err + } + if *localOnly { + logf("LOCAL-ONLY: the publish policy was not applied; this bundle is unsigned and must not be published") + } else { + full := len(next.Programs) + next = pol.Filter(next, removed) + logf("publish policy: %d of %d programs publishable", len(next.Programs), full) + } + snap, delta, _, err := bundle.Write(*out, prev, next, bundle.WriteOptions{Now: now, + Collector: bundle.Collector{Version: version, Commit: os.Getenv("GITHUB_SHA"), LocalOnly: *localOnly}}) + if err != nil { + return err + } + st := snap.Stats + logf("built sequence %d: %d programs (%d open, %d bounty), %d targets, %d rejected targets, %d changes", + snap.Sequence, st.Programs, st.Open, st.Bounty, st.Targets, st.Rejected, st.Changes) + if delta != nil { + logf("delta from %d: %d programs (+%d added, %d closed, %d dropped)", delta.BaseSequence, delta.Stats.Programs, + delta.Stats.Added, delta.Stats.Closed, delta.Stats.Dropped) + } + return nil +} + +func cmdSign(args []string) error { + fs := flag.NewFlagSet("sign", flag.ContinueOnError) + dir := fs.String("dir", "", "bundle directory") + keyset := fs.String("keyset", "keys/keyset.dsse.json", "key set envelope") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + if err := fs.Parse(args); err != nil { + return err + } + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(os.Getenv("PROGRAMFEED_SIGNING_KEY"))) + if err != nil || len(seed) != ed25519.SeedSize { + return errors.New("sign: PROGRAMFEED_SIGNING_KEY must be a base64 32-byte Ed25519 seed") + } + ks, err := os.ReadFile(*keyset) + if err != nil { + return err + } + return bundle.Sign(*dir, ed25519.NewKeyFromSeed(seed), ks, *root, time.Now()) +} + +func cmdVerify(args []string) error { + fs := flag.NewFlagSet("verify", flag.ContinueOnError) + dir := fs.String("dir", "", "bundle directory") + root := fs.String("root-key-id", os.Getenv("PROGRAMFEED_ROOT_KEY_ID"), "pinned root key id") + applied := fs.Uint64("applied", 0, "last applied sequence") + expired := fs.Bool("allow-expired", false, "accept an expired bundle") + if err := fs.Parse(args); err != nil { + return err + } + v, err := bundle.VerifyDir(*dir, bundle.VerifyOptions{PinnedRoot: *root, AppliedSeq: *applied, Now: time.Now(), AllowExpired: *expired}) + if err != nil { + return err + } + logf("verified sequence %d (snapshot %d programs, %d targets; key set v%d)", v.Latest.Sequence, + v.Snapshot.Stats.Programs, v.Snapshot.Stats.Targets, v.KeySet.Version) + return nil +} + +type multi []string + +func (m *multi) String() string { return strings.Join(*m, ",") } +func (m *multi) Set(v string) error { *m = append(*m, v); return nil } + +func cmdKeys(args []string) error { + if len(args) == 0 { + return errors.New("keys: root-init|gen|sign-keyset") + } + switch args[0] { + case "root-init", "gen": + fs := flag.NewFlagSet(args[0], flag.ContinueOnError) + out := fs.String("out", "", "file to write the private seed to (mode 0600; keep a root offline)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + if *out == "" { + return errors.New("keys: --out is required") + } + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return err + } + f, err := os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := f.WriteString(base64.StdEncoding.EncodeToString(priv.Seed()) + "\n"); err != nil { + f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + fmt.Printf("key id: %s\npublic key: %s\nseed file: %s\n", dsse.KeyID(pub), base64.StdEncoding.EncodeToString(pub), *out) + return nil + case "sign-keyset": + fs := flag.NewFlagSet("sign-keyset", flag.ContinueOnError) + rootFile := fs.String("root", "", "root seed file") + ver := fs.Uint64("version", 0, "key set version (higher than the current one)") + days := fs.Int("days", 180, "validity in days (at most 180)") + out := fs.String("out", "keys/keyset.dsse.json", "output") + var keys multi + fs.Var(&keys, "key", "base64 public key of an online signing key (repeatable)") + if err := fs.Parse(args[1:]); err != nil { + return err + } + raw, err := os.ReadFile(*rootFile) + if err != nil { + return err + } + seed, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))) + if err != nil || len(seed) != ed25519.SeedSize { + return errors.New("keys: the root file is not a base64 32-byte seed") + } + pubs := make([]ed25519.PublicKey, 0, len(keys)) + for _, k := range keys { + b, err := base64.StdEncoding.DecodeString(k) + if err != nil || len(b) != ed25519.PublicKeySize { + return fmt.Errorf("keys: %q is not a base64 Ed25519 public key", k) + } + pubs = append(pubs, b) + } + env, err := dsse.SignKeySet(ed25519.NewKeyFromSeed(seed), *ver, time.Now(), time.Duration(*days)*24*time.Hour, pubs) + if err != nil { + return err + } + return os.WriteFile(*out, env, 0o644) + } + return fmt.Errorf("keys: unknown command %q", args[0]) +} diff --git a/cmd/programfeed/main_test.go b/cmd/programfeed/main_test.go new file mode 100644 index 0000000..2ef0870 --- /dev/null +++ b/cmd/programfeed/main_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "testing" + + "github.com/openctemio/programfeed/policy" +) + +func names(t *testing.T, list string, only []string, local bool) []string { + t.Helper() + srcs, err := sources(list, "", only, local) + if err != nil { + return nil + } + var out []string + for _, s := range srcs { + out = append(out, s.Meta().Name) + } + return out +} + +// A signed build does not even fetch dataset platforms the policy does not +// publish; a local-only build reads everything. +func TestSourceSelectionFollowsPolicy(t *testing.T) { + pub := policy.Default().PublishablePlatforms() + if got := names(t, "bounty-targets,pd-bounty,diodb", pub, false); len(got) != 2 || got[0] != "pd-bounty" || got[1] != "disclose" { + t.Errorf("signed build: %v (bounty-targets has no publishable platform and must not be fetched)", got) + } + if got := names(t, "bounty-targets,pd-bounty", pub, true); len(got) != 2 || got[0] != "bounty-targets" { + t.Errorf("local-only build: %v", got) + } + if got := names(t, "bounty-targets", pub, false); got != nil { + t.Errorf("nothing to fetch must be an error, got %v", got) + } +} diff --git a/docs/outreach/README.md b/docs/outreach/README.md new file mode 100644 index 0000000..346ab8d --- /dev/null +++ b/docs/outreach/README.md @@ -0,0 +1,16 @@ +# Outreach to platforms + +Drafts of permission requests, one per platform. **Nothing here has been sent.** The owner decides when and to whom. + +A platform stays `publish: false` in [`policy/platforms.json`](../../policy/platforms.json) until a written permission is on file. Then, in one pull request: set `publish: true`, set the `attribution` text the platform asked for, and link the permission in the pull request description. + +| Platform | Draft | Why we ask | +|---|---|---| +| HackerOne | [hackerone.md](hackerone.md) | the Hacker API needs an account token; no redistribution grant | +| Bugcrowd | [bugcrowd.md](bugcrowd.md) | website terms forbid redistribution | +| Intigriti | [intigriti.md](intigriti.md) | researcher API needs a token; no redistribution grant | +| YesWeHack | [yeswehack.md](yeswehack.md) | legal notices forbid redistributing platform data | +| Immunefi | [immunefi.md](immunefi.md) | terms forbid automated access without written consent | +| HackenProof | [hackenproof.md](hackenproof.md) | no official listing or redistribution statement | + +Signed off in every draft as "OpenCTEM team, info@openctem.io". diff --git a/docs/outreach/bugcrowd.md b/docs/outreach/bugcrowd.md new file mode 100644 index 0000000..2e7f468 --- /dev/null +++ b/docs/outreach/bugcrowd.md @@ -0,0 +1,48 @@ +# Permission request to Bugcrowd + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of Bugcrowd) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** Bugcrowd (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing Bugcrowd public programs in the OpenCTEM program feed + +Hello Bugcrowd team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from Bugcrowd. Until we hear from you we publish nothing from Bugcrowd. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public engagement directory and the scope each public engagement publishes (engagement name, engagement page, bounty or no bounty, in-scope and out-of-scope targets, safe-harbour field, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +Your website terms do not allow copying or redistribution of website content without permission, so we do not publish anything of yours today. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on Bugcrowd and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names Bugcrowd as the original platform and links to the original program page, so users go to Bugcrowd for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today the data reaches us through a public aggregate dataset on GitHub and we read it at a pinned commit. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of Bugcrowd, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include Bugcrowd's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case Bugcrowd stays unpublished and we remove any local copies we hold. + +If you would rather provide a feed or an API for public engagements, we will use that instead. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/docs/outreach/hackenproof.md b/docs/outreach/hackenproof.md new file mode 100644 index 0000000..74d4904 --- /dev/null +++ b/docs/outreach/hackenproof.md @@ -0,0 +1,48 @@ +# Permission request to HackenProof + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of HackenProof) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** HackenProof (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing HackenProof public programs in the OpenCTEM program feed + +Hello HackenProof team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from HackenProof. Until we hear from you we publish nothing from HackenProof. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public program directory and the scope each public program publishes (program name, program page, bounty or no bounty, in-scope and out-of-scope assets, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +We have not found an official machine-readable listing or a statement on redistribution, so we do not publish anything of yours today. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on HackenProof and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names HackenProof as the original platform and links to the original program page, so users go to HackenProof for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today we do not read this platform at all. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of HackenProof, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include HackenProof's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case HackenProof stays unpublished and we remove any local copies we hold. + +If you offer a listing or API for public programs, we will use that and nothing else. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/docs/outreach/hackerone.md b/docs/outreach/hackerone.md new file mode 100644 index 0000000..6a99350 --- /dev/null +++ b/docs/outreach/hackerone.md @@ -0,0 +1,48 @@ +# Permission request to HackerOne + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of HackerOne) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** HackerOne (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing HackerOne public programs in the OpenCTEM program feed + +Hello HackerOne team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from HackerOne. Until we hear from you we publish nothing from HackerOne. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public program directory and the scope each public program publishes (program name, program page, bounty or no bounty, in-scope and out-of-scope assets, safe-harbour flag, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +We understand the Hacker API is available to account holders only and we do not use it. The public program listings are currently the only form in which this information reaches us. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on HackerOne and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names HackerOne as the original platform and links to the original program page, so users go to HackerOne for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today the data reaches us through a public aggregate dataset on GitHub and we read it at a pinned commit. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of HackerOne, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include HackerOne's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case HackerOne stays unpublished and we remove any local copies we hold. + +If a partner or public-programs data path exists that you would prefer us to use instead of the public pages, we will use that and nothing else. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/docs/outreach/immunefi.md b/docs/outreach/immunefi.md new file mode 100644 index 0000000..1b6f14d --- /dev/null +++ b/docs/outreach/immunefi.md @@ -0,0 +1,48 @@ +# Permission request to Immunefi + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of Immunefi) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** Immunefi (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing Immunefi public programs in the OpenCTEM program feed + +Hello Immunefi team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from Immunefi. Until we hear from you we publish nothing from Immunefi. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public bug-bounty directory and the assets in scope of each public bounty (project name, bounty page, rewards present or not, in-scope assets such as websites and smart-contract addresses, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +Your terms of use do not allow automated access to the website without written consent, so we do not read your website and we do not publish anything of yours today. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on Immunefi and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names Immunefi as the original platform and links to the original program page, so users go to Immunefi for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today we do not read this platform at all. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of Immunefi, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include Immunefi's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case Immunefi stays unpublished and we remove any local copies we hold. + +If you can share written consent for reading your public bounty listing, or point us to a data path made for this, we would use only that. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/docs/outreach/intigriti.md b/docs/outreach/intigriti.md new file mode 100644 index 0000000..0d40535 --- /dev/null +++ b/docs/outreach/intigriti.md @@ -0,0 +1,48 @@ +# Permission request to Intigriti + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of Intigriti) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** Intigriti (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing Intigriti public programs in the OpenCTEM program feed + +Hello Intigriti team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from Intigriti. Until we hear from you we publish nothing from Intigriti. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public program directory and the scope each public program publishes (program name, program page, bounty range, in-scope and out-of-scope endpoints, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +We understand the researcher API requires a personal access token and we do not use it. We do not publish anything of yours today. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on Intigriti and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names Intigriti as the original platform and links to the original program page, so users go to Intigriti for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today the data reaches us through a public aggregate dataset on GitHub and we read it at a pinned commit. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of Intigriti, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include Intigriti's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case Intigriti stays unpublished and we remove any local copies we hold. + +If you offer a partner or public data path, we will use it instead of any other route. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/docs/outreach/yeswehack.md b/docs/outreach/yeswehack.md new file mode 100644 index 0000000..d4debb2 --- /dev/null +++ b/docs/outreach/yeswehack.md @@ -0,0 +1,48 @@ +# Permission request to YesWeHack + +> **DRAFT. Not sent.** Review, confirm the recipient (partnerships or legal contact of YesWeHack) and send from info@openctem.io. Keep the reply with the written permission in the repository history before changing `policy/platforms.json`. + +**To:** YesWeHack (partnerships / legal contact, to be confirmed before sending) +**From:** OpenCTEM team, info@openctem.io +**Subject:** Request for permission: referencing YesWeHack public programs in the OpenCTEM program feed + +Hello YesWeHack team, + +We are writing to ask for your written permission, or to be pointed to an official data path, before we publish anything that comes from YesWeHack. Until we hear from you we publish nothing from YesWeHack. + +## Who we are + +OpenCTEM (https://openctem.io, source at https://github.com/openctemio) is an open-source continuous threat exposure management platform that organisations run themselves. One of its modules lets an organisation follow public bug-bounty and vulnerability-disclosure programs: who runs a program, where the policy is, whether it pays, what scope the program publishes and how that scope changes. The collector that feeds it is open source: https://github.com/openctemio/programfeed. + +## What data we would use + +We would include the public program directory and the scope each public program publishes (program name, program page, bounty range, in-scope and out-of-scope targets, status). We would use only what is already visible to any visitor without an account: no private programs, no report data, no researcher data, no account-only fields. + +Your legal notices do not allow redistribution of platform data without permission, so we do not publish anything of yours today. + +## How it would be used + +- **Passive monitoring.** Subscribers read program information and are notified when a public program changes its scope or rules. The feed does not test, scan or contact anything. +- **Never an authorisation.** A scope entry in the feed states what the program lists. It is never presented as permission to test. Before any active testing of a program's assets, OpenCTEM requires the user to read the program's terms on YesWeHack and attest that they accepted them and are eligible; targets taken from a listing are suggestions the user confirms. +- **Link back, attribution.** Every record names YesWeHack as the original platform and links to the original program page, so users go to YesWeHack for the authoritative, current terms. We show the attribution text you ask for wherever the data appears. +- **Careful reading.** Today the data reaches us through a public aggregate dataset on GitHub and we read it at a pinned commit. If you grant permission we would read only what you specify, identify ourselves with a descriptive User-Agent and contact address, honour robots.txt, use conditional requests and keep to at most one request per second. +- **Signed and verifiable.** The feed is published as a signed bundle; every record carries its provenance (source, commit, original platform, original program page, time read). + +## Removal at any time + +Any request to remove a program, or all of YesWeHack, is handled under https://github.com/openctemio/programfeed/blob/main/TAKEDOWN.md: we acknowledge within 2 business days, stop publishing the records within 72 hours and delete earlier published copies. Contact: security@openctem.io. + +## What we ask + +1. Written permission to include YesWeHack's public program listings (program name, program page, bounty yes/no, published scope, status) in the OpenCTEM program feed, which is publicly redistributed, under the attribution text and any conditions you specify; **or** +2. an official API or partner path for public program data, with its terms; **or** +3. a clear no, in which case YesWeHack stays unpublished and we remove any local copies we hold. + +If you would prefer a feed or API for public programs, we will use it instead. + +If it helps, we are glad to share a sample of exactly what a record looks like, or to schedule a short call. + +Thank you for your time. + +OpenCTEM team +info@openctem.io diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fa474c2 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/openctemio/programfeed + +go 1.26.0 + +toolchain go1.26.9 + +require golang.org/x/net v0.60.0 + +require golang.org/x/text v0.42.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1ed037a --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go new file mode 100644 index 0000000..2454d6d --- /dev/null +++ b/internal/bundle/bundle.go @@ -0,0 +1,419 @@ +// Package bundle builds, signs, reads and verifies program bundles: JSON +// Lines files per record type, a signed manifest per snapshot and delta, +// and a signed latest pointer. The layout, envelopes, key set and checks +// are the ones of the OpenCTEM vulnerability feed, so one verifier serves +// both (only the payload types and record files differ). +package bundle + +import ( + "bufio" + "compress/gzip" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +// Schemas, payload types and file names. +const ( + ManifestSchema = "openctem.programfeed/v1" + LatestSchema = "openctem.programfeed.latest/v1" + ManifestPayloadType = "application/vnd.openctem.programfeed.manifest+json" + LatestPayloadType = "application/vnd.openctem.programfeed.latest+json" + + KindSnapshot = "snapshot" + KindDelta = "delta" + + LatestFile = "latest.dsse.json" + KeySetFile = "keyset.dsse.json" + ManifestFile = "%s.manifest.dsse.json" // snapshot | delta + + Validity = 7 * 24 * time.Hour +) + +// Caps (the importer applies the same). +const ( + MaxManifestBytes = 1 << 20 + MaxFileBytes = 128 << 20 + MaxDecompressedBytes = 1 << 30 + MaxRecordBytes = 4 << 20 +) + +// Record files of a bundle, in import order. A snapshot holds every +// program and this run's change log; a delta holds the programs that +// changed (complete new records) and the same change log. +var recordFiles = []string{"programs", "changes"} + +// FileName is the name of a record file of a kind. +func FileName(kind, records string) string { return kind + "-" + records + ".jsonl.gz" } + +// Source names one upstream source, its terms and how much it gave. +type Source struct { + Name string `json:"name"` + URL string `json:"url"` + AsOf time.Time `json:"as_of"` // last successful read + Terms string `json:"terms"` + Licence string `json:"licence"` + Attribution string `json:"attribution"` + Programs int `json:"programs"` +} + +// File is one record file of a manifest. +type File struct { + Name string `json:"name"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Records int `json:"records"` +} + +// Stats summarises a bundle. +type Stats struct { + Programs int `json:"programs"` + Open int `json:"open"` + Bounty int `json:"bounty"` + Targets int `json:"targets"` + Rejected int `json:"rejected"` + Changes int `json:"changes"` + Added int `json:"added"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` +} + +// Collector identifies the build. +type Collector struct { + Version string `json:"version"` + Commit string `json:"commit"` + // LocalOnly marks a bundle built without the publish policy (it may + // hold records of platforms that do not allow redistribution). It is + // never signed and never published. + LocalOnly bool `json:"local_only,omitempty"` +} + +// Manifest is the signed description of a snapshot or a delta. +type Manifest struct { + Schema string `json:"schema"` + Sequence uint64 `json:"sequence"` + Kind string `json:"kind"` + BaseSequence uint64 `json:"base_sequence,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` + Sources []Source `json:"sources"` + Files []File `json:"files"` + Stats Stats `json:"stats"` + Collector Collector `json:"collector"` +} + +// Latest is the signed pointer to the newest bundle. +type Latest struct { + Schema string `json:"schema"` + Sequence uint64 `json:"sequence"` + Tag string `json:"tag"` + Snapshot string `json:"snapshot"` + Delta string `json:"delta,omitempty"` + BaseSequence uint64 `json:"base_sequence,omitempty"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` +} + +// Tag is the release tag of a sequence. +func Tag(sequence uint64) string { return fmt.Sprintf("v1-%d", sequence) } + +// Snapshot is the complete program set plus the change log of the run +// that produced it. +type Snapshot struct { + Sequence uint64 + Sources []Source + Programs map[string]model.Program + Changes []model.Change +} + +// NewSnapshot returns an empty snapshot. +func NewSnapshot() *Snapshot { return &Snapshot{Programs: map[string]model.Program{}} } + +// Source returns a source by name. +func (s *Snapshot) Source(name string) (Source, bool) { + for _, src := range s.Sources { + if src.Name == name { + return src, true + } + } + return Source{}, false +} + +// SortedIDs returns the program ids in order. +func (s *Snapshot) SortedIDs() []string { + ids := make([]string, 0, len(s.Programs)) + for id := range s.Programs { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// ChangedIDs returns the ids of programs present in the snapshot that the +// change log touches, in order. +func (s *Snapshot) ChangedIDs() []string { + seen := map[string]bool{} + var ids []string + for _, c := range s.Changes { + if _, ok := s.Programs[c.Program]; ok && !seen[c.Program] { + seen[c.Program] = true + ids = append(ids, c.Program) + } + } + sort.Strings(ids) + return ids +} + +// Validate checks every record of the snapshot. +func (s *Snapshot) Validate() error { + if len(s.Programs) > model.MaxPrograms { + return fmt.Errorf("%d programs, over the cap of %d", len(s.Programs), model.MaxPrograms) + } + for _, id := range s.SortedIDs() { + p := s.Programs[id] + if p.ID != id { + return fmt.Errorf("program %s filed under %s", p.ID, id) + } + if err := p.Validate(); err != nil { + return err + } + } + for i := range s.Changes { + c := &s.Changes[i] + if err := c.Validate(); err != nil { + return err + } + if c.Sequence != s.Sequence { + return fmt.Errorf("change of %s has sequence %d, the snapshot is %d", c.Program, c.Sequence, s.Sequence) + } + if _, ok := s.Programs[c.Program]; ok == (c.Kind == model.ChangeProgramDropped) { + return fmt.Errorf("change %s of %s: a dropped program must be absent, any other present", c.Kind, c.Program) + } + } + return nil +} + +func (s *Snapshot) stats(ids []string) Stats { + st := Stats{Programs: len(ids), Changes: len(s.Changes)} + for _, id := range ids { + p := s.Programs[id] + if p.Status == model.StatusOpen { + st.Open++ + } + if p.OffersBounty { + st.Bounty++ + } + st.Targets += len(p.InScope) + len(p.OutOfScope) + st.Rejected += len(p.Rejected) + } + for _, c := range s.Changes { + switch c.Kind { + case model.ChangeProgramAdded: + st.Added++ + case model.ChangeProgramClosed: + st.Closed++ + case model.ChangeProgramDropped: + st.Dropped++ + } + } + return st +} + +// WriteOptions describe a build. +type WriteOptions struct { + Now time.Time + Collector Collector +} + +// Write writes the snapshot of next and, when prev is not nil, the delta +// from prev, with unsigned manifests (payloads) and the latest pointer +// payload, into dir. +func Write(dir string, prev, next *Snapshot, opt WriteOptions) (snap, delta *Manifest, latest *Latest, err error) { + if err := next.Validate(); err != nil { + return nil, nil, nil, fmt.Errorf("refusing to publish an invalid bundle: %w", err) + } + if prev != nil && next.Sequence <= prev.Sequence { + return nil, nil, nil, fmt.Errorf("sequence %d is not after %d", next.Sequence, prev.Sequence) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, nil, nil, err + } + now := opt.Now.UTC().Truncate(time.Second) + all := next.SortedIDs() + snap = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindSnapshot, CreatedAt: now, + ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(all)} + if snap.Files, err = writeFiles(dir, KindSnapshot, next, all); err != nil { + return nil, nil, nil, err + } + latest = &Latest{Schema: LatestSchema, Sequence: next.Sequence, Tag: Tag(next.Sequence), + Snapshot: fmt.Sprintf(ManifestFile, KindSnapshot), CreatedAt: now, ExpiresAt: now.Add(Validity)} + if prev != nil { + ids := next.ChangedIDs() + delta = &Manifest{Schema: ManifestSchema, Sequence: next.Sequence, Kind: KindDelta, BaseSequence: prev.Sequence, + CreatedAt: now, ExpiresAt: now.Add(Validity), Sources: next.Sources, Collector: opt.Collector, Stats: next.stats(ids)} + if delta.Files, err = writeFiles(dir, KindDelta, next, ids); err != nil { + return nil, nil, nil, err + } + latest.Delta, latest.BaseSequence = fmt.Sprintf(ManifestFile, KindDelta), prev.Sequence + } + if err := writeJSON(filepath.Join(dir, "snapshot.manifest.json"), snap); err != nil { + return nil, nil, nil, err + } + if delta != nil { + if err := writeJSON(filepath.Join(dir, "delta.manifest.json"), delta); err != nil { + return nil, nil, nil, err + } + } + if err := writeJSON(filepath.Join(dir, "latest.json"), latest); err != nil { + return nil, nil, nil, err + } + return snap, delta, latest, nil +} + +func writeFiles(dir, kind string, s *Snapshot, ids []string) ([]File, error) { + var files []File + for _, rf := range recordFiles { + name := FileName(kind, rf) + var n int + f, err := writeGz(filepath.Join(dir, name), func(enc *json.Encoder) error { + switch rf { + case "programs": + for _, id := range ids { + if err := enc.Encode(s.Programs[id]); err != nil { + return err + } + n++ + } + case "changes": + for _, c := range s.Changes { + if err := enc.Encode(c); err != nil { + return err + } + n++ + } + } + return nil + }) + if err != nil { + return nil, err + } + f.Name, f.Records = name, n + files = append(files, f) + } + return files, nil +} + +func writeGz(path string, fill func(*json.Encoder) error) (File, error) { + out, err := os.Create(path) + if err != nil { + return File{}, err + } + h := sha256.New() + counter := &countWriter{} + gz, _ := gzip.NewWriterLevel(io.MultiWriter(out, h, counter), gzip.BestCompression) + gz.ModTime = time.Unix(0, 0) + bw := bufio.NewWriterSize(gz, 1<<20) + enc := json.NewEncoder(bw) + enc.SetEscapeHTML(false) + if err := fill(enc); err != nil { + out.Close() + return File{}, err + } + if err := bw.Flush(); err != nil { + out.Close() + return File{}, err + } + if err := gz.Close(); err != nil { + out.Close() + return File{}, err + } + if err := out.Close(); err != nil { + return File{}, err + } + if counter.n > MaxFileBytes { + return File{}, fmt.Errorf("%s is %d bytes, over the %d cap", filepath.Base(path), counter.n, MaxFileBytes) + } + return File{SHA256: hex.EncodeToString(h.Sum(nil)), Size: counter.n}, nil +} + +type countWriter struct{ n int64 } + +func (c *countWriter) Write(p []byte) (int, error) { + c.n += int64(len(p)) + return len(p), nil +} + +func writeJSON(path string, v any) error { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, b, 0o644) +} + +// Sign wraps the unsigned manifests and latest pointer in dir in DSSE +// envelopes signed by priv, and copies the key set next to them. The +// signing key must be listed in the key set. +func Sign(dir string, priv ed25519.PrivateKey, keyset []byte, pinnedRoot string, now time.Time) error { + if raw, err := os.ReadFile(filepath.Join(dir, "snapshot.manifest.json")); err == nil { + var m struct { + Collector Collector `json:"collector"` + } + if json.Unmarshal(raw, &m) == nil && m.Collector.LocalOnly { + return errors.New("refusing to sign a local-only bundle: it was built without the publish policy") + } + } + ks, err := dsse.VerifyKeySet(keyset, pinnedRoot, 0, now) + if err != nil { + return err + } + pub, _ := priv.Public().(ed25519.PublicKey) + listed := false + for _, k := range ks.Keys { + if k.KeyID == dsse.KeyID(pub) { + listed = true + } + } + if !listed { + return fmt.Errorf("the signing key %s is not in the key set", dsse.KeyID(pub)) + } + pairs := []struct{ in, out, typ string }{ + {"snapshot.manifest.json", fmt.Sprintf(ManifestFile, KindSnapshot), ManifestPayloadType}, + {"delta.manifest.json", fmt.Sprintf(ManifestFile, KindDelta), ManifestPayloadType}, + {"latest.json", LatestFile, LatestPayloadType}, + } + for _, p := range pairs { + payload, err := os.ReadFile(filepath.Join(dir, p.in)) + if errors.Is(err, os.ErrNotExist) && p.in == "delta.manifest.json" { + continue + } + if err != nil { + return err + } + env, err := dsse.Sign(priv, p.typ, payload) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, p.out), env, 0o644); err != nil { + return err + } + if err := os.Remove(filepath.Join(dir, p.in)); err != nil { + return err + } + } + return os.WriteFile(filepath.Join(dir, KeySetFile), keyset, 0o644) +} + +// Stats summarises the whole snapshot. +func (s *Snapshot) Stats() Stats { return s.stats(s.SortedIDs()) } diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go new file mode 100644 index 0000000..a2c4762 --- /dev/null +++ b/internal/bundle/bundle_test.go @@ -0,0 +1,274 @@ +package bundle + +import ( + "compress/gzip" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +type keys struct { + rootID string + priv ed25519.PrivateKey + keyset []byte +} + +func newKeys(t *testing.T) keys { + t.Helper() + rootPub, root, _ := ed25519.GenerateKey(nil) + pub, priv, _ := ed25519.GenerateKey(nil) + ks, err := dsse.SignKeySet(root, 1, time.Now().Add(-time.Hour), 90*24*time.Hour, []ed25519.PublicKey{pub}) + if err != nil { + t.Fatal(err) + } + return keys{rootID: dsse.KeyID(rootPub), priv: priv, keyset: ks} +} + +var t0 = time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + +func program(id, target string) model.Program { + p := model.Program{ + ID: id, Source: "disclose", Platform: "self-hosted", Name: id, URL: "https://openctem.io/security", + Type: model.TypeVDP, Status: model.StatusOpen, + InScope: []model.Target{{Type: model.TargetDomain, Value: target, Confidence: model.ConfidenceInferred}}, + OutOfScope: []model.Target{}, + Terms: model.Terms{URL: "https://openctem.io/security"}, + Provenance: model.Provenance{Source: "disclose", SourceURL: "https://openctem.io/list.json", FetchedAt: t0}, + FirstSeen: t0, LastSeen: t0, LastChanged: t0, + } + model.Normalize(&p) + return p +} + +func snapshot(seq uint64, target string) *Snapshot { + s := NewSnapshot() + s.Sequence = seq + s.Sources = []Source{{Name: "disclose", URL: "https://openctem.io/list.json", AsOf: t0, Licence: "CC0-1.0"}} + a, b := program("disclose:a", "a.openctem.io"), program("disclose:b", target) + s.Programs[a.ID], s.Programs[b.ID] = a, b + tg := b.InScope[0] + s.Changes = []model.Change{{Sequence: seq, Program: b.ID, Kind: model.ChangeTargetAdded, Scope: "in", Target: &tg}} + return s +} + +func build(t *testing.T, k keys, prev, next *Snapshot) string { + t.Helper() + dir := t.TempDir() + if _, _, _, err := Write(dir, prev, next, WriteOptions{Now: time.Now()}); err != nil { + t.Fatal(err) + } + if err := Sign(dir, k.priv, k.keyset, k.rootID, time.Now()); err != nil { + t.Fatal(err) + } + return dir +} + +func TestRoundTrip(t *testing.T) { + k := newKeys(t) + prev, next := snapshot(1, "b.openctem.io"), snapshot(2, "c.openctem.io") + dir := build(t, k, prev, next) + v, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 1, Now: time.Now()}) + if err != nil { + t.Fatal(err) + } + if v.Latest.Sequence != 2 || v.Latest.Tag != "v1-2" || v.Delta == nil || v.Delta.BaseSequence != 1 { + t.Fatalf("%+v", v.Latest) + } + if v.Snapshot.Stats.Programs != 2 || v.Delta.Stats.Programs != 1 || v.Delta.Stats.Changes != 1 { + t.Fatalf("stats %+v / %+v", v.Snapshot.Stats, v.Delta.Stats) + } + got, err := ReadRecords(dir, v.Snapshot) + if err != nil || len(got.Programs) != 2 || got.Programs["disclose:b"].InScope[0].Value != "c.openctem.io" { + t.Fatalf("snapshot %v %+v", err, got) + } + d, err := ReadRecords(dir, *v.Delta) + if err != nil || len(d.Programs) != 1 || len(d.Changes) != 1 { + t.Fatalf("delta %v %+v", err, d) + } + dir2 := build(t, k, prev, next) + for _, f := range v.Snapshot.Files { + a, _ := os.ReadFile(filepath.Join(dir, f.Name)) + b, _ := os.ReadFile(filepath.Join(dir2, f.Name)) + if string(a) != string(b) { + t.Errorf("%s differs between builds", f.Name) + } + } +} + +func TestVerifyRefuses(t *testing.T) { + k := newKeys(t) + other := newKeys(t) + now := time.Now() + cases := map[string]struct { + mut func(t *testing.T, dir string) + opt VerifyOptions + }{ + "rollback": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 5, Now: now}}, + "replay same": {nil, VerifyOptions{PinnedRoot: k.rootID, AppliedSeq: 3, Now: now}}, + "expired": {nil, VerifyOptions{PinnedRoot: k.rootID, Now: now.Add(8 * 24 * time.Hour)}}, + "other root": {nil, VerifyOptions{PinnedRoot: other.rootID, Now: now}}, + "no pinned root": {nil, VerifyOptions{Now: now}}, + "key set rolled back": {nil, VerifyOptions{PinnedRoot: k.rootID, MinKeySetVer: 2, Now: now}}, + "tampered file": {func(t *testing.T, dir string) { + p := filepath.Join(dir, FileName(KindSnapshot, "programs")) + b, _ := os.ReadFile(p) + b[len(b)/2] ^= 0xff + _ = os.WriteFile(p, b, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "swapped key set": {func(t *testing.T, dir string) { + _ = os.WriteFile(filepath.Join(dir, KeySetFile), other.keyset, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "vulnerability feed key set": {func(t *testing.T, dir string) { + // A key set of another feed (other payload type) is refused even + // under the same root. + raw, _ := os.ReadFile(filepath.Join(dir, KeySetFile)) + var env dsse.Envelope + _ = json.Unmarshal(raw, &env) + env.PayloadType = "application/vnd.openctem.vulnfeed.keyset+json" + b, _ := json.Marshal(env) + _ = os.WriteFile(filepath.Join(dir, KeySetFile), b, 0o644) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "pointer signed by a stranger": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, other.priv, func(m map[string]any) {}) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "pointer names another sequence": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["sequence"] = 9; m["tag"] = "v1-9" }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "manifest valid for a year": {func(t *testing.T, dir string) { + resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) { + m["expires_at"] = now.Add(365 * 24 * time.Hour).UTC().Format(time.RFC3339) + }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + "unknown field": {func(t *testing.T, dir string) { + resignJSON(t, dir, LatestFile, LatestPayloadType, k.priv, func(m map[string]any) { m["extra"] = 1 }) + }, VerifyOptions{PinnedRoot: k.rootID, Now: now}}, + } + for name, c := range cases { + dir := build(t, k, snapshot(2, "b.openctem.io"), snapshot(3, "c.openctem.io")) + if c.mut != nil { + c.mut(t, dir) + } + if _, err := VerifyDir(dir, c.opt); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +// Poisoned records under a valid signature and correct hashes are refused: +// the importer re-validates every record. +func TestPoisonedRecordsRefused(t *testing.T) { + k := newKeys(t) + good := program("disclose:x", "x.openctem.io") + line := func(mut func(m map[string]any)) string { + b, _ := json.Marshal(good) + var m map[string]any + _ = json.Unmarshal(b, &m) + mut(m) + out, _ := json.Marshal(m) + return string(out) + } + target := func(typ, value string) func(m map[string]any) { + return func(m map[string]any) { + m["in_scope"] = []any{map[string]any{"type": typ, "value": value, "confidence": "published"}} + } + } + cases := map[string]string{ + "public suffix wildcard": line(target("wildcard", "*.com")), + "private range": line(target("cidr", "10.0.0.0/8")), + "huge range": line(target("cidr", "1.0.0.0/8")), + "metadata address": line(target("ip", "169.254.169.254")), + "bare TLD": line(target("domain", "com")), + "javascript url": line(func(m map[string]any) { m["url"] = "javascript:alert(1)" }), + "closed without time": line(func(m map[string]any) { m["status"] = "closed" }), + "extra field": line(func(m map[string]any) { m["sql"] = "x" }), + "duplicate": line(func(map[string]any) {}) + "\n" + line(func(map[string]any) {}), + } + for name, l := range cases { + dir := build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), l+"\n", strings.Count(l, "\n")+1) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil { + t.Errorf("%s: accepted", name) + } + } + // A change naming a program that is not in the bundle is refused. + dir := build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "changes"), `{"sequence":1,"program":"disclose:ghost","kind":"program_added"}`+"\n", 1) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil { + t.Error("change of an absent program accepted") + } + // A wrong record count is refused too. + dir = build(t, k, nil, snapshot(1, "b.openctem.io")) + replaceFile(t, dir, k, FileName(KindSnapshot, "programs"), line(func(map[string]any) {})+"\n", 5) + if _, err := VerifyDir(dir, VerifyOptions{PinnedRoot: k.rootID, Now: time.Now()}); err == nil || !strings.Contains(err.Error(), "records") { + t.Errorf("record count: %v", err) + } +} + +func TestWriteRefusesInvalidSnapshot(t *testing.T) { + s := snapshot(1, "b.openctem.io") + p := s.Programs["disclose:a"] + p.InScope = []model.Target{{Type: model.TargetWildcard, Value: "*.com", Confidence: model.ConfidencePublished}} + s.Programs["disclose:a"] = p + if _, _, _, err := Write(t.TempDir(), nil, s, WriteOptions{Now: time.Now()}); err == nil { + t.Fatal("invalid snapshot written") + } +} + +func TestSignRefusesKeyOutsideKeySet(t *testing.T) { + k := newKeys(t) + _, stranger, _ := ed25519.GenerateKey(nil) + dir := t.TempDir() + if _, _, _, err := Write(dir, nil, snapshot(1, "b.openctem.io"), WriteOptions{Now: time.Now()}); err != nil { + t.Fatal(err) + } + if err := Sign(dir, stranger, k.keyset, k.rootID, time.Now()); err == nil { + t.Fatal("signed with a key outside the key set") + } +} + +// replaceFile rewrites one record file, fixes its hash/size/count in the +// manifest and re-signs the manifest: only record validation can catch it. +func replaceFile(t *testing.T, dir string, k keys, name, content string, records int) { + t.Helper() + p := filepath.Join(dir, name) + f, _ := os.Create(p) + gz := gzip.NewWriter(f) + _, _ = gz.Write([]byte(content)) + _ = gz.Close() + _ = f.Close() + b, _ := os.ReadFile(p) + sum := sha256.Sum256(b) + resignJSON(t, dir, "snapshot.manifest.dsse.json", ManifestPayloadType, k.priv, func(m map[string]any) { + for _, fi := range m["files"].([]any) { + fm := fi.(map[string]any) + if fm["name"] == name { + fm["sha256"], fm["size"], fm["records"] = hex.EncodeToString(sum[:]), len(b), records + } + } + }) +} + +func resignJSON(t *testing.T, dir, file, payloadType string, priv ed25519.PrivateKey, mut func(map[string]any)) { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + t.Fatal(err) + } + var env dsse.Envelope + _ = json.Unmarshal(raw, &env) + var m map[string]any + _ = json.Unmarshal(env.Payload, &m) + mut(m) + payload, _ := json.Marshal(m) + out, _ := dsse.Sign(priv, payloadType, payload) + _ = os.WriteFile(filepath.Join(dir, file), out, 0o644) +} diff --git a/internal/bundle/verify.go b/internal/bundle/verify.go new file mode 100644 index 0000000..bd7e4dd --- /dev/null +++ b/internal/bundle/verify.go @@ -0,0 +1,277 @@ +package bundle + +import ( + "bufio" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/openctemio/programfeed/internal/dsse" + "github.com/openctemio/programfeed/internal/model" +) + +// Verified is a bundle directory whose signatures, freshness and file +// hashes checked out. +type Verified struct { + KeySet *dsse.KeySet + Latest Latest + Snapshot Manifest + Delta *Manifest +} + +// VerifyOptions are the consumer's state. +type VerifyOptions struct { + PinnedRoot string + MinKeySetVer uint64 + AppliedSeq uint64 + Now time.Time + AllowExpired bool // the previous release, read while building + SkipRecordChk bool +} + +// VerifyDir checks a bundle the way the platform importer does: key set +// (pinned root, version), latest pointer and manifests (key set, schema, +// sequence, expiry), every file's size and hash, and every record. +func VerifyDir(dir string, opt VerifyOptions) (*Verified, error) { + ksRaw, err := readCapped(filepath.Join(dir, KeySetFile), dsse.MaxKeySetBytes) + if err != nil { + return nil, err + } + ks, err := dsse.VerifyKeySet(ksRaw, opt.PinnedRoot, opt.MinKeySetVer, opt.Now) + if err != nil { + return nil, err + } + v := &Verified{KeySet: ks} + if err := verifyJSON(ks, filepath.Join(dir, LatestFile), LatestPayloadType, &v.Latest); err != nil { + return nil, err + } + l := v.Latest + switch { + case l.Schema != LatestSchema: + return nil, fmt.Errorf("latest: schema %q", l.Schema) + case l.Sequence <= opt.AppliedSeq: + return nil, fmt.Errorf("latest: sequence %d is not after the applied %d (rollback refused)", l.Sequence, opt.AppliedSeq) + case !opt.AllowExpired && !l.ExpiresAt.After(opt.Now): + return nil, fmt.Errorf("latest: expired at %s", l.ExpiresAt.Format(time.RFC3339)) + case l.Tag != Tag(l.Sequence): + return nil, fmt.Errorf("latest: tag %q does not name sequence %d", l.Tag, l.Sequence) + } + if err := v.loadManifest(dir, l.Snapshot, KindSnapshot, &v.Snapshot, opt); err != nil { + return nil, err + } + if l.Delta != "" { + var d Manifest + if err := v.loadManifest(dir, l.Delta, KindDelta, &d, opt); err != nil { + return nil, err + } + if d.BaseSequence != l.BaseSequence || d.BaseSequence >= d.Sequence { + return nil, fmt.Errorf("delta: base sequence %d does not match the pointer", d.BaseSequence) + } + v.Delta = &d + } + return v, nil +} + +func (v *Verified) loadManifest(dir, name, kind string, m *Manifest, opt VerifyOptions) error { + if name != fmt.Sprintf(ManifestFile, kind) { + return fmt.Errorf("%s manifest name %q", kind, name) + } + if err := verifyJSON(v.KeySet, filepath.Join(dir, name), ManifestPayloadType, m); err != nil { + return err + } + switch { + case m.Schema != ManifestSchema: + return fmt.Errorf("%s: schema %q", kind, m.Schema) + case m.Kind != kind: + return fmt.Errorf("%s: kind %q", kind, m.Kind) + case m.Sequence != v.Latest.Sequence: + return fmt.Errorf("%s: sequence %d, the pointer says %d", kind, m.Sequence, v.Latest.Sequence) + case !opt.AllowExpired && !m.ExpiresAt.After(opt.Now): + return fmt.Errorf("%s: expired", kind) + case m.ExpiresAt.Sub(m.CreatedAt) > Validity: + return fmt.Errorf("%s: valid for more than %s", kind, Validity) + case len(m.Files) != len(recordFiles): + return fmt.Errorf("%s: %d files", kind, len(m.Files)) + } + for i, f := range m.Files { + if f.Name != FileName(kind, recordFiles[i]) { + return fmt.Errorf("%s: unexpected file %q", kind, f.Name) + } + if err := checkFile(filepath.Join(dir, f.Name), f); err != nil { + return err + } + } + if opt.SkipRecordChk { + return nil + } + _, err := ReadRecords(dir, *m) + return err +} + +func verifyJSON(ks *dsse.KeySet, path, payloadType string, v any) error { + raw, err := readCapped(path, MaxManifestBytes) + if err != nil { + return err + } + payload, err := dsse.VerifyWith(ks, raw, payloadType, MaxManifestBytes) + if err != nil { + return fmt.Errorf("%s: %w", filepath.Base(path), err) + } + if err := dsse.DecodeStrict(payload, v); err != nil { + return fmt.Errorf("%s: %w", filepath.Base(path), err) + } + return nil +} + +func readCapped(path string, max int64) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, max+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > max { + return nil, fmt.Errorf("%s is over %d bytes", filepath.Base(path), max) + } + return b, nil +} + +func checkFile(path string, f File) error { + if f.Size > MaxFileBytes || f.Size < 0 { + return fmt.Errorf("%s: size %d over the cap", f.Name, f.Size) + } + fh, err := os.Open(path) + if err != nil { + return err + } + defer fh.Close() + h := sha256.New() + n, err := io.Copy(h, io.LimitReader(fh, MaxFileBytes+1)) + if err != nil { + return err + } + if n != f.Size || hex.EncodeToString(h.Sum(nil)) != f.SHA256 { + return fmt.Errorf("%s: size or SHA-256 does not match the manifest", f.Name) + } + return nil +} + +// ReadRecords reads and validates the record files of a verified manifest +// into a snapshot (for a delta: only the changed programs, plus the change +// log). +func ReadRecords(dir string, m Manifest) (*Snapshot, error) { + s := NewSnapshot() + s.Sequence, s.Sources = m.Sequence, m.Sources + counts := map[string]int{} + for _, f := range m.Files { + var kind string + for _, rf := range recordFiles { + if f.Name == FileName(m.Kind, rf) { + kind = rf + } + } + err := eachLine(filepath.Join(dir, f.Name), func(line []byte) error { + counts[kind]++ + switch kind { + case "programs": + var p model.Program + if err := strictLine(line, &p); err != nil { + return err + } + if err := p.Validate(); err != nil { + return err + } + if _, dup := s.Programs[p.ID]; dup { + return fmt.Errorf("program %s listed twice", p.ID) + } + if len(s.Programs) >= model.MaxPrograms { + return fmt.Errorf("over %d programs", model.MaxPrograms) + } + s.Programs[p.ID] = p + case "changes": + var c model.Change + if err := strictLine(line, &c); err != nil { + return err + } + if err := c.Validate(); err != nil { + return err + } + if len(s.Changes) >= 10*model.MaxPrograms { + return errors.New("too many changes") + } + s.Changes = append(s.Changes, c) + default: + return errors.New("unexpected record file") + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("%s: %w", f.Name, err) + } + if counts[kind] != f.Records { + return nil, fmt.Errorf("%s: %d records, the manifest says %d", f.Name, counts[kind], f.Records) + } + } + if err := s.Validate(); err != nil { + return nil, err + } + if m.Kind == KindDelta { + if ids := s.ChangedIDs(); len(ids) != len(s.Programs) { + return nil, errors.New("delta: a program that no change names") + } + } + return s, nil +} + +func strictLine(line []byte, v any) error { return dsse.DecodeStrict(line, v) } + +func eachLine(path string, fn func([]byte) error) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(io.LimitReader(f, MaxFileBytes)) + if err != nil { + return err + } + defer gz.Close() + sc := bufio.NewScanner(&limited{r: gz, left: MaxDecompressedBytes}) + sc.Buffer(make([]byte, 64<<10), MaxRecordBytes) + for sc.Scan() { + if len(sc.Bytes()) == 0 { + continue + } + if err := fn(sc.Bytes()); err != nil { + return err + } + } + return sc.Err() +} + +// limited fails once more than left bytes are read (a decompression bomb). +type limited struct { + r io.Reader + left int64 +} + +func (l *limited) Read(p []byte) (int, error) { + if l.left <= 0 { + return 0, errors.New("decompressed data over the cap") + } + if int64(len(p)) > l.left { + p = p[:l.left] + } + n, err := l.r.Read(p) + l.left -= int64(n) + return n, err +} diff --git a/internal/collect/collect.go b/internal/collect/collect.go new file mode 100644 index 0000000..03c8e00 --- /dev/null +++ b/internal/collect/collect.go @@ -0,0 +1,334 @@ +// Package collect builds the next snapshot: the previous one merged with +// what each source says now, and the change log between them. +package collect + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" + "github.com/openctemio/programfeed/internal/source" +) + +// DropAfter is how long a closed program stays in the feed before it is +// dropped (so subscribers see the closure first). +const DropAfter = 180 * 24 * time.Hour + +// Mass-change guard: a source that suddenly reports fewer than half of the +// programs it had (and had at least massMinimum) is treated as broken for +// this run, and its previous records are kept. +const ( + massShare = 0.5 + massMinimum = 20 +) + +// Options of a build. +type Options struct { + Now time.Time + // AllowMassChange lets a source drop more than the guard allows (a + // person checked the source). + AllowMassChange bool + Log func(format string, args ...any) +} + +// Build returns the next snapshot. prev may be nil (first build). It fails +// only when no source could be read. +func Build(ctx context.Context, prev *bundle.Snapshot, sources []source.Source, client *fetch.Client, opt Options) (*bundle.Snapshot, error) { + if prev == nil { + prev = bundle.NewSnapshot() + } + log := opt.Log + if log == nil { + log = func(string, ...any) {} + } + now := opt.Now.UTC().Truncate(time.Second) + next := bundle.NewSnapshot() + next.Sequence = prev.Sequence + 1 + ok := 0 + var failures []string + seenSource := map[string]bool{} + for _, src := range sources { + meta := src.Meta() + if seenSource[meta.Name] { + return nil, fmt.Errorf("source %q configured twice", meta.Name) + } + seenSource[meta.Name] = true + old := programsOf(prev, meta.Name) + res, err := src.Collect(ctx, client) + if err == nil { + err = guard(res, old, opt.AllowMassChange) + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + log("%s: %v; keeping its %d previous records", meta.Name, err, len(old)) + failures = append(failures, meta.Name+": "+err.Error()) + for _, p := range old { + next.Programs[p.ID] = p + } + if s, had := prev.Source(meta.Name); had { + s.Programs = len(old) + next.Sources = append(next.Sources, s) + } + continue + } + ok++ + n := merge(prev, next, meta.Name, old, res, now, log) + next.Sources = append(next.Sources, bundle.Source{Name: meta.Name, URL: meta.URL, AsOf: now, Terms: meta.Terms, + Licence: meta.Licence, Attribution: meta.Attribution, Programs: n}) + log("%s: %d programs (%d skipped records, %d unreadable kept)", meta.Name, n, res.Skipped, len(res.Unknown)) + } + // Programs of sources no longer configured are kept as they are; a + // source is removed from the feed on purpose, in code review. + for id, p := range prev.Programs { + if !seenSource[p.Source] { + next.Programs[id] = p + } + } + for _, s := range prev.Sources { + if !seenSource[s.Name] { + next.Sources = append(next.Sources, s) + } + } + if ok == 0 { + return nil, fmt.Errorf("no source could be read (%s)", strings.Join(failures, "; ")) + } + sort.Slice(next.Sources, func(i, j int) bool { return next.Sources[i].Name < next.Sources[j].Name }) + sortChanges(next.Changes) + return next, next.Validate() +} + +func programsOf(s *bundle.Snapshot, name string) map[string]model.Program { + out := map[string]model.Program{} + for id, p := range s.Programs { + if p.Source == name { + out[id] = p + } + } + return out +} + +func guard(res *source.Result, old map[string]model.Program, allow bool) error { + open := 0 + for _, p := range old { + if p.Status != model.StatusClosed { + open++ + } + } + got := len(res.Programs) + len(res.Unknown) + if !allow && open >= massMinimum && float64(got) < float64(open)*massShare { + return fmt.Errorf("refusing: %d programs now, %d before (rerun with --allow-mass-change after checking the source)", got, open) + } + return nil +} + +// merge folds one source's result into next and returns how many programs +// of the source next holds. +func merge(prev, next *bundle.Snapshot, name string, old map[string]model.Program, res *source.Result, now time.Time, log func(string, ...any)) int { + seq := next.Sequence + add := func(c model.Change) { + c.Sequence = seq + next.Changes = append(next.Changes, c) + } + seen := map[string]bool{} + for _, id := range res.Unknown { + if p, had := old[id]; had { + next.Programs[id] = p + seen[id] = true + } + } + for _, p := range res.Programs { + if seen[p.ID] { + continue + } + if p.Source != name { + log("%s: program %s claims source %q; skipped", name, p.ID, p.Source) + continue + } + model.Normalize(&p) + was, had := old[p.ID] + p.LastSeen = now + if had { + p.FirstSeen = was.FirstSeen + changes := diff(was, p) + if len(changes) > 0 { + p.LastChanged = now + } else { + p.LastChanged = was.LastChanged + } + if err := p.Validate(); err != nil { + log("%s: %v; keeping the previous record", name, err) + next.Programs[p.ID] = was + seen[p.ID] = true + continue + } + for _, c := range changes { + add(c) + } + } else { + p.FirstSeen, p.LastChanged = now, now + if err := p.Validate(); err != nil { + log("%s: %v; skipped", name, err) + continue + } + add(model.Change{Program: p.ID, Kind: model.ChangeProgramAdded}) + } + next.Programs[p.ID] = p + seen[p.ID] = true + } + ids := make([]string, 0, len(old)) + for id := range old { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + if seen[id] { + continue + } + p := old[id] + switch { + case p.Status != model.StatusClosed: + at := now + p.Status, p.ClosedAt, p.LastChanged = model.StatusClosed, &at, now + next.Programs[id] = p + add(model.Change{Program: id, Kind: model.ChangeProgramClosed}) + case now.Sub(*p.ClosedAt) > DropAfter: + add(model.Change{Program: id, Kind: model.ChangeProgramDropped}) + default: + next.Programs[id] = p + } + seen[id] = true + } + n := 0 + for _, p := range next.Programs { + if p.Source == name { + n++ + } + } + return n +} + +// diff lists what changed between two records of one program. +func diff(a, b model.Program) []model.Change { + var out []model.Change + id := b.ID + switch { + case a.Status == model.StatusClosed && b.Status != model.StatusClosed: + out = append(out, model.Change{Program: id, Kind: model.ChangeProgramReopened}) + case a.Status != b.Status: + out = append(out, model.Change{Program: id, Kind: model.ChangeStatus, Fields: []string{"status"}}) + } + out = append(out, diffTargets(id, "in", a.InScope, b.InScope)...) + out = append(out, diffTargets(id, "out", a.OutOfScope, b.OutOfScope)...) + if !reflect.DeepEqual(a.Rules, b.Rules) { + out = append(out, model.Change{Program: id, Kind: model.ChangeRules, Fields: diffFields(a.Rules, b.Rules)}) + } + if a.Terms.URL != b.Terms.URL || a.Terms.SHA256 != b.Terms.SHA256 { + out = append(out, model.Change{Program: id, Kind: model.ChangeTerms}) + } + var fields []string + for name, eq := range map[string]bool{ + "name": a.Name == b.Name, "url": a.URL == b.URL, "platform": a.Platform == b.Platform, "type": a.Type == b.Type, + "scope_published": a.ScopePublished == b.ScopePublished, "contact": a.Contact == b.Contact, + "rejected": reflect.DeepEqual(a.Rejected, b.Rejected), + } { + if !eq { + fields = append(fields, name) + } + } + if len(fields) > 0 { + sort.Strings(fields) + out = append(out, model.Change{Program: id, Kind: model.ChangeProgramModified, Fields: fields}) + } + return out +} + +func diffTargets(id, scope string, a, b []model.Target) []model.Change { + key := func(t model.Target) string { return t.Type + "\x00" + t.Value } + before := map[string]model.Target{} + for _, t := range a { + before[key(t)] = t + } + var out []model.Change + after := map[string]bool{} + for _, t := range b { + t := t + after[key(t)] = true + was, had := before[key(t)] + switch { + case !had: + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetAdded, Scope: scope, Target: &t}) + case !reflect.DeepEqual(was, t): + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetModified, Scope: scope, Target: &t, Fields: diffFields(was, t)}) + } + } + for _, t := range a { + t := t + if !after[key(t)] { + out = append(out, model.Change{Program: id, Kind: model.ChangeTargetRemoved, Scope: scope, Target: &t}) + } + } + return out +} + +// diffFields names the JSON fields that differ between two structs. +func diffFields(a, b any) []string { + var ma, mb map[string]json.RawMessage + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + _ = json.Unmarshal(ja, &ma) + _ = json.Unmarshal(jb, &mb) + var out []string + for k, v := range ma { + if string(mb[k]) != string(v) { + out = append(out, k) + } + } + for k := range mb { + if _, ok := ma[k]; !ok { + out = append(out, k) + } + } + sort.Strings(out) + if len(out) > model.MaxChangeFields { + out = out[:model.MaxChangeFields] + } + return out +} + +var kindOrder = map[string]int{ + model.ChangeProgramAdded: 0, model.ChangeProgramReopened: 1, model.ChangeStatus: 2, model.ChangeProgramModified: 3, + model.ChangeRules: 4, model.ChangeTerms: 5, model.ChangeTargetAdded: 6, model.ChangeTargetModified: 7, + model.ChangeTargetRemoved: 8, model.ChangeProgramClosed: 9, model.ChangeProgramDropped: 10, +} + +func sortChanges(cs []model.Change) { + sort.SliceStable(cs, func(i, j int) bool { + a, b := cs[i], cs[j] + if a.Program != b.Program { + return a.Program < b.Program + } + if kindOrder[a.Kind] != kindOrder[b.Kind] { + return kindOrder[a.Kind] < kindOrder[b.Kind] + } + if a.Scope != b.Scope { + return a.Scope < b.Scope + } + var ta, tb string + if a.Target != nil { + ta = a.Target.Type + "\x00" + a.Target.Value + } + if b.Target != nil { + tb = b.Target.Type + "\x00" + b.Target.Value + } + return ta < tb + }) +} diff --git a/internal/collect/collect_test.go b/internal/collect/collect_test.go new file mode 100644 index 0000000..c92fc4c --- /dev/null +++ b/internal/collect/collect_test.go @@ -0,0 +1,196 @@ +package collect + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" + "github.com/openctemio/programfeed/internal/source" +) + +type fake struct { + name string + res *source.Result + err error +} + +func (f fake) Meta() source.Meta { + return source.Meta{Name: f.name, URL: "https://openctem.io/" + f.name, Terms: "https://openctem.io/terms", Licence: "CC0-1.0"} +} + +func (f fake) Collect(context.Context, *fetch.Client) (*source.Result, error) { + if f.err != nil { + return nil, f.err + } + // Return copies: the collector normalises in place. + out := *f.res + out.Programs = append([]model.Program(nil), f.res.Programs...) + for i := range out.Programs { + p := &out.Programs[i] + p.InScope = append([]model.Target(nil), p.InScope...) + } + return &out, nil +} + +var day0 = time.Date(2026, 10, 1, 2, 0, 0, 0, time.UTC) + +func prog(src, slug string, targets ...string) model.Program { + p := model.Program{ + ID: src + ":" + slug, Source: src, Platform: "self-hosted", Name: slug, URL: "https://" + slug + ".org/security", + Type: model.TypeVDP, Status: model.StatusOpen, InScope: []model.Target{}, OutOfScope: []model.Target{}, + Terms: model.Terms{URL: "https://" + slug + ".org/security"}, + Provenance: model.Provenance{Source: src, SourceURL: "https://openctem.io/" + src, FetchedAt: day0}, + } + for _, t := range targets { + typ := model.TargetDomain + if strings.HasPrefix(t, "*.") { + typ = model.TargetWildcard + } + p.InScope = append(p.InScope, model.Target{Type: typ, Value: t}) + } + return p +} + +func run(t *testing.T, prev *bundle.Snapshot, at time.Time, allowMass bool, srcs ...source.Source) *bundle.Snapshot { + t.Helper() + next, err := Build(context.Background(), prev, srcs, nil, Options{Now: at, AllowMassChange: allowMass}) + if err != nil { + t.Fatal(err) + } + return next +} + +func kinds(s *bundle.Snapshot) string { + var out []string + for _, c := range s.Changes { + k := c.Program + " " + c.Kind + if c.Target != nil { + k += " " + c.Scope + ":" + c.Target.Value + } + out = append(out, k) + } + return strings.Join(out, "\n") +} + +func TestBuildLifecycle(t *testing.T) { + s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com"), + prog("d", "beta", "beta.org"), + prog("d", "acme", "acme.org"), // duplicate id: first wins + }}}) + if s1.Sequence != 1 || len(s1.Programs) != 2 || kinds(s1) != "d:acme program_added\nd:beta program_added" { + t.Fatalf("first build:\n%s", kinds(s1)) + } + acme := s1.Programs["d:acme"] + if len(acme.InScope) != 1 || len(acme.Rejected) != 1 || !acme.FirstSeen.Equal(day0) { + t.Fatalf("acme: %+v", acme) + } + + // Day 1: acme gains a target and loses none, beta disappears. + day1 := day0.Add(24 * time.Hour) + s2 := run(t, s1, day1, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com", "api.acme.org"), + }}}) + want := "d:acme target_added in:api.acme.org\nd:beta program_closed" + if s2.Sequence != 2 || kinds(s2) != want { + t.Fatalf("day 1:\n%s\nwant\n%s", kinds(s2), want) + } + if b := s2.Programs["d:beta"]; b.Status != model.StatusClosed || b.ClosedAt == nil || !b.LastSeen.Equal(day0) { + t.Errorf("beta: %+v", b) + } + if a := s2.Programs["d:acme"]; !a.FirstSeen.Equal(day0) || !a.LastChanged.Equal(day1) || !a.LastSeen.Equal(day1) { + t.Errorf("acme timestamps: %+v", a) + } + + // Day 2: nothing changes: no change entries, last_changed stays. + day2 := day1.Add(24 * time.Hour) + s3 := run(t, s2, day2, false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org", "*.com", "api.acme.org"), + }}}) + if len(s3.Changes) != 0 || !s3.Programs["d:acme"].LastChanged.Equal(day1) { + t.Fatalf("day 2:\n%s", kinds(s3)) + } + + // Beta comes back with a different target set; acme loses a target. + s4 := run(t, s3, day2.Add(time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{ + prog("d", "acme", "acme.org"), + prog("d", "beta", "beta.org", "www.beta.org"), + }}}) + want = "d:acme program_modified\nd:acme target_removed in:api.acme.org\nd:beta program_reopened\nd:beta target_added in:www.beta.org" + if kinds(s4) != want { + t.Fatalf("reopen:\n%s\nwant\n%s", kinds(s4), want) + } + + // A program closed for longer than DropAfter is dropped. + s5 := run(t, s4, day2.Add(2*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}}) + s6 := run(t, s5, day2.Add(DropAfter+3*time.Hour), false, fake{name: "d", res: &source.Result{Programs: []model.Program{prog("d", "acme", "acme.org")}}}) + if _, still := s6.Programs["d:beta"]; still || kinds(s6) != "d:beta program_dropped" { + t.Fatalf("drop:\n%s", kinds(s6)) + } +} + +func TestBuildKeepsRecordsOfFailingSources(t *testing.T) { + s1 := run(t, nil, day0, false, + fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "one", "one.org"), prog("a", "two", "two.org")}}}, + fake{name: "b", res: &source.Result{Programs: []model.Program{prog("b", "three", "three.org")}}}) + day1 := day0.Add(24 * time.Hour) + // Source a is down, source b reports "two"'s host unknown... and b's + // program unknown: both keep their records unchanged. + s2 := run(t, s1, day1, false, + fake{name: "a", err: errors.New("HTTP 503")}, + fake{name: "b", res: &source.Result{Unknown: []string{"b:three"}}}) + if len(s2.Programs) != 3 || len(s2.Changes) != 0 { + t.Fatalf("kept %d, changes:\n%s", len(s2.Programs), kinds(s2)) + } + for _, p := range s2.Programs { + if p.Status != model.StatusOpen || !p.LastSeen.Equal(day0) { + t.Errorf("%s: %+v", p.ID, p) + } + } + if src, _ := s2.Source("a"); !src.AsOf.Equal(day0) { + t.Errorf("a failing source keeps its old as_of: %+v", src) + } + // Every source failing fails the build. + if _, err := Build(context.Background(), s1, []source.Source{fake{name: "a", err: errors.New("down")}}, nil, Options{Now: day1}); err == nil { + t.Error("a build with no readable source succeeded") + } +} + +func TestMassChangeGuard(t *testing.T) { + var many []model.Program + for i := 0; i < 40; i++ { + many = append(many, prog("d", fmt.Sprintf("p%02d", i), fmt.Sprintf("p%02d.org", i))) + } + s1 := run(t, nil, day0, false, fake{name: "d", res: &source.Result{Programs: many}}) + few := &source.Result{Programs: many[:5]} + other := fake{name: "o", res: &source.Result{Programs: []model.Program{prog("o", "x", "x.org")}}} + // With another readable source, the guarded source keeps its records. + s2 := run(t, s1, day0.Add(time.Hour), false, fake{name: "d", res: few}, other) + if len(s2.Programs) != 41 || s2.Stats().Closed != 0 { + t.Fatalf("guard did not hold: %+v", s2.Stats()) + } + // Alone, it fails the build and says why. + if _, err := Build(context.Background(), s1, []source.Source{fake{name: "d", res: few}}, nil, Options{Now: day0.Add(time.Hour)}); err == nil || !strings.Contains(err.Error(), "allow-mass-change") { + t.Fatalf("guard alone: %v", err) + } + s3 := run(t, s1, day0.Add(time.Hour), true, fake{name: "d", res: few}) + if s3.Stats().Closed != 35 { + t.Fatalf("override: %+v", s3.Stats()) + } +} + +func TestSourceCannotWriteAnotherSourcesPrograms(t *testing.T) { + evil := prog("a", "victim", "victim.org") // claims source a, served by b + s := run(t, nil, day0, false, + fake{name: "a", res: &source.Result{Programs: []model.Program{prog("a", "real", "real.org")}}}, + fake{name: "b", res: &source.Result{Programs: []model.Program{evil, prog("b", "ok", "ok.org")}}}) + if _, ok := s.Programs["a:victim"]; ok || len(s.Programs) != 2 { + t.Fatalf("programs: %v", s.SortedIDs()) + } +} diff --git a/internal/dsse/dsse.go b/internal/dsse/dsse.go new file mode 100644 index 0000000..ac32416 --- /dev/null +++ b/internal/dsse/dsse.go @@ -0,0 +1,251 @@ +// Package dsse signs and verifies DSSE v1 envelopes with Ed25519, and the +// key sets that name the keys allowed to sign bundles. +// +// The envelope, pre-authentication encoding and key ids are the ones the +// OpenCTEM platform already verifies for signed jobs: JSON +// {"payloadType", "payload" (base64), "signatures": [{"keyid", "sig" (base64)}]}, +// PAE "DSSEv1 ", key id +// "SHA256:" + hex(sha256(raw public key)). +package dsse + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "time" +) + +// Envelope is a DSSE v1 envelope. +type Envelope struct { + PayloadType string `json:"payloadType"` + Payload []byte `json:"payload"` + Signatures []Signature `json:"signatures"` +} + +// Signature is one signature of an envelope. +type Signature struct { + KeyID string `json:"keyid"` + Sig []byte `json:"sig"` +} + +// PAE is the DSSE v1 pre-authentication encoding. +func PAE(payloadType string, payload []byte) []byte { + var b bytes.Buffer + b.WriteString("DSSEv1 ") + b.WriteString(strconv.Itoa(len(payloadType))) + b.WriteByte(' ') + b.WriteString(payloadType) + b.WriteByte(' ') + b.WriteString(strconv.Itoa(len(payload))) + b.WriteByte(' ') + b.Write(payload) + return b.Bytes() +} + +// KeyID names a public key. +func KeyID(pub ed25519.PublicKey) string { + sum := sha256.Sum256(pub) + return "SHA256:" + hex.EncodeToString(sum[:]) +} + +// Sign wraps payload in an envelope signed by priv. +func Sign(priv ed25519.PrivateKey, payloadType string, payload []byte) ([]byte, error) { + if len(priv) != ed25519.PrivateKeySize { + return nil, errors.New("dsse: an Ed25519 private key is required") + } + pub, _ := priv.Public().(ed25519.PublicKey) + return json.Marshal(Envelope{ + PayloadType: payloadType, + Payload: payload, + Signatures: []Signature{{KeyID: KeyID(pub), Sig: ed25519.Sign(priv, PAE(payloadType, payload))}}, + }) +} + +// Open parses an envelope of the expected type (at most maxBytes) without +// checking signatures. +func Open(envelope []byte, payloadType string, maxBytes int) (*Envelope, error) { + if len(envelope) == 0 || len(envelope) > maxBytes { + return nil, fmt.Errorf("dsse: envelope is empty or over %d bytes", maxBytes) + } + var env Envelope + if err := json.Unmarshal(envelope, &env); err != nil { + return nil, fmt.Errorf("dsse: envelope: %w", err) + } + if env.PayloadType != payloadType { + return nil, fmt.Errorf("dsse: payload type %q, want %q", env.PayloadType, payloadType) + } + return &env, nil +} + +// VerifiedBy reports whether env carries a valid signature of pub. +func (env *Envelope) VerifiedBy(pub ed25519.PublicKey) bool { + id := KeyID(pub) + pae := PAE(env.PayloadType, env.Payload) + for _, s := range env.Signatures { + if s.KeyID == id && ed25519.Verify(pub, pae, s.Sig) { + return true + } + } + return false +} + +// DecodeStrict decodes JSON into v, refusing unknown fields and trailing data. +func DecodeStrict(data []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return errors.New("trailing data") + } + return nil +} + +// Key set (offline root → online keys). +const ( + KeySetPayloadType = "application/vnd.openctem.programfeed.keyset+json" + KeySetKind = "openctem.programfeed.keyset/v1" + MaxKeySetValidity = 180 * 24 * time.Hour + MaxKeySetBytes = 64 << 10 + MaxKeySetKeys = 8 + maxClockSkew = 5 * time.Minute +) + +// PublicKey describes one online key. +type PublicKey struct { + KeyID string `json:"keyid"` + Algorithm string `json:"algorithm"` + PublicKey string `json:"public_key"` +} + +// NewPublicKey describes pub. +func NewPublicKey(pub ed25519.PublicKey) PublicKey { + return PublicKey{KeyID: KeyID(pub), Algorithm: "ed25519", PublicKey: base64.StdEncoding.EncodeToString(pub)} +} + +// Decode returns the raw key after checking the id matches. +func (k PublicKey) Decode() (ed25519.PublicKey, error) { + if k.Algorithm != "ed25519" { + return nil, fmt.Errorf("key %s: unsupported algorithm %q", k.KeyID, k.Algorithm) + } + raw, err := base64.StdEncoding.DecodeString(k.PublicKey) + if err != nil || len(raw) != ed25519.PublicKeySize || KeyID(raw) != k.KeyID { + return nil, fmt.Errorf("key %s: not the 32-byte Ed25519 key of its id", k.KeyID) + } + return raw, nil +} + +// KeySet is the list of online keys an offline root allows to sign. +type KeySet struct { + Kind string `json:"kind"` + Version uint64 `json:"version"` + IssuedAt time.Time `json:"issued_at"` + NotAfter time.Time `json:"not_after"` + Keys []PublicKey `json:"keys"` + RootKeyID string `json:"root_keyid"` + RootPublicKey string `json:"root_public_key"` +} + +func (ks *KeySet) check() error { + switch { + case ks.Kind != KeySetKind: + return fmt.Errorf("key set: kind %q", ks.Kind) + case ks.Version == 0: + return errors.New("key set: version must be 1 or more") + case ks.IssuedAt.IsZero() || !ks.NotAfter.After(ks.IssuedAt) || ks.NotAfter.Sub(ks.IssuedAt) > MaxKeySetValidity: + return fmt.Errorf("key set: validity must be positive and at most %s", MaxKeySetValidity) + case len(ks.Keys) == 0 || len(ks.Keys) > MaxKeySetKeys: + return fmt.Errorf("key set: 1 to %d keys", MaxKeySetKeys) + } + root, err := base64.StdEncoding.DecodeString(ks.RootPublicKey) + if err != nil || len(root) != ed25519.PublicKeySize || KeyID(root) != ks.RootKeyID { + return errors.New("key set: root_public_key is not the key of root_keyid") + } + seen := map[string]bool{} + for _, k := range ks.Keys { + if _, err := k.Decode(); err != nil { + return fmt.Errorf("key set: %w", err) + } + if seen[k.KeyID] || k.KeyID == ks.RootKeyID { + return fmt.Errorf("key set: key %s listed twice or is the root", k.KeyID) + } + seen[k.KeyID] = true + } + return nil +} + +// SignKeySet signs a key set with the offline root. +func SignKeySet(root ed25519.PrivateKey, version uint64, issuedAt time.Time, validity time.Duration, keys []ed25519.PublicKey) ([]byte, error) { + if len(root) != ed25519.PrivateKeySize { + return nil, errors.New("key set: an Ed25519 root key is required") + } + rootPub, _ := root.Public().(ed25519.PublicKey) + issuedAt = issuedAt.UTC().Truncate(time.Second) + ks := KeySet{Kind: KeySetKind, Version: version, IssuedAt: issuedAt, NotAfter: issuedAt.Add(validity), + RootKeyID: KeyID(rootPub), RootPublicKey: base64.StdEncoding.EncodeToString(rootPub)} + for _, k := range keys { + ks.Keys = append(ks.Keys, NewPublicKey(k)) + } + if err := ks.check(); err != nil { + return nil, err + } + payload, err := json.Marshal(ks) + if err != nil { + return nil, err + } + return Sign(root, KeySetPayloadType, payload) +} + +// VerifyKeySet checks a key set: signed by its root, that root is the pinned +// one, valid now, and its version is not below minVersion. +func VerifyKeySet(envelope []byte, pinnedRoot string, minVersion uint64, now time.Time) (*KeySet, error) { + env, err := Open(envelope, KeySetPayloadType, MaxKeySetBytes) + if err != nil { + return nil, err + } + var ks KeySet + if err := DecodeStrict(env.Payload, &ks); err != nil { + return nil, fmt.Errorf("key set: %w", err) + } + if err := ks.check(); err != nil { + return nil, err + } + if pinnedRoot == "" || ks.RootKeyID != pinnedRoot { + return nil, fmt.Errorf("key set: signed by root %s, the pinned root is %q", ks.RootKeyID, pinnedRoot) + } + root, _ := base64.StdEncoding.DecodeString(ks.RootPublicKey) + if !env.VerifiedBy(root) { + return nil, errors.New("key set: bad root signature") + } + if ks.Version < minVersion { + return nil, fmt.Errorf("key set: version %d is below %d (rolled back)", ks.Version, minVersion) + } + if ks.IssuedAt.After(now.Add(maxClockSkew)) || !ks.NotAfter.After(now.Add(-maxClockSkew)) { + return nil, fmt.Errorf("key set: not valid at %s (valid %s to %s)", now.UTC().Format(time.RFC3339), + ks.IssuedAt.Format(time.RFC3339), ks.NotAfter.Format(time.RFC3339)) + } + return &ks, nil +} + +// VerifyWith checks envelope against the key set and returns its payload. +func VerifyWith(ks *KeySet, envelope []byte, payloadType string, maxBytes int) ([]byte, error) { + env, err := Open(envelope, payloadType, maxBytes) + if err != nil { + return nil, err + } + for _, k := range ks.Keys { + pub, err := k.Decode() + if err == nil && env.VerifiedBy(pub) { + return env.Payload, nil + } + } + return nil, errors.New("dsse: not signed by a key of the key set") +} diff --git a/internal/dsse/dsse_test.go b/internal/dsse/dsse_test.go new file mode 100644 index 0000000..7df7c93 --- /dev/null +++ b/internal/dsse/dsse_test.go @@ -0,0 +1,124 @@ +package dsse + +import ( + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" +) + +func key(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return pub, priv +} + +func TestPAEAndEnvelopeFormat(t *testing.T) { + if got := string(PAE("t/x", []byte("hello"))); got != "DSSEv1 3 t/x 5 hello" { + t.Fatalf("PAE = %q", got) + } + _, priv := key(t) + env, err := Sign(priv, "t/x", []byte(`{"a":1}`)) + if err != nil { + t.Fatal(err) + } + var raw map[string]any + if err := json.Unmarshal(env, &raw); err != nil { + t.Fatal(err) + } + sigs, _ := raw["signatures"].([]any) + sig, _ := sigs[0].(map[string]any) + if raw["payloadType"] != "t/x" || raw["payload"] != "eyJhIjoxfQ==" || !strings.HasPrefix(sig["keyid"].(string), "SHA256:") || sig["sig"] == nil { + t.Fatalf("envelope %s", env) + } +} + +func TestKeySetAndVerify(t *testing.T) { + rootPub, root := key(t) + pub, priv := key(t) + now := time.Now() + ksEnv, err := SignKeySet(root, 3, now, 90*24*time.Hour, []ed25519.PublicKey{pub}) + if err != nil { + t.Fatal(err) + } + ks, err := VerifyKeySet(ksEnv, KeyID(rootPub), 2, now) + if err != nil { + t.Fatal(err) + } + env, _ := Sign(priv, "t/x", []byte("payload")) + if got, err := VerifyWith(ks, env, "t/x", 1<<10); err != nil || string(got) != "payload" { + t.Fatalf("VerifyWith: %q %v", got, err) + } + + otherPub, other := key(t) + _ = otherPub + forged, _ := Sign(other, "t/x", []byte("payload")) + cases := map[string]func() error{ + "another root pinned": func() error { _, err := VerifyKeySet(ksEnv, KeyID(otherPub), 0, now); return err }, + "no root pinned": func() error { _, err := VerifyKeySet(ksEnv, "", 0, now); return err }, + "rolled back": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 4, now); return err }, + "expired": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(91*24*time.Hour)); return err }, + "not yet valid": func() error { _, err := VerifyKeySet(ksEnv, KeyID(rootPub), 0, now.Add(-time.Hour)); return err }, + "unknown signer": func() error { _, err := VerifyWith(ks, forged, "t/x", 1<<10); return err }, + "wrong type": func() error { _, err := VerifyWith(ks, env, "t/y", 1<<10); return err }, + "too big": func() error { _, err := VerifyWith(ks, env, "t/x", 10); return err }, + "tampered payload": func() error { + var e Envelope + _ = json.Unmarshal(env, &e) + e.Payload = []byte("payloae") + b, _ := json.Marshal(e) + _, err := VerifyWith(ks, b, "t/x", 1<<10) + return err + }, + "key set signed by an online key": func() error { + bad, _ := SignKeySet(priv, 1, now, time.Hour, []ed25519.PublicKey{otherPub}) + _, err := VerifyKeySet(bad, KeyID(rootPub), 0, now) + return err + }, + } + for name, fn := range cases { + if fn() == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestSignKeySetRefuses(t *testing.T) { + rootPub, root := key(t) + pub, _ := key(t) + now := time.Now() + for name, fn := range map[string]func() error{ + "too long": func() error { + _, err := SignKeySet(root, 1, now, 181*24*time.Hour, []ed25519.PublicKey{pub}) + return err + }, + "version 0": func() error { _, err := SignKeySet(root, 0, now, time.Hour, []ed25519.PublicKey{pub}); return err }, + "no keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, nil); return err }, + "root as online": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{rootPub}); return err }, + "duplicate keys": func() error { _, err := SignKeySet(root, 1, now, time.Hour, []ed25519.PublicKey{pub, pub}); return err }, + "not a root key": func() error { + _, err := SignKeySet(ed25519.PrivateKey{1}, 1, now, time.Hour, []ed25519.PublicKey{pub}) + return err + }, + } { + if fn() == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestDecodeStrict(t *testing.T) { + var v struct{ A int } + if DecodeStrict([]byte(`{"A":1}`), &v) != nil || v.A != 1 { + t.Fatal("valid") + } + for _, s := range []string{`{"A":1,"B":2}`, `{"A":1} {}`, `[`} { + if DecodeStrict([]byte(s), &v) == nil { + t.Errorf("%s accepted", s) + } + } +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go new file mode 100644 index 0000000..bdede14 --- /dev/null +++ b/internal/fetch/fetch.go @@ -0,0 +1,295 @@ +// Package fetch is the collector's only way to the network: a polite HTTP +// client that names itself and its contact, honours robots.txt, spaces +// requests to one host, makes conditional requests from an on-disk cache, +// caps response sizes and refuses to connect to non-public addresses (a +// seed list or a redirect must not point the collector at internal +// services). +package fetch + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "os" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/openctemio/programfeed/internal/model" +) + +// Product is the robots.txt product token of the collector. +const Product = "openctem-programfeed" + +// UserAgent names the collector, where it lives and who to contact. +func UserAgent(version string) string { + return Product + "/" + version + " (+https://github.com/openctemio/programfeed; security@openctem.io)" +} + +// Errors callers act on. +var ( + ErrDisallowed = errors.New("robots.txt disallows this URL") + ErrNotFound = errors.New("not found") + ErrTooLarge = errors.New("response over the size cap") +) + +const maxRobotsBytes = 512 << 10 + +// Options configure a client. +type Options struct { + UserAgent string + CacheDir string // conditional requests; empty disables the cache + MinInterval time.Duration // between two requests to one host + Timeout time.Duration // per request + // AllowPrivate lets tests talk to httptest servers on loopback. Never + // set in the collector. + AllowPrivate bool + // DialOverride sends every connection to this address (tests only, + // with AllowPrivate). + DialOverride string +} + +// Client is safe for concurrent use. +type Client struct { + opt Options + http *http.Client + + mu sync.Mutex + next map[string]time.Time // host -> earliest next request + robots map[string]*robotsRules +} + +// Response is a fetched body. +type Response struct { + URL string // final URL after redirects + Body []byte + FetchedAt time.Time + Cached bool // 304: the body is the cached one +} + +// New returns a client. +func New(opt Options) *Client { + if opt.MinInterval == 0 { + opt.MinInterval = time.Second + } + if opt.Timeout == 0 { + opt.Timeout = 30 * time.Second + } + c := &Client{opt: opt, next: map[string]time.Time{}, robots: map[string]*robotsRules{}} + dialer := &net.Dialer{Timeout: 10 * time.Second, Control: func(_, address string, _ syscall.RawConn) error { + if opt.AllowPrivate { + return nil + } + ap, err := netip.ParseAddrPort(address) + if err != nil || !model.PublicAddr(ap.Addr()) { + return fmt.Errorf("refusing to connect to non-public address %s", address) + } + return nil + }} + dial := dialer.DialContext + if opt.DialOverride != "" && opt.AllowPrivate { + dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, opt.DialOverride) + } + } + transport := &http.Transport{ + Proxy: nil, + DialContext: dial, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: opt.Timeout, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 30 * time.Second, + } + c.http = &http.Client{Transport: transport, Timeout: opt.Timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 3 { + return errors.New("more than 3 redirects") + } + if req.URL.Scheme != "https" && !(opt.AllowPrivate && req.URL.Scheme == "http") { + return errors.New("redirect to a non-https URL") + } + // A robots.txt fetch is not itself subject to robots.txt (a + // site that redirects everything would otherwise recurse). + if req.Context().Value(robotsFetch{}) == nil { + if ok, err := c.allowed(req.Context(), req.URL); err != nil || !ok { + return ErrDisallowed + } + } + return c.wait(req.Context(), req.URL.Host) + }} + return c +} + +// Get fetches rawURL (at most maxBytes) after checking robots.txt. +func (c *Client) Get(ctx context.Context, rawURL string, maxBytes int64) (*Response, error) { + u, err := url.Parse(rawURL) + if err != nil || (u.Scheme != "https" && !(c.opt.AllowPrivate && u.Scheme == "http")) || u.Host == "" || u.User != nil { + return nil, fmt.Errorf("fetch: refusing URL %q", rawURL) + } + ok, err := c.allowed(ctx, u) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("%s: %w", rawURL, ErrDisallowed) + } + return c.get(ctx, u, maxBytes, true) +} + +type cacheEntry struct { + URL string `json:"url"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` + Body []byte `json:"body"` +} + +func (c *Client) cachePath(u string) string { + if c.opt.CacheDir == "" { + return "" + } + sum := sha256.Sum256([]byte(u)) + return filepath.Join(c.opt.CacheDir, hex.EncodeToString(sum[:])+".json") +} + +func (c *Client) get(ctx context.Context, u *url.URL, maxBytes int64, useCache bool) (*Response, error) { + if err := c.wait(ctx, u.Host); err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.opt.UserAgent) + req.Header.Set("From", "security@openctem.io") + var cached *cacheEntry + path := "" + if useCache { + path = c.cachePath(u.String()) + } + if path != "" { + if b, err := os.ReadFile(path); err == nil { + var e cacheEntry + if json.Unmarshal(b, &e) == nil && e.URL == u.String() && int64(len(e.Body)) <= maxBytes { + cached = &e + if e.ETag != "" { + req.Header.Set("If-None-Match", e.ETag) + } + if e.LastModified != "" { + req.Header.Set("If-Modified-Since", e.LastModified) + } + } + } + } + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + now := time.Now().UTC().Truncate(time.Second) + switch { + case resp.StatusCode == http.StatusNotModified && cached != nil: + return &Response{URL: u.String(), Body: cached.Body, FetchedAt: now, Cached: true}, nil + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return nil, fmt.Errorf("%s: %w", u, ErrNotFound) + case resp.StatusCode != http.StatusOK: + return nil, &StatusError{URL: u.String(), Code: resp.StatusCode} + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > maxBytes { + return nil, fmt.Errorf("%s: %w (%d bytes)", u, ErrTooLarge, maxBytes) + } + if path != "" && (resp.Header.Get("ETag") != "" || resp.Header.Get("Last-Modified") != "") { + e := cacheEntry{URL: u.String(), ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), Body: body} + if b, err := json.Marshal(e); err == nil { + _ = os.MkdirAll(c.opt.CacheDir, 0o755) + _ = os.WriteFile(path, b, 0o644) + } + } + return &Response{URL: resp.Request.URL.String(), Body: body, FetchedAt: now}, nil +} + +// wait blocks until a request to host is polite. +func (c *Client) wait(ctx context.Context, host string) error { + c.mu.Lock() + now := time.Now() + at := c.next[host] + if at.Before(now) { + at = now + } + c.next[host] = at.Add(c.opt.MinInterval) + c.mu.Unlock() + if d := time.Until(at); d > 0 { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + } + return nil +} + +// allowed checks robots.txt of u's origin (fetched once per run). A missing +// robots.txt (4xx) allows everything; an unreachable one (network error, +// 5xx) disallows everything, as RFC 9309 asks. +func (c *Client) allowed(ctx context.Context, u *url.URL) (bool, error) { + origin := u.Scheme + "://" + u.Host + c.mu.Lock() + r, ok := c.robots[origin] + c.mu.Unlock() + if !ok { + ru, _ := url.Parse(origin + "/robots.txt") + resp, err := c.get(context.WithValue(ctx, robotsFetch{}, true), ru, maxRobotsBytes, false) + switch { + case err == nil: + r = parseRobots(string(resp.Body), Product) + case errors.Is(err, ErrNotFound) || isStatus(err, 400, 499): + r = &robotsRules{} + case errors.Is(err, ErrTooLarge): + r = &robotsRules{} + case ctx.Err() != nil: + return false, ctx.Err() + default: + r = &robotsRules{disallowAll: true} + } + c.mu.Lock() + c.robots[origin] = r + c.mu.Unlock() + } + path := u.EscapedPath() + if path == "" { + path = "/" + } + if u.RawQuery != "" { + path += "?" + u.RawQuery + } + return r.allows(path), nil +} + +type robotsFetch struct{} + +// StatusError is an unexpected HTTP status. +type StatusError struct { + URL string + Code int +} + +func (e *StatusError) Error() string { return fmt.Sprintf("%s: HTTP %d", e.URL, e.Code) } + +func isStatus(err error, lo, hi int) bool { + var se *StatusError + return errors.As(err, &se) && se.Code >= lo && se.Code <= hi +} diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go new file mode 100644 index 0000000..e575f47 --- /dev/null +++ b/internal/fetch/fetch_test.go @@ -0,0 +1,154 @@ +package fetch + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestRobots(t *testing.T) { + body := ` +User-agent: * +Disallow: /private +Allow: /private/ok + +User-agent: openctem-programfeed +User-agent: other +Disallow: /mine/*.json$ +Allow: / + +User-agent: badbot +Disallow: / +` + r := parseRobots(body, Product) + cases := map[string]bool{ + "/": true, + "/private": true, // our group applies, not "*" + "/mine/a.json": false, + "/mine/a.json?x": true, + "/mine/dir/b.txt": true, + } + for p, want := range cases { + if got := r.allows(p); got != want { + t.Errorf("%s: %v, want %v", p, got, want) + } + } + star := parseRobots(body, "someone-else") + if star.allows("/private/x") || !star.allows("/private/ok/1") || !star.allows("/public") { + t.Error("star group rules") + } + if !parseRobots("", Product).allows("/anything") { + t.Error("empty robots.txt must allow") + } +} + +func newClient(cacheDir string) *Client { + return New(Options{UserAgent: UserAgent("test"), CacheDir: cacheDir, MinInterval: time.Millisecond, AllowPrivate: true}) +} + +func TestGetPoliteAndConditional(t *testing.T) { + var hits, notModified atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/robots.txt": + _, _ = w.Write([]byte("User-agent: *\nDisallow: /secret\n")) + case "/data.json": + hits.Add(1) + if !strings.Contains(r.UserAgent(), "security@openctem.io") || r.Header.Get("From") != "security@openctem.io" { + w.WriteHeader(http.StatusForbidden) + return + } + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"v1"`) + _, _ = w.Write([]byte(`{"ok":true}`)) + case "/big": + _, _ = w.Write([]byte(strings.Repeat("x", 2048))) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + ctx := context.Background() + c := newClient(t.TempDir()) + for i := 0; i < 2; i++ { + r, err := c.Get(ctx, srv.URL+"/data.json", 1024) + if err != nil || string(r.Body) != `{"ok":true}` || r.Cached != (i == 1) { + t.Fatalf("get %d: %v %+v", i, err, r) + } + } + if hits.Load() != 2 || notModified.Load() != 1 { + t.Errorf("hits %d, 304s %d", hits.Load(), notModified.Load()) + } + if _, err := c.Get(ctx, srv.URL+"/secret/x", 1024); !errors.Is(err, ErrDisallowed) { + t.Errorf("robots: %v", err) + } + if _, err := c.Get(ctx, srv.URL+"/big", 1024); !errors.Is(err, ErrTooLarge) { + t.Errorf("cap: %v", err) + } + if _, err := c.Get(ctx, srv.URL+"/nope", 1024); !errors.Is(err, ErrNotFound) { + t.Errorf("404: %v", err) + } +} + +func TestRefusesPrivateAddresses(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("hi")) })) + defer srv.Close() + // The collector's client (AllowPrivate false) must not reach loopback, + // whatever the URL says; it also refuses plain http. + c := New(Options{UserAgent: UserAgent("test"), MinInterval: time.Millisecond}) + if _, err := c.Get(context.Background(), srv.URL+"/", 1024); err == nil { + t.Fatal("fetched a loopback URL") + } + httpsURL := strings.Replace(srv.URL, "http://", "https://", 1) + if _, err := c.Get(context.Background(), httpsURL+"/", 1024); err == nil { + t.Fatal("connected to a loopback address") + } + if _, err := c.Get(context.Background(), "https://user:pw@openctem.io/", 1024); err == nil { + t.Fatal("accepted credentials in URL") + } +} + +func TestUnreachableRobotsDisallows(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/robots.txt" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte("data")) + })) + defer srv.Close() + if _, err := newClient("").Get(context.Background(), srv.URL+"/x", 1024); !errors.Is(err, ErrDisallowed) { + t.Fatalf("5xx robots.txt: %v", err) + } +} + +// A site that redirects every path (robots.txt included) to its home page +// must not make the robots.txt check recurse. +func TestRedirectEverythingTerminates(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/home" { + _, _ = w.Write([]byte("home")) + return + } + http.Redirect(w, r, "/home", http.StatusFound) + })) + defer srv.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r, err := newClient("").Get(ctx, srv.URL+"/.well-known/security.txt", 1024) + if ctx.Err() != nil { + t.Fatal("robots.txt check recursed") + } + if err != nil || r.URL != srv.URL+"/home" { + t.Fatalf("%v %+v", err, r) + } +} diff --git a/internal/fetch/robots.go b/internal/fetch/robots.go new file mode 100644 index 0000000..e7ee102 --- /dev/null +++ b/internal/fetch/robots.go @@ -0,0 +1,116 @@ +package fetch + +import ( + "strings" +) + +// robotsRules is the group of a robots.txt that applies to the collector +// (RFC 9309): the group naming our product token, else the "*" group. +type robotsRules struct { + disallowAll bool + rules []robotsRule +} + +type robotsRule struct { + allow bool + pattern string +} + +func parseRobots(body, product string) *robotsRules { + product = strings.ToLower(product) + type group struct { + agents []string + rules []robotsRule + } + var groups []*group + var cur *group + lastWasAgent := false + for _, line := range strings.Split(body, "\n") { + if i := strings.IndexByte(line, '#'); i >= 0 { + line = line[:i] + } + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + k, v = strings.ToLower(strings.TrimSpace(k)), strings.TrimSpace(v) + switch k { + case "user-agent": + if cur == nil || !lastWasAgent { + cur = &group{} + groups = append(groups, cur) + } + cur.agents = append(cur.agents, strings.ToLower(v)) + lastWasAgent = true + case "allow", "disallow": + lastWasAgent = false + if cur == nil || v == "" { + continue + } + cur.rules = append(cur.rules, robotsRule{allow: k == "allow", pattern: v}) + default: + lastWasAgent = false + } + } + var mine, star []robotsRule + matched := false + for _, g := range groups { + for _, a := range g.agents { + switch { + case a == product: + mine = append(mine, g.rules...) + matched = true + case a == "*": + star = append(star, g.rules...) + } + } + } + if matched { + return &robotsRules{rules: mine} + } + return &robotsRules{rules: star} +} + +// allows applies the longest matching rule; on a tie, allow wins. +func (r *robotsRules) allows(path string) bool { + if r.disallowAll { + return false + } + best, allow := -1, true + for _, rule := range r.rules { + if !match(rule.pattern, path) { + continue + } + n := len(rule.pattern) + if n > best || (n == best && rule.allow) { + best, allow = n, rule.allow + } + } + return allow +} + +// match implements robots.txt patterns: "*" matches any run, a trailing +// "$" anchors the end, otherwise a prefix match. +func match(pattern, path string) bool { + anchored := strings.HasSuffix(pattern, "$") + pattern = strings.TrimSuffix(pattern, "$") + parts := strings.Split(pattern, "*") + if !strings.HasPrefix(path, parts[0]) { + return false + } + pos := len(parts[0]) + for i, part := range parts[1:] { + if i == len(parts)-2 && anchored { + return len(path)-pos >= len(part) && strings.HasSuffix(path, part) + } + j := strings.Index(path[pos:], part) + if j < 0 { + return false + } + pos += j + len(part) + } + if anchored { + return pos == len(path) + } + return true +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 0000000..a525779 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,448 @@ +// Package model is the feed's record model and its validation. +// +// The rules are the ones the OpenCTEM importer applies: a bundle with one +// record the importer would refuse is refused whole, so the collector +// refuses to publish it first. Targets are different: a target the scope +// rules refuse (a public suffix, a private range, a huge CIDR …) does not +// fail the bundle; it is moved to the program's rejected list with the +// reason, so subscribers can see what a program claimed and why the feed +// did not pass it on. +package model + +import ( + "errors" + "fmt" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +// Limits (the importer applies the same). +const ( + MaxNameLen = 200 + MaxTextLen = 4000 + MaxURLLen = 2048 + MaxValueLen = 512 + MaxTargets = 2000 // per list (in scope, out of scope, rejected) + MaxListItems = 32 // testing restrictions, required headers, languages + MaxPrograms = 100000 + MaxChangeFields = 16 +) + +// Program types and statuses. +const ( + TypeBounty = "bounty" + TypeVDP = "vdp" + + StatusOpen = "open" + StatusPaused = "paused" + StatusClosed = "closed" +) + +// Safe-harbour values. +const ( + SafeHarbourFull = "full" + SafeHarbourPartial = "partial" + SafeHarbourNone = "none" + SafeHarbourUnknown = "unknown" +) + +// Target types. +const ( + TargetDomain = "domain" + TargetWildcard = "wildcard" + TargetIP = "ip" + TargetCIDR = "cidr" + TargetURL = "url" + TargetMobileApp = "mobile_app" + TargetSourceRepo = "source_repo" + TargetOther = "other" +) + +// Target confidence: published = the program's own structured scope; +// inferred = derived by the collector (for example the host a program's +// security.txt was served from). An inferred target is a suggestion, never +// an authorisation to test. +const ( + ConfidencePublished = "published" + ConfidenceInferred = "inferred" + // ConfidencePublishedByPlatform: scope the hosting platform published + // for the program, read through a public aggregate dataset. It states + // what the program lists; it is never permission to test. + ConfidencePublishedByPlatform = "published_by_platform" +) + +var ( + idRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$`) + sha256RE = regexp.MustCompile(`^[0-9a-f]{64}$`) + severities = map[string]bool{"": true, "none": true, "low": true, "medium": true, "high": true, "critical": true} + types = map[string]bool{TypeBounty: true, TypeVDP: true} + statuses = map[string]bool{StatusOpen: true, StatusPaused: true, StatusClosed: true} + harbours = map[string]bool{SafeHarbourFull: true, SafeHarbourPartial: true, SafeHarbourNone: true, SafeHarbourUnknown: true} + targetType = map[string]bool{TargetDomain: true, TargetWildcard: true, TargetIP: true, TargetCIDR: true, TargetURL: true, + TargetMobileApp: true, TargetSourceRepo: true, TargetOther: true} + confidences = map[string]bool{ConfidencePublished: true, ConfidenceInferred: true, ConfidencePublishedByPlatform: true} +) + +// Program is one public bug-bounty or vulnerability-disclosure program. +type Program struct { + ID string `json:"id"` // ":", stable, owned by one source + Source string `json:"source"` // the adapter that produced it + Platform string `json:"platform"` // where the program is hosted ("self-hosted" or a platform name) + Name string `json:"name"` + URL string `json:"url"` // the program's policy page + Type string `json:"type"` + Status string `json:"status"` + OffersBounty bool `json:"offers_bounty"` + ScopePublished bool `json:"scope_published"` // the source carries the program's own structured scope + InScope []Target `json:"in_scope"` + OutOfScope []Target `json:"out_of_scope"` + Rejected []Rejected `json:"rejected"` + Rules Rules `json:"rules"` + Terms Terms `json:"terms"` + Contact Contact `json:"contact"` + Provenance Provenance `json:"provenance"` + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + LastChanged time.Time `json:"last_changed"` + ClosedAt *time.Time `json:"closed_at,omitempty"` +} + +// Target is one scope entry. +type Target struct { + Type string `json:"type"` + Value string `json:"value"` + Confidence string `json:"confidence"` + Eligible *bool `json:"eligible_for_bounty,omitempty"` + MaxSev string `json:"max_severity,omitempty"` + Notes string `json:"notes,omitempty"` +} + +// Rejected is a target the scope rules refused, with the reason. +type Rejected struct { + Scope string `json:"scope"` // in | out + Type string `json:"type"` + Value string `json:"value"` + Reason string `json:"reason"` +} + +// Rules summarises how a program may be tested. +type Rules struct { + Summary string `json:"summary,omitempty"` + TestingRestrictions []string `json:"testing_restrictions"` + RateLimit string `json:"rate_limit,omitempty"` + RequiredHeaders []string `json:"required_headers"` + SafeHarbour string `json:"safe_harbour"` + DisclosureDays int `json:"disclosure_days,omitempty"` + Languages []string `json:"languages"` +} + +// Terms names the program's terms and, when the collector read them, the +// SHA-256 of the exact bytes it read. +type Terms struct { + URL string `json:"url"` + SHA256 string `json:"sha256,omitempty"` + FetchedAt *time.Time `json:"fetched_at,omitempty"` +} + +// Contact is the program's reporting channel. +type Contact struct { + Email string `json:"email,omitempty"` + URL string `json:"url,omitempty"` +} + +// Provenance records where and when a record was read. +type Provenance struct { + Source string `json:"source"` + SourceURL string `json:"source_url"` + FetchedAt time.Time `json:"fetched_at"` + // Set for records read through an aggregate dataset: the dataset + // repository ("owner/name"), the commit the record was read at, and + // the platform and program page the dataset took it from. + Dataset string `json:"dataset,omitempty"` + DatasetCommit string `json:"dataset_commit,omitempty"` + OriginalPlatform string `json:"original_platform,omitempty"` + OriginalURL string `json:"original_url,omitempty"` +} + +// Validate checks a normalised program. +func (p *Program) Validate() error { + fail := func(format string, args ...any) error { + return fmt.Errorf("program %q: %s", p.ID, fmt.Sprintf(format, args...)) + } + switch { + case !idRE.MatchString(p.ID): + return fail("id must be : in lower case") + case !strings.HasPrefix(p.ID, p.Source+":"): + return fail("id is not under its source %q", p.Source) + case p.Provenance.Source != p.Source: + return fail("provenance source %q", p.Provenance.Source) + case !safeText(p.Name, MaxNameLen) || strings.TrimSpace(p.Name) == "": + return fail("name is empty, too long or has control characters") + case !safeText(p.Platform, 64) || p.Platform == "": + return fail("platform") + case !types[p.Type]: + return fail("type %q", p.Type) + case !statuses[p.Status]: + return fail("status %q", p.Status) + case (p.Status == StatusClosed) != (p.ClosedAt != nil): + return fail("closed_at must be set exactly when the status is closed") + case p.OffersBounty != (p.Type == TypeBounty): + return fail("offers_bounty must match the type") + case p.FirstSeen.IsZero() || p.LastSeen.Before(p.FirstSeen) || p.LastChanged.Before(p.FirstSeen): + return fail("first_seen must not be after last_seen or last_changed") + case p.Provenance.FetchedAt.IsZero(): + return fail("provenance fetched_at") + } + if err := checkURL(p.URL, true); err != nil { + return fail("url: %v", err) + } + if err := checkURL(p.Provenance.SourceURL, true); err != nil { + return fail("provenance source_url: %v", err) + } + if err := checkURL(p.Terms.URL, true); err != nil { + return fail("terms url: %v", err) + } + if err := p.Provenance.validateDataset(); err != nil { + return fail("provenance: %v", err) + } + if p.Terms.SHA256 != "" && !sha256RE.MatchString(p.Terms.SHA256) { + return fail("terms sha256") + } + if (p.Terms.SHA256 == "") != (p.Terms.FetchedAt == nil) { + return fail("terms sha256 and fetched_at go together") + } + if p.Contact.Email != "" { + if a, err := mail.ParseAddress(p.Contact.Email); err != nil || a.Address != p.Contact.Email || len(p.Contact.Email) > 254 { + return fail("contact email") + } + } + if p.Contact.URL != "" { + if err := checkURL(p.Contact.URL, false); err != nil { + return fail("contact url: %v", err) + } + } + if err := p.Rules.validate(); err != nil { + return fail("%v", err) + } + for _, list := range []struct { + name string + ts []Target + }{{"in_scope", p.InScope}, {"out_of_scope", p.OutOfScope}} { + if list.ts == nil { + return fail("%s must be a list", list.name) + } + if len(list.ts) > MaxTargets { + return fail("%s has more than %d targets", list.name, MaxTargets) + } + for i, t := range list.ts { + if err := t.Validate(); err != nil { + return fail("%s: %v", list.name, err) + } + if i > 0 && !targetLess(list.ts[i-1], t) { + return fail("%s is not sorted or has duplicates", list.name) + } + } + } + if p.Rejected == nil || len(p.Rejected) > MaxTargets { + return fail("rejected must be a list of at most %d", MaxTargets) + } + for _, r := range p.Rejected { + if (r.Scope != "in" && r.Scope != "out") || !safeText(r.Type, 32) || !safeText(r.Value, MaxValueLen) || !safeText(r.Reason, 200) || r.Reason == "" { + return fail("rejected entry %q", r.Value) + } + } + return nil +} + +var ( + datasetRE = regexp.MustCompile(`^[A-Za-z0-9._-]{1,100}/[A-Za-z0-9._-]{1,100}$`) + commitRE = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +// validateDataset checks the aggregate-dataset fields: they are set +// together or not at all. +func (v Provenance) validateDataset() error { + if v.Dataset == "" && v.DatasetCommit == "" && v.OriginalPlatform == "" && v.OriginalURL == "" { + return nil + } + switch { + case !datasetRE.MatchString(v.Dataset): + return errors.New("dataset must be owner/name") + case !commitRE.MatchString(v.DatasetCommit): + return errors.New("dataset_commit must be a 40-character commit id") + case v.OriginalPlatform == "" || !safeText(v.OriginalPlatform, 64): + return errors.New("original_platform") + } + if err := checkURL(v.OriginalURL, true); err != nil { + return fmt.Errorf("original_url: %v", err) + } + return nil +} + +func (r Rules) validate() error { + if !safeText(r.Summary, MaxTextLen) || !safeText(r.RateLimit, 200) { + return errors.New("rules: summary or rate_limit too long or has control characters") + } + if !harbours[r.SafeHarbour] { + return fmt.Errorf("rules: safe_harbour %q", r.SafeHarbour) + } + if r.DisclosureDays < 0 || r.DisclosureDays > 3650 { + return errors.New("rules: disclosure_days") + } + for name, l := range map[string][]string{"testing_restrictions": r.TestingRestrictions, "required_headers": r.RequiredHeaders, "languages": r.Languages} { + if l == nil || len(l) > MaxListItems { + return fmt.Errorf("rules: %s must be a list of at most %d", name, MaxListItems) + } + for _, s := range l { + if s == "" || !safeText(s, 500) { + return fmt.Errorf("rules: %s entry", name) + } + } + } + return nil +} + +// Validate checks a normalised target (Normalize produces only valid ones). +func (t Target) Validate() error { + if !targetType[t.Type] || !confidences[t.Confidence] || !severities[t.MaxSev] || !safeText(t.Notes, 500) { + return fmt.Errorf("target %q: type, confidence, max_severity or notes", t.Value) + } + n, reason := NormalizeTarget(t.Type, t.Value) + if reason != "" || n != t.Value { + return fmt.Errorf("target %q is not a normalised %s (%s)", t.Value, t.Type, reason) + } + return nil +} + +func targetLess(a, b Target) bool { + if a.Type != b.Type { + return a.Type < b.Type + } + return a.Value < b.Value +} + +// SortTargets orders targets canonically and drops exact duplicates +// (same type and value; the first wins). +func SortTargets(ts []Target) []Target { + sort.SliceStable(ts, func(i, j int) bool { return targetLess(ts[i], ts[j]) }) + out := ts[:0] + for i, t := range ts { + if i > 0 && t.Type == ts[i-1].Type && t.Value == ts[i-1].Value { + continue + } + out = append(out, t) + } + return out +} + +// safeText: valid UTF-8, at most max bytes, no control characters other +// than newline and tab, no bidi overrides. +func safeText(s string, max int) bool { + if len(s) > max || !utf8.ValidString(s) { + return false + } + for _, r := range s { + if (unicode.IsControl(r) && r != '\n' && r != '\t') || (r >= 0x202A && r <= 0x202E) || (r >= 0x2066 && r <= 0x2069) { + return false + } + } + return true +} + +// checkURL accepts absolute http(s) URLs without credentials; mailto is +// allowed where links may be mail links. +func checkURL(s string, httpOnly bool) error { + if s == "" || len(s) > MaxURLLen || !safeText(s, MaxURLLen) || strings.ContainsAny(s, " \n\t") { + return errors.New("empty, too long or has spaces/control characters") + } + u, err := url.Parse(s) + if err != nil { + return err + } + switch { + case u.Scheme == "mailto" && !httpOnly: + return nil + case u.Scheme != "https" && u.Scheme != "http": + return fmt.Errorf("scheme %q", u.Scheme) + case u.User != nil: + return errors.New("credentials in URL") + case u.Host == "": + return errors.New("no host") + } + return nil +} + +// CleanText trims a source string, folds whitespace runs and cuts it to max +// bytes on a rune boundary; it returns "" for text with control +// characters. +func CleanText(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + if !safeText(s, len(s)) { + return "" + } + if len(s) <= max { + return s + } + cut := max + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] +} + +// Change is one entry of a run's change log. +type Change struct { + Sequence uint64 `json:"sequence"` + Program string `json:"program"` + Kind string `json:"kind"` + Scope string `json:"scope,omitempty"` // in | out, for target changes + Target *Target `json:"target,omitempty"` // the target after (before, for removed) + Fields []string `json:"fields,omitempty"` // for program_modified / target_modified +} + +// Change kinds. +const ( + ChangeProgramAdded = "program_added" + ChangeProgramClosed = "program_closed" + ChangeProgramReopened = "program_reopened" + ChangeProgramDropped = "program_dropped" + ChangeProgramModified = "program_modified" + ChangeStatus = "status_changed" + ChangeRules = "rules_changed" + ChangeTerms = "terms_changed" + ChangeTargetAdded = "target_added" + ChangeTargetRemoved = "target_removed" + ChangeTargetModified = "target_modified" +) + +var changeKinds = map[string]bool{ChangeProgramAdded: true, ChangeProgramClosed: true, ChangeProgramReopened: true, + ChangeProgramDropped: true, ChangeProgramModified: true, ChangeStatus: true, ChangeRules: true, ChangeTerms: true, + ChangeTargetAdded: true, ChangeTargetRemoved: true, ChangeTargetModified: true} + +// Validate checks a change entry. +func (c *Change) Validate() error { + if !idRE.MatchString(c.Program) || !changeKinds[c.Kind] || c.Sequence == 0 || len(c.Fields) > MaxChangeFields { + return fmt.Errorf("change %q %q: program, kind, sequence or fields", c.Program, c.Kind) + } + isTarget := strings.HasPrefix(c.Kind, "target_") + if isTarget != (c.Target != nil) || isTarget != (c.Scope == "in" || c.Scope == "out") || (!isTarget && c.Scope != "") { + return fmt.Errorf("change %q %q: target and scope go with target changes only", c.Program, c.Kind) + } + if c.Target != nil { + if err := c.Target.Validate(); err != nil { + return err + } + } + for _, f := range c.Fields { + if !safeText(f, 64) || f == "" { + return fmt.Errorf("change %q: field name", c.Program) + } + } + return nil +} diff --git a/internal/model/model_test.go b/internal/model/model_test.go new file mode 100644 index 0000000..93f34ed --- /dev/null +++ b/internal/model/model_test.go @@ -0,0 +1,239 @@ +package model + +import ( + "strings" + "testing" + "time" +) + +func TestNormalizeTarget(t *testing.T) { + ok := []struct{ typ, in, want string }{ + {TargetDomain, "OpenCTEM.ORG.", "openctem.org"}, + {TargetDomain, "api.openctem.io", "api.openctem.io"}, + {TargetDomain, "bücher.de", "xn--bcher-kva.de"}, + {TargetDomain, "foo.example.co.uk", "foo.example.co.uk"}, + {TargetWildcard, "*.OpenCTEM.io", "*.openctem.io"}, + {TargetWildcard, "*.example.co.uk", "*.example.co.uk"}, + {TargetIP, "8.8.8.8", "8.8.8.8"}, + {TargetIP, "::ffff:8.8.4.4", "8.8.4.4"}, + {TargetIP, "2606:4700::1111", "2606:4700::1111"}, + {TargetCIDR, "8.8.8.1/24", "8.8.8.0/24"}, + {TargetCIDR, "203.1.0.0/16", "203.1.0.0/16"}, + {TargetCIDR, "2606:4700::/48", "2606:4700::/48"}, + {TargetURL, "HTTPS://App.OpenCTEM.io/login#x", "https://app.openctem.io/login"}, + {TargetURL, "https://openctem.io:8443", "https://openctem.io:8443/"}, + {TargetMobileApp, " com.example.app ", "com.example.app"}, + {TargetSourceRepo, "https://github.com/openctemio/openctem", "https://github.com/openctemio/openctem"}, + } + for _, c := range ok { + got, reason := NormalizeTarget(c.typ, c.in) + if reason != "" || got != c.want { + t.Errorf("%s %q = %q (%s), want %q", c.typ, c.in, got, reason, c.want) + } + } + bad := []struct{ typ, in string }{ + {TargetDomain, "com"}, + {TargetDomain, "co.uk"}, + {TargetDomain, "github.io"}, // private public suffix + {TargetDomain, "localhost"}, + {TargetDomain, "printer.local"}, + {TargetDomain, "corp.internal"}, + {TargetDomain, "example.com"}, // documentation + {TargetDomain, "10.0.0.1"}, + {TargetDomain, "a..b.com"}, + {TargetDomain, "-bad.com"}, + {TargetDomain, "under_score.com"}, + {TargetDomain, "x.123"}, + {TargetDomain, "foo.*.com"}, + {TargetDomain, strings.Repeat("a", 64) + ".com"}, + {TargetDomain, "bad‮.com"}, + {TargetWildcard, "*.com"}, + {TargetWildcard, "*.co.uk"}, + {TargetWildcard, "*.github.io"}, + {TargetWildcard, "*"}, + {TargetWildcard, "*.*.example.org"}, + {TargetWildcard, "example.org"}, + {TargetIP, "10.1.2.3"}, + {TargetIP, "127.0.0.1"}, + {TargetIP, "169.254.169.254"}, + {TargetIP, "100.64.0.1"}, + {TargetIP, "192.0.2.10"}, + {TargetIP, "::1"}, + {TargetIP, "fd00::1"}, + {TargetIP, "2001:db8::1"}, + {TargetIP, "fe80::1%eth0"}, + {TargetIP, "224.0.0.1"}, + {TargetIP, "not-an-ip"}, + {TargetCIDR, "0.0.0.0/0"}, + {TargetCIDR, "8.0.0.0/8"}, + {TargetCIDR, "8.8.0.0/15"}, + {TargetCIDR, "2606::/16"}, + {TargetCIDR, "10.0.0.0/24"}, + {TargetCIDR, "172.16.5.0/24"}, + {TargetCIDR, "::ffff:8.8.8.0/120"}, + {TargetURL, "javascript:alert(1)"}, + {TargetURL, "ftp://openctem.io/"}, + {TargetURL, "https://user:pw@openctem.io/"}, + {TargetURL, "https://127.0.0.1/"}, + {TargetURL, "https://[::1]/"}, + {TargetURL, "https://*.openctem.io/"}, + {TargetURL, "https://com/"}, + {TargetURL, "https://openctem.io:99999/"}, + {TargetOther, ""}, + {TargetOther, "a\x00b"}, + {"subnet", "1.2.3.0/24"}, + } + for _, c := range bad { + if got, reason := NormalizeTarget(c.typ, c.in); reason == "" { + t.Errorf("%s %q accepted as %q", c.typ, c.in, got) + } + } +} + +func FuzzNormalizeTarget(f *testing.F) { + for _, s := range []string{"example.org", "*.a.b.co.uk", "8.8.8.0/24", "https://a.io/x?y#z", "::ffff:1.2.3.4", "xn--bcher-kva.de"} { + f.Add(s) + } + types := []string{TargetDomain, TargetWildcard, TargetIP, TargetCIDR, TargetURL, TargetOther} + f.Fuzz(func(t *testing.T, s string) { + for _, typ := range types { + n, reason := NormalizeTarget(typ, s) + if reason != "" { + continue + } + // Normalising is idempotent, so every accepted target validates. + if n2, r2 := NormalizeTarget(typ, n); r2 != "" || n2 != n { + t.Fatalf("%s %q -> %q -> %q (%s)", typ, s, n, n2, r2) + } + } + }) +} + +func validProgram() Program { + now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) + return Program{ + ID: "disclose:openctem", Source: "disclose", Platform: "self-hosted", Name: "OpenCTEM", + URL: "https://openctem.io/security", Type: TypeVDP, Status: StatusOpen, + InScope: []Target{{Type: TargetWildcard, Value: "*.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}, {Type: TargetIP, Value: "8.8.8.8"}}, + OutOfScope: []Target{{Type: TargetDomain, Value: "status.openctem.io"}, {Type: TargetDomain, Value: "openctem.io"}}, + Rules: Rules{TestingRestrictions: []string{"no DoS", "no DoS"}}, + Terms: Terms{URL: "https://openctem.io/security"}, + Provenance: Provenance{Source: "disclose", SourceURL: "https://example.org/list.json", FetchedAt: now}, + FirstSeen: now, LastSeen: now, LastChanged: now, + } +} + +func TestNormalizeProgram(t *testing.T) { + p := validProgram() + p.InScope = append(p.InScope, Target{Type: TargetWildcard, Value: "*.com"}, Target{Type: TargetCIDR, Value: "10.0.0.0/8"}, + Target{Type: TargetDomain, Value: "OPENCTEM.io"}) + Normalize(&p) + if err := p.Validate(); err != nil { + t.Fatal(err) + } + var in []string + for _, x := range p.InScope { + in = append(in, x.Type+":"+x.Value) + if x.Confidence != ConfidencePublished { + t.Errorf("confidence %q", x.Confidence) + } + } + // Sorted, deduplicated, and openctem.io dropped because it is also out of scope. + if got := strings.Join(in, " "); got != "ip:8.8.8.8 wildcard:*.openctem.io" { + t.Errorf("in scope = %s", got) + } + reasons := map[string]string{} + for _, r := range p.Rejected { + reasons[r.Value] = r.Reason + } + if reasons["*.com"] == "" || reasons["10.0.0.0/8"] == "" || reasons["openctem.io"] != "also listed out of scope" { + t.Errorf("rejected = %+v", p.Rejected) + } + if len(p.Rules.TestingRestrictions) != 1 || p.Rules.SafeHarbour != SafeHarbourUnknown { + t.Errorf("rules = %+v", p.Rules) + } +} + +func TestProgramValidateRefuses(t *testing.T) { + cases := map[string]func(p *Program){ + "bad id": func(p *Program) { p.ID = "Disclose:X" }, + "id of other source": func(p *Program) { p.ID = "other:x" }, + "control in name": func(p *Program) { p.Name = "a\x07b" }, + "empty name": func(p *Program) { p.Name = " " }, + "bad type": func(p *Program) { p.Type = "paid" }, + "bounty mismatch": func(p *Program) { p.OffersBounty = true }, + "closed without time": func(p *Program) { p.Status = StatusClosed }, + "javascript url": func(p *Program) { p.URL = "javascript:alert(1)" }, + "terms hash no time": func(p *Program) { p.Terms.SHA256 = strings.Repeat("a", 64) }, + "bad email": func(p *Program) { p.Contact.Email = "x " }, + "unsorted targets": func(p *Program) { + p.InScope = []Target{{Type: TargetDomain, Value: "b.org", Confidence: ConfidencePublished}, {Type: TargetDomain, Value: "a.org", Confidence: ConfidencePublished}} + }, + "unnormalised target": func(p *Program) { + p.InScope = []Target{{Type: TargetDomain, Value: "A.org", Confidence: ConfidencePublished}} + }, + "public suffix target": func(p *Program) { + p.InScope = []Target{{Type: TargetWildcard, Value: "*.com", Confidence: ConfidencePublished}} + }, + "nil list": func(p *Program) { p.OutOfScope = nil }, + "first seen": func(p *Program) { p.FirstSeen = p.LastSeen.Add(time.Hour) }, + "bad harbour": func(p *Program) { p.Rules.SafeHarbour = "maybe" }, + "no provenance": func(p *Program) { p.Provenance.FetchedAt = time.Time{} }, + } + for name, mut := range cases { + p := validProgram() + Normalize(&p) + mut(&p) + if err := p.Validate(); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestChangeValidate(t *testing.T) { + tgt := Target{Type: TargetDomain, Value: "openctem.io", Confidence: ConfidenceInferred} + good := []Change{ + {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &tgt}, + } + for _, c := range good { + if err := c.Validate(); err != nil { + t.Error(err) + } + } + bad := []Change{ + {Sequence: 0, Program: "disclose:x", Kind: ChangeProgramAdded}, + {Sequence: 1, Program: "disclose:x", Kind: "deleted"}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeProgramAdded, Scope: "in", Target: &tgt}, + {Sequence: 1, Program: "disclose:x", Kind: ChangeTargetAdded, Scope: "in", Target: &Target{Type: TargetWildcard, Value: "*.com", Confidence: ConfidenceInferred}}, + } + for i, c := range bad { + if err := c.Validate(); err == nil { + t.Errorf("bad change %d accepted", i) + } + } +} + +func TestDatasetProvenance(t *testing.T) { + ok := Provenance{Dataset: "o/r", DatasetCommit: strings.Repeat("a", 40), OriginalPlatform: "hackerone", OriginalURL: "https://hackerone.com/x"} + if err := ok.validateDataset(); err != nil { + t.Fatal(err) + } + if err := (Provenance{}).validateDataset(); err != nil { + t.Fatal(err) + } + for name, mut := range map[string]func(*Provenance){ + "commit": func(p *Provenance) { p.DatasetCommit = "main" }, + "dataset": func(p *Provenance) { p.Dataset = "o/r/extra" }, + "platform": func(p *Provenance) { p.OriginalPlatform = "" }, + "url": func(p *Provenance) { p.OriginalURL = "javascript:alert(1)" }, + "partial": func(p *Provenance) { *p = Provenance{Dataset: "o/r"} }, + } { + p := ok + mut(&p) + if p.validateDataset() == nil { + t.Errorf("%s: accepted", name) + } + } +} diff --git a/internal/model/normalize.go b/internal/model/normalize.go new file mode 100644 index 0000000..28585b1 --- /dev/null +++ b/internal/model/normalize.go @@ -0,0 +1,103 @@ +package model + +import ( + "sort" + "strings" +) + +// Normalize canonicalises the targets of a program an adapter produced: +// every target is normalised or moved to Rejected with the reason, lists are +// deduplicated and sorted, an in-scope target that is also out of scope is +// dropped from in scope (out of scope wins), and lists are capped. Text +// fields are cleaned. It does not set the timestamps; the collector does. +func Normalize(p *Program) { + p.Name = CleanText(p.Name, MaxNameLen) + p.Platform = CleanText(p.Platform, 64) + p.Rules.Summary = CleanText(p.Rules.Summary, MaxTextLen) + p.Rules.RateLimit = CleanText(p.Rules.RateLimit, 200) + p.Rules.TestingRestrictions = cleanList(p.Rules.TestingRestrictions) + p.Rules.RequiredHeaders = cleanList(p.Rules.RequiredHeaders) + p.Rules.Languages = cleanList(p.Rules.Languages) + if p.Rules.SafeHarbour == "" { + p.Rules.SafeHarbour = SafeHarbourUnknown + } + p.OffersBounty = p.Type == TypeBounty + var rejected []Rejected + norm := func(scope string, in []Target) []Target { + out := make([]Target, 0, len(in)) + for _, t := range in { + v, reason := NormalizeTarget(t.Type, t.Value) + if reason != "" { + rejected = append(rejected, Rejected{Scope: scope, Type: CleanText(t.Type, 32), Value: CleanText(t.Value, MaxValueLen), Reason: reason}) + continue + } + t.Value = v + if t.Confidence == "" { + t.Confidence = ConfidencePublished + } + if !severities[t.MaxSev] { + t.MaxSev = "" + } + t.Notes = CleanText(t.Notes, 500) + out = append(out, t) + } + out = SortTargets(out) + if len(out) > MaxTargets { + for _, t := range out[MaxTargets:] { + rejected = append(rejected, Rejected{Scope: scope, Type: t.Type, Value: t.Value, Reason: "over the per-program target cap"}) + } + out = out[:MaxTargets] + } + return out + } + p.OutOfScope = norm("out", p.OutOfScope) + in := norm("in", p.InScope) + excluded := map[string]bool{} + for _, t := range p.OutOfScope { + excluded[t.Type+"\x00"+t.Value] = true + } + p.InScope = in[:0] + for _, t := range in { + if excluded[t.Type+"\x00"+t.Value] { + rejected = append(rejected, Rejected{Scope: "in", Type: t.Type, Value: t.Value, Reason: "also listed out of scope"}) + continue + } + p.InScope = append(p.InScope, t) + } + for i := range rejected { + if rejected[i].Value == "" { + rejected[i].Value = "(unprintable)" + } + if rejected[i].Type == "" { + rejected[i].Type = "unknown" + } + } + sort.SliceStable(rejected, func(i, j int) bool { + a, b := rejected[i], rejected[j] + return strings.Join([]string{a.Scope, a.Type, a.Value, a.Reason}, "\x00") < strings.Join([]string{b.Scope, b.Type, b.Value, b.Reason}, "\x00") + }) + if len(rejected) > MaxTargets { + rejected = rejected[:MaxTargets] + } + if rejected == nil { + rejected = []Rejected{} + } + p.Rejected = rejected +} + +func cleanList(in []string) []string { + out := []string{} + seen := map[string]bool{} + for _, s := range in { + s = CleanText(s, 500) + if s == "" || seen[s] { + continue + } + seen[s] = true + out = append(out, s) + if len(out) == MaxListItems { + break + } + } + return out +} diff --git a/internal/model/schema_test.go b/internal/model/schema_test.go new file mode 100644 index 0000000..d4e7c5f --- /dev/null +++ b/internal/model/schema_test.go @@ -0,0 +1,60 @@ +package model + +import ( + "encoding/json" + "os" + "reflect" + "sort" + "strings" + "testing" +) + +// The published JSON Schema documents every field the feed writes, and +// nothing else. +func TestSchemaMatchesModel(t *testing.T) { + type obj struct { + Properties map[string]json.RawMessage `json:"properties"` + Defs map[string]obj `json:"$defs"` + } + read := func(name string) obj { + b, err := os.ReadFile("../../schema/" + name) + if err != nil { + t.Fatal(err) + } + var o obj + if err := json.Unmarshal(b, &o); err != nil { + t.Fatal(err) + } + return o + } + prog, change := read("program.schema.json"), read("change.schema.json") + cases := []struct { + name string + schema obj + v any + }{ + {"program", prog, Program{}}, + {"target", prog.Defs["target"], Target{}}, + {"rejected", prog.Defs["rejected"], Rejected{}}, + {"rules", prog.Defs["rules"], Rules{}}, + {"terms", prog.Defs["terms"], Terms{}}, + {"contact", prog.Defs["contact"], Contact{}}, + {"provenance", prog.Defs["provenance"], Provenance{}}, + {"change", change, Change{}}, + } + for _, c := range cases { + var want, got []string + rt := reflect.TypeOf(c.v) + for i := 0; i < rt.NumField(); i++ { + want = append(want, strings.Split(rt.Field(i).Tag.Get("json"), ",")[0]) + } + for k := range c.schema.Properties { + got = append(got, k) + } + sort.Strings(want) + sort.Strings(got) + if strings.Join(want, ",") != strings.Join(got, ",") { + t.Errorf("%s: schema has %v, model has %v", c.name, got, want) + } + } +} diff --git a/internal/model/scope.go b/internal/model/scope.go new file mode 100644 index 0000000..cece441 --- /dev/null +++ b/internal/model/scope.go @@ -0,0 +1,206 @@ +package model + +import ( + "net/netip" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/idna" + "golang.org/x/net/publicsuffix" +) + +// Scope caps: a CIDR wider than these is refused (a program does not own +// a /8, and a wide range in a feed would point scanners at strangers). +const ( + MinIPv4Prefix = 16 + MinIPv6Prefix = 48 +) + +// Ranges no program can own on the public internet (RFC 6890 and friends). +var reserved = func() []netip.Prefix { + var out []netip.Prefix + for _, s := range []string{ + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", + "192.0.0.0/24", "192.0.2.0/24", "192.88.99.0/24", "192.168.0.0/16", "198.18.0.0/15", "198.51.100.0/24", + "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", "255.255.255.255/32", + "::/128", "::1/128", "::ffff:0:0/96", "64:ff9b::/96", "64:ff9b:1::/48", "100::/64", "2001::/23", + "2001:db8::/32", "2002::/16", "3fff::/20", "5f00::/16", "fc00::/7", "fe80::/10", "ff00::/8", + } { + out = append(out, netip.MustParsePrefix(s)) + } + return out +}() + +// Names that never resolve on the public internet, or are reserved for +// documentation. +var reservedSuffixes = []string{"local", "localhost", "test", "example", "invalid", "internal", "onion", "arpa", + "home.arpa", "lan", "corp", "intranet", "private", "example.com", "example.net", "example.org"} + +var lookup = idna.New(idna.MapForLookup(), idna.Transitional(false), idna.StrictDomainName(true), idna.BidiRule()) + +// NormalizeTarget returns the canonical form of a target value of the given +// type, or a non-empty reason it is refused. +func NormalizeTarget(typ, value string) (string, string) { + v := strings.TrimSpace(value) + if v == "" || len(v) > MaxValueLen || !safeText(v, MaxValueLen) { + return "", "empty, too long or has control characters" + } + switch typ { + case TargetDomain: + return normDomain(v) + case TargetWildcard: + if !strings.HasPrefix(v, "*.") { + return "", "a wildcard must start with *." + } + d, reason := normDomain(v[2:]) + if reason != "" { + return "", reason + } + return "*." + d, "" + case TargetIP: + a, err := netip.ParseAddr(v) + if err != nil || a.Zone() != "" { + return "", "not an IP address" + } + a = a.Unmap() + if r := reservedAddr(a); r != "" { + return "", r + } + return a.String(), "" + case TargetCIDR: + p, err := netip.ParsePrefix(v) + if err != nil { + return "", "not a CIDR" + } + if p.Addr().Is4In6() { + return "", "IPv4-mapped CIDR" + } + p = p.Masked() + if (p.Addr().Is4() && p.Bits() < MinIPv4Prefix) || (p.Addr().Is6() && p.Bits() < MinIPv6Prefix) { + return "", "CIDR wider than /" + strconv.Itoa(MinIPv4Prefix) + " (IPv4) or /" + strconv.Itoa(MinIPv6Prefix) + " (IPv6)" + } + for _, r := range reserved { + if r.Overlaps(p) { + return "", "private or reserved range" + } + } + return p.String(), "" + case TargetURL: + return normURL(v) + case TargetMobileApp, TargetSourceRepo, TargetOther: + s := CleanText(v, MaxValueLen) + if s == "" { + return "", "empty or has control characters" + } + return s, "" + } + return "", "unknown target type" +} + +func reservedAddr(a netip.Addr) string { + if !a.IsGlobalUnicast() || a.IsPrivate() { + return "private or reserved address" + } + for _, r := range reserved { + if r.Contains(a) { + return "private or reserved address" + } + } + return "" +} + +func normDomain(v string) (string, string) { + d := strings.TrimSuffix(strings.ToLower(v), ".") + if strings.Contains(d, "*") { + return "", "wildcard inside a domain" + } + if _, err := netip.ParseAddr(d); err == nil { + return "", "an IP address, not a domain" + } + ascii, err := lookup.ToASCII(d) + if err != nil { + return "", "not a valid domain name" + } + if len(ascii) > 253 || !strings.Contains(ascii, ".") { + return "", "not a fully qualified domain name" + } + labels := strings.Split(ascii, ".") + for _, l := range labels { + if l == "" || len(l) > 63 || l[0] == '-' || l[len(l)-1] == '-' { + return "", "not a valid domain name" + } + for _, c := range l { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' { + return "", "not a valid domain name" + } + } + } + if tld := labels[len(labels)-1]; strings.Trim(tld, "0123456789") == "" { + return "", "numeric top-level label" + } + for _, s := range reservedSuffixes { + if ascii == s || strings.HasSuffix(ascii, "."+s) { + return "", "reserved or non-public name" + } + } + if ps, _ := publicsuffix.PublicSuffix(ascii); ps == ascii { + return "", "a public suffix" + } + if _, err := publicsuffix.EffectiveTLDPlusOne(ascii); err != nil { + return "", "a public suffix" + } + return ascii, "" +} + +func normURL(v string) (string, string) { + u, err := url.Parse(v) + if err != nil { + return "", "not a URL" + } + u.Scheme = strings.ToLower(u.Scheme) + if u.Scheme != "https" && u.Scheme != "http" { + return "", "not an http(s) URL" + } + if u.User != nil { + return "", "credentials in URL" + } + host := u.Hostname() + var h string + if a, err := netip.ParseAddr(host); err == nil { + a = a.Unmap() + if r := reservedAddr(a); r != "" || a.Zone() != "" { + return "", "private or reserved address" + } + h = a.String() + if a.Is6() { + h = "[" + h + "]" + } + } else { + d, reason := normDomain(host) + if reason != "" { + return "", reason + } + h = d + } + if p := u.Port(); p != "" { + n, err := strconv.Atoi(p) + if err != nil || n < 1 || n > 65535 { + return "", "bad port" + } + h += ":" + strconv.Itoa(n) + } + u.Host, u.Fragment, u.RawFragment = h, "", "" + if u.Path == "" { + u.Path = "/" + } + s := u.String() + if len(s) > MaxValueLen { + return "", "too long" + } + return s, "" +} + +// PublicAddr reports whether a is a public unicast address a program could +// own (the fetcher also refuses to connect anywhere else). +func PublicAddr(a netip.Addr) bool { return reservedAddr(a.Unmap()) == "" } diff --git a/internal/source/bountytargets.go b/internal/source/bountytargets.go new file mode 100644 index 0000000..50261d5 --- /dev/null +++ b/internal/source/bountytargets.go @@ -0,0 +1,379 @@ +package source + +import ( + "context" + "encoding/json" + "net/url" + "sort" + "strings" + "time" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// BountyTargetsRepo is the dataset that mirrors the public program +// listings of several bug-bounty platforms as one JSON file per platform. +const BountyTargetsRepo = "arkadiyt/bounty-targets-data" + +// BountyTargets reads that dataset. Every record keeps the platform it was +// taken from and that platform's program page; targets are marked +// published_by_platform: the platform listed them for the program, which +// is not permission to test them. Whether a platform's records may be +// published is decided by the publish policy, not here. +type BountyTargets struct { + Dataset githubDataset + // Only limits the run to these platforms (nil = every platform the + // adapter knows). A restricted platform that cannot be published is + // not even fetched in a signed build. + Only []string +} + +// BountyTargetsPlatforms are the platforms the adapter knows, in order. +func BountyTargetsPlatforms() []string { + out := make([]string, len(btdPlatforms)) + for i, p := range btdPlatforms { + out[i] = p.name + } + return out +} + +// NewBountyTargets returns the adapter for the real dataset. +func NewBountyTargets(only []string) BountyTargets { + return BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, Ref: "main"}, Only: only} +} + +// Meta implements Source. +func (b BountyTargets) Meta() Meta { + return Meta{ + Name: "bounty-targets", + URL: "https://github.com/" + b.repo(), + Terms: "https://github.com/" + b.repo(), + Licence: "MIT (dataset code); the records stay subject to the terms of the platform they name", + Attribution: "Program and scope listings aggregated by the " + b.repo() + " project from the public program pages of " + + "the platforms named in each record.", + } +} + +func (b BountyTargets) repo() string { + if b.Dataset.Repo != "" { + return b.Dataset.Repo + } + return BountyTargetsRepo +} + +type btdPlatform struct { + platformSpec + file string + parse func(raw json.RawMessage) (*dsProgram, bool) +} + +var btdPlatforms = []btdPlatform{ + {platformSpec{"bugcrowd", []string{"bugcrowd.com"}}, "bugcrowd_data.json", parseBugcrowd}, + {platformSpec{"hackerone", []string{"hackerone.com"}}, "hackerone_data.json", parseHackerOne}, + {platformSpec{"intigriti", []string{"intigriti.com"}}, "intigriti_data.json", parseIntigriti}, + {platformSpec{"yeswehack", []string{"yeswehack.com"}}, "yeswehack_data.json", parseYesWeHack}, +} + +// dsProgram is a program as the platform parsers describe it. +type dsProgram struct { + key string // handle, unique within the platform + name string + url string + bounty bool + paused bool + harbour string + in, out []dsTarget +} + +type dsTarget struct { + kind string + value string + eligible *bool + maxSev string + notes string +} + +// Collect implements Source. Any platform file that cannot be read fails +// the whole source, so one run never mixes commits or half a dataset. +func (b BountyTargets) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + commit, err := b.Dataset.resolve(ctx, c) + if err != nil { + return nil, err + } + meta := b.Meta() + res := &Result{} + for _, plat := range btdPlatforms { + if !wanted(b.Only, plat.name) { + continue + } + fileURL := b.Dataset.rawURL(commit, "data/"+plat.file) + resp, err := c.Get(ctx, fileURL, maxDatasetBytes) + if err != nil { + return nil, err + } + raw, err := decodeRecords(resp.Body, plat.name) + if err != nil { + return nil, err + } + var progs []*dsProgram + keys := make([]string, 0, len(raw)) + for _, r := range raw { + p, ok := plat.parse(r) + if !ok || p.key == "" || strings.TrimSpace(p.name) == "" || !plat.onPlatform(p.url) { + res.Skipped++ + continue + } + progs = append(progs, p) + keys = append(keys, p.key) + } + slugs := UniqueSlugs(keys) + for _, p := range progs { + res.Programs = append(res.Programs, p.toProgram(meta, plat.name, slugs[p.key], b.repo(), commit, fileURL, resp.FetchedAt)) + } + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + return res, nil +} + +func (p *dsProgram) toProgram(meta Meta, platform, slug, repo, commit, fileURL string, fetched time.Time) model.Program { + prog := model.Program{ + ID: meta.Name + ":" + platform + "-" + slug, + Source: meta.Name, + Platform: platform, + Name: p.name, + URL: strings.TrimSpace(p.url), + Type: model.TypeVDP, + Status: model.StatusOpen, + ScopePublished: true, + InScope: targets(p.in), + OutOfScope: targets(p.out), + Rules: model.Rules{SafeHarbour: harbour(p.harbour), TestingRestrictions: []string{}, RequiredHeaders: []string{}, + Languages: []string{}}, + Terms: model.Terms{URL: strings.TrimSpace(p.url)}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: fileURL, FetchedAt: fetched, + Dataset: repo, DatasetCommit: commit, OriginalPlatform: platform, OriginalURL: strings.TrimSpace(p.url)}, + } + if p.bounty { + prog.Type = model.TypeBounty + } + if p.paused { + prog.Status = model.StatusPaused + } + return prog +} + +func targets(in []dsTarget) []model.Target { + out := make([]model.Target, 0, len(in)) + for _, t := range in { + typ, val := classify(t.kind, t.value) + out = append(out, model.Target{Type: typ, Value: val, Confidence: model.ConfidencePublishedByPlatform, + Eligible: t.eligible, MaxSev: strings.ToLower(t.maxSev), Notes: t.notes}) + } + return out +} + +// lastSegment is the last non-empty path segment of a URL. +func lastSegment(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "" + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + return parts[len(parts)-1] +} + +// ---- HackerOne ---- + +func parseHackerOne(raw json.RawMessage) (*dsProgram, bool) { + var r struct { + Handle string `json:"handle"` + Name string `json:"name"` + URL string `json:"url"` + Bounty bool `json:"offers_bounties"` + State string `json:"submission_state"` + Targets struct { + In []h1Target `json:"in_scope"` + Out []h1Target `json:"out_of_scope"` + } `json:"targets"` + } + if json.Unmarshal(raw, &r) != nil { + return nil, false + } + p := &dsProgram{key: r.Handle, name: r.Name, url: r.URL, bounty: r.Bounty, paused: r.State != "open"} + for _, t := range r.Targets.In { + p.in = append(p.in, t.target()) + } + for _, t := range r.Targets.Out { + p.out = append(p.out, t.target()) + } + return p, true +} + +type h1Target struct { + Identifier string `json:"asset_identifier"` + Type string `json:"asset_type"` + Eligible *bool `json:"eligible_for_bounty"` + MaxSeverity string `json:"max_severity"` + Instruction string `json:"instruction"` +} + +func (t h1Target) target() dsTarget { + kind := kindAuto + switch t.Type { + case "SOURCE_CODE": + kind = kindSource + case "GOOGLE_PLAY_APP_ID", "APPLE_STORE_APP_ID", "TESTFLIGHT", "OTHER_APK", "WINDOWS_APP_STORE_APP_ID": + kind = kindMobile + case "HARDWARE", "DOWNLOADABLE_EXECUTABLES", "AI_MODEL", "SMART_CONTRACT": + kind = kindOpaque + } + return dsTarget{kind: kind, value: t.Identifier, eligible: t.Eligible, maxSev: t.MaxSeverity, notes: t.Instruction} +} + +// ---- Bugcrowd ---- + +func parseBugcrowd(raw json.RawMessage) (*dsProgram, bool) { + var r struct { + Name string `json:"name"` + URL string `json:"url"` + SafeHarbor string `json:"safe_harbor"` + MaxPayout *float64 `json:"max_payout"` + Targets struct { + In []bcTarget `json:"in_scope"` + Out []bcTarget `json:"out_of_scope"` + } `json:"targets"` + } + if json.Unmarshal(raw, &r) != nil { + return nil, false + } + p := &dsProgram{key: lastSegment(r.URL), name: r.Name, url: r.URL, harbour: r.SafeHarbor, + bounty: r.MaxPayout != nil && *r.MaxPayout > 0} + for _, t := range r.Targets.In { + p.in = append(p.in, t.target()) + } + for _, t := range r.Targets.Out { + p.out = append(p.out, t.target()) + } + return p, true +} + +type bcTarget struct { + Type string `json:"type"` + Target string `json:"target"` + Name string `json:"name"` +} + +func (t bcTarget) target() dsTarget { + kind := kindAuto + switch t.Type { + case "ios", "android": + kind = kindMobile + case "hardware", "iot": + kind = kindOpaque + } + v := t.Target + if v == "" { + v = t.Name + } + return dsTarget{kind: kind, value: v} +} + +// ---- Intigriti ---- + +func parseIntigriti(raw json.RawMessage) (*dsProgram, bool) { + var r struct { + Company string `json:"company_handle"` + Handle string `json:"handle"` + Name string `json:"name"` + URL string `json:"url"` + Status string `json:"status"` + Level string `json:"confidentiality_level"` + MaxBount *struct { + Value float64 `json:"value"` + } `json:"max_bounty"` + Targets struct { + In []intTarget `json:"in_scope"` + Out []intTarget `json:"out_of_scope"` + } `json:"targets"` + } + if json.Unmarshal(raw, &r) != nil || r.Company == "" || r.Handle == "" || (r.Level != "" && r.Level != "public") { + return nil, false + } + p := &dsProgram{key: r.Company + "-" + r.Handle, name: r.Name, url: r.URL, paused: r.Status != "open", + bounty: r.MaxBount != nil && r.MaxBount.Value > 0} + for _, t := range r.Targets.In { + p.in = append(p.in, t.target()) + } + for _, t := range r.Targets.Out { + p.out = append(p.out, t.target()) + } + return p, true +} + +type intTarget struct { + Type string `json:"type"` + Endpoint string `json:"endpoint"` + Description string `json:"description"` + Impact string `json:"impact"` +} + +func (t intTarget) target() dsTarget { + kind := kindAuto + switch t.Type { + case "ios", "android": + kind = kindMobile + case "device": + kind = kindOpaque + } + out := dsTarget{kind: kind, value: t.Endpoint, notes: t.Description} + if strings.EqualFold(strings.TrimSpace(t.Impact), "No Bounty") { + no := false + out.eligible = &no + } + return out +} + +// ---- YesWeHack ---- + +func parseYesWeHack(raw json.RawMessage) (*dsProgram, bool) { + var r struct { + ID string `json:"id"` + Name string `json:"name"` + Public *bool `json:"public"` + Disabled bool `json:"disabled"` + MaxBounty *float64 `json:"max_bounty"` + Targets struct { + In []ywhTarget `json:"in_scope"` + Out []ywhTarget `json:"out_of_scope"` + } `json:"targets"` + } + if json.Unmarshal(raw, &r) != nil || r.ID == "" || (r.Public != nil && !*r.Public) { + return nil, false + } + p := &dsProgram{key: r.ID, name: r.Name, url: "https://yeswehack.com/programs/" + url.PathEscape(r.ID), paused: r.Disabled, + bounty: r.MaxBounty != nil && *r.MaxBounty > 0} + for _, t := range r.Targets.In { + p.in = append(p.in, t.target()) + } + for _, t := range r.Targets.Out { + p.out = append(p.out, t.target()) + } + return p, true +} + +type ywhTarget struct { + Type string `json:"type"` + Target string `json:"target"` +} + +func (t ywhTarget) target() dsTarget { + kind := kindAuto + switch { + case strings.HasPrefix(t.Type, "mobile-application"): + kind = kindMobile + case t.Type == "ai-model" || t.Type == "challenge": + kind = kindOpaque + } + return dsTarget{kind: kind, value: t.Target} +} diff --git a/internal/source/dataset.go b/internal/source/dataset.go new file mode 100644 index 0000000..8aeb223 --- /dev/null +++ b/internal/source/dataset.go @@ -0,0 +1,184 @@ +package source + +import ( + "context" + "encoding/json" + "fmt" + "net/netip" + "net/url" + "regexp" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// Shared parts of the adapters that read public aggregate datasets on +// GitHub: datasets that copy the program listings of bug-bounty platforms +// into JSON files. A dataset record is never trusted more than the +// dataset: the program page must sit on the platform's own host, every +// record names the dataset repository, the commit it was read at and the +// original platform and page, and every target is normalised by the same +// scope rules as any other source. +// +// Reading a dataset does not make its records publishable: the platform a +// record came from has its own publish policy (see the policy package). + +const ( + githubAPI = "https://api.github.com" + githubRaw = "https://raw.githubusercontent.com" + + maxDatasetBytes = 32 << 20 + maxDatasetRecords = 20000 + maxCommitListSize = 1 << 20 +) + +// githubDataset locates files of one dataset repository at one commit. +type githubDataset struct { + Repo string // owner/name + Ref string // branch to resolve when Commit is empty + Commit string // pinned commit; resolved from Ref when empty + APIBase string // githubAPI unless a test overrides it + RawBase string // githubRaw unless a test overrides it +} + +func (g githubDataset) apiBase() string { + if g.APIBase != "" { + return g.APIBase + } + return githubAPI +} + +func (g githubDataset) rawBase() string { + if g.RawBase != "" { + return g.RawBase + } + return githubRaw +} + +// resolve returns the commit every file of this run is read at. All files +// of a run come from one commit, and the raw URLs contain it, so the bytes +// read are the bytes of that commit. +func (g githubDataset) resolve(ctx context.Context, c *fetch.Client) (string, error) { + if g.Commit != "" { + if !commitHex.MatchString(g.Commit) { + return "", fmt.Errorf("%s: pinned commit %q is not a 40-character commit id", g.Repo, g.Commit) + } + return g.Commit, nil + } + ref := g.Ref + if ref == "" { + ref = "main" + } + u := fmt.Sprintf("%s/repos/%s/commits?per_page=1&sha=%s", g.apiBase(), g.Repo, url.QueryEscape(ref)) + resp, err := c.Get(ctx, u, maxCommitListSize) + if err != nil { + return "", fmt.Errorf("%s: resolving %s: %w", g.Repo, ref, err) + } + var list []struct { + SHA string `json:"sha"` + } + if err := json.Unmarshal(resp.Body, &list); err != nil || len(list) != 1 || !commitHex.MatchString(list[0].SHA) { + return "", fmt.Errorf("%s: %s does not resolve to a commit", g.Repo, ref) + } + return list[0].SHA, nil +} + +func (g githubDataset) rawURL(commit, path string) string { + return fmt.Sprintf("%s/%s/%s/%s", g.rawBase(), g.Repo, commit, path) +} + +var commitHex = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// platformSpec describes one platform an aggregate dataset covers. +type platformSpec struct { + name string + hosts []string // hosts of the platform's own program pages +} + +// onPlatform reports whether a program page is an https URL on one of the +// platform's hosts. +func (p platformSpec) onPlatform(raw string) bool { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Scheme != "https" || u.User != nil || u.Port() != "" { + return false + } + h := strings.ToLower(u.Hostname()) + for _, ok := range p.hosts { + if h == ok || strings.HasSuffix(h, "."+ok) { + return true + } + } + return false +} + +// wanted reports whether only is empty or names the platform. +func wanted(only []string, platform string) bool { + if len(only) == 0 { + return true + } + for _, o := range only { + if o == platform { + return true + } + } + return false +} + +var hostnameRE = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?(\.[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?)+$`) + +// Kinds of asset a platform declares, as far as the feed distinguishes them. +const ( + kindAuto = "" + kindMobile = "mobile" + kindSource = "source" + kindOpaque = "opaque" // hardware, devices and the like: kept as text +) + +// classify turns an asset a platform lists into a typed target. The +// platform's declared kind decides only where the value cannot say what it +// is (an app id looks like a host name); everything else follows the shape +// of the value. The scope rules refuse what is not acceptable afterwards. +func classify(kind, value string) (string, string) { + v := strings.TrimSpace(value) + switch kind { + case kindMobile: + return model.TargetMobileApp, v + case kindSource: + return model.TargetSourceRepo, v + case kindOpaque: + return model.TargetOther, v + } + switch { + case v == "" || strings.ContainsAny(v, " \t\r\n"): + return model.TargetOther, v + case strings.HasPrefix(v, "http://") || strings.HasPrefix(v, "https://"): + return model.TargetURL, v + case strings.HasPrefix(v, "*."): + return model.TargetWildcard, v + } + if _, err := netip.ParsePrefix(v); err == nil { + return model.TargetCIDR, v + } + if _, err := netip.ParseAddr(v); err == nil { + return model.TargetIP, v + } + if hostnameRE.MatchString(v) { + return model.TargetDomain, v + } + return model.TargetOther, v +} + +// decodeRecords splits a JSON array into records, enforcing the dataset +// caps. A file that is not an array is an error: the dataset changed shape +// and the whole source is kept as it was. +func decodeRecords(body []byte, what string) ([]json.RawMessage, error) { + var raw []json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("%s: %w", what, err) + } + if len(raw) > maxDatasetRecords { + return nil, fmt.Errorf("%s: %d records, over the cap of %d", what, len(raw), maxDatasetRecords) + } + return raw, nil +} diff --git a/internal/source/dataset_test.go b/internal/source/dataset_test.go new file mode 100644 index 0000000..a5a99b8 --- /dev/null +++ b/internal/source/dataset_test.go @@ -0,0 +1,261 @@ +package source + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/openctemio/programfeed/internal/model" +) + +const testCommit = "0123456789abcdef0123456789abcdef01234567" + +// datasetServer serves the recorded fixtures the way GitHub does and +// records the paths requested. +func datasetServer(t *testing.T) (*httptest.Server, func() []string) { + t.Helper() + var mu sync.Mutex + var seen []string + files := map[string]string{ + "/repos/arkadiyt/bounty-targets-data/commits": "commits.json", + "/arkadiyt/bounty-targets-data/" + testCommit + "/data/hackerone_data.json": "hackerone_data.json", + "/arkadiyt/bounty-targets-data/" + testCommit + "/data/bugcrowd_data.json": "bugcrowd_data.json", + "/arkadiyt/bounty-targets-data/" + testCommit + "/data/intigriti_data.json": "intigriti_data.json", + "/arkadiyt/bounty-targets-data/" + testCommit + "/data/yeswehack_data.json": "yeswehack_data.json", + "/repos/projectdiscovery/public-bugbounty-programs/commits": "commits.json", + "/projectdiscovery/public-bugbounty-programs/" + testCommit + "/dist/data.json": "pd-programs.json", + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + seen = append(seen, r.URL.Path) + mu.Unlock() + if r.URL.Path == "/robots.txt" { + http.NotFound(w, r) + return + } + f, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + b, err := os.ReadFile(filepath.Join("testdata", "datasets", f)) + if err != nil { + t.Error(err) + } + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), seen...) } +} + +func normalized(ps []model.Program) map[string]model.Program { + out := map[string]model.Program{} + for _, p := range ps { + model.Normalize(&p) + out[p.ID] = p + } + return out +} + +func TestBountyTargets(t *testing.T) { + srv, _ := datasetServer(t) + b := BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, Ref: "main", APIBase: srv.URL, RawBase: srv.URL}} + res, err := b.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + // h1: acme, quiet (wrong-host and handle-less skipped, plus a non-object); bc: 2; intigriti: umbrella, old; ywh: initrode, closed-one. + if len(res.Programs) != 8 || res.Skipped != 5 { + t.Fatalf("%d programs, %d skipped", len(res.Programs), res.Skipped) + } + ps := normalized(res.Programs) + acme := ps["bounty-targets:hackerone-acme"] + if acme.Platform != "hackerone" || acme.Type != model.TypeBounty || acme.Status != model.StatusOpen || !acme.ScopePublished || + acme.URL != "https://hackerone.com/acme" { + t.Fatalf("acme: %+v", acme) + } + pv := acme.Provenance + if pv.Dataset != BountyTargetsRepo || pv.DatasetCommit != testCommit || pv.OriginalPlatform != "hackerone" || + pv.OriginalURL != "https://hackerone.com/acme" || !strings.HasSuffix(pv.SourceURL, "/"+testCommit+"/data/hackerone_data.json") { + t.Errorf("provenance: %+v", pv) + } + got := map[string]string{} + for _, tg := range acme.InScope { + if tg.Confidence != model.ConfidencePublishedByPlatform { + t.Errorf("target %s has confidence %s", tg.Value, tg.Confidence) + } + got[tg.Value] = tg.Type + } + want := map[string]string{"*.acmecorp.net": "wildcard", "api.acme.io": "domain", "https://app.acme.io/v1": "url", + "151.101.8.0/24": "cidr", "com.acme.app": "mobile_app", "https://github.com/acme/tool": "source_repo"} + if len(got) != len(want) { + t.Errorf("in scope %v", got) + } + for v, typ := range want { + if got[v] != typ { + t.Errorf("%s: %q, want %q", v, got[v], typ) + } + } + rej := map[string]string{} + for _, r := range acme.Rejected { + rej[r.Value] = r.Reason + } + if rej["10.0.0.0/8"] == "" || rej["co.uk"] == "" { + t.Errorf("a private range and a public suffix must be rejected: %v", rej) + } + if len(acme.OutOfScope) != 1 || acme.OutOfScope[0].Value != "blog.acme.io" { + t.Errorf("out of scope %+v", acme.OutOfScope) + } + if quiet := ps["bounty-targets:hackerone-quiet"]; quiet.Status != model.StatusPaused || quiet.Type != model.TypeVDP { + t.Errorf("quiet: %+v", quiet) + } + if _, bad := ps["bounty-targets:hackerone-phish"]; bad { + t.Error("a program page outside the platform host must be skipped") + } + + gx := ps["bounty-targets:bugcrowd-globex"] + if gx.Rules.SafeHarbour != model.SafeHarbourFull || gx.Type != model.TypeBounty || gx.URL != "https://bugcrowd.com/engagements/globex" { + t.Errorf("globex: %+v", gx) + } + types := map[string]string{} + for _, tg := range gx.InScope { + types[tg.Value] = tg.Type + } + if types["Globex iOS"] != "mobile_app" || types["https://www.globex.io/"] != "url" || types["Globex Router X1"] != "other" || types["151.101.1.7"] != "ip" || types["api.globex.io"] != "domain" { + t.Errorf("globex targets: %v", types) + } + if len(gx.Rejected) != 1 || gx.Rejected[0].Value != "10.1.2.3" { + t.Errorf("documentation address must be rejected: %+v", gx.Rejected) + } + if v := ps["bounty-targets:bugcrowd-initech-vdp"]; v.Type != model.TypeVDP || v.Rules.SafeHarbour != model.SafeHarbourUnknown { + t.Errorf("initech: %+v", v) + } + + um := ps["bounty-targets:intigriti-umbrella-umbrellaprogram"] + if um.Type != model.TypeBounty || len(um.InScope) != 5 || len(um.OutOfScope) != 1 { + t.Fatalf("umbrella: %+v", um) + } + for _, tg := range um.InScope { + if tg.Value == "portal.umbrella.eu" && (tg.Eligible == nil || *tg.Eligible) { + t.Error("impact No Bounty must mean not eligible") + } + } + if _, bad := ps["bounty-targets:intigriti-hidden-secret"]; bad { + t.Error("a non-public program must be skipped") + } + if ps["bounty-targets:intigriti-old-gone"].Status != model.StatusPaused { + t.Error("a suspended program is paused") + } + + in := ps["bounty-targets:yeswehack-initrode"] + if in.URL != "https://yeswehack.com/programs/initrode" || in.Type != model.TypeBounty { + t.Errorf("initrode: %+v", in) + } + if len(in.Rejected) != 1 || in.Rejected[0].Value != "admin.internal" { + t.Errorf("an internal name must be rejected: %+v", in.Rejected) + } + if ps["bounty-targets:yeswehack-closed-one"].Status != model.StatusPaused { + t.Error("a disabled program is paused") + } + for id, p := range ps { + if err := p.Validate(); err == nil { + continue + } else if p.FirstSeen.IsZero() { + continue // timestamps are set by the collector + } else { + t.Errorf("%s: %v", id, err) + } + } +} + +func TestBountyTargetsOnlyDoesNotFetchOtherPlatforms(t *testing.T) { + srv, seen := datasetServer(t) + b := BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, Commit: testCommit, APIBase: srv.URL, RawBase: srv.URL}, Only: []string{"yeswehack"}} + res, err := b.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + if len(res.Programs) != 2 { + t.Errorf("%d programs", len(res.Programs)) + } + for _, p := range seen() { + if strings.Contains(p, "hackerone") || strings.Contains(p, "bugcrowd") || strings.Contains(p, "intigriti") { + t.Errorf("fetched %s although the platform is not wanted", p) + } + if strings.Contains(p, "/commits") { + t.Error("a pinned commit must not be resolved again") + } + } +} + +func TestBountyTargetsBadInputs(t *testing.T) { + srv, _ := datasetServer(t) + c := testClient(srv) + if _, err := (BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, Commit: "main", APIBase: srv.URL, RawBase: srv.URL}}).Collect(context.Background(), c); err == nil { + t.Error("a branch name is not a pinned commit") + } + // A dataset that changed shape fails the whole source. + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/commits"): + _, _ = w.Write([]byte(`[{"sha":"` + testCommit + `"}]`)) + case r.URL.Path == "/robots.txt": + http.NotFound(w, r) + default: + _, _ = w.Write([]byte(`{"programs": []}`)) + } + })) + defer bad.Close() + if _, err := (BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, APIBase: bad.URL, RawBase: bad.URL}}).Collect(context.Background(), testClient(bad)); err == nil { + t.Error("an object where an array is expected must fail the source") + } + notSHA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/robots.txt" { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(`[{"sha":"../../etc"}]`)) + })) + defer notSHA.Close() + if _, err := (BountyTargets{Dataset: githubDataset{Repo: BountyTargetsRepo, APIBase: notSHA.URL, RawBase: notSHA.URL}}).Collect(context.Background(), testClient(notSHA)); err == nil { + t.Error("a commit that is not 40 hex characters must be refused") + } +} + +func TestPDBounty(t *testing.T) { + srv, _ := datasetServer(t) + d := PDBounty{Dataset: githubDataset{Repo: PDBountyRepo, Ref: "main", APIBase: srv.URL, RawBase: srv.URL}} + res, err := d.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + if len(res.Programs) != 3 || res.Skipped != 2 { + t.Fatalf("%d programs, %d skipped", len(res.Programs), res.Skipped) + } + ps := normalized(res.Programs) + acme := ps["pd-bounty:acme"] + if acme.Platform != "hackerone" || acme.Type != model.TypeBounty || acme.ScopePublished || len(acme.InScope) != 2 { + t.Fatalf("acme: %+v", acme) + } + for _, tg := range acme.InScope { + if tg.Confidence != model.ConfidenceInferred { + t.Errorf("a community-curated domain is inferred, got %s", tg.Confidence) + } + } + if acme.Provenance.DatasetCommit != testCommit || acme.Provenance.OriginalPlatform != "hackerone" || acme.Provenance.Dataset != PDBountyRepo { + t.Errorf("provenance: %+v", acme.Provenance) + } + if ps["pd-bounty:4chan"].Platform != "self-hosted" || ps["pd-bounty:hosted-web3"].Platform != "immunefi" { + t.Error("platform is derived from the program page host") + } + only := PDBounty{Dataset: d.Dataset, Only: []string{"self-hosted"}} + res, err = only.Collect(context.Background(), testClient(srv)) + if err != nil || len(res.Programs) != 1 { + t.Errorf("only self-hosted: %v %d", err, len(res.Programs)) + } +} diff --git a/internal/source/diodb.go b/internal/source/diodb.go new file mode 100644 index 0000000..e9720d7 --- /dev/null +++ b/internal/source/diodb.go @@ -0,0 +1,225 @@ +package source + +import ( + "context" + "encoding/json" + "fmt" + "net/mail" + "net/url" + "sort" + "strconv" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// DioDBURL is the disclose.io program list (CC0-1.0). +const DioDBURL = "https://raw.githubusercontent.com/disclose/diodb/master/program-list.json" + +const ( + maxDioDBBytes = 32 << 20 + maxDioDBRecords = 50000 +) + +// DioDB reads the disclose.io program list: a public directory of +// self-hosted and platform-hosted programs with their policy URL, contact, +// bounty and safe-harbour flags. It carries no structured scope; the only +// target it yields is the host of a program's own security.txt, as an +// inferred target. +type DioDB struct { + URL string // DioDBURL unless a test overrides it +} + +// Meta implements Source. +func (d DioDB) Meta() Meta { + return Meta{ + Name: "disclose", + URL: d.url(), + Terms: "https://github.com/disclose/diodb/blob/master/LICENSE", + Licence: "CC0-1.0", + Attribution: "Program directory data from the disclose.io diodb project (CC0-1.0).", + } +} + +func (d DioDB) url() string { + if d.URL != "" { + return d.URL + } + return DioDBURL +} + +type dioRecord struct { + ProgramName string `json:"program_name"` + PolicyURL string `json:"policy_url"` + PolicyURLStatus string `json:"policy_url_status"` + ContactEmail string `json:"contact_email"` + ContactURL string `json:"contact_url"` + OffersBounty json.RawMessage `json:"offers_bounty"` + SafeHarbor json.RawMessage `json:"safe_harbor"` + SecurityTxtURL json.RawMessage `json:"securitytxt_url"` + PreferredLanguage json.RawMessage `json:"preferred_languages"` + DisclosureDays json.RawMessage `json:"disclosure_timeline_days"` +} + +// Hosts of platforms that host programs on their own site: a policy URL on +// one of them names the platform, never the program's own domain. +var platformHosts = map[string]string{ + "hackerone.com": "hackerone", "bugcrowd.com": "bugcrowd", "intigriti.com": "intigriti", + "yeswehack.com": "yeswehack", "federacy.com": "federacy", "openbugbounty.org": "openbugbounty", + "hackenproof.com": "hackenproof", "immunefi.com": "immunefi", "huntr.com": "huntr", "huntr.dev": "huntr", + "whitehub.net": "whitehub", "bugbase.ai": "bugbase", "zerocopter.com": "zerocopter", "hackrate.com": "hackrate", +} + +// Collect implements Source. +func (d DioDB) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + resp, err := c.Get(ctx, d.url(), maxDioDBBytes) + if err != nil { + return nil, err + } + var raw []json.RawMessage + if err := json.Unmarshal(resp.Body, &raw); err != nil { + return nil, fmt.Errorf("diodb: %w", err) + } + if len(raw) > maxDioDBRecords { + return nil, fmt.Errorf("diodb: %d records, over the cap of %d", len(raw), maxDioDBRecords) + } + res := &Result{} + recs := make([]dioRecord, 0, len(raw)) + names := make([]string, 0, len(raw)) + for _, r := range raw { + var rec dioRecord + if json.Unmarshal(r, &rec) != nil || strings.TrimSpace(rec.ProgramName) == "" || !httpURL(rec.PolicyURL) { + res.Skipped++ + continue + } + recs = append(recs, rec) + names = append(names, rec.ProgramName) + } + slugs := UniqueSlugs(names) + meta := d.Meta() + for _, rec := range recs { + p := model.Program{ + ID: meta.Name + ":" + slugs[rec.ProgramName], + Source: meta.Name, + Platform: platformOf(rec.PolicyURL), + Name: rec.ProgramName, + URL: strings.TrimSpace(rec.PolicyURL), + Type: model.TypeVDP, + Status: model.StatusOpen, + InScope: []model.Target{}, + OutOfScope: []model.Target{}, + Rules: model.Rules{ + SafeHarbour: harbour(str(rec.SafeHarbor)), + TestingRestrictions: []string{}, + RequiredHeaders: []string{}, + Languages: splitList(str(rec.PreferredLanguage)), + }, + Terms: model.Terms{URL: strings.TrimSpace(rec.PolicyURL)}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: meta.URL, FetchedAt: resp.FetchedAt}, + } + switch strings.ToLower(str(rec.OffersBounty)) { + case "yes", "true", "partial": + p.Type = model.TypeBounty + } + if strings.EqualFold(rec.PolicyURLStatus, "dead") { + p.Status = model.StatusPaused + p.Rules.Summary = "The policy URL did not answer when the directory last checked it." + } + if n, err := strconv.Atoi(str(rec.DisclosureDays)); err == nil && n > 0 && n <= 3650 { + p.Rules.DisclosureDays = n + } + if a, err := mail.ParseAddress(strings.TrimSpace(rec.ContactEmail)); err == nil && len(a.Address) <= 254 { + p.Contact.Email = a.Address + } + if u := strings.TrimSpace(rec.ContactURL); httpURL(u) || strings.HasPrefix(u, "mailto:") { + p.Contact.URL = u + } + if host := securityTxtHost(str(rec.SecurityTxtURL)); host != "" { + p.InScope = append(p.InScope, model.Target{Type: model.TargetDomain, Value: host, Confidence: model.ConfidenceInferred, + Notes: "Host that serves the program's security.txt; not a published scope."}) + } + res.Programs = append(res.Programs, p) + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + return res, nil +} + +// str reads a JSON value that the directory stores as a string, a bool, a +// number or null. +func str(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return strings.TrimSpace(s) + } + var b bool + if json.Unmarshal(raw, &b) == nil { + return strconv.FormatBool(b) + } + var n json.Number + if json.Unmarshal(raw, &n) == nil { + return n.String() + } + return "" +} + +func httpURL(s string) bool { + u, err := url.Parse(strings.TrimSpace(s)) + return err == nil && (u.Scheme == "https" || u.Scheme == "http") && u.Host != "" && u.User == nil +} + +func hostOf(s string) string { + u, err := url.Parse(strings.TrimSpace(s)) + if err != nil { + return "" + } + return strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.") +} + +func platformOf(policyURL string) string { + h := hostOf(policyURL) + for host, name := range platformHosts { + if h == host || strings.HasSuffix(h, "."+host) { + return name + } + } + return "self-hosted" +} + +// securityTxtHost returns the host of a first-party security.txt URL, or +// "" when the URL is not one (wrong path, or hosted on a platform). +func securityTxtHost(s string) string { + u, err := url.Parse(strings.TrimSpace(s)) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.User != nil || u.Port() != "" { + return "" + } + if u.Path != "/.well-known/security.txt" && u.Path != "/security.txt" { + return "" + } + if platformOf(s) != "self-hosted" { + return "" + } + return strings.ToLower(u.Hostname()) +} + +func harbour(s string) string { + switch strings.ToLower(s) { + case "full", "yes": + return model.SafeHarbourFull + case "partial": + return model.SafeHarbourPartial + case "none", "no": + return model.SafeHarbourNone + } + return model.SafeHarbourUnknown +} + +func splitList(s string) []string { + out := []string{} + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" { + out = append(out, p) + } + } + return out +} diff --git a/internal/source/pdbounty.go b/internal/source/pdbounty.go new file mode 100644 index 0000000..8cc12de --- /dev/null +++ b/internal/source/pdbounty.go @@ -0,0 +1,120 @@ +package source + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// PDBountyRepo is a community-curated list of public bug-bounty and +// disclosure programs with the domains each one covers. +const PDBountyRepo = "projectdiscovery/public-bugbounty-programs" + +// PDBounty reads that list. It carries the program page and a plain list of +// domains per program; the domains are curated by the community, not +// published by the program, so they are inferred targets (suggestions). The +// platform is derived from the program page; the publish policy decides +// per platform. +type PDBounty struct { + Dataset githubDataset + // Only keeps programs of these platforms (nil = all). + Only []string +} + +// NewPDBounty returns the adapter for the real dataset. +func NewPDBounty(only []string) PDBounty { + return PDBounty{Dataset: githubDataset{Repo: PDBountyRepo, Ref: "main"}, Only: only} +} + +func (d PDBounty) repo() string { + if d.Dataset.Repo != "" { + return d.Dataset.Repo + } + return PDBountyRepo +} + +// Meta implements Source. +func (d PDBounty) Meta() Meta { + return Meta{ + Name: "pd-bounty", + URL: "https://github.com/" + d.repo(), + Terms: "https://github.com/" + d.repo() + "/blob/main/LICENSE.md", + Licence: "MIT (list); program pages stay subject to the terms of the platform they name", + Attribution: "Program list curated by the " + d.repo() + " community project.", + } +} + +const pdFile = "dist/data.json" + +// Collect implements Source. +func (d PDBounty) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + commit, err := d.Dataset.resolve(ctx, c) + if err != nil { + return nil, err + } + fileURL := d.Dataset.rawURL(commit, pdFile) + resp, err := c.Get(ctx, fileURL, maxDatasetBytes) + if err != nil { + return nil, err + } + var doc struct { + Programs []json.RawMessage `json:"programs"` + } + if err := json.Unmarshal(resp.Body, &doc); err != nil { + return nil, fmt.Errorf("pd-bounty: %w", err) + } + if len(doc.Programs) > maxDatasetRecords { + return nil, fmt.Errorf("pd-bounty: %d records, over the cap of %d", len(doc.Programs), maxDatasetRecords) + } + type rec struct { + Name string `json:"name"` + URL string `json:"url"` + Bounty bool `json:"bounty"` + Domains []string `json:"domains"` + } + res := &Result{} + var recs []rec + var names []string + for _, raw := range doc.Programs { + var r rec + if json.Unmarshal(raw, &r) != nil || strings.TrimSpace(r.Name) == "" || !httpURL(r.URL) { + res.Skipped++ + continue + } + recs = append(recs, r) + names = append(names, r.Name) + } + slugs := UniqueSlugs(names) + meta := d.Meta() + for _, r := range recs { + platform := platformOf(r.URL) + if !wanted(d.Only, platform) { + continue + } + p := model.Program{ + ID: meta.Name + ":" + slugs[r.Name], Source: meta.Name, Platform: platform, Name: r.Name, URL: strings.TrimSpace(r.URL), + Type: model.TypeVDP, Status: model.StatusOpen, InScope: []model.Target{}, OutOfScope: []model.Target{}, + Rules: model.Rules{SafeHarbour: model.SafeHarbourUnknown, TestingRestrictions: []string{}, RequiredHeaders: []string{}, + Languages: []string{}}, + Terms: model.Terms{URL: strings.TrimSpace(r.URL)}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: fileURL, FetchedAt: resp.FetchedAt, Dataset: d.repo(), + DatasetCommit: commit, OriginalPlatform: platform, OriginalURL: strings.TrimSpace(r.URL)}, + } + if r.Bounty { + p.Type = model.TypeBounty + } + for _, dom := range r.Domains { + typ, val := classify(kindAuto, dom) + p.InScope = append(p.InScope, model.Target{Type: typ, Value: val, Confidence: model.ConfidenceInferred, + Notes: "Domain listed for the program by a community-curated list; not published by the program."}) + } + res.Programs = append(res.Programs, p) + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + return res, nil +} diff --git a/internal/source/securitytxt.go b/internal/source/securitytxt.go new file mode 100644 index 0000000..299c793 --- /dev/null +++ b/internal/source/securitytxt.go @@ -0,0 +1,411 @@ +package source + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "sort" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// DotGovFederalURL is CISA's list of US federal .gov domains (CC0-1.0). +const DotGovFederalURL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-federal.csv" + +const ( + maxSeedBytes = 8 << 20 + maxSeedDomains = 20000 + maxSecurityTxtBytes = 32 << 10 +) + +// Seed is one domain to look for a security.txt on, with the name of the +// organisation that holds it. +type Seed struct { + Domain string + Org string +} + +// SecurityTxt reads RFC 9116 security.txt files from a seed list of +// domains. A file with a Contact and a Policy field is a self-hosted +// disclosure program; the domain it was served from is an inferred target +// (security.txt does not publish a scope). +type SecurityTxt struct { + // SeedCSV is a CSV whose first column is a domain and third column the + // organisation (the dotgov-data layout); DotGovFederalURL by default. + SeedCSV string + // Extra are additional seeds (from the repository's seed file). + Extra []Seed + // Workers is the number of domains read at once (16 by default; each + // host still gets at most one request per MinInterval). + Workers int + // Progress, when set, is called after every 100 domains. + Progress func(done, total int) + // Scheme is "https" except in tests. + Scheme string + // Now is the time used to judge Expires (time.Now by default). + Now func() time.Time +} + +// Meta implements Source. +func (s SecurityTxt) Meta() Meta { + return Meta{ + Name: "securitytxt", + URL: s.seedURL(), + Terms: "https://www.rfc-editor.org/rfc/rfc9116", + Licence: "Seed list: CC0-1.0 (cisagov/dotgov-data). security.txt files: published by each domain for automated retrieval (RFC 9116).", + Attribution: "Domain seed list from CISA's dotgov-data (CC0-1.0); program data from each domain's own security.txt.", + } +} + +func (s SecurityTxt) seedURL() string { + if s.SeedCSV != "" { + return s.SeedCSV + } + return DotGovFederalURL +} + +// Collect implements Source. +func (s SecurityTxt) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { + resp, err := c.Get(ctx, s.seedURL(), maxSeedBytes) + if err != nil { + return nil, fmt.Errorf("seed list: %w", err) + } + seeds, err := ParseSeedCSV(resp.Body) + if err != nil { + return nil, err + } + seeds = append(seeds, s.Extra...) + byDomain := map[string]Seed{} + for _, sd := range seeds { + d, reason := model.NormalizeTarget(model.TargetDomain, sd.Domain) + if reason != "" || len(d) > 120 { + continue + } + if _, dup := byDomain[d]; !dup { + byDomain[d] = Seed{Domain: d, Org: sd.Org} + } + } + if len(byDomain) > maxSeedDomains { + return nil, fmt.Errorf("seed list: %d domains, over the cap of %d", len(byDomain), maxSeedDomains) + } + domains := make([]string, 0, len(byDomain)) + for d := range byDomain { + domains = append(domains, d) + } + sort.Strings(domains) + + workers := s.Workers + if workers <= 0 { + workers = 16 + } + now := time.Now + if s.Now != nil { + now = s.Now + } + meta := s.Meta() + res := &Result{} + var mu sync.Mutex + done := 0 + jobs := make(chan string) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for d := range jobs { + p, known, err := s.one(ctx, c, meta, byDomain[d], now()) + mu.Lock() + done++ + if s.Progress != nil && done%100 == 0 { + s.Progress(done, len(domains)) + } + switch { + case p != nil: + res.Programs = append(res.Programs, *p) + case err != nil && !known: + res.Unknown = append(res.Unknown, meta.Name+":"+d) + default: + res.Skipped++ + } + mu.Unlock() + } + }() + } + for _, d := range domains { + select { + case jobs <- d: + case <-ctx.Done(): + } + } + close(jobs) + wg.Wait() + if ctx.Err() != nil { + return nil, ctx.Err() + } + sort.Slice(res.Programs, func(i, j int) bool { return res.Programs[i].ID < res.Programs[j].ID }) + sort.Strings(res.Unknown) + return res, nil +} + +// one reads one domain. known reports whether the answer is definite (a +// file, or a definite "no file"); an indefinite failure keeps the +// previous record. +func (s SecurityTxt) one(ctx context.Context, c *fetch.Client, meta Meta, sd Seed, now time.Time) (*model.Program, bool, error) { + scheme := s.Scheme + if scheme == "" { + scheme = "https" + } + u := scheme + "://" + sd.Domain + "/.well-known/security.txt" + resp, err := c.Get(ctx, u, maxSecurityTxtBytes) + if err != nil { + var se *fetch.StatusError + definite := errors.Is(err, fetch.ErrNotFound) || errors.Is(err, fetch.ErrDisallowed) || + errors.Is(err, fetch.ErrTooLarge) || (errors.As(err, &se) && se.Code >= 400 && se.Code < 500) + return nil, definite, err + } + f, err := ParseSecurityTxt(resp.Body) + if err != nil { + return nil, true, err + } + if !ownedBy(f, resp.URL, sd.Domain) { + return nil, true, errors.New("security.txt is served for another host") + } + policy := firstHTTP(f.Policy) + if policy == "" { + return nil, true, errors.New("no Policy field") + } + sum := sha256.Sum256(resp.Body) + fetched := resp.FetchedAt + name := sd.Org + if name == "" { + name = sd.Domain + } else { + name += " (" + sd.Domain + ")" + } + p := &model.Program{ + ID: meta.Name + ":" + sd.Domain, + Source: meta.Name, + Platform: "self-hosted", + Name: name, + URL: policy, + Type: model.TypeVDP, + Status: model.StatusOpen, + InScope: []model.Target{{Type: model.TargetDomain, Value: sd.Domain, Confidence: model.ConfidenceInferred, + Notes: "Domain whose security.txt names this policy; not a published scope."}}, + OutOfScope: []model.Target{}, + Rules: model.Rules{SafeHarbour: model.SafeHarbourUnknown, TestingRestrictions: []string{}, + RequiredHeaders: []string{}, Languages: splitList(strings.Join(f.Languages, ","))}, + Terms: model.Terms{URL: u, SHA256: hex.EncodeToString(sum[:]), FetchedAt: &fetched}, + Provenance: model.Provenance{Source: meta.Name, SourceURL: u, FetchedAt: resp.FetchedAt}, + } + if !f.Expires.IsZero() && f.Expires.Before(now) { + p.Status = model.StatusPaused + p.Rules.Summary = "The security.txt expired on " + f.Expires.UTC().Format("2006-01-02") + "." + } + for _, ct := range f.Contact { + switch { + case strings.HasPrefix(ct, "mailto:") && p.Contact.Email == "": + if addr := strings.TrimPrefix(ct, "mailto:"); validEmail(addr) { + p.Contact.Email = addr + } + case strings.HasPrefix(ct, "https://") && p.Contact.URL == "": + p.Contact.URL = ct + } + } + return p, true, nil +} + +// SecurityTxtFile holds the fields of a security.txt the collector uses. +type SecurityTxtFile struct { + Contact []string + Policy []string + Canonical []string + Languages []string + Expires time.Time + CVDPolicy []string // draft-behring-cvd-policy; recorded, not yet parsed +} + +// ParseSecurityTxt parses an RFC 9116 file (a cleartext-signed one too; the +// signature is not checked). It refuses HTML and files without a Contact. +func ParseSecurityTxt(body []byte) (*SecurityTxtFile, error) { + if !utf8.Valid(body) { + return nil, errors.New("security.txt: not UTF-8") + } + text := string(body) + if l := strings.ToLower(strings.TrimSpace(text)); strings.HasPrefix(l, "<") || strings.Contains(l, " 2048 { + continue + } + switch strings.ToLower(strings.TrimSpace(k)) { + case "contact": + f.Contact = appendCapped(f.Contact, v) + case "policy": + f.Policy = appendCapped(f.Policy, v) + case "canonical": + f.Canonical = appendCapped(f.Canonical, v) + case "preferred-languages": + f.Languages = appendCapped(f.Languages, v) + case "cvd-policy": + f.CVDPolicy = appendCapped(f.CVDPolicy, v) + case "expires": + if t, err := time.Parse(time.RFC3339, v); err == nil && f.Expires.IsZero() { + f.Expires = t + } + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("security.txt: %w", err) + } + if len(f.Contact) == 0 { + return nil, errors.New("security.txt: no Contact field") + } + return f, nil +} + +func appendCapped(l []string, v string) []string { + if len(l) >= 16 { + return l + } + return append(l, v) +} + +func stripClearsign(text string) string { + _, rest, ok := strings.Cut(text, "\n\n") + if !ok { + _, rest, _ = strings.Cut(text, "\r\n\r\n") + } + if i := strings.Index(rest, "-----BEGIN PGP SIGNATURE-----"); i >= 0 { + rest = rest[:i] + } + var b strings.Builder + for _, line := range strings.Split(rest, "\n") { + b.WriteString(strings.TrimPrefix(line, "- ")) + b.WriteByte('\n') + } + return b.String() +} + +// ownedBy: the file was served from the seed domain, or one of its +// Canonical URLs is on the seed domain. A file reached through a redirect +// to another organisation's host, with no Canonical naming the seed domain, +// is that other host's file. +func ownedBy(f *SecurityTxtFile, finalURL, domain string) bool { + if hostIs(finalURL, domain) { + return true + } + for _, c := range f.Canonical { + if hostIs(c, domain) { + return true + } + } + return false +} + +func hostIs(raw, domain string) bool { + u, err := url.Parse(raw) + return err == nil && strings.EqualFold(strings.TrimSuffix(u.Hostname(), "."), domain) +} + +func firstHTTP(l []string) string { + for _, v := range l { + if u, err := url.Parse(v); err == nil && u.Scheme == "https" && u.Host != "" && u.User == nil && len(v) <= model.MaxURLLen && !strings.ContainsAny(v, " \t") { + return v + } + } + return "" +} + +func validEmail(s string) bool { + if len(s) > 254 || strings.ContainsAny(s, " <>\"") { + return false + } + at := strings.LastIndexByte(s, '@') + return at > 0 && at < len(s)-3 && strings.Contains(s[at:], ".") +} + +// ParseSeedCSV reads a seed CSV: header row, domain in column 1, the +// organisation in column 3 (and a sub-organisation in column 4, if any). +func ParseSeedCSV(body []byte) ([]Seed, error) { + r := csv.NewReader(bytes.NewReader(body)) + r.FieldsPerRecord = -1 + r.ReuseRecord = true + var out []Seed + first := true + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("seed list: %w", err) + } + if first { + first = false + if len(rec) > 0 && strings.Contains(strings.ToLower(rec[0]), "domain") { + continue + } + } + if len(rec) == 0 || strings.TrimSpace(rec[0]) == "" { + continue + } + sd := Seed{Domain: strings.TrimSpace(rec[0])} + if len(rec) > 2 { + sd.Org = strings.TrimSpace(rec[2]) + } + if len(rec) > 3 && strings.TrimSpace(rec[3]) != "" { + sd.Org += " / " + strings.TrimSpace(rec[3]) + } + out = append(out, sd) + if len(out) > maxSeedDomains { + return nil, fmt.Errorf("seed list: over %d domains", maxSeedDomains) + } + } + return out, nil +} + +// ParseSeedFile reads the repository seed file: one domain per line, +// optionally followed by whitespace and an organisation name; # comments. +func ParseSeedFile(body []byte) []Seed { + var out []Seed + for _, line := range strings.Split(string(body), "\n") { + if i := strings.IndexByte(line, '#'); i >= 0 { + line = line[:i] + } + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + out = append(out, Seed{Domain: fields[0], Org: strings.Join(fields[1:], " ")}) + } + return out +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000..696a1bf --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,78 @@ +// Package source holds the adapters, one per upstream source. An adapter +// reads its source through the polite fetch client only, and turns what it +// read into programs; it never validates scope itself (the collector +// normalises every program the same way) and never sets timestamps other +// than provenance. +package source + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// Meta describes a source and its terms; it is carried in every manifest. +type Meta struct { + Name string // id prefix of its programs, [a-z0-9-] + URL string + Terms string + Licence string + Attribution string +} + +// Result is what one run of an adapter produced. +type Result struct { + Programs []model.Program + // Unknown lists ids whose state could not be read this run (a host + // that did not answer, for example). The collector keeps their + // previous record instead of closing them. + Unknown []string + // Skipped counts source records that were not programs (no policy, + // malformed record …), for the log. + Skipped int +} + +// Source is one adapter. +type Source interface { + Meta() Meta + // Collect reads the whole source. An error means the source as a + // whole could not be read; the collector then keeps every previous + // record of the source unchanged. + Collect(ctx context.Context, c *fetch.Client) (*Result, error) +} + +var slugRE = regexp.MustCompile(`[^a-z0-9]+`) + +// Slug turns a name into an id slug ([a-z0-9-], at most 100 characters). +func Slug(name string) string { + s := strings.Trim(slugRE.ReplaceAllString(strings.ToLower(name), "-"), "-") + if len(s) > 100 { + s = strings.TrimRight(s[:100], "-") + } + return s +} + +// UniqueSlugs gives every name a stable slug; names whose slugs collide +// (or are empty) all get a short hash of the name appended, so the result +// does not depend on the order of the source. +func UniqueSlugs(names []string) map[string]string { + count := map[string]int{} + for _, n := range names { + count[Slug(n)]++ + } + out := make(map[string]string, len(names)) + for _, n := range names { + s := Slug(n) + if count[s] > 1 || s == "" { + sum := sha256.Sum256([]byte(n)) + s = strings.TrimLeft(s+"-"+hex.EncodeToString(sum[:4]), "-") + } + out[n] = s + } + return out +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go new file mode 100644 index 0000000..6557a3c --- /dev/null +++ b/internal/source/source_test.go @@ -0,0 +1,200 @@ +package source + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" +) + +// testClient sends every request, whatever its host, to srv. +func testClient(srv *httptest.Server) *fetch.Client { + return fetch.New(fetch.Options{UserAgent: fetch.UserAgent("test"), MinInterval: time.Millisecond, AllowPrivate: true, + DialOverride: srv.Listener.Addr().String(), Timeout: 5 * time.Second}) +} + +func byID(ps []model.Program) map[string]model.Program { + out := map[string]model.Program{} + for _, p := range ps { + out[p.ID] = p + } + return out +} + +func TestDioDB(t *testing.T) { + body, err := os.ReadFile("testdata/diodb.json") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/list.json" { + _, _ = w.Write(body) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + res, err := DioDB{URL: srv.URL + "/list.json"}.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + if len(res.Programs) != 12 || res.Skipped != 5 { + t.Fatalf("%d programs, %d skipped", len(res.Programs), res.Skipped) + } + ps := byID(res.Programs) + ada := ps["disclose:ada-health"] + if len(ada.InScope) != 1 || ada.InScope[0].Value != "ada.com" || ada.InScope[0].Confidence != model.ConfidenceInferred || ada.ScopePublished { + t.Errorf("ada: %+v", ada.InScope) + } + yt := ps["disclose:youtube"] + if yt.Type != model.TypeBounty || yt.Platform != "self-hosted" || len(yt.InScope) != 0 || yt.Rules.SafeHarbour != model.SafeHarbourPartial { + t.Errorf("youtube: %+v", yt) + } + ph := ps["disclose:platform-hosted"] + if ph.Platform != "hackerone" || ph.Type != model.TypeBounty || len(ph.InScope) != 0 { + t.Errorf("a security.txt on a platform host must not become a target: %+v", ph) + } + if len(ps["disclose:odd-path-txt"].InScope) != 0 { + t.Error("a non-standard security.txt path must not become a target") + } + dead := ps["disclose:dead-link"] + if dead.Status != model.StatusPaused || dead.Type != model.TypeBounty || dead.Contact != (model.Contact{}) || + dead.Rules.DisclosureDays != 90 || strings.Join(dead.Rules.Languages, ",") != "en,de" { + t.Errorf("dead link: %+v", dead) + } + dups := 0 + for id := range ps { + if strings.HasPrefix(id, "disclose:duplicate-") { + dups++ + } + } + if dups != 2 { + t.Errorf("colliding slugs must both get a hash suffix (%d)", dups) + } + // The public-suffix target is passed on; the collector's Normalize + // rejects it. + pst := ps["disclose:public-suffix-txt"] + model.Normalize(&pst) + if len(pst.InScope) != 0 || len(pst.Rejected) != 1 { + t.Errorf("public suffix: %+v / %+v", pst.InScope, pst.Rejected) + } +} + +const goodTxt = `# Our policy +Contact: mailto:security@agency.gov +Contact: https://agency.gov/report +Expires: 2099-01-01T00:00:00Z +Policy: https://agency.gov/vulnerability-disclosure-policy +Preferred-Languages: en, es +Canonical: https://agency.gov/.well-known/security.txt +` + +func TestParseSecurityTxt(t *testing.T) { + f, err := ParseSecurityTxt([]byte(goodTxt)) + if err != nil || len(f.Contact) != 2 || f.Policy[0] != "https://agency.gov/vulnerability-disclosure-policy" || f.Expires.Year() != 2099 { + t.Fatalf("%v %+v", err, f) + } + signed := "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA256\n\n" + strings.ReplaceAll(goodTxt, "# Our", "- # Our") + + "-----BEGIN PGP SIGNATURE-----\nabc\n-----END PGP SIGNATURE-----\n" + if f, err := ParseSecurityTxt([]byte(signed)); err != nil || len(f.Policy) != 1 { + t.Errorf("signed: %v", err) + } + for name, body := range map[string]string{ + "html": "Contact: x", + "no contact": "Policy: https://a.gov/p\n", + "binary": "\xff\xfe\x00", + } { + if _, err := ParseSecurityTxt([]byte(body)); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestSecurityTxt(t *testing.T) { + seed := "Domain name,Domain type,Organization name,Suborganization name\n" + + "agency.gov,Federal,Agency,Office\nnopolicy.gov,Federal,NoPolicy,\nhtml.gov,Federal,Html,\nmissing.gov,Federal,Missing,\n" + + "down.gov,Federal,Down,\nredirect.gov,Federal,Redirect,\nexpired.gov,Federal,Expired,\nrobots.gov,Federal,Robots,\n" + + "localhost,Federal,Bad,\n10.0.0.1,Federal,Bad,\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host, _, _ := net.SplitHostPort(r.Host) + if host == "" { + host = r.Host + } + if r.URL.Path == "/robots.txt" { + if host == "robots.gov" { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /.well-known/\n")) + return + } + http.NotFound(w, r) + return + } + if r.URL.Path == "/seed.csv" { + _, _ = w.Write([]byte(seed)) + return + } + switch host { + case "agency.gov", "robots.gov": + _, _ = w.Write([]byte(goodTxt)) + case "nopolicy.gov": + _, _ = w.Write([]byte("Contact: mailto:a@nopolicy.gov\n")) + case "html.gov": + _, _ = w.Write([]byte("not found")) + case "down.gov": + w.WriteHeader(http.StatusBadGateway) + case "redirect.gov": + http.Redirect(w, r, "http://other.gov/.well-known/security.txt", http.StatusFound) + case "other.gov": + _, _ = w.Write([]byte("Contact: mailto:a@other.gov\nPolicy: https://other.gov/p\n")) + case "expired.gov": + _, _ = w.Write([]byte("Contact: mailto:a@expired.gov\nExpires: 2020-01-01T00:00:00Z\nPolicy: https://expired.gov/p\n")) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + st := SecurityTxt{SeedCSV: "http://seed.example.net/seed.csv", Scheme: "http", Workers: 3} + res, err := st.Collect(context.Background(), testClient(srv)) + if err != nil { + t.Fatal(err) + } + ps := byID(res.Programs) + if len(ps) != 2 { + t.Fatalf("programs: %v", keys(ps)) + } + a := ps["securitytxt:agency.gov"] + if a.Name != "Agency / Office (agency.gov)" || a.URL != "https://agency.gov/vulnerability-disclosure-policy" || + a.Contact.Email != "security@agency.gov" || a.Contact.URL != "https://agency.gov/report" || a.Terms.SHA256 == "" || + len(a.InScope) != 1 || a.InScope[0].Value != "agency.gov" || a.InScope[0].Confidence != model.ConfidenceInferred || a.Status != model.StatusOpen { + t.Errorf("agency: %+v", a) + } + if ps["securitytxt:expired.gov"].Status != model.StatusPaused { + t.Error("an expired security.txt must give a paused program") + } + // A host that did not answer is unknown (its previous record is kept); + // a definite "no file" is not. + if strings.Join(res.Unknown, ",") != "securitytxt:down.gov" { + t.Errorf("unknown = %v", res.Unknown) + } +} + +func keys(m map[string]model.Program) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} + +func TestSlugs(t *testing.T) { + s := UniqueSlugs([]string{"Acme Corp", "acme-corp", "Zeta", "!!!"}) + if s["Zeta"] != "zeta" || s["Acme Corp"] == s["acme-corp"] || !strings.HasPrefix(s["Acme Corp"], "acme-corp-") || s["!!!"] == "" { + t.Errorf("%v", s) + } +} diff --git a/internal/source/testdata/datasets/bugcrowd_data.json b/internal/source/testdata/datasets/bugcrowd_data.json new file mode 100644 index 0000000..23dea0a --- /dev/null +++ b/internal/source/testdata/datasets/bugcrowd_data.json @@ -0,0 +1,13 @@ +[ +{"name":"Globex Bug Bounty","url":"https://bugcrowd.com/engagements/globex","allows_disclosure":true,"managed_by_bugcrowd":true,"safe_harbor":"full","max_payout":5000,"targets":{ + "in_scope":[ + {"type":"website","target":"https://www.globex.io","uri":"https://www.globex.io","name":"www","ipAddress":null}, + {"type":"api","target":"api.globex.io","uri":"","name":"api","ipAddress":null}, + {"type":"ios","target":"Globex iOS","uri":"","name":"Globex iOS","ipAddress":null}, + {"type":"hardware","target":"Globex Router X1","uri":"","name":"Globex Router X1","ipAddress":null}, + {"type":"ip_address","target":"151.101.1.7","uri":"","name":"x","ipAddress":null}, + {"type":"ip_address","target":"10.1.2.3","uri":"","name":"x","ipAddress":null} + ], + "out_of_scope":[{"type":"website","target":"status.globex.io","uri":"","name":"status","ipAddress":null}]}}, +{"name":"Initech VDP","url":"https://bugcrowd.com/initech-vdp","allows_disclosure":false,"managed_by_bugcrowd":true,"safe_harbor":null,"max_payout":0,"targets":{"in_scope":[{"type":"website","target":"initech-corp.net","uri":"","name":"x","ipAddress":null}],"out_of_scope":[]}} +] diff --git a/internal/source/testdata/datasets/commits.json b/internal/source/testdata/datasets/commits.json new file mode 100644 index 0000000..fdf9766 --- /dev/null +++ b/internal/source/testdata/datasets/commits.json @@ -0,0 +1 @@ +[{"sha":"0123456789abcdef0123456789abcdef01234567","commit":{"message":"data update"}}] diff --git a/internal/source/testdata/datasets/hackerone_data.json b/internal/source/testdata/datasets/hackerone_data.json new file mode 100644 index 0000000..35d752e --- /dev/null +++ b/internal/source/testdata/datasets/hackerone_data.json @@ -0,0 +1,18 @@ +[ +{"handle":"acme","id":1,"name":"Acme","offers_bounties":true,"submission_state":"open","url":"https://hackerone.com/acme","targets":{ + "in_scope":[ + {"asset_identifier":"*.acmecorp.net","asset_type":"WILDCARD","eligible_for_bounty":true,"instruction":"","max_severity":"critical"}, + {"asset_identifier":"api.acme.io","asset_type":"URL","eligible_for_bounty":true,"instruction":"Rate limit 5 req/s","max_severity":"high"}, + {"asset_identifier":"https://app.acme.io/v1","asset_type":"URL","eligible_for_bounty":false,"instruction":"","max_severity":"medium"}, + {"asset_identifier":"151.101.8.0/24","asset_type":"CIDR","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, + {"asset_identifier":"10.0.0.0/8","asset_type":"CIDR","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, + {"asset_identifier":"co.uk","asset_type":"URL","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, + {"asset_identifier":"com.acme.app","asset_type":"GOOGLE_PLAY_APP_ID","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, + {"asset_identifier":"https://github.com/acme/tool","asset_type":"SOURCE_CODE","eligible_for_bounty":true,"instruction":"","max_severity":"critical"} + ], + "out_of_scope":[{"asset_identifier":"blog.acme.io","asset_type":"URL","eligible_for_bounty":false,"instruction":"","max_severity":null}]}}, +{"handle":"quiet","id":2,"name":"Quiet VDP","offers_bounties":false,"submission_state":"paused","url":"https://hackerone.com/quiet","targets":{"in_scope":[],"out_of_scope":[]}}, +{"handle":"phish","id":3,"name":"Wrong Host","offers_bounties":true,"submission_state":"open","url":"https://evil.example.net/phish","targets":{"in_scope":[],"out_of_scope":[]}}, +{"handle":"","id":4,"name":"No handle","url":"https://hackerone.com/x"}, +"not an object" +] diff --git a/internal/source/testdata/datasets/intigriti_data.json b/internal/source/testdata/datasets/intigriti_data.json new file mode 100644 index 0000000..a6554b0 --- /dev/null +++ b/internal/source/testdata/datasets/intigriti_data.json @@ -0,0 +1,13 @@ +[ +{"id":"a1","name":"Umbrella","company_handle":"umbrella","handle":"umbrellaprogram","url":"https://www.intigriti.com/programs/umbrella/umbrellaprogram/detail","status":"open","confidentiality_level":"public","min_bounty":{"value":50,"currency":"EUR"},"max_bounty":{"value":2000,"currency":"EUR"},"targets":{ + "in_scope":[ + {"type":"wildcard","endpoint":"*.umbrella.eu","description":"Main platform","impact":"Tier 1"}, + {"type":"url","endpoint":"portal.umbrella.eu","description":null,"impact":"No Bounty"}, + {"type":"iprange","endpoint":"151.101.0.0/22","description":null,"impact":"Tier 2"}, + {"type":"android","endpoint":"eu.umbrella.app","description":null,"impact":"Tier 2"}, + {"type":"other","endpoint":"All other digital assets owned by Umbrella","description":null,"impact":"Tier 3"} + ], + "out_of_scope":[{"type":"wildcard","endpoint":"*.shop.umbrella.eu","description":null,"impact":"Out of scope"}]}}, +{"id":"a2","name":"Private one","company_handle":"hidden","handle":"secret","url":"https://www.intigriti.com/programs/hidden/secret/detail","status":"open","confidentiality_level":"application","max_bounty":{"value":100,"currency":"EUR"},"targets":{"in_scope":[],"out_of_scope":[]}}, +{"id":"a3","name":"Closed","company_handle":"old","handle":"gone","url":"https://www.intigriti.com/programs/old/gone/detail","status":"suspended","confidentiality_level":"public","max_bounty":{"value":0,"currency":"EUR"},"targets":{"in_scope":[],"out_of_scope":[]}} +] diff --git a/internal/source/testdata/datasets/pd-programs.json b/internal/source/testdata/datasets/pd-programs.json new file mode 100644 index 0000000..8efda88 --- /dev/null +++ b/internal/source/testdata/datasets/pd-programs.json @@ -0,0 +1,7 @@ +{"programs":[ +{"name":"Acme","url":"https://hackerone.com/acme","bounty":true,"domains":["acme.io","acmecorp.net"]}, +{"name":"4chan","url":"https://www.4chan.org/security","bounty":false,"swag":true,"domains":["4chan.org","4cdn.org"]}, +{"name":"Hosted Web3","url":"https://immunefi.com/bug-bounty/hosted/","bounty":true,"domains":[]}, +{"name":"","url":"https://example.com"}, +{"name":"No URL","url":"ftp://x"} +]} diff --git a/internal/source/testdata/datasets/yeswehack_data.json b/internal/source/testdata/datasets/yeswehack_data.json new file mode 100644 index 0000000..d2e4055 --- /dev/null +++ b/internal/source/testdata/datasets/yeswehack_data.json @@ -0,0 +1,12 @@ +[ +{"id":"initrode","name":"Initrode","public":true,"disabled":false,"managed":null,"min_bounty":50,"max_bounty":5000,"targets":{ + "in_scope":[ + {"target":"*.initrode.com","type":"wildcard"}, + {"target":"https://cockpit.initrode.com/","type":"web-application"}, + {"target":"Initrode Android","type":"mobile-application-android"}, + {"target":"admin.internal","type":"web-application"} + ], + "out_of_scope":[{"target":"Any social engineering","type":"other"}]}}, +{"id":"closed-one","name":"Closed one","public":true,"disabled":true,"max_bounty":0,"targets":{"in_scope":[],"out_of_scope":[]}}, +{"id":"private-one","name":"Private","public":false,"max_bounty":10,"targets":{"in_scope":[],"out_of_scope":[]}} +] diff --git a/internal/source/testdata/diodb.json b/internal/source/testdata/diodb.json new file mode 100644 index 0000000..68d716c --- /dev/null +++ b/internal/source/testdata/diodb.json @@ -0,0 +1,138 @@ +[ + { + "program_name": ".nz Registry", + "policy_url": "https://registry.internetnz.nz/about/vulnerability-disclosure-policy/", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": false, + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive", + "contact_email": "security@internetnz.net.nz" + }, + { + "program_name": "Ashby", + "policy_url": "https://www.ashbyhq.com/resources/vulnerability-disclosure", + "contact_email": "security@ashbyhq.com", + "offers_bounty": "no", + "offers_swag": false, + "pgp_key": "", + "hall_of_fame": "", + "hiring": "", + "launch_date": "", + "policy_url_status": "alive", + "preferred_languages": "", + "public_disclosure": "", + "safe_harbor": "", + "securitytxt_url": "" + }, + { + "program_name": "Firebase", + "policy_url": "https://g.co/vrp", + "contact_url": "https://g.co/vulnz", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": true, + "hall_of_fame": "", + "safe_harbor": "none", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive" + }, + { + "program_name": "YouTube", + "policy_url": "https://g.co/vrp", + "contact_url": "https://g.co/vulnz", + "launch_date": "", + "offers_bounty": "yes", + "offers_swag": "", + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "", + "hiring": "", + "securitytxt_url": "", + "preferred_languages": "", + "policy_url_status": "alive" + }, + { + "program_name": "Ada Health", + "policy_url": "https://ada.com/security-vulnerability-disclosure-policy/", + "contact_url": "https://ada.com/security-vulnerability-disclosure-policy/", + "launch_date": "", + "offers_bounty": "no", + "offers_swag": false, + "hall_of_fame": "", + "safe_harbor": "partial", + "public_disclosure": "", + "pgp_key": "https://ada.com/pgp-key.txt", + "hiring": "https:/ada.com/careers/", + "securitytxt_url": "https://ada.com/.well-known/security.txt", + "preferred_languages": "en, de", + "policy_url_status": "alive", + "contact_email": "security@ada.com" + }, + { + "program_name": "", + "policy_url": "https://nameless.org/security" + }, + { + "program_name": "Script Policy", + "policy_url": "javascript:alert(1)" + }, + { + "program_name": "No Policy" + }, + { + "program_name": "Platform Hosted", + "policy_url": "https://hackerone.com/platform-hosted", + "securitytxt_url": "https://hackerone.com/.well-known/security.txt", + "offers_bounty": true, + "safe_harbor": "full" + }, + { + "program_name": "Public Suffix Txt", + "policy_url": "https://blog.github.io/policy", + "securitytxt_url": "https://github.io/.well-known/security.txt" + }, + { + "program_name": "Odd Path Txt", + "policy_url": "https://oddpath.org/policy", + "securitytxt_url": "https://oddpath.org/files/security.txt" + }, + { + "program_name": "Dead Link", + "policy_url": "https://dead.org/policy", + "policy_url_status": "dead", + "offers_bounty": "partial", + "contact_email": "not an email", + "contact_url": "javascript:x", + "disclosure_timeline_days": "90", + "preferred_languages": "en, de ,," + }, + { + "program_name": "Duplicate", + "policy_url": "https://dup-one.org/policy" + }, + { + "program_name": "duplicate!", + "policy_url": "https://dup-two.org/policy" + }, + { + "program_name": "Bell\u0007Name", + "policy_url": "https://bell.org/policy" + }, + "not an object", + { + "program_name": 42, + "policy_url": "https://number.org/policy" + } +] diff --git a/keys/README.md b/keys/README.md new file mode 100644 index 0000000..6be20a1 --- /dev/null +++ b/keys/README.md @@ -0,0 +1,49 @@ +# Keys + +This directory holds only public material: + +| File | What | +|---|---| +| `root-keyid.txt` | key id of the offline root (`SHA256:…`); platforms pin it | +| `keyset.dsse.json` | the current key set, signed by the root | + +## First setup (maintainer, on an offline machine) + +```sh +go build -o programfeed ./cmd/programfeed + +# 1. The offline root. Keep root.key offline (hardware-backed storage or an +# encrypted offline medium, plus a backup). It never goes to CI. +./programfeed keys root-init --out root.key # prints the root key id + +# 2. The online signing key. +./programfeed keys gen --out signing.key # prints its public key + +# 3. The key set: the root allows the online key for up to 180 days. +./programfeed keys sign-keyset --root root.key --key --version 1 --days 180 --out keys/keyset.dsse.json +``` + +Then: + +- commit `keys/root-keyid.txt` (the root key id) and `keys/keyset.dsse.json`; +- create the GitHub environment **`publish`**: deployment branches limited + to the default branch; add the secret **`PROGRAMFEED_SIGNING_KEY`** with the + content of `signing.key`; delete `signing.key` from the machine; +- give OpenCTEM platforms the root key id (`PROGRAMFEED_ROOT_KEY_ID`). + +Use a root distinct from the vulnerability feed's root: one compromised root +then cannot sign for both feeds. + +Until `keys/root-keyid.txt` and `keys/keyset.dsse.json` are committed, the +scheduled publish workflow stops at its first step with an error saying so. + +## Rotation and revocation + +Before the key set expires, or at once if the online key may have leaked: +generate a new online key, sign a key set with a **higher version** listing +only the keys still trusted, commit it and replace the environment secret. +Platforms that have seen the new version refuse the older key set from then +on. + +If the root itself is compromised, generate a new root and have every +platform pin its new key id. diff --git a/policy/platforms.json b/policy/platforms.json new file mode 100644 index 0000000..bf6d052 --- /dev/null +++ b/policy/platforms.json @@ -0,0 +1,75 @@ +{ + "takedown_contact": "security@openctem.io", + "platforms": { + "self-hosted": { + "publish": true, + "attribution": "Programs that publish their own policy or security.txt.", + "reason": "The policy page is the program owner own page; the feed links to it and records facts about it." + }, + "hackerone": { + "publish": false, + "attribution": "Program data from HackerOne public program pages.", + "reason": "General Terms allow use only in a permitted manner and no redistribution grant exists. Needs written permission." + }, + "bugcrowd": { + "publish": false, + "attribution": "Program data from Bugcrowd public program pages.", + "reason": "Website terms forbid copying and redistribution of any portion of the website. Needs written permission." + }, + "intigriti": { + "publish": false, + "attribution": "Program data from Intigriti public program pages.", + "reason": "No redistribution grant found; the official API needs an account token. Needs written permission." + }, + "yeswehack": { + "publish": false, + "attribution": "Program data from YesWeHack public program pages.", + "reason": "Legal notices forbid redistributing the platform data. Needs written permission." + }, + "immunefi": { + "publish": false, + "attribution": "Program data from Immunefi public program pages.", + "reason": "Terms of use forbid automated access without written consent. Needs written permission." + }, + "hackenproof": { + "publish": false, + "attribution": "Program data from HackenProof public program pages.", + "reason": "No official machine-readable listing or redistribution grant. Needs written permission." + }, + "federacy": { + "publish": false, + "attribution": "Program data from Federacy public program pages.", + "reason": "No redistribution grant found." + }, + "openbugbounty": { + "publish": false, + "attribution": "Program data from Open Bug Bounty.", + "reason": "Listing is behind bot protection; no redistribution grant found." + }, + "huntr": { + "publish": false, + "attribution": "Program data from huntr.", + "reason": "No redistribution grant found." + }, + "whitehub": { + "publish": false, + "attribution": "Program data from WhiteHub.", + "reason": "No official listing or redistribution grant found." + }, + "bugbase": { + "publish": false, + "attribution": "Program data from BugBase.", + "reason": "No official listing or redistribution grant found." + }, + "zerocopter": { + "publish": false, + "attribution": "Program data from Zerocopter.", + "reason": "No redistribution grant found." + }, + "hackrate": { + "publish": false, + "attribution": "Program data from Hackrate.", + "reason": "No redistribution grant found." + } + } +} diff --git a/policy/policy.go b/policy/policy.go new file mode 100644 index 0000000..45aff0c --- /dev/null +++ b/policy/policy.go @@ -0,0 +1,207 @@ +// Package policy decides, per original platform, whether records read from +// a source may go into the signed public feed. +// +// The collector may read every source, but only platforms with +// publish=true reach the signed output. A platform that is not listed is +// not published. The rule is applied to the finished snapshot (Filter) and +// to the previous release (PrunePrev), so switching a platform to +// publish=false removes its records from the feed on the next run instead +// of keeping them as "unchanged". +package policy + +import ( + "bytes" + _ "embed" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/model" +) + +//go:embed platforms.json +var defaultJSON []byte + +// SelfHosted is the platform name of programs that host their own policy. +const SelfHosted = "self-hosted" + +var nameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) + +// Platform is the publish policy of one original platform. +type Platform struct { + Publish bool `json:"publish"` + Attribution string `json:"attribution"` + TakedownContact string `json:"takedown_contact,omitempty"` // defaults to the policy-wide contact + Reason string `json:"reason,omitempty"` +} + +// Policy is the whole configuration. +type Policy struct { + TakedownContact string `json:"takedown_contact"` + Platforms map[string]Platform `json:"platforms"` + // ExcludedPrograms are program ids removed on request of the program + // owner (TAKEDOWN.md); they are never published, whatever their platform. + ExcludedPrograms []string `json:"excluded_programs,omitempty"` +} + +// Default returns the policy compiled into the binary. +func Default() *Policy { + p, err := Parse(defaultJSON) + if err != nil { + panic("policy: embedded platforms.json: " + err.Error()) + } + return p +} + +// Parse reads and checks a policy; unknown fields are an error. +func Parse(b []byte) (*Policy, error) { + var p Policy + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return nil, err + } + if !strings.Contains(p.TakedownContact, "@") { + return nil, errors.New("takedown_contact is required") + } + if len(p.Platforms) == 0 { + return nil, errors.New("no platforms") + } + for _, id := range p.ExcludedPrograms { + if !strings.Contains(id, ":") { + return nil, fmt.Errorf("excluded program %q is not a program id", id) + } + } + for name, pl := range p.Platforms { + if !nameRE.MatchString(name) { + return nil, fmt.Errorf("platform name %q", name) + } + if pl.Publish && strings.TrimSpace(pl.Attribution) == "" { + return nil, fmt.Errorf("platform %s: publish=true needs an attribution text", name) + } + } + return &p, nil +} + +// Publishable reports whether records of the platform may be published. +// Unlisted platforms are not. +func (p *Policy) Publishable(platform string) bool { + return p.Platforms[platform].Publish +} + +// PublishablePlatforms lists the platforms with publish=true, sorted. +func (p *Policy) PublishablePlatforms() []string { + var out []string + for name, pl := range p.Platforms { + if pl.Publish { + out = append(out, name) + } + } + sort.Strings(out) + return out +} + +func (p *Policy) contact(platform string) string { + if c := p.Platforms[platform].TakedownContact; c != "" { + return c + } + return p.TakedownContact +} + +// ok reports whether a program may be published: the platform it is +// hosted on and, for dataset records, the original platform both must be +// publishable. +func (p *Policy) ok(prog model.Program) bool { + for _, id := range p.ExcludedPrograms { + if id == prog.ID { + return false + } + } + if !p.Publishable(prog.Platform) { + return false + } + op := prog.Provenance.OriginalPlatform + return op == "" || p.Publishable(op) +} + +// PrunePrev returns the previous release without the programs the policy +// does not allow, and the ids it removed. Without this a platform switched +// to publish=false would stay in the feed as an unchanged record. +func (p *Policy) PrunePrev(prev *bundle.Snapshot) (*bundle.Snapshot, []string) { + if prev == nil { + return nil, nil + } + out := *prev + out.Programs = make(map[string]model.Program, len(prev.Programs)) + var removed []string + for id, prog := range prev.Programs { + if p.ok(prog) { + out.Programs[id] = prog + } else { + removed = append(removed, id) + } + } + sort.Strings(removed) + return &out, removed +} + +// Filter returns the publishable part of a full snapshot: the programs the +// policy allows, the changes that concern them, a program_dropped change +// for every id in removed (publishable earlier, not any more), and the +// source entries recounted, with the attribution of every platform that +// is present and the takedown contact. +func (p *Policy) Filter(full *bundle.Snapshot, removed []string) *bundle.Snapshot { + out := bundle.NewSnapshot() + out.Sequence = full.Sequence + platforms := map[string]map[string]bool{} // source -> platforms present + count := map[string]int{} + for id, prog := range full.Programs { + if !p.ok(prog) { + continue + } + out.Programs[id] = prog + count[prog.Source]++ + if platforms[prog.Source] == nil { + platforms[prog.Source] = map[string]bool{} + } + platforms[prog.Source][prog.Platform] = true + } + dropped := map[string]bool{} + for _, c := range full.Changes { + if _, in := out.Programs[c.Program]; in || c.Kind == model.ChangeProgramDropped { + out.Changes = append(out.Changes, c) + if c.Kind == model.ChangeProgramDropped { + dropped[c.Program] = true + } + } + } + for _, id := range removed { + if _, still := out.Programs[id]; still || dropped[id] { + continue + } + out.Changes = append(out.Changes, model.Change{Sequence: full.Sequence, Program: id, Kind: model.ChangeProgramDropped}) + } + sort.SliceStable(out.Changes, func(i, j int) bool { return out.Changes[i].Program < out.Changes[j].Program }) + for _, s := range full.Sources { + s.Programs = count[s.Name] + var names []string + for name := range platforms[s.Name] { + if name != SelfHosted { + names = append(names, name) + } + } + sort.Strings(names) + for _, name := range names { + s.Attribution += " " + strings.TrimSpace(p.Platforms[name].Attribution) + } + if len(names) > 0 { + s.Attribution += " Removal requests: " + p.contact(names[0]) + " (see TAKEDOWN.md)." + } + out.Sources = append(out.Sources, s) + } + return out +} diff --git a/policy/policy_test.go b/policy/policy_test.go new file mode 100644 index 0000000..8b1ccca --- /dev/null +++ b/policy/policy_test.go @@ -0,0 +1,250 @@ +package policy + +import ( + "compress/gzip" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/openctemio/programfeed/internal/bundle" + "github.com/openctemio/programfeed/internal/collect" + "github.com/openctemio/programfeed/internal/fetch" + "github.com/openctemio/programfeed/internal/model" + "github.com/openctemio/programfeed/internal/source" +) + +var day = time.Date(2026, 10, 10, 2, 0, 0, 0, time.UTC) + +type fake struct { + name string + progs []model.Program +} + +func (f fake) Meta() source.Meta { + return source.Meta{Name: f.name, URL: "https://openctem.io/" + f.name, Terms: "https://openctem.io/terms", Licence: "test", + Attribution: "Test data."} +} + +func (f fake) Collect(context.Context, *fetch.Client) (*source.Result, error) { + out := &source.Result{} + for _, p := range f.progs { + p.InScope = append([]model.Target(nil), p.InScope...) + out.Programs = append(out.Programs, p) + } + return out, nil +} + +// prog builds a dataset record of the platform; hosted names the platform +// recorded in Platform (differs from the original platform only in the +// "mislabelled" case). +func prog(src, platform, slug string) model.Program { + u := "https://" + platform + ".example.net/" + slug + p := model.Program{ + ID: src + ":" + slug, Source: src, Platform: platform, Name: "Program " + slug, URL: u, Type: model.TypeBounty, + Status: model.StatusOpen, ScopePublished: true, + InScope: []model.Target{{Type: model.TargetDomain, Value: slug + ".restricted-secret-corp.net", Confidence: model.ConfidencePublishedByPlatform}}, + OutOfScope: []model.Target{}, Terms: model.Terms{URL: u}, + Provenance: model.Provenance{Source: src, SourceURL: "https://openctem.io/" + src, FetchedAt: day, + Dataset: "o/r", DatasetCommit: strings.Repeat("a", 40), OriginalPlatform: platform, OriginalURL: u}, + } + return p +} + +func build(t *testing.T, pol *Policy, prev *bundle.Snapshot, srcs ...source.Source) (dir string, next *bundle.Snapshot) { + t.Helper() + var removed []string + prev, removed = pol.PrunePrev(prev) + full, err := collect.Build(context.Background(), prev, srcs, nil, collect.Options{Now: day}) + if err != nil { + t.Fatal(err) + } + next = pol.Filter(full, removed) + dir = t.TempDir() + if _, _, _, err := bundle.Write(dir, prev, next, bundle.WriteOptions{Now: day}); err != nil { + t.Fatal(err) + } + return dir, next +} + +// output returns everything the signed output would contain: every +// record file, uncompressed, plus the manifests. +func output(t *testing.T, dir string) string { + t.Helper() + var sb strings.Builder + entries, _ := os.ReadDir(dir) + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatal(err) + } + if strings.HasSuffix(e.Name(), ".gz") { + zr, err := gzip.NewReader(strings.NewReader(string(b))) + if err != nil { + t.Fatal(err) + } + if b, err = io.ReadAll(zr); err != nil { + t.Fatal(err) + } + } + sb.Write(b) + } + return sb.String() +} + +func TestDefaultPolicy(t *testing.T) { + p := Default() + if got := p.PublishablePlatforms(); len(got) != 1 || got[0] != SelfHosted { + t.Fatalf("only self-hosted programs may be published by default, got %v", got) + } + for _, name := range []string{"hackerone", "bugcrowd", "intigriti", "yeswehack", "immunefi", "hackenproof"} { + if pl, ok := p.Platforms[name]; !ok || pl.Publish || pl.Reason == "" { + t.Errorf("%s must be listed with publish=false and a reason: %+v", name, pl) + } + } + if p.Publishable("a-platform-nobody-listed") { + t.Error("an unlisted platform must not be published") + } + if p.TakedownContact != "security@openctem.io" { + t.Errorf("takedown contact %q", p.TakedownContact) + } +} + +func TestParseRefuses(t *testing.T) { + for name, doc := range map[string]string{ + "no attribution": `{"takedown_contact":"a@b.io","platforms":{"x":{"publish":true}}}`, + "no contact": `{"platforms":{"x":{"publish":false}}}`, + "unknown field": `{"takedown_contact":"a@b.io","platforms":{"x":{"publish":false,"publsh":true}}}`, + "bad name": `{"takedown_contact":"a@b.io","platforms":{"X Y":{"publish":false}}}`, + "no platforms": `{"takedown_contact":"a@b.io","platforms":{}}`, + } { + if _, err := Parse([]byte(doc)); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +// Restricted platforms are read (the collector sees them) but never reach +// the signed output: not in the records, the change log, the manifests or +// the source counts. +func TestRestrictedPlatformsNeverReachSignedOutput(t *testing.T) { + pol := Default() + var progs []model.Program + for name, pl := range pol.Platforms { + if !pl.Publish { + progs = append(progs, prog("bounty-targets", name, name+"-prog")) + } + } + // A record whose Platform field says self-hosted but whose original + // platform is restricted must not slip through. + sneaky := prog("bounty-targets", "hackerone", "sneaky") + sneaky.Platform = "self-hosted" + progs = append(progs, sneaky) + progs = append(progs, prog("bounty-targets", "self-hosted", "own-policy")) + progs[len(progs)-1].InScope = []model.Target{{Type: model.TargetDomain, Value: "own.openctem.io", Confidence: model.ConfidenceInferred}} + progs[len(progs)-1].Provenance = model.Provenance{Source: "bounty-targets", SourceURL: "https://openctem.io/bounty-targets", FetchedAt: day} + + dir, next := build(t, pol, nil, fake{name: "bounty-targets", progs: progs}) + if len(next.Programs) != 1 || next.Programs["bounty-targets:own-policy"].ID == "" { + t.Fatalf("published %v", next.SortedIDs()) + } + out := output(t, dir) + if strings.Contains(out, "restricted-secret-corp") { + t.Error("a target of a restricted platform reached the output") + } + for name, pl := range pol.Platforms { + if !pl.Publish && (strings.Contains(out, name+"-prog") || strings.Contains(out, "\"original_platform\":\""+name+"\"")) { + t.Errorf("records of %s reached the output", name) + } + } + if strings.Contains(out, "sneaky") { + t.Error("a record with a restricted original platform reached the output") + } + if got := next.Sources[0].Programs; got != 1 { + t.Errorf("source count %d, want 1", got) + } + for _, c := range next.Changes { + if c.Program != "bounty-targets:own-policy" { + t.Errorf("change of a restricted program: %+v", c) + } + } +} + +func TestPublishedPlatformCarriesAttributionAndTakedown(t *testing.T) { + pol, err := Parse([]byte(`{"takedown_contact":"security@openctem.io","platforms":{ + "self-hosted":{"publish":true,"attribution":"Own pages."}, + "goodco":{"publish":true,"attribution":"Data from GoodCo, used with permission."}}}`)) + if err != nil { + t.Fatal(err) + } + _, next := build(t, pol, nil, fake{name: "bounty-targets", progs: []model.Program{prog("bounty-targets", "goodco", "g1")}}) + if len(next.Programs) != 1 { + t.Fatal("a publishable platform must be published") + } + a := next.Sources[0].Attribution + if !strings.Contains(a, "GoodCo, used with permission") || !strings.Contains(a, "security@openctem.io") { + t.Errorf("attribution %q", a) + } +} + +// Switching a platform to publish=false removes its programs from the +// next release with a program_dropped change; they do not linger as +// unchanged records. +func TestSwitchingPlatformOffDropsItsPrograms(t *testing.T) { + on, _ := Parse([]byte(`{"takedown_contact":"security@openctem.io","platforms":{ + "self-hosted":{"publish":true,"attribution":"Own pages."}, + "goodco":{"publish":true,"attribution":"GoodCo."}}}`)) + off, _ := Parse([]byte(`{"takedown_contact":"security@openctem.io","platforms":{ + "self-hosted":{"publish":true,"attribution":"Own pages."}, + "goodco":{"publish":false}}}`)) + src := fake{name: "bounty-targets", progs: []model.Program{prog("bounty-targets", "goodco", "g1"), prog("bounty-targets", "goodco", "g2")}} + _, first := build(t, on, nil, src) + if len(first.Programs) != 2 { + t.Fatal("setup") + } + dir, second := build(t, off, first, src) + if len(second.Programs) != 0 { + t.Fatalf("still published: %v", second.SortedIDs()) + } + dropped := 0 + for _, c := range second.Changes { + if c.Kind == model.ChangeProgramDropped { + dropped++ + } + } + if dropped != 2 { + t.Errorf("%d program_dropped changes, want 2", dropped) + } + if strings.Contains(output(t, dir), "restricted-secret-corp") { + t.Error("a switched-off platform still has records in the output") + } +} + +func TestSignRefusesLocalOnlyBundle(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "snapshot.manifest.json"), []byte(`{"collector":{"local_only":true}}`), 0o644); err != nil { + t.Fatal(err) + } + if err := bundle.Sign(dir, nil, nil, "", day); err == nil || !strings.Contains(err.Error(), "local-only") { + t.Errorf("signing a local-only bundle: %v", err) + } +} + +func TestExcludedProgramIsNeverPublished(t *testing.T) { + pol, err := Parse([]byte(`{"takedown_contact":"security@openctem.io","excluded_programs":["disclose:gone"],"platforms":{ + "self-hosted":{"publish":true,"attribution":"Own pages."}}}`)) + if err != nil { + t.Fatal(err) + } + keep, gone := prog("disclose", "self-hosted", "kept"), prog("disclose", "self-hosted", "gone") + _, next := build(t, pol, nil, fake{name: "disclose", progs: []model.Program{keep, gone}}) + if len(next.Programs) != 1 || next.Programs["disclose:kept"].ID == "" { + t.Fatalf("published %v", next.SortedIDs()) + } + if _, err := Parse([]byte(`{"takedown_contact":"a@b.io","excluded_programs":["nocolon"],"platforms":{"x":{"publish":false}}}`)); err == nil { + t.Error("an excluded entry that is not a program id was accepted") + } +} diff --git a/schema/change.schema.json b/schema/change.schema.json new file mode 100644 index 0000000..3a11ebe --- /dev/null +++ b/schema/change.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openctemio/programfeed/schema/change.schema.json", + "title": "Change", + "description": "One entry of a run's change log: one line of changes.jsonl.gz. Entries are sorted by program, then kind, then scope and target.", + "type": "object", + "additionalProperties": false, + "required": ["sequence", "program", "kind"], + "properties": { + "sequence": {"type": "integer", "minimum": 1, "description": "The bundle sequence that made the change."}, + "program": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$"}, + "kind": {"enum": ["program_added", "program_reopened", "status_changed", "program_modified", "rules_changed", "terms_changed", + "target_added", "target_modified", "target_removed", "program_closed", "program_dropped"], + "description": "program_dropped: the program is no longer in the feed (closed for more than 180 days); every other kind names a program present in the same bundle."}, + "scope": {"enum": ["in", "out"], "description": "Target changes only."}, + "target": {"$ref": "program.schema.json#/$defs/target", "description": "Target changes only: the target after the change (before, for target_removed)."}, + "fields": {"type": "array", "maxItems": 16, "items": {"type": "string"}, "description": "For program_modified, rules_changed, status_changed and target_modified: the JSON fields that changed."} + } +} diff --git a/schema/program.schema.json b/schema/program.schema.json new file mode 100644 index 0000000..89792b4 --- /dev/null +++ b/schema/program.schema.json @@ -0,0 +1,108 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openctemio/programfeed/schema/program.schema.json", + "title": "Program", + "description": "One public bug-bounty or vulnerability-disclosure program: one line of programs.jsonl.gz.", + "type": "object", + "additionalProperties": false, + "required": ["id", "source", "platform", "name", "url", "type", "status", "offers_bounty", "scope_published", + "in_scope", "out_of_scope", "rejected", "rules", "terms", "contact", "provenance", "first_seen", "last_seen", "last_changed"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,31}:[a-z0-9][a-z0-9._-]{0,127}$", + "description": ":. Stable; owned by exactly one source adapter."}, + "source": {"type": "string", "description": "The adapter that produced the record (the id prefix)."}, + "platform": {"type": "string", "maxLength": 64, "description": "Where the program is hosted: \"self-hosted\" or the name of the hosting platform."}, + "name": {"type": "string", "minLength": 1, "maxLength": 200}, + "url": {"type": "string", "format": "uri", "maxLength": 2048, "description": "The program's policy page (http or https)."}, + "type": {"enum": ["bounty", "vdp"]}, + "status": {"enum": ["open", "paused", "closed"], + "description": "closed: the source no longer lists the program (kept 180 days, then dropped). paused: listed but not currently accepting (dead policy link, expired security.txt)."}, + "offers_bounty": {"type": "boolean", "description": "Always equal to type == bounty."}, + "scope_published": {"type": "boolean", "description": "True when the source carries the program's own structured scope. When false, in_scope holds at most inferred suggestions."}, + "in_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}, "description": "Sorted by (type, value), no duplicates. Out of scope wins: a value listed in both appears only in out_of_scope."}, + "out_of_scope": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/target"}}, + "rejected": {"type": "array", "maxItems": 2000, "items": {"$ref": "#/$defs/rejected"}, "description": "Targets the program listed that the feed's scope rules refused, with the reason."}, + "rules": {"$ref": "#/$defs/rules"}, + "terms": {"$ref": "#/$defs/terms"}, + "contact": {"$ref": "#/$defs/contact"}, + "provenance": {"$ref": "#/$defs/provenance"}, + "first_seen": {"type": "string", "format": "date-time"}, + "last_seen": {"type": "string", "format": "date-time", "description": "Last run in which the source listed the program."}, + "last_changed": {"type": "string", "format": "date-time", "description": "Last run in which any content (not timestamps) changed."}, + "closed_at": {"type": "string", "format": "date-time", "description": "Present exactly when status is closed."} + }, + "$defs": { + "target": { + "type": "object", + "additionalProperties": false, + "required": ["type", "value", "confidence"], + "properties": { + "type": {"enum": ["domain", "wildcard", "ip", "cidr", "url", "mobile_app", "source_repo", "other"]}, + "value": {"type": "string", "maxLength": 512, + "description": "Canonical form: domain = lower-case A-label, not a public suffix, not a reserved name; wildcard = \"*.\" + such a domain; ip = public unicast address; cidr = masked public prefix of at least /16 (IPv4) or /48 (IPv6); url = http(s) without credentials or fragment, host checked as above."}, + "confidence": {"enum": ["published", "published_by_platform", "inferred"], + "description": "published: the program's own scope. published_by_platform: scope the hosting platform published for the program, read through a public aggregate dataset; it states what the program lists and is never permission to test.inferred: derived by the collector (for example the host serving the program's security.txt); a suggestion, never an authorisation to test."}, + "eligible_for_bounty": {"type": "boolean"}, + "max_severity": {"enum": ["none", "low", "medium", "high", "critical"]}, + "notes": {"type": "string", "maxLength": 500} + } + }, + "rejected": { + "type": "object", + "additionalProperties": false, + "required": ["scope", "type", "value", "reason"], + "properties": { + "scope": {"enum": ["in", "out"]}, + "type": {"type": "string", "maxLength": 32}, + "value": {"type": "string", "maxLength": 512}, + "reason": {"type": "string", "minLength": 1, "maxLength": 200} + } + }, + "rules": { + "type": "object", + "additionalProperties": false, + "required": ["testing_restrictions", "required_headers", "safe_harbour", "languages"], + "properties": { + "summary": {"type": "string", "maxLength": 4000}, + "testing_restrictions": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}}, + "rate_limit": {"type": "string", "maxLength": 200}, + "required_headers": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}}, + "safe_harbour": {"enum": ["full", "partial", "none", "unknown"]}, + "disclosure_days": {"type": "integer", "minimum": 0, "maximum": 3650}, + "languages": {"type": "array", "maxItems": 32, "items": {"type": "string", "minLength": 1, "maxLength": 500}} + } + }, + "terms": { + "type": "object", + "additionalProperties": false, + "required": ["url"], + "properties": { + "url": {"type": "string", "format": "uri", "maxLength": 2048}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "SHA-256 of the exact bytes of the terms document the collector read at fetched_at. Absent when the collector did not read the terms."}, + "fetched_at": {"type": "string", "format": "date-time", "description": "Present exactly when sha256 is."} + } + }, + "contact": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": {"type": "string", "maxLength": 254}, + "url": {"type": "string", "maxLength": 2048, "description": "http(s) or mailto."} + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "required": ["source", "source_url", "fetched_at"], + "properties": { + "source": {"type": "string"}, + "source_url": {"type": "string", "format": "uri", "maxLength": 2048}, + "fetched_at": {"type": "string", "format": "date-time"}, + "dataset": {"type": "string", "pattern": "^[A-Za-z0-9._-]{1,100}/[A-Za-z0-9._-]{1,100}$", "description": "Aggregate dataset repository the record was read through (owner/name)."}, + "dataset_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "original_platform": {"type": "string", "maxLength": 64, "description": "The platform the dataset took the record from."}, + "original_url": {"type": "string", "format": "uri", "maxLength": 2048, "description": "The program page on the original platform."} + } + } + } +} diff --git a/seeds/securitytxt.txt b/seeds/securitytxt.txt new file mode 100644 index 0000000..d5c0417 --- /dev/null +++ b/seeds/securitytxt.txt @@ -0,0 +1,3 @@ +# Extra domains whose security.txt the securitytxt adapter reads, one per line, +# optionally followed by the organisation name. Only add domains whose owner +# publishes a vulnerability disclosure policy for public use.