feat: collector that builds, signs and verifies public program bundles

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+      - name: Format
+        run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; }
+      - name: Vet
+        run: go vet ./...
+      - name: Test (recorded fixtures, no network)
+        run: go test -race ./...
+      - name: Fuzz the scope normaliser briefly
+        run: go test ./internal/model -run '^$' -fuzz '^FuzzNormalizeTarget$' -fuzztime 20s
+      - name: Known vulnerabilities
+        run: go run golang.org/x/vuln/cmd/govulncheck@latest ./...

.github/workflows/publish.yml

@@ -0,0 +1,135 @@
+name: Publish bundle
+
+# Daily: build the next program bundle from the previous release and the
+# sources, sign it, verify it and publish it as a GitHub Release.
+#
+# Two jobs on purpose. "build" talks to the sources and never sees the
+# signing key. "publish" runs in the protected "publish" environment (only
+# the default branch can deploy to it), holds the signing key and nothing
+# else, and only signs, verifies and uploads what "build" produced.
+
+on:
+  schedule:
+    - cron: "41 3 * * *"
+  workflow_dispatch:
+    inputs:
+      allow_mass_change:
+        description: "Accept a source that lost more than half of its programs (only after checking the source)"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: publish
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    outputs:
+      ready: ${{ steps.keys.outputs.ready }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Key ceremony done?
+        id: keys
+        run: |
+          if [ -s keys/root-keyid.txt ] && [ -s keys/keyset.dsse.json ]; then
+            echo "ready=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::warning::No keys yet (keys/root-keyid.txt, keys/keyset.dsse.json): nothing is published until the key ceremony in keys/README.md is done"
+            echo "ready=false" >> "$GITHUB_OUTPUT"
+          fi
+      - if: steps.keys.outputs.ready == 'true'
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+      - if: steps.keys.outputs.ready == 'true'
+        name: Test
+        run: go test ./...
+      - if: steps.keys.outputs.ready == 'true'
+        name: Build the CLI
+        run: go build -trimpath -ldflags "-X main.version=${GITHUB_SHA::12}" -o programfeed ./cmd/programfeed
+      - if: steps.keys.outputs.ready == 'true'
+        name: Restore the HTTP cache (conditional requests only; every body is re-validated)
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: httpcache
+          key: httpcache-${{ github.run_id }}
+          restore-keys: httpcache-
+      - if: steps.keys.outputs.ready == 'true'
+        name: Download the previous release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p prev
+          if gh release view --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+            gh release download --repo "$GITHUB_REPOSITORY" --dir prev --pattern 'snapshot*' --pattern 'latest.dsse.json' --pattern 'keyset.dsse.json'
+            echo "PREV=--prev=prev" >> "$GITHUB_ENV"
+          fi
+      - if: steps.keys.outputs.ready == 'true'
+        name: Build the bundle
+        env:
+          MASS: ${{ inputs.allow_mass_change && '--allow-mass-change' || '' }}
+        run: ./programfeed build --out out --cache httpcache --root-key-id "$(cat keys/root-keyid.txt)" $PREV $MASS
+      - if: steps.keys.outputs.ready == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: bundle
+          path: out/
+          retention-days: 3
+          if-no-files-found: error
+
+  publish:
+    needs: build
+    if: needs.build.outputs.ready == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment: publish
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+      - name: Build the CLI
+        run: go build -trimpath -o programfeed ./cmd/programfeed
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: bundle
+          path: out
+      - name: Sign
+        env:
+          PROGRAMFEED_SIGNING_KEY: ${{ secrets.PROGRAMFEED_SIGNING_KEY }}
+        run: ./programfeed sign --dir out --keyset keys/keyset.dsse.json --root-key-id "$(cat keys/root-keyid.txt)"
+      - name: Verify as the platform does
+        run: ./programfeed verify --dir out --root-key-id "$(cat keys/root-keyid.txt)"
+      - name: Publish the release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          tag="$(python3 -c 'import json,base64,sys; e=json.load(open("out/latest.dsse.json")); print(json.loads(base64.b64decode(e["payload"]))["tag"])')"
+          cat > notes.md <<'NOTES'
+          Public program bundle for OpenCTEM: a signed snapshot of public bug-bounty and vulnerability-disclosure programs, and the delta from the previous release.
+
+          Targets marked "inferred" are suggestions, not a program's published scope. See NOTICE for the terms of every source.
+          NOTES
+          gh release create "$tag" --repo "$GITHUB_REPOSITORY" --title "$tag" --notes-file notes.md out/*
+      - name: Keep the last 60 releases
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release list --repo "$GITHUB_REPOSITORY" --limit 1000 --json tagName,createdAt \
+            --jq 'sort_by(.createdAt) | reverse | .[60:] | .[].tagName' |
+          while read -r tag; do
+            gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+          done

.gitignore

@@ -0,0 +1,5 @@
+/programfeed
+/out/
+/prev/
+/httpcache/
+*.key

CONTRIBUTING.md

@@ -0,0 +1,83 @@
+# Contributing
+
+Thanks for helping. Two kinds of contributions are common: adding seed
+domains for the security.txt adapter, and adding a new source adapter.
+
+## Adding seed domains
+
+Add lines to `seeds/securitytxt.txt` (`domain [organisation name]`). Only add
+domains whose owner publishes a vulnerability disclosure policy for public
+use. The domain must already serve `/.well-known/security.txt` with a
+`Contact` and a `Policy` field, or the collector skips it.
+
+## Adding a source adapter
+
+### 1. Check that the source may be collected
+
+Before writing code, open an issue with:
+
+- the official API or machine-readable listing (documentation link);
+- whether it needs authentication (the feed never logs in, uses cookies or
+  tokens, runs JavaScript or bypasses bot protection: a source that needs
+  any of that is not accepted);
+- its terms of service and robots.txt, quoted: they must allow automated
+  collection **and redistribution**, since the feed is published openly
+  (or the publisher must have given written permission, linked in the
+  issue);
+- the licence of the data and the attribution it requires;
+- its rate limits.
+
+### 2. Write the adapter
+
+An adapter is one file in `internal/source/` implementing `source.Source`:
+
+```go
+type Source interface {
+	Meta() Meta                                             // name, URL, terms, licence, attribution
+	Collect(ctx context.Context, c *fetch.Client) (*Result, error)
+}
+```
+
+Rules:
+
+- Use only the `*fetch.Client` it is given (it handles the User-Agent,
+  robots.txt, rate limits, conditional requests, size caps and the
+  public-address check). Pass a size cap that fits the source.
+- Program ids are `<Meta().Name>:<slug>`; use `source.UniqueSlugs` for names.
+  Ids must be stable across runs. The collector refuses programs outside the
+  adapter's own prefix.
+- Fill `Provenance` (source, source URL, fetched-at time). Leave
+  `first_seen`, `last_seen`, `last_changed` and `closed_at` to the collector.
+- Put the program's own scope in `InScope`/`OutOfScope` with
+  `Confidence: published` and set `ScopePublished`. Anything the adapter
+  derives itself is `Confidence: inferred`, with a note saying how.
+- Do not filter scope yourself: the collector normalises every target and
+  records refused ones with the reason.
+- Return an error only when the source as a whole could not be read (the
+  collector then keeps every previous record). Put ids whose state is
+  unknown this run (a host that did not answer) in `Result.Unknown`.
+- Hostile input is expected: decode defensively, cap counts, never trust a
+  field's type.
+
+Register it in `cmd/programfeed/main.go` (`sources`), and add its row to the
+sources table in `README.md` and its terms to `NOTICE`.
+
+### 3. Test it with recorded fixtures
+
+Tests never touch the network. Record a small, representative response
+under `internal/source/testdata/` (trimmed, no personal data beyond what the
+source publishes), serve it with `httptest`, and point the fetch client at
+it (`fetch.Options{AllowPrivate: true, DialOverride: ...}`, see
+`source_test.go`). Include poisoned entries: malformed records, wrong field
+types, control characters, public-suffix or private-range scope.
+
+### 4. Check before opening the pull request
+
+```sh
+gofmt -l .
+go vet ./...
+go test -race ./...
+go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+```
+
+Pull requests target `main`.

NOTICE

@@ -0,0 +1,25 @@
+programfeed
+Copyright 2026 The OpenCTEM Authors
+
+This software is licensed under the Apache License, Version 2.0 (LICENSE).
+
+The bundles this software publishes contain data from third-party sources,
+under their own terms:
+
+disclose.io diodb (https://github.com/disclose/diodb)
+  Program directory data, dedicated to the public domain under CC0 1.0
+  Universal: https://github.com/disclose/diodb/blob/master/LICENSE
+
+CISA dotgov-data (https://github.com/cisagov/dotgov-data)
+  The list of US federal .gov domains used as the security.txt seed list,
+  dedicated to the public domain under CC0 1.0 Universal.
+
+security.txt files (RFC 9116)
+  Each program record from the securitytxt adapter is derived from the
+  security.txt file its domain publishes for automated retrieval; the
+  record's provenance names the file. The policies those files link to
+  belong to their publishers and are not copied into the bundles.
+
+Program names, policies and terms remain the property of the organisations
+that publish them. Listing a program in the feed does not imply that the
+organisation endorses OpenCTEM, nor that any testing is authorised.