diff --git a/README.md b/README.md index 3afc529..7b98c54 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,22 @@ Key rules: - Scope entries are typed (`domain`, `wildcard`, `ip`, `cidr`, `url`, `mobile_app`, `source_repo`, `other`), canonical, sorted and deduplicated; out of scope wins over in scope. +- Schema 1.1 (additive; `openctem.programfeed/v1` is unchanged, every new + field is optional and omitted when the platform did not publish it, never + guessed). A scope entry may carry `asset_type` (the platform's own asset + type, normalised: `domain`, `wildcard`, `url`, `api`, `ip`, `cidr`, + `android_app`, `ios_app`, `mobile_app`, `source_code`, `executable`, + `hardware`, `smart_contract`, `ai_model`, `other`; `type` stays the coarse + kind), `ports` (ports or ranges such as `8000-8100`), `protocol` + (`tcp`/`udp`), `path_prefix` (url targets), `environment` + (`production`/`staging`/`other`), `instructions` (the platform's testing + text, at most 1000 bytes), `requires` (headers, test accounts), alongside + `eligible_for_bounty` and `max_severity`. The aggregate datasets currently + publish `asset_type`, `instructions`, eligibility and severity (and the + URL path becomes `path_prefix`); the other fields are in the schema for + sources that publish them. `notes` is now only the collector's own remark. + Readers that validate with `additionalProperties: false` must adopt the + 1.1 schema. - `confidence`: `published` (the program's own scope), `published_by_platform` (scope the hosting platform published, read through a public dataset; the record's `provenance` names the dataset, commit, original platform and diff --git a/cmd/programfeed/main.go b/cmd/programfeed/main.go index 473f674..2b8edb8 100644 --- a/cmd/programfeed/main.go +++ b/cmd/programfeed/main.go @@ -148,8 +148,8 @@ func cmdBuild(ctx context.Context, args []string) error { return fmt.Errorf("policy: %w", err) } } - if *localOnly && *prevDir != "" { - return errors.New("build: a local-only bundle is rebuilt from scratch; --prev is not allowed with --local-only") + if !*localOnly && *prevDir != "" && bundle.IsLocalOnly(*prevDir) { + return errors.New("build: --prev is a local-only bundle; a publishable build needs a signed release") } srcs, err := sources(*names, *seedFile, pol.PublishablePlatforms(), *localOnly) if err != nil { @@ -157,7 +157,15 @@ func cmdBuild(ctx context.Context, args []string) error { } now := time.Now().UTC() var prev *bundle.Snapshot - if *prevDir != "" { + if *prevDir != "" && *localOnly { + // Continue the previous local bundle's sequence (snapshot-only, + // unsigned): the platform importer refuses non-increasing sequences. + var err error + if prev, err = bundle.ReadLocal(*prevDir); err != nil { + return fmt.Errorf("previous local bundle: %w", err) + } + logf("previous local bundle: sequence %d, %d programs", prev.Sequence, len(prev.Programs)) + } else if *prevDir != "" { v, err := bundle.VerifyDir(*prevDir, bundle.VerifyOptions{PinnedRoot: *root, Now: now, AllowExpired: true, SkipRecordChk: true}) if err != nil { return fmt.Errorf("previous release: %w", err) @@ -183,7 +191,11 @@ func cmdBuild(ctx context.Context, args []string) error { next = pol.Filter(next, removed) logf("publish policy: %d of %d programs publishable", len(next.Programs), full) } - snap, delta, _, err := bundle.Write(*out, prev, next, bundle.WriteOptions{Now: now, + writePrev := prev + if *localOnly { + writePrev = nil // snapshot-only: no delta is written for a local bundle + } + snap, delta, _, err := bundle.Write(*out, writePrev, next, bundle.WriteOptions{Now: now, Collector: bundle.Collector{Version: version, Commit: os.Getenv("GITHUB_SHA"), LocalOnly: *localOnly}}) if err != nil { return err diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go index a2c4762..1ec5b41 100644 --- a/internal/bundle/bundle_test.go +++ b/internal/bundle/bundle_test.go @@ -272,3 +272,42 @@ func resignJSON(t *testing.T, dir, file, payloadType string, priv ed25519.Privat out, _ := dsse.Sign(priv, payloadType, payload) _ = os.WriteFile(filepath.Join(dir, file), out, 0o644) } + +func TestReadLocalContinuesSequence(t *testing.T) { + dir := t.TempDir() + if _, _, _, err := Write(dir, nil, snapshot(1, "b.openctem.io"), WriteOptions{Now: time.Now(), Collector: Collector{LocalOnly: true}}); err != nil { + t.Fatal(err) + } + if !IsLocalOnly(dir) { + t.Fatal("IsLocalOnly") + } + prev, err := ReadLocal(dir) + if err != nil || prev.Sequence != 1 || len(prev.Programs) != 2 { + t.Fatalf("ReadLocal: %v %+v", err, prev) + } + // A tampered record file is refused. + if err := os.WriteFile(filepath.Join(dir, FileName(KindSnapshot, recordFiles[0])), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := ReadLocal(dir); err == nil { + t.Fatal("tampered local bundle accepted") + } +} + +func TestReadLocalRefusesSignedAndNonLocal(t *testing.T) { + k := newKeys(t) + signed := build(t, k, nil, snapshot(1, "b.openctem.io")) + if _, err := ReadLocal(signed); err == nil { + t.Fatal("signed release accepted as a local prev") + } + if IsLocalOnly(signed) { + t.Fatal("signed release reported local-only") + } + plain := t.TempDir() + if _, _, _, err := Write(plain, nil, snapshot(1, "b.openctem.io"), WriteOptions{Now: time.Now()}); err != nil { + t.Fatal(err) + } + if _, err := ReadLocal(plain); err == nil { + t.Fatal("non-local unsigned bundle accepted") + } +} diff --git a/internal/bundle/verify.go b/internal/bundle/verify.go index bd7e4dd..c848c92 100644 --- a/internal/bundle/verify.go +++ b/internal/bundle/verify.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -275,3 +276,55 @@ func (l *limited) Read(p []byte) (int, error) { l.left -= int64(n) return n, err } + +// ReadLocal reads the previous UNSIGNED local-only bundle in dir (written by +// a --local-only build) so the next local build can continue its sequence. +// It checks schema, tag, file sizes and hashes and every record, and refuses +// anything that is not a local-only bundle (a signed release has DSSE +// envelopes in place of the plain pointer and manifest). +func ReadLocal(dir string) (*Snapshot, error) { + var l Latest + raw, err := readCapped(filepath.Join(dir, "latest.json"), 1<<20) + if err != nil { + return nil, err + } + if json.Unmarshal(raw, &l) != nil || l.Schema != LatestSchema { + return nil, errors.New("not an unsigned local-only bundle (signed releases cannot seed a local build)") + } + if l.Tag != Tag(l.Sequence) || l.Snapshot != fmt.Sprintf(ManifestFile, KindSnapshot) { + return nil, fmt.Errorf("latest: tag %q or snapshot %q does not match sequence %d", l.Tag, l.Snapshot, l.Sequence) + } + raw, err = readCapped(filepath.Join(dir, "snapshot.manifest.json"), 1<<20) + if err != nil { + return nil, err + } + var m Manifest + if json.Unmarshal(raw, &m) != nil || m.Schema != ManifestSchema || m.Kind != KindSnapshot { + return nil, errors.New("snapshot manifest: not an unsigned manifest") + } + if !m.Collector.LocalOnly { + return nil, errors.New("previous bundle is not local-only; a local build continues only a local-only bundle") + } + if m.Sequence != l.Sequence || len(m.Files) != len(recordFiles) { + return nil, errors.New("snapshot manifest does not match the pointer") + } + for i, f := range m.Files { + if f.Name != FileName(KindSnapshot, recordFiles[i]) { + return nil, fmt.Errorf("unexpected file %q", f.Name) + } + if err := checkFile(filepath.Join(dir, f.Name), f); err != nil { + return nil, err + } + } + return ReadRecords(dir, m) +} + +// IsLocalOnly reports whether dir holds an unsigned local-only bundle. +func IsLocalOnly(dir string) bool { + raw, err := os.ReadFile(filepath.Join(dir, "snapshot.manifest.json")) + if err != nil { + return false + } + var m Manifest + return json.Unmarshal(raw, &m) == nil && m.Collector.LocalOnly +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index bdede14..7280834 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -49,7 +49,10 @@ type Options struct { UserAgent string CacheDir string // conditional requests; empty disables the cache MinInterval time.Duration // between two requests to one host - Timeout time.Duration // per request + Timeout time.Duration // response-header timeout, and the default whole-request deadline + // IdleTimeout fails a response body that delivers no byte for this long + // (default 20s), so a large download may take long but never stalls. + IdleTimeout time.Duration // AllowPrivate lets tests talk to httptest servers on loopback. Never // set in the collector. AllowPrivate bool @@ -84,6 +87,9 @@ func New(opt Options) *Client { if opt.Timeout == 0 { opt.Timeout = 30 * time.Second } + if opt.IdleTimeout == 0 { + opt.IdleTimeout = 20 * time.Second + } c := &Client{opt: opt, next: map[string]time.Time{}, robots: map[string]*robotsRules{}} dialer := &net.Dialer{Timeout: 10 * time.Second, Control: func(_, address string, _ syscall.RawConn) error { if opt.AllowPrivate { @@ -109,7 +115,7 @@ func New(opt Options) *Client { MaxIdleConnsPerHost: 2, IdleConnTimeout: 30 * time.Second, } - c.http = &http.Client{Transport: transport, Timeout: opt.Timeout, + c.http = &http.Client{Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 3 { return errors.New("more than 3 redirects") @@ -129,6 +135,30 @@ func New(opt Options) *Client { return c } +type deadlineKey struct{} + +// WithDeadline raises (or lowers) the whole-request deadline of fetches made +// with the returned context, for sources that download large files. The +// header, connect and idle-body timeouts still apply. +func WithDeadline(ctx context.Context, d time.Duration) context.Context { + return context.WithValue(ctx, deadlineKey{}, d) +} + +// idleReader resets the idle timer whenever bytes arrive. +type idleReader struct { + r io.Reader + timer *time.Timer + idle time.Duration +} + +func (i *idleReader) Read(p []byte) (int, error) { + n, err := i.r.Read(p) + if n > 0 { + i.timer.Reset(i.idle) + } + return n, err +} + // Get fetches rawURL (at most maxBytes) after checking robots.txt. func (c *Client) Get(ctx context.Context, rawURL string, maxBytes int64) (*Response, error) { u, err := url.Parse(rawURL) @@ -164,6 +194,16 @@ func (c *Client) get(ctx context.Context, u *url.URL, maxBytes int64, useCache b if err := c.wait(ctx, u.Host); err != nil { return nil, err } + total := c.opt.Timeout + if d, ok := ctx.Value(deadlineKey{}).(time.Duration); ok && d > 0 { + total = d + } + ctx, cancelTotal := context.WithTimeout(ctx, total) + defer cancelTotal() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + idle := time.AfterFunc(c.opt.IdleTimeout, cancel) // a stalled body fails + defer idle.Stop() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err @@ -203,7 +243,7 @@ func (c *Client) get(ctx context.Context, u *url.URL, maxBytes int64, useCache b case resp.StatusCode != http.StatusOK: return nil, &StatusError{URL: u.String(), Code: resp.StatusCode} } - body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + body, err := io.ReadAll(io.LimitReader(&idleReader{r: resp.Body, timer: idle, idle: c.opt.IdleTimeout}, maxBytes+1)) if err != nil { return nil, err } diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go index e575f47..8f79d1f 100644 --- a/internal/fetch/fetch_test.go +++ b/internal/fetch/fetch_test.go @@ -152,3 +152,51 @@ func TestRedirectEverythingTerminates(t *testing.T) { t.Fatalf("%v %+v", err, r) } } + +func slowServer(chunks int, gap time.Duration, stallAfter int) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/robots.txt" { + http.NotFound(w, r) + return + } + fl := w.(http.Flusher) + for i := 0; i < chunks; i++ { + if stallAfter >= 0 && i == stallAfter { + select { + case <-r.Context().Done(): + case <-time.After(5 * time.Second): + } + return + } + _, _ = w.Write([]byte(strings.Repeat("x", 1024))) + fl.Flush() + time.Sleep(gap) + } + })) +} + +func TestSlowButSteadyBodyPasses(t *testing.T) { + srv := slowServer(20, 25*time.Millisecond, -1) // ~500ms total, over Timeout + defer srv.Close() + c := New(Options{UserAgent: "t", MinInterval: time.Millisecond, AllowPrivate: true, + Timeout: 200 * time.Millisecond, IdleTimeout: 300 * time.Millisecond}) + if _, err := c.Get(context.Background(), srv.URL+"/big", 1<<20); err == nil { + t.Fatal("the default deadline must still bound the request") + } + resp, err := c.Get(WithDeadline(context.Background(), 5*time.Second), srv.URL+"/big", 1<<20) + if err != nil || len(resp.Body) != 20*1024 { + t.Fatalf("err=%v", err) + } +} + +func TestStalledBodyFails(t *testing.T) { + srv := slowServer(20, time.Millisecond, 3) + defer srv.Close() + c := New(Options{UserAgent: "t", MinInterval: time.Millisecond, AllowPrivate: true, + Timeout: time.Second, IdleTimeout: 150 * time.Millisecond}) + start := time.Now() + _, err := c.Get(WithDeadline(context.Background(), time.Minute), srv.URL+"/big", 1<<20) + if err == nil || time.Since(start) > 3*time.Second { + t.Fatalf("a stall must fail fast: err=%v after %v", err, time.Since(start)) + } +} diff --git a/internal/model/model.go b/internal/model/model.go index a525779..6fde29f 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -16,6 +16,7 @@ import ( "net/url" "regexp" "sort" + "strconv" "strings" "time" "unicode" @@ -64,6 +65,42 @@ const ( TargetOther = "other" ) +// Asset types: the normalised kind of asset a platform lists, finer than +// Type (which stays coarse so older readers keep working). An absent +// asset_type means the source did not say. +const ( + AssetDomain = "domain" + AssetWildcard = "wildcard" + AssetURL = "url" + AssetAPI = "api" + AssetIP = "ip" + AssetCIDR = "cidr" + AssetAndroidApp = "android_app" // store id or APK + AssetIOSApp = "ios_app" // store id, TestFlight or IPA + AssetMobileApp = "mobile_app" // platform not stated + AssetSourceCode = "source_code" + AssetExecutable = "executable" + AssetHardware = "hardware" + AssetSmartContract = "smart_contract" + AssetAIModel = "ai_model" + AssetOther = "other" +) + +// Target environments and protocols. +const ( + EnvProduction = "production" + EnvStaging = "staging" + EnvOther = "other" +) + +// Limits of the optional target fields. +const ( + MaxInstructionsLen = 1000 + MaxRequiresLen = 500 + MaxPathPrefixLen = 256 + MaxPorts = 32 +) + // Target confidence: published = the program's own structured scope; // inferred = derived by the collector (for example the host a program's // security.txt was served from). An inferred target is a suggestion, never @@ -86,7 +123,12 @@ var ( harbours = map[string]bool{SafeHarbourFull: true, SafeHarbourPartial: true, SafeHarbourNone: true, SafeHarbourUnknown: true} targetType = map[string]bool{TargetDomain: true, TargetWildcard: true, TargetIP: true, TargetCIDR: true, TargetURL: true, TargetMobileApp: true, TargetSourceRepo: true, TargetOther: true} - confidences = map[string]bool{ConfidencePublished: true, ConfidenceInferred: true, ConfidencePublishedByPlatform: true} + assetTypes = map[string]bool{AssetDomain: true, AssetWildcard: true, AssetURL: true, AssetAPI: true, AssetIP: true, + AssetCIDR: true, AssetAndroidApp: true, AssetIOSApp: true, AssetMobileApp: true, AssetSourceCode: true, + AssetExecutable: true, AssetHardware: true, AssetSmartContract: true, AssetAIModel: true, AssetOther: true} + environments = map[string]bool{EnvProduction: true, EnvStaging: true, EnvOther: true} + portRE = regexp.MustCompile(`^[0-9]{1,5}(-[0-9]{1,5})?$`) + confidences = map[string]bool{ConfidencePublished: true, ConfidenceInferred: true, ConfidencePublishedByPlatform: true} ) // Program is one public bug-bounty or vulnerability-disclosure program. @@ -120,7 +162,31 @@ type Target struct { Confidence string `json:"confidence"` Eligible *bool `json:"eligible_for_bounty,omitempty"` MaxSev string `json:"max_severity,omitempty"` - Notes string `json:"notes,omitempty"` + Notes string `json:"notes,omitempty"` // the collector's remark (for example why a target is inferred) + + // Optional, only when the platform published them (an absent field is + // unknown, never a guess). + AssetType string `json:"asset_type,omitempty"` + Ports []string `json:"ports,omitempty"` // "443" or "8000-8100" + Protocol string `json:"protocol,omitempty"` + PathPrefix string `json:"path_prefix,omitempty"` // url targets: the path the scope is limited to + Environment string `json:"environment,omitempty"` + Instructions string `json:"instructions,omitempty"` // the platform's testing instructions for this asset + Requires string `json:"requires,omitempty"` // headers, test accounts or other prerequisites +} + +// ValidPort reports whether s is a port ("443") or a port range ("8000-8100"). +func ValidPort(s string) bool { + if !portRE.MatchString(s) { + return false + } + lo, hi := s, s + if i := strings.IndexByte(s, '-'); i >= 0 { + lo, hi = s[:i], s[i+1:] + } + a, _ := strconv.Atoi(lo) + b, _ := strconv.Atoi(hi) + return a >= 1 && b <= 65535 && a <= b } // Rejected is a target the scope rules refused, with the reason. @@ -313,6 +379,17 @@ func (t Target) Validate() error { if !targetType[t.Type] || !confidences[t.Confidence] || !severities[t.MaxSev] || !safeText(t.Notes, 500) { return fmt.Errorf("target %q: type, confidence, max_severity or notes", t.Value) } + if (t.AssetType != "" && !assetTypes[t.AssetType]) || (t.Environment != "" && !environments[t.Environment]) || + (t.Protocol != "" && t.Protocol != "tcp" && t.Protocol != "udp") || + !safeText(t.Instructions, MaxInstructionsLen) || !safeText(t.Requires, MaxRequiresLen) || len(t.Ports) > MaxPorts || + (t.PathPrefix != "" && (t.Type != TargetURL || !strings.HasPrefix(t.PathPrefix, "/") || !safeText(t.PathPrefix, MaxPathPrefixLen) || strings.ContainsAny(t.PathPrefix, " ?#"))) { + return fmt.Errorf("target %q: asset_type, environment, protocol, ports, path_prefix, instructions or requires", t.Value) + } + for _, p := range t.Ports { + if !ValidPort(p) { + return fmt.Errorf("target %q: port %q", t.Value, p) + } + } n, reason := NormalizeTarget(t.Type, t.Value) if reason != "" || n != t.Value { return fmt.Errorf("target %q is not a normalised %s (%s)", t.Value, t.Type, reason) diff --git a/internal/model/model_test.go b/internal/model/model_test.go index 93f34ed..8e13442 100644 --- a/internal/model/model_test.go +++ b/internal/model/model_test.go @@ -237,3 +237,35 @@ func TestDatasetProvenance(t *testing.T) { } } } + +func TestTargetOptionalFields(t *testing.T) { + good := Target{Type: TargetURL, Value: "https://a.openctem.io/v1", Confidence: ConfidencePublishedByPlatform, AssetType: AssetAPI, + Ports: []string{"443", "8000-8100"}, Protocol: "tcp", PathPrefix: "/v1", Environment: EnvStaging, Instructions: "x", Requires: "header X-Test"} + if err := good.Validate(); err != nil { + t.Fatal(err) + } + for name, mut := range map[string]func(*Target){ + "asset": func(x *Target) { x.AssetType = "gadget" }, + "env": func(x *Target) { x.Environment = "prod" }, + "protocol": func(x *Target) { x.Protocol = "icmp" }, + "port": func(x *Target) { x.Ports = []string{"0"} }, + "range": func(x *Target) { x.Ports = []string{"9000-80"} }, + "high": func(x *Target) { x.Ports = []string{"70000"} }, + "path": func(x *Target) { x.PathPrefix = "v1" }, + "long": func(x *Target) { x.Instructions = strings.Repeat("a", MaxInstructionsLen+1) }, + } { + bad := good + mut(&bad) + if bad.Validate() == nil { + t.Errorf("%s accepted", name) + } + } + // Normalize drops what is outside the grammar instead of guessing. + p := Program{InScope: []Target{{Type: TargetDomain, Value: "a.openctem.io", AssetType: "gadget", Protocol: "icmp", + Ports: []string{"80", "x", "80", "443"}, PathPrefix: "/p", Environment: "prod"}}} + Normalize(&p) + g := p.InScope[0] + if g.AssetType != "" || g.Protocol != "" || g.Environment != "" || g.PathPrefix != "" || strings.Join(g.Ports, ",") != "443,80" { + t.Errorf("normalised: %+v", g) + } +} diff --git a/internal/model/normalize.go b/internal/model/normalize.go index 28585b1..81166de 100644 --- a/internal/model/normalize.go +++ b/internal/model/normalize.go @@ -39,6 +39,7 @@ func Normalize(p *Program) { t.MaxSev = "" } t.Notes = CleanText(t.Notes, 500) + t = cleanTargetExtras(t) out = append(out, t) } out = SortTargets(out) @@ -101,3 +102,34 @@ func cleanList(in []string) []string { } return out } + +// cleanTargetExtras cleans the optional target fields: text is folded and +// cut, anything outside its enum or grammar is dropped (never guessed). +func cleanTargetExtras(t Target) Target { + t.Instructions = CleanText(t.Instructions, MaxInstructionsLen) + t.Requires = CleanText(t.Requires, MaxRequiresLen) + if !assetTypes[t.AssetType] { + t.AssetType = "" + } + if !environments[t.Environment] { + t.Environment = "" + } + if t.Protocol != "tcp" && t.Protocol != "udp" { + t.Protocol = "" + } + if t.Type != TargetURL || !strings.HasPrefix(t.PathPrefix, "/") || len(t.PathPrefix) > MaxPathPrefixLen || + !safeText(t.PathPrefix, MaxPathPrefixLen) || strings.ContainsAny(t.PathPrefix, " ?#") { + t.PathPrefix = "" + } + var ports []string + seen := map[string]bool{} + for _, p := range t.Ports { + if ValidPort(p) && !seen[p] && len(ports) < MaxPorts { + seen[p] = true + ports = append(ports, p) + } + } + sort.Strings(ports) + t.Ports = ports + return t +} diff --git a/internal/source/bountytargets.go b/internal/source/bountytargets.go index 50261d5..5f8cf8b 100644 --- a/internal/source/bountytargets.go +++ b/internal/source/bountytargets.go @@ -91,7 +91,10 @@ type dsTarget struct { value string eligible *bool maxSev string - notes string + // asset is the platform's own asset type mapped to the normalised enum + // ("" = not stated; the classified type then decides). + asset string + instructions string } // Collect implements Source. Any platform file that cannot be read fails @@ -108,7 +111,7 @@ func (b BountyTargets) Collect(ctx context.Context, c *fetch.Client) (*Result, e continue } fileURL := b.Dataset.rawURL(commit, "data/"+plat.file) - resp, err := c.Get(ctx, fileURL, maxDatasetBytes) + resp, err := c.Get(fetch.WithDeadline(ctx, datasetDeadline), fileURL, maxDatasetBytes) if err != nil { return nil, err } @@ -167,12 +170,37 @@ func targets(in []dsTarget) []model.Target { out := make([]model.Target, 0, len(in)) for _, t := range in { typ, val := classify(t.kind, t.value) + asset := t.asset + if asset == "" { + asset = assetOfType[typ] + } out = append(out, model.Target{Type: typ, Value: val, Confidence: model.ConfidencePublishedByPlatform, - Eligible: t.eligible, MaxSev: strings.ToLower(t.maxSev), Notes: t.notes}) + Eligible: t.eligible, MaxSev: strings.ToLower(t.maxSev), AssetType: asset, + PathPrefix: pathPrefix(typ, val), Instructions: t.instructions}) } return out } +// assetOfType is the asset type implied by a classified target type when +// the platform did not state one. +var assetOfType = map[string]string{ + model.TargetDomain: model.AssetDomain, model.TargetWildcard: model.AssetWildcard, model.TargetIP: model.AssetIP, + model.TargetCIDR: model.AssetCIDR, model.TargetURL: model.AssetURL, model.TargetMobileApp: model.AssetMobileApp, + model.TargetSourceRepo: model.AssetSourceCode, model.TargetOther: model.AssetOther, +} + +// pathPrefix is the path a URL target is limited to; "" for a bare host. +func pathPrefix(typ, val string) string { + if typ != model.TargetURL { + return "" + } + u, err := url.Parse(val) + if err != nil || u.Path == "" || u.Path == "/" { + return "" + } + return u.Path +} + // lastSegment is the last non-empty path segment of a URL. func lastSegment(raw string) string { u, err := url.Parse(strings.TrimSpace(raw)) @@ -223,12 +251,20 @@ func (t h1Target) target() dsTarget { switch t.Type { case "SOURCE_CODE": kind = kindSource - case "GOOGLE_PLAY_APP_ID", "APPLE_STORE_APP_ID", "TESTFLIGHT", "OTHER_APK", "WINDOWS_APP_STORE_APP_ID": + case "GOOGLE_PLAY_APP_ID", "APPLE_STORE_APP_ID", "TESTFLIGHT", "OTHER_APK", "OTHER_IPA", "WINDOWS_APP_STORE_APP_ID": kind = kindMobile case "HARDWARE", "DOWNLOADABLE_EXECUTABLES", "AI_MODEL", "SMART_CONTRACT": kind = kindOpaque } - return dsTarget{kind: kind, value: t.Identifier, eligible: t.Eligible, maxSev: t.MaxSeverity, notes: t.Instruction} + asset := map[string]string{ + "URL": model.AssetURL, "WILDCARD": model.AssetWildcard, "CIDR": model.AssetCIDR, "IP_ADDRESS": model.AssetIP, + "API": model.AssetAPI, "SOURCE_CODE": model.AssetSourceCode, "DOWNLOADABLE_EXECUTABLES": model.AssetExecutable, + "HARDWARE": model.AssetHardware, "SMART_CONTRACT": model.AssetSmartContract, "AI_MODEL": model.AssetAIModel, + "GOOGLE_PLAY_APP_ID": model.AssetAndroidApp, "OTHER_APK": model.AssetAndroidApp, + "APPLE_STORE_APP_ID": model.AssetIOSApp, "TESTFLIGHT": model.AssetIOSApp, "OTHER_IPA": model.AssetIOSApp, + "WINDOWS_APP_STORE_APP_ID": model.AssetMobileApp, "OTHER": model.AssetOther, + }[t.Type] + return dsTarget{kind: kind, value: t.Identifier, eligible: t.Eligible, maxSev: t.MaxSeverity, asset: asset, instructions: t.Instruction} } // ---- Bugcrowd ---- @@ -272,11 +308,15 @@ func (t bcTarget) target() dsTarget { case "hardware", "iot": kind = kindOpaque } + asset := map[string]string{ + "website": model.AssetURL, "api": model.AssetAPI, "ios": model.AssetIOSApp, "android": model.AssetAndroidApp, + "hardware": model.AssetHardware, "iot": model.AssetHardware, "ip_address": model.AssetIP, "other": model.AssetOther, + }[t.Type] // "network" and anything new: the classified type decides v := t.Target if v == "" { v = t.Name } - return dsTarget{kind: kind, value: v} + return dsTarget{kind: kind, value: v, asset: asset} } // ---- Intigriti ---- @@ -326,7 +366,11 @@ func (t intTarget) target() dsTarget { case "device": kind = kindOpaque } - out := dsTarget{kind: kind, value: t.Endpoint, notes: t.Description} + asset := map[string]string{ + "url": model.AssetURL, "wildcard": model.AssetWildcard, "iprange": model.AssetCIDR, "ios": model.AssetIOSApp, + "android": model.AssetAndroidApp, "device": model.AssetHardware, "other": model.AssetOther, + }[t.Type] + out := dsTarget{kind: kind, value: t.Endpoint, asset: asset, instructions: t.Description} if strings.EqualFold(strings.TrimSpace(t.Impact), "No Bounty") { no := false out.eligible = &no @@ -375,5 +419,10 @@ func (t ywhTarget) target() dsTarget { case t.Type == "ai-model" || t.Type == "challenge": kind = kindOpaque } - return dsTarget{kind: kind, value: t.Target} + asset := map[string]string{ + "web-application": model.AssetURL, "api": model.AssetAPI, "wildcard": model.AssetWildcard, + "mobile-application-ios": model.AssetIOSApp, "mobile-application-android": model.AssetAndroidApp, + "mobile-application": model.AssetMobileApp, "ai-model": model.AssetAIModel, "other": model.AssetOther, + }[t.Type] // "application", "challenge" and anything new: the classified type decides + return dsTarget{kind: kind, value: t.Target, asset: asset} } diff --git a/internal/source/dataset.go b/internal/source/dataset.go index 8aeb223..8a7ae8e 100644 --- a/internal/source/dataset.go +++ b/internal/source/dataset.go @@ -8,6 +8,7 @@ import ( "net/url" "regexp" "strings" + "time" "github.com/openctemio/programfeed/internal/fetch" "github.com/openctemio/programfeed/internal/model" @@ -28,7 +29,10 @@ const ( githubAPI = "https://api.github.com" githubRaw = "https://raw.githubusercontent.com" - maxDatasetBytes = 32 << 20 + maxDatasetBytes = 32 << 20 + // datasetDeadline bounds one large dataset download (the idle-read guard + // still fails a stalled one early). + datasetDeadline = 5 * time.Minute maxDatasetRecords = 20000 maxCommitListSize = 1 << 20 ) diff --git a/internal/source/dataset_test.go b/internal/source/dataset_test.go index a5a99b8..f707340 100644 --- a/internal/source/dataset_test.go +++ b/internal/source/dataset_test.go @@ -92,7 +92,30 @@ func TestBountyTargets(t *testing.T) { got[tg.Value] = tg.Type } want := map[string]string{"*.acmecorp.net": "wildcard", "api.acme.io": "domain", "https://app.acme.io/v1": "url", - "151.101.8.0/24": "cidr", "com.acme.app": "mobile_app", "https://github.com/acme/tool": "source_repo"} + "151.101.8.0/24": "cidr", "com.acme.app": "mobile_app", "https://github.com/acme/tool": "source_repo", + "https://api.acme.io/graphql": "url", "0x00000000000000000000000000000000000acme1": "other"} + assets := map[string]string{"*.acmecorp.net": "wildcard", "api.acme.io": "url", "https://app.acme.io/v1": "url", "151.101.8.0/24": "cidr", + "com.acme.app": "android_app", "https://github.com/acme/tool": "source_code", "https://api.acme.io/graphql": "api", + "0x00000000000000000000000000000000000acme1": "smart_contract"} + for _, tg := range acme.InScope { + if tg.AssetType != assets[tg.Value] { + t.Errorf("%s: asset_type %q, want %q", tg.Value, tg.AssetType, assets[tg.Value]) + } + switch tg.Value { + case "https://app.acme.io/v1": + if tg.PathPrefix != "/v1" { + t.Errorf("path_prefix %q", tg.PathPrefix) + } + case "api.acme.io": + if tg.Instructions != "Rate limit 5 req/s" || tg.Notes != "" || tg.PathPrefix != "" { + t.Errorf("instructions/notes: %+v", tg) + } + } + // The datasets publish none of these: they stay empty, never guessed. + if len(tg.Ports) > 0 || tg.Protocol != "" || tg.Environment != "" || tg.Requires != "" { + t.Errorf("%s: unexpected optional fields %+v", tg.Value, tg) + } + } if len(got) != len(want) { t.Errorf("in scope %v", got) } diff --git a/internal/source/diodb.go b/internal/source/diodb.go index e9720d7..bc7c5a5 100644 --- a/internal/source/diodb.go +++ b/internal/source/diodb.go @@ -73,7 +73,7 @@ var platformHosts = map[string]string{ // Collect implements Source. func (d DioDB) Collect(ctx context.Context, c *fetch.Client) (*Result, error) { - resp, err := c.Get(ctx, d.url(), maxDioDBBytes) + resp, err := c.Get(fetch.WithDeadline(ctx, datasetDeadline), d.url(), maxDioDBBytes) if err != nil { return nil, err } diff --git a/internal/source/pdbounty.go b/internal/source/pdbounty.go index 8cc12de..218908d 100644 --- a/internal/source/pdbounty.go +++ b/internal/source/pdbounty.go @@ -58,7 +58,7 @@ func (d PDBounty) Collect(ctx context.Context, c *fetch.Client) (*Result, error) return nil, err } fileURL := d.Dataset.rawURL(commit, pdFile) - resp, err := c.Get(ctx, fileURL, maxDatasetBytes) + resp, err := c.Get(fetch.WithDeadline(ctx, datasetDeadline), fileURL, maxDatasetBytes) if err != nil { return nil, err } diff --git a/internal/source/testdata/datasets/hackerone_data.json b/internal/source/testdata/datasets/hackerone_data.json index 35d752e..28aff6c 100644 --- a/internal/source/testdata/datasets/hackerone_data.json +++ b/internal/source/testdata/datasets/hackerone_data.json @@ -8,7 +8,9 @@ {"asset_identifier":"10.0.0.0/8","asset_type":"CIDR","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, {"asset_identifier":"co.uk","asset_type":"URL","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, {"asset_identifier":"com.acme.app","asset_type":"GOOGLE_PLAY_APP_ID","eligible_for_bounty":true,"instruction":"","max_severity":"high"}, - {"asset_identifier":"https://github.com/acme/tool","asset_type":"SOURCE_CODE","eligible_for_bounty":true,"instruction":"","max_severity":"critical"} + {"asset_identifier":"https://github.com/acme/tool","asset_type":"SOURCE_CODE","eligible_for_bounty":true,"instruction":"","max_severity":"critical"}, + {"asset_identifier":"https://api.acme.io/graphql","asset_type":"API","eligible_for_bounty":true,"instruction":"Use the sandbox account","max_severity":"high"}, + {"asset_identifier":"0x00000000000000000000000000000000000acme1","asset_type":"SMART_CONTRACT","eligible_for_bounty":true,"instruction":"","max_severity":"critical"} ], "out_of_scope":[{"asset_identifier":"blog.acme.io","asset_type":"URL","eligible_for_bounty":false,"instruction":"","max_severity":null}]}}, {"handle":"quiet","id":2,"name":"Quiet VDP","offers_bounties":false,"submission_state":"paused","url":"https://hackerone.com/quiet","targets":{"in_scope":[],"out_of_scope":[]}}, diff --git a/schema/program.schema.json b/schema/program.schema.json index 89792b4..12e8af5 100644 --- a/schema/program.schema.json +++ b/schema/program.schema.json @@ -44,7 +44,16 @@ "description": "published: the program's own scope. published_by_platform: scope the hosting platform published for the program, read through a public aggregate dataset; it states what the program lists and is never permission to test.inferred: derived by the collector (for example the host serving the program's security.txt); a suggestion, never an authorisation to test."}, "eligible_for_bounty": {"type": "boolean"}, "max_severity": {"enum": ["none", "low", "medium", "high", "critical"]}, - "notes": {"type": "string", "maxLength": 500} + "notes": {"type": "string", "maxLength": 500, "description": "The collector's remark about the target (for example why it is inferred)."}, + "asset_type": {"enum": ["domain", "wildcard", "url", "api", "ip", "cidr", "android_app", "ios_app", "mobile_app", "source_code", "executable", "hardware", "smart_contract", "ai_model", "other"], + "description": "Schema 1.1. The platform's own asset type mapped to a normalised kind; finer than type. Absent when the source did not say."}, + "ports": {"type": "array", "maxItems": 32, "items": {"type": "string", "pattern": "^[0-9]{1,5}(-[0-9]{1,5})?$"}, + "description": "Schema 1.1. Ports or port ranges the scope is limited to, when the platform published them."}, + "protocol": {"enum": ["tcp", "udp"], "description": "Schema 1.1."}, + "path_prefix": {"type": "string", "maxLength": 256, "pattern": "^/", "description": "Schema 1.1. url targets only: the path the scope is limited to."}, + "environment": {"enum": ["production", "staging", "other"], "description": "Schema 1.1."}, + "instructions": {"type": "string", "maxLength": 1000, "description": "Schema 1.1. The platform's testing instructions for this asset."}, + "requires": {"type": "string", "maxLength": 500, "description": "Schema 1.1. Headers, test accounts or other prerequisites the platform lists."} } }, "rejected": {