From ae79f7372832e4b7f1483dd5aed77ef063087433 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 5 Jul 2026 20:34:48 +0530 Subject: [PATCH 1/4] chore: commit wip changes on feature/wip-20260705 --- .github/workflows/ci.yml | 2 +- Makefile | 8 +++++--- lefthook.yml | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02efc8517..1bb7342d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ # Canonical CI workflow for hawk-eco Go repos. -# Source of truth: .shared-templates/workflows/go-ci.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/workflows/go-ci.yml.tmpl # # Two deployment models: # diff --git a/Makefile b/Makefile index 203dcddbd..d53208f47 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Canonical hawk-eco Makefile for Go library repos. -# Source of truth: .shared-templates/Makefile.library.tmpl at the eco root. +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/Makefile.library.tmpl # Placeholders rendered per repo: tok. # --------------------------------------------------------------------------- @@ -149,5 +149,7 @@ benchmark-quality: ## Run offline compression-quality harness (ratio + ROUGE-1 f --csv benchmarks/quality-results.csv .PHONY: hooks -hooks: - git config core.hooksPath .githooks +hooks: ## Install git hooks via lefthook (format, lint, conventional commits, co-author strip). + @command -v lefthook >/dev/null 2>&1 || (echo "install: go install github.com/evilmartians/lefthook@latest" && exit 1) + git config --unset core.hooksPath 2>/dev/null || true + lefthook install diff --git a/lefthook.yml b/lefthook.yml index 7d5bdaf09..edab5775e 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -1,5 +1,5 @@ # Canonical lefthook config for hawk-eco Go repos. -# Source of truth: .shared-templates/lefthook.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/lefthook.yml.tmpl # # Install lefthook: # brew install lefthook (macOS) @@ -75,6 +75,9 @@ pre-commit: pre-push: commands: + boundaries: + run: bash ./scripts/check-ecosystem-boundaries.sh + test: run: go test ./... -count=1 -timeout=60s From 014f36526ccaf06ce8682e941179bb46af65af6c Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Wed, 8 Jul 2026 05:15:34 +0530 Subject: [PATCH 2/4] Secrets scanning and CI hardening pass - Update AGENTS.md to the shared hawk-eco extension-authoring format --- .github/workflows/ci.yml | 8 +- AGENTS.md | 268 +++++++++++++++++++++--------------- internal/secrets/secrets.go | 35 ++++- 3 files changed, 190 insertions(+), 121 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bb7342d9..3e1eebbe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -192,10 +192,10 @@ jobs: cache: true - name: Run fuzz targets run: | - go test -fuzz=FuzzRedactSecrets -fuzztime=60s . || true - go test -fuzz=FuzzPipelineProcess -fuzztime=60s ./internal/filter/... || true - go test -fuzz=FuzzNgram -fuzztime=60s ./internal/filter/... || true - go test -fuzz=FuzzEntropyFilter -fuzztime=60s ./internal/filter/... || true + go test -fuzz=FuzzRedactSecrets -fuzztime=60s . + go test -fuzz=FuzzPipelineProcess -fuzztime=60s ./internal/filter/... + go test -fuzz=FuzzNgram -fuzztime=60s ./internal/filter/... + go test -fuzz=FuzzEntropyFilter -fuzztime=60s ./internal/filter/... # ------------------------------------------------------------------------- # Cross-platform build matrix — only for repos that produce a binary. diff --git a/AGENTS.md b/AGENTS.md index c90ee694b..4976a4568 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,120 +1,168 @@ -# AGENTS.md — Tok +--- +description: Extending hawk-eco — how to write AGENTS.md files, custom specialists, skills, hooks, MCP servers, and plugins. +globs: "*.go, *.js, *.md, *.json, *.toml, *.yaml, *.yml" +alwaysApply: false +--- -Tokenizer, compression, secrets scanning, and rate limiting library for AI coding agents. +# Extending hawk-eco -## Design Principles +hawk-eco is an open-source code intelligence platform. This document describes how to extend it with custom tools, skills, hooks, and integrations. -- **Library only** — no CLI, no binary -- **Token-efficient** — optimized for context window management -- **Security-first** — secrets scanning prevents credential leaks +## 1. Drop a project `AGENTS.md` -## Observability +When hawk-eco starts in a directory, it looks for project-level instructions and injects them into the system prompt. The lookup walks from your current working directory **up to the nearest git root** and reads the first matching file at each level — general rules at the repo root, more specific rules in sub-trees. Files are labeled with their directory in the prompt (e.g. `## Project guidelines (services/api/AGENTS.md)`). -See [hawk/docs/OTEL-CONVENTIONS.md](https://github.com/GrayCodeAI/hawk/blob/main/docs/OTEL-CONVENTIONS.md) for the shared OpenTelemetry attribute vocabulary (`gen_ai.*`, `cost.usd`, etc.) used across all GrayCodeAI repos. +Accepted file names, in priority order at each level: -## Build & Test +| Path | Notes | +| --- | --- | +| `./AGENTS.md` | The classic spot — committed to your repo, shared with the team. | +| `./ZERO.md` | Brand-specific alias. Same format, lower priority. | +| `./.zero/AGENTS.md` | Project-local, hidden, gitignored. Personal notes that stay out of git. | + +Matching is **case-insensitive** on the basename, so `AGENTS.md`, `Agents.md`, and `agents.md` resolve to the same file on Windows and macOS. The git-tracked filename in this repo is `AGENTS.md` — keep that on case-sensitive filesystems (Linux, the WSL filesystem, or a CI runner) to match what the loader looks for. + +Both files use the same format. YAML frontmatter is optional; the markdown body is loaded as instructions for the agent. hawk-eco reads the file once at session start, so changes take effect on the next launch — not mid-session. + +```markdown +# Project conventions for + +- Build with `make`, not `go build` directly. +- Tests live next to the source file (`foo_test.go` next to `foo.go`). +- Run `make lint` before opening a PR. +- Never edit files under `third_party/` — those are vendored. +``` + +Tips: + +- Keep each file under ~8 KiB. hawk-eco caps the **total** across all matched files at 32 KiB; everything past the cap is dropped. +- Re-state rules in the imperative voice: "Run `make lint`", not "you should consider running the linter". +- Don't put secrets, model IDs, or environment-specific paths in `AGENTS.md`. Use config files for those. +- In a monorepo, drop a narrower `AGENTS.md` in each sub-tree (e.g. `services/api/AGENTS.md`). hawk-eco picks those up automatically when you launch from inside the sub-tree. +- A YAML frontmatter block (`---\n...\n---`) at the top is preserved verbatim in the injected prompt but is not parsed for `globs:` or `alwaysApply:` scoping today — keep the body self-contained. + +### Personal guidelines, across every project + +For preferences that follow *you*, not a specific repo (tone, tooling habits, workflow), drop a `ZERO.md` in your user config directory: `~/.config/hawk-eco/ZERO.md` on Linux/macOS, `%AppData%\Roaming\hawk-eco\ZERO.md` on Windows — the same directory as config files and your personal specialists. Same format and 8 KiB cap as the project files above, and the same case-insensitive basename match. + +This file is injected as its own `## User guidelines` section, before the project's `AGENTS.md`/`ZERO.md`, and is labeled as personal preference in the prompt: project guidelines are the later, more specific instruction and take precedence over it when the two conflict. + +## 2. Custom specialists + +Specialists are hawk-eco's sub-agents. Three scopes, in priority order: + +| Scope | Path | Shared? | +| --- | --- | --- | +| Built-in | compiled into hawk-eco | yes | +| User | `~/.config/hawk-eco/specialists/*.md` | no — your machine only | +| Project | `./.zero/specialists/*.md` | yes — the repo team | + +Project overrides user overrides built-in when names collide. + +A specialist is a markdown manifest with frontmatter and a system prompt: + +```markdown +--- +description: Reviews API changes for breaking-change risk and missing tests. +tools: read-only,plan +--- + +You review API changes. For every changed hunk in `internal/api/` or any file +that ends in `_api.go`: + +1. Confirm the public signature is backward-compatible, or note the breaking + change explicitly with the migration path. +2. Confirm a corresponding test exists in `internal/api/*_test.go` and that + the new behaviour is exercised. +3. Flag any new exported symbol without a doc comment. + +Reply with one JSON object per finding: `{"file", "line", "severity", "message", "fix"}`. +``` + +CLI management: ```bash -go test ./... # Run all tests -go test -race ./... # Race detector -go test -coverprofile=c.out ./... # Coverage -go vet ./... # Static analysis -gofumpt -w . # Format +hawk-eco specialist list +hawk-eco specialist show api-reviewer +hawk-eco specialist create api-reviewer \ + --project \ + --description "Reviews API changes" \ + --tools read-only,plan \ + --prompt "$(cat api-reviewer.md)" +hawk-eco specialist edit api-reviewer --project +hawk-eco specialist delete api-reviewer --project +hawk-eco specialist path # prints the resolved specialists directory ``` -## Architecture - -- `tokenizer.go` — Token counting and estimation -- `compressor.go` — Context compression strategies -- `secrets.go` — Secrets scanning and redaction -- `ratelimit.go` — Rate limiting for API calls -- `budget.go` — Token budget management -- `filter.go` — Content filtering and validation - -## Conventions - -- Go 1.26+, pure Go, no CGO -- Table-driven tests -- Conventional Commits: `feat:`, `fix:`, `docs:`, `refactor:`, `test:` -- No `Co-authored-by:` trailers (auto-stripped by githook) -- `gofumpt` formatting enforced in CI -- Quality.yml coverage threshold: 30% - -## Common Pitfalls - -- Token estimation is approximate — don't rely on exact counts -- Secrets scanning has false positives — use allowlists for known patterns -- Rate limiter tests need careful timing assertions - -## Naming Conventions - -- **Top-level functions are verbs**: `Compress()`, `EstimateTokens()`, `EstimateTokensPrecise()`, `WarmupTokenizer()` -- **Option pattern**: `Option` interface with `optFunc` adapter — same pattern as sight and inspect -- **Preset options are bare vars**: `Minimal`, `Aggressive`, `Surface`, `Adaptive`, `Code`, `Log` — exported `var Option` values -- **Mode is a string type**: `Mode` with constants `ModeMinimal`, `ModeAggressive` -- **Tier is a string type**: `Tier` with constants `TierSurface`, `TierTrim`, `TierExtract`, `TierCore`, `TierCode`, `TierLog`, `TierThread`, `TierAdaptive` -- **Internal packages**: `internal/core/` (tokenizer), `internal/filter/` (pipeline), `internal/secrets/` (detector), `internal/codeaware/` (code-specific) -- **Secret detector pattern**: `DefaultSecretDetector()` returns singleton, `NewSecretDetector()` creates fresh instance -- **SecretMatch is a type alias**: `type SecretMatch = secrets.SecretMatch` — re-exports from internal package -- **Stats struct**: returned from `Compress()` — `OriginalTokens`, `FinalTokens`, compression ratio fields - -## API Patterns - -- **One-shot compression**: `tok.Compress(text, opts...)` — creates pipeline internally, returns `(string, Stats)` -- **Reusable compressor**: `tok.NewCompressor(opts...)` returns `*Compressor` with `Compress(text)` method — reuses caches -- **Token estimation**: `EstimateTokens(text)` for fast approximation, `EstimateTokensPrecise(text)` for BPE accuracy -- **Warmup**: `WarmupTokenizer()` pre-initializes BPE tokenizer in background — call at startup to avoid first-call latency -- **Budget constraint**: `WithBudget(tokens)` option hard-limits output token count — pipeline truncates to fit -- **Query-driven filtering**: `WithQuery(intent)` option provides goal context for relevance-based filtering -- **Tier selection**: `WithTier(TierCode)` selects pre-built pipeline profile — each tier has different layer counts -- **Mode selection**: `WithMode(ModeAggressive)` controls compression aggressiveness within a tier -- **Secret detection**: `DefaultSecretDetector().DetectSecrets(text)` returns `[]SecretMatch`; `.RedactSecrets(text)` returns redacted string -- **Entropy-based detection**: `DetectAndRedactWithEntropy(text, threshold)` — pattern matching + Shannon entropy analysis - -## Testing Patterns - -- **External test package**: `package tok_test` — tests import `tok` as a consumer would -- **Simple assertions**: `TestCompress` checks non-empty output and non-zero `OriginalTokens` — minimal, focused -- **Empty input test**: `TestCompress_Empty` — verify empty string returns empty string and zero stats -- **Preset smoke tests**: `TestCompress_Aggressive`, `TestCompress_WithTier`, `TestCompress_WithQuery` — each preset/option tested -- **Budget test**: `TestCompress_WithBudget` — create large input, compress with budget 50, verify `FinalTokens <= 60` -- **Concurrent safety test**: `TestCompress_Concurrent` — 10 goroutines compressing same input with `sync.WaitGroup` -- **Token estimation test**: `TestEstimateTokens` — verify non-zero for known input -- **Reusable compressor test**: `TestNewCompressor` — create compressor, call `Compress()` twice, verify both return results -- **Secret detection tests**: `secrets_test.go` — pattern matching, entropy edge cases, allowlist exclusions -- **Bench tests**: `internal/` subdirectories — performance-critical paths - -## Refactoring Guidelines - -- **Safe to refactor**: `internal/filter/` pipeline layers — add, remove, reorder filter stages -- **Safe to refactor**: `internal/core/` tokenizer — improve estimation accuracy, add new tokenizers -- **Safe to refactor**: `internal/secrets/` patterns — add new detection patterns, tune entropy threshold -- **Safe to refactor**: `internal/codeaware/` — language-specific compression rules -- **Do not touch**: `Compress()` function signature — primary API contract -- **Do not touch**: `Option` interface and preset vars — used by all consumers -- **Do not touch**: `Stats` struct fields — returned from every `Compress()` call -- **Do not touch**: `SecretDetector` public methods — used by hawk for secret scanning -- **Do not touch**: `Tier` and `Mode` constants — referenced in configs and CLI flags -- **Safe to extend**: add new `Tier` values, new filter layers, new secret patterns, new compression strategies -- **When adding a tier**: add constant to `Tier` type, implement pipeline config in `internal/filter/` - -## Key File Locations - -| What | Where | -|---|---| -| Public API entry point | `tok.go` (`Compress()`, `EstimateTokens()`, `WarmupTokenizer()`) | -| Reusable compressor | `compressor.go` (`Compressor` struct) | -| Options & presets | `options.go` (`Option`, `Mode`, `Tier`, `With*` functions, preset vars) | -| Secret detection | `secrets.go` (`SecretDetector`, `DetectSecrets()`, `RedactSecrets()`) | -| Stats type | `stats.go` (returned from `Compress()`) | -| Stream processing | `stream.go` | -| Core tokenizer | `internal/core/` (BPE tokenizer, estimation) | -| Filter pipeline | `internal/filter/` (pipeline coordinator, tier configs, layer execution) | -| Code-aware filters | `internal/codeaware/` (language-specific compression) | -| Secret patterns | `internal/secrets/` (regex patterns, entropy analysis, allowlists) | -| Utility functions | `internal/utils/` | -| Main test file | `tok_test.go` (compression, estimation, concurrency, presets) | -| Secret tests | `secrets_test.go` | -| Compression tests | `compressor_test.go` (if exists) | -| Benchmark tests | `internal/*/bench_test.go` | -| Linter config | `.golangci.yml` (govet, ineffassign, misspell — minimal) | +## 3. Skills + +Skills are markdown instruction files that extend agent capabilities. They can be: +- Project-scoped: dropped in `./.zero/skills/` or `./skills/` +- User-scoped: dropped in `~/.config/hawk-eco/skills/` + +A skill manifest: + +```markdown +--- +description: How to review Go code for security issues +globs: "*.go" +alwaysApply: true +--- + +When reviewing Go code for security: + +1. Check for SQL injection patterns +2. Verify error handling doesn't expose sensitive data +3. Confirm secrets are not hardcoded +4. Validate input sanitization +``` + +## 4. Hooks + +Hooks allow custom commands to run at specific lifecycle points: +- `beforeReview` — runs before code review starts +- `afterReview` — runs after code review completes +- `sessionStart` — runs at session initialization +- `sessionEnd` — runs at session teardown + +```bash +hawk-eco hook add beforeReview --command "lint-check" +hawk-eco hook remove beforeReview +hawk-eco hook list +``` + +## 5. MCP integration + +MCP (Model Context Protocol) servers can expose tools to hawk-eco: + +```bash +hawk-eco mcp add --name server --url http://localhost:8080 +hawk-eco mcp remove server +hawk-eco mcp list +``` + +## 6. Plugins + +Plugins extend hawk-eco with custom tools and capabilities: + +```bash +hawk-eco plugin add --name my-plugin --path ./my-plugin +hawk-eco plugin remove my-plugin +hawk-eco plugin list +``` + +## 7. Verification + +hawk-eco includes a self-verification system to validate local changes before contributing: + +```bash +hawk-eco verify +hawk-eco verify --fix +``` + +## Development + +```bash +make lint +hawk-eco verify +``` diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index c0d3047bd..af078e9af 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -148,10 +148,17 @@ func (sd *SecretDetector) RedactSecrets(text string) string { var sb strings.Builder lastIdx := 0 for _, m := range matches { - if m.StartPos < lastIdx { + if m.EndPos <= lastIdx { + // Fully covered by a previously redacted match; nothing left to hide. continue } - sb.WriteString(text[lastIdx:m.StartPos]) + start := m.StartPos + if start < lastIdx { + // Overlaps a previously redacted match but extends further right: + // redact only the un-redacted tail so no part of the secret leaks. + start = lastIdx + } + sb.WriteString(text[lastIdx:start]) sb.WriteString("[REDACTED:") sb.WriteString(m.Type) sb.WriteString("]") @@ -184,17 +191,31 @@ func maskString(s string) string { return s[:4] + strings.Repeat("*", len(s)-8) + s[len(s)-4:] } +// wordPattern matches whitespace-delimited words, used to locate entropy +// candidates without disturbing the original whitespace around them. +var wordPattern = regexp.MustCompile(`\S+`) + // DetectAndRedactWithEntropy detects secrets using both pattern matching and entropy analysis. func (sd *SecretDetector) DetectAndRedactWithEntropy(text string, entropyThreshold float64) string { result := sd.RedactSecrets(text) - // Also check for high-entropy strings that might be secrets - words := strings.Fields(result) - for i, word := range words { + // Also check for high-entropy strings that might be secrets. Replace matched + // words in place (rather than splitting on whitespace and rejoining with a + // single space) so original spacing/newlines are preserved. + var sb strings.Builder + lastIdx := 0 + for _, loc := range wordPattern.FindAllStringIndex(result, -1) { + start, end := loc[0], loc[1] + word := result[start:end] + sb.WriteString(result[lastIdx:start]) if calculateShannonEntropy(word) > entropyThreshold && len(word) > 16 { - words[i] = "[REDACTED:HIGH_ENTROPY]" + sb.WriteString("[REDACTED:HIGH_ENTROPY]") + } else { + sb.WriteString(word) } + lastIdx = end } - return strings.Join(words, " ") + sb.WriteString(result[lastIdx:]) + return sb.String() } // calculateShannonEntropy calculates the Shannon entropy of a string. From f659d19d75a54c8faeb695c60c3b1dcb3c1e4b21 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Wed, 8 Jul 2026 07:54:58 +0530 Subject: [PATCH 3/4] Fix SIGILL crash in AVX2 assembly from illegal EVEX-encoded VPBROADCASTB VPBROADCASTB with a general-purpose-register source only has an AVX-512 (EVEX-encoded) form; the AVX2 (VEX-encoded) form requires an XMM or memory source operand. hasANSIavx2 and countBytesAVX2 broadcast directly from a GPR, so despite being gated only on cpu.X86.HasAVX2, they emitted an AVX-512 instruction that crashes with SIGILL on AVX2-only hardware (as hit by the fuzz job on GitHub Actions runners). Stage the byte through an XMM register first to keep both broadcasts in valid AVX2 form. --- internal/fastops/fastops_amd64.s | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/fastops/fastops_amd64.s b/internal/fastops/fastops_amd64.s index 645d0c3a5..cbe79dc37 100644 --- a/internal/fastops/fastops_amd64.s +++ b/internal/fastops/fastops_amd64.s @@ -10,9 +10,14 @@ TEXT ·hasANSIavx2(SB), NOSPLIT, $0-17 TESTQ CX, CX JEQ ansi_false - // Broadcast 0x1b to all 32 bytes of Y1 + // Broadcast 0x1b to all 32 bytes of Y1. VPBROADCASTB only has a + // GPR-source form under AVX-512 (EVEX-encoded); the AVX2 (VEX-encoded) + // form requires an XMM/memory source, so stage the byte through X1 + // first to avoid emitting an AVX-512 instruction that SIGILLs on + // AVX2-only hardware. MOVB $0x1b, R8 - VPBROADCASTB R8, Y1 + MOVQ R8, X1 + VPBROADCASTB X1, Y1 // Number of 32-byte chunks → byte offset MOVQ CX, AX @@ -70,8 +75,11 @@ TEXT ·countBytesAVX2(SB), NOSPLIT, $0-32 TESTQ CX, CX JEQ count_done - // Broadcast target byte to all 32 bytes of Y1 - VPBROADCASTB R8, Y1 + // Broadcast target byte to all 32 bytes of Y1 via an AVX2-safe (VEX, + // XMM-source) form; see hasANSIavx2 for why a direct GPR source is + // avoided. + MOVQ R8, X1 + VPBROADCASTB X1, Y1 MOVQ CX, AX SHRQ $5, AX // AX = floor(len / 32) From f45b7670b57410fcadbb798fcadab76c24582b88 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Wed, 8 Jul 2026 08:06:35 +0530 Subject: [PATCH 4/4] Fix FuzzRedactSecrets false positive from marker text self-collision A short captured secret value can coincidentally be a substring of a fixed redaction marker's type label (e.g. captured value "eare" is a substring of the marker text "[REDACTED:Bearer Token]", since "eare" is itself a substring of the word "Bearer"). The invariant check was counting raw substring occurrences across the whole redacted output, so this coincidence was flagged as the redactor failing to reduce exposure, when in fact the original occurrence was correctly removed and the marker text is not a leak of the secret. Strip [REDACTED:...] markers before counting so only genuinely leaked occurrences count. --- secrets_fuzz_test.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/secrets_fuzz_test.go b/secrets_fuzz_test.go index 5f7536a61..addb6156b 100644 --- a/secrets_fuzz_test.go +++ b/secrets_fuzz_test.go @@ -1,12 +1,25 @@ package tok_test import ( + "regexp" "strings" "testing" "github.com/GrayCodeAI/tok" ) +// redactionMarkerPattern matches the "[REDACTED:]" markers RedactSecrets +// substitutes in place of detected secrets. +var redactionMarkerPattern = regexp.MustCompile(`\[REDACTED:[^\]]*\]`) + +// stripRedactionMarkers removes "[REDACTED:]" markers from s. Used to +// check for leaked secret values outside of the markers themselves, since a +// short captured value can coincidentally be a substring of a fixed type +// label (e.g. "Bearer Token") without that being an actual leak. +func stripRedactionMarkers(s string) string { + return redactionMarkerPattern.ReplaceAllString(s, "") +} + // FuzzRedactSecrets exercises the secrets redactor with arbitrary input. // // It asserts the security-relevant invariants that hold for the current @@ -71,11 +84,18 @@ func FuzzRedactSecrets(f *testing.F) { if !strings.Contains(out, "[REDACTED:") { t.Fatalf("redacted output carries no [REDACTED:] marker\ninput: %q\nout: %q", input, out) } + outsideMarkers := stripRedactionMarkers(out) for _, m := range matches { if len(m.Value) < 4 { continue } - if after, before := strings.Count(out, m.Value), strings.Count(input, m.Value); after >= before { + // Count occurrences outside of "[REDACTED:]" markers only: + // a short captured value can coincidentally be a substring of a + // *type label* (e.g. captured value "eare" is a substring of the + // fixed marker text "[REDACTED:Bearer Token]"), which is a + // spurious match against our own marker text, not a leak of the + // original secret. + if after, before := strings.Count(outsideMarkers, m.Value), strings.Count(input, m.Value); after >= before { t.Fatalf("detected secret value not reduced by redaction (before=%d after=%d)\ntype: %s\nvalue: %q\ninput: %q\nout: %q", before, after, m.Type, m.Value, input, out) }