diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02efc8517..3e1eebbe1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ # Canonical CI workflow for hawk-eco Go repos. -# Source of truth: .shared-templates/workflows/go-ci.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/workflows/go-ci.yml.tmpl # # Two deployment models: # @@ -192,10 +192,10 @@ jobs: cache: true - name: Run fuzz targets run: | - go test -fuzz=FuzzRedactSecrets -fuzztime=60s . || true - go test -fuzz=FuzzPipelineProcess -fuzztime=60s ./internal/filter/... || true - go test -fuzz=FuzzNgram -fuzztime=60s ./internal/filter/... || true - go test -fuzz=FuzzEntropyFilter -fuzztime=60s ./internal/filter/... || true + go test -fuzz=FuzzRedactSecrets -fuzztime=60s . + go test -fuzz=FuzzPipelineProcess -fuzztime=60s ./internal/filter/... + go test -fuzz=FuzzNgram -fuzztime=60s ./internal/filter/... + go test -fuzz=FuzzEntropyFilter -fuzztime=60s ./internal/filter/... # ------------------------------------------------------------------------- # Cross-platform build matrix — only for repos that produce a binary. diff --git a/AGENTS.md b/AGENTS.md index c90ee694b..4976a4568 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,120 +1,168 @@ -# AGENTS.md — Tok +--- +description: Extending hawk-eco — how to write AGENTS.md files, custom specialists, skills, hooks, MCP servers, and plugins. +globs: "*.go, *.js, *.md, *.json, *.toml, *.yaml, *.yml" +alwaysApply: false +--- -Tokenizer, compression, secrets scanning, and rate limiting library for AI coding agents. +# Extending hawk-eco -## Design Principles +hawk-eco is an open-source code intelligence platform. This document describes how to extend it with custom tools, skills, hooks, and integrations. -- **Library only** — no CLI, no binary -- **Token-efficient** — optimized for context window management -- **Security-first** — secrets scanning prevents credential leaks +## 1. Drop a project `AGENTS.md` -## Observability +When hawk-eco starts in a directory, it looks for project-level instructions and injects them into the system prompt. The lookup walks from your current working directory **up to the nearest git root** and reads the first matching file at each level — general rules at the repo root, more specific rules in sub-trees. Files are labeled with their directory in the prompt (e.g. `## Project guidelines (services/api/AGENTS.md)`). -See [hawk/docs/OTEL-CONVENTIONS.md](https://github.com/GrayCodeAI/hawk/blob/main/docs/OTEL-CONVENTIONS.md) for the shared OpenTelemetry attribute vocabulary (`gen_ai.*`, `cost.usd`, etc.) used across all GrayCodeAI repos. +Accepted file names, in priority order at each level: -## Build & Test +| Path | Notes | +| --- | --- | +| `./AGENTS.md` | The classic spot — committed to your repo, shared with the team. | +| `./ZERO.md` | Brand-specific alias. Same format, lower priority. | +| `./.zero/AGENTS.md` | Project-local, hidden, gitignored. Personal notes that stay out of git. | + +Matching is **case-insensitive** on the basename, so `AGENTS.md`, `Agents.md`, and `agents.md` resolve to the same file on Windows and macOS. The git-tracked filename in this repo is `AGENTS.md` — keep that on case-sensitive filesystems (Linux, the WSL filesystem, or a CI runner) to match what the loader looks for. + +Both files use the same format. YAML frontmatter is optional; the markdown body is loaded as instructions for the agent. hawk-eco reads the file once at session start, so changes take effect on the next launch — not mid-session. + +```markdown +# Project conventions for + +- Build with `make`, not `go build` directly. +- Tests live next to the source file (`foo_test.go` next to `foo.go`). +- Run `make lint` before opening a PR. +- Never edit files under `third_party/` — those are vendored. +``` + +Tips: + +- Keep each file under ~8 KiB. hawk-eco caps the **total** across all matched files at 32 KiB; everything past the cap is dropped. +- Re-state rules in the imperative voice: "Run `make lint`", not "you should consider running the linter". +- Don't put secrets, model IDs, or environment-specific paths in `AGENTS.md`. Use config files for those. +- In a monorepo, drop a narrower `AGENTS.md` in each sub-tree (e.g. `services/api/AGENTS.md`). hawk-eco picks those up automatically when you launch from inside the sub-tree. +- A YAML frontmatter block (`---\n...\n---`) at the top is preserved verbatim in the injected prompt but is not parsed for `globs:` or `alwaysApply:` scoping today — keep the body self-contained. + +### Personal guidelines, across every project + +For preferences that follow *you*, not a specific repo (tone, tooling habits, workflow), drop a `ZERO.md` in your user config directory: `~/.config/hawk-eco/ZERO.md` on Linux/macOS, `%AppData%\Roaming\hawk-eco\ZERO.md` on Windows — the same directory as config files and your personal specialists. Same format and 8 KiB cap as the project files above, and the same case-insensitive basename match. + +This file is injected as its own `## User guidelines` section, before the project's `AGENTS.md`/`ZERO.md`, and is labeled as personal preference in the prompt: project guidelines are the later, more specific instruction and take precedence over it when the two conflict. + +## 2. Custom specialists + +Specialists are hawk-eco's sub-agents. Three scopes, in priority order: + +| Scope | Path | Shared? | +| --- | --- | --- | +| Built-in | compiled into hawk-eco | yes | +| User | `~/.config/hawk-eco/specialists/*.md` | no — your machine only | +| Project | `./.zero/specialists/*.md` | yes — the repo team | + +Project overrides user overrides built-in when names collide. + +A specialist is a markdown manifest with frontmatter and a system prompt: + +```markdown +--- +description: Reviews API changes for breaking-change risk and missing tests. +tools: read-only,plan +--- + +You review API changes. For every changed hunk in `internal/api/` or any file +that ends in `_api.go`: + +1. Confirm the public signature is backward-compatible, or note the breaking + change explicitly with the migration path. +2. Confirm a corresponding test exists in `internal/api/*_test.go` and that + the new behaviour is exercised. +3. Flag any new exported symbol without a doc comment. + +Reply with one JSON object per finding: `{"file", "line", "severity", "message", "fix"}`. +``` + +CLI management: ```bash -go test ./... # Run all tests -go test -race ./... # Race detector -go test -coverprofile=c.out ./... # Coverage -go vet ./... # Static analysis -gofumpt -w . # Format +hawk-eco specialist list +hawk-eco specialist show api-reviewer +hawk-eco specialist create api-reviewer \ + --project \ + --description "Reviews API changes" \ + --tools read-only,plan \ + --prompt "$(cat api-reviewer.md)" +hawk-eco specialist edit api-reviewer --project +hawk-eco specialist delete api-reviewer --project +hawk-eco specialist path # prints the resolved specialists directory ``` -## Architecture - -- `tokenizer.go` — Token counting and estimation -- `compressor.go` — Context compression strategies -- `secrets.go` — Secrets scanning and redaction -- `ratelimit.go` — Rate limiting for API calls -- `budget.go` — Token budget management -- `filter.go` — Content filtering and validation - -## Conventions - -- Go 1.26+, pure Go, no CGO -- Table-driven tests -- Conventional Commits: `feat:`, `fix:`, `docs:`, `refactor:`, `test:` -- No `Co-authored-by:` trailers (auto-stripped by githook) -- `gofumpt` formatting enforced in CI -- Quality.yml coverage threshold: 30% - -## Common Pitfalls - -- Token estimation is approximate — don't rely on exact counts -- Secrets scanning has false positives — use allowlists for known patterns -- Rate limiter tests need careful timing assertions - -## Naming Conventions - -- **Top-level functions are verbs**: `Compress()`, `EstimateTokens()`, `EstimateTokensPrecise()`, `WarmupTokenizer()` -- **Option pattern**: `Option` interface with `optFunc` adapter — same pattern as sight and inspect -- **Preset options are bare vars**: `Minimal`, `Aggressive`, `Surface`, `Adaptive`, `Code`, `Log` — exported `var Option` values -- **Mode is a string type**: `Mode` with constants `ModeMinimal`, `ModeAggressive` -- **Tier is a string type**: `Tier` with constants `TierSurface`, `TierTrim`, `TierExtract`, `TierCore`, `TierCode`, `TierLog`, `TierThread`, `TierAdaptive` -- **Internal packages**: `internal/core/` (tokenizer), `internal/filter/` (pipeline), `internal/secrets/` (detector), `internal/codeaware/` (code-specific) -- **Secret detector pattern**: `DefaultSecretDetector()` returns singleton, `NewSecretDetector()` creates fresh instance -- **SecretMatch is a type alias**: `type SecretMatch = secrets.SecretMatch` — re-exports from internal package -- **Stats struct**: returned from `Compress()` — `OriginalTokens`, `FinalTokens`, compression ratio fields - -## API Patterns - -- **One-shot compression**: `tok.Compress(text, opts...)` — creates pipeline internally, returns `(string, Stats)` -- **Reusable compressor**: `tok.NewCompressor(opts...)` returns `*Compressor` with `Compress(text)` method — reuses caches -- **Token estimation**: `EstimateTokens(text)` for fast approximation, `EstimateTokensPrecise(text)` for BPE accuracy -- **Warmup**: `WarmupTokenizer()` pre-initializes BPE tokenizer in background — call at startup to avoid first-call latency -- **Budget constraint**: `WithBudget(tokens)` option hard-limits output token count — pipeline truncates to fit -- **Query-driven filtering**: `WithQuery(intent)` option provides goal context for relevance-based filtering -- **Tier selection**: `WithTier(TierCode)` selects pre-built pipeline profile — each tier has different layer counts -- **Mode selection**: `WithMode(ModeAggressive)` controls compression aggressiveness within a tier -- **Secret detection**: `DefaultSecretDetector().DetectSecrets(text)` returns `[]SecretMatch`; `.RedactSecrets(text)` returns redacted string -- **Entropy-based detection**: `DetectAndRedactWithEntropy(text, threshold)` — pattern matching + Shannon entropy analysis - -## Testing Patterns - -- **External test package**: `package tok_test` — tests import `tok` as a consumer would -- **Simple assertions**: `TestCompress` checks non-empty output and non-zero `OriginalTokens` — minimal, focused -- **Empty input test**: `TestCompress_Empty` — verify empty string returns empty string and zero stats -- **Preset smoke tests**: `TestCompress_Aggressive`, `TestCompress_WithTier`, `TestCompress_WithQuery` — each preset/option tested -- **Budget test**: `TestCompress_WithBudget` — create large input, compress with budget 50, verify `FinalTokens <= 60` -- **Concurrent safety test**: `TestCompress_Concurrent` — 10 goroutines compressing same input with `sync.WaitGroup` -- **Token estimation test**: `TestEstimateTokens` — verify non-zero for known input -- **Reusable compressor test**: `TestNewCompressor` — create compressor, call `Compress()` twice, verify both return results -- **Secret detection tests**: `secrets_test.go` — pattern matching, entropy edge cases, allowlist exclusions -- **Bench tests**: `internal/` subdirectories — performance-critical paths - -## Refactoring Guidelines - -- **Safe to refactor**: `internal/filter/` pipeline layers — add, remove, reorder filter stages -- **Safe to refactor**: `internal/core/` tokenizer — improve estimation accuracy, add new tokenizers -- **Safe to refactor**: `internal/secrets/` patterns — add new detection patterns, tune entropy threshold -- **Safe to refactor**: `internal/codeaware/` — language-specific compression rules -- **Do not touch**: `Compress()` function signature — primary API contract -- **Do not touch**: `Option` interface and preset vars — used by all consumers -- **Do not touch**: `Stats` struct fields — returned from every `Compress()` call -- **Do not touch**: `SecretDetector` public methods — used by hawk for secret scanning -- **Do not touch**: `Tier` and `Mode` constants — referenced in configs and CLI flags -- **Safe to extend**: add new `Tier` values, new filter layers, new secret patterns, new compression strategies -- **When adding a tier**: add constant to `Tier` type, implement pipeline config in `internal/filter/` - -## Key File Locations - -| What | Where | -|---|---| -| Public API entry point | `tok.go` (`Compress()`, `EstimateTokens()`, `WarmupTokenizer()`) | -| Reusable compressor | `compressor.go` (`Compressor` struct) | -| Options & presets | `options.go` (`Option`, `Mode`, `Tier`, `With*` functions, preset vars) | -| Secret detection | `secrets.go` (`SecretDetector`, `DetectSecrets()`, `RedactSecrets()`) | -| Stats type | `stats.go` (returned from `Compress()`) | -| Stream processing | `stream.go` | -| Core tokenizer | `internal/core/` (BPE tokenizer, estimation) | -| Filter pipeline | `internal/filter/` (pipeline coordinator, tier configs, layer execution) | -| Code-aware filters | `internal/codeaware/` (language-specific compression) | -| Secret patterns | `internal/secrets/` (regex patterns, entropy analysis, allowlists) | -| Utility functions | `internal/utils/` | -| Main test file | `tok_test.go` (compression, estimation, concurrency, presets) | -| Secret tests | `secrets_test.go` | -| Compression tests | `compressor_test.go` (if exists) | -| Benchmark tests | `internal/*/bench_test.go` | -| Linter config | `.golangci.yml` (govet, ineffassign, misspell — minimal) | +## 3. Skills + +Skills are markdown instruction files that extend agent capabilities. They can be: +- Project-scoped: dropped in `./.zero/skills/` or `./skills/` +- User-scoped: dropped in `~/.config/hawk-eco/skills/` + +A skill manifest: + +```markdown +--- +description: How to review Go code for security issues +globs: "*.go" +alwaysApply: true +--- + +When reviewing Go code for security: + +1. Check for SQL injection patterns +2. Verify error handling doesn't expose sensitive data +3. Confirm secrets are not hardcoded +4. Validate input sanitization +``` + +## 4. Hooks + +Hooks allow custom commands to run at specific lifecycle points: +- `beforeReview` — runs before code review starts +- `afterReview` — runs after code review completes +- `sessionStart` — runs at session initialization +- `sessionEnd` — runs at session teardown + +```bash +hawk-eco hook add beforeReview --command "lint-check" +hawk-eco hook remove beforeReview +hawk-eco hook list +``` + +## 5. MCP integration + +MCP (Model Context Protocol) servers can expose tools to hawk-eco: + +```bash +hawk-eco mcp add --name server --url http://localhost:8080 +hawk-eco mcp remove server +hawk-eco mcp list +``` + +## 6. Plugins + +Plugins extend hawk-eco with custom tools and capabilities: + +```bash +hawk-eco plugin add --name my-plugin --path ./my-plugin +hawk-eco plugin remove my-plugin +hawk-eco plugin list +``` + +## 7. Verification + +hawk-eco includes a self-verification system to validate local changes before contributing: + +```bash +hawk-eco verify +hawk-eco verify --fix +``` + +## Development + +```bash +make lint +hawk-eco verify +``` diff --git a/Makefile b/Makefile index 203dcddbd..d53208f47 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Canonical hawk-eco Makefile for Go library repos. -# Source of truth: .shared-templates/Makefile.library.tmpl at the eco root. +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/Makefile.library.tmpl # Placeholders rendered per repo: tok. # --------------------------------------------------------------------------- @@ -149,5 +149,7 @@ benchmark-quality: ## Run offline compression-quality harness (ratio + ROUGE-1 f --csv benchmarks/quality-results.csv .PHONY: hooks -hooks: - git config core.hooksPath .githooks +hooks: ## Install git hooks via lefthook (format, lint, conventional commits, co-author strip). + @command -v lefthook >/dev/null 2>&1 || (echo "install: go install github.com/evilmartians/lefthook@latest" && exit 1) + git config --unset core.hooksPath 2>/dev/null || true + lefthook install diff --git a/internal/fastops/fastops_amd64.s b/internal/fastops/fastops_amd64.s index 645d0c3a5..cbe79dc37 100644 --- a/internal/fastops/fastops_amd64.s +++ b/internal/fastops/fastops_amd64.s @@ -10,9 +10,14 @@ TEXT ·hasANSIavx2(SB), NOSPLIT, $0-17 TESTQ CX, CX JEQ ansi_false - // Broadcast 0x1b to all 32 bytes of Y1 + // Broadcast 0x1b to all 32 bytes of Y1. VPBROADCASTB only has a + // GPR-source form under AVX-512 (EVEX-encoded); the AVX2 (VEX-encoded) + // form requires an XMM/memory source, so stage the byte through X1 + // first to avoid emitting an AVX-512 instruction that SIGILLs on + // AVX2-only hardware. MOVB $0x1b, R8 - VPBROADCASTB R8, Y1 + MOVQ R8, X1 + VPBROADCASTB X1, Y1 // Number of 32-byte chunks → byte offset MOVQ CX, AX @@ -70,8 +75,11 @@ TEXT ·countBytesAVX2(SB), NOSPLIT, $0-32 TESTQ CX, CX JEQ count_done - // Broadcast target byte to all 32 bytes of Y1 - VPBROADCASTB R8, Y1 + // Broadcast target byte to all 32 bytes of Y1 via an AVX2-safe (VEX, + // XMM-source) form; see hasANSIavx2 for why a direct GPR source is + // avoided. + MOVQ R8, X1 + VPBROADCASTB X1, Y1 MOVQ CX, AX SHRQ $5, AX // AX = floor(len / 32) diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index c0d3047bd..af078e9af 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -148,10 +148,17 @@ func (sd *SecretDetector) RedactSecrets(text string) string { var sb strings.Builder lastIdx := 0 for _, m := range matches { - if m.StartPos < lastIdx { + if m.EndPos <= lastIdx { + // Fully covered by a previously redacted match; nothing left to hide. continue } - sb.WriteString(text[lastIdx:m.StartPos]) + start := m.StartPos + if start < lastIdx { + // Overlaps a previously redacted match but extends further right: + // redact only the un-redacted tail so no part of the secret leaks. + start = lastIdx + } + sb.WriteString(text[lastIdx:start]) sb.WriteString("[REDACTED:") sb.WriteString(m.Type) sb.WriteString("]") @@ -184,17 +191,31 @@ func maskString(s string) string { return s[:4] + strings.Repeat("*", len(s)-8) + s[len(s)-4:] } +// wordPattern matches whitespace-delimited words, used to locate entropy +// candidates without disturbing the original whitespace around them. +var wordPattern = regexp.MustCompile(`\S+`) + // DetectAndRedactWithEntropy detects secrets using both pattern matching and entropy analysis. func (sd *SecretDetector) DetectAndRedactWithEntropy(text string, entropyThreshold float64) string { result := sd.RedactSecrets(text) - // Also check for high-entropy strings that might be secrets - words := strings.Fields(result) - for i, word := range words { + // Also check for high-entropy strings that might be secrets. Replace matched + // words in place (rather than splitting on whitespace and rejoining with a + // single space) so original spacing/newlines are preserved. + var sb strings.Builder + lastIdx := 0 + for _, loc := range wordPattern.FindAllStringIndex(result, -1) { + start, end := loc[0], loc[1] + word := result[start:end] + sb.WriteString(result[lastIdx:start]) if calculateShannonEntropy(word) > entropyThreshold && len(word) > 16 { - words[i] = "[REDACTED:HIGH_ENTROPY]" + sb.WriteString("[REDACTED:HIGH_ENTROPY]") + } else { + sb.WriteString(word) } + lastIdx = end } - return strings.Join(words, " ") + sb.WriteString(result[lastIdx:]) + return sb.String() } // calculateShannonEntropy calculates the Shannon entropy of a string. diff --git a/lefthook.yml b/lefthook.yml index 7d5bdaf09..edab5775e 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -1,5 +1,5 @@ # Canonical lefthook config for hawk-eco Go repos. -# Source of truth: .shared-templates/lefthook.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/lefthook.yml.tmpl # # Install lefthook: # brew install lefthook (macOS) @@ -75,6 +75,9 @@ pre-commit: pre-push: commands: + boundaries: + run: bash ./scripts/check-ecosystem-boundaries.sh + test: run: go test ./... -count=1 -timeout=60s diff --git a/secrets_fuzz_test.go b/secrets_fuzz_test.go index 5f7536a61..addb6156b 100644 --- a/secrets_fuzz_test.go +++ b/secrets_fuzz_test.go @@ -1,12 +1,25 @@ package tok_test import ( + "regexp" "strings" "testing" "github.com/GrayCodeAI/tok" ) +// redactionMarkerPattern matches the "[REDACTED:]" markers RedactSecrets +// substitutes in place of detected secrets. +var redactionMarkerPattern = regexp.MustCompile(`\[REDACTED:[^\]]*\]`) + +// stripRedactionMarkers removes "[REDACTED:]" markers from s. Used to +// check for leaked secret values outside of the markers themselves, since a +// short captured value can coincidentally be a substring of a fixed type +// label (e.g. "Bearer Token") without that being an actual leak. +func stripRedactionMarkers(s string) string { + return redactionMarkerPattern.ReplaceAllString(s, "") +} + // FuzzRedactSecrets exercises the secrets redactor with arbitrary input. // // It asserts the security-relevant invariants that hold for the current @@ -71,11 +84,18 @@ func FuzzRedactSecrets(f *testing.F) { if !strings.Contains(out, "[REDACTED:") { t.Fatalf("redacted output carries no [REDACTED:] marker\ninput: %q\nout: %q", input, out) } + outsideMarkers := stripRedactionMarkers(out) for _, m := range matches { if len(m.Value) < 4 { continue } - if after, before := strings.Count(out, m.Value), strings.Count(input, m.Value); after >= before { + // Count occurrences outside of "[REDACTED:]" markers only: + // a short captured value can coincidentally be a substring of a + // *type label* (e.g. captured value "eare" is a substring of the + // fixed marker text "[REDACTED:Bearer Token]"), which is a + // spurious match against our own marker text, not a leak of the + // original secret. + if after, before := strings.Count(outsideMarkers, m.Value), strings.Count(input, m.Value); after >= before { t.Fatalf("detected secret value not reduced by redaction (before=%d after=%d)\ntype: %s\nvalue: %q\ninput: %q\nout: %q", before, after, m.Type, m.Value, input, out) }