Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.1.0
0.1.2
2 changes: 1 addition & 1 deletion benchmarks/quality/quality.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ type TierSummary struct {

// LoadSamples reads a samples JSON file (the benchmarks/samples.json shape).
func LoadSamples(path string) ([]Sample, error) {
data, err := os.ReadFile(path)
data, err := os.ReadFile(path) // #nosec G304 -- path is a benchmark fixture path supplied by the harness caller (e.g. benchmarks/samples.json), not externally supplied
if err != nil {
return nil, fmt.Errorf("read samples: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/cache/git_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ func (w *GitWatcher) hashFiles(workingDir string, files []string) (map[string]st

// hashFile hashes a single file's content using streaming
func (w *GitWatcher) hashFile(path string) (string, error) {
f, err := os.Open(path)
f, err := os.Open(path) // #nosec G304 -- path is joined from the watcher's working directory and a git-reported repo-relative file path, not externally supplied
if err != nil {
return "", err
}
Expand Down
2 changes: 1 addition & 1 deletion internal/cache/multilevel.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func (mc *MultiLevelCache) Get(key string) (string, bool) {
// L2: Disk
hash := hashKey(key)
path := filepath.Join(mc.l2Dir, hash)
data, err := os.ReadFile(path)
data, err := os.ReadFile(path) // #nosec G304 -- path is a hash-derived filename under the cache's configured l2Dir, not externally supplied
if err == nil {
val := string(data)
mc.promoteToL1(key, val)
Expand Down
1 change: 1 addition & 0 deletions internal/cache/query_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ func (c *QueryCache) Invalidate(predicate func(*CacheEntry) bool) error {
args[i] = key
}

// #nosec G201 -- IN clause is placeholder tokens only; values parameterized
query := fmt.Sprintf("DELETE FROM query_cache WHERE key IN (%s)",
strings.Join(placeholders, ","))
_, err = c.db.ExecContext(context.Background(), query, args...)
Expand Down
2 changes: 1 addition & 1 deletion internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,7 @@ func (c *Config) Save(path string) (retErr error) {
}

// Create file with restrictive permissions (owner read/write only)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) // #nosec G304 -- path is the configuration file location supplied by the caller (CLI flag/default config path), not externally supplied
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion internal/core/estimator.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (c *tokenCache) getShard(key uint64) *tokenCacheShard {

func hashText(text string) uint64 {
h := fnv.New64a()
h.Write([]byte(text))
_, _ = h.Write([]byte(text)) // #nosec G104 -- hash.Hash.Write never returns an error
return h.Sum64()
}

Expand Down
2 changes: 1 addition & 1 deletion internal/core/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (r *OSCommandRunner) Run(ctx context.Context, args []string) (string, int,
return hint, 127, err
}

cmd := exec.CommandContext(ctx, cmdPath, args[1:]...)
cmd := exec.CommandContext(ctx, cmdPath, args[1:]...) // #nosec G204 -- cmdPath resolved via validated exec.LookPath; not a shell, args passed directly
cmd.Env = r.Env

var buf bytes.Buffer
Expand Down
16 changes: 12 additions & 4 deletions internal/fastops/fastops_amd64.s
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,14 @@ TEXT ·hasANSIavx2(SB), NOSPLIT, $0-17
TESTQ CX, CX
JEQ ansi_false

// Broadcast 0x1b to all 32 bytes of Y1
// Broadcast 0x1b to all 32 bytes of Y1. VPBROADCASTB only has a
// GPR-source form under AVX-512 (EVEX-encoded); the AVX2 (VEX-encoded)
// form requires an XMM/memory source, so stage the byte through X1
// first to avoid emitting an AVX-512 instruction that SIGILLs on
// AVX2-only hardware.
MOVB $0x1b, R8
VPBROADCASTB R8, Y1
MOVQ R8, X1
VPBROADCASTB X1, Y1

// Number of 32-byte chunks → byte offset
MOVQ CX, AX
Expand Down Expand Up @@ -70,8 +75,11 @@ TEXT ·countBytesAVX2(SB), NOSPLIT, $0-32
TESTQ CX, CX
JEQ count_done

// Broadcast target byte to all 32 bytes of Y1
VPBROADCASTB R8, Y1
// Broadcast target byte to all 32 bytes of Y1 via an AVX2-safe (VEX,
// XMM-source) form; see hasANSIavx2 for why a direct GPR source is
// avoided.
MOVQ R8, X1
VPBROADCASTB X1, Y1

MOVQ CX, AX
SHRQ $5, AX // AX = floor(len / 32)
Expand Down
2 changes: 1 addition & 1 deletion internal/fastops/simd.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func FastLower(s string) string {
// safe: b is a private copy created above; no other reference to its
// backing array exists and it is never mutated after this point, so
// aliasing it as a string preserves string immutability.
return unsafe.String(unsafe.SliceData(b), len(b))
return unsafe.String(unsafe.SliceData(b), len(b)) // #nosec G103 -- audited: b is a private copy exclusively owned here and never mutated after this point (see comment above)
}

// FastEqual compares strings with early exit
Expand Down
3 changes: 2 additions & 1 deletion internal/filter/dedup.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@ func SimHash(content string) uint64 {
for i := 0; i+2 < len(content); i++ {
ngram := content[i : i+3]
h := fnv.New64a()
h.Write([]byte(ngram))
_, _ = h.Write([]byte(ngram)) // #nosec G104 -- hash.Hash.Write never returns an error
hash := h.Sum64()

for j := 0; j < 64; j++ {
// #nosec G115 -- j is a bounded non-negative loop index (0-63)
if hash&(1<<uint(j)) != 0 {
v[j]++
} else {
Expand Down
2 changes: 1 addition & 1 deletion internal/filter/kv_cache_aligner.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,6 @@ func isStableLine(line string) bool {
// computeCacheKey computes a hash key for a content prefix.
func computeCacheKey(content string) string {
h := fnv.New64a()
h.Write([]byte(content))
_, _ = h.Write([]byte(content)) // #nosec G104 -- hash.Hash.Write never returns an error
return string(h.Sum(nil))
}
2 changes: 1 addition & 1 deletion internal/filter/llm_compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (lc *LLMCompressor) Compress(content string, maxTokens int) (string, int, i
ctx, cancel := context.WithTimeout(context.Background(), lc.timeout)
defer cancel()

cmd := exec.CommandContext(ctx, lc.binPath)
cmd := exec.CommandContext(ctx, lc.binPath) // #nosec G204 -- binPath is a caller-supplied absolute path validated executable by isExecutable at construction
cmd.Stdin = strings.NewReader(string(reqBytes))
cmd.Env = append(os.Environ(), fmt.Sprintf("TOK_LLM_TIMEOUT=%d", int(lc.timeout.Seconds())))

Expand Down
1 change: 1 addition & 0 deletions internal/filter/path_shim_injector.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ func (psi *PATHShimInjector) Install(commands []string) error {
shimContent := fmt.Sprintf(`#!/bin/sh
exec tok %s "$@"
`, cmd)
// #nosec G306 -- shim must be executable (owner-only exec bit) so exec.LookPath/subprocess exec can run it
if err := os.WriteFile(shimPath, []byte(shimContent), 0o700); err != nil {
return fmt.Errorf("failed to write shim for %q: %w", cmd, err)
}
Expand Down
6 changes: 3 additions & 3 deletions internal/filter/zerocopy.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func (z *ZeroCopyBuffer) String() string {
// safe: the string header aliases z.data's backing array. The invariant
// (documented above) is that the backing array is not mutated while the
// returned string is live — Append/Reset after String() violate it.
return unsafe.String(unsafe.SliceData(z.data), len(z.data))
return unsafe.String(unsafe.SliceData(z.data), len(z.data)) // #nosec G103 -- audited: aliases z.data's backing array under the documented no-mutation-while-live invariant (see doc comment above)
}

func (z *ZeroCopyBuffer) Reset() {
Expand All @@ -58,7 +58,7 @@ func StringToBytes(s string) []byte {
// The invariant is that the backing array is never mutated through the
// returned slice — callers treat it as read-only, preserving string
// immutability.
return unsafe.Slice(unsafe.StringData(s), len(s))
return unsafe.Slice(unsafe.StringData(s), len(s)) // #nosec G103 -- audited: aliases s's backing array under the documented read-only invariant (see doc comment above)
}

// BytesToString converts []byte to string without allocation.
Expand All @@ -72,5 +72,5 @@ func BytesToString(b []byte) string {
// safe: the string header aliases b's backing array. The invariant is that
// the backing array is not mutated (through b or any other alias) while the
// returned string is live, preserving string immutability.
return unsafe.String(unsafe.SliceData(b), len(b))
return unsafe.String(unsafe.SliceData(b), len(b)) // #nosec G103 -- audited: aliases b's backing array under the documented no-mutation-while-live invariant (see doc comment above)
}
2 changes: 1 addition & 1 deletion internal/utils/logger.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func InitLogger(logPath string, level LogLevel) error {
logFile = nil
}

file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) // #nosec G304 -- logPath is the configured log file location (from CLI/config), not externally supplied
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions jsoncrunch.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ func CompressJSON(text string, maxItems int) string {
}
}
h := fnv.New64a()
h.Write([]byte(text))
rng := rand.New(rand.NewSource(int64(h.Sum64())))
_, _ = h.Write([]byte(text)) // #nosec G104 -- hash.Hash.Write never returns an error
rng := rand.New(rand.NewSource(int64(h.Sum64()))) // #nosec G404 G115 -- non-cryptographic use (deterministic shuffle seed from content hash); hash-mixing conversion, overflow is harmless
rng.Shuffle(len(middle), func(i, j int) { middle[i], middle[j] = middle[j], middle[i] })
need := maxItems - len(keep)
if need > len(middle) {
Expand Down
22 changes: 21 additions & 1 deletion secrets_fuzz_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,25 @@
package tok_test

import (
"regexp"
"strings"
"testing"

"github.com/GrayCodeAI/tok"
)

// redactionMarkerPattern matches the "[REDACTED:<type>]" markers RedactSecrets
// substitutes in place of detected secrets.
var redactionMarkerPattern = regexp.MustCompile(`\[REDACTED:[^\]]*\]`)

// stripRedactionMarkers removes "[REDACTED:<type>]" markers from s. Used to
// check for leaked secret values outside of the markers themselves, since a
// short captured value can coincidentally be a substring of a fixed type
// label (e.g. "Bearer Token") without that being an actual leak.
func stripRedactionMarkers(s string) string {
return redactionMarkerPattern.ReplaceAllString(s, "")
}

// FuzzRedactSecrets exercises the secrets redactor with arbitrary input.
//
// It asserts the security-relevant invariants that hold for the current
Expand Down Expand Up @@ -71,11 +84,18 @@ func FuzzRedactSecrets(f *testing.F) {
if !strings.Contains(out, "[REDACTED:") {
t.Fatalf("redacted output carries no [REDACTED:] marker\ninput: %q\nout: %q", input, out)
}
outsideMarkers := stripRedactionMarkers(out)
for _, m := range matches {
if len(m.Value) < 4 {
continue
}
if after, before := strings.Count(out, m.Value), strings.Count(input, m.Value); after >= before {
// Count occurrences outside of "[REDACTED:<type>]" markers only:
// a short captured value can coincidentally be a substring of a
// *type label* (e.g. captured value "eare" is a substring of the
// fixed marker text "[REDACTED:Bearer Token]"), which is a
// spurious match against our own marker text, not a leak of the
// original secret.
if after, before := strings.Count(outsideMarkers, m.Value), strings.Count(input, m.Value); after >= before {
t.Fatalf("detected secret value not reduced by redaction (before=%d after=%d)\ntype: %s\nvalue: %q\ninput: %q\nout: %q",
before, after, m.Type, m.Value, input, out)
}
Expand Down
Loading