From 31a2b1fd40549146c2e6744482ee31339a72df91 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:00:25 +0530 Subject: [PATCH 01/20] chore: add self-scan config excluding test fixtures KeyWatch's own test suite embeds fake example credentials (the AWS documentation example keys, dummy tokens in generated hook scripts). Exclude tests/ from self-scans so the pre-commit staged scan and manual scans of this repository report only real findings. --- .keywatch.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .keywatch.toml diff --git a/.keywatch.toml b/.keywatch.toml new file mode 100644 index 0000000..eaf9869 --- /dev/null +++ b/.keywatch.toml @@ -0,0 +1,7 @@ +# KeyWatch self-scan configuration. +# +# The test suite embeds fake example credentials as fixtures (the well-known +# AWS documentation example keys, dummy tokens in generated hook scripts). +# Exclude them so scanning this repository — including the pre-commit hook's +# staged scan — reports only real findings. +exclude = ["tests/*"] From 60de57a738be5d551268b572aa22aac53dbd88da Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:01:16 +0530 Subject: [PATCH 02/20] fix(detectors): raise Base64Detector entropy threshold to 4.2 At entropy 3.0 the detector flagged long CamelCase identifiers (measured <= ~4.13 bits) throughout ordinary source code, flooding reports with LOW findings. Real base64 payloads measure >= ~4.27, so 4.2 separates the two. The stdin integration fixture loses the double-count of the AWS example key (entropy ~4.0), which the dedicated AWS detector still reports as HIGH. Also exclude detectors.toml from self-scans: the detector definitions themselves are secret-shaped strings. --- .keywatch.toml | 4 +++- detectors.toml | 5 ++++- tests/scanner_tests.rs | 6 ++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.keywatch.toml b/.keywatch.toml index eaf9869..eadce4b 100644 --- a/.keywatch.toml +++ b/.keywatch.toml @@ -4,4 +4,6 @@ # AWS documentation example keys, dummy tokens in generated hook scripts). # Exclude them so scanning this repository — including the pre-commit hook's # staged scan — reports only real findings. -exclude = ["tests/*"] +# detectors.toml is excluded because the detector definitions themselves are +# secret-shaped strings (pattern regexes, allowlist examples). +exclude = ["tests/*", "detectors.toml"] diff --git a/detectors.toml b/detectors.toml index 3358c59..9c36867 100644 --- a/detectors.toml +++ b/detectors.toml @@ -111,7 +111,10 @@ name = "Base64Detector" pattern = "\\b[A-Za-z0-9+/]{20,}[=]{0,2}\\b" finding_type = "Base64 Encoded String" severity = "LOW" -entropy = 3.0 +# 4.2 sits between long CamelCase identifiers (measured <= ~4.13) and real +# base64 payloads (>= ~4.27), so ordinary source code no longer floods +# reports with LOW findings. +entropy = 4.2 [[detectors]] name = "HighEntropyDetector" diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 8195ec4..c0962fb 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -1069,8 +1069,10 @@ fn test_stdin_scanning_integration() -> Result<(), String> { let combined = format!("{}{}", stdout, stderr); assert!( - combined.contains("3 potential secret(s)"), - "Should detect 3 secrets from stdin input\nstdout:\n{}\nstderr:\n{}", + // AWS key + password. The Base64Detector no longer double-counts the + // AWS key itself: its entropy (~4.0) is below the 4.2 threshold. + combined.contains("2 potential secret(s)"), + "Should detect 2 secrets from stdin input\nstdout:\n{}\nstderr:\n{}", stdout, stderr ); From ad8aca079e98298abd15dffda5fbd59e7fa14484 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:04:05 +0530 Subject: [PATCH 03/20] feat(scan): add --staged mode scanning only added lines of the staged diff The pre-commit hook previously scanned whole staged files, so any pre-existing finding blocked commits even for deletion-only or identical-to-upstream changes. scan --staged parses 'git diff --cached -U0' in Rust, scans only added lines, and attributes findings to real file paths and post-image line numbers so --exclude globs and --baseline entries compose with hook scans. The git invocation is hardened against user config that would break diff parsing or silently hide findings: --no-color/color.ui=false (color.ui=always previously ANSI-wrapped every line, making the parser match nothing), --literal-pathspecs (a staged filename containing glob metacharacters was fnmatch-expanded and skipped), and explicit diff.mnemonicPrefix/noprefix/quotePath overrides (wrong prefixes broke path attribution). Files git renders as binary (e.g. '-diff' in .gitattributes) are surfaced in excluded_files instead of passing silently, non-UTF-8 content is decoded lossily instead of aborting the scan, and a git failure exits 2 so callers fail closed. Scan validation is reworked into one exhaustive match over (git_history, staged, stdin). Pre-existing fake fixtures in scanner unit tests get inline keywatch:ignore markers so self-scans stay clean. --- src/cli.rs | 42 +-- src/scanner.rs | 465 +++++++++++++++++++++++++++++++--- src/scanner/error.rs | 18 +- tests/cli_validation_tests.rs | 80 ++++++ tests/scanner_tests.rs | 242 +++++++++++++++++- 5 files changed, 798 insertions(+), 49 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index c50550d..8cb55ef 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -7,6 +7,8 @@ use std::fmt::{Display, Formatter}; pub enum CliValidationError { GitHistoryWithStdin, GitHistoryWithMultiplePaths, + StagedWithStdin, + StagedWithGitHistory, StdinWithPaths, MissingScanInput, PreCommitRepositoryFilters, @@ -22,10 +24,15 @@ impl Display for CliValidationError { Self::GitHistoryWithMultiplePaths => { formatter.write_str("Cannot specify more than one path with --git-history") } - Self::StdinWithPaths => formatter.write_str("Cannot specify both --stdin and paths"), - Self::MissingScanInput => { - formatter.write_str("Must specify paths, use --stdin, or use --git-history") + Self::StagedWithStdin => { + formatter.write_str("Cannot specify both --staged and --stdin") + } + Self::StagedWithGitHistory => { + formatter.write_str("Cannot specify both --staged and --git-history") } + Self::StdinWithPaths => formatter.write_str("Cannot specify both --stdin and paths"), + Self::MissingScanInput => formatter + .write_str("Must specify paths, use --stdin, --staged, or --git-history"), Self::PreCommitRepositoryFilters => formatter.write_str( "--allowed-repos and --blocked-repos are only supported for pre-push hooks", ), @@ -87,6 +94,10 @@ pub struct ScanArgs { #[arg(long, default_value_t = false)] pub git_history: bool, + /// Scan only the lines staged for commit (paths narrow the staged diff) + #[arg(long, default_value_t = false)] + pub staged: bool, + /// Output the result to a file #[arg(short, long)] pub output: Option, @@ -126,22 +137,21 @@ pub struct ScanArgs { impl ScanArgs { pub fn validate(&self) -> Result<(), CliValidationError> { - if self.git_history { - if self.stdin { - return Err(CliValidationError::GitHistoryWithStdin); + match (self.git_history, self.staged, self.stdin) { + (true, true, _) => Err(CliValidationError::StagedWithGitHistory), + (true, _, true) => Err(CliValidationError::GitHistoryWithStdin), + (true, false, false) if self.paths.len() > 1 => { + Err(CliValidationError::GitHistoryWithMultiplePaths) } - if self.paths.len() > 1 { - return Err(CliValidationError::GitHistoryWithMultiplePaths); + (false, true, true) => Err(CliValidationError::StagedWithStdin), + (false, false, true) if !self.paths.is_empty() => { + Err(CliValidationError::StdinWithPaths) } - return Ok(()); - } - if self.stdin && !self.paths.is_empty() { - return Err(CliValidationError::StdinWithPaths); - } - if !self.stdin && self.paths.is_empty() { - return Err(CliValidationError::MissingScanInput); + (false, false, false) if self.paths.is_empty() => { + Err(CliValidationError::MissingScanInput) + } + _ => Ok(()), } - Ok(()) } } diff --git a/src/scanner.rs b/src/scanner.rs index f790535..0a48927 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -208,11 +208,14 @@ pub fn run_scan( .args([ "-c", "diff.external=", + "-c", + "color.ui=false", "log", "-p", "-U0", "--no-ext-diff", "--no-textconv", + "--no-color", ]) .stdout(std::process::Stdio::piped()) .spawn() @@ -243,6 +246,62 @@ pub fn run_scan( return Ok((findings, metadata)); } + if args.staged { + let exclude_patterns = compile_exclude_patterns(args, config)?; + let mut command = std::process::Command::new("git"); + // The parser depends on undecorated `diff --git`/`@@`/`+` framing and + // literal `a/`/`b/` path prefixes, so user git config that colors, + // re-prefixes, quotes, or glob-expands must be overridden here — a + // stray `color.ui = always` would otherwise hide every added line. + command.args([ + "--literal-pathspecs", + "-c", + "diff.external=", + "-c", + "color.ui=false", + "-c", + "diff.mnemonicPrefix=false", + "-c", + "diff.noprefix=false", + "-c", + "core.quotePath=false", + "diff", + "--cached", + "-U0", + "--no-ext-diff", + "--no-textconv", + "--no-color", + "--", + ]); + command.args(&args.paths); + let mut child = command + .stdout(std::process::Stdio::piped()) + .spawn() + .map_err(|source| ScannerError::RunGitDiff { source })?; + + let stdout = child.stdout.take().ok_or(ScannerError::CaptureGitStdout)?; + let reader = BufReader::new(stdout); + let scan_result = scan_staged_diff( + reader, + &exclude_patterns, + &multiline_detectors, + &line_detectors, + ); + if scan_result.is_err() { + // Reap git instead of leaving it writing into a closed pipe. + let _ = child.kill(); + } + let wait_result = child.wait(); + + let (findings, metadata) = scan_result?; + let status = wait_result.map_err(|source| ScannerError::GitProcess { source })?; + if !status.success() { + return Err(ScannerError::GitDiffNonZero); + } + + return Ok((findings, metadata)); + } + if args.stdin { let stdin = std::io::stdin(); let reader = BufReader::new(stdin); @@ -288,36 +347,7 @@ pub fn run_scan( } let unique_paths: Vec<_> = unique_paths.into_iter().collect(); - let mut exclude_patterns: Vec = args - .exclude - .as_ref() - .map(|exclude_str| { - exclude_str - .split(',') - .filter(|pattern| !pattern.trim().is_empty()) - .map(|pattern| { - Pattern::new(pattern.trim()).map_err(|source| { - ScannerError::InvalidExcludePattern { - pattern: pattern.to_string(), - source, - } - }) - }) - .collect::, _>>() - }) - .transpose()? - .unwrap_or_default(); - - if let Some(excludes) = config.and_then(|cfg| cfg.exclude.as_ref()) { - for pattern_str in excludes { - exclude_patterns.push(Pattern::new(pattern_str).map_err(|source| { - ScannerError::InvalidConfigExcludePattern { - pattern: pattern_str.to_string(), - source, - } - })?); - } - } + let exclude_patterns = compile_exclude_patterns(args, config)?; let results: Vec<(Vec, usize, usize, Option)> = unique_paths .into_par_iter() @@ -388,6 +418,235 @@ pub fn run_scan( Ok((findings, metadata)) } +fn compile_exclude_patterns( + args: &ScanArgs, + config: Option<&KeywatchConfig>, +) -> Result, ScannerError> { + let mut exclude_patterns: Vec = args + .exclude + .as_ref() + .map(|exclude_str| { + exclude_str + .split(',') + .filter(|pattern| !pattern.trim().is_empty()) + .map(|pattern| { + Pattern::new(pattern.trim()).map_err(|source| { + ScannerError::InvalidExcludePattern { + pattern: pattern.to_string(), + source, + } + }) + }) + .collect::, _>>() + }) + .transpose()? + .unwrap_or_default(); + + if let Some(excludes) = config.and_then(|cfg| cfg.exclude.as_ref()) { + for pattern_str in excludes { + exclude_patterns.push(Pattern::new(pattern_str).map_err(|source| { + ScannerError::InvalidConfigExcludePattern { + pattern: pattern_str.to_string(), + source, + } + })?); + } + } + + Ok(exclude_patterns) +} + +fn parse_hunk_new_start(header: &str) -> usize { + header + .split_whitespace() + .find(|token| token.starts_with('+')) + .and_then(|token| { + token[1..] + .split(',') + .next() + .and_then(|start| start.parse().ok()) + }) + .unwrap_or(0) +} + +fn parse_diff_target_path(target: &str) -> Option { + let target = target.trim_end(); + if target == "/dev/null" { + return None; + } + let target = target.trim_matches('"'); + let target = target.strip_prefix("b/").unwrap_or(target); + Some(target.to_string()) +} + +fn flush_staged_hunk( + path: Option<&str>, + hunk_start: usize, + hunk_added: &mut Vec, + multiline_detectors: &[&Detector], + findings: &mut Vec, +) { + if hunk_added.is_empty() { + return; + } + if let Some(path) = path { + let chunk = hunk_added.join("\n"); + scan_multiline_chunk( + &chunk, + hunk_start.saturating_sub(1), + path, + multiline_detectors, + findings, + ); + } + hunk_added.clear(); +} + +/// Best-effort path from a `Binary files a/x and b/x differ` marker: the +/// post-image side, for surfacing skipped files in `excluded_files`. +fn parse_binary_marker_path(marker: &str) -> String { + marker + .strip_suffix(" differ") + .and_then(|paths| paths.rsplit(" and ").next()) + .map(|target| target.strip_prefix("b/").unwrap_or(target)) + .unwrap_or(marker) + .to_string() +} + +/// Scans only the added lines of the staged diff, attributing findings to the +/// real file path and post-image line number so `--baseline` entries match. +/// Hunk state is tracked because added content may itself start with "+". +/// Lines are decoded lossily so one non-UTF-8 file cannot abort the scan. +fn scan_staged_diff( + mut reader: ReaderType, + exclude_patterns: &[Pattern], + multiline_detectors: &[&Detector], + line_detectors: &[&Detector], +) -> Result<(Vec, ScanMetadata), ScannerError> { + let mut findings = Vec::new(); + let mut total_lines = 0; + let mut scanned_files: std::collections::BTreeSet = std::collections::BTreeSet::new(); + let mut excluded_files: Vec = Vec::new(); + let mut current_path: Option = None; + let mut in_hunk = false; + let mut next_line_number = 0; + let mut hunk_start = 0; + let mut hunk_added: Vec = Vec::new(); + let mut raw_line: Vec = Vec::new(); + + loop { + raw_line.clear(); + let bytes_read = + reader + .read_until(b'\n', &mut raw_line) + .map_err(|source| ScannerError::ReadStream { + path: "".to_string(), + source, + })?; + if bytes_read == 0 { + break; + } + if raw_line.last() == Some(&b'\n') { + raw_line.pop(); + } + if raw_line.last() == Some(&b'\r') { + raw_line.pop(); + } + let line = String::from_utf8_lossy(&raw_line); + + if in_hunk { + if let Some(content) = line.strip_prefix('+') { + let line_number = next_line_number; + next_line_number += 1; + let Some(path) = current_path.as_deref() else { + continue; + }; + total_lines += 1; + scanned_files.insert(path.to_string()); + scan_line_detectors( + content, + line_number, + path, + line_detectors, + is_inline_suppressed(content), + &mut findings, + ); + hunk_added.push(content.to_string()); + continue; + } + if line.starts_with('-') || line.starts_with('\\') { + continue; + } + } + + if line.starts_with("@@") { + flush_staged_hunk( + current_path.as_deref(), + hunk_start, + &mut hunk_added, + multiline_detectors, + &mut findings, + ); + hunk_start = parse_hunk_new_start(&line); + next_line_number = hunk_start; + in_hunk = true; + continue; + } + + if line.starts_with("diff ") { + flush_staged_hunk( + current_path.as_deref(), + hunk_start, + &mut hunk_added, + multiline_detectors, + &mut findings, + ); + in_hunk = false; + current_path = None; + continue; + } + + if let Some(target) = line.strip_prefix("+++ ") { + current_path = match parse_diff_target_path(target) { + Some(path) if matches_exclude_patterns(&path, &[], exclude_patterns) => { + excluded_files.push(path); + None + } + other => other, + }; + continue; + } + + // A `-diff` gitattribute (or a true binary) yields no hunks; surface + // the skipped file instead of silently reporting it as clean. + if let Some(marker) = line.strip_prefix("Binary files ") { + excluded_files.push(parse_binary_marker_path(marker)); + } + } + + flush_staged_hunk( + current_path.as_deref(), + hunk_start, + &mut hunk_added, + multiline_detectors, + &mut findings, + ); + + findings.sort_by(|a, b| { + a.file_path + .cmp(&b.file_path) + .then(a.line_number.cmp(&b.line_number)) + }); + + let metadata = ScanMetadata { + files_scanned: scanned_files.len(), + total_lines, + excluded_files, + }; + + Ok((findings, metadata)) +} + fn collect_files(dir_path: &str, files: &mut Vec<(String, Option)>, root: &str) { if let Ok(entries) = fs::read_dir(dir_path) { for entry in entries.flatten() { @@ -553,4 +812,150 @@ mod tests { "Should find all 2500 secrets across chunks" ); } + + #[test] + fn test_scan_staged_diff_preserves_plus_prefixed_added_content() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let diff = "diff --git a/notes.txt b/notes.txt\n\ + index aabbcc0..ddeeff1 100644\n\ + --- /dev/null\n\ + +++ b/notes.txt\n\ + @@ -0,0 +1,3 @@\n\ + +SECRET_ONE plain\n\ + +++SECRET_TWO starts with pluses\n\ + +@@ SECRET_THREE looks like a hunk header\n"; + + let (findings, metadata) = + scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + + let summary: Vec<(String, usize)> = findings + .iter() + .map(|finding| (finding.matched_content.clone(), finding.line_number)) + .collect(); + assert_eq!( + summary, + vec![ + ("SECRET_ONE".to_string(), 1), + ("SECRET_TWO".to_string(), 2), + ("SECRET_THREE".to_string(), 3), + ], + "added lines starting with '+', '@@' must be scanned as content" + ); + assert!(findings.iter().all(|f| f.file_path == "notes.txt")); + assert_eq!(metadata.files_scanned, 1); + } + + #[test] + fn test_scan_staged_diff_ignores_deleted_files_and_removed_lines() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let diff = "diff --git a/gone.txt b/gone.txt\n\ + deleted file mode 100644\n\ + --- a/gone.txt\n\ + +++ /dev/null\n\ + @@ -1,2 +0,0 @@\n\ + -SECRET_GONE\n\ + -goodbye\n"; + + let (findings, metadata) = + scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + + assert!(findings.is_empty(), "removed lines must not be scanned"); + assert_eq!(metadata.files_scanned, 0); + } + + #[test] + fn test_scan_staged_diff_attributes_multiple_files() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let diff = "diff --git a/first.txt b/first.txt\n\ + --- a/first.txt\n\ + +++ b/first.txt\n\ + @@ -0,0 +7 @@\n\ + +SECRET_A\n\ + diff --git a/second.txt b/second.txt\n\ + --- a/second.txt\n\ + +++ b/second.txt\n\ + @@ -0,0 +2 @@\n\ + +SECRET_B\n"; + + let (findings, metadata) = + scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + + let summary: Vec<(String, usize)> = findings + .iter() + .map(|finding| (finding.file_path.clone(), finding.line_number)) + .collect(); + assert_eq!( + summary, + vec![("first.txt".to_string(), 7), ("second.txt".to_string(), 2)] + ); + assert_eq!(metadata.files_scanned, 2); + } + + #[test] + fn test_scan_staged_diff_surfaces_binary_files_as_excluded() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let diff = "diff --git a/img.png b/img.png\n\ + index aabbcc0..ddeeff1 100644\n\ + Binary files a/img.png and b/img.png differ\n"; + + let (findings, metadata) = + scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + + assert!(findings.is_empty()); + assert_eq!( + metadata.excluded_files, + vec!["img.png".to_string()], + "files git renders as binary must be surfaced, not silently clean" + ); + } + + #[test] + fn test_scan_staged_diff_survives_non_utf8_content() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let mut diff: Vec = Vec::new(); + diff.extend_from_slice(b"diff --git a/legacy.csv b/legacy.csv\n"); + diff.extend_from_slice(b"--- a/legacy.csv\n"); + diff.extend_from_slice(b"+++ b/legacy.csv\n"); + diff.extend_from_slice(b"@@ -0,0 +1,2 @@\n"); + diff.extend_from_slice(b"+caf\xE9 latin-1 line\n"); + diff.extend_from_slice(b"+SECRET_AFTER_BINARYISH\n"); + + let (findings, _) = + scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + + assert_eq!( + findings.len(), + 1, + "non-UTF-8 content must not abort the scan" + ); + assert_eq!(findings[0].line_number, 2); + } + + #[test] + fn test_scan_staged_diff_multiline_detector_uses_hunk_start_offset() { + let detector = make_detector("Block", r"(?s)BEGIN KEY.*END KEY", "Test", "HIGH"); + let multiline_detectors = vec![&detector]; + let diff = "diff --git a/key.pem b/key.pem\n\ + --- a/key.pem\n\ + +++ b/key.pem\n\ + @@ -0,0 +5,3 @@\n\ + +BEGIN KEY\n\ + +material\n\ + +END KEY\n"; + + let (findings, _) = + scan_staged_diff(Cursor::new(diff), &[], &multiline_detectors, &[]).unwrap(); + + assert_eq!(findings.len(), 1); + assert_eq!( + findings[0].line_number, 5, + "multiline findings must use the hunk's post-image start line" + ); + assert_eq!(findings[0].file_path, "key.pem"); + } } diff --git a/src/scanner/error.rs b/src/scanner/error.rs index 9c73fe9..9afe7df 100644 --- a/src/scanner/error.rs +++ b/src/scanner/error.rs @@ -19,11 +19,15 @@ pub enum ScannerError { RunGitLog { source: io::Error, }, + RunGitDiff { + source: io::Error, + }, CaptureGitStdout, GitProcess { source: io::Error, }, GitLogNonZero, + GitDiffNonZero, InvalidExcludePattern { pattern: String, source: glob::PatternError, @@ -45,11 +49,17 @@ impl fmt::Display for ScannerError { ScannerError::RunGitLog { source } => { write!(formatter, "Failed to run git log: {}", source) } + ScannerError::RunGitDiff { source } => { + write!(formatter, "Failed to run git diff: {}", source) + } ScannerError::CaptureGitStdout => write!(formatter, "Failed to capture git stdout"), ScannerError::GitProcess { source } => { write!(formatter, "git process error: {}", source) } ScannerError::GitLogNonZero => write!(formatter, "git log exited with non-zero status"), + ScannerError::GitDiffNonZero => { + write!(formatter, "git diff exited with non-zero status") + } ScannerError::InvalidExcludePattern { pattern, source } => { write!( formatter, @@ -72,8 +82,12 @@ impl StdError for ScannerError { ScannerError::DetectorInit { source } => Some(source), ScannerError::Config { source } => Some(source), ScannerError::ReadStream { source, .. } => Some(source), - ScannerError::RunGitLog { source } => Some(source), - ScannerError::CaptureGitStdout | ScannerError::GitLogNonZero => None, + ScannerError::RunGitLog { source } | ScannerError::RunGitDiff { source } => { + Some(source) + } + ScannerError::CaptureGitStdout + | ScannerError::GitLogNonZero + | ScannerError::GitDiffNonZero => None, ScannerError::GitProcess { source } => Some(source), ScannerError::InvalidExcludePattern { source, .. } | ScannerError::InvalidConfigExcludePattern { source, .. } => Some(source), diff --git a/tests/cli_validation_tests.rs b/tests/cli_validation_tests.rs index cde1fea..3f39f09 100644 --- a/tests/cli_validation_tests.rs +++ b/tests/cli_validation_tests.rs @@ -7,6 +7,7 @@ fn test_stdin_with_path_validation_returns_typed_error() { paths: vec!["secret.txt".to_string()], stdin: true, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -32,3 +33,82 @@ fn test_run_cli_error_wraps_cli_validation_display() { assert_eq!(error.to_string(), "Cannot specify both --stdin and paths"); } + +#[test] +fn test_staged_with_stdin_validation_returns_typed_error() { + let options = ScanArgs { + paths: vec![], + stdin: true, + git_history: false, + staged: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + config: None, + no_config_discovery: false, + format: OutputFormat::Json, + }; + + let error = options + .validate() + .expect_err("staged with stdin should be rejected"); + + assert_eq!(error, CliValidationError::StagedWithStdin); + assert_eq!( + error.to_string(), + "Cannot specify both --staged and --stdin" + ); +} + +#[test] +fn test_staged_with_git_history_validation_returns_typed_error() { + let options = ScanArgs { + paths: vec![], + stdin: false, + git_history: true, + staged: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + config: None, + no_config_discovery: false, + format: OutputFormat::Json, + }; + + let error = options + .validate() + .expect_err("staged with git-history should be rejected"); + + assert_eq!(error, CliValidationError::StagedWithGitHistory); + assert_eq!( + error.to_string(), + "Cannot specify both --staged and --git-history" + ); +} + +#[test] +fn test_staged_allows_zero_or_many_paths() { + let options = ScanArgs { + paths: vec!["a.txt".to_string(), "b.txt".to_string()], + stdin: false, + git_history: false, + staged: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + config: None, + no_config_discovery: false, + format: OutputFormat::Json, + }; + + assert!(options.validate().is_ok(), "staged paths narrow the diff"); +} diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index c0962fb..d1251dc 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -58,8 +58,11 @@ fn commit_file(path: &Path, file_name: &str, contents: &str, message: &str) -> R return Err("git add failed".to_string()); } + // --no-verify keeps fixture commits hermetic on machines where a global + // core.hooksPath installs a secret-scanning pre-commit hook; these tests + // exercise git-history scanning, not hooks. let status = Command::new("git") - .args(["commit", "-m", message, "--quiet"]) + .args(["commit", "-m", message, "--quiet", "--no-verify"]) .current_dir(path) .status() .map_err(|error| format!("git commit: {error}"))?; @@ -108,6 +111,7 @@ sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\n\ paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -141,6 +145,7 @@ Stripe: sk_test_51ABCDEF12345678901234567890\n\ paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -175,6 +180,7 @@ AZURE_STORAGE=DefaultEndpointsProtocol=https;AccountName=examplestore; paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -210,6 +216,7 @@ b3BlbnNzaC1ldi0xLjAAABgQDQD2FGB3V2t4=\n\ paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -239,6 +246,7 @@ fn test_multiple_detections_in_line() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -280,6 +288,7 @@ fn test_directory_scan_with_exclusions() { paths: vec![test_dir.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -320,6 +329,7 @@ fn test_exclude_pattern_filtering() { paths: vec![test_dir.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: Some("*.log".to_string()), @@ -350,6 +360,7 @@ fn test_invalid_cli_exclude_pattern_returns_typed_error() { paths: vec![".".to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: Some("[".to_string()), @@ -398,6 +409,7 @@ fn test_dot_github_directory_is_scanned() { paths: vec![test_dir.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -426,6 +438,7 @@ fn test_scan_no_secrets() { paths: vec![temp_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -455,6 +468,7 @@ fn test_non_utf8_file_handling() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -488,6 +502,7 @@ fn test_multiple_files_scan() { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -522,6 +537,7 @@ fn test_duplicate_paths_are_scanned_once() { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -570,6 +586,7 @@ fn test_mixed_file_and_directory_paths_are_scanned_once() { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -610,6 +627,7 @@ fn test_nonexistent_paths_are_ignored_without_counting_as_scanned() { paths: vec![missing_path.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -657,6 +675,7 @@ fn test_explicit_symlink_path_is_skipped() -> Result<(), String> { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -702,6 +721,7 @@ fn test_recursive_symlink_path_is_skipped() -> Result<(), String> { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -740,6 +760,7 @@ fn test_detect_aadhaar() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -776,6 +797,7 @@ fn test_detect_voter_id() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -809,6 +831,7 @@ fn test_detect_pan_card() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -842,6 +865,7 @@ fn test_detect_abha() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -876,6 +900,7 @@ fn test_multiple_indian_ids() { paths: vec![test_file.to_str().unwrap().to_string()], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -935,6 +960,7 @@ fn test_overlapping_scan_roots_with_exclusions() { ], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: Some("subdir/secret.txt".to_string()), @@ -979,6 +1005,7 @@ AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\n paths: vec![path_str], stdin: false, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -1026,6 +1053,7 @@ fn test_stdin_args_validation() { paths: vec![], stdin: true, git_history: false, + staged: false, output: None, verbose: false, exclude: None, @@ -1086,6 +1114,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { paths: vec![], stdin: false, git_history: true, + staged: false, output: None, verbose: false, exclude: None, @@ -1100,6 +1129,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { paths: vec!["/tmp/requested-root".to_string()], stdin: false, git_history: true, + staged: false, output: None, verbose: false, exclude: None, @@ -1117,6 +1147,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { ], stdin: false, git_history: true, + staged: false, output: None, verbose: false, exclude: None, @@ -1243,3 +1274,212 @@ fn test_git_history_does_not_execute_textconv_helpers() -> Result<(), String> { let _ = fs::remove_dir_all(&repo_dir); Ok(()) } + +fn run_staged_scan(current_dir: &Path, extra_args: &[&str]) -> Result { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "--staged"]) + .args(extra_args) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(current_dir) + .output() + .map_err(|error| format!("run key-watch scan --staged: {error}")) +} + +fn stage_file(path: &Path, file_name: &str, contents: &str) -> Result<(), String> { + let file_path = path.join(file_name); + fs::write(&file_path, contents).map_err(|error| format!("write {file_name}: {error}"))?; + + let status = Command::new("git") + .args(["add", file_name]) + .current_dir(path) + .status() + .map_err(|error| format!("git add: {error}"))?; + if !status.success() { + return Err("git add failed".to_string()); + } + + Ok(()) +} + +#[test] +fn test_staged_scan_ignores_findings_on_unchanged_lines() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_unchanged_lines"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file( + &repo_dir, + "secrets.txt", + "AWS Key: AKIAABCDEFGHIJKLMNOP\n", + "initial", + )?; + stage_file( + &repo_dir, + "secrets.txt", + "AWS Key: AKIAABCDEFGHIJKLMNOP\nplain documentation line\n", + )?; + + let output = run_staged_scan(&repo_dir, &[])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + matches!(output.status.code(), Some(0)), + "pre-existing secret on an unchanged line must not block\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_ignores_deletion_only_changes() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_deletion_only"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file( + &repo_dir, + "secrets.txt", + "keep this line\nAWS Key: AKIAABCDEFGHIJKLMNOP\n", + "initial", + )?; + stage_file(&repo_dir, "secrets.txt", "keep this line\n")?; + + let output = run_staged_scan(&repo_dir, &[])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + matches!(output.status.code(), Some(0)), + "deletion-only staged change must not block\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_reports_added_secret_with_real_path_and_line() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_added_secret"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file(&repo_dir, "config.txt", "line one\nline two\n", "initial")?; + stage_file( + &repo_dir, + "config.txt", + "line one\nline two\nAWS Key: AKIAABCDEFGHIJKLMNOP\n", + )?; + + let output = run_staged_scan(&repo_dir, &["--verbose"])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + matches!(output.status.code(), Some(1)), + "a staged secret must block\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + assert!( + stdout.contains("\"file_path\": \"config.txt\""), + "findings must carry the real file path, not \nstdout:\n{}", + stdout + ); + assert!( + stdout.contains("\"line_number\": 3"), + "findings must carry the post-image line number\nstdout:\n{}", + stdout + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_respects_exclude_patterns() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_exclude"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file(&repo_dir, "fixture.snap", "clean\n", "initial")?; + stage_file( + &repo_dir, + "fixture.snap", + "clean\nAWS Key: AKIAABCDEFGHIJKLMNOP\n", + )?; + + let output = run_staged_scan(&repo_dir, &["--exclude", "*.snap"])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + matches!(output.status.code(), Some(0)), + "excluded staged paths must not be scanned\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_composes_with_baseline() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_baseline"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file(&repo_dir, "config.txt", "line one\n", "initial")?; + stage_file( + &repo_dir, + "config.txt", + "line one\nAWS Key: AKIAABCDEFGHIJKLMNOP\n", + )?; + + let update = run_staged_scan( + &repo_dir, + &["--baseline", "baseline.json", "--update-baseline"], + )?; + assert!( + matches!(update.status.code(), Some(0)), + "baseline update should succeed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&update.stdout), + String::from_utf8_lossy(&update.stderr) + ); + + let output = run_staged_scan(&repo_dir, &["--baseline", "baseline.json"])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + matches!(output.status.code(), Some(0)), + "baselined staged findings must be suppressed\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + From 4284ef24347dfaf165028c13b1b7caba1e7edc32 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:01 +0530 Subject: [PATCH 04/20] perf(scan): single-pass Aho-Corasick keyword prefilter Profiling a 36 MB corpus showed ~75% of scan time in the keyword prefilter: has_keywords lowercased the whole line once per detector (~119x per line) and re-lowercased every keyword on every call. Keywords are now lowercased once at Detector construction, each line is lowercased once into a reusable buffer, and all detector keywords are folded into one Aho-Corasick automaton whose single overlapping-search pass per line marks candidate detectors. has_keywords now expects pre-lowercased content. aho-corasick was already in the dependency tree via regex. Measured on the same corpus with identical findings: stdin stream scan 7.5s -> 0.83s (9x), directory scan 1.10s -> 0.27s (4x). --- Cargo.lock | 1 + Cargo.toml | 1 + src/detector.rs | 10 ++- src/scanner.rs | 180 +++++++++++++++++++++++++++++++--------- tests/detector_tests.rs | 19 ++++- 5 files changed, 167 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 70515b1..01c169f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -318,6 +318,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" name = "key-watch" version = "2.0.1" dependencies = [ + "aho-corasick", "clap", "dirs", "glob", diff --git a/Cargo.toml b/Cargo.toml index 8cd6149..9fc3742 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ keywords = ["secret-scanner", "security", "credentials", "lint"] license = "GPL-3.0-only" [dependencies] +aho-corasick = "1.1" clap = { version = "4.6.1", features = ["derive"] } regex = "1.12.4" toml = "1.1.2" diff --git a/src/detector.rs b/src/detector.rs index c19d677..0344513 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -111,19 +111,21 @@ impl Detector { finding_type: finding_type.to_string(), severity: parsed_severity, allowlist: compiled_allowlist, - keywords: keywords.to_vec(), + keywords: keywords.iter().map(|keyword| keyword.to_lowercase()).collect(), entropy_threshold, }) } - pub fn has_keywords(&self, content: &str) -> bool { + /// `lowercase_content` must already be lowercased. Keywords are stored + /// lowercased at construction so callers can lowercase once per line + /// instead of once per detector. + pub fn has_keywords(&self, lowercase_content: &str) -> bool { if self.keywords.is_empty() { return true; } - let lowercase_content = content.to_lowercase(); self.keywords .iter() - .any(|keyword| lowercase_content.contains(&keyword.to_lowercase())) + .any(|keyword| lowercase_content.contains(keyword.as_str())) } pub fn has_sufficient_entropy(&self, matched: &str) -> bool { diff --git a/src/scanner.rs b/src/scanner.rs index 0a48927..95d9ff8 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -2,6 +2,7 @@ use crate::cli::ScanArgs; use crate::config::KeywatchConfig; use crate::detector::{Detector, initialize_detectors, initialize_trusted_detectors}; use crate::report::{Finding, ScanMetadata}; +use aho_corasick::AhoCorasick; use glob::Pattern; use rayon::prelude::*; use std::fs; @@ -25,33 +26,138 @@ fn is_allowlisted(matched: &str, detector: &Detector) -> bool { .any(|pattern| pattern.is_match(matched)) } +/// Lowercases `src` into `buf` without allocating a fresh string per line. +fn to_lowercase_into(src: &str, buf: &mut String) { + buf.clear(); + buf.extend(src.chars().flat_map(char::to_lowercase)); +} + +/// Folds every distinct detector keyword into one Aho-Corasick automaton so a +/// line is checked against all keywords in a single pass instead of one +/// substring search per keyword per detector. +struct KeywordPrefilter { + automaton: Option, + /// Automaton pattern index -> indices of detectors owning that keyword. + owners: Vec>, + /// Detectors without keywords always run their regex. + unconditional: Vec, + detector_count: usize, +} + +impl KeywordPrefilter { + fn new(line_detectors: &[&Detector]) -> Self { + let mut patterns: Vec<&str> = Vec::new(); + let mut pattern_indices: std::collections::HashMap<&str, usize> = + std::collections::HashMap::new(); + let mut owners: Vec> = Vec::new(); + let mut unconditional = Vec::new(); + + for (detector_index, detector) in line_detectors.iter().enumerate() { + if detector.keywords.is_empty() { + unconditional.push(detector_index); + continue; + } + for keyword in &detector.keywords { + let pattern_index = + *pattern_indices.entry(keyword.as_str()).or_insert_with(|| { + patterns.push(keyword.as_str()); + owners.push(Vec::new()); + patterns.len() - 1 + }); + owners[pattern_index].push(detector_index); + } + } + + let automaton = if patterns.is_empty() { + None + } else { + // Plain literal patterns cannot hit the automaton's size limits. + Some(AhoCorasick::new(&patterns).expect("build keyword automaton")) + }; + + Self { + automaton, + owners, + unconditional, + detector_count: line_detectors.len(), + } + } + + /// Marks the detectors whose keywords occur in `lowered_line` in + /// `candidates`, a scratch buffer reused across lines. + fn candidates_into(&self, lowered_line: &str, candidates: &mut Vec) { + candidates.clear(); + candidates.resize(self.detector_count, false); + for &detector_index in &self.unconditional { + candidates[detector_index] = true; + } + if let Some(automaton) = &self.automaton { + // Overlapping search: leftmost-first would report only one of two + // keywords sharing a prefix (e.g. "key" hides "keystone"). + for keyword_match in automaton.find_overlapping_iter(lowered_line) { + for &detector_index in &self.owners[keyword_match.pattern().as_usize()] { + candidates[detector_index] = true; + } + } + } + } +} + +struct LineScanContext<'detectors> { + line_detectors: &'detectors [&'detectors Detector], + prefilter: KeywordPrefilter, +} + +impl<'detectors> LineScanContext<'detectors> { + fn new(line_detectors: &'detectors [&'detectors Detector]) -> Self { + Self { + line_detectors, + prefilter: KeywordPrefilter::new(line_detectors), + } + } +} + +/// Per-line scratch buffers, reused across lines to avoid allocating in the +/// hot loop. Each scanning loop owns one (they are not shared across threads). +#[derive(Default)] +struct LineScratch { + lowered_line: String, + candidates: Vec, +} + fn scan_line_detectors( line: &str, line_number: usize, path: &str, - line_detectors: &[&Detector], - line_is_suppressed: bool, + context: &LineScanContext<'_>, + scratch: &mut LineScratch, findings: &mut Vec, ) { - if line_is_suppressed { + to_lowercase_into(line, &mut scratch.lowered_line); + if scratch.lowered_line.contains(INLINE_SUPPRESS) { return; } - for detector in line_detectors { - if detector.has_keywords(line) { - for mat in detector.regex.find_iter(line) { - if !is_allowlisted(mat.as_str(), detector) - && detector.has_sufficient_entropy(mat.as_str()) - { - findings.push(Finding { - file_path: path.to_string(), - line_number, - matched_content: mat.as_str().to_string(), - finding_type: detector.finding_type.clone(), - severity: detector.severity, - plugin_name: detector.name.clone(), - }); - } + context + .prefilter + .candidates_into(&scratch.lowered_line, &mut scratch.candidates); + + for (detector_index, detector) in context.line_detectors.iter().enumerate() { + if !scratch.candidates[detector_index] { + continue; + } + for mat in detector.regex.find_iter(line) { + if !is_allowlisted(mat.as_str(), detector) + && detector.has_sufficient_entropy(mat.as_str()) + { + findings.push(Finding { + file_path: path.to_string(), + line_number, + matched_content: mat.as_str().to_string(), + finding_type: detector.finding_type.clone(), + severity: detector.severity, + plugin_name: detector.name.clone(), + }); } } } @@ -64,8 +170,12 @@ fn scan_multiline_chunk( multiline_detectors: &[&Detector], findings: &mut Vec, ) { + if multiline_detectors.is_empty() { + return; + } + let lowered_chunk = chunk.to_lowercase(); for detector in multiline_detectors { - if detector.has_keywords(chunk) { + if detector.has_keywords(&lowered_chunk) { for mat in detector.regex.find_iter(chunk) { let line_in_chunk = chunk[..mat.start()].matches('\n').count() + 1; let line_content = chunk @@ -96,23 +206,17 @@ fn scan_content( content: &str, path: &str, multiline_detectors: &[&Detector], - line_detectors: &[&Detector], + context: &LineScanContext<'_>, ) -> (Vec, usize) { let mut findings = Vec::new(); let mut total_lines = 0; scan_multiline_chunk(content, 0, path, multiline_detectors, &mut findings); + let mut scratch = LineScratch::default(); for (line_idx, line) in content.lines().enumerate() { total_lines += 1; - scan_line_detectors( - line, - line_idx + 1, - path, - line_detectors, - is_inline_suppressed(line), - &mut findings, - ); + scan_line_detectors(line, line_idx + 1, path, context, &mut scratch, &mut findings); } (findings, total_lines) @@ -127,10 +231,12 @@ fn scan_stream( const CHUNK_SIZE: usize = 1000; const OVERLAP_LINES: usize = 50; + let context = LineScanContext::new(line_detectors); let mut findings = Vec::new(); let mut total_lines = 0; let mut buffer: Vec = Vec::with_capacity(CHUNK_SIZE + OVERLAP_LINES); let mut line_offset = 0; + let mut scratch = LineScratch::default(); for line_result in reader.lines() { let line = line_result.map_err(|source| ScannerError::ReadStream { @@ -139,14 +245,7 @@ fn scan_stream( })?; total_lines += 1; - scan_line_detectors( - &line, - total_lines, - path, - line_detectors, - is_inline_suppressed(&line), - &mut findings, - ); + scan_line_detectors(&line, total_lines, path, &context, &mut scratch, &mut findings); buffer.push(line); @@ -348,6 +447,7 @@ pub fn run_scan( let unique_paths: Vec<_> = unique_paths.into_iter().collect(); let exclude_patterns = compile_exclude_patterns(args, config)?; + let line_scan_context = LineScanContext::new(&line_detectors); let results: Vec<(Vec, usize, usize, Option)> = unique_paths .into_par_iter() @@ -383,7 +483,7 @@ pub fn run_scan( }; let (file_findings, file_lines) = - scan_content(&full_content, &path, &multiline_detectors, &line_detectors); + scan_content(&full_content, &path, &multiline_detectors, &line_scan_context); (file_findings, 1, file_lines, None) }) @@ -523,6 +623,7 @@ fn scan_staged_diff( multiline_detectors: &[&Detector], line_detectors: &[&Detector], ) -> Result<(Vec, ScanMetadata), ScannerError> { + let context = LineScanContext::new(line_detectors); let mut findings = Vec::new(); let mut total_lines = 0; let mut scanned_files: std::collections::BTreeSet = std::collections::BTreeSet::new(); @@ -532,6 +633,7 @@ fn scan_staged_diff( let mut next_line_number = 0; let mut hunk_start = 0; let mut hunk_added: Vec = Vec::new(); + let mut scratch = LineScratch::default(); let mut raw_line: Vec = Vec::new(); loop { @@ -567,8 +669,8 @@ fn scan_staged_diff( content, line_number, path, - line_detectors, - is_inline_suppressed(content), + &context, + &mut scratch, &mut findings, ); hunk_added.push(content.to_string()); diff --git a/tests/detector_tests.rs b/tests/detector_tests.rs index 59a6ef1..489a74a 100644 --- a/tests/detector_tests.rs +++ b/tests/detector_tests.rs @@ -80,7 +80,24 @@ fn test_keywords_prefilter_skips_non_matching_content() -> Result<(), DetectorEr assert!(!detector.has_keywords("some random text without the keyword")); assert!(detector.has_keywords("this text contains apikey in it")); - assert!(detector.has_keywords("APIKEY in uppercase")); + Ok(()) +} + +#[test] +fn test_keywords_are_lowercased_at_construction() -> Result<(), DetectorError> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &["ApiKey".to_string()], + None, + )?; + + // Callers lowercase content once per line; uppercase keyword definitions + // must still match that lowered content. + assert!(detector.has_keywords("this text contains apikey in it")); Ok(()) } From 4a47aa163a391b424a3b3dcbb5a60b6f64333087 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:14 +0530 Subject: [PATCH 05/20] fix(scan): never scan the --baseline file itself Exercising the baseline flow against public secret-corpus repos showed a feedback loop: the baseline file stores finding hashes that trigger detectors, so a baselined scan re-flagged its own baseline (46 -> 90 findings on trufflesecurity/test_keys) and every --update-baseline re-ingested them, growing the file indefinitely (90 -> 180 -> ...). The --baseline path is now appended to the exclude patterns as an escaped literal glob. --- src/scanner.rs | 9 +++++++++ tests/scanner_tests.rs | 44 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/scanner.rs b/src/scanner.rs index 95d9ff8..dfc61a2 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -553,6 +553,15 @@ fn compile_exclude_patterns( } } + // Never scan the baseline file itself: it stores finding hashes that + // trigger detectors, and each --update-baseline would re-ingest them, + // growing the baseline on every run. + if let Some(baseline_path) = &args.baseline { + let literal = Pattern::escape(baseline_path); + exclude_patterns + .push(Pattern::new(&literal).expect("escaped literal is a valid glob pattern")); + } + Ok(exclude_patterns) } diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index d1251dc..973dfb5 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -1483,3 +1483,47 @@ fn test_staged_scan_composes_with_baseline() -> Result<(), String> { Ok(()) } +#[test] +fn test_baseline_file_itself_is_never_scanned() { + let dir = unique_temp_dir("baseline_self_exclusion"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("create temp dir"); + fs::write( + dir.join("secrets.txt"), + "aws_access_key_id = AKIAIOSFODNN7EXAMPLE\n", + ) + .expect("write secret file"); + + let run = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "."]) + .args(extra) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&dir) + .output() + .expect("run key-watch") + }; + + let update = run(&["--baseline", "baseline.json", "--update-baseline"]); + assert!(update.status.success(), "baseline update should succeed"); + + // Two consecutive baselined scans must be clean and must not grow the + // baseline by re-scanning the baseline file's own hash strings. + let first = run(&["--baseline", "baseline.json"]); + let size_before = fs::metadata(dir.join("baseline.json")).unwrap().len(); + let second_update = run(&["--baseline", "baseline.json", "--update-baseline"]); + assert!(second_update.status.success()); + let size_after = fs::metadata(dir.join("baseline.json")).unwrap().len(); + + assert!( + String::from_utf8_lossy(&first.stdout).contains("No secrets found."), + "baselined findings must be suppressed, got:\n{}", + String::from_utf8_lossy(&first.stdout) + ); + assert_eq!( + size_before, size_after, + "re-updating the baseline must not ingest the baseline file itself" + ); + + let _ = fs::remove_dir_all(&dir); +} From 57c6fff36d7245310cee30d777dd242cecb9af58 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:24 +0530 Subject: [PATCH 06/20] feat(config): walk up ancestors for config discovery, bounded by trust Config discovery previously checked only the first scan path's immediate directory, so a repository-root .keywatch.toml never applied to nested paths like proto/payment.proto. Discovery now walks ancestor directories, nearest config first. The walk stops at the first directory containing .git (the repository root) or at the home directory: .keywatch.toml can add rules, disable detectors, and exclude paths, so trusting one from an arbitrary writable ancestor (e.g. /tmp/.keywatch.toml above CI checkouts) would let it silently weaken every scan below it. --- src/config/mod.rs | 67 +++++++++++++++++++--------- src/config/tests/discovery.rs | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/config/mod.rs b/src/config/mod.rs index fea7c08..df1b5d9 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -13,10 +13,11 @@ mod tests; /// User-facing configuration that extends and overrides the built-in detectors. /// -/// Loaded from `--config ` or discovered from the first CLI scan path -/// (directory itself, or parent directory for a file path). Merges with -/// `detectors.toml`: custom rules are appended, overrides are applied by -/// detector name. +/// Loaded from `--config ` or discovered by walking up from the first +/// CLI scan path (the directory itself, or a file's parent) through its +/// ancestors, stopping at the enclosing repository root or the home +/// directory. Merges with `detectors.toml`: custom rules are appended, +/// overrides are applied by detector name. #[derive(Deserialize, Default)] pub struct KeywatchConfig { pub rules: Option>, @@ -143,9 +144,14 @@ impl KeywatchConfig { /// /// Resolution order: /// 1. `explicit_path` — used as-is; returns `Err` if the file is absent. - /// 2. First CLI scan path that is a directory — search that directory. - /// 3. First CLI scan path that is a file — search its parent directory. - /// 4. Current working directory — only when `scan_paths` is empty. + /// 2. First CLI scan path — the directory itself for a directory, or the + /// parent for a file — then each ancestor directory, nearest first. + /// 3. Current working directory (and its ancestors) — only when + /// `scan_paths` is empty. + /// + /// The ancestor walk stops at the first directory containing `.git` (the + /// repository root) or at the home directory, so configuration is never + /// read from outside the tree being scanned. /// /// Candidate filenames tried in order: `.keywatch.toml`, `keywatch.toml`, /// `.kw.toml`. @@ -163,20 +169,18 @@ impl KeywatchConfig { scan_paths: &[String], cwd: &Path, ) -> Result, ConfigError> { - let config_path: Option = if let Some(path) = explicit_path { - if !Path::new(path).exists() { + let config_path = match explicit_path { + Some(path) if !Path::new(path).exists() => { return Err(ConfigError::NotFound { path: path.to_string(), }); } - Some(path.to_string()) - } else { - find_config_candidates(scan_paths, cwd) + Some(path) => Some(path.to_string()), + None => find_config_candidates(scan_paths, cwd), }; - let config_path = match config_path { - Some(path) => path, - None => return Ok(None), + let Some(config_path) = config_path else { + return Ok(None); }; let contents = fs::read_to_string(&config_path).map_err(|source| ConfigError::Read { @@ -231,9 +235,32 @@ fn find_config_candidates(scan_paths: &[String], cwd: &Path) -> Option { None => cwd.to_path_buf(), }; - CONFIG_NAMES - .iter() - .map(|name| search_dir.join(name)) - .find(|candidate_path| candidate_path.exists()) - .and_then(|candidate_path| candidate_path.to_str().map(str::to_string)) + let search_dir = if search_dir.is_absolute() { + search_dir + } else { + cwd.join(search_dir) + }; + + let home = std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .map(PathBuf::from); + + // Walk up from the scan target so a repo-root config applies to nested + // paths like `proto/payment.proto`, not just files beside the config. + // Stop at the repository root or the home directory: config above either + // is outside the scanned tree's trust boundary. + for dir in search_dir.ancestors() { + let found = CONFIG_NAMES + .iter() + .map(|name| dir.join(name)) + .find(|candidate_path| candidate_path.exists()) + .and_then(|candidate_path| candidate_path.to_str().map(str::to_string)); + if found.is_some() { + return found; + } + if dir.join(".git").exists() || home.as_deref() == Some(dir) { + return None; + } + } + None } diff --git a/src/config/tests/discovery.rs b/src/config/tests/discovery.rs index 44a4b38..87ce6a1 100644 --- a/src/config/tests/discovery.rs +++ b/src/config/tests/discovery.rs @@ -47,6 +47,88 @@ fn test_config_discovery_file_parent() { assert!(names.contains(&"FileParentRule".to_string())); } +#[test] +fn test_config_discovery_walks_up_to_ancestor_directories() { + let dir = TempDir::new().unwrap(); + write_file( + &dir, + ".keywatch.toml", + &minimal_rule_toml("RootRule", r"\\bROOT\\b", "HIGH"), + ); + let nested = dir.path().join("proto").join("payments"); + std::fs::create_dir_all(&nested).unwrap(); + let scan_file = nested.join("payment.proto"); + std::fs::write(&scan_file, "message Payment {}").unwrap(); + + let paths = vec![scan_file.to_str().unwrap().to_string()]; + let config = KeywatchConfig::load_for_paths(None, &paths) + .unwrap() + .expect("root config should apply to nested scan paths"); + let names: Vec<_> = config + .rules + .unwrap() + .into_iter() + .map(|rule| rule.name) + .collect(); + assert!(names.contains(&"RootRule".to_string())); +} + +#[test] +fn test_config_discovery_stops_at_repository_root() { + let dir = TempDir::new().unwrap(); + write_file( + &dir, + ".keywatch.toml", + &minimal_rule_toml("OutsideRule", r"\\bOUT\\b", "HIGH"), + ); + let repo_root = dir.path().join("repo"); + std::fs::create_dir_all(repo_root.join(".git")).unwrap(); + let scan_file = repo_root.join("secrets.txt"); + std::fs::write(&scan_file, "some content").unwrap(); + + let paths = vec![scan_file.to_str().unwrap().to_string()]; + let config = KeywatchConfig::load_for_paths(None, &paths).unwrap(); + assert!( + config.is_none(), + "config above the repository root must not be trusted" + ); +} + +#[test] +fn test_config_discovery_nearest_config_wins_over_ancestor() { + let dir = TempDir::new().unwrap(); + write_file( + &dir, + ".keywatch.toml", + &minimal_rule_toml("RootRule", r"\\bROOT\\b", "HIGH"), + ); + let nested = dir.path().join("nested"); + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write( + nested.join(".keywatch.toml"), + minimal_rule_toml("NearRule", r"\\bNEAR\\b", "LOW"), + ) + .unwrap(); + let scan_file = nested.join("secrets.txt"); + std::fs::write(&scan_file, "some content").unwrap(); + + let paths = vec![scan_file.to_str().unwrap().to_string()]; + let config = KeywatchConfig::load_for_paths(None, &paths) + .unwrap() + .expect("nearest config should be found"); + let names: Vec<_> = config + .rules + .unwrap() + .into_iter() + .map(|rule| rule.name) + .collect(); + assert!(names.contains(&"NearRule".to_string()), "nearest wins"); + assert!( + !names.contains(&"RootRule".to_string()), + "ancestor config must not shadow the nearest one" + ); +} + #[test] fn test_explicit_config_takes_precedence() { let dir = TempDir::new().unwrap(); From 5b429954f2e73cc1ab2fff4a4e415fb4e771e08d Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:34 +0530 Subject: [PATCH 07/20] feat(hooks): pre-commit hook delegates to scan --staged The pre-commit template previously listed staged files in bash and scanned each whole file, so pre-existing findings on unchanged lines blocked deletion-only and identical-to-upstream commits. The template is now a single 'key-watch scan --staged --exclude ...' call: added-line extraction, exclusion, and path attribution live in Rust, the per-file loop / awk hunk parser / symlink checks are gone, and a failing file listing can no longer fail open. Exit handling uses a case statement; any scanner exit above 1 (e.g. git failure) fails the hook closed. The pre-commit framework integration (.pre-commit-hooks.yaml) switches to the same command and stops passing filenames, which the staged scan already covers. hook install/uninstall messages abbreviate the home directory as ~, and hook unit tests pin core.hooksPath locally so a machine-wide global hooks path cannot leak into fixtures. --- .pre-commit-hooks.yaml | 12 ++-- src/hooks.rs | 124 ++++++++++++++++++++++------------------ templates/pre-commit.sh | 40 ++++++------- tests/hooks_tests.rs | 67 +++++++++++++--------- 4 files changed, 133 insertions(+), 110 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 63157c1..a0bb054 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,19 +1,19 @@ - id: keywatch-scan name: KeyWatch scan - description: Scan staged files for secrets, API keys, tokens, and credentials - entry: key-watch scan + description: Scan staged changes for secrets, API keys, tokens, and credentials + entry: key-watch scan --staged language: rust files: '' - pass_filenames: true + pass_filenames: false verbose: false args: ['--exit-mode=strict'] - id: keywatch-scan-system name: KeyWatch scan (system) - description: Scan staged files for secrets, API keys, tokens, and credentials - entry: key-watch scan + description: Scan staged changes for secrets, API keys, tokens, and credentials + entry: key-watch scan --staged language: system files: '' - pass_filenames: true + pass_filenames: false verbose: false args: ['--exit-mode=strict'] diff --git a/src/hooks.rs b/src/hooks.rs index 70708c8..3723384 100644 --- a/src/hooks.rs +++ b/src/hooks.rs @@ -19,6 +19,18 @@ fn shell_escape(input: &str) -> String { format!("'{}'", input.replace('\'', "'\"'\"'")) } +/// Renders a path for terminal output, abbreviating the home directory as `~`. +fn display_path(path: &Path) -> String { + let home = env::var_os("HOME") + .or_else(|| env::var_os("USERPROFILE")) + .map(PathBuf::from); + match home.as_deref().and_then(|home| path.strip_prefix(home).ok()) { + Some(rest) if rest.as_os_str().is_empty() => "~".to_string(), + Some(rest) => format!("~/{}", rest.display()), + None => path.display().to_string(), + } +} + fn build_repo_section(allowed: Option<&str>, blocked: Option<&str>) -> String { let escaped_allowed = allowed.map(shell_escape); let escaped_blocked = blocked.map(shell_escape); @@ -119,15 +131,19 @@ pub fn install_hook(args: &HookInstallArgs) -> Result<(), HookError> { if install_target.configured_global_path { println!( "Configured git --global core.hooksPath to {}", - install_target.hooks_dir.display() + display_path(&install_target.hooks_dir) ); } - if install_target.is_global { - println!("Installed global {hook_type_str} hook at {hook_path}"); + let scope = if install_target.is_global { + "global " } else { - println!("Installed {hook_type_str} hook at {hook_path}"); - } + "" + }; + println!( + "Installed {scope}{hook_type_str} hook at {}", + display_path(&install_target.path) + ); println!( "The hook will run automatically during git {}.", hook_type_str.replace('-', " ") @@ -144,15 +160,15 @@ pub fn uninstall_hook(args: &HookUninstallArgs) -> Result<(), HookError> { let hook_type_str = args.hook_type.as_str(); let install_target = resolve_hook_uninstall_target(hook_type_str, args.global)?; + let scope = if install_target.is_global { + "global" + } else { + "local" + }; if !install_target.path.exists() { - let scope = if install_target.is_global { - "global" - } else { - "local" - }; println!( "No {scope} {hook_type_str} hook found at {}", - install_target.path.display() + display_path(&install_target.path) ); return Ok(()); } @@ -168,17 +184,10 @@ pub fn uninstall_hook(args: &HookUninstallArgs) -> Result<(), HookError> { source, })?; - if install_target.is_global { - println!( - "Removed global {hook_type_str} hook at {}", - install_target.path.display() - ); - } else { - println!( - "Removed {hook_type_str} hook at {}", - install_target.path.display() - ); - } + println!( + "Removed {scope} {hook_type_str} hook at {}", + display_path(&install_target.path) + ); Ok(()) } @@ -306,22 +315,18 @@ fn read_global_hooks_path() -> Result, HookError> { .output() .map_err(|source| HookError::ReadGlobalHooksPath { source })?; - if output.status.success() { - let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); - return Ok(if value.is_empty() { - None - } else { - Some(PathBuf::from(value)) - }); - } - - if output.status.code() == Some(1) { + match output.status.code() { + _ if output.status.success() => { + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + Ok((!value.is_empty()).then(|| PathBuf::from(value))) + } // Exit code 1 means the key is not set. - return Ok(None); + Some(1) => Ok(None), + _ => { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + Err(HookError::ReadGlobalHooksPathFromGit { stderr }) + } } - - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - Err(HookError::ReadGlobalHooksPathFromGit { stderr }) } fn managed_global_hooks_dir( @@ -330,25 +335,26 @@ fn managed_global_hooks_dir( appdata: Option, userprofile: Option, ) -> Result { - if let Some(xdg) = xdg_config_home { - return Ok(PathBuf::from(xdg).join("key-watch").join("hooks")); - } - if let Some(home) = home { - return Ok(PathBuf::from(home) - .join(".config") - .join("key-watch") - .join("hooks")); - } - if let Some(appdata) = appdata { - return Ok(PathBuf::from(appdata).join("key-watch").join("hooks")); - } - if let Some(userprofile) = userprofile { - return Ok(PathBuf::from(userprofile) - .join(".config") - .join("key-watch") - .join("hooks")); - } - Err(HookError::MissingGlobalHooksBaseDir) + xdg_config_home + .map(|xdg| PathBuf::from(xdg).join("key-watch").join("hooks")) + .or_else(|| { + home.map(|home| { + PathBuf::from(home) + .join(".config") + .join("key-watch") + .join("hooks") + }) + }) + .or_else(|| Some(PathBuf::from(appdata?).join("key-watch").join("hooks"))) + .or_else(|| { + userprofile.map(|profile| { + PathBuf::from(profile) + .join(".config") + .join("key-watch") + .join("hooks") + }) + }) + .ok_or(HookError::MissingGlobalHooksBaseDir) } fn configure_global_hooks_path(hooks_dir: &Path) -> Result<(), HookError> { @@ -436,6 +442,14 @@ mod tests { .status() .expect("run git init"); assert!(status.success(), "git init should succeed"); + // Pin the local hooks path so a machine-wide core.hooksPath does not + // redirect hook resolution away from this repository. + let status = Command::new("git") + .args(["config", "core.hooksPath", ".git/hooks"]) + .current_dir(path) + .status() + .expect("run git config"); + assert!(status.success(), "git config should succeed"); } #[test] diff --git a/templates/pre-commit.sh b/templates/pre-commit.sh index 95b76cb..95feb94 100644 --- a/templates/pre-commit.sh +++ b/templates/pre-commit.sh @@ -10,27 +10,21 @@ if ! command -v "$KEYWATCH_BIN" >/dev/null 2>&1; then exit 1 fi -while IFS= read -r -d '' file; do - if [ -z "$file" ]; then - continue - fi - if [ -L "$file" ]; then - continue - fi - if [ ! -f "$file" ]; then - continue - fi - "$KEYWATCH_BIN" scan "$file" --exclude "$EXCLUDE_PATTERNS" >/dev/null 2>&1 - EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - continue - fi - if [ $EXIT_CODE -eq 1 ]; then - echo "ERROR: Secret detected in $file" +# scan --staged reads only the added lines of git diff --cached, so findings +# on unchanged lines never block a commit. A git failure inside the scanner +# exits with code 2, which fails the hook closed. +"$KEYWATCH_BIN" scan --staged --exclude "$EXCLUDE_PATTERNS" >/dev/null 2>&1 +EXIT_CODE=$? +case $EXIT_CODE in + 0) + exit 0 + ;; + 1) + echo "ERROR: Secret detected in staged changes. Run '$KEYWATCH_BIN scan --staged' to inspect." exit 1 - fi - echo "Error: key-watch failed on $file (exit code: $EXIT_CODE)" >&2 - exit 1 -done < <(git diff --cached --name-only -z) - -exit 0 + ;; + *) + echo "Error: $KEYWATCH_BIN scan --staged failed (exit code: $EXIT_CODE)" >&2 + exit 1 + ;; +esac diff --git a/tests/hooks_tests.rs b/tests/hooks_tests.rs index 4730519..3f74125 100644 --- a/tests/hooks_tests.rs +++ b/tests/hooks_tests.rs @@ -143,18 +143,13 @@ fn test_hook_generation_pre_commit() { hook.contains("KEYWATCH_BIN='"), "Should shell-quote binary assignment" ); - assert!(hook.contains("--exclude"), "Should pass exclude patterns"); assert!( hook.contains("EXCLUDE_PATTERNS='*.log,*.tmp'"), "Should preserve comma-separated exclude patterns" ); assert!( - hook.contains("scan \"$file\""), - "Should use scan subcommand" - ); - assert!( - hook.contains("[ -L \"$file\" ]"), - "Should skip staged symlinks" + hook.contains("scan --staged --exclude \"$EXCLUDE_PATTERNS\""), + "Should delegate staged-diff scanning and excludes to scan --staged" ); assert!( hook.contains(">/dev/null 2>&1"), @@ -213,11 +208,8 @@ fn test_pre_commit_failure_output_does_not_print_matched_secret() { let temp_dir = unique_temp_dir("pre_commit_secret_output"); let _ = fs::remove_dir_all(&temp_dir); fs::create_dir_all(&temp_dir).expect("create temp dir"); - fs::write(temp_dir.join("secret.txt"), "secret").expect("write staged file"); - let hook = generate_pre_commit_hook(&hook_install_args(HookType::PreCommit, None, None, None)); - let git_script = - "#!/bin/bash\nif [ \"$1\" = \"diff\" ]; then printf 'secret.txt\\0'; exit 0; fi\nexit 1\n"; + let git_script = "#!/bin/bash\nexit 1\n"; let keywatch_script = "#!/bin/bash\nprintf 'MATCHED_SECRET_VALUE\\n'\nprintf 'MATCHED_SECRET_VALUE\\n' >&2\nexit 1\n"; let output = run_hook(&hook, &[], git_script, keywatch_script, &temp_dir); let combined = format!( @@ -227,7 +219,7 @@ fn test_pre_commit_failure_output_does_not_print_matched_secret() { ); assert_eq!(output.status.code(), Some(1)); - assert!(combined.contains("ERROR: Secret detected in secret.txt")); + assert!(combined.contains("ERROR: Secret detected in staged changes")); assert!( !combined.contains("MATCHED_SECRET_VALUE"), "Generated pre-commit hook must not print matched secrets" @@ -238,27 +230,50 @@ fn test_pre_commit_failure_output_does_not_print_matched_secret() { #[cfg(unix)] #[test] -fn test_pre_commit_skips_staged_symlinks() { - let temp_dir = unique_temp_dir("pre_commit_symlink_skip"); - let marker = temp_dir.join("scanner-ran"); +fn test_pre_commit_delegates_to_staged_scan() { + let temp_dir = unique_temp_dir("pre_commit_staged_delegation"); + let marker = temp_dir.join("scanner-args"); let _ = fs::remove_dir_all(&temp_dir); fs::create_dir_all(&temp_dir).expect("create temp dir"); - fs::write(temp_dir.join("target.txt"), "secret").expect("write target"); - std::os::unix::fs::symlink("target.txt", temp_dir.join("linked.txt")).expect("create symlink"); - let hook = generate_pre_commit_hook(&hook_install_args(HookType::PreCommit, None, None, None)); - let git_script = - "#!/bin/bash\nif [ \"$1\" = \"diff\" ]; then printf 'linked.txt\\0'; exit 0; fi\nexit 1\n"; - let keywatch_script = format!("#!/bin/bash\nprintf ran > '{}'\nexit 1\n", marker.display()); + let hook = generate_pre_commit_hook(&hook_install_args( + HookType::PreCommit, + None, + None, + Some("*.log,*.tmp"), + )); + let git_script = "#!/bin/bash\nexit 1\n"; + let keywatch_script = keywatch_script_that_records_args(&marker); let output = run_hook(&hook, &[], git_script, &keywatch_script, &temp_dir); - assert!( - output.status.success(), - "symlink-only pre-commit should pass" + assert!(output.status.success(), "clean staged scan should pass"); + assert_eq!( + fs::read_to_string(&marker) + .expect("read scanner args") + .trim_end(), + "scan --staged --exclude *.log,*.tmp", + "hook must delegate added-line extraction and excludes to scan --staged" ); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_commit_fails_closed_when_staged_scan_errors() { + let temp_dir = unique_temp_dir("pre_commit_staged_scan_error"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_commit_hook(&hook_install_args(HookType::PreCommit, None, None, None)); + let git_script = "#!/bin/bash\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 2\n"; + let output = run_hook(&hook, &[], git_script, keywatch_script, &temp_dir); + + assert_eq!(output.status.code(), Some(1)); assert!( - !marker.exists(), - "scanner should not run for staged symlinks" + String::from_utf8_lossy(&output.stderr).contains("exit code: 2"), + "hook must fail closed when the staged scan cannot run" ); fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); From 0d78694b1b322f8c06ba83611ed03fe2461d0560 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:05:43 +0530 Subject: [PATCH 08/20] refactor: prefer match over if/else chains in report output Output selection becomes a match over the finding count with a verbose guard, the severity gate uses matches! and i32::from instead of a boolean if/else. --- src/lib.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e8644c9..aa988f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,19 +85,17 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { } .map_err(|source| RunCliError::ReportSerialize { source })?; - if args.verbose { - println!("{report_out}"); - } else if findings_count == 0 { - println!("No secrets found."); - } else { - println!( + match findings_count { + _ if args.verbose => println!("{report_out}"), + 0 => println!("No secrets found."), + count => println!( "WARNING: {} potential secret(s) detected (CRITICAL: {}, HIGH: {}, MEDIUM: {}, LOW: {})", - findings_count, + count, severity_counts.0, severity_counts.1, severity_counts.2, severity_counts.3 - ); + ), } if let Some(ref output_path) = args.output { @@ -153,9 +151,9 @@ fn calculate_exit_code(findings: &[Finding], exit_mode: &ExitMode) -> i32 { ExitMode::Always => 0, ExitMode::Critical => { let has_critical_or_high = findings.iter().any(|finding| { - finding.severity == Severity::Critical || finding.severity == Severity::High + matches!(finding.severity, Severity::Critical | Severity::High) }); - if has_critical_or_high { 1 } else { 0 } + i32::from(has_critical_or_high) } ExitMode::Strict => 1, } From 9f01a7002b8487209c4cf920a5fd1002f86c1030 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:06:53 +0530 Subject: [PATCH 09/20] fix(detectors): allowlist $PWD working-directory references in PasswordDetector '--volume "$PWD:/workspace:ro"' style docker/make snippets were flagged as HIGH Password findings. Uppercase 'PWD:' matches are allowlisted (the shell working-directory variable); lowercase 'pwd = ...' assignments are still detected. --- detectors.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detectors.toml b/detectors.toml index 9c36867..16bfb26 100644 --- a/detectors.toml +++ b/detectors.toml @@ -53,6 +53,9 @@ pattern = "(?i)(password|passwd|pwd)\\s*[:=]\\s*(?-i)[\"']?([^\"'\\n]+)[\"']?" finding_type = "Password" severity = "HIGH" keywords = ["password", "passwd", "pwd"] +# $PWD is the shell working-directory variable (docker --volume "$PWD:...", +# Makefiles), never a password literal. +allowlist = ["^\\$?PWD:"] [[detectors]] name = "EmailDetector" From c1163fa1b50b702bce23746445696c62a2b94d76 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:06:53 +0530 Subject: [PATCH 10/20] docs: document staged scanning, config discovery, and hook behavior Covers scan --staged semantics (pathspecs, binary handling, the multi-hunk secret caveat with pre-push as backstop), bounded walk-up config discovery, the core.hooksPath side effect and per-repo restore, and changelog entries for the fixes and the keyword prefilter performance work. --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ README.md | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bde35e..51cd04e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,32 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- **Staged diff scanning** — `scan --staged [path...]` scans only the added lines of `git diff --cached`, attributing findings to real file paths (which makes `--baseline` and `--exclude` compose) and post-image line numbers + +### Fixed + +- `PasswordDetector` no longer flags `$PWD:` volume mounts and similar working-directory references (uppercase `PWD:` is allowlisted; lowercase `pwd = ...` assignments are still detected) +- `Base64Detector` entropy threshold raised from 3.0 to 4.2 so long CamelCase identifiers in ordinary source code no longer flood reports with LOW findings; real base64 payloads (entropy >= ~4.27) are still detected +- Staged and git-history scans now force `--no-color`, literal pathspecs, and standard `a/`/`b/` diff prefixes, so user git config (`color.ui = always`, `diff.mnemonicPrefix`, `diff.noprefix`, `core.quotePath`) can no longer break diff parsing — a `color.ui = always` config previously made `scan --staged` silently report zero findings +- Files git renders as binary (including text files marked `-diff` in `.gitattributes`) are now surfaced in the report's `excluded_files` instead of being silently treated as clean by `scan --staged` +- Non-UTF-8 staged content is decoded lossily instead of aborting the whole staged scan with an error +- The `--baseline` file is automatically excluded from scanning; previously its stored hashes were re-flagged as findings and every `--update-baseline` grew the file indefinitely + +### Performance + +- Detector keywords are lowercased once at construction and every line is lowercased once (not once per detector), and all keywords are matched in a single Aho-Corasick pass per line — directory scans ~2.9x faster, stdin/git-history streams ~9x faster on a 36 MB corpus, identical findings + +### Changed + +- Pre-commit hooks now run `key-watch scan --staged` instead of whole-file scans, so only the lines a commit adds are scanned and findings on unchanged lines no longer block commits +- Pre-commit `--exclude` patterns are now matched against staged file paths (forwarded to `scan --staged --exclude`) +- The `pre-commit` framework integration (`.pre-commit-hooks.yaml`) now runs `key-watch scan --staged`, so framework users get diff-based scanning instead of whole-file scans +- Config discovery now walks up parent directories from the scan target, so a repository-root `.keywatch.toml` applies to nested paths; the nearest config wins and the walk stops at the repository root or home directory so config outside the scanned tree is never trusted +- The `pre-commit` framework hooks no longer pass filenames (`scan --staged` already scans exactly the staged set, and pathspec-glob interpretation of literal filenames could skip files) +- Hook install/uninstall messages abbreviate the home directory as `~` + ## [2.0.1] - 2026-08-02 ### Fixed diff --git a/README.md b/README.md index a9a0f73..56b995f 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,9 @@ cat secrets.txt | key-watch scan --stdin # Scan git history for committed secrets key-watch scan --git-history +# Scan only the lines staged for commit +key-watch scan --staged + # Verbose output (JSON) key-watch scan secrets.txt --verbose @@ -182,6 +185,7 @@ key-watch verify-integrity - `scan --format ` - Choose the report format written to stdout or the output file - `scan --stdin` - Read content from stdin instead of files - `scan --git-history` - Scan git history (`git log -p`) for committed secrets +- `scan --staged [path...]` - Scan only the added lines of the staged diff (run from inside the repository; paths narrow the diff as git pathspecs); findings keep real file paths and line numbers, so `--baseline` and `--exclude` compose. Files git renders as binary (e.g. `-diff` in `.gitattributes`) are listed in `excluded_files` rather than scanned. Note: a multi-line secret added across separate commits can span hunks the diff scan never sees together — the pre-push whole-tree scan remains the backstop for that case - `scan --output ` - Save report to file - `scan --verbose` - Print full JSON output - `scan --exclude ` - Comma-separated glob patterns to exclude @@ -242,11 +246,14 @@ password = 'known-test-password' # keywatch:ignore - `hook uninstall pre-commit|pre-push` removes a KeyWatch hook from the same target - `hook install ... --global` installs into Git's global hooks directory - `hook uninstall ... --global` removes the hook from Git's global hooks directory +- Pre-commit hooks run `key-watch scan --staged`, so only the lines a commit adds are scanned and findings on unchanged lines never block a commit +- Pre-commit `--exclude` patterns are forwarded to `scan --staged --exclude` and matched against staged file paths - Local hook paths are resolved via `git rev-parse --git-path hooks`, so installs work in worktrees and submodules too - If `core.hooksPath` is already configured, KeyWatch installs into that directory - Otherwise KeyWatch creates a managed hooks directory and configures `git config --global core.hooksPath` - KeyWatch refuses to overwrite a non-KeyWatch global hook file - KeyWatch also refuses to remove a non-KeyWatch global hook file +- A global `core.hooksPath` makes Git ignore every repository's own `.git/hooks/` scripts (husky, lefthook, plain hook files). To restore a repository's local hooks, run `git config core.hooksPath .git/hooks` inside it — the repo-local setting overrides the global one, and the KeyWatch hook then no longer runs in that repository ## Architecture From 7692e0c727e6477e8d2350c7a52ea3b7d0b86a19 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:17:34 +0530 Subject: [PATCH 11/20] style: apply cargo fmt and allow 'caf' in typos config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latin-1 test fixture spells café as raw bytes (caf\xE9), which the spell checker read as a typo for 'calf'. --- src/cli.rs | 5 +++-- src/detector.rs | 5 ++++- src/hooks.rs | 5 ++++- src/lib.rs | 12 ++++-------- src/scanner.rs | 40 ++++++++++++++++++++++++++++------------ typos.toml | 4 ++++ 6 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 8cb55ef..bbd8fbe 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -31,8 +31,9 @@ impl Display for CliValidationError { formatter.write_str("Cannot specify both --staged and --git-history") } Self::StdinWithPaths => formatter.write_str("Cannot specify both --stdin and paths"), - Self::MissingScanInput => formatter - .write_str("Must specify paths, use --stdin, --staged, or --git-history"), + Self::MissingScanInput => { + formatter.write_str("Must specify paths, use --stdin, --staged, or --git-history") + } Self::PreCommitRepositoryFilters => formatter.write_str( "--allowed-repos and --blocked-repos are only supported for pre-push hooks", ), diff --git a/src/detector.rs b/src/detector.rs index 0344513..9f80418 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -111,7 +111,10 @@ impl Detector { finding_type: finding_type.to_string(), severity: parsed_severity, allowlist: compiled_allowlist, - keywords: keywords.iter().map(|keyword| keyword.to_lowercase()).collect(), + keywords: keywords + .iter() + .map(|keyword| keyword.to_lowercase()) + .collect(), entropy_threshold, }) } diff --git a/src/hooks.rs b/src/hooks.rs index 3723384..3b65d68 100644 --- a/src/hooks.rs +++ b/src/hooks.rs @@ -24,7 +24,10 @@ fn display_path(path: &Path) -> String { let home = env::var_os("HOME") .or_else(|| env::var_os("USERPROFILE")) .map(PathBuf::from); - match home.as_deref().and_then(|home| path.strip_prefix(home).ok()) { + match home + .as_deref() + .and_then(|home| path.strip_prefix(home).ok()) + { Some(rest) if rest.as_os_str().is_empty() => "~".to_string(), Some(rest) => format!("~/{}", rest.display()), None => path.display().to_string(), diff --git a/src/lib.rs b/src/lib.rs index aa988f3..c538f51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,11 +90,7 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { 0 => println!("No secrets found."), count => println!( "WARNING: {} potential secret(s) detected (CRITICAL: {}, HIGH: {}, MEDIUM: {}, LOW: {})", - count, - severity_counts.0, - severity_counts.1, - severity_counts.2, - severity_counts.3 + count, severity_counts.0, severity_counts.1, severity_counts.2, severity_counts.3 ), } @@ -150,9 +146,9 @@ fn calculate_exit_code(findings: &[Finding], exit_mode: &ExitMode) -> i32 { match exit_mode { ExitMode::Always => 0, ExitMode::Critical => { - let has_critical_or_high = findings.iter().any(|finding| { - matches!(finding.severity, Severity::Critical | Severity::High) - }); + let has_critical_or_high = findings + .iter() + .any(|finding| matches!(finding.severity, Severity::Critical | Severity::High)); i32::from(has_critical_or_high) } ExitMode::Strict => 1, diff --git a/src/scanner.rs b/src/scanner.rs index dfc61a2..42e4986 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -58,12 +58,11 @@ impl KeywordPrefilter { continue; } for keyword in &detector.keywords { - let pattern_index = - *pattern_indices.entry(keyword.as_str()).or_insert_with(|| { - patterns.push(keyword.as_str()); - owners.push(Vec::new()); - patterns.len() - 1 - }); + let pattern_index = *pattern_indices.entry(keyword.as_str()).or_insert_with(|| { + patterns.push(keyword.as_str()); + owners.push(Vec::new()); + patterns.len() - 1 + }); owners[pattern_index].push(detector_index); } } @@ -216,7 +215,14 @@ fn scan_content( let mut scratch = LineScratch::default(); for (line_idx, line) in content.lines().enumerate() { total_lines += 1; - scan_line_detectors(line, line_idx + 1, path, context, &mut scratch, &mut findings); + scan_line_detectors( + line, + line_idx + 1, + path, + context, + &mut scratch, + &mut findings, + ); } (findings, total_lines) @@ -245,7 +251,14 @@ fn scan_stream( })?; total_lines += 1; - scan_line_detectors(&line, total_lines, path, &context, &mut scratch, &mut findings); + scan_line_detectors( + &line, + total_lines, + path, + &context, + &mut scratch, + &mut findings, + ); buffer.push(line); @@ -482,8 +495,12 @@ pub fn run_scan( Err(_) => return (Vec::new(), 0, 0, None), }; - let (file_findings, file_lines) = - scan_content(&full_content, &path, &multiline_detectors, &line_scan_context); + let (file_findings, file_lines) = scan_content( + &full_content, + &path, + &multiline_detectors, + &line_scan_context, + ); (file_findings, 1, file_lines, None) }) @@ -1036,8 +1053,7 @@ mod tests { diff.extend_from_slice(b"+caf\xE9 latin-1 line\n"); diff.extend_from_slice(b"+SECRET_AFTER_BINARYISH\n"); - let (findings, _) = - scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + let (findings, _) = scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); assert_eq!( findings.len(), diff --git a/typos.toml b/typos.toml index 7ca515b..d796726 100644 --- a/typos.toml +++ b/typos.toml @@ -5,3 +5,7 @@ extend-ignore-identifiers-re = [ # Fly.io token prefix pattern (fo1_) "fo1_", ] + +[default.extend-words] +# "caf\xE9" (café as latin-1 bytes) in a non-UTF-8 test fixture +caf = "caf" From 326c433e0e55c7d43621b31b840c832da33cda20 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:10:44 +0530 Subject: [PATCH 12/20] feat(baseline): auto-discover .keywatch-baseline.json and add update workflow Following the gitleaks/.gitleaksignore and ggshield/.gitguardian.yaml convention of a committed, conventionally named ignore file: when --baseline is not passed, KeyWatch walks up from the scan target (bounded at the repository root or home directory, same as config discovery) for .keywatch-baseline.json. Hook scans therefore use a committed repo baseline with no configuration; --no-baseline-discovery opts out and --update-baseline creates the conventional file when none exists. The resolved baseline is excluded from scanning like an explicit one. Baseline fingerprints normalize a leading ./ so 'scan .' and 'scan nested/file' produce matching entries. Re-baselining is deliberately not automatic (neither gitleaks nor ggshield auto-update; doing so would silently accept new secrets): a workflow_dispatch-only update-baseline workflow regenerates the file and opens a pull request for review. --- .github/workflows/update-baseline.yml | 51 +++++++++++ CHANGELOG.md | 2 + README.md | 5 +- src/baseline.rs | 26 +++++- src/cli.rs | 7 +- src/config/mod.rs | 18 +++- src/lib.rs | 13 +++ tests/cli_validation_tests.rs | 4 + tests/scanner_tests.rs | 127 ++++++++++++++++++++++++++ 9 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/update-baseline.yml diff --git a/.github/workflows/update-baseline.yml b/.github/workflows/update-baseline.yml new file mode 100644 index 0000000..24e6545 --- /dev/null +++ b/.github/workflows/update-baseline.yml @@ -0,0 +1,51 @@ +name: Update baseline + +# Manually triggered on purpose: automatically re-baselining on every push +# would silently accept newly introduced secrets. This regenerates +# .keywatch-baseline.json and opens a pull request so each newly baselined +# finding gets human review. (Neither gitleaks nor ggshield auto-update +# their baselines for the same reason.) +on: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update-baseline: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - name: Build key-watch + run: cargo build --release --locked + + - name: Regenerate baseline + run: ./target/release/key-watch scan . --update-baseline + + - name: Open pull request if the baseline changed + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ -z "$(git status --porcelain -- .keywatch-baseline.json)" ]; then + echo "Baseline unchanged; nothing to do." + exit 0 + fi + branch="chore/update-baseline-${{ github.run_id }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git add .keywatch-baseline.json + git commit -m "chore: update keywatch baseline" + git push origin "$branch" + gh pr create \ + --title "chore: update keywatch baseline" \ + --body "Regenerated by the update-baseline workflow. Review every newly baselined finding before merging: each entry is a finding KeyWatch will stop reporting." \ + --base "${{ github.ref_name }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 51cd04e..322ca39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file. - **Staged diff scanning** — `scan --staged [path...]` scans only the added lines of `git diff --cached`, attributing findings to real file paths (which makes `--baseline` and `--exclude` compose) and post-image line numbers +- **Baseline discovery** — without `--baseline`, a `.keywatch-baseline.json` is discovered by walking up from the scan target (bounded at the repository root or home directory), so hook scans use a committed repo baseline automatically; `--no-baseline-discovery` opts out and `--update-baseline` creates the conventional file when none exists. Baseline fingerprints ignore a leading `./` so `scan .` and `scan nested/file` entries match. A manually triggered `update-baseline` workflow regenerates the baseline via a reviewable pull request + ### Fixed - `PasswordDetector` no longer flags `$PWD:` volume mounts and similar working-directory references (uppercase `PWD:` is allowlisted; lowercase `pwd = ...` assignments are still detected) diff --git a/README.md b/README.md index 56b995f..2a2f78e 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,9 @@ key-watch verify-integrity - `scan --verbose` - Print full JSON output - `scan --exclude ` - Comma-separated glob patterns to exclude - `scan --exit-mode ` - Exit behavior: `always` (always pass), `critical` (fail on HIGH/CRITICAL only), `strict` (fail on any finding, default) -- `scan --baseline ` - Suppress known findings from a previous scan -- `scan --update-baseline` - Update baseline with current findings (requires `--baseline`) +- `scan --baseline ` - Suppress known findings from a previous scan. Without this flag, a `.keywatch-baseline.json` is discovered automatically by walking up from the scan target (bounded at the repository root or home directory), so hook scans pick up a committed repo baseline with no configuration +- `scan --no-baseline-discovery` - Ignore a discovered baseline (an explicit `--baseline` still loads) +- `scan --update-baseline` - Update the baseline with current findings; creates `.keywatch-baseline.json` when no baseline exists. The baseline stores fingerprints (path + finding type + SHA-256 of the match + detector), never secrets, and is meant to be committed. The `update-baseline` workflow can regenerate it via a reviewable pull request - `hook install [--global]` - Install a git hook - `hook uninstall [--global]` - Remove a git hook - `hook install pre-push --allowed-repos ` - Whitelist repos for pre-push hooks diff --git a/src/baseline.rs b/src/baseline.rs index d64e722..6eb764c 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -16,6 +16,18 @@ use crate::report::Finding; /// Application releases do not invalidate existing baselines. const BASELINE_VERSION: &str = "1.0"; +/// Conventional baseline filename, discovered automatically (like config) +/// when `--baseline` is not passed explicitly. +pub const DEFAULT_BASELINE_NAME: &str = ".keywatch-baseline.json"; + +/// Walks up from the first scan path (or the current directory) looking for +/// [`DEFAULT_BASELINE_NAME`], bounded at the repository root or home +/// directory. Returns `None` when no baseline file exists in the tree. +pub fn discover_baseline_path(scan_paths: &[String]) -> Option { + let cwd = std::env::current_dir().ok()?; + crate::config::find_file_upwards(scan_paths, &cwd, &[DEFAULT_BASELINE_NAME]) +} + /// Domain-separation prefix for fingerprint hashes so that baseline hashes /// cannot collide with plain SHA-256 of the matched content. const HASH_DOMAIN_SEPARATOR: &str = "keywatch-baseline-v1"; @@ -35,10 +47,20 @@ struct BaselineFingerprint { plugin_name: String, } +/// Fingerprint paths ignore a leading `./` so `scan .` and +/// `scan nested/file` produce matching baseline entries. +fn normalize_fingerprint_path(path: &str) -> String { + let mut path = path; + while let Some(stripped) = path.strip_prefix("./") { + path = stripped; + } + path.to_string() +} + impl BaselineFingerprint { fn from_finding(finding: &Finding) -> Self { Self { - file_path: finding.file_path.clone(), + file_path: normalize_fingerprint_path(&finding.file_path), finding_type: finding.finding_type.clone(), matched_content_hash: hash_content(&finding.matched_content), plugin_name: finding.plugin_name.clone(), @@ -49,7 +71,7 @@ impl BaselineFingerprint { impl From<&BaselineEntry> for BaselineFingerprint { fn from(entry: &BaselineEntry) -> Self { Self { - file_path: entry.file_path.clone(), + file_path: normalize_fingerprint_path(&entry.file_path), finding_type: entry.finding_type.clone(), matched_content_hash: entry.matched_content_hash.clone(), plugin_name: entry.plugin_name.clone(), diff --git a/src/cli.rs b/src/cli.rs index bbd8fbe..f729e1f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -82,7 +82,7 @@ pub enum Command { VerifyIntegrity, } -#[derive(Args, Debug)] +#[derive(Args, Debug, Clone)] pub struct ScanArgs { /// Paths to scan (files or directories) pub paths: Vec, @@ -116,9 +116,14 @@ pub struct ScanArgs { pub exit_mode: ExitMode, /// Path to a baseline file for suppressing known findings + /// (defaults to a discovered .keywatch-baseline.json) #[arg(long)] pub baseline: Option, + /// Disable automatic baseline discovery (an explicit --baseline still loads) + #[arg(long, default_value_t = false)] + pub no_baseline_discovery: bool, + /// Update the baseline file with current findings instead of scanning #[arg(long)] pub update_baseline: bool, diff --git a/src/config/mod.rs b/src/config/mod.rs index df1b5d9..52ec3d2 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -219,6 +219,18 @@ fn default_severity() -> Severity { const CONFIG_NAMES: [&str; 3] = [".keywatch.toml", "keywatch.toml", ".kw.toml"]; fn find_config_candidates(scan_paths: &[String], cwd: &Path) -> Option { + find_file_upwards(scan_paths, cwd, &CONFIG_NAMES) +} + +/// Walks up from the first scan path (or `cwd`) looking for one of `names`, +/// nearest directory first, stopping at the repository root or the home +/// directory so nothing outside the scanned tree's trust boundary is used. +/// Shared by config discovery and baseline discovery. +pub(crate) fn find_file_upwards( + scan_paths: &[String], + cwd: &Path, + names: &[&str], +) -> Option { let search_dir: PathBuf = match scan_paths.first() { Some(first) => { let scan_path = Path::new(first); @@ -245,12 +257,8 @@ fn find_config_candidates(scan_paths: &[String], cwd: &Path) -> Option { .or_else(|| std::env::var_os("USERPROFILE")) .map(PathBuf::from); - // Walk up from the scan target so a repo-root config applies to nested - // paths like `proto/payment.proto`, not just files beside the config. - // Stop at the repository root or the home directory: config above either - // is outside the scanned tree's trust boundary. for dir in search_dir.ancestors() { - let found = CONFIG_NAMES + let found = names .iter() .map(|name| dir.join(name)) .find(|candidate_path| candidate_path.exists()) diff --git a/src/lib.rs b/src/lib.rs index c538f51..e088b03 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,6 +46,19 @@ pub fn run_cli() -> Result<(), RunCliError> { fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { let start = Instant::now(); + // Resolve the baseline like config: an explicit --baseline wins, otherwise + // discover .keywatch-baseline.json in the scanned tree. --update-baseline + // with nothing discovered creates the conventional file in the current + // directory. The resolved path also gets excluded from scanning. + let mut args = args.clone(); + if args.baseline.is_none() && !args.no_baseline_discovery { + args.baseline = baseline::discover_baseline_path(&args.paths); + if args.baseline.is_none() && args.update_baseline { + args.baseline = Some(baseline::DEFAULT_BASELINE_NAME.to_string()); + } + } + let args = &args; + let config = if args.config.is_some() || !args.no_config_discovery { config::KeywatchConfig::load_for_paths(args.config.as_deref(), &args.paths)? } else { diff --git a/tests/cli_validation_tests.rs b/tests/cli_validation_tests.rs index 3f39f09..1c02091 100644 --- a/tests/cli_validation_tests.rs +++ b/tests/cli_validation_tests.rs @@ -14,6 +14,7 @@ fn test_stdin_with_path_validation_returns_typed_error() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -47,6 +48,7 @@ fn test_staged_with_stdin_validation_returns_typed_error() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -76,6 +78,7 @@ fn test_staged_with_git_history_validation_returns_typed_error() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -105,6 +108,7 @@ fn test_staged_allows_zero_or_many_paths() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 973dfb5..4447a96 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -118,6 +118,7 @@ sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\n\ exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -152,6 +153,7 @@ Stripe: sk_test_51ABCDEF12345678901234567890\n\ exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -187,6 +189,7 @@ AZURE_STORAGE=DefaultEndpointsProtocol=https;AccountName=examplestore; exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -223,6 +226,7 @@ b3BlbnNzaC1ldi0xLjAAABgQDQD2FGB3V2t4=\n\ exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -253,6 +257,7 @@ fn test_multiple_detections_in_line() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -295,6 +300,7 @@ fn test_directory_scan_with_exclusions() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -336,6 +342,7 @@ fn test_exclude_pattern_filtering() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -367,6 +374,7 @@ fn test_invalid_cli_exclude_pattern_returns_typed_error() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -416,6 +424,7 @@ fn test_dot_github_directory_is_scanned() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -445,6 +454,7 @@ fn test_scan_no_secrets() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -475,6 +485,7 @@ fn test_non_utf8_file_handling() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -509,6 +520,7 @@ fn test_multiple_files_scan() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -544,6 +556,7 @@ fn test_duplicate_paths_are_scanned_once() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -593,6 +606,7 @@ fn test_mixed_file_and_directory_paths_are_scanned_once() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -634,6 +648,7 @@ fn test_nonexistent_paths_are_ignored_without_counting_as_scanned() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -682,6 +697,7 @@ fn test_explicit_symlink_path_is_skipped() -> Result<(), String> { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -728,6 +744,7 @@ fn test_recursive_symlink_path_is_skipped() -> Result<(), String> { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -767,6 +784,7 @@ fn test_detect_aadhaar() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -804,6 +822,7 @@ fn test_detect_voter_id() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -838,6 +857,7 @@ fn test_detect_pan_card() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -872,6 +892,7 @@ fn test_detect_abha() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -907,6 +928,7 @@ fn test_multiple_indian_ids() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -967,6 +989,7 @@ fn test_overlapping_scan_roots_with_exclusions() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1012,6 +1035,7 @@ AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\n exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1060,6 +1084,7 @@ fn test_stdin_args_validation() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1121,6 +1146,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1136,6 +1162,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1154,6 +1181,7 @@ fn test_git_history_args_validation_allows_zero_or_one_path() { exit_mode: ExitMode::Strict, baseline: None, update_baseline: false, + no_baseline_discovery: true, config: None, no_config_discovery: false, format: OutputFormat::Json, @@ -1527,3 +1555,102 @@ fn test_baseline_file_itself_is_never_scanned() { let _ = fs::remove_dir_all(&dir); } + +#[test] +fn test_baseline_auto_discovered_from_repo_root() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("baseline_auto_discovery"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + fs::create_dir_all(repo_dir.join("nested")).map_err(|e| e.to_string())?; + fs::write( + repo_dir.join("nested/config.txt"), + "AWS Key: AKIAABCDEFGHIJKLMNOP\n", + ) + .map_err(|e| e.to_string())?; + + let run = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(extra) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&repo_dir) + .output() + .expect("run key-watch") + }; + + // No baseline anywhere: --update-baseline creates the conventional file. + let update = run(&["scan", ".", "--update-baseline"]); + assert!( + update.status.success(), + "default-name update should succeed" + ); + assert!( + repo_dir.join(".keywatch-baseline.json").exists(), + "update should create .keywatch-baseline.json" + ); + + // A nested scan discovers the repo-root baseline and comes back clean. + let scan = run(&["scan", "nested/config.txt"]); + assert!( + String::from_utf8_lossy(&scan.stdout).contains("No secrets found."), + "discovered baseline should suppress known findings, got:\n{}", + String::from_utf8_lossy(&scan.stdout) + ); + + // Discovery can be turned off. + let no_discovery = run(&["scan", "nested/config.txt", "--no-baseline-discovery"]); + assert_eq!( + no_discovery.status.code(), + Some(1), + "--no-baseline-discovery must ignore the repo baseline" + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_uses_discovered_baseline() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + let repo_dir = unique_temp_dir("staged_auto_baseline"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file(&repo_dir, "config.txt", "clean line\n", "initial")?; + stage_file( + &repo_dir, + "config.txt", + "clean line\nAWS Key: AKIAABCDEFGHIJKLMNOP\n", + )?; + + let run = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(extra) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&repo_dir) + .output() + .expect("run key-watch") + }; + + let update = run(&["scan", "--staged", "--update-baseline"]); + assert!( + update.status.success(), + "staged baseline update should succeed" + ); + + // The hook's exact invocation now picks the baseline up automatically. + let staged = run(&["scan", "--staged"]); + assert!( + matches!(staged.status.code(), Some(0)), + "staged scan should discover the repo baseline\nstdout:\n{}", + String::from_utf8_lossy(&staged.stdout) + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} From 3549d486c8e555b927a7f09e9209d31168e70258 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:24:11 +0530 Subject: [PATCH 13/20] fix(baseline): exclude the baseline file by canonical path The self-exclusion used a literal glob built from the --baseline string, which only matched when the caller passed a relative path. A discovered baseline resolves to an absolute path while scanned files stay relative, so a full-tree scan read the baseline back and re-flagged every stored hash as a finding (a 1043-entry baseline reported 1043 LOW findings). Exclusion now compares canonicalized paths. --- src/scanner.rs | 33 ++++++++++++++++++--------- tests/scanner_tests.rs | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/src/scanner.rs b/src/scanner.rs index 42e4986..b4598a8 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -7,7 +7,7 @@ use glob::Pattern; use rayon::prelude::*; use std::fs; use std::io::{BufRead, BufReader}; -use std::path::Path; +use std::path::{Path, PathBuf}; mod error; @@ -460,6 +460,7 @@ pub fn run_scan( let unique_paths: Vec<_> = unique_paths.into_iter().collect(); let exclude_patterns = compile_exclude_patterns(args, config)?; + let excluded_baseline = baseline_exclusion(args); let line_scan_context = LineScanContext::new(&line_detectors); let results: Vec<(Vec, usize, usize, Option)> = unique_paths @@ -469,7 +470,9 @@ pub fn run_scan( return (Vec::new(), 0, 0, Some(path)); } - if matches_exclude_patterns(&path, &roots, &exclude_patterns) { + if matches_exclude_patterns(&path, &roots, &exclude_patterns) + || is_baseline_file(&path, excluded_baseline.as_ref()) + { return (Vec::new(), 0, 0, Some(path)); } @@ -570,18 +573,26 @@ fn compile_exclude_patterns( } } - // Never scan the baseline file itself: it stores finding hashes that - // trigger detectors, and each --update-baseline would re-ingest them, - // growing the baseline on every run. - if let Some(baseline_path) = &args.baseline { - let literal = Pattern::escape(baseline_path); - exclude_patterns - .push(Pattern::new(&literal).expect("escaped literal is a valid glob pattern")); - } - Ok(exclude_patterns) } +/// Canonical path of the baseline file, so scans never read it. It stores +/// finding hashes that themselves trip detectors, and each +/// `--update-baseline` would re-ingest them, growing the file every run. +/// Compared canonically because a discovered baseline is absolute while the +/// scanned path may be relative (`./.keywatch-baseline.json`). +fn baseline_exclusion(args: &ScanArgs) -> Option { + let baseline_path = args.baseline.as_ref()?; + fs::canonicalize(baseline_path).ok() +} + +fn is_baseline_file(path: &str, baseline: Option<&PathBuf>) -> bool { + let Some(baseline) = baseline else { + return false; + }; + fs::canonicalize(path).is_ok_and(|candidate| candidate == *baseline) +} + fn parse_hunk_new_start(header: &str) -> usize { header .split_whitespace() diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 4447a96..30f1203 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -1654,3 +1654,55 @@ fn test_staged_scan_uses_discovered_baseline() -> Result<(), String> { let _ = fs::remove_dir_all(&repo_dir); Ok(()) } + +#[test] +fn test_discovered_baseline_file_is_never_scanned() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + // A discovered baseline resolves to an absolute path while scanned files + // are relative, so self-exclusion must compare canonical paths. + let repo_dir = unique_temp_dir("discovered_baseline_self_scan"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + fs::write( + repo_dir.join("secrets.txt"), + "aws_access_key_id = AKIAIOSFODNN7EXAMPLE\n", + ) + .map_err(|e| e.to_string())?; + + let run = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "."]) + .args(extra) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&repo_dir) + .output() + .expect("run key-watch") + }; + + assert!(run(&["--update-baseline"]).status.success()); + let size_before = fs::metadata(repo_dir.join(".keywatch-baseline.json")) + .map_err(|e| e.to_string())? + .len(); + + let rescan = run(&[]); + assert!( + String::from_utf8_lossy(&rescan.stdout).contains("No secrets found."), + "scanning the tree must not re-flag the discovered baseline's own hashes, got:\n{}", + String::from_utf8_lossy(&rescan.stdout) + ); + + assert!(run(&["--update-baseline"]).status.success()); + let size_after = fs::metadata(repo_dir.join(".keywatch-baseline.json")) + .map_err(|e| e.to_string())? + .len(); + assert_eq!( + size_before, size_after, + "re-updating a discovered baseline must not ingest the baseline itself" + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} From 7f4326e1577687da9581e206e22c2ae9a36c0345 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:48:59 +0530 Subject: [PATCH 14/20] chore: dogfood a committed baseline instead of inline ignore comments Suppressing KeyWatch's own fake test fixtures with inline keywatch:ignore comments meant editing source purely to satisfy the scanner. A committed .keywatch-baseline.json does the same job without touching the code, which is the convention gitleaks and ggshield use for their own repositories. This also drops the blanket tests/* path exclusion: the baseline pins the exact fingerprint of each known fixture, so a newly introduced secret in tests/ is still reported, whereas an excluded path hides everything under it forever. Committing the baseline surfaced a gap in the staged scan, fixed here: the baseline file was skipped by path scans but not by scan --staged, so staging a baseline flagged all of its stored hashes as added lines and blocked the commit. --- .keywatch-baseline.json | 1412 +++++++++++++++++++++++++++++++++++++++ .keywatch.toml | 15 +- src/scanner.rs | 24 +- tests/scanner_tests.rs | 46 ++ 4 files changed, 1482 insertions(+), 15 deletions(-) create mode 100644 .keywatch-baseline.json diff --git a/.keywatch-baseline.json b/.keywatch-baseline.json new file mode 100644 index 0000000..71bc3c0 --- /dev/null +++ b/.keywatch-baseline.json @@ -0,0 +1,1412 @@ +{ + "version": "1.0", + "entries": [ + { + "file_path": "./.github/workflows/ci.yml", + "line_number": 46, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./.github/workflows/docker-publish.yml", + "line_number": 42, + "finding_type": "Password", + "matched_content_hash": "4e433d275810f054b78ebef6ba3971128253dc8fb930dee5f57c5c846b6d42ea", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./Cargo.lock", + "line_number": 9, + "finding_type": "Random String", + "matched_content_hash": "c8d1c3250631a9e1d023dd19a2a08bb870ed58b1097399b417a90f14d8b879d0", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 18, + "finding_type": "Random String", + "matched_content_hash": "33eb33ba107f160af28967a66382f97144224347f82f83ed6a4597be39f81135", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 33, + "finding_type": "Random String", + "matched_content_hash": "370b19237b0e67b69d28abc80f0b9cb8daf63333f6de649304e4b943be0e8031", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 39, + "finding_type": "Random String", + "matched_content_hash": "da591a00311f9d4821ec6e5d16c3378e9096759c49db9c5dcfb7ed964cc4583b", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 48, + "finding_type": "Random String", + "matched_content_hash": "cf9c67a00674f63b9a98cce5e5e95475c05a6bc37b5847b387d656f5bf38a792", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 57, + "finding_type": "Random String", + "matched_content_hash": "a54f7d8294cebd637550eb00fc90200de9fc5b59fe7a87de9a2239f1b06ff8e0", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 68, + "finding_type": "Random String", + "matched_content_hash": "1ec491434d71773bc0b63657e3e07632a1836a6ad089083eaad8798e601056a7", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 74, + "finding_type": "Random String", + "matched_content_hash": "f62bb4c31019b93a9189c8c6f9629357c1d5690a3cfe217ceb037f409d19a584", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 83, + "finding_type": "Random String", + "matched_content_hash": "3f156fa0d5ef2ac02cf31659eb860c6e6eee2377a8308a1a7c5b59dbc0080f70", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 89, + "finding_type": "Random String", + "matched_content_hash": "5a7fcd4b40b4e4ea2f02afac071ee1178c4a493be9fcdc1f829e5c063d157dd2", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 99, + "finding_type": "Random String", + "matched_content_hash": "3c5b70444607fcb2158a3f3342608e80522322a0c8a9474256d5a11d6e9bf44a", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 111, + "finding_type": "Random String", + "matched_content_hash": "82abbc51d83a81e951a7236b75bf99a2f1ad7cdbef9f973a92e22047bf66f64f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 123, + "finding_type": "Random String", + "matched_content_hash": "8e21663c1445dcc5577f3b9634312bf4be1beed362850af9f0f5d6ee739f2d5e", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 129, + "finding_type": "Random String", + "matched_content_hash": "417ae24b2c82afb5ea7bcb077b8135a92dc4610e8ecd59a84f5f357990d6dc78", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 135, + "finding_type": "Random String", + "matched_content_hash": "fc08b3d5b2beb633fca936bb90a93fff282a02b248c3e2eb8e50f2dc4d910e44", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 141, + "finding_type": "Random String", + "matched_content_hash": "6b11c0a2c2282f33ff93724fbaa51b7fef9095f0c4999176c3f6f7f3aacb1eba", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 150, + "finding_type": "Random String", + "matched_content_hash": "e7c9a693d2ef31690569c03ce5c0edec0c87b72e1a2c6f914680384e501ebea0", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 160, + "finding_type": "Random String", + "matched_content_hash": "f150b6fab76528bb09b8eecd1afd00079e3ad3c560f15c34f81f3aa8c0191c8d", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 169, + "finding_type": "Random String", + "matched_content_hash": "b29c48f46f4936941ebc1096fc091d94043bde96734b30cbd4b5c80cf77081f7", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 175, + "finding_type": "Random String", + "matched_content_hash": "d5b3cde488632bcc2a7ab589a01cc094211b634494fe16a15db54efc6417b806", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 184, + "finding_type": "Random String", + "matched_content_hash": "371c9558953a2734f61cb896b12635f922dd8ecfeaee0028bbf50d4a175f4353", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 195, + "finding_type": "Random String", + "matched_content_hash": "d0ad5d95f66287e168f8ba968156efc8ff02f1f6c7f8a9916160c56de50b6b02", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 204, + "finding_type": "Random String", + "matched_content_hash": "af070f6fb020dc67c6ecebf6e156744215ca2de910c64c70b87693d4b190e582", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 216, + "finding_type": "Random String", + "matched_content_hash": "7c8d7968efe25482769a60d0e7eb4ca89b9f8a3a52a56c46c5a1f76e1220d014", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 222, + "finding_type": "Random String", + "matched_content_hash": "31487886ae8abf681f91f628d49931d711807523bbd5d07654e8d748a647db93", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 228, + "finding_type": "Random String", + "matched_content_hash": "1457870a87e534c0226a3e87d749b987b5d418b9c2d59aed3f6d70bd1533df03", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 238, + "finding_type": "Random String", + "matched_content_hash": "7d0f79f2c1291b58191aad3cf0902e45ac813f6a75959d9b4bea24ad1ab7953f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 244, + "finding_type": "Random String", + "matched_content_hash": "60c48091657bd1d25b5bb3baae17927907d996d674b8383ae1542447ca485d9c", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 255, + "finding_type": "Random String", + "matched_content_hash": "f81293058acb5c579279f1ffd36da29b37c27b3264672c926ce8ce41b8db688b", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 266, + "finding_type": "Random String", + "matched_content_hash": "d4dc06aca115ba3231671b6fabca0e002918a04b181880399e201e3f9166a6f5", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 272, + "finding_type": "Random String", + "matched_content_hash": "f2bb5c7af3a24236cdf2655efcf0abee61cb7c9db9ca7531f04fdc03392df3ac", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 278, + "finding_type": "Random String", + "matched_content_hash": "60149c530a8d2a5ad767839642c169a1ff3a6f7f1f4504dd2374314db44e7a72", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 284, + "finding_type": "Random String", + "matched_content_hash": "91403991975cf43935efa745c90457f0dfc78fada76a7e9dc397807ae6ddc14c", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 290, + "finding_type": "Random String", + "matched_content_hash": "a4dadec8783080f16011968f68106152b4ef05f77620c6a330fcdcffb974b14e", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 299, + "finding_type": "Random String", + "matched_content_hash": "93e8b37755c946a6385d1cd22a72530b6bc47f82de6cf0513f4594c3d76f0de7", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 309, + "finding_type": "Random String", + "matched_content_hash": "3f5e171ec2ceec6b7fe0514cf88582084ef10d7fab1be8a86f59cf76ac853415", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 315, + "finding_type": "Random String", + "matched_content_hash": "0f5cffacb91ff3fd3ef6321ab04a0219350b8036b4bd7c3ce59f468be286f71f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 339, + "finding_type": "Random String", + "matched_content_hash": "bb53833f654b5be3d7ec25711178e011ac56bb20b7c2667149210b8777751afc", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 345, + "finding_type": "Random String", + "matched_content_hash": "7dbe98e83dfaef207d3a4249c2516b9b43802fc4e7a49fd937e1f00ef7d55306", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 354, + "finding_type": "Random String", + "matched_content_hash": "f860869b093e1e91294416d75b01e59c9ec15ac9a04f7be7e9c387398c013006", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 360, + "finding_type": "Random String", + "matched_content_hash": "32437fb9cffca3f9abb3b4eff8451721893f8aaee3787b92410430b7d4400161", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 366, + "finding_type": "Random String", + "matched_content_hash": "5aecc8081e0db0f58678c133d404b5e21a30d77fe0158426ebd3b8e0657f5b11", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 372, + "finding_type": "Random String", + "matched_content_hash": "d1de37a1f7f5929eb802e9036d41cd1770b34cf9457d41f4c84a1b8adb167656", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 378, + "finding_type": "Random String", + "matched_content_hash": "a8a1789b0b86434f7025a208a36ad25ed1f63cba52ce8fc93accc4ed29bc0507", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 384, + "finding_type": "Random String", + "matched_content_hash": "8e38bacf79922659ae17e176d20ee82d74f5e1f387ee6c87327c049206dd166e", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 393, + "finding_type": "Random String", + "matched_content_hash": "13f73828067665fdaa84d6eadb2f9e78a227243373c697ee30c3cbaec703f85c", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 402, + "finding_type": "Random String", + "matched_content_hash": "7a47291e29eeb7fe52f62692e78bf3f9314d437bc9843709b70aaf066969df90", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 408, + "finding_type": "Random String", + "matched_content_hash": "2bc6471fa1a1c83d0d4ff1a83f797ce2a5184cc84b7d7ed77f777961b76dc70d", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 418, + "finding_type": "Random String", + "matched_content_hash": "cd607357cfa9cc395fd66b1a29f34cd5afd616c36eabe3de6aed4c7c18fd2b26", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 428, + "finding_type": "Random String", + "matched_content_hash": "7ef556c5c17bdcf5888308b2b6447da7458ac415f4385a5ee0af18dacebf2e12", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 439, + "finding_type": "Random String", + "matched_content_hash": "67ca470700433b58407e27275fece3b6cd025fec33b6ab38918aafc0cede8fc6", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 451, + "finding_type": "Random String", + "matched_content_hash": "035f4b9c3a14fa6d7e3c26e5f534720e238da761c96674b9c0b29ad65dd3c36f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 462, + "finding_type": "Random String", + "matched_content_hash": "0ab5508f4caa9319318fcd6028bef1298848dcb529bac7a5e2abd8e9a993395f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 468, + "finding_type": "Random String", + "matched_content_hash": "7706a5b27cf99b14a13c4484391215bdfc49a1902f6b1148d67b1efdc2eb2d6f", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 481, + "finding_type": "Random String", + "matched_content_hash": "e8f521072549cca3e83fc6506c0fd55f39c8e402738f7206d153ff47a8a32997", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 491, + "finding_type": "Random String", + "matched_content_hash": "0b7d3ab93e23cb7adef796cfb80929727a3852974e17a0dc5095841eceeb3c82", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 500, + "finding_type": "Random String", + "matched_content_hash": "5e9edb82257f05da28e1594e808d2aadf3cd1cd71cd73416f241d0290eb84e64", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 511, + "finding_type": "Random String", + "matched_content_hash": "a1691de1bcb099bdf0ae5a35eae604073911c38d458be89d8b4b7dc03e8efd24", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 524, + "finding_type": "Random String", + "matched_content_hash": "6e390335cd3bb60382d4296111ec37bfaa5babcef8282f4822f13decb12036c4", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 533, + "finding_type": "Random String", + "matched_content_hash": "2a9a13fb8ea7019fd3bc9a41836d894c15e193d3a4d8074cda657047f0b8186c", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 544, + "finding_type": "Random String", + "matched_content_hash": "a386bab7a0c34b10c11346dd289cba5a83f785d54298bdfc360edaa5a64156df", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 550, + "finding_type": "Random String", + "matched_content_hash": "14f72c9ea36dd39c9f02654739f97c2ad0a7f2a87c5182f8c408cafa69619fc9", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 561, + "finding_type": "Random String", + "matched_content_hash": "4f6e923095b0cf803643e3e60a94fbf48d18e87552860ee85ffd06fd8aae2657", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 574, + "finding_type": "Random String", + "matched_content_hash": "e564e5e576c326dc436666249b7b1cbcb6dfc10e65f4c34c86dd58f4281d48f7", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 583, + "finding_type": "Random String", + "matched_content_hash": "21d093030e2f60564be7e1f156f86ed27506b7a41e90ac0d25accb77128296b5", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 594, + "finding_type": "Random String", + "matched_content_hash": "a1a3501f74b6e3838ff19d6c6039299d3156ec7bf872b72ec538183d739b87a6", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 609, + "finding_type": "Random String", + "matched_content_hash": "555808c70baac48da4fb3e614f6d9a7b19f6742ca0565890fbca2245f7edf025", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 618, + "finding_type": "Random String", + "matched_content_hash": "a39b99d0f63dc4048f376a668997a9df8a2d4f8c76d826611f177955d74640a8", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 627, + "finding_type": "Random String", + "matched_content_hash": "90a67ab65795b169fed64ad757a916de4d04c9bb084c923f111b0c8385e1b885", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 633, + "finding_type": "Random String", + "matched_content_hash": "a2442da3498d252d9a36b7e7a2265cbc3046dcadb1d30505944af00f79978515", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 639, + "finding_type": "Random String", + "matched_content_hash": "f25eeaca1d26fca5ebd395ce085c16c7317015de4c43b6359bc06fa4be3f902c", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 645, + "finding_type": "Random String", + "matched_content_hash": "44e4e53ad62323823f8bc3a5c35c962187bdbaf89be46414a23783834910dbee", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 651, + "finding_type": "Random String", + "matched_content_hash": "3b8c7590b9b628c6540d5f8c3b204921f02243a53abda9e767797cd9b6a94f55", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 657, + "finding_type": "Random String", + "matched_content_hash": "9560a34bab7016b3f62cab254be31cb3e0b0d73ef0505163eb59ecffe543be32", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 663, + "finding_type": "Random String", + "matched_content_hash": "733dd83061d241cd637bcaa908b08aae34be8bd36010d82bce87789d2776a44b", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 672, + "finding_type": "Random String", + "matched_content_hash": "108b563afb32bf298e14587a98ffab0829649cc6fe900823b1994fceb7fe0547", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.lock", + "line_number": 678, + "finding_type": "Random String", + "matched_content_hash": "5998851de721febfdd446d7ebf849bc5db8139d7111aec11efb4e69d7ee61bce", + "plugin_name": "RandomString" + }, + { + "file_path": "./Cargo.toml", + "line_number": 6, + "finding_type": "Email Address", + "matched_content_hash": "dd58c0e81231a24fa00886e14227fc41fa36882ab3965aaa37d1ef013bf3e8f9", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 1, + "finding_type": "Phone Number", + "matched_content_hash": "0deb55c5d91b45627d90d836ab67021855ae0fd993bf3ad309829ad75f3c445d", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 7, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "c7499b2b8fb43780aa044148fb9ecdcb6795c4604db9ad468644fa974cb87ae6", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Phone Number", + "matched_content_hash": "16391bbd0912e3a14a7904a19e59e0e90a26915649298a90cbf169270f0b3d36", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "4b235f293c86102514fce21c17c77c9e46da4e34ca3f36642dde269bf593a42a", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "9cb066ba3697cbcd17f51c83125e42b3b09256edd013145402d9066613eba929", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "3f469ae3bade9899dc4902162eade7e691ea17aeef930ef0f12c82f99a3b2ded", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "118aa0afdb39a144fd8285e2b9174fac00c729cd5f8a66eca1ddf975997a37fc", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "8e3cb34aae11dc73b875cc925ef3b875c7fc154ca52c133f90159fb65aeb80ee", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "318e014fa33dc923021432ac2c388ccb64d2e886e32e82095d590caf37412da3", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "f359f7aacd4e316cadc07e5a2561ed82ffdd4afc171581390d638f248c958735", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "91e0e33ed1c5151578d8fac6d4f139bab71ac57c9feb4f68ecda5fc49e3e8bbf", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/cli-modules.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "7363f682607f5d1c84d39f90fe9559d9f16b0b21fc287c006b07ed038c43ffe1", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 1, + "finding_type": "Phone Number", + "matched_content_hash": "2044d9eee458d1db70b32a0a58303a41a3334aeab1009e4c7da50789d14016c6", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 7, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "c77955587c1ef977a8ef77ed7c143faf1c8922f858da143af3e146a3f95fe841", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Phone Number", + "matched_content_hash": "16391bbd0912e3a14a7904a19e59e0e90a26915649298a90cbf169270f0b3d36", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "83929a84ca0439480236425c060e4e784dfd74f72d7f3cc5dbad2ea4c0bb18e1", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "56dda2c9c468a9ed6a08bec9cb49ddd201e98b179951477abc5188f4a47a1d6e", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "045c378999d40cdba4d8400c824996e20c6e324d85078accd2619d8d13bb6dde", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "aafcd1d46e6640f70204ee1979df445d2c2f30975f2db678a52ae1c335613dbf", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "e7741b12ff6c08334c644e0593b38a566bf878e8bd6550855ec268f097654c29", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "fa9cafeea120bb26d43f09cb134a5b71ba21c0d8178533aaa29ff72db4a83816", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/detector-config-trust.svg", + "line_number": 92, + "finding_type": "Random String", + "matched_content_hash": "de239f05cccb06dd9750a39355706433951533d49d597dd94ab1413ca76af498", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 1, + "finding_type": "Phone Number", + "matched_content_hash": "3dd6d01f94aa20b694e951442cb53bca1869c85e989c673f6bfb64264cb4688d", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 7, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "f3753f090549b3529cacad629e2bfcbd8864d45851010ffedc5159fc3bb74e91", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 14, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "ba34e96d0f5312b12c26d734c3cd0c5f468e318de78c5fb54c13f5b6df6515a8", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "7ae7f412f4e7e09aa2a1ff3b3971856db6fb02b3a0831655fd2e8d06c50e08b7", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "792a73cb3e6c1c8bcf043169c3a04b5150a44aee82cc9dc3a1465e9cc87d4635", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "e2c8b447bec2515ba87584b65ab76809a08c289254e76b78aace09c1b60706e0", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "6b8142a88eac27c3dc92f2954c69a7b0879469fa695ee01c674d28a528031e1e", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "16391bbd0912e3a14a7904a19e59e0e90a26915649298a90cbf169270f0b3d36", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "d23bcc094744d55fc2703893908052f38b15db057959bae98bfc888224d393c9", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "72f3317324e1695d075e798056d689c3aad0c6e766d79885c9af88686577406f", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Phone Number", + "matched_content_hash": "836cfe1bfda35b788dbb5a4a57540a0679450b674b4c88a3f03a08a02d18e967", + "plugin_name": "PhoneNumberDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Credit Card Number", + "matched_content_hash": "6709932b262fff137fb5b0e129ace37e7b0e7f7f6ae835b4f6e6db75db34704c", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Credit Card Number", + "matched_content_hash": "fd7252084fb5689a6d3ed4f47bf05ff768cb730cabd9c8fbac789fb582c6a7f5", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Credit Card Number", + "matched_content_hash": "e145d9cf0cc5e695541faf4a13e889503fdcd301129cad4564acbe5f74d4ed3b", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Credit Card Number", + "matched_content_hash": "281d66846b079e5f043527db6f1572b1b08dc766e25a6a03dbdf31775f73c92f", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Credit Card Number", + "matched_content_hash": "99563df9c71a2221e67007f5d1c688ca172852db3a5720ab4f29ce34a428cc7a", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "b5458e408e59b20b5f8bb66bb996cf3229bba0f4260ab705070fe7bf7029afa4", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "863a91ad1862061e7a75d0c35635387e87c46d03f419fedf85776419a0857c12", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "0f3980221f6fd3f6b36a6c1ad85672be41f74a19ca07fe476139e93dca97a2f1", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "60f82ca7405be5f3a6c7f4b68b73ac5860326d1803f68cb2c59a6f8cf385b5a4", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "ba9a801e888aa9b07340db3e28892741e9d96ee7603dd608769900ef4c0ccb8b", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "37af6e0b2eabc53664239dccdc3cbc94143615480a8125c127c0886b7919d7ed", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "d11ae8f43eb29643b289cadb28ef5ffbaf4455ea37b89d7c2858ccbd270171be", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Random String", + "matched_content_hash": "888b0d80b6cb228df1b6bf35bc6201908bc74a15d2012ce3b1417d043d4d169c", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Random String", + "matched_content_hash": "6b6adc15e5af26b2c97bfd0a9efa20983d3da2bd27d9cb3b9fd20e33b6bb00a3", + "plugin_name": "RandomString" + }, + { + "file_path": "./docs/architecture/scan-pipeline.svg", + "line_number": 99, + "finding_type": "Random String", + "matched_content_hash": "fb1a63b387f03e5a3c47f9524380935ddbcf2fc2fa4cea99d6dcd00d9bb7dd3a", + "plugin_name": "RandomString" + }, + { + "file_path": "./src/baseline.rs", + "line_number": 270, + "finding_type": "Random String", + "matched_content_hash": "c40c9141cd77d55f1ba9d21fb045f8e7c5e900c16f2e563779e7ed75cee6b15b", + "plugin_name": "RandomString" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 851, + "finding_type": "AWS Access Key", + "matched_content_hash": "3f733150de7916d4778298d7f90493889c38b76876b80c058e439851ce60cb2b", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 851, + "finding_type": "Password", + "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 851, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 907, + "finding_type": "Private Key Content", + "matched_content_hash": "f91082d1cbd2032b5ea19f2bbf6b3f88e02c12ae320fc9e08e6dc0f73d82bdd1", + "plugin_name": "PrivateKeyContentDetector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 907, + "finding_type": "SSH Private Key", + "matched_content_hash": "678a65e8968aabff441076ae306e13d4fc85b1d36d8036a10d2100c1dc40d251", + "plugin_name": "SSHPrivateKeyDetector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 907, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "3dde06bf268892d4210f4b0bf1402ecc6a8ad1e015f8204cdc31667762572ef5", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./src/scanner.rs", + "line_number": 930, + "finding_type": "Password", + "matched_content_hash": "f260ab98a91b6cf1495f7d0048606e54f4ea955195e06f0523068b49a9611b44", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/baseline_tests.rs", + "line_number": 23, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/exit_tests.rs", + "line_number": 32, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/exit_tests.rs", + "line_number": 145, + "finding_type": "AWS Access Key", + "matched_content_hash": "3f733150de7916d4778298d7f90493889c38b76876b80c058e439851ce60cb2b", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/exit_tests.rs", + "line_number": 164, + "finding_type": "Email Address", + "matched_content_hash": "fb050d7953f797a143e2098851ef2428b88397c27a6def4e6f4bef5baf4fb6b0", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 291, + "finding_type": "Email Address", + "matched_content_hash": "3fea8b798a88a2e9649a3ea443b8c944734f11128bbd451cd3dfde4c4eebe4b7", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 320, + "finding_type": "Email Address", + "matched_content_hash": "a925ccefb3339d83f6d9d23811f6c4a878eabf6df2a389a1c98fa13db0d336a6", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 346, + "finding_type": "Email Address", + "matched_content_hash": "79a3e36bd5010edb6d96aae4cbd22eced96e2991b2e1c1ae0bf2cdd5ab29c589", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 534, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "554736805d6e994bf786b21a30653266d29aa65775bbc17eb9f910d2556da91b", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 566, + "finding_type": "Random String", + "matched_content_hash": "4363c4decefe728cc24bed26b6280008f641dd2ae766ecbc77887df9a99d050a", + "plugin_name": "RandomString" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 635, + "finding_type": "Random String", + "matched_content_hash": "a4934d46da939da52885c506ac5465a46e416c40d2a58be0bb2f5d8fc748c48c", + "plugin_name": "RandomString" + }, + { + "file_path": "./tests/hooks_tests.rs", + "line_number": 796, + "finding_type": "AWS Access Key", + "matched_content_hash": "3f733150de7916d4778298d7f90493889c38b76876b80c058e439851ce60cb2b", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/report_tests.rs", + "line_number": 86, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "01c80fce098d3bb4634fe8110c31070d44fc509417b970a68733152f066a0089", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/run_cli_error_tests.rs", + "line_number": 9, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 34, + "finding_type": "Email Address", + "matched_content_hash": "07976d47040b0974eaade10d0da9a9253abeaa82912a2c18e744f0590c90f637", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 101, + "finding_type": "AWS Access Key", + "matched_content_hash": "3f733150de7916d4778298d7f90493889c38b76876b80c058e439851ce60cb2b", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 102, + "finding_type": "Password", + "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 102, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "f6bd37622d846ac435a7b7dcbde2347d59494dfd3c3a787604e8b91491a39c95", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 103, + "finding_type": "Email Address", + "matched_content_hash": "fb050d7953f797a143e2098851ef2428b88397c27a6def4e6f4bef5baf4fb6b0", + "plugin_name": "EmailDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 105, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "bcb5cfad271ffb8c4784236991957a45dfb2a0fba4b27e87951e76e78dcac232", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 105, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "c281ba6691c2e2772aebdbd6a7a05c869f7627d7c69eab07edfa130cee7bda70", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 105, + "finding_type": "SendGrid API Key", + "matched_content_hash": "324af8a2810ec90cd74766d9fa88ddb6957d311ffa77a7a4f1e533f230e6745c", + "plugin_name": "SendGridAPIKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 106, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "703296baa6f0ae75d7b4c6e41b9908603d1273c9db9a28d6fefd4e23d8f3c8b5", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 106, + "finding_type": "OpenAI API Key", + "matched_content_hash": "7a0d70456feea263871762c537e5e6141866b456eadc938eb9840e59eb08d1a9", + "plugin_name": "OpenAIAPIKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 106, + "finding_type": "Kimi/Moonshot API Key", + "matched_content_hash": "7a0d70456feea263871762c537e5e6141866b456eadc938eb9840e59eb08d1a9", + "plugin_name": "KimiMoonshotAPIKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 140, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "39336d01282aa7877cf262c7e980c64c3e6cbfa711933be49eb0bed651a12ce4", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 141, + "finding_type": "Stripe API Key", + "matched_content_hash": "c371dd8bd98e42a547fdc2378597152357423ba782bbb8ecbbafd0508ec8b825", + "plugin_name": "StripeAPIKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 175, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "8450e62d795b679b8f440e183ba0af22b8534d0cb06db8629d0b3ff98a030928", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 175, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "587ba5528b57a5d889136bbf1ea490763b8d652e61da4f5d78d8a1a81ba8edcb", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 176, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "6c6d557cc63eda114745424f991b2019fdfde2844412a1f58fb5527f0e0bef9f", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 210, + "finding_type": "Private Key Content", + "matched_content_hash": "b94c156724c086556087773aad998c0791dc79172caa4b3184a7330cfa7c1d76", + "plugin_name": "PrivateKeyContentDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 210, + "finding_type": "SSH Private Key", + "matched_content_hash": "678a65e8968aabff441076ae306e13d4fc85b1d36d8036a10d2100c1dc40d251", + "plugin_name": "SSHPrivateKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 210, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "4587ecda3342a38feff73eef527c2582add9f98a892b185b54b470bb91bc631e", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 212, + "finding_type": "SSH Private Key", + "matched_content_hash": "ca54604a9ed82ad96e5f5d68450002ef41112a7058469bbbdde94f83b267dca5", + "plugin_name": "SSHPrivateKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 213, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "131407c23861d5a739223f303ce0b62808fc0dabe0301622d9836dc3c4999fcb", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 246, + "finding_type": "Password", + "matched_content_hash": "a8a14cd33571f4a245d8f4e1b1c9f0ffd05d915db5573d1c66394e9540242d93", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 288, + "finding_type": "Password", + "matched_content_hash": "9f380d00b53e154b8421fb61924bbe35b6f01dccd608f5298f0e35764ee0b654", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 331, + "finding_type": "Password", + "matched_content_hash": "c505b74b9daf07fb40c38c619a4c0fdf5d807e85223142d6cd19923e7b84e15d", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 332, + "finding_type": "Password", + "matched_content_hash": "ca52bf35c9f7b4506aaee67e2aede5c5b09b9b258e850a74e7af0c078e932960", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 507, + "finding_type": "Password", + "matched_content_hash": "d1456485b2f315b81fcaffef101a4b398ea558b82bfe6ecee763ef7c327608e7", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 507, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "d1456485b2f315b81fcaffef101a4b398ea558b82bfe6ecee763ef7c327608e7", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 543, + "finding_type": "Password", + "matched_content_hash": "795c0808e6128ceee90a9dada1ad2ad243d07534def7532d5adf7e82a7237374", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 543, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "795c0808e6128ceee90a9dada1ad2ad243d07534def7532d5adf7e82a7237374", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 593, + "finding_type": "Password", + "matched_content_hash": "ead01d09cbf0176f24f604a4a4f955b32c34b155fe614e81ecc52f2c6fe89a3f", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 593, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "ead01d09cbf0176f24f604a4a4f955b32c34b155fe614e81ecc52f2c6fe89a3f", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 773, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "d72c959a2a51ac75ee2eb9db2b5d041f33ea32bea37515473e0487c2c3a3dce2", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 773, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "6d997bbfce7b131f5970fa00faeb56eaccc723239297811aa8c8770d11b7b19a", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 811, + "finding_type": "Voter ID (EPIC)", + "matched_content_hash": "e530110a549dd903837a0e2af06b406b01367b4572d07bffee9387334c027f20", + "plugin_name": "VoterIDDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 811, + "finding_type": "Voter ID (EPIC)", + "matched_content_hash": "8d6bf65189e5bdf52955b4a1592eb9c2fb0560709152317a1dabc36971c38f30", + "plugin_name": "VoterIDDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 846, + "finding_type": "PAN Card Number", + "matched_content_hash": "75d7a58ed9996980fee805fb623d81193cdb5e9efddf2fa12b595171ca16530e", + "plugin_name": "PANCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 846, + "finding_type": "PAN Card Number", + "matched_content_hash": "4dd22fad4bf96036ceea29ca253be8d72851cc289d7e694c9b208b7370812ee6", + "plugin_name": "PANCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 881, + "finding_type": "Credit Card Number", + "matched_content_hash": "25b618101c3bfdb4438894aa540e108cd150903bd24da57ac24f0d4174c0fe7e", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 881, + "finding_type": "Credit Card Number", + "matched_content_hash": "397d0bc36d77ccfc6f74c2eaa6615bb75c3fe3bdc3df26e6f53f413cfcdacdb3", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 881, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "881524bd6d254d60a115d72efeb134b7f82fe0fc9fa0f0feac806edce4f2889d", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 881, + "finding_type": "ABHA Health ID", + "matched_content_hash": "25b618101c3bfdb4438894aa540e108cd150903bd24da57ac24f0d4174c0fe7e", + "plugin_name": "ABHADetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 881, + "finding_type": "ABHA Health ID", + "matched_content_hash": "397d0bc36d77ccfc6f74c2eaa6615bb75c3fe3bdc3df26e6f53f413cfcdacdb3", + "plugin_name": "ABHADetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 917, + "finding_type": "Credit Card Number", + "matched_content_hash": "89deb2b58721d6303daa4a62f2dee30b9a388afb905fc4d3595ec88ca875fb15", + "plugin_name": "CreditCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 917, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "2a9d3ae086b4485e7a056b74ea2dc7ad45855144971e2fff00b402a8dd4244a0", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 917, + "finding_type": "Aadhaar Card Number", + "matched_content_hash": "8d7ab03162973f8bc2baf7c8556af862841d23c4d66f7f18f8cf5c5777f8b263", + "plugin_name": "AadhaarCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 917, + "finding_type": "PAN Card Number", + "matched_content_hash": "18fe123b5e00bbed7228c4ededc8a4e6e0bfd59169af97ec2e62b595eac11dde", + "plugin_name": "PANCardDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 917, + "finding_type": "ABHA Health ID", + "matched_content_hash": "89deb2b58721d6303daa4a62f2dee30b9a388afb905fc4d3595ec88ca875fb15", + "plugin_name": "ABHADetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1054, + "finding_type": "Google API Key", + "matched_content_hash": "aec6660855470791a5b4f5a6c1ac4b61e61d2b942b1cdec400fb7811743798d0", + "plugin_name": "GoogleAPIKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1054, + "finding_type": "Random String", + "matched_content_hash": "cbe1ce18b874bc08437699d863cd422c49e104462e7d5ac6bd390acc0d7c973a", + "plugin_name": "RandomString" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1112, + "finding_type": "Password", + "matched_content_hash": "67ef748345ad7f183084a7449ec05906e648c882400b9d940f2fadec23a7b197", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1242, + "finding_type": "AWS Access Key", + "matched_content_hash": "05c0aace2b76ca255ed3a7a953016d981477226dccc3b0e709d00174c8bc48b5", + "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1521, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" + } + ] +} \ No newline at end of file diff --git a/.keywatch.toml b/.keywatch.toml index eadce4b..fe17d2e 100644 --- a/.keywatch.toml +++ b/.keywatch.toml @@ -1,9 +1,10 @@ # KeyWatch self-scan configuration. # -# The test suite embeds fake example credentials as fixtures (the well-known -# AWS documentation example keys, dummy tokens in generated hook scripts). -# Exclude them so scanning this repository — including the pre-commit hook's -# staged scan — reports only real findings. -# detectors.toml is excluded because the detector definitions themselves are -# secret-shaped strings (pattern regexes, allowlist examples). -exclude = ["tests/*", "detectors.toml"] +# target/ is build output. detectors.toml is excluded because the detector +# definitions themselves are secret-shaped strings (pattern regexes, +# allowlist examples) and the file changes whenever a detector is added. +# +# Everything else — including the fake credentials used as test fixtures — is +# covered by .keywatch-baseline.json rather than path exclusions, so a newly +# introduced secret in those files is still reported. +exclude = ["target/**", "detectors.toml"] diff --git a/src/scanner.rs b/src/scanner.rs index b4598a8..f834a11 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -309,6 +309,9 @@ pub fn run_scan( .iter() .partition(|detector| detector.regex.as_str().contains("(?s)")); + // Resolved once: every scan mode must skip the baseline file itself. + let excluded_baseline = baseline_exclusion(args); + if args.git_history { let git_root = args .paths @@ -396,6 +399,7 @@ pub fn run_scan( let scan_result = scan_staged_diff( reader, &exclude_patterns, + excluded_baseline.as_ref(), &multiline_detectors, &line_detectors, ); @@ -460,7 +464,6 @@ pub fn run_scan( let unique_paths: Vec<_> = unique_paths.into_iter().collect(); let exclude_patterns = compile_exclude_patterns(args, config)?; - let excluded_baseline = baseline_exclusion(args); let line_scan_context = LineScanContext::new(&line_detectors); let results: Vec<(Vec, usize, usize, Option)> = unique_paths @@ -657,6 +660,7 @@ fn parse_binary_marker_path(marker: &str) -> String { fn scan_staged_diff( mut reader: ReaderType, exclude_patterns: &[Pattern], + excluded_baseline: Option<&PathBuf>, multiline_detectors: &[&Detector], line_detectors: &[&Detector], ) -> Result<(Vec, ScanMetadata), ScannerError> { @@ -747,7 +751,10 @@ fn scan_staged_diff( if let Some(target) = line.strip_prefix("+++ ") { current_path = match parse_diff_target_path(target) { - Some(path) if matches_exclude_patterns(&path, &[], exclude_patterns) => { + Some(path) + if matches_exclude_patterns(&path, &[], exclude_patterns) + || is_baseline_file(&path, excluded_baseline) => + { excluded_files.push(path); None } @@ -966,7 +973,7 @@ mod tests { +@@ SECRET_THREE looks like a hunk header\n"; let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); let summary: Vec<(String, usize)> = findings .iter() @@ -998,7 +1005,7 @@ mod tests { -goodbye\n"; let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); assert!(findings.is_empty(), "removed lines must not be scanned"); assert_eq!(metadata.files_scanned, 0); @@ -1020,7 +1027,7 @@ mod tests { +SECRET_B\n"; let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); let summary: Vec<(String, usize)> = findings .iter() @@ -1042,7 +1049,7 @@ mod tests { Binary files a/img.png and b/img.png differ\n"; let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); assert!(findings.is_empty()); assert_eq!( @@ -1064,7 +1071,8 @@ mod tests { diff.extend_from_slice(b"+caf\xE9 latin-1 line\n"); diff.extend_from_slice(b"+SECRET_AFTER_BINARYISH\n"); - let (findings, _) = scan_staged_diff(Cursor::new(diff), &[], &[], &line_detectors).unwrap(); + let (findings, _) = + scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); assert_eq!( findings.len(), @@ -1087,7 +1095,7 @@ mod tests { +END KEY\n"; let (findings, _) = - scan_staged_diff(Cursor::new(diff), &[], &multiline_detectors, &[]).unwrap(); + scan_staged_diff(Cursor::new(diff), &[], None, &multiline_detectors, &[]).unwrap(); assert_eq!(findings.len(), 1); assert_eq!( diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 30f1203..44ad85e 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -1706,3 +1706,49 @@ fn test_discovered_baseline_file_is_never_scanned() -> Result<(), String> { let _ = fs::remove_dir_all(&repo_dir); Ok(()) } + +#[test] +fn test_staged_scan_skips_the_baseline_file() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + // Committing a baseline must not trip the hook: its stored hashes are + // added lines in the staged diff and would otherwise be flagged. + let repo_dir = unique_temp_dir("staged_skips_baseline"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + fs::write( + repo_dir.join("secrets.txt"), + "aws_access_key_id = AKIAIOSFODNN7EXAMPLE\n", + ) + .map_err(|e| e.to_string())?; + + let run = |args: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(args) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&repo_dir) + .output() + .expect("run key-watch") + }; + + assert!(run(&["scan", ".", "--update-baseline"]).status.success()); + + let status = Command::new("git") + .args(["add", "secrets.txt", ".keywatch-baseline.json"]) + .current_dir(&repo_dir) + .status() + .map_err(|e| e.to_string())?; + assert!(status.success(), "git add should succeed"); + + let staged = run(&["scan", "--staged"]); + assert!( + matches!(staged.status.code(), Some(0)), + "staging the baseline must not fail the hook\nstdout:\n{}", + String::from_utf8_lossy(&staged.stdout) + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} From 76b1bc7aa66a760a92271b0ef8a2430a46d279a0 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:04:13 +0530 Subject: [PATCH 15/20] refactor: address review feedback in scanner and hooks - both git-backed scan modes share one scan_git_output helper that owns spawning, streaming, reaping and status checking, replacing the duplicated kill/wait/early-return blocks - the keyword automaton no longer expect()s: if it cannot be built every keyword detector runs unconditionally, which is slower but can never miss a secret - exclude patterns are collected by iterating the flag directly, so an absent --exclude reads as 'no patterns' rather than a defaulted value - hook scope labels come from a const fn and $HOME is resolved once per process instead of on every path render --- src/hooks.rs | 34 +++++----- src/scanner.rs | 177 ++++++++++++++++++++++++++----------------------- 2 files changed, 112 insertions(+), 99 deletions(-) diff --git a/src/hooks.rs b/src/hooks.rs index 3b65d68..7aa78a3 100644 --- a/src/hooks.rs +++ b/src/hooks.rs @@ -4,6 +4,7 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; +use std::sync::LazyLock; mod error; pub use error::HookError; @@ -19,12 +20,21 @@ fn shell_escape(input: &str) -> String { format!("'{}'", input.replace('\'', "'\"'\"'")) } +/// The user's home directory, resolved once per process. +static HOME_DIR: LazyLock> = LazyLock::new(|| { + env::var_os("HOME") + .or_else(|| env::var_os("USERPROFILE")) + .map(PathBuf::from) +}); + +/// `"global"` or `"local"`, for messages describing a hook's scope. +const fn scope_label(is_global: bool) -> &'static str { + if is_global { "global" } else { "local" } +} + /// Renders a path for terminal output, abbreviating the home directory as `~`. fn display_path(path: &Path) -> String { - let home = env::var_os("HOME") - .or_else(|| env::var_os("USERPROFILE")) - .map(PathBuf::from); - match home + match HOME_DIR .as_deref() .and_then(|home| path.strip_prefix(home).ok()) { @@ -138,13 +148,9 @@ pub fn install_hook(args: &HookInstallArgs) -> Result<(), HookError> { ); } - let scope = if install_target.is_global { - "global " - } else { - "" - }; println!( - "Installed {scope}{hook_type_str} hook at {}", + "Installed {} {hook_type_str} hook at {}", + scope_label(install_target.is_global), display_path(&install_target.path) ); println!( @@ -163,11 +169,7 @@ pub fn uninstall_hook(args: &HookUninstallArgs) -> Result<(), HookError> { let hook_type_str = args.hook_type.as_str(); let install_target = resolve_hook_uninstall_target(hook_type_str, args.global)?; - let scope = if install_target.is_global { - "global" - } else { - "local" - }; + let scope = scope_label(install_target.is_global); if !install_target.path.exists() { println!( "No {scope} {hook_type_str} hook found at {}", @@ -410,7 +412,7 @@ fn ensure_hook_target_is_keywatch_managed( return Ok(()); } - let scope = if is_global { "global" } else { "local" }; + let scope = scope_label(is_global); Err(HookError::RefuseExistingHook { action, scope, diff --git a/src/scanner.rs b/src/scanner.rs index f834a11..690f3dd 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -67,11 +67,14 @@ impl KeywordPrefilter { } } - let automaton = if patterns.is_empty() { - None - } else { - // Plain literal patterns cannot hit the automaton's size limits. - Some(AhoCorasick::new(&patterns).expect("build keyword automaton")) + // If the automaton cannot be built, every keyword detector runs + // unconditionally: slower, but it can never miss a secret. + let automaton = match AhoCorasick::new(&patterns) { + Ok(automaton) if !patterns.is_empty() => Some(automaton), + _ => { + unconditional = (0..line_detectors.len()).collect(); + None + } }; Self { @@ -91,8 +94,6 @@ impl KeywordPrefilter { candidates[detector_index] = true; } if let Some(automaton) = &self.automaton { - // Overlapping search: leftmost-first would report only one of two - // keywords sharing a prefix (e.g. "key" hides "keystone"). for keyword_match in automaton.find_overlapping_iter(lowered_line) { for &detector_index in &self.owners[keyword_match.pattern().as_usize()] { candidates[detector_index] = true; @@ -290,6 +291,37 @@ fn scan_stream( Ok((findings, total_lines)) } +/// Runs `command`, feeds its stdout to `scan`, and reaps the child process. +/// +/// Both git-backed scan modes share this so the process lifetime is handled +/// in exactly one place: on a scan error the child is killed rather than left +/// writing into a closed pipe, and it is always waited on before the status +/// is checked. +fn scan_git_output( + mut command: std::process::Command, + nonzero_status: ScannerError, + scan: impl FnOnce(BufReader) -> Result, + spawn_failed: impl FnOnce(std::io::Error) -> ScannerError, +) -> Result { + let mut child = command + .stdout(std::process::Stdio::piped()) + .spawn() + .map_err(spawn_failed)?; + + let stdout = child.stdout.take().ok_or(ScannerError::CaptureGitStdout)?; + let scanned = scan(BufReader::new(stdout)); + if scanned.is_err() { + let _ = child.kill(); + } + + let status = child + .wait() + .map_err(|source| ScannerError::GitProcess { source })?; + let scanned = scanned?; + + status.success().then_some(scanned).ok_or(nonzero_status) +} + pub fn run_scan( args: &ScanArgs, config: Option<&KeywatchConfig>, @@ -318,39 +350,33 @@ pub fn run_scan( .first() .map(Path::new) .unwrap_or_else(|| Path::new(".")); - let mut child = std::process::Command::new("git") - .current_dir(git_root) - .args([ - "-c", - "diff.external=", - "-c", - "color.ui=false", - "log", - "-p", - "-U0", - "--no-ext-diff", - "--no-textconv", - "--no-color", - ]) - .stdout(std::process::Stdio::piped()) - .spawn() - .map_err(|source| ScannerError::RunGitLog { source })?; - - let stdout = child.stdout.take().ok_or(ScannerError::CaptureGitStdout)?; - let reader = BufReader::new(stdout); - let (findings, total_lines) = scan_stream( - reader, - "", - &multiline_detectors, - &line_detectors, - )?; + let mut command = std::process::Command::new("git"); + command.current_dir(git_root).args([ + "-c", + "diff.external=", + "-c", + "color.ui=false", + "log", + "-p", + "-U0", + "--no-ext-diff", + "--no-textconv", + "--no-color", + ]); - let status = child - .wait() - .map_err(|source| ScannerError::GitProcess { source })?; - if !status.success() { - return Err(ScannerError::GitLogNonZero); - } + let (findings, total_lines) = scan_git_output( + command, + ScannerError::GitLogNonZero, + |reader| { + scan_stream( + reader, + "", + &multiline_detectors, + &line_detectors, + ) + }, + |source| ScannerError::RunGitLog { source }, + )?; let metadata = ScanMetadata { files_scanned: 1, @@ -389,33 +415,21 @@ pub fn run_scan( "--", ]); command.args(&args.paths); - let mut child = command - .stdout(std::process::Stdio::piped()) - .spawn() - .map_err(|source| ScannerError::RunGitDiff { source })?; - - let stdout = child.stdout.take().ok_or(ScannerError::CaptureGitStdout)?; - let reader = BufReader::new(stdout); - let scan_result = scan_staged_diff( - reader, - &exclude_patterns, - excluded_baseline.as_ref(), - &multiline_detectors, - &line_detectors, - ); - if scan_result.is_err() { - // Reap git instead of leaving it writing into a closed pipe. - let _ = child.kill(); - } - let wait_result = child.wait(); - - let (findings, metadata) = scan_result?; - let status = wait_result.map_err(|source| ScannerError::GitProcess { source })?; - if !status.success() { - return Err(ScannerError::GitDiffNonZero); - } - return Ok((findings, metadata)); + return scan_git_output( + command, + ScannerError::GitDiffNonZero, + |reader| { + scan_staged_diff( + reader, + &exclude_patterns, + excluded_baseline.as_ref(), + &multiline_detectors, + &line_detectors, + ) + }, + |source| ScannerError::RunGitDiff { source }, + ); } if args.stdin { @@ -545,25 +559,22 @@ fn compile_exclude_patterns( args: &ScanArgs, config: Option<&KeywatchConfig>, ) -> Result, ScannerError> { - let mut exclude_patterns: Vec = args + let mut exclude_patterns: Vec = Vec::new(); + + for pattern in args .exclude - .as_ref() - .map(|exclude_str| { - exclude_str - .split(',') - .filter(|pattern| !pattern.trim().is_empty()) - .map(|pattern| { - Pattern::new(pattern.trim()).map_err(|source| { - ScannerError::InvalidExcludePattern { - pattern: pattern.to_string(), - source, - } - }) - }) - .collect::, _>>() - }) - .transpose()? - .unwrap_or_default(); + .iter() + .flat_map(|patterns| patterns.split(',')) + .map(str::trim) + .filter(|pattern| !pattern.is_empty()) + { + exclude_patterns.push(Pattern::new(pattern).map_err(|source| { + ScannerError::InvalidExcludePattern { + pattern: pattern.to_string(), + source, + } + })?); + } if let Some(excludes) = config.and_then(|cfg| cfg.exclude.as_ref()) { for pattern_str in excludes { From e57d6490b99290293272fe1e092672cd1bf8fdf5 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:04:13 +0530 Subject: [PATCH 16/20] fix: do not panic when stdout is closed println! panics if the reader goes away, so 'key-watch scan . | head' aborted mid-report. Output now goes through a locked writer that treats a broken pipe as a normal end of output and propagates real I/O errors. --- src/lib.rs | 34 +++++++++++++++++++++++++--------- src/run_error.rs | 6 +++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e088b03..cfa411b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,6 +43,22 @@ pub fn run_cli() -> Result<(), RunCliError> { } } +/// Writes a line to stdout. +/// +/// `println!` panics if stdout is closed, which happens routinely when output +/// is piped (`key-watch scan . | head`). A closed pipe is a normal way for a +/// reader to stop listening, so it is reported as success; anything else is a +/// real I/O failure and is propagated. +fn emit(line: &str) -> Result<(), RunCliError> { + use std::io::Write; + + let mut stdout = std::io::stdout().lock(); + match writeln!(stdout, "{line}") { + Err(error) if error.kind() == std::io::ErrorKind::BrokenPipe => Ok(()), + other => other.map_err(|source| RunCliError::WriteOutput { source }), + } +} + fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { let start = Instant::now(); @@ -79,7 +95,7 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { let mut baseline = baseline::Baseline::load(std::path::Path::new(baseline_path))?; baseline.update_with_findings(&findings); baseline.save(std::path::Path::new(baseline_path))?; - println!("Baseline updated: {}", baseline_path); + emit(&format!("Baseline updated: {baseline_path}"))?; return Ok(()); } @@ -98,14 +114,15 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), RunCliError> { } .map_err(|source| RunCliError::ReportSerialize { source })?; - match findings_count { - _ if args.verbose => println!("{report_out}"), - 0 => println!("No secrets found."), - count => println!( + let summary = match findings_count { + _ if args.verbose => report_out.clone(), + 0 => "No secrets found.".to_string(), + count => format!( "WARNING: {} potential secret(s) detected (CRITICAL: {}, HIGH: {}, MEDIUM: {}, LOW: {})", count, severity_counts.0, severity_counts.1, severity_counts.2, severity_counts.3 ), - } + }; + emit(&summary)?; if let Some(ref output_path) = args.output { utils::write_to_file(output_path, &report_out).map_err(|source| { @@ -146,9 +163,8 @@ fn verify_binary_integrity() -> Result<(), RunCliError> { } } - println!("Binary integrity verified: {:?}", exe_path); - println!("Size: {} bytes", metadata.len()); - Ok(()) + emit(&format!("Binary integrity verified: {exe_path:?}"))?; + emit(&format!("Size: {} bytes", metadata.len())) } fn calculate_exit_code(findings: &[Finding], exit_mode: &ExitMode) -> i32 { diff --git a/src/run_error.rs b/src/run_error.rs index f77fce3..c0b7823 100644 --- a/src/run_error.rs +++ b/src/run_error.rs @@ -19,6 +19,7 @@ pub enum RunCliError { MissingBaselineForUpdate, ReportSerialize { source: serde_json::Error }, ReportWrite { path: String, source: io::Error }, + WriteOutput { source: io::Error }, ExecutablePath { source: io::Error }, ExecutableMetadata { source: io::Error }, } @@ -37,6 +38,9 @@ impl Display for RunCliError { Self::ReportSerialize { source } => { write!(formatter, "Failed to serialize report: {source}") } + Self::WriteOutput { source } => { + write!(formatter, "Failed to write output: {}", source) + } Self::ReportWrite { path, source } => { write!(formatter, "Failed to write report to '{path}': {source}") } @@ -60,7 +64,7 @@ impl StdError for RunCliError { Self::Hooks { source } => Some(source), Self::MissingBaselineForUpdate => None, Self::ReportSerialize { source } => Some(source), - Self::ReportWrite { source, .. } => Some(source), + Self::ReportWrite { source, .. } | Self::WriteOutput { source } => Some(source), Self::ExecutablePath { source } => Some(source), Self::ExecutableMetadata { source } => Some(source), } From 5cb7ccb9cbbe3608e38812a25110390d96c235e2 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:04:13 +0530 Subject: [PATCH 17/20] docs: condense the unreleased changelog to one line per change --- CHANGELOG.md | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 322ca39..7f70c1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,31 +6,27 @@ All notable changes to this project will be documented in this file. ### Added -- **Staged diff scanning** — `scan --staged [path...]` scans only the added lines of `git diff --cached`, attributing findings to real file paths (which makes `--baseline` and `--exclude` compose) and post-image line numbers +- `scan --staged` scans only the lines a commit adds +- Baselines are auto-discovered from `.keywatch-baseline.json`; `--no-baseline-discovery` opts out +- `update-baseline` workflow regenerates the baseline via a pull request -- **Baseline discovery** — without `--baseline`, a `.keywatch-baseline.json` is discovered by walking up from the scan target (bounded at the repository root or home directory), so hook scans use a committed repo baseline automatically; `--no-baseline-discovery` opts out and `--update-baseline` creates the conventional file when none exists. Baseline fingerprints ignore a leading `./` so `scan .` and `scan nested/file` entries match. A manually triggered `update-baseline` workflow regenerates the baseline via a reviewable pull request +### Changed + +- Pre-commit hooks scan the staged diff instead of whole files +- Config discovery searches parent directories up to the repository root +- Hook messages abbreviate the home directory as `~` ### Fixed -- `PasswordDetector` no longer flags `$PWD:` volume mounts and similar working-directory references (uppercase `PWD:` is allowlisted; lowercase `pwd = ...` assignments are still detected) -- `Base64Detector` entropy threshold raised from 3.0 to 4.2 so long CamelCase identifiers in ordinary source code no longer flood reports with LOW findings; real base64 payloads (entropy >= ~4.27) are still detected -- Staged and git-history scans now force `--no-color`, literal pathspecs, and standard `a/`/`b/` diff prefixes, so user git config (`color.ui = always`, `diff.mnemonicPrefix`, `diff.noprefix`, `core.quotePath`) can no longer break diff parsing — a `color.ui = always` config previously made `scan --staged` silently report zero findings -- Files git renders as binary (including text files marked `-diff` in `.gitattributes`) are now surfaced in the report's `excluded_files` instead of being silently treated as clean by `scan --staged` -- Non-UTF-8 staged content is decoded lossily instead of aborting the whole staged scan with an error -- The `--baseline` file is automatically excluded from scanning; previously its stored hashes were re-flagged as findings and every `--update-baseline` grew the file indefinitely +- `scan --staged` no longer misses findings under `color.ui = always` or custom diff prefixes +- Binary and non-UTF-8 files no longer pass silently or abort a staged scan +- The baseline file is no longer scanned as input to itself +- `Base64Detector` no longer flags long identifiers; `PasswordDetector` no longer flags `$PWD:` +- Piping output to a closed reader no longer panics ### Performance -- Detector keywords are lowercased once at construction and every line is lowercased once (not once per detector), and all keywords are matched in a single Aho-Corasick pass per line — directory scans ~2.9x faster, stdin/git-history streams ~9x faster on a 36 MB corpus, identical findings - -### Changed - -- Pre-commit hooks now run `key-watch scan --staged` instead of whole-file scans, so only the lines a commit adds are scanned and findings on unchanged lines no longer block commits -- Pre-commit `--exclude` patterns are now matched against staged file paths (forwarded to `scan --staged --exclude`) -- The `pre-commit` framework integration (`.pre-commit-hooks.yaml`) now runs `key-watch scan --staged`, so framework users get diff-based scanning instead of whole-file scans -- Config discovery now walks up parent directories from the scan target, so a repository-root `.keywatch.toml` applies to nested paths; the nearest config wins and the walk stops at the repository root or home directory so config outside the scanned tree is never trusted -- The `pre-commit` framework hooks no longer pass filenames (`scan --staged` already scans exactly the staged set, and pathspec-glob interpretation of literal filenames could skip files) -- Hook install/uninstall messages abbreviate the home directory as `~` +- Keyword matching uses a single Aho-Corasick pass per line: ~3x faster file scans, ~9x faster streams ## [2.0.1] - 2026-08-02 From 66f745c71eb7688682057e5254889feaa035f33d Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:12:21 +0530 Subject: [PATCH 18/20] fix: match baseline and exclude paths regardless of separator The Windows CI failures came from path spelling: 'scan .' records '.\secrets.txt' while the staged diff and explicit paths use forward slashes, so no baseline fingerprint ever matched and both baseline tests failed. Fingerprint paths now fold backslashes to forward slashes along with the leading './', which also makes a baseline generated on one platform usable on another. Exclude globs had the same gap: patterns are written with forward slashes, so 'target/**' never matched a scanned 'target\foo'. Paths are now matched in forward-slashed form. The new unit tests feed both spellings directly, so this stays covered on every platform rather than only on Windows CI. --- .keywatch-baseline.json | 7 +++++ src/baseline.rs | 69 ++++++++++++++++++++++++++++++++++++----- src/scanner.rs | 11 +++++-- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/.keywatch-baseline.json b/.keywatch-baseline.json index 71bc3c0..2d4fbf8 100644 --- a/.keywatch-baseline.json +++ b/.keywatch-baseline.json @@ -1407,6 +1407,13 @@ "finding_type": "AWS Access Key", "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./src/baseline.rs", + "line_number": 309, + "finding_type": "AWS Access Key", + "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", + "plugin_name": "AWSKeyDetector" } ] } \ No newline at end of file diff --git a/src/baseline.rs b/src/baseline.rs index 6eb764c..33767dd 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -47,14 +47,22 @@ struct BaselineFingerprint { plugin_name: String, } -/// Fingerprint paths ignore a leading `./` so `scan .` and -/// `scan nested/file` produce matching baseline entries. +/// Normalizes a path into the form stored in fingerprints: forward slashes +/// and no leading `./`. +/// +/// The same finding reaches the baseline spelled several ways — `scan .` +/// yields `./nested/file` (`.\nested\file` on Windows), an explicit path +/// yields what the user typed, and the staged diff always yields +/// forward-slashed repo-relative paths. Folding them together is what lets +/// one committed baseline serve every scan mode, and lets a baseline +/// generated on one platform work on another. fn normalize_fingerprint_path(path: &str) -> String { - let mut path = path; - while let Some(stripped) = path.strip_prefix("./") { - path = stripped; + let separators_folded = path.replace('\\', "/"); + let mut normalized = separators_folded.as_str(); + while let Some(stripped) = normalized.strip_prefix("./") { + normalized = stripped; } - path.to_string() + normalized.to_string() } impl BaselineFingerprint { @@ -261,7 +269,8 @@ impl Default for Baseline { #[cfg(test)] mod tests { - use super::hash_content; + use super::{Baseline, hash_content, normalize_fingerprint_path}; + use crate::report::Finding; #[test] fn hash_content_uses_expected_lowercase_hex() { @@ -270,4 +279,50 @@ mod tests { "49969dbf750f1c12188f4646dc9b5ff608ceb35d74e5de79fad99e88ebd445d6" ); } + + #[test] + fn normalize_fingerprint_path_folds_platform_spellings() { + // Windows path forms must fold to the same key as the staged diff's + // forward-slashed, repo-relative paths, or a baseline generated by + // `scan .` suppresses nothing on Windows. + for spelling in [ + "nested/config.txt", + "./nested/config.txt", + ".\\nested\\config.txt", + "nested\\config.txt", + ] { + assert_eq!( + normalize_fingerprint_path(spelling), + "nested/config.txt", + "unexpected normalization of {spelling:?}" + ); + } + } + + #[test] + fn baseline_matches_findings_across_path_spellings() { + let recorded = Finding { + file_path: ".\\secrets.txt".to_string(), + line_number: 1, + finding_type: "AWS".to_string(), + severity: crate::report::Severity::High, + matched_content: "AKIAIOSFODNN7EXAMPLE".to_string(), + plugin_name: "AWSKeyDetector".to_string(), + }; + let baseline = Baseline::from_findings(&[recorded]); + + let seen_again = Finding { + file_path: "secrets.txt".to_string(), + line_number: 9, + finding_type: "AWS".to_string(), + severity: crate::report::Severity::High, + matched_content: "AKIAIOSFODNN7EXAMPLE".to_string(), + plugin_name: "AWSKeyDetector".to_string(), + }; + + assert!( + baseline.filter_findings(vec![seen_again]).is_empty(), + "a baseline entry must suppress the same finding under any path spelling" + ); + } } diff --git a/src/scanner.rs b/src/scanner.rs index 690f3dd..a0e5acb 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -837,7 +837,11 @@ fn matches_exclude_patterns( scan_roots: &[Option], patterns: &[Pattern], ) -> bool { - let path = Path::new(path); + // Exclude patterns are written with forward slashes, so paths are matched + // in that form: on Windows a scanned path is `target\\foo` and would + // otherwise never match `target/**`. + let forward_slashed = path.replace('\\', "/"); + let path = Path::new(forward_slashed.as_str()); patterns.iter().any(|pattern| { pattern.matches_path(path) @@ -848,8 +852,9 @@ fn matches_exclude_patterns( || scan_roots.iter().any(|root_opt| { root_opt .as_deref() - .and_then(|root| path.strip_prefix(root).ok()) - .is_some_and(|relative| pattern.matches_path(relative)) + .map(|root| root.replace('\\', "/")) + .and_then(|root| path.strip_prefix(&root).ok().map(Path::to_path_buf)) + .is_some_and(|relative| pattern.matches_path(&relative)) }) }) } From c391a60a9d8627bc5461debf515b572e5772aa45 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:40:49 +0530 Subject: [PATCH 19/20] fix: close two detection bypasses found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A repository could disable the pre-commit hook entirely: a detectors.toml committed at its root replaces KeyWatch's detector set, and unlike pre-push the pre-commit template never passed --no-config-discovery. Reproduced end to end — a staged AWS secret was committed with no hook output. Both the generated hook and the pre-commit framework entries now use built-in detectors. Suppression still works through the baseline and inline markers, which the hook continues to honour, so this repository's own detectors.toml suppression moves from .keywatch.toml to the baseline. A '-diff' gitattribute made git emit only 'Binary files ... differ', so the added lines never reached the scanner and the file was reported clean. Undiffable paths are now read back from the index with git cat-file and scanned directly; genuinely binary blobs (NUL bytes) are still skipped, and excluded paths are not re-read. Base64Detector matched from 20 characters while its 4.2 entropy threshold drops 82% of random 20-character base64. Measurement shows base64 and CamelCase identifiers overlap in entropy below ~28 chars under every normalization tried, so the pattern floor moves to 28 where the two separate; known short credential formats have their own detectors with no entropy gate. Also fixes the CI push trigger, which pointed at 'main' while the default branch is 'master', so post-merge CI never ran and the Rust cache was never saved. --- .github/workflows/ci.yml | 2 +- .keywatch-baseline.json | 42 +++++++++++ .keywatch.toml | 14 ++-- .pre-commit-hooks.yaml | 4 +- CHANGELOG.md | 7 +- detectors.toml | 12 ++-- src/scanner.rs | 152 +++++++++++++++++++++++++++++++++------ templates/pre-commit.sh | 9 ++- tests/hooks_tests.rs | 22 +++--- tests/scanner_tests.rs | 90 +++++++++++++++++++++++ 10 files changed, 308 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02f7180..f92274e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI-push on: push: branches: - - main + - master merge_group: types: - checks_requested diff --git a/.keywatch-baseline.json b/.keywatch-baseline.json index 2d4fbf8..b250a80 100644 --- a/.keywatch-baseline.json +++ b/.keywatch-baseline.json @@ -1414,6 +1414,48 @@ "finding_type": "AWS Access Key", "matched_content_hash": "bc377e3c9554437ff10fd844193b505dced77a6af5a662e74232e814d284a1a2", "plugin_name": "AWSKeyDetector" + }, + { + "file_path": "./detectors.toml", + "line_number": 100, + "finding_type": "Certificate", + "matched_content_hash": "648ee3671e3bc408c3bce7d0ce237d9716da2c360a5c57c235a21067ac11eea9", + "plugin_name": "CertificateDetector" + }, + { + "file_path": "./detectors.toml", + "line_number": 249, + "finding_type": "Private Key Content", + "matched_content_hash": "f59444f5756fc6dd701c599452c1752e64a97ecb0447ce41a1e19eb15293ef9a", + "plugin_name": "PrivateKeyContentDetector" + }, + { + "file_path": "./detectors.toml", + "line_number": 249, + "finding_type": "SSH Private Key", + "matched_content_hash": "678a65e8968aabff441076ae306e13d4fc85b1d36d8036a10d2100c1dc40d251", + "plugin_name": "SSHPrivateKeyDetector" + }, + { + "file_path": "./detectors.toml", + "line_number": 666, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "e64a42fa536515c68483f9b16cae91558748e69c9e7119bc8f32964322a67fe2", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./detectors.toml", + "line_number": 666, + "finding_type": "Base64 Encoded String", + "matched_content_hash": "613dfb068a3be178fc4a3592d7233cdfb4f7005ab1582ec9676f875e15adcbb5", + "plugin_name": "Base64Detector" + }, + { + "file_path": "./tests/scanner_tests.rs", + "line_number": 1777, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "3490ca8089f67ef5e1c7455855266a1337d5817e69935e870adb73d049810429", + "plugin_name": "GenericKeyValueDetector" } ] } \ No newline at end of file diff --git a/.keywatch.toml b/.keywatch.toml index fe17d2e..7b0b776 100644 --- a/.keywatch.toml +++ b/.keywatch.toml @@ -1,10 +1,10 @@ # KeyWatch self-scan configuration. # -# target/ is build output. detectors.toml is excluded because the detector -# definitions themselves are secret-shaped strings (pattern regexes, -# allowlist examples) and the file changes whenever a detector is added. -# -# Everything else — including the fake credentials used as test fixtures — is -# covered by .keywatch-baseline.json rather than path exclusions, so a newly +# Only build output is excluded by path. Everything else — including the fake +# credentials used as test fixtures and the secret-shaped pattern strings in +# detectors.toml — is suppressed by .keywatch-baseline.json instead, so a newly # introduced secret in those files is still reported. -exclude = ["target/**", "detectors.toml"] +# +# Note this file is not consulted by the pre-commit hook, which runs with +# --no-config-discovery so that a repository cannot weaken its own scan. +exclude = ["target/**"] diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index a0bb054..596287b 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,7 +1,7 @@ - id: keywatch-scan name: KeyWatch scan description: Scan staged changes for secrets, API keys, tokens, and credentials - entry: key-watch scan --staged + entry: key-watch scan --staged --no-config-discovery language: rust files: '' pass_filenames: false @@ -11,7 +11,7 @@ - id: keywatch-scan-system name: KeyWatch scan (system) description: Scan staged changes for secrets, API keys, tokens, and credentials - entry: key-watch scan --staged + entry: key-watch scan --staged --no-config-discovery language: system files: '' pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f70c1f..6ada36f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,10 +18,13 @@ All notable changes to this project will be documented in this file. ### Fixed +- Hooks use built-in detectors, so a `detectors.toml` committed to a scanned repository can no longer replace the detector set and disable its own scan +- Files git renders as binary (including text marked `-diff` in `.gitattributes`) are read from the index instead of being reported clean +- `Base64Detector` matches from 28 characters, the length where entropy can actually separate base64 from identifiers - `scan --staged` no longer misses findings under `color.ui = always` or custom diff prefixes -- Binary and non-UTF-8 files no longer pass silently or abort a staged scan +- Non-UTF-8 files no longer abort a staged scan - The baseline file is no longer scanned as input to itself -- `Base64Detector` no longer flags long identifiers; `PasswordDetector` no longer flags `$PWD:` +- `PasswordDetector` no longer flags `$PWD:` - Piping output to a closed reader no longer panics ### Performance diff --git a/detectors.toml b/detectors.toml index 16bfb26..6e6034f 100644 --- a/detectors.toml +++ b/detectors.toml @@ -111,12 +111,16 @@ keywords = ["mysql://", "postgres://", "mongodb://", "redis://"] [[detectors]] name = "Base64Detector" -pattern = "\\b[A-Za-z0-9+/]{20,}[=]{0,2}\\b" +# 28 chars, not 20: below that, entropy cannot separate base64 from ordinary +# CamelCase identifiers (a random 20-char base64 string averages 4.04 bits, +# a 20-char identifier 3.68 — the distributions overlap), so a threshold low +# enough to catch short base64 floods reports with identifiers, and one high +# enough to reject identifiers drops 4 out of 5 short base64 strings. At 28+ +# the two separate cleanly. Known short credential formats are covered by +# their own detectors, which have no entropy gate. +pattern = "\\b[A-Za-z0-9+/]{28,}[=]{0,2}\\b" finding_type = "Base64 Encoded String" severity = "LOW" -# 4.2 sits between long CamelCase identifiers (measured <= ~4.13) and real -# base64 payloads (>= ~4.27), so ordinary source code no longer floods -# reports with LOW findings. entropy = 4.2 [[detectors]] diff --git a/src/scanner.rs b/src/scanner.rs index a0e5acb..6dc5591 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -416,7 +416,7 @@ pub fn run_scan( ]); command.args(&args.paths); - return scan_git_output( + let staged = scan_git_output( command, ScannerError::GitDiffNonZero, |reader| { @@ -429,7 +429,28 @@ pub fn run_scan( ) }, |source| ScannerError::RunGitDiff { source }, - ); + )?; + + let StagedScan { + mut findings, + mut metadata, + undiffable_files, + } = staged; + + let (blob_findings, blob_lines, skipped) = + scan_undiffable_blobs(&undiffable_files, &multiline_detectors, &line_detectors)?; + findings.extend(blob_findings); + metadata.total_lines += blob_lines; + metadata.files_scanned += undiffable_files.len() - skipped.len(); + metadata.excluded_files.extend(skipped); + + findings.sort_by(|a, b| { + a.file_path + .cmp(&b.file_path) + .then(a.line_number.cmp(&b.line_number)) + }); + + return Ok((findings, metadata)); } if args.stdin { @@ -674,12 +695,13 @@ fn scan_staged_diff( excluded_baseline: Option<&PathBuf>, multiline_detectors: &[&Detector], line_detectors: &[&Detector], -) -> Result<(Vec, ScanMetadata), ScannerError> { +) -> Result { let context = LineScanContext::new(line_detectors); let mut findings = Vec::new(); let mut total_lines = 0; let mut scanned_files: std::collections::BTreeSet = std::collections::BTreeSet::new(); let mut excluded_files: Vec = Vec::new(); + let mut undiffable_files: Vec = Vec::new(); let mut current_path: Option = None; let mut in_hunk = false; let mut next_line_number = 0; @@ -774,10 +796,18 @@ fn scan_staged_diff( continue; } - // A `-diff` gitattribute (or a true binary) yields no hunks; surface - // the skipped file instead of silently reporting it as clean. + // A `-diff` gitattribute (or a true binary) yields no hunks. The diff + // tells us nothing about the content, so record the path and read the + // staged blob directly rather than reporting the file as clean. if let Some(marker) = line.strip_prefix("Binary files ") { - excluded_files.push(parse_binary_marker_path(marker)); + let path = parse_binary_marker_path(marker); + if matches_exclude_patterns(&path, &[], exclude_patterns) + || is_baseline_file(&path, excluded_baseline) + { + excluded_files.push(path); + } else { + undiffable_files.push(path); + } } } @@ -801,7 +831,59 @@ fn scan_staged_diff( excluded_files, }; - Ok((findings, metadata)) + Ok(StagedScan { + findings, + metadata, + undiffable_files, + }) +} + +/// Result of parsing a staged diff. `undiffable_files` are paths git rendered +/// as binary (a real binary, or text marked `-diff` in .gitattributes); their +/// content never appears in the diff and must be read from the index instead. +struct StagedScan { + findings: Vec, + metadata: ScanMetadata, + undiffable_files: Vec, +} + +/// Scans the staged blob of each undiffable path via `git cat-file`. +/// +/// Without this a `.gitattributes` entry like `*.env -diff` would hide a +/// staged secret completely: git emits only "Binary files ... differ" and the +/// scan would report the file as clean. +fn scan_undiffable_blobs( + paths: &[String], + multiline_detectors: &[&Detector], + line_detectors: &[&Detector], +) -> Result<(Vec, usize, Vec), ScannerError> { + let context = LineScanContext::new(line_detectors); + let mut findings = Vec::new(); + let mut total_lines = 0; + let mut skipped = Vec::new(); + + for path in paths { + let output = std::process::Command::new("git") + .args(["cat-file", "blob", &format!(":{path}")]) + .output() + .map_err(|source| ScannerError::RunGitDiff { source })?; + if !output.status.success() { + skipped.push(path.clone()); + continue; + } + // Genuinely binary content (NUL bytes) is skipped, matching file mode. + if output.stdout.contains(&0) { + skipped.push(path.clone()); + continue; + } + let content = String::from_utf8_lossy(&output.stdout); + let (blob_findings, blob_lines) = + scan_content(&content, path, multiline_detectors, &context); + findings.extend(blob_findings); + total_lines += blob_lines; + } + + Ok((findings, total_lines, skipped)) } fn collect_files(dir_path: &str, files: &mut Vec<(String, Option)>, root: &str) { @@ -988,8 +1070,9 @@ mod tests { +++SECRET_TWO starts with pluses\n\ +@@ SECRET_THREE looks like a hunk header\n"; - let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); + let StagedScan { + findings, metadata, .. + } = scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); let summary: Vec<(String, usize)> = findings .iter() @@ -1020,8 +1103,9 @@ mod tests { -SECRET_GONE\n\ -goodbye\n"; - let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); + let StagedScan { + findings, metadata, .. + } = scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); assert!(findings.is_empty(), "removed lines must not be scanned"); assert_eq!(metadata.files_scanned, 0); @@ -1042,8 +1126,9 @@ mod tests { @@ -0,0 +2 @@\n\ +SECRET_B\n"; - let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); + let StagedScan { + findings, metadata, .. + } = scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); let summary: Vec<(String, usize)> = findings .iter() @@ -1057,21 +1142,46 @@ mod tests { } #[test] - fn test_scan_staged_diff_surfaces_binary_files_as_excluded() { + fn test_scan_staged_diff_queues_binary_files_for_blob_reading() { let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); let line_detectors = vec![&detector]; let diff = "diff --git a/img.png b/img.png\n\ index aabbcc0..ddeeff1 100644\n\ Binary files a/img.png and b/img.png differ\n"; - let (findings, metadata) = - scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); + let staged = scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); - assert!(findings.is_empty()); + assert!(staged.findings.is_empty()); + assert!( + staged.metadata.excluded_files.is_empty(), + "an undiffable file is not 'excluded' — its blob still gets scanned" + ); assert_eq!( - metadata.excluded_files, + staged.undiffable_files, vec!["img.png".to_string()], - "files git renders as binary must be surfaced, not silently clean" + "binary-rendered files must be queued for a direct blob read, or a \ + '-diff' gitattribute hides a staged secret entirely" + ); + } + + #[test] + fn test_scan_staged_diff_respects_excludes_for_binary_files() { + let detector = make_detector("Line", r"SECRET_\w+", "Test", "HIGH"); + let line_detectors = vec![&detector]; + let diff = "diff --git a/vendor/blob.bin b/vendor/blob.bin\n\ + Binary files a/vendor/blob.bin and b/vendor/blob.bin differ\n"; + let pattern = Pattern::new("vendor/**").unwrap(); + + let staged = + scan_staged_diff(Cursor::new(diff), &[pattern], None, &[], &line_detectors).unwrap(); + + assert!( + staged.undiffable_files.is_empty(), + "an excluded path must not be re-read from the index" + ); + assert_eq!( + staged.metadata.excluded_files, + vec!["vendor/blob.bin".to_string()] ); } @@ -1087,7 +1197,7 @@ mod tests { diff.extend_from_slice(b"+caf\xE9 latin-1 line\n"); diff.extend_from_slice(b"+SECRET_AFTER_BINARYISH\n"); - let (findings, _) = + let StagedScan { findings, .. } = scan_staged_diff(Cursor::new(diff), &[], None, &[], &line_detectors).unwrap(); assert_eq!( @@ -1110,7 +1220,7 @@ mod tests { +material\n\ +END KEY\n"; - let (findings, _) = + let StagedScan { findings, .. } = scan_staged_diff(Cursor::new(diff), &[], None, &multiline_detectors, &[]).unwrap(); assert_eq!(findings.len(), 1); diff --git a/templates/pre-commit.sh b/templates/pre-commit.sh index 95feb94..4d6a1f4 100644 --- a/templates/pre-commit.sh +++ b/templates/pre-commit.sh @@ -13,7 +13,14 @@ fi # scan --staged reads only the added lines of git diff --cached, so findings # on unchanged lines never block a commit. A git failure inside the scanner # exits with code 2, which fails the hook closed. -"$KEYWATCH_BIN" scan --staged --exclude "$EXCLUDE_PATTERNS" >/dev/null 2>&1 +# +# --no-config-discovery makes the hook use KeyWatch's built-in detectors, as +# the pre-push hook already does. Without it a detectors.toml committed to the +# scanned repository REPLACES the detector set, so a repository could disable +# secret detection for everyone who clones it. Suppress findings with a +# committed baseline or an inline keywatch:ignore marker, both of which the +# hook still honours. +"$KEYWATCH_BIN" scan --staged --no-config-discovery --exclude "$EXCLUDE_PATTERNS" >/dev/null 2>&1 EXIT_CODE=$? case $EXIT_CODE in 0) diff --git a/tests/hooks_tests.rs b/tests/hooks_tests.rs index 3f74125..8a42032 100644 --- a/tests/hooks_tests.rs +++ b/tests/hooks_tests.rs @@ -148,9 +148,14 @@ fn test_hook_generation_pre_commit() { "Should preserve comma-separated exclude patterns" ); assert!( - hook.contains("scan --staged --exclude \"$EXCLUDE_PATTERNS\""), + hook.contains("scan --staged") && hook.contains("--exclude \"$EXCLUDE_PATTERNS\""), "Should delegate staged-diff scanning and excludes to scan --staged" ); + assert!( + hook.contains("--no-config-discovery"), + "Hook must use built-in detectors so a scanned repository cannot \ + replace the detector set and disable its own scan" + ); assert!( hook.contains(">/dev/null 2>&1"), "Should suppress scanner output before concise failure" @@ -247,13 +252,14 @@ fn test_pre_commit_delegates_to_staged_scan() { let output = run_hook(&hook, &[], git_script, &keywatch_script, &temp_dir); assert!(output.status.success(), "clean staged scan should pass"); - assert_eq!( - fs::read_to_string(&marker) - .expect("read scanner args") - .trim_end(), - "scan --staged --exclude *.log,*.tmp", - "hook must delegate added-line extraction and excludes to scan --staged" - ); + let recorded = fs::read_to_string(&marker).expect("read scanner args"); + let recorded = recorded.trim_end(); + for expected in ["scan", "--staged", "--no-config-discovery", "*.log,*.tmp"] { + assert!( + recorded.contains(expected), + "hook invocation {recorded:?} should contain {expected:?}" + ); + } fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); } diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 44ad85e..9137372 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -1752,3 +1752,93 @@ fn test_staged_scan_skips_the_baseline_file() -> Result<(), String> { let _ = fs::remove_dir_all(&repo_dir); Ok(()) } + +#[test] +fn test_repo_detectors_toml_cannot_disable_hook_scan() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + // A repository that ships its own detectors.toml replaces the detector + // set. The hook runs with --no-config-discovery precisely so a scanned + // repository cannot switch off detection for everyone who clones it. + let repo_dir = unique_temp_dir("hostile_detectors_toml"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + fs::write( + repo_dir.join("detectors.toml"), + "[[detectors]]\nname = \"Noop\"\npattern = \"\\\\bqqqzzz1234\\\\b\"\n\ + finding_type = \"Noop\"\nseverity = \"LOW\"\n", + ) + .map_err(|e| e.to_string())?; + stage_file( + &repo_dir, + "leak.txt", + "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\n", + )?; + let status = Command::new("git") + .args(["add", "detectors.toml"]) + .current_dir(&repo_dir) + .status() + .map_err(|e| e.to_string())?; + assert!(status.success()); + + let run = |extra: &[&str]| { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "--staged", "--no-baseline-discovery"]) + .args(extra) + .current_dir(&repo_dir) + .output() + .expect("run key-watch") + }; + + assert_eq!( + run(&["--no-config-discovery"]).status.code(), + Some(1), + "with built-in detectors the staged secret must still be reported" + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} + +#[test] +fn test_staged_scan_reads_blobs_git_renders_as_binary() -> Result<(), String> { + if !git_available() { + return Ok(()); + } + + // `*.env -diff` makes git emit only "Binary files ... differ", so the + // added lines never appear in the diff. The scan must read the staged + // blob instead of reporting the file as clean. + let repo_dir = unique_temp_dir("staged_undiffable_blob"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + fs::write(repo_dir.join(".gitattributes"), "*.env -diff\n").map_err(|e| e.to_string())?; + stage_file( + &repo_dir, + "secrets.env", + "aws_access_key_id = AKIAIOSFODNN7EXAMPLE\n", + )?; + + let output = Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "--staged", "--no-baseline-discovery", "--verbose"]) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(&repo_dir) + .output() + .expect("run key-watch"); + let stdout = String::from_utf8_lossy(&output.stdout); + + assert_eq!( + output.status.code(), + Some(1), + "a '-diff' gitattribute must not hide a staged secret\nstdout:\n{stdout}" + ); + assert!( + stdout.contains("\"file_path\": \"secrets.env\""), + "the finding must be attributed to the real path\nstdout:\n{stdout}" + ); + + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} From 6508181e6954640ccf94cec36cb3b78fbdffc63e Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:37:31 +0530 Subject: [PATCH 20/20] fix(detectors): stop flagging code identifiers as credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GenericKeyValueDetector treats the value side's quotes as optional, so 'let payment_method_token = card_token;' matched as key = value with the identifier 'card_token' clearing the 2.5 entropy gate — reported HIGH on a line containing no literal at all. RandomString had the same shape, flagging quoted snake_case serde attributes. Both now allowlist an all-lowercase snake_case value. Quoted literals end in a quote and generated secrets carry digits or capitals, so 'api_key = "sk_live_..."', 'API_KEY=abc123def456789' and 'token = api_key_2024' all still report. --- .keywatch-baseline.json | 35 +++++++++++++++++++++++++++++++++++ CHANGELOG.md | 1 + detectors.toml | 10 ++++++++++ tests/detector_tests.rs | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+) diff --git a/.keywatch-baseline.json b/.keywatch-baseline.json index b250a80..2c82f2d 100644 --- a/.keywatch-baseline.json +++ b/.keywatch-baseline.json @@ -1456,6 +1456,41 @@ "finding_type": "Generic Key/Secret", "matched_content_hash": "3490ca8089f67ef5e1c7455855266a1337d5817e69935e870adb73d049810429", "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/detector_tests.rs", + "line_number": 259, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "78eb69b065478e9702683ad7731074c1b46c2ab9c59ef29e9a38754005970888", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/detector_tests.rs", + "line_number": 270, + "finding_type": "Stripe API Key", + "matched_content_hash": "f5e0f4b16462b5c0a472bc8650167795b846f2b7f825a6a3d9804a8aca2e7864", + "plugin_name": "StripeAPIKeyDetector" + }, + { + "file_path": "./tests/detector_tests.rs", + "line_number": 271, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "64ac92e0d5a31bef34682076a022571a87ebbaa61a1c74d7a43bbfa354087548", + "plugin_name": "GenericKeyValueDetector" + }, + { + "file_path": "./tests/detector_tests.rs", + "line_number": 272, + "finding_type": "Password", + "matched_content_hash": "4018438f1eaf55ea32baee0422d118935b7a6a4d0a9cb3ce86e41a5ec5827e4b", + "plugin_name": "PasswordDetector" + }, + { + "file_path": "./tests/detector_tests.rs", + "line_number": 273, + "finding_type": "Generic Key/Secret", + "matched_content_hash": "650dfe7ddce0129182c60029ba66733efa412ba0824523dd6b096482c4eb8ac9", + "plugin_name": "GenericKeyValueDetector" } ] } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ada36f..6a7ca7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. - `scan --staged` no longer misses findings under `color.ui = always` or custom diff prefixes - Non-UTF-8 files no longer abort a staged scan - The baseline file is no longer scanned as input to itself +- `GenericKeyValueDetector` and `RandomString` no longer flag code identifiers (`let payment_method_token = card_token`, snake_case serde attributes) - `PasswordDetector` no longer flags `$PWD:` - Piping output to a closed reader no longer panics diff --git a/detectors.toml b/detectors.toml index 6e6034f..5079d4e 100644 --- a/detectors.toml +++ b/detectors.toml @@ -94,6 +94,12 @@ finding_type = "Generic Key/Secret" severity = "HIGH" keywords = ["api_key", "secret", "password", "token", "access_key", "credential"] entropy = 2.5 +# An unquoted, all-lowercase snake_case value is a variable reference, not a +# credential: `let payment_method_token = card_token;` reads identically to +# `token = <10 random chars>` to the pattern above. Requiring no quotes, no +# digits and no capitals keeps real values flagged — quoted literals end in a +# quote, and generated secrets carry digits or mixed case. +allowlist = ["[:=]\\s*[a-z]+(?:_[a-z]+)+$"] [[detectors]] name = "CertificateDetector" @@ -157,6 +163,10 @@ pattern = "\"[a-zA-Z0-9\\-_=]{35,}\"" finding_type = "Random String" severity = "LOW" entropy = 3.5 +# Long all-lowercase snake_case string literals are identifiers — serde +# attributes, config keys, enum names — not random values. A generated secret +# of this length carries digits or capitals. +allowlist = ["^\"[a-z]+(?:_[a-z]+)+\"$"] [[detectors]] diff --git a/tests/detector_tests.rs b/tests/detector_tests.rs index 489a74a..5cc1b07 100644 --- a/tests/detector_tests.rs +++ b/tests/detector_tests.rs @@ -236,3 +236,42 @@ fn test_detector_init_error_duplicate_name_display() { assert_eq!(error.to_string(), "duplicate detector name 'Dup'"); } + +#[test] +fn test_generic_key_value_ignores_unquoted_identifier_assignments() { + let detectors = key_watch::detector::initialize_detectors().expect("load detectors"); + let generic = detectors + .iter() + .find(|d| d.name == "GenericKeyValueDetector") + .expect("GenericKeyValueDetector should exist"); + + let is_reported = |line: &str| { + generic.regex.find_iter(line).any(|m| { + !generic.allowlist.iter().any(|a| a.is_match(m.as_str())) + && generic.has_sufficient_entropy(m.as_str()) + }) + }; + + // Rust/Python/Go variable bindings are not credentials. + for code in [ + " let payment_method_token = card_token.clone();", + "let secret = client_secret;", + "token = payment_method_token", + ] { + assert!( + !is_reported(code), + "should not flag identifier assignment: {code}" + ); + } + + // Quoted literals, and unquoted values carrying digits or capitals, must + // still be reported. + for secret in [ + "api_key = \"sk_live_51abcdefghij\"", + "API_KEY=abc123def456789", + "password = \"hunter2hunter2\"", + "token = api_key_2024", + ] { + assert!(is_reported(secret), "should flag credential: {secret}"); + } +}