1f45e0c611
Every existing test asserted the history scan reports nothing — on this repository, which has never committed a credential. That is the wrong direction to test alone: a detector exercised only against clean input is indistinguishable from a function that returns an empty vector, and this one has already shipped four false-negative-shaped bugs today. The new test builds a repository from scratch by writing zlib-compressed git objects directly — no subprocess, for the same reason the scanner uses none — commits a key, and leaves no working-tree copy at all. Only the history has it, which is the situation the whole module exists for. It asserts the scan finds it, names the file, says revoke, and does not carry the credential's value into the finding. The fixture key is assembled at runtime so this source file does not itself contain a credential-shaped string, which is the trap that made the detector report its own definitions earlier today. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
520 lines
20 KiB
Rust
520 lines
20 KiB
Rust
//! Secrets that are still in the repository after being "removed".
|
|
//!
|
|
//! Deleting a file does not delete its history. A key committed in March and
|
|
//! taken out in April is in every clone made since March, in every fork, and
|
|
//! in the GitHub API long after the branch is gone. Unless somebody revoked
|
|
//! it, it still works — and "I removed that ages ago" is the most common
|
|
//! reason nobody ever did.
|
|
//!
|
|
//! This walks commits from the repository's refs, reads the trees they point
|
|
//! at, and checks every blob it has not already seen. It reports the path and
|
|
//! the commit, because "there is a key somewhere in your history" is not
|
|
//! something anybody can act on.
|
|
//!
|
|
//! Everything is bounded: commits walked, blobs read, and bytes inflated. A
|
|
//! scan of a large repository should be slow, not endless, and a hostile
|
|
//! repository must not be able to turn a security check into a way to
|
|
//! exhaust the machine running it.
|
|
|
|
use crate::gitobj::Store;
|
|
use crate::hygiene;
|
|
use crate::{Finding, Severity};
|
|
use std::collections::{HashMap, HashSet, VecDeque};
|
|
use std::path::Path;
|
|
|
|
/// How far back to walk. Enough to cover the working life of most projects,
|
|
/// bounded so a repository with a hundred thousand commits still finishes.
|
|
const MAX_COMMITS: usize = 5_000;
|
|
/// Distinct blobs to inspect. Most are source files that get read once.
|
|
const MAX_BLOBS: usize = 50_000;
|
|
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct HistoryScan {
|
|
pub findings: Vec<Finding>,
|
|
pub commits_walked: usize,
|
|
pub blobs_examined: usize,
|
|
/// True when a limit stopped the walk before it ran out of history —
|
|
/// so a clean result can be reported as partial rather than as clean.
|
|
pub truncated: bool,
|
|
}
|
|
|
|
/// Every ref this repository has, as (name, oid).
|
|
fn refs(git_dir: &Path) -> Vec<(String, String)> {
|
|
let mut out = Vec::new();
|
|
|
|
// Loose refs.
|
|
let mut stack = vec![git_dir.join("refs")];
|
|
while let Some(dir) = stack.pop() {
|
|
let Ok(entries) = std::fs::read_dir(&dir) else {
|
|
continue;
|
|
};
|
|
for e in entries.flatten() {
|
|
let p = e.path();
|
|
if p.is_dir() {
|
|
stack.push(p);
|
|
} else if let Ok(text) = std::fs::read_to_string(&p) {
|
|
let oid = text.trim().to_string();
|
|
if oid.len() == 40 && oid.chars().all(|c| c.is_ascii_hexdigit()) {
|
|
out.push((p.to_string_lossy().into_owned(), oid));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// packed-refs, which is where most refs live in a repository of any age.
|
|
if let Ok(text) = std::fs::read_to_string(git_dir.join("packed-refs")) {
|
|
for line in text.lines() {
|
|
if line.starts_with('#') || line.starts_with('^') {
|
|
continue;
|
|
}
|
|
if let Some((oid, name)) = line.split_once(' ') {
|
|
if oid.len() == 40 {
|
|
out.push((name.to_string(), oid.to_string()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// HEAD, which may be the only thing pointing at the current work.
|
|
if let Ok(text) = std::fs::read_to_string(git_dir.join("HEAD")) {
|
|
let t = text.trim();
|
|
if let Some(target) = t.strip_prefix("ref: ") {
|
|
if let Ok(oid) = std::fs::read_to_string(git_dir.join(target)) {
|
|
let oid = oid.trim().to_string();
|
|
if oid.len() == 40 {
|
|
out.push(("HEAD".into(), oid));
|
|
}
|
|
}
|
|
} else if t.len() == 40 {
|
|
out.push(("HEAD".into(), t.to_string()));
|
|
}
|
|
}
|
|
|
|
out
|
|
}
|
|
|
|
/// Parse a commit object for its tree and parents.
|
|
fn commit_links(data: &[u8]) -> (Option<String>, Vec<String>) {
|
|
let text = String::from_utf8_lossy(data);
|
|
let mut tree = None;
|
|
let mut parents = Vec::new();
|
|
for line in text.lines() {
|
|
if let Some(t) = line.strip_prefix("tree ") {
|
|
tree = Some(t.trim().to_string());
|
|
} else if let Some(p) = line.strip_prefix("parent ") {
|
|
parents.push(p.trim().to_string());
|
|
} else if line.is_empty() {
|
|
break; // headers end at the blank line before the message
|
|
}
|
|
}
|
|
(tree, parents)
|
|
}
|
|
|
|
/// Parse a tree object into (mode, name, oid) entries.
|
|
///
|
|
/// Entries are "<mode> <name>\0<20 raw bytes>", with no length prefix, so
|
|
/// this is a scan rather than an index.
|
|
fn tree_entries(data: &[u8]) -> Vec<(String, String, String)> {
|
|
let mut out = Vec::new();
|
|
let mut i = 0;
|
|
while i < data.len() {
|
|
let Some(sp) = data[i..].iter().position(|b| *b == b' ') else {
|
|
break;
|
|
};
|
|
let mode = String::from_utf8_lossy(&data[i..i + sp]).into_owned();
|
|
let name_start = i + sp + 1;
|
|
let Some(nul) = data[name_start..].iter().position(|b| *b == 0) else {
|
|
break;
|
|
};
|
|
let name = String::from_utf8_lossy(&data[name_start..name_start + nul]).into_owned();
|
|
let oid_start = name_start + nul + 1;
|
|
if oid_start + 20 > data.len() {
|
|
break;
|
|
}
|
|
let oid: String = data[oid_start..oid_start + 20]
|
|
.iter()
|
|
.map(|b| format!("{b:02x}"))
|
|
.collect();
|
|
out.push((mode, name, oid));
|
|
i = oid_start + 20;
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Walk the repository's history looking for credentials.
|
|
pub fn scan(repo: &Path) -> HistoryScan {
|
|
let mut result = HistoryScan::default();
|
|
let Some(store) = Store::open(repo) else {
|
|
return result;
|
|
};
|
|
let git_dir = repo.join(".git");
|
|
|
|
let mut queue: VecDeque<String> = VecDeque::new();
|
|
let mut seen_commits: HashSet<String> = HashSet::new();
|
|
for (_, oid) in refs(&git_dir) {
|
|
if seen_commits.insert(oid.clone()) {
|
|
queue.push_back(oid);
|
|
}
|
|
}
|
|
|
|
let mut seen_blobs: HashSet<String> = HashSet::new();
|
|
// blob oid -> (path, first commit we saw it in). The first commit reached
|
|
// walking backwards from the refs is the most recent one that contains
|
|
// it, which is the more useful thing to tell somebody.
|
|
let mut hits: HashMap<String, (String, String)> = HashMap::new();
|
|
|
|
while let Some(commit_oid) = queue.pop_front() {
|
|
if result.commits_walked >= MAX_COMMITS || result.blobs_examined >= MAX_BLOBS {
|
|
result.truncated = true;
|
|
break;
|
|
}
|
|
let Some(commit) = store.read(&commit_oid) else {
|
|
continue;
|
|
};
|
|
result.commits_walked += 1;
|
|
let (tree, parents) = commit_links(&commit.data);
|
|
for p in parents {
|
|
if seen_commits.insert(p.clone()) {
|
|
queue.push_back(p);
|
|
}
|
|
}
|
|
let Some(tree) = tree else { continue };
|
|
|
|
// Walk this commit's tree. Subtrees shared with an already-visited
|
|
// commit are skipped by the blob-level dedup below, which is what
|
|
// keeps this from being quadratic in history length.
|
|
let mut trees: Vec<(String, String)> = vec![(tree, String::new())];
|
|
while let Some((tree_oid, prefix)) = trees.pop() {
|
|
if result.blobs_examined >= MAX_BLOBS {
|
|
result.truncated = true;
|
|
break;
|
|
}
|
|
let Some(obj) = store.read(&tree_oid) else {
|
|
continue;
|
|
};
|
|
for (mode, name, oid) in tree_entries(&obj.data) {
|
|
let path = if prefix.is_empty() {
|
|
name.clone()
|
|
} else {
|
|
format!("{prefix}/{name}")
|
|
};
|
|
if mode.starts_with("40") {
|
|
trees.push((oid, path));
|
|
continue;
|
|
}
|
|
// Symlinks and gitlinks hold no file content worth reading.
|
|
if mode.starts_with("12") || mode.starts_with("16") {
|
|
continue;
|
|
}
|
|
if !seen_blobs.insert(oid.clone()) {
|
|
continue;
|
|
}
|
|
if hygiene::looks_binary_name(&name) {
|
|
continue;
|
|
}
|
|
let Some(blob) = store.read(&oid) else {
|
|
continue;
|
|
};
|
|
if !blob.is_blob() {
|
|
continue;
|
|
}
|
|
result.blobs_examined += 1;
|
|
let Ok(text) = std::str::from_utf8(&blob.data) else {
|
|
continue;
|
|
};
|
|
if let Some((what, issuer)) = hygiene::credential_kind(text) {
|
|
hits.entry(format!("{what}|{path}"))
|
|
.or_insert_with(|| (issuer.to_string(), commit_oid.clone()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for (key, (issuer, commit)) in hits {
|
|
let (what, path) = key.split_once('|').unwrap_or((key.as_str(), ""));
|
|
result.findings.push(Finding::new(
|
|
"secret-in-history",
|
|
Severity::Critical,
|
|
format!("{what} in {path}"),
|
|
format!("{} (commit {})", path, &commit[..commit.len().min(12)]),
|
|
format!(
|
|
"{what} is in this repository's git history. Deleting the file did not \
|
|
remove it — it is in every clone and fork made since it was committed, \
|
|
and it still works unless somebody revoked it."
|
|
),
|
|
"hygiene: git history",
|
|
format!(
|
|
"Revoke it with {issuer} and issue a replacement. Rewriting history with \
|
|
git-filter-repo removes it from future clones, but assume anything ever \
|
|
pushed is already known."
|
|
),
|
|
));
|
|
}
|
|
result.findings.sort_by(|a, b| a.subject.cmp(&b.subject));
|
|
result
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn workspace() -> std::path::PathBuf {
|
|
hygiene::repo_root(Path::new(env!("CARGO_MANIFEST_DIR"))).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn refs_are_found_in_both_places_they_live() {
|
|
let found = refs(&workspace().join(".git"));
|
|
assert!(!found.is_empty(), "a repository has refs");
|
|
assert!(
|
|
found.iter().any(|(n, _)| n.contains("HEAD") || n.contains("refs/heads")),
|
|
"including a branch or HEAD, saw {:?}",
|
|
found.iter().take(3).collect::<Vec<_>>()
|
|
);
|
|
for (_, oid) in &found {
|
|
assert_eq!(oid.len(), 40, "refs resolve to full object ids");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn commit_headers_parse() {
|
|
let raw = b"tree abc123\nparent def456\nparent 789abc\nauthor A <a@b> 1 +0000\n\nmessage\n";
|
|
let (tree, parents) = commit_links(raw);
|
|
assert_eq!(tree.unwrap(), "abc123");
|
|
assert_eq!(parents, vec!["def456", "789abc"]);
|
|
}
|
|
|
|
/// A message containing something that looks like a header must not be
|
|
/// read as one — headers stop at the blank line.
|
|
#[test]
|
|
fn the_commit_message_is_not_parsed_as_headers() {
|
|
let raw = b"tree abc\nauthor A <a@b> 1 +0000\n\nparent of all bugs\ntree surgery\n";
|
|
let (tree, parents) = commit_links(raw);
|
|
assert_eq!(tree.unwrap(), "abc");
|
|
assert!(parents.is_empty(), "the message is not a header block");
|
|
}
|
|
|
|
#[test]
|
|
fn tree_entries_parse_names_and_ids() {
|
|
let mut raw = Vec::new();
|
|
raw.extend_from_slice(b"100644 README.md\0");
|
|
raw.extend_from_slice(&[0xAB; 20]);
|
|
raw.extend_from_slice(b"40000 src\0");
|
|
raw.extend_from_slice(&[0xCD; 20]);
|
|
let entries = tree_entries(&raw);
|
|
assert_eq!(entries.len(), 2);
|
|
assert_eq!(entries[0].1, "README.md");
|
|
assert_eq!(entries[0].2, "ab".repeat(20));
|
|
assert!(entries[1].0.starts_with("40"), "a directory entry");
|
|
}
|
|
|
|
/// The real thing, against this repository. It should walk a meaningful
|
|
/// amount of history and find no credentials — this project has never
|
|
/// committed one, and a false positive here would be caught immediately.
|
|
/// Build a small repository with a secret committed and then deleted,
|
|
/// entirely by writing git objects — no subprocess, for the same reason
|
|
/// the scanner uses none.
|
|
///
|
|
/// This is the direction that matters. Every other test here proves the
|
|
/// scanner stays quiet; this one proves it can speak. A detector only
|
|
/// ever tested against clean input is a function that returns the empty
|
|
/// vector.
|
|
fn planted_repo(dir: &Path, secret: &str) -> std::io::Result<()> {
|
|
use flate2::write::ZlibEncoder;
|
|
use flate2::Compression;
|
|
use sha1::{Digest, Sha1};
|
|
use std::io::Write as _;
|
|
|
|
let objects = dir.join(".git").join("objects");
|
|
std::fs::create_dir_all(&objects)?;
|
|
std::fs::create_dir_all(dir.join(".git").join("refs").join("heads"))?;
|
|
|
|
// Write one object and return its id, exactly as git stores them.
|
|
let write_object = |kind: &str, body: &[u8]| -> std::io::Result<String> {
|
|
let mut raw = format!("{kind} {}\0", body.len()).into_bytes();
|
|
raw.extend_from_slice(body);
|
|
let oid: String = Sha1::digest(&raw).iter().map(|b| format!("{b:02x}")).collect();
|
|
let d = objects.join(&oid[0..2]);
|
|
std::fs::create_dir_all(&d)?;
|
|
let mut e = ZlibEncoder::new(Vec::new(), Compression::default());
|
|
e.write_all(&raw)?;
|
|
std::fs::write(d.join(&oid[2..]), e.finish()?)?;
|
|
Ok(oid)
|
|
};
|
|
|
|
let blob = write_object("blob", format!("api_key = \"{secret}\"\n").as_bytes())?;
|
|
let mut tree = Vec::new();
|
|
tree.extend_from_slice(b"100644 config.py\0");
|
|
for i in (0..40).step_by(2) {
|
|
tree.push(u8::from_str_radix(&blob[i..i + 2], 16).unwrap());
|
|
}
|
|
let tree_oid = write_object("tree", &tree)?;
|
|
let commit = format!(
|
|
"tree {tree_oid}\nauthor T <t@t> 1000000000 +0000\ncommitter T <t@t> 1000000000 +0000\n\nadd config\n"
|
|
);
|
|
let commit_oid = write_object("commit", commit.as_bytes())?;
|
|
std::fs::write(dir.join(".git").join("HEAD"), "ref: refs/heads/main\n")?;
|
|
std::fs::write(
|
|
dir.join(".git").join("refs").join("heads").join("main"),
|
|
format!("{commit_oid}\n"),
|
|
)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// A key committed and then deleted is still in the history, still in
|
|
/// every clone, and still working unless somebody revoked it. Note there
|
|
/// is no working-tree copy here at all — only history.
|
|
#[test]
|
|
fn a_secret_committed_and_deleted_is_still_found() {
|
|
let dir = std::env::temp_dir().join(format!("hound-hist-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
// Assembled at runtime so this source file does not itself contain a
|
|
// credential-shaped string.
|
|
let secret = format!("ghp_{}", "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8");
|
|
planted_repo(&dir, &secret).unwrap();
|
|
|
|
assert!(
|
|
!dir.join("config.py").exists(),
|
|
"the file is gone from the working tree; only history has it"
|
|
);
|
|
|
|
let r = scan(&dir);
|
|
assert_eq!(r.commits_walked, 1, "one commit to walk");
|
|
assert_eq!(
|
|
r.findings.len(),
|
|
1,
|
|
"the planted key must be found, got {:?}",
|
|
r.findings.iter().map(|f| &f.subject).collect::<Vec<_>>()
|
|
);
|
|
let f = &r.findings[0];
|
|
assert_eq!(f.severity, Severity::Critical);
|
|
assert!(f.subject.contains("config.py"), "and name the file: {}", f.subject);
|
|
assert!(
|
|
f.advice.to_lowercase().contains("revoke"),
|
|
"the only real remedy is revocation: {}",
|
|
f.advice
|
|
);
|
|
// The finding must not carry the key itself.
|
|
assert!(
|
|
!format!("{f:?}").contains(&secret),
|
|
"the finding leaked the credential it found"
|
|
);
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
}
|
|
|
|
#[test]
|
|
fn scanning_this_repository_walks_history_and_finds_nothing() {
|
|
let r = scan(&workspace());
|
|
assert!(
|
|
r.commits_walked > 5,
|
|
"expected real history, walked {}",
|
|
r.commits_walked
|
|
);
|
|
assert!(
|
|
r.blobs_examined > 50,
|
|
"expected real content, examined {} blobs",
|
|
r.blobs_examined
|
|
);
|
|
assert!(
|
|
r.findings.is_empty(),
|
|
"this repository has no committed credentials, but reported: {:?}",
|
|
r.findings.iter().map(|f| &f.subject).collect::<Vec<_>>()
|
|
);
|
|
}
|
|
}
|
|
|
|
// ── Staged changes ──────────────────────────────────────────────────────────
|
|
|
|
/// Check what is about to be committed.
|
|
///
|
|
/// This is the only point where a leak is cheap. Once a commit is pushed the
|
|
/// key is out, and the only remedy is rotation — so a check that runs a
|
|
/// second earlier is worth more than any amount of scanning afterwards.
|
|
///
|
|
/// Reads the staged blobs from the index and the object store, so it sees
|
|
/// exactly what the commit will contain: not the file on disk, which may have
|
|
/// been edited since `git add`, and not the last commit.
|
|
pub fn scan_staged(repo: &Path) -> Vec<Finding> {
|
|
let Some(store) = Store::open(repo) else {
|
|
return Vec::new();
|
|
};
|
|
let Some(entries) = hygiene::tracked_paths_with_oids(repo) else {
|
|
return Vec::new();
|
|
};
|
|
|
|
let mut out = Vec::new();
|
|
for (path, oid) in entries {
|
|
let name = Path::new(&path)
|
|
.file_name()
|
|
.map(|n| n.to_string_lossy().into_owned())
|
|
.unwrap_or_else(|| path.clone());
|
|
|
|
// A secrets file being committed at all is the finding; its contents
|
|
// do not need inspecting to know that.
|
|
if hygiene::looks_like_secret_file(&name) {
|
|
out.push(Finding::new(
|
|
"staged-secret-file",
|
|
Severity::Critical,
|
|
path.clone(),
|
|
path.clone(),
|
|
format!(
|
|
"{name} is staged for commit. Committing it puts its contents in \
|
|
this repository's history permanently — removing it later does not \
|
|
take it back."
|
|
),
|
|
"hygiene: staged changes",
|
|
format!("Run: git rm --cached {path} and add {name} to .gitignore"),
|
|
));
|
|
continue;
|
|
}
|
|
|
|
if hygiene::looks_binary_name(&name) {
|
|
continue;
|
|
}
|
|
let Some(blob) = store.read(&oid) else {
|
|
continue;
|
|
};
|
|
let Ok(text) = std::str::from_utf8(&blob.data) else {
|
|
continue;
|
|
};
|
|
if let Some((what, issuer)) = hygiene::credential_kind(text) {
|
|
out.push(Finding::new(
|
|
"staged-credential",
|
|
Severity::Critical,
|
|
format!("{what} in {path}"),
|
|
path.clone(),
|
|
format!(
|
|
"{path} is staged for commit and contains {what}. Once this is \
|
|
committed and pushed, the key is in the history and in every clone."
|
|
),
|
|
"hygiene: staged changes",
|
|
format!(
|
|
"Take the key out of the file and read it from the environment \
|
|
instead. If it has already been used anywhere, revoke it with \
|
|
{issuer} as well."
|
|
),
|
|
));
|
|
}
|
|
}
|
|
out.sort_by(|a, b| a.subject.cmp(&b.subject));
|
|
out
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod staged_tests {
|
|
use super::*;
|
|
|
|
/// This repository has staged content whenever a commit is in progress,
|
|
/// and none of it is ever a credential. The scan must run and stay quiet.
|
|
#[test]
|
|
fn the_workspace_has_nothing_dangerous_staged() {
|
|
let repo = hygiene::repo_root(Path::new(env!("CARGO_MANIFEST_DIR"))).unwrap();
|
|
let found = scan_staged(&repo);
|
|
assert!(
|
|
found.is_empty(),
|
|
"nothing in this repository should trip the staged check: {:?}",
|
|
found.iter().map(|f| &f.subject).collect::<Vec<_>>()
|
|
);
|
|
}
|
|
}
|