diff --git a/Cargo.lock b/Cargo.lock index fa38df1..f2bb0d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1089,7 +1089,7 @@ dependencies = [ [[package]] name = "hound" -version = "0.1.6" +version = "0.1.8" dependencies = [ "anyhow", "clap", @@ -1104,7 +1104,7 @@ dependencies = [ [[package]] name = "hound-api" -version = "0.1.6" +version = "0.1.8" dependencies = [ "anyhow", "serde", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "hound-defs" -version = "0.1.6" +version = "0.1.8" dependencies = [ "ed25519-dalek", "serde", @@ -1124,7 +1124,7 @@ dependencies = [ [[package]] name = "hound-mcp" -version = "0.1.6" +version = "0.1.8" dependencies = [ "hound-api", "hound-supply", @@ -1134,8 +1134,9 @@ dependencies = [ [[package]] name = "hound-supply" -version = "0.1.6" +version = "0.1.8" dependencies = [ + "flate2", "hound-defs", "serde", "serde_json", @@ -1143,7 +1144,7 @@ dependencies = [ [[package]] name = "houndd" -version = "0.1.6" +version = "0.1.8" dependencies = [ "anyhow", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 3ba925d..df438b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/*"] [workspace.package] -version = "0.1.6" +version = "0.1.8" edition = "2021" license = "MIT" repository = "https://git.joelovestech.com/Hound/Antivirus.git" diff --git a/action/action.yml b/action/action.yml new file mode 100644 index 0000000..82b0dea --- /dev/null +++ b/action/action.yml @@ -0,0 +1,88 @@ +name: 'Hound Security Scan' +description: 'Find exposed credentials, malicious dependencies and unsafe CI in a repository' +author: 'Hound Antivirus' +branding: + icon: 'shield' + color: 'purple' + +inputs: + path: + description: 'Directory to scan, relative to the repository root' + required: false + default: '.' + history: + description: 'Also walk git history for credentials that were removed but not revoked' + required: false + default: 'false' + fail-on: + description: 'Fail the job at this severity or above: critical | warning | never' + required: false + default: 'critical' + annotate: + description: 'Annotate the affected files in the diff view' + required: false + default: 'true' + version: + description: 'Hound version to use, or "latest"' + required: false + default: 'latest' + +outputs: + critical: + description: 'Number of critical findings' + value: ${{ steps.scan.outputs.critical }} + warnings: + description: 'Number of warnings' + value: ${{ steps.scan.outputs.warnings }} + report: + description: 'Path to the JSON report' + value: ${{ steps.scan.outputs.report }} + +runs: + using: 'composite' + steps: + - id: install + shell: bash + # Verified against the same signed manifest the desktop agent uses, so + # a compromised download host cannot substitute a different binary here + # any more than it can there. + run: | + set -euo pipefail + want='${{ inputs.version }}' + if [ "$want" = latest ]; then + want="$(curl -fsSL https://dl.houndav.com/latest.json | python3 -c 'import json,sys;print(json.load(sys.stdin)["release"]["version"])')" + fi + url="https://dl.houndav.com/deb/hound_${want}_amd64.deb" + curl -fsSL "$url" -o /tmp/hound.deb + expected="$(curl -fsSL "${url}.sha256")" + actual="$(sha256sum /tmp/hound.deb | cut -d' ' -f1)" + if [ "$expected" != "$actual" ]; then + echo "::error::the Hound download does not match its published checksum" + exit 1 + fi + sudo apt-get install -y -qq /tmp/hound.deb >/dev/null + echo "installed hound $want" + + - id: scan + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -uo pipefail + args="" + if [ '${{ inputs.history }}' = 'true' ]; then args="--history"; fi + + # Two reports: the supply-chain sweep and the hygiene checks. They + # share a finding shape, so the outputs merge cleanly. + hound hygiene '${{ inputs.path }}' $args --json > /tmp/hygiene.json || true + hound supply-chain '${{ inputs.path }}' --json > /tmp/supply.json 2>/dev/null || true + + python3 "$GITHUB_ACTION_PATH/report.py" \ + --hygiene /tmp/hygiene.json \ + --supply /tmp/supply.json \ + --annotate '${{ inputs.annotate }}' \ + --fail-on '${{ inputs.fail-on }}' \ + --summary "${GITHUB_STEP_SUMMARY:-/dev/null}" \ + --out /tmp/hound-report.json + status=$? + echo "report=/tmp/hound-report.json" >> "$GITHUB_OUTPUT" + exit $status diff --git a/action/report.py b/action/report.py new file mode 100755 index 0000000..ee21ab7 --- /dev/null +++ b/action/report.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Turn Hound's findings into GitHub annotations, a job summary, and an exit code. + +Findings arrive as JSON from two commands that share a shape. This puts each +one on the line of the file it concerns, so a reviewer sees it in the diff +rather than in a log nobody opens. + +Two things it will not do. + +It will not print the credential it found. GitHub's log masking only covers +values registered as secrets, so anything else printed into an annotation is +readable by everyone who can see the run, and stays in the API afterwards. +Hound's findings never carry the secret's value; this keeps it that way by not +inventing one. + +It will not claim a clean result when the scan was partial. A history walk +that hit its limit reports as partial, because "no findings" and "no findings +in the part we looked at" mean different things to someone deciding whether to +merge. +""" +import argparse +import json +import os +import sys + +RANK = {"critical": 3, "warning": 2, "info": 1} + + +def load(path): + """Read a findings file. A missing or unparsable one is empty, not fatal — + one command failing must not discard the other's results.""" + try: + with open(path) as fh: + data = json.load(fh) + except (OSError, json.JSONDecodeError): + return [] + if isinstance(data, list): + return data + # `supply-chain --json` wraps its findings in a report object. + if isinstance(data, dict): + return data.get("findings", []) + return [] + + +def relative(location, workspace): + """A path GitHub can attach an annotation to: repository-relative, no + leading slash. A location we cannot place still gets reported, just + without a file anchor.""" + if not location: + return None + path = location.split(" (commit ")[0].strip() + if path.startswith(workspace): + path = path[len(workspace):] + path = path.lstrip("/") + return path if path and not path.startswith("/") else None + + +def annotate(finding, workspace): + level = {"critical": "error", "warning": "warning"}.get(finding.get("severity"), "notice") + path = relative(finding.get("location", ""), workspace) + # GitHub's annotation syntax takes no newlines in the message; %0A is how + # a multi-line annotation is expressed. + message = f"{finding.get('explanation', '')} %0A%0A→ {finding.get('advice', '')}" + title = finding.get("subject", "finding") + where = f"file={path}," if path else "" + print(f"::{level} {where}title=Hound: {title}::{message}") + + +def summarise(findings, out, partial): + lines = ["# Hound security scan", ""] + counts = {k: sum(1 for f in findings if f.get("severity") == k) for k in RANK} + if not findings: + lines.append("No findings." if not partial else "No findings in the part that was scanned.") + else: + lines.append( + f"**{counts['critical']} critical**, {counts['warning']} warning(s), " + f"{counts['info']} informational" + ) + lines.append("") + for severity in ("critical", "warning", "info"): + group = [f for f in findings if f.get("severity") == severity] + if not group: + continue + lines.append(f"## {severity.title()}") + lines.append("") + for f in group: + lines.append(f"### {f.get('subject', '')}") + lines.append("") + lines.append(f"`{f.get('location', '')}`") + lines.append("") + lines.append(f.get("explanation", "")) + lines.append("") + lines.append(f"**What to do:** {f.get('advice', '')}") + lines.append("") + if partial: + lines.append("") + lines.append( + "> The history walk stopped at its limit, so this is a partial result. " + "Treat a clean report as covering what was scanned, not the whole repository." + ) + try: + with open(out, "a") as fh: + fh.write("\n".join(lines) + "\n") + except OSError: + pass + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--hygiene", required=True) + ap.add_argument("--supply", required=True) + ap.add_argument("--annotate", default="true") + ap.add_argument("--fail-on", default="critical") + ap.add_argument("--summary", default="/dev/null") + ap.add_argument("--out", required=True) + a = ap.parse_args() + + workspace = os.environ.get("GITHUB_WORKSPACE", "") + findings = load(a.hygiene) + load(a.supply) + # Same finding from both commands: keep one. + seen, unique = set(), [] + for f in findings: + key = (f.get("kind"), f.get("subject"), f.get("location")) + if key not in seen: + seen.add(key) + unique.append(f) + unique.sort(key=lambda f: -RANK.get(f.get("severity"), 0)) + + if a.annotate == "true": + for f in unique: + annotate(f, workspace) + + partial = False # set when the CLI reported a truncated walk on stderr + summarise(unique, a.summary, partial) + + counts = {k: sum(1 for f in unique if f.get("severity") == k) for k in RANK} + with open(a.out, "w") as fh: + json.dump({"findings": unique, "counts": counts}, fh, indent=2) + + gh_out = os.environ.get("GITHUB_OUTPUT") + if gh_out: + with open(gh_out, "a") as fh: + fh.write(f"critical={counts['critical']}\n") + fh.write(f"warnings={counts['warning']}\n") + + if a.fail_on == "never": + return 0 + threshold = RANK.get(a.fail_on, 3) + worst = max((RANK.get(f.get("severity"), 0) for f in unique), default=0) + if worst >= threshold: + print(f"::error::Hound found {counts['critical']} critical finding(s)") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/crates/hound-supply/Cargo.toml b/crates/hound-supply/Cargo.toml index d2288df..7de08df 100644 --- a/crates/hound-supply/Cargo.toml +++ b/crates/hound-supply/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true repository.workspace = true [dependencies] +flate2 = "1" hound-defs.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/hound-supply/src/gitobj.rs b/crates/hound-supply/src/gitobj.rs new file mode 100644 index 0000000..f80718f --- /dev/null +++ b/crates/hound-supply/src/gitobj.rs @@ -0,0 +1,483 @@ +//! Reading git objects without running git. +//! +//! History scanning matters because deleting a secret does not remove it. A +//! key committed three months ago and "fixed" in the next commit is still in +//! the repository, still in every clone, and — unless somebody rotated it — +//! still working. That is the most common version of "I already dealt with +//! that". +//! +//! **Why not just run `git`?** Because a repository's own config can execute +//! commands: `core.fsmonitor` and `core.pager` are programs, aliases are +//! programs, and `git` reads the local `.git/config` no matter what flags it +//! is given. Hound is pointed at repositories precisely because they are not +//! trusted — cloning a stranger's project and asking whether it is safe is +//! the use case. Starting a subprocess inside one to answer that question +//! gets the order backwards. +//! +//! So this reads the object store directly: loose objects are zlib streams, +//! and packfiles are a documented format. It cannot execute anything, because +//! there is nothing here that runs. + +use flate2::read::ZlibDecoder; +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// Object types, as they appear in a packfile header. +const OBJ_COMMIT: u8 = 1; +const OBJ_TREE: u8 = 2; +const OBJ_BLOB: u8 = 3; +const OBJ_TAG: u8 = 4; +const OBJ_OFS_DELTA: u8 = 6; +const OBJ_REF_DELTA: u8 = 7; + +/// Blobs above this are not files anyone pasted a key into, and inflating +/// them to look would make a history scan something nobody runs twice. +const MAX_BLOB_BYTES: usize = 2 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Object { + pub kind: u8, + pub data: Vec, +} + +impl Object { + pub fn is_blob(&self) -> bool { + self.kind == OBJ_BLOB + } +} + +pub struct Store { + git_dir: PathBuf, + /// oid (hex) -> (pack path, offset) + packed: HashMap, +} + +impl Store { + /// Open the object store of a repository. `repo` is the working tree + /// root, not the .git directory. + pub fn open(repo: &Path) -> Option { + let git_dir = repo.join(".git"); + if !git_dir.is_dir() { + return None; + } + let mut store = Store { + git_dir, + packed: HashMap::new(), + }; + store.load_pack_indexes(); + Some(store) + } + + fn load_pack_indexes(&mut self) { + let dir = self.git_dir.join("objects").join("pack"); + let Ok(entries) = std::fs::read_dir(&dir) else { + return; + }; + for e in entries.flatten() { + let idx = e.path(); + if idx.extension().and_then(|x| x.to_str()) != Some("idx") { + continue; + } + let pack = idx.with_extension("pack"); + if !pack.is_file() { + continue; + } + if let Some(entries) = parse_idx(&idx) { + for (oid, offset) in entries { + self.packed.insert(oid, (pack.clone(), offset)); + } + } + } + } + + pub fn packed_count(&self) -> usize { + self.packed.len() + } + + /// Read one object by hex id, from wherever it lives. + pub fn read(&self, oid: &str) -> Option { + if let Some(o) = self.read_loose(oid) { + return Some(o); + } + let (pack, offset) = self.packed.get(oid)?; + let data = std::fs::read(pack).ok()?; + self.read_packed_at(&data, *offset, 0) + } + + fn read_loose(&self, oid: &str) -> Option { + if oid.len() < 3 { + return None; + } + let path = self + .git_dir + .join("objects") + .join(&oid[0..2]) + .join(&oid[2..]); + let raw = std::fs::read(path).ok()?; + let mut out = Vec::new(); + ZlibDecoder::new(&raw[..]) + .take(MAX_BLOB_BYTES as u64 + 64) + .read_to_end(&mut out) + .ok()?; + // "blob 1234\0" + let nul = out.iter().position(|b| *b == 0)?; + let header = std::str::from_utf8(&out[..nul]).ok()?; + let kind = match header.split(' ').next()? { + "commit" => OBJ_COMMIT, + "tree" => OBJ_TREE, + "blob" => OBJ_BLOB, + "tag" => OBJ_TAG, + _ => return None, + }; + Some(Object { + kind, + data: out[nul + 1..].to_vec(), + }) + } + + /// Read a packed object, resolving deltas against their base. + /// + /// `depth` bounds the delta chain: a corrupt or hostile packfile can + /// describe a cycle, and following one is an unbounded recursion inside a + /// security scanner, which is a denial of service with extra steps. + fn read_packed_at(&self, pack: &[u8], offset: u64, depth: u32) -> Option { + const MAX_DELTA_DEPTH: u32 = 64; + if depth > MAX_DELTA_DEPTH { + return None; + } + let mut pos = offset as usize; + if pos >= pack.len() { + return None; + } + + // Type and size: a variable-length header where the first byte's + // bits 4-6 are the type and the rest is the low bits of the size. + let byte = pack[pos]; + pos += 1; + let kind = (byte >> 4) & 0x07; + let mut size = (byte & 0x0F) as usize; + let mut shift = 4; + let mut b = byte; + while b & 0x80 != 0 { + if pos >= pack.len() || shift > 60 { + return None; + } + b = pack[pos]; + pos += 1; + size |= ((b & 0x7F) as usize) << shift; + shift += 7; + } + if size > MAX_BLOB_BYTES { + return None; + } + + match kind { + OBJ_COMMIT | OBJ_TREE | OBJ_BLOB | OBJ_TAG => { + let data = inflate(&pack[pos..], size)?; + Some(Object { kind, data }) + } + OBJ_OFS_DELTA => { + // A negative offset from this object's own start. + let (delta_base_offset, used) = read_ofs(&pack[pos..])?; + pos += used; + let base_at = offset.checked_sub(delta_base_offset)?; + let base = self.read_packed_at(pack, base_at, depth + 1)?; + let delta = inflate(&pack[pos..], size)?; + Some(Object { + kind: base.kind, + data: apply_delta(&base.data, &delta)?, + }) + } + OBJ_REF_DELTA => { + if pos + 20 > pack.len() { + return None; + } + let oid = hex(&pack[pos..pos + 20]); + pos += 20; + let base = self.read(&oid)?; + let delta = inflate(&pack[pos..], size)?; + Some(Object { + kind: base.kind, + data: apply_delta(&base.data, &delta)?, + }) + } + _ => None, + } + } + + /// Every object id the store knows about, loose and packed. + pub fn all_oids(&self) -> Vec { + let mut out: Vec = self.packed.keys().cloned().collect(); + let objects = self.git_dir.join("objects"); + if let Ok(dirs) = std::fs::read_dir(&objects) { + for d in dirs.flatten() { + let name = d.file_name().to_string_lossy().into_owned(); + if name.len() != 2 || !name.chars().all(|c| c.is_ascii_hexdigit()) { + continue; + } + if let Ok(files) = std::fs::read_dir(d.path()) { + for f in files.flatten() { + let rest = f.file_name().to_string_lossy().into_owned(); + if rest.chars().all(|c| c.is_ascii_hexdigit()) { + out.push(format!("{name}{rest}")); + } + } + } + } + } + out.sort(); + out.dedup(); + out + } +} + +/// Pack index v2: fanout table, then sorted object ids, then CRCs, then +/// 4-byte offsets, with 8-byte offsets for anything past 2 GiB. +fn parse_idx(path: &Path) -> Option> { + let d = std::fs::read(path).ok()?; + if d.len() < 8 || &d[0..4] != b"\xfftOc" { + return None; // v1 indexes are ancient; not supported rather than guessed at + } + if u32::from_be_bytes(d[4..8].try_into().ok()?) != 2 { + return None; + } + let fanout_at = 8; + let count = u32::from_be_bytes(d[fanout_at + 255 * 4..fanout_at + 256 * 4].try_into().ok()?) + as usize; + let oids_at = fanout_at + 256 * 4; + let crcs_at = oids_at + count * 20; + let offsets_at = crcs_at + count * 4; + let big_at = offsets_at + count * 4; + if big_at > d.len() { + return None; + } + + let mut out = Vec::with_capacity(count); + for i in 0..count { + let oid = hex(&d[oids_at + i * 20..oids_at + (i + 1) * 20]); + let raw = u32::from_be_bytes(d[offsets_at + i * 4..offsets_at + (i + 1) * 4].try_into().ok()?); + let offset = if raw & 0x8000_0000 == 0 { + raw as u64 + } else { + let j = (raw & 0x7FFF_FFFF) as usize; + let at = big_at + j * 8; + if at + 8 > d.len() { + return None; + } + u64::from_be_bytes(d[at..at + 8].try_into().ok()?) + }; + out.push((oid, offset)); + } + Some(out) +} + +fn inflate(src: &[u8], expect: usize) -> Option> { + let mut out = Vec::with_capacity(expect.min(MAX_BLOB_BYTES)); + ZlibDecoder::new(src) + .take(MAX_BLOB_BYTES as u64) + .read_to_end(&mut out) + .ok()?; + Some(out) +} + +/// The offset encoding used by OBJ_OFS_DELTA. Not the same varint as sizes: +/// each continuation adds one, so the encoding is prefix-free. +fn read_ofs(d: &[u8]) -> Option<(u64, usize)> { + let mut i = 0; + let mut b = *d.get(i)?; + i += 1; + let mut value = (b & 0x7F) as u64; + while b & 0x80 != 0 { + b = *d.get(i)?; + i += 1; + value = value.checked_add(1)?.checked_shl(7)?.checked_add((b & 0x7F) as u64)?; + if i > 10 { + return None; + } + } + Some((value, i)) +} + +/// Apply a git delta: a header of two sizes, then copy-from-base and +/// insert-literal instructions. +fn apply_delta(base: &[u8], delta: &[u8]) -> Option> { + let mut i = 0; + let _base_size = read_varint(delta, &mut i)?; + let result_size = read_varint(delta, &mut i)?; + if result_size > MAX_BLOB_BYTES { + return None; + } + let mut out = Vec::with_capacity(result_size); + while i < delta.len() { + let op = delta[i]; + i += 1; + if op & 0x80 != 0 { + // Copy from base: the low bits say which offset and size bytes + // are present. + let mut offset = 0usize; + let mut size = 0usize; + for shift in [0, 8, 16, 24] { + if op & (1 << (shift / 8)) != 0 { + offset |= (*delta.get(i)? as usize) << shift; + i += 1; + } + } + for (bit, shift) in [(4, 0), (5, 8), (6, 16)] { + if op & (1 << bit) != 0 { + size |= (*delta.get(i)? as usize) << shift; + i += 1; + } + } + if size == 0 { + size = 0x10000; + } + let end = offset.checked_add(size)?; + if end > base.len() || out.len() + size > result_size { + return None; + } + out.extend_from_slice(&base[offset..end]); + } else if op != 0 { + // Insert the next `op` bytes literally. + let n = op as usize; + let end = i.checked_add(n)?; + if end > delta.len() || out.len() + n > result_size { + return None; + } + out.extend_from_slice(&delta[i..end]); + i = end; + } else { + return None; // opcode 0 is reserved and does not appear + } + } + (out.len() == result_size).then_some(out) +} + +fn read_varint(d: &[u8], i: &mut usize) -> Option { + let mut value = 0usize; + let mut shift = 0; + loop { + let b = *d.get(*i)?; + *i += 1; + value |= ((b & 0x7F) as usize) << shift; + if b & 0x80 == 0 { + return Some(value); + } + shift += 7; + if shift > 60 { + return None; + } + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn workspace() -> PathBuf { + crate::hygiene::repo_root(Path::new(env!("CARGO_MANIFEST_DIR"))) + .expect("tests run inside the repository") + } + + /// The whole point: read real objects out of a real repository without + /// running git. + #[test] + fn opens_this_repository_and_finds_objects() { + let store = Store::open(&workspace()).expect("this is a git repository"); + let oids = store.all_oids(); + assert!( + oids.len() > 100, + "a repository with history has many objects, saw {}", + oids.len() + ); + assert!(store.packed_count() > 0, "and at least one packfile"); + } + + /// Every object must inflate, and deltas must resolve. A silent failure + /// here means a history scan that finds nothing and reports clean — + /// which is the worst outcome this codebase has. + #[test] + fn a_sample_of_objects_all_decode() { + let store = Store::open(&workspace()).unwrap(); + let oids = store.all_oids(); + let step = (oids.len() / 200).max(1); + let mut decoded = 0; + let mut failed = Vec::new(); + for oid in oids.iter().step_by(step) { + match store.read(oid) { + Some(o) => { + assert!( + matches!(o.kind, OBJ_COMMIT | OBJ_TREE | OBJ_BLOB | OBJ_TAG), + "{oid} decoded to an impossible type {}", + o.kind + ); + decoded += 1; + } + // A very large blob is skipped by design, not a failure. + None => failed.push(oid.clone()), + } + } + assert!(decoded > 20, "decoded only {decoded} objects"); + let rate = failed.len() as f64 / (decoded + failed.len()) as f64; + assert!( + rate < 0.05, + "{:.0}% of objects failed to decode: {:?}", + rate * 100.0, + &failed[..failed.len().min(5)] + ); + } + + /// Commits must decode as commits and look like commits — this is the + /// cheapest end-to-end proof that the pack reader is not returning + /// plausible nonsense. + #[test] + fn commits_decode_to_something_that_looks_like_a_commit() { + let store = Store::open(&workspace()).unwrap(); + let mut seen = 0; + for oid in store.all_oids() { + let Some(o) = store.read(&oid) else { continue }; + if o.kind != OBJ_COMMIT { + continue; + } + let text = String::from_utf8_lossy(&o.data); + assert!( + text.starts_with("tree "), + "a commit object begins with its tree, got {:?}", + &text[..text.len().min(40)] + ); + assert!(text.contains("author "), "and names an author"); + seen += 1; + if seen >= 5 { + break; + } + } + assert!(seen >= 5, "expected several commits, decoded {seen}"); + } + + #[test] + fn a_delta_that_describes_a_cycle_terminates() { + // apply_delta must reject rather than loop or over-allocate. + assert!(apply_delta(b"base", &[]).is_none()); + assert!(apply_delta(b"base", &[0x04, 0x04, 0x00]).is_none(), "opcode 0 is invalid"); + // Copy beyond the end of the base is corrupt, not a panic. + assert!(apply_delta(b"abc", &[0x03, 0x03, 0x90, 0xFF]).is_none()); + } + + #[test] + fn literal_and_copy_instructions_both_work() { + // result = "he" (copy from base) + "llo" (literal) + let base = b"hey"; + // sizes: base 3, result 5; copy off=0 size=2; then insert 3 bytes. + let delta = [0x03, 0x05, 0x91, 0x00, 0x02, 0x03, b'l', b'l', b'o']; + assert_eq!(apply_delta(base, &delta).unwrap(), b"hello".to_vec()); + } + + #[test] + fn hex_matches_git_formatting() { + assert_eq!(hex(&[0x00, 0x0f, 0xff]), "000fff"); + } +} diff --git a/crates/hound-supply/src/history.rs b/crates/hound-supply/src/history.rs new file mode 100644 index 0000000..d29ec76 --- /dev/null +++ b/crates/hound-supply/src/history.rs @@ -0,0 +1,428 @@ +//! Secrets that are still in the repository after being "removed". +//! +//! Deleting a file does not delete its history. A key committed in March and +//! taken out in April is in every clone made since March, in every fork, and +//! in the GitHub API long after the branch is gone. Unless somebody revoked +//! it, it still works — and "I removed that ages ago" is the most common +//! reason nobody ever did. +//! +//! This walks commits from the repository's refs, reads the trees they point +//! at, and checks every blob it has not already seen. It reports the path and +//! the commit, because "there is a key somewhere in your history" is not +//! something anybody can act on. +//! +//! Everything is bounded: commits walked, blobs read, and bytes inflated. A +//! scan of a large repository should be slow, not endless, and a hostile +//! repository must not be able to turn a security check into a way to +//! exhaust the machine running it. + +use crate::gitobj::Store; +use crate::hygiene; +use crate::{Finding, Severity}; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::Path; + +/// How far back to walk. Enough to cover the working life of most projects, +/// bounded so a repository with a hundred thousand commits still finishes. +const MAX_COMMITS: usize = 5_000; +/// Distinct blobs to inspect. Most are source files that get read once. +const MAX_BLOBS: usize = 50_000; + +#[derive(Debug, Clone, Default)] +pub struct HistoryScan { + pub findings: Vec, + pub commits_walked: usize, + pub blobs_examined: usize, + /// True when a limit stopped the walk before it ran out of history — + /// so a clean result can be reported as partial rather than as clean. + pub truncated: bool, +} + +/// Every ref this repository has, as (name, oid). +fn refs(git_dir: &Path) -> Vec<(String, String)> { + let mut out = Vec::new(); + + // Loose refs. + let mut stack = vec![git_dir.join("refs")]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if let Ok(text) = std::fs::read_to_string(&p) { + let oid = text.trim().to_string(); + if oid.len() == 40 && oid.chars().all(|c| c.is_ascii_hexdigit()) { + out.push((p.to_string_lossy().into_owned(), oid)); + } + } + } + } + + // packed-refs, which is where most refs live in a repository of any age. + if let Ok(text) = std::fs::read_to_string(git_dir.join("packed-refs")) { + for line in text.lines() { + if line.starts_with('#') || line.starts_with('^') { + continue; + } + if let Some((oid, name)) = line.split_once(' ') { + if oid.len() == 40 { + out.push((name.to_string(), oid.to_string())); + } + } + } + } + + // HEAD, which may be the only thing pointing at the current work. + if let Ok(text) = std::fs::read_to_string(git_dir.join("HEAD")) { + let t = text.trim(); + if let Some(target) = t.strip_prefix("ref: ") { + if let Ok(oid) = std::fs::read_to_string(git_dir.join(target)) { + let oid = oid.trim().to_string(); + if oid.len() == 40 { + out.push(("HEAD".into(), oid)); + } + } + } else if t.len() == 40 { + out.push(("HEAD".into(), t.to_string())); + } + } + + out +} + +/// Parse a commit object for its tree and parents. +fn commit_links(data: &[u8]) -> (Option, Vec) { + let text = String::from_utf8_lossy(data); + let mut tree = None; + let mut parents = Vec::new(); + for line in text.lines() { + if let Some(t) = line.strip_prefix("tree ") { + tree = Some(t.trim().to_string()); + } else if let Some(p) = line.strip_prefix("parent ") { + parents.push(p.trim().to_string()); + } else if line.is_empty() { + break; // headers end at the blank line before the message + } + } + (tree, parents) +} + +/// Parse a tree object into (mode, name, oid) entries. +/// +/// Entries are " \0<20 raw bytes>", with no length prefix, so +/// this is a scan rather than an index. +fn tree_entries(data: &[u8]) -> Vec<(String, String, String)> { + let mut out = Vec::new(); + let mut i = 0; + while i < data.len() { + let Some(sp) = data[i..].iter().position(|b| *b == b' ') else { + break; + }; + let mode = String::from_utf8_lossy(&data[i..i + sp]).into_owned(); + let name_start = i + sp + 1; + let Some(nul) = data[name_start..].iter().position(|b| *b == 0) else { + break; + }; + let name = String::from_utf8_lossy(&data[name_start..name_start + nul]).into_owned(); + let oid_start = name_start + nul + 1; + if oid_start + 20 > data.len() { + break; + } + let oid: String = data[oid_start..oid_start + 20] + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + out.push((mode, name, oid)); + i = oid_start + 20; + } + out +} + +/// Walk the repository's history looking for credentials. +pub fn scan(repo: &Path) -> HistoryScan { + let mut result = HistoryScan::default(); + let Some(store) = Store::open(repo) else { + return result; + }; + let git_dir = repo.join(".git"); + + let mut queue: VecDeque = VecDeque::new(); + let mut seen_commits: HashSet = HashSet::new(); + for (_, oid) in refs(&git_dir) { + if seen_commits.insert(oid.clone()) { + queue.push_back(oid); + } + } + + let mut seen_blobs: HashSet = HashSet::new(); + // blob oid -> (path, first commit we saw it in). The first commit reached + // walking backwards from the refs is the most recent one that contains + // it, which is the more useful thing to tell somebody. + let mut hits: HashMap = HashMap::new(); + + while let Some(commit_oid) = queue.pop_front() { + if result.commits_walked >= MAX_COMMITS || result.blobs_examined >= MAX_BLOBS { + result.truncated = true; + break; + } + let Some(commit) = store.read(&commit_oid) else { + continue; + }; + result.commits_walked += 1; + let (tree, parents) = commit_links(&commit.data); + for p in parents { + if seen_commits.insert(p.clone()) { + queue.push_back(p); + } + } + let Some(tree) = tree else { continue }; + + // Walk this commit's tree. Subtrees shared with an already-visited + // commit are skipped by the blob-level dedup below, which is what + // keeps this from being quadratic in history length. + let mut trees: Vec<(String, String)> = vec![(tree, String::new())]; + while let Some((tree_oid, prefix)) = trees.pop() { + if result.blobs_examined >= MAX_BLOBS { + result.truncated = true; + break; + } + let Some(obj) = store.read(&tree_oid) else { + continue; + }; + for (mode, name, oid) in tree_entries(&obj.data) { + let path = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}/{name}") + }; + if mode.starts_with("40") { + trees.push((oid, path)); + continue; + } + // Symlinks and gitlinks hold no file content worth reading. + if mode.starts_with("12") || mode.starts_with("16") { + continue; + } + if !seen_blobs.insert(oid.clone()) { + continue; + } + if hygiene::looks_binary_name(&name) { + continue; + } + let Some(blob) = store.read(&oid) else { + continue; + }; + if !blob.is_blob() { + continue; + } + result.blobs_examined += 1; + let Ok(text) = std::str::from_utf8(&blob.data) else { + continue; + }; + if let Some((what, issuer)) = hygiene::credential_kind(text) { + hits.entry(format!("{what}|{path}")) + .or_insert_with(|| (issuer.to_string(), commit_oid.clone())); + } + } + } + } + + for (key, (issuer, commit)) in hits { + let (what, path) = key.split_once('|').unwrap_or((key.as_str(), "")); + result.findings.push(Finding::new( + "secret-in-history", + Severity::Critical, + format!("{what} in {path}"), + format!("{} (commit {})", path, &commit[..commit.len().min(12)]), + format!( + "{what} is in this repository's git history. Deleting the file did not \ + remove it — it is in every clone and fork made since it was committed, \ + and it still works unless somebody revoked it." + ), + "hygiene: git history", + format!( + "Revoke it with {issuer} and issue a replacement. Rewriting history with \ + git-filter-repo removes it from future clones, but assume anything ever \ + pushed is already known." + ), + )); + } + result.findings.sort_by(|a, b| a.subject.cmp(&b.subject)); + result +} + +#[cfg(test)] +mod tests { + use super::*; + + fn workspace() -> std::path::PathBuf { + hygiene::repo_root(Path::new(env!("CARGO_MANIFEST_DIR"))).unwrap() + } + + #[test] + fn refs_are_found_in_both_places_they_live() { + let found = refs(&workspace().join(".git")); + assert!(!found.is_empty(), "a repository has refs"); + assert!( + found.iter().any(|(n, _)| n.contains("HEAD") || n.contains("refs/heads")), + "including a branch or HEAD, saw {:?}", + found.iter().take(3).collect::>() + ); + for (_, oid) in &found { + assert_eq!(oid.len(), 40, "refs resolve to full object ids"); + } + } + + #[test] + fn commit_headers_parse() { + let raw = b"tree abc123\nparent def456\nparent 789abc\nauthor A 1 +0000\n\nmessage\n"; + let (tree, parents) = commit_links(raw); + assert_eq!(tree.unwrap(), "abc123"); + assert_eq!(parents, vec!["def456", "789abc"]); + } + + /// A message containing something that looks like a header must not be + /// read as one — headers stop at the blank line. + #[test] + fn the_commit_message_is_not_parsed_as_headers() { + let raw = b"tree abc\nauthor A 1 +0000\n\nparent of all bugs\ntree surgery\n"; + let (tree, parents) = commit_links(raw); + assert_eq!(tree.unwrap(), "abc"); + assert!(parents.is_empty(), "the message is not a header block"); + } + + #[test] + fn tree_entries_parse_names_and_ids() { + let mut raw = Vec::new(); + raw.extend_from_slice(b"100644 README.md\0"); + raw.extend_from_slice(&[0xAB; 20]); + raw.extend_from_slice(b"40000 src\0"); + raw.extend_from_slice(&[0xCD; 20]); + let entries = tree_entries(&raw); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].1, "README.md"); + assert_eq!(entries[0].2, "ab".repeat(20)); + assert!(entries[1].0.starts_with("40"), "a directory entry"); + } + + /// The real thing, against this repository. It should walk a meaningful + /// amount of history and find no credentials — this project has never + /// committed one, and a false positive here would be caught immediately. + #[test] + fn scanning_this_repository_walks_history_and_finds_nothing() { + let r = scan(&workspace()); + assert!( + r.commits_walked > 5, + "expected real history, walked {}", + r.commits_walked + ); + assert!( + r.blobs_examined > 50, + "expected real content, examined {} blobs", + r.blobs_examined + ); + assert!( + r.findings.is_empty(), + "this repository has no committed credentials, but reported: {:?}", + r.findings.iter().map(|f| &f.subject).collect::>() + ); + } +} + +// ── Staged changes ────────────────────────────────────────────────────────── + +/// Check what is about to be committed. +/// +/// This is the only point where a leak is cheap. Once a commit is pushed the +/// key is out, and the only remedy is rotation — so a check that runs a +/// second earlier is worth more than any amount of scanning afterwards. +/// +/// Reads the staged blobs from the index and the object store, so it sees +/// exactly what the commit will contain: not the file on disk, which may have +/// been edited since `git add`, and not the last commit. +pub fn scan_staged(repo: &Path) -> Vec { + let Some(store) = Store::open(repo) else { + return Vec::new(); + }; + let Some(entries) = hygiene::tracked_paths_with_oids(repo) else { + return Vec::new(); + }; + + let mut out = Vec::new(); + for (path, oid) in entries { + let name = Path::new(&path) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.clone()); + + // A secrets file being committed at all is the finding; its contents + // do not need inspecting to know that. + if hygiene::looks_like_secret_file(&name) { + out.push(Finding::new( + "staged-secret-file", + Severity::Critical, + path.clone(), + path.clone(), + format!( + "{name} is staged for commit. Committing it puts its contents in \ + this repository's history permanently — removing it later does not \ + take it back." + ), + "hygiene: staged changes", + format!("Run: git rm --cached {path} and add {name} to .gitignore"), + )); + continue; + } + + if hygiene::looks_binary_name(&name) { + continue; + } + let Some(blob) = store.read(&oid) else { + continue; + }; + let Ok(text) = std::str::from_utf8(&blob.data) else { + continue; + }; + if let Some((what, issuer)) = hygiene::credential_kind(text) { + out.push(Finding::new( + "staged-credential", + Severity::Critical, + format!("{what} in {path}"), + path.clone(), + format!( + "{path} is staged for commit and contains {what}. Once this is \ + committed and pushed, the key is in the history and in every clone." + ), + "hygiene: staged changes", + format!( + "Take the key out of the file and read it from the environment \ + instead. If it has already been used anywhere, revoke it with \ + {issuer} as well." + ), + )); + } + } + out.sort_by(|a, b| a.subject.cmp(&b.subject)); + out +} + +#[cfg(test)] +mod staged_tests { + use super::*; + + /// This repository has staged content whenever a commit is in progress, + /// and none of it is ever a credential. The scan must run and stay quiet. + #[test] + fn the_workspace_has_nothing_dangerous_staged() { + let repo = hygiene::repo_root(Path::new(env!("CARGO_MANIFEST_DIR"))).unwrap(); + let found = scan_staged(&repo); + assert!( + found.is_empty(), + "nothing in this repository should trip the staged check: {:?}", + found.iter().map(|f| &f.subject).collect::>() + ); + } +} diff --git a/crates/hound-supply/src/hygiene.rs b/crates/hound-supply/src/hygiene.rs index 916a233..6b8871f 100644 --- a/crates/hound-supply/src/hygiene.rs +++ b/crates/hound-supply/src/hygiene.rs @@ -32,37 +32,72 @@ use std::path::{Path, PathBuf}; /// issuer actually uses. Each entry is a real, documented format — no /// entropy heuristics, because a false "you have leaked a key" is expensive /// in a way a missed one is not: people stop reading after the second one. -const CREDENTIALS: &[(&str, &str, &str)] = &[ +/// What characters a credential's body may contain after its prefix. +/// +/// This is what separates a token from an identifier. `npm_` followed by +/// forty characters looked like a token until it turned out to be a test +/// function called `npm_paths_yield_the_package_not_the_path` — snake_case +/// runs long, and underscores are what make it long. Real tokens are base62, +/// with hyphens only where an issuer uses them as separators. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Body { + /// Letters and digits only. An underscore ends the run. + Alnum, + /// Letters, digits and hyphens — Slack and a few others. + AlnumDash, + /// The prefix is the whole signal (PEM headers). + Whole, +} + +/// (prefix, minimum body characters, body charset, what, issuer) +/// +/// The length requirement is not decoration. A bare prefix match reported +/// `npm_config_cache` as an npm token and `AKIA` in this very file as an AWS +/// key — the module says findings must be actionable, and the first version +/// of it was not. A real token is a prefix followed by a long run of base62; +/// requiring that is the difference between a detector and a grep. +const CREDENTIALS: &[(&str, usize, Body, &str, &str)] = &[ // (needle, what it is, who to tell) - ("AKIA", "an AWS access key", "AWS"), - ("ASIA", "a temporary AWS access key", "AWS"), - ("ghp_", "a GitHub personal access token", "GitHub"), - ("gho_", "a GitHub OAuth token", "GitHub"), - ("ghs_", "a GitHub server token", "GitHub"), - ("ghu_", "a GitHub user token", "GitHub"), - ("github_pat_", "a GitHub fine-grained token", "GitHub"), - ("sk-ant-", "an Anthropic API key", "Anthropic"), - ("sk-proj-", "an OpenAI project key", "OpenAI"), - ("sk_live_", "a live Stripe secret key", "Stripe"), - ("rk_live_", "a live Stripe restricted key", "Stripe"), - ("xoxb-", "a Slack bot token", "Slack"), - ("xoxp-", "a Slack user token", "Slack"), - ("xapp-", "a Slack app token", "Slack"), - ("glpat-", "a GitLab personal access token", "GitLab"), - ("dop_v1_", "a DigitalOcean token", "DigitalOcean"), - ("SG.", "a SendGrid API key", "SendGrid"), - ("npm_", "an npm access token", "npm"), - ("pypi-AgEIcHlwaS5vcmc", "a PyPI upload token", "PyPI"), - ("AIza", "a Google API key", "Google"), - ("-----BEGIN RSA PRIVATE KEY", "an RSA private key", "whoever it authenticates to"), - ("-----BEGIN OPENSSH PRIVATE KEY", "an SSH private key", "whoever it authenticates to"), - ("-----BEGIN DSA PRIVATE KEY", "a DSA private key", "whoever it authenticates to"), - ("-----BEGIN EC PRIVATE KEY", "an EC private key", "whoever it authenticates to"), - ("-----BEGIN PGP PRIVATE KEY", "a PGP private key", "whoever it authenticates to"), + ("AKIA", 16, Body::Alnum, "an AWS access key", "AWS"), + ("ASIA", 16, Body::Alnum, "a temporary AWS access key", "AWS"), + ("ghp_", 36, Body::Alnum, "a GitHub personal access token", "GitHub"), + ("gho_", 36, Body::Alnum, "a GitHub OAuth token", "GitHub"), + ("ghs_", 36, Body::Alnum, "a GitHub server token", "GitHub"), + ("ghu_", 36, Body::Alnum, "a GitHub user token", "GitHub"), + ("github_pat_", 40, Body::Alnum, "a GitHub fine-grained token", "GitHub"), + ("sk-ant-", 40, Body::AlnumDash, "an Anthropic API key", "Anthropic"), + ("sk-proj-", 40, Body::AlnumDash, "an OpenAI project key", "OpenAI"), + ("sk_live_", 24, Body::Alnum, "a live Stripe secret key", "Stripe"), + ("rk_live_", 24, Body::Alnum, "a live Stripe restricted key", "Stripe"), + ("xoxb-", 20, Body::AlnumDash, "a Slack bot token", "Slack"), + ("xoxp-", 20, Body::AlnumDash, "a Slack user token", "Slack"), + ("xapp-", 20, Body::AlnumDash, "a Slack app token", "Slack"), + ("glpat-", 20, Body::AlnumDash, "a GitLab personal access token", "GitLab"), + ("dop_v1_", 32, Body::Alnum, "a DigitalOcean token", "DigitalOcean"), + ("SG.", 60, Body::Alnum, "a SendGrid API key", "SendGrid"), + ("npm_", 36, Body::Alnum, "an npm access token", "npm"), + ("pypi-AgEIcHlwaS5vcmc", 40, Body::Alnum, "a PyPI upload token", "PyPI"), + ("AIza", 35, Body::Alnum, "a Google API key", "Google"), + (concat!("-----BEGIN RSA PRIVATE ", "KEY-----"), 0, Body::Whole, "an RSA private key", "whoever it authenticates to"), + (concat!("-----BEGIN OPENSSH PRIVATE ", "KEY-----"), 0, Body::Whole, "an SSH private key", "whoever it authenticates to"), + (concat!("-----BEGIN DSA PRIVATE ", "KEY-----"), 0, Body::Whole, "a DSA private key", "whoever it authenticates to"), + (concat!("-----BEGIN EC PRIVATE ", "KEY-----"), 0, Body::Whole, "an EC private key", "whoever it authenticates to"), + (concat!("-----BEGIN PGP PRIVATE ", "KEY-----"), 0, Body::Whole, "a PGP private key", "whoever it authenticates to"), ]; /// Files that hold credentials by convention. Presence is not a problem; /// being committed, or world-readable, is. +/// Credentials that vendors publish as examples. AWS's own documentation +/// uses AKIAIOSFODNN7EXAMPLE, so it appears in tutorials, test fixtures and +/// this file. Reporting it as a leak is how a scanner earns a reputation for +/// crying wolf. +const KNOWN_EXAMPLES: &[&str] = &[ + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "ASIAIOSFODNN7EXAMPLE", + "AKIAI44QH8DHBEXAMPLE", +]; + const SECRET_FILENAMES: &[&str] = &[ ".env", ".env.local", @@ -90,6 +125,19 @@ const EXAMPLE_MARKERS: &[&str] = &[ ".example", ".sample", ".template", ".dist", "example.", "sample.", ]; +/// Extensions whose contents are not text worth searching for a token. +/// Shared with the history scanner, which cannot stat a file to guess. +pub fn looks_binary_name(name: &str) -> bool { + const BINARY: &[&str] = &[ + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", + ".xz", ".zst", ".tar", ".mp4", ".mp3", ".wav", ".woff", ".woff2", ".ttf", + ".so", ".dylib", ".dll", ".class", ".jar", ".wasm", ".pyc", ".o", ".a", + ".bin", ".pack", ".idx", ".lock", + ]; + let lower = name.to_ascii_lowercase(); + BINARY.iter().any(|e| lower.ends_with(e)) +} + pub fn looks_like_secret_file(name: &str) -> bool { let lower = name.to_ascii_lowercase(); if EXAMPLE_MARKERS.iter().any(|m| lower.contains(m)) { @@ -111,10 +159,38 @@ pub fn looks_like_secret_file(name: &str) -> bool { /// moved the key somewhere new — into a log, a CI artefact, an assistant's /// context — which is the thing we are trying to prevent. pub fn credential_kind(text: &str) -> Option<(&'static str, &'static str)> { - CREDENTIALS - .iter() - .find(|(needle, _, _)| text.contains(needle)) - .map(|(_, what, issuer)| (*what, *issuer)) + for (needle, min_len, body, what, issuer) in CREDENTIALS { + if *body == Body::Whole { + // A PEM header with nothing after it is not a key — it is a + // detector definition, a code comment, or documentation. This + // file's own history contained exactly that and reported itself. + // A real key is followed by base64. + for (i, _) in text.match_indices(needle) { + if has_pem_body(&text[i + needle.len()..]) { + return Some((what, issuer)); + } + } + continue; + } + for (i, _) in text.match_indices(needle) { + let after = &text[i + needle.len()..]; + let run: String = after + .chars() + .take_while(|c| { + c.is_ascii_alphanumeric() || (*body == Body::AlnumDash && *c == '-') + }) + .collect(); + if run.len() < *min_len { + continue; + } + let whole = format!("{needle}{run}"); + if KNOWN_EXAMPLES.iter().any(|e| whole.starts_with(e)) { + continue; + } + return Some((what, issuer)); + } + } + None } /// Everything git is tracking, if this is a git repository. @@ -124,6 +200,15 @@ pub fn credential_kind(text: &str) -> Option<(&'static str, &'static str)> { /// trust, and running a subprocess in it is exactly the thing a hostile /// repository wants. Parsing the index is boring and cannot execute anything. pub fn tracked_paths(repo: &Path) -> Option> { + Some(tracked_paths_with_oids(repo)?.into_iter().map(|(p, _)| p).collect()) +} + +/// The index, with each path's staged blob id. +/// +/// The blob is what a commit will actually contain — not the file on disk, +/// which may have been edited since `git add`. Checking the wrong one is how +/// a pre-commit hook passes a commit that carries a key. +pub fn tracked_paths_with_oids(repo: &Path) -> Option> { let index = repo.join(".git").join("index"); let data = std::fs::read(index).ok()?; if data.len() < 12 || &data[0..4] != b"DIRC" { @@ -139,7 +224,7 @@ pub fn tracked_paths(repo: &Path) -> Option> { return None; } - let mut out = HashSet::new(); + let mut out = Vec::new(); let mut pos = 12; for _ in 0..count { // 62 bytes of fixed fields, then a NUL-terminated path, then padding @@ -160,8 +245,13 @@ pub fn tracked_paths(repo: &Path) -> Option> { if end > data.len() { break; } + // The 20-byte object id sits at offset 40 in the fixed header. + let oid: String = data[pos + 40..pos + 60] + .iter() + .map(|b| format!("{b:02x}")) + .collect(); if let Ok(name) = std::str::from_utf8(&data[start..end]) { - out.insert(name.to_string()); + out.push((name.to_string(), oid)); } let entry_len = 62 + (end - start) + 1; pos += (entry_len + 7) & !7; @@ -189,6 +279,23 @@ pub fn gitignore_covers(repo: &Path, name: &str) -> bool { ignored_by(&gitignore_lines(repo), name) } +/// Does a PEM header have actual key material after it? +/// +/// Base64 in long unbroken runs, within a few lines of the header. Anything +/// less is a mention of a key rather than a key. +fn has_pem_body(after: &str) -> bool { + const MIN_RUN: usize = 40; + after + .lines() + .take(8) + .any(|line| { + let t = line.trim(); + t.len() >= MIN_RUN + && t.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '+' || c == '/' || c == '=') + }) +} + fn ignored_by(lines: &[String], name: &str) -> bool { lines.iter().any(|l| { let pat = l.trim_start_matches('/').trim_end_matches('/'); @@ -299,14 +406,28 @@ mod tests { #[test] fn known_credential_formats_are_recognised() { + // Assembled at runtime, deliberately. A fixture that looks like a + // real credential IS a credential-shaped string, and writing one as a + // literal would put it in this file — where the history scanner would + // then find it, correctly, forever. + let body = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S9t0"; for (text, expect_issuer) in [ - ("AKIAIOSFODNN7EXAMPLE", "AWS"), - ("ghp_16CharsOfNonsenseHere0000000000000", "GitHub"), - ("sk-ant-api03-abcdef", "Anthropic"), - ("xoxb-1234-5678-abcdef", "Slack"), - ("-----BEGIN OPENSSH PRIVATE KEY-----", "whoever it authenticates to"), + (format!("AKIA{}", &body[..16]), "AWS"), + (format!("ghp_{body}"), "GitHub"), + (format!("sk-ant-api03-{body}"), "Anthropic"), + (format!("xoxb-{}-{}-{}", "123456789012", "1234567890123", &body[..20]), "Slack"), + // A header plus key material — the header alone is documentation, + // which the test below covers. + ( + format!( + "{d}BEGIN OPENSSH PRIVATE {k}{d}\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAAB\n", + d = "-----", + k = "KEY" + ), + "whoever it authenticates to", + ), ] { - let got = credential_kind(text).unwrap_or_else(|| panic!("missed {text}")); + let got = credential_kind(&text).unwrap_or_else(|| panic!("missed {text}")); assert_eq!(got.1, expect_issuer); } } @@ -314,6 +435,66 @@ mod tests { /// The rule that keeps this usable: no entropy guessing. A random-looking /// string is not a credential, and reporting it as one teaches people to /// ignore the report. + /// The first version matched a bare prefix, and so reported + /// `npm_config_cache` as an npm token and the string "AKIA" in this very + /// file as an AWS key. A history scan of this repository lit up with six + /// findings, all of them our own detector definitions. A credential is a + /// prefix followed by a long run of base62; anything shorter is a word. + /// The detector's own definitions must not be detections. Hound has hit + /// this before — the engine flagged its own binary because the rule pack + /// was embedded verbatim — and a security tool that reports itself is a + /// security tool nobody trusts. + #[test] + fn this_files_own_definitions_are_not_credentials() { + let me = include_str!("hygiene.rs"); + assert!( + credential_kind(me).is_none(), + "the detector detected itself" + ); + } + + /// A header on its own is documentation. This mattered concretely: an + /// earlier commit of this file listed the PEM headers as plain literals, + /// and the history scanner reported the detector as a leaked SSH key. + #[test] + fn a_pem_header_without_key_material_is_not_a_key() { + let dashes = "-----"; + let header = format!("{dashes}BEGIN OPENSSH PRIVATE KEY{dashes}"); + assert!( + credential_kind(&header).is_none(), + "a bare header is a mention, not a key" + ); + assert!( + credential_kind(&format!("// see {header} for the format")).is_none(), + "and so is a comment about one" + ); + let real = format!( + "{header}\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAB\n" + ); + assert!( + credential_kind(&real).is_some(), + "a header followed by key material is a key" + ); + } + + #[test] + fn a_prefix_without_a_credential_after_it_is_not_a_credential() { + for text in [ + "npm_config_cache", + "npm_package_version", + "const AKIA_PREFIX = 'AKIA';", + "// ghp_ tokens start with this", + "sk_live_ is the Stripe prefix", + "AIza", + "xoxb-", + ] { + assert!( + credential_kind(text).is_none(), + "{text:?} is a mention, not a credential" + ); + } + } + #[test] fn random_looking_text_is_not_a_credential() { for text in [ @@ -333,7 +514,8 @@ mod tests { /// a log, a CI artefact, an assistant's context window. #[test] fn a_finding_never_contains_the_secret() { - let secret = "ghp_supersecretvalue000000000000000000"; + let secret = format!("ghp_{}", "supersecretvalue00000000000000000000"); + let secret = secret.as_str(); let (what, issuer) = credential_kind(secret).unwrap(); let f = hardcoded_credential(Path::new("/tmp/app.py"), what, issuer, true); let whole = format!("{f:?}"); diff --git a/crates/hound-supply/src/lib.rs b/crates/hound-supply/src/lib.rs index 5320fec..c910689 100644 --- a/crates/hound-supply/src/lib.rs +++ b/crates/hound-supply/src/lib.rs @@ -17,7 +17,9 @@ //! is a finding that gets ignored, and an ignored finding is worse than //! none because it also costs trust. +pub mod gitobj; pub mod hygiene; +pub mod history; pub mod injection; pub mod installscript; pub mod lockfile; diff --git a/crates/hound/src/main.rs b/crates/hound/src/main.rs index 6f424ae..c80d4f2 100644 --- a/crates/hound/src/main.rs +++ b/crates/hound/src/main.rs @@ -35,6 +35,29 @@ struct Cli { enum Cmd { /// Show engine status (daemon version, engine, signature-DB age) Status, + /// Check a repository for exposed credentials and bad opsec + Hygiene { + /// Repository or directory to check (default: the current directory) + #[arg(default_value = ".")] + path: String, + /// Check only what is staged for commit — for a pre-commit hook + #[arg(long)] + staged: bool, + /// Also walk git history for secrets that were "removed" + #[arg(long)] + history: bool, + /// Emit machine-readable JSON instead of human text + #[arg(long)] + json: bool, + }, + /// Install or remove the pre-commit hook that blocks leaking a secret + Hook { + #[arg(value_parser = ["install", "remove"])] + action: String, + /// Repository to act on (default: the current directory) + #[arg(default_value = ".")] + path: String, + }, /// Add or remove the file-manager right-click entry for this user ContextMenu { #[arg(value_parser = ["install", "remove"])] @@ -421,6 +444,109 @@ fn client(sock: &Option) -> Result { /// Returns the process exit code. fn run(client: &Client, cmd: &Cmd) -> Result { match cmd { + Cmd::Hygiene { + path, + staged, + history, + json, + } => { + // Runs in this process rather than through the daemon: a hook has + // to work before Hound is installed as a service, in a container, + // and in CI — and it needs no privileges to read a repository the + // caller already owns. + let root = std::path::Path::new(path); + let repo = hound_supply::hygiene::repo_root(root); + let mut findings = Vec::new(); + + if *staged { + let Some(repo) = repo.as_deref() else { + eprintln!("{} not a git repository: {path}", "!".yellow()); + return Ok(0); + }; + findings.extend(hound_supply::history::scan_staged(repo)); + } else { + findings.extend(hound_supply::sweep::sweep(root).findings.into_iter().filter( + |f| f.source.starts_with("hygiene"), + )); + if *history { + let Some(repo) = repo.as_deref() else { + eprintln!("{} not a git repository, skipping history", "!".yellow()); + return Ok(0); + }; + let h = hound_supply::history::scan(repo); + if h.truncated { + eprintln!( + "{} history walk stopped at its limit after {} commits — treat \ + a clean result as partial", + "!".yellow(), + h.commits_walked + ); + } + findings.extend(h.findings); + } + } + + if *json { + println!("{}", serde_json::to_string_pretty(&findings)?); + } else if findings.is_empty() { + println!( + "{} nothing exposed{}", + "✔".green().bold(), + if *staged { " in what you are committing" } else { "" } + ); + } else { + for f in &findings { + let mark = match f.severity.as_str() { + "critical" => "✘".red().bold().to_string(), + "warning" => "!".yellow().bold().to_string(), + _ => "·".dimmed().to_string(), + }; + println!("{mark} {}", f.subject.bold()); + println!(" {}", f.explanation); + println!(" {} {}", "→".cyan(), f.advice); + println!(); + } + } + let critical = findings.iter().filter(|f| f.severity.as_str() == "critical").count(); + Ok(if critical > 0 { 1 } else { 0 }) + } + Cmd::Hook { action, path } => { + let root = std::path::Path::new(path); + let repo = hound_supply::hygiene::repo_root(root) + .ok_or_else(|| anyhow::anyhow!("not a git repository: {path}"))?; + let hook = repo.join(".git").join("hooks").join("pre-commit"); + if action == "remove" { + let _ = std::fs::remove_file(&hook); + println!("{} pre-commit hook removed", "✔".green().bold()); + return Ok(0); + } + if hook.exists() { + let existing = std::fs::read_to_string(&hook).unwrap_or_default(); + if !existing.contains("hound hygiene") { + // Overwriting somebody's hook to install a security check + // is not a trade anyone agreed to. + anyhow::bail!( + "{} already exists and is not ours. Add this line to it instead:\n \ + hound hygiene --staged || exit 1", + hook.display() + ); + } + } + std::fs::create_dir_all(hook.parent().expect("hooks dir"))?; + std::fs::write( + &hook, + "#!/bin/sh\n\ + # Installed by `hound hook install`.\n\ + # Blocks a commit that would put a credential in this repository.\n\ + # Bypass once with: git commit --no-verify\n\ + hound hygiene --staged || exit 1\n", + )?; + std::fs::set_permissions(&hook, std::os::unix::fs::PermissionsExt::from_mode(0o755))?; + println!("{} pre-commit hook installed in {}", "✔".green().bold(), repo.display()); + println!(" A commit that would leak a credential is now refused."); + println!(" Bypass a single commit with: {}", "git commit --no-verify".yellow()); + Ok(0) + } Cmd::ContextMenu { action } => { // Nemo, Caja and Dolphin read menu entries from system // directories, so the package installs those. GNOME Files and @@ -538,7 +664,7 @@ fn run(client: &Client, cmd: &Cmd) -> Result { println!(" OS: {}", st.os); println!( " Engine: {}", - "NOT FOUND — sudo apt install clamav".red() + "NOT FOUND — the scanning engine failed to load".red() ); } Ok(0) @@ -1100,9 +1226,9 @@ fn print_update_human(u: &UpdateResult) { if !u.ok { println!( " {}", - "hint: plain-user freshclam needs write access to /var/lib/clamav \ - and /var/log/clamav — run the update from the GUI (polkit) or as \ - a user in the clamav group." + "hint: installing definitions needs write access to /var/lib/hound \ + — run it with sudo, or from the desktop app, which asks polkit \ + for permission." .dimmed() ); } @@ -1265,6 +1391,11 @@ mod tests { for argv in [ vec!["hound", "status"], vec!["hound", "selfcheck"], + vec!["hound", "hygiene"], + vec!["hound", "hygiene", "--staged"], + vec!["hound", "hygiene", ".", "--history"], + vec!["hound", "hook", "install"], + vec!["hound", "hook", "remove"], vec!["hound", "context-menu", "install"], vec!["hound", "context-menu", "remove"], vec!["hound", "scan", "/tmp"], diff --git a/dist/hound_0.1.7_amd64.deb b/dist/hound_0.1.7_amd64.deb new file mode 100644 index 0000000..bfe40a4 Binary files /dev/null and b/dist/hound_0.1.7_amd64.deb differ diff --git a/dist/hound_0.1.8_amd64.deb b/dist/hound_0.1.8_amd64.deb new file mode 100644 index 0000000..b3a1420 Binary files /dev/null and b/dist/hound_0.1.8_amd64.deb differ diff --git a/gui/dist/app.js b/gui/dist/app.js index 131dc25..9687978 100644 --- a/gui/dist/app.js +++ b/gui/dist/app.js @@ -239,7 +239,7 @@ async function doUpdate() { $("update-panel").classList.remove("hidden"); const log = $("update-log"); log.className = "log"; - log.textContent = "Running freshclam — this can take a minute…\n"; + log.textContent = "Checking for new definitions…\n"; try { const u = await invoke("update"); log.textContent = u.output.trim(); diff --git a/gui/dist/index.html b/gui/dist/index.html index 030cefa..6a54eb3 100644 --- a/gui/dist/index.html +++ b/gui/dist/index.html @@ -220,7 +220,7 @@