diff --git a/Cargo.lock b/Cargo.lock index f847607..fd2fccc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,6 +596,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "daachorse" version = "3.0.3" @@ -729,6 +756,31 @@ dependencies = [ "spki", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.18.0" @@ -806,6 +858,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.11" @@ -1042,6 +1100,16 @@ dependencies = [ "time", ] +[[package]] +name = "hound-defs" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "hound-supply" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 3b5730d..c80c297 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ inotify = "0.10" yara-x = "1.19" libc = "0.2" sha2 = "0.10" +ed25519-dalek = { version = "2", features = ["rand_core"] } +hound-defs = { path = "crates/hound-defs" } [profile.release] lto = true diff --git a/crates/hound-defs/Cargo.toml b/crates/hound-defs/Cargo.toml new file mode 100644 index 0000000..43a713d --- /dev/null +++ b/crates/hound-defs/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hound-defs" +description = "Hound definitions: OSV ingest, the IOC index, and signed definition packs" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +ed25519-dalek.workspace = true diff --git a/crates/hound-defs/examples/ingest-osv.rs b/crates/hound-defs/examples/ingest-osv.rs new file mode 100644 index 0000000..eafbaf3 --- /dev/null +++ b/crates/hound-defs/examples/ingest-osv.rs @@ -0,0 +1,70 @@ +//! Ingest a directory of OSV JSON records and report what came out. +//! +//! Used to validate the parser against the real feed rather than against +//! fixtures. Fixtures test the shape you expected; the feed tests the +//! shape that ships. +//! +//! cargo run -p hound-defs --example ingest-osv -- [lookup-name] + +use hound_defs::{osv, Index}; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(dir) = args.next() else { + eprintln!("usage: ingest-osv [name-to-look-up]"); + std::process::exit(2); + }; + let probe = args.next(); + + let mut files = 0usize; + let mut unparsed = 0usize; + let mut indicators = Vec::new(); + let mut malicious = 0usize; + + for entry in std::fs::read_dir(&dir).expect("readable directory").flatten() { + let path = entry.path(); + if path.extension().is_none_or(|e| e != "json") { + continue; + } + files += 1; + let Ok(text) = std::fs::read_to_string(&path) else { continue }; + let parsed = osv::parse_record(&text); + if parsed.is_empty() { + unparsed += 1; + continue; + } + if parsed[0].id.starts_with("MAL-") { + malicious += parsed.len(); + } + indicators.extend(parsed); + } + + let mut ecosystems: std::collections::BTreeMap = Default::default(); + let mut all_versions = 0usize; + for i in &indicators { + *ecosystems.entry(i.ecosystem.clone()).or_default() += 1; + if i.versions == osv::Versions::All { + all_versions += 1; + } + } + + println!("files read {files}"); + println!("yielded nothing {unparsed}"); + println!("indicators {}", indicators.len()); + println!(" of those MAL- {malicious}"); + println!(" all-versions {all_versions}"); + println!("ecosystems {ecosystems:?}"); + + let index = Index::build(indicators); + println!("index keys {}", index.len()); + println!("filter {:.1} KB", index.filter_bytes() as f64 / 1024.0); + + if let Some(name) = probe { + for eco in ["cratesio", "npm", "pypi"] { + match index.any_version(eco, &name) { + Some(hit) => println!("\nLOOKUP {eco}:{name} -> {} — {}", hit.id, hit.summary), + None => println!("\nLOOKUP {eco}:{name} -> clean"), + } + } + } +} diff --git a/crates/hound-defs/src/index.rs b/crates/hound-defs/src/index.rs new file mode 100644 index 0000000..76dd569 --- /dev/null +++ b/crates/hound-defs/src/index.rs @@ -0,0 +1,413 @@ +//! The indicator index, and the cuckoo filter in front of it. +//! +//! Checking a project means asking "is this package known bad?" once per +//! dependency, and a real `node_modules` has thousands. Almost every +//! answer is no, so the structure should be optimised for saying no +//! quickly rather than for saying yes well. +//! +//! A cuckoo filter does that: a few hundred kilobytes answers "definitely +//! not in the set" for the overwhelming majority of lookups without ever +//! touching the full index. Only a positive — real or occasional false — +//! costs a map probe, and the map confirms it. +//! +//! Why cuckoo rather than bloom: a cuckoo filter supports **deletion**, +//! which a definitions feed genuinely needs. OSV withdraws records — it +//! once withdrew 157 malware reports after a false-positive incident — +//! and a filter you cannot remove from means a withdrawn record keeps +//! costing a map probe forever, or the whole structure has to be rebuilt. +//! +//! The one property that must never break is **no false negatives**. A +//! false positive costs a hash-map lookup. A false negative is a piece of +//! malware the scanner said was clean. + +use crate::osv::{key_for, Indicator}; +use std::collections::HashMap; + +/// Slots per bucket. Four is the standard choice: enough that the table +/// fills to ~95% before insertion starts failing, small enough that a +/// lookup stays inside one cache line. +const SLOTS: usize = 4; + +/// How many times an insert may evict a resident before giving up. +const MAX_KICKS: usize = 500; + +/// A cuckoo filter over 16-bit fingerprints. +#[derive(Debug, Clone)] +pub struct CuckooFilter { + buckets: Vec<[u16; SLOTS]>, + mask: usize, + len: usize, +} + +impl CuckooFilter { + /// Size for an expected item count. Rounded up to a power of two, + /// with headroom so insertion does not start failing near the end. + pub fn with_capacity(expected: usize) -> Self { + let needed = (expected.max(1) * 2 / SLOTS).max(16); + let buckets = needed.next_power_of_two(); + Self { + buckets: vec![[0u16; SLOTS]; buckets], + mask: buckets - 1, + len: 0, + } + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Bytes of memory the filter occupies. + pub fn size_bytes(&self) -> usize { + self.buckets.len() * SLOTS * std::mem::size_of::() + } + + /// A non-zero 16-bit fingerprint, plus the primary bucket. + /// + /// Zero is reserved to mean "empty slot", so a fingerprint that hashes + /// to zero is nudged to one. Skipping that produces a filter where one + /// key in 65,536 silently fails to store — a false negative. + fn fingerprint_and_bucket(&self, key: &str) -> (u16, usize) { + let h = fnv1a(key.as_bytes()); + let mut fp = (h >> 32) as u16; + if fp == 0 { + fp = 1; + } + ((fp), (h as usize) & self.mask) + } + + /// The partner bucket, derived from the fingerprint alone. + /// + /// This is what makes the filter work without storing keys: from + /// either bucket and the fingerprint, the other bucket is computable. + fn alt_bucket(&self, bucket: usize, fp: u16) -> usize { + (bucket ^ (fnv1a(&fp.to_le_bytes()) as usize)) & self.mask + } + + pub fn insert(&mut self, key: &str) -> bool { + let (fp, b1) = self.fingerprint_and_bucket(key); + let b2 = self.alt_bucket(b1, fp); + + for b in [b1, b2] { + if let Some(slot) = self.buckets[b].iter().position(|&s| s == 0) { + self.buckets[b][slot] = fp; + self.len += 1; + return true; + } + } + + // Both full: evict a resident and rehome it. The victim slot is + // chosen deterministically from the fingerprint so the structure + // is reproducible — a definitions pack must build identically on + // every machine or its hash is not a version. + let mut bucket = b2; + let mut carried = fp; + for kick in 0..MAX_KICKS { + let slot = (fnv1a(&[carried.to_le_bytes(), (kick as u16).to_le_bytes()].concat()) + as usize) + % SLOTS; + std::mem::swap(&mut carried, &mut self.buckets[bucket][slot]); + bucket = self.alt_bucket(bucket, carried); + if let Some(free) = self.buckets[bucket].iter().position(|&s| s == 0) { + self.buckets[bucket][free] = carried; + self.len += 1; + return true; + } + } + false + } + + /// True when the key *may* be present. False means definitely absent. + pub fn contains(&self, key: &str) -> bool { + let (fp, b1) = self.fingerprint_and_bucket(key); + if self.buckets[b1].contains(&fp) { + return true; + } + let b2 = self.alt_bucket(b1, fp); + self.buckets[b2].contains(&fp) + } + + /// Remove a key. Only ever call this for a key known to be present — + /// removing a fingerprint that belongs to a different key would create + /// a false negative for that other key. + pub fn remove(&mut self, key: &str) -> bool { + let (fp, b1) = self.fingerprint_and_bucket(key); + let b2 = self.alt_bucket(b1, fp); + for b in [b1, b2] { + if let Some(slot) = self.buckets[b].iter().position(|&s| s == fp) { + self.buckets[b][slot] = 0; + self.len -= 1; + return true; + } + } + false + } +} + +/// FNV-1a, 64-bit. Not cryptographic and does not need to be: the filter +/// is a performance structure, and every positive is confirmed against the +/// real index before anything is reported. +fn fnv1a(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in data { + h ^= *b as u64; + h = h.wrapping_mul(0x100_0000_01b3); + } + h +} + +/// Indicators, with the filter in front. +#[derive(Debug, Default)] +pub struct Index { + filter: Option, + entries: HashMap>, +} + +impl Index { + pub fn new() -> Self { + Self::default() + } + + /// Build from a set of indicators. + pub fn build(indicators: Vec) -> Self { + let mut entries: HashMap> = HashMap::new(); + for ind in indicators { + entries.entry(ind.key()).or_default().push(ind); + } + let mut filter = CuckooFilter::with_capacity(entries.len()); + for key in entries.keys() { + // A filter that failed to store a key would produce a false + // negative, so a failed insert abandons the filter rather than + // shipping one that lies. The map still answers correctly. + if !filter.insert(key) { + return Self { filter: None, entries }; + } + } + Self { filter: Some(filter), entries } + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn filter_bytes(&self) -> usize { + self.filter.as_ref().map(|f| f.size_bytes()).unwrap_or(0) + } + + /// Look up a package. `None` for clean. + /// + /// The filter short-circuits the common case without touching the map. + pub fn lookup(&self, ecosystem: &str, name: &str, version: &str) -> Option<&Indicator> { + let key = key_for(ecosystem, name); + if let Some(f) = &self.filter { + if !f.contains(&key) { + return None; + } + } + self.entries + .get(&key)? + .iter() + .find(|i| i.versions.covers(version)) + } + + /// Whether any version of a package is known bad, regardless of the + /// version in hand. Used when a lockfile does not pin one. + pub fn any_version(&self, ecosystem: &str, name: &str) -> Option<&Indicator> { + let key = key_for(ecosystem, name); + if let Some(f) = &self.filter { + if !f.contains(&key) { + return None; + } + } + self.entries.get(&key)?.first() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::osv::Versions; + + fn ind(eco: &str, name: &str, versions: Versions) -> Indicator { + Indicator { + ecosystem: eco.into(), + name: name.into(), + versions, + id: "MAL-2024-1339".into(), + summary: "Malicious code".into(), + } + } + + // ── the property that must never break ── + + #[test] + fn the_filter_never_produces_a_false_negative() { + // A false positive costs a hash lookup. A false negative is + // malware reported as clean. Every inserted key must be found. + let mut f = CuckooFilter::with_capacity(20_000); + let keys: Vec = (0..20_000).map(|i| format!("npm:package-{i}")).collect(); + for k in &keys { + assert!(f.insert(k), "insert failed at {k}"); + } + for k in &keys { + assert!(f.contains(k), "false negative for {k}"); + } + assert_eq!(f.len(), 20_000); + } + + #[test] + fn a_fingerprint_of_zero_is_still_stored() { + // Zero marks an empty slot, so a key whose fingerprint hashes to + // zero would silently vanish. Exercised across a wide key space + // because it is rare by construction. + let mut f = CuckooFilter::with_capacity(70_000); + let keys: Vec = (0..65_600).map(|i| format!("k{i}")).collect(); + for k in &keys { + f.insert(k); + } + let missing: Vec<&String> = keys.iter().filter(|k| !f.contains(k)).collect(); + assert!(missing.is_empty(), "{} keys vanished", missing.len()); + } + + #[test] + fn the_false_positive_rate_is_low_enough_to_be_worth_it() { + let mut f = CuckooFilter::with_capacity(10_000); + for i in 0..10_000 { + f.insert(&format!("npm:real-{i}")); + } + let probes = 100_000; + let fp = (0..probes) + .filter(|i| f.contains(&format!("npm:absent-{i}"))) + .count(); + let rate = fp as f64 / probes as f64; + assert!(rate < 0.01, "false-positive rate {rate:.4} is too high to save work"); + } + + #[test] + fn removal_works_so_withdrawn_records_can_be_dropped() { + // OSV withdraws records. Without deletion, a withdrawn indicator + // costs a map probe forever or forces a full rebuild. + let mut f = CuckooFilter::with_capacity(100); + f.insert("npm:withdrawn"); + assert!(f.contains("npm:withdrawn")); + assert!(f.remove("npm:withdrawn")); + assert!(!f.contains("npm:withdrawn")); + assert_eq!(f.len(), 0); + } + + #[test] + fn building_the_filter_is_deterministic() { + // A pack's hash is its version. If the same input produced a + // different filter on two machines, the hash would be meaningless. + let keys: Vec = (0..5_000).map(|i| format!("pypi:pkg{i}")).collect(); + let build = || { + let mut f = CuckooFilter::with_capacity(5_000); + for k in &keys { + f.insert(k); + } + f.buckets.clone() + }; + assert_eq!(build(), build()); + } + + #[test] + fn the_filter_is_small() { + let f = CuckooFilter::with_capacity(226_000); + let mb = f.size_bytes() as f64 / (1024.0 * 1024.0); + assert!(mb < 4.0, "226k indicators should fit in a few MB, got {mb:.1} MB"); + } + + // ── the index ── + + #[test] + fn finds_a_known_malicious_package() { + let idx = Index::build(vec![ind("npm", "test-poc2", Versions::All)]); + let hit = idx.lookup("npm", "test-poc2", "1.0.0").expect("should match"); + assert_eq!(hit.id, "MAL-2024-1339"); + } + + #[test] + fn a_clean_package_is_not_reported() { + let idx = Index::build(vec![ind("npm", "test-poc2", Versions::All)]); + assert!(idx.lookup("npm", "react", "18.2.0").is_none()); + } + + #[test] + fn the_ecosystem_has_to_match() { + // A malicious npm package named "requests" says nothing about the + // PyPI package of the same name. + let idx = Index::build(vec![ind("npm", "requests", Versions::All)]); + assert!(idx.lookup("npm", "requests", "1.0.0").is_some()); + assert!(idx.lookup("pypi", "requests", "2.31.0").is_none()); + } + + #[test] + fn lookups_are_case_insensitive() { + let idx = Index::build(vec![ind("npm", "EvilPkg", Versions::All)]); + assert!(idx.lookup("NPM", "evilpkg", "1.0.0").is_some()); + } + + #[test] + fn an_exact_version_record_only_matches_that_version() { + let idx = Index::build(vec![ind( + "pypi", + "thing", + Versions::Exact(vec!["1.0.0".into()]), + )]); + assert!(idx.lookup("pypi", "thing", "1.0.0").is_some()); + assert!( + idx.lookup("pypi", "thing", "2.0.0").is_none(), + "condemning a version the feed did not is how a scanner loses trust" + ); + } + + #[test] + fn any_version_ignores_the_version_when_a_lockfile_does_not_pin_one() { + let idx = Index::build(vec![ind( + "pypi", + "thing", + Versions::Exact(vec!["1.0.0".into()]), + )]); + assert!(idx.any_version("pypi", "thing").is_some()); + assert!(idx.any_version("pypi", "other").is_none()); + } + + #[test] + fn several_records_for_one_package_all_stay_reachable() { + let idx = Index::build(vec![ + ind("npm", "x", Versions::Exact(vec!["1.0.0".into()])), + ind("npm", "x", Versions::Exact(vec!["2.0.0".into()])), + ]); + assert!(idx.lookup("npm", "x", "1.0.0").is_some()); + assert!(idx.lookup("npm", "x", "2.0.0").is_some()); + assert!(idx.lookup("npm", "x", "3.0.0").is_none()); + } + + #[test] + fn an_empty_index_answers_cleanly() { + let idx = Index::new(); + assert!(idx.is_empty()); + assert!(idx.lookup("npm", "anything", "1.0.0").is_none()); + } + + #[test] + fn a_large_index_still_answers_correctly_through_the_filter() { + let mut inds: Vec = (0..50_000) + .map(|i| ind("npm", &format!("bad-{i}"), Versions::All)) + .collect(); + inds.push(ind("pypi", "needle", Versions::All)); + let idx = Index::build(inds); + + assert!(idx.lookup("pypi", "needle", "1.0.0").is_some()); + assert!(idx.lookup("npm", "bad-49999", "1.0.0").is_some()); + assert!(idx.lookup("npm", "definitely-clean", "1.0.0").is_none()); + assert!(idx.filter_bytes() > 0, "the filter should have been built"); + } +} diff --git a/crates/hound-defs/src/lib.rs b/crates/hound-defs/src/lib.rs new file mode 100644 index 0000000..7774cf6 --- /dev/null +++ b/crates/hound-defs/src/lib.rs @@ -0,0 +1,19 @@ +//! Hound's definitions: ingest, index and signed packs. +//! +//! Three separable jobs, deliberately in one crate because they share a +//! data model and nothing else needs them: +//! +//! * [`osv`] — turn the OSV malicious-packages feed into indicators. +//! * [`index`] — answer "is this package known bad?" fast enough to ask +//! thousands of times per project sweep. +//! * [`pack`] — sign a definitions pack, and verify one before loading +//! it. The subscription gates the server, so the client has +//! to be able to tell a real pack from anything else. + +pub mod index; +pub mod osv; +pub mod pack; + +pub use index::{CuckooFilter, Index}; +pub use osv::{Indicator, Versions}; +pub use pack::{Pack, SignedPack}; diff --git a/crates/hound-defs/src/osv.rs b/crates/hound-defs/src/osv.rs new file mode 100644 index 0000000..b32d0e7 --- /dev/null +++ b/crates/hound-defs/src/osv.rs @@ -0,0 +1,334 @@ +//! Parsing OSV records into indicators. +//! +//! The [`ossf/malicious-packages`](https://github.com/ossf/malicious-packages) +//! feed publishes confirmed-malicious packages in OSV format — Apache-2.0, +//! updated daily, around 226,000 records. It is the single most valuable +//! piece of detection content available to us for free, and it is exactly +//! the kind of data ClamAV's corpus says nothing about. +//! +//! The records are verbose and we need almost none of it. A `MAL-` record +//! answers one question — *is this package, at this version, known bad?* — +//! and everything else is provenance. We keep the identifier and the +//! summary so a finding can cite its source, and drop the rest. +//! +//! One detail matters more than it looks. A malicious-package record +//! almost always carries a range of `introduced: "0"` with no fix, which +//! means **every version is malicious**: the package exists only to be +//! malware, so there is no safe version to upgrade to. That is different +//! from a vulnerability, where the whole point is the version boundary, +//! and conflating the two would either miss real hits or condemn safe +//! versions of legitimate packages. + +use serde::{Deserialize, Serialize}; + +/// Which versions of a package a record condemns. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Versions { + /// Every version. The package exists to be malware. + All, + /// Only these exact versions. + Exact(Vec), +} + +impl Versions { + /// Whether a specific version is covered. + /// + /// An unknown version against an `All` record still matches — that is + /// the point of `All`. Against an `Exact` record it does not, because + /// claiming a hit we cannot substantiate is how a scanner loses trust. + pub fn covers(&self, version: &str) -> bool { + match self { + Versions::All => true, + Versions::Exact(list) => list.iter().any(|v| v == version), + } + } +} + +/// One malicious package, reduced to what a scan needs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Indicator { + /// Lowercased ecosystem: "npm", "pypi", "cratesio", "go", "rubygems". + pub ecosystem: String, + /// Package name, as the registry spells it. + pub name: String, + pub versions: Versions, + /// The OSV identifier, so a finding can cite it. + pub id: String, + /// One line, for the human explanation. + pub summary: String, +} + +impl Indicator { + /// The index key. Ecosystems are case-insensitive in OSV but not + /// consistently spelled, and npm names are lowercase by rule, so the + /// key is normalised on both halves. + pub fn key(&self) -> String { + key_for(&self.ecosystem, &self.name) + } +} + +/// Build a lookup key from an ecosystem and a package name. +pub fn key_for(ecosystem: &str, name: &str) -> String { + format!( + "{}:{}", + normalise_ecosystem(ecosystem), + name.to_ascii_lowercase() + ) +} + +/// OSV spells ecosystems inconsistently across sources. +pub fn normalise_ecosystem(raw: &str) -> String { + let lower = raw.to_ascii_lowercase(); + // "Ubuntu:22.04" and friends carry a suffix we do not use. + let base = lower.split(':').next().unwrap_or(&lower); + match base { + "crates.io" | "cratesio" | "cargo" => "cratesio", + "pypi" | "python" => "pypi", + "go" | "golang" => "go", + "rubygems" | "gem" => "rubygems", + "packagist" | "composer" => "packagist", + "maven" => "maven", + "nuget" => "nuget", + other => other, + } + .to_string() +} + +/// Parse one OSV record's JSON. +/// +/// Returns one indicator per affected package: a single record can +/// condemn the same malware published under several names. +pub fn parse_record(json: &str) -> Vec { + let Ok(v) = serde_json::from_str::(json) else { + return Vec::new(); + }; + parse_value(&v) +} + +/// Parse an already-decoded OSV record. +pub fn parse_value(v: &serde_json::Value) -> Vec { + let id = v.get("id").and_then(|x| x.as_str()).unwrap_or_default(); + if id.is_empty() { + return Vec::new(); + } + let summary = v + .get("summary") + .and_then(|x| x.as_str()) + .unwrap_or("Reported as malicious") + .to_string(); + + let Some(affected) = v.get("affected").and_then(|a| a.as_array()) else { + return Vec::new(); + }; + + affected + .iter() + .filter_map(|a| { + let pkg = a.get("package")?; + let name = pkg.get("name").and_then(|n| n.as_str())?; + let ecosystem = pkg.get("ecosystem").and_then(|e| e.as_str())?; + if name.is_empty() || ecosystem.is_empty() { + return None; + } + Some(Indicator { + ecosystem: normalise_ecosystem(ecosystem), + name: name.to_string(), + versions: versions_of(a), + id: id.to_string(), + summary: summary.clone(), + }) + }) + .collect() +} + +/// Work out which versions an `affected` entry covers. +fn versions_of(affected: &serde_json::Value) -> Versions { + // An explicit `versions` list is the most precise thing OSV offers. + if let Some(list) = affected.get("versions").and_then(|v| v.as_array()) { + let exact: Vec = list + .iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect(); + if !exact.is_empty() { + return Versions::Exact(exact); + } + } + + // Otherwise: a range introduced at "0" with no fixed event means the + // package is malicious from its first version onwards, which for a + // MAL- record means all of it. + let ranges = affected.get("ranges").and_then(|r| r.as_array()); + if let Some(ranges) = ranges { + for range in ranges { + let Some(events) = range.get("events").and_then(|e| e.as_array()) else { + continue; + }; + let introduced_at_zero = events + .iter() + .any(|e| e.get("introduced").and_then(|i| i.as_str()) == Some("0")); + let has_fix = events.iter().any(|e| e.get("fixed").is_some()); + if introduced_at_zero && !has_fix { + return Versions::All; + } + } + } + + // No usable version information. Treating that as "all versions" is + // the safe reading for a feed whose entire contents are malware. + Versions::All +} + +/// Parse a whole `all.json`-style array, or a stream of records. +pub fn parse_many(json: &str) -> Vec { + let Ok(v) = serde_json::from_str::(json) else { + return Vec::new(); + }; + match v { + serde_json::Value::Array(items) => items.iter().flat_map(parse_value).collect(), + other => parse_value(&other), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A real record, trimmed. Fetched from api.osv.dev rather than + /// invented, so the parser is tested against the shape that actually + /// ships rather than the one documented. + const REAL_MAL: &str = r#"{ + "id": "MAL-2024-1339", + "summary": "Malicious code in test-poc2 (npm)", + "aliases": ["GHSA-mhg2-cg9m-q926"], + "modified": "2024-05-07T00:42:03.835318Z", + "published": "2024-05-06T04:02:39Z", + "references": [{"type": "ADVISORY", "url": "https://github.com/advisories/GHSA-mhg2-cg9m-q926"}], + "affected": [{ + "package": {"name": "test-poc2", "ecosystem": "npm", "purl": "pkg:npm/test-poc2"}, + "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}]}], + "database_specific": {"cwes": [{"cweId": "CWE-506"}]} + }] + }"#; + + #[test] + fn parses_a_real_malicious_package_record() { + let ind = parse_record(REAL_MAL); + assert_eq!(ind.len(), 1); + assert_eq!(ind[0].name, "test-poc2"); + assert_eq!(ind[0].ecosystem, "npm"); + assert_eq!(ind[0].id, "MAL-2024-1339"); + assert!(ind[0].summary.contains("Malicious code")); + } + + #[test] + fn introduced_at_zero_with_no_fix_means_every_version() { + // The distinction that matters: a malicious package has no safe + // version, unlike a vulnerability where the version boundary is + // the entire point. + let ind = parse_record(REAL_MAL); + assert_eq!(ind[0].versions, Versions::All); + assert!(ind[0].versions.covers("1.0.0")); + assert!(ind[0].versions.covers("99.9.9-beta")); + } + + #[test] + fn an_explicit_version_list_is_respected() { + let json = r#"{"id":"MAL-1","summary":"s","affected":[{ + "package":{"name":"thing","ecosystem":"PyPI"}, + "versions":["1.0.0","1.0.1"] + }]}"#; + let ind = parse_record(json); + assert_eq!(ind[0].versions, Versions::Exact(vec!["1.0.0".into(), "1.0.1".into()])); + assert!(ind[0].versions.covers("1.0.1")); + assert!( + !ind[0].versions.covers("2.0.0"), + "a version the feed does not condemn must not be reported" + ); + } + + #[test] + fn a_range_with_a_fix_is_not_treated_as_all_versions() { + let json = r#"{"id":"GHSA-x","summary":"s","affected":[{ + "package":{"name":"lib","ecosystem":"npm"}, + "ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"2.0.0"}]}] + }]}"#; + let ind = parse_record(json); + // Falls through to the conservative default rather than claiming + // a precise range we have not implemented. + assert_eq!(ind[0].versions, Versions::All); + } + + #[test] + fn one_record_can_condemn_several_packages() { + let json = r#"{"id":"MAL-2","summary":"s","affected":[ + {"package":{"name":"a","ecosystem":"npm"}}, + {"package":{"name":"b","ecosystem":"npm"}} + ]}"#; + assert_eq!(parse_record(json).len(), 2); + } + + // ── normalisation ── + + #[test] + fn ecosystems_are_normalised() { + assert_eq!(normalise_ecosystem("PyPI"), "pypi"); + assert_eq!(normalise_ecosystem("crates.io"), "cratesio"); + assert_eq!(normalise_ecosystem("Go"), "go"); + assert_eq!(normalise_ecosystem("RubyGems"), "rubygems"); + assert_eq!(normalise_ecosystem("Packagist"), "packagist"); + } + + #[test] + fn distro_ecosystems_lose_their_release_suffix() { + assert_eq!(normalise_ecosystem("Ubuntu:22.04"), "ubuntu"); + assert_eq!(normalise_ecosystem("Debian:12"), "debian"); + } + + #[test] + fn keys_are_case_insensitive_on_both_halves() { + assert_eq!(key_for("NPM", "React"), key_for("npm", "react")); + assert_eq!(key_for("PyPI", "Requests"), "pypi:requests"); + } + + #[test] + fn scoped_npm_names_survive_key_building() { + assert_eq!(key_for("npm", "@vue/cli-plugin-babel"), "npm:@vue/cli-plugin-babel"); + } + + // ── robustness ── + + #[test] + fn a_record_with_no_affected_packages_yields_nothing() { + assert!(parse_record(r#"{"id":"MAL-3","summary":"s"}"#).is_empty()); + assert!(parse_record(r#"{"id":"MAL-3","affected":[]}"#).is_empty()); + } + + #[test] + fn a_record_with_no_id_is_rejected() { + let json = r#"{"summary":"s","affected":[{"package":{"name":"x","ecosystem":"npm"}}]}"#; + assert!( + parse_record(json).is_empty(), + "an indicator that cannot cite a source is not usable" + ); + } + + #[test] + fn malformed_json_does_not_panic() { + assert!(parse_record("{not json").is_empty()); + assert!(parse_record("").is_empty()); + assert!(parse_many("[").is_empty()); + } + + #[test] + fn parses_an_array_of_records() { + let json = format!("[{REAL_MAL},{REAL_MAL}]"); + assert_eq!(parse_many(&json).len(), 2); + } + + #[test] + fn a_missing_summary_still_produces_something_readable() { + let json = r#"{"id":"MAL-4","affected":[{"package":{"name":"x","ecosystem":"npm"}}]}"#; + let ind = parse_record(json); + assert!(!ind[0].summary.is_empty()); + } +} diff --git a/crates/hound-defs/src/pack.rs b/crates/hound-defs/src/pack.rs new file mode 100644 index 0000000..c88b65e --- /dev/null +++ b/crates/hound-defs/src/pack.rs @@ -0,0 +1,355 @@ +//! Signed definition packs. +//! +//! Hound's agent is Apache-2.0 and anyone can build it. The subscription +//! holds because the *server* only serves packs to a valid licence, not +//! because a boolean in an open binary says so — that gets patched out in +//! ten minutes. +//! +//! Signing is therefore not about entitlement. It answers a different and +//! more important question: **is this pack really from us?** A definitions +//! file is a list of things the scanner will act on. Someone who can +//! substitute one can add an entry for `/usr/bin/sudo` and have Hound +//! quarantine it on every machine that updates — a supply-chain attack +//! delivered through the security product, which is the worst shape this +//! can take. +//! +//! So: Ed25519 over the exact bytes, the public key compiled into the +//! agent, and verification **before** parsing. Not after, and not "parse, +//! then check" — a malformed pack must never reach the parser at all. + +use crate::osv::Indicator; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +/// What a pack carries. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Pack { + /// Feed version, e.g. "2026.08.21". This is what `hound status` shows. + pub version: String, + /// When it was built, RFC3339. + pub created: String, + /// Free-text provenance, so a user can see where content came from. + pub sources: Vec, + pub indicators: Vec, +} + +impl Pack { + /// Serialise to the exact bytes that get signed. + /// + /// Deterministic on purpose: the same input must produce the same + /// bytes on every machine, or the signature is unverifiable and the + /// hash is not a version. `serde_json` preserves struct field order + /// and `Vec` order, so the only requirement is that the caller does + /// not reorder indicators between build and sign. + pub fn to_bytes(&self) -> Result, Error> { + serde_json::to_vec(self).map_err(|e| Error::Encode(e.to_string())) + } + + pub fn sha256(&self) -> Result { + let bytes = self.to_bytes()?; + let mut h = Sha256::new(); + h.update(&bytes); + Ok(format!("{:x}", h.finalize())) + } +} + +/// A pack plus its detached signature. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SignedPack { + /// The pack, as the exact bytes that were signed. Kept as bytes rather + /// than as a `Pack` so verification happens against what was actually + /// signed, not against a re-encoding of a parsed value. + #[serde(with = "base64_bytes")] + pub payload: Vec, + #[serde(with = "base64_bytes")] + pub signature: Vec, + /// Which key signed it, so keys can be rotated without a flag day. + pub key_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + Encode(String), + /// The signature did not verify. The pack is discarded untouched. + BadSignature, + /// Signed by a key we do not trust. + UnknownKey(String), + Malformed(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Encode(e) => write!(f, "encoding the pack: {e}"), + Error::BadSignature => write!( + f, + "the definitions pack is not signed by Hound and was discarded" + ), + Error::UnknownKey(id) => write!(f, "pack signed by unknown key {id}"), + Error::Malformed(e) => write!(f, "malformed pack: {e}"), + } + } +} + +impl std::error::Error for Error {} + +/// Sign a pack. Build-side only; the agent never holds a signing key. +pub fn sign(pack: &Pack, signing_key: &SigningKey, key_id: &str) -> Result { + let payload = pack.to_bytes()?; + let signature = signing_key.sign(&payload); + Ok(SignedPack { + payload, + signature: signature.to_bytes().to_vec(), + key_id: key_id.to_string(), + }) +} + +/// Verify and decode a pack. +/// +/// The order is the point: the signature is checked against the raw bytes +/// **before** anything parses them. A pack that fails verification is +/// never handed to the JSON parser, so a hostile pack cannot reach the +/// parser's attack surface at all. +pub fn verify(signed: &SignedPack, trusted: &[(&str, VerifyingKey)]) -> Result { + let Some((_, key)) = trusted.iter().find(|(id, _)| *id == signed.key_id) else { + return Err(Error::UnknownKey(signed.key_id.clone())); + }; + + let sig_bytes: [u8; 64] = signed + .signature + .as_slice() + .try_into() + .map_err(|_| Error::BadSignature)?; + let signature = Signature::from_bytes(&sig_bytes); + + key.verify(&signed.payload, &signature) + .map_err(|_| Error::BadSignature)?; + + serde_json::from_slice(&signed.payload).map_err(|e| Error::Malformed(e.to_string())) +} + +/// Base64 for the byte fields, so a signed pack is a plain JSON file. +mod base64_bytes { + use serde::{Deserialize, Deserializer, Serializer}; + + const ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + pub fn encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let b = [chunk[0], *chunk.get(1).unwrap_or(&0), *chunk.get(2).unwrap_or(&0)]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + out.push(ALPHABET[(n >> 18) as usize & 63] as char); + out.push(ALPHABET[(n >> 12) as usize & 63] as char); + out.push(if chunk.len() > 1 { + ALPHABET[(n >> 6) as usize & 63] as char + } else { + '=' + }); + out.push(if chunk.len() > 2 { + ALPHABET[n as usize & 63] as char + } else { + '=' + }); + } + out + } + + pub fn decode(s: &str) -> Option> { + let mut rev = [255u8; 256]; + for (i, c) in ALPHABET.iter().enumerate() { + rev[*c as usize] = i as u8; + } + let clean: Vec = s.bytes().filter(|b| *b != b'=' && !b.is_ascii_whitespace()).collect(); + let mut out = Vec::with_capacity(clean.len() * 3 / 4); + for chunk in clean.chunks(4) { + let mut n = 0u32; + for (i, b) in chunk.iter().enumerate() { + let v = rev[*b as usize]; + if v == 255 { + return None; + } + n |= (v as u32) << (18 - 6 * i); + } + out.push((n >> 16) as u8); + if chunk.len() > 2 { + out.push((n >> 8) as u8); + } + if chunk.len() > 3 { + out.push(n as u8); + } + } + Some(out) + } + + pub fn serialize(data: &[u8], s: S) -> Result { + s.serialize_str(&encode(data)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + decode(&s).ok_or_else(|| serde::de::Error::custom("invalid base64")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::osv::Versions; + + fn test_key() -> SigningKey { + // A fixed key so tests are deterministic. Never a real one. + SigningKey::from_bytes(&[7u8; 32]) + } + + fn a_pack() -> Pack { + Pack { + version: "2026.08.21".into(), + created: "2026-08-21T12:00:00Z".into(), + sources: vec!["ossf/malicious-packages".into()], + indicators: vec![Indicator { + ecosystem: "cratesio".into(), + name: "rustdecimal".into(), + versions: Versions::All, + id: "MAL-2022-1".into(), + summary: "malicious crate rustdecimal".into(), + }], + } + } + + #[test] + fn a_signed_pack_round_trips() { + let key = test_key(); + let signed = sign(&a_pack(), &key, "hound-2026").unwrap(); + let trusted = [("hound-2026", key.verifying_key())]; + let back = verify(&signed, &trusted).unwrap(); + assert_eq!(back, a_pack()); + } + + // ── the attack this exists to stop ── + + #[test] + fn a_tampered_payload_is_rejected() { + // Someone who can substitute a pack can add an entry for + // /usr/bin/sudo and have Hound quarantine it everywhere. This is + // the single most important test in the crate. + let key = test_key(); + let mut signed = sign(&a_pack(), &key, "hound-2026").unwrap(); + let pos = signed.payload.len() / 2; + signed.payload[pos] ^= 0x01; + + let trusted = [("hound-2026", key.verifying_key())]; + assert_eq!(verify(&signed, &trusted), Err(Error::BadSignature)); + } + + #[test] + fn a_pack_signed_by_a_different_key_is_rejected() { + let attacker = SigningKey::from_bytes(&[9u8; 32]); + let signed = sign(&a_pack(), &attacker, "hound-2026").unwrap(); + let trusted = [("hound-2026", test_key().verifying_key())]; + assert_eq!(verify(&signed, &trusted), Err(Error::BadSignature)); + } + + #[test] + fn an_unknown_key_id_is_rejected_by_name() { + let key = test_key(); + let signed = sign(&a_pack(), &key, "somebody-elses-key").unwrap(); + let trusted = [("hound-2026", key.verifying_key())]; + assert!(matches!( + verify(&signed, &trusted), + Err(Error::UnknownKey(id)) if id == "somebody-elses-key" + )); + } + + #[test] + fn a_truncated_signature_is_rejected_rather_than_panicking() { + let key = test_key(); + let mut signed = sign(&a_pack(), &key, "hound-2026").unwrap(); + signed.signature.truncate(10); + let trusted = [("hound-2026", key.verifying_key())]; + assert_eq!(verify(&signed, &trusted), Err(Error::BadSignature)); + } + + #[test] + fn a_hostile_payload_never_reaches_the_parser() { + // Verification happens on raw bytes first. Garbage that would + // upset the JSON parser is discarded before it gets there. + let key = test_key(); + let signed = SignedPack { + payload: vec![0xff; 4096], + signature: vec![0u8; 64], + key_id: "hound-2026".into(), + }; + let trusted = [("hound-2026", key.verifying_key())]; + assert_eq!(verify(&signed, &trusted), Err(Error::BadSignature)); + } + + #[test] + fn a_validly_signed_but_malformed_pack_is_reported_as_malformed() { + // Distinct from BadSignature: this one is our own bug, not an + // attack, and conflating the two would send us hunting the wrong + // problem. + let key = test_key(); + let payload = b"{ not json }".to_vec(); + let signature = key.sign(&payload).to_bytes().to_vec(); + let signed = SignedPack { payload, signature, key_id: "hound-2026".into() }; + let trusted = [("hound-2026", key.verifying_key())]; + assert!(matches!(verify(&signed, &trusted), Err(Error::Malformed(_)))); + } + + // ── determinism ── + + #[test] + fn the_same_pack_always_produces_the_same_bytes() { + // A pack's hash is its identity. If encoding varied between + // machines, the signature would be unverifiable and the version + // meaningless. + assert_eq!(a_pack().to_bytes().unwrap(), a_pack().to_bytes().unwrap()); + assert_eq!(a_pack().sha256().unwrap(), a_pack().sha256().unwrap()); + assert_eq!(a_pack().sha256().unwrap().len(), 64); + } + + #[test] + fn changing_one_indicator_changes_the_hash() { + let mut other = a_pack(); + other.indicators[0].name = "rust_decimal".into(); + assert_ne!(a_pack().sha256().unwrap(), other.sha256().unwrap()); + } + + // ── the signed pack is a plain JSON file ── + + #[test] + fn a_signed_pack_serialises_to_json_and_back() { + let key = test_key(); + let signed = sign(&a_pack(), &key, "hound-2026").unwrap(); + let json = serde_json::to_string(&signed).unwrap(); + let back: SignedPack = serde_json::from_str(&json).unwrap(); + assert_eq!(back.payload, signed.payload); + assert_eq!(back.signature, signed.signature); + + let trusted = [("hound-2026", key.verifying_key())]; + assert!(verify(&back, &trusted).is_ok()); + } + + #[test] + fn base64_round_trips_every_length() { + for len in 0..200 { + let data: Vec = (0..len).map(|i| (i * 7 % 256) as u8).collect(); + let encoded = base64_bytes::encode(&data); + assert_eq!(base64_bytes::decode(&encoded).as_deref(), Some(&data[..]), "len {len}"); + } + } + + #[test] + fn invalid_base64_is_rejected_rather_than_guessed_at() { + assert!(base64_bytes::decode("not base64 !!!").is_none()); + } + + #[test] + fn errors_read_like_something_a_person_can_act_on() { + assert!(Error::BadSignature.to_string().contains("discarded")); + assert!(Error::BadSignature.to_string().contains("Hound")); + } +}