diff --git a/crates/hound-defs/examples/build-community.rs b/crates/hound-defs/examples/build-community.rs new file mode 100644 index 0000000..5763a42 --- /dev/null +++ b/crates/hound-defs/examples/build-community.rs @@ -0,0 +1,115 @@ +//! Build the free community pack: a snapshot of the most recent public OSV +//! malicious packages, so a Free install detects current threats out of the +//! box. +//! +//! build-community [created] [limit] +//! +//! This is the free half of the feed. It is deliberately a *recent* subset, +//! not the full corpus: Pro is not "access to public data" — anyone can pull +//! OSV — it is the full 235k history, the daily/near-real-time freshness, and +//! the curated threat pack. The community pack costs us nothing to give away +//! (it is public data) and makes Free genuinely useful. +//! +//! "Recent" is approximated by sorting indicator ids descending: the OSSF +//! feed mints `MAL-YYYY-NNNNN`, so the newest advisories sort first. The pack +//! is built from the already-published, already-verified feed, so there is a +//! single source of truth and no second ingest to drift. + +use ed25519_dalek::{SigningKey, VerifyingKey}; +use hound_defs::{pack, Indicator, Pack}; +use std::collections::BTreeSet; +use std::path::Path; + +/// How many indicators the free pack carries. A few thousand recent ones — +/// enough to be useful, small enough that the full feed is clearly more. +const DEFAULT_LIMIT: usize = 5000; + +fn die(m: impl std::fmt::Display) -> ! { + eprintln!("build-community: {m}"); + std::process::exit(1); +} + +fn load_key(path: &str) -> SigningKey { + let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}"))); + let seed: [u8; 32] = bytes + .get(..32) + .and_then(|s| s.try_into().ok()) + .unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key"))); + SigningKey::from_bytes(&seed) +} + +/// Every indicator in the published full feed (baselines + deltas), verified +/// against our own key. Community packs are excluded so this does not feed on +/// itself. +fn all_published(dir: &Path, pubkey: &VerifyingKey) -> Vec { + let trusted = [("hound-2026", *pubkey)]; + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue }; + if !name.ends_with(".pack") || name.starts_with("community-") { + continue; + } + let Ok(text) = std::fs::read_to_string(&path) else { continue }; + let Ok(signed) = serde_json::from_str::(&text) else { continue }; + let Ok(p) = pack::verify(&signed, &trusted) else { + eprintln!("build-community: WARNING ignoring unverifiable {name}"); + continue; + }; + for ind in p.indicators { + let key = (ind.ecosystem.clone(), ind.name.clone(), ind.id.clone()); + if seen.insert(key) { + out.push(ind); + } + } + } + out +} + +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + if a.len() < 4 { + die("usage: build-community [created] [limit]"); + } + let (published, out_path, key_path, version) = + (Path::new(&a[0]), &a[1], &a[2], &a[3]); + let created = a.get(4).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into()); + let limit: usize = a.get(5).and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_LIMIT); + + let key = load_key(key_path); + let mut all = all_published(published, &key.verifying_key()); + if all.is_empty() { + die("the published feed has no indicators to draw a community pack from"); + } + + // Most recent first (MAL-YYYY-NNNNN sorts by recency descending), then + // take the head. A stable secondary sort on the full identity keeps the + // output deterministic when ids collide. + all.sort_by(|x, y| { + y.id.cmp(&x.id).then_with(|| { + (&x.ecosystem, &x.name).cmp(&(&y.ecosystem, &y.name)) + }) + }); + all.truncate(limit); + + // Re-sort into the canonical build order so the pack bytes are stable. + all.sort_by(|x, y| { + (&x.ecosystem, &x.name, &x.id).cmp(&(&y.ecosystem, &y.name, &y.id)) + }); + + let p = Pack { + version: version.clone(), + created, + sources: vec![ + "ossf/malicious-packages (Apache-2.0), recent subset".to_string(), + "osv.dev".to_string(), + ], + indicators: all.clone(), + }; + let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into()); + let signed = pack::sign(&p, &key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}"))); + std::fs::write(out_path, serde_json::to_string(&signed).unwrap()) + .unwrap_or_else(|e| die(format!("writing {out_path}: {e}"))); + eprintln!("wrote {out_path} — {} recent indicators, version {version}", all.len()); +} diff --git a/crates/hound-defs/examples/build-delta.rs b/crates/hound-defs/examples/build-delta.rs new file mode 100644 index 0000000..30e9ef8 --- /dev/null +++ b/crates/hound-defs/examples/build-delta.rs @@ -0,0 +1,182 @@ +//! Incremental definition builder: emit a small daily delta instead of a +//! fresh 44 MB pack every night. +//! +//! build-delta [created] +//! +//! The problem it solves: the client already downloads only packs whose +//! sha256 it does not have, but the old nightly rebuilt the full +//! per-ecosystem pack every day, so its hash changed daily and every client +//! re-fetched the whole thing. Here each day's *new* indicators go in a +//! small delta pack; the baseline stays byte-identical between rebuilds, so +//! a client fetches it once and then only the daily deltas (kilobytes). +//! +//! What it does, for one ecosystem: +//! 1. Parse today's full indicator set from the OSV export. +//! 2. Read the currently-published baseline + deltas to learn which +//! indicators the feed already carries. +//! 3. Decide: +//! - no baseline yet, or too many deltas piled up → REBASELINE: +//! write a fresh full baseline (the client re-fetches it once and +//! the old files are dropped from the index, so it self-cleans). +//! - new indicators since yesterday → DELTA: write just those. +//! - nothing new → UNCHANGED: write nothing. +//! 4. Print one decision line for the publish script to act on. +//! +//! It only ever writes into the staging dir and only ever reads the +//! published dir — moving files into place and rebuilding the index is the +//! publish script's job, so this stays a pure, testable computation. + +use ed25519_dalek::{SigningKey, VerifyingKey}; +use hound_defs::{osv, pack, Indicator, Pack}; +use std::collections::BTreeSet; +use std::path::Path; + +/// Fold the deltas back into a fresh baseline once this many have piled up, +/// so a new install never has to replay an unbounded chain and the served +/// directory stays small. One baseline plus at most this many deltas. +const REBASELINE_AFTER: usize = 14; + +fn die(msg: impl std::fmt::Display) -> ! { + eprintln!("build-delta: {msg}"); + std::process::exit(1); +} + +fn load_key(path: &str) -> SigningKey { + let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}"))); + let seed: [u8; 32] = bytes + .get(..32) + .and_then(|s| s.try_into().ok()) + .unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key"))); + SigningKey::from_bytes(&seed) +} + +/// A stable identity for an indicator, matching build-pack's dedup key. +fn key_of(i: &Indicator) -> (String, String, String) { + (i.ecosystem.clone(), i.name.clone(), i.id.clone()) +} + +/// Every `.pack` in `dir` whose name is `-…`, verified against +/// our own key, flattened to the indicator identities they already carry. +/// Also returns how many delta files were seen, for the re-baseline call. +fn published_state( + dir: &Path, + eco: &str, + pubkey: &VerifyingKey, +) -> (BTreeSet<(String, String, String)>, usize, bool) { + let trusted = [("hound-2026", *pubkey)]; + let mut known = BTreeSet::new(); + let mut deltas = 0usize; + let mut has_baseline = false; + let prefix = format!("{eco}-"); + + for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue }; + if !name.ends_with(".pack") || !name.starts_with(&prefix) { + continue; + } + // Guard against "go-" matching "golang-": the char after the prefix + // must start a version (a digit). + if !name[prefix.len()..].starts_with(|c: char| c.is_ascii_digit()) { + continue; + } + let Ok(text) = std::fs::read_to_string(&path) else { continue }; + let Ok(signed) = serde_json::from_str::(&text) else { continue }; + let Ok(p) = pack::verify(&signed, &trusted) else { + eprintln!("build-delta: WARNING ignoring unverifiable {name}"); + continue; + }; + for ind in p.indicators { + known.insert(key_of(&ind)); + } + if name.contains(".delta.pack") { + deltas += 1; + } else { + has_baseline = true; + } + } + (known, deltas, has_baseline) +} + +/// Parse today's full indicator set for the ecosystem from its OSV export. +fn todays_indicators(osv_dir: &Path) -> Vec { + let mut out = Vec::new(); + for entry in std::fs::read_dir(osv_dir) + .unwrap_or_else(|e| die(format!("reading {}: {e}", osv_dir.display()))) + .flatten() + { + let path = entry.path(); + if path.extension().is_none_or(|e| e != "json") { + continue; + } + if let Ok(text) = std::fs::read_to_string(&path) { + out.extend(osv::parse_record(&text)); + } + } + out.sort_by(|a, b| key_of(a).cmp(&key_of(b))); + out.dedup_by(|a, b| key_of(a) == key_of(b)); + out +} + +fn write_pack(pack_data: &Pack, key: &SigningKey, out: &Path) { + let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into()); + let signed = pack::sign(pack_data, key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}"))); + std::fs::write(out, serde_json::to_string(&signed).unwrap()) + .unwrap_or_else(|e| die(format!("writing {}: {e}", out.display()))); +} + +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + if a.len() < 6 { + die("usage: build-delta [created]"); + } + let (osv_dir, published, eco, staging, key_path, version) = + (Path::new(&a[0]), Path::new(&a[1]), &a[2], Path::new(&a[3]), &a[4], &a[5]); + let created = a.get(6).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into()); + + let key = load_key(key_path); + let pubkey = key.verifying_key(); + + let today = todays_indicators(osv_dir); + if today.is_empty() { + // The ingest produced nothing for this ecosystem; never publish an + // empty pack (a client that installed it would show zero indicators). + println!("UNCHANGED {eco} (no records ingested)"); + return; + } + + let (known, deltas, has_baseline) = published_state(published, eco, &pubkey); + + let sources = vec!["ossf/malicious-packages (Apache-2.0)".to_string(), "osv.dev".to_string()]; + let rebaseline = !has_baseline || deltas >= REBASELINE_AFTER; + + if rebaseline { + let out = staging.join(format!("{eco}-{version}.pack")); + write_pack( + &Pack { version: version.clone(), created, sources, indicators: today.clone() }, + &key, + &out, + ); + // The publish script archives every existing {eco}-* file when it + // sees REBASELINE, so the fresh baseline stands alone. + println!("REBASELINE {eco} {} indicators -> {}", today.len(), out.display()); + return; + } + + let new: Vec = today + .into_iter() + .filter(|i| !known.contains(&key_of(i))) + .collect(); + if new.is_empty() { + println!("UNCHANGED {eco} (no new indicators)"); + return; + } + + let out = staging.join(format!("{eco}-{version}.delta.pack")); + write_pack( + &Pack { version: version.clone(), created, sources, indicators: new.clone() }, + &key, + &out, + ); + println!("DELTA {eco} {} new indicator(s) -> {}", new.len(), out.display()); +} diff --git a/crates/houndd/src/engine.rs b/crates/houndd/src/engine.rs index 54b1956..130fc08 100644 --- a/crates/houndd/src/engine.rs +++ b/crates/houndd/src/engine.rs @@ -48,6 +48,16 @@ pub trait ScanEngine: Send + Sync { /// combined stdout+stderr tail) for the last attempt made. fn update(&self) -> Result<(bool, String, String)>; + /// Recompile the YARA ruleset from disk, picking up any newly installed + /// rules packs, without the rest of `update`'s work (no freshclam, no + /// definitions fetch). The daily scheduler calls this after a defs run + /// that installed or pruned rules packs, so a new threat pack goes live + /// without waiting for a restart. Engines with no on-disk ruleset — the + /// ClamAV and fake backends — do nothing. + fn reload_rules(&self) -> Result<()> { + Ok(()) + } + /// Scan bytes already in hand, returning a detection name. /// /// This exists for the execution gate, which is handed an open diff --git a/crates/houndd/src/main.rs b/crates/houndd/src/main.rs index 22f0f7b..40356c0 100644 --- a/crates/houndd/src/main.rs +++ b/crates/houndd/src/main.rs @@ -845,24 +845,27 @@ fn dispatch(req: &hound_api::Request, st: &DaemonState) -> Result { /// One definitions check, using the same install path as `hound update` so /// there is no second implementation to drift. Returns how many packs landed. fn scheduled_defs_update(st: &DaemonState) -> Result { - // The full feed is what a Pro subscription pays for. Checked here, at - // fetch time rather than at boot, so installing a licence takes effect - // on the next cycle without a restart — and a lapse stops the fetch - // without touching what is already on disk. - if !st + // The free community pack is fetched for everyone; the full feed and the + // threat pack only with a Pro licence. Checked here, at fetch time rather + // than at boot, so installing a licence takes effect on the next cycle + // without a restart — and a lapse drops the machine cleanly back to the + // community pack on the next run. + let full_feed = st .license .current() .effective - .allows(hound_api::license::Capability::FullSupplyChainFeed) - { - return Ok(0); - } + .allows(hound_api::license::Capability::FullSupplyChainFeed); let keys = defs::trusted_keys(); let trusted: Vec<(&str, ed25519_dalek::VerifyingKey)> = keys.iter().map(|(id, k)| (id.as_str(), *k)).collect(); - let outcome = update::run(&update::install_dir(), &trusted)?; - if !outcome.installed.is_empty() { + let outcome = update::run(&update::install_dir(), &trusted, full_feed)?; + if outcome.changed() { st.defs.reload(); + // A rules pack may have landed or been pruned; recompile so a new + // threat pack goes live without waiting for a restart. + if let Err(e) = engine::engine().reload_rules() { + eprintln!("scheduler: rules reload failed: {e}"); + } } Ok(outcome.installed.len()) } @@ -1086,38 +1089,39 @@ fn update(st: &DaemonState) -> Result { .current() .effective .allows(hound_api::license::Capability::FullSupplyChainFeed); - if !feed_licensed { - // Said once, plainly, in the update output — and never as a nag - // anywhere else. The built-in rules still reload below, and any - // packs already on disk stay loaded. - lines.push(format!( - "definitions: {}", - hound_api::license::Capability::FullSupplyChainFeed.explain_absence() - )); - } else { - match update::run(&update::install_dir(), &trusted) { - Ok(outcome) => { - lines.push(format!("definitions: {}", outcome.summary())); - lines.extend(outcome.log.iter().map(|l| format!(" {l}"))); - if !outcome.installed.is_empty() { - let loaded = st.defs.reload(); - lines.push(format!( - " loaded {} indicators from {} pack(s) [{}]", - loaded.indicators, - loaded.packs.len(), - loaded.version - )); - } - let sev = if outcome.rejected.is_empty() { "info" } else { "warn" }; - st.events - .push("update", sev, format!("definitions: {}", outcome.summary())); + + // Always run: the free community pack updates for everyone, and only the + // full feed and threat pack are gated on the licence. + match update::run(&update::install_dir(), &trusted, feed_licensed) { + Ok(outcome) => { + lines.push(format!("definitions: {}", outcome.summary())); + lines.extend(outcome.log.iter().map(|l| format!(" {l}"))); + if outcome.changed() { + let loaded = st.defs.reload(); + lines.push(format!( + " loaded {} indicators from {} pack(s) [{}]", + loaded.indicators, + loaded.packs.len(), + loaded.version + )); } - Err(e) => { - // Reported, not fatal. - lines.push(format!("definitions: could not update — {e}")); - st.events - .push("update", "warn", format!("definitions update failed: {e}")); + if !feed_licensed { + // Said once, plainly — never as a nag anywhere else. The + // community pack and every heuristic keep working. + lines.push(format!( + " {}", + hound_api::license::Capability::FullSupplyChainFeed.explain_absence() + )); } + let sev = if outcome.rejected.is_empty() { "info" } else { "warn" }; + st.events + .push("update", sev, format!("definitions: {}", outcome.summary())); + } + Err(e) => { + // Reported, not fatal. + lines.push(format!("definitions: could not update — {e}")); + st.events + .push("update", "warn", format!("definitions update failed: {e}")); } } diff --git a/crates/houndd/src/native.rs b/crates/houndd/src/native.rs index b17f381..b780e6e 100644 --- a/crates/houndd/src/native.rs +++ b/crates/houndd/src/native.rs @@ -208,6 +208,14 @@ impl ScanEngine for HoundEngine { Err(e) => Ok((false, "reload rules".to_string(), format!("{e}\n"))), } } + + fn reload_rules(&self) -> Result<()> { + // A newly installed threat pack changes what the ruleset detects, so + // verdicts cached under the old rules must not be trusted. + self.rules.reload()?; + self.cache.clear(); + Ok(()) + } } impl HoundEngine { diff --git a/crates/houndd/src/update.rs b/crates/houndd/src/update.rs index 32f9858..4937120 100644 --- a/crates/houndd/src/update.rs +++ b/crates/houndd/src/update.rs @@ -58,10 +58,19 @@ pub struct IndexEntry { #[derive(Debug, Clone, Deserialize, Default)] pub struct Index { + /// The full malicious-package feed — every ecosystem, baselines and + /// deltas. This is the Pro feed: fetched only with a licence. #[serde(default)] pub packs: Vec, - /// Signed YARA rules packs — the curated threat pack channel. Absent - /// from older indexes, which is an empty list, which is fine. + /// The free community pack: a snapshot of the most recent public OSV + /// malicious packages, so a Free install detects current threats out of + /// the box. Always fetched, licence or not. Pro is not "access to this + /// data" — it is the full corpus, the daily/near-real-time freshness and + /// the curated threat pack on top. + #[serde(default)] + pub community: Vec, + /// Signed YARA rules packs — the curated threat pack channel (Pro). + /// Absent from older indexes, which is an empty list, which is fine. #[serde(default)] pub rules: Vec, } @@ -72,6 +81,8 @@ pub struct Outcome { pub installed: Vec, pub already_current: Vec, pub rejected: Vec, + /// Superseded packs removed because the index stopped listing them. + pub pruned: Vec, pub log: Vec, } @@ -85,15 +96,32 @@ impl Outcome { self.rejected.len() ); } + let pruned = if self.pruned.is_empty() { + String::new() + } else { + format!(", {} superseded pack(s) removed", self.pruned.len()) + }; if self.installed.is_empty() { - return format!("definitions are up to date ({} pack(s))", self.already_current.len()); + if self.pruned.is_empty() { + return format!("definitions are up to date ({} pack(s))", self.already_current.len()); + } + return format!( + "definitions are up to date ({} pack(s){pruned})", + self.already_current.len() + ); } format!( - "{} pack(s) installed, {} already current", + "{} pack(s) installed, {} already current{pruned}", self.installed.len(), self.already_current.len() ) } + + /// Whether anything changed on disk — an install or a prune. Either + /// means the in-memory indicator set is now stale and must be reloaded. + pub fn changed(&self) -> bool { + !self.installed.is_empty() || !self.pruned.is_empty() + } } /// A filename from a remote index is untrusted input. @@ -157,8 +185,19 @@ pub fn fetch_index(base: &str) -> Result { serde_json::from_slice(&bytes).context("the definitions index is not valid JSON") } -/// Fetch, verify and install everything the index offers that we lack. -pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Result { +/// Fetch, verify and install what this installation is entitled to. +/// +/// `full_feed` is the Pro entitlement (`Capability::FullSupplyChainFeed`). +/// The free community pack is fetched regardless — a Free install still +/// detects current public malware. The full per-ecosystem feed and the +/// curated threat pack are fetched only with the entitlement, and pruned +/// away when it is absent, so a machine that stops paying drops cleanly to +/// exactly what a fresh Free install has: the community pack, no less. +pub fn run( + dir: &Path, + trusted: &[(&str, ed25519_dalek::VerifyingKey)], + full_feed: bool, +) -> Result { let base = base_url(); let mut out = Outcome::default(); @@ -169,12 +208,90 @@ pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Resul .with_context(|| format!("creating {}", dir.display()))?; let index = fetch_index(&base)?; - if index.packs.is_empty() { + if index.packs.is_empty() && index.community.is_empty() { out.log.push(format!("{base} offers no packs")); return Ok(out); } - for entry in &index.packs { + // The free community pack, always. + install_packs(&base, &index.community, dir, trusted, &mut out); + + // The full Pro feed, only with the entitlement. + if full_feed { + install_packs(&base, &index.packs, dir, trusted, &mut out); + install_rules_packs(&base, &index, trusted, &mut out); + } + + // Prune definition packs the installation should no longer hold. Two + // reasons a pack goes away: the index stopped listing it (a rebaseline + // folded the deltas into a fresh baseline), or the entitlement lapsed + // (the full feed is Pro; a Free machine keeps only the community pack). + // The keep-set is exactly what we are entitled to and the index still + // offers; everything else is deleted. Only pack-named files are touched. + let mut keep: std::collections::HashSet<&str> = + index.community.iter().map(|e| e.file.as_str()).collect(); + if full_feed { + keep.extend(index.packs.iter().map(|e| e.file.as_str())); + } + for name in prune(dir, ".pack", &keep) { + out.pruned.push(name); + } + + // The threat pack is Pro; without the entitlement, prune every rpack + // envelope AND every Hound-managed extract so a lapsed Pro drops to + // exactly the free ruleset — the built-in rules only. Custom `.yar` a + // user placed here themselves is left alone: only `hound-pack-*.yar` + // extracts are removed. + let rules_dir = rules_install_dir(); + let keep_rules: std::collections::HashSet<&str> = if full_feed { + index.rules.iter().map(|e| e.file.as_str()).collect() + } else { + std::collections::HashSet::new() + }; + for name in prune(&rules_dir, ".rpack", &keep_rules) { + out.pruned.push(name); + } + if !full_feed { + for name in prune_managed_extracts(&rules_dir) { + out.pruned.push(name); + } + } + + Ok(out) +} + +/// Remove Hound-installed rule extracts (`hound-pack-*.yar`) and the pack +/// VERSION marker. Used on downgrade: the threat pack is Pro, and a machine +/// without the entitlement keeps only the built-in rules compiled into the +/// binary. A user's own `.yar` files never match the prefix and are safe. +fn prune_managed_extracts(dir: &Path) -> Vec { + let mut removed = Vec::new(); + let Ok(entries) = std::fs::read_dir(dir) else { return removed }; + for entry in entries.flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue }; + let is_managed = name.starts_with("hound-pack-") && name.ends_with(".yar"); + if !is_managed && name != "VERSION" { + continue; + } + if std::fs::remove_file(&path).is_ok() { + removed.push(name.to_string()); + } + } + removed +} + +/// Fetch, verify and install a list of definition packs into `dir`. Shared +/// by the community channel and the full feed so both get the same +/// verify-before-install discipline and the same reporting. +fn install_packs( + base: &str, + entries: &[IndexEntry], + dir: &Path, + trusted: &[(&str, ed25519_dalek::VerifyingKey)], + out: &mut Outcome, +) { + for entry in entries { if !safe_pack_name(&entry.file) { out.rejected.push(entry.file.clone()); out.log @@ -242,10 +359,29 @@ pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Resul )); out.installed.push(entry.file.clone()); } +} - install_rules_packs(&base, &index, trusted, &mut out); - - Ok(out) +/// Delete files in `dir` ending in `extension` whose basename is not in +/// `keep`. Returns the names removed. Refuses to touch anything that is not +/// a plain pack filename, so a stray or hostile name cannot turn this into +/// an arbitrary delete. +fn prune(dir: &Path, extension: &str, keep: &std::collections::HashSet<&str>) -> Vec { + let mut removed = Vec::new(); + let Ok(entries) = std::fs::read_dir(dir) else { return removed }; + for entry in entries.flatten() { + let path = entry.path(); + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue }; + if !name.ends_with(extension) || !safe_name(name, extension) { + continue; + } + if keep.contains(name) { + continue; + } + if std::fs::remove_file(&path).is_ok() { + removed.push(name.to_string()); + } + } + removed } /// Fetch and install the signed YARA rules packs the index offers. @@ -333,7 +469,10 @@ fn install_rules_packs( std::fs::write(&tmp, bytes)?; std::fs::rename(&tmp, path) }; - let yar = dir.join(format!("{}.yar", rules.name)); + // Hound-managed extracts carry a distinguishing prefix so a + // downgrade can remove them without touching any custom `.yar` a + // user dropped in the rules directory themselves. + let yar = dir.join(format!("hound-pack-{}.yar", rules.name)); if let Err(e) = install(&yar, rules.yara.as_bytes()) .and_then(|_| install(&dir.join("VERSION"), rules.version.as_bytes())) .and_then(|_| install(&dest, &bytes)) @@ -351,6 +490,8 @@ fn install_rules_packs( )); out.installed.push(entry.file.clone()); } + // Pruning of superseded / unentitled `.rpack` envelopes happens in + // `run`, in one place, so the tier logic lives together. } /// A pack name that may become an installed basename: plain, short, no @@ -509,7 +650,7 @@ mod tests { #[test] fn refuses_to_run_with_an_empty_trust_store() { let dir = std::env::temp_dir().join(format!("hound-upd-nokey-{}", std::process::id())); - let e = run(&dir, &[]).unwrap_err(); + let e = run(&dir, &[], true).unwrap_err(); assert!(e.to_string().contains("no signing key")); } @@ -540,5 +681,108 @@ mod tests { // An index with no packs array is empty rather than fatal. let empty: Index = serde_json::from_slice(b"{}").unwrap(); assert!(empty.packs.is_empty()); + assert!(empty.rules.is_empty()); + } + + #[test] + fn an_index_with_deltas_is_just_more_packs() { + // The delta feed needs no schema change: a baseline and its daily + // deltas are all ordinary pack entries, and the client fetches + // whichever it lacks. + let idx: Index = serde_json::from_slice( + br#"{"packs":[ + {"file":"npm-2026.08.01.pack","sha256":"aa","version":"2026.08.01"}, + {"file":"npm-2026.08.21.delta.pack","sha256":"bb","version":"2026.08.21"} + ]}"#, + ) + .unwrap(); + assert_eq!(idx.packs.len(), 2); + assert!(safe_pack_name("npm-2026.08.21.delta.pack")); + } + + // ── pruning: what makes the delta feed self-cleaning ── + + #[test] + fn prune_removes_only_unlisted_packs_and_nothing_else() { + let dir = std::env::temp_dir().join(format!("hound-prune-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + + // A current baseline, a stale superseded baseline, and files that + // are not packs and must never be touched. + std::fs::write(dir.join("npm-2026.08.21.pack"), b"keep").unwrap(); + std::fs::write(dir.join("npm-2026.07.01.pack"), b"stale").unwrap(); + std::fs::write(dir.join("VERSION"), b"2026.08.21").unwrap(); + std::fs::write(dir.join("notes.txt"), b"leave me").unwrap(); + + let keep: std::collections::HashSet<&str> = + ["npm-2026.08.21.pack"].into_iter().collect(); + let removed = prune(&dir, ".pack", &keep); + + assert_eq!(removed, vec!["npm-2026.07.01.pack".to_string()]); + assert!(dir.join("npm-2026.08.21.pack").exists(), "the listed pack stays"); + assert!(!dir.join("npm-2026.07.01.pack").exists(), "the stale pack is gone"); + assert!(dir.join("VERSION").exists(), "non-pack files are never touched"); + assert!(dir.join("notes.txt").exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn prune_leaves_everything_when_the_index_lists_it_all() { + let dir = std::env::temp_dir().join(format!("hound-prune2-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("npm-2026.08.21.pack"), b"a").unwrap(); + std::fs::write(dir.join("npm-2026.08.22.delta.pack"), b"b").unwrap(); + let keep: std::collections::HashSet<&str> = + ["npm-2026.08.21.pack", "npm-2026.08.22.delta.pack"].into_iter().collect(); + assert!(prune(&dir, ".pack", &keep).is_empty()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn changed_tracks_installs_and_prunes() { + let mut o = Outcome::default(); + assert!(!o.changed()); + o.pruned.push("old.pack".into()); + assert!(o.changed(), "a prune alone must trigger a reload"); + assert!(o.summary().contains("removed")); + } + + // ── Option C: the free community channel and the downgrade path ── + + #[test] + fn the_index_splits_community_from_the_paid_feed() { + let idx: Index = serde_json::from_slice( + br#"{"packs":[{"file":"npm-2026.08.21.pack","sha256":"a","version":"x"}], + "community":[{"file":"community-2026.08.21.pack","sha256":"b","version":"x"}], + "rules":[{"file":"hound-linux-2026.08.21.rpack","sha256":"c","version":"x"}]}"#, + ) + .unwrap(); + assert_eq!(idx.packs.len(), 1); + assert_eq!(idx.community.len(), 1); + assert_eq!(idx.rules.len(), 1); + } + + #[test] + fn downgrade_removes_managed_extracts_but_spares_custom_rules() { + let dir = std::env::temp_dir().join(format!("hound-ext-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + // Hound-managed extract + version marker, plus a user's own rule. + std::fs::write(dir.join("hound-pack-hound-linux.yar"), b"rule x {condition:true}").unwrap(); + std::fs::write(dir.join("VERSION"), b"2026.08.21").unwrap(); + std::fs::write(dir.join("my-custom.yar"), b"rule c {condition:true}").unwrap(); + + let mut removed = prune_managed_extracts(&dir); + removed.sort(); + assert_eq!(removed, vec!["VERSION".to_string(), "hound-pack-hound-linux.yar".to_string()]); + assert!(!dir.join("hound-pack-hound-linux.yar").exists()); + assert!(!dir.join("VERSION").exists()); + assert!( + dir.join("my-custom.yar").exists(), + "a user's own rule must never be pruned as a downgrade side effect" + ); + let _ = std::fs::remove_dir_all(&dir); } } diff --git a/tools/publish-rules-pack.sh b/tools/publish-rules-pack.sh index 3d8f69f..c573d10 100755 --- a/tools/publish-rules-pack.sh +++ b/tools/publish-rules-pack.sh @@ -45,35 +45,38 @@ chmod 644 "$DEST/.$base.tmp" mv -f "$DEST/.$base.tmp" "$DEST/$base" log "published $base" -# Rebuild the index over everything on disk — definition packs and rules -# packs both. Same logic as refresh-definitions.sh so the two agree. +# Rebuild the index over everything on disk — every definition pack +# (baselines and deltas) and every rules pack. Identical logic to +# refresh-definitions.sh so running either keeps the delta feed intact; +# version comes from inside the signed payload, not the filename. python3 - "$DEST" <<'PY' -import hashlib, json, os, sys +import base64, hashlib, json, os, sys dest = sys.argv[1] -def newest_by_family(suffix, strip): - newest = {} - for f in sorted(os.listdir(dest)): - if f.endswith(suffix): - newest[f.rsplit("-", 1)[0]] = f - out = [] - for _, f in sorted(newest.items()): - p = os.path.join(dest, f) - out.append({ - "file": f, - "sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(), - "size": os.path.getsize(p), - "version": f.rsplit("-", 1)[1][:-strip], - }) - return out -packs = newest_by_family(".pack", 5) -rules = newest_by_family(".rpack", 6) +def entry(f): + raw = open(os.path.join(dest, f), "rb").read() + try: + version = json.loads(base64.b64decode(json.loads(raw)["payload"])).get("version", "") + except Exception: + version = "" + return {"file": f, "sha256": hashlib.sha256(raw).hexdigest(), + "size": len(raw), "version": version} +packs, community, rules = [], [], [] +for f in sorted(os.listdir(dest)): + if not os.path.isfile(os.path.join(dest, f)): + continue + if f.endswith(".rpack"): + rules.append(entry(f)) + elif f.startswith("community-") and f.endswith(".pack"): + community.append(entry(f)) + elif f.endswith(".pack"): + packs.append(entry(f)) tmp = os.path.join(dest, ".index.json.tmp") with open(tmp, "w") as fh: - json.dump({"packs": packs, "rules": rules}, fh, indent=2) + json.dump({"packs": packs, "community": community, "rules": rules}, fh, indent=2) fh.write("\n") os.chmod(tmp, 0o644) os.replace(tmp, os.path.join(dest, "index.json")) -print(f"index.json lists {len(packs)} definition pack(s) and {len(rules)} rules pack(s)") +print(f"index.json: {len(packs)} feed, {len(community)} community, {len(rules)} rules pack(s)") PY log "done" diff --git a/tools/refresh-definitions.sh b/tools/refresh-definitions.sh index 246d9a5..45db986 100755 --- a/tools/refresh-definitions.sh +++ b/tools/refresh-definitions.sh @@ -1,10 +1,13 @@ #!/usr/bin/env bash # -# Rebuild the definition feed from OSV and publish it. +# Rebuild the definition feed from OSV and publish it — incrementally. # -# The client-side update machinery is only worth having if the source it -# points at actually moves. Everything below already existed as separate -# manual steps; this is the thing that runs them on a schedule. +# The old version rebuilt the full per-ecosystem pack every night, so its +# sha256 changed daily and every client re-downloaded the whole 44 MB npm +# pack even when a handful of records had been added. This version publishes +# an immutable baseline plus small daily deltas: the client (which already +# fetches only packs whose sha256 it lacks) downloads the baseline once and +# then kilobytes a day. See crates/hound-defs/examples/build-delta.rs. # # Publishing is atomic per pack: each is written to a temporary name in the # destination directory and renamed into place, so an agent fetching mid-run @@ -12,8 +15,12 @@ # what tells an agent a pack exists — writing it first would advertise files # that are not there yet. # -# Old packs are kept. An agent that has not checked in for a while still has -# a URL that resolves, and disk is cheaper than a failed update. +# When build-delta decides to REBASELINE an ecosystem (no baseline yet, or +# too many deltas piled up), every existing file for that ecosystem is moved +# to archive/ before the fresh baseline lands; the index then lists only the +# new baseline, and clients delete their now-unlisted copies. That is what +# keeps both the server directory and every client's defs dir from growing +# without bound. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -21,23 +28,26 @@ KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}" DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}" WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}" VERSION="$(date -u +%Y.%m.%d)" +CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } [ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; } [ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; } +mkdir -p "$DEST/archive" mkdir -p "$WORK" -log "refreshing definitions for $VERSION" +log "refreshing definitions for $VERSION (incremental)" # The OSV exports are large and change slowly; a failure to fetch one # ecosystem must not discard the others. python3 "$ROOT/tools/ingest-osv.py" "$WORK" -BUILDER="$ROOT/target/release/examples/build-pack" -if [ ! -x "$BUILDER" ]; then - log "building the pack builder" - ( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack ) +BUILDER="$ROOT/target/release/examples/build-delta" +COMMUNITY_BUILDER="$ROOT/target/release/examples/build-community" +if [ ! -x "$BUILDER" ] || [ ! -x "$COMMUNITY_BUILDER" ]; then + log "building the definition builders" + ( cd "$ROOT" && cargo build --release -p hound-defs --example build-delta --example build-community ) fi # Outside $WORK on purpose: the loop below treats every directory in $WORK as @@ -46,7 +56,7 @@ fi STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")" trap 'rm -rf "$STAGE"' EXIT -published=0 +changed=0 for dir in "$WORK"/*/; do eco="$(basename "$dir")" # Lower-case, and '.' is not wanted in a filename component. @@ -59,70 +69,109 @@ for dir in "$WORK"/*/; do log "$eco: no malicious records, skipping" continue fi - pack="$STAGE/${name}-${VERSION}.pack" - if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then - log "$eco: built $(basename "$pack") from $count record(s)" - published=$((published + 1)) - else - log "$eco: BUILD FAILED — keeping the previous pack" - fi + + # build-delta reads the published dir, decides, and writes to staging. + decision="$("$BUILDER" "$dir" "$DEST" "$name" "$STAGE" "$KEY" "$VERSION" "$CREATED" 2>&1)" || { + log "$eco: BUILD FAILED — keeping the previous pack(s): $decision" + continue + } + log "$eco: $decision" + + case "$decision" in + REBASELINE*) + # Retire every existing file for this ecosystem, then install the + # fresh baseline. Matches '-' so 'go' never touches + # another ecosystem's files. + for old in "$DEST/${name}-"[0-9]*.pack; do + [ -e "$old" ] && mv -f "$old" "$DEST/archive/" + done + changed=1 ;; + DELTA*) + changed=1 ;; + UNCHANGED*) + : ;; + esac done -if [ "$published" -eq 0 ]; then - # Never publish an empty feed. An agent that installs it would report a - # clean machine with no indicators loaded, which is worse than one that - # keeps yesterday's. - log "nothing built; leaving the published feed untouched" - exit 1 -fi - -# Packs first, then the index that advertises them. +# Install whatever landed in staging (baselines and deltas), atomically. for pack in "$STAGE"/*.pack; do + [ -e "$pack" ] || continue base="$(basename "$pack")" cp "$pack" "$DEST/.$base.tmp" chmod 644 "$DEST/.$base.tmp" mv -f "$DEST/.$base.tmp" "$DEST/$base" done -python3 - "$DEST" "$VERSION" <<'PY' -import hashlib, json, os, sys -dest, version = sys.argv[1], sys.argv[2] +# Rebuild the free community pack from the freshly-published feed: a recent +# subset a Free install can use. Rebuilt only when the feed actually changed +# — Pro's value is DAILY freshness, so the free snapshot lags on purpose. +if [ "$changed" -eq 1 ]; then + cpack="$STAGE/community-${VERSION}.pack" + if "$COMMUNITY_BUILDER" "$DEST" "$cpack" "$KEY" "$VERSION" "$CREATED" "${HOUND_COMMUNITY_LIMIT:-5000}"; then + for old in "$DEST"/community-*.pack; do + [ -e "$old" ] && mv -f "$old" "$DEST/archive/" + done + base="$(basename "$cpack")" + cp "$cpack" "$DEST/.$base.tmp"; chmod 644 "$DEST/.$base.tmp"; mv -f "$DEST/.$base.tmp" "$DEST/$base" + log "community: published $base" + else + log "community: build failed — keeping the previous community pack" + fi +fi -# One entry per family: the newest file. Older ones stay on disk so existing -# URLs keep resolving, but the index only ever advertises current data. -def newest_by_family(suffix, strip): - newest = {} - for f in sorted(os.listdir(dest)): - if f.endswith(suffix): - newest[f.rsplit("-", 1)[0]] = f - out = [] - for _, f in sorted(newest.items()): - p = os.path.join(dest, f) - out.append({ - "file": f, - "sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(), - "size": os.path.getsize(p), - "version": f.rsplit("-", 1)[1][:-strip], - }) - return out +# Never publish an empty feed: if nothing changed AND the destination has no +# packs at all, something is wrong — leave whatever is there untouched. +if [ "$changed" -eq 0 ] && [ -z "$(find "$DEST" -maxdepth 1 -name '*.pack' -print -quit)" ]; then + log "nothing to publish and no existing feed; leaving it untouched" + exit 1 +fi +if [ "$changed" -eq 0 ]; then + log "no ecosystem changed today; feed already current, refreshing index only" +fi -# Definition packs (.pack) are rebuilt daily by this script. Rules packs -# (.rpack — the curated threat pack) are built and published separately by -# tools/publish-rules-pack.sh, but the index advertises both, so it is -# rebuilt from whatever .rpack files are on disk rather than dropping them. -packs = newest_by_family(".pack", 5) -rules = newest_by_family(".rpack", 6) +python3 - "$DEST" <<'PY' +import base64, hashlib, json, os, sys +dest = sys.argv[1] + +# List EVERY current pack in the served directory — every baseline and every +# delta. The client fetches whatever it is missing and deletes what the index +# stops listing, so "current" is exactly "present here" (archive/ is a +# subdirectory and is not walked). The version comes from inside the signed +# payload rather than from the filename, so the delta naming +# (npm-2026.08.22.delta.pack) needs no special parsing. +def entry(f): + p = os.path.join(dest, f) + raw = open(p, "rb").read() + try: + version = json.loads(base64.b64decode(json.loads(raw)["payload"])).get("version", "") + except Exception: + version = "" + return {"file": f, "sha256": hashlib.sha256(raw).hexdigest(), + "size": len(raw), "version": version} + +# Route by name: community-*.pack is the free channel; every other .pack is +# the full Pro feed; .rpack is the curated threat pack (Pro). +packs, community, rules = [], [], [] +for f in sorted(os.listdir(dest)): + if not os.path.isfile(os.path.join(dest, f)): + continue + if f.endswith(".rpack"): + rules.append(entry(f)) + elif f.startswith("community-") and f.endswith(".pack"): + community.append(entry(f)) + elif f.endswith(".pack"): + packs.append(entry(f)) tmp = os.path.join(dest, ".index.json.tmp") with open(tmp, "w") as fh: - json.dump({"packs": packs, "rules": rules}, fh, indent=2) + json.dump({"packs": packs, "community": community, "rules": rules}, fh, indent=2) fh.write("\n") os.chmod(tmp, 0o644) os.replace(tmp, os.path.join(dest, "index.json")) -print(f"index.json lists {len(packs)} definition pack(s) and {len(rules)} rules pack(s)") +print(f"index.json: {len(packs)} feed pack(s), {len(community)} community pack(s), {len(rules)} rules pack(s)") PY -log "published $published pack(s) for $VERSION" +log "definitions refresh complete for $VERSION" # Clear the extracted records. They are the bulk of the scratch — 1.2 GB # after a single run — and the daily timer regenerates them every time. The