//! Incremental definition builder: emit a small daily delta instead of a //! fresh 44 MB pack every night. //! //! build-delta [created] //! //! The problem it solves: the client already downloads only packs whose //! sha256 it does not have, but the old nightly rebuilt the full //! per-ecosystem pack every day, so its hash changed daily and every client //! re-fetched the whole thing. Here each day's *new* indicators go in a //! small delta pack; the baseline stays byte-identical between rebuilds, so //! a client fetches it once and then only the daily deltas (kilobytes). //! //! What it does, for one ecosystem: //! 1. Parse today's full indicator set from the OSV export. //! 2. Read the currently-published baseline + deltas to learn which //! indicators the feed already carries. //! 3. Decide: //! - no baseline yet, or too many deltas piled up → REBASELINE: //! write a fresh full baseline (the client re-fetches it once and //! the old files are dropped from the index, so it self-cleans). //! - new indicators since yesterday → DELTA: write just those. //! - nothing new → UNCHANGED: write nothing. //! 4. Print one decision line for the publish script to act on. //! //! It only ever writes into the staging dir and only ever reads the //! published dir — moving files into place and rebuilding the index is the //! publish script's job, so this stays a pure, testable computation. use ed25519_dalek::{SigningKey, VerifyingKey}; use hound_defs::{osv, pack, Indicator, Pack}; use std::collections::BTreeSet; use std::path::Path; /// Fold the deltas back into a fresh baseline once this many have piled up, /// so a new install never has to replay an unbounded chain and the served /// directory stays small. One baseline plus at most this many deltas. const REBASELINE_AFTER: usize = 14; fn die(msg: impl std::fmt::Display) -> ! { eprintln!("build-delta: {msg}"); std::process::exit(1); } fn load_key(path: &str) -> SigningKey { let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}"))); let seed: [u8; 32] = bytes .get(..32) .and_then(|s| s.try_into().ok()) .unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key"))); SigningKey::from_bytes(&seed) } /// A stable identity for an indicator, matching build-pack's dedup key. fn key_of(i: &Indicator) -> (String, String, String) { (i.ecosystem.clone(), i.name.clone(), i.id.clone()) } /// Every `.pack` in `dir` whose name is `-…`, verified against /// our own key, flattened to the indicator identities they already carry. /// Also returns how many delta files were seen, for the re-baseline call. fn published_state( dir: &Path, eco: &str, pubkey: &VerifyingKey, ) -> (BTreeSet<(String, String, String)>, usize, bool) { let trusted = [("hound-2026", *pubkey)]; let mut known = BTreeSet::new(); let mut deltas = 0usize; let mut has_baseline = false; let prefix = format!("{eco}-"); for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() { let path = entry.path(); let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue }; if !name.ends_with(".pack") || !name.starts_with(&prefix) { continue; } // Guard against "go-" matching "golang-": the char after the prefix // must start a version (a digit). if !name[prefix.len()..].starts_with(|c: char| c.is_ascii_digit()) { continue; } let Ok(text) = std::fs::read_to_string(&path) else { continue }; let Ok(signed) = serde_json::from_str::(&text) else { continue }; let Ok(p) = pack::verify(&signed, &trusted) else { eprintln!("build-delta: WARNING ignoring unverifiable {name}"); continue; }; for ind in p.indicators { known.insert(key_of(&ind)); } if name.contains(".delta.pack") { deltas += 1; } else { has_baseline = true; } } (known, deltas, has_baseline) } /// Parse today's full indicator set for the ecosystem from its OSV export. fn todays_indicators(osv_dir: &Path) -> Vec { let mut out = Vec::new(); for entry in std::fs::read_dir(osv_dir) .unwrap_or_else(|e| die(format!("reading {}: {e}", osv_dir.display()))) .flatten() { let path = entry.path(); if path.extension().is_none_or(|e| e != "json") { continue; } if let Ok(text) = std::fs::read_to_string(&path) { out.extend(osv::parse_record(&text)); } } out.sort_by(|a, b| key_of(a).cmp(&key_of(b))); out.dedup_by(|a, b| key_of(a) == key_of(b)); out } fn write_pack(pack_data: &Pack, key: &SigningKey, out: &Path) { let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into()); let signed = pack::sign(pack_data, key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}"))); std::fs::write(out, serde_json::to_string(&signed).unwrap()) .unwrap_or_else(|e| die(format!("writing {}: {e}", out.display()))); } fn main() { let a: Vec = std::env::args().skip(1).collect(); if a.len() < 6 { die("usage: build-delta [created]"); } let (osv_dir, published, eco, staging, key_path, version) = (Path::new(&a[0]), Path::new(&a[1]), &a[2], Path::new(&a[3]), &a[4], &a[5]); let created = a.get(6).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into()); let key = load_key(key_path); let pubkey = key.verifying_key(); let today = todays_indicators(osv_dir); if today.is_empty() { // The ingest produced nothing for this ecosystem; never publish an // empty pack (a client that installed it would show zero indicators). println!("UNCHANGED {eco} (no records ingested)"); return; } let (known, deltas, has_baseline) = published_state(published, eco, &pubkey); let sources = vec!["ossf/malicious-packages (Apache-2.0)".to_string(), "osv.dev".to_string()]; let rebaseline = !has_baseline || deltas >= REBASELINE_AFTER; if rebaseline { let out = staging.join(format!("{eco}-{version}.pack")); write_pack( &Pack { version: version.clone(), created, sources, indicators: today.clone() }, &key, &out, ); // The publish script archives every existing {eco}-* file when it // sees REBASELINE, so the fresh baseline stands alone. println!("REBASELINE {eco} {} indicators -> {}", today.len(), out.display()); return; } let new: Vec = today .into_iter() .filter(|i| !known.contains(&key_of(i))) .collect(); if new.is_empty() { println!("UNCHANGED {eco} (no new indicators)"); return; } let out = staging.join(format!("{eco}-{version}.delta.pack")); write_pack( &Pack { version: version.clone(), created, sources, indicators: new.clone() }, &key, &out, ); println!("DELTA {eco} {} new indicator(s) -> {}", new.len(), out.display()); }