Antivirus/crates/hound-defs/examples/build-delta.rs

//! Incremental definition builder: emit a small daily delta instead of a
//! fresh 44 MB pack every night.
//!
//!     build-delta <osv-dir> <published-dir> <ecosystem> <staging-dir> <key> <version> [created]
//!
//! The problem it solves: the client already downloads only packs whose
//! sha256 it does not have, but the old nightly rebuilt the full
//! per-ecosystem pack every day, so its hash changed daily and every client
//! re-fetched the whole thing. Here each day's *new* indicators go in a
//! small delta pack; the baseline stays byte-identical between rebuilds, so
//! a client fetches it once and then only the daily deltas (kilobytes).
//!
//! What it does, for one ecosystem:
//!   1. Parse today's full indicator set from the OSV export.
//!   2. Read the currently-published baseline + deltas to learn which
//!      indicators the feed already carries.
//!   3. Decide:
//!        - no baseline yet, or too many deltas piled up → REBASELINE:
//!          write a fresh full baseline (the client re-fetches it once and
//!          the old files are dropped from the index, so it self-cleans).
//!        - new indicators since yesterday → DELTA: write just those.
//!        - nothing new → UNCHANGED: write nothing.
//!   4. Print one decision line for the publish script to act on.
//!
//! It only ever writes into the staging dir and only ever reads the
//! published dir — moving files into place and rebuilding the index is the
//! publish script's job, so this stays a pure, testable computation.

use ed25519_dalek::{SigningKey, VerifyingKey};
use hound_defs::{osv, pack, Indicator, Pack};
use std::collections::BTreeSet;
use std::path::Path;

/// Fold the deltas back into a fresh baseline once this many have piled up,
/// so a new install never has to replay an unbounded chain and the served
/// directory stays small. One baseline plus at most this many deltas.
const REBASELINE_AFTER: usize = 14;

fn die(msg: impl std::fmt::Display) -> ! {
    eprintln!("build-delta: {msg}");
    std::process::exit(1);
}

fn load_key(path: &str) -> SigningKey {
    let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}")));
    let seed: [u8; 32] = bytes
        .get(..32)
        .and_then(|s| s.try_into().ok())
        .unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key")));
    SigningKey::from_bytes(&seed)
}

/// A stable identity for an indicator, matching build-pack's dedup key.
fn key_of(i: &Indicator) -> (String, String, String) {
    (i.ecosystem.clone(), i.name.clone(), i.id.clone())
}

/// Every `.pack` in `dir` whose name is `<eco>-<digit>…`, verified against
/// our own key, flattened to the indicator identities they already carry.
/// Also returns how many delta files were seen, for the re-baseline call.
fn published_state(
    dir: &Path,
    eco: &str,
    pubkey: &VerifyingKey,
) -> (BTreeSet<(String, String, String)>, usize, bool) {
    let trusted = [("hound-2026", *pubkey)];
    let mut known = BTreeSet::new();
    let mut deltas = 0usize;
    let mut has_baseline = false;
    let prefix = format!("{eco}-");

    for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() {
        let path = entry.path();
        let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
        if !name.ends_with(".pack") || !name.starts_with(&prefix) {
            continue;
        }
        // Guard against "go-" matching "golang-": the char after the prefix
        // must start a version (a digit).
        if !name[prefix.len()..].starts_with(|c: char| c.is_ascii_digit()) {
            continue;
        }
        let Ok(text) = std::fs::read_to_string(&path) else { continue };
        let Ok(signed) = serde_json::from_str::<pack::SignedPack>(&text) else { continue };
        let Ok(p) = pack::verify(&signed, &trusted) else {
            eprintln!("build-delta: WARNING ignoring unverifiable {name}");
            continue;
        };
        for ind in p.indicators {
            known.insert(key_of(&ind));
        }
        if name.contains(".delta.pack") {
            deltas += 1;
        } else {
            has_baseline = true;
        }
    }
    (known, deltas, has_baseline)
}

/// Parse today's full indicator set for the ecosystem from its OSV export.
fn todays_indicators(osv_dir: &Path) -> Vec<Indicator> {
    let mut out = Vec::new();
    for entry in std::fs::read_dir(osv_dir)
        .unwrap_or_else(|e| die(format!("reading {}: {e}", osv_dir.display())))
        .flatten()
    {
        let path = entry.path();
        if path.extension().is_none_or(|e| e != "json") {
            continue;
        }
        if let Ok(text) = std::fs::read_to_string(&path) {
            out.extend(osv::parse_record(&text));
        }
    }
    out.sort_by(|a, b| key_of(a).cmp(&key_of(b)));
    out.dedup_by(|a, b| key_of(a) == key_of(b));
    out
}

fn write_pack(pack_data: &Pack, key: &SigningKey, out: &Path) {
    let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into());
    let signed = pack::sign(pack_data, key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}")));
    std::fs::write(out, serde_json::to_string(&signed).unwrap())
        .unwrap_or_else(|e| die(format!("writing {}: {e}", out.display())));
}

fn main() {
    let a: Vec<String> = std::env::args().skip(1).collect();
    if a.len() < 6 {
        die("usage: build-delta <osv-dir> <published-dir> <ecosystem> <staging-dir> <key> <version> [created]");
    }
    let (osv_dir, published, eco, staging, key_path, version) =
        (Path::new(&a[0]), Path::new(&a[1]), &a[2], Path::new(&a[3]), &a[4], &a[5]);
    let created = a.get(6).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into());

    let key = load_key(key_path);
    let pubkey = key.verifying_key();

    let today = todays_indicators(osv_dir);
    if today.is_empty() {
        // The ingest produced nothing for this ecosystem; never publish an
        // empty pack (a client that installed it would show zero indicators).
        println!("UNCHANGED {eco} (no records ingested)");
        return;
    }

    let (known, deltas, has_baseline) = published_state(published, eco, &pubkey);

    let sources = vec!["ossf/malicious-packages (Apache-2.0)".to_string(), "osv.dev".to_string()];
    let rebaseline = !has_baseline || deltas >= REBASELINE_AFTER;

    if rebaseline {
        let out = staging.join(format!("{eco}-{version}.pack"));
        write_pack(
            &Pack { version: version.clone(), created, sources, indicators: today.clone() },
            &key,
            &out,
        );
        // The publish script archives every existing {eco}-* file when it
        // sees REBASELINE, so the fresh baseline stands alone.
        println!("REBASELINE {eco} {} indicators -> {}", today.len(), out.display());
        return;
    }

    let new: Vec<Indicator> = today
        .into_iter()
        .filter(|i| !known.contains(&key_of(i)))
        .collect();
    if new.is_empty() {
        println!("UNCHANGED {eco} (no new indicators)");
        return;
    }

    let out = staging.join(format!("{eco}-{version}.delta.pack"));
    write_pack(
        &Pack { version: version.clone(), created, sources, indicators: new.clone() },
        &key,
        &out,
    );
    println!("DELTA {eco} {} new indicator(s) -> {}", new.len(), out.display());
}