defs: incremental delta feed + free community tier (Option C)
Two changes to how definitions are distributed, sharing one mechanism. Incremental updates. The old nightly rebuilt the full per-ecosystem pack every day, so its hash changed and every client re-downloaded 44 MB of npm daily. Now the server publishes an immutable baseline plus small daily delta packs (crates/hound-defs/examples/build-delta.rs); the client — which already fetches only packs whose sha256 it lacks — pulls the baseline once and then kilobytes a day. When deltas pile up the builder folds them into a fresh baseline and drops the old files from the index; the client prunes whatever the index stops listing, so both the server dir and every client's defs dir stay bounded. Verified end to end: day-2 fetched only the delta (baseline untouched), day-3 rebaseline pruned the superseded packs. Free community tier. Free now gets a recent subset of the public OSV feed (build-community.rs, ~5,000 newest indicators) so a Free install detects current threats out of the box — not just the heuristics. Pro is the full 235k corpus, daily/near-real-time freshness, and the curated threat pack. The client always fetches the community channel and gates the full feed + threat pack on the licence; a lapse prunes both back to exactly what a fresh Free install has — community pack + built-in rules — while leaving any custom .yar the user placed themselves untouched. reload_rules() lets a new threat pack go live without a daemon restart. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
dev
parent
2cd68ebc57
commit
e92e865ff5
8 changed files with 748 additions and 133 deletions
115
crates/hound-defs/examples/build-community.rs
Normal file
115
crates/hound-defs/examples/build-community.rs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
//! Build the free community pack: a snapshot of the most recent public OSV
|
||||
//! malicious packages, so a Free install detects current threats out of the
|
||||
//! box.
|
||||
//!
|
||||
//! build-community <published-dir> <out.pack> <key> <version> [created] [limit]
|
||||
//!
|
||||
//! This is the free half of the feed. It is deliberately a *recent* subset,
|
||||
//! not the full corpus: Pro is not "access to public data" — anyone can pull
|
||||
//! OSV — it is the full 235k history, the daily/near-real-time freshness, and
|
||||
//! the curated threat pack. The community pack costs us nothing to give away
|
||||
//! (it is public data) and makes Free genuinely useful.
|
||||
//!
|
||||
//! "Recent" is approximated by sorting indicator ids descending: the OSSF
|
||||
//! feed mints `MAL-YYYY-NNNNN`, so the newest advisories sort first. The pack
|
||||
//! is built from the already-published, already-verified feed, so there is a
|
||||
//! single source of truth and no second ingest to drift.
|
||||
|
||||
use ed25519_dalek::{SigningKey, VerifyingKey};
|
||||
use hound_defs::{pack, Indicator, Pack};
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
/// How many indicators the free pack carries. A few thousand recent ones —
|
||||
/// enough to be useful, small enough that the full feed is clearly more.
|
||||
const DEFAULT_LIMIT: usize = 5000;
|
||||
|
||||
fn die(m: impl std::fmt::Display) -> ! {
|
||||
eprintln!("build-community: {m}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
fn load_key(path: &str) -> SigningKey {
|
||||
let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}")));
|
||||
let seed: [u8; 32] = bytes
|
||||
.get(..32)
|
||||
.and_then(|s| s.try_into().ok())
|
||||
.unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key")));
|
||||
SigningKey::from_bytes(&seed)
|
||||
}
|
||||
|
||||
/// Every indicator in the published full feed (baselines + deltas), verified
|
||||
/// against our own key. Community packs are excluded so this does not feed on
|
||||
/// itself.
|
||||
fn all_published(dir: &Path, pubkey: &VerifyingKey) -> Vec<Indicator> {
|
||||
let trusted = [("hound-2026", *pubkey)];
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut out = Vec::new();
|
||||
for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
|
||||
if !name.ends_with(".pack") || name.starts_with("community-") {
|
||||
continue;
|
||||
}
|
||||
let Ok(text) = std::fs::read_to_string(&path) else { continue };
|
||||
let Ok(signed) = serde_json::from_str::<pack::SignedPack>(&text) else { continue };
|
||||
let Ok(p) = pack::verify(&signed, &trusted) else {
|
||||
eprintln!("build-community: WARNING ignoring unverifiable {name}");
|
||||
continue;
|
||||
};
|
||||
for ind in p.indicators {
|
||||
let key = (ind.ecosystem.clone(), ind.name.clone(), ind.id.clone());
|
||||
if seen.insert(key) {
|
||||
out.push(ind);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let a: Vec<String> = std::env::args().skip(1).collect();
|
||||
if a.len() < 4 {
|
||||
die("usage: build-community <published-dir> <out.pack> <key> <version> [created] [limit]");
|
||||
}
|
||||
let (published, out_path, key_path, version) =
|
||||
(Path::new(&a[0]), &a[1], &a[2], &a[3]);
|
||||
let created = a.get(4).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into());
|
||||
let limit: usize = a.get(5).and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_LIMIT);
|
||||
|
||||
let key = load_key(key_path);
|
||||
let mut all = all_published(published, &key.verifying_key());
|
||||
if all.is_empty() {
|
||||
die("the published feed has no indicators to draw a community pack from");
|
||||
}
|
||||
|
||||
// Most recent first (MAL-YYYY-NNNNN sorts by recency descending), then
|
||||
// take the head. A stable secondary sort on the full identity keeps the
|
||||
// output deterministic when ids collide.
|
||||
all.sort_by(|x, y| {
|
||||
y.id.cmp(&x.id).then_with(|| {
|
||||
(&x.ecosystem, &x.name).cmp(&(&y.ecosystem, &y.name))
|
||||
})
|
||||
});
|
||||
all.truncate(limit);
|
||||
|
||||
// Re-sort into the canonical build order so the pack bytes are stable.
|
||||
all.sort_by(|x, y| {
|
||||
(&x.ecosystem, &x.name, &x.id).cmp(&(&y.ecosystem, &y.name, &y.id))
|
||||
});
|
||||
|
||||
let p = Pack {
|
||||
version: version.clone(),
|
||||
created,
|
||||
sources: vec![
|
||||
"ossf/malicious-packages (Apache-2.0), recent subset".to_string(),
|
||||
"osv.dev".to_string(),
|
||||
],
|
||||
indicators: all.clone(),
|
||||
};
|
||||
let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into());
|
||||
let signed = pack::sign(&p, &key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}")));
|
||||
std::fs::write(out_path, serde_json::to_string(&signed).unwrap())
|
||||
.unwrap_or_else(|e| die(format!("writing {out_path}: {e}")));
|
||||
eprintln!("wrote {out_path} — {} recent indicators, version {version}", all.len());
|
||||
}
|
||||
182
crates/hound-defs/examples/build-delta.rs
Normal file
182
crates/hound-defs/examples/build-delta.rs
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
//! Incremental definition builder: emit a small daily delta instead of a
|
||||
//! fresh 44 MB pack every night.
|
||||
//!
|
||||
//! build-delta <osv-dir> <published-dir> <ecosystem> <staging-dir> <key> <version> [created]
|
||||
//!
|
||||
//! The problem it solves: the client already downloads only packs whose
|
||||
//! sha256 it does not have, but the old nightly rebuilt the full
|
||||
//! per-ecosystem pack every day, so its hash changed daily and every client
|
||||
//! re-fetched the whole thing. Here each day's *new* indicators go in a
|
||||
//! small delta pack; the baseline stays byte-identical between rebuilds, so
|
||||
//! a client fetches it once and then only the daily deltas (kilobytes).
|
||||
//!
|
||||
//! What it does, for one ecosystem:
|
||||
//! 1. Parse today's full indicator set from the OSV export.
|
||||
//! 2. Read the currently-published baseline + deltas to learn which
|
||||
//! indicators the feed already carries.
|
||||
//! 3. Decide:
|
||||
//! - no baseline yet, or too many deltas piled up → REBASELINE:
|
||||
//! write a fresh full baseline (the client re-fetches it once and
|
||||
//! the old files are dropped from the index, so it self-cleans).
|
||||
//! - new indicators since yesterday → DELTA: write just those.
|
||||
//! - nothing new → UNCHANGED: write nothing.
|
||||
//! 4. Print one decision line for the publish script to act on.
|
||||
//!
|
||||
//! It only ever writes into the staging dir and only ever reads the
|
||||
//! published dir — moving files into place and rebuilding the index is the
|
||||
//! publish script's job, so this stays a pure, testable computation.
|
||||
|
||||
use ed25519_dalek::{SigningKey, VerifyingKey};
|
||||
use hound_defs::{osv, pack, Indicator, Pack};
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
/// Fold the deltas back into a fresh baseline once this many have piled up,
|
||||
/// so a new install never has to replay an unbounded chain and the served
|
||||
/// directory stays small. One baseline plus at most this many deltas.
|
||||
const REBASELINE_AFTER: usize = 14;
|
||||
|
||||
fn die(msg: impl std::fmt::Display) -> ! {
|
||||
eprintln!("build-delta: {msg}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
fn load_key(path: &str) -> SigningKey {
|
||||
let bytes = std::fs::read(path).unwrap_or_else(|e| die(format!("reading key {path}: {e}")));
|
||||
let seed: [u8; 32] = bytes
|
||||
.get(..32)
|
||||
.and_then(|s| s.try_into().ok())
|
||||
.unwrap_or_else(|| die(format!("{path} is not at least a 32-byte key")));
|
||||
SigningKey::from_bytes(&seed)
|
||||
}
|
||||
|
||||
/// A stable identity for an indicator, matching build-pack's dedup key.
|
||||
fn key_of(i: &Indicator) -> (String, String, String) {
|
||||
(i.ecosystem.clone(), i.name.clone(), i.id.clone())
|
||||
}
|
||||
|
||||
/// Every `.pack` in `dir` whose name is `<eco>-<digit>…`, verified against
|
||||
/// our own key, flattened to the indicator identities they already carry.
|
||||
/// Also returns how many delta files were seen, for the re-baseline call.
|
||||
fn published_state(
|
||||
dir: &Path,
|
||||
eco: &str,
|
||||
pubkey: &VerifyingKey,
|
||||
) -> (BTreeSet<(String, String, String)>, usize, bool) {
|
||||
let trusted = [("hound-2026", *pubkey)];
|
||||
let mut known = BTreeSet::new();
|
||||
let mut deltas = 0usize;
|
||||
let mut has_baseline = false;
|
||||
let prefix = format!("{eco}-");
|
||||
|
||||
for entry in std::fs::read_dir(dir).into_iter().flatten().flatten() {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
|
||||
if !name.ends_with(".pack") || !name.starts_with(&prefix) {
|
||||
continue;
|
||||
}
|
||||
// Guard against "go-" matching "golang-": the char after the prefix
|
||||
// must start a version (a digit).
|
||||
if !name[prefix.len()..].starts_with(|c: char| c.is_ascii_digit()) {
|
||||
continue;
|
||||
}
|
||||
let Ok(text) = std::fs::read_to_string(&path) else { continue };
|
||||
let Ok(signed) = serde_json::from_str::<pack::SignedPack>(&text) else { continue };
|
||||
let Ok(p) = pack::verify(&signed, &trusted) else {
|
||||
eprintln!("build-delta: WARNING ignoring unverifiable {name}");
|
||||
continue;
|
||||
};
|
||||
for ind in p.indicators {
|
||||
known.insert(key_of(&ind));
|
||||
}
|
||||
if name.contains(".delta.pack") {
|
||||
deltas += 1;
|
||||
} else {
|
||||
has_baseline = true;
|
||||
}
|
||||
}
|
||||
(known, deltas, has_baseline)
|
||||
}
|
||||
|
||||
/// Parse today's full indicator set for the ecosystem from its OSV export.
|
||||
fn todays_indicators(osv_dir: &Path) -> Vec<Indicator> {
|
||||
let mut out = Vec::new();
|
||||
for entry in std::fs::read_dir(osv_dir)
|
||||
.unwrap_or_else(|e| die(format!("reading {}: {e}", osv_dir.display())))
|
||||
.flatten()
|
||||
{
|
||||
let path = entry.path();
|
||||
if path.extension().is_none_or(|e| e != "json") {
|
||||
continue;
|
||||
}
|
||||
if let Ok(text) = std::fs::read_to_string(&path) {
|
||||
out.extend(osv::parse_record(&text));
|
||||
}
|
||||
}
|
||||
out.sort_by(|a, b| key_of(a).cmp(&key_of(b)));
|
||||
out.dedup_by(|a, b| key_of(a) == key_of(b));
|
||||
out
|
||||
}
|
||||
|
||||
fn write_pack(pack_data: &Pack, key: &SigningKey, out: &Path) {
|
||||
let key_id = std::env::var("HOUND_KEY_ID").unwrap_or_else(|_| "hound-2026".into());
|
||||
let signed = pack::sign(pack_data, key, &key_id).unwrap_or_else(|e| die(format!("signing: {e}")));
|
||||
std::fs::write(out, serde_json::to_string(&signed).unwrap())
|
||||
.unwrap_or_else(|e| die(format!("writing {}: {e}", out.display())));
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let a: Vec<String> = std::env::args().skip(1).collect();
|
||||
if a.len() < 6 {
|
||||
die("usage: build-delta <osv-dir> <published-dir> <ecosystem> <staging-dir> <key> <version> [created]");
|
||||
}
|
||||
let (osv_dir, published, eco, staging, key_path, version) =
|
||||
(Path::new(&a[0]), Path::new(&a[1]), &a[2], Path::new(&a[3]), &a[4], &a[5]);
|
||||
let created = a.get(6).cloned().unwrap_or_else(|| "1970-01-01T00:00:00Z".into());
|
||||
|
||||
let key = load_key(key_path);
|
||||
let pubkey = key.verifying_key();
|
||||
|
||||
let today = todays_indicators(osv_dir);
|
||||
if today.is_empty() {
|
||||
// The ingest produced nothing for this ecosystem; never publish an
|
||||
// empty pack (a client that installed it would show zero indicators).
|
||||
println!("UNCHANGED {eco} (no records ingested)");
|
||||
return;
|
||||
}
|
||||
|
||||
let (known, deltas, has_baseline) = published_state(published, eco, &pubkey);
|
||||
|
||||
let sources = vec!["ossf/malicious-packages (Apache-2.0)".to_string(), "osv.dev".to_string()];
|
||||
let rebaseline = !has_baseline || deltas >= REBASELINE_AFTER;
|
||||
|
||||
if rebaseline {
|
||||
let out = staging.join(format!("{eco}-{version}.pack"));
|
||||
write_pack(
|
||||
&Pack { version: version.clone(), created, sources, indicators: today.clone() },
|
||||
&key,
|
||||
&out,
|
||||
);
|
||||
// The publish script archives every existing {eco}-* file when it
|
||||
// sees REBASELINE, so the fresh baseline stands alone.
|
||||
println!("REBASELINE {eco} {} indicators -> {}", today.len(), out.display());
|
||||
return;
|
||||
}
|
||||
|
||||
let new: Vec<Indicator> = today
|
||||
.into_iter()
|
||||
.filter(|i| !known.contains(&key_of(i)))
|
||||
.collect();
|
||||
if new.is_empty() {
|
||||
println!("UNCHANGED {eco} (no new indicators)");
|
||||
return;
|
||||
}
|
||||
|
||||
let out = staging.join(format!("{eco}-{version}.delta.pack"));
|
||||
write_pack(
|
||||
&Pack { version: version.clone(), created, sources, indicators: new.clone() },
|
||||
&key,
|
||||
&out,
|
||||
);
|
||||
println!("DELTA {eco} {} new indicator(s) -> {}", new.len(), out.display());
|
||||
}
|
||||
|
|
@ -48,6 +48,16 @@ pub trait ScanEngine: Send + Sync {
|
|||
/// combined stdout+stderr tail) for the last attempt made.
|
||||
fn update(&self) -> Result<(bool, String, String)>;
|
||||
|
||||
/// Recompile the YARA ruleset from disk, picking up any newly installed
|
||||
/// rules packs, without the rest of `update`'s work (no freshclam, no
|
||||
/// definitions fetch). The daily scheduler calls this after a defs run
|
||||
/// that installed or pruned rules packs, so a new threat pack goes live
|
||||
/// without waiting for a restart. Engines with no on-disk ruleset — the
|
||||
/// ClamAV and fake backends — do nothing.
|
||||
fn reload_rules(&self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Scan bytes already in hand, returning a detection name.
|
||||
///
|
||||
/// This exists for the execution gate, which is handed an open
|
||||
|
|
|
|||
|
|
@ -845,24 +845,27 @@ fn dispatch(req: &hound_api::Request, st: &DaemonState) -> Result<Value> {
|
|||
/// One definitions check, using the same install path as `hound update` so
|
||||
/// there is no second implementation to drift. Returns how many packs landed.
|
||||
fn scheduled_defs_update(st: &DaemonState) -> Result<usize> {
|
||||
// The full feed is what a Pro subscription pays for. Checked here, at
|
||||
// fetch time rather than at boot, so installing a licence takes effect
|
||||
// on the next cycle without a restart — and a lapse stops the fetch
|
||||
// without touching what is already on disk.
|
||||
if !st
|
||||
// The free community pack is fetched for everyone; the full feed and the
|
||||
// threat pack only with a Pro licence. Checked here, at fetch time rather
|
||||
// than at boot, so installing a licence takes effect on the next cycle
|
||||
// without a restart — and a lapse drops the machine cleanly back to the
|
||||
// community pack on the next run.
|
||||
let full_feed = st
|
||||
.license
|
||||
.current()
|
||||
.effective
|
||||
.allows(hound_api::license::Capability::FullSupplyChainFeed)
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
.allows(hound_api::license::Capability::FullSupplyChainFeed);
|
||||
let keys = defs::trusted_keys();
|
||||
let trusted: Vec<(&str, ed25519_dalek::VerifyingKey)> =
|
||||
keys.iter().map(|(id, k)| (id.as_str(), *k)).collect();
|
||||
let outcome = update::run(&update::install_dir(), &trusted)?;
|
||||
if !outcome.installed.is_empty() {
|
||||
let outcome = update::run(&update::install_dir(), &trusted, full_feed)?;
|
||||
if outcome.changed() {
|
||||
st.defs.reload();
|
||||
// A rules pack may have landed or been pruned; recompile so a new
|
||||
// threat pack goes live without waiting for a restart.
|
||||
if let Err(e) = engine::engine().reload_rules() {
|
||||
eprintln!("scheduler: rules reload failed: {e}");
|
||||
}
|
||||
}
|
||||
Ok(outcome.installed.len())
|
||||
}
|
||||
|
|
@ -1086,38 +1089,39 @@ fn update(st: &DaemonState) -> Result<hound_api::UpdateResult> {
|
|||
.current()
|
||||
.effective
|
||||
.allows(hound_api::license::Capability::FullSupplyChainFeed);
|
||||
if !feed_licensed {
|
||||
// Said once, plainly, in the update output — and never as a nag
|
||||
// anywhere else. The built-in rules still reload below, and any
|
||||
// packs already on disk stay loaded.
|
||||
lines.push(format!(
|
||||
"definitions: {}",
|
||||
hound_api::license::Capability::FullSupplyChainFeed.explain_absence()
|
||||
));
|
||||
} else {
|
||||
match update::run(&update::install_dir(), &trusted) {
|
||||
Ok(outcome) => {
|
||||
lines.push(format!("definitions: {}", outcome.summary()));
|
||||
lines.extend(outcome.log.iter().map(|l| format!(" {l}")));
|
||||
if !outcome.installed.is_empty() {
|
||||
let loaded = st.defs.reload();
|
||||
lines.push(format!(
|
||||
" loaded {} indicators from {} pack(s) [{}]",
|
||||
loaded.indicators,
|
||||
loaded.packs.len(),
|
||||
loaded.version
|
||||
));
|
||||
}
|
||||
let sev = if outcome.rejected.is_empty() { "info" } else { "warn" };
|
||||
st.events
|
||||
.push("update", sev, format!("definitions: {}", outcome.summary()));
|
||||
|
||||
// Always run: the free community pack updates for everyone, and only the
|
||||
// full feed and threat pack are gated on the licence.
|
||||
match update::run(&update::install_dir(), &trusted, feed_licensed) {
|
||||
Ok(outcome) => {
|
||||
lines.push(format!("definitions: {}", outcome.summary()));
|
||||
lines.extend(outcome.log.iter().map(|l| format!(" {l}")));
|
||||
if outcome.changed() {
|
||||
let loaded = st.defs.reload();
|
||||
lines.push(format!(
|
||||
" loaded {} indicators from {} pack(s) [{}]",
|
||||
loaded.indicators,
|
||||
loaded.packs.len(),
|
||||
loaded.version
|
||||
));
|
||||
}
|
||||
Err(e) => {
|
||||
// Reported, not fatal.
|
||||
lines.push(format!("definitions: could not update — {e}"));
|
||||
st.events
|
||||
.push("update", "warn", format!("definitions update failed: {e}"));
|
||||
if !feed_licensed {
|
||||
// Said once, plainly — never as a nag anywhere else. The
|
||||
// community pack and every heuristic keep working.
|
||||
lines.push(format!(
|
||||
" {}",
|
||||
hound_api::license::Capability::FullSupplyChainFeed.explain_absence()
|
||||
));
|
||||
}
|
||||
let sev = if outcome.rejected.is_empty() { "info" } else { "warn" };
|
||||
st.events
|
||||
.push("update", sev, format!("definitions: {}", outcome.summary()));
|
||||
}
|
||||
Err(e) => {
|
||||
// Reported, not fatal.
|
||||
lines.push(format!("definitions: could not update — {e}"));
|
||||
st.events
|
||||
.push("update", "warn", format!("definitions update failed: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -208,6 +208,14 @@ impl ScanEngine for HoundEngine {
|
|||
Err(e) => Ok((false, "reload rules".to_string(), format!("{e}\n"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn reload_rules(&self) -> Result<()> {
|
||||
// A newly installed threat pack changes what the ruleset detects, so
|
||||
// verdicts cached under the old rules must not be trusted.
|
||||
self.rules.reload()?;
|
||||
self.cache.clear();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl HoundEngine {
|
||||
|
|
|
|||
|
|
@ -58,10 +58,19 @@ pub struct IndexEntry {
|
|||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
pub struct Index {
|
||||
/// The full malicious-package feed — every ecosystem, baselines and
|
||||
/// deltas. This is the Pro feed: fetched only with a licence.
|
||||
#[serde(default)]
|
||||
pub packs: Vec<IndexEntry>,
|
||||
/// Signed YARA rules packs — the curated threat pack channel. Absent
|
||||
/// from older indexes, which is an empty list, which is fine.
|
||||
/// The free community pack: a snapshot of the most recent public OSV
|
||||
/// malicious packages, so a Free install detects current threats out of
|
||||
/// the box. Always fetched, licence or not. Pro is not "access to this
|
||||
/// data" — it is the full corpus, the daily/near-real-time freshness and
|
||||
/// the curated threat pack on top.
|
||||
#[serde(default)]
|
||||
pub community: Vec<IndexEntry>,
|
||||
/// Signed YARA rules packs — the curated threat pack channel (Pro).
|
||||
/// Absent from older indexes, which is an empty list, which is fine.
|
||||
#[serde(default)]
|
||||
pub rules: Vec<IndexEntry>,
|
||||
}
|
||||
|
|
@ -72,6 +81,8 @@ pub struct Outcome {
|
|||
pub installed: Vec<String>,
|
||||
pub already_current: Vec<String>,
|
||||
pub rejected: Vec<String>,
|
||||
/// Superseded packs removed because the index stopped listing them.
|
||||
pub pruned: Vec<String>,
|
||||
pub log: Vec<String>,
|
||||
}
|
||||
|
||||
|
|
@ -85,15 +96,32 @@ impl Outcome {
|
|||
self.rejected.len()
|
||||
);
|
||||
}
|
||||
let pruned = if self.pruned.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(", {} superseded pack(s) removed", self.pruned.len())
|
||||
};
|
||||
if self.installed.is_empty() {
|
||||
return format!("definitions are up to date ({} pack(s))", self.already_current.len());
|
||||
if self.pruned.is_empty() {
|
||||
return format!("definitions are up to date ({} pack(s))", self.already_current.len());
|
||||
}
|
||||
return format!(
|
||||
"definitions are up to date ({} pack(s){pruned})",
|
||||
self.already_current.len()
|
||||
);
|
||||
}
|
||||
format!(
|
||||
"{} pack(s) installed, {} already current",
|
||||
"{} pack(s) installed, {} already current{pruned}",
|
||||
self.installed.len(),
|
||||
self.already_current.len()
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether anything changed on disk — an install or a prune. Either
|
||||
/// means the in-memory indicator set is now stale and must be reloaded.
|
||||
pub fn changed(&self) -> bool {
|
||||
!self.installed.is_empty() || !self.pruned.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// A filename from a remote index is untrusted input.
|
||||
|
|
@ -157,8 +185,19 @@ pub fn fetch_index(base: &str) -> Result<Index> {
|
|||
serde_json::from_slice(&bytes).context("the definitions index is not valid JSON")
|
||||
}
|
||||
|
||||
/// Fetch, verify and install everything the index offers that we lack.
|
||||
pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Result<Outcome> {
|
||||
/// Fetch, verify and install what this installation is entitled to.
|
||||
///
|
||||
/// `full_feed` is the Pro entitlement (`Capability::FullSupplyChainFeed`).
|
||||
/// The free community pack is fetched regardless — a Free install still
|
||||
/// detects current public malware. The full per-ecosystem feed and the
|
||||
/// curated threat pack are fetched only with the entitlement, and pruned
|
||||
/// away when it is absent, so a machine that stops paying drops cleanly to
|
||||
/// exactly what a fresh Free install has: the community pack, no less.
|
||||
pub fn run(
|
||||
dir: &Path,
|
||||
trusted: &[(&str, ed25519_dalek::VerifyingKey)],
|
||||
full_feed: bool,
|
||||
) -> Result<Outcome> {
|
||||
let base = base_url();
|
||||
let mut out = Outcome::default();
|
||||
|
||||
|
|
@ -169,12 +208,90 @@ pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Resul
|
|||
.with_context(|| format!("creating {}", dir.display()))?;
|
||||
|
||||
let index = fetch_index(&base)?;
|
||||
if index.packs.is_empty() {
|
||||
if index.packs.is_empty() && index.community.is_empty() {
|
||||
out.log.push(format!("{base} offers no packs"));
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
for entry in &index.packs {
|
||||
// The free community pack, always.
|
||||
install_packs(&base, &index.community, dir, trusted, &mut out);
|
||||
|
||||
// The full Pro feed, only with the entitlement.
|
||||
if full_feed {
|
||||
install_packs(&base, &index.packs, dir, trusted, &mut out);
|
||||
install_rules_packs(&base, &index, trusted, &mut out);
|
||||
}
|
||||
|
||||
// Prune definition packs the installation should no longer hold. Two
|
||||
// reasons a pack goes away: the index stopped listing it (a rebaseline
|
||||
// folded the deltas into a fresh baseline), or the entitlement lapsed
|
||||
// (the full feed is Pro; a Free machine keeps only the community pack).
|
||||
// The keep-set is exactly what we are entitled to and the index still
|
||||
// offers; everything else is deleted. Only pack-named files are touched.
|
||||
let mut keep: std::collections::HashSet<&str> =
|
||||
index.community.iter().map(|e| e.file.as_str()).collect();
|
||||
if full_feed {
|
||||
keep.extend(index.packs.iter().map(|e| e.file.as_str()));
|
||||
}
|
||||
for name in prune(dir, ".pack", &keep) {
|
||||
out.pruned.push(name);
|
||||
}
|
||||
|
||||
// The threat pack is Pro; without the entitlement, prune every rpack
|
||||
// envelope AND every Hound-managed extract so a lapsed Pro drops to
|
||||
// exactly the free ruleset — the built-in rules only. Custom `.yar` a
|
||||
// user placed here themselves is left alone: only `hound-pack-*.yar`
|
||||
// extracts are removed.
|
||||
let rules_dir = rules_install_dir();
|
||||
let keep_rules: std::collections::HashSet<&str> = if full_feed {
|
||||
index.rules.iter().map(|e| e.file.as_str()).collect()
|
||||
} else {
|
||||
std::collections::HashSet::new()
|
||||
};
|
||||
for name in prune(&rules_dir, ".rpack", &keep_rules) {
|
||||
out.pruned.push(name);
|
||||
}
|
||||
if !full_feed {
|
||||
for name in prune_managed_extracts(&rules_dir) {
|
||||
out.pruned.push(name);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Remove Hound-installed rule extracts (`hound-pack-*.yar`) and the pack
|
||||
/// VERSION marker. Used on downgrade: the threat pack is Pro, and a machine
|
||||
/// without the entitlement keeps only the built-in rules compiled into the
|
||||
/// binary. A user's own `.yar` files never match the prefix and are safe.
|
||||
fn prune_managed_extracts(dir: &Path) -> Vec<String> {
|
||||
let mut removed = Vec::new();
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return removed };
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
|
||||
let is_managed = name.starts_with("hound-pack-") && name.ends_with(".yar");
|
||||
if !is_managed && name != "VERSION" {
|
||||
continue;
|
||||
}
|
||||
if std::fs::remove_file(&path).is_ok() {
|
||||
removed.push(name.to_string());
|
||||
}
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Fetch, verify and install a list of definition packs into `dir`. Shared
|
||||
/// by the community channel and the full feed so both get the same
|
||||
/// verify-before-install discipline and the same reporting.
|
||||
fn install_packs(
|
||||
base: &str,
|
||||
entries: &[IndexEntry],
|
||||
dir: &Path,
|
||||
trusted: &[(&str, ed25519_dalek::VerifyingKey)],
|
||||
out: &mut Outcome,
|
||||
) {
|
||||
for entry in entries {
|
||||
if !safe_pack_name(&entry.file) {
|
||||
out.rejected.push(entry.file.clone());
|
||||
out.log
|
||||
|
|
@ -242,10 +359,29 @@ pub fn run(dir: &Path, trusted: &[(&str, ed25519_dalek::VerifyingKey)]) -> Resul
|
|||
));
|
||||
out.installed.push(entry.file.clone());
|
||||
}
|
||||
}
|
||||
|
||||
install_rules_packs(&base, &index, trusted, &mut out);
|
||||
|
||||
Ok(out)
|
||||
/// Delete files in `dir` ending in `extension` whose basename is not in
|
||||
/// `keep`. Returns the names removed. Refuses to touch anything that is not
|
||||
/// a plain pack filename, so a stray or hostile name cannot turn this into
|
||||
/// an arbitrary delete.
|
||||
fn prune(dir: &Path, extension: &str, keep: &std::collections::HashSet<&str>) -> Vec<String> {
|
||||
let mut removed = Vec::new();
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return removed };
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
|
||||
if !name.ends_with(extension) || !safe_name(name, extension) {
|
||||
continue;
|
||||
}
|
||||
if keep.contains(name) {
|
||||
continue;
|
||||
}
|
||||
if std::fs::remove_file(&path).is_ok() {
|
||||
removed.push(name.to_string());
|
||||
}
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
/// Fetch and install the signed YARA rules packs the index offers.
|
||||
|
|
@ -333,7 +469,10 @@ fn install_rules_packs(
|
|||
std::fs::write(&tmp, bytes)?;
|
||||
std::fs::rename(&tmp, path)
|
||||
};
|
||||
let yar = dir.join(format!("{}.yar", rules.name));
|
||||
// Hound-managed extracts carry a distinguishing prefix so a
|
||||
// downgrade can remove them without touching any custom `.yar` a
|
||||
// user dropped in the rules directory themselves.
|
||||
let yar = dir.join(format!("hound-pack-{}.yar", rules.name));
|
||||
if let Err(e) = install(&yar, rules.yara.as_bytes())
|
||||
.and_then(|_| install(&dir.join("VERSION"), rules.version.as_bytes()))
|
||||
.and_then(|_| install(&dest, &bytes))
|
||||
|
|
@ -351,6 +490,8 @@ fn install_rules_packs(
|
|||
));
|
||||
out.installed.push(entry.file.clone());
|
||||
}
|
||||
// Pruning of superseded / unentitled `.rpack` envelopes happens in
|
||||
// `run`, in one place, so the tier logic lives together.
|
||||
}
|
||||
|
||||
/// A pack name that may become an installed basename: plain, short, no
|
||||
|
|
@ -509,7 +650,7 @@ mod tests {
|
|||
#[test]
|
||||
fn refuses_to_run_with_an_empty_trust_store() {
|
||||
let dir = std::env::temp_dir().join(format!("hound-upd-nokey-{}", std::process::id()));
|
||||
let e = run(&dir, &[]).unwrap_err();
|
||||
let e = run(&dir, &[], true).unwrap_err();
|
||||
assert!(e.to_string().contains("no signing key"));
|
||||
}
|
||||
|
||||
|
|
@ -540,5 +681,108 @@ mod tests {
|
|||
// An index with no packs array is empty rather than fatal.
|
||||
let empty: Index = serde_json::from_slice(b"{}").unwrap();
|
||||
assert!(empty.packs.is_empty());
|
||||
assert!(empty.rules.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_index_with_deltas_is_just_more_packs() {
|
||||
// The delta feed needs no schema change: a baseline and its daily
|
||||
// deltas are all ordinary pack entries, and the client fetches
|
||||
// whichever it lacks.
|
||||
let idx: Index = serde_json::from_slice(
|
||||
br#"{"packs":[
|
||||
{"file":"npm-2026.08.01.pack","sha256":"aa","version":"2026.08.01"},
|
||||
{"file":"npm-2026.08.21.delta.pack","sha256":"bb","version":"2026.08.21"}
|
||||
]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(idx.packs.len(), 2);
|
||||
assert!(safe_pack_name("npm-2026.08.21.delta.pack"));
|
||||
}
|
||||
|
||||
// ── pruning: what makes the delta feed self-cleaning ──
|
||||
|
||||
#[test]
|
||||
fn prune_removes_only_unlisted_packs_and_nothing_else() {
|
||||
let dir = std::env::temp_dir().join(format!("hound-prune-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
|
||||
// A current baseline, a stale superseded baseline, and files that
|
||||
// are not packs and must never be touched.
|
||||
std::fs::write(dir.join("npm-2026.08.21.pack"), b"keep").unwrap();
|
||||
std::fs::write(dir.join("npm-2026.07.01.pack"), b"stale").unwrap();
|
||||
std::fs::write(dir.join("VERSION"), b"2026.08.21").unwrap();
|
||||
std::fs::write(dir.join("notes.txt"), b"leave me").unwrap();
|
||||
|
||||
let keep: std::collections::HashSet<&str> =
|
||||
["npm-2026.08.21.pack"].into_iter().collect();
|
||||
let removed = prune(&dir, ".pack", &keep);
|
||||
|
||||
assert_eq!(removed, vec!["npm-2026.07.01.pack".to_string()]);
|
||||
assert!(dir.join("npm-2026.08.21.pack").exists(), "the listed pack stays");
|
||||
assert!(!dir.join("npm-2026.07.01.pack").exists(), "the stale pack is gone");
|
||||
assert!(dir.join("VERSION").exists(), "non-pack files are never touched");
|
||||
assert!(dir.join("notes.txt").exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_leaves_everything_when_the_index_lists_it_all() {
|
||||
let dir = std::env::temp_dir().join(format!("hound-prune2-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("npm-2026.08.21.pack"), b"a").unwrap();
|
||||
std::fs::write(dir.join("npm-2026.08.22.delta.pack"), b"b").unwrap();
|
||||
let keep: std::collections::HashSet<&str> =
|
||||
["npm-2026.08.21.pack", "npm-2026.08.22.delta.pack"].into_iter().collect();
|
||||
assert!(prune(&dir, ".pack", &keep).is_empty());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_tracks_installs_and_prunes() {
|
||||
let mut o = Outcome::default();
|
||||
assert!(!o.changed());
|
||||
o.pruned.push("old.pack".into());
|
||||
assert!(o.changed(), "a prune alone must trigger a reload");
|
||||
assert!(o.summary().contains("removed"));
|
||||
}
|
||||
|
||||
// ── Option C: the free community channel and the downgrade path ──
|
||||
|
||||
#[test]
|
||||
fn the_index_splits_community_from_the_paid_feed() {
|
||||
let idx: Index = serde_json::from_slice(
|
||||
br#"{"packs":[{"file":"npm-2026.08.21.pack","sha256":"a","version":"x"}],
|
||||
"community":[{"file":"community-2026.08.21.pack","sha256":"b","version":"x"}],
|
||||
"rules":[{"file":"hound-linux-2026.08.21.rpack","sha256":"c","version":"x"}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(idx.packs.len(), 1);
|
||||
assert_eq!(idx.community.len(), 1);
|
||||
assert_eq!(idx.rules.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downgrade_removes_managed_extracts_but_spares_custom_rules() {
|
||||
let dir = std::env::temp_dir().join(format!("hound-ext-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
// Hound-managed extract + version marker, plus a user's own rule.
|
||||
std::fs::write(dir.join("hound-pack-hound-linux.yar"), b"rule x {condition:true}").unwrap();
|
||||
std::fs::write(dir.join("VERSION"), b"2026.08.21").unwrap();
|
||||
std::fs::write(dir.join("my-custom.yar"), b"rule c {condition:true}").unwrap();
|
||||
|
||||
let mut removed = prune_managed_extracts(&dir);
|
||||
removed.sort();
|
||||
assert_eq!(removed, vec!["VERSION".to_string(), "hound-pack-hound-linux.yar".to_string()]);
|
||||
assert!(!dir.join("hound-pack-hound-linux.yar").exists());
|
||||
assert!(!dir.join("VERSION").exists());
|
||||
assert!(
|
||||
dir.join("my-custom.yar").exists(),
|
||||
"a user's own rule must never be pruned as a downgrade side effect"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,35 +45,38 @@ chmod 644 "$DEST/.$base.tmp"
|
|||
mv -f "$DEST/.$base.tmp" "$DEST/$base"
|
||||
log "published $base"
|
||||
|
||||
# Rebuild the index over everything on disk — definition packs and rules
|
||||
# packs both. Same logic as refresh-definitions.sh so the two agree.
|
||||
# Rebuild the index over everything on disk — every definition pack
|
||||
# (baselines and deltas) and every rules pack. Identical logic to
|
||||
# refresh-definitions.sh so running either keeps the delta feed intact;
|
||||
# version comes from inside the signed payload, not the filename.
|
||||
python3 - "$DEST" <<'PY'
|
||||
import hashlib, json, os, sys
|
||||
import base64, hashlib, json, os, sys
|
||||
dest = sys.argv[1]
|
||||
def newest_by_family(suffix, strip):
|
||||
newest = {}
|
||||
for f in sorted(os.listdir(dest)):
|
||||
if f.endswith(suffix):
|
||||
newest[f.rsplit("-", 1)[0]] = f
|
||||
out = []
|
||||
for _, f in sorted(newest.items()):
|
||||
p = os.path.join(dest, f)
|
||||
out.append({
|
||||
"file": f,
|
||||
"sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(),
|
||||
"size": os.path.getsize(p),
|
||||
"version": f.rsplit("-", 1)[1][:-strip],
|
||||
})
|
||||
return out
|
||||
packs = newest_by_family(".pack", 5)
|
||||
rules = newest_by_family(".rpack", 6)
|
||||
def entry(f):
|
||||
raw = open(os.path.join(dest, f), "rb").read()
|
||||
try:
|
||||
version = json.loads(base64.b64decode(json.loads(raw)["payload"])).get("version", "")
|
||||
except Exception:
|
||||
version = ""
|
||||
return {"file": f, "sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"size": len(raw), "version": version}
|
||||
packs, community, rules = [], [], []
|
||||
for f in sorted(os.listdir(dest)):
|
||||
if not os.path.isfile(os.path.join(dest, f)):
|
||||
continue
|
||||
if f.endswith(".rpack"):
|
||||
rules.append(entry(f))
|
||||
elif f.startswith("community-") and f.endswith(".pack"):
|
||||
community.append(entry(f))
|
||||
elif f.endswith(".pack"):
|
||||
packs.append(entry(f))
|
||||
tmp = os.path.join(dest, ".index.json.tmp")
|
||||
with open(tmp, "w") as fh:
|
||||
json.dump({"packs": packs, "rules": rules}, fh, indent=2)
|
||||
json.dump({"packs": packs, "community": community, "rules": rules}, fh, indent=2)
|
||||
fh.write("\n")
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, os.path.join(dest, "index.json"))
|
||||
print(f"index.json lists {len(packs)} definition pack(s) and {len(rules)} rules pack(s)")
|
||||
print(f"index.json: {len(packs)} feed, {len(community)} community, {len(rules)} rules pack(s)")
|
||||
PY
|
||||
|
||||
log "done"
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Rebuild the definition feed from OSV and publish it.
|
||||
# Rebuild the definition feed from OSV and publish it — incrementally.
|
||||
#
|
||||
# The client-side update machinery is only worth having if the source it
|
||||
# points at actually moves. Everything below already existed as separate
|
||||
# manual steps; this is the thing that runs them on a schedule.
|
||||
# The old version rebuilt the full per-ecosystem pack every night, so its
|
||||
# sha256 changed daily and every client re-downloaded the whole 44 MB npm
|
||||
# pack even when a handful of records had been added. This version publishes
|
||||
# an immutable baseline plus small daily deltas: the client (which already
|
||||
# fetches only packs whose sha256 it lacks) downloads the baseline once and
|
||||
# then kilobytes a day. See crates/hound-defs/examples/build-delta.rs.
|
||||
#
|
||||
# Publishing is atomic per pack: each is written to a temporary name in the
|
||||
# destination directory and renamed into place, so an agent fetching mid-run
|
||||
|
|
@ -12,8 +15,12 @@
|
|||
# what tells an agent a pack exists — writing it first would advertise files
|
||||
# that are not there yet.
|
||||
#
|
||||
# Old packs are kept. An agent that has not checked in for a while still has
|
||||
# a URL that resolves, and disk is cheaper than a failed update.
|
||||
# When build-delta decides to REBASELINE an ecosystem (no baseline yet, or
|
||||
# too many deltas piled up), every existing file for that ecosystem is moved
|
||||
# to archive/ before the fresh baseline lands; the index then lists only the
|
||||
# new baseline, and clients delete their now-unlisted copies. That is what
|
||||
# keeps both the server directory and every client's defs dir from growing
|
||||
# without bound.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
|
@ -21,23 +28,26 @@ KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}"
|
|||
DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}"
|
||||
WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}"
|
||||
VERSION="$(date -u +%Y.%m.%d)"
|
||||
CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
||||
|
||||
[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; }
|
||||
[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; }
|
||||
mkdir -p "$DEST/archive"
|
||||
|
||||
mkdir -p "$WORK"
|
||||
log "refreshing definitions for $VERSION"
|
||||
log "refreshing definitions for $VERSION (incremental)"
|
||||
|
||||
# The OSV exports are large and change slowly; a failure to fetch one
|
||||
# ecosystem must not discard the others.
|
||||
python3 "$ROOT/tools/ingest-osv.py" "$WORK"
|
||||
|
||||
BUILDER="$ROOT/target/release/examples/build-pack"
|
||||
if [ ! -x "$BUILDER" ]; then
|
||||
log "building the pack builder"
|
||||
( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack )
|
||||
BUILDER="$ROOT/target/release/examples/build-delta"
|
||||
COMMUNITY_BUILDER="$ROOT/target/release/examples/build-community"
|
||||
if [ ! -x "$BUILDER" ] || [ ! -x "$COMMUNITY_BUILDER" ]; then
|
||||
log "building the definition builders"
|
||||
( cd "$ROOT" && cargo build --release -p hound-defs --example build-delta --example build-community )
|
||||
fi
|
||||
|
||||
# Outside $WORK on purpose: the loop below treats every directory in $WORK as
|
||||
|
|
@ -46,7 +56,7 @@ fi
|
|||
STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")"
|
||||
trap 'rm -rf "$STAGE"' EXIT
|
||||
|
||||
published=0
|
||||
changed=0
|
||||
for dir in "$WORK"/*/; do
|
||||
eco="$(basename "$dir")"
|
||||
# Lower-case, and '.' is not wanted in a filename component.
|
||||
|
|
@ -59,70 +69,109 @@ for dir in "$WORK"/*/; do
|
|||
log "$eco: no malicious records, skipping"
|
||||
continue
|
||||
fi
|
||||
pack="$STAGE/${name}-${VERSION}.pack"
|
||||
if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then
|
||||
log "$eco: built $(basename "$pack") from $count record(s)"
|
||||
published=$((published + 1))
|
||||
else
|
||||
log "$eco: BUILD FAILED — keeping the previous pack"
|
||||
fi
|
||||
|
||||
# build-delta reads the published dir, decides, and writes to staging.
|
||||
decision="$("$BUILDER" "$dir" "$DEST" "$name" "$STAGE" "$KEY" "$VERSION" "$CREATED" 2>&1)" || {
|
||||
log "$eco: BUILD FAILED — keeping the previous pack(s): $decision"
|
||||
continue
|
||||
}
|
||||
log "$eco: $decision"
|
||||
|
||||
case "$decision" in
|
||||
REBASELINE*)
|
||||
# Retire every existing file for this ecosystem, then install the
|
||||
# fresh baseline. Matches '<name>-<digit>' so 'go' never touches
|
||||
# another ecosystem's files.
|
||||
for old in "$DEST/${name}-"[0-9]*.pack; do
|
||||
[ -e "$old" ] && mv -f "$old" "$DEST/archive/"
|
||||
done
|
||||
changed=1 ;;
|
||||
DELTA*)
|
||||
changed=1 ;;
|
||||
UNCHANGED*)
|
||||
: ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$published" -eq 0 ]; then
|
||||
# Never publish an empty feed. An agent that installs it would report a
|
||||
# clean machine with no indicators loaded, which is worse than one that
|
||||
# keeps yesterday's.
|
||||
log "nothing built; leaving the published feed untouched"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Packs first, then the index that advertises them.
|
||||
# Install whatever landed in staging (baselines and deltas), atomically.
|
||||
for pack in "$STAGE"/*.pack; do
|
||||
[ -e "$pack" ] || continue
|
||||
base="$(basename "$pack")"
|
||||
cp "$pack" "$DEST/.$base.tmp"
|
||||
chmod 644 "$DEST/.$base.tmp"
|
||||
mv -f "$DEST/.$base.tmp" "$DEST/$base"
|
||||
done
|
||||
|
||||
python3 - "$DEST" "$VERSION" <<'PY'
|
||||
import hashlib, json, os, sys
|
||||
dest, version = sys.argv[1], sys.argv[2]
|
||||
# Rebuild the free community pack from the freshly-published feed: a recent
|
||||
# subset a Free install can use. Rebuilt only when the feed actually changed
|
||||
# — Pro's value is DAILY freshness, so the free snapshot lags on purpose.
|
||||
if [ "$changed" -eq 1 ]; then
|
||||
cpack="$STAGE/community-${VERSION}.pack"
|
||||
if "$COMMUNITY_BUILDER" "$DEST" "$cpack" "$KEY" "$VERSION" "$CREATED" "${HOUND_COMMUNITY_LIMIT:-5000}"; then
|
||||
for old in "$DEST"/community-*.pack; do
|
||||
[ -e "$old" ] && mv -f "$old" "$DEST/archive/"
|
||||
done
|
||||
base="$(basename "$cpack")"
|
||||
cp "$cpack" "$DEST/.$base.tmp"; chmod 644 "$DEST/.$base.tmp"; mv -f "$DEST/.$base.tmp" "$DEST/$base"
|
||||
log "community: published $base"
|
||||
else
|
||||
log "community: build failed — keeping the previous community pack"
|
||||
fi
|
||||
fi
|
||||
|
||||
# One entry per family: the newest file. Older ones stay on disk so existing
|
||||
# URLs keep resolving, but the index only ever advertises current data.
|
||||
def newest_by_family(suffix, strip):
|
||||
newest = {}
|
||||
for f in sorted(os.listdir(dest)):
|
||||
if f.endswith(suffix):
|
||||
newest[f.rsplit("-", 1)[0]] = f
|
||||
out = []
|
||||
for _, f in sorted(newest.items()):
|
||||
p = os.path.join(dest, f)
|
||||
out.append({
|
||||
"file": f,
|
||||
"sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(),
|
||||
"size": os.path.getsize(p),
|
||||
"version": f.rsplit("-", 1)[1][:-strip],
|
||||
})
|
||||
return out
|
||||
# Never publish an empty feed: if nothing changed AND the destination has no
|
||||
# packs at all, something is wrong — leave whatever is there untouched.
|
||||
if [ "$changed" -eq 0 ] && [ -z "$(find "$DEST" -maxdepth 1 -name '*.pack' -print -quit)" ]; then
|
||||
log "nothing to publish and no existing feed; leaving it untouched"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$changed" -eq 0 ]; then
|
||||
log "no ecosystem changed today; feed already current, refreshing index only"
|
||||
fi
|
||||
|
||||
# Definition packs (.pack) are rebuilt daily by this script. Rules packs
|
||||
# (.rpack — the curated threat pack) are built and published separately by
|
||||
# tools/publish-rules-pack.sh, but the index advertises both, so it is
|
||||
# rebuilt from whatever .rpack files are on disk rather than dropping them.
|
||||
packs = newest_by_family(".pack", 5)
|
||||
rules = newest_by_family(".rpack", 6)
|
||||
python3 - "$DEST" <<'PY'
|
||||
import base64, hashlib, json, os, sys
|
||||
dest = sys.argv[1]
|
||||
|
||||
# List EVERY current pack in the served directory — every baseline and every
|
||||
# delta. The client fetches whatever it is missing and deletes what the index
|
||||
# stops listing, so "current" is exactly "present here" (archive/ is a
|
||||
# subdirectory and is not walked). The version comes from inside the signed
|
||||
# payload rather than from the filename, so the delta naming
|
||||
# (npm-2026.08.22.delta.pack) needs no special parsing.
|
||||
def entry(f):
|
||||
p = os.path.join(dest, f)
|
||||
raw = open(p, "rb").read()
|
||||
try:
|
||||
version = json.loads(base64.b64decode(json.loads(raw)["payload"])).get("version", "")
|
||||
except Exception:
|
||||
version = ""
|
||||
return {"file": f, "sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"size": len(raw), "version": version}
|
||||
|
||||
# Route by name: community-*.pack is the free channel; every other .pack is
|
||||
# the full Pro feed; .rpack is the curated threat pack (Pro).
|
||||
packs, community, rules = [], [], []
|
||||
for f in sorted(os.listdir(dest)):
|
||||
if not os.path.isfile(os.path.join(dest, f)):
|
||||
continue
|
||||
if f.endswith(".rpack"):
|
||||
rules.append(entry(f))
|
||||
elif f.startswith("community-") and f.endswith(".pack"):
|
||||
community.append(entry(f))
|
||||
elif f.endswith(".pack"):
|
||||
packs.append(entry(f))
|
||||
|
||||
tmp = os.path.join(dest, ".index.json.tmp")
|
||||
with open(tmp, "w") as fh:
|
||||
json.dump({"packs": packs, "rules": rules}, fh, indent=2)
|
||||
json.dump({"packs": packs, "community": community, "rules": rules}, fh, indent=2)
|
||||
fh.write("\n")
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, os.path.join(dest, "index.json"))
|
||||
print(f"index.json lists {len(packs)} definition pack(s) and {len(rules)} rules pack(s)")
|
||||
print(f"index.json: {len(packs)} feed pack(s), {len(community)} community pack(s), {len(rules)} rules pack(s)")
|
||||
PY
|
||||
|
||||
log "published $published pack(s) for $VERSION"
|
||||
log "definitions refresh complete for $VERSION"
|
||||
|
||||
# Clear the extracted records. They are the bulk of the scratch — 1.2 GB
|
||||
# after a single run — and the daily timer regenerates them every time. The
|
||||
|
|
|
|||
Loading…
Reference in a new issue