Antivirus/tools/refresh-definitions.sh

#!/usr/bin/env bash
#
# Rebuild the definition feed from OSV and publish it — incrementally.
#
# The old version rebuilt the full per-ecosystem pack every night, so its
# sha256 changed daily and every client re-downloaded the whole 44 MB npm
# pack even when a handful of records had been added. This version publishes
# an immutable baseline plus small daily deltas: the client (which already
# fetches only packs whose sha256 it lacks) downloads the baseline once and
# then kilobytes a day. See crates/hound-defs/examples/build-delta.rs.
#
# Publishing is atomic per pack: each is written to a temporary name in the
# destination directory and renamed into place, so an agent fetching mid-run
# never sees a half-written pack. index.json is written last, because it is
# what tells an agent a pack exists — writing it first would advertise files
# that are not there yet.
#
# When build-delta decides to REBASELINE an ecosystem (no baseline yet, or
# too many deltas piled up), every existing file for that ecosystem is moved
# to archive/ before the fresh baseline lands; the index then lists only the
# new baseline, and clients delete their now-unlisted copies. That is what
# keeps both the server directory and every client's defs dir from growing
# without bound.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}"
DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}"
WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}"
VERSION="$(date -u +%Y.%m.%d)"
CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; }

[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; }
[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; }
mkdir -p "$DEST/archive"

mkdir -p "$WORK"
log "refreshing definitions for $VERSION (incremental)"

# The OSV exports are large and change slowly; a failure to fetch one
# ecosystem must not discard the others.
python3 "$ROOT/tools/ingest-osv.py" "$WORK"

BUILDER="$ROOT/target/release/examples/build-delta"
COMMUNITY_BUILDER="$ROOT/target/release/examples/build-community"
if [ ! -x "$BUILDER" ] || [ ! -x "$COMMUNITY_BUILDER" ]; then
  log "building the definition builders"
  ( cd "$ROOT" && cargo build --release -p hound-defs --example build-delta --example build-community )
fi

# Outside $WORK on purpose: the loop below treats every directory in $WORK as
# an ecosystem, and a staging directory in there gets built into a pack named
# after the mktemp suffix. Which is exactly what happened the first time.
STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")"
trap 'rm -rf "$STAGE"' EXIT

changed=0
for dir in "$WORK"/*/; do
  eco="$(basename "$dir")"
  # Lower-case, and '.' is not wanted in a filename component.
  name="$(echo "$eco" | tr '[:upper:]' '[:lower:]' | tr '.' '-')"
  case "$name" in
    stage*|.*) log "$eco: not an ecosystem, skipping"; continue ;;
  esac
  count="$(find "$dir" -maxdepth 1 -name '*.json' | wc -l)"
  if [ "$count" -eq 0 ]; then
    log "$eco: no malicious records, skipping"
    continue
  fi

  # build-delta reads the published dir, decides, and writes to staging.
  decision="$("$BUILDER" "$dir" "$DEST" "$name" "$STAGE" "$KEY" "$VERSION" "$CREATED" 2>&1)" || {
    log "$eco: BUILD FAILED — keeping the previous pack(s): $decision"
    continue
  }
  log "$eco: $decision"

  case "$decision" in
    REBASELINE*)
      # Retire every existing file for this ecosystem, then install the
      # fresh baseline. Matches '<name>-<digit>' so 'go' never touches
      # another ecosystem's files.
      for old in "$DEST/${name}-"[0-9]*.pack; do
        [ -e "$old" ] && mv -f "$old" "$DEST/archive/"
      done
      changed=1 ;;
    DELTA*)
      changed=1 ;;
    UNCHANGED*)
      : ;;
  esac
done

# Install whatever landed in staging (baselines and deltas), atomically.
for pack in "$STAGE"/*.pack; do
  [ -e "$pack" ] || continue
  base="$(basename "$pack")"
  cp "$pack" "$DEST/.$base.tmp"
  chmod 644 "$DEST/.$base.tmp"
  mv -f "$DEST/.$base.tmp" "$DEST/$base"
done

# Rebuild the free community pack from the freshly-published feed: a recent
# subset a Free install can use. Rebuilt only when the feed actually changed
# — Pro's value is DAILY freshness, so the free snapshot lags on purpose.
if [ "$changed" -eq 1 ]; then
  cpack="$STAGE/community-${VERSION}.pack"
  if "$COMMUNITY_BUILDER" "$DEST" "$cpack" "$KEY" "$VERSION" "$CREATED" "${HOUND_COMMUNITY_LIMIT:-5000}"; then
    for old in "$DEST"/community-*.pack; do
      [ -e "$old" ] && mv -f "$old" "$DEST/archive/"
    done
    base="$(basename "$cpack")"
    cp "$cpack" "$DEST/.$base.tmp"; chmod 644 "$DEST/.$base.tmp"; mv -f "$DEST/.$base.tmp" "$DEST/$base"
    log "community: published $base"
  else
    log "community: build failed — keeping the previous community pack"
  fi
fi

# Never publish an empty feed: if nothing changed AND the destination has no
# packs at all, something is wrong — leave whatever is there untouched.
if [ "$changed" -eq 0 ] && [ -z "$(find "$DEST" -maxdepth 1 -name '*.pack' -print -quit)" ]; then
  log "nothing to publish and no existing feed; leaving it untouched"
  exit 1
fi
if [ "$changed" -eq 0 ]; then
  log "no ecosystem changed today; feed already current, refreshing index only"
fi

python3 - "$DEST" <<'PY'
import base64, hashlib, json, os, sys
dest = sys.argv[1]

# List EVERY current pack in the served directory — every baseline and every
# delta. The client fetches whatever it is missing and deletes what the index
# stops listing, so "current" is exactly "present here" (archive/ is a
# subdirectory and is not walked). The version comes from inside the signed
# payload rather than from the filename, so the delta naming
# (npm-2026.08.22.delta.pack) needs no special parsing.
def entry(f):
    p = os.path.join(dest, f)
    raw = open(p, "rb").read()
    try:
        version = json.loads(base64.b64decode(json.loads(raw)["payload"])).get("version", "")
    except Exception:
        version = ""
    return {"file": f, "sha256": hashlib.sha256(raw).hexdigest(),
            "size": len(raw), "version": version}

# Route by name: community-*.pack is the free channel; every other .pack is
# the full Pro feed; .rpack is the curated threat pack (Pro).
packs, community, rules = [], [], []
for f in sorted(os.listdir(dest)):
    if not os.path.isfile(os.path.join(dest, f)):
        continue
    if f.endswith(".rpack"):
        rules.append(entry(f))
    elif f.startswith("community-") and f.endswith(".pack"):
        community.append(entry(f))
    elif f.endswith(".pack"):
        packs.append(entry(f))

tmp = os.path.join(dest, ".index.json.tmp")
with open(tmp, "w") as fh:
    json.dump({"packs": packs, "community": community, "rules": rules}, fh, indent=2)
    fh.write("\n")
os.chmod(tmp, 0o644)
os.replace(tmp, os.path.join(dest, "index.json"))
print(f"index.json: {len(packs)} feed pack(s), {len(community)} community pack(s), {len(rules)} rules pack(s)")
PY

log "definitions refresh complete for $VERSION"

# Clear the extracted records. They are the bulk of the scratch — 1.2 GB
# after a single run — and the daily timer regenerates them every time. The
# downloaded zips stay, because ingest-osv.py reuses them and fetching is the
# slow part; the JSON extracted from them costs seconds to rebuild and is
# pure waste to keep. Left alone, a nightly job quietly fills the disk of the
# machine that publishes a security feed.
find "$WORK" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +
log "scratch now $(du -sh "$WORK" 2>/dev/null | cut -f1)"