Antivirus/tools/refresh-definitions.sh

#!/usr/bin/env bash
#
# Rebuild the definition feed from OSV and publish it.
#
# The client-side update machinery is only worth having if the source it
# points at actually moves. Everything below already existed as separate
# manual steps; this is the thing that runs them on a schedule.
#
# Publishing is atomic per pack: each is written to a temporary name in the
# destination directory and renamed into place, so an agent fetching mid-run
# never sees a half-written pack. index.json is written last, because it is
# what tells an agent a pack exists — writing it first would advertise files
# that are not there yet.
#
# Old packs are kept. An agent that has not checked in for a while still has
# a URL that resolves, and disk is cheaper than a failed update.
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}"
DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}"
WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}"
VERSION="$(date -u +%Y.%m.%d)"

log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; }

[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; }
[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; }

mkdir -p "$WORK"
log "refreshing definitions for $VERSION"

# The OSV exports are large and change slowly; a failure to fetch one
# ecosystem must not discard the others.
python3 "$ROOT/tools/ingest-osv.py" "$WORK"

BUILDER="$ROOT/target/release/examples/build-pack"
if [ ! -x "$BUILDER" ]; then
  log "building the pack builder"
  ( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack )
fi

# Outside $WORK on purpose: the loop below treats every directory in $WORK as
# an ecosystem, and a staging directory in there gets built into a pack named
# after the mktemp suffix. Which is exactly what happened the first time.
STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")"
trap 'rm -rf "$STAGE"' EXIT

published=0
for dir in "$WORK"/*/; do
  eco="$(basename "$dir")"
  # Lower-case, and '.' is not wanted in a filename component.
  name="$(echo "$eco" | tr '[:upper:]' '[:lower:]' | tr '.' '-')"
  case "$name" in
    stage*|.*) log "$eco: not an ecosystem, skipping"; continue ;;
  esac
  count="$(find "$dir" -maxdepth 1 -name '*.json' | wc -l)"
  if [ "$count" -eq 0 ]; then
    log "$eco: no malicious records, skipping"
    continue
  fi
  pack="$STAGE/${name}-${VERSION}.pack"
  if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then
    log "$eco: built $(basename "$pack") from $count record(s)"
    published=$((published + 1))
  else
    log "$eco: BUILD FAILED — keeping the previous pack"
  fi
done

if [ "$published" -eq 0 ]; then
  # Never publish an empty feed. An agent that installs it would report a
  # clean machine with no indicators loaded, which is worse than one that
  # keeps yesterday's.
  log "nothing built; leaving the published feed untouched"
  exit 1
fi

# Packs first, then the index that advertises them.
for pack in "$STAGE"/*.pack; do
  base="$(basename "$pack")"
  cp "$pack" "$DEST/.$base.tmp"
  chmod 644 "$DEST/.$base.tmp"
  mv -f "$DEST/.$base.tmp" "$DEST/$base"
done

python3 - "$DEST" "$VERSION" <<'PY'
import hashlib, json, os, sys
dest, version = sys.argv[1], sys.argv[2]

# One entry per family: the newest file. Older ones stay on disk so existing
# URLs keep resolving, but the index only ever advertises current data.
def newest_by_family(suffix, strip):
    newest = {}
    for f in sorted(os.listdir(dest)):
        if f.endswith(suffix):
            newest[f.rsplit("-", 1)[0]] = f
    out = []
    for _, f in sorted(newest.items()):
        p = os.path.join(dest, f)
        out.append({
            "file": f,
            "sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(),
            "size": os.path.getsize(p),
            "version": f.rsplit("-", 1)[1][:-strip],
        })
    return out

# Definition packs (.pack) are rebuilt daily by this script. Rules packs
# (.rpack — the curated threat pack) are built and published separately by
# tools/publish-rules-pack.sh, but the index advertises both, so it is
# rebuilt from whatever .rpack files are on disk rather than dropping them.
packs = newest_by_family(".pack", 5)
rules = newest_by_family(".rpack", 6)

tmp = os.path.join(dest, ".index.json.tmp")
with open(tmp, "w") as fh:
    json.dump({"packs": packs, "rules": rules}, fh, indent=2)
    fh.write("\n")
os.chmod(tmp, 0o644)
os.replace(tmp, os.path.join(dest, "index.json"))
print(f"index.json lists {len(packs)} definition pack(s) and {len(rules)} rules pack(s)")
PY

log "published $published pack(s) for $VERSION"

# Clear the extracted records. They are the bulk of the scratch — 1.2 GB
# after a single run — and the daily timer regenerates them every time. The
# downloaded zips stay, because ingest-osv.py reuses them and fetching is the
# slow part; the JSON extracted from them costs seconds to rebuild and is
# pure waste to keep. Left alone, a nightly job quietly fills the disk of the
# machine that publishes a security feed.
find "$WORK" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +
log "scratch now $(du -sh "$WORK" 2>/dev/null | cut -f1)"