diff --git a/packaging/systemd/hound-defs-refresh.service b/packaging/systemd/hound-defs-refresh.service new file mode 100644 index 0000000..385c25b --- /dev/null +++ b/packaging/systemd/hound-defs-refresh.service @@ -0,0 +1,23 @@ +[Unit] +Description=Rebuild and publish the Hound definition feed +Documentation=https://houndav.com +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=anon +# Runs as the publisher, not as root: it needs the signing key and write +# access to the published directory, and nothing else. A build pipeline that +# runs as root to write a web directory is a larger target than the thing it +# protects. +WorkingDirectory=/home/anon/agents/hound/antivirus +ExecStart=/home/anon/agents/hound/antivirus/tools/refresh-definitions.sh +# The OSV exports total a few hundred MB and npm alone is a quarter of a +# million records; a slow morning should not look like a hang. +TimeoutStartSec=45min +Nice=10 +IOSchedulingClass=idle + +[Install] +WantedBy=multi-user.target diff --git a/packaging/systemd/hound-defs-refresh.timer b/packaging/systemd/hound-defs-refresh.timer new file mode 100644 index 0000000..a71d2ff --- /dev/null +++ b/packaging/systemd/hound-defs-refresh.timer @@ -0,0 +1,17 @@ +[Unit] +Description=Refresh the Hound definition feed daily +Documentation=https://houndav.com + +[Timer] +# OSV rebuilds its exports daily. Early UTC means the feed is current before +# most of the working day starts, wherever the user is. +OnCalendar=*-*-* 05:20:00 UTC +# A machine that was off at 05:20 still gets a rebuild rather than silently +# skipping a day — which is the failure that produces a stale feed nobody +# notices. +Persistent=true +# Spread the load if this ever runs on more than one builder. +RandomizedDelaySec=20min + +[Install] +WantedBy=timers.target diff --git a/tools/refresh-definitions.sh b/tools/refresh-definitions.sh new file mode 100755 index 0000000..5efa733 --- /dev/null +++ b/tools/refresh-definitions.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Rebuild the definition feed from OSV and publish it. +# +# The client-side update machinery is only worth having if the source it +# points at actually moves. Everything below already existed as separate +# manual steps; this is the thing that runs them on a schedule. +# +# Publishing is atomic per pack: each is written to a temporary name in the +# destination directory and renamed into place, so an agent fetching mid-run +# never sees a half-written pack. index.json is written last, because it is +# what tells an agent a pack exists — writing it first would advertise files +# that are not there yet. +# +# Old packs are kept. An agent that has not checked in for a while still has +# a URL that resolves, and disk is cheaper than a failed update. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}" +DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}" +WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}" +VERSION="$(date -u +%Y.%m.%d)" + +log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } + +[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; } +[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; } + +mkdir -p "$WORK" +log "refreshing definitions for $VERSION" + +# The OSV exports are large and change slowly; a failure to fetch one +# ecosystem must not discard the others. +python3 "$ROOT/tools/ingest-osv.py" "$WORK" + +BUILDER="$ROOT/target/release/examples/build-pack" +if [ ! -x "$BUILDER" ]; then + log "building the pack builder" + ( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack ) +fi + +# Outside $WORK on purpose: the loop below treats every directory in $WORK as +# an ecosystem, and a staging directory in there gets built into a pack named +# after the mktemp suffix. Which is exactly what happened the first time. +STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")" +trap 'rm -rf "$STAGE"' EXIT + +published=0 +for dir in "$WORK"/*/; do + eco="$(basename "$dir")" + # Lower-case, and '.' is not wanted in a filename component. + name="$(echo "$eco" | tr '[:upper:]' '[:lower:]' | tr '.' '-')" + case "$name" in + stage*|.*) log "$eco: not an ecosystem, skipping"; continue ;; + esac + count="$(find "$dir" -maxdepth 1 -name '*.json' | wc -l)" + if [ "$count" -eq 0 ]; then + log "$eco: no malicious records, skipping" + continue + fi + pack="$STAGE/${name}-${VERSION}.pack" + if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then + log "$eco: built $(basename "$pack") from $count record(s)" + published=$((published + 1)) + else + log "$eco: BUILD FAILED — keeping the previous pack" + fi +done + +if [ "$published" -eq 0 ]; then + # Never publish an empty feed. An agent that installs it would report a + # clean machine with no indicators loaded, which is worse than one that + # keeps yesterday's. + log "nothing built; leaving the published feed untouched" + exit 1 +fi + +# Packs first, then the index that advertises them. +for pack in "$STAGE"/*.pack; do + base="$(basename "$pack")" + cp "$pack" "$DEST/.$base.tmp" + chmod 644 "$DEST/.$base.tmp" + mv -f "$DEST/.$base.tmp" "$DEST/$base" +done + +python3 - "$DEST" "$VERSION" <<'PY' +import hashlib, json, os, sys +dest, version = sys.argv[1], sys.argv[2] +packs = [] +# One pack per ecosystem: the newest. Older ones stay on disk so existing +# URLs keep resolving, but the index only ever advertises current data. +newest = {} +for f in sorted(os.listdir(dest)): + if not f.endswith(".pack"): + continue + eco = f.rsplit("-", 1)[0] + newest[eco] = f +for eco, f in sorted(newest.items()): + p = os.path.join(dest, f) + packs.append({ + "file": f, + "sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(), + "size": os.path.getsize(p), + "version": f.rsplit("-", 1)[1][:-5], + }) +tmp = os.path.join(dest, ".index.json.tmp") +with open(tmp, "w") as fh: + json.dump({"packs": packs}, fh, indent=2) + fh.write("\n") +os.chmod(tmp, 0o644) +os.replace(tmp, os.path.join(dest, "index.json")) +print(f"index.json lists {len(packs)} pack(s)") +PY + +log "published $published pack(s) for $VERSION"