From 86c33e4648a396cf8b81ebbe8835c837291c7f9e Mon Sep 17 00:00:00 2001 From: dev Date: Fri, 21 Aug 2026 12:31:54 -0500 Subject: [PATCH] defs: rebuild and publish the feed daily instead of by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every pack in the published feed was dated 2026.08.21 because that is when I last ran the ingest by hand. The client-side update machinery shipped in 0.1.2 works well against a source that never moves, which means it would have kept every installation confidently green on data that aged a day for every day that passed — and the staleness warnings added in the same release would have started telling users the truth about a problem we caused. tools/refresh-definitions.sh runs the steps that already existed (ingest-osv.py, build-pack, index.json) with the properties a published feed needs: - Never publishes an empty feed. If every pack fails to build it exits non-zero and leaves the previous one live. An agent that installed an empty feed would report a clean machine with no indicators loaded, which is worse than one keeping yesterday's. - Packs are written before index.json, and each lands via rename. The index is what tells an agent a pack exists, so writing it first would advertise files that are not there yet, and a rename means a fetch mid-run never sees a half-written pack. - Old packs stay on disk; the index advertises only the newest per ecosystem. An agent that has been offline for a while still has a URL that resolves. - One ecosystem failing does not cost the others. The first dry run built a pack called stage-bfsqqa-2026.08.21.pack: the staging directory was inside the work directory, and the loop treats every directory in there as an ecosystem. Staging now lives outside it, and directory names are filtered as well. The timer runs at 05:20 UTC with Persistent=true, so a builder that was off does a catch-up rather than silently skipping a day — that being the exact failure that produces a stale feed nobody notices. The service runs as the publishing user rather than root: it needs the signing key and write access to one directory, and a build pipeline running as root to write a web directory is a bigger target than the thing it protects. Verified: the timer is enabled, a real run republished all six packs and 235,577 indicators, and a client installed from the result. Co-Authored-By: Claude Opus 5 --- packaging/systemd/hound-defs-refresh.service | 23 ++++ packaging/systemd/hound-defs-refresh.timer | 17 +++ tools/refresh-definitions.sh | 116 +++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 packaging/systemd/hound-defs-refresh.service create mode 100644 packaging/systemd/hound-defs-refresh.timer create mode 100755 tools/refresh-definitions.sh diff --git a/packaging/systemd/hound-defs-refresh.service b/packaging/systemd/hound-defs-refresh.service new file mode 100644 index 0000000..385c25b --- /dev/null +++ b/packaging/systemd/hound-defs-refresh.service @@ -0,0 +1,23 @@ +[Unit] +Description=Rebuild and publish the Hound definition feed +Documentation=https://houndav.com +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=anon +# Runs as the publisher, not as root: it needs the signing key and write +# access to the published directory, and nothing else. A build pipeline that +# runs as root to write a web directory is a larger target than the thing it +# protects. +WorkingDirectory=/home/anon/agents/hound/antivirus +ExecStart=/home/anon/agents/hound/antivirus/tools/refresh-definitions.sh +# The OSV exports total a few hundred MB and npm alone is a quarter of a +# million records; a slow morning should not look like a hang. +TimeoutStartSec=45min +Nice=10 +IOSchedulingClass=idle + +[Install] +WantedBy=multi-user.target diff --git a/packaging/systemd/hound-defs-refresh.timer b/packaging/systemd/hound-defs-refresh.timer new file mode 100644 index 0000000..a71d2ff --- /dev/null +++ b/packaging/systemd/hound-defs-refresh.timer @@ -0,0 +1,17 @@ +[Unit] +Description=Refresh the Hound definition feed daily +Documentation=https://houndav.com + +[Timer] +# OSV rebuilds its exports daily. Early UTC means the feed is current before +# most of the working day starts, wherever the user is. +OnCalendar=*-*-* 05:20:00 UTC +# A machine that was off at 05:20 still gets a rebuild rather than silently +# skipping a day — which is the failure that produces a stale feed nobody +# notices. +Persistent=true +# Spread the load if this ever runs on more than one builder. +RandomizedDelaySec=20min + +[Install] +WantedBy=timers.target diff --git a/tools/refresh-definitions.sh b/tools/refresh-definitions.sh new file mode 100755 index 0000000..5efa733 --- /dev/null +++ b/tools/refresh-definitions.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Rebuild the definition feed from OSV and publish it. +# +# The client-side update machinery is only worth having if the source it +# points at actually moves. Everything below already existed as separate +# manual steps; this is the thing that runs them on a schedule. +# +# Publishing is atomic per pack: each is written to a temporary name in the +# destination directory and renamed into place, so an agent fetching mid-run +# never sees a half-written pack. index.json is written last, because it is +# what tells an agent a pack exists — writing it first would advertise files +# that are not there yet. +# +# Old packs are kept. An agent that has not checked in for a while still has +# a URL that resolves, and disk is cheaper than a failed update. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}" +DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}" +WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}" +VERSION="$(date -u +%Y.%m.%d)" + +log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } + +[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; } +[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; } + +mkdir -p "$WORK" +log "refreshing definitions for $VERSION" + +# The OSV exports are large and change slowly; a failure to fetch one +# ecosystem must not discard the others. +python3 "$ROOT/tools/ingest-osv.py" "$WORK" + +BUILDER="$ROOT/target/release/examples/build-pack" +if [ ! -x "$BUILDER" ]; then + log "building the pack builder" + ( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack ) +fi + +# Outside $WORK on purpose: the loop below treats every directory in $WORK as +# an ecosystem, and a staging directory in there gets built into a pack named +# after the mktemp suffix. Which is exactly what happened the first time. +STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")" +trap 'rm -rf "$STAGE"' EXIT + +published=0 +for dir in "$WORK"/*/; do + eco="$(basename "$dir")" + # Lower-case, and '.' is not wanted in a filename component. + name="$(echo "$eco" | tr '[:upper:]' '[:lower:]' | tr '.' '-')" + case "$name" in + stage*|.*) log "$eco: not an ecosystem, skipping"; continue ;; + esac + count="$(find "$dir" -maxdepth 1 -name '*.json' | wc -l)" + if [ "$count" -eq 0 ]; then + log "$eco: no malicious records, skipping" + continue + fi + pack="$STAGE/${name}-${VERSION}.pack" + if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then + log "$eco: built $(basename "$pack") from $count record(s)" + published=$((published + 1)) + else + log "$eco: BUILD FAILED — keeping the previous pack" + fi +done + +if [ "$published" -eq 0 ]; then + # Never publish an empty feed. An agent that installs it would report a + # clean machine with no indicators loaded, which is worse than one that + # keeps yesterday's. + log "nothing built; leaving the published feed untouched" + exit 1 +fi + +# Packs first, then the index that advertises them. +for pack in "$STAGE"/*.pack; do + base="$(basename "$pack")" + cp "$pack" "$DEST/.$base.tmp" + chmod 644 "$DEST/.$base.tmp" + mv -f "$DEST/.$base.tmp" "$DEST/$base" +done + +python3 - "$DEST" "$VERSION" <<'PY' +import hashlib, json, os, sys +dest, version = sys.argv[1], sys.argv[2] +packs = [] +# One pack per ecosystem: the newest. Older ones stay on disk so existing +# URLs keep resolving, but the index only ever advertises current data. +newest = {} +for f in sorted(os.listdir(dest)): + if not f.endswith(".pack"): + continue + eco = f.rsplit("-", 1)[0] + newest[eco] = f +for eco, f in sorted(newest.items()): + p = os.path.join(dest, f) + packs.append({ + "file": f, + "sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(), + "size": os.path.getsize(p), + "version": f.rsplit("-", 1)[1][:-5], + }) +tmp = os.path.join(dest, ".index.json.tmp") +with open(tmp, "w") as fh: + json.dump({"packs": packs}, fh, indent=2) + fh.write("\n") +os.chmod(tmp, 0o644) +os.replace(tmp, os.path.join(dest, "index.json")) +print(f"index.json lists {len(packs)} pack(s)") +PY + +log "published $published pack(s) for $VERSION"