125 lines
4.6 KiB
Bash
Executable file
125 lines
4.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# Rebuild the definition feed from OSV and publish it.
|
|
#
|
|
# The client-side update machinery is only worth having if the source it
|
|
# points at actually moves. Everything below already existed as separate
|
|
# manual steps; this is the thing that runs them on a schedule.
|
|
#
|
|
# Publishing is atomic per pack: each is written to a temporary name in the
|
|
# destination directory and renamed into place, so an agent fetching mid-run
|
|
# never sees a half-written pack. index.json is written last, because it is
|
|
# what tells an agent a pack exists — writing it first would advertise files
|
|
# that are not there yet.
|
|
#
|
|
# Old packs are kept. An agent that has not checked in for a while still has
|
|
# a URL that resolves, and disk is cheaper than a failed update.
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
KEY="${HOUND_DEFS_KEY:-$HOME/agents/hound/.secrets/defs-signing.key}"
|
|
DEST="${HOUND_DEFS_DIR:-/srv/houndav/defs}"
|
|
WORK="${HOUND_DEFS_WORK:-/var/tmp/hound-defs}"
|
|
VERSION="$(date -u +%Y.%m.%d)"
|
|
|
|
log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
|
|
[ -f "$KEY" ] || { echo "no signing key at $KEY" >&2; exit 1; }
|
|
[ -d "$DEST" ] || { echo "no destination directory $DEST" >&2; exit 1; }
|
|
|
|
mkdir -p "$WORK"
|
|
log "refreshing definitions for $VERSION"
|
|
|
|
# The OSV exports are large and change slowly; a failure to fetch one
|
|
# ecosystem must not discard the others.
|
|
python3 "$ROOT/tools/ingest-osv.py" "$WORK"
|
|
|
|
BUILDER="$ROOT/target/release/examples/build-pack"
|
|
if [ ! -x "$BUILDER" ]; then
|
|
log "building the pack builder"
|
|
( cd "$ROOT" && cargo build --release -p hound-defs --example build-pack )
|
|
fi
|
|
|
|
# Outside $WORK on purpose: the loop below treats every directory in $WORK as
|
|
# an ecosystem, and a staging directory in there gets built into a pack named
|
|
# after the mktemp suffix. Which is exactly what happened the first time.
|
|
STAGE="$(mktemp -d "${TMPDIR:-/var/tmp}/hound-defs-stage.XXXXXX")"
|
|
trap 'rm -rf "$STAGE"' EXIT
|
|
|
|
published=0
|
|
for dir in "$WORK"/*/; do
|
|
eco="$(basename "$dir")"
|
|
# Lower-case, and '.' is not wanted in a filename component.
|
|
name="$(echo "$eco" | tr '[:upper:]' '[:lower:]' | tr '.' '-')"
|
|
case "$name" in
|
|
stage*|.*) log "$eco: not an ecosystem, skipping"; continue ;;
|
|
esac
|
|
count="$(find "$dir" -maxdepth 1 -name '*.json' | wc -l)"
|
|
if [ "$count" -eq 0 ]; then
|
|
log "$eco: no malicious records, skipping"
|
|
continue
|
|
fi
|
|
pack="$STAGE/${name}-${VERSION}.pack"
|
|
if "$BUILDER" "$dir" "$pack" "$KEY" "$VERSION" >/dev/null 2>&1; then
|
|
log "$eco: built $(basename "$pack") from $count record(s)"
|
|
published=$((published + 1))
|
|
else
|
|
log "$eco: BUILD FAILED — keeping the previous pack"
|
|
fi
|
|
done
|
|
|
|
if [ "$published" -eq 0 ]; then
|
|
# Never publish an empty feed. An agent that installs it would report a
|
|
# clean machine with no indicators loaded, which is worse than one that
|
|
# keeps yesterday's.
|
|
log "nothing built; leaving the published feed untouched"
|
|
exit 1
|
|
fi
|
|
|
|
# Packs first, then the index that advertises them.
|
|
for pack in "$STAGE"/*.pack; do
|
|
base="$(basename "$pack")"
|
|
cp "$pack" "$DEST/.$base.tmp"
|
|
chmod 644 "$DEST/.$base.tmp"
|
|
mv -f "$DEST/.$base.tmp" "$DEST/$base"
|
|
done
|
|
|
|
python3 - "$DEST" "$VERSION" <<'PY'
|
|
import hashlib, json, os, sys
|
|
dest, version = sys.argv[1], sys.argv[2]
|
|
packs = []
|
|
# One pack per ecosystem: the newest. Older ones stay on disk so existing
|
|
# URLs keep resolving, but the index only ever advertises current data.
|
|
newest = {}
|
|
for f in sorted(os.listdir(dest)):
|
|
if not f.endswith(".pack"):
|
|
continue
|
|
eco = f.rsplit("-", 1)[0]
|
|
newest[eco] = f
|
|
for eco, f in sorted(newest.items()):
|
|
p = os.path.join(dest, f)
|
|
packs.append({
|
|
"file": f,
|
|
"sha256": hashlib.sha256(open(p, "rb").read()).hexdigest(),
|
|
"size": os.path.getsize(p),
|
|
"version": f.rsplit("-", 1)[1][:-5],
|
|
})
|
|
tmp = os.path.join(dest, ".index.json.tmp")
|
|
with open(tmp, "w") as fh:
|
|
json.dump({"packs": packs}, fh, indent=2)
|
|
fh.write("\n")
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, os.path.join(dest, "index.json"))
|
|
print(f"index.json lists {len(packs)} pack(s)")
|
|
PY
|
|
|
|
log "published $published pack(s) for $VERSION"
|
|
|
|
# Clear the extracted records. They are the bulk of the scratch — 1.2 GB
|
|
# after a single run — and the daily timer regenerates them every time. The
|
|
# downloaded zips stay, because ingest-osv.py reuses them and fetching is the
|
|
# slow part; the JSON extracted from them costs seconds to rebuild and is
|
|
# pure waste to keep. Left alone, a nightly job quietly fills the disk of the
|
|
# machine that publishes a security feed.
|
|
find "$WORK" -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} +
|
|
log "scratch now $(du -sh "$WORK" 2>/dev/null | cut -f1)"
|