defs: 235,577 real indicators, published and fetched over TLS

The machinery has been real for a while and the content was 19 crates.io
records. Now it is the whole malicious-packages feed:

  npm         220,324
  PyPI         11,702
  RubyGems      3,512
  Go               19
  crates.io        19
  Packagist         1
  ────────────────────
              235,577 indicators, 45 MB across six signed packs

The striking number is npm: 220,323 of its 227,149 OSV records are
malicious packages rather than vulnerability advisories. NINETY-SEVEN
PERCENT. That is the whole thesis in one statistic — the dominant
security fact about the npm ecosystem is not that libraries have bugs,
it is that the registry is full of things that exist only to be malware.
It is also why the malicious-versus-vulnerability classifier had to come
first: without it this pack would have been 227,149 indicators, and
7,000 of them would have been ordinary libraries.

Verified end to end from a genuinely empty definitions directory:

  before:  defs: no verified packs were found
  update:  6 pack(s) installed, 0 already current
           loaded 235577 indicators from 6 pack(s) [2026.08.21]
  elapsed: 1.4 seconds for 45 MB, verified and loaded
  RSS:     214 MB (unit cap is 1G)

And it detects. A lockfile with 500 ordinary packages and three real
malicious ones — ineldua, dian-kue20-riris, restart-rocket-jasmine-jwt,
picked at random from the feed rather than chosen to work — produced
three criticals and no false positives, in 11 milliseconds. The cuckoo
filter is doing exactly what it was built for: 500 clean lookups never
touch the map.

tools/ingest-osv.py does the coarse filter so build-pack does not walk a
quarter of a million files it will discard. It mirrors the Rust
classifier deliberately, and the Rust side still applies its own — a
record slipping through the Python changes nothing.

`hound status` now reports definitions, and reports them when there are
none, with the reason. Silence would let somebody believe they were
protected by a feed they never received.

Known and not yet solved: every agent downloads 43 MB of npm pack on
first update, and the whole pack again whenever it changes. Incremental
updates and a CDN in front of defs.houndav.com are both wanted. The
subdomain split exists precisely so the second one is a DNS change.

362 tests pass.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

crates/hound-api/src/lib.rs

@@ -108,6 +108,9 @@ pub struct Status {
     /// Execution-gate state.
     #[serde(default)]
     pub gate: GateStatus,
+    /// Loaded definition packs.
+    #[serde(default)]
+    pub defs: DefsStatus,
 }
 
 /// One thing on this machine that can make code run again after a reboot.
@@ -155,6 +158,20 @@ pub struct PersistenceReport {
     pub changes: Vec<PersistenceChange>,
 }
 
+/// Loaded definitions, for the tray and `hound status`.
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct DefsStatus {
+    /// Feed version, e.g. "2026.08.21". Empty when nothing loaded.
+    #[serde(default)]
+    pub version: String,
+    pub indicators: u64,
+    pub packs: u64,
+    /// Why nothing loaded, when nothing did. An operator who believes
+    /// they have definitions and does not is worse off than one who knows.
+    #[serde(default)]
+    pub detail: String,
+}
+
 /// Execution-gate state, for the tray and `hound status`.
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct GateStatus {

crates/hound/src/main.rs

@@ -305,6 +305,24 @@ fn wrap(text: &str, width: usize) -> Vec<String> {
     lines
 }
 
+/// Report the loaded definitions.
+///
+/// Prints even when nothing is loaded, and says why. Silence would let
+/// somebody believe they were protected by a feed they never received.
+fn print_defs(d: &hound_api::DefsStatus) {
+    if d.indicators == 0 {
+        let why = if d.detail.is_empty() { "none loaded" } else { &d.detail };
+        println!("  Defs:      {}", why.dimmed());
+        return;
+    }
+    println!(
+        "  Defs:      {} indicators from {} pack(s) [{}]",
+        d.indicators.to_string().green(),
+        d.packs,
+        d.version
+    );
+}
+
 /// Report the execution gate.
 ///
 /// An armed gate is the most consequential thing the daemon is doing, and
@@ -383,6 +401,7 @@ fn run(client: &Client, cmd: &Cmd) -> Result<i32> {
                         db.file, db.updated_at
                     );
                 }
+                print_defs(&st.defs);
                 print_gate(&st.gate);
             } else {
                 println!(

crates/houndd/src/main.rs

@@ -676,6 +676,15 @@ fn status(st: &DaemonState) -> Result<hound_api::Status> {
         realtime: st.realtime.status(),
         quarantined: st.quarantine.count(),
         gate: gate_status(st),
+        defs: {
+            let d = st.defs.current();
+            hound_api::DefsStatus {
+                version: d.version.clone(),
+                indicators: d.indicators as u64,
+                packs: d.packs.len() as u64,
+                detail: d.detail.clone(),
+            }
+        },
     })
 }
 

tools/ingest-osv.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Fetch OSV ecosystem exports and extract the malicious-package records.
+
+Hound's index holds malicious packages only, never vulnerability
+advisories — they mean opposite things, and conflating them once had
+Hound report tokio as malware. See crates/hound-defs/src/osv.rs.
+
+This tool does the coarse filter so `build-pack` does not have to walk a
+quarter of a million files it will discard. The Rust parser still applies
+its own classifier, so a record slipping through here changes nothing.
+
+    tools/ingest-osv.py <work-dir> [ecosystem ...]
+
+Leaves <work-dir>/<ecosystem>/*.json ready for build-pack.
+"""
+import json
+import os
+import sys
+import urllib.request
+import zipfile
+
+BASE = "https://osv-vulnerabilities.storage.googleapis.com"
+DEFAULT = ["npm", "PyPI", "crates.io", "RubyGems", "Go", "Packagist"]
+
+
+def is_malicious(record: dict) -> bool:
+    """Mirror of osv::is_malicious_record in the Rust side."""
+    if record.get("id", "").startswith("MAL-"):
+        return True
+    if "malicious-packages-origins" in record.get("database_specific", {}):
+        return True
+    summary = record.get("summary", "").lower()
+    return summary.startswith("malicious code in") or summary.startswith("malicious package")
+
+
+def fetch(eco: str, work: str) -> str:
+    path = os.path.join(work, f"{eco}.zip")
+    if os.path.exists(path) and os.path.getsize(path) > 0:
+        print(f"{eco}: using cached {os.path.getsize(path) // 1048576} MB")
+        return path
+    url = f"{BASE}/{eco}/all.zip"
+    print(f"{eco}: fetching {url}")
+    urllib.request.urlretrieve(url, path)
+    return path
+
+
+def extract(eco: str, zip_path: str, work: str) -> int:
+    out_dir = os.path.join(work, eco.replace(".", "-").lower())
+    os.makedirs(out_dir, exist_ok=True)
+    kept = skipped = 0
+    with zipfile.ZipFile(zip_path) as z:
+        for name in z.namelist():
+            if not name.endswith(".json"):
+                continue
+            raw = z.read(name)
+            try:
+                record = json.loads(raw)
+            except json.JSONDecodeError:
+                skipped += 1
+                continue
+            if not is_malicious(record):
+                skipped += 1
+                continue
+            with open(os.path.join(out_dir, os.path.basename(name)), "wb") as f:
+                f.write(raw)
+            kept += 1
+    print(f"{eco}: {kept} malicious, {skipped} advisories skipped -> {out_dir}")
+    return kept
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print(__doc__)
+        return 2
+    work = sys.argv[1]
+    ecosystems = sys.argv[2:] or DEFAULT
+    os.makedirs(work, exist_ok=True)
+
+    total = 0
+    for eco in ecosystems:
+        try:
+            total += extract(eco, fetch(eco, work), work)
+        except Exception as e:  # a missing export must not stop the rest
+            print(f"{eco}: skipped ({e})")
+    print(f"\ntotal malicious records: {total}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())