//! Hound's own engine: yara-x, in process, with a verdict cache. //! //! This is the Phase 0 replacement for forking `clamscan`. The old path //! spent 6.5 seconds and ~1.5 GB of RSS on a 68-byte file because every //! invocation reloaded a 169 MB signature database. Here the ruleset is //! compiled once at daemon start, the scanner is reused across every //! file in a walk, and an unchanged file that has been seen before never //! reaches the matcher at all. //! //! Deliberate choices worth knowing about: //! //! * **Symlinks are never followed.** A directory walk that follows links //! can loop, can be steered outside the requested tree by anyone who //! can create a link, and re-scans the same inode repeatedly. We stat //! with `symlink_metadata` and skip links entirely. //! * **Pseudo-filesystems are skipped.** Reading `/proc` and `/sys` is //! meaningless here and reading some of their files blocks forever. //! * **Oversized files are counted, not read.** Reporting them as scanned //! would be a lie; skipping them silently would be worse. They are //! counted separately and surfaced in the summary. use anyhow::{Context, Result}; use hound_api::{DbFile, Found, ScanResult}; use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::cache::{FileKey, VerdictCache}; use crate::engine::{to_rfc3339, ScanEngine}; use crate::rules::{RuleSet, RuleStore}; /// Files larger than this are skipped. Malware that matters is rarely /// this big, and reading disk images on every scan makes the product /// unusable on the developer machines we are targeting. const DEFAULT_MAX_FILE_BYTES: u64 = 100 * 1024 * 1024; /// How many file verdicts to remember. ~200k entries is a few tens of MB /// and comfortably covers a working developer tree plus the system. const CACHE_CAPACITY: usize = 200_000; /// Directory prefixes that are never worth walking. const SKIP_PREFIXES: &[&str] = &[ "/proc", "/sys", "/dev", "/run", "/var/lib/hound/vault", ]; pub struct HoundEngine { rules: RuleStore, cache: VerdictCache, max_file_bytes: u64, } impl HoundEngine { /// Compile the ruleset and build the cache. Called once, from the /// engine factory, at daemon start. pub fn new() -> Result { let rules = RuleStore::load()?; let max_file_bytes = std::env::var("HOUNDD_MAX_FILE_BYTES") .ok() .and_then(|v| v.parse().ok()) .unwrap_or(DEFAULT_MAX_FILE_BYTES); Ok(Self { rules, cache: VerdictCache::new(CACHE_CAPACITY), max_file_bytes, }) } /// Cache statistics, for `hound status` and the benchmark harness. pub fn cache_stats(&self) -> (usize, u64, u64) { self.cache.stats() } } /// Everything a walk accumulated. Kept separate from the wire type so the /// skipped count can be reported without changing the public API shape. #[derive(Default)] struct Tally { scanned: u64, infected: u64, skipped_large: u64, found: Vec, } impl ScanEngine for HoundEngine { fn name(&self) -> &'static str { "hound" } fn probe(&self) -> (bool, String, Option) { let set = self.rules.current(); let summary = format!( "{} rules loaded from {} source(s) [hound {}]", set.count, set.sources.len(), set.version ); ( true, summary, Some(DbFile { file: set.version.clone(), updated_at: to_rfc3339(set.loaded_at), }), ) } fn scan(&self, path: &str, recursive: bool) -> Result { let root = std::fs::canonicalize(path).with_context(|| format!("no such path: {path}"))?; let set = self.rules.current(); // One scanner for the whole walk. Constructing it per file would // reintroduce a chunk of the per-invocation cost we just removed. let mut scanner = yara_x::Scanner::new(&set.rules); let mut tally = Tally::default(); let mut queue: Vec = vec![root.clone()]; while let Some(current) = queue.pop() { let Ok(md) = std::fs::symlink_metadata(¤t) else { continue; }; if md.is_symlink() { continue; } if md.is_dir() { // The root is always descended into; deeper levels only // when the caller asked for a recursive scan. if current != root && !recursive { continue; } if is_skipped_dir(¤t) { continue; } for entry in std::fs::read_dir(¤t).into_iter().flatten().flatten() { queue.push(entry.path()); } continue; } if !md.is_file() { continue; } if md.len() > self.max_file_bytes { tally.skipped_large += 1; continue; } self.scan_one(¤t, &md, &set, &mut scanner, &mut tally); } let clean = tally.scanned.saturating_sub(tally.infected); Ok(ScanResult { scanned: tally.scanned, clean, infected: tally.infected, skipped: tally.skipped_large, found: tally.found, }) } fn scan_bytes(&self, bytes: &[u8]) -> Option { let set = self.rules.current(); let mut scanner = yara_x::Scanner::new(&set.rules); scanner .scan(bytes) .ok()? .matching_rules() .next() .map(|r| RuleSet::detection_name(&r)) } fn update(&self) -> Result<(bool, String, String)> { let before = self.rules.current().count; match self.rules.reload() { Ok(set) => { // Verdicts reached under the old rules say nothing about // the new ones. self.cache.clear(); Ok(( true, "reload rules".to_string(), format!( "OK: {} rules loaded ({}), was {before}\nsources: {}\n", set.count, set.version, set.sources.join(", ") ), )) } Err(e) => Ok((false, "reload rules".to_string(), format!("{e}\n"))), } } } impl HoundEngine { /// Scan a single regular file, consulting the cache first. fn scan_one( &self, path: &Path, md: &std::fs::Metadata, set: &Arc, scanner: &mut yara_x::Scanner, tally: &mut Tally, ) { let key = FileKey::from_metadata(md); if let Some(verdict) = self.cache.get(&key) { tally.scanned += 1; if let Some(name) = verdict { tally.infected += 1; tally.found.push(Found { path: path.to_string_lossy().into_owned(), virus: name.to_string(), }); } return; } let Ok(bytes) = std::fs::read(path) else { // Unreadable is not clean, so it is not cached and not // counted as scanned. return; }; tally.scanned += 1; let Ok(results) = scanner.scan(&bytes) else { return; }; // A file can trip several rules; report it once, under the first // match, exactly as the ClamAV path did with --allmatch. let hit = results .matching_rules() .next() .map(|r| RuleSet::detection_name(&r)); match hit { Some(name) => { tally.infected += 1; tally.found.push(Found { path: path.to_string_lossy().into_owned(), virus: name.clone(), }); self.cache.put(key, Some(name.into())); } None => self.cache.put(key, None), } let _ = set; } } /// Pseudo-filesystems and our own vault: never walked. fn is_skipped_dir(path: &Path) -> bool { SKIP_PREFIXES.iter().any(|p| path.starts_with(p)) } #[cfg(test)] mod tests { use super::*; use std::fs; fn tmpdir(tag: &str) -> PathBuf { let d = std::env::temp_dir().join(format!( "hound-native-{tag}-{}-{:?}", std::process::id(), std::thread::current().id() )); let _ = fs::remove_dir_all(&d); fs::create_dir_all(&d).unwrap(); d } const EICAR: &str = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*"; #[test] fn finds_eicar_in_a_directory() { let d = tmpdir("eicar"); fs::write(d.join("clean.txt"), b"nothing to see").unwrap(); fs::write(d.join("eicar.com"), EICAR).unwrap(); let e = HoundEngine::new().unwrap(); let r = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(r.scanned, 2); assert_eq!(r.infected, 1); assert_eq!(r.clean, 1); assert_eq!(r.found[0].virus, "EICAR-Test-Signature"); assert!(r.found[0].path.ends_with("eicar.com")); let _ = fs::remove_dir_all(&d); } #[test] fn clean_tree_is_clean() { let d = tmpdir("clean"); for i in 0..20 { fs::write(d.join(format!("f{i}.txt")), format!("file number {i}")).unwrap(); } let e = HoundEngine::new().unwrap(); let r = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(r.scanned, 20); assert_eq!(r.infected, 0); assert!(r.is_clean()); let _ = fs::remove_dir_all(&d); } #[test] fn non_recursive_stops_at_the_top() { let d = tmpdir("shallow"); fs::write(d.join("top.txt"), b"top").unwrap(); let sub = d.join("sub"); fs::create_dir_all(&sub).unwrap(); fs::write(sub.join("eicar.com"), EICAR).unwrap(); let e = HoundEngine::new().unwrap(); let r = e.scan(d.to_str().unwrap(), false).unwrap(); assert_eq!(r.scanned, 1, "must not descend when recursive is false"); assert_eq!(r.infected, 0); let _ = fs::remove_dir_all(&d); } #[test] fn symlinks_are_not_followed() { let d = tmpdir("symlink"); let real = d.join("real"); fs::create_dir_all(&real).unwrap(); fs::write(real.join("eicar.com"), EICAR).unwrap(); // A link pointing back at the parent would loop forever if followed. std::os::unix::fs::symlink(&d, d.join("loop")).unwrap(); let e = HoundEngine::new().unwrap(); let r = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(r.scanned, 1, "the linked tree must not be walked twice"); assert_eq!(r.infected, 1); let _ = fs::remove_dir_all(&d); } #[test] fn oversized_files_are_skipped_not_scanned() { let d = tmpdir("large"); fs::write(d.join("big.bin"), vec![0u8; 4096]).unwrap(); fs::write(d.join("small.txt"), b"ok").unwrap(); let mut e = HoundEngine::new().unwrap(); e.max_file_bytes = 1024; let r = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(r.scanned, 1, "the 4 KB file must not be counted as scanned"); assert_eq!(r.skipped, 1, "and it must be reported, not silently dropped"); let _ = fs::remove_dir_all(&d); } #[test] fn second_scan_is_served_from_cache() { let d = tmpdir("cache"); for i in 0..30 { fs::write(d.join(format!("f{i}.txt")), format!("contents {i}")).unwrap(); } let e = HoundEngine::new().unwrap(); let first = e.scan(d.to_str().unwrap(), true).unwrap(); let (_, hits_after_first, _) = e.cache_stats(); assert_eq!(hits_after_first, 0, "a cold walk cannot hit"); let second = e.scan(d.to_str().unwrap(), true).unwrap(); let (_, hits, _) = e.cache_stats(); assert_eq!(first.scanned, second.scanned); assert_eq!(hits, 30, "every file should be served from cache the second time"); let _ = fs::remove_dir_all(&d); } #[test] fn editing_a_file_invalidates_its_cache_entry() { let d = tmpdir("invalidate"); let f = d.join("mutable.txt"); fs::write(&f, b"harmless").unwrap(); let e = HoundEngine::new().unwrap(); let first = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(first.infected, 0); // Rewrite the same path with EICAR. Size and mtime both change. fs::write(&f, EICAR).unwrap(); let second = e.scan(d.to_str().unwrap(), true).unwrap(); assert_eq!(second.infected, 1, "a rewritten file must be rescanned"); let _ = fs::remove_dir_all(&d); } #[test] fn scanning_a_single_file_works() { let d = tmpdir("single"); let f = d.join("eicar.com"); fs::write(&f, EICAR).unwrap(); let e = HoundEngine::new().unwrap(); let r = e.scan(f.to_str().unwrap(), false).unwrap(); assert_eq!(r.scanned, 1); assert_eq!(r.infected, 1); let _ = fs::remove_dir_all(&d); } #[test] fn missing_path_is_an_error() { let e = HoundEngine::new().unwrap(); assert!(e.scan("/definitely/not/here/at/all", true).is_err()); } #[test] fn probe_reports_the_ruleset() { let e = HoundEngine::new().unwrap(); let (present, summary, db) = e.probe(); assert!(present); assert!(summary.contains("rules loaded")); assert!(db.unwrap().file.starts_with("builtin-")); } #[test] fn update_reloads_and_clears_the_cache() { let d = tmpdir("update"); fs::write(d.join("a.txt"), b"a").unwrap(); let e = HoundEngine::new().unwrap(); e.scan(d.to_str().unwrap(), true).unwrap(); assert!(e.cache_stats().0 > 0); let (ok, label, log) = e.update().unwrap(); assert!(ok, "reload should succeed: {log}"); assert_eq!(label, "reload rules"); assert_eq!(e.cache_stats().0, 0, "stale verdicts must be dropped"); let _ = fs::remove_dir_all(&d); } /// The Phase 0 exit criterion, asserted so it can never quietly /// regress. The path this replaced took 6.5 seconds *per file* /// because it reloaded a 169 MB database on every invocation. /// /// The bound is deliberately loose (2 s for 400 files, against a /// measured ~9 ms) because CI machines are slow and shared, and a /// flaky performance test gets deleted rather than fixed. It is /// tight enough to catch the only regression that matters: someone /// reintroducing per-file setup cost. #[test] fn four_hundred_files_scan_in_under_two_seconds() { let d = tmpdir("perf"); for i in 0..399 { fs::write(d.join(format!("f{i}.bin")), format!("payload {i}").repeat(64)).unwrap(); } fs::write(d.join("eicar.com"), EICAR).unwrap(); let e = HoundEngine::new().unwrap(); let started = std::time::Instant::now(); let r = e.scan(d.to_str().unwrap(), true).unwrap(); let elapsed = started.elapsed(); assert_eq!(r.scanned, 400); assert_eq!(r.infected, 1, "EICAR must still be caught at speed"); assert!( elapsed < std::time::Duration::from_secs(2), "400 files took {elapsed:?} — per-file setup cost is back" ); eprintln!("perf: 400 files cold in {elapsed:?}"); let _ = fs::remove_dir_all(&d); } #[test] fn scan_bytes_matches_without_touching_disk() { let e = HoundEngine::new().unwrap(); assert_eq!( e.scan_bytes(EICAR.as_bytes()).as_deref(), Some("EICAR-Test-Signature") ); assert!(e.scan_bytes(b"an ordinary sentence").is_none()); } #[test] fn skips_pseudo_filesystems() { assert!(is_skipped_dir(Path::new("/proc/1"))); assert!(is_skipped_dir(Path::new("/sys/kernel"))); assert!(!is_skipped_dir(Path::new("/home/joe/src"))); } }