Antivirus/crates/houndd/src/native.rs

//! Hound's own engine: yara-x, in process, with a verdict cache.
//!
//! This is the Phase 0 replacement for forking `clamscan`. The old path
//! spent 6.5 seconds and ~1.5 GB of RSS on a 68-byte file because every
//! invocation reloaded a 169 MB signature database. Here the ruleset is
//! compiled once at daemon start, the scanner is reused across every
//! file in a walk, and an unchanged file that has been seen before never
//! reaches the matcher at all.
//!
//! Deliberate choices worth knowing about:
//!
//! * **Symlinks are never followed.** A directory walk that follows links
//!   can loop, can be steered outside the requested tree by anyone who
//!   can create a link, and re-scans the same inode repeatedly. We stat
//!   with `symlink_metadata` and skip links entirely.
//! * **Pseudo-filesystems are skipped.** Reading `/proc` and `/sys` is
//!   meaningless here and reading some of their files blocks forever.
//! * **Oversized files are counted, not read.** Reporting them as scanned
//!   would be a lie; skipping them silently would be worse. They are
//!   counted separately and surfaced in the summary.

use anyhow::{Context, Result};
use hound_api::{DbFile, Found, ScanResult};
use std::path::{Path, PathBuf};
use std::sync::Arc;

use crate::cache::{FileKey, VerdictCache};
use crate::engine::{to_rfc3339, ScanEngine};
use crate::rules::{RuleSet, RuleStore};

/// Files larger than this are skipped. Malware that matters is rarely
/// this big, and reading disk images on every scan makes the product
/// unusable on the developer machines we are targeting.
const DEFAULT_MAX_FILE_BYTES: u64 = 100 * 1024 * 1024;

/// How many file verdicts to remember. ~200k entries is a few tens of MB
/// and comfortably covers a working developer tree plus the system.
const CACHE_CAPACITY: usize = 200_000;

/// Directory prefixes that are never worth walking.
const SKIP_PREFIXES: &[&str] = &[
    "/proc",
    "/sys",
    "/dev",
    "/run",
    "/var/lib/hound/vault",
];

pub struct HoundEngine {
    rules: RuleStore,
    cache: VerdictCache,
    max_file_bytes: u64,
}

impl HoundEngine {
    /// Compile the ruleset and build the cache. Called once, from the
    /// engine factory, at daemon start.
    pub fn new() -> Result<Self> {
        let rules = RuleStore::load()?;
        let max_file_bytes = std::env::var("HOUNDD_MAX_FILE_BYTES")
            .ok()
            .and_then(|v| v.parse().ok())
            .unwrap_or(DEFAULT_MAX_FILE_BYTES);
        Ok(Self {
            rules,
            cache: VerdictCache::new(CACHE_CAPACITY),
            max_file_bytes,
        })
    }

    /// Cache statistics, for `hound status` and the benchmark harness.
    pub fn cache_stats(&self) -> (usize, u64, u64) {
        self.cache.stats()
    }
}

/// Everything a walk accumulated. Kept separate from the wire type so the
/// skipped count can be reported without changing the public API shape.
#[derive(Default)]
struct Tally {
    scanned: u64,
    infected: u64,
    skipped_large: u64,
    found: Vec<Found>,
}

impl ScanEngine for HoundEngine {
    fn name(&self) -> &'static str {
        "hound"
    }

    fn probe(&self) -> (bool, String, Option<DbFile>) {
        let set = self.rules.current();
        let summary = format!(
            "{} rules loaded from {} source(s) [hound {}]",
            set.count,
            set.sources.len(),
            set.version
        );
        (
            true,
            summary,
            Some(DbFile {
                file: set.version.clone(),
                updated_at: to_rfc3339(set.loaded_at),
            }),
        )
    }

    fn scan(&self, path: &str, recursive: bool) -> Result<ScanResult> {
        let root = std::fs::canonicalize(path).with_context(|| format!("no such path: {path}"))?;
        let set = self.rules.current();

        // One scanner for the whole walk. Constructing it per file would
        // reintroduce a chunk of the per-invocation cost we just removed.
        let mut scanner = yara_x::Scanner::new(&set.rules);
        let mut tally = Tally::default();

        let mut queue: Vec<PathBuf> = vec![root.clone()];
        while let Some(current) = queue.pop() {
            let Ok(md) = std::fs::symlink_metadata(&current) else {
                continue;
            };

            if md.is_symlink() {
                continue;
            }

            if md.is_dir() {
                // The root is always descended into; deeper levels only
                // when the caller asked for a recursive scan.
                if current != root && !recursive {
                    continue;
                }
                if is_skipped_dir(&current) {
                    continue;
                }
                for entry in std::fs::read_dir(&current).into_iter().flatten().flatten() {
                    queue.push(entry.path());
                }
                continue;
            }

            if !md.is_file() {
                continue;
            }
            if md.len() > self.max_file_bytes {
                tally.skipped_large += 1;
                continue;
            }

            self.scan_one(&current, &md, &set, &mut scanner, &mut tally);
        }

        let clean = tally.scanned.saturating_sub(tally.infected);
        Ok(ScanResult {
            scanned: tally.scanned,
            clean,
            infected: tally.infected,
            skipped: tally.skipped_large,
            found: tally.found,
        })
    }

    fn scan_bytes(&self, bytes: &[u8]) -> Option<String> {
        let set = self.rules.current();
        let mut scanner = yara_x::Scanner::new(&set.rules);
        scanner
            .scan(bytes)
            .ok()?
            .matching_rules()
            .next()
            .map(|r| RuleSet::detection_name(&r))
    }

    fn update(&self) -> Result<(bool, String, String)> {
        let before = self.rules.current().count;
        match self.rules.reload() {
            Ok(set) => {
                // Verdicts reached under the old rules say nothing about
                // the new ones.
                self.cache.clear();
                Ok((
                    true,
                    "reload rules".to_string(),
                    format!(
                        "OK: {} rules loaded ({}), was {before}\nsources: {}\n",
                        set.count,
                        set.version,
                        set.sources.join(", ")
                    ),
                ))
            }
            Err(e) => Ok((false, "reload rules".to_string(), format!("{e}\n"))),
        }
    }
}

impl HoundEngine {
    /// Scan a single regular file, consulting the cache first.
    fn scan_one(
        &self,
        path: &Path,
        md: &std::fs::Metadata,
        set: &Arc<RuleSet>,
        scanner: &mut yara_x::Scanner,
        tally: &mut Tally,
    ) {
        let key = FileKey::from_metadata(md);

        if let Some(verdict) = self.cache.get(&key) {
            tally.scanned += 1;
            if let Some(name) = verdict {
                tally.infected += 1;
                tally.found.push(Found {
                    path: path.to_string_lossy().into_owned(),
                    virus: name.to_string(),
                });
            }
            return;
        }

        let Ok(bytes) = std::fs::read(path) else {
            // Unreadable is not clean, so it is not cached and not
            // counted as scanned.
            return;
        };

        tally.scanned += 1;

        let Ok(results) = scanner.scan(&bytes) else {
            return;
        };

        // A file can trip several rules; report it once, under the first
        // match, exactly as the ClamAV path did with --allmatch.
        let hit = results
            .matching_rules()
            .next()
            .map(|r| RuleSet::detection_name(&r));

        match hit {
            Some(name) => {
                tally.infected += 1;
                tally.found.push(Found {
                    path: path.to_string_lossy().into_owned(),
                    virus: name.clone(),
                });
                self.cache.put(key, Some(name.into()));
            }
            None => self.cache.put(key, None),
        }
        let _ = set;
    }
}

/// Pseudo-filesystems and our own vault: never walked.
fn is_skipped_dir(path: &Path) -> bool {
    SKIP_PREFIXES.iter().any(|p| path.starts_with(p))
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;

    fn tmpdir(tag: &str) -> PathBuf {
        let d = std::env::temp_dir().join(format!(
            "hound-native-{tag}-{}-{:?}",
            std::process::id(),
            std::thread::current().id()
        ));
        let _ = fs::remove_dir_all(&d);
        fs::create_dir_all(&d).unwrap();
        d
    }

    const EICAR: &str = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*";

    #[test]
    fn finds_eicar_in_a_directory() {
        let d = tmpdir("eicar");
        fs::write(d.join("clean.txt"), b"nothing to see").unwrap();
        fs::write(d.join("eicar.com"), EICAR).unwrap();

        let e = HoundEngine::new().unwrap();
        let r = e.scan(d.to_str().unwrap(), true).unwrap();

        assert_eq!(r.scanned, 2);
        assert_eq!(r.infected, 1);
        assert_eq!(r.clean, 1);
        assert_eq!(r.found[0].virus, "EICAR-Test-Signature");
        assert!(r.found[0].path.ends_with("eicar.com"));
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn clean_tree_is_clean() {
        let d = tmpdir("clean");
        for i in 0..20 {
            fs::write(d.join(format!("f{i}.txt")), format!("file number {i}")).unwrap();
        }
        let e = HoundEngine::new().unwrap();
        let r = e.scan(d.to_str().unwrap(), true).unwrap();
        assert_eq!(r.scanned, 20);
        assert_eq!(r.infected, 0);
        assert!(r.is_clean());
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn non_recursive_stops_at_the_top() {
        let d = tmpdir("shallow");
        fs::write(d.join("top.txt"), b"top").unwrap();
        let sub = d.join("sub");
        fs::create_dir_all(&sub).unwrap();
        fs::write(sub.join("eicar.com"), EICAR).unwrap();

        let e = HoundEngine::new().unwrap();
        let r = e.scan(d.to_str().unwrap(), false).unwrap();
        assert_eq!(r.scanned, 1, "must not descend when recursive is false");
        assert_eq!(r.infected, 0);
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn symlinks_are_not_followed() {
        let d = tmpdir("symlink");
        let real = d.join("real");
        fs::create_dir_all(&real).unwrap();
        fs::write(real.join("eicar.com"), EICAR).unwrap();
        // A link pointing back at the parent would loop forever if followed.
        std::os::unix::fs::symlink(&d, d.join("loop")).unwrap();

        let e = HoundEngine::new().unwrap();
        let r = e.scan(d.to_str().unwrap(), true).unwrap();
        assert_eq!(r.scanned, 1, "the linked tree must not be walked twice");
        assert_eq!(r.infected, 1);
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn oversized_files_are_skipped_not_scanned() {
        let d = tmpdir("large");
        fs::write(d.join("big.bin"), vec![0u8; 4096]).unwrap();
        fs::write(d.join("small.txt"), b"ok").unwrap();

        let mut e = HoundEngine::new().unwrap();
        e.max_file_bytes = 1024;
        let r = e.scan(d.to_str().unwrap(), true).unwrap();
        assert_eq!(r.scanned, 1, "the 4 KB file must not be counted as scanned");
        assert_eq!(r.skipped, 1, "and it must be reported, not silently dropped");
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn second_scan_is_served_from_cache() {
        let d = tmpdir("cache");
        for i in 0..30 {
            fs::write(d.join(format!("f{i}.txt")), format!("contents {i}")).unwrap();
        }
        let e = HoundEngine::new().unwrap();

        let first = e.scan(d.to_str().unwrap(), true).unwrap();
        let (_, hits_after_first, _) = e.cache_stats();
        assert_eq!(hits_after_first, 0, "a cold walk cannot hit");

        let second = e.scan(d.to_str().unwrap(), true).unwrap();
        let (_, hits, _) = e.cache_stats();

        assert_eq!(first.scanned, second.scanned);
        assert_eq!(hits, 30, "every file should be served from cache the second time");
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn editing_a_file_invalidates_its_cache_entry() {
        let d = tmpdir("invalidate");
        let f = d.join("mutable.txt");
        fs::write(&f, b"harmless").unwrap();

        let e = HoundEngine::new().unwrap();
        let first = e.scan(d.to_str().unwrap(), true).unwrap();
        assert_eq!(first.infected, 0);

        // Rewrite the same path with EICAR. Size and mtime both change.
        fs::write(&f, EICAR).unwrap();
        let second = e.scan(d.to_str().unwrap(), true).unwrap();
        assert_eq!(second.infected, 1, "a rewritten file must be rescanned");
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn scanning_a_single_file_works() {
        let d = tmpdir("single");
        let f = d.join("eicar.com");
        fs::write(&f, EICAR).unwrap();
        let e = HoundEngine::new().unwrap();
        let r = e.scan(f.to_str().unwrap(), false).unwrap();
        assert_eq!(r.scanned, 1);
        assert_eq!(r.infected, 1);
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn missing_path_is_an_error() {
        let e = HoundEngine::new().unwrap();
        assert!(e.scan("/definitely/not/here/at/all", true).is_err());
    }

    #[test]
    fn probe_reports_the_ruleset() {
        let e = HoundEngine::new().unwrap();
        let (present, summary, db) = e.probe();
        assert!(present);
        assert!(summary.contains("rules loaded"));
        assert!(db.unwrap().file.starts_with("builtin-"));
    }

    #[test]
    fn update_reloads_and_clears_the_cache() {
        let d = tmpdir("update");
        fs::write(d.join("a.txt"), b"a").unwrap();
        let e = HoundEngine::new().unwrap();
        e.scan(d.to_str().unwrap(), true).unwrap();
        assert!(e.cache_stats().0 > 0);

        let (ok, label, log) = e.update().unwrap();
        assert!(ok, "reload should succeed: {log}");
        assert_eq!(label, "reload rules");
        assert_eq!(e.cache_stats().0, 0, "stale verdicts must be dropped");
        let _ = fs::remove_dir_all(&d);
    }

    /// The Phase 0 exit criterion, asserted so it can never quietly
    /// regress. The path this replaced took 6.5 seconds *per file*
    /// because it reloaded a 169 MB database on every invocation.
    ///
    /// The bound is deliberately loose (2 s for 400 files, against a
    /// measured ~9 ms) because CI machines are slow and shared, and a
    /// flaky performance test gets deleted rather than fixed. It is
    /// tight enough to catch the only regression that matters: someone
    /// reintroducing per-file setup cost.
    #[test]
    fn four_hundred_files_scan_in_under_two_seconds() {
        let d = tmpdir("perf");
        for i in 0..399 {
            fs::write(d.join(format!("f{i}.bin")), format!("payload {i}").repeat(64)).unwrap();
        }
        fs::write(d.join("eicar.com"), EICAR).unwrap();

        let e = HoundEngine::new().unwrap();
        let started = std::time::Instant::now();
        let r = e.scan(d.to_str().unwrap(), true).unwrap();
        let elapsed = started.elapsed();

        assert_eq!(r.scanned, 400);
        assert_eq!(r.infected, 1, "EICAR must still be caught at speed");
        assert!(
            elapsed < std::time::Duration::from_secs(2),
            "400 files took {elapsed:?} — per-file setup cost is back"
        );
        eprintln!("perf: 400 files cold in {elapsed:?}");
        let _ = fs::remove_dir_all(&d);
    }

    #[test]
    fn scan_bytes_matches_without_touching_disk() {
        let e = HoundEngine::new().unwrap();
        assert_eq!(
            e.scan_bytes(EICAR.as_bytes()).as_deref(),
            Some("EICAR-Test-Signature")
        );
        assert!(e.scan_bytes(b"an ordinary sentence").is_none());
    }

    #[test]
    fn skips_pseudo_filesystems() {
        assert!(is_skipped_dir(Path::new("/proc/1")));
        assert!(is_skipped_dir(Path::new("/sys/kernel")));
        assert!(!is_skipped_dir(Path::new("/home/joe/src")));
    }
}