Antivirus/crates/hound-defs/src/osv.rs

//! Parsing OSV records into indicators.
//!
//! The [`ossf/malicious-packages`](https://github.com/ossf/malicious-packages)
//! feed publishes confirmed-malicious packages in OSV format — Apache-2.0,
//! updated daily, around 226,000 records. It is the single most valuable
//! piece of detection content available to us for free, and it is exactly
//! the kind of data ClamAV's corpus says nothing about.
//!
//! The records are verbose and we need almost none of it. A `MAL-` record
//! answers one question — *is this package, at this version, known bad?* —
//! and everything else is provenance. We keep the identifier and the
//! summary so a finding can cite its source, and drop the rest.
//!
//! **This index holds malicious packages only, never vulnerabilities.**
//! The two live side by side in OSV — the crates.io export is 1,524 GHSA
//! and 1,206 RUSTSEC advisories against 19 malicious-package records —
//! and they mean opposite things. A malicious package should not exist at
//! all; a vulnerable one is a legitimate library with a bug, and most of
//! its versions are fine.
//!
//! Ingesting both cost a false positive on `tokio` during development:
//! the advisory `GHSA-2grh-hm3w-w7hv` describes a race condition fixed in
//! 1.8.1, and Hound reported tokio 1.38.0 — a version released years
//! later — as malware. Anything that flags tokio is a product nobody
//! trusts again. So a record has to *prove* it is a malicious-package
//! report before it becomes an indicator, and anything unrecognised is
//! dropped rather than guessed at.
//!
//! Within that set, a record almost always carries `introduced: "0"` with
//! no fix, meaning every version is malicious: the package exists only to
//! be malware, so there is no safe version to upgrade to.

use serde::{Deserialize, Serialize};

/// Which versions of a package a record condemns.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Versions {
    /// Every version. The package exists to be malware.
    All,
    /// Only these exact versions.
    Exact(Vec<String>),
}

impl Versions {
    /// Whether a specific version is covered.
    ///
    /// An unknown version against an `All` record still matches — that is
    /// the point of `All`. Against an `Exact` record it does not, because
    /// claiming a hit we cannot substantiate is how a scanner loses trust.
    pub fn covers(&self, version: &str) -> bool {
        match self {
            Versions::All => true,
            Versions::Exact(list) => list.iter().any(|v| v == version),
        }
    }
}

/// One malicious package, reduced to what a scan needs.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Indicator {
    /// Lowercased ecosystem: "npm", "pypi", "cratesio", "go", "rubygems".
    pub ecosystem: String,
    /// Package name, as the registry spells it.
    pub name: String,
    pub versions: Versions,
    /// The OSV identifier, so a finding can cite it.
    pub id: String,
    /// One line, for the human explanation.
    pub summary: String,
}

impl Indicator {
    /// The index key. Ecosystems are case-insensitive in OSV but not
    /// consistently spelled, and npm names are lowercase by rule, so the
    /// key is normalised on both halves.
    pub fn key(&self) -> String {
        key_for(&self.ecosystem, &self.name)
    }
}

/// Build a lookup key from an ecosystem and a package name.
pub fn key_for(ecosystem: &str, name: &str) -> String {
    format!(
        "{}:{}",
        normalise_ecosystem(ecosystem),
        name.to_ascii_lowercase()
    )
}

/// OSV spells ecosystems inconsistently across sources.
pub fn normalise_ecosystem(raw: &str) -> String {
    let lower = raw.to_ascii_lowercase();
    // "Ubuntu:22.04" and friends carry a suffix we do not use.
    let base = lower.split(':').next().unwrap_or(&lower);
    match base {
        "crates.io" | "cratesio" | "cargo" => "cratesio",
        "pypi" | "python" => "pypi",
        "go" | "golang" => "go",
        "rubygems" | "gem" => "rubygems",
        "packagist" | "composer" => "packagist",
        "maven" => "maven",
        "nuget" => "nuget",
        other => other,
    }
    .to_string()
}

/// Parse one OSV record's JSON.
///
/// Returns one indicator per affected package: a single record can
/// condemn the same malware published under several names.
pub fn parse_record(json: &str) -> Vec<Indicator> {
    let Ok(v) = serde_json::from_str::<serde_json::Value>(json) else {
        return Vec::new();
    };
    parse_value(&v)
}

/// Whether a record reports a malicious package rather than a vulnerability.
///
/// Three independent signals, any of which is sufficient, because the feed
/// is assembled from several sources that do not agree on conventions:
///
/// * a `MAL-` identifier, which is what the malicious-packages feed mints;
/// * `database_specific.malicious-packages-origins`, which that feed
///   attaches when it imports from GHSA or elsewhere;
/// * a summary of the form "Malicious code in …", GHSA's own wording for
///   its malware advisories.
///
/// Anything else is a vulnerability and is not our business here.
pub fn is_malicious_record(v: &serde_json::Value) -> bool {
    let id = v.get("id").and_then(|x| x.as_str()).unwrap_or_default();
    if id.starts_with("MAL-") {
        return true;
    }
    if v.get("database_specific")
        .and_then(|d| d.get("malicious-packages-origins"))
        .is_some()
    {
        return true;
    }
    let summary = v
        .get("summary")
        .and_then(|x| x.as_str())
        .unwrap_or_default()
        .to_ascii_lowercase();
    summary.starts_with("malicious code in") || summary.starts_with("malicious package")
}

/// Parse an already-decoded OSV record.
///
/// Returns nothing for a vulnerability advisory. That is not an oversight
/// — see the module note.
pub fn parse_value(v: &serde_json::Value) -> Vec<Indicator> {
    let id = v.get("id").and_then(|x| x.as_str()).unwrap_or_default();
    if id.is_empty() {
        return Vec::new();
    }
    if !is_malicious_record(v) {
        return Vec::new();
    }
    let summary = v
        .get("summary")
        .and_then(|x| x.as_str())
        .unwrap_or("Reported as malicious")
        .to_string();

    let Some(affected) = v.get("affected").and_then(|a| a.as_array()) else {
        return Vec::new();
    };

    affected
        .iter()
        .filter_map(|a| {
            let pkg = a.get("package")?;
            let name = pkg.get("name").and_then(|n| n.as_str())?;
            let ecosystem = pkg.get("ecosystem").and_then(|e| e.as_str())?;
            if name.is_empty() || ecosystem.is_empty() {
                return None;
            }
            Some(Indicator {
                ecosystem: normalise_ecosystem(ecosystem),
                name: name.to_string(),
                versions: versions_of(a),
                id: id.to_string(),
                summary: summary.clone(),
            })
        })
        .collect()
}

/// Work out which versions an `affected` entry covers.
fn versions_of(affected: &serde_json::Value) -> Versions {
    // An explicit `versions` list is the most precise thing OSV offers.
    if let Some(list) = affected.get("versions").and_then(|v| v.as_array()) {
        let exact: Vec<String> = list
            .iter()
            .filter_map(|v| v.as_str().map(str::to_string))
            .collect();
        if !exact.is_empty() {
            return Versions::Exact(exact);
        }
    }

    // Otherwise: a range introduced at "0" with no fixed event means the
    // package is malicious from its first version onwards, which for a
    // MAL- record means all of it.
    let ranges = affected.get("ranges").and_then(|r| r.as_array());
    if let Some(ranges) = ranges {
        for range in ranges {
            let Some(events) = range.get("events").and_then(|e| e.as_array()) else {
                continue;
            };
            let introduced_at_zero = events
                .iter()
                .any(|e| e.get("introduced").and_then(|i| i.as_str()) == Some("0"));
            let has_fix = events.iter().any(|e| e.get("fixed").is_some());
            if introduced_at_zero && !has_fix {
                return Versions::All;
            }
        }
    }

    // No usable version information. For a record we have already
    // confirmed is a malicious-package report, "all versions" is the
    // correct reading: the package exists to be malware.
    Versions::All
}

/// Parse a whole `all.json`-style array, or a stream of records.
pub fn parse_many(json: &str) -> Vec<Indicator> {
    let Ok(v) = serde_json::from_str::<serde_json::Value>(json) else {
        return Vec::new();
    };
    match v {
        serde_json::Value::Array(items) => items.iter().flat_map(parse_value).collect(),
        other => parse_value(&other),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A real record, trimmed. Fetched from api.osv.dev rather than
    /// invented, so the parser is tested against the shape that actually
    /// ships rather than the one documented.
    const REAL_MAL: &str = r#"{
      "id": "MAL-2024-1339",
      "summary": "Malicious code in test-poc2 (npm)",
      "aliases": ["GHSA-mhg2-cg9m-q926"],
      "modified": "2024-05-07T00:42:03.835318Z",
      "published": "2024-05-06T04:02:39Z",
      "references": [{"type": "ADVISORY", "url": "https://github.com/advisories/GHSA-mhg2-cg9m-q926"}],
      "affected": [{
        "package": {"name": "test-poc2", "ecosystem": "npm", "purl": "pkg:npm/test-poc2"},
        "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}]}],
        "database_specific": {"cwes": [{"cweId": "CWE-506"}]}
      }]
    }"#;

    #[test]
    fn parses_a_real_malicious_package_record() {
        let ind = parse_record(REAL_MAL);
        assert_eq!(ind.len(), 1);
        assert_eq!(ind[0].name, "test-poc2");
        assert_eq!(ind[0].ecosystem, "npm");
        assert_eq!(ind[0].id, "MAL-2024-1339");
        assert!(ind[0].summary.contains("Malicious code"));
    }

    #[test]
    fn introduced_at_zero_with_no_fix_means_every_version() {
        // The distinction that matters: a malicious package has no safe
        // version, unlike a vulnerability where the version boundary is
        // the entire point.
        let ind = parse_record(REAL_MAL);
        assert_eq!(ind[0].versions, Versions::All);
        assert!(ind[0].versions.covers("1.0.0"));
        assert!(ind[0].versions.covers("99.9.9-beta"));
    }

    #[test]
    fn an_explicit_version_list_is_respected() {
        let json = r#"{"id":"MAL-1","summary":"s","affected":[{
            "package":{"name":"thing","ecosystem":"PyPI"},
            "versions":["1.0.0","1.0.1"]
        }]}"#;
        let ind = parse_record(json);
        assert_eq!(ind[0].versions, Versions::Exact(vec!["1.0.0".into(), "1.0.1".into()]));
        assert!(ind[0].versions.covers("1.0.1"));
        assert!(
            !ind[0].versions.covers("2.0.0"),
            "a version the feed does not condemn must not be reported"
        );
    }

    // ── the false positive this classifier exists to prevent ──

    /// The real advisory, trimmed. Ingesting the whole crates.io export
    /// put this in the index and Hound reported tokio 1.38.0 as malware.
    const REAL_TOKIO_ADVISORY: &str = r#"{
      "id": "GHSA-2grh-hm3w-w7hv",
      "summary": "Race condition in tokio",
      "affected": [{
        "package": {"name": "tokio", "ecosystem": "crates.io"},
        "ranges": [{"type": "SEMVER", "events": [{"introduced": "1.8.0"}, {"fixed": "1.8.1"}]}]
      }]
    }"#;

    #[test]
    fn a_vulnerability_advisory_is_not_a_malicious_package() {
        assert!(
            parse_record(REAL_TOKIO_ADVISORY).is_empty(),
            "tokio has advisories like every large library; flagging it as malware \
             would end the product"
        );
        assert!(!is_malicious_record(
            &serde_json::from_str(REAL_TOKIO_ADVISORY).unwrap()
        ));
    }

    #[test]
    fn rustsec_advisories_are_not_malicious_packages_either() {
        let json = r#"{"id":"RUSTSEC-2021-0079","summary":"Integer overflow in hyper",
            "affected":[{"package":{"name":"hyper","ecosystem":"crates.io"},
            "ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"0.14.10"}]}]}]}"#;
        assert!(parse_record(json).is_empty());
    }

    #[test]
    fn a_mal_identifier_is_enough_to_classify() {
        assert!(is_malicious_record(&serde_json::from_str(REAL_MAL).unwrap()));
    }

    #[test]
    fn a_ghsa_malware_advisory_is_recognised_by_its_wording() {
        // GHSA mints its own ids for malware, so the MAL- prefix is not
        // always there. The malicious-packages feed imports these.
        let json = r#"{"id":"GHSA-mhg2-cg9m-q926","summary":"Malicious code in test-poc2 (npm)",
            "affected":[{"package":{"name":"test-poc2","ecosystem":"npm"}}]}"#;
        assert_eq!(parse_record(json).len(), 1);
    }

    #[test]
    fn the_origins_marker_is_recognised() {
        let json = r#"{"id":"GHSA-xyz","summary":"Something",
            "database_specific":{"malicious-packages-origins":[{"source":"ghsa-malware"}]},
            "affected":[{"package":{"name":"evil","ecosystem":"npm"}}]}"#;
        assert_eq!(parse_record(json).len(), 1);
    }

    #[test]
    fn an_unclassifiable_record_is_dropped_rather_than_guessed_at() {
        let json = r#"{"id":"SOMETHING-1","summary":"who knows",
            "affected":[{"package":{"name":"lib","ecosystem":"npm"}}]}"#;
        assert!(
            parse_record(json).is_empty(),
            "an indicator we cannot justify must not be created"
        );
    }

    #[test]
    fn one_record_can_condemn_several_packages() {
        let json = r#"{"id":"MAL-2","summary":"s","affected":[
            {"package":{"name":"a","ecosystem":"npm"}},
            {"package":{"name":"b","ecosystem":"npm"}}
        ]}"#;
        assert_eq!(parse_record(json).len(), 2);
    }

    // ── normalisation ──

    #[test]
    fn ecosystems_are_normalised() {
        assert_eq!(normalise_ecosystem("PyPI"), "pypi");
        assert_eq!(normalise_ecosystem("crates.io"), "cratesio");
        assert_eq!(normalise_ecosystem("Go"), "go");
        assert_eq!(normalise_ecosystem("RubyGems"), "rubygems");
        assert_eq!(normalise_ecosystem("Packagist"), "packagist");
    }

    #[test]
    fn distro_ecosystems_lose_their_release_suffix() {
        assert_eq!(normalise_ecosystem("Ubuntu:22.04"), "ubuntu");
        assert_eq!(normalise_ecosystem("Debian:12"), "debian");
    }

    #[test]
    fn keys_are_case_insensitive_on_both_halves() {
        assert_eq!(key_for("NPM", "React"), key_for("npm", "react"));
        assert_eq!(key_for("PyPI", "Requests"), "pypi:requests");
    }

    #[test]
    fn scoped_npm_names_survive_key_building() {
        assert_eq!(key_for("npm", "@vue/cli-plugin-babel"), "npm:@vue/cli-plugin-babel");
    }

    // ── robustness ──

    #[test]
    fn a_record_with_no_affected_packages_yields_nothing() {
        assert!(parse_record(r#"{"id":"MAL-3","summary":"s"}"#).is_empty());
        assert!(parse_record(r#"{"id":"MAL-3","affected":[]}"#).is_empty());
    }

    #[test]
    fn a_record_with_no_id_is_rejected() {
        let json = r#"{"summary":"s","affected":[{"package":{"name":"x","ecosystem":"npm"}}]}"#;
        assert!(
            parse_record(json).is_empty(),
            "an indicator that cannot cite a source is not usable"
        );
    }

    #[test]
    fn malformed_json_does_not_panic() {
        assert!(parse_record("{not json").is_empty());
        assert!(parse_record("").is_empty());
        assert!(parse_many("[").is_empty());
    }

    #[test]
    fn parses_an_array_of_records() {
        let json = format!("[{REAL_MAL},{REAL_MAL}]");
        assert_eq!(parse_many(&json).len(), 2);
    }

    #[test]
    fn a_missing_summary_still_produces_something_readable() {
        let json = r#"{"id":"MAL-4","affected":[{"package":{"name":"x","ecosystem":"npm"}}]}"#;
        let ind = parse_record(json);
        assert!(!ind[0].summary.is_empty());
    }
}