neiam /gitgud

defmodule GitGud.Scripts.OsvAuditTest do
  @moduledoc """
  `scripts/osv-audit`, run for real against a stand-in osv-scanner: a
  shell script that prints the report this test hands it, exits as told,
  and writes down the arguments it got. No network.

  What it covers is the sorting — a fixed version listed for the
  package means "fix now", none means "blocked" — and what each outcome
  exits with and says in `$GITHUB_OUTPUT`, which a job's
  `gitgud-conclusion` output reads.
  """
  use ExUnit.Case, async: true

  @script Path.expand("../../scripts/osv-audit", __DIR__)

  setup do
    if System.find_executable("python3") && System.find_executable("sh") do
      dir = Path.join(System.tmp_dir!(), "osv-audit-#{System.unique_integer([:positive])}")
      File.mkdir_p!(dir)
      on_exit(fn -> File.rm_rf!(dir) end)
      {:ok, dir: dir}
    else
      {:skip, "python3 or sh not on PATH"}
    end
  end

  defp vuln(id, name, events, ecosystem \\ "crates.io") do
    %{
      "id" => id,
      "affected" => [
        %{
          "package" => %{"name" => name, "ecosystem" => ecosystem},
          "ranges" => [%{"type" => "SEMVER", "events" => events}]
        }
      ]
    }
  end

  defp package(name, version, vulns, ecosystem \\ "crates.io") do
    %{
      "package" => %{"name" => name, "version" => version, "ecosystem" => ecosystem},
      "vulnerabilities" => vulns
    }
  end

  defp report(packages) do
    %{"results" => [%{"source" => %{"path" => "Cargo.lock"}, "packages" => packages}]}
  end

  # Run the script in `dir` with a stand-in scanner that prints `report`
  # and exits `code`. Returns {output, exit status, GITHUB_OUTPUT, args}.
  defp audit(dir, report, code, args \\ ["-r", "."]) do
    File.write!(Path.join(dir, "report.json"), Jason.encode!(report))
    scanner = Path.join(dir, "osv-scanner")

    File.write!(scanner, """
    #!/bin/sh
    printf '%s\\n' "$@" > "#{dir}/args"
    cat "#{dir}/report.json"
    exit #{code}
    """)

    File.chmod!(scanner, 0o755)
    github_output = Path.join(dir, "github_output")
    File.rm(github_output)

    {out, status} =
      System.cmd("python3", [@script | args],
        cd: dir,
        env: [{"OSV_SCANNER", scanner}, {"GITHUB_OUTPUT", github_output}],
        stderr_to_stdout: true
      )

    args_seen =
      case File.read(Path.join(dir, "args")) do
        {:ok, text} -> String.split(text, "\n", trim: true)
        _ -> nil
      end

    {out, status, File.read(github_output), args_seen}
  end

  test "a fixable finding fails, and both kinds are listed", %{dir: dir} do
    report =
      report([
        package("hashbrown", "0.15.0", [
          vuln("RUSTSEC-1", "hashbrown", [%{"introduced" => "0"}, %{"fixed" => "0.15.1"}])
        ]),
        package("block-buffer", "0.10.4", [
          vuln("GHSA-2", "block-buffer", [%{"introduced" => "0"}])
        ])
      ])

    assert {out, 1, {:ok, "conclusion=failure\n"}, args} = audit(dir, report, 1)
    assert args == ~w(scan source --format json -r .)

    [fix_now, blocked] = String.split(out, "No fix available / blocked")
    assert fix_now =~ "Fix now: 1"
    assert fix_now =~ ~r/hashbrown\s+0\.15\.0\s+RUSTSEC-1\s+0\.15\.1/
    assert blocked =~ ~r/block-buffer\s+0\.10\.4\s+GHSA-2\s+no fixed version/
  end

  test "only unfixable findings: a warning, exit 0", %{dir: dir} do
    report =
      report([
        package("block-buffer", "0.10.4", [
          vuln("GHSA-2", "block-buffer", [%{"introduced" => "0"}])
        ])
      ])

    assert {out, 0, {:ok, "conclusion=warning\n"}, _} = audit(dir, report, 1)
    assert out =~ "Fix now: 0"
    assert out =~ "No fix available / blocked: 1"
  end

  test "nothing found: success", %{dir: dir} do
    assert {out, 0, {:ok, "conclusion=success\n"}, _} = audit(dir, %{"results" => []}, 0)
    assert out =~ "No known vulnerabilities"
  end

  test "a fix is the one for the range the version is in", %{dir: dir} do
    events = [
      %{"introduced" => "0"},
      %{"fixed" => "1.2.5"},
      %{"introduced" => "2.0.0"},
      %{"fixed" => "2.0.3"}
    ]

    report = report([package("ring", "2.0.1", [vuln("RUSTSEC-3", "ring", events)])])

    assert {out, 1, _, _} = audit(dir, report, 1)
    assert out =~ ~r/ring\s+2\.0\.1\s+RUSTSEC-3\s+2\.0\.3/
  end

  test "an advisory naming the crate with `_` still counts; another ecosystem's doesn't",
       %{dir: dir} do
    report =
      report([
        package("block-buffer", "0.10.4", [
          vuln("GHSA-4", "block_buffer", [%{"introduced" => "0"}, %{"fixed" => "0.12.1"}])
        ]),
        package("serde", "1.0.0", [
          vuln("GHSA-5", "serde", [%{"introduced" => "0"}, %{"fixed" => "1.0.1"}], "npm")
        ])
      ])

    assert {out, 1, _, _} = audit(dir, report, 1)
    assert out =~ ~r/block-buffer\s+0\.10\.4\s+GHSA-4\s+0\.12\.1/
    assert out =~ ~r/serde\s+1\.0\.0\s+GHSA-5\s+no fixed version/
  end

  test "osv-scanner's config is passed when it's here and not given", %{dir: dir} do
    File.write!(Path.join(dir, "osv-scanner.toml"), "")

    assert {_, 0, _, args} = audit(dir, %{"results" => []}, 0, ["--lockfile", "Cargo.lock"])
    assert args == ~w(scan source --format json --config osv-scanner.toml --lockfile Cargo.lock)

    assert {_, 0, _, args} = audit(dir, %{"results" => []}, 0, ["--config", "x.toml", "-r", "."])
    assert args == ~w(scan source --format json --config x.toml -r .)
  end

  test "a scanner error is passed on, and writes no conclusion", %{dir: dir} do
    assert {_, 127, {:error, :enoent}, _} = audit(dir, %{}, 127)
  end

  test "no lockfiles is an error unless allowed", %{dir: dir} do
    assert {_, 128, {:error, :enoent}, _} = audit(dir, %{}, 128)

    assert {_, 0, {:ok, "conclusion=success\n"}, _} =
             audit(dir, %{}, 128, ["-r", ".", "--allow-no-lockfiles"])
  end

  test "--format is refused: the script reads the JSON itself", %{dir: dir} do
    assert {out, 2, _, nil} = audit(dir, %{}, 0, ["--format", "table"])
    assert out =~ "--format"
  end
end