3.1 KiB · text History b868907
defmodule GitGud.Workflows.StaleJobReaperTest do
  @moduledoc """
  The reaper takes back jobs whose runner has gone quiet — by the last
  report (`heartbeat_at`), not by when the job started, so a job longer
  than the window keeps its runner, its reports and its job token.
  """
  use GitGud.DataCase, async: false

  alias GitGud.Workflows
  alias GitGud.Workflows.RunnerJwt
  alias GitGud.Workflows.Workers.StaleJobReaper
  alias GitGud.Workflows.WorkflowJob
  alias GitGud.Workflows.WorkflowRun

  import GitGud.ForgeFixtures

  setup do
    {_user, repo} = repository_fixture()

    run =
      %WorkflowRun{repository_id: repo.id}
      |> WorkflowRun.create_changeset(%{
        number: 1,
        workflow_path: ".github/workflows/ci.yml",
        trigger: "push",
        head_sha: :crypto.strong_rand_bytes(20)
      })
      |> Repo.insert!()

    {:ok, run: run}
  end

  defp ago(seconds), do: DateTime.add(DateTime.utc_now(:second), -seconds, :second)

  defp running_job(run, started_at, heartbeat_at) do
    %WorkflowJob{workflow_run_id: run.id}
    |> WorkflowJob.create_changeset(%{
      job_id: "build-#{System.unique_integer([:positive])}",
      definition: %{}
    })
    |> Ecto.Changeset.change(
      status: "running",
      started_at: started_at,
      heartbeat_at: heartbeat_at
    )
    |> Repo.insert!()
  end

  defp reap!, do: :ok = StaleJobReaper.perform(%Oban.Job{})

  test "a long job whose runner still reports keeps running, and its token works", %{run: run} do
    job = running_job(run, ago(600), ago(10))
    token = RunnerJwt.mint(job, run)

    reap!()

    assert %WorkflowJob{status: "running"} = Repo.get!(WorkflowJob, job.id)
    assert {:ok, _claims} = RunnerJwt.verify_active(token)
  end

  test "a job whose runner went quiet is queued again", %{run: run} do
    job = running_job(run, ago(900), ago(600))

    reap!()

    assert %WorkflowJob{status: "queued", runner_id: nil, started_at: nil, heartbeat_at: nil} =
             Repo.get!(WorkflowJob, job.id)
  end

  test "a job claimed before heartbeats goes by when it started", %{run: run} do
    old = running_job(run, ago(600), nil)
    fresh = running_job(run, ago(30), nil)

    reap!()

    assert Repo.get!(WorkflowJob, old.id).status == "queued"
    assert Repo.get!(WorkflowJob, fresh.id).status == "running"
  end

  describe "heartbeat/1" do
    test "moves a stale heartbeat to now", %{run: run} do
      job = running_job(run, ago(600), ago(120))

      :ok = Workflows.heartbeat(job)

      assert DateTime.diff(DateTime.utc_now(), Repo.get!(WorkflowJob, job.id).heartbeat_at) < 5
    end

    test "writes at most every 30 seconds", %{run: run} do
      recent = ago(10)
      job = running_job(run, ago(600), recent)

      :ok = Workflows.heartbeat(job)

      assert Repo.get!(WorkflowJob, job.id).heartbeat_at == recent
    end

    test "leaves a job that isn't running alone", %{run: run} do
      job = running_job(run, ago(600), nil)
      job |> Ecto.Changeset.change(status: "failure") |> Repo.update!()

      :ok = Workflows.heartbeat(job)

      assert Repo.get!(WorkflowJob, job.id).heartbeat_at == nil
    end
  end
end