defmodule GitGud.Workflows.StaleJobReaperTest do
@moduledoc """
The reaper takes back jobs whose runner has gone quiet — by the last
report (`heartbeat_at`), not by when the job started, so a job longer
than the window keeps its runner, its reports and its job token.
"""
use GitGud.DataCase, async: false
alias GitGud.Workflows
alias GitGud.Workflows.RunnerJwt
alias GitGud.Workflows.Workers.StaleJobReaper
alias GitGud.Workflows.WorkflowJob
alias GitGud.Workflows.WorkflowRun
import GitGud.ForgeFixtures
setup do
{_user, repo} = repository_fixture()
run =
%WorkflowRun{repository_id: repo.id}
|> WorkflowRun.create_changeset(%{
number: 1,
workflow_path: ".github/workflows/ci.yml",
trigger: "push",
head_sha: :crypto.strong_rand_bytes(20)
})
|> Repo.insert!()
{:ok, run: run}
end
defp ago(seconds), do: DateTime.add(DateTime.utc_now(:second), -seconds, :second)
defp running_job(run, started_at, heartbeat_at) do
%WorkflowJob{workflow_run_id: run.id}
|> WorkflowJob.create_changeset(%{
job_id: "build-#{System.unique_integer([:positive])}",
definition: %{}
})
|> Ecto.Changeset.change(
status: "running",
started_at: started_at,
heartbeat_at: heartbeat_at
)
|> Repo.insert!()
end
defp reap!, do: :ok = StaleJobReaper.perform(%Oban.Job{})
test "a long job whose runner still reports keeps running, and its token works", %{run: run} do
job = running_job(run, ago(600), ago(10))
token = RunnerJwt.mint(job, run)
reap!()
assert %WorkflowJob{status: "running"} = Repo.get!(WorkflowJob, job.id)
assert {:ok, _claims} = RunnerJwt.verify_active(token)
end
test "a job whose runner went quiet is queued again", %{run: run} do
job = running_job(run, ago(900), ago(600))
reap!()
assert %WorkflowJob{status: "queued", runner_id: nil, started_at: nil, heartbeat_at: nil} =
Repo.get!(WorkflowJob, job.id)
end
test "a job claimed before heartbeats goes by when it started", %{run: run} do
old = running_job(run, ago(600), nil)
fresh = running_job(run, ago(30), nil)
reap!()
assert Repo.get!(WorkflowJob, old.id).status == "queued"
assert Repo.get!(WorkflowJob, fresh.id).status == "running"
end
describe "heartbeat/1" do
test "moves a stale heartbeat to now", %{run: run} do
job = running_job(run, ago(600), ago(120))
:ok = Workflows.heartbeat(job)
assert DateTime.diff(DateTime.utc_now(), Repo.get!(WorkflowJob, job.id).heartbeat_at) < 5
end
test "writes at most every 30 seconds", %{run: run} do
recent = ago(10)
job = running_job(run, ago(600), recent)
:ok = Workflows.heartbeat(job)
assert Repo.get!(WorkflowJob, job.id).heartbeat_at == recent
end
test "leaves a job that isn't running alone", %{run: run} do
job = running_job(run, ago(600), nil)
job |> Ecto.Changeset.change(status: "failure") |> Repo.update!()
:ok = Workflows.heartbeat(job)
assert Repo.get!(WorkflowJob, job.id).heartbeat_at == nil
end
end
end
neiam
/gitgud
Git Gud
public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages
⭐
1
Log in to mark this repository.
3.1 KiB · text
History
b868907