3.9 KiB · text History b868907
defmodule GitGud.Workflows.QueuedJobTimeoutTest do
  @moduledoc """
  A job no runner takes — a `runs-on` nobody has a runner for — fails
  once it has been queued past the timeout, and its run and dependents
  settle as for any failure.
  """
  use GitGud.DataCase, async: false

  alias GitGud.Workflows
  alias GitGud.Workflows.Workers.QueuedJobTimeout
  alias GitGud.Workflows.Workers.StaleJobReaper
  alias GitGud.Workflows.WorkflowJob
  alias GitGud.Workflows.WorkflowRun

  import GitGud.ForgeFixtures

  setup do
    {_user, repo} = repository_fixture()

    run =
      %WorkflowRun{repository_id: repo.id}
      |> WorkflowRun.create_changeset(%{
        number: 1,
        workflow_path: ".github/workflows/ci.yml",
        trigger: "push",
        head_sha: :crypto.strong_rand_bytes(20)
      })
      |> Ecto.Changeset.change(status: "queued")
      |> Repo.insert!()

    {:ok, run: run}
  end

  defp hours_ago(h), do: DateTime.add(DateTime.utc_now(:second), -h * 3600, :second)

  defp job(run, status, updated_at, attrs \\ %{}) do
    %WorkflowJob{workflow_run_id: run.id}
    |> WorkflowJob.create_changeset(
      Map.merge(
        %{
          job_id: "build-#{System.unique_integer([:positive])}",
          runs_on: %{"labels" => ["riscv64"]},
          definition: %{}
        },
        attrs
      )
    )
    |> Ecto.Changeset.change(status: status, inserted_at: updated_at, updated_at: updated_at)
    |> Repo.insert!()
  end

  defp time_out!, do: :ok = QueuedJobTimeout.perform(%Oban.Job{})

  test "a job queued past the timeout fails, and says why; so does its run", %{run: run} do
    stuck = job(run, "queued", hours_ago(25))

    time_out!()

    failed = Repo.get!(WorkflowJob, stuck.id)
    assert failed.status == "failure"
    assert failed.completed_at
    assert failed.status_reason =~ "riscv64"
    assert Repo.get!(WorkflowRun, run.id).status == "failure"
  end

  test "a job queued within the timeout stays queued", %{run: run} do
    fresh = job(run, "queued", hours_ago(23))

    time_out!()

    assert Repo.get!(WorkflowJob, fresh.id).status == "queued"
  end

  test "blocked, waiting, awaiting_approval and running jobs aren't timed", %{run: run} do
    others =
      for status <- ~w(blocked waiting awaiting_approval running),
          do: job(run, status, hours_ago(100))

    time_out!()

    for j <- others, do: assert(Repo.get!(WorkflowJob, j.id).status == j.status)
  end

  test "a job that needs the timed-out one is skipped", %{run: run} do
    stuck = job(run, "queued", hours_ago(25), %{job_id: "build"})
    after_it = job(run, "blocked", hours_ago(25), %{job_id: "publish", needs: ["build"]})

    time_out!()

    assert Repo.get!(WorkflowJob, stuck.id).status == "failure"
    assert Repo.get!(WorkflowJob, after_it.id).status == "skipped"
  end

  test "a job a runner claimed meanwhile isn't failed", %{run: run} do
    stale_copy = job(run, "queued", hours_ago(25))

    Repo.get!(WorkflowJob, stale_copy.id)
    |> Ecto.Changeset.change(status: "running")
    |> Repo.update!()

    assert {:error, _} = Workflows.update_job_status(stale_copy, "failure", from: ["queued"])
    assert Repo.get!(WorkflowJob, stale_copy.id).status == "running"
  end

  test "nil turns the timeout off", %{run: run} do
    stuck = job(run, "queued", hours_ago(24 * 30))
    previous = Application.get_env(:git_gud, QueuedJobTimeout)
    Application.put_env(:git_gud, QueuedJobTimeout, queued_timeout_hours: nil)
    on_exit(fn -> Application.put_env(:git_gud, QueuedJobTimeout, previous || []) end)

    time_out!()

    assert Repo.get!(WorkflowJob, stuck.id).status == "queued"
  end

  test "a job the stale-job reaper queues again starts a fresh clock", %{run: run} do
    reaped =
      job(run, "running", hours_ago(25))
      |> Ecto.Changeset.change(started_at: hours_ago(25), heartbeat_at: hours_ago(25))
      |> Repo.update!()

    :ok = StaleJobReaper.perform(%Oban.Job{})
    time_out!()

    assert Repo.get!(WorkflowJob, reaped.id).status == "queued"
  end
end