iterating
99d66f7 · gmorell · 2026-05-22 15:37
Files changed
modified
config/config.exs
+10
−1
@@ -47,10 +47,19 @@ config :git_gud, Oban,
| 47 | 47 | # keys don't drift even when nobody's touching the actor. |
| 48 | 48 | {"@hourly", GitGud.Federation.Workers.RefreshRemoteActors}, |
| 49 | 49 | # Prune `actions/cache` entries by age + per-repo size cap. |
| 50 | − {"@hourly", GitGud.Workflows.Workers.CacheEviction} | |
| 50 | + {"@hourly", GitGud.Workflows.Workers.CacheEviction}, | |
| 51 | + # Requeue workflow_jobs stuck in `running` after a claim-then- | |
| 52 | + # can't-send (network blip, encoder bug, runner crash). Threshold | |
| 53 | + # is configurable via :stale_after_minutes; default 5 minutes. | |
| 54 | + {"* * * * *", GitGud.Workflows.Workers.StaleJobReaper} | |
| 51 | 55 | ]} |
| 52 | 56 | ] |
| 53 | 57 | |
| 58 | +# Stale-job reaper threshold. Lower = faster recovery, higher = more | |
| 59 | +# tolerant of slow runners. Tune per deployment. | |
| 60 | +config :git_gud, GitGud.Workflows.Workers.StaleJobReaper, | |
| 61 | + stale_after_minutes: 5 | |
| 62 | + | |
| 54 | 63 | # Federation retention. nil = keep forever. Tune per deployment. |
| 55 | 64 | config :git_gud, GitGud.Federation.Workers.Retention, |
| 56 | 65 | inbox_deliveries_days: nil, |
added
lib/git_gud/workflows/workers/stale_job_reaper.ex
+83
−0
@@ -0,0 +1,83 @@
| 1 | +defmodule GitGud.Workflows.Workers.StaleJobReaper do | |
| 2 | + @moduledoc """ | |
| 3 | + Resets `workflow_jobs` rows that have been `running` past the | |
| 4 | + heartbeat threshold without an `UpdateTask` from the runner. | |
| 5 | + | |
| 6 | + Why this exists: `Workflows.claim_next_job/1` flips a job to | |
| 7 | + `running` inside the transaction *before* the controller has | |
| 8 | + successfully sent the Task payload back to the runner. If the | |
| 9 | + response fails (network blip, runner crash, protobuf encoding | |
| 10 | + bug — see the runner-controller respond path), the row is | |
| 11 | + stranded with `status: "running"` and no one is actually | |
| 12 | + working on it. Subsequent `FetchTask` polls won't re-claim it | |
| 13 | + (the SQL filter is `status = "queued"`), so the work blocks | |
| 14 | + indefinitely. | |
| 15 | + | |
| 16 | + Configure with: | |
| 17 | + | |
| 18 | + config :git_gud, GitGud.Workflows.Workers.StaleJobReaper, | |
| 19 | + stale_after_minutes: 5 | |
| 20 | + | |
| 21 | + Runs every minute via the Oban cron plugin (`config.exs`). | |
| 22 | + """ | |
| 23 | + | |
| 24 | + use Oban.Worker, queue: :ci, max_attempts: 3 | |
| 25 | + | |
| 26 | + import Ecto.Query | |
| 27 | + | |
| 28 | + alias GitGud.Repo | |
| 29 | + alias GitGud.Workflows.WorkflowJob | |
| 30 | + alias GitGud.Workflows.WorkflowRun | |
| 31 | + | |
| 32 | + @default_stale_after_minutes 5 | |
| 33 | + | |
| 34 | + @impl Oban.Worker | |
| 35 | + def perform(%Oban.Job{}) do | |
| 36 | + minutes = | |
| 37 | + Application.get_env(:git_gud, __MODULE__, []) | |
| 38 | + |> Keyword.get(:stale_after_minutes, @default_stale_after_minutes) | |
| 39 | + | |
| 40 | + cutoff = DateTime.utc_now() |> DateTime.add(-minutes * 60, :second) | |
| 41 | + | |
| 42 | + {count, ids} = | |
| 43 | + from(j in WorkflowJob, | |
| 44 | + where: | |
| 45 | + j.status == "running" and | |
| 46 | + (is_nil(j.started_at) or j.started_at < ^cutoff), | |
| 47 | + select: j.id | |
| 48 | + ) | |
| 49 | + |> Repo.update_all( | |
| 50 | + set: [ | |
| 51 | + status: "queued", | |
| 52 | + runner_id: nil, | |
| 53 | + started_at: nil | |
| 54 | + ] | |
| 55 | + ) | |
| 56 | + | |
| 57 | + if count > 0 do | |
| 58 | + require Logger | |
| 59 | + | |
| 60 | + Logger.warning( | |
| 61 | + "stale_job_reaper requeued #{count} job(s): #{inspect(ids)} " <> | |
| 62 | + "(no UpdateTask in >= #{minutes}m)" | |
| 63 | + ) | |
| 64 | + | |
| 65 | + # Bring affected parent runs back to `queued` so the UI doesn't | |
| 66 | + # show "running" for a run that has only re-queued children. | |
| 67 | + run_ids = | |
| 68 | + from(j in WorkflowJob, | |
| 69 | + where: j.id in ^ids, | |
| 70 | + select: j.workflow_run_id, | |
| 71 | + distinct: true | |
| 72 | + ) | |
| 73 | + |> Repo.all() | |
| 74 | + | |
| 75 | + from(r in WorkflowRun, | |
| 76 | + where: r.id in ^run_ids and r.status == "running" | |
| 77 | + ) | |
| 78 | + |> Repo.update_all(set: [status: "queued"]) | |
| 79 | + end | |
| 80 | + | |
| 81 | + :ok | |
| 82 | + end | |
| 83 | +end |
modified
lib/git_gud_web/controllers/runner_controller.ex
+8
−0
@@ -326,7 +326,15 @@ defmodule GitGudWeb.RunnerController do
| 326 | 326 | {:protobuf, echoed_ct} -> |
| 327 | 327 | body = msg.__struct__.encode(msg) |> IO.iodata_to_binary() |
| 328 | 328 | |
| 329 | + require Logger | |
| 330 | + | |
| 331 | + Logger.info( | |
| 332 | + "runner-respond struct=#{inspect(msg.__struct__)} body_bytes=#{byte_size(body)} " <> | |
| 333 | + "task_present=#{inspect(match?(%{task: t} when not is_nil(t), msg))}" | |
| 334 | + ) | |
| 335 | + | |
| 329 | 336 | conn |
| 337 | + |> put_resp_header("x-debug-body-size", Integer.to_string(byte_size(body))) | |
| 330 | 338 | |> put_resp_content_type(echoed_ct, nil) |
| 331 | 339 | |> send_resp(200, body) |
| 332 | 340 |
Parents: 8740f46