GITGUD

neiam neiam /gitgud

Git Gud

public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages

⭐ 1

Log in to mark this repository.

Package retention, and where the registry disk went

aa9e861 · Gabriel Morell · 2026-09-12 14:22

42 files +2543 -4
Message 
{commit_body(@commit)}

Files changed

modified config/config.exs
+3 −0
@@ -60,6 +60,9 @@ config :git_gud, Oban,
60 60 # Re-sync repositories that opted into a scheduled upstream
61 61 # fetch. Anonymous — a private upstream is synced by hand.
62 62 {"@hourly", GitGud.Repositories.Workers.UpstreamSync},
63 + # Retention pass over the package registries. Reports rather
64 + # than deletes until `packages.retention_enforced` is on.
65 + {"@daily", GitGud.Packages.Workers.Prune},
63 66 # Prune `actions/cache` entries by age + per-repo size cap.
64 67 {"@hourly", GitGud.Workflows.Workers.CacheEviction},
65 68 # Requeue workflow_jobs stuck in `running` after a claim-then-
modified lib/git_gud/packages.ex
+6 −1
@@ -102,7 +102,12 @@ defmodule GitGud.Packages do
102 102 def list_versions(%Package{id: pid}) do
103 103 PackageVersion
104 104 |> where([v], v.package_id == ^pid)
105 − |> order_by([v], desc: v.pushed_at)
105 + # `id` breaks the tie: `pushed_at` has second granularity, so two
106 + # versions published in the same second otherwise come back in
107 + # whatever order the plan happened to produce — which changes when
108 + # an index is added, and leaves order-sensitive formats (cargo's
109 + # index is one) flapping between requests.
110 + |> order_by([v], desc: v.pushed_at, desc: v.id)
106 111 |> Repo.all()
107 112 end
108 113
added lib/git_gud/packages/disk_usage.ex
+157 −0
@@ -0,0 +1,157 @@
1 +defmodule GitGud.Packages.DiskUsage do
2 + @moduledoc """
3 + Where the registry's disk went.
4 +
5 + Every number here is one aggregate query rather than a walk: an
6 + instance with a few hundred thousand files should be able to render
7 + the admin page without thinking about it.
8 +
9 + `SUM` over a bigint column is `numeric` in Postgres, which Ecto loads
10 + as a `Decimal` — so every sum here is cast back to an integer in the
11 + query rather than leaving each caller to discover that the hard way.
12 +
13 + Sizes come from `package_files.size` where a format has files, and
14 + from `package_versions.size` where it doesn't — OCI's bytes live in
15 + zot, but they are still bytes this instance is answerable for, so
16 + they are counted and labelled rather than left out of the total.
17 + """
18 +
19 + import Ecto.Query, warn: false
20 +
21 + alias GitGud.Organizations.Organization
22 + alias GitGud.Packages.Package
23 + alias GitGud.Packages.PackageFile
24 + alias GitGud.Packages.PackageVersion
25 + alias GitGud.Repo
26 + alias GitGud.Repositories.Repository
27 +
28 + @typedoc "Bytes and counts for one grouping."
29 + @type row :: %{
30 + key: String.t() | integer(),
31 + label: String.t(),
32 + bytes: non_neg_integer(),
33 + versions: non_neg_integer(),
34 + packages: non_neg_integer()
35 + }
36 +
37 + @doc """
38 + Instance-wide totals, one row per format, largest first.
39 +
40 + The shape every view starts from: what is on this disk, and which
41 + ecosystem put it there.
42 + """
43 + @spec by_format(keyword()) :: [row()]
44 + def by_format(opts \\ []) do
45 + base()
46 + |> scope(opts)
47 + |> group_by([p], p.format)
48 + |> select([p, v, f], %{
49 + key: p.format,
50 + bytes: type(coalesce(sum(coalesce(f.size, v.size)), 0), :integer),
51 + versions: count(v.id, :distinct),
52 + packages: count(p.id, :distinct)
53 + })
54 + |> Repo.all()
55 + |> Enum.map(&Map.put(&1, :label, &1.key))
56 + |> Enum.sort_by(& &1.bytes, :desc)
57 + end
58 +
59 + @doc "One row per repository, largest first. Scoped like `by_format/1`."
60 + @spec by_repository(keyword()) :: [row()]
61 + def by_repository(opts \\ []) do
62 + base()
63 + |> scope(opts)
64 + |> join(:inner, [p], r in Repository, on: r.id == p.repository_id)
65 + |> group_by([p, v, f, r], [r.id, r.name])
66 + |> select([p, v, f, r], %{
67 + key: r.id,
68 + label: r.name,
69 + bytes: type(coalesce(sum(coalesce(f.size, v.size)), 0), :integer),
70 + versions: count(v.id, :distinct),
71 + packages: count(p.id, :distinct)
72 + })
73 + |> Repo.all()
74 + |> Enum.sort_by(& &1.bytes, :desc)
75 + end
76 +
77 + @doc """
78 + The biggest packages, largest first — what to look at when the
79 + number at the top of the page is a surprise.
80 + """
81 + @spec top_packages(keyword()) :: [map()]
82 + def top_packages(opts \\ []) do
83 + limit = Keyword.get(opts, :limit, 20)
84 +
85 + base()
86 + |> scope(opts)
87 + |> join(:inner, [p], r in Repository, on: r.id == p.repository_id)
88 + |> group_by([p, v, f, r], [p.id, p.name, p.format, r.id, r.name])
89 + |> select([p, v, f, r], %{
90 + package_id: p.id,
91 + name: p.name,
92 + format: p.format,
93 + repository_id: r.id,
94 + repository: r.name,
95 + bytes: type(coalesce(sum(coalesce(f.size, v.size)), 0), :integer),
96 + versions: count(v.id, :distinct)
97 + })
98 + |> order_by([p, v, f], desc: coalesce(sum(coalesce(f.size, v.size)), 0))
99 + |> limit(^limit)
100 + |> Repo.all()
101 + end
102 +
103 + @doc "One number: everything in scope."
104 + @spec total(keyword()) :: %{
105 + bytes: non_neg_integer(),
106 + packages: non_neg_integer(),
107 + versions: non_neg_integer()
108 + }
109 + def total(opts \\ []) do
110 + base()
111 + |> scope(opts)
112 + |> select([p, v, f], %{
113 + bytes: type(coalesce(sum(coalesce(f.size, v.size)), 0), :integer),
114 + versions: count(v.id, :distinct),
115 + packages: count(p.id, :distinct)
116 + })
117 + |> Repo.one()
118 + end
119 +
120 + # Files left-joined so a version with no file rows (OCI) still counts
121 + # once, at the version's own size.
122 + defp base do
123 + from(p in Package,
124 + left_join: v in PackageVersion,
125 + on: v.package_id == p.id,
126 + left_join: f in PackageFile,
127 + on: f.package_version_id == v.id
128 + )
129 + end
130 +
131 + defp scope(query, opts) do
132 + Enum.reduce(opts, query, fn
133 + {:repository, %Repository{id: id}}, q -> where(q, [p], p.repository_id == ^id)
134 + {:repository_id, id}, q -> where(q, [p], p.repository_id == ^id)
135 + {:organization, %Organization{id: id}}, q -> scope_org(q, id)
136 + {:organization_id, id}, q -> scope_org(q, id)
137 + {:format, format}, q -> where(q, [p], p.format == ^format)
138 + _, q -> q
139 + end)
140 + end
141 +
142 + defp scope_org(query, org_id) do
143 + repos = from(r in Repository, where: r.organization_id == ^org_id, select: r.id)
144 + where(query, [p], p.repository_id in subquery(repos))
145 + end
146 +
147 + @doc """
148 + Human bytes. Binary units, because that is what a disk is measured
149 + in and what every other size in this app uses.
150 + """
151 + @spec humanize(integer() | nil) :: String.t()
152 + def humanize(nil), do: "0 B"
153 + def humanize(bytes) when bytes < 1024, do: "#{bytes} B"
154 + def humanize(bytes) when bytes < 1_048_576, do: "#{Float.round(bytes / 1024, 1)} KiB"
155 + def humanize(bytes) when bytes < 1_073_741_824, do: "#{Float.round(bytes / 1_048_576, 1)} MiB"
156 + def humanize(bytes), do: "#{Float.round(bytes / 1_073_741_824, 2)} GiB"
157 +end
modified lib/git_gud/packages/formats/cargo.ex
+4 −1
@@ -178,7 +178,10 @@ defmodule GitGud.Packages.Formats.Cargo do
178 178 defp index_body(%Package{} = package) do
179 179 package
180 180 |> Packages.list_versions()
181 − |> Enum.sort_by(& &1.pushed_at, DateTime)
181 + # Oldest first, and stable within a second — cargo caches these
182 + # lines, so the same set in a different order is a cache that
183 + # thrashes for no reason.
184 + |> Enum.sort_by(&{DateTime.to_unix(&1.pushed_at), &1.id})
182 185 |> Enum.map(&[Jason.encode_to_iodata!(index_entry(package, &1)), "\n"])
183 186 end
184 187
modified lib/git_gud/packages/package.ex
+21 −0
@@ -20,6 +20,12 @@ defmodule GitGud.Packages.Package do
20 20 field :name, :string
21 21 field :normalized_name, :string
22 22 field :latest_version, :string
23 + # nil on any of these means "inherit the instance policy" — which
24 + # is what lets an admin move the ceiling without rewriting rows.
25 + # See `GitGud.Packages.Retention`.
26 + field :retention_mode, :string
27 + field :retention_keep, :integer
28 + field :retention_bytes, :integer
23 29
24 30 belongs_to :repository, Repository
25 31 has_many :versions, PackageVersion
@@ -37,6 +43,21 @@ defmodule GitGud.Packages.Package do
37 43 |> unique_constraint([:repository_id, :format, :normalized_name])
38 44 end
39 45
46 + @doc """
47 + Changeset for a package's own retention policy.
48 +
49 + Bounds aren't checked here — "stricter than the instance" is a
50 + comparison against a value this module can't see. `Retention.set_policy/2`
51 + owns that, and this only casts.
52 + """
53 + def retention_changeset(package, attrs) do
54 + package
55 + |> cast(attrs, [:retention_mode, :retention_keep, :retention_bytes])
56 + |> validate_inclusion(:retention_mode, ~w(keep_last quota))
57 + |> validate_number(:retention_keep, greater_than: 0)
58 + |> validate_number(:retention_bytes, greater_than_or_equal_to: 0)
59 + end
60 +
40 61 # The normalized name is derived, not supplied: letting a caller set
41 62 # it independently of `name` is how you get two rows that a client
42 63 # sees as one package.
modified lib/git_gud/packages/package_version.ex
+5 −0

Click to load diff…

added lib/git_gud/packages/pruner.ex
+221 −0

Click to load diff…

added lib/git_gud/packages/retention.ex
+350 −0

Click to load diff…

added lib/git_gud/packages/workers/prune.ex
+51 −0

Click to load diff…

added lib/git_gud/packages/zot.ex
+72 −0

Click to load diff…

added lib/git_gud/settings.ex
+167 −0

Click to load diff…

added lib/git_gud/settings/instance_setting.ex
+27 −0

Click to load diff…

added lib/git_gud_web/live/admin_packages_live/index.ex
+321 −0

Click to load diff…

added lib/git_gud_web/live/org_live/packages.ex
+186 −0

Click to load diff…

modified lib/git_gud_web/live/repo_live/packages.ex
+199 −2

Click to load diff…

modified lib/git_gud_web/router.ex
+2 −0

Click to load diff…

added priv/repo/migrations/20260912100000_create_instance_settings.exs
+26 −0

Click to load diff…

added priv/repo/migrations/20260912100100_add_package_retention.exs
+35 −0

Click to load diff…

added test/git_gud/packages/pruner_test.exs
+163 −0

Click to load diff…

added test/git_gud/packages/retention_test.exs
+239 −0

Click to load diff…

added test/git_gud_web/live/admin_packages_test.exs
+116 −0

Click to load diff…

added test/git_gud_web/live/package_retention_ui_test.exs
+172 −0

Click to load diff…

Parents: 1ecec4a