GITGUD

neiam neiam /gitgud

Git Gud

public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages

⭐ 1

Log in to mark this repository.

Make the F-Droid backfill callable from a release

a30d17e · Gabriel Morell · 2026-09-19 04:59

✓ build ✓ hello
14 files +240 -107
Message 
{commit_body(@commit)}

Files changed

modified README.md
+1 −0
@@ -602,6 +602,7 @@ kubectl -n git-gud exec deployment/git-gud -- \
602 602 | `keygen()` | Mint the RS256 keypair for the OCI token endpoint, printing both PEMs with `===PRIV===` / `===PUB===` delimiters so they can be piped into `kubectl create secret` without touching a laptop |
603 603 | `configure_repos()` | Backfill the per-repo git config `actions/checkout` needs (`uploadpack.allow*SHA1InWant`). New repos get it at creation; this fixes older ones |
604 604 | `backfill_language_tags()` | Derive each repo's language tag from the stats already computed for it. New stats tag their repo as they're written, so this is for repos whose stats predate the tag |
605 +| `backfill_fdroid_icons()` | Fill in the app name and launcher icon the F-Droid index shows, by re-reading published APKs. Both are read at upload, so apps published before the adapter could resolve `resources.arsc` have neither. Takes `all: true` for every version rather than each app's newest, and `dry_run: true` to report without writing |
605 606 | `sentry_test()` | Send a synchronous test event to verify the DSN |
606 607
607 608 All of them are idempotent and safe against a live instance: each
modified lib/git_gud/packages/formats/fdroid.ex
+121 −0
@@ -249,6 +249,127 @@ defmodule GitGud.Packages.Formats.Fdroid do
249 249 }
250 250 end
251 251
252 + @doc """
253 + Re-read published APKs and record the app name and launcher icon that
254 + the index shows.
255 +
256 + Both are read at upload and stored with the file, so an app published
257 + before the adapter could resolve `resources.arsc` carries neither and
258 + nothing re-reads it afterwards: the entry shows a package id and no
259 + icon until the app publishes again. For an app that isn't building
260 + today, that is never.
261 +
262 + Options:
263 +
264 + * `:all` — every version rather than the newest of each app. The
265 + index takes the name and icon from the highest versionCode, so
266 + the default reads one APK per app instead of pulling twenty out
267 + of storage to change nothing anyone can see.
268 + * `:dry_run` — report without writing.
269 + * `:on_file` — called with a report map per APK read.
270 +
271 + Adds a label and an icon and nothing else: everything else the file
272 + records was written when it was published, and a re-read is the wrong
273 + place to discover a disagreement about, say, a versionCode that the
274 + index orders by and the filename encodes.
275 +
276 + Returns `{read, written}`. Idempotent — an icon already in storage is
277 + stored under the same content-addressed key.
278 + """
279 + @spec backfill_icons(keyword()) :: {non_neg_integer(), non_neg_integer()}
280 + def backfill_icons(opts \\ []) do
281 + all? = Keyword.get(opts, :all, false)
282 + dry? = Keyword.get(opts, :dry_run, false)
283 + on_file = Keyword.get(opts, :on_file, fn _ -> :ok end)
284 +
285 + backfill_candidates(all?)
286 + |> Enum.reduce({0, 0}, fn file, {read, written} ->
287 + case backfill_file(file, dry?) do
288 + {:ok, report} ->
289 + on_file.(report)
290 + {read + 1, if(dry?, do: written, else: written + 1)}
291 +
292 + {:error, report} ->
293 + on_file.(report)
294 + {read + 1, written}
295 + end
296 + end)
297 + end
298 +
299 + # Every published APK, or the newest of each app. "Newest" is by
300 + # versionCode, which is what the index sorts on; a file whose metadata
301 + # predates this adapter has none and sorts last rather than breaking
302 + # the comparison.
303 + defp backfill_candidates(true), do: published_apk_files()
304 +
305 + defp backfill_candidates(false) do
306 + published_apk_files()
307 + |> Enum.group_by(& &1.package_version.package_id)
308 + |> Enum.map(fn {_package_id, files} -> Enum.max_by(files, &backfill_version_code/1) end)
309 + |> Enum.sort_by(& &1.id)
310 + end
311 +
312 + defp published_apk_files do
313 + import Ecto.Query
314 +
315 + from(f in PackageFile,
316 + join: v in assoc(f, :package_version),
317 + join: p in assoc(v, :package),
318 + where: p.format == ^@format,
319 + preload: [package_version: {v, package: p}]
320 + )
321 + |> GitGud.Repo.all()
322 + end
323 +
324 + defp backfill_version_code(%PackageFile{metadata: %{"version_code" => code}})
325 + when is_integer(code),
326 + do: code
327 +
328 + defp backfill_version_code(%PackageFile{}), do: -1
329 +
330 + # Only these come from the re-read.
331 + @backfilled ~w(label icon_key icon_sha256 icon_size icon_ext)
332 +
333 + defp backfill_file(%PackageFile{} = file, dry?) do
334 + with {:ok, body} <- Packages.read_file(file),
335 + {:ok, info} <- Apk.read(body) do
336 + icon =
337 + case Apk.icon(body) do
338 + {:ok, bytes, ext} -> {byte_size(bytes), ext}
339 + :none -> :none
340 + end
341 +
342 + report = %{
343 + package: info.package,
344 + filename: file.filename,
345 + label: info.label,
346 + icon: icon,
347 + written?: not dry?
348 + }
349 +
350 + unless dry? do
351 + additions = info |> apk_metadata(body) |> Map.take(@backfilled)
352 +
353 + file
354 + |> Ecto.Changeset.change(metadata: Map.merge(file.metadata || %{}, additions))
355 + |> GitGud.Repo.update!()
356 +
357 + # The version carries the same map at publish time; kept in step
358 + # so the two don't disagree about what an APK holds.
359 + file.package_version
360 + |> Ecto.Changeset.change(
361 + metadata: Map.merge(file.package_version.metadata || %{}, additions)
362 + )
363 + |> GitGud.Repo.update!()
364 + end
365 +
366 + {:ok, report}
367 + else
368 + {:error, reason} ->
369 + {:error, %{filename: file.filename, error: reason}}
370 + end
371 + end
372 +
252 373 @doc """
253 374 Everything the index knows about an APK: what the manifest and
254 375 resource table say, plus the launcher icon, stored.
modified lib/git_gud/release.ex
+64 −0
@@ -115,6 +115,70 @@ defmodule GitGud.Release do
115 115 :ok
116 116 end
117 117
118 + @doc """
119 + Backfill the app name and launcher icon the F-Droid index shows for
120 + each published Android app.
121 +
122 + Both are read out of an APK at upload, so an app published before the
123 + adapter could resolve `resources.arsc` has neither, and nothing
124 + re-reads an APK afterwards — the entry stays a package id with no
125 + icon until the app publishes again. Re-running its build doesn't
126 + help: versionCode is the CI run number, a re-run reuses it, and the
127 + registry refuses the same versionCode with different bytes.
128 +
129 + Run in a deployed pod (no Mix in a release):
130 +
131 + kubectl -n git-gud exec deployment/git-gud -- \\
132 + bin/git_gud eval 'GitGud.Release.backfill_fdroid_icons()'
133 +
134 + # every version, not just each app's newest
135 + bin/git_gud eval 'GitGud.Release.backfill_fdroid_icons(all: true)'
136 +
137 + # report without writing
138 + bin/git_gud eval 'GitGud.Release.backfill_fdroid_icons(dry_run: true)'
139 +
140 + `eval` is enough because blobs are read through `GitGud.Storage`,
141 + which is the disk implementation by default. An instance configured
142 + for S3 wants `rpc` instead, so the call lands on the running node
143 + where ExAws is already started.
144 +
145 + Idempotent, and safe against a live instance: `with_repo/2` starts the
146 + repo for the duration and stops it after, so this never contends with
147 + the running server for a port the way booting the application would.
148 +
149 + The walk is `GitGud.Packages.Formats.Fdroid.backfill_icons/1`, shared
150 + with `mix gitgud.backfill_fdroid_icons` — two copies would drift.
151 + """
152 + def backfill_fdroid_icons(opts \\ []) do
153 + load_app()
154 +
155 + {:ok, {read, written}, _} =
156 + Ecto.Migrator.with_repo(GitGud.Repo, fn _repo ->
157 + GitGud.Packages.Formats.Fdroid.backfill_icons(
158 + Keyword.put(opts, :on_file, &IO.puts(format_backfill(&1)))
159 + )
160 + end)
161 +
162 + IO.puts("\n#{read} APK(s) read, #{written} updated.")
163 + :ok
164 + end
165 +
166 + defp format_backfill(%{error: reason, filename: filename}),
167 + do: "#{filename}: could not read — #{inspect(reason)}"
168 +
169 + defp format_backfill(%{package: package, filename: filename} = report) do
170 + icon =
171 + case report.icon do
172 + {size, ext} -> "#{size} byte .#{ext} icon"
173 + :none -> "no icon"
174 + end
175 +
176 + name = report.label || "no name (falls back to #{package})"
177 + suffix = if report.written?, do: "", else: " — not written"
178 +
179 + "#{filename}: #{name}, #{icon}#{suffix}"
180 + end
181 +
118 182 @doc """
119 183 Send a test event to Sentry to verify the DSN + integration:
120 184
modified lib/mix/tasks/gitgud.backfill_fdroid_icons.ex
+27 −94
@@ -18,123 +18,56 @@ defmodule Mix.Tasks.Gitgud.BackfillFdroidIcons do
18 18 re-reading twenty older builds would pull twenty APKs out of storage
19 19 to change nothing anyone can see.
20 20
21 − Safe to run twice: an icon already in storage is stored under the
22 − same content-addressed key, and the write is idempotent.
23 −
24 − It only ever adds a name and an icon. Everything else the file
25 − records was written when it was published, and stays.
21 + Starts the Repo alone rather than the whole application, so it can be
22 + run against a live instance — booting the app would fight the running
23 + server for its fixed SSH and metrics ports.
24 +
25 + The walk itself is `GitGud.Packages.Formats.Fdroid.backfill_icons/1`,
26 + shared with `GitGud.Release.backfill_fdroid_icons/1` — a release
27 + ships no Mix, so a deployed instance needs the other entry point, and
28 + two copies of this would drift.
26 29 """
27 30 @shortdoc "Fill in F-Droid app names and icons from published APKs"
28 31
29 32 use Mix.Task
30 33
31 − import Ecto.Query
32 −
33 − alias GitGud.Packages
34 34 alias GitGud.Packages.Formats.Fdroid
35 − alias GitGud.Packages.Formats.Fdroid.Apk
36 − alias GitGud.Packages.PackageFile
37 − alias GitGud.Repo
38 −
39 − @format "fdroid"
40 −
41 − # Only these are taken from the re-read. Everything else the file
42 − # already records — the package name, the versionCode, the signer —
43 − # is what it was published as, and a backfill is the wrong place to
44 − # renumber a published version: the index orders by versionCode and
45 − # the filename encodes it.
46 − @backfilled ~w(label icon_key icon_sha256 icon_size icon_ext)
47 35
48 36 @impl true
49 37 def run(args) do
50 38 {opts, _rest, _invalid} =
51 39 OptionParser.parse(args, strict: [all: :boolean, dry_run: :boolean])
52 40
53 − Mix.Task.run("app.start")
41 + Mix.Task.run("app.config")
42 + {:ok, _} = Application.ensure_all_started(:ecto_sql)
43 + {:ok, _} = Application.ensure_all_started(:postgrex)
54 44
55 − all? = Keyword.get(opts, :all, false)
56 − dry? = Keyword.get(opts, :dry_run, false)
57 −
58 − files = candidates(all?)
59 −
60 − if files == [] do
61 − Mix.shell().info("No APKs published.")
62 − else
63 − Mix.shell().info("#{length(files)} APK(s) to read#{if dry?, do: " (dry run)", else: ""}")
64 − Enum.each(files, &backfill(&1, dry?))
65 − end
66 − end
45 + started? =
46 + case GitGud.Repo.start_link() do
47 + {:ok, _pid} -> true
48 + {:error, {:already_started, _pid}} -> false
49 + end
67 50
68 − # Every published APK, or just the newest of each app. "Newest" is by
69 − # versionCode, which is what the index sorts on — a file whose
70 − # metadata predates this adapter has none, and sorts last rather than
71 − # crashing the comparison.
72 − defp candidates(true), do: apk_files()
73 −
74 − defp candidates(false) do
75 − apk_files()
76 − |> Enum.group_by(& &1.package_version.package_id)
77 − |> Enum.map(fn {_package_id, files} -> Enum.max_by(files, &version_code/1) end)
78 − |> Enum.sort_by(& &1.id)
79 − end
51 + {read, written} =
52 + Fdroid.backfill_icons(Keyword.put(opts, :on_file, &Mix.shell().info(report(&1))))
80 53
81 − defp apk_files do
82 − from(f in PackageFile,
83 − join: v in assoc(f, :package_version),
84 − join: p in assoc(v, :package),
85 − where: p.format == ^@format,
86 − preload: [package_version: {v, package: p}]
87 − )
88 − |> Repo.all()
89 − end
54 + Mix.shell().info("\n#{read} APK(s) read, #{written} updated.")
90 55
91 − defp version_code(%PackageFile{metadata: metadata}) do
92 − case metadata do
93 − %{"version_code" => code} when is_integer(code) -> code
94 − _ -> -1
95 − end
56 + if started?, do: GitGud.Repo.stop()
96 57 end
97 58
98 − defp backfill(%PackageFile{} = file, dry?) do
99 − label = "#{file.package_version.package.name} #{file.filename}"
100 −
101 − with {:ok, body} <- Packages.read_file(file),
102 − {:ok, info} <- Apk.read(body) do
103 − report(label, info, body, dry?)
104 −
105 − unless dry? do
106 − additions = info |> Fdroid.apk_metadata(body) |> Map.take(@backfilled)
107 − metadata = Map.merge(file.metadata || %{}, additions)
108 −
109 − file
110 − |> Ecto.Changeset.change(metadata: metadata)
111 − |> Repo.update!()
112 −
113 − # The version carries the same map at publish time. Kept in
114 − # step here so the two don't disagree about what an APK holds.
115 − file.package_version
116 − |> Ecto.Changeset.change(
117 − metadata: Map.merge(file.package_version.metadata || %{}, metadata)
118 − )
119 − |> Repo.update!()
120 − end
121 − else
122 − {:error, reason} ->
123 − Mix.shell().error("#{label}: could not read — #{inspect(reason)}")
124 − end
125 − end
59 + defp report(%{error: reason, filename: filename}),
60 + do: "#{filename}: could not read — #{inspect(reason)}"
126 61
127 − # Reported before anything is written, so a dry run and a real run
128 − # print the same lines and the difference is only whether they stick.
129 − defp report(label, info, body, dry?) do
62 + defp report(%{package: package, filename: filename} = file) do
130 63 icon =
131 − case Apk.icon(body) do
132 − {:ok, bytes, ext} -> "#{byte_size(bytes)} byte .#{ext} icon"
64 + case file.icon do
65 + {size, ext} -> "#{size} byte .#{ext} icon"
133 66 :none -> "no icon"
134 67 end
135 68
136 − name = info[:label] || "no name (falls back to #{info[:package]})"
69 + name = file.label || "no name (falls back to #{package})"
137 70
138 − Mix.shell().info("#{label}: #{name}, #{icon}#{if dry?, do: " — not written", else: ""}")
71 + "#{filename}: #{name}, #{icon}#{if file.written?, do: "", else: " — not written"}"
139 72 end
140 73 end
modified test/git_gud/packages/fdroid_backfill_test.exs
+27 −13
@@ -1,6 +1,8 @@
1 1 defmodule GitGud.Packages.Fdroid.BackfillTest do
2 2 @moduledoc """
3 − `mix gitgud.backfill_fdroid_icons`.
3 + `Fdroid.backfill_icons/1` — the walk behind both
4 + `GitGud.Release.backfill_fdroid_icons/1`, which is what a release can
5 + call, and `mix gitgud.backfill_fdroid_icons`.
4 6
5 7 The situation it exists for: an APK published before the adapter
6 8 could read `resources.arsc` carries no icon and no label, and nothing
@@ -14,9 +16,9 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
14 16
15 17 alias GitGud.ApkFixtures
16 18 alias GitGud.Packages
19 + alias GitGud.Packages.Formats.Fdroid
17 20 alias GitGud.Packages.PackageFile
18 21 alias GitGud.Repo
19 − alias Mix.Tasks.Gitgud.BackfillFdroidIcons
20 22
21 23 setup do
22 24 {_user, repo} = repository_fixture()
@@ -48,8 +50,17 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
48 50
49 51 defp reload(file), do: Repo.get!(PackageFile, file.id)
50 52
51 − defp run(args) do
52 − ExUnit.CaptureIO.capture_io(fn -> BackfillFdroidIcons.run(args) end)
53 + # Reports are collected rather than printed: the callback is what
54 + # both entry points render, so asserting on it tests the thing they
55 + # share instead of one of their two formatters.
56 + defp run(opts \\ []) do
57 + {:ok, reports} = Agent.start_link(fn -> [] end)
58 +
59 + on_file = fn report -> Agent.update(reports, &[report | &1]) end
60 +
61 + {read, written} = Fdroid.backfill_icons(Keyword.put(opts, :on_file, on_file))
62 +
63 + %{read: read, written: written, reports: Agent.get(reports, &Enum.reverse/1)}
53 64 end
54 65
55 66 test "fills in the label and the icon", %{repo: repo} do
@@ -58,7 +69,10 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
58 69 refute Map.has_key?(file.metadata, "label")
59 70 refute Map.has_key?(file.metadata, "icon_key")
60 71
61 − run([])
72 + assert %{read: 1, written: 1, reports: [report]} = run()
73 +
74 + assert report.label == "GitGud"
75 + assert {_size, "png"} = report.icon
62 76
63 77 metadata = reload(file).metadata
64 78
@@ -78,7 +92,7 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
78 92 # version, which the index orders by and the filename encodes.
79 93 file = publish_legacy(repo, 224)
80 94
81 − run([])
95 + run()
82 96
83 97 metadata = reload(file).metadata
84 98
@@ -92,7 +106,7 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
92 106 old = publish_legacy(repo, 1)
93 107 new = publish_legacy(repo, 224)
94 108
95 − run([])
109 + run()
96 110
97 111 # The index takes an app's name and icon from its highest
98 112 # versionCode, so the older builds are twenty APKs of storage read
@@ -105,7 +119,7 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
105 119 old = publish_legacy(repo, 1)
106 120 new = publish_legacy(repo, 224)
107 121
108 − run(["--all"])
122 + run(all: true)
109 123
110 124 assert reload(old).metadata["label"] == "GitGud"
111 125 assert reload(new).metadata["label"] == "GitGud"
@@ -114,19 +128,19 @@ defmodule GitGud.Packages.Fdroid.BackfillTest do
114 128 test "--dry-run reports without writing", %{repo: repo} do
115 129 file = publish_legacy(repo, 224)
116 130
117 − output = run(["--dry-run"])
131 + assert %{read: 1, written: 0, reports: [report]} = run(dry_run: true)
118 132
119 − assert output =~ "GitGud"
120 − assert output =~ "not written"
133 + assert report.label == "GitGud"
134 + refute report.written?
121 135 refute Map.has_key?(reload(file).metadata, "label")
122 136 end
123 137
124 138 test "is safe to run twice", %{repo: repo} do
125 139 file = publish_legacy(repo, 224)
126 140
127 − run([])
141 + run()
128 142 first = reload(file).metadata
129 − run([])
143 + run()
130 144
131 145 assert reload(file).metadata == first
132 146 end

Parents: 8808a78