GITGUD

neiam neiam /gitgud

Git Gud

public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages

⭐ 1

Log in to mark this repository.

Detect a repository's licence, and publish it to F-Droid

53f3a6c · Gabriel Morell · 2026-09-22 13:33

✓ build ✓ hello
30 files +656 -1
Message 
{commit_body(@commit)}

Files changed

modified README.md
+1 −0
@@ -603,6 +603,7 @@ kubectl -n git-gud exec deployment/git-gud -- \
603 603 | `configure_repos()` | Backfill the per-repo git config `actions/checkout` needs (`uploadpack.allow*SHA1InWant`). New repos get it at creation; this fixes older ones |
604 604 | `backfill_language_tags()` | Derive each repo's language tag from the stats already computed for it. New stats tag their repo as they're written, so this is for repos whose stats predate the tag |
605 605 | `backfill_fdroid_icons()` | Fill in the app name and launcher icon the F-Droid index shows, by re-reading published APKs. Both are read at upload, so apps published before the adapter could resolve `resources.arsc` have neither. Takes `all: true` for every version rather than each app's newest, and `dry_run: true` to report without writing |
606 +| `backfill_licenses()` | Re-detect every repository's licence from the licence file at its default branch's tip. A push refreshes it, so this is for repositories that predate detection or that nobody pushes to |
606 607 | `sentry_test()` | Send a synchronous test event to verify the DSN |
607 608
608 609 All of them are idempotent and safe against a live instance: each
modified lib/git_gud/packages/formats/fdroid.ex
+16 −1
@@ -57,6 +57,10 @@ defmodule GitGud.Packages.Formats.Fdroid do
57 57 spell those fields identically and neither localizes them, unlike
58 58 `name`, `summary` and `description`.
59 59
60 + `license` is the SPDX id detected from the repository's licence file
61 + (`GitGud.Repositories.License`), which v2 omits when detection
62 + couldn't tell and v1 spells `Unknown`, having no way to say it.
63 +
60 64 `webSite` is deliberately left unset: on a forge it would be the
61 65 repository page again, and F-Droid would show the same link twice
62 66 under two labels.
@@ -445,6 +449,7 @@ defmodule GitGud.Packages.Formats.Fdroid do
445 449 # Both index versions spell these the same way and neither localizes
446 450 # them, unlike `name`, `summary` and `description`.
447 451 #
452 + #
448 453 # Taken from the newest APK's repository: an app is published from
449 454 # one, and `_all` aggregates a namespace where in principle two could
450 455 # publish the same package name. The version a phone would install is
@@ -465,6 +470,10 @@ defmodule GitGud.Packages.Formats.Fdroid do
465 470 }
466 471 end
467 472
473 + # The SPDX id `GitGud.Repositories.License` detected for the
474 + # repository this app is published from, or nil when it couldn't tell.
475 + defp license(group), do: Enum.max_by(group, & &1.info["version_code"]).repo.license
476 +
468 477 defp repo_page_url(%Repository{} = repo),
469 478 do: "#{forge_url()}/r/#{Storage.repo_handle(repo)}/#{repo.name}"
470 479
@@ -593,6 +602,10 @@ defmodule GitGud.Packages.Formats.Fdroid do
593 602 "added" => added,
594 603 "lastUpdated" => updated,
595 604 "name" => %{"en-US" => app_label(name, group)},
605 + # Dropped when detection couldn't tell, which is the
606 + # difference between "we don't know" and claiming a licence
607 + # the repository doesn't state.
608 + "license" => license(group),
596 609 "summary" => localized(summary(group)),
597 610 "description" => localized(full_description(group)),
598 611 "icon" => app_icon_file(group)
@@ -676,7 +689,9 @@ defmodule GitGud.Packages.Formats.Fdroid do
676 689 "name" => app_label(name, group),
677 690 "summary" => summary(group) || "",
678 691 "description" => full_description(group) || "",
679 − "license" => "Unknown",
692 + # v1 has no notion of "we looked and couldn't tell", and a client
693 + # renders whatever string it is given.
694 + "license" => license(group) || "Unknown",
680 695 "categories" => [],
681 696 "added" => added,
682 697 "lastUpdated" => updated,
modified lib/git_gud/release.ex
+34 −0
@@ -179,6 +179,40 @@ defmodule GitGud.Release do
179 179 "#{filename}: #{name}, #{icon}#{suffix}"
180 180 end
181 181
182 + @doc """
183 + Re-detect every repository's licence from its default branch.
184 +
185 + A push refreshes a repository's licence, so this is for the ones that
186 + predate detection — and for a repository nobody is pushing to, which
187 + would otherwise never be looked at again.
188 +
189 + Run in a deployed pod (no Mix in a release):
190 +
191 + kubectl -n git-gud exec deployment/git-gud -- \\
192 + bin/git_gud eval 'GitGud.Release.backfill_licenses()'
193 +
194 + Idempotent, and safe against a live instance: `with_repo/2` starts the
195 + repo for the duration and stops it after.
196 +
197 + The walk is `GitGud.Repositories.backfill_licenses/1`, shared with
198 + `mix gitgud.backfill_licenses`.
199 + """
200 + def backfill_licenses do
201 + load_app()
202 +
203 + {:ok, {seen, detected}, _} =
204 + Ecto.Migrator.with_repo(GitGud.Repo, fn _repo ->
205 + GitGud.Repositories.backfill_licenses(
206 + on_repo: fn %{repository: repo, license: license} ->
207 + IO.puts("#{repo.name}: #{license || "(none detected)"}")
208 + end
209 + )
210 + end)
211 +
212 + IO.puts("\n#{seen} repositories, #{detected} with a licence.")
213 + :ok
214 + end
215 +
182 216 @doc """
183 217 Send a test event to Sentry to verify the DSN + integration:
184 218
modified lib/git_gud/repositories.ex
+96 −0
@@ -21,6 +21,7 @@ defmodule GitGud.Repositories do
21 21 alias GitGud.Repositories.GitCommitLanguageStats
22 22 alias GitGud.Repositories.GitRef
23 23 alias GitGud.Repositories.GitTree
24 + alias GitGud.Repositories.License
24 25 alias GitGud.Repositories.HookReceiver
25 26 alias GitGud.Repositories.Repository
26 27 alias GitGud.Repositories.RepositoryRoot
@@ -2067,6 +2068,101 @@ defmodule GitGud.Repositories do
2067 2068 end)
2068 2069 end
2069 2070
2071 + @doc """
2072 + Detect the licence at `sha` and record it on the repository.
2073 +
2074 + Reads the root tree only: a licence lives beside the README, and
2075 + walking a whole tree to find one would be work for nothing. Candidate
2076 + files are read in full — an AGPL text is 34 KB, and `@max_license_bytes`
2077 + is the line past which a file called `LICENSE` is something else.
2078 +
2079 + Returns `{:ok, spdx | nil}`. A repository whose licence file is
2080 + unrecognised keeps `nil`, which says "we looked and couldn't tell" —
2081 + better than a guess in a field a package index publishes.
2082 + """
2083 + @max_license_bytes 512 * 1024
2084 +
2085 + def detect_and_store_license(%Repository{} = repo, sha) when is_binary(sha) do
2086 + with {:ok, commit} <- get_commit(repo, sha),
2087 + {:ok, entries} <- get_tree(repo, commit.tree_sha) do
2088 + files =
2089 + entries
2090 + |> Enum.filter(fn entry ->
2091 + entry.type == "blob" and License.candidate?(entry.name) and
2092 + (is_nil(entry.size) or entry.size <= @max_license_bytes)
2093 + end)
2094 + |> Enum.flat_map(fn entry ->
2095 + case get_blob_bytes(repo, entry.sha) do
2096 + {:ok, bytes} when byte_size(bytes) <= @max_license_bytes -> [{entry.name, bytes}]
2097 + _ -> []
2098 + end
2099 + end)
2100 +
2101 + license = License.detect_all(files)
2102 +
2103 + file =
2104 + if license do
2105 + files |> Enum.map(&elem(&1, 0)) |> Enum.sort() |> List.first()
2106 + end
2107 +
2108 + store_license(repo, license, file)
2109 + {:ok, license}
2110 + end
2111 + end
2112 +
2113 + # Written straight rather than through a changeset: this is derived
2114 + # state, cast from the git repository and never from a user.
2115 + #
2116 + # Unconditionally, and not "only when it changed": the struct a caller
2117 + # hands in is whatever it loaded, which may predate the last
2118 + # detection, and comparing against a stale copy is how a licence
2119 + # deleted from HEAD stays on the repository forever. One UPDATE per
2120 + # push is not worth that.
2121 + #
2122 + # `updated_at` is left alone. This is the repository being read, not
2123 + # edited, and `pushed_at` already records that something happened.
2124 + defp store_license(%Repository{id: rid}, license, file) do
2125 + from(r in Repository, where: r.id == ^rid)
2126 + |> Repo.update_all(set: [license: license, license_file: file])
2127 +
2128 + :ok
2129 + end
2130 +
2131 + @doc """
2132 + Re-detect every repository's licence from its default branch.
2133 +
2134 + For repositories that predate detection: new pushes refresh it, and
2135 + one that isn't being pushed to would never otherwise be looked at.
2136 +
2137 + Shared by `mix gitgud.backfill_licenses` and
2138 + `GitGud.Release.backfill_licenses/0`.
2139 + """
2140 + def backfill_licenses(opts \\ []) do
2141 + on_repo = Keyword.get(opts, :on_repo, fn _ -> :ok end)
2142 +
2143 + Repository
2144 + |> Repo.all()
2145 + |> Enum.reduce({0, 0}, fn repo, {seen, detected} ->
2146 + license =
2147 + with {:ok, sha} <- default_branch_sha(repo),
2148 + {:ok, license} <- detect_and_store_license(repo, sha) do
2149 + license
2150 + else
2151 + _ -> nil
2152 + end
2153 +
2154 + on_repo.(%{repository: repo, license: license})
2155 + {seen + 1, if(license, do: detected + 1, else: detected)}
2156 + end)
2157 + end
2158 +
2159 + defp default_branch_sha(%Repository{default_branch: branch, disk_path: path})
2160 + when is_binary(branch) and branch != "" do
2161 + Git.resolve(path, "refs/heads/" <> branch)
2162 + end
2163 +
2164 + defp default_branch_sha(_repo), do: :error
2165 +
2070 2166 @doc """
2071 2167 Walk the tree at `sha` and bucket blob bytes by `detected_language`.
2072 2168 Writes the result into `git_commit_language_stats`. Synchronous —
added lib/git_gud/repositories/license.ex
+153 −0
@@ -0,0 +1,153 @@
1 +defmodule GitGud.Repositories.License do
2 + @moduledoc """
3 + Which licence a repository is under, from the file it keeps it in.
4 +
5 + Not a full text-matching identifier. `licensee` and its kind compare
6 + a candidate against the normalised body of every licence SPDX knows,
7 + which means shipping those bodies — a megabyte of corpus to answer a
8 + question that, in practice, a title line already answers. This
9 + matches distinctive phrases instead: a title, plus something from the
10 + body to corroborate it, so a README that merely *mentions* the GPL
11 + isn't mistaken for one.
12 +
13 + Three rules keep it honest:
14 +
15 + * An `SPDX-License-Identifier:` in the file wins outright. It is the
16 + author saying exactly what they meant, and no inference beats it.
17 + * Order is most-specific first. Affero and Lesser both contain
18 + "GNU General Public License", so the plain GPL patterns can only
19 + match once those have been ruled out.
20 + * Nothing recognised means `nil`, not a guess. "Unknown" is a true
21 + statement about a licence file we can't read; naming the wrong
22 + licence is a false one, and this ends up in a package index where
23 + people rely on it.
24 +
25 + ## Why the ids have no `-only` / `-or-later`
26 +
27 + SPDX splits the GPL family that way, and the licence text can't tell
28 + you which: "either version 3 of the License, or (at your option) any
29 + later version" appears in the GPL's own *appendix*, in every copy,
30 + whatever the project chose. Inferring the suffix from the file would
31 + be reading boilerplate as intent. The deprecated family id is what is
32 + actually known — unless the author wrote an `SPDX-License-Identifier`
33 + down, which is exactly the case where the suffix is trustworthy.
34 +
35 + ## Dual licensing
36 +
37 + A repository with `LICENSE-MIT` and `LICENSE-APACHE` is under both,
38 + and SPDX has a spelling for that: `MIT OR Apache-2.0`. Distinct
39 + findings are joined rather than one of them being picked.
40 + """
41 +
42 + # Filenames that hold a licence, matched case-insensitively against
43 + # the root tree. `COPYING.LESSER` is the LGPL's conventional second
44 + # file, so the extension is not restricted.
45 + @filenames ~w(license licence copying unlicense copyright)
46 +
47 + # {spdx id, [phrases that must all appear]}. Ordered: the first match
48 + # wins, so a longer name is checked before the one it contains.
49 + @signatures [
50 + {"AGPL-3.0", ["gnu affero general public license", "version 3"]},
51 + {"LGPL-3.0", ["gnu lesser general public license", "version 3"]},
52 + {"LGPL-2.1", ["gnu lesser general public license", "version 2.1"]},
53 + {"GPL-3.0", ["gnu general public license", "version 3"]},
54 + {"GPL-2.0", ["gnu general public license", "version 2"]},
55 + {"Apache-2.0", ["apache license", "version 2.0"]},
56 + {"MPL-2.0", ["mozilla public license", "version 2.0"]},
57 + {"EPL-2.0", ["eclipse public license", "v 2.0"]},
58 + {"CC0-1.0", ["cc0 1.0 universal"]},
59 + {"Unlicense", ["this is free and unencumbered software released into the public domain"]},
60 + {"WTFPL", ["do what the fuck you want to public license"]},
61 + {"ISC", ["permission to use, copy, modify, and/or distribute this software"]},
62 + {"MIT",
63 + ["permission is hereby granted, free of charge", "the software is provided \"as is\""]},
64 + {"BSD-3-Clause",
65 + ["redistribution and use in source and binary forms", "neither the name of"]},
66 + {"BSD-2-Clause", ["redistribution and use in source and binary forms"]}
67 + ]
68 +
69 + @doc """
70 + True for a filename that would hold a licence.
71 +
72 + A name, not a path: this is applied to the entries of a repository's
73 + root tree, where a licence lives. `src/license.rs` is a source file
74 + that happens to be called that, and is none of this function's
75 + business.
76 + """
77 + @spec candidate?(String.t()) :: boolean()
78 + def candidate?(name) when is_binary(name) do
79 + base = String.downcase(name)
80 +
81 + not String.contains?(base, "/") and
82 + Enum.any?(@filenames, fn candidate ->
83 + base == candidate or String.starts_with?(base, candidate <> ".") or
84 + String.starts_with?(base, candidate <> "-")
85 + end)
86 + end
87 +
88 + def candidate?(_), do: false
89 +
90 + @doc """
91 + The SPDX id `text` is, or nil.
92 + """
93 + @spec detect(binary()) :: String.t() | nil
94 + def detect(text) when is_binary(text) do
95 + case declared(text) do
96 + nil -> matched(text)
97 + id -> id
98 + end
99 + end
100 +
101 + def detect(_), do: nil
102 +
103 + @doc """
104 + One SPDX expression for a whole set of licence files.
105 +
106 + Takes `{name, contents}` pairs — the root tree, filtered — and gives
107 + back what the repository is under: a single id, an `A OR B` for a
108 + dual-licensed one, or nil.
109 + """
110 + @spec detect_all([{String.t(), binary()}]) :: String.t() | nil
111 + def detect_all(files) when is_list(files) do
112 + files
113 + |> Enum.filter(fn {name, _contents} -> candidate?(name) end)
114 + |> Enum.sort_by(fn {name, _contents} -> name end)
115 + |> Enum.map(fn {_name, contents} -> detect(contents) end)
116 + |> Enum.reject(&is_nil/1)
117 + |> Enum.uniq()
118 + |> case do
119 + [] -> nil
120 + [one] -> one
121 + several -> Enum.join(Enum.sort(several), " OR ")
122 + end
123 + end
124 +
125 + # `SPDX-License-Identifier: AGPL-3.0-or-later`, anywhere in the file.
126 + # Bounded to one line and to characters an id can contain, so a prose
127 + # mention can't drag a paragraph in.
128 + defp declared(text) do
129 + case Regex.run(~r/SPDX-License-Identifier:\s*([A-Za-z0-9\-\.\+ ]+)/, text,
130 + capture: :all_but_first
131 + ) do
132 + [id] -> id |> String.trim() |> String.trim_trailing(".")
133 + _ -> nil
134 + end
135 + end
136 +
137 + defp matched(text) do
138 + normalized = normalize(text)
139 +
140 + Enum.find_value(@signatures, fn {id, phrases} ->
141 + if Enum.all?(phrases, &String.contains?(normalized, &1)), do: id
142 + end)
143 + end
144 +
145 + # Case and layout carry no meaning here, and a licence file is
146 + # wrapped at whatever width its author chose — so a phrase that spans
147 + # a line break has to compare equal to the same phrase that doesn't.
148 + defp normalize(text) do
149 + text
150 + |> String.downcase()
151 + |> String.replace(~r/\s+/u, " ")
152 + end
153 +end
modified lib/git_gud/repositories/repository.ex
+5 −0

Click to load diff…

modified lib/git_gud/repositories/workers/language_stats.ex
+20 −0

Click to load diff…

added lib/mix/tasks/gitgud.backfill_licenses.ex
+48 −0

Click to load diff…

added priv/repo/migrations/20260922120000_add_repository_license.exs
+25 −0

Click to load diff…

modified test/git_gud/packages/fdroid_format_test.exs
+28 −0

Click to load diff…

added test/git_gud/repositories/license_detection_test.exs
+93 −0

Click to load diff…

added test/git_gud/repositories/license_test.exs
+121 −0

Click to load diff…

modified test/support/fixtures/forge_fixtures.ex
+16 −0

Click to load diff…

Parents: 7e05eb8