apt: namespace-wide repositories at _all

202c083 · Gabriel Morell · 2026-09-11 15:13

17 files +756 -107
Message
{commit_body(@commit)}

Files changed

modified lib/git_gud/packages.ex
+28 −6
@@ -402,12 +402,9 @@ defmodule GitGud.Packages do
402 402 end
403 403
404 404 @doc """
405 − A namespace's existing key for `format`, or `nil`.
406 −
407 − Never generates. The owner-facing key endpoint uses this so that
408 − asking for a key only ever confirms a namespace that has actually
409 − published something — a 404 otherwise, rather than an oracle that
410 − answers for every handle anyone cares to try.
405 + A namespace's existing key for `format`, or `nil`. Never generates —
406 + for callers that want to know whether one exists rather than to have
407 + one.
411 408 """
412 409 def get_signing_key(owner, format) when is_struct(owner) do
413 410 Repo.get_by(SigningKey, SigningKey.owner_fields(owner) ++ [format: format])
@@ -422,6 +419,31 @@ defmodule GitGud.Packages do
422 419 repo.organization || repo.owner
423 420 end
424 421
422 + @doc """
423 + Every repository in a namespace, in name order.
424 +
425 + Unfiltered — the caller decides who may see what, because "who is
426 + asking" is a registry-auth question and this module has no opinion
427 + about it. Personal repositories are those the user owns *outside* an
428 + organization; an org-owned repo belongs to the org's namespace even
429 + though a user created it.
430 + """
431 + def namespace_repositories(%GitGud.Organizations.Organization{id: oid}) do
432 + Repository
433 + |> where([r], r.organization_id == ^oid)
434 + |> order_by([r], asc: r.name)
435 + |> Repo.all()
436 + |> Repo.preload([:owner, :organization])
437 + end
438 +
439 + def namespace_repositories(%GitGud.Accounts.User{id: uid}) do
440 + Repository
441 + |> where([r], r.owner_id == ^uid and is_nil(r.organization_id))
442 + |> order_by([r], asc: r.name)
443 + |> Repo.all()
444 + |> Repo.preload([:owner, :organization])
445 + end
446 +
425 447 defp generate_signing_key(owner, format) do
426 448 {:ok, public_pem, private_pem} = generate_keypair(owner, format)
427 449 {:ok, wrapped} = GitGud.Secrets.encrypt(private_pem)
modified lib/git_gud/packages/format.ex
+26 −1
@@ -113,7 +113,32 @@ defmodule GitGud.Packages.Format do
113 113 """
114 114 @callback normalize(String.t()) :: String.t()
115 115
116 − @optional_callbacks normalize: 1
116 + @doc """
117 + Serve a namespace-wide index at `/pkg/:format/:owner/#{"_all"}/*rest`,
118 + aggregating every repository in the namespace the caller may read.
119 +
120 + Optional. It earns its place where the namespace is the unit a
121 + consumer actually configures — apt, where five repositories mean
122 + five `deb` lines and five keyring entries for what an administrator
123 + thinks of as one publisher. A format without it 404s there, which is
124 + the honest answer.
125 +
126 + `repos` is already filtered by the caller's read access, so an
127 + adapter renders what it is given and makes no visibility decisions
128 + of its own.
129 + """
130 + @callback namespace_index(namespace :: struct(), repos :: [Repository.t()], ctx()) ::
131 + index_result()
132 +
133 + @optional_callbacks normalize: 1, namespace_index: 3
134 +
135 + @doc """
136 + The repository-slot marker that means "the whole namespace".
137 +
138 + It can never collide with a repository: `Repository`'s name format
139 + requires a leading alphanumeric.
140 + """
141 + def namespace_marker, do: "_all"
117 142
118 143 @doc "Format identifiers this forge knows about."
119 144 def formats, do: @formats
modified lib/git_gud/packages/formats/deb.ex
+129 −81
@@ -1,19 +1,40 @@
1 1 defmodule GitGud.Packages.Formats.Deb do
2 2 @moduledoc """
3 − A Debian repository, per git repository.
3 + A Debian repository, per git repository *and* per namespace.
4 4
5 − curl -fsSL https://<host>/pkg/deb/<owner>/<repo>/key.asc \\
6 − | sudo tee /etc/apt/keyrings/<repo>.asc > /dev/null
7 − echo "deb [signed-by=/etc/apt/keyrings/<repo>.asc] \\
8 − https://<host>/pkg/deb/<owner>/<repo> stable main" \\
9 − | sudo tee /etc/apt/sources.list.d/<repo>.list
5 + # everything acme publishes — one line, one key
6 + deb [signed-by=/etc/apt/keyrings/acme.asc] \\
7 + https://<host>/pkg/deb/acme/_all stable main
8 +
9 + # or one repository on its own
10 + deb [signed-by=/etc/apt/keyrings/acme.asc] \\
11 + https://<host>/pkg/deb/acme/tools stable main
10 12
11 13 There is no daemon to embed here. An apt repository is static files
12 14 over plain HTTP — which is why this is last rather than hard: the
13 15 only genuinely new thing is the OpenPGP signature, and that lives in
14 16 `GitGud.Packages.Apt.Native`.
15 17
16 − Paths below the repository root:
18 + ## `_all`, and why the URL looks like that
19 +
20 + A namespace usually *is* the unit a machine wants. Five repositories
21 + under one organization are five `deb` lines and five `Packages`
22 + fetches, for what an administrator thinks of as "acme's packages".
23 + `_all` collapses them into one repository that aggregates every
24 + repository in the namespace the caller may read.
25 +
26 + It sits in the repository slot because a repository name must start
27 + with an alphanumeric (`Repository`'s name format), so `_all` can
28 + never be a repository and the two can share a URL space. The same
29 + marker is available to any other format that later wants a
30 + namespace-wide index.
31 +
32 + A namespace repository is filtered per caller: anonymous sees only
33 + public repositories, a user sees what they can read, a deploy token
34 + sees its own repository plus anything public. The `Release` is signed
35 + at request time, so a filtered view is still a correctly signed one.
36 +
37 + Paths below either root:
17 38
18 39 /key.asc the signing key
19 40 /dists/<dist>/InRelease clearsigned, preferred
@@ -21,8 +42,8 @@ defmodule GitGud.Packages.Formats.Deb do
21 42 /dists/<dist>/Release.gpg detached, for older apt
22 43 /dists/<dist>/<comp>/binary-<arch>/Packages
23 44 /dists/<dist>/<comp>/binary-<arch>/Packages.xz
24 − /pool/<comp>/<p>/<pkg>/<file>.deb
25 − /upload?dist=&component= POST a .deb
45 + /pool/<comp>/<repo>/<p>/<pkg>/<file>.deb
46 + /upload?dist=&component= POST a .deb (per repo)
26 47
27 48 ## What the spec actually requires
28 49
@@ -68,93 +89,121 @@ defmodule GitGud.Packages.Formats.Deb do
68 89 # ── reads ───────────────────────────────────────────────────────────
69 90
70 91 @impl true
71 − def index(repo, ctx)
92 + def index(repo, ctx), do: serve([repo], repo_label(repo), repo, ctx)
93 +
94 + @doc """
95 + The namespace-wide repository at `_all`: the same documents, built
96 + from every repository in the namespace the caller may read.
72 97
73 − def index(repo, %{path: ["key.asc"]}) do
74 − with {:ok, %{public_pem: armored}} <- Packages.signing_key(repo, @format) do
98 + `signing_scope` is the namespace itself, so the aggregate is signed
99 + with exactly the key that signs each repository individually — one
100 + keyring entry covers both ways of consuming it.
101 + """
102 + @impl true
103 + def namespace_index(namespace, repos, ctx),
104 + do: serve(repos, Storage.owner_handle(namespace), namespace, ctx)
105 +
106 + # `signing_scope` is whatever `Packages.signing_key/2` should key on —
107 + # a repository resolves to its namespace anyway, so both callers end
108 + # up at the same key.
109 + defp serve(repos, label, signing_scope, ctx)
110 +
111 + defp serve(_repos, _label, scope, %{path: ["key.asc"]}) do
112 + with {:ok, %{public_pem: armored}} <- Packages.signing_key(scope, @format) do
75 113 {:ok, "application/pgp-keys", armored}
76 114 end
77 115 end
78 116
79 − def index(repo, %{path: ["dists", dist, "InRelease"]}) do
80 − with {:ok, %{private_pem: secret}} <- Packages.signing_key(repo, @format),
81 − {:ok, signed} <- Native.sign_clearsign(secret, release_document(repo, dist)) do
82 − {:ok, "text/plain; charset=utf-8", signed}
83 − else
84 − {:error, reason} -> {:error, {500, "could not sign the Release: #{inspect(reason)}"}}
85 − end
117 + defp serve(repos, label, scope, %{path: ["dists", dist, "InRelease"]}) do
118 + sign(scope, &Native.sign_clearsign/2, release_document(repos, label, dist),
119 + content_type: "text/plain; charset=utf-8"
120 + )
86 121 end
87 122
88 − def index(repo, %{path: ["dists", dist, "Release"]}),
89 − do: {:ok, "text/plain; charset=utf-8", release_document(repo, dist)}
123 + defp serve(repos, label, _scope, %{path: ["dists", dist, "Release"]}),
124 + do: {:ok, "text/plain; charset=utf-8", release_document(repos, label, dist)}
90 125
91 − def index(repo, %{path: ["dists", dist, "Release.gpg"]}) do
92 − with {:ok, %{private_pem: secret}} <- Packages.signing_key(repo, @format),
93 − {:ok, signature} <- Native.sign_detached(secret, release_document(repo, dist)) do
94 − {:ok, "application/pgp-signature", signature}
95 − else
96 − {:error, reason} -> {:error, {500, "could not sign the Release: #{inspect(reason)}"}}
97 − end
126 + defp serve(repos, label, scope, %{path: ["dists", dist, "Release.gpg"]}) do
127 + sign(scope, &Native.sign_detached/2, release_document(repos, label, dist),
128 + content_type: "application/pgp-signature"
129 + )
98 130 end
99 131
100 − def index(repo, %{path: ["dists", dist, component, "binary-" <> arch, "Packages"]}),
101 − do: {:ok, "text/plain; charset=utf-8", packages_document(repo, dist, component, arch)}
132 + defp serve(repos, _label, _scope, %{
133 + path: ["dists", dist, component, "binary-" <> arch, "Packages"]
134 + }),
135 + do: {:ok, "text/plain; charset=utf-8", packages_document(repos, dist, component, arch)}
102 136
103 − def index(repo, %{path: ["dists", dist, component, "binary-" <> arch, "Packages.xz"]}) do
104 − case Native.xz_compress(packages_document(repo, dist, component, arch)) do
137 + defp serve(repos, _label, _scope, %{
138 + path: ["dists", dist, component, "binary-" <> arch, "Packages.xz"]
139 + }) do
140 + case Native.xz_compress(packages_document(repos, dist, component, arch)) do
105 141 {:ok, compressed} -> {:ok, "application/x-xz", compressed}
106 142 {:error, reason} -> {:error, {500, "could not compress the index: #{reason}"}}
107 143 end
108 144 end
109 145
110 − def index(repo, %{path: ["pool" | _] = path}) do
111 − filename = List.last(path)
146 + defp serve(repos, _label, _scope, %{path: ["pool" | _] = path}) do
147 + # Matched against the whole pool path, not just the filename: two
148 + # repositories in one namespace may well both publish
149 + # `widget_1.0_amd64.deb`, and serving the wrong one would be a
150 + # silent substitution rather than an error.
151 + wanted = Enum.join(path, "/")
112 152
113 − case find_deb(repo, filename) do
114 − %PackageFile{} = file -> {:blob, file}
153 + repos
154 + |> debs()
155 + |> Enum.find(&(pool_path(&1) == wanted))
156 + |> case do
157 + %{file: %PackageFile{} = file} -> {:blob, file}
115 158 nil -> {:error, :not_found}
116 159 end
117 160 end
118 161
119 − def index(_repo, _ctx), do: {:error, :not_found}
162 + defp serve(_repos, _label, _scope, _ctx), do: {:error, :not_found}
120 163
121 − defp find_deb(repo, filename) do
122 − repo
123 − |> Packages.list_packages(@format)
124 − |> Enum.find_value(fn package -> Packages.get_package_file(package, filename) end)
164 + defp sign(scope, signer, document, content_type: content_type) do
165 + with {:ok, %{private_pem: secret}} <- Packages.signing_key(scope, @format),
166 + {:ok, signed} <- signer.(secret, document) do
167 + {:ok, content_type, signed}
168 + else
169 + {:error, reason} -> {:error, {500, "could not sign the Release: #{inspect(reason)}"}}
170 + end
125 171 end
126 172
127 173 # ── the indices ─────────────────────────────────────────────────────
128 174
129 − # Every .deb in the repository, flattened with the fields the
130 − # indices key on. One query per package rather than one join is an
131 − # N+1, but a repository's package list is small and this keeps the
132 − # substrate's API unchanged.
133 − defp debs(repo) do
134 − repo
135 − |> Packages.list_packages(@format)
136 − |> Enum.flat_map(&Packages.list_package_files/1)
137 − |> Enum.map(fn %PackageFile{} = file ->
138 − metadata = file.metadata || %{}
139 −
140 − %{
141 − file: file,
142 − dist: metadata["dist"] || @default_dist,
143 − component: metadata["component"] || @default_component,
144 − architecture: metadata["architecture"] || @arch_all,
145 − control: metadata["control"] || %{}
146 − }
175 + # Every .deb across `repos`, flattened with the fields the indices
176 + # key on. One query per package rather than one join is an N+1, but a
177 + # namespace's package list is small and this keeps the substrate's
178 + # API unchanged.
179 + defp debs(repos) do
180 + Enum.flat_map(repos, fn %Repository{} = repo ->
181 + repo
182 + |> Packages.list_packages(@format)
183 + |> Enum.flat_map(&Packages.list_package_files/1)
184 + |> Enum.map(fn %PackageFile{} = file ->
185 + metadata = file.metadata || %{}
186 +
187 + %{
188 + file: file,
189 + repo_name: repo.name,
190 + dist: metadata["dist"] || @default_dist,
191 + component: metadata["component"] || @default_component,
192 + architecture: metadata["architecture"] || @arch_all,
193 + control: metadata["control"] || %{}
194 + }
195 + end)
147 196 end)
148 197 end
149 198
150 − defp packages_document(repo, dist, component, arch) do
151 − repo
199 + defp packages_document(repos, dist, component, arch) do
200 + repos
152 201 |> debs()
153 202 |> Enum.filter(fn deb ->
154 203 deb.dist == dist and deb.component == component and
155 204 (deb.architecture == arch or deb.architecture == @arch_all)
156 205 end)
157 − |> Enum.sort_by(&{&1.control["Package"], &1.control["Version"]})
206 + |> Enum.sort_by(&{&1.control["Package"], &1.control["Version"], &1.repo_name})
158 207 |> Enum.map_join("\n", &packages_stanza/1)
159 208 end
160 209
@@ -197,37 +246,36 @@ defmodule GitGud.Packages.Formats.Deb do
197 246 @doc """
198 247 The pool path for a package, relative to the repository root.
199 248
200 − Debian shards on the first letter, except that `lib*` gets its own
201 − two-character shard — `libfoo` under `libf/`, not `l/` — because
202 − otherwise `l/` holds most of the archive.
249 + Qualified by the source repository, which a stock Debian pool is
250 + not. It has to be: a namespace repository aggregates several, and two
251 + of them publishing `widget_1.0_amd64.deb` would otherwise be one path
252 + with two sets of bytes. `Filename` is opaque to apt, and qualifying
253 + it in *both* views means the per-repository and namespace indices
254 + name the same file identically — a client that switches between them
255 + re-downloads nothing.
256 +
257 + Within that, Debian's own sharding: the first letter, except `lib*`
258 + gets its own two-character shard — `libfoo` under `libf/`, not `l/` —
259 + because otherwise `l/` holds most of the archive.
203 260 """
204 261 def pool_path(deb) do
205 262 name = deb.control["Package"] || "unknown"
206 − Path.join(["pool", deb.component, shard(name), name, deb.file.filename])
263 + Path.join(["pool", deb.component, deb.repo_name, shard(name), name, deb.file.filename])
207 264 end
208 265
209 266 defp shard("lib" <> <<first::utf8>> <> _rest), do: "lib" <> <<first::utf8>>
210 267 defp shard(<<first::utf8>> <> _rest), do: <<first::utf8>>
211 268 defp shard(_), do: "unknown"
212 269
213 − defp release_document(repo, dist) do
214 − debs = repo |> debs() |> Enum.filter(&(&1.dist == dist))
270 + defp release_document(repos, label, dist) do
271 + debs = repos |> debs() |> Enum.filter(&(&1.dist == dist))
215 272
216 273 components = debs |> Enum.map(& &1.component) |> Enum.uniq() |> Enum.sort()
217 274 components = if components == [], do: [@default_component], else: components
218 275
219 − architectures =
220 − debs
221 − |> Enum.map(& &1.architecture)
222 − |> Enum.uniq()
223 − |> Enum.sort()
224 −
225 − # `all` on its own is not something a client can install from;
226 − # list a concrete architecture alongside so the index is usable.
276 + architectures = debs |> Enum.map(& &1.architecture) |> Enum.uniq() |> Enum.sort()
227 277 architectures = if architectures == [], do: [@arch_all], else: architectures
228 278
229 − label = repo_label(repo)
230 −
231 279 header =
232 280 [
233 281 {"Origin", label},
@@ -241,15 +289,15 @@ defmodule GitGud.Packages.Formats.Deb do
241 289 ]
242 290 |> Enum.map_join(fn {name, value} -> "#{name}: #{value}\n" end)
243 291
244 − header <> "SHA256:\n" <> checksums(repo, dist, components, architectures)
292 + header <> "SHA256:\n" <> checksums(repos, dist, components, architectures)
245 293 end
246 294
247 295 # Every index file the Release covers, with the size and SHA-256 of
248 296 # each — compressed *and* uncompressed, which the format requires
249 297 # even though apt will only fetch one of them.
250 − defp checksums(repo, dist, components, architectures) do
298 + defp checksums(repos, dist, components, architectures) do
251 299 for component <- components, arch <- architectures, into: "" do
252 − body = packages_document(repo, dist, component, arch)
300 + body = packages_document(repos, dist, component, arch)
253 301 base = "#{component}/binary-#{arch}/Packages"
254 302
255 303 compressed =
modified lib/git_gud/packages/usage.ex
+23 −2
@@ -88,6 +88,18 @@ defmodule GitGud.Packages.Usage do
88 88 @spec key_url(context(), String.t()) :: String.t()
89 89 def key_url(ctx, format), do: "#{ctx.root}/pkg/_keys/#{format}/#{ctx.handle}"
90 90
91 + @doc """
92 + The namespace-wide registry root — every repository in the namespace
93 + the caller may read, as one repository.
94 +
95 + For apt this is usually the line you actually want: one `deb` entry
96 + and one keyring file for what an administrator thinks of as one
97 + publisher, rather than one of each per repository.
98 + """
99 + @spec namespace_url(context(), String.t()) :: String.t()
100 + def namespace_url(ctx, format),
101 + do: "#{ctx.root}/pkg/#{format}/#{ctx.handle}/#{GitGud.Packages.Format.namespace_marker()}"
102 +
91 103 @doc """
92 104 Instructions for `format`, or `nil` if there is nothing to say.
93 105 """
@@ -305,7 +317,7 @@ defmodule GitGud.Packages.Usage do
305 317 %{
306 318 consume: [
307 319 step(
308 − "Trust #{ctx.handle}'s key — once per machine, for every repository it publishes",
320 + "Trust #{ctx.handle}'s key — once per machine, for everything it publishes",
309 321 "shell",
310 322 """
311 323 sudo install -d -m 0755 /etc/apt/keyrings
@@ -314,7 +326,16 @@ defmodule GitGud.Packages.Usage do
314 326 """
315 327 ),
316 328 step(
317 − "Then add this repository",
329 + "Add everything #{ctx.handle} publishes, as one repository",
330 + "shell",
331 + """
332 + echo "deb [signed-by=/etc/apt/keyrings/#{ctx.handle}.asc] #{namespace_url(ctx, "deb")} stable main" \\
333 + | sudo tee /etc/apt/sources.list.d/#{ctx.handle}.list
334 + sudo apt-get update
335 + """
336 + ),
337 + step(
338 + "…or just this repository",
318 339 "shell",
319 340 """
320 341 echo "deb [signed-by=/etc/apt/keyrings/#{ctx.handle}.asc] #{url} stable main" \\
modified lib/git_gud_web/controllers/package_registry_controller.ex
+67 −7
@@ -25,7 +25,6 @@ defmodule GitGudWeb.PackageRegistryController do
25 25 alias GitGud.Packages.Auth
26 26 alias GitGud.Packages.Format
27 27 alias GitGud.Packages.PackageFile
28 − alias GitGud.Packages.SigningKey
29 28 alias GitGud.Repositories
30 29
31 30 # Upload bodies we read ourselves. Well above a fat manylinux wheel;
@@ -47,16 +46,22 @@ defmodule GitGudWeb.PackageRegistryController do
47 46 instructions can say "once per organization" and mean it.
48 47
49 48 Unauthenticated, and deliberately so: a public key is not a secret,
50 − and apt and mix both need it before any credential is configured. It
51 − is served only where one already exists — generating on demand here
52 − would turn the endpoint into an oracle that confirms any handle
53 − anyone cares to type, whereas this only answers for a namespace that
54 − has actually published something.
49 + and apt and mix both need it before any credential is configured —
50 + pinning the key is literally the first line of the setup, before any
51 + `Release` has been fetched.
52 +
53 + Which is also why it generates on demand rather than 404ing until
54 + something has been signed: otherwise the first line of the
55 + instructions fails for a namespace that hasn't served a `Release`
56 + yet. A handle that doesn't resolve gets a 404 and no key, so probing
57 + creates nothing; a handle that does resolve is already discoverable
58 + through `/orgs` and `/u/:handle`, and a public key tells an attacker
59 + nothing further.
55 60 """
56 61 def owner_key(conn, %{"format" => format, "owner" => owner}) do
57 62 with {:ok, _mod} <- adapter(format),
58 63 {:ok, namespace} <- Repositories.resolve_namespace(owner),
59 − %SigningKey{public_key: key} <- Packages.get_signing_key(namespace, format) do
64 + {:ok, %{public_pem: key}} <- Packages.signing_key(namespace, format) do
60 65 conn
61 66 |> put_resp_content_type_raw(key_content_type(format))
62 67 |> send_resp(200, key)
@@ -68,6 +73,61 @@ defmodule GitGudWeb.PackageRegistryController do
68 73 defp key_content_type("deb"), do: "application/pgp-keys"
69 74 defp key_content_type(_format), do: "text/plain; charset=utf-8"
70 75
76 + # `_all` in the repository slot means the whole namespace. It can
77 + # never be a repository — a repository name must start with an
78 + # alphanumeric — so the two share a URL space without ambiguity.
79 + defp dispatch(conn, %{"format" => format, "owner" => owner, "name" => "_all"} = params, action) do
80 + with {:ok, mod} <- adapter(format),
81 + # `function_exported?/3` answers false for a module that
82 + # simply hasn't been loaded yet, so the ensure_loaded? is
83 + # load-bearing — without it this 404s on the first request
84 + # after a boot and works ever after, which is the worst
85 + # possible failure shape.
86 + true <-
87 + (Code.ensure_loaded?(mod) and function_exported?(mod, :namespace_index, 3)) or
88 + :unsupported,
89 + {:ok, namespace} <- Repositories.resolve_namespace(owner) do
90 + principal = Auth.authenticate(conn)
91 +
92 + # Filtered here, once, so the adapter renders what it is given
93 + # and makes no visibility decisions of its own. An anonymous
94 + # caller gets the public repositories; a token gets its own plus
95 + # those; a user gets what they can read.
96 + repos =
97 + namespace
98 + |> Packages.namespace_repositories()
99 + |> Enum.filter(&Auth.authorize(principal, &1, :read))
100 +
101 + ctx = %{
102 + path: List.wrap(params["rest"]),
103 + conn: conn,
104 + method: conn.method,
105 + trailing_slash?: String.ends_with?(conn.request_path, "/"),
106 + base_url: base_url(format, owner, "_all"),
107 + principal: principal
108 + }
109 +
110 + case action do
111 + :read ->
112 + respond(conn, format, mod.namespace_index(namespace, repos, ctx))
113 +
114 + # Publishing names a repository. There is no sensible one to
115 + # pick here, and guessing would put a package somewhere the
116 + # publisher didn't ask for.
117 + :write ->
118 + send_error(conn, format, 405, "publish to a repository, not to the namespace")
119 + end
120 + else
121 + :unsupported ->
122 + send_error(conn, format, 404, "#{format} has no namespace-wide index")
123 +
124 + # Both `adapter/1` and `resolve_namespace/1` fail as a bare
125 + # `:error`, and the caller gets the same 404 either way.
126 + :error ->
127 + send_error(conn, format, 404, "no such format or namespace")
128 + end
129 + end
130 +
71 131 defp dispatch(conn, %{"format" => format, "owner" => owner, "name" => name} = params, action) do
72 132 with {:ok, mod} <- adapter(format),
73 133 {:ok, repo} <- fetch_repo(owner, name),
modified test/git_gud/package_signing_keys_test.exs
+21 −3
@@ -112,11 +112,29 @@ defmodule GitGud.PackageSigningKeysTest do
112 112 end
113 113 end
114 114
115 − test "404s for a namespace that has never published — not an oracle", %{conn: conn} do
115 + test "mints the key on first ask — pinning it is the first setup step", %{conn: conn} do
116 116 {_user, org, _one, _two} = org_with_two_repos()
117 117
118 − assert get(conn, "/pkg/_keys/deb/#{org.handle}").status == 404
119 − assert get(build_conn(), "/pkg/_keys/deb/nobody-at-all").status == 404
118 + refute Packages.get_signing_key(org, "deb")
119 +
120 + conn = get(conn, "/pkg/_keys/deb/#{org.handle}")
121 +
122 + assert conn.status == 200
123 + assert conn.resp_body =~ "-----BEGIN PGP PUBLIC KEY BLOCK-----"
124 + assert Packages.get_signing_key(org, "deb")
125 + end
126 +
127 + test "and hands back the same key every time after that", %{conn: conn} do
128 + {_user, org, _one, _two} = org_with_two_repos()
129 +
130 + first = get(conn, "/pkg/_keys/deb/#{org.handle}").resp_body
131 + second = get(build_conn(), "/pkg/_keys/deb/#{org.handle}").resp_body
132 +
133 + assert first == second
134 + end
135 +
136 + test "a handle that doesn't resolve 404s, and mints nothing", %{conn: conn} do
137 + assert get(conn, "/pkg/_keys/deb/nobody-at-all").status == 404
120 138 end
121 139
122 140 test "404s for a format the forge doesn't serve", %{conn: conn} do
added test/git_gud_web/apt_namespace_test.exs
+424 −0
@@ -0,0 +1,424 @@
1 +defmodule GitGudWeb.AptNamespaceTest do
2 + @moduledoc """
3 + The namespace-wide apt repository at `_all`.
4 +
5 + A namespace is usually the unit a machine wants: five repositories
6 + under one organization are five `deb` lines for what an administrator
7 + thinks of as one publisher. These assert that the aggregate is a real
8 + apt repository — signed, complete, and filtered by who is asking.
9 +
10 + Organizations and users are the same code path; both are namespaces,
11 + and both are covered here.
12 + """
13 + use GitGudWeb.ConnCase, async: false
14 +
15 + alias GitGud.Packages
16 + alias GitGud.Packages.Apt.Native
17 + alias GitGud.Repositories
18 + alias GitGud.Repositories.Storage
19 +
20 + import GitGud.AccountsFixtures
21 + import GitGud.ForgeFixtures
22 +
23 + # A .deb: an ar archive of debian-binary, control.tar.gz and
24 + # data.tar.gz, in that order.
25 + defp deb(fields) do
26 + control = Enum.map_join(fields, fn {k, v} -> "#{k}: #{v}\n" end)
27 +
28 + ar([
29 + {"debian-binary", "2.0\n"},
30 + {"control.tar.gz", :zlib.gzip(tar([{~c"./control", control}]))},
31 + {"data.tar.gz", :zlib.gzip(tar([{~c"./usr/bin/thing", "#!/bin/sh\n"}]))}
32 + ])
33 + end
34 +
35 + defp tar(members) do
36 + path = Path.join(System.tmp_dir!(), "t-#{System.unique_integer([:positive])}.tar")
37 + :ok = :erl_tar.create(String.to_charlist(path), members, [])
38 + File.read!(path)
39 + end
40 +
41 + defp ar(members) do
42 + body =
43 + Enum.map_join(members, fn {name, data} ->
44 + header =
45 + String.pad_trailing(name, 16) <>
46 + String.pad_trailing("0", 12) <>
47 + String.pad_trailing("0", 6) <>
48 + String.pad_trailing("0", 6) <>
49 + String.pad_trailing("100644", 8) <>
50 + String.pad_trailing(Integer.to_string(byte_size(data)), 10) <>
51 + <<0x60, 0x0A>>
52 +
53 + header <> data <> if(rem(byte_size(data), 2) == 1, do: "\n", else: "")
54 + end)
55 +
56 + "!<arch>\n" <> body
57 + end
58 +
59 + defp publish(repo, package, opts \\ []) do
60 + fields = [
61 + Package: package,
62 + Version: Keyword.get(opts, :version, "1.0.0-1"),
63 + Architecture: Keyword.get(opts, :arch, "amd64"),
64 + Maintainer: "Someone <someone@example.com>",
65 + Description: "the #{package} package"
66 + ]
67 +
68 + body = deb(fields)
69 +
70 + {:ok, _p, _v, _f} =
71 + Packages.publish_file(repo, "deb", package, Keyword.get(opts, :version, "1.0.0-1"), %{
72 + filename:
73 + "#{package}_#{Keyword.get(opts, :version, "1.0.0-1")}_#{Keyword.get(opts, :arch, "amd64")}.deb",
74 + body: body,
75 + metadata: %{
76 + "dist" => Keyword.get(opts, :dist, "stable"),
77 + "component" => "main",
78 + "architecture" => Keyword.get(opts, :arch, "amd64"),
79 + "control" => Map.new(fields, fn {k, v} -> {to_string(k), v} end)
80 + }
81 + })
82 +
83 + body
84 + end
85 +
86 + defp org_fixture(user) do
87 + {:ok, org} =
88 + GitGud.Organizations.create_organization(user, %{
89 + "handle" => "org#{System.unique_integer([:positive])}",
90 + "name" => "Test Org"
91 + })
92 +
93 + org
94 + end
95 +
96 + defp org_repo(org, user, visibility \\ "public") do
97 + {:ok, repo} =
98 + Repositories.create_repository_for_org(org, user, %{
99 + "name" => unique_repo_name(),
100 + "visibility" => visibility
101 + })
102 +
103 + GitGud.Repo.preload(repo, [:owner, :organization])
104 + end
105 +
106 + setup do
107 + user = user_fixture()
108 + org = org_fixture(user)
109 + tools = org_repo(org, user)
110 + agents = org_repo(org, user)
111 +
112 + {:ok, user: user, org: org, tools: tools, agents: agents, root: "/pkg/deb/#{org.handle}/_all"}
113 + end
114 +
115 + describe "aggregation" do
116 + test "one index carries packages from every repository in the org", %{
117 + conn: conn,
118 + tools: tools,
119 + agents: agents,
120 + root: root
121 + } do
122 + publish(tools, "widget")
123 + publish(agents, "gadget")
124 +
125 + body = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
126 +
127 + assert body =~ "Package: widget"
128 + assert body =~ "Package: gadget"
129 + end
130 +
131 + test "each entry points into its own repository's pool", %{
132 + conn: conn,
133 + tools: tools,
134 + agents: agents,
135 + root: root
136 + } do
137 + publish(tools, "widget")
138 + publish(agents, "gadget")
139 +
140 + body = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
141 +
142 + assert body =~ "Filename: pool/main/#{tools.name}/w/widget/widget_1.0.0-1_amd64.deb"
143 + assert body =~ "Filename: pool/main/#{agents.name}/g/gadget/gadget_1.0.0-1_amd64.deb"
144 + end
145 +
146 + test "two repositories publishing the same filename don't collide", %{
147 + conn: conn,
148 + tools: tools,
149 + agents: agents,
150 + root: root
151 + } do
152 + # Same package name and version in both, different bytes.
153 + from_tools = publish(tools, "widget", version: "1.0.0-1")
154 + from_agents = publish(agents, "widget", version: "1.0.0-1", arch: "arm64")
155 +
156 + refute from_tools == from_agents
157 +
158 + index = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
159 + [_, tools_path] = Regex.run(~r/^Filename: (\S+)$/m, index)
160 +
161 + assert get(build_conn(), root <> "/" <> tools_path).resp_body == from_tools
162 +
163 + arm = get(build_conn(), root <> "/dists/stable/main/binary-arm64/Packages").resp_body
164 + [_, agents_path] = Regex.run(~r/^Filename: (\S+)$/m, arm)
165 +
166 + refute tools_path == agents_path
167 + assert get(build_conn(), root <> "/" <> agents_path).resp_body == from_agents
168 + end
169 +
170 + test "Release spans the architectures and suites of the whole namespace", %{
171 + conn: conn,
172 + tools: tools,
173 + agents: agents,
174 + org: org,
175 + root: root
176 + } do
177 + publish(tools, "widget", arch: "amd64")
178 + publish(agents, "gadget", arch: "arm64")
179 +
180 + release = get(conn, root <> "/dists/stable/Release").resp_body
181 +
182 + assert release =~ ~r/^Origin: #{org.handle}$/m
183 + assert release =~ ~r/^Architectures: amd64 arm64$/m
184 + assert release =~ ~r/^Suite: stable$/m
185 + end
186 +
187 + test "a suite only one repository uses still resolves", %{
188 + conn: conn,
189 + tools: tools,
190 + root: root
191 + } do
192 + publish(tools, "widget", dist: "trixie")
193 +
194 + body = get(conn, root <> "/dists/trixie/main/binary-amd64/Packages").resp_body
195 + assert body =~ "Package: widget"
196 +
197 + assert get(build_conn(), root <> "/dists/stable/main/binary-amd64/Packages").resp_body == ""
198 + end
199 +
200 + test "every path Release lists is served, at the right size and digest", %{
201 + conn: conn,
202 + tools: tools,
203 + agents: agents,
204 + root: root
205 + } do
206 + publish(tools, "widget")
207 + publish(agents, "gadget", arch: "all")
208 +
209 + release = get(conn, root <> "/dists/stable/Release").resp_body
210 +
211 + listed =
212 + Regex.scan(~r/^ ([0-9a-f]{64}) +(\d+) +(\S+)$/m, release)
213 + |> Map.new(fn [_, digest, size, path] -> {path, {digest, String.to_integer(size)}} end)
214 +
215 + assert map_size(listed) > 0
216 +
217 + for {path, {digest, size}} <- listed do
218 + served = get(build_conn(), root <> "/dists/stable/" <> path)
219 +
220 + assert served.status == 200, "#{path} is listed in Release but not served"
221 + assert byte_size(served.resp_body) == size, "#{path} size disagrees with Release"
222 +
223 + assert Base.encode16(:crypto.hash(:sha256, served.resp_body), case: :lower) == digest,
224 + "#{path} checksum disagrees with Release"
225 + end
226 + end
227 + end
228 +
229 + describe "signing" do
230 + test "the aggregate is signed with the same key as each repository", %{
231 + conn: conn,
232 + org: org,
233 + tools: tools,
234 + root: root
235 + } do
236 + publish(tools, "widget")
237 +
238 + key = get(conn, "/pkg/_keys/deb/#{org.handle}").resp_body
239 + release = get(build_conn(), root <> "/dists/stable/Release").resp_body
240 + signature = get(build_conn(), root <> "/dists/stable/Release.gpg").resp_body
241 +
242 + assert {:ok, true} = Native.verify_detached(key, release, signature)
243 +
244 + # …and the per-repository view verifies against the very same key,
245 + # which is the whole point: one keyring entry either way.
246 + repo_root = "/pkg/deb/#{Storage.repo_handle(tools)}/#{tools.name}"
247 + repo_release = get(build_conn(), repo_root <> "/dists/stable/Release").resp_body
248 + repo_sig = get(build_conn(), repo_root <> "/dists/stable/Release.gpg").resp_body
249 +
250 + assert {:ok, true} = Native.verify_detached(key, repo_release, repo_sig)
251 + end
252 +
253 + test "InRelease is clearsigned", %{conn: conn, tools: tools, root: root} do
254 + publish(tools, "widget")
255 +
256 + body = get(conn, root <> "/dists/stable/InRelease").resp_body
257 +
258 + assert body =~ "-----BEGIN PGP SIGNED MESSAGE-----"
259 + assert body =~ "Package" or body =~ "Suite: stable"
260 + end
261 +
262 + test "key.asc under _all is the namespace key", %{conn: conn, org: org, root: root} do
263 + from_all = get(conn, root <> "/key.asc").resp_body
264 +
265 + assert from_all =~ "-----BEGIN PGP PUBLIC KEY BLOCK-----"
266 + assert get(build_conn(), "/pkg/_keys/deb/#{org.handle}").resp_body == from_all
267 + end
268 + end
269 +
270 + describe "visibility" do
271 + test "an anonymous caller sees only the public repositories", %{
272 + conn: conn,
273 + org: org,
274 + user: user,
275 + tools: tools,
276 + root: root
277 + } do
278 + private = org_repo(org, user, "private")
279 +
280 + publish(tools, "public-thing")
281 + publish(private, "secret-thing")
282 +
283 + body = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
284 +
285 + assert body =~ "Package: public-thing"
286 + refute body =~ "Package: secret-thing"
287 + end
288 +
289 + test "a repository's own deploy token still doesn't widen the view", %{
290 + org: org,
291 + user: user,
292 + tools: tools,
293 + root: root
294 + } do
295 + other_private = org_repo(org, user, "private")
296 + publish(tools, "public-thing")
297 + publish(other_private, "secret-thing")
298 +
299 + # A token for a *different* private repo in the same namespace.
300 + {:ok, _t, token} = Packages.create_registry_token(tools, "ci", ["pull"])
301 +
302 + body =
303 + build_conn()
304 + |> put_req_header("authorization", token)
305 + |> get(root <> "/dists/stable/main/binary-amd64/Packages")
306 + |> Map.get(:resp_body)
307 +
308 + assert body =~ "Package: public-thing"
309 + refute body =~ "Package: secret-thing"
310 + end
311 +
312 + test "a private package's pool path 404s for a caller who can't read it", %{
313 + conn: conn,
314 + org: org,
315 + user: user,
316 + root: root
317 + } do
318 + private = org_repo(org, user, "private")
319 + publish(private, "secret-thing")
320 +
321 + path = "pool/main/#{private.name}/s/secret-thing/secret-thing_1.0.0-1_amd64.deb"
322 +
323 + assert get(conn, root <> "/" <> path).status == 404
324 + end
325 +
326 + test "an empty namespace still serves a valid, signed Release", %{conn: conn, root: root} do
327 + release = get(conn, root <> "/dists/stable/Release").resp_body
328 +
329 + assert release =~ ~r/^Components: main$/m
330 + assert release =~ ~r/^SHA256:$/m
331 + assert get(build_conn(), root <> "/dists/stable/InRelease").status == 200
332 + end
333 + end
334 +
335 + describe "user namespaces work the same way" do
336 + setup do
337 + user = user_fixture()
338 + {_u, one} = repository_fixture(%{owner: user, visibility: "public"})
339 + {_u, two} = repository_fixture(%{owner: user, visibility: "public"})
340 +
341 + preload = &GitGud.Repo.preload(&1, [:owner, :organization])
342 +
343 + {:ok,
344 + solo: user,
345 + one: preload.(one),
346 + two: preload.(two),
347 + solo_root: "/pkg/deb/#{Storage.owner_handle(user)}/_all"}
348 + end
349 +
350 + test "one index spans a user's own repositories", %{
351 + conn: conn,
352 + one: one,
353 + two: two,
354 + solo_root: root
355 + } do
356 + publish(one, "alpha")
357 + publish(two, "beta")
358 +
359 + body = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
360 +
361 + assert body =~ "Package: alpha"
362 + assert body =~ "Package: beta"
363 + end
364 +
365 + test "and it is signed by that user's key", %{
366 + conn: conn,
367 + solo: solo,
368 + one: one,
369 + solo_root: root
370 + } do
371 + publish(one, "alpha")
372 +
373 + key = get(conn, "/pkg/_keys/deb/#{Storage.owner_handle(solo)}").resp_body
374 + release = get(build_conn(), root <> "/dists/stable/Release").resp_body
375 + signature = get(build_conn(), root <> "/dists/stable/Release.gpg").resp_body
376 +
377 + assert {:ok, true} = Native.verify_detached(key, release, signature)
378 + end
379 +
380 + test "an org-owned repository is not in its creator's personal namespace", %{
381 + conn: conn,
382 + solo: solo,
383 + solo_root: root
384 + } do
385 + org = org_fixture(solo)
386 + org_owned = org_repo(org, solo)
387 + publish(org_owned, "org-thing")
388 +
389 + body = get(conn, root <> "/dists/stable/main/binary-amd64/Packages").resp_body
390 +
391 + refute body =~ "Package: org-thing"
392 + end
393 + end
394 +
395 + describe "the marker" do
396 + test "publishing to _all is refused — a package needs a repository", %{
397 + conn: conn,
398 + root: root
399 + } do
400 + conn =
401 + conn
402 + |> put_req_header("content-type", "application/octet-stream")
403 + |> post(root <> "/upload", deb(Package: "widget", Version: "1.0"))
404 +
405 + assert conn.status == 405
406 + assert conn.resp_body =~ "publish to a repository"
407 + end
408 +
409 + test "a format with no namespace index 404s there", %{conn: conn, org: org} do
410 + assert get(conn, "/pkg/pypi/#{org.handle}/_all/simple/").status == 404
411 + end
412 +
413 + test "an unknown namespace 404s", %{conn: conn} do
414 + assert get(conn, "/pkg/deb/nobody-at-all/_all/dists/stable/Release").status == 404
415 + end
416 +
417 + test "a repository can never be called _all", %{user: user} do
418 + assert {:error, changeset} =
419 + Repositories.create_repository(user, %{"name" => "_all", "visibility" => "public"})
420 +
421 + assert Keyword.has_key?(changeset.errors, :name)
422 + end
423 + end
424 +end
modified test/git_gud_web/apt_registry_test.exs
+38 −7
@@ -265,6 +265,7 @@ defmodule GitGudWeb.AptRegistryTest do
265 265 describe "Packages" do
266 266 test "carries what a stanza requires, Description-md5 included", %{
267 267 conn: conn,
268 + repo: repo,
268 269 root: root,
269 270 token: token
270 271 } do
@@ -275,7 +276,12 @@ defmodule GitGudWeb.AptRegistryTest do
275 276 assert body =~ ~r/^Package: widget$/m
276 277 assert body =~ ~r/^Version: 1\.0\.0-1$/m
277 278 assert body =~ ~r/^Architecture: amd64$/m
278 − assert body =~ ~r/^Filename: pool\/main\/w\/widget\/widget_1\.0\.0-1_amd64\.deb$/m
279 +
280 + # The pool path is qualified by the source repository so that a
281 + # namespace index aggregating several can't collide.
282 + assert body =~
283 + "Filename: pool/main/#{repo.name}/w/widget/widget_1.0.0-1_amd64.deb\n"
284 +
279 285 assert body =~ ~r/^Size: \d+$/m
280 286 assert body =~ ~r/^SHA256: [0-9a-f]{64}$/m
281 287 # Mandatory as of APT 2.7.7.
@@ -284,6 +290,7 @@ defmodule GitGudWeb.AptRegistryTest do
284 290
285 291 test "a lib package gets its own two-character pool shard", %{
286 292 conn: conn,
293 + repo: repo,
287 294 root: root,
288 295 token: token
289 296 } do
@@ -291,7 +298,8 @@ defmodule GitGudWeb.AptRegistryTest do
291 298
292 299 body = get(build_conn(), root <> "/dists/stable/main/binary-amd64/Packages").resp_body
293 300
294 − assert body =~ "Filename: pool/main/libw/libwidget/libwidget_1.0.0-1_amd64.deb"
301 + assert body =~
302 + "Filename: pool/main/#{repo.name}/libw/libwidget/libwidget_1.0.0-1_amd64.deb"
295 303 end
296 304
297 305 test "arch:all appears in every architecture's index", %{
@@ -337,7 +345,12 @@ defmodule GitGudWeb.AptRegistryTest do
337 345 end
338 346
339 347 describe "pool" do
340 − test "serves the .deb bytes", %{conn: conn, root: root, token: token} do
348 + test "serves the .deb bytes at exactly the path the index names", %{
349 + conn: conn,
350 + repo: repo,
351 + root: root,
352 + token: token
353 + } do
341 354 body = deb(control_fields())
342 355
343 356 conn
@@ -345,15 +358,33 @@ defmodule GitGudWeb.AptRegistryTest do
345 358 |> put_req_header("content-type", "application/octet-stream")
346 359 |> post(root <> "/upload", body)
347 360
348 − served =
349 − get(build_conn(), root <> "/pool/main/w/widget/widget_1.0.0-1_amd64.deb")
361 + # Take the path out of the index rather than reconstructing it —
362 + # a Filename apt can't fetch is the failure that matters.
363 + index = get(build_conn(), root <> "/dists/stable/main/binary-amd64/Packages").resp_body
364 + [_, filename] = Regex.run(~r/^Filename: (\S+)$/m, index)
365 +
366 + assert filename == "pool/main/#{repo.name}/w/widget/widget_1.0.0-1_amd64.deb"
367 +
368 + served = get(build_conn(), root <> "/" <> filename)
350 369
351 370 assert served.status == 200
352 371 assert served.resp_body == body
353 372 end
354 373
355 − test "404 for a .deb nobody uploaded", %{conn: conn, root: root} do
356 − assert get(conn, root <> "/pool/main/g/ghost/ghost_1.0_amd64.deb").status == 404
374 + test "404 for a .deb nobody uploaded", %{conn: conn, repo: repo, root: root} do
375 + assert get(conn, root <> "/pool/main/#{repo.name}/g/ghost/ghost_1.0_amd64.deb").status ==
376 + 404
377 + end
378 +
379 + test "404 for a path that names another repository's pool", %{
380 + conn: conn,
381 + root: root,
382 + token: token
383 + } do
384 + upload(conn, root, token, control_fields())
385 +
386 + assert get(build_conn(), root <> "/pool/main/elsewhere/w/widget/widget_1.0.0-1_amd64.deb").status ==
387 + 404
357 388 end
358 389 end
359 390

Parents: aab4e3f