Package registries phase 1: PyPI

ed33862 · Gabriel Morell · 2026-09-11 14:16

16 files +1078 -98
Message
{commit_body(@commit)}

Files changed

modified lib/git_gud/packages/format.ex
+1 −1
@@ -45,7 +45,7 @@ defmodule GitGud.Packages.Format do
45 45 # distribution API itself, so `oci` exists only as a `format` value
46 46 # the packages UI can list alongside everything else. Each remaining
47 47 # format appears here as its adapter lands.
48 − @default_adapters %{}
48 + @default_adapters %{"pypi" => GitGud.Packages.Formats.Pypi}
49 49
50 50 @typedoc """
51 51 Request context handed to an adapter.
added lib/git_gud/packages/formats/pypi.ex
+473 −0
@@ -0,0 +1,473 @@
1 +defmodule GitGud.Packages.Formats.Pypi do
2 + @moduledoc """
3 + A PEP 503 / PEP 691 Simple Repository, per git repository.
4 +
5 + pip install --index-url https://<host>/pkg/pypi/<owner>/<repo>/simple/ widget
6 + twine upload --repository-url https://<host>/pkg/pypi/<owner>/<repo>/ dist/*
7 +
8 + Paths below the repository root:
9 +
10 + / twine's upload target (POST)
11 + /simple/ the project list
12 + /simple/<name>/ a project's files
13 + /files/<filename> an artifact's bytes
14 +
15 + ## The parts that bite
16 +
17 + **Normalisation.** PEP 503 defines exactly one canonical form,
18 + `re.sub(r"[-_.]+", "-", name).lower()`, and it must be applied before
19 + *any* lookup — `Flask_Login`, `flask.login` and `FLASK--LOGIN` are
20 + one project. Clients request the normalised URL directly; we also
21 + redirect an unnormalised one, which the PEP permits and humans
22 + appreciate.
23 +
24 + **Trailing slashes are load-bearing.** Every index URL ends in `/`,
25 + and an unslashed one must redirect rather than render — relative
26 + hrefs resolve against the wrong base otherwise. Plug drops the
27 + trailing slash when it splits the path, so the controller passes the
28 + raw fact through as `:trailing_slash?`.
29 +
30 + **Two serialisations, one data structure.** PEP 691 adds a JSON
31 + rendering of exactly the same information, negotiated on `Accept`:
32 +
33 + application/vnd.pypi.simple.v1+json
34 + application/vnd.pypi.simple.v1+html
35 + text/html (legacy alias for v1+html)
36 +
37 + with `406` when nothing matches. Both come off `project_files/1`
38 + here; only the rendering differs.
39 + """
40 +
41 + @behaviour GitGud.Packages.Format
42 +
43 + alias GitGud.Packages
44 + alias GitGud.Packages.Package
45 + alias GitGud.Packages.PackageFile
46 +
47 + @format "pypi"
48 + @api_version "1.0"
49 +
50 + @json "application/vnd.pypi.simple.v1+json"
51 + @html "application/vnd.pypi.simple.v1+html"
52 + @legacy_html "text/html"
53 +
54 + # Most preferred last-resort first: a client that sends `*/*` (curl,
55 + # a browser) gets HTML, which is the legacy behaviour and readable.
56 + @serialisations [@json, @html, @legacy_html]
57 +
58 + @doc """
59 + PEP 503 name normalisation, which is the whole of it:
60 +
61 + re.sub(r"[-_.]+", "-", name).lower()
62 + """
63 + @impl true
64 + def normalize(name) when is_binary(name) do
65 + name
66 + |> String.replace(~r/[-_.]+/, "-")
67 + |> String.downcase()
68 + end
69 +
70 + @impl true
71 + def content_type(filename) do
72 + cond do
73 + String.ends_with?(filename, ".whl") -> "application/zip"
74 + String.ends_with?(filename, ".zip") -> "application/zip"
75 + String.ends_with?(filename, ".egg") -> "application/zip"
76 + String.ends_with?(filename, ".gz") -> "application/gzip"
77 + String.ends_with?(filename, ".bz2") -> "application/x-bzip2"
78 + true -> "application/octet-stream"
79 + end
80 + end
81 +
82 + # ── reads ───────────────────────────────────────────────────────────
83 +
84 + @impl true
85 + def index(repo, ctx)
86 +
87 + # The repository root is twine's upload target, not a page. Send a
88 + # human (or a curious client) on to the index.
89 + def index(_repo, %{path: []} = ctx), do: {:redirect, ctx.base_url <> "/simple/"}
90 +
91 + def index(_repo, %{path: ["simple"], trailing_slash?: false} = ctx),
92 + do: {:redirect, ctx.base_url <> "/simple/"}
93 +
94 + def index(repo, %{path: ["simple"]} = ctx) do
95 + projects = Packages.list_packages(repo, @format)
96 +
97 + serve(ctx, fn
98 + :json -> root_json(projects)
99 + :html -> root_html(projects)
100 + end)
101 + end
102 +
103 + def index(repo, %{path: ["simple", name]} = ctx) do
104 + normalized = normalize(name)
105 +
106 + cond do
107 + # PEP 503: a server *may* redirect an unnormalised name. Doing so
108 + # means a hand-typed URL works and a client's cache key stays the
109 + # canonical one.
110 + normalized != name ->
111 + {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"}
112 +
113 + not ctx.trailing_slash? ->
114 + {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"}
115 +
116 + true ->
117 + case Packages.get_package(repo, @format, normalized) do
118 + nil ->
119 + {:error, :not_found}
120 +
121 + %Package{} = package ->
122 + files = project_files(package, ctx.base_url)
123 +
124 + serve(ctx, fn
125 + :json -> project_json(normalized, files)
126 + :html -> project_html(normalized, files)
127 + end)
128 + end
129 + end
130 + end
131 +
132 + def index(repo, %{path: ["files", filename]}) do
133 + # A filename is unique within a project but the download URL
134 + # doesn't name the project, so this searches the repository's whole
135 + # pypi namespace. Distribution filenames embed the project name, so
136 + # collisions across projects don't happen in practice.
137 + case find_file(repo, filename) do
138 + %PackageFile{} = file -> {:blob, file}
139 + nil -> {:error, :not_found}
140 + end
141 + end
142 +
143 + def index(_repo, _ctx), do: {:error, :not_found}
144 +
145 + defp find_file(repo, filename) do
146 + repo
147 + |> Packages.list_packages(@format)
148 + |> Enum.find_value(fn package -> Packages.get_package_file(package, filename) end)
149 + end
150 +
151 + # ── the one data structure both serialisations render ───────────────
152 +
153 + defp project_files(%Package{} = package, base_url) do
154 + package
155 + |> Packages.list_package_files()
156 + |> Enum.map(fn %PackageFile{} = file ->
157 + meta = file.metadata || %{}
158 +
159 + %{
160 + filename: file.filename,
161 + url: "#{base_url}/files/#{URI.encode(file.filename)}#sha256=#{file.sha256}",
162 + sha256: file.sha256,
163 + requires_python: meta["requires_python"],
164 + # PEP 592: `false` for a live file, the *reason string* for a
165 + # yanked one — and an empty reason is still a yank, so this is
166 + # deliberately not `reason || false`.
167 + yanked:
168 + if(file.package_version.yanked,
169 + do: file.package_version.yanked_reason || true,
170 + else: false
171 + )
172 + }
173 + end)
174 + end
175 +
176 + # ── content negotiation ─────────────────────────────────────────────
177 +
178 + defp serve(ctx, render) do
179 + case negotiate(ctx.conn) do
180 + {:ok, @json} -> {:ok, @json, Jason.encode_to_iodata!(render.(:json))}
181 + {:ok, media} -> {:ok, media, render.(:html)}
182 + :error -> {:error, :not_acceptable}
183 + end
184 + end
185 +
186 + @doc """
187 + Pick a serialisation for an `Accept` header, or `:error` for 406.
188 +
189 + A request with no `Accept` at all is answered as though it asked for
190 + `text/html` — that is what every pre-PEP-691 client did, and what
191 + PEP 691 tells servers to assume.
192 + """
193 + def negotiate(conn) do
194 + case Plug.Conn.get_req_header(conn, "accept") do
195 + [] ->
196 + {:ok, @legacy_html}
197 +
198 + headers ->
199 + headers
200 + |> Enum.join(",")
201 + |> parse_accept()
202 + |> best_match()
203 + end
204 + end
205 +
206 + # `media/type;q=0.5` → {media_range, q}, highest q first. Ties keep
207 + # the order the client wrote them in, which is the convention for
208 + # equal-quality ranges.
209 + defp parse_accept(header) do
210 + header
211 + |> String.split(",")
212 + |> Enum.map(&String.trim/1)
213 + |> Enum.reject(&(&1 == ""))
214 + |> Enum.with_index()
215 + |> Enum.map(fn {entry, index} ->
216 + [range | params] = String.split(entry, ";")
217 + {String.trim(range) |> String.downcase(), quality(params), index}
218 + end)
219 + |> Enum.sort_by(fn {_range, q, index} -> {-q, index} end)
220 + end
221 +
222 + defp quality(params) do
223 + Enum.find_value(params, 1.0, fn param ->
224 + case param |> String.trim() |> String.split("=", parts: 2) do
225 + ["q", value] ->
226 + case Float.parse(String.trim(value)) do
227 + {q, _} -> q
228 + :error -> 1.0
229 + end
230 +
231 + _ ->
232 + nil
233 + end
234 + end)
235 + end
236 +
237 + defp best_match(ranges) do
238 + # `q=0` is an explicit refusal, not a weak preference.
239 + ranges
240 + |> Enum.reject(fn {_range, q, _} -> q == 0.0 end)
241 + |> Enum.find_value(:error, fn {range, _q, _} ->
242 + case Enum.find(@serialisations, &matches?(range, &1)) do
243 + nil -> nil
244 + # `text/html` is only an alias for the HTML serialisation; the
245 + # response still advertises what it actually is.
246 + @legacy_html -> {:ok, @legacy_html}
247 + media -> {:ok, media}
248 + end
249 + end)
250 + end
251 +
252 + defp matches?("*/*", _media), do: true
253 +
254 + defp matches?(range, media) do
255 + case String.split(range, "/", parts: 2) do
256 + [type, "*"] -> String.starts_with?(media, type <> "/")
257 + _ -> range == media
258 + end
259 + end
260 +
261 + # ── PEP 691 (JSON) ──────────────────────────────────────────────────
262 +
263 + defp root_json(projects) do
264 + %{
265 + "meta" => %{"api-version" => @api_version},
266 + "projects" => Enum.map(projects, &%{"name" => &1.normalized_name})
267 + }
268 + end
269 +
270 + defp project_json(name, files) do
271 + %{
272 + "meta" => %{"api-version" => @api_version},
273 + "name" => name,
274 + "files" => Enum.map(files, &file_json/1)
275 + }
276 + end
277 +
278 + defp file_json(file) do
279 + # `hashes` is required even when empty — a client may not assume
280 + # the key is absent, so this is never conditionally built.
281 + base = %{
282 + "filename" => file.filename,
283 + "url" => file.url,
284 + "hashes" => %{"sha256" => file.sha256},
285 + "yanked" => file.yanked
286 + }
287 +
288 + case file.requires_python do
289 + nil -> base
290 + "" -> base
291 + spec -> Map.put(base, "requires-python", spec)
292 + end
293 + end
294 +
295 + # ── PEP 503 (HTML) ──────────────────────────────────────────────────
296 +
297 + defp root_html(projects) do
298 + links =
299 + Enum.map(projects, fn project ->
300 + ~s( <a href="#{esc(project.normalized_name)}/">#{esc(project.normalized_name)}</a><br />\n)
301 + end)
302 +
303 + page("Simple index", links)
304 + end
305 +
306 + defp project_html(name, files) do
307 + links =
308 + Enum.map(files, fn file ->
309 + ~s( <a href="#{esc(file.url)}"#{html_attrs(file)}>#{esc(file.filename)}</a><br />\n)
310 + end)
311 +
312 + page("Links for #{name}", [~s( <h1>Links for #{esc(name)}</h1>\n) | links])
313 + end
314 +
315 + defp html_attrs(file) do
316 + [
317 + requires_python_attr(file.requires_python),
318 + yanked_attr(file.yanked)
319 + ]
320 + |> Enum.reject(&is_nil/1)
321 + |> Enum.join()
322 + end
323 +
324 + defp requires_python_attr(nil), do: nil
325 + defp requires_python_attr(""), do: nil
326 + defp requires_python_attr(spec), do: ~s( data-requires-python="#{esc(spec)}")
327 +
328 + defp yanked_attr(false), do: nil
329 + defp yanked_attr(true), do: ~s( data-yanked="")
330 + defp yanked_attr(reason), do: ~s( data-yanked="#{esc(reason)}")
331 +
332 + defp page(title, body) do
333 + [
334 + "<!DOCTYPE html>\n<html>\n <head>\n",
335 + " <meta name=\"pypi:repository-version\" content=\"#{@api_version}\" />\n",
336 + " <title>#{esc(title)}</title>\n </head>\n <body>\n",
337 + body,
338 + " </body>\n</html>\n"
339 + ]
340 + end
341 +
342 + # PEP 503 is served to a browser as well as to pip, and a project
343 + # name reaches us from an upload form.
344 + defp esc(value) do
345 + value
346 + |> to_string()
347 + |> String.replace("&", "&amp;")
348 + |> String.replace("<", "&lt;")
349 + |> String.replace(">", "&gt;")
350 + |> String.replace("\"", "&quot;")
351 + end
352 +
353 + # ── uploads ─────────────────────────────────────────────────────────
354 +
355 + @impl true
356 + def store(repo, ctx, params)
357 +
358 + def store(repo, %{method: "POST"} = ctx, %{} = params) do
359 + with {:ok, name} <- require_field(params, "name"),
360 + {:ok, version} <- require_field(params, "version"),
361 + {:ok, upload} <- require_upload(params),
362 + {:ok, body} <- read_upload(upload),
363 + :ok <- verify_digest(params["sha256_digest"], body) do
364 + attrs = %{
365 + filename: Path.basename(upload.filename),
366 + body: body,
367 + content_type: content_type(upload.filename),
368 + metadata: file_metadata(params),
369 + version_metadata: version_metadata(params),
370 + uploader: uploader(ctx)
371 + }
372 +
373 + case Packages.publish_file(repo, @format, name, version, attrs) do
374 + # twine reads nothing but the status. An empty 200 is what
375 + # warehouse sends.
376 + {:ok, _package, _version, _file} -> {:ok, "text/plain; charset=utf-8", ""}
377 + {:error, :file_exists} -> {:error, :file_exists}
378 + {:error, reason} -> {:error, changeset_message(reason)}
379 + end
380 + end
381 + end
382 +
383 + def store(_repo, %{method: "POST"}, _body),
384 + do: {:error, {400, "expected a multipart/form-data upload"}}
385 +
386 + def store(_repo, %{method: method}, _params),
387 + do: {:error, {405, "#{method} is not supported by the PyPI upload API"}}
388 +
389 + defp require_field(params, key) do
390 + case params[key] do
391 + value when is_binary(value) and value != "" -> {:ok, value}
392 + _ -> {:error, {400, "missing required form field `#{key}`"}}
393 + end
394 + end
395 +
396 + defp require_upload(params) do
397 + case params["content"] do
398 + %Plug.Upload{} = upload -> {:ok, upload}
399 + _ -> {:error, {400, "missing the `content` file part"}}
400 + end
401 + end
402 +
403 + defp read_upload(%Plug.Upload{path: path}) do
404 + case File.read(path) do
405 + {:ok, body} -> {:ok, body}
406 + {:error, reason} -> {:error, {400, "could not read the upload: #{inspect(reason)}"}}
407 + end
408 + end
409 +
410 + # twine computes this itself and sends it alongside. Checking it
411 + # costs one hash and turns a truncated upload into a clear error
412 + # instead of a corrupt wheel nobody notices until install time.
413 + defp verify_digest(nil, _body), do: :ok
414 + defp verify_digest("", _body), do: :ok
415 +
416 + defp verify_digest(expected, body) do
417 + actual = Base.encode16(:crypto.hash(:sha256, body), case: :lower)
418 +
419 + if String.downcase(expected) == actual,
420 + do: :ok,
421 + else: {:error, {400, "sha256_digest does not match the uploaded bytes"}}
422 + end
423 +
424 + defp file_metadata(params) do
425 + %{
426 + "requires_python" => blank_to_nil(params["requires_python"]),
427 + "filetype" => blank_to_nil(params["filetype"]),
428 + "pyversion" => blank_to_nil(params["pyversion"])
429 + }
430 + |> Enum.reject(fn {_k, v} -> is_nil(v) end)
431 + |> Map.new()
432 + end
433 +
434 + # Core metadata twine sends along with the distribution, kept so the
435 + # packages UI can describe a release without unzipping a wheel.
436 + defp version_metadata(params) do
437 + %{
438 + "summary" => blank_to_nil(params["summary"]),
439 + "metadata_version" => blank_to_nil(params["metadata_version"]),
440 + "requires_python" => blank_to_nil(params["requires_python"]),
441 + "home_page" => blank_to_nil(params["home_page"]),
442 + "license" => blank_to_nil(params["license"])
443 + }
444 + |> Enum.reject(fn {_k, v} -> is_nil(v) end)
445 + |> Map.new()
446 + end
447 +
448 + defp blank_to_nil(value) when is_binary(value) do
449 + case String.trim(value) do
450 + "" -> nil
451 + trimmed -> trimmed
452 + end
453 + end
454 +
455 + defp blank_to_nil(_), do: nil
456 +
457 + defp uploader(%{principal: {:user, user}}), do: user
458 + defp uploader(_ctx), do: nil
459 +
460 + defp changeset_message(%Ecto.Changeset{} = changeset) do
461 + errors =
462 + Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} ->
463 + Enum.reduce(opts, msg, fn {k, v}, acc ->
464 + String.replace(acc, "%{#{k}}", to_string(v))
465 + end)
466 + end)
467 +
468 + {400,
469 + Enum.map_join(errors, "; ", fn {field, msgs} -> "#{field} #{Enum.join(msgs, ", ")}" end)}
470 + end
471 +
472 + defp changeset_message(reason), do: {400, inspect(reason)}
473 +end
modified lib/git_gud_web/endpoint.ex
+4 −0
@@ -49,6 +49,10 @@ defmodule GitGudWeb.Endpoint do
49 49 # consumes the raw request body itself.
50 50 plug GitGudWeb.Plugs.GitRouter
51 51
52 + # Package-registry uploads, parsed with their own (much larger) size
53 + # limit before the general parser below claims the body.
54 + plug GitGudWeb.Plugs.PackageUploadParser
55 +
52 56 plug Plug.Parsers,
53 57 parsers: [:urlencoded, :multipart, :json],
54 58 pass: ["*/*"],
modified lib/git_gud_web/live/repo_live/packages.ex
+150 −97
@@ -1,9 +1,14 @@
1 1 defmodule GitGudWeb.RepoLive.Packages do
2 2 @moduledoc """
3 − Lists container/package versions mirrored from zot for a repository.
4 −
5 − This is a UI over the data zot pushes through `/_webhooks/zot`.
6 − Click a package row to see all of its versions inline.
3 + Everything a repository has published that you can actually install,
4 + grouped by registry format.
5 +
6 + Container images are mirrored from zot via `/_webhooks/zot`;
7 + everything else was uploaded to this forge's own registries under
8 + `/pkg/:format/:owner/:name`. Each format carries the line you paste
9 + to point a client at it, and each version the line you paste to
10 + install that version — the page is only useful if you can copy from
11 + it without going to look anything up.
7 12 """
8 13
9 14 use GitGudWeb, :live_view
@@ -12,59 +17,78 @@ defmodule GitGudWeb.RepoLive.Packages do
12 17 alias GitGud.Repositories
13 18 alias GitGud.Repositories.Storage
14 19
20 + # Display order: containers first (the oldest surface here), then the
21 + # language registries in the order they landed.
22 + @format_order ~w(oci pypi hex cargo deb)
23 +
24 + @format_labels %{
25 + "oci" => "Container images",
26 + "pypi" => "Python",
27 + "hex" => "Hex",
28 + "cargo" => "Crates",
29 + "deb" => "Debian packages"
30 + }
31 +
15 32 @impl true
16 33 def mount(%{"owner" => owner, "name" => name}, _session, socket) do
17 34 repo =
18 35 Repositories.get_repository_by_path!(owner, name)
19 36 |> GitGud.Repo.preload([:owner, :organization])
20 37
21 − packages = Packages.list_packages(repo)
38 + handle = Storage.repo_handle(repo)
22 39
23 40 {:ok,
24 41 socket
25 42 |> assign(:repo, repo)
26 − |> assign(:handle, Storage.repo_handle(repo))
43 + |> assign(:handle, handle)
27 44 |> GitGudWeb.RepoLive.Header.assign_chrome(repo)
28 − |> assign(:packages, packages)
45 + |> assign(:groups, grouped_packages(repo))
29 46 |> assign(:expanded_id, nil)
30 47 |> assign(:versions_by_id, %{})
31 − |> assign(:registry_host, Application.get_env(:git_gud, :registry_host))
48 + |> assign(:registry, %{
49 + handle: handle,
50 + repo_name: repo.name,
51 + # The OCI registry is a separate host (zot); everything else is
52 + # served by this forge.
53 + host: Application.get_env(:git_gud, :registry_host),
54 + root: registry_root()
55 + })
32 56 |> assign(:page_title, "Packages — #{owner}/#{name}")}
33 57 end
34 58
35 − # Reconstruct the zot repository path the webhook deflated: a package
36 − # named after the repo was pushed as `<owner>/<repo>`; a sub-image as
37 − # `<owner>/<repo>/<image>`. (See PackagesWebhookController.)
38 − defp pull_ref(handle, repo, pkg, version, host) do
39 − base =
40 − if pkg.name == repo.name,
41 − do: "#{handle}/#{repo.name}",
42 − else: "#{handle}/#{repo.name}/#{pkg.name}"
59 + defp grouped_packages(repo) do
60 + repo
61 + |> Packages.list_packages()
62 + |> Enum.group_by(& &1.format)
63 + |> Enum.sort_by(fn {format, _} -> Enum.find_index(@format_order, &(&1 == format)) || 99 end)
64 + end
43 65
44 − "#{host}/#{base}:#{version}"
66 + defp registry_root do
67 + (Application.get_env(:git_gud, :package_registry_url) || GitGudWeb.Endpoint.url())
68 + |> String.trim_trailing("/")
45 69 end
46 70
47 71 @impl true
48 72 def handle_event("toggle", %{"id" => id}, socket) do
49 73 id = String.to_integer(id)
50 74
51 − cond do
52 − socket.assigns.expanded_id == id ->
53 − {:noreply, assign(socket, :expanded_id, nil)}
54 −
55 − true ->
56 − package = Enum.find(socket.assigns.packages, &(&1.id == id))
57 −
58 − versions =
59 − case Map.get(socket.assigns.versions_by_id, id) do
60 − nil -> Packages.list_versions(package)
61 − cached -> cached
62 − end
63 −
64 − {:noreply,
65 − socket
66 − |> assign(:expanded_id, id)
67 − |> assign(:versions_by_id, Map.put(socket.assigns.versions_by_id, id, versions))}
75 + if socket.assigns.expanded_id == id do
76 + {:noreply, assign(socket, :expanded_id, nil)}
77 + else
78 + package =
79 + socket.assigns.groups
80 + |> Enum.flat_map(fn {_format, packages} -> packages end)
81 + |> Enum.find(&(&1.id == id))
82 +
83 + versions =
84 + Map.get_lazy(socket.assigns.versions_by_id, id, fn ->
85 + Packages.list_versions(package)
86 + end)
87 +
88 + {:noreply,
89 + socket
90 + |> assign(:expanded_id, id)
91 + |> assign(:versions_by_id, Map.put(socket.assigns.versions_by_id, id, versions))}
68 92 end
69 93 end
70 94
@@ -72,7 +96,7 @@ defmodule GitGudWeb.RepoLive.Packages do
72 96 def render(assigns) do
73 97 ~H"""
74 98 <Layouts.app flash={@flash} current_scope={@current_scope}>
75 − <div class="space-y-4">
99 + <div class="space-y-6">
76 100 <GitGudWeb.RepoLive.Header.header
77 101 repo={@repo}
78 102 handle={@handle}
@@ -85,84 +109,113 @@ defmodule GitGudWeb.RepoLive.Packages do
85 109 <header>
86 110 <h1 class="text-xl font-semibold">Packages</h1>
87 111 <p class="text-xs opacity-70 mt-1">
88 − Mirrored from the OCI registry. Authoritative storage lives in zot;
89 − this view tracks pushes via webhook so you can see what's been
90 − published without leaving the forge.
112 + Artifacts published from this repository, by format. Container
113 + images live in the OCI registry; everything else is served from
114 + this forge under <code class="font-mono">/pkg</code>.
91 115 </p>
92 116 </header>
93 117
94 − <ul
95 − id="packages"
96 − class="divide-y divide-base-300 border border-base-300 rounded"
118 + <p
119 + :if={@groups == []}
120 + class="px-3 py-6 text-sm opacity-60 text-center border border-base-300 rounded"
97 121 >
98 − <li :if={@packages == []} class="px-3 py-6 text-sm opacity-60 text-center">
99 − No packages published yet.
100 − </li>
101 −
102 − <li :for={pkg <- @packages} class="px-3 py-2">
103 − <button
104 − type="button"
105 − phx-click="toggle"
106 − phx-value-id={pkg.id}
107 − class="w-full flex items-baseline justify-between gap-3 text-left"
108 − >
109 − <div>
110 − <p class="font-mono text-sm">{pkg.name}</p>
111 − <p class="text-xs opacity-60">
112 − <span class="badge badge-xs">{pkg.kind}</span>
113 − <span :if={pkg.latest_version} class="ml-1">
122 + Nothing published yet.
123 + </p>
124 +
125 + <section :for={{format, packages} <- @groups} class="space-y-2">
126 + <div class="flex items-baseline justify-between gap-3">
127 + <h2 class="text-sm font-semibold">{format_label(format)}</h2>
128 + <span class="text-xs opacity-50">{length(packages)}</span>
129 + </div>
130 +
131 + <code
132 + :if={configure_line(format, @registry)}
133 + class="block bg-base-200 rounded px-2 py-1 select-all text-[11px] overflow-x-auto"
134 + >
135 + {configure_line(format, @registry)}
136 + </code>
137 +
138 + <ul class="divide-y divide-base-300 border border-base-300 rounded">
139 + <li :for={pkg <- packages} class="px-3 py-2">
140 + <button
141 + type="button"
142 + phx-click="toggle"
143 + phx-value-id={pkg.id}
144 + class="w-full flex items-baseline justify-between gap-3 text-left"
145 + >
146 + <div>
147 + <p class="font-mono text-sm">{pkg.name}</p>
148 + <p :if={pkg.latest_version} class="text-xs opacity-60">
114 149 latest: <span class="font-mono">{pkg.latest_version}</span>
115 − </span>
116 − </p>
117 − </div>
118 − <.icon
119 − name={if @expanded_id == pkg.id, do: "hero-chevron-up", else: "hero-chevron-down"}
120 − class="size-4 opacity-50"
121 − />
122 − </button>
123 −
124 − <ul
125 − :if={@expanded_id == pkg.id}
126 − class="mt-2 ml-4 border-l border-base-300 pl-3 space-y-1 text-xs"
127 − >
128 − <li :for={v <- Map.get(@versions_by_id, pkg.id, [])} class="space-y-1">
129 − <div class="flex items-baseline justify-between gap-2">
130 − <span class="font-mono">{v.version}</span>
131 − <span class="opacity-60 truncate">{short_digest(v.digest)}</span>
132 − <span class="opacity-50">{format_size(v.size)}</span>
133 − <span class="opacity-50">{format_dt(v.pushed_at)}</span>
150 + </p>
134 151 </div>
135 − <code
136 − :if={@registry_host}
137 − class="block bg-base-200 rounded px-2 py-1 select-all text-[11px]"
138 − >
139 − podman pull {pull_ref(@handle, @repo, pkg, v.version, @registry_host)}
140 − </code>
141 − </li>
142 − <li :if={Map.get(@versions_by_id, pkg.id, []) == []} class="opacity-60">
143 − No versions.
144 − </li>
145 − </ul>
146 − </li>
147 − </ul>
152 + <.icon
153 + name={if @expanded_id == pkg.id, do: "hero-chevron-up", else: "hero-chevron-down"}
154 + class="size-4 opacity-50"
155 + />
156 + </button>
157 +
158 + <ul
159 + :if={@expanded_id == pkg.id}
160 + class="mt-2 ml-4 border-l border-base-300 pl-3 space-y-2 text-xs"
161 + >
162 + <li :for={v <- Map.get(@versions_by_id, pkg.id, [])} class="space-y-1">
163 + <div class="flex items-baseline justify-between gap-2">
164 + <span class="font-mono">
165 + {v.version}
166 + <span :if={v.yanked} class="badge badge-xs badge-warning ml-1">yanked</span>
167 + </span>
168 + <span :if={v.digest} class="opacity-60 truncate">{short_digest(v.digest)}</span>
169 + <span class="opacity-50">{format_dt(v.pushed_at)}</span>
170 + </div>
171 + <code
172 + :if={install_line(format, @registry, pkg, v.version)}
173 + class="block bg-base-200 rounded px-2 py-1 select-all text-[11px] overflow-x-auto"
174 + >
175 + {install_line(format, @registry, pkg, v.version)}
176 + </code>
177 + </li>
178 + <li :if={Map.get(@versions_by_id, pkg.id, []) == []} class="opacity-60">
179 + No versions.
180 + </li>
181 + </ul>
182 + </li>
183 + </ul>
184 + </section>
148 185 </div>
149 186 </Layouts.app>
150 187 """
151 188 end
152 189
153 − defp short_digest(nil), do: ""
190 + defp format_label(format), do: Map.get(@format_labels, format, format)
191 +
192 + # The line that points a client at this repository's registry. OCI
193 + # has none — the pull reference carries the host itself.
194 + defp configure_line("pypi", %{handle: handle, repo_name: name, root: root}),
195 + do: "pip config set global.index-url #{root}/pkg/pypi/#{handle}/#{name}/simple/"
196 +
197 + defp configure_line(_format, _registry), do: nil
198 +
199 + # Reconstruct the zot repository path the webhook deflated: a package
200 + # named after the repo was pushed as `<owner>/<repo>`; a sub-image as
201 + # `<owner>/<repo>/<image>`. (See PackagesWebhookController.)
202 + defp install_line("oci", %{handle: handle, repo_name: name, host: host}, pkg, version)
203 + when is_binary(host) do
204 + base = if pkg.name == name, do: "#{handle}/#{name}", else: "#{handle}/#{name}/#{pkg.name}"
205 +
206 + "podman pull #{host}/#{base}:#{version}"
207 + end
208 +
209 + defp install_line("pypi", %{handle: handle, repo_name: name, root: root}, pkg, version) do
210 + "pip install --index-url #{root}/pkg/pypi/#{handle}/#{name}/simple/ #{pkg.name}==#{version}"
211 + end
212 +
213 + defp install_line(_format, _registry, _pkg, _version), do: nil
154 214
155 215 defp short_digest("sha256:" <> rest),
156 216 do: "sha256:" <> binary_part(rest, 0, min(12, byte_size(rest)))
157 217
158 218 defp short_digest(d) when is_binary(d), do: binary_part(d, 0, min(20, byte_size(d)))
159 − defp short_digest(_), do: ""
160 −
161 − defp format_size(nil), do: ""
162 − defp format_size(0), do: ""
163 − defp format_size(b) when b < 1024, do: "#{b} B"
164 − defp format_size(b) when b < 1_048_576, do: "#{Float.round(b / 1024, 1)} KB"
165 − defp format_size(b), do: "#{Float.round(b / 1_048_576, 1)} MB"
166 219
167 220 defp format_dt(nil), do: ""
168 221 defp format_dt(dt), do: Calendar.strftime(dt, "%Y-%m-%d %H:%M")
added lib/git_gud_web/plugs/package_upload_parser.ex
+38 −0
@@ -0,0 +1,38 @@
1 +defmodule GitGudWeb.Plugs.PackageUploadParser do
2 + @moduledoc """
3 + Parses package-registry upload bodies before the endpoint's own
4 + `Plug.Parsers` gets to them, so publishing a fat wheel isn't capped
5 + by the 8 MB limit that suits ordinary form posts.
6 +
7 + Only multipart is handled here — that is twine's framing. Cargo's
8 + length-prefixed `application/octet-stream` and Hex's raw tarball
9 + fall through `pass: ["*/*"]` untouched and are read off the socket
10 + by `GitGudWeb.PackageRegistryController`.
11 +
12 + `Plug.Parsers` is a no-op once `body_params` has been fetched, so
13 + the endpoint's instance leaves whatever this one did alone.
14 + """
15 +
16 + @behaviour Plug
17 +
18 + # Comfortably past a manylinux wheel with a bundled CUDA runtime.
19 + # Anything genuinely larger belongs in LFS or the OCI registry.
20 + @max_upload_bytes 200_000_000
21 +
22 + @parser_opts Plug.Parsers.init(
23 + parsers: [:multipart],
24 + pass: ["*/*"],
25 + length: @max_upload_bytes
26 + )
27 +
28 + @impl true
29 + def init(opts), do: opts
30 +
31 + @impl true
32 + def call(%Plug.Conn{path_info: ["pkg" | _], method: method} = conn, _opts)
33 + when method in ["POST", "PUT", "PATCH"] do
34 + Plug.Parsers.call(conn, @parser_opts)
35 + end
36 +
37 + def call(conn, _opts), do: conn
38 +end
added test/git_gud_web/pypi_registry_test.exs
+412 −0
@@ -0,0 +1,412 @@
1 +defmodule GitGudWeb.PypiRegistryTest do
2 + @moduledoc """
3 + Phase 1: the PEP 503 / PEP 691 Simple Repository, over HTTP.
4 +
5 + Everything asserted here is something a real client depends on —
6 + pip's content negotiation, its trailing-slash assumptions, and
7 + twine's upload framing.
8 + """
9 + use GitGudWeb.ConnCase, async: false
10 +
11 + alias GitGud.Packages
12 + alias GitGud.Packages.Formats.Pypi
13 + alias GitGud.Repositories.Storage
14 +
15 + import GitGud.ForgeFixtures
16 +
17 + @json "application/vnd.pypi.simple.v1+json"
18 + @html "application/vnd.pypi.simple.v1+html"
19 +
20 + setup do
21 + {user, repo} = repository_fixture(%{visibility: "public"})
22 + {:ok, _t, token} = Packages.create_registry_token(repo, "ci", ["pull", "push"])
23 +
24 + {:ok,
25 + user: user,
26 + repo: repo,
27 + token: token,
28 + root: "/pkg/pypi/#{Storage.owner_handle(user)}/#{repo.name}"}
29 + end
30 +
31 + defp publish(repo, name, version, filename, body \\ "artifact bytes") do
32 + {:ok, _p, _v, f} =
33 + Packages.publish_file(repo, "pypi", name, version, %{filename: filename, body: body})
34 +
35 + f
36 + end
37 +
38 + describe "normalisation (PEP 503)" do
39 + test "collapses runs of -_. and lowercases" do
40 + assert Pypi.normalize("Flask_Login") == "flask-login"
41 + assert Pypi.normalize("zope.interface") == "zope-interface"
42 + assert Pypi.normalize("A--_.-B") == "a-b"
43 + assert Pypi.normalize("already-normal") == "already-normal"
44 + end
45 + end
46 +
47 + describe "GET /simple/ (project list)" do
48 + test "lists normalised project names as HTML by default", %{
49 + conn: conn,
50 + repo: repo,
51 + root: root
52 + } do
53 + publish(repo, "Flask_Login", "1.0", "Flask_Login-1.0.tar.gz")
54 +
55 + conn = get(conn, root <> "/simple/")
56 +
57 + assert conn.status == 200
58 + assert get_resp_header(conn, "content-type") == ["text/html"]
59 + assert conn.resp_body =~ ~s(<a href="flask-login/">flask-login</a>)
60 + end
61 +
62 + test "serves the JSON serialisation when asked for it", %{
63 + conn: conn,
64 + repo: repo,
65 + root: root
66 + } do
67 + publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
68 +
69 + conn =
70 + conn
71 + |> put_req_header("accept", @json)
72 + |> get(root <> "/simple/")
73 +
74 + assert get_resp_header(conn, "content-type") == [@json]
75 +
76 + assert %{
77 + "meta" => %{"api-version" => "1.0"},
78 + "projects" => [%{"name" => "widget"}]
79 + } = Jason.decode!(conn.resp_body)
80 + end
81 +
82 + test "honours pip's real Accept header, which prefers JSON", %{
83 + conn: conn,
84 + repo: repo,
85 + root: root
86 + } do
87 + publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
88 +
89 + conn =
90 + conn
91 + |> put_req_header(
92 + "accept",
93 + "#{@json}, #{@html};q=0.2, text/html;q=0.01"
94 + )
95 + |> get(root <> "/simple/")
96 +
97 + assert get_resp_header(conn, "content-type") == [@json]
98 + end
99 +
100 + test "406 when the client accepts nothing we serve", %{conn: conn, root: root} do
101 + conn =
102 + conn
103 + |> put_req_header("accept", "application/vnd.pypi.simple.v99+xml")
104 + |> get(root <> "/simple/")
105 +
106 + assert conn.status == 406
107 + end
108 +
109 + test "a q=0 range is a refusal, not a weak preference", %{conn: conn, root: root} do
110 + conn =
111 + conn
112 + |> put_req_header("accept", "text/html;q=0, #{@html};q=0")
113 + |> get(root <> "/simple/")
114 +
115 + assert conn.status == 406
116 + end
117 +
118 + test "redirects the unslashed URL", %{conn: conn, root: root} do
119 + conn = get(conn, root <> "/simple")
120 +
121 + assert conn.status == 301
122 + assert [location] = get_resp_header(conn, "location")
123 + assert String.ends_with?(location, "/simple/")
124 + end
125 +
126 + test "the repository root sends you to the index", %{conn: conn, root: root} do
127 + conn = get(conn, root <> "/")
128 +
129 + assert conn.status == 301
130 + assert [location] = get_resp_header(conn, "location")
131 + assert String.ends_with?(location, "/simple/")
132 + end
133 + end
134 +
135 + describe "GET /simple/<project>/ (file list)" do
136 + test "anchors the filename and carries the sha256 fragment", %{
137 + conn: conn,
138 + repo: repo,
139 + root: root
140 + } do
141 + file = publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
142 +
143 + conn = get(conn, root <> "/simple/widget/")
144 +
145 + assert conn.status == 200
146 + assert conn.resp_body =~ "widget-1.0.tar.gz</a>"
147 + assert conn.resp_body =~ "#sha256=#{file.sha256}"
148 + end
149 +
150 + test "JSON file entries always carry `hashes`", %{conn: conn, repo: repo, root: root} do
151 + file = publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
152 +
153 + conn =
154 + conn
155 + |> put_req_header("accept", @json)
156 + |> get(root <> "/simple/widget/")
157 +
158 + assert %{"name" => "widget", "files" => [entry]} = Jason.decode!(conn.resp_body)
159 + assert entry["filename"] == "widget-1.0.tar.gz"
160 + assert entry["hashes"] == %{"sha256" => file.sha256}
161 + assert entry["yanked"] == false
162 + assert entry["url"] =~ "/files/widget-1.0.tar.gz#sha256=#{file.sha256}"
163 + end
164 +
165 + test "an unnormalised name redirects to the canonical URL", %{
166 + conn: conn,
167 + repo: repo,
168 + root: root
169 + } do
170 + publish(repo, "Flask_Login", "1.0", "Flask_Login-1.0.tar.gz")
171 +
172 + conn = get(conn, root <> "/simple/Flask_Login/")
173 +
174 + assert conn.status == 301
175 + assert [location] = get_resp_header(conn, "location")
176 + assert String.ends_with?(location, "/simple/flask-login/")
177 + end
178 +
179 + test "an unslashed project URL redirects", %{conn: conn, repo: repo, root: root} do
180 + publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
181 +
182 + conn = get(conn, root <> "/simple/widget")
183 +
184 + assert conn.status == 301
185 + assert [location] = get_resp_header(conn, "location")
186 + assert String.ends_with?(location, "/simple/widget/")
187 + end
188 +
189 + test "404 for a project nobody has published", %{conn: conn, root: root} do
190 + assert get(conn, root <> "/simple/nothing-here/").status == 404
191 + end
192 +
193 + test "a yanked release reports its reason (PEP 592)", %{conn: conn, repo: repo, root: root} do
194 + publish(repo, "widget", "1.0", "widget-1.0.tar.gz")
195 + pkg = Packages.get_package(repo, "pypi", "widget")
196 + {:ok, _} = Packages.set_yanked(Packages.get_version(pkg, "1.0"), true, "bad build")
197 +
198 + json =
199 + conn
200 + |> put_req_header("accept", @json)
201 + |> get(root <> "/simple/widget/")
202 + |> Map.get(:resp_body)
203 + |> Jason.decode!()
204 +
205 + assert [%{"yanked" => "bad build"}] = json["files"]
206 +
207 + assert build_conn()
208 + |> get(root <> "/simple/widget/")
209 + |> Map.get(:resp_body) =~ ~s(data-yanked="bad build")
210 + end
211 +
212 + test "requires-python rides along in both serialisations", %{
213 + conn: conn,
214 + repo: repo,
215 + root: root
216 + } do
217 + {:ok, _, _, _} =
218 + Packages.publish_file(repo, "pypi", "widget", "1.0", %{
219 + filename: "widget-1.0.tar.gz",
220 + body: "sdist",
221 + metadata: %{"requires_python" => ">=3.9"}
222 + })
223 +
224 + html = get(conn, root <> "/simple/widget/").resp_body
225 + assert html =~ ~s(data-requires-python="&gt;=3.9")
226 +
227 + json =
228 + build_conn()
229 + |> put_req_header("accept", @json)
230 + |> get(root <> "/simple/widget/")
231 + |> Map.get(:resp_body)
232 + |> Jason.decode!()
233 +
234 + assert [%{"requires-python" => ">=3.9"}] = json["files"]
235 + end
236 + end
237 +
238 + describe "GET /files/<filename>" do
239 + test "serves the stored bytes", %{conn: conn, repo: repo, root: root} do
240 + publish(repo, "widget", "1.0", "widget-1.0.tar.gz", "the real sdist")
241 +
242 + conn = get(conn, root <> "/files/widget-1.0.tar.gz")
243 +
244 + assert conn.status == 200
245 + assert conn.resp_body == "the real sdist"
246 + end
247 +
248 + test "404 for a filename that was never published", %{conn: conn, root: root} do
249 + assert get(conn, root <> "/files/ghost-9.9.tar.gz").status == 404
250 + end
251 + end
252 +
253 + describe "POST / (twine upload)" do
254 + test "publishes a distribution", %{conn: conn, repo: repo, root: root, token: token} do
255 + conn = upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes")
256 +
257 + assert conn.status == 200
258 +
259 + pkg = Packages.get_package(repo, "pypi", "widget")
260 + assert pkg.latest_version == "1.0.0"
261 +
262 + assert [file] = Packages.list_files(Packages.get_version(pkg, "1.0.0"))
263 + assert file.filename == "widget-1.0.0.tar.gz"
264 + assert {:ok, "sdist bytes"} = Packages.read_file(file)
265 + end
266 +
267 + test "rejects an upload whose sha256_digest doesn't match", %{
268 + conn: conn,
269 + root: root,
270 + token: token
271 + } do
272 + conn =
273 + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes",
274 + sha256_digest: String.duplicate("0", 64)
275 + )
276 +
277 + assert conn.status == 400
278 + assert conn.resp_body =~ "sha256_digest"
279 + end
280 +
281 + test "accepts the digest twine actually computes", %{conn: conn, root: root, token: token} do
282 + digest = Base.encode16(:crypto.hash(:sha256, "sdist bytes"), case: :lower)
283 +
284 + conn =
285 + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes",
286 + sha256_digest: digest
287 + )
288 +
289 + assert conn.status == 200
290 + end
291 +
292 + test "409 on re-uploading a filename with different bytes", %{
293 + conn: conn,
294 + root: root,
295 + token: token
296 + } do
297 + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "first")
298 +
299 + conn = upload(build_conn(), root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "second")
300 +
301 + assert conn.status == 409
302 + end
303 +
304 + test "400 without a name", %{conn: conn, root: root, token: token} do
305 + conn =
306 + conn
307 + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{token}"))
308 + |> post(root <> "/", %{"version" => "1.0.0"})
309 +
310 + assert conn.status == 400
311 + assert conn.resp_body =~ "name"
312 + end
313 +
314 + test "an anonymous upload is challenged, not silently dropped", %{conn: conn, root: root} do
315 + conn = upload(conn, root, nil, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist")
316 +
317 + assert conn.status == 401
318 + assert [challenge] = get_resp_header(conn, "www-authenticate")
319 + assert challenge =~ "Basic realm="
320 + end
321 +
322 + test "a pull-only token may read but not publish", %{conn: conn, repo: repo, root: root} do
323 + {:ok, _t, ro} = Packages.create_registry_token(repo, "readonly", ["pull"])
324 +
325 + assert upload(conn, root, ro, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist").status ==
326 + 403
327 +
328 + assert build_conn()
329 + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{ro}"))
330 + |> get(root <> "/simple/")
331 + |> Map.get(:status) == 200
332 + end
333 + end
334 +
335 + describe "visibility" do
336 + setup do
337 + {user, private} = repository_fixture(%{visibility: "private"})
338 + {:ok, _t, token} = Packages.create_registry_token(private, "ci", ["pull"])
339 +
340 + {:ok,
341 + private: private,
342 + private_token: token,
343 + private_root: "/pkg/pypi/#{Storage.owner_handle(user)}/#{private.name}"}
344 + end
345 +
346 + test "a private index challenges an anonymous client", %{
347 + conn: conn,
348 + private_root: root
349 + } do
350 + conn = get(conn, root <> "/simple/")
351 +
352 + assert conn.status == 401
353 + assert [_] = get_resp_header(conn, "www-authenticate")
354 + end
355 +
356 + test "and serves it to a token holder — this is what pip does next", %{
357 + conn: conn,
358 + private: private,
359 + private_root: root,
360 + private_token: token
361 + } do
362 + publish(private, "secret", "1.0", "secret-1.0.tar.gz")
363 +
364 + conn =
365 + conn
366 + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{token}"))
367 + |> get(root <> "/simple/")
368 +
369 + assert conn.status == 200
370 + assert conn.resp_body =~ "secret"
371 + end
372 + end
373 +
374 + test "an unknown format 404s rather than reaching an adapter", %{conn: conn, root: root} do
375 + assert get(conn, String.replace(root, "/pypi/", "/nonsense/") <> "/simple/").status == 404
376 + end
377 +
378 + test "a missing repository 404s", %{conn: conn} do
379 + assert get(conn, "/pkg/pypi/nobody/nothing/simple/").status == 404
380 + end
381 +
382 + # twine's framing: multipart/form-data with the distribution in
383 + # `content` and the core metadata in sibling fields.
384 + defp upload(conn, root, token, name, version, filename, body, opts \\ []) do
385 + path = Path.join(System.tmp_dir!(), "upload-#{System.unique_integer([:positive])}")
386 + File.write!(path, body)
387 +
388 + params =
389 + %{
390 + ":action" => "file_upload",
391 + "protocol_version" => "1",
392 + "name" => name,
393 + "version" => version,
394 + "filetype" => "sdist",
395 + "metadata_version" => "2.1",
396 + "content" => %Plug.Upload{
397 + path: path,
398 + filename: filename,
399 + content_type: "application/octet-stream"
400 + }
401 + }
402 + |> Map.merge(Map.new(opts, fn {k, v} -> {to_string(k), v} end))
403 +
404 + conn =
405 + if token,
406 + do:
407 + put_req_header(conn, "authorization", "Basic " <> Base.encode64("__token__:#{token}")),
408 + else: conn
409 +
410 + post(conn, root <> "/", params)
411 + end
412 +end

Parents: d70ff9f