neiam
/gitgud
Git Gud
public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages
⭐
1
Log in to mark this repository.
Package registries phase 1: PyPI
ed33862 · Gabriel Morell · 2026-09-11 14:16
Message
{commit_body(@commit)}
Files changed
modified
lib/git_gud/packages/format.ex
+1
−1
@@ -45,7 +45,7 @@ defmodule GitGud.Packages.Format do
| 45 | 45 | # distribution API itself, so `oci` exists only as a `format` value |
| 46 | 46 | # the packages UI can list alongside everything else. Each remaining |
| 47 | 47 | # format appears here as its adapter lands. |
| 48 | − @default_adapters %{} | |
| 48 | + @default_adapters %{"pypi" => GitGud.Packages.Formats.Pypi} | |
| 49 | 49 | |
| 50 | 50 | @typedoc """ |
| 51 | 51 | Request context handed to an adapter. |
added
lib/git_gud/packages/formats/pypi.ex
+473
−0
@@ -0,0 +1,473 @@
| 1 | +defmodule GitGud.Packages.Formats.Pypi do | |
| 2 | + @moduledoc """ | |
| 3 | + A PEP 503 / PEP 691 Simple Repository, per git repository. | |
| 4 | + | |
| 5 | + pip install --index-url https://<host>/pkg/pypi/<owner>/<repo>/simple/ widget | |
| 6 | + twine upload --repository-url https://<host>/pkg/pypi/<owner>/<repo>/ dist/* | |
| 7 | + | |
| 8 | + Paths below the repository root: | |
| 9 | + | |
| 10 | + / twine's upload target (POST) | |
| 11 | + /simple/ the project list | |
| 12 | + /simple/<name>/ a project's files | |
| 13 | + /files/<filename> an artifact's bytes | |
| 14 | + | |
| 15 | + ## The parts that bite | |
| 16 | + | |
| 17 | + **Normalisation.** PEP 503 defines exactly one canonical form, | |
| 18 | + `re.sub(r"[-_.]+", "-", name).lower()`, and it must be applied before | |
| 19 | + *any* lookup — `Flask_Login`, `flask.login` and `FLASK--LOGIN` are | |
| 20 | + one project. Clients request the normalised URL directly; we also | |
| 21 | + redirect an unnormalised one, which the PEP permits and humans | |
| 22 | + appreciate. | |
| 23 | + | |
| 24 | + **Trailing slashes are load-bearing.** Every index URL ends in `/`, | |
| 25 | + and an unslashed one must redirect rather than render — relative | |
| 26 | + hrefs resolve against the wrong base otherwise. Plug drops the | |
| 27 | + trailing slash when it splits the path, so the controller passes the | |
| 28 | + raw fact through as `:trailing_slash?`. | |
| 29 | + | |
| 30 | + **Two serialisations, one data structure.** PEP 691 adds a JSON | |
| 31 | + rendering of exactly the same information, negotiated on `Accept`: | |
| 32 | + | |
| 33 | + application/vnd.pypi.simple.v1+json | |
| 34 | + application/vnd.pypi.simple.v1+html | |
| 35 | + text/html (legacy alias for v1+html) | |
| 36 | + | |
| 37 | + with `406` when nothing matches. Both come off `project_files/1` | |
| 38 | + here; only the rendering differs. | |
| 39 | + """ | |
| 40 | + | |
| 41 | + @behaviour GitGud.Packages.Format | |
| 42 | + | |
| 43 | + alias GitGud.Packages | |
| 44 | + alias GitGud.Packages.Package | |
| 45 | + alias GitGud.Packages.PackageFile | |
| 46 | + | |
| 47 | + @format "pypi" | |
| 48 | + @api_version "1.0" | |
| 49 | + | |
| 50 | + @json "application/vnd.pypi.simple.v1+json" | |
| 51 | + @html "application/vnd.pypi.simple.v1+html" | |
| 52 | + @legacy_html "text/html" | |
| 53 | + | |
| 54 | + # Most preferred last-resort first: a client that sends `*/*` (curl, | |
| 55 | + # a browser) gets HTML, which is the legacy behaviour and readable. | |
| 56 | + @serialisations [@json, @html, @legacy_html] | |
| 57 | + | |
| 58 | + @doc """ | |
| 59 | + PEP 503 name normalisation, which is the whole of it: | |
| 60 | + | |
| 61 | + re.sub(r"[-_.]+", "-", name).lower() | |
| 62 | + """ | |
| 63 | + @impl true | |
| 64 | + def normalize(name) when is_binary(name) do | |
| 65 | + name | |
| 66 | + |> String.replace(~r/[-_.]+/, "-") | |
| 67 | + |> String.downcase() | |
| 68 | + end | |
| 69 | + | |
| 70 | + @impl true | |
| 71 | + def content_type(filename) do | |
| 72 | + cond do | |
| 73 | + String.ends_with?(filename, ".whl") -> "application/zip" | |
| 74 | + String.ends_with?(filename, ".zip") -> "application/zip" | |
| 75 | + String.ends_with?(filename, ".egg") -> "application/zip" | |
| 76 | + String.ends_with?(filename, ".gz") -> "application/gzip" | |
| 77 | + String.ends_with?(filename, ".bz2") -> "application/x-bzip2" | |
| 78 | + true -> "application/octet-stream" | |
| 79 | + end | |
| 80 | + end | |
| 81 | + | |
| 82 | + # ── reads ─────────────────────────────────────────────────────────── | |
| 83 | + | |
| 84 | + @impl true | |
| 85 | + def index(repo, ctx) | |
| 86 | + | |
| 87 | + # The repository root is twine's upload target, not a page. Send a | |
| 88 | + # human (or a curious client) on to the index. | |
| 89 | + def index(_repo, %{path: []} = ctx), do: {:redirect, ctx.base_url <> "/simple/"} | |
| 90 | + | |
| 91 | + def index(_repo, %{path: ["simple"], trailing_slash?: false} = ctx), | |
| 92 | + do: {:redirect, ctx.base_url <> "/simple/"} | |
| 93 | + | |
| 94 | + def index(repo, %{path: ["simple"]} = ctx) do | |
| 95 | + projects = Packages.list_packages(repo, @format) | |
| 96 | + | |
| 97 | + serve(ctx, fn | |
| 98 | + :json -> root_json(projects) | |
| 99 | + :html -> root_html(projects) | |
| 100 | + end) | |
| 101 | + end | |
| 102 | + | |
| 103 | + def index(repo, %{path: ["simple", name]} = ctx) do | |
| 104 | + normalized = normalize(name) | |
| 105 | + | |
| 106 | + cond do | |
| 107 | + # PEP 503: a server *may* redirect an unnormalised name. Doing so | |
| 108 | + # means a hand-typed URL works and a client's cache key stays the | |
| 109 | + # canonical one. | |
| 110 | + normalized != name -> | |
| 111 | + {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"} | |
| 112 | + | |
| 113 | + not ctx.trailing_slash? -> | |
| 114 | + {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"} | |
| 115 | + | |
| 116 | + true -> | |
| 117 | + case Packages.get_package(repo, @format, normalized) do | |
| 118 | + nil -> | |
| 119 | + {:error, :not_found} | |
| 120 | + | |
| 121 | + %Package{} = package -> | |
| 122 | + files = project_files(package, ctx.base_url) | |
| 123 | + | |
| 124 | + serve(ctx, fn | |
| 125 | + :json -> project_json(normalized, files) | |
| 126 | + :html -> project_html(normalized, files) | |
| 127 | + end) | |
| 128 | + end | |
| 129 | + end | |
| 130 | + end | |
| 131 | + | |
| 132 | + def index(repo, %{path: ["files", filename]}) do | |
| 133 | + # A filename is unique within a project but the download URL | |
| 134 | + # doesn't name the project, so this searches the repository's whole | |
| 135 | + # pypi namespace. Distribution filenames embed the project name, so | |
| 136 | + # collisions across projects don't happen in practice. | |
| 137 | + case find_file(repo, filename) do | |
| 138 | + %PackageFile{} = file -> {:blob, file} | |
| 139 | + nil -> {:error, :not_found} | |
| 140 | + end | |
| 141 | + end | |
| 142 | + | |
| 143 | + def index(_repo, _ctx), do: {:error, :not_found} | |
| 144 | + | |
| 145 | + defp find_file(repo, filename) do | |
| 146 | + repo | |
| 147 | + |> Packages.list_packages(@format) | |
| 148 | + |> Enum.find_value(fn package -> Packages.get_package_file(package, filename) end) | |
| 149 | + end | |
| 150 | + | |
| 151 | + # ── the one data structure both serialisations render ─────────────── | |
| 152 | + | |
| 153 | + defp project_files(%Package{} = package, base_url) do | |
| 154 | + package | |
| 155 | + |> Packages.list_package_files() | |
| 156 | + |> Enum.map(fn %PackageFile{} = file -> | |
| 157 | + meta = file.metadata || %{} | |
| 158 | + | |
| 159 | + %{ | |
| 160 | + filename: file.filename, | |
| 161 | + url: "#{base_url}/files/#{URI.encode(file.filename)}#sha256=#{file.sha256}", | |
| 162 | + sha256: file.sha256, | |
| 163 | + requires_python: meta["requires_python"], | |
| 164 | + # PEP 592: `false` for a live file, the *reason string* for a | |
| 165 | + # yanked one — and an empty reason is still a yank, so this is | |
| 166 | + # deliberately not `reason || false`. | |
| 167 | + yanked: | |
| 168 | + if(file.package_version.yanked, | |
| 169 | + do: file.package_version.yanked_reason || true, | |
| 170 | + else: false | |
| 171 | + ) | |
| 172 | + } | |
| 173 | + end) | |
| 174 | + end | |
| 175 | + | |
| 176 | + # ── content negotiation ───────────────────────────────────────────── | |
| 177 | + | |
| 178 | + defp serve(ctx, render) do | |
| 179 | + case negotiate(ctx.conn) do | |
| 180 | + {:ok, @json} -> {:ok, @json, Jason.encode_to_iodata!(render.(:json))} | |
| 181 | + {:ok, media} -> {:ok, media, render.(:html)} | |
| 182 | + :error -> {:error, :not_acceptable} | |
| 183 | + end | |
| 184 | + end | |
| 185 | + | |
| 186 | + @doc """ | |
| 187 | + Pick a serialisation for an `Accept` header, or `:error` for 406. | |
| 188 | + | |
| 189 | + A request with no `Accept` at all is answered as though it asked for | |
| 190 | + `text/html` — that is what every pre-PEP-691 client did, and what | |
| 191 | + PEP 691 tells servers to assume. | |
| 192 | + """ | |
| 193 | + def negotiate(conn) do | |
| 194 | + case Plug.Conn.get_req_header(conn, "accept") do | |
| 195 | + [] -> | |
| 196 | + {:ok, @legacy_html} | |
| 197 | + | |
| 198 | + headers -> | |
| 199 | + headers | |
| 200 | + |> Enum.join(",") | |
| 201 | + |> parse_accept() | |
| 202 | + |> best_match() | |
| 203 | + end | |
| 204 | + end | |
| 205 | + | |
| 206 | + # `media/type;q=0.5` → {media_range, q}, highest q first. Ties keep | |
| 207 | + # the order the client wrote them in, which is the convention for | |
| 208 | + # equal-quality ranges. | |
| 209 | + defp parse_accept(header) do | |
| 210 | + header | |
| 211 | + |> String.split(",") | |
| 212 | + |> Enum.map(&String.trim/1) | |
| 213 | + |> Enum.reject(&(&1 == "")) | |
| 214 | + |> Enum.with_index() | |
| 215 | + |> Enum.map(fn {entry, index} -> | |
| 216 | + [range | params] = String.split(entry, ";") | |
| 217 | + {String.trim(range) |> String.downcase(), quality(params), index} | |
| 218 | + end) | |
| 219 | + |> Enum.sort_by(fn {_range, q, index} -> {-q, index} end) | |
| 220 | + end | |
| 221 | + | |
| 222 | + defp quality(params) do | |
| 223 | + Enum.find_value(params, 1.0, fn param -> | |
| 224 | + case param |> String.trim() |> String.split("=", parts: 2) do | |
| 225 | + ["q", value] -> | |
| 226 | + case Float.parse(String.trim(value)) do | |
| 227 | + {q, _} -> q | |
| 228 | + :error -> 1.0 | |
| 229 | + end | |
| 230 | + | |
| 231 | + _ -> | |
| 232 | + nil | |
| 233 | + end | |
| 234 | + end) | |
| 235 | + end | |
| 236 | + | |
| 237 | + defp best_match(ranges) do | |
| 238 | + # `q=0` is an explicit refusal, not a weak preference. | |
| 239 | + ranges | |
| 240 | + |> Enum.reject(fn {_range, q, _} -> q == 0.0 end) | |
| 241 | + |> Enum.find_value(:error, fn {range, _q, _} -> | |
| 242 | + case Enum.find(@serialisations, &matches?(range, &1)) do | |
| 243 | + nil -> nil | |
| 244 | + # `text/html` is only an alias for the HTML serialisation; the | |
| 245 | + # response still advertises what it actually is. | |
| 246 | + @legacy_html -> {:ok, @legacy_html} | |
| 247 | + media -> {:ok, media} | |
| 248 | + end | |
| 249 | + end) | |
| 250 | + end | |
| 251 | + | |
| 252 | + defp matches?("*/*", _media), do: true | |
| 253 | + | |
| 254 | + defp matches?(range, media) do | |
| 255 | + case String.split(range, "/", parts: 2) do | |
| 256 | + [type, "*"] -> String.starts_with?(media, type <> "/") | |
| 257 | + _ -> range == media | |
| 258 | + end | |
| 259 | + end | |
| 260 | + | |
| 261 | + # ── PEP 691 (JSON) ────────────────────────────────────────────────── | |
| 262 | + | |
| 263 | + defp root_json(projects) do | |
| 264 | + %{ | |
| 265 | + "meta" => %{"api-version" => @api_version}, | |
| 266 | + "projects" => Enum.map(projects, &%{"name" => &1.normalized_name}) | |
| 267 | + } | |
| 268 | + end | |
| 269 | + | |
| 270 | + defp project_json(name, files) do | |
| 271 | + %{ | |
| 272 | + "meta" => %{"api-version" => @api_version}, | |
| 273 | + "name" => name, | |
| 274 | + "files" => Enum.map(files, &file_json/1) | |
| 275 | + } | |
| 276 | + end | |
| 277 | + | |
| 278 | + defp file_json(file) do | |
| 279 | + # `hashes` is required even when empty — a client may not assume | |
| 280 | + # the key is absent, so this is never conditionally built. | |
| 281 | + base = %{ | |
| 282 | + "filename" => file.filename, | |
| 283 | + "url" => file.url, | |
| 284 | + "hashes" => %{"sha256" => file.sha256}, | |
| 285 | + "yanked" => file.yanked | |
| 286 | + } | |
| 287 | + | |
| 288 | + case file.requires_python do | |
| 289 | + nil -> base | |
| 290 | + "" -> base | |
| 291 | + spec -> Map.put(base, "requires-python", spec) | |
| 292 | + end | |
| 293 | + end | |
| 294 | + | |
| 295 | + # ── PEP 503 (HTML) ────────────────────────────────────────────────── | |
| 296 | + | |
| 297 | + defp root_html(projects) do | |
| 298 | + links = | |
| 299 | + Enum.map(projects, fn project -> | |
| 300 | + ~s( <a href="#{esc(project.normalized_name)}/">#{esc(project.normalized_name)}</a><br />\n) | |
| 301 | + end) | |
| 302 | + | |
| 303 | + page("Simple index", links) | |
| 304 | + end | |
| 305 | + | |
| 306 | + defp project_html(name, files) do | |
| 307 | + links = | |
| 308 | + Enum.map(files, fn file -> | |
| 309 | + ~s( <a href="#{esc(file.url)}"#{html_attrs(file)}>#{esc(file.filename)}</a><br />\n) | |
| 310 | + end) | |
| 311 | + | |
| 312 | + page("Links for #{name}", [~s( <h1>Links for #{esc(name)}</h1>\n) | links]) | |
| 313 | + end | |
| 314 | + | |
| 315 | + defp html_attrs(file) do | |
| 316 | + [ | |
| 317 | + requires_python_attr(file.requires_python), | |
| 318 | + yanked_attr(file.yanked) | |
| 319 | + ] | |
| 320 | + |> Enum.reject(&is_nil/1) | |
| 321 | + |> Enum.join() | |
| 322 | + end | |
| 323 | + | |
| 324 | + defp requires_python_attr(nil), do: nil | |
| 325 | + defp requires_python_attr(""), do: nil | |
| 326 | + defp requires_python_attr(spec), do: ~s( data-requires-python="#{esc(spec)}") | |
| 327 | + | |
| 328 | + defp yanked_attr(false), do: nil | |
| 329 | + defp yanked_attr(true), do: ~s( data-yanked="") | |
| 330 | + defp yanked_attr(reason), do: ~s( data-yanked="#{esc(reason)}") | |
| 331 | + | |
| 332 | + defp page(title, body) do | |
| 333 | + [ | |
| 334 | + "<!DOCTYPE html>\n<html>\n <head>\n", | |
| 335 | + " <meta name=\"pypi:repository-version\" content=\"#{@api_version}\" />\n", | |
| 336 | + " <title>#{esc(title)}</title>\n </head>\n <body>\n", | |
| 337 | + body, | |
| 338 | + " </body>\n</html>\n" | |
| 339 | + ] | |
| 340 | + end | |
| 341 | + | |
| 342 | + # PEP 503 is served to a browser as well as to pip, and a project | |
| 343 | + # name reaches us from an upload form. | |
| 344 | + defp esc(value) do | |
| 345 | + value | |
| 346 | + |> to_string() | |
| 347 | + |> String.replace("&", "&") | |
| 348 | + |> String.replace("<", "<") | |
| 349 | + |> String.replace(">", ">") | |
| 350 | + |> String.replace("\"", """) | |
| 351 | + end | |
| 352 | + | |
| 353 | + # ── uploads ───────────────────────────────────────────────────────── | |
| 354 | + | |
| 355 | + @impl true | |
| 356 | + def store(repo, ctx, params) | |
| 357 | + | |
| 358 | + def store(repo, %{method: "POST"} = ctx, %{} = params) do | |
| 359 | + with {:ok, name} <- require_field(params, "name"), | |
| 360 | + {:ok, version} <- require_field(params, "version"), | |
| 361 | + {:ok, upload} <- require_upload(params), | |
| 362 | + {:ok, body} <- read_upload(upload), | |
| 363 | + :ok <- verify_digest(params["sha256_digest"], body) do | |
| 364 | + attrs = %{ | |
| 365 | + filename: Path.basename(upload.filename), | |
| 366 | + body: body, | |
| 367 | + content_type: content_type(upload.filename), | |
| 368 | + metadata: file_metadata(params), | |
| 369 | + version_metadata: version_metadata(params), | |
| 370 | + uploader: uploader(ctx) | |
| 371 | + } | |
| 372 | + | |
| 373 | + case Packages.publish_file(repo, @format, name, version, attrs) do | |
| 374 | + # twine reads nothing but the status. An empty 200 is what | |
| 375 | + # warehouse sends. | |
| 376 | + {:ok, _package, _version, _file} -> {:ok, "text/plain; charset=utf-8", ""} | |
| 377 | + {:error, :file_exists} -> {:error, :file_exists} | |
| 378 | + {:error, reason} -> {:error, changeset_message(reason)} | |
| 379 | + end | |
| 380 | + end | |
| 381 | + end | |
| 382 | + | |
| 383 | + def store(_repo, %{method: "POST"}, _body), | |
| 384 | + do: {:error, {400, "expected a multipart/form-data upload"}} | |
| 385 | + | |
| 386 | + def store(_repo, %{method: method}, _params), | |
| 387 | + do: {:error, {405, "#{method} is not supported by the PyPI upload API"}} | |
| 388 | + | |
| 389 | + defp require_field(params, key) do | |
| 390 | + case params[key] do | |
| 391 | + value when is_binary(value) and value != "" -> {:ok, value} | |
| 392 | + _ -> {:error, {400, "missing required form field `#{key}`"}} | |
| 393 | + end | |
| 394 | + end | |
| 395 | + | |
| 396 | + defp require_upload(params) do | |
| 397 | + case params["content"] do | |
| 398 | + %Plug.Upload{} = upload -> {:ok, upload} | |
| 399 | + _ -> {:error, {400, "missing the `content` file part"}} | |
| 400 | + end | |
| 401 | + end | |
| 402 | + | |
| 403 | + defp read_upload(%Plug.Upload{path: path}) do | |
| 404 | + case File.read(path) do | |
| 405 | + {:ok, body} -> {:ok, body} | |
| 406 | + {:error, reason} -> {:error, {400, "could not read the upload: #{inspect(reason)}"}} | |
| 407 | + end | |
| 408 | + end | |
| 409 | + | |
| 410 | + # twine computes this itself and sends it alongside. Checking it | |
| 411 | + # costs one hash and turns a truncated upload into a clear error | |
| 412 | + # instead of a corrupt wheel nobody notices until install time. | |
| 413 | + defp verify_digest(nil, _body), do: :ok | |
| 414 | + defp verify_digest("", _body), do: :ok | |
| 415 | + | |
| 416 | + defp verify_digest(expected, body) do | |
| 417 | + actual = Base.encode16(:crypto.hash(:sha256, body), case: :lower) | |
| 418 | + | |
| 419 | + if String.downcase(expected) == actual, | |
| 420 | + do: :ok, | |
| 421 | + else: {:error, {400, "sha256_digest does not match the uploaded bytes"}} | |
| 422 | + end | |
| 423 | + | |
| 424 | + defp file_metadata(params) do | |
| 425 | + %{ | |
| 426 | + "requires_python" => blank_to_nil(params["requires_python"]), | |
| 427 | + "filetype" => blank_to_nil(params["filetype"]), | |
| 428 | + "pyversion" => blank_to_nil(params["pyversion"]) | |
| 429 | + } | |
| 430 | + |> Enum.reject(fn {_k, v} -> is_nil(v) end) | |
| 431 | + |> Map.new() | |
| 432 | + end | |
| 433 | + | |
| 434 | + # Core metadata twine sends along with the distribution, kept so the | |
| 435 | + # packages UI can describe a release without unzipping a wheel. | |
| 436 | + defp version_metadata(params) do | |
| 437 | + %{ | |
| 438 | + "summary" => blank_to_nil(params["summary"]), | |
| 439 | + "metadata_version" => blank_to_nil(params["metadata_version"]), | |
| 440 | + "requires_python" => blank_to_nil(params["requires_python"]), | |
| 441 | + "home_page" => blank_to_nil(params["home_page"]), | |
| 442 | + "license" => blank_to_nil(params["license"]) | |
| 443 | + } | |
| 444 | + |> Enum.reject(fn {_k, v} -> is_nil(v) end) | |
| 445 | + |> Map.new() | |
| 446 | + end | |
| 447 | + | |
| 448 | + defp blank_to_nil(value) when is_binary(value) do | |
| 449 | + case String.trim(value) do | |
| 450 | + "" -> nil | |
| 451 | + trimmed -> trimmed | |
| 452 | + end | |
| 453 | + end | |
| 454 | + | |
| 455 | + defp blank_to_nil(_), do: nil | |
| 456 | + | |
| 457 | + defp uploader(%{principal: {:user, user}}), do: user | |
| 458 | + defp uploader(_ctx), do: nil | |
| 459 | + | |
| 460 | + defp changeset_message(%Ecto.Changeset{} = changeset) do | |
| 461 | + errors = | |
| 462 | + Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} -> | |
| 463 | + Enum.reduce(opts, msg, fn {k, v}, acc -> | |
| 464 | + String.replace(acc, "%{#{k}}", to_string(v)) | |
| 465 | + end) | |
| 466 | + end) | |
| 467 | + | |
| 468 | + {400, | |
| 469 | + Enum.map_join(errors, "; ", fn {field, msgs} -> "#{field} #{Enum.join(msgs, ", ")}" end)} | |
| 470 | + end | |
| 471 | + | |
| 472 | + defp changeset_message(reason), do: {400, inspect(reason)} | |
| 473 | +end |
modified
lib/git_gud_web/endpoint.ex
+4
−0
@@ -49,6 +49,10 @@ defmodule GitGudWeb.Endpoint do
| 49 | 49 | # consumes the raw request body itself. |
| 50 | 50 | plug GitGudWeb.Plugs.GitRouter |
| 51 | 51 | |
| 52 | + # Package-registry uploads, parsed with their own (much larger) size | |
| 53 | + # limit before the general parser below claims the body. | |
| 54 | + plug GitGudWeb.Plugs.PackageUploadParser | |
| 55 | + | |
| 52 | 56 | plug Plug.Parsers, |
| 53 | 57 | parsers: [:urlencoded, :multipart, :json], |
| 54 | 58 | pass: ["*/*"], |
modified
lib/git_gud_web/live/repo_live/packages.ex
+150
−97
@@ -1,9 +1,14 @@
| 1 | 1 | defmodule GitGudWeb.RepoLive.Packages do |
| 2 | 2 | @moduledoc """ |
| 3 | − Lists container/package versions mirrored from zot for a repository. | |
| 4 | − | |
| 5 | − This is a UI over the data zot pushes through `/_webhooks/zot`. | |
| 6 | − Click a package row to see all of its versions inline. | |
| 3 | + Everything a repository has published that you can actually install, | |
| 4 | + grouped by registry format. | |
| 5 | + | |
| 6 | + Container images are mirrored from zot via `/_webhooks/zot`; | |
| 7 | + everything else was uploaded to this forge's own registries under | |
| 8 | + `/pkg/:format/:owner/:name`. Each format carries the line you paste | |
| 9 | + to point a client at it, and each version the line you paste to | |
| 10 | + install that version — the page is only useful if you can copy from | |
| 11 | + it without going to look anything up. | |
| 7 | 12 | """ |
| 8 | 13 | |
| 9 | 14 | use GitGudWeb, :live_view |
@@ -12,59 +17,78 @@ defmodule GitGudWeb.RepoLive.Packages do
| 12 | 17 | alias GitGud.Repositories |
| 13 | 18 | alias GitGud.Repositories.Storage |
| 14 | 19 | |
| 20 | + # Display order: containers first (the oldest surface here), then the | |
| 21 | + # language registries in the order they landed. | |
| 22 | + @format_order ~w(oci pypi hex cargo deb) | |
| 23 | + | |
| 24 | + @format_labels %{ | |
| 25 | + "oci" => "Container images", | |
| 26 | + "pypi" => "Python", | |
| 27 | + "hex" => "Hex", | |
| 28 | + "cargo" => "Crates", | |
| 29 | + "deb" => "Debian packages" | |
| 30 | + } | |
| 31 | + | |
| 15 | 32 | @impl true |
| 16 | 33 | def mount(%{"owner" => owner, "name" => name}, _session, socket) do |
| 17 | 34 | repo = |
| 18 | 35 | Repositories.get_repository_by_path!(owner, name) |
| 19 | 36 | |> GitGud.Repo.preload([:owner, :organization]) |
| 20 | 37 | |
| 21 | − packages = Packages.list_packages(repo) | |
| 38 | + handle = Storage.repo_handle(repo) | |
| 22 | 39 | |
| 23 | 40 | {:ok, |
| 24 | 41 | socket |
| 25 | 42 | |> assign(:repo, repo) |
| 26 | − |> assign(:handle, Storage.repo_handle(repo)) | |
| 43 | + |> assign(:handle, handle) | |
| 27 | 44 | |> GitGudWeb.RepoLive.Header.assign_chrome(repo) |
| 28 | − |> assign(:packages, packages) | |
| 45 | + |> assign(:groups, grouped_packages(repo)) | |
| 29 | 46 | |> assign(:expanded_id, nil) |
| 30 | 47 | |> assign(:versions_by_id, %{}) |
| 31 | − |> assign(:registry_host, Application.get_env(:git_gud, :registry_host)) | |
| 48 | + |> assign(:registry, %{ | |
| 49 | + handle: handle, | |
| 50 | + repo_name: repo.name, | |
| 51 | + # The OCI registry is a separate host (zot); everything else is | |
| 52 | + # served by this forge. | |
| 53 | + host: Application.get_env(:git_gud, :registry_host), | |
| 54 | + root: registry_root() | |
| 55 | + }) | |
| 32 | 56 | |> assign(:page_title, "Packages — #{owner}/#{name}")} |
| 33 | 57 | end |
| 34 | 58 | |
| 35 | − # Reconstruct the zot repository path the webhook deflated: a package | |
| 36 | − # named after the repo was pushed as `<owner>/<repo>`; a sub-image as | |
| 37 | − # `<owner>/<repo>/<image>`. (See PackagesWebhookController.) | |
| 38 | − defp pull_ref(handle, repo, pkg, version, host) do | |
| 39 | − base = | |
| 40 | − if pkg.name == repo.name, | |
| 41 | − do: "#{handle}/#{repo.name}", | |
| 42 | − else: "#{handle}/#{repo.name}/#{pkg.name}" | |
| 59 | + defp grouped_packages(repo) do | |
| 60 | + repo | |
| 61 | + |> Packages.list_packages() | |
| 62 | + |> Enum.group_by(& &1.format) | |
| 63 | + |> Enum.sort_by(fn {format, _} -> Enum.find_index(@format_order, &(&1 == format)) || 99 end) | |
| 64 | + end | |
| 43 | 65 | |
| 44 | − "#{host}/#{base}:#{version}" | |
| 66 | + defp registry_root do | |
| 67 | + (Application.get_env(:git_gud, :package_registry_url) || GitGudWeb.Endpoint.url()) | |
| 68 | + |> String.trim_trailing("/") | |
| 45 | 69 | end |
| 46 | 70 | |
| 47 | 71 | @impl true |
| 48 | 72 | def handle_event("toggle", %{"id" => id}, socket) do |
| 49 | 73 | id = String.to_integer(id) |
| 50 | 74 | |
| 51 | − cond do | |
| 52 | − socket.assigns.expanded_id == id -> | |
| 53 | − {:noreply, assign(socket, :expanded_id, nil)} | |
| 54 | − | |
| 55 | − true -> | |
| 56 | − package = Enum.find(socket.assigns.packages, &(&1.id == id)) | |
| 57 | − | |
| 58 | − versions = | |
| 59 | − case Map.get(socket.assigns.versions_by_id, id) do | |
| 60 | − nil -> Packages.list_versions(package) | |
| 61 | − cached -> cached | |
| 62 | − end | |
| 63 | − | |
| 64 | − {:noreply, | |
| 65 | − socket | |
| 66 | − |> assign(:expanded_id, id) | |
| 67 | − |> assign(:versions_by_id, Map.put(socket.assigns.versions_by_id, id, versions))} | |
| 75 | + if socket.assigns.expanded_id == id do | |
| 76 | + {:noreply, assign(socket, :expanded_id, nil)} | |
| 77 | + else | |
| 78 | + package = | |
| 79 | + socket.assigns.groups | |
| 80 | + |> Enum.flat_map(fn {_format, packages} -> packages end) | |
| 81 | + |> Enum.find(&(&1.id == id)) | |
| 82 | + | |
| 83 | + versions = | |
| 84 | + Map.get_lazy(socket.assigns.versions_by_id, id, fn -> | |
| 85 | + Packages.list_versions(package) | |
| 86 | + end) | |
| 87 | + | |
| 88 | + {:noreply, | |
| 89 | + socket | |
| 90 | + |> assign(:expanded_id, id) | |
| 91 | + |> assign(:versions_by_id, Map.put(socket.assigns.versions_by_id, id, versions))} | |
| 68 | 92 | end |
| 69 | 93 | end |
| 70 | 94 |
@@ -72,7 +96,7 @@ defmodule GitGudWeb.RepoLive.Packages do
| 72 | 96 | def render(assigns) do |
| 73 | 97 | ~H""" |
| 74 | 98 | <Layouts.app flash={@flash} current_scope={@current_scope}> |
| 75 | − <div class="space-y-4"> | |
| 99 | + <div class="space-y-6"> | |
| 76 | 100 | <GitGudWeb.RepoLive.Header.header |
| 77 | 101 | repo={@repo} |
| 78 | 102 | handle={@handle} |
@@ -85,84 +109,113 @@ defmodule GitGudWeb.RepoLive.Packages do
| 85 | 109 | <header> |
| 86 | 110 | <h1 class="text-xl font-semibold">Packages</h1> |
| 87 | 111 | <p class="text-xs opacity-70 mt-1"> |
| 88 | − Mirrored from the OCI registry. Authoritative storage lives in zot; | |
| 89 | − this view tracks pushes via webhook so you can see what's been | |
| 90 | − published without leaving the forge. | |
| 112 | + Artifacts published from this repository, by format. Container | |
| 113 | + images live in the OCI registry; everything else is served from | |
| 114 | + this forge under <code class="font-mono">/pkg</code>. | |
| 91 | 115 | </p> |
| 92 | 116 | </header> |
| 93 | 117 | |
| 94 | − <ul | |
| 95 | − id="packages" | |
| 96 | − class="divide-y divide-base-300 border border-base-300 rounded" | |
| 118 | + <p | |
| 119 | + :if={@groups == []} | |
| 120 | + class="px-3 py-6 text-sm opacity-60 text-center border border-base-300 rounded" | |
| 97 | 121 | > |
| 98 | − <li :if={@packages == []} class="px-3 py-6 text-sm opacity-60 text-center"> | |
| 99 | − No packages published yet. | |
| 100 | − </li> | |
| 101 | − | |
| 102 | − <li :for={pkg <- @packages} class="px-3 py-2"> | |
| 103 | − <button | |
| 104 | − type="button" | |
| 105 | − phx-click="toggle" | |
| 106 | − phx-value-id={pkg.id} | |
| 107 | − class="w-full flex items-baseline justify-between gap-3 text-left" | |
| 108 | − > | |
| 109 | − <div> | |
| 110 | − <p class="font-mono text-sm">{pkg.name}</p> | |
| 111 | − <p class="text-xs opacity-60"> | |
| 112 | − <span class="badge badge-xs">{pkg.kind}</span> | |
| 113 | − <span :if={pkg.latest_version} class="ml-1"> | |
| 122 | + Nothing published yet. | |
| 123 | + </p> | |
| 124 | + | |
| 125 | + <section :for={{format, packages} <- @groups} class="space-y-2"> | |
| 126 | + <div class="flex items-baseline justify-between gap-3"> | |
| 127 | + <h2 class="text-sm font-semibold">{format_label(format)}</h2> | |
| 128 | + <span class="text-xs opacity-50">{length(packages)}</span> | |
| 129 | + </div> | |
| 130 | + | |
| 131 | + <code | |
| 132 | + :if={configure_line(format, @registry)} | |
| 133 | + class="block bg-base-200 rounded px-2 py-1 select-all text-[11px] overflow-x-auto" | |
| 134 | + > | |
| 135 | + {configure_line(format, @registry)} | |
| 136 | + </code> | |
| 137 | + | |
| 138 | + <ul class="divide-y divide-base-300 border border-base-300 rounded"> | |
| 139 | + <li :for={pkg <- packages} class="px-3 py-2"> | |
| 140 | + <button | |
| 141 | + type="button" | |
| 142 | + phx-click="toggle" | |
| 143 | + phx-value-id={pkg.id} | |
| 144 | + class="w-full flex items-baseline justify-between gap-3 text-left" | |
| 145 | + > | |
| 146 | + <div> | |
| 147 | + <p class="font-mono text-sm">{pkg.name}</p> | |
| 148 | + <p :if={pkg.latest_version} class="text-xs opacity-60"> | |
| 114 | 149 | latest: <span class="font-mono">{pkg.latest_version}</span> |
| 115 | − </span> | |
| 116 | − </p> | |
| 117 | − </div> | |
| 118 | − <.icon | |
| 119 | − name={if @expanded_id == pkg.id, do: "hero-chevron-up", else: "hero-chevron-down"} | |
| 120 | − class="size-4 opacity-50" | |
| 121 | − /> | |
| 122 | − </button> | |
| 123 | − | |
| 124 | − <ul | |
| 125 | − :if={@expanded_id == pkg.id} | |
| 126 | − class="mt-2 ml-4 border-l border-base-300 pl-3 space-y-1 text-xs" | |
| 127 | − > | |
| 128 | − <li :for={v <- Map.get(@versions_by_id, pkg.id, [])} class="space-y-1"> | |
| 129 | − <div class="flex items-baseline justify-between gap-2"> | |
| 130 | − <span class="font-mono">{v.version}</span> | |
| 131 | − <span class="opacity-60 truncate">{short_digest(v.digest)}</span> | |
| 132 | − <span class="opacity-50">{format_size(v.size)}</span> | |
| 133 | − <span class="opacity-50">{format_dt(v.pushed_at)}</span> | |
| 150 | + </p> | |
| 134 | 151 | </div> |
| 135 | − <code | |
| 136 | − :if={@registry_host} | |
| 137 | − class="block bg-base-200 rounded px-2 py-1 select-all text-[11px]" | |
| 138 | − > | |
| 139 | − podman pull {pull_ref(@handle, @repo, pkg, v.version, @registry_host)} | |
| 140 | − </code> | |
| 141 | − </li> | |
| 142 | − <li :if={Map.get(@versions_by_id, pkg.id, []) == []} class="opacity-60"> | |
| 143 | − No versions. | |
| 144 | − </li> | |
| 145 | − </ul> | |
| 146 | − </li> | |
| 147 | − </ul> | |
| 152 | + <.icon | |
| 153 | + name={if @expanded_id == pkg.id, do: "hero-chevron-up", else: "hero-chevron-down"} | |
| 154 | + class="size-4 opacity-50" | |
| 155 | + /> | |
| 156 | + </button> | |
| 157 | + | |
| 158 | + <ul | |
| 159 | + :if={@expanded_id == pkg.id} | |
| 160 | + class="mt-2 ml-4 border-l border-base-300 pl-3 space-y-2 text-xs" | |
| 161 | + > | |
| 162 | + <li :for={v <- Map.get(@versions_by_id, pkg.id, [])} class="space-y-1"> | |
| 163 | + <div class="flex items-baseline justify-between gap-2"> | |
| 164 | + <span class="font-mono"> | |
| 165 | + {v.version} | |
| 166 | + <span :if={v.yanked} class="badge badge-xs badge-warning ml-1">yanked</span> | |
| 167 | + </span> | |
| 168 | + <span :if={v.digest} class="opacity-60 truncate">{short_digest(v.digest)}</span> | |
| 169 | + <span class="opacity-50">{format_dt(v.pushed_at)}</span> | |
| 170 | + </div> | |
| 171 | + <code | |
| 172 | + :if={install_line(format, @registry, pkg, v.version)} | |
| 173 | + class="block bg-base-200 rounded px-2 py-1 select-all text-[11px] overflow-x-auto" | |
| 174 | + > | |
| 175 | + {install_line(format, @registry, pkg, v.version)} | |
| 176 | + </code> | |
| 177 | + </li> | |
| 178 | + <li :if={Map.get(@versions_by_id, pkg.id, []) == []} class="opacity-60"> | |
| 179 | + No versions. | |
| 180 | + </li> | |
| 181 | + </ul> | |
| 182 | + </li> | |
| 183 | + </ul> | |
| 184 | + </section> | |
| 148 | 185 | </div> |
| 149 | 186 | </Layouts.app> |
| 150 | 187 | """ |
| 151 | 188 | end |
| 152 | 189 | |
| 153 | − defp short_digest(nil), do: "" | |
| 190 | + defp format_label(format), do: Map.get(@format_labels, format, format) | |
| 191 | + | |
| 192 | + # The line that points a client at this repository's registry. OCI | |
| 193 | + # has none — the pull reference carries the host itself. | |
| 194 | + defp configure_line("pypi", %{handle: handle, repo_name: name, root: root}), | |
| 195 | + do: "pip config set global.index-url #{root}/pkg/pypi/#{handle}/#{name}/simple/" | |
| 196 | + | |
| 197 | + defp configure_line(_format, _registry), do: nil | |
| 198 | + | |
| 199 | + # Reconstruct the zot repository path the webhook deflated: a package | |
| 200 | + # named after the repo was pushed as `<owner>/<repo>`; a sub-image as | |
| 201 | + # `<owner>/<repo>/<image>`. (See PackagesWebhookController.) | |
| 202 | + defp install_line("oci", %{handle: handle, repo_name: name, host: host}, pkg, version) | |
| 203 | + when is_binary(host) do | |
| 204 | + base = if pkg.name == name, do: "#{handle}/#{name}", else: "#{handle}/#{name}/#{pkg.name}" | |
| 205 | + | |
| 206 | + "podman pull #{host}/#{base}:#{version}" | |
| 207 | + end | |
| 208 | + | |
| 209 | + defp install_line("pypi", %{handle: handle, repo_name: name, root: root}, pkg, version) do | |
| 210 | + "pip install --index-url #{root}/pkg/pypi/#{handle}/#{name}/simple/ #{pkg.name}==#{version}" | |
| 211 | + end | |
| 212 | + | |
| 213 | + defp install_line(_format, _registry, _pkg, _version), do: nil | |
| 154 | 214 | |
| 155 | 215 | defp short_digest("sha256:" <> rest), |
| 156 | 216 | do: "sha256:" <> binary_part(rest, 0, min(12, byte_size(rest))) |
| 157 | 217 | |
| 158 | 218 | defp short_digest(d) when is_binary(d), do: binary_part(d, 0, min(20, byte_size(d))) |
| 159 | − defp short_digest(_), do: "" | |
| 160 | − | |
| 161 | − defp format_size(nil), do: "" | |
| 162 | − defp format_size(0), do: "" | |
| 163 | − defp format_size(b) when b < 1024, do: "#{b} B" | |
| 164 | − defp format_size(b) when b < 1_048_576, do: "#{Float.round(b / 1024, 1)} KB" | |
| 165 | − defp format_size(b), do: "#{Float.round(b / 1_048_576, 1)} MB" | |
| 166 | 219 | |
| 167 | 220 | defp format_dt(nil), do: "" |
| 168 | 221 | defp format_dt(dt), do: Calendar.strftime(dt, "%Y-%m-%d %H:%M") |
added
lib/git_gud_web/plugs/package_upload_parser.ex
+38
−0
@@ -0,0 +1,38 @@
| 1 | +defmodule GitGudWeb.Plugs.PackageUploadParser do | |
| 2 | + @moduledoc """ | |
| 3 | + Parses package-registry upload bodies before the endpoint's own | |
| 4 | + `Plug.Parsers` gets to them, so publishing a fat wheel isn't capped | |
| 5 | + by the 8 MB limit that suits ordinary form posts. | |
| 6 | + | |
| 7 | + Only multipart is handled here — that is twine's framing. Cargo's | |
| 8 | + length-prefixed `application/octet-stream` and Hex's raw tarball | |
| 9 | + fall through `pass: ["*/*"]` untouched and are read off the socket | |
| 10 | + by `GitGudWeb.PackageRegistryController`. | |
| 11 | + | |
| 12 | + `Plug.Parsers` is a no-op once `body_params` has been fetched, so | |
| 13 | + the endpoint's instance leaves whatever this one did alone. | |
| 14 | + """ | |
| 15 | + | |
| 16 | + @behaviour Plug | |
| 17 | + | |
| 18 | + # Comfortably past a manylinux wheel with a bundled CUDA runtime. | |
| 19 | + # Anything genuinely larger belongs in LFS or the OCI registry. | |
| 20 | + @max_upload_bytes 200_000_000 | |
| 21 | + | |
| 22 | + @parser_opts Plug.Parsers.init( | |
| 23 | + parsers: [:multipart], | |
| 24 | + pass: ["*/*"], | |
| 25 | + length: @max_upload_bytes | |
| 26 | + ) | |
| 27 | + | |
| 28 | + @impl true | |
| 29 | + def init(opts), do: opts | |
| 30 | + | |
| 31 | + @impl true | |
| 32 | + def call(%Plug.Conn{path_info: ["pkg" | _], method: method} = conn, _opts) | |
| 33 | + when method in ["POST", "PUT", "PATCH"] do | |
| 34 | + Plug.Parsers.call(conn, @parser_opts) | |
| 35 | + end | |
| 36 | + | |
| 37 | + def call(conn, _opts), do: conn | |
| 38 | +end |
added
test/git_gud_web/pypi_registry_test.exs
+412
−0
@@ -0,0 +1,412 @@
| 1 | +defmodule GitGudWeb.PypiRegistryTest do | |
| 2 | + @moduledoc """ | |
| 3 | + Phase 1: the PEP 503 / PEP 691 Simple Repository, over HTTP. | |
| 4 | + | |
| 5 | + Everything asserted here is something a real client depends on — | |
| 6 | + pip's content negotiation, its trailing-slash assumptions, and | |
| 7 | + twine's upload framing. | |
| 8 | + """ | |
| 9 | + use GitGudWeb.ConnCase, async: false | |
| 10 | + | |
| 11 | + alias GitGud.Packages | |
| 12 | + alias GitGud.Packages.Formats.Pypi | |
| 13 | + alias GitGud.Repositories.Storage | |
| 14 | + | |
| 15 | + import GitGud.ForgeFixtures | |
| 16 | + | |
| 17 | + @json "application/vnd.pypi.simple.v1+json" | |
| 18 | + @html "application/vnd.pypi.simple.v1+html" | |
| 19 | + | |
| 20 | + setup do | |
| 21 | + {user, repo} = repository_fixture(%{visibility: "public"}) | |
| 22 | + {:ok, _t, token} = Packages.create_registry_token(repo, "ci", ["pull", "push"]) | |
| 23 | + | |
| 24 | + {:ok, | |
| 25 | + user: user, | |
| 26 | + repo: repo, | |
| 27 | + token: token, | |
| 28 | + root: "/pkg/pypi/#{Storage.owner_handle(user)}/#{repo.name}"} | |
| 29 | + end | |
| 30 | + | |
| 31 | + defp publish(repo, name, version, filename, body \\ "artifact bytes") do | |
| 32 | + {:ok, _p, _v, f} = | |
| 33 | + Packages.publish_file(repo, "pypi", name, version, %{filename: filename, body: body}) | |
| 34 | + | |
| 35 | + f | |
| 36 | + end | |
| 37 | + | |
| 38 | + describe "normalisation (PEP 503)" do | |
| 39 | + test "collapses runs of -_. and lowercases" do | |
| 40 | + assert Pypi.normalize("Flask_Login") == "flask-login" | |
| 41 | + assert Pypi.normalize("zope.interface") == "zope-interface" | |
| 42 | + assert Pypi.normalize("A--_.-B") == "a-b" | |
| 43 | + assert Pypi.normalize("already-normal") == "already-normal" | |
| 44 | + end | |
| 45 | + end | |
| 46 | + | |
| 47 | + describe "GET /simple/ (project list)" do | |
| 48 | + test "lists normalised project names as HTML by default", %{ | |
| 49 | + conn: conn, | |
| 50 | + repo: repo, | |
| 51 | + root: root | |
| 52 | + } do | |
| 53 | + publish(repo, "Flask_Login", "1.0", "Flask_Login-1.0.tar.gz") | |
| 54 | + | |
| 55 | + conn = get(conn, root <> "/simple/") | |
| 56 | + | |
| 57 | + assert conn.status == 200 | |
| 58 | + assert get_resp_header(conn, "content-type") == ["text/html"] | |
| 59 | + assert conn.resp_body =~ ~s(<a href="flask-login/">flask-login</a>) | |
| 60 | + end | |
| 61 | + | |
| 62 | + test "serves the JSON serialisation when asked for it", %{ | |
| 63 | + conn: conn, | |
| 64 | + repo: repo, | |
| 65 | + root: root | |
| 66 | + } do | |
| 67 | + publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 68 | + | |
| 69 | + conn = | |
| 70 | + conn | |
| 71 | + |> put_req_header("accept", @json) | |
| 72 | + |> get(root <> "/simple/") | |
| 73 | + | |
| 74 | + assert get_resp_header(conn, "content-type") == [@json] | |
| 75 | + | |
| 76 | + assert %{ | |
| 77 | + "meta" => %{"api-version" => "1.0"}, | |
| 78 | + "projects" => [%{"name" => "widget"}] | |
| 79 | + } = Jason.decode!(conn.resp_body) | |
| 80 | + end | |
| 81 | + | |
| 82 | + test "honours pip's real Accept header, which prefers JSON", %{ | |
| 83 | + conn: conn, | |
| 84 | + repo: repo, | |
| 85 | + root: root | |
| 86 | + } do | |
| 87 | + publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 88 | + | |
| 89 | + conn = | |
| 90 | + conn | |
| 91 | + |> put_req_header( | |
| 92 | + "accept", | |
| 93 | + "#{@json}, #{@html};q=0.2, text/html;q=0.01" | |
| 94 | + ) | |
| 95 | + |> get(root <> "/simple/") | |
| 96 | + | |
| 97 | + assert get_resp_header(conn, "content-type") == [@json] | |
| 98 | + end | |
| 99 | + | |
| 100 | + test "406 when the client accepts nothing we serve", %{conn: conn, root: root} do | |
| 101 | + conn = | |
| 102 | + conn | |
| 103 | + |> put_req_header("accept", "application/vnd.pypi.simple.v99+xml") | |
| 104 | + |> get(root <> "/simple/") | |
| 105 | + | |
| 106 | + assert conn.status == 406 | |
| 107 | + end | |
| 108 | + | |
| 109 | + test "a q=0 range is a refusal, not a weak preference", %{conn: conn, root: root} do | |
| 110 | + conn = | |
| 111 | + conn | |
| 112 | + |> put_req_header("accept", "text/html;q=0, #{@html};q=0") | |
| 113 | + |> get(root <> "/simple/") | |
| 114 | + | |
| 115 | + assert conn.status == 406 | |
| 116 | + end | |
| 117 | + | |
| 118 | + test "redirects the unslashed URL", %{conn: conn, root: root} do | |
| 119 | + conn = get(conn, root <> "/simple") | |
| 120 | + | |
| 121 | + assert conn.status == 301 | |
| 122 | + assert [location] = get_resp_header(conn, "location") | |
| 123 | + assert String.ends_with?(location, "/simple/") | |
| 124 | + end | |
| 125 | + | |
| 126 | + test "the repository root sends you to the index", %{conn: conn, root: root} do | |
| 127 | + conn = get(conn, root <> "/") | |
| 128 | + | |
| 129 | + assert conn.status == 301 | |
| 130 | + assert [location] = get_resp_header(conn, "location") | |
| 131 | + assert String.ends_with?(location, "/simple/") | |
| 132 | + end | |
| 133 | + end | |
| 134 | + | |
| 135 | + describe "GET /simple/<project>/ (file list)" do | |
| 136 | + test "anchors the filename and carries the sha256 fragment", %{ | |
| 137 | + conn: conn, | |
| 138 | + repo: repo, | |
| 139 | + root: root | |
| 140 | + } do | |
| 141 | + file = publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 142 | + | |
| 143 | + conn = get(conn, root <> "/simple/widget/") | |
| 144 | + | |
| 145 | + assert conn.status == 200 | |
| 146 | + assert conn.resp_body =~ "widget-1.0.tar.gz</a>" | |
| 147 | + assert conn.resp_body =~ "#sha256=#{file.sha256}" | |
| 148 | + end | |
| 149 | + | |
| 150 | + test "JSON file entries always carry `hashes`", %{conn: conn, repo: repo, root: root} do | |
| 151 | + file = publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 152 | + | |
| 153 | + conn = | |
| 154 | + conn | |
| 155 | + |> put_req_header("accept", @json) | |
| 156 | + |> get(root <> "/simple/widget/") | |
| 157 | + | |
| 158 | + assert %{"name" => "widget", "files" => [entry]} = Jason.decode!(conn.resp_body) | |
| 159 | + assert entry["filename"] == "widget-1.0.tar.gz" | |
| 160 | + assert entry["hashes"] == %{"sha256" => file.sha256} | |
| 161 | + assert entry["yanked"] == false | |
| 162 | + assert entry["url"] =~ "/files/widget-1.0.tar.gz#sha256=#{file.sha256}" | |
| 163 | + end | |
| 164 | + | |
| 165 | + test "an unnormalised name redirects to the canonical URL", %{ | |
| 166 | + conn: conn, | |
| 167 | + repo: repo, | |
| 168 | + root: root | |
| 169 | + } do | |
| 170 | + publish(repo, "Flask_Login", "1.0", "Flask_Login-1.0.tar.gz") | |
| 171 | + | |
| 172 | + conn = get(conn, root <> "/simple/Flask_Login/") | |
| 173 | + | |
| 174 | + assert conn.status == 301 | |
| 175 | + assert [location] = get_resp_header(conn, "location") | |
| 176 | + assert String.ends_with?(location, "/simple/flask-login/") | |
| 177 | + end | |
| 178 | + | |
| 179 | + test "an unslashed project URL redirects", %{conn: conn, repo: repo, root: root} do | |
| 180 | + publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 181 | + | |
| 182 | + conn = get(conn, root <> "/simple/widget") | |
| 183 | + | |
| 184 | + assert conn.status == 301 | |
| 185 | + assert [location] = get_resp_header(conn, "location") | |
| 186 | + assert String.ends_with?(location, "/simple/widget/") | |
| 187 | + end | |
| 188 | + | |
| 189 | + test "404 for a project nobody has published", %{conn: conn, root: root} do | |
| 190 | + assert get(conn, root <> "/simple/nothing-here/").status == 404 | |
| 191 | + end | |
| 192 | + | |
| 193 | + test "a yanked release reports its reason (PEP 592)", %{conn: conn, repo: repo, root: root} do | |
| 194 | + publish(repo, "widget", "1.0", "widget-1.0.tar.gz") | |
| 195 | + pkg = Packages.get_package(repo, "pypi", "widget") | |
| 196 | + {:ok, _} = Packages.set_yanked(Packages.get_version(pkg, "1.0"), true, "bad build") | |
| 197 | + | |
| 198 | + json = | |
| 199 | + conn | |
| 200 | + |> put_req_header("accept", @json) | |
| 201 | + |> get(root <> "/simple/widget/") | |
| 202 | + |> Map.get(:resp_body) | |
| 203 | + |> Jason.decode!() | |
| 204 | + | |
| 205 | + assert [%{"yanked" => "bad build"}] = json["files"] | |
| 206 | + | |
| 207 | + assert build_conn() | |
| 208 | + |> get(root <> "/simple/widget/") | |
| 209 | + |> Map.get(:resp_body) =~ ~s(data-yanked="bad build") | |
| 210 | + end | |
| 211 | + | |
| 212 | + test "requires-python rides along in both serialisations", %{ | |
| 213 | + conn: conn, | |
| 214 | + repo: repo, | |
| 215 | + root: root | |
| 216 | + } do | |
| 217 | + {:ok, _, _, _} = | |
| 218 | + Packages.publish_file(repo, "pypi", "widget", "1.0", %{ | |
| 219 | + filename: "widget-1.0.tar.gz", | |
| 220 | + body: "sdist", | |
| 221 | + metadata: %{"requires_python" => ">=3.9"} | |
| 222 | + }) | |
| 223 | + | |
| 224 | + html = get(conn, root <> "/simple/widget/").resp_body | |
| 225 | + assert html =~ ~s(data-requires-python=">=3.9") | |
| 226 | + | |
| 227 | + json = | |
| 228 | + build_conn() | |
| 229 | + |> put_req_header("accept", @json) | |
| 230 | + |> get(root <> "/simple/widget/") | |
| 231 | + |> Map.get(:resp_body) | |
| 232 | + |> Jason.decode!() | |
| 233 | + | |
| 234 | + assert [%{"requires-python" => ">=3.9"}] = json["files"] | |
| 235 | + end | |
| 236 | + end | |
| 237 | + | |
| 238 | + describe "GET /files/<filename>" do | |
| 239 | + test "serves the stored bytes", %{conn: conn, repo: repo, root: root} do | |
| 240 | + publish(repo, "widget", "1.0", "widget-1.0.tar.gz", "the real sdist") | |
| 241 | + | |
| 242 | + conn = get(conn, root <> "/files/widget-1.0.tar.gz") | |
| 243 | + | |
| 244 | + assert conn.status == 200 | |
| 245 | + assert conn.resp_body == "the real sdist" | |
| 246 | + end | |
| 247 | + | |
| 248 | + test "404 for a filename that was never published", %{conn: conn, root: root} do | |
| 249 | + assert get(conn, root <> "/files/ghost-9.9.tar.gz").status == 404 | |
| 250 | + end | |
| 251 | + end | |
| 252 | + | |
| 253 | + describe "POST / (twine upload)" do | |
| 254 | + test "publishes a distribution", %{conn: conn, repo: repo, root: root, token: token} do | |
| 255 | + conn = upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes") | |
| 256 | + | |
| 257 | + assert conn.status == 200 | |
| 258 | + | |
| 259 | + pkg = Packages.get_package(repo, "pypi", "widget") | |
| 260 | + assert pkg.latest_version == "1.0.0" | |
| 261 | + | |
| 262 | + assert [file] = Packages.list_files(Packages.get_version(pkg, "1.0.0")) | |
| 263 | + assert file.filename == "widget-1.0.0.tar.gz" | |
| 264 | + assert {:ok, "sdist bytes"} = Packages.read_file(file) | |
| 265 | + end | |
| 266 | + | |
| 267 | + test "rejects an upload whose sha256_digest doesn't match", %{ | |
| 268 | + conn: conn, | |
| 269 | + root: root, | |
| 270 | + token: token | |
| 271 | + } do | |
| 272 | + conn = | |
| 273 | + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes", | |
| 274 | + sha256_digest: String.duplicate("0", 64) | |
| 275 | + ) | |
| 276 | + | |
| 277 | + assert conn.status == 400 | |
| 278 | + assert conn.resp_body =~ "sha256_digest" | |
| 279 | + end | |
| 280 | + | |
| 281 | + test "accepts the digest twine actually computes", %{conn: conn, root: root, token: token} do | |
| 282 | + digest = Base.encode16(:crypto.hash(:sha256, "sdist bytes"), case: :lower) | |
| 283 | + | |
| 284 | + conn = | |
| 285 | + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist bytes", | |
| 286 | + sha256_digest: digest | |
| 287 | + ) | |
| 288 | + | |
| 289 | + assert conn.status == 200 | |
| 290 | + end | |
| 291 | + | |
| 292 | + test "409 on re-uploading a filename with different bytes", %{ | |
| 293 | + conn: conn, | |
| 294 | + root: root, | |
| 295 | + token: token | |
| 296 | + } do | |
| 297 | + upload(conn, root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "first") | |
| 298 | + | |
| 299 | + conn = upload(build_conn(), root, token, "widget", "1.0.0", "widget-1.0.0.tar.gz", "second") | |
| 300 | + | |
| 301 | + assert conn.status == 409 | |
| 302 | + end | |
| 303 | + | |
| 304 | + test "400 without a name", %{conn: conn, root: root, token: token} do | |
| 305 | + conn = | |
| 306 | + conn | |
| 307 | + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{token}")) | |
| 308 | + |> post(root <> "/", %{"version" => "1.0.0"}) | |
| 309 | + | |
| 310 | + assert conn.status == 400 | |
| 311 | + assert conn.resp_body =~ "name" | |
| 312 | + end | |
| 313 | + | |
| 314 | + test "an anonymous upload is challenged, not silently dropped", %{conn: conn, root: root} do | |
| 315 | + conn = upload(conn, root, nil, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist") | |
| 316 | + | |
| 317 | + assert conn.status == 401 | |
| 318 | + assert [challenge] = get_resp_header(conn, "www-authenticate") | |
| 319 | + assert challenge =~ "Basic realm=" | |
| 320 | + end | |
| 321 | + | |
| 322 | + test "a pull-only token may read but not publish", %{conn: conn, repo: repo, root: root} do | |
| 323 | + {:ok, _t, ro} = Packages.create_registry_token(repo, "readonly", ["pull"]) | |
| 324 | + | |
| 325 | + assert upload(conn, root, ro, "widget", "1.0.0", "widget-1.0.0.tar.gz", "sdist").status == | |
| 326 | + 403 | |
| 327 | + | |
| 328 | + assert build_conn() | |
| 329 | + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{ro}")) | |
| 330 | + |> get(root <> "/simple/") | |
| 331 | + |> Map.get(:status) == 200 | |
| 332 | + end | |
| 333 | + end | |
| 334 | + | |
| 335 | + describe "visibility" do | |
| 336 | + setup do | |
| 337 | + {user, private} = repository_fixture(%{visibility: "private"}) | |
| 338 | + {:ok, _t, token} = Packages.create_registry_token(private, "ci", ["pull"]) | |
| 339 | + | |
| 340 | + {:ok, | |
| 341 | + private: private, | |
| 342 | + private_token: token, | |
| 343 | + private_root: "/pkg/pypi/#{Storage.owner_handle(user)}/#{private.name}"} | |
| 344 | + end | |
| 345 | + | |
| 346 | + test "a private index challenges an anonymous client", %{ | |
| 347 | + conn: conn, | |
| 348 | + private_root: root | |
| 349 | + } do | |
| 350 | + conn = get(conn, root <> "/simple/") | |
| 351 | + | |
| 352 | + assert conn.status == 401 | |
| 353 | + assert [_] = get_resp_header(conn, "www-authenticate") | |
| 354 | + end | |
| 355 | + | |
| 356 | + test "and serves it to a token holder — this is what pip does next", %{ | |
| 357 | + conn: conn, | |
| 358 | + private: private, | |
| 359 | + private_root: root, | |
| 360 | + private_token: token | |
| 361 | + } do | |
| 362 | + publish(private, "secret", "1.0", "secret-1.0.tar.gz") | |
| 363 | + | |
| 364 | + conn = | |
| 365 | + conn | |
| 366 | + |> put_req_header("authorization", "Basic " <> Base.encode64("__token__:#{token}")) | |
| 367 | + |> get(root <> "/simple/") | |
| 368 | + | |
| 369 | + assert conn.status == 200 | |
| 370 | + assert conn.resp_body =~ "secret" | |
| 371 | + end | |
| 372 | + end | |
| 373 | + | |
| 374 | + test "an unknown format 404s rather than reaching an adapter", %{conn: conn, root: root} do | |
| 375 | + assert get(conn, String.replace(root, "/pypi/", "/nonsense/") <> "/simple/").status == 404 | |
| 376 | + end | |
| 377 | + | |
| 378 | + test "a missing repository 404s", %{conn: conn} do | |
| 379 | + assert get(conn, "/pkg/pypi/nobody/nothing/simple/").status == 404 | |
| 380 | + end | |
| 381 | + | |
| 382 | + # twine's framing: multipart/form-data with the distribution in | |
| 383 | + # `content` and the core metadata in sibling fields. | |
| 384 | + defp upload(conn, root, token, name, version, filename, body, opts \\ []) do | |
| 385 | + path = Path.join(System.tmp_dir!(), "upload-#{System.unique_integer([:positive])}") | |
| 386 | + File.write!(path, body) | |
| 387 | + | |
| 388 | + params = | |
| 389 | + %{ | |
| 390 | + ":action" => "file_upload", | |
| 391 | + "protocol_version" => "1", | |
| 392 | + "name" => name, | |
| 393 | + "version" => version, | |
| 394 | + "filetype" => "sdist", | |
| 395 | + "metadata_version" => "2.1", | |
| 396 | + "content" => %Plug.Upload{ | |
| 397 | + path: path, | |
| 398 | + filename: filename, | |
| 399 | + content_type: "application/octet-stream" | |
| 400 | + } | |
| 401 | + } | |
| 402 | + |> Map.merge(Map.new(opts, fn {k, v} -> {to_string(k), v} end)) | |
| 403 | + | |
| 404 | + conn = | |
| 405 | + if token, | |
| 406 | + do: | |
| 407 | + put_req_header(conn, "authorization", "Basic " <> Base.encode64("__token__:#{token}")), | |
| 408 | + else: conn | |
| 409 | + | |
| 410 | + post(conn, root <> "/", params) | |
| 411 | + end | |
| 412 | +end |
Parents: d70ff9f