GITGUD

neiam neiam /gitgud

Git Gud

public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages

⭐ 1

Log in to mark this repository.

Package registries phase 3: Cargo

173e2a6 · Gabriel Morell · 2026-09-11 14:31

15 files +1021 -13
Message 
{commit_body(@commit)}

Files changed

modified lib/git_gud/packages/format.ex
+2 −1
@@ -47,7 +47,8 @@ defmodule GitGud.Packages.Format do
47 47 # format appears here as its adapter lands.
48 48 @default_adapters %{
49 49 "pypi" => GitGud.Packages.Formats.Pypi,
50 − "hex" => GitGud.Packages.Formats.Hex
50 + "hex" => GitGud.Packages.Formats.Hex,
51 + "cargo" => GitGud.Packages.Formats.Cargo
51 52 }
52 53
53 54 @typedoc """
added lib/git_gud/packages/formats/cargo.ex
+471 −0
@@ -0,0 +1,471 @@
1 +defmodule GitGud.Packages.Formats.Cargo do
2 + @moduledoc """
3 + A Cargo registry — sparse HTTP index plus the publish API.
4 +
5 + # .cargo/config.toml
6 + [registries.forge]
7 + index = "sparse+https://<host>/pkg/cargo/<owner>/<repo>/"
8 +
9 + cargo publish --registry forge
10 +
11 + The git-index form is deliberately skipped. It is tempting here of
12 + all places — this *is* a git server — but sparse has been the default
13 + since Rust 1.68, it is a plain tree of files over HTTP, and it avoids
14 + a whole second write path that would have to keep a repository in
15 + sync with the database.
16 +
17 + Paths below the repository root:
18 +
19 + /config.json index configuration
20 + /1/{name}, /2/{name}, /3/{c}/{name},
21 + /{c1c2}/{c3c4}/{name} the index entries
22 + /api/v1/crates/{n}/{v}/download the .crate bytes
23 + /api/v1/crates/new PUT to publish
24 + /api/v1/crates/{n}/{v}/yank DELETE to yank
25 + /api/v1/crates/{n}/{v}/unyank PUT to restore
26 + /api/v1/crates/{n}/owners GET/PUT/DELETE
27 + /api/v1/crates?q=&per_page= search
28 +
29 + ## The prefix layout
30 +
31 + Index files are sharded by the lowercased crate name's length: one
32 + and two-character names sit under `1/` and `2/`, three-character ones
33 + under `3/{first}/`, and everything else under `{first two}/{next
34 + two}/` — so `cargo` lives at `ca/rg/cargo`. Requests are checked
35 + against the shard the name actually belongs to, so a crate can't be
36 + served from a path it doesn't hash to.
37 +
38 + ## Caching is not a nicety here
39 +
40 + The index is read constantly — every dependency of every build, on
41 + every resolve — so `ETag`/`If-None-Match` is implemented and returns
42 + a real `304`. Bodies are small; the round trips are the cost.
43 +
44 + ## Publish framing
45 +
46 + `PUT /api/v1/crates/new` sends four things concatenated: a
47 + little-endian u32 length, that many bytes of JSON metadata, another
48 + u32, and the `.crate` tarball. Cargo labels this
49 + `Content-Type: application/json`, which it is not — hence the
50 + package upload parser running ahead of the endpoint's JSON parser, so
51 + nothing tries to decode the tarball as JSON.
52 + """
53 +
54 + @behaviour GitGud.Packages.Format
55 +
56 + alias GitGud.Packages
57 + alias GitGud.Packages.Package
58 + alias GitGud.Packages.PackageFile
59 + alias GitGud.Packages.PackageVersion
60 +
61 + @format "cargo"
62 + @json "application/json"
63 + # Cargo's index parser is newline-delimited JSON, not a JSON array.
64 + @index_type "text/plain; charset=utf-8"
65 +
66 + @impl true
67 + def content_type(_filename), do: "application/x-tar"
68 +
69 + # ── reads ───────────────────────────────────────────────────────────
70 +
71 + @impl true
72 + def index(repo, ctx)
73 +
74 + def index(repo, %{path: ["config.json"], base_url: base} = ctx) do
75 + config = %{
76 + "dl" => base <> "/api/v1/crates",
77 + "api" => base,
78 + # Only when it is actually true: setting it on a public registry
79 + # makes cargo demand a token for every fetch, including from
80 + # builds that have no business holding one.
81 + "auth-required" => repo.visibility != "public"
82 + }
83 +
84 + cached(ctx, @json, Jason.encode_to_iodata!(config))
85 + end
86 +
87 + def index(repo, %{path: ["api", "v1", "crates", name, version, "download"]}) do
88 + with %Package{} = package <- Packages.get_package(repo, @format, name),
89 + %PackageFile{} = file <-
90 + Packages.get_package_file(package, crate_filename(name, version)) do
91 + {:blob, file}
92 + else
93 + _ -> {:error, :not_found}
94 + end
95 + end
96 +
97 + def index(repo, %{path: ["api", "v1", "crates"], conn: conn} = ctx) do
98 + params = Plug.Conn.fetch_query_params(conn).query_params
99 + query = String.downcase(params["q"] || "")
100 + per_page = to_integer(params["per_page"], 10)
101 +
102 + crates =
103 + repo
104 + |> Packages.list_packages(@format)
105 + |> Enum.filter(&String.contains?(&1.normalized_name, query))
106 + |> Enum.take(per_page)
107 + |> Enum.map(fn package ->
108 + %{
109 + "name" => package.name,
110 + "max_version" => package.latest_version,
111 + "description" => description(package)
112 + }
113 + end)
114 +
115 + body = %{"crates" => crates, "meta" => %{"total" => length(crates)}}
116 + cached(ctx, @json, Jason.encode_to_iodata!(body))
117 + end
118 +
119 + def index(repo, %{path: ["api", "v1", "crates", name, "owners"]}) do
120 + repo = GitGud.Repo.preload(repo, :owner)
121 +
122 + owners = [
123 + %{
124 + "id" => repo.owner_id,
125 + "login" => owner_login(repo),
126 + "name" => owner_login(repo)
127 + }
128 + ]
129 +
130 + case Packages.get_package(repo, @format, name) do
131 + nil -> {:error, :not_found}
132 + %Package{} -> {:ok, @json, Jason.encode_to_iodata!(%{"users" => owners})}
133 + end
134 + end
135 +
136 + # Everything else is an index path, and the shard it arrived under
137 + # has to be the shard its name belongs to.
138 + def index(repo, %{path: path} = ctx) when path != [] do
139 + name = List.last(path)
140 +
141 + if index_path(name) == path do
142 + case Packages.get_package(repo, @format, name) do
143 + nil -> {:error, :not_found}
144 + %Package{} = package -> cached(ctx, @index_type, index_body(package))
145 + end
146 + else
147 + {:error, :not_found}
148 + end
149 + end
150 +
151 + def index(_repo, _ctx), do: {:error, :not_found}
152 +
153 + @doc """
154 + The sparse-index path for a crate name, as segments.
155 +
156 + iex> index_path("a") #=> ["1", "a"]
157 + iex> index_path("ab") #=> ["2", "ab"]
158 + iex> index_path("abc") #=> ["3", "a", "abc"]
159 + iex> index_path("cargo") #=> ["ca", "rg", "cargo"]
160 + """
161 + def index_path(name) when is_binary(name) do
162 + lower = String.downcase(name)
163 +
164 + case String.length(lower) do
165 + 0 -> []
166 + 1 -> ["1", lower]
167 + 2 -> ["2", lower]
168 + 3 -> ["3", String.slice(lower, 0, 1), lower]
169 + _ -> [String.slice(lower, 0, 2), String.slice(lower, 2, 2), lower]
170 + end
171 + end
172 +
173 + # ── the index document ──────────────────────────────────────────────
174 +
175 + # One JSON object per version, newline-separated, oldest first.
176 + # Only `yanked` may ever change after a line is written; everything
177 + # else is frozen at publish, because cargo caches these by version.
178 + defp index_body(%Package{} = package) do
179 + package
180 + |> Packages.list_versions()
181 + |> Enum.sort_by(& &1.pushed_at, DateTime)
182 + |> Enum.map(&[Jason.encode_to_iodata!(index_entry(package, &1)), "\n"])
183 + end
184 +
185 + defp index_entry(%Package{} = package, %PackageVersion{} = version) do
186 + metadata = version.metadata || %{}
187 + {features, features2} = split_features(metadata["features"] || %{})
188 +
189 + entry = %{
190 + "name" => package.name,
191 + "vers" => version.version,
192 + "deps" => metadata["deps"] || [],
193 + "cksum" => metadata["cksum"],
194 + "features" => features,
195 + "yanked" => version.yanked,
196 + "links" => metadata["links"],
197 + # `v: 2` is what tells cargo to look at `features2` at all. Emit
198 + # it only when there is something there to look at, so older
199 + # cargo keeps reading crates that don't need the new syntax.
200 + "v" => if(features2 == %{}, do: 1, else: 2)
201 + }
202 +
203 + entry
204 + |> maybe_put("features2", if(features2 == %{}, do: nil, else: features2))
205 + |> maybe_put("rust_version", metadata["rust_version"])
206 + end
207 +
208 + @doc """
209 + Split a feature table into the pre-1.60 half and the half that needs
210 + `features2`.
211 +
212 + A feature belongs in `features2` if any of its values uses the
213 + `dep:name` or `pkg?/feat` syntax. Older cargo ignores `features2`
214 + entirely, which is the point: it sees a crate whose remaining
215 + features it can still understand rather than a parse error.
216 + """
217 + def split_features(features) when is_map(features) do
218 + Enum.reduce(features, {%{}, %{}}, fn {name, values}, {old, new} ->
219 + values = List.wrap(values)
220 +
221 + if Enum.any?(values, &new_feature_syntax?/1),
222 + do: {old, Map.put(new, name, values)},
223 + else: {Map.put(old, name, values), new}
224 + end)
225 + end
226 +
227 + def split_features(_), do: {%{}, %{}}
228 +
229 + defp new_feature_syntax?(value) when is_binary(value),
230 + do: String.starts_with?(value, "dep:") or String.contains?(value, "?/")
231 +
232 + defp new_feature_syntax?(_), do: false
233 +
234 + defp description(%Package{} = package) do
235 + case Packages.list_versions(package) do
236 + [%PackageVersion{metadata: %{"description" => description}} | _] -> description
237 + _ -> nil
238 + end
239 + end
240 +
241 + defp owner_login(repo) do
242 + case repo.owner do
243 + %{handle: handle} when is_binary(handle) and handle != "" -> handle
244 + %{email: email} when is_binary(email) -> email |> String.split("@") |> List.first()
245 + _ -> "owner"
246 + end
247 + end
248 +
249 + # ── conditional requests ────────────────────────────────────────────
250 +
251 + # The index is refetched constantly, so a 304 is the common case
252 + # rather than an optimisation. The body's own hash is the validator:
253 + # it changes exactly when the answer changes, which `updated_at`
254 + # timestamps across several rows do not.
255 + defp cached(ctx, content_type, body) do
256 + etag =
257 + ~s("#{:crypto.hash(:sha256, body) |> Base.encode16(case: :lower) |> binary_part(0, 32)}")
258 +
259 + if etag in if_none_match(ctx.conn) do
260 + :not_modified
261 + else
262 + {:ok, content_type, body, [{"etag", etag}, {"cache-control", "public, max-age=0"}]}
263 + end
264 + end
265 +
266 + defp if_none_match(conn) do
267 + conn
268 + |> Plug.Conn.get_req_header("if-none-match")
269 + |> Enum.flat_map(&String.split(&1, ","))
270 + |> Enum.map(&String.trim/1)
271 + # A revalidation that survived a proxy may come back weak.
272 + |> Enum.map(&String.replace_prefix(&1, "W/", ""))
273 + end
274 +
275 + # ── writes ──────────────────────────────────────────────────────────
276 +
277 + @impl true
278 + def store(repo, ctx, body)
279 +
280 + def store(repo, %{method: "PUT", path: ["api", "v1", "crates", "new"]} = ctx, body)
281 + when is_binary(body) do
282 + with {:ok, metadata, crate} <- unframe(body),
283 + {:ok, name} <- fetch(metadata, "name"),
284 + {:ok, version} <- fetch(metadata, "vers") do
285 + attrs = %{
286 + filename: crate_filename(name, version),
287 + body: crate,
288 + content_type: content_type(name),
289 + version_metadata: version_metadata(metadata, crate),
290 + uploader: uploader(ctx)
291 + }
292 +
293 + case Packages.publish_file(repo, @format, name, version, attrs) do
294 + {:ok, _package, _version, _file} ->
295 + # Cargo prints whatever comes back under `warnings.other`,
296 + # and requires all three keys to be present.
297 + {:ok, @json,
298 + Jason.encode_to_iodata!(%{
299 + "warnings" => %{"invalid_categories" => [], "invalid_badges" => [], "other" => []}
300 + })}
301 +
302 + {:error, :file_exists} ->
303 + {:error, {409, "#{name} #{version} is already published"}}
304 +
305 + {:error, reason} ->
306 + {:error, {400, inspect(reason)}}
307 + end
308 + end
309 + end
310 +
311 + def store(
312 + repo,
313 + %{method: "DELETE", path: ["api", "v1", "crates", name, version, "yank"]},
314 + _body
315 + ),
316 + do: set_yanked(repo, name, version, true)
317 +
318 + def store(
319 + repo,
320 + %{method: "PUT", path: ["api", "v1", "crates", name, version, "unyank"]},
321 + _body
322 + ),
323 + do: set_yanked(repo, name, version, false)
324 +
325 + # Ownership lives in the forge's own permission model — a repository's
326 + # owner and organization members. Accepting a change here would make
327 + # cargo's view of who may publish diverge from the forge's, so the
328 + # endpoint answers truthfully instead of pretending.
329 + def store(_repo, %{method: method, path: ["api", "v1", "crates", _name, "owners"]}, _body)
330 + when method in ["PUT", "DELETE"] do
331 + {:error,
332 + {403, "crate ownership follows the repository's collaborators; change it in the forge"}}
333 + end
334 +
335 + def store(_repo, %{method: method, path: path}, _body),
336 + do: {:error, {404, "no #{method} endpoint at /#{Enum.join(path, "/")}"}}
337 +
338 + defp set_yanked(repo, name, version, yanked?) do
339 + with %Package{} = package <- Packages.get_package(repo, @format, name),
340 + %PackageVersion{} = row <- Packages.get_version(package, version),
341 + {:ok, _} <- Packages.set_yanked(row, yanked?) do
342 + {:ok, @json, Jason.encode_to_iodata!(%{"ok" => true})}
343 + else
344 + nil -> {:error, {404, "#{name} #{version} is not published here"}}
345 + {:error, reason} -> {:error, {400, inspect(reason)}}
346 + end
347 + end
348 +
349 + @doc """
350 + Split the publish body into its metadata and its `.crate`.
351 +
352 + u32 little-endian metadata length
353 + <n bytes> JSON metadata
354 + u32 little-endian .crate length
355 + <n bytes> the tarball
356 + """
357 + def unframe(body) do
358 + case body do
359 + <<meta_len::little-unsigned-32, rest::binary>> when byte_size(rest) >= meta_len ->
360 + <<meta_json::binary-size(^meta_len), rest::binary>> = rest
361 +
362 + case rest do
363 + <<crate_len::little-unsigned-32, crate::binary>> when byte_size(crate) >= crate_len ->
364 + <<crate::binary-size(^crate_len), _trailing::binary>> = crate
365 + decode_metadata(meta_json, crate)
366 +
367 + _ ->
368 + {:error, {400, "the publish body's crate length prefix is truncated"}}
369 + end
370 +
371 + _ ->
372 + {:error, {400, "the publish body's metadata length prefix is truncated"}}
373 + end
374 + end
375 +
376 + defp decode_metadata(json, crate) do
377 + case Jason.decode(json) do
378 + {:ok, metadata} when is_map(metadata) -> {:ok, metadata, crate}
379 + _ -> {:error, {400, "the publish body's metadata is not a JSON object"}}
380 + end
381 + end
382 +
383 + defp version_metadata(metadata, crate) do
384 + %{
385 + "cksum" => Base.encode16(:crypto.hash(:sha256, crate), case: :lower),
386 + "deps" => Enum.map(List.wrap(metadata["deps"]), &index_dependency/1),
387 + "features" => metadata["features"] || %{},
388 + "links" => metadata["links"],
389 + "rust_version" => rust_version(metadata["rust_version"]),
390 + "description" => metadata["description"],
391 + "documentation" => metadata["documentation"],
392 + "homepage" => metadata["homepage"],
393 + "repository" => metadata["repository"],
394 + "license" => metadata["license"],
395 + "keywords" => metadata["keywords"] || [],
396 + "categories" => metadata["categories"] || []
397 + }
398 + end
399 +
400 + # The publish payload and the index disagree on how to name a
401 + # dependency, and the translation is the whole of it: publish sends
402 + # the real crate in `name` with any rename in `explicit_name_in_toml`;
403 + # the index wants the *renamed* name in `name` and the real crate in
404 + # `package`. Getting this backwards resolves the wrong crate.
405 + defp index_dependency(dep) when is_map(dep) do
406 + real = dep["name"]
407 + renamed = blank_to_nil(dep["explicit_name_in_toml"])
408 +
409 + %{
410 + "name" => renamed || real,
411 + "package" => if(renamed, do: real),
412 + "req" => dep["version_req"] || dep["req"],
413 + "features" => dep["features"] || [],
414 + "optional" => dep["optional"] || false,
415 + "default_features" => Map.get(dep, "default_features", true),
416 + "target" => dep["target"],
417 + "kind" => dep["kind"] || "normal",
418 + "registry" => dep["registry"]
419 + }
420 + end
421 +
422 + defp index_dependency(_), do: %{}
423 +
424 + # `rust_version` in the index is a bare version, never a requirement —
425 + # `1.60`, not `=1.60`. Cargo compares it directly.
426 + defp rust_version(nil), do: nil
427 +
428 + defp rust_version(value) when is_binary(value) do
429 + case String.trim_leading(value, "=") |> String.trim() do
430 + "" -> nil
431 + trimmed -> String.replace_leading(trimmed, "^", "")
432 + end
433 + end
434 +
435 + defp rust_version(_), do: nil
436 +
437 + # ── helpers ─────────────────────────────────────────────────────────
438 +
439 + defp crate_filename(name, version), do: "#{name}-#{version}.crate"
440 +
441 + defp fetch(metadata, key) do
442 + case metadata[key] do
443 + value when is_binary(value) and value != "" -> {:ok, value}
444 + _ -> {:error, {400, "the publish metadata has no `#{key}`"}}
445 + end
446 + end
447 +
448 + defp maybe_put(map, _key, nil), do: map
449 + defp maybe_put(map, key, value), do: Map.put(map, key, value)
450 +
451 + defp blank_to_nil(value) when is_binary(value) do
452 + case String.trim(value) do
453 + "" -> nil
454 + trimmed -> trimmed
455 + end
456 + end
457 +
458 + defp blank_to_nil(_), do: nil
459 +
460 + defp to_integer(value, default) when is_binary(value) do
461 + case Integer.parse(value) do
462 + {n, _} when n > 0 -> min(n, 100)
463 + _ -> default
464 + end
465 + end
466 +
467 + defp to_integer(_, default), do: default
468 +
469 + defp uploader(%{principal: {:user, user}}), do: user
470 + defp uploader(_ctx), do: nil
471 +end
modified lib/git_gud_web/live/repo_live/packages.ex
+10 −0
@@ -205,6 +205,12 @@ defmodule GitGudWeb.RepoLive.Packages do
205 205 "mix hex.repo add #{hex_repo} #{url} --public-key=<(curl -s #{url}/public_key)"
206 206 end
207 207
208 + # Sparse, not git — the default since Rust 1.68 and a plain tree of
209 + # files over HTTP.
210 + defp configure_line("cargo", %{handle: handle, repo_name: name, root: root}) do
211 + ~s([registries.#{name}] index = "sparse+#{root}/pkg/cargo/#{handle}/#{name}/")
212 + end
213 +
208 214 defp configure_line(_format, _registry), do: nil
209 215
210 216 # Reconstruct the zot repository path the webhook deflated: a package
@@ -225,6 +231,10 @@ defmodule GitGudWeb.RepoLive.Packages do
225 231 ~s[{:#{pkg.name}, "~> #{version}", repo: "#{hex_repo}"}]
226 232 end
227 233
234 + defp install_line("cargo", %{repo_name: name}, pkg, version) do
235 + "cargo add #{pkg.name}@#{version} --registry #{name}"
236 + end
237 +
228 238 defp install_line(_format, _registry, _pkg, _version), do: nil
229 239
230 240 defp short_digest("sha256:" <> rest),
modified lib/git_gud_web/plugs/package_upload_parser.ex
+34 −12
@@ -1,16 +1,27 @@
1 1 defmodule GitGudWeb.Plugs.PackageUploadParser do
2 2 @moduledoc """
3 − Parses package-registry upload bodies before the endpoint's own
4 − `Plug.Parsers` gets to them, so publishing a fat wheel isn't capped
5 − by the 8 MB limit that suits ordinary form posts.
6 −
7 − Only multipart is handled here — that is twine's framing. Cargo's
8 − length-prefixed `application/octet-stream` and Hex's raw tarball
9 − fall through `pass: ["*/*"]` untouched and are read off the socket
10 − by `GitGudWeb.PackageRegistryController`.
11 −
12 − `Plug.Parsers` is a no-op once `body_params` has been fetched, so
13 − the endpoint's instance leaves whatever this one did alone.
3 + Decides who owns the request body for a package-registry upload,
4 + ahead of the endpoint's general `Plug.Parsers`.
5 +
6 + Two shapes arrive:
7 +
8 + * **multipart** — twine's framing. Parsed here rather than
9 + downstream so it gets its own, much larger, size limit; 8 MB
10 + suits an ordinary form post and not a wheel with a bundled CUDA
11 + runtime.
12 +
13 + * **everything else** — Cargo's length-prefixed publish body and
14 + Hex's raw tarball. These are claimed for the controller to read
15 + off the socket. Claiming them matters: Cargo labels its binary
16 + publish body `Content-Type: application/json`, and without this
17 + the endpoint's JSON parser would try to decode a tarball and
18 + fail the request before it ever reached an adapter.
19 +
20 + `Plug.Parsers` treats a fetched `body_params` as "someone else
21 + handled this" and no-ops, which is how the claim is made. It does
22 + *not* set `body_params` itself when a body merely passes its `pass`
23 + list, so setting it here is the whole mechanism rather than a
24 + belt-and-braces extra.
14 25 """
15 26
16 27 @behaviour Plug
@@ -31,8 +42,19 @@ defmodule GitGudWeb.Plugs.PackageUploadParser do
31 42 @impl true
32 43 def call(%Plug.Conn{path_info: ["pkg" | _], method: method} = conn, _opts)
33 44 when method in ["POST", "PUT", "PATCH"] do
34 − Plug.Parsers.call(conn, @parser_opts)
45 + if multipart?(conn),
46 + do: Plug.Parsers.call(conn, @parser_opts),
47 + else: %{conn | body_params: %{}}
35 48 end
36 49
37 50 def call(conn, _opts), do: conn
51 +
52 + defp multipart?(conn) do
53 + with [value | _] <- Plug.Conn.get_req_header(conn, "content-type"),
54 + {:ok, "multipart", _subtype, _params} <- Plug.Conn.Utils.content_type(value) do
55 + true
56 + else
57 + _ -> false
58 + end
59 + end
38 60 end
added test/git_gud_web/cargo_registry_test.exs
+504 −0
@@ -0,0 +1,504 @@
1 +defmodule GitGudWeb.CargoRegistryTest do
2 + @moduledoc """
3 + Phase 3: the Cargo sparse index and publish API, over HTTP.
4 +
5 + The interesting assertions are the ones where a mistake is silent
6 + rather than loud — the prefix shard, the dependency rename
7 + translation, `features2`/`v: 2`, and the 304.
8 + """
9 + use GitGudWeb.ConnCase, async: false
10 +
11 + alias GitGud.Packages
12 + alias GitGud.Packages.Formats.Cargo
13 + alias GitGud.Repositories.Storage
14 +
15 + import GitGud.ForgeFixtures
16 +
17 + setup do
18 + {user, repo} = repository_fixture(%{visibility: "public"})
19 + {:ok, _t, token} = Packages.create_registry_token(repo, "ci", ["pull", "push"])
20 +
21 + {:ok,
22 + user: user,
23 + repo: repo,
24 + token: token,
25 + root: "/pkg/cargo/#{Storage.owner_handle(user)}/#{repo.name}"}
26 + end
27 +
28 + # `cargo publish`'s body: u32 LE length, JSON metadata, u32 LE
29 + # length, the .crate tarball.
30 + defp publish_body(metadata, crate) do
31 + json = Jason.encode!(metadata)
32 +
33 + <<byte_size(json)::little-unsigned-32, json::binary, byte_size(crate)::little-unsigned-32,
34 + crate::binary>>
35 + end
36 +
37 + defp publish(conn, root, token, metadata, crate \\ "crate bytes") do
38 + conn =
39 + if token, do: put_req_header(conn, "authorization", token), else: conn
40 +
41 + conn
42 + # Cargo really does label this JSON. It is not JSON.
43 + |> put_req_header("content-type", "application/json")
44 + |> put_req_header("accept", "application/json")
45 + |> put(root <> "/api/v1/crates/new", publish_body(metadata, crate))
46 + end
47 +
48 + defp meta(name, vers, extra \\ %{}) do
49 + Map.merge(%{"name" => name, "vers" => vers, "deps" => [], "features" => %{}}, extra)
50 + end
51 +
52 + defp index_lines(conn, root, name) do
53 + path = Enum.join(Cargo.index_path(name), "/")
54 + conn = get(conn, root <> "/" <> path)
55 +
56 + {conn,
57 + conn.resp_body
58 + |> String.split("\n", trim: true)
59 + |> Enum.map(&Jason.decode!/1)}
60 + end
61 +
62 + describe "index_path/1" do
63 + test "shards by the lowercased name's length" do
64 + assert Cargo.index_path("a") == ["1", "a"]
65 + assert Cargo.index_path("ab") == ["2", "ab"]
66 + assert Cargo.index_path("abc") == ["3", "a", "abc"]
67 + assert Cargo.index_path("cargo") == ["ca", "rg", "cargo"]
68 + assert Cargo.index_path("Serde") == ["se", "rd", "serde"]
69 + end
70 + end
71 +
72 + describe "config.json" do
73 + test "points cargo at the download and API roots", %{conn: conn, root: root} do
74 + conn = get(conn, root <> "/config.json")
75 +
76 + assert conn.status == 200
77 + config = Jason.decode!(conn.resp_body)
78 + assert String.ends_with?(config["dl"], "/api/v1/crates")
79 + assert String.ends_with?(config["api"], root)
80 + end
81 +
82 + test "a public registry does not demand a token for every fetch", %{conn: conn, root: root} do
83 + assert Jason.decode!(get(conn, root <> "/config.json").resp_body)["auth-required"] == false
84 + end
85 +
86 + test "a private one does" do
87 + {user, private} = repository_fixture(%{visibility: "private"})
88 + {:ok, _t, token} = Packages.create_registry_token(private, "ci", ["pull"])
89 + root = "/pkg/cargo/#{Storage.owner_handle(user)}/#{private.name}"
90 +
91 + conn =
92 + build_conn()
93 + |> put_req_header("authorization", token)
94 + |> get(root <> "/config.json")
95 +
96 + assert Jason.decode!(conn.resp_body)["auth-required"] == true
97 + end
98 + end
99 +
100 + describe "the index" do
101 + test "serves one JSON object per version, newline-delimited", %{
102 + conn: conn,
103 + root: root,
104 + token: token
105 + } do
106 + publish(conn, root, token, meta("widget", "1.0.0"))
107 + publish(build_conn(), root, token, meta("widget", "1.1.0"))
108 +
109 + {conn, lines} = index_lines(build_conn(), root, "widget")
110 +
111 + assert conn.status == 200
112 + assert get_resp_header(conn, "content-type") == ["text/plain; charset=utf-8"]
113 + assert Enum.map(lines, & &1["vers"]) == ["1.0.0", "1.1.0"]
114 + end
115 +
116 + test "cksum is the sha256 of the .crate", %{conn: conn, root: root, token: token} do
117 + publish(conn, root, token, meta("widget", "1.0.0"), "the real crate")
118 +
119 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
120 +
121 + assert entry["cksum"] ==
122 + Base.encode16(:crypto.hash(:sha256, "the real crate"), case: :lower)
123 + end
124 +
125 + test "a crate served from the wrong shard is a 404", %{conn: conn, root: root, token: token} do
126 + publish(conn, root, token, meta("widget", "1.0.0"))
127 +
128 + # widget's real shard is wi/dg/widget.
129 + assert get(build_conn(), root <> "/wi/dg/widget").status == 200
130 + assert get(build_conn(), root <> "/3/w/widget").status == 404
131 + assert get(build_conn(), root <> "/xx/yy/widget").status == 404
132 + end
133 +
134 + test "404 for a crate nobody published", %{conn: conn, root: root} do
135 + assert get(conn, root <> "/no/th/nothing").status == 404
136 + end
137 +
138 + test "only `yanked` changes after publication", %{
139 + conn: conn,
140 + repo: repo,
141 + root: root,
142 + token: token
143 + } do
144 + publish(conn, root, token, meta("widget", "1.0.0"))
145 + {_conn, [before]} = index_lines(build_conn(), root, "widget")
146 +
147 + pkg = Packages.get_package(repo, "cargo", "widget")
148 + {:ok, _} = Packages.set_yanked(Packages.get_version(pkg, "1.0.0"), true)
149 +
150 + {_conn, [after_yank]} = index_lines(build_conn(), root, "widget")
151 +
152 + assert before["yanked"] == false
153 + assert after_yank["yanked"] == true
154 + assert Map.delete(before, "yanked") == Map.delete(after_yank, "yanked")
155 + end
156 + end
157 +
158 + describe "features2 and v" do
159 + test "a crate with only old-style features stays at v: 1", %{
160 + conn: conn,
161 + root: root,
162 + token: token
163 + } do
164 + publish(
165 + conn,
166 + root,
167 + token,
168 + meta("widget", "1.0.0", %{"features" => %{"default" => ["std"], "std" => []}})
169 + )
170 +
171 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
172 +
173 + assert entry["v"] == 1
174 + refute Map.has_key?(entry, "features2")
175 + assert entry["features"] == %{"default" => ["std"], "std" => []}
176 + end
177 +
178 + test "`dep:` and `pkg?/feat` move to features2 and bump v to 2", %{
179 + conn: conn,
180 + root: root,
181 + token: token
182 + } do
183 + features = %{
184 + "default" => ["std"],
185 + "serde" => ["dep:serde"],
186 + "json" => ["serde?/derive"]
187 + }
188 +
189 + publish(conn, root, token, meta("widget", "1.0.0", %{"features" => features}))
190 +
191 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
192 +
193 + assert entry["v"] == 2
194 + # Old cargo reads `features` and ignores `features2` — so the
195 + # features it can still understand must stay where it looks.
196 + assert entry["features"] == %{"default" => ["std"]}
197 + assert entry["features2"] == %{"serde" => ["dep:serde"], "json" => ["serde?/derive"]}
198 + end
199 +
200 + test "split_features/1 partitions on the new syntax" do
201 + assert Cargo.split_features(%{"a" => ["b"], "c" => ["dep:d"], "e" => ["f?/g"]}) ==
202 + {%{"a" => ["b"]}, %{"c" => ["dep:d"], "e" => ["f?/g"]}}
203 + end
204 + end
205 +
206 + describe "dependencies" do
207 + test "a renamed dependency swaps name and package", %{
208 + conn: conn,
209 + root: root,
210 + token: token
211 + } do
212 + deps = [
213 + %{
214 + "name" => "serde",
215 + "explicit_name_in_toml" => "serde1",
216 + "version_req" => "^1.0",
217 + "features" => ["derive"],
218 + "optional" => true,
219 + "default_features" => false,
220 + "target" => "cfg(unix)",
221 + "kind" => "normal",
222 + "registry" => "https://example.invalid"
223 + }
224 + ]
225 +
226 + publish(conn, root, token, meta("widget", "1.0.0", %{"deps" => deps}))
227 +
228 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
229 +
230 + assert [dep] = entry["deps"]
231 + # The index names the dependency as the crate refers to it…
232 + assert dep["name"] == "serde1"
233 + # …and says which crate that actually is.
234 + assert dep["package"] == "serde"
235 + # `version_req` on the way in, `req` on the way out.
236 + assert dep["req"] == "^1.0"
237 + assert dep["features"] == ["derive"]
238 + assert dep["optional"] == true
239 + assert dep["default_features"] == false
240 + assert dep["target"] == "cfg(unix)"
241 + assert dep["kind"] == "normal"
242 + end
243 +
244 + test "an unrenamed dependency has a null package", %{conn: conn, root: root, token: token} do
245 + deps = [%{"name" => "serde", "version_req" => "^1.0"}]
246 + publish(conn, root, token, meta("widget", "1.0.0", %{"deps" => deps}))
247 +
248 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
249 +
250 + assert [%{"name" => "serde", "package" => nil, "default_features" => true}] = entry["deps"]
251 + end
252 + end
253 +
254 + describe "rust_version" do
255 + test "is stored without an operator", %{conn: conn, root: root, token: token} do
256 + publish(conn, root, token, meta("widget", "1.0.0", %{"rust_version" => "=1.60"}))
257 +
258 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
259 +
260 + assert entry["rust_version"] == "1.60"
261 + end
262 +
263 + test "is absent when the crate doesn't declare one", %{conn: conn, root: root, token: token} do
264 + publish(conn, root, token, meta("widget", "1.0.0"))
265 +
266 + {_conn, [entry]} = index_lines(build_conn(), root, "widget")
267 +
268 + refute Map.has_key?(entry, "rust_version")
269 + end
270 + end
271 +
272 + describe "caching" do
273 + test "an unchanged index revalidates to 304", %{conn: conn, root: root, token: token} do
274 + publish(conn, root, token, meta("widget", "1.0.0"))
275 +
276 + {first, _} = index_lines(build_conn(), root, "widget")
277 + assert [etag] = get_resp_header(first, "etag")
278 +
279 + revalidated =
280 + build_conn()
281 + |> put_req_header("if-none-match", etag)
282 + |> get(root <> "/wi/dg/widget")
283 +
284 + assert revalidated.status == 304
285 + assert revalidated.resp_body == ""
286 + end
287 +
288 + test "a weak validator from a proxy still matches", %{conn: conn, root: root, token: token} do
289 + publish(conn, root, token, meta("widget", "1.0.0"))
290 + {first, _} = index_lines(build_conn(), root, "widget")
291 + [etag] = get_resp_header(first, "etag")
292 +
293 + revalidated =
294 + build_conn()
295 + |> put_req_header("if-none-match", "W/" <> etag)
296 + |> get(root <> "/wi/dg/widget")
297 +
298 + assert revalidated.status == 304
299 + end
300 +
301 + test "publishing a new version invalidates it", %{conn: conn, root: root, token: token} do
302 + publish(conn, root, token, meta("widget", "1.0.0"))
303 + {first, _} = index_lines(build_conn(), root, "widget")
304 + [etag] = get_resp_header(first, "etag")
305 +
306 + publish(build_conn(), root, token, meta("widget", "1.1.0"))
307 +
308 + refreshed =
309 + build_conn()
310 + |> put_req_header("if-none-match", etag)
311 + |> get(root <> "/wi/dg/widget")
312 +
313 + assert refreshed.status == 200
314 + end
315 + end
316 +
317 + describe "download" do
318 + test "serves the .crate bytes", %{conn: conn, root: root, token: token} do
319 + publish(conn, root, token, meta("widget", "1.0.0"), "the real crate")
320 +
321 + conn = get(build_conn(), root <> "/api/v1/crates/widget/1.0.0/download")
322 +
323 + assert conn.status == 200
324 + assert conn.resp_body == "the real crate"
325 + end
326 +
327 + test "404 for a version that was never published", %{conn: conn, root: root, token: token} do
328 + publish(conn, root, token, meta("widget", "1.0.0"))
329 +
330 + assert get(build_conn(), root <> "/api/v1/crates/widget/9.9.9/download").status == 404
331 + end
332 + end
333 +
334 + describe "publish" do
335 + test "answers with the warnings object cargo expects", %{
336 + conn: conn,
337 + root: root,
338 + token: token
339 + } do
340 + conn = publish(conn, root, token, meta("widget", "1.0.0"))
341 +
342 + assert conn.status == 200
343 +
344 + assert %{"warnings" => %{"invalid_categories" => [], "invalid_badges" => [], "other" => []}} =
345 + Jason.decode!(conn.resp_body)
346 + end
347 +
348 + test "a truncated metadata prefix is a cargo-shaped error", %{
349 + conn: conn,
350 + root: root,
351 + token: token
352 + } do
353 + conn =
354 + conn
355 + |> put_req_header("authorization", token)
356 + |> put_req_header("content-type", "application/json")
357 + |> put(root <> "/api/v1/crates/new", <<255, 255, 255, 255, 1, 2>>)
358 +
359 + assert conn.status == 400
360 + assert %{"errors" => [%{"detail" => detail}]} = Jason.decode!(conn.resp_body)
361 + assert detail =~ "metadata length prefix"
362 + end
363 +
364 + test "a truncated crate prefix is caught too", %{conn: conn, root: root, token: token} do
365 + json = Jason.encode!(meta("widget", "1.0.0"))
366 +
367 + conn =
368 + conn
369 + |> put_req_header("authorization", token)
370 + |> put_req_header("content-type", "application/json")
371 + |> put(
372 + root <> "/api/v1/crates/new",
373 + <<byte_size(json)::little-unsigned-32, json::binary, 255, 255, 255, 255>>
374 + )
375 +
376 + assert conn.status == 400
377 + assert Jason.decode!(conn.resp_body)["errors"] != []
378 + end
379 +
380 + test "409 on republishing the same version with different bytes", %{
381 + conn: conn,
382 + root: root,
383 + token: token
384 + } do
385 + publish(conn, root, token, meta("widget", "1.0.0"), "first")
386 + conn = publish(build_conn(), root, token, meta("widget", "1.0.0"), "second")
387 +
388 + assert conn.status == 409
389 + end
390 +
391 + test "an anonymous publish is refused", %{conn: conn, root: root} do
392 + assert publish(conn, root, nil, meta("widget", "1.0.0")).status == 401
393 + end
394 + end
395 +
396 + describe "yank and unyank" do
397 + test "round-trip without removing the bytes", %{
398 + conn: conn,
399 + repo: repo,
400 + root: root,
401 + token: token
402 + } do
403 + publish(conn, root, token, meta("widget", "1.0.0"), "the real crate")
404 +
405 + yanked =
406 + build_conn()
407 + |> put_req_header("authorization", token)
408 + |> delete(root <> "/api/v1/crates/widget/1.0.0/yank")
409 +
410 + assert yanked.status == 200
411 + assert Jason.decode!(yanked.resp_body) == %{"ok" => true}
412 +
413 + pkg = Packages.get_package(repo, "cargo", "widget")
414 + assert Packages.get_version(pkg, "1.0.0").yanked
415 +
416 + # A build that already pinned this version must still resolve it.
417 + assert get(build_conn(), root <> "/api/v1/crates/widget/1.0.0/download").resp_body ==
418 + "the real crate"
419 +
420 + unyanked =
421 + build_conn()
422 + |> put_req_header("authorization", token)
423 + |> put(root <> "/api/v1/crates/widget/1.0.0/unyank")
424 +
425 + assert unyanked.status == 200
426 + refute Packages.get_version(pkg, "1.0.0").yanked
427 + end
428 +
429 + test "yanking needs write access", %{conn: conn, repo: repo, root: root, token: token} do
430 + publish(conn, root, token, meta("widget", "1.0.0"))
431 + {:ok, _t, readonly} = Packages.create_registry_token(repo, "ro", ["pull"])
432 +
433 + conn =
434 + build_conn()
435 + |> put_req_header("authorization", readonly)
436 + |> delete(root <> "/api/v1/crates/widget/1.0.0/yank")
437 +
438 + assert conn.status == 403
439 + end
440 +
441 + test "404 for an unpublished version", %{conn: conn, root: root, token: token} do
442 + conn =
443 + conn
444 + |> put_req_header("authorization", token)
445 + |> delete(root <> "/api/v1/crates/ghost/1.0.0/yank")
446 +
447 + assert conn.status == 404
448 + end
449 + end
450 +
451 + describe "search and owners" do
452 + test "search matches on the normalized name", %{conn: conn, root: root, token: token} do
453 + publish(conn, root, token, meta("widget", "1.0.0", %{"description" => "a widget"}))
454 + publish(build_conn(), root, token, meta("gadget", "2.0.0"))
455 +
456 + body = get(build_conn(), root <> "/api/v1/crates?q=WID").resp_body |> Jason.decode!()
457 +
458 + assert [%{"name" => "widget", "max_version" => "1.0.0", "description" => "a widget"}] =
459 + body["crates"]
460 +
461 + assert body["meta"]["total"] == 1
462 + end
463 +
464 + test "search with no query lists everything, capped by per_page", %{
465 + conn: conn,
466 + root: root,
467 + token: token
468 + } do
469 + for n <- ["alpha", "beta", "gamma"],
470 + do: publish(build_conn(), root, token, meta(n, "1.0.0"))
471 +
472 + body = get(conn, root <> "/api/v1/crates?per_page=2").resp_body |> Jason.decode!()
473 +
474 + assert length(body["crates"]) == 2
475 + end
476 +
477 + test "owners reports the repository's owner", %{conn: conn, root: root, token: token} do
478 + publish(conn, root, token, meta("widget", "1.0.0"))
479 +
480 + body =
481 + get(build_conn(), root <> "/api/v1/crates/widget/owners").resp_body |> Jason.decode!()
482 +
483 + assert [%{"login" => login}] = body["users"]
484 + assert is_binary(login)
485 + end
486 +
487 + test "changing owners is refused — the forge is the source of truth", %{
488 + conn: conn,
489 + root: root,
490 + token: token
491 + } do
492 + publish(conn, root, token, meta("widget", "1.0.0"))
493 +
494 + conn =
495 + build_conn()
496 + |> put_req_header("authorization", token)
497 + |> put_req_header("content-type", "application/json")
498 + |> put(root <> "/api/v1/crates/widget/owners", Jason.encode!(%{"users" => ["someone"]}))
499 +
500 + assert conn.status == 403
501 + assert Jason.decode!(conn.resp_body)["errors"] != []
502 + end
503 + end
504 +end

Parents: 4869a7f