defmodule GitGud.Packages.Formats.Deb do
@moduledoc """
A Debian repository, per git repository *and* per namespace.
# everything acme publishes — one line, one key
deb [signed-by=/etc/apt/keyrings/acme.asc] \\
https://<host>/pkg/deb/acme/_all stable main
# or one repository on its own
deb [signed-by=/etc/apt/keyrings/acme.asc] \\
https://<host>/pkg/deb/acme/tools stable main
There is no daemon to embed here. An apt repository is static files
over plain HTTP — which is why this is last rather than hard: the
only genuinely new thing is the OpenPGP signature, and that lives in
`GitGud.Packages.Apt.Native`.
## `_all`, and why the URL looks like that
A namespace usually *is* the unit a machine wants. Five repositories
under one organization are five `deb` lines and five `Packages`
fetches, for what an administrator thinks of as "acme's packages".
`_all` collapses them into one repository that aggregates every
repository in the namespace the caller may read.
It sits in the repository slot because a repository name must start
with an alphanumeric (`Repository`'s name format), so `_all` can
never be a repository and the two can share a URL space. The same
marker is available to any other format that later wants a
namespace-wide index.
A namespace repository is filtered per caller: anonymous sees only
public repositories, a user sees what they can read, a deploy token
sees its own repository plus anything public. The `Release` is signed
at request time, so a filtered view is still a correctly signed one.
Paths below either root:
/key.asc the signing key
/dists/<dist>/InRelease clearsigned, preferred
/dists/<dist>/Release the same, unsigned
/dists/<dist>/Release.gpg detached, for older apt
/dists/<dist>/<comp>/binary-<arch>/Packages
/dists/<dist>/<comp>/binary-<arch>/Packages.xz
/pool/<comp>/<repo>/<p>/<pkg>/<file>.deb
/upload?dist=&component= POST a .deb (per repo)
## What the spec actually requires
* `Release` must carry Suite and/or Codename, Architectures,
Components, Date, and SHA256. Clients are required to reject
MD5Sum and SHA1 as security, so SHA256 is the floor and nothing
weaker is offered for that purpose.
* Checksums and sizes are listed for the *uncompressed* index too,
not only the compressed one.
* A `Packages` stanza needs Package, Filename, Size and a checksum
— and `Description-md5`, which apt has required since 2.7.7.
* Servers should offer xz; clients are required to support it.
## arch:all
Packages built `Architecture: all` are served twice: in
`binary-all/Packages` and folded into every concrete architecture's
index. Debian does both, and an apt that only looks at
`binary-amd64` still finds them.
"""
@behaviour GitGud.Packages.Format
alias GitGud.Packages
alias GitGud.Packages.Apt.Native
alias GitGud.Packages.Formats.Deb.Control
alias GitGud.Packages.PackageFile
alias GitGud.Repositories.Repository
alias GitGud.Repositories.Storage
@format "deb"
@default_dist "stable"
@default_component "main"
# `Architecture: all` is not an architecture, it is the absence of
# one, and it appears in every architecture's index.
@arch_all "all"
@impl true
def content_type(_filename), do: "application/vnd.debian.binary-package"
# ── reads ───────────────────────────────────────────────────────────
@impl true
def index(repo, ctx), do: serve([repo], repo_label(repo), repo, ctx)
@doc """
The namespace-wide repository at `_all`: the same documents, built
from every repository in the namespace the caller may read.
`signing_scope` is the namespace itself, so the aggregate is signed
with exactly the key that signs each repository individually — one
keyring entry covers both ways of consuming it.
"""
@impl true
def namespace_index(namespace, repos, ctx),
do: serve(repos, Storage.owner_handle(namespace), namespace, ctx)
# `signing_scope` is whatever `Packages.signing_key/2` should key on —
# a repository resolves to its namespace anyway, so both callers end
# up at the same key.
defp serve(repos, label, signing_scope, ctx)
defp serve(_repos, _label, scope, %{path: ["key.asc"]}) do
with {:ok, %{public_pem: armored}} <- Packages.signing_key(scope, @format) do
{:ok, "application/pgp-keys", armored}
end
end
defp serve(repos, label, scope, %{path: ["dists", dist, "InRelease"]}) do
sign(scope, &Native.sign_clearsign/2, release_document(repos, label, dist),
content_type: "text/plain; charset=utf-8"
)
end
defp serve(repos, label, _scope, %{path: ["dists", dist, "Release"]}),
do: {:ok, "text/plain; charset=utf-8", release_document(repos, label, dist)}
defp serve(repos, label, scope, %{path: ["dists", dist, "Release.gpg"]}) do
sign(scope, &Native.sign_detached/2, release_document(repos, label, dist),
content_type: "application/pgp-signature"
)
end
defp serve(repos, _label, _scope, %{
path: ["dists", dist, component, "binary-" <> arch, "Packages"]
}),
do: {:ok, "text/plain; charset=utf-8", packages_document(repos, dist, component, arch)}
defp serve(repos, _label, _scope, %{
path: ["dists", dist, component, "binary-" <> arch, "Packages.xz"]
}) do
case Native.xz_compress(packages_document(repos, dist, component, arch)) do
{:ok, compressed} -> {:ok, "application/x-xz", compressed}
{:error, reason} -> {:error, {500, "could not compress the index: #{reason}"}}
end
end
defp serve(repos, label, _scope, %{path: ["pool" | _] = path} = ctx) do
# Matched against the whole pool path, not just the filename: two
# repositories in one namespace may well both publish
# `widget_1.0_amd64.deb`, and serving the wrong one would be a
# silent substitution rather than an error.
wanted = Enum.join(path, "/")
repos
|> debs()
|> Enum.find(&(pool_path(&1) == wanted))
|> case do
%{file: %PackageFile{} = file} -> {:blob, file}
nil -> listing(repos, label, ctx)
end
end
# Anything that isn't a file this repository serves may still be a
# directory of them.
defp serve(repos, label, _scope, ctx), do: listing(repos, label, ctx)
defp sign(scope, signer, document, content_type: content_type) do
with {:ok, %{private_pem: secret}} <- Packages.signing_key(scope, @format),
{:ok, signed} <- signer.(secret, document) do
{:ok, content_type, signed}
else
{:error, reason} -> {:error, {500, "could not sign the Release: #{inspect(reason)}"}}
end
end
# ── browsing ────────────────────────────────────────────────────────
# A Debian mirror is a tree of files over plain HTTP, and every one of
# them is conventionally browsable — `deb.debian.org/debian/` is an
# autoindex you can click down through. Ours only ever answered exact
# paths, so pointing a browser at a repository got a 404 and there was
# no way to see what was published without already knowing its name.
#
# apt never requests a directory, so none of this is on its path; it
# exists for the person checking whether their upload landed.
#
# The listing is derived from the paths this repository actually
# serves, never from the filesystem or from a separate enumeration —
# so it cannot advertise something that would 404, and it cannot leak
# a repository the caller couldn't already read, because `repos` was
# filtered before it got here.
defp listing(repos, label, %{path: prefix} = ctx) do
case children(served_paths(repos), prefix) do
# An empty listing below the root is a path that doesn't exist,
# not an empty directory — the tree has no empty nodes in it.
[] when prefix != [] -> {:error, :not_found}
entries -> {:ok, "text/html; charset=utf-8", index_page(label, ctx, prefix, entries)}
end
end
# Every path this repository answers to, with the size and date where
# we know them for free. Index documents are generated per request, so
# sizing them here would mean building all of them to draw a table.
defp served_paths(repos) do
debs = debs(repos)
dists =
debs
|> Enum.group_by(& &1.dist)
|> Enum.flat_map(fn {dist, in_dist} ->
components = in_dist |> Enum.map(& &1.component) |> Enum.uniq()
architectures = in_dist |> Enum.map(& &1.architecture) |> Enum.uniq()
~w(InRelease Release Release.gpg)
|> Enum.map(&"dists/#{dist}/#{&1}")
|> Enum.concat(
for component <- components,
arch <- architectures,
name <- ["Packages", "Packages.xz"] do
"dists/#{dist}/#{component}/binary-#{arch}/#{name}"
end
)
end)
Enum.map(["key.asc" | dists], &%{path: &1, size: nil, at: nil}) ++
Enum.map(debs, &%{path: pool_path(&1), size: &1.file.size, at: &1.file.inserted_at})
end
# The immediate children of `prefix`: a path with exactly one segment
# left is a file, more than one means a directory we've now seen.
defp children(paths, prefix) do
depth = length(prefix)
paths
|> Enum.flat_map(fn entry ->
segments = String.split(entry.path, "/")
if List.starts_with?(segments, prefix) do
case Enum.drop(segments, depth) do
[name] -> [%{name: name, dir?: false, size: entry.size, at: entry.at}]
[name | _] -> [%{name: name, dir?: true, size: nil, at: nil}]
[] -> []
end
else
[]
end
end)
|> Enum.uniq_by(& &1.name)
|> Enum.sort_by(&{not &1.dir?, &1.name})
end
# A plain document rather than the app layout — this is served from
# the registry URL, which a browser may well reach with a Basic-auth
# prompt and no session at all — but it still wears the forge's
# colours.
#
# It pulls in `app.css` for the daisyUI theme definitions and repeats
# the root layout's theme bootstrap, so `data-theme` is set before
# first paint and the page matches whatever the viewer picked. Styling
# then goes through the theme's own custom properties rather than
# Tailwind utility classes: this module lives under `lib/git_gud`, and
# `app.css` only scans `lib/git_gud_web`, so utility classes written
# here would be purged out of the bundle and silently do nothing.
#
# The bootstrap is read-only — no picker on this page, so it never
# writes `phx:theme` — but it does listen for `storage`, so changing
# the theme in another tab recolours this one.
defp index_page(label, ctx, prefix, entries) do
here = Enum.join([ctx.base_url | prefix], "/")
rows =
Enum.map(entries, fn entry ->
href = here <> "/" <> URI.encode(entry.name) <> if(entry.dir?, do: "/", else: "")
[
"<tr><td><a class=\"",
if(entry.dir?, do: "dir", else: "file"),
"\" href=\"",
esc(href),
"\">",
esc(entry.name),
if(entry.dir?, do: "/", else: ""),
"</a></td><td class=\"n\">",
format_size(entry.size),
"</td><td class=\"n\">",
format_date(entry.at),
"</td></tr>\n"
]
end)
parent =
if prefix == [] do
[]
else
up = Enum.join([ctx.base_url | Enum.drop(prefix, -1)], "/") <> "/"
["<tr><td><a class=\"dir\" href=\"", esc(up), "\">../</a></td><td></td><td></td></tr>\n"]
end
title = "/" <> Enum.join(prefix, "/")
[
"""
<!DOCTYPE html>
<html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>#{esc(label)} #{esc(title)}</title>
<link rel="stylesheet" href="#{esc(stylesheet_path())}">
<script>
(() => {
const SYSTEM_DARK = "#{GitGud.Themes.system_dark()}";
const SYSTEM_LIGHT = "#{GitGud.Themes.system_light()}";
const mq = window.matchMedia("(prefers-color-scheme: dark)");
const systemTheme = () => mq.matches ? SYSTEM_DARK : SYSTEM_LIGHT;
const apply = (theme) =>
document.documentElement.setAttribute("data-theme", theme || systemTheme());
apply(localStorage.getItem("phx:theme"));
mq.addEventListener("change", () => {
if (!localStorage.getItem("phx:theme")) apply(null);
});
window.addEventListener("storage", (e) => {
if (e.key === "phx:theme") apply(e.newValue);
});
})();
</script>
<style>
body {
background: var(--color-base-100);
color: var(--color-base-content);
font-family: ui-monospace, "B612 Mono", SFMono-Regular, Menlo, monospace;
font-size: 14px;
line-height: 1.6;
margin: 0 auto;
max-width: 60rem;
min-height: 100vh;
padding: 2rem 1rem;
}
h1 { font-size: 1rem; font-weight: 600; margin: 0 0 .25rem; }
h1 .path { opacity: .55; }
p.sub { margin: 0 0 1.5rem; opacity: .6; font-size: .8rem; }
table { border-collapse: collapse; width: 100%; }
td { padding: .25rem .75rem .25rem 0; border-bottom: 1px solid var(--color-base-300); }
td.n { text-align: right; white-space: nowrap; opacity: .6; font-size: .8rem; }
a { color: inherit; text-decoration: none; }
a:hover { text-decoration: underline; }
a.dir { color: var(--color-primary); }
</style></head><body>
<h1>#{esc(label)}<span class="path">#{esc(title)}</span></h1>
<p class="sub">Debian repository · apt reads this tree; you can browse it.</p>
<table>
""",
parent,
rows,
"</table></body></html>\n"
]
end
# Digest-aware, so the stylesheet resolves in a release the same way
# the app layout's does.
defp stylesheet_path, do: GitGudWeb.Endpoint.static_path("/assets/css/app.css")
defp format_size(nil), do: ""
defp format_size(bytes) when bytes < 1024, do: "#{bytes} B"
defp format_size(bytes) when bytes < 1_048_576, do: "#{Float.round(bytes / 1024, 1)} KB"
defp format_size(bytes), do: "#{Float.round(bytes / 1_048_576, 1)} MB"
defp format_date(nil), do: ""
defp format_date(at), do: Calendar.strftime(at, "%Y-%m-%d %H:%M")
defp esc(value) do
value
|> to_string()
|> String.replace("&", "&")
|> String.replace("<", "<")
|> String.replace(">", ">")
|> String.replace("\"", """)
end
# ── the indices ─────────────────────────────────────────────────────
# Every .deb across `repos`, flattened with the fields the indices
# key on. One query per package rather than one join is an N+1, but a
# namespace's package list is small and this keeps the substrate's
# API unchanged.
defp debs(repos) do
Enum.flat_map(repos, fn %Repository{} = repo ->
repo
|> Packages.list_packages(@format)
|> Enum.flat_map(&Packages.list_package_files/1)
|> Enum.map(fn %PackageFile{} = file ->
metadata = file.metadata || %{}
%{
file: file,
repo_name: repo.name,
dist: metadata["dist"] || @default_dist,
component: metadata["component"] || @default_component,
architecture: metadata["architecture"] || @arch_all,
control: metadata["control"] || %{}
}
end)
end)
end
defp packages_document(repos, dist, component, arch) do
repos
|> debs()
|> Enum.filter(fn deb ->
deb.dist == dist and deb.component == component and
(deb.architecture == arch or deb.architecture == @arch_all)
end)
|> Enum.sort_by(&{&1.control["Package"], &1.control["Version"], &1.repo_name})
|> Enum.map_join("\n", &packages_stanza/1)
end
# Field order follows dpkg-scanpackages, which is what every
# repository in the wild looks like. `MD5sum` here and `MD5Sum` in
# Release is not a typo — the two files really do differ, and apt's
# parser is case-insensitive either way.
defp packages_stanza(deb) do
control = deb.control
description = control["Description"]
[
{"Package", control["Package"]},
{"Version", control["Version"]},
{"Architecture", deb.architecture},
{"Maintainer", control["Maintainer"]},
{"Installed-Size", control["Installed-Size"]},
{"Depends", control["Depends"]},
{"Pre-Depends", control["Pre-Depends"]},
{"Recommends", control["Recommends"]},
{"Suggests", control["Suggests"]},
{"Conflicts", control["Conflicts"]},
{"Breaks", control["Breaks"]},
{"Replaces", control["Replaces"]},
{"Provides", control["Provides"]},
{"Section", control["Section"]},
{"Priority", control["Priority"]},
{"Homepage", control["Homepage"]},
{"Filename", pool_path(deb)},
{"Size", deb.file.size},
{"MD5sum", deb.file.md5},
{"SHA256", deb.file.sha256},
{"Description", description},
{"Description-md5", Control.description_md5(description)}
]
|> Enum.reject(fn {_name, value} -> value in [nil, ""] end)
|> Enum.map_join(fn {name, value} -> "#{name}: #{value}\n" end)
end
@doc """
The pool path for a package, relative to the repository root.
Qualified by the source repository, which a stock Debian pool is
not. It has to be: a namespace repository aggregates several, and two
of them publishing `widget_1.0_amd64.deb` would otherwise be one path
with two sets of bytes. `Filename` is opaque to apt, and qualifying
it in *both* views means the per-repository and namespace indices
name the same file identically — a client that switches between them
re-downloads nothing.
Within that, Debian's own sharding: the first letter, except `lib*`
gets its own two-character shard — `libfoo` under `libf/`, not `l/` —
because otherwise `l/` holds most of the archive.
"""
def pool_path(deb) do
name = deb.control["Package"] || "unknown"
Path.join(["pool", deb.component, deb.repo_name, shard(name), name, deb.file.filename])
end
defp shard("lib" <> <<first::utf8>> <> _rest), do: "lib" <> <<first::utf8>>
defp shard(<<first::utf8>> <> _rest), do: <<first::utf8>>
defp shard(_), do: "unknown"
defp release_document(repos, label, dist) do
debs = repos |> debs() |> Enum.filter(&(&1.dist == dist))
components = debs |> Enum.map(& &1.component) |> Enum.uniq() |> Enum.sort()
components = if components == [], do: [@default_component], else: components
architectures = debs |> Enum.map(& &1.architecture) |> Enum.uniq() |> Enum.sort()
architectures = if architectures == [], do: [@arch_all], else: architectures
header =
[
{"Origin", label},
{"Label", label},
{"Suite", dist},
{"Codename", dist},
{"Date", http_date()},
{"Architectures", Enum.join(architectures, " ")},
{"Components", Enum.join(components, " ")},
{"Description", "#{label} — packages published from the forge"}
]
|> Enum.map_join(fn {name, value} -> "#{name}: #{value}\n" end)
header <> "SHA256:\n" <> checksums(repos, dist, components, architectures)
end
# Every index file the Release covers, with the size and SHA-256 of
# each — compressed *and* uncompressed, which the format requires
# even though apt will only fetch one of them.
defp checksums(repos, dist, components, architectures) do
for component <- components, arch <- architectures, into: "" do
body = packages_document(repos, dist, component, arch)
base = "#{component}/binary-#{arch}/Packages"
compressed =
case Native.xz_compress(body) do
{:ok, bytes} -> [{base <> ".xz", bytes}]
{:error, _} -> []
end
Enum.map_join([{base, body} | compressed], fn {path, bytes} ->
digest = :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower)
# Field width matters only for readability; apt splits on
# whitespace.
" #{digest} #{byte_size(bytes)} #{path}\n"
end)
end
end
defp repo_label(%Repository{} = repo) do
repo = GitGud.Repo.preload(repo, [:owner, :organization])
"#{Storage.repo_handle(repo)}/#{repo.name}"
end
# RFC 1123 in UTC, which is what dpkg writes and apt parses.
defp http_date do
DateTime.utc_now()
|> Calendar.strftime("%a, %d %b %Y %H:%M:%S UTC")
end
@doc "Default distribution when an upload doesn't name one."
def default_dist, do: @default_dist
@doc "Default component when an upload doesn't name one."
def default_component, do: @default_component
# ── uploads ─────────────────────────────────────────────────────────
@impl true
def store(repo, ctx, body)
def store(repo, %{method: "POST", path: path, conn: conn} = ctx, body)
when is_binary(body) and (path == ["upload"] or path == []) do
params = Plug.Conn.fetch_query_params(conn).query_params
dist = blank_to_nil(params["dist"]) || @default_dist
component = blank_to_nil(params["component"]) || @default_component
with {:ok, control} <- Control.parse(body) do
name = control["Package"]
version = control["Version"]
architecture = control["Architecture"] || @arch_all
attrs = %{
filename: "#{name}_#{version}_#{architecture}.deb",
body: body,
content_type: content_type(name),
metadata: %{
"dist" => dist,
"component" => component,
"architecture" => architecture,
"control" => control
},
version_metadata: %{"control" => control},
uploader: uploader(ctx)
}
case Packages.publish_file(repo, @format, name, version, attrs) do
{:ok, _package, _version, _file} ->
{:ok, "application/json",
Jason.encode_to_iodata!(%{
"package" => name,
"version" => version,
"architecture" => architecture,
"dist" => dist,
"component" => component
})}
{:error, :file_exists} ->
{:error, :file_exists}
{:error, reason} ->
{:error, {400, inspect(reason)}}
end
end
end
def store(_repo, %{method: "POST"}, _body),
do: {:error, {400, "POST the raw .deb to /upload"}}
def store(_repo, %{method: method}, _body),
do: {:error, {405, "#{method} is not supported by the apt repository"}}
defp blank_to_nil(value) when is_binary(value) do
case String.trim(value) do
"" -> nil
trimmed -> trimmed
end
end
defp blank_to_nil(_), do: nil
defp uploader(%{principal: {:user, user}}), do: user
defp uploader(_ctx), do: nil
end
neiam
/gitgud
Git Gud
public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages
⭐
1
Log in to mark this repository.
22.8 KiB · text
History
da63596