neiam /gitgud

defmodule GitGud.Packages.Formats.Pypi do
  @moduledoc """
  A PEP 503 / PEP 691 Simple Repository, per git repository.

      pip install --index-url https://<host>/pkg/pypi/<owner>/<repo>/simple/ widget
      twine upload --repository-url https://<host>/pkg/pypi/<owner>/<repo>/ dist/*

  Paths below the repository root:

      /                       twine's upload target (POST)
      /simple/                the project list
      /simple/<name>/         a project's files
      /files/<filename>       an artifact's bytes

  ## The parts that bite

  **Normalisation.** PEP 503 defines exactly one canonical form,
  `re.sub(r"[-_.]+", "-", name).lower()`, and it must be applied before
  *any* lookup — `Flask_Login`, `flask.login` and `FLASK--LOGIN` are
  one project. Clients request the normalised URL directly; we also
  redirect an unnormalised one, which the PEP permits and humans
  appreciate.

  **Trailing slashes are load-bearing.** Every index URL ends in `/`,
  and an unslashed one must redirect rather than render — relative
  hrefs resolve against the wrong base otherwise. Plug drops the
  trailing slash when it splits the path, so the controller passes the
  raw fact through as `:trailing_slash?`.

  **Two serialisations, one data structure.** PEP 691 adds a JSON
  rendering of exactly the same information, negotiated on `Accept`:

      application/vnd.pypi.simple.v1+json
      application/vnd.pypi.simple.v1+html
      text/html                            (legacy alias for v1+html)

  with `406` when nothing matches. Both come off `project_files/1`
  here; only the rendering differs.
  """

  @behaviour GitGud.Packages.Format

  alias GitGud.Packages
  alias GitGud.Packages.Package
  alias GitGud.Packages.PackageFile

  @format "pypi"
  @api_version "1.0"

  @json "application/vnd.pypi.simple.v1+json"
  @html "application/vnd.pypi.simple.v1+html"
  @legacy_html "text/html"

  # Most preferred last-resort first: a client that sends `*/*` (curl,
  # a browser) gets HTML, which is the legacy behaviour and readable.
  @serialisations [@json, @html, @legacy_html]

  @doc """
  PEP 503 name normalisation, which is the whole of it:

      re.sub(r"[-_.]+", "-", name).lower()
  """
  @impl true
  def normalize(name) when is_binary(name) do
    name
    |> String.replace(~r/[-_.]+/, "-")
    |> String.downcase()
  end

  @impl true
  def content_type(filename) do
    cond do
      String.ends_with?(filename, ".whl") -> "application/zip"
      String.ends_with?(filename, ".zip") -> "application/zip"
      String.ends_with?(filename, ".egg") -> "application/zip"
      String.ends_with?(filename, ".gz") -> "application/gzip"
      String.ends_with?(filename, ".bz2") -> "application/x-bzip2"
      true -> "application/octet-stream"
    end
  end

  # ── reads ───────────────────────────────────────────────────────────

  @impl true
  def index(repo, ctx)

  # The repository root is twine's upload target, not a page. Send a
  # human (or a curious client) on to the index.
  def index(_repo, %{path: []} = ctx), do: {:redirect, ctx.base_url <> "/simple/"}

  def index(_repo, %{path: ["simple"], trailing_slash?: false} = ctx),
    do: {:redirect, ctx.base_url <> "/simple/"}

  def index(repo, %{path: ["simple"]} = ctx) do
    projects = Packages.list_packages(repo, @format)

    serve(ctx, fn
      :json -> root_json(projects)
      :html -> root_html(projects)
    end)
  end

  def index(repo, %{path: ["simple", name]} = ctx) do
    normalized = normalize(name)

    cond do
      # PEP 503: a server *may* redirect an unnormalised name. Doing so
      # means a hand-typed URL works and a client's cache key stays the
      # canonical one.
      normalized != name ->
        {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"}

      not ctx.trailing_slash? ->
        {:redirect, ctx.base_url <> "/simple/" <> normalized <> "/"}

      true ->
        case Packages.get_package(repo, @format, normalized) do
          nil ->
            {:error, :not_found}

          %Package{} = package ->
            files = project_files(package, ctx.base_url)

            serve(ctx, fn
              :json -> project_json(normalized, files)
              :html -> project_html(normalized, files)
            end)
        end
    end
  end

  def index(repo, %{path: ["files", filename]}) do
    # A filename is unique within a project but the download URL
    # doesn't name the project, so this searches the repository's whole
    # pypi namespace. Distribution filenames embed the project name, so
    # collisions across projects don't happen in practice.
    case find_file(repo, filename) do
      %PackageFile{} = file -> {:blob, file}
      nil -> {:error, :not_found}
    end
  end

  def index(_repo, _ctx), do: {:error, :not_found}

  defp find_file(repo, filename) do
    repo
    |> Packages.list_packages(@format)
    |> Enum.find_value(fn package -> Packages.get_package_file(package, filename) end)
  end

  # ── the one data structure both serialisations render ───────────────

  defp project_files(%Package{} = package, base_url) do
    package
    |> Packages.list_package_files()
    |> Enum.map(fn %PackageFile{} = file ->
      meta = file.metadata || %{}

      %{
        filename: file.filename,
        url: "#{base_url}/files/#{URI.encode(file.filename)}#sha256=#{file.sha256}",
        sha256: file.sha256,
        requires_python: meta["requires_python"],
        # PEP 592: `false` for a live file, the *reason string* for a
        # yanked one — and an empty reason is still a yank, so this is
        # deliberately not `reason || false`.
        yanked:
          if(file.package_version.yanked,
            do: file.package_version.yanked_reason || true,
            else: false
          )
      }
    end)
  end

  # ── content negotiation ─────────────────────────────────────────────

  defp serve(ctx, render) do
    case negotiate(ctx.conn) do
      {:ok, @json} -> {:ok, @json, Jason.encode_to_iodata!(render.(:json))}
      {:ok, media} -> {:ok, media, render.(:html)}
      :error -> {:error, :not_acceptable}
    end
  end

  @doc """
  Pick a serialisation for an `Accept` header, or `:error` for 406.

  A request with no `Accept` at all is answered as though it asked for
  `text/html` — that is what every pre-PEP-691 client did, and what
  PEP 691 tells servers to assume.
  """
  def negotiate(conn) do
    case Plug.Conn.get_req_header(conn, "accept") do
      [] ->
        {:ok, @legacy_html}

      headers ->
        headers
        |> Enum.join(",")
        |> parse_accept()
        |> best_match()
    end
  end

  # `media/type;q=0.5` → {media_range, q}, highest q first. Ties keep
  # the order the client wrote them in, which is the convention for
  # equal-quality ranges.
  defp parse_accept(header) do
    header
    |> String.split(",")
    |> Enum.map(&String.trim/1)
    |> Enum.reject(&(&1 == ""))
    |> Enum.with_index()
    |> Enum.map(fn {entry, index} ->
      [range | params] = String.split(entry, ";")
      {String.trim(range) |> String.downcase(), quality(params), index}
    end)
    |> Enum.sort_by(fn {_range, q, index} -> {-q, index} end)
  end

  defp quality(params) do
    Enum.find_value(params, 1.0, fn param ->
      case param |> String.trim() |> String.split("=", parts: 2) do
        ["q", value] ->
          case Float.parse(String.trim(value)) do
            {q, _} -> q
            :error -> 1.0
          end

        _ ->
          nil
      end
    end)
  end

  defp best_match(ranges) do
    # `q=0` is an explicit refusal, not a weak preference.
    ranges
    |> Enum.reject(fn {_range, q, _} -> q == 0.0 end)
    |> Enum.find_value(:error, fn {range, _q, _} ->
      case Enum.find(@serialisations, &matches?(range, &1)) do
        nil -> nil
        # `text/html` is only an alias for the HTML serialisation; the
        # response still advertises what it actually is.
        @legacy_html -> {:ok, @legacy_html}
        media -> {:ok, media}
      end
    end)
  end

  defp matches?("*/*", _media), do: true

  defp matches?(range, media) do
    case String.split(range, "/", parts: 2) do
      [type, "*"] -> String.starts_with?(media, type <> "/")
      _ -> range == media
    end
  end

  # ── PEP 691 (JSON) ──────────────────────────────────────────────────

  defp root_json(projects) do
    %{
      "meta" => %{"api-version" => @api_version},
      "projects" => Enum.map(projects, &%{"name" => &1.normalized_name})
    }
  end

  defp project_json(name, files) do
    %{
      "meta" => %{"api-version" => @api_version},
      "name" => name,
      "files" => Enum.map(files, &file_json/1)
    }
  end

  defp file_json(file) do
    # `hashes` is required even when empty — a client may not assume
    # the key is absent, so this is never conditionally built.
    base = %{
      "filename" => file.filename,
      "url" => file.url,
      "hashes" => %{"sha256" => file.sha256},
      "yanked" => file.yanked
    }

    case file.requires_python do
      nil -> base
      "" -> base
      spec -> Map.put(base, "requires-python", spec)
    end
  end

  # ── PEP 503 (HTML) ──────────────────────────────────────────────────

  defp root_html(projects) do
    links =
      Enum.map(projects, fn project ->
        ~s(    <a href="#{esc(project.normalized_name)}/">#{esc(project.normalized_name)}</a><br />\n)
      end)

    page("Simple index", links)
  end

  defp project_html(name, files) do
    links =
      Enum.map(files, fn file ->
        ~s(    <a href="#{esc(file.url)}"#{html_attrs(file)}>#{esc(file.filename)}</a><br />\n)
      end)

    page("Links for #{name}", [~s(    <h1>Links for #{esc(name)}</h1>\n) | links])
  end

  defp html_attrs(file) do
    [
      requires_python_attr(file.requires_python),
      yanked_attr(file.yanked)
    ]
    |> Enum.reject(&is_nil/1)
    |> Enum.join()
  end

  defp requires_python_attr(nil), do: nil
  defp requires_python_attr(""), do: nil
  defp requires_python_attr(spec), do: ~s( data-requires-python="#{esc(spec)}")

  defp yanked_attr(false), do: nil
  defp yanked_attr(true), do: ~s( data-yanked="")
  defp yanked_attr(reason), do: ~s( data-yanked="#{esc(reason)}")

  defp page(title, body) do
    [
      "<!DOCTYPE html>\n<html>\n  <head>\n",
      "    <meta name=\"pypi:repository-version\" content=\"#{@api_version}\" />\n",
      "    <title>#{esc(title)}</title>\n  </head>\n  <body>\n",
      body,
      "  </body>\n</html>\n"
    ]
  end

  # PEP 503 is served to a browser as well as to pip, and a project
  # name reaches us from an upload form.
  defp esc(value) do
    value
    |> to_string()
    |> String.replace("&", "&amp;")
    |> String.replace("<", "&lt;")
    |> String.replace(">", "&gt;")
    |> String.replace("\"", "&quot;")
  end

  # ── uploads ─────────────────────────────────────────────────────────

  @impl true
  def store(repo, ctx, params)

  def store(repo, %{method: "POST"} = ctx, %{} = params) do
    with {:ok, name} <- require_field(params, "name"),
         {:ok, version} <- require_field(params, "version"),
         {:ok, upload} <- require_upload(params),
         {:ok, body} <- read_upload(upload),
         :ok <- verify_digest(params["sha256_digest"], body) do
      attrs = %{
        filename: Path.basename(upload.filename),
        body: body,
        content_type: content_type(upload.filename),
        metadata: file_metadata(params),
        version_metadata: version_metadata(params),
        uploader: uploader(ctx)
      }

      case Packages.publish_file(repo, @format, name, version, attrs) do
        # twine reads nothing but the status. An empty 200 is what
        # warehouse sends.
        {:ok, _package, _version, _file} -> {:ok, "text/plain; charset=utf-8", ""}
        {:error, :file_exists} -> {:error, :file_exists}
        {:error, reason} -> {:error, changeset_message(reason)}
      end
    end
  end

  def store(_repo, %{method: "POST"}, _body),
    do: {:error, {400, "expected a multipart/form-data upload"}}

  def store(_repo, %{method: method}, _params),
    do: {:error, {405, "#{method} is not supported by the PyPI upload API"}}

  defp require_field(params, key) do
    case params[key] do
      value when is_binary(value) and value != "" -> {:ok, value}
      _ -> {:error, {400, "missing required form field `#{key}`"}}
    end
  end

  defp require_upload(params) do
    case params["content"] do
      %Plug.Upload{} = upload -> {:ok, upload}
      _ -> {:error, {400, "missing the `content` file part"}}
    end
  end

  defp read_upload(%Plug.Upload{path: path}) do
    case File.read(path) do
      {:ok, body} -> {:ok, body}
      {:error, reason} -> {:error, {400, "could not read the upload: #{inspect(reason)}"}}
    end
  end

  # twine computes this itself and sends it alongside. Checking it
  # costs one hash and turns a truncated upload into a clear error
  # instead of a corrupt wheel nobody notices until install time.
  defp verify_digest(nil, _body), do: :ok
  defp verify_digest("", _body), do: :ok

  defp verify_digest(expected, body) do
    actual = Base.encode16(:crypto.hash(:sha256, body), case: :lower)

    if String.downcase(expected) == actual,
      do: :ok,
      else: {:error, {400, "sha256_digest does not match the uploaded bytes"}}
  end

  defp file_metadata(params) do
    %{
      "requires_python" => blank_to_nil(params["requires_python"]),
      "filetype" => blank_to_nil(params["filetype"]),
      "pyversion" => blank_to_nil(params["pyversion"])
    }
    |> Enum.reject(fn {_k, v} -> is_nil(v) end)
    |> Map.new()
  end

  # Core metadata twine sends along with the distribution, kept so the
  # packages UI can describe a release without unzipping a wheel.
  defp version_metadata(params) do
    %{
      "summary" => blank_to_nil(params["summary"]),
      "metadata_version" => blank_to_nil(params["metadata_version"]),
      "requires_python" => blank_to_nil(params["requires_python"]),
      "home_page" => blank_to_nil(params["home_page"]),
      "license" => blank_to_nil(params["license"])
    }
    |> Enum.reject(fn {_k, v} -> is_nil(v) end)
    |> Map.new()
  end

  defp blank_to_nil(value) when is_binary(value) do
    case String.trim(value) do
      "" -> nil
      trimmed -> trimmed
    end
  end

  defp blank_to_nil(_), do: nil

  defp uploader(%{principal: {:user, user, _scopes}}), do: user
  defp uploader(_ctx), do: nil

  defp changeset_message(%Ecto.Changeset{} = changeset) do
    errors =
      Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} ->
        Enum.reduce(opts, msg, fn {k, v}, acc ->
          String.replace(acc, "%{#{k}}", to_string(v))
        end)
      end)

    {400,
     Enum.map_join(errors, "; ", fn {field, msgs} -> "#{field} #{Enum.join(msgs, ", ")}" end)}
  end

  defp changeset_message(reason), do: {400, inspect(reason)}
end