neiam
/final_track
public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages
⭐
Log in to mark this repository.
Covers: fall back to the main artist; posts wait for a pending cover
34723be · gmorell · 2026-11-18 21:08
Message
{commit_body(@commit)}
Files changed
modified
config/runtime.exs
+3
−0
@@ -51,6 +51,9 @@ if config_env() == :prod do
| 51 | 51 | config :final_track, FinalTrack.Covers, |
| 52 | 52 | root: System.get_env("FINAL_TRACK_COVERS_DIR", "/var/lib/final_track/covers"), |
| 53 | 53 | auto_fetch: System.get_env("FINAL_TRACK_AUTO_FETCH_COVERS", "true") == "true", |
| 54 | + # Between MusicBrainz requests, per pod. MusicBrainz allows ~1/s per IP | |
| 55 | + # and every replica runs its own queue: 2.2s keeps two pods under it. | |
| 56 | + interval_ms: String.to_integer(System.get_env("FINAL_TRACK_COVER_INTERVAL_MS", "2200")), | |
| 54 | 57 | user_agent: |
| 55 | 58 | System.get_env( |
| 56 | 59 | "FINAL_TRACK_MUSICBRAINZ_USER_AGENT", |
modified
config/test.exs
+2
−1
@@ -16,7 +16,8 @@ config :final_track, FinalTrack.Covers,
| 16 | 16 | config :final_track, FinalTrack.Federation, |
| 17 | 17 | enabled: true, |
| 18 | 18 | delivery: :inline, |
| 19 | − post_delay_ms: 0 | |
| 19 | + post_delay_ms: 0, | |
| 20 | + cover_waits: 0 | |
| 20 | 21 | |
| 21 | 22 | config :final_track, FinalTrack.Federation.HTTP, |
| 22 | 23 | guard_addresses: false, |
modified
lib/final_track/covers.ex
+21
−0
@@ -31,6 +31,27 @@ defmodule FinalTrack.Covers do
| 31 | 31 | |
| 32 | 32 | def maybe_fetch(_album), do: :ok |
| 33 | 33 | |
| 34 | + @doc """ | |
| 35 | + Queues another lookup for every album that has no cover and never matched | |
| 36 | + on MusicBrainz (no `mbid`), e.g. after the lookup learned something new. | |
| 37 | + Returns how many were queued. From a release: | |
| 38 | + | |
| 39 | + bin/final_track rpc 'IO.inspect(FinalTrack.Covers.retry_misses())' | |
| 40 | + """ | |
| 41 | + def retry_misses do | |
| 42 | + import Ecto.Query, only: [from: 2] | |
| 43 | + | |
| 44 | + ids = | |
| 45 | + from(a in Album, | |
| 46 | + where: is_nil(a.cover_key) and is_nil(a.mbid) and not is_nil(a.cover_checked_at), | |
| 47 | + select: a.id | |
| 48 | + ) | |
| 49 | + |> FinalTrack.Repo.all() | |
| 50 | + | |
| 51 | + Enum.each(ids, &Queue.enqueue/1) | |
| 52 | + length(ids) | |
| 53 | + end | |
| 54 | + | |
| 34 | 55 | @doc "Queues a lookup for `album` now, whatever it has (a person asked for it)." |
| 35 | 56 | def request_fetch(%Album{id: id}), do: Queue.enqueue(id) |
| 36 | 57 |
modified
lib/final_track/covers/fetcher.ex
+35
−1
@@ -26,12 +26,46 @@ defmodule FinalTrack.Covers.Fetcher do
| 26 | 26 | but the archive has no front cover, or another `{:error, reason}`. |
| 27 | 27 | """ |
| 28 | 28 | def fetch(%Album{} = album) do |
| 29 | − with {:ok, mbid} <- release_group(album.title, album.artist.name), | |
| 29 | + with {:ok, mbid} <- find_release_group(album.title, album.artist.name), | |
| 30 | 30 | {:ok, bytes} <- front_cover(mbid) do |
| 31 | 31 | {:ok, %{bytes: bytes, mbid: mbid}} |
| 32 | 32 | end |
| 33 | 33 | end |
| 34 | 34 | |
| 35 | + # Players often put the whole credit in the artist field ("Justice & Tame | |
| 36 | + # Impala", "X feat. Y"), and MusicBrainz knows the album under the main | |
| 37 | + # artist. The full name goes first, so a duo whose name has an "&" in it | |
| 38 | + # (Simon & Garfunkel) is still found as itself. | |
| 39 | + defp find_release_group(title, artist) do | |
| 40 | + artist | |
| 41 | + |> artist_candidates() | |
| 42 | + |> Enum.with_index() | |
| 43 | + |> Enum.reduce_while({:error, :no_match}, fn {candidate, i}, _ -> | |
| 44 | + # Each extra search is another MusicBrainz request: keep to its pace. | |
| 45 | + if i > 0, do: Process.sleep(pause_ms()) | |
| 46 | + | |
| 47 | + case release_group(title, candidate) do | |
| 48 | + {:error, :no_match} = miss -> {:cont, miss} | |
| 49 | + other -> {:halt, other} | |
| 50 | + end | |
| 51 | + end) | |
| 52 | + end | |
| 53 | + | |
| 54 | + @separators ~r/\s+(?:&|feat\.?|ft\.?|featuring|with|x|vs\.?|and)\s+|\s*[,;\/]\s*/iu | |
| 55 | + | |
| 56 | + @doc """ | |
| 57 | + The artist names worth searching for: the full credit, then its main | |
| 58 | + artist when the credit names several ("Justice & Tame Impala" → "Justice"). | |
| 59 | + """ | |
| 60 | + def artist_candidates(artist) do | |
| 61 | + main = @separators |> Regex.split(artist, parts: 2) |> hd() |> String.trim() | |
| 62 | + Enum.uniq(Enum.reject([artist, main], &(&1 == ""))) | |
| 63 | + end | |
| 64 | + | |
| 65 | + defp pause_ms, | |
| 66 | + do: | |
| 67 | + Application.get_env(:final_track, FinalTrack.Covers, []) |> Keyword.get(:interval_ms, 1100) | |
| 68 | + | |
| 35 | 69 | @doc "The best MusicBrainz release-group id for an album, if any." |
| 36 | 70 | def release_group(title, artist) do |
| 37 | 71 | query = ~s(releasegroup:"#{escape(title)}" AND artist:"#{escape(artist)}") |
modified
lib/final_track/federation.ex
+18
−3
@@ -178,13 +178,28 @@ defmodule FinalTrack.Federation do
| 178 | 178 | end |
| 179 | 179 | |
| 180 | 180 | # Built at send time: nil (nothing sent) if the play went private meanwhile. |
| 181 | − defp create_for(post_id) do | |
| 181 | + # If the play's album is still waiting on its first cover lookup, give it | |
| 182 | + # a little longer: a post without its cover is a post Pixelfed drops (it | |
| 183 | + # doesn't store notes without media), and the cover is usually seconds | |
| 184 | + # away. Then it goes as it is. | |
| 185 | + defp create_for(post_id, waits \\ cover_waits()) do | |
| 182 | 186 | case get_post(post_id) do |
| 183 | − {:ok, post} -> Objects.create(post, post.account) | |
| 184 | − _ -> nil | |
| 187 | + {:ok, %{scrobble: %{album: %{cover_key: nil, cover_checked_at: nil}}}} when waits > 0 -> | |
| 188 | + Process.sleep(cover_wait_ms()) | |
| 189 | + create_for(post_id, waits - 1) | |
| 190 | + | |
| 191 | + {:ok, post} -> | |
| 192 | + Objects.create(post, post.account) | |
| 193 | + | |
| 194 | + _ -> | |
| 195 | + nil | |
| 185 | 196 | end |
| 186 | 197 | end |
| 187 | 198 | |
| 199 | + # Up to 4 × 15s more, on top of post_delay_ms. | |
| 200 | + defp cover_waits, do: Keyword.get(config(), :cover_waits, 4) | |
| 201 | + defp cover_wait_ms, do: Keyword.get(config(), :cover_wait_ms, 15_000) | |
| 202 | + | |
| 188 | 203 | @doc """ |
| 189 | 204 | A post that may be shown: its account on, its play not private. |
| 190 | 205 | `{:ok, post}` (scrobble, account and user loaded), `{:error, :gone}` for |
modified
test/final_track/covers_test.exs
+90
−0
@@ -88,6 +88,80 @@ defmodule FinalTrack.CoversTest do
| 88 | 88 | assert_receive {:cover_fetch_failed, ^other_id, :unavailable} |
| 89 | 89 | end |
| 90 | 90 | |
| 91 | + test "artist candidates: the full credit, then its main artist" do | |
| 92 | + alias FinalTrack.Covers.Fetcher | |
| 93 | + | |
| 94 | + assert Fetcher.artist_candidates("Justice & Tame Impala") == [ | |
| 95 | + "Justice & Tame Impala", | |
| 96 | + "Justice" | |
| 97 | + ] | |
| 98 | + | |
| 99 | + assert Fetcher.artist_candidates("Daft Punk feat. Pharrell Williams") == [ | |
| 100 | + "Daft Punk feat. Pharrell Williams", | |
| 101 | + "Daft Punk" | |
| 102 | + ] | |
| 103 | + | |
| 104 | + assert Fetcher.artist_candidates("Calvin Harris ft Rihanna") == [ | |
| 105 | + "Calvin Harris ft Rihanna", | |
| 106 | + "Calvin Harris" | |
| 107 | + ] | |
| 108 | + | |
| 109 | + assert Fetcher.artist_candidates("Skrillex, Fred again..") == [ | |
| 110 | + "Skrillex, Fred again..", | |
| 111 | + "Skrillex" | |
| 112 | + ] | |
| 113 | + | |
| 114 | + assert Fetcher.artist_candidates("Disclosure x Kehlani") == [ | |
| 115 | + "Disclosure x Kehlani", | |
| 116 | + "Disclosure" | |
| 117 | + ] | |
| 118 | + | |
| 119 | + assert Fetcher.artist_candidates("Pink Floyd") == ["Pink Floyd"] | |
| 120 | + # Names that merely contain an x or an "and" aren't split. | |
| 121 | + assert Fetcher.artist_candidates("Charli XCX") == ["Charli XCX"] | |
| 122 | + assert Fetcher.artist_candidates("Andrew Bird") == ["Andrew Bird"] | |
| 123 | + end | |
| 124 | + | |
| 125 | + test "fetch: a collaboration credit falls back to the main artist" do | |
| 126 | + test = self() | |
| 127 | + artist = Catalog.ensure_artist!("Justice & Tame Impala") | |
| 128 | + album = artist |> Catalog.ensure_album!("Hyperdrama") |> Map.put(:artist, artist) | |
| 129 | + | |
| 130 | + stub( | |
| 131 | + fn conn -> | |
| 132 | + query = conn.query_params["query"] | |
| 133 | + send(test, {:searched, query}) | |
| 134 | + | |
| 135 | + if query =~ ~s(artist:"Justice"), | |
| 136 | + do: found().(conn), | |
| 137 | + else: Req.Test.json(conn, %{"release-groups" => []}) | |
| 138 | + end, | |
| 139 | + &Plug.Conn.send_resp(&1, 200, png_bytes()) | |
| 140 | + ) | |
| 141 | + | |
| 142 | + assert {:ok, %{cover_source: "coverartarchive"}} = Covers.fetch_cover_now(album.id) | |
| 143 | + assert_received {:searched, ~s(releasegroup:"Hyperdrama" AND artist:"Justice & Tame Impala")} | |
| 144 | + assert_received {:searched, ~s(releasegroup:"Hyperdrama" AND artist:"Justice")} | |
| 145 | + end | |
| 146 | + | |
| 147 | + test "fetch: a duo with & in its name is found as itself, without a second search" do | |
| 148 | + test = self() | |
| 149 | + artist = Catalog.ensure_artist!("Simon & Garfunkel") | |
| 150 | + album = artist |> Catalog.ensure_album!("Bookends") |> Map.put(:artist, artist) | |
| 151 | + | |
| 152 | + stub( | |
| 153 | + fn conn -> | |
| 154 | + send(test, {:searched, conn.query_params["query"]}) | |
| 155 | + found().(conn) | |
| 156 | + end, | |
| 157 | + &Plug.Conn.send_resp(&1, 200, png_bytes()) | |
| 158 | + ) | |
| 159 | + | |
| 160 | + assert {:ok, _} = Covers.fetch_cover_now(album.id) | |
| 161 | + assert_received {:searched, ~s(releasegroup:"Bookends" AND artist:"Simon & Garfunkel")} | |
| 162 | + refute_received {:searched, _} | |
| 163 | + end | |
| 164 | + | |
| 91 | 165 | test "fetch: a weak MusicBrainz match is no match", %{album: album} do |
| 92 | 166 | stub(found(60), &Plug.Conn.send_resp(&1, 200, png_bytes())) |
| 93 | 167 | assert {:error, :no_match} = Covers.fetch_cover_now(album.id) |
@@ -120,6 +194,22 @@ defmodule FinalTrack.CoversTest do
| 120 | 194 | assert {:error, :not_found} = Store.fetch(second.cover_key) |
| 121 | 195 | end |
| 122 | 196 | |
| 197 | + test "retry_misses/0 counts the albums that never matched", %{album: album} do | |
| 198 | + {:ok, _} = Catalog.mark_cover_checked(album) | |
| 199 | + other = Catalog.ensure_album!(album.artist, "Meddle") | |
| 200 | + {:ok, _} = Catalog.mark_cover_checked(other, %{mbid: "known-but-no-art"}) | |
| 201 | + | |
| 202 | + # Only the first: the second matched, the archive just has no art. | |
| 203 | + stub(&Req.Test.json(&1, %{"release-groups" => []}), &Plug.Conn.send_resp(&1, 404, "")) | |
| 204 | + # The queue is global: let it use this test's stub (made first) and DB. | |
| 205 | + queue = Process.whereis(FinalTrack.Covers.Queue) | |
| 206 | + Ecto.Adapters.SQL.Sandbox.allow(FinalTrack.Repo, self(), queue) | |
| 207 | + Req.Test.allow(FinalTrack.Covers.Fetcher, self(), queue) | |
| 208 | + assert Covers.retry_misses() == 1 | |
| 209 | + album_id = album.id | |
| 210 | + assert_receive {:cover_fetch_failed, ^album_id, :no_match} | |
| 211 | + end | |
| 212 | + | |
| 123 | 213 | test "maybe_fetch/1 only queues what was never tried, and only when enabled", %{album: album} do |
| 124 | 214 | # Test config has auto_fetch off: nothing is queued. |
| 125 | 215 | assert :ok = Covers.maybe_fetch(album) |
Parents: dcfbf0f