GITGUD

neiam neiam /apollos_crib

It's Apollo's Crib

public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages

⭐

Log in to mark this repository.

A tester for realtime URLs, and it found a real one

afda479 · Gabriel Morell · 2026-08-25 21:52

15 files +458 -0
Message 
{commit_body(@commit)}

Files changed

modified ?
+0 −0

Binary file — not rendered.

modified ?
+0 −0

Binary file — not rendered.

added apps/room_gtfs/lib/room_gtfs/feed_tester.ex
+388 −0
@@ -0,0 +1,388 @@
1 +defmodule RoomGtfs.FeedTester do
2 + @moduledoc """
3 + Fetches a source's realtime URLs and says what is wrong with them.
4 +
5 + Every check here exists because the mistake it catches was actually made, and
6 + because none of them announce themselves. A realtime feed that is wrong does
7 + not error -- it decodes, reports plausible numbers, and quietly matches
8 + nothing, and the symptom is "why are there no live times" a week later.
9 +
10 + What it looks for, roughly in order of how long it would otherwise take to
11 + work out:
12 +
13 + * **Realtime that does not match the static feed.** The worst one. Trip and
14 + stop ids only mean something against the schedule they came from, and
15 + 511's realtime uses 511's stop numbering while an operator's own download
16 + uses the operator's. Trips then match, stops do not, and arrivals find the
17 + trip and no times for it -- a half-working state that looks like a bug in
18 + the app.
19 +
20 + * **A `.proto` definition configured as a feed.** The MTA publishes both,
21 + one describes the other, and pasting the wrong one gives a decode failure
22 + naming a truncated URL.
23 +
24 + * **A prefixed feed with no operator code.** 511's regional feed keys trips
25 + `SF:12074394_M21`; without `rt_agency` nothing matches, and with the wrong
26 + one everything is filtered away. Both look identical from the outside: no
27 + realtime.
28 +
29 + * **An endpoint that answers 200 with an error page**, which several do.
30 +
31 + Read-only, and deliberately not cached: the whole point is to see what the
32 + URL says right now.
33 + """
34 +
35 + import Ecto.Query
36 +
37 + alias RoomSanctum.Repo
38 +
39 + # Enough to be confident about a percentage without pulling a whole feed's
40 + # worth of ids through the database.
41 + @sample 300
42 +
43 + @doc """
44 + Test every realtime URL a source has, one entry per distinct URL.
45 +
46 + Returns `[%{url:, kinds:, findings:, ...}]` where `findings` is a list of
47 + `{severity, message}` with severity in `:error`, `:warn`, `:info`, `:ok`.
48 + """
49 + def run(%{type: :gtfs} = source) do
50 + source.config
51 + |> RoomGtfs.Worker.RT.rt_groups()
52 + |> Enum.map(&check_group(&1, source))
53 + end
54 +
55 + def run(_source), do: []
56 +
57 + defp check_group({url, kinds}, source) do
58 + base = %{url: display_url(url), kinds: kinds, entities: %{}, findings: []}
59 +
60 + case RoomGtfs.Worker.RT.fetch_rt_url(url) do
61 + {:ok, feed} ->
62 + base
63 + |> Map.put(:ok, true)
64 + |> check_contents(feed, kinds, source)
65 +
66 + {:error, reason} ->
67 + base
68 + |> Map.put(:ok, false)
69 + |> add(:error, describe_failure(reason, url))
70 + end
71 + end
72 +
73 + defp check_contents(result, feed, kinds, source) do
74 + counts = %{
75 + tu: Enum.count(feed.entity, & &1.trip_update),
76 + vp: Enum.count(feed.entity, & &1.vehicle),
77 + sa: Enum.count(feed.entity, & &1.alert)
78 + }
79 +
80 + result
81 + |> Map.put(:entities, counts)
82 + |> Map.put(:total, length(feed.entity))
83 + |> check_empty(feed, counts, kinds)
84 + |> check_age(feed)
85 + |> check_extensions(feed)
86 + |> check_agency(feed, source)
87 + |> check_against_static(feed, source)
88 + |> finish()
89 + end
90 +
91 + defp check_empty(result, _feed, counts, kinds) do
92 + cond do
93 + result.total == 0 ->
94 + add(result, :warn, "decoded, but carries no entities at all right now")
95 +
96 + Enum.all?(kinds, &(counts[&1] == 0)) ->
97 + add(
98 + result,
99 + :error,
100 + "carries nothing of the kind it is configured for -- it has " <>
101 + describe_counts(counts) <> ", but this URL is set as " <> kind_names(kinds)
102 + )
103 +
104 + true ->
105 + Enum.reduce(kinds, result, fn kind, acc ->
106 + if counts[kind] == 0 do
107 + add(acc, :warn, "no #{kind_name(kind)} in this feed, though it is configured for them")
108 + else
109 + acc
110 + end
111 + end)
112 + end
113 + end
114 +
115 + defp check_age(result, feed) do
116 + case feed.header && feed.header.timestamp do
117 + nil ->
118 + result
119 +
120 + ts ->
121 + age = System.system_time(:second) - ts
122 +
123 + cond do
124 + age > 3600 -> add(result, :error, "feed timestamp is #{div(age, 60)} minutes old")
125 + age > 300 -> add(result, :warn, "feed timestamp is #{div(age, 60)} minutes old")
126 + true -> result
127 + end
128 + end
129 + end
130 +
131 + defp check_extensions(result, feed) do
132 + unnamed =
133 + Enum.reduce(feed.entity, length(feed.header.__unknown_fields__), fn e, acc ->
134 + acc + length(e.__unknown_fields__) + unknown(e.trip_update) + unknown(e.vehicle) +
135 + unknown(e.alert)
136 + end)
137 +
138 + result = Map.put(result, :extensions, unnamed)
139 +
140 + if unnamed > 0 do
141 + add(
142 + result,
143 + :info,
144 + "#{unnamed} fields no loaded extension can name -- kept, not decoded, and not an error"
145 + )
146 + else
147 + result
148 + end
149 + end
150 +
151 + defp unknown(nil), do: 0
152 + defp unknown(%{__unknown_fields__: fields}), do: length(fields)
153 + defp unknown(_other), do: 0
154 +
155 + # A feed that names its operators, against a source that may or may not say
156 + # which one it wants. Skipped for a feed with no trips in it at all -- a
157 + # service-alerts feed carries none, and asking why it prefixes nothing would
158 + # report a problem with a perfectly ordinary feed.
159 + defp check_agency(result, feed, source) do
160 + if Enum.any?(feed.entity, &(trip_id(&1) != nil)) do
161 + do_check_agency(result, feed, source)
162 + else
163 + result
164 + end
165 + end
166 +
167 + defp do_check_agency(result, feed, source) do
168 + prefixes =
169 + feed.entity
170 + |> Enum.map(&trip_id/1)
171 + |> Enum.reject(&is_nil/1)
172 + |> Enum.map(fn id ->
173 + case String.split(id, ":", parts: 2) do
174 + [prefix, _rest] -> prefix
175 + _ -> nil
176 + end
177 + end)
178 + |> Enum.reject(&is_nil/1)
179 + |> Enum.uniq()
180 + |> Enum.sort()
181 +
182 + configured = Map.get(source.config, :rt_agency)
183 + result = Map.put(result, :prefixes, prefixes)
184 +
185 + cond do
186 + prefixes != [] and configured in [nil, ""] ->
187 + add(
188 + result,
189 + :error,
190 + "trip ids are prefixed (#{Enum.join(Enum.take(prefixes, 6), ", ")}#{if length(prefixes) > 6, do: ", …", else: ""}) " <>
191 + "but no operator code is set, so nothing will match the schedule"
192 + )
193 +
194 + configured not in [nil, ""] and prefixes == [] ->
195 + add(
196 + result,
197 + :warn,
198 + "operator code #{configured} is set but this feed prefixes nothing, so everything will be filtered out"
199 + )
200 +
201 + configured not in [nil, ""] and configured not in prefixes ->
202 + add(
203 + result,
204 + :error,
205 + "operator code #{configured} appears nowhere in this feed, which carries " <>
206 + Enum.join(Enum.take(prefixes, 8), ", ")
207 + )
208 +
209 + true ->
210 + result
211 + end
212 + end
213 +
214 + # The check worth having. Ids only mean anything against the schedule they
215 + # came from.
216 + defp check_against_static(result, feed, source) do
217 + agency = Map.get(source.config, :rt_agency)
218 +
219 + # Narrowed the way the poller narrows it. Sampling the raw regional feed
220 + # would draw mostly other operators' entities and report 0% against a
221 + # perfectly correct source -- which is what it did before this line.
222 + feed = RoomGtfs.Worker.RT.for_this_agency(feed, %{inst: source})
223 +
224 + trips =
225 + feed.entity
226 + |> Enum.map(&trip_id/1)
227 + |> Enum.reject(&is_nil/1)
228 + |> Enum.map(&unprefix(&1, agency))
229 + |> Enum.uniq()
230 + |> Enum.take(@sample)
231 +
232 + stops =
233 + feed.entity
234 + |> Enum.filter(& &1.trip_update)
235 + |> Enum.flat_map(& &1.trip_update.stop_time_update)
236 + |> Enum.map(& &1.stop_id)
237 + |> Enum.reject(&is_nil/1)
238 + |> Enum.uniq()
239 + |> Enum.take(@sample)
240 +
241 + result
242 + |> match_against(:trips, trips, source.id, &known_trips/2)
243 + |> match_against(:stops, stops, source.id, &known_stops/2)
244 + end
245 +
246 + defp match_against(result, _label, [], _source_id, _fun), do: result
247 +
248 + defp match_against(result, label, ids, source_id, fun) do
249 + known = fun.(source_id, ids)
250 + pct = round(known / length(ids) * 100)
251 + result = Map.put(result, :"#{label}_match", pct)
252 +
253 + cond do
254 + pct == 0 and label == :trips and partial_trip_ids?(source_id, ids) ->
255 + add(
256 + result,
257 + :error,
258 + "its trip ids are the tail end of the schedule's -- realtime sends " <>
259 + "\"098600_5..S03R\" where the schedule says " <>
260 + "\"ASP26GEN-1038-Sunday-00_098600_5..S03R\". NYCT publishes partial ids like this. " <>
261 + "Nothing matches on an exact comparison, so there are no live times"
262 + )
263 +
264 + pct == 0 ->
265 + add(
266 + result,
267 + :error,
268 + "none of its #{label} exist in this offering's schedule -- realtime and static are from " <>
269 + "different publishers, or the schedule has not been imported"
270 + )
271 +
272 + pct < 60 ->
273 + # Not a fault. One feed often covers more than one offering: MTA's bus
274 + # realtime is citywide while the schedules are split by borough, so
275 + # Brooklyn legitimately sees about a third of it and a small borough
276 + # sees a few percent. The number worth acting on is zero, and warning
277 + # about the rest would cry wolf at every correctly split source.
278 + add(
279 + result,
280 + :ok,
281 + "#{pct}% of its #{label} are in this offering's schedule -- expected to be a " <>
282 + "fraction where one feed serves several offerings, as MTA's citywide bus feed " <>
283 + "does for the boroughs"
284 + )
285 +
286 + true ->
287 + add(result, :ok, "#{pct}% of its #{label} match the schedule")
288 + end
289 + end
290 +
291 + # Distinguishes "these ids are from somewhere else entirely" from "these ids
292 + # are the same ids, shortened" -- which look identical at 0% and are nothing
293 + # alike to fix.
294 + defp partial_trip_ids?(source_id, ids) do
295 + # Patterns built here rather than in SQL: `LIKE ANY(array)` is valid Postgres,
296 + # `LIKE '%' || ANY(array)` is not -- ANY only compares, it does not
297 + # concatenate.
298 + patterns = ids |> Enum.take(40) |> Enum.map(&("%" <> &1))
299 +
300 + matched =
301 + Repo.one(
302 + from(t in "gtfs_trips",
303 + where:
304 + t.source_id == ^to_int(source_id) and fragment("? LIKE ANY(?)", t.trip_id, ^patterns),
305 + select: count(t.id)
306 + )
307 + ) || 0
308 +
309 + matched > 0
310 + end
311 +
312 + defp known_trips(source_id, ids) do
313 + Repo.one(
314 + from(t in "gtfs_trips",
315 + where: t.source_id == ^to_int(source_id) and t.trip_id in ^ids,
316 + select: count(fragment("DISTINCT ?", t.trip_id))
317 + )
318 + ) || 0
319 + end
320 +
321 + defp known_stops(source_id, ids) do
322 + Repo.one(
323 + from(s in "gtfs_stops",
324 + where: s.source_id == ^to_int(source_id) and s.stop_id in ^ids,
325 + select: count(fragment("DISTINCT ?", s.stop_id))
326 + )
327 + ) || 0
328 + end
329 +
330 + defp to_int(id) when is_integer(id), do: id
331 + defp to_int(id) when is_binary(id), do: String.to_integer(id)
332 +
333 + defp trip_id(%{trip_update: %{trip: %{trip_id: id}}}) when is_binary(id), do: id
334 + defp trip_id(%{vehicle: %{trip: %{trip_id: id}}}) when is_binary(id), do: id
335 + defp trip_id(_entity), do: nil
336 +
337 + defp unprefix(id, agency) when is_binary(agency) and agency != "",
338 + do: String.replace_prefix(id, agency <> ":", "")
339 +
340 + defp unprefix(id, _agency), do: id
341 +
342 + defp describe_failure(:not_protobuf, url) do
343 + if String.ends_with?(url, ".proto") do
344 + "did not decode as GTFS-realtime, and the URL ends in .proto -- that is the " <>
345 + "schema describing a feed, not a feed"
346 + else
347 + "answered with something that is not GTFS-realtime, most likely an error page"
348 + end
349 + end
350 +
351 + defp describe_failure(:decode_failed, url), do: describe_failure(:not_protobuf, url)
352 + defp describe_failure(:bad_status, _url), do: "did not answer with HTTP 200"
353 + defp describe_failure(%HTTPoison.Error{reason: reason}, _url), do: "could not be reached: #{inspect(reason)}"
354 + defp describe_failure(reason, _url), do: "failed: #{inspect(reason)}"
355 +
356 + defp describe_counts(counts) do
357 + counts
358 + |> Enum.reject(fn {_k, n} -> n == 0 end)
359 + |> case do
360 + [] -> "nothing"
361 + some -> Enum.map_join(some, ", ", fn {k, n} -> "#{n} #{kind_name(k)}" end)
362 + end
363 + end
364 +
365 + defp kind_names(kinds), do: Enum.map_join(kinds, ", ", &kind_name/1)
366 +
367 + defp kind_name(:tu), do: "trip updates"
368 + defp kind_name(:vp), do: "vehicle positions"
369 + defp kind_name(:sa), do: "service alerts"
370 +
371 + # Credentials show up in these URLs, and a test result is the kind of thing
372 + # that gets pasted into a chat window.
373 + defp display_url(url) do
374 + Regex.replace(~r/([?&](?:api_key|apikey|key|token)=)([^&]{4})[^&]*/i, url, "\\1\\2****")
375 + end
376 +
377 + defp add(result, severity, message),
378 + do: Map.update!(result, :findings, &(&1 ++ [{severity, message}]))
379 +
380 + # A group with nothing to say is worth saying so about.
381 + defp finish(result) do
382 + if Enum.any?(result.findings, fn {sev, _} -> sev in [:error, :warn] end) do
383 + result
384 + else
385 + add(result, :ok, "looks healthy")
386 + end
387 + end
388 +end
modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/show.ex
+28 −0
@@ -26,6 +26,7 @@ defmodule RoomSanctumWeb.SourceLive.Show do
26 26 |> assign(:tester_selected, nil)
27 27 |> assign(:tester_selected_name, nil)
28 28 |> assign(:tester_selected_data, %{})
29 + |> assign(:rt_test, nil)
29 30 |> assign(:vehicle_positions, [])
30 31 |> assign(:free_bikes, [])
31 32 |> assign(:stations, [])
@@ -319,6 +320,20 @@ defmodule RoomSanctumWeb.SourceLive.Show do
319 320
320 321 defp station_id_of(_), do: nil
321 322
323 + defp rt_kind_name(:tu), do: "trip updates"
324 + defp rt_kind_name(:vp), do: "vehicle positions"
325 + defp rt_kind_name(:sa), do: "service alerts"
326 +
327 + defp rt_finding_class(:error), do: "text-error"
328 + defp rt_finding_class(:warn), do: "text-warning"
329 + defp rt_finding_class(:ok), do: "text-success"
330 + defp rt_finding_class(_severity), do: "opacity-70"
331 +
332 + defp rt_finding_icon(:error), do: "fa-circle-exclamation"
333 + defp rt_finding_icon(:warn), do: "fa-triangle-exclamation"
334 + defp rt_finding_icon(:ok), do: "fa-circle-check"
335 + defp rt_finding_icon(_severity), do: "fa-circle-info"
336 +
322 337 defp page_title(:show), do: "Offering Detail"
323 338 defp page_title(:edit), do: "Modify Offering"
324 339
@@ -802,6 +817,19 @@ defmodule RoomSanctumWeb.SourceLive.Show do
802 817 {:noreply, socket}
803 818 end
804 819
820 + # Fetches the source's realtime URLs and reports what is wrong with them. Runs
821 + # on demand rather than on load: it makes real requests, and some feeds are
822 + # metered.
823 + @impl true
824 + def handle_event("do-rt-test", _params, socket) do
825 + {:noreply, assign(socket, :rt_test, RoomGtfs.FeedTester.run(socket.assigns.source))}
826 + end
827 +
828 + @impl true
829 + def handle_event("dismiss-rt-test", _params, socket) do
830 + {:noreply, assign(socket, :rt_test, nil)}
831 + end
832 +
805 833 @impl true
806 834 def handle_event("do-stats", %{"type" => type, "id" => id}, socket) do
807 835 # IO.inspect({type, id})
modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/show.html.heex
+42 −0
@@ -288,6 +288,48 @@ gl_available_projects={@gitlab_available_projects}
288 288 </div>
289 289 </div>
290 290
291 + <div :if={@source.type == :gtfs} class="flex">
292 + <div class="card card-compact w-full bg-base-100 shadow-xl m-2">
293 + <div class="card-body">
294 + <div class="flex items-center justify-between">
295 + <h3 class="card-title">Realtime feeds</h3>
296 + <div class="flex gap-2">
297 + <button class="btn btn-secondary btn-sm" phx-click="do-rt-test">
298 + <i class="fa-solid fa-stethoscope mr-1"></i> Test
299 + </button>
300 + <button :if={@rt_test} class="btn btn-ghost btn-sm" phx-click="dismiss-rt-test">
301 + <i class="fa-solid fa-close"></i>
302 + </button>
303 + </div>
304 + </div>
305 +
306 + <p :if={is_nil(@rt_test)} class="text-xs opacity-60">
307 + Fetches each realtime URL and checks it against this offering's schedule.
308 + Makes one request per feed, so mind a metered source.
309 + </p>
310 +
311 + <p :if={@rt_test == []} class="text-sm opacity-70">
312 + No realtime URLs configured for this offering.
313 + </p>
314 +
315 + <div :for={group <- @rt_test || []} class="mt-2">
316 + <div class="text-xs font-mono opacity-70 break-all"><%= group.url %></div>
317 + <div class="text-xs opacity-60 mb-1">
318 + <%= Enum.map_join(group.kinds, ", ", &rt_kind_name/1) %>
319 + <%= if Map.get(group, :total) do %>
320 + &middot; <%= group.total %> entities
321 + <% end %>
322 + </div>
323 + <div :for={{severity, message} <- group.findings}
324 + class={"text-sm flex gap-2 items-start #{rt_finding_class(severity)}"}>
325 + <i class={"fa-solid fa-fw mt-0.5 #{rt_finding_icon(severity)}"}></i>
326 + <span><%= message %></span>
327 + </div>
328 + </div>
329 + </div>
330 + </div>
331 + </div>
332 +
291 333 <%= if @stats != %{} do %>
292 334 <div class="flex">
293 335 <div class="card card-compact w-full bg-base-100 shadow-xl m-2">

Parents: b6d9a98