neiam /apollos_crib

It's Apollo's Crib

public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages

Say when a gtfs-rt url is not returning protobuf

a953f33 · Gabriel Morell · 2026-08-11 18:37

7 files +136 -12

Message

{commit_body(@commit)}

Files changed

modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex

+77 −12

@@ -391,18 +391,72 @@ defmodule RoomGtfs.Worker.RT do

391	391
392	392	def fetch_rt_url(url) do
393	393	case HTTPoison.get(url, [], follow_redirect: true) do
	394	+ {:ok, %{status_code: 200} = result} ->
	395	+ decode_rt(url, result)
	396	+
394	397	{:ok, result} ->
395		− try do
396		− {:ok, result.body \|> TransitRealtime.FeedMessage.decode()}
397		− rescue
398		− e ->
399		− Logger.warning("failed to decode gtfs-rt protobuf from #{url}: #{inspect(e)}")
400		− {:error, :decode_failed}
401		− end
402		− {:error, error} -> {:error, error}
	398	+ Logger.warning(
	399	+ "gtfs-rt url #{url} answered HTTP #{result.status_code}: #{body_hint(result.body)}"
	400	+ )
	401	+
	402	+ {:error, :bad_status}
	403	+
	404	+ {:error, error} ->
	405	+ {:error, error}
	406	+ end
	407	+ end
	408	+
	409	+ # A dead endpoint rarely says so with a status code. MBTA's CDN answers a
	410	+ # missing object with 200 and an S3 AccessDenied document, which the decoder
	411	+ # then reads as protobuf and reports as "closing group 7 but no groups are
	412	+ # open" -- true, and useless for working out that the URL is wrong.
	413	+ defp decode_rt(url, %{body: body} = result) do
	414	+ if protobuf_response?(result) do
	415	+ try do
	416	+ {:ok, TransitRealtime.FeedMessage.decode(body)}
	417	+ rescue
	418	+ e ->
	419	+ Logger.warning("failed to decode gtfs-rt protobuf from #{url}: #{inspect(e)}")
	420	+ {:error, :decode_failed}
	421	+ end
	422	+ else
	423	+ Logger.warning("gtfs-rt url #{url} did not return protobuf: #{body_hint(body)}")
	424	+ {:error, :not_protobuf}
	425	+ end
	426	+ end
	427	+
	428	+ # Trust the content type when the server states one, since publishers vary
	429	+ # (application/x-protobuf, application/octet-stream). With no type at all,
	430	+ # fall back to rejecting bodies that are plainly markup.
	431	+ def protobuf_response?(%{body: body} = result) do
	432	+ case content_type(result) do
	433	+ nil -> not markup?(body)
	434	+ type -> String.contains?(type, "protobuf") or String.contains?(type, "octet-stream")
403	435	end
404	436	end
405	437
	438	+ defp content_type(%{headers: headers}) do
	439	+ Enum.find_value(headers, fn {name, value} ->
	440	+ if String.downcase(name) == "content-type", do: String.downcase(value)
	441	+ end)
	442	+ end
	443	+
	444	+ defp content_type(_result), do: nil
	445	+
	446	+ defp markup?(body) when is_binary(body) do
	447	+ body \|> String.trim_leading() \|> String.starts_with?(["<?xml", "<!DOCTYPE", "<html", "<HTML"])
	448	+ end
	449	+
	450	+ defp markup?(_body), do: false
	451	+
	452	+ # Enough of the body to recognise an error page, without pouring a 350kB
	453	+ # feed into the log.
	454	+ defp body_hint(body) when is_binary(body) do
	455	+ body \|> String.slice(0, 160) \|> String.replace(~r/\s+/, " ") \|> String.trim()
	456	+ end
	457	+
	458	+ defp body_hint(_body), do: "(no body)"
	459	+
406	460	def handle_cast(:refresh_db_cfg, state) do
407	461	inst = Configuration.get_source!(state.id)
408	462	{:noreply, state \|> Map.put(:inst, inst)}

@@ -1033,13 +1087,24 @@ end

1033	1087	# Only ever fills blanks: a URL already in the config was put there
1034	1088	# deliberately and outranks whatever the feed advertises.
1035	1089	defp apply_linked_datasets(cfg, urls) do
1036		− fill =
1037		− urls
1038		− \|> Enum.reject(fn {field, _url} ->
	1090	+ {configured, blank} =
	1091	+ Enum.split_with(urls, fn {field, _url} ->
1039	1092	existing = Map.get(cfg.config, field)
1040	1093	is_binary(existing) and String.trim(existing) != ""
1041	1094	end)
1042		− \|> Map.new()
	1095	+
	1096	+ # A URL set by hand still wins -- it may be a proxy, or carry a key the
	1097	+ # feed cannot know about. But when the feed names a different one it is
	1098	+ # worth saying so out loud: a stale URL usually fails as a decode error
	1099	+ # somewhere else entirely, with nothing pointing back here.
	1100	+ for {field, url} <- configured, Map.get(cfg.config, field) != url do
	1101	+ Logger.warning(
	1102	+ "GTFS::#{cfg.id} #{field} is set to #{Map.get(cfg.config, field)} " <>
	1103	+ "but the feed advertises #{url}"
	1104	+ )
	1105	+ end
	1106	+
	1107	+ fill = Map.new(blank)
1043	1108
1044	1109	if fill == %{} do
1045	1110	:ok

added apps/room_gtfs/test/rt_response_test.exs

+59 −0

@@ -0,0 +1,59 @@

	1	+defmodule RoomGtfs.RtResponseTest do
	2	+ @moduledoc """
	3	+ A dead GTFS-RT endpoint rarely says so with a status code. MBTA's CDN answers
	4	+ a missing object with HTTP 200 and an S3 AccessDenied document; fed to the
	5	+ protobuf decoder that surfaces as "closing group 7 but no groups are open",
	6	+ which is true and tells you nothing about the URL being wrong.
	7	+ """
	8	+ use ExUnit.Case, async: true
	9	+
	10	+ alias RoomGtfs.Worker.RT
	11	+
	12	+ defp response(content_type, body) do
	13	+ headers = if content_type, do: [{"Content-Type", content_type}], else: []
	14	+ %{headers: headers, body: body}
	15	+ end
	16	+
	17	+ test "protobuf content types are accepted" do
	18	+ for type <- ["application/x-protobuf", "application/octet-stream", "application/protobuf"] do
	19	+ assert RT.protobuf_response?(response(type, <<10, 5>>)), type
	20	+ end
	21	+ end
	22	+
	23	+ test "the header is matched case-insensitively and past any charset" do
	24	+ assert RT.protobuf_response?(%{
	25	+ headers: [{"content-type", "Application/X-Protobuf; charset=binary"}],
	26	+ body: <<10>>
	27	+ })
	28	+ end
	29	+
	30	+ test "an error document is rejected however it is served" do
	31	+ refute RT.protobuf_response?(response("application/xml", ~s(<?xml version="1.0"?><Error/>)))
	32	+ refute RT.protobuf_response?(response("text/html; charset=utf-8", "<!DOCTYPE HTML><h1>500</h1>"))
	33	+ refute RT.protobuf_response?(response("application/json", ~s({"error":"nope"})))
	34	+ end
	35	+
	36	+ test "the s3 access-denied body that started this is rejected" do
	37	+ body = """
	38	+ <?xml version="1.0" encoding="UTF-8"?>
	39	+ <Error><Code>AccessDenied</Code><Message>not authorized</Message></Error>
	40	+ """
	41	+
	42	+ refute RT.protobuf_response?(response("application/xml", body))
	43	+ end
	44	+
	45	+ test "with no content type at all, markup is still rejected" do
	46	+ refute RT.protobuf_response?(response(nil, ~s(<?xml version="1.0"?><Error/>)))
	47	+ refute RT.protobuf_response?(response(nil, "<!DOCTYPE HTML>"))
	48	+ refute RT.protobuf_response?(response(nil, " \n <html><body>oops</body></html>"))
	49	+ end
	50	+
	51	+ test "with no content type, binary is given the benefit of the doubt" do
	52	+ assert RT.protobuf_response?(response(nil, <<10, 5, 1, 2, 3>>))
	53	+ end
	54	+
	55	+ test "a stated protobuf type wins over a body that looks like markup" do
	56	+ # a feed whose first bytes happen to resemble markup is still protobuf
	57	+ assert RT.protobuf_response?(response("application/x-protobuf", "<?xml"))
	58	+ end
	59	+end

Parents: db17c28