neiam /apollos_crib

It's Apollo's Crib

Let a hundred-megabyte timetable finish downloading

3c00594 · Gabriel Morell · 2026-09-04 15:07

6 files +43 -3

Message

{commit_body(@commit)}

modified apps/room_gtfs/lib/room_gtfs/application.ex

+18 −2

@@ -31,8 +31,24 @@ defmodule RoomGtfs.Application do

31	31	# start callback, it is.
32	32	:ok = Protobuf.load_extensions()
33	33
34		− children = [RoomGtfs.FeedCache, RoomGtfs.RTIndex]
	34	+ children = [static_pool(), RoomGtfs.FeedCache, RoomGtfs.RTIndex]
35	35
36	36	Supervisor.start_link(children, strategy: :one_for_one, name: RoomGtfs.Supervisor)
37	37	end
38		−end
	38	+
	39	+ # The connection pool the static feed downloads use.
	40	+ #
	41	+ # Started here with a size rather than left for hackney to create on demand,
	42	+ # because the size is the point. A static feed is the whole timetable and
	43	+ # they run to a hundred megabytes; four at once is already several hundred
	44	+ # megabytes in flight, and each one holds its connection for minutes. Left
	45	+ # unbounded, a scheduled refresh of every source at once would try to hold
	46	+ # them all.
	47	+ #
	48	+ # Separate from hackney's default pool for the same reason. Everything else
	49	+ # in this umbrella -- realtime feeds, GitHub, the weather -- shares that one,
	50	+ # and a bulk transfer sitting in it starves requests that take milliseconds.
	51	+ defp static_pool do
	52	+ :hackney_pool.child_spec(:gtfs_static, max_connections: 4, timeout: 300_000)
	53	+ end
	54	+end

modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex

+26 −2

@@ -736,6 +736,11 @@ defmodule RoomGtfs.Worker do

736	736	end
737	737
738	738	defmodule RoomGtfs.Worker.RT do
	739	+
	740	+ # A realtime feed is a protobuf of a few hundred kilobytes, fetched on a
	741	+ # tick. Explicit rather than left to HTTPoison's five seconds, which is the
	742	+ # right number here but should not be a coincidence.
	743	+ @rt_http [follow_redirect: true, recv_timeout: 10_000, timeout: 8_000]
739	744	use GenServer
740	745	@registry :zeus
741	746

@@ -1147,7 +1152,7 @@ defmodule RoomGtfs.Worker.RT do

1147	1152
1148	1153
1149	1154	def fetch_rt_url(url) do
1150		− case HTTPoison.get(url, [], follow_redirect: true) do
	1155	+ case HTTPoison.get(url, [], @rt_http) do
1151	1156	{:ok, %{status_code: 200} = result} ->
1152	1157	decode_rt(url, result)
1153	1158

@@ -1602,6 +1607,25 @@ defmodule RoomGtfs.Worker.RT do

1602	1607	end
1603	1608
1604	1609	defmodule RoomGtfs.Worker.Static do
	1610	+
	1611	+ # A static feed is the whole timetable, and they are enormous: Chicago's is
	1612	+ # 95 MiB, which is half a minute of transfer on a good link and several on a
	1613	+ # bad one. Left on HTTPoison's five second default it could not finish, and
	1614	+ # had not been finishing -- every run ended in `%HTTPoison.Error{reason:
	1615	+ # :timeout}` and the source simply never updated.
	1616	+ #
	1617	+ # The pool matters as much as the number. Every HTTP client in this umbrella
	1618	+ # -- realtime feeds, GitHub, GitLab, the weather -- shares hackney's default
	1619	+ # pool, and a hundred megabytes takes a connection out of it for minutes.
	1620	+ # That is what the `:checkout_timeout`s on unrelated requests were: not those
	1621	+ # services being slow, but this download holding the pool. A pool of its own
	1622	+ # means a bulk transfer can only ever starve another bulk transfer.
	1623	+ @static_http [
	1624	+ follow_redirect: true,
	1625	+ recv_timeout: 300_000,
	1626	+ timeout: 30_000,
	1627	+ hackney: [pool: :gtfs_static]
	1628	+ ]
1605	1629	use GenServer
1606	1630	require Logger
1607	1631	@registry :zeus

@@ -2258,7 +2282,7 @@ end

2258	2282	Logger.info("GTFS::#{id} updating static info")
2259	2283	bcast(id, :downloading, 1, 11)
2260	2284
2261		− case HTTPoison.get(cfg.config.url, [], follow_redirect: true) do
	2285	+ case HTTPoison.get(cfg.config.url, [], @static_http) do
2262	2286	{:ok, result} ->
2263	2287	bcast(id, :extracting, 2, 11)
2264	2288

Parents: c514686