neiam /apollos_crib

It's Apollo's Crib

Report shapes while it is being written, and in the stats

3ba048f · Gabriel Morell · 2026-08-24 05:09

11 files +51 -26

Message

{commit_body(@commit)}

modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex

+44 −25

@@ -245,6 +245,7 @@ defmodule RoomGtfs.Worker do

245	245	calendars: Storage.count_calendars(id),
246	246	directions: Storage.count_directions(id),
247	247	routes: Storage.count_routes(id),
	248	+ shapes: Storage.count_shapes(id),
248	249	stops: Storage.count_stops(id),
249	250	stop_times: Storage.count_stop_times(id),
250	251	trips: Storage.count_trips(id),

@@ -727,6 +728,21 @@ defmodule RoomGtfs.Worker.Static do

727	728	alias RoomSanctum.Repo
728	729	alias RoomSanctum.Storage.GTFS
729	730
	731	+ # The files that get loaded, and the only ones file_to_atom/1 and
	732	+ # file_to_order/1 accept. Everything else in a feed -- calendar_dates.txt,
	733	+ # feed_info.txt, linked_datasets.txt -- is either handled separately or
	734	+ # ignored.
	735	+ @import_files ~w(
	736	+ agency.txt
	737	+ calendar.txt
	738	+ directions.txt
	739	+ routes.txt
	740	+ shapes.txt
	741	+ stops.txt
	742	+ stop_times.txt
	743	+ trips.txt
	744	+ )
	745	+
730	746	def start_link(opts) do
731	747	GenServer.start_link(__MODULE__, opts, name: via_tuple("gtfs-st" <> opts[:name]))
732	748	end

@@ -1241,16 +1257,23 @@ end

1241	1257	end
1242	1258	end
1243	1259
	1260	+ # The order the progress bar counts in, which has to be the order the files are
	1261	+ # actually written in or the bar moves backwards. Entries come from the zip's
	1262	+ # own directory, which for every feed seen here is alphabetical -- so shapes is
	1263	+ # written fifth, between routes and stops, and was previously numbered tenth.
	1264	+ # Being numbered last meant its step arrived as "10 of 10", which the source
	1265	+ # page reads as completion: the bar hit 100% and flipped to idle while three
	1266	+ # more files, stop_times included, were still to load.
1244	1267	def file_to_order(filename) do
1245	1268	case filename do
1246	1269	"agency.txt" -> 3
1247	1270	"calendar.txt" -> 4
1248	1271	"directions.txt" -> 5
1249	1272	"routes.txt" -> 6
1250		− "stops.txt" -> 7
1251		− "stop_times.txt" -> 8
1252		− "trips.txt" -> 9
1253		− "shapes.txt" -> 10
	1273	+ "shapes.txt" -> 7
	1274	+ "stops.txt" -> 8
	1275	+ "stop_times.txt" -> 9
	1276	+ "trips.txt" -> 10
1254	1277	end
1255	1278	end
1256	1279

@@ -1293,11 +1316,11 @@ end

1293	1316
1294	1317	defp do_import_static(id, cfg) do
1295	1318	Logger.info("GTFS::#{id} updating static info")
1296		− bcast(id, :downloading, 1, 10)
	1319	+ bcast(id, :downloading, 1, 11)
1297	1320
1298	1321	case HTTPoison.get(cfg.config.url, [], follow_redirect: true) do
1299	1322	{:ok, result} ->
1300		− bcast(id, :extracting, 2, 10)
	1323	+ bcast(id, :extracting, 2, 11)
1301	1324
1302	1325	case result.body \|> Unzip.InMem.new() \|> Unzip.new() do
1303	1326	{:ok, unzip} ->

@@ -1317,27 +1340,23 @@ end

1317	1340	end
1318	1341
1319	1342	try do
	1343	+ # Sorted, because `Unzip.list_entries/1` maps over `cd_list`, which
	1344	+ # is a map -- so the order it hands back is neither the zip's nor
	1345	+ # alphabetical, it is whatever the hash gives. Left to that, the
	1346	+ # progress bar walks backwards: stop_times (step 9) came out ahead
	1347	+ # of stops (step 8) on the MTA feeds. Sorting by the same function
	1348	+ # that numbers the steps makes the order the bar claims the order
	1349	+ # the files are actually written in, for every feed.
1320	1350	files
	1351	+ \|> Enum.filter(&(&1.file_name in @import_files))
	1352	+ \|> Enum.sort_by(&file_to_order(&1.file_name))
1321	1353	\|> Enum.map(fn e ->
1322		− if Enum.member?(
1323		− [
1324		− "agency.txt",
1325		− "calendar.txt",
1326		− "directions.txt",
1327		− "routes.txt",
1328		− "stops.txt",
1329		− "stop_times.txt",
1330		− "trips.txt",
1331		− "shapes.txt",
1332		− ],
1333		− e.file_name
1334		− ) do
1335		−
1336		− bcast(id, file_to_atom(e.file_name), file_to_order(e.file_name), 10)
1337		−
1338		− Unzip.file_stream!(unzip, e.file_name)
1339		− \|> write_file(file_to_atom(e.file_name), id, nil)
1340		− end
	1354	+ type = file_to_atom(e.file_name)
	1355	+
	1356	+ bcast(id, type, file_to_order(e.file_name), 11)
	1357	+
	1358	+ Unzip.file_stream!(unzip, e.file_name)
	1359	+ \|> write_file(type, id, nil)
1341	1360	end)
1342	1361	rescue
1343	1362	e ->

modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/show.ex

+7 −1

@@ -1165,7 +1165,13 @@ defmodule RoomSanctumWeb.SourceLive.Show do

1165	1165	@impl true
1166	1166	def handle_info({type, id, :done} = info, socket) do
1167	1167	source = Configuration.get_source!(socket.assigns.source_id)
1168		− {:noreply, socket \|> assign(:source, source) \|> put_flash(:info, "Completed")}
	1168	+
	1169	+ {:noreply,
	1170	+ socket
	1171	+ \|> assign(:source, source)
	1172	+ \|> assign(:status, :idle)
	1173	+ \|> assign(:status_val, 0)
	1174	+ \|> put_flash(:info, "Completed")}
1169	1175	end
1170	1176
1171	1177	# Handle vehicle position updates for source-specific display

Parents: 51bc3c5