neiam /apollos_crib

It's Apollo's Crib

public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages

Find a GTFS feed that is inside a folder in its zip

4c3c2fc · Gabriel Morell · 2026-09-07 03:40

10 files +102 -4

Message

{commit_body(@commit)}

Files changed

modified ?

+0 −0

Binary file — not rendered.

modified ?

+0 −0

Binary file — not rendered.

modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex

+53 −4

@@ -2138,6 +2138,54 @@ end

2138	2138	end
2139	2139	end
2140	2140
	2141	+ @doc false
	2142	+ # The entries of a zip that are actually this feed's GTFS files.
	2143	+ #
	2144	+ # Three things go wrong in real feeds, and all of them end the same way --
	2145	+ # matching the whole path against a list of bare names finds nothing, and an
	2146	+ # import that loads no files looks exactly like one that worked.
	2147	+ #
	2148	+ # * The files are inside a folder. Delhi publishes
	2149	+ # `delhi_buses_static_gtfs_v1-2/stops.txt`, which is not `stops.txt`.
	2150	+ #
	2151	+ # * A zip built on a Mac carries `__MACOSX/._stops.txt` beside every real
	2152	+ # file -- an AppleDouble resource fork, not a GTFS file. Matched on
	2153	+ # basename alone these are still excluded, since `._stops.txt` is not
	2154	+ # `stops.txt`, but they are dropped explicitly rather than by luck.
	2155	+ #
	2156	+ # * A zip can hold more than one feed, each in its own folder. Taking every
	2157	+ # `stops.txt` would merge two agencies into one source, so the directory
	2158	+ # with the most GTFS files wins and the rest are left alone.
	2159	+ def gtfs_entries(files, id \\ nil) do
	2160	+ by_dir =
	2161	+ files
	2162	+ \|> Enum.reject(&apple_double?/1)
	2163	+ \|> Enum.filter(&(Path.basename(&1.file_name) in @import_files))
	2164	+ \|> Enum.group_by(&Path.dirname(&1.file_name))
	2165	+
	2166	+ case by_dir \|> Enum.sort_by(fn {dir, entries} -> {-length(entries), dir} end) do
	2167	+ [] ->
	2168	+ []
	2169	+
	2170	+ [{dir, entries} \| rest] ->
	2171	+ if dir != "." do
	2172	+ Logger.info("GTFS::#{id} feed is nested under #{dir}/")
	2173	+ end
	2174	+
	2175	+ for {other, _} <- rest do
	2176	+ Logger.info("GTFS::#{id} ignoring a second feed under #{other}/")
	2177	+ end
	2178	+
	2179	+ entries
	2180	+ end
	2181	+ end
	2182	+
	2183	+ # `__MACOSX/._trips.txt` and friends: metadata a Mac's zip tool leaves
	2184	+ # behind, which is not a GTFS file however much its name looks like one.
	2185	+ defp apple_double?(%{file_name: name}) do
	2186	+ String.starts_with?(name, "__MACOSX/") or String.starts_with?(Path.basename(name), "._")
	2187	+ end
	2188	+
2141	2189	defp file_to_atom(filename) do
2142	2190	case filename do
2143	2191	"agency.txt" ->

@@ -2313,12 +2361,13 @@ end

2313	2361	# that numbers the steps makes the order the bar claims the order
2314	2362	# the files are actually written in, for every feed.
2315	2363	files
2316		− \|> Enum.filter(&(&1.file_name in @import_files))
2317		− \|> Enum.sort_by(&file_to_order(&1.file_name))
	2364	+ \|> gtfs_entries(id)
	2365	+ \|> Enum.sort_by(&file_to_order(Path.basename(&1.file_name)))
2318	2366	\|> Enum.map(fn e ->
2319		− type = file_to_atom(e.file_name)
	2367	+ name = Path.basename(e.file_name)
	2368	+ type = file_to_atom(name)
2320	2369
2321		− bcast(id, type, file_to_order(e.file_name), 12)
	2370	+ bcast(id, type, file_to_order(name), 12)
2322	2371
2323	2372	Unzip.file_stream!(unzip, e.file_name)
2324	2373	\|> write_file(type, id, nil)

added apps/room_gtfs/test/gtfs_entries_test.exs

+49 −0

@@ -0,0 +1,49 @@

	1	+defmodule RoomGtfs.GtfsEntriesTest do
	2	+ use ExUnit.Case, async: true
	3	+
	4	+ alias RoomGtfs.Worker.Static
	5	+
	6	+ defp picked(names) do
	7	+ names
	8	+ \|> Enum.map(&%{file_name: &1})
	9	+ \|> Static.gtfs_entries()
	10	+ \|> Enum.map(& &1.file_name)
	11	+ \|> Enum.sort()
	12	+ end
	13	+
	14	+ test "a feed at the root of the zip is unchanged" do
	15	+ assert picked(["stops.txt", "trips.txt", "feed_info.txt"]) == ["stops.txt", "trips.txt"]
	16	+ end
	17	+
	18	+ test "a feed inside a folder is found" do
	19	+ # Delhi publishes exactly this shape, and matching the whole path against a
	20	+ # list of bare names found nothing at all -- an import that loaded no files
	21	+ # and reported no error, which looks identical to one that worked.
	22	+ assert picked(["delhi_v1-2/stops.txt", "delhi_v1-2/trips.txt"]) ==
	23	+ ["delhi_v1-2/stops.txt", "delhi_v1-2/trips.txt"]
	24	+ end
	25	+
	26	+ test "AppleDouble junk from a Mac's zip tool is left out" do
	27	+ names = ["ncrtc/stops.txt", "__MACOSX/ncrtc/._stops.txt", "__MACOSX/._ncrtc", "ncrtc/trips.txt"]
	28	+
	29	+ assert picked(names) == ["ncrtc/stops.txt", "ncrtc/trips.txt"]
	30	+ end
	31	+
	32	+ test "two feeds in one zip do not become one source" do
	33	+ # Taking every stops.txt would merge two agencies. The fuller folder wins.
	34	+ names = ["agency_a/stops.txt", "agency_a/trips.txt", "agency_a/routes.txt", "agency_b/stops.txt"]
	35	+
	36	+ assert picked(names) ==
	37	+ ["agency_a/routes.txt", "agency_a/stops.txt", "agency_a/trips.txt"]
	38	+ end
	39	+
	40	+ test "a tie between folders is resolved the same way every time" do
	41	+ # Not by whichever the zip's hash-ordered entry list happened to give.
	42	+ assert picked(["b/stops.txt", "a/stops.txt"]) == ["a/stops.txt"]
	43	+ end
	44	+
	45	+ test "a zip with nothing recognisable selects nothing rather than raising" do
	46	+ assert picked(["readme.md", "__MACOSX/._readme.md"]) == []
	47	+ assert picked([]) == []
	48	+ end
	49	+end

Parents: 5730d3d