GITGUD

neiam neiam /apollos_crib

It's Apollo's Crib

public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages

⭐

Log in to mark this repository.

Match NYCT's partial trip ids, behind a flag

8aca56e · Gabriel Morell · 2026-08-25 22:00

19 files +131 -25
Message 
{commit_body(@commit)}

Files changed

modified ?
+0 −0

Binary file — not rendered.

modified ?
+0 −0

Binary file — not rendered.

modified apps/room_gtfs/lib/room_gtfs/feed_tester.ex
+37 −17
@@ -239,26 +239,27 @@ defmodule RoomGtfs.FeedTester do
239 239 |> Enum.take(@sample)
240 240
241 241 result
242 − |> match_against(:trips, trips, source.id, &known_trips/2)
243 − |> match_against(:stops, stops, source.id, &known_stops/2)
242 + |> match_against(:trips, trips, source, &known_trips/2)
243 + |> match_against(:stops, stops, source, &known_stops/2)
244 244 end
245 245
246 − defp match_against(result, _label, [], _source_id, _fun), do: result
246 + defp match_against(result, _label, [], _source, _fun), do: result
247 247
248 − defp match_against(result, label, ids, source_id, fun) do
249 − known = fun.(source_id, ids)
248 + defp match_against(result, label, ids, source, fun) do
249 + known = fun.(source, ids)
250 250 pct = round(known / length(ids) * 100)
251 251 result = Map.put(result, :"#{label}_match", pct)
252 252
253 253 cond do
254 − pct == 0 and label == :trips and partial_trip_ids?(source_id, ids) ->
254 + pct == 0 and label == :trips and partial_trip_ids?(source.id, ids) ->
255 255 add(
256 256 result,
257 257 :error,
258 258 "its trip ids are the tail end of the schedule's -- realtime sends " <>
259 259 "\"098600_5..S03R\" where the schedule says " <>
260 − "\"ASP26GEN-1038-Sunday-00_098600_5..S03R\". NYCT publishes partial ids like this. " <>
261 − "Nothing matches on an exact comparison, so there are no live times"
260 + "\"ASP26GEN-1038-Sunday-00_098600_5..S03R\". NYCT publishes partial ids like " <>
261 + "this, and nothing matches on an exact comparison, so there are no live times. " <>
262 + "Tick \"Realtime sends partial trip ids\" on this offering to match on the tail"
262 263 )
263 264
264 265 pct == 0 ->
@@ -309,19 +310,38 @@ defmodule RoomGtfs.FeedTester do
309 310 matched > 0
310 311 end
311 312
312 − defp known_trips(source_id, ids) do
313 − Repo.one(
314 − from(t in "gtfs_trips",
315 − where: t.source_id == ^to_int(source_id) and t.trip_id in ^ids,
316 − select: count(fragment("DISTINCT ?", t.trip_id))
317 − )
318 − ) || 0
313 + # Counted the way the poller matches, so a source configured for partial ids is
314 + # not reported as broken for behaving exactly as configured.
315 + defp known_trips(source, ids) do
316 + if Map.get(source.config, :rt_trip_id_suffix) do
317 + # Counting realtime ids that found a schedule entry, not schedule entries
318 + # that were found: several scheduled trips share one realtime id across
319 + # service days, and counting those would read as more than 100%.
320 + Repo.one(
321 + from(r in fragment("SELECT unnest(?::text[]) AS id", ^ids),
322 + where:
323 + fragment(
324 + "EXISTS (SELECT 1 FROM gtfs_trips t WHERE t.source_id = ? AND t.trip_id LIKE '%' || ?)",
325 + ^to_int(source.id),
326 + r.id
327 + ),
328 + select: count()
329 + )
330 + ) || 0
331 + else
332 + Repo.one(
333 + from(t in "gtfs_trips",
334 + where: t.source_id == ^to_int(source.id) and t.trip_id in ^ids,
335 + select: count(fragment("DISTINCT ?", t.trip_id))
336 + )
337 + ) || 0
338 + end
319 339 end
320 340
321 − defp known_stops(source_id, ids) do
341 + defp known_stops(source, ids) do
322 342 Repo.one(
323 343 from(s in "gtfs_stops",
324 − where: s.source_id == ^to_int(source_id) and s.stop_id in ^ids,
344 + where: s.source_id == ^to_int(source.id) and s.stop_id in ^ids,
325 345 select: count(fragment("DISTINCT ?", s.stop_id))
326 346 )
327 347 ) || 0
modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex
+70 −7
@@ -101,11 +101,50 @@ defmodule RoomGtfs.Worker do
101 101 |> GenServer.call({:query_vehicle_positions, trips}, 30_000)
102 102 end
103 103
104 + @doc """
105 + Whether a realtime trip id refers to a scheduled trip id.
106 +
107 + Usually they are the same string. NYCT sends the tail of it -- realtime
108 + "098600_5..S03R" against a schedule that says
109 + "ASP26GEN-1038-Sunday-00_098600_5..S03R", the leading part naming which
110 + published schedule the trip belongs to -- so an exact comparison finds
111 + nothing and those arrivals never show a live time.
112 +
113 + Off unless the source says so. A suffix comparison is looser and can match a
114 + trip it should not; a feed that needs it should declare it rather than have it
115 + guessed.
116 + """
117 + def trip_id_match?(config, scheduled, realtime)
118 +
119 + def trip_id_match?(_config, scheduled, realtime) when scheduled == realtime, do: true
120 +
121 + def trip_id_match?(%{rt_trip_id_suffix: true}, scheduled, realtime)
122 + when is_binary(scheduled) and is_binary(realtime) do
123 + String.ends_with?(scheduled, realtime)
124 + end
125 +
126 + def trip_id_match?(_config, _scheduled, _realtime), do: false
127 +
128 + @doc false
129 + # A realtime id against a whole list of scheduled ones. Exact matching gets a
130 + # MapSet; suffix matching cannot, and the lists it runs against are the trips
131 + # calling at one stop, so a scan is the right size of answer.
132 + def any_trip_match?(%{rt_trip_id_suffix: true} = config, scheduled_ids, realtime) do
133 + Enum.any?(scheduled_ids, &trip_id_match?(config, &1, realtime))
134 + end
135 +
136 + def any_trip_match?(_config, scheduled_ids, realtime) do
137 + realtime in scheduled_ids
138 + end
139 +
104 140 def query_stop(id, query) do
105 141 inst = Configuration.get_source!(id)
106 142 res = Storage.get_upcoming_arrivals_for_stop(id, query.stop) |> Storage.fix_arrival_times
107 143
108 − case inst.config.url_rt_tu do
144 + # Not `url_rt_tu` alone: a source whose trip updates arrive in a combined
145 + # feed has that field blank and its URL in url_rt_shared, and gating on the
146 + # specific field skipped realtime entirely for exactly those sources.
147 + case RoomGtfs.Worker.RT.rt_url_for(inst.config, :tu) do
109 148 nil ->
110 149 res
111 150
@@ -119,7 +158,9 @@ defmodule RoomGtfs.Worker do
119 158 rtvals ->
120 159 res
121 160 |> Enum.map(fn x ->
122 − case Enum.find(rtvals, fn v -> x.trip_id == v.trip_update.trip.trip_id end) do
161 + case Enum.find(rtvals, fn v ->
162 + trip_id_match?(inst.config, x.trip_id, v.trip_update.trip.trip_id)
163 + end) do
123 164 nil ->
124 165 x
125 166
@@ -880,9 +921,24 @@ defmodule RoomGtfs.Worker.RT do
880 921 # subway feeds are 67 trip updates, 45 vehicle positions and an alert in a
881 922 # single URL -- names it once instead of three times. Specific beats general,
882 923 # so a combined feed plus a dedicated alerts feed works too.
883 − defp rt_url(config, :sa), do: Map.get(config, :url_rt_sa) || Map.get(config, :url_rt_shared)
884 − defp rt_url(config, :tu), do: Map.get(config, :url_rt_tu) || Map.get(config, :url_rt_shared)
885 − defp rt_url(config, :vp), do: Map.get(config, :url_rt_vp) || Map.get(config, :url_rt_shared)
924 + @doc """
925 + Where a kind's feed comes from: its own URL, or the combined one behind it.
926 +
927 + Public because the question is asked outside the poller too -- `query_stop/2`
928 + has to know whether trip updates exist at all before it goes looking for them,
929 + and asking `url_rt_tu` directly gets the wrong answer for a combined feed.
930 + """
931 + def rt_url_for(config, kind)
932 +
933 + def rt_url_for(config, :sa), do: blank(Map.get(config, :url_rt_sa)) || blank(Map.get(config, :url_rt_shared))
934 + def rt_url_for(config, :tu), do: blank(Map.get(config, :url_rt_tu)) || blank(Map.get(config, :url_rt_shared))
935 + def rt_url_for(config, :vp), do: blank(Map.get(config, :url_rt_vp)) || blank(Map.get(config, :url_rt_shared))
936 +
937 + defp blank(nil), do: nil
938 + defp blank(""), do: nil
939 + defp blank(value), do: value
940 +
941 + defp rt_url(config, kind), do: rt_url_for(config, kind)
886 942
887 943 defp entity_field(:sa), do: :alert
888 944 defp entity_field(:tu), do: :trip_update
@@ -943,7 +999,11 @@ defmodule RoomGtfs.Worker.RT do
943 999 # files, which is why this held up until a subway source was added.
944 1000 |> Enum.filter(fn x ->
945 1001 x.trip_update && x.trip_update.trip &&
946 − Enum.member?(trips, x.trip_update.trip.trip_id)
1002 + RoomGtfs.Worker.any_trip_match?(
1003 + state.inst && state.inst.config,
1004 + trips,
1005 + x.trip_update.trip.trip_id
1006 + )
947 1007 end)
948 1008 |> Enum.map(fn x ->
949 1009 x
@@ -969,7 +1029,10 @@ defmodule RoomGtfs.Worker.RT do
969 1029 vehicles =
970 1030 state.rt_vp
971 1031 |> vehicle_positions_from(state.stops)
972 − |> Enum.filter(&(&1.trip_id && &1.trip_id in trips))
1032 + |> Enum.filter(
1033 + &(&1.trip_id &&
1034 + RoomGtfs.Worker.any_trip_match?(state.inst && state.inst.config, trips, &1.trip_id))
1035 + )
973 1036
974 1037 {:reply, vehicles, state}
975 1038 end
modified apps/room_sanctum/lib/room_sanctum/configuration/configs/gtfs.ex
+11 −1
@@ -39,6 +39,16 @@ defmodule RoomSanctum.Configuration.Configs.GTFS do
39 39 # and says what prefix to strip. Left blank for a single-agency feed, which
40 40 # is nearly all of them.
41 41 field :rt_agency, :string
42 +
43 + # Whether realtime trip ids are the tail of the schedule's rather than the
44 + # whole thing. NYCT does this: realtime says "098600_5..S03R" where
45 + # gtfs_subway.zip says "ASP26GEN-1038-Sunday-00_098600_5..S03R", the leading
46 + # part being which published schedule the trip belongs to.
47 + #
48 + # Off by default, and deliberately not inferred. A suffix comparison is
49 + # looser than an exact one and can match a trip it should not; a feed that
50 + # needs it should say so rather than have it guessed from a bad day's data.
51 + field :rt_trip_id_suffix, :boolean, default: false
42 52 end
43 53
44 54 def changeset(source, params) do
@@ -46,7 +56,7 @@ defmodule RoomSanctum.Configuration.Configs.GTFS do
46 56 |> cast(
47 57 params,
48 58 ~w(url url_rt_sa url_rt_tu url_rt_vp url_rt_shared tz rt_agency
49 − rt_period_tu rt_period_vp rt_period_sa)a
59 + rt_trip_id_suffix rt_period_tu rt_period_vp rt_period_sa)a
50 60 )
51 61 |> validate_required([:url, :tz])
52 62 |> validate_periods()
modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/form_component.html.heex
+13 −0
@@ -125,6 +125,19 @@
125 125 Roughly <span class="font-bold"><%= @rt_requests %></span> requests an hour.
126 126 511.org allows 60 on a default token.
127 127 </div>
128 + <div class="form-control">
129 + <.input field={gtfs_form[:rt_trip_id_suffix]} type="checkbox"
130 + label="Realtime sends partial trip ids"/>
131 + <label class="label">
132 + <span class="label-text-alt opacity-70">
133 + For a feed whose trip ids are the tail of the schedule's &mdash; NYCT
134 + subway sends <code>098600_5..S03R</code> where the schedule says
135 + <code>ASP26GEN-1038-Sunday-00_098600_5..S03R</code>. Leave off unless the
136 + feed tester says otherwise; matching on a suffix is looser than matching
137 + exactly.
138 + </span>
139 + </label>
140 + </div>
128 141 <div class="form-control">
129 142 <.input field={gtfs_form[:rt_agency]} type="text" label="Realtime operator code"/>
130 143 <label class="label">

Parents: afda479