neiam
/apollos_crib
It's Apollo's Crib
public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages
⭐
Log in to mark this repository.
Join a stop's arrivals within one feed, and index for it
c319a5a · Gabriel Morell · 2026-08-29 22:01
Message
{commit_body(@commit)}
Files changed
modified
apps/room_sanctum/lib/room_sanctum/storage.ex
+29
−6
@@ -1292,14 +1292,37 @@ defmodule RoomSanctum.Storage do
| 1292 | 1292 | |> conditional_where(source_id, stop_id, timestamp_time_hour, timestamp_time) |
| 1293 | 1293 | |> order_by([st], asc: st.arrival_time) |
| 1294 | 1294 | |> limit(^limit) |
| 1295 | − |> join(:left, [st], s in Stop, as: :stop, on: s.stop_id == st.stop_id) | |
| 1296 | − |> join(:left, [st], t in Trip, as: :trip, on: t.trip_id == st.trip_id) | |
| 1297 | − |> join(:left, [_st, trip: t], r in Route, as: :route, on: t.route_id == r.route_id) | |
| 1298 | − |> join(:left, [_st, trip: t], d in Direction, | |
| 1295 | + # Every join carries source_id, and not only because a trip_id is unique | |
| 1296 | + # within a feed rather than across them -- 3,491 stop_ids, 438 route_ids | |
| 1297 | + # and 185 trip_ids are shared by two feeds in the dev database, so without | |
| 1298 | + # it a stop's arrivals pick up another agency's routes and calendars, and | |
| 1299 | + # one arrival comes back several times over. It is also the whole of the | |
| 1300 | + # performance story: every unique index on these tables leads with | |
| 1301 | + # source_id, so joining on the bare id can use none of them, and matching | |
| 1302 | + # a stop's arrivals meant a sequential scan of gtfs_trips -- 271k rows -- | |
| 1303 | + # hashed afresh on every request. | |
| 1304 | + |> join(:left, [st], s in Stop, | |
| 1305 | + as: :stop, | |
| 1306 | + on: s.stop_id == st.stop_id and s.source_id == st.source_id | |
| 1307 | + ) | |
| 1308 | + |> join(:left, [st], t in Trip, | |
| 1309 | + as: :trip, | |
| 1310 | + on: t.trip_id == st.trip_id and t.source_id == st.source_id | |
| 1311 | + ) | |
| 1312 | + |> join(:left, [st, trip: t], r in Route, | |
| 1313 | + as: :route, | |
| 1314 | + on: t.route_id == r.route_id and r.source_id == st.source_id | |
| 1315 | + ) | |
| 1316 | + |> join(:left, [st, trip: t], d in Direction, | |
| 1299 | 1317 | as: :direction, |
| 1300 | − on: t.direction_id == d.direction_id and t.route_id == d.route_id | |
| 1318 | + on: | |
| 1319 | + t.direction_id == d.direction_id and t.route_id == d.route_id and | |
| 1320 | + d.source_id == st.source_id | |
| 1321 | + ) | |
| 1322 | + |> join(:left, [st, trip: t], c in Calendar, | |
| 1323 | + as: :calendar, | |
| 1324 | + on: t.service_id == c.service_id and c.source_id == st.source_id | |
| 1301 | 1325 | ) |
| 1302 | − |> join(:left, [_st, trip: t], c in Calendar, as: :calendar, on: t.service_id == c.service_id) | |
| 1303 | 1326 | |> select( |
| 1304 | 1327 | [st, s, t, r, d, c], |
| 1305 | 1328 | # [c,d,r,t,s,st], |
added
apps/room_sanctum/priv/repo/migrations/20260829210000_index_stop_times_by_arrival.exs
+49
−0
@@ -0,0 +1,49 @@
| 1 | +defmodule RoomSanctum.Repo.Migrations.IndexStopTimesByArrival do | |
| 2 | + use Ecto.Migration | |
| 3 | + | |
| 4 | + # CREATE INDEX takes a lock that blocks writes for as long as it runs, and | |
| 5 | + # gtfs_stop_times is millions of rows per feed -- 14s to build on a dev copy, | |
| 6 | + # longer in production with imports running. Concurrently instead, which | |
| 7 | + # needs its own transaction. | |
| 8 | + @disable_ddl_transaction true | |
| 9 | + # The migration lock is itself a transaction, and CREATE INDEX CONCURRENTLY | |
| 10 | + # cannot run inside one -- without this the migration rolls back before it | |
| 11 | + # reaches the index. | |
| 12 | + @disable_migration_lock true | |
| 13 | + @timeout :infinity | |
| 14 | + | |
| 15 | + # if_not_exists throughout, because nothing here is rolled back on failure: | |
| 16 | + # a concurrent build runs outside the transaction that records the migration, | |
| 17 | + # so a run that dies partway leaves the index behind and un-recorded, and the | |
| 18 | + # retry has to be able to step over it. If a build is interrupted rather than | |
| 19 | + # failed, Postgres leaves an INVALID index that IF NOT EXISTS will step over | |
| 20 | + # too and never finish -- check | |
| 21 | + # `select indexrelid::regclass from pg_index where not indisvalid` after any | |
| 22 | + # interrupted run, and drop what it names before retrying. | |
| 23 | + | |
| 24 | + def up do | |
| 25 | + # "the next arrivals at this stop" filters on (source_id, stop_id) and a | |
| 26 | + # one-hour window of arrival_time, then orders by arrival_time and takes | |
| 27 | + # 16. Leading with the two equalities and ending on arrival_time lets one | |
| 28 | + # index scan answer the range, the ordering and the limit together: the | |
| 29 | + # scan stops after 16 rows instead of reading every arrival the stop has | |
| 30 | + # ever had -- 6,817 of them at the busiest stop in the dev feed -- and | |
| 31 | + # sorting them to throw all but 16 away. | |
| 32 | + create_if_not_exists index(:gtfs_stop_times, [:source_id, :stop_id, :arrival_time], | |
| 33 | + concurrently: true | |
| 34 | + ) | |
| 35 | + | |
| 36 | + # Redundant now: a prefix of the index above, which serves every lookup it | |
| 37 | + # served (get_trips_for_stop included) without a second 134MB copy to keep | |
| 38 | + # written on import. | |
| 39 | + drop_if_exists index(:gtfs_stop_times, [:source_id, :stop_id], concurrently: true) | |
| 40 | + end | |
| 41 | + | |
| 42 | + def down do | |
| 43 | + create_if_not_exists index(:gtfs_stop_times, [:source_id, :stop_id], concurrently: true) | |
| 44 | + | |
| 45 | + drop_if_exists index(:gtfs_stop_times, [:source_id, :stop_id, :arrival_time], | |
| 46 | + concurrently: true | |
| 47 | + ) | |
| 48 | + end | |
| 49 | +end |
Parents: a640fb2