neiam
/apollos_crib
It's Apollo's Crib
public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages
⭐
Log in to mark this repository.
Ask a source's counts together, and hold them for a minute
75067f6 · Gabriel Morell · 2026-09-02 19:16
Message
{commit_body(@commit)}
Files changed
modified
apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex
+62
−21
@@ -606,30 +606,71 @@ defmodule RoomGtfs.Worker do
| 606 | 606 | |
| 607 | 607 | defp via_tuple(name), do: {:via, Registry, {@registry, name}} |
| 608 | 608 | |
| 609 | + @doc """ | |
| 610 | + The row counts behind a source, for the stats card. | |
| 611 | + | |
| 612 | + Every one of these is an exact `count(*)` over a table filtered to one | |
| 613 | + source, and two of those tables hold millions of rows -- five million stop | |
| 614 | + times and half a million shape points for one MBTA feed. Asked one after | |
| 615 | + another that is several seconds of database work before anything renders, | |
| 616 | + which is what made the button time out. | |
| 617 | + | |
| 618 | + So: asked together, and the answer held for a minute. They change when a feed | |
| 619 | + imports and not otherwise, so the worst a stale one can be is a count from | |
| 620 | + before the import currently running -- which the queue page is there to show. | |
| 621 | + """ | |
| 609 | 622 | def source_stats(id) do |
| 610 | − %{ | |
| 611 | − calendars: Storage.count_calendars(id), | |
| 612 | − # The three numbers that say whether service-day filtering is working for | |
| 613 | − # this source, which a count of rows on its own does not. | |
| 614 | − # | |
| 615 | − # calendar_dates -- did calendar_dates.txt import at all | |
| 616 | − # services_today -- zero means the filter stands down entirely and | |
| 617 | − # the board shows every pattern, running or not | |
| 618 | − # trips_no_service -- trips kept because neither file mentions their | |
| 619 | − # service; a large share means the same thing | |
| 620 | − calendar_dates: Storage.count_calendar_dates(id), | |
| 621 | − services_today: services_today(id), | |
| 622 | − trips_no_service: Storage.count_trips_without_service(id), | |
| 623 | − directions: Storage.count_directions(id), | |
| 624 | − routes: Storage.count_routes(id), | |
| 625 | − shapes: Storage.count_shapes(id), | |
| 626 | − stops: Storage.count_stops(id), | |
| 627 | − stop_times: Storage.count_stop_times(id), | |
| 628 | − trips: Storage.count_trips(id), | |
| 629 | − rt: RoomGtfs.Worker.RT.stats(id), | |
| 630 | − } | |
| 623 | + case RoomSanctum.QueryCache.fetch({:source_stats, id}, fn -> gather_stats(id) end, 60_000) do | |
| 624 | + {:ok, stats} -> stats | |
| 625 | + # Someone else is already gathering them. Saying so beats waiting several | |
| 626 | + # seconds to say the same thing. | |
| 627 | + :busy -> %{gathering: true} | |
| 628 | + end | |
| 631 | 629 | end |
| 632 | 630 | |
| 631 | + @stat_names [ | |
| 632 | + :calendars, | |
| 633 | + :calendar_dates, | |
| 634 | + :services_today, | |
| 635 | + :trips_no_service, | |
| 636 | + :directions, | |
| 637 | + :routes, | |
| 638 | + :shapes, | |
| 639 | + :stops, | |
| 640 | + :stop_times, | |
| 641 | + :trips, | |
| 642 | + :rt | |
| 643 | + ] | |
| 644 | + | |
| 645 | + defp gather_stats(id) do | |
| 646 | + [ | |
| 647 | + fn -> Storage.count_calendars(id) end, | |
| 648 | + fn -> Storage.count_calendar_dates(id) end, | |
| 649 | + fn -> services_today(id) end, | |
| 650 | + fn -> Storage.count_trips_without_service(id) end, | |
| 651 | + fn -> Storage.count_directions(id) end, | |
| 652 | + fn -> Storage.count_routes(id) end, | |
| 653 | + fn -> Storage.count_shapes(id) end, | |
| 654 | + fn -> Storage.count_stops(id) end, | |
| 655 | + fn -> Storage.count_stop_times(id) end, | |
| 656 | + fn -> Storage.count_trips(id) end, | |
| 657 | + fn -> RoomGtfs.Worker.RT.stats(id) end | |
| 658 | + ] | |
| 659 | + |> Task.async_stream(& &1.(), | |
| 660 | + max_concurrency: 4, | |
| 661 | + timeout: 20_000, | |
| 662 | + on_timeout: :kill_task | |
| 663 | + ) | |
| 664 | + |> Enum.zip(@stat_names) | |
| 665 | + |> Enum.map(fn | |
| 666 | + {{:ok, value}, name} -> {name, value} | |
| 667 | + # One count that will not answer should not cost the other ten. | |
| 668 | + {{:exit, _reason}, name} -> {name, :unavailable} | |
| 669 | + end) | |
| 670 | + |> Enum.into(%{}) | |
| 671 | + end | |
| 672 | + | |
| 673 | + | |
| 633 | 674 | # In the source's own timezone: a board in Boston rolls over to the next |
| 634 | 675 | # service day at midnight there, not at midnight UTC, and for part of every |
| 635 | 676 | # day the two disagree about which services are running. |
modified
apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/show.ex
+26
−10
@@ -241,6 +241,23 @@ defmodule RoomSanctumWeb.SourceLive.Show do
| 241 | 241 | |
| 242 | 242 | # "Weather at Home" reads better in a list of queries than "Home" alone, |
| 243 | 243 | # which is what the map's + query produces for a stop. |
| 244 | + # The realtime figures are a map of maps, one per feed kind, flattened so the | |
| 245 | + # card can list them. A kind that could not be reached comes back as an atom | |
| 246 | + # rather than a map, and so does the whole reply when the RT worker itself is | |
| 247 | + # not answering -- neither should take the stats card down. | |
| 248 | + defp flatten_rt_stats(rt) when is_map(rt) do | |
| 249 | + rt | |
| 250 | + |> Enum.flat_map(fn {feed, info} -> | |
| 251 | + case info do | |
| 252 | + m when is_map(m) -> Enum.map(m, fn {k, v} -> {:"#{feed}_#{k}", v} end) | |
| 253 | + other -> [{feed, other}] | |
| 254 | + end | |
| 255 | + end) | |
| 256 | + |> Map.new() | |
| 257 | + end | |
| 258 | + | |
| 259 | + defp flatten_rt_stats(other), do: %{realtime: other || :unavailable} | |
| 260 | + | |
| 244 | 261 | defp query_name_for(type, %{name: name}), do: "#{String.capitalize(to_string(type))} at #{name}" |
| 245 | 262 | defp query_name_for(type, _foci), do: to_string(type) |
| 246 | 263 |
@@ -837,16 +854,15 @@ defmodule RoomSanctumWeb.SourceLive.Show do
| 837 | 854 | stats = |
| 838 | 855 | case type do |
| 839 | 856 | "gtfs" -> |
| 840 | − stats = RoomGtfs.Worker.source_stats(id) | |
| 841 | − rt_flat = stats.rt | |
| 842 | − |> Enum.flat_map(fn {feed, info} -> | |
| 843 | − case info do | |
| 844 | − m when is_map(m) -> Enum.map(m, fn {k, v} -> {:"#{feed}_#{k}", v} end) | |
| 845 | − other -> [{feed, other}] | |
| 846 | − end | |
| 847 | − end) | |
| 848 | − |> Map.new() | |
| 849 | − %{gtfs: Map.delete(stats, :rt), rt: rt_flat} | |
| 857 | + case RoomGtfs.Worker.source_stats(id) do | |
| 858 | + # Somebody else is already counting. The numbers land on the next | |
| 859 | + # press rather than after several seconds of waiting for theirs. | |
| 860 | + %{gathering: true} -> | |
| 861 | + %{gtfs: %{counting: "in progress, try again in a moment"}} | |
| 862 | + | |
| 863 | + stats -> | |
| 864 | + %{gtfs: Map.delete(stats, :rt), rt: flatten_rt_stats(Map.get(stats, :rt))} | |
| 865 | + end | |
| 850 | 866 | "gbfs" -> %{gbfs: RoomGbfs.Worker.source_stats(id), system: RoomGbfs.Worker.sys_info_as_stats(id), free: RoomGbfs.Worker.free_stats(id)} |
| 851 | 867 | _otherwise -> %{} |
| 852 | 868 | end |
Parents: 4758a10