GITGUD

neiam neiam /apollos_crib

It's Apollo's Crib

public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages

⭐

Log in to mark this repository.

A poll interval per realtime kind, on a slider

3b86b77 · Gabriel Morell · 2026-08-25 21:36

19 files +266 -72
Message 
{commit_body(@commit)}

Files changed

modified ?
+0 −0

Binary file — not rendered.

modified ?
+0 −0

Binary file — not rendered.

modified apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex
+76 −55
@@ -268,6 +268,11 @@ defmodule RoomGtfs.Worker.RT do
268 268 # Static stops only change when a feed is reimported.
269 269 @stops_ttl :timer.minutes(5)
270 270
271 + # What a kind polls at when the source has not said. One request a minute per
272 + # kind is fresh enough for a bus and cheap enough for a metered feed to be
273 + # worth a second look before lowering.
274 + @default_rt_period :timer.seconds(60)
275 +
271 276 def start_link(opts) do
272 277 GenServer.start_link(__MODULE__, opts, name: via_tuple("gtfs-rt" <> opts[:name]))
273 278 end
@@ -302,7 +307,10 @@ defmodule RoomGtfs.Worker.RT do
302 307 )
303 308
304 309 Periodic.start_link(
305 − every: :timer.seconds(30),
310 + # 15s because that is the finest interval a source can be set to, and the
311 + # tick is the floor on all of them. Every kind that is not due returns
312 + # immediately, so the extra ticks cost a message and a map lookup.
313 + every: :timer.seconds(15),
306 314 run: fn -> RoomGtfs.Worker.RT.update_realtime_data(opts[:name]) end,
307 315 initial_delay: :timer.seconds(60)
308 316 )
@@ -317,7 +325,9 @@ defmodule RoomGtfs.Worker.RT do
317 325 # stop_id => {lat, lon}, for feeds that report a station instead of a
318 326 # coordinate. See vehicle_positions_from/2.
319 327 stops: %{},
320 − stops_at: nil
328 + stops_at: nil,
329 + # kind => when it was last polled; each has its own interval.
330 + rt_polled_at: %{}
321 331 }}
322 332 end
323 333
@@ -332,11 +342,8 @@ defmodule RoomGtfs.Worker.RT do
332 342 # A source polling every three minutes can happily use data up to three
333 343 # minutes old, and several sources sharing one URL -- which is the whole point
334 344 # of the regional feed -- then fetch it once between them instead of once each.
335 − defp cached_get(state, url) do
336 − case rt_period_ms(state) do
337 − nil -> FeedCache.get(url)
338 − period -> FeedCache.get(url, period - :timer.seconds(5))
339 − end
345 + defp cached_get(state, kind, url) do
346 + FeedCache.get(url, max(rt_period_ms(state, kind) - :timer.seconds(5), :timer.seconds(1)))
340 347 end
341 348
342 349 defp record_health(state, kind, url, result, took) do
@@ -748,34 +755,44 @@ defmodule RoomGtfs.Worker.RT do
748 755 end
749 756
750 757 def handle_cast(:update_realtime, state) do
751 − if rt_due?(state) do
752 − do_update_realtime(Map.put(state, :rt_polled_at, System.monotonic_time(:millisecond)))
753 − else
754 − {:noreply, state}
758 + do_update_realtime(state)
759 + end
760 +
761 + # Each kind keeps its own clock. Trip updates go stale in seconds; service
762 + # alerts change a few times a day, and polling them as often as vehicle
763 + # positions buys nothing and costs a request -- which matters where the feed
764 + # is metered, and is simply waste where it is not.
765 + #
766 + # Gated here rather than by rescheduling the Periodic, which cannot be changed
767 + # once started and would have to be torn down and rebuilt whenever the config
768 + # is refreshed -- every four seconds, as it happens.
769 + defp due?(state, kind) do
770 + case state |> Map.get(:rt_polled_at, %{}) |> Map.get(kind) do
771 + nil -> true
772 + at -> System.monotonic_time(:millisecond) - at >= rt_period_ms(state, kind)
755 773 end
756 774 end
757 775
758 − # The poller ticks every 30s for everyone; a source with its own interval is
759 − # gated here rather than by rescheduling the Periodic, which cannot be changed
760 − # once started and would have to be torn down and rebuilt on every config
761 − # refresh.
762 − defp rt_due?(state) do
763 − case rt_period_ms(state) do
764 − nil ->
765 − true
776 + defp mark_polled(state, kind) do
777 + polled = state |> Map.get(:rt_polled_at, %{}) |> Map.put(kind, System.monotonic_time(:millisecond))
778 + Map.put(state, :rt_polled_at, polled)
779 + end
766 780
767 − period ->
768 − case Map.get(state, :rt_polled_at) do
769 − nil -> true
770 − at -> System.monotonic_time(:millisecond) - at >= period
771 − end
781 + defp rt_period_ms(state, kind) do
782 + state.inst
783 + |> case do
784 + %{config: config} -> Map.get(config, period_field(kind))
785 + _ -> nil
786 + end
787 + |> case do
788 + seconds when is_integer(seconds) -> :timer.seconds(seconds)
789 + _ -> @default_rt_period
772 790 end
773 791 end
774 792
775 − defp rt_period_ms(%{inst: %{config: %{rt_period: seconds}}}) when is_integer(seconds),
776 − do: :timer.seconds(seconds)
777 −
778 − defp rt_period_ms(_state), do: nil
793 + defp period_field(:tu), do: :rt_period_tu
794 + defp period_field(:vp), do: :rt_period_vp
795 + defp period_field(:sa), do: :rt_period_sa
779 796
780 797 defp do_update_realtime(state) do
781 798 state =
@@ -799,35 +816,39 @@ defmodule RoomGtfs.Worker.RT do
799 816 defp rt_url(config, :vp), do: Map.get(config, :url_rt_vp) || Map.get(config, :url_rt_shared)
800 817
801 818 defp fetch_kind(kind, state) do
802 − case rt_url(state.inst.config, kind) do
803 − nil ->
804 − state
819 + url = rt_url(state.inst.config, kind)
805 820
806 − url ->
807 − started = System.monotonic_time(:millisecond)
808 − result = cached_get(state, url)
809 − took = System.monotonic_time(:millisecond) - started
810 −
811 − case result do
812 − {:ok, feed} when is_struct(feed, TransitRealtime.FeedMessage) ->
813 − # Narrowed before it is recorded, so health reports what this kind
814 − # actually got. Recording the raw feed would have every kind of a
815 − # combined source claim the same entity count -- exactly the reading
816 − # that means "the same URL is in all three fields by mistake", and
817 − # would say it about a correct configuration.
818 − feed = feed |> for_this_agency(state) |> slice(entity_field(kind))
819 − record_health(state, kind, url, {:ok, feed}, took)
820 − store_feed(state, kind, feed)
821 −
822 − {:error, error} ->
823 − record_health(state, kind, url, result, took)
824 −
825 − Logger.info(
826 − "failed to fetch gtfs-rt url[#{kind}] for '#{state.inst.name}', reason: #{inspect(error)}"
827 − )
821 + cond do
822 + is_nil(url) -> state
823 + not due?(state, kind) -> state
824 + true -> do_fetch_kind(kind, url, mark_polled(state, kind))
825 + end
826 + end
828 827
829 − state
830 − end
828 + defp do_fetch_kind(kind, url, state) do
829 + started = System.monotonic_time(:millisecond)
830 + result = cached_get(state, kind, url)
831 + took = System.monotonic_time(:millisecond) - started
832 +
833 + case result do
834 + {:ok, feed} when is_struct(feed, TransitRealtime.FeedMessage) ->
835 + # Narrowed before it is recorded, so health reports what this kind
836 + # actually got. Recording the raw feed would have every kind of a
837 + # combined source claim the same entity count -- exactly the reading
838 + # that means "the same URL is in all three fields by mistake", and
839 + # would say it about a correct configuration.
840 + feed = feed |> for_this_agency(state) |> slice(entity_field(kind))
841 + record_health(state, kind, url, {:ok, feed}, took)
842 + store_feed(state, kind, feed)
843 +
844 + {:error, error} ->
845 + record_health(state, kind, url, result, took)
846 +
847 + Logger.info(
848 + "failed to fetch gtfs-rt url[#{kind}] for '#{state.inst.name}', reason: #{inspect(error)}"
849 + )
850 +
851 + state
831 852 end
832 853 end
833 854
modified apps/room_sanctum/lib/room_sanctum/configuration/configs/gtfs.ex
+25 −11
@@ -22,12 +22,16 @@ defmodule RoomSanctum.Configuration.Configs.GTFS do
22 22
23 23 field :tz, :string
24 24
25 − # Seconds between realtime polls. Nil polls at the default 30s, which is
26 − # what an unmetered feed wants. A metered one does not: 511.org allows 60
27 − # requests an hour on a default token, and three feed kinds polled every 30
28 − # seconds is 360 -- so those sources set this and trade freshness for
29 − # staying inside the budget.
30 − field :rt_period, :integer
25 + # Seconds between realtime polls, per kind, because the three kinds do not
26 + # deserve the same cadence. Trip updates go stale in seconds; service alerts
27 + # change a few times a day and polling them as often as vehicle positions is
28 + # pure waste -- and waste is not free where a feed is metered. 511.org
29 + # allows 60 requests an hour on a default token.
30 + #
31 + # Nil is the default interval, not "never": see RoomGtfs.Worker.RT.
32 + field :rt_period_tu, :integer
33 + field :rt_period_vp, :integer
34 + field :rt_period_sa, :integer
31 35
32 36 # Operator code for a feed that carries several agencies at once, e.g. "SF"
33 37 # for a source reading 511's regional feed. Realtime entities there are
@@ -39,11 +43,21 @@ defmodule RoomSanctum.Configuration.Configs.GTFS do
39 43
40 44 def changeset(source, params) do
41 45 source
42 − |> cast(params, ~w(url url_rt_sa url_rt_tu url_rt_vp url_rt_shared tz rt_period rt_agency)a)
46 + |> cast(
47 + params,
48 + ~w(url url_rt_sa url_rt_tu url_rt_vp url_rt_shared tz rt_agency
49 + rt_period_tu rt_period_vp rt_period_sa)a
50 + )
43 51 |> validate_required([:url, :tz])
44 − # Below about ten seconds the poller cannot keep up and the feed will not
45 − # have changed anyway; the upper bound is a day, past which it is not
46 − # realtime by any reading.
47 − |> validate_number(:rt_period, greater_than_or_equal_to: 10, less_than_or_equal_to: 86_400)
52 + |> validate_periods()
53 + end
54 +
55 + # Below about ten seconds the poller cannot keep up and the feed will not have
56 + # changed anyway; the upper bound is a day, past which it is not realtime by
57 + # any reading.
58 + defp validate_periods(changeset) do
59 + Enum.reduce([:rt_period_tu, :rt_period_vp, :rt_period_sa], changeset, fn field, acc ->
60 + validate_number(acc, field, greater_than_or_equal_to: 10, less_than_or_equal_to: 86_400)
61 + end)
48 62 end
49 63 end
modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/form_component.ex
+133 −0
@@ -36,6 +36,110 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
36 36 end
37 37 end
38 38
39 + # Realtime poll-interval slider, one per feed kind. The three do not deserve
40 + # the same cadence: trip updates go stale in seconds, service alerts change a
41 + # few times a day, and on a metered feed the difference is the whole budget.
42 + #
43 + # The top of the range exists for 511.org, which allows 60 requests an hour on
44 + # a default token -- three kinds at 240s is 45, and at 90s it would be 120.
45 + @rt_period_steps [
46 + {"15s", 15},
47 + {"30s", 30},
48 + {"45s", 45},
49 + {"60s", 60},
50 + {"75s", 75},
51 + {"90s", 90},
52 + {"2m", 120},
53 + {"3m", 180},
54 + {"4m", 240}
55 + ]
56 +
57 + # Index 3 -- 60s. Fresh enough for a bus, and cheap enough that a metered feed
58 + # is worth a second look before it is lowered.
59 + @rt_period_default_idx 3
60 +
61 + def rt_period_steps, do: @rt_period_steps
62 +
63 + def rt_period_kinds,
64 + do: [{:tu, "Trip updates"}, {:vp, "Vehicle positions"}, {:sa, "Service alerts"}]
65 +
66 + def rt_secs_to_idx(nil), do: @rt_period_default_idx
67 +
68 + def rt_secs_to_idx(secs) when is_integer(secs) do
69 + @rt_period_steps
70 + |> Enum.with_index()
71 + |> Enum.min_by(fn {{_, s}, _} -> abs(s - secs) end)
72 + |> elem(1)
73 + end
74 +
75 + def rt_secs_to_idx(_other), do: @rt_period_default_idx
76 +
77 + def rt_idx_to_secs(idx) when is_binary(idx), do: rt_idx_to_secs(String.to_integer(idx))
78 +
79 + def rt_idx_to_secs(idx) do
80 + {_, secs} = Enum.at(@rt_period_steps, idx, Enum.at(@rt_period_steps, @rt_period_default_idx))
81 + secs
82 + end
83 +
84 + def rt_period_label(idx) do
85 + {label, _secs} =
86 + Enum.at(@rt_period_steps, idx, Enum.at(@rt_period_steps, @rt_period_default_idx))
87 +
88 + label
89 + end
90 +
91 + @doc """
92 + Requests an hour the three intervals add up to, counting each distinct URL
93 + once.
94 +
95 + A source with one combined feed polls one URL however many kinds read from
96 + it, so counting per kind would overstate it threefold -- and the number is
97 + only worth showing because some feeds are metered, where an overstatement is
98 + as unhelpful as no number at all.
99 + """
100 + def rt_requests_per_hour(config, idxs) when is_map(config) do
101 + for {kind, _label} <- rt_period_kinds(), reduce: %{} do
102 + acc ->
103 + case rt_url_for(config, kind) do
104 + nil -> acc
105 + url -> Map.update(acc, url, rt_idx_to_secs(idxs[kind]), &min(&1, rt_idx_to_secs(idxs[kind])))
106 + end
107 + end
108 + |> Enum.map(fn {_url, secs} -> div(3600, secs) end)
109 + |> Enum.sum()
110 + end
111 +
112 + def rt_requests_per_hour(_config, _idxs), do: 0
113 +
114 + defp rt_url_for(config, kind) do
115 + specific =
116 + case kind do
117 + :tu -> Map.get(config, :url_rt_tu)
118 + :vp -> Map.get(config, :url_rt_vp)
119 + :sa -> Map.get(config, :url_rt_sa)
120 + end
121 +
122 + blank(specific) || blank(Map.get(config, :url_rt_shared))
123 + end
124 +
125 + defp blank(nil), do: nil
126 + defp blank(""), do: nil
127 + defp blank(value), do: value
128 +
129 + defp convert_rt_period_params(source_params) do
130 + Enum.reduce(rt_period_kinds(), source_params, fn {kind, _label}, params ->
131 + case get_in(params, ["config", "rt_idx_#{kind}"]) do
132 + nil ->
133 + params
134 +
135 + idx ->
136 + params
137 + |> put_in(["config", "rt_period_#{kind}"], rt_idx_to_secs(idx))
138 + |> update_in(["config"], &Map.delete(&1, "rt_idx_#{kind}"))
139 + end
140 + end)
141 + end
142 +
39 143 # GitHub poll-interval slider: index → label/seconds
40 144 @github_poll_steps [
41 145 {"15s", 15},
@@ -249,6 +353,20 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
249 353 |> Enum.map(&{&1.name, &1.id})
250 354 run_period_idx = seconds_to_idx(source.meta && source.meta.run_period)
251 355
356 + rt_idxs =
357 + Map.new(rt_period_kinds(), fn {kind, _label} ->
358 + secs =
359 + case source.config do
360 + %RoomSanctum.Configuration.Configs.GTFS{} = cfg ->
361 + Map.get(cfg, String.to_existing_atom("rt_period_#{kind}"))
362 +
363 + _other ->
364 + nil
365 + end
366 +
367 + {kind, rt_secs_to_idx(secs)}
368 + end)
369 +
252 370 gh_secs =
253 371 case source.config do
254 372 %RoomSanctum.Configuration.Configs.Github{poll_seconds: secs}
@@ -281,6 +399,8 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
281 399 |> assign(:mailbox_sel, mailbox_sel)
282 400 |> assign(:tint_opts, RoomSanctum.Tints.all())
283 401 |> assign(:run_period_idx, run_period_idx)
402 + |> assign(:rt_idxs, rt_idxs)
403 + |> assign(:rt_requests, rt_requests_per_hour(source.config, rt_idxs))
284 404 |> assign(:github_poll_idx, github_secs_to_idx(gh_secs))
285 405 |> assign(:github_estimate, gh_estimate)
286 406 |> assign(:gitlab_poll_idx, gitlab_secs_to_idx(gl_secs))
@@ -324,6 +444,7 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
324 444 source_params
325 445 |> inj_uid(socket)
326 446 |> convert_run_period_params()
447 + |> convert_rt_period_params()
327 448 |> convert_github_poll_params()
328 449 |> convert_gitlab_poll_params()
329 450
@@ -365,9 +486,20 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
365 486 {socket.assigns[:gitlab_poll_idx] || 1, socket.assigns[:gitlab_estimate] || %{watched: 0, calls: 0, source: :unknown, cycles: 0}}
366 487 end
367 488
489 + # Read back from the converted params rather than the changeset: a slider
490 + # the user has just dragged is in the params, and its label has to move with
491 + # it or the control reads as broken.
492 + rt_idxs =
493 + Map.new(rt_period_kinds(), fn {kind, _label} ->
494 + secs = get_in(source_params, ["config", "rt_period_#{kind}"])
495 + {kind, rt_secs_to_idx(secs)}
496 + end)
497 +
368 498 {:noreply,
369 499 socket
370 500 |> assign(:run_period_idx, run_period_idx)
501 + |> assign(:rt_idxs, rt_idxs)
502 + |> assign(:rt_requests, rt_requests_per_hour(config_for_estimate, rt_idxs))
371 503 |> assign(:github_poll_idx, gh_idx)
372 504 |> assign(:github_estimate, gh_estimate)
373 505 |> assign(:gitlab_poll_idx, gl_idx)
@@ -380,6 +512,7 @@ defmodule RoomSanctumWeb.SourceLive.FormComponent do
380 512 source_params
381 513 |> inj_uid(socket)
382 514 |> convert_run_period_params()
515 + |> convert_rt_period_params()
383 516 |> convert_github_poll_params()
384 517 |> convert_gitlab_poll_params()
385 518
modified apps/room_sanctum/lib/room_sanctum_web/live/cfg_source_live/form_component.html.heex
+32 −6
@@ -89,15 +89,41 @@
89 89 </span>
90 90 </label>
91 91 </div>
92 − <div class="form-control">
93 − <.input field={gtfs_form[:rt_period]} type="number" label="Realtime poll interval (seconds)"/>
92 + <%= for {kind, title} <- RoomSanctumWeb.SourceLive.FormComponent.rt_period_kinds() do %>
93 + <div class="form-control mt-2">
94 94 <label class="label">
95 − <span class="label-text-alt opacity-70">
96 − Blank polls every 30s. Raise it for a feed with a request quota &mdash;
97 − 511.org allows 60 requests an hour, which three feed kinds exhaust in
98 − twenty minutes at the default.
95 + <span class="label-text">
96 + <%= title %> interval &mdash;
97 + <span class="font-bold">
98 + <%= RoomSanctumWeb.SourceLive.FormComponent.rt_period_label(@rt_idxs[kind]) %>
99 + </span>
99 100 </span>
100 101 </label>
102 + <input type="range"
103 + name={"source[config][rt_idx_#{kind}]"}
104 + id={"source_config_rt_idx_#{kind}"}
105 + min="0"
106 + max={length(RoomSanctumWeb.SourceLive.FormComponent.rt_period_steps()) - 1}
107 + step="1"
108 + value={@rt_idxs[kind]}
109 + class="range range-sm range-secondary" />
110 + <div class="flex justify-between text-xs px-1 mt-1">
111 + <%= for {label, _secs} <- RoomSanctumWeb.SourceLive.FormComponent.rt_period_steps() do %>
112 + <span><%= label %></span>
113 + <% end %>
114 + </div>
115 + </div>
116 + <% end %>
117 + <p class="text-xs text-base-content/60 mt-1">
118 + The three kinds go stale at different rates: trip updates in seconds, service
119 + alerts a few times a day. Feeds sharing one URL are fetched once between them.
120 + </p>
121 + <div class="text-xs mt-2 opacity-80">
122 + <i class="fa-solid fa-fw fa-gauge-high"></i>
123 + Roughly
124 + <span class="font-bold"><%= @rt_requests %></span>
125 + requests an hour, counting each distinct URL once.
126 + 511.org allows 60 on a default token.
101 127 </div>
102 128 <div class="form-control">
103 129 <.input field={gtfs_form[:rt_agency]} type="text" label="Realtime operator code"/>

Parents: 58b065b