neiam
/apollos_crib
It's Apollo's Crib
public · Issues · Pulls 1 · Labels · Forks · Compare · Actions failure · Badges · Packages
⭐
Log in to mark this repository.
Give the GTFS truncates longer than Ecto's default 15s
51bc3c5 · Gabriel Morell · 2026-08-24 05:00
Message
{commit_body(@commit)}
Files changed
modified
apps/room_gtfs/lib/room_gtfs/gtfs_worker.ex
+6
−0
@@ -937,6 +937,12 @@ end
| 937 | 937 | end |
| 938 | 938 | |
| 939 | 939 | # IO.inspect({table, columns, pg_cols}) |
| 940 | + # `Repo.config/0` carries Ecto's `timeout: 15000`, which looks like it would | |
| 941 | + # cap the bulk statements below. It does not: DBConnection takes the | |
| 942 | + # *transaction's* timeout as the default for statements run inside it, and | |
| 943 | + # the transaction is opened with `timeout: :infinity`. Verified rather than | |
| 944 | + # assumed -- a 3s query on a connection started with `timeout: 1000` runs | |
| 945 | + # fine inside an `:infinity` transaction, and fails inside a 1s one. | |
| 940 | 946 | opts = RoomSanctum.Repo.config() |
| 941 | 947 | {:ok, pid} = Postgrex.start_link(opts) |
| 942 | 948 |
modified
apps/room_sanctum/lib/room_sanctum/storage.ex
+21
−8
@@ -101,9 +101,22 @@ defmodule RoomSanctum.Storage do
| 101 | 101 | Repo.delete(agency) |
| 102 | 102 | end |
| 103 | 103 | |
| 104 | + # A GTFS import deletes a whole feed before reloading it, and gtfs_stop_times | |
| 105 | + # runs to a couple of million rows per source. Ecto's default 15s is not | |
| 106 | + # nearly enough for that delete on a database that is doing anything else, | |
| 107 | + # and when it is exceeded the error blames the pool -- | |
| 108 | + # "client ... timed out because it queued and checked out the connection for | |
| 109 | + # longer than 15000ms" -- which reads like connection starvation rather than | |
| 110 | + # a statement that simply needed longer than a quarter of a minute. | |
| 111 | + # | |
| 112 | + # Only the GTFS truncates carry this. The GBFS ones below delete hundreds of | |
| 113 | + # rows, not millions, and a query there running past 15s means something is | |
| 114 | + # wrong rather than something is big. | |
| 115 | + @gtfs_truncate_timeout :timer.minutes(15) | |
| 116 | + | |
| 104 | 117 | def truncate_agency(source_id) do |
| 105 | 118 | from(p in Agency, where: p.source_id == ^source_id) |
| 106 | − |> Repo.delete_all() | |
| 119 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 107 | 120 | end |
| 108 | 121 | |
| 109 | 122 | @doc """ |
@@ -222,7 +235,7 @@ defmodule RoomSanctum.Storage do
| 222 | 235 | |
| 223 | 236 | def truncate_calendar(source_id) do |
| 224 | 237 | from(p in Calendar, where: p.source_id == ^source_id) |
| 225 | − |> Repo.delete_all() | |
| 238 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 226 | 239 | end |
| 227 | 240 | |
| 228 | 241 | @doc """ |
@@ -341,7 +354,7 @@ defmodule RoomSanctum.Storage do
| 341 | 354 | |
| 342 | 355 | def truncate_direction(source_id) do |
| 343 | 356 | from(p in Direction, where: p.source_id == ^source_id) |
| 344 | − |> Repo.delete_all() | |
| 357 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 345 | 358 | end |
| 346 | 359 | |
| 347 | 360 | @doc """ |
@@ -736,7 +749,7 @@ defmodule RoomSanctum.Storage do
| 736 | 749 | |
| 737 | 750 | def truncate_shape(source_id) do |
| 738 | 751 | from(p in Shape, where: p.source_id == ^source_id) |
| 739 | − |> Repo.delete_all() | |
| 752 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 740 | 753 | end |
| 741 | 754 | |
| 742 | 755 | def count_shapes(source_id) do |
@@ -746,7 +759,7 @@ defmodule RoomSanctum.Storage do
| 746 | 759 | |
| 747 | 760 | def truncate_route(source_id) do |
| 748 | 761 | from(p in Route, where: p.source_id == ^source_id) |
| 749 | − |> Repo.delete_all() | |
| 762 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 750 | 763 | end |
| 751 | 764 | |
| 752 | 765 | @doc """ |
@@ -865,7 +878,7 @@ defmodule RoomSanctum.Storage do
| 865 | 878 | |
| 866 | 879 | def truncate_stop_time(source_id) do |
| 867 | 880 | from(p in StopTime, where: p.source_id == ^source_id) |
| 868 | − |> Repo.delete_all() | |
| 881 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 869 | 882 | end |
| 870 | 883 | |
| 871 | 884 | defp convert_gtfs_time(time) do |
@@ -1023,7 +1036,7 @@ defmodule RoomSanctum.Storage do
| 1023 | 1036 | |
| 1024 | 1037 | def truncate_stop(source_id) do |
| 1025 | 1038 | from(p in Stop, where: p.source_id == ^source_id) |
| 1026 | − |> Repo.delete_all() | |
| 1039 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 1027 | 1040 | end |
| 1028 | 1041 | |
| 1029 | 1042 | @doc """ |
@@ -1162,7 +1175,7 @@ defmodule RoomSanctum.Storage do
| 1162 | 1175 | |
| 1163 | 1176 | def truncate_trip(source_id) do |
| 1164 | 1177 | from(p in Trip, where: p.source_id == ^source_id) |
| 1165 | − |> Repo.delete_all() | |
| 1178 | + |> Repo.delete_all(timeout: @gtfs_truncate_timeout) | |
| 1166 | 1179 | end |
| 1167 | 1180 | |
| 1168 | 1181 | @doc """ |
Parents: ae13c56