Serve Prometheus metrics on 9568

d2c66e6 · Gabriel Morell · 2026-08-23 17:51

6 files +107 -0
Message
{commit_body(@commit)}

Files changed

modified config/config.exs
+14 −0
@@ -259,6 +259,20 @@ config :sentry,
259 259 enable_source_code_context: true,
260 260 root_source_code_paths: [File.cwd!()]
261 261
262 +# Prometheus metrics, exposed on port 9568 by GitGud.MetricsPlug and scraped
263 +# from the pod annotations in argo-01-bos/git-gud/app.yml.
264 +#
265 +# `grafana: :disabled` is load-bearing, not boilerplate: PromEx would otherwise
266 +# upload its own dashboards on boot, and the dashboards for this fleet are
267 +# committed in deployment/grafana and pushed from there. `metrics_server:
268 +# :disabled` because that option is Cowboy-based — see GitGud.MetricsPlug.
269 +config :git_gud, GitGud.PromEx,
270 + disabled: false,
271 + manual_metrics_start_delay: :no_delay,
272 + drop_metrics_groups: [],
273 + grafana: :disabled,
274 + metrics_server: :disabled
275 +
262 276 # Import environment specific config. This must remain at the bottom
263 277 # of this file so it overrides the configuration defined above.
264 278 import_config "#{config_env()}.exs"
modified lib/git_gud/application.ex
+3 −0
@@ -19,6 +19,9 @@ defmodule GitGud.Application do
19 19 children =
20 20 [
21 21 GitGudWeb.Telemetry,
22 + GitGud.PromEx,
23 + # The metrics listener, on its own port — see GitGud.MetricsPlug.
24 + {Bandit, plug: GitGud.MetricsPlug, scheme: :http, port: 9568},
22 25 GitGud.Repo,
23 26 {DNSCluster, query: Application.get_env(:git_gud, :dns_cluster_query) || :ignore}
24 27 ] ++
added lib/git_gud/metrics_plug.ex
+27 −0
@@ -0,0 +1,27 @@
1 +defmodule GitGud.MetricsPlug do
2 + @moduledoc """
3 + Serves `/metrics` on port 9568, separately from the site.
4 +
5 + Not mounted in the endpoint on purpose. The Ingress matches `/` as a prefix,
6 + so anything in the Phoenix router is reachable from the internetthat is
7 + how the Rust services ended up serving `/metrics` publicly. A second listener
8 + on a port the Ingress does not name keeps it to in-cluster scrapers, and the
9 + scrape annotations on the pod point here rather than at 4000.
10 +
11 + This runs on Bandit, the same server the endpoint uses. PromEx ships its own
12 + `metrics_server` option which would do the same job, but it is built on
13 + Plug.Cowboyusing it would mean compiling and running Cowboy alongside
14 + Bandit for one route.
15 + """
16 +
17 + use Plug.Builder
18 +
19 + plug PromEx.Plug, prom_ex_module: GitGud.PromEx
20 +
21 + # `PromEx.Plug` passes anything that is not /metrics straight through rather
22 + # than halting, so without this the listener would answer every other path
23 + # with 200 and an empty body.
24 + plug :not_found
25 +
26 + defp not_found(conn, _opts), do: send_resp(conn, 404, "")
27 +end
added lib/git_gud/prom_ex.ex
+52 −0
@@ -0,0 +1,52 @@
1 +defmodule GitGud.PromEx do
2 + @moduledoc """
3 + Prometheus metrics for git_gud, served on port 9568 by `GitGud.MetricsPlug`.
4 +
5 + ## Why the prefix is `phx` rather than `git_gud`
6 +
7 + PromEx names metrics `[otp_app, :prom_ex, plugin]` by default, which would
8 + give `git_gud_prom_ex_phoenix_http_requests_total` here and a *different*
9 + name in every other Phoenix app in this fleetsix near-identical apps that
10 + no single dashboard could describe.
11 +
12 + Every plugin below overrides that to `[:phx, plugin]`, so all six emit
13 + identical names and the Kubernetes `namespace` label is what tells them
14 + apart. That is the same shape the Rust services have, where one dashboard is
15 + cloned per app with nothing changed but the namespace. Two apps sharing a
16 + namespace would collide; none do.
17 +
18 + ## Grafana upload is off
19 +
20 + PromEx can push its own bundled dashboards to Grafana on boot. That would
21 + fight `deployment/grafana`, where git is the source of truth and `sync.sh`
22 + pushes dashboards the other wayan uploaded dashboard would either be
23 + reverted by the next sync or quietly win over the committed one. Disabled
24 + explicitly in `config/config.exs` rather than left to the default, because
25 + this is the kind of thing that gets turned on by a copied config snippet.
26 + """
27 +
28 + use PromEx, otp_app: :git_gud
29 +
30 + alias PromEx.Plugins
31 +
32 + @impl true
33 + def plugins do
34 + [
35 + # Uptime and build info, so a restart is visible without reading the
36 + # process metrics.
37 + {Plugins.Application, metric_prefix: [:phx, :application]},
38 + # The BEAM itself: run queue, memory by kind, process and port counts.
39 + # This is most of what is worth knowing about an Elixir app and none of
40 + # it is visible from the outside.
41 + {Plugins.Beam, metric_prefix: [:phx, :beam]},
42 + # `path` is the *route pattern* from the router, not the request path, so
43 + # the label stays bounded no matter how many distinct URLs are hit.
44 + {Plugins.Phoenix,
45 + router: GitGudWeb.Router, endpoint: GitGudWeb.Endpoint, metric_prefix: [:phx, :phoenix]},
46 + {Plugins.Ecto, repos: [GitGud.Repo], metric_prefix: [:phx, :ecto]},
47 + # Queue depth by state is the one that matters: jobs piling up in
48 + # `available` means the workers are behind.
49 + {Plugins.Oban, metric_prefix: [:phx, :oban]}
50 + ]
51 + end
52 +end
modified mix.exs
+6 −0
@@ -71,6 +71,12 @@ defmodule GitGud.MixProject do
71 71 {:sentry, "~> 10.8"},
72 72 {:telemetry_metrics, "~> 1.0"},
73 73 {:telemetry_poller, "~> 1.0"},
74 + # Pinned to a minor, not just a major: the metric names these six apps
75 + # share are what makes one dashboard describe all of them, and a
76 + # plugin rename between minors would only show up as an empty panel.
77 + # 1.12 wants ecto >= 3.14, which angler's lock does not have — lift
78 + # this across the whole fleet at once or not at all.
79 + {:prom_ex, "~> 1.11.0"},
74 80 {:gettext, "~> 1.0"},
75 81 {:jason, "~> 1.2"},
76 82 {:dns_cluster, "~> 0.2.0"},
modified mix.lock
+5 −0
@@ -2,6 +2,7 @@
2 2 "asn1_compiler": {:hex, :asn1_compiler, "0.1.1", "64a4e52b59d1f225878445ace2c75cd2245b13a5a81182304fd9dc5acfc8994e", [:mix], [], "hexpm", "c250d24c22f1a3f305d88864400f9ac2df55c6886e1e3a030e2946efeb94695e"},
3 3 "bandit": {:hex, :bandit, "1.11.0", "dbdd9c9963f146ee9da9860d1ee5b0ffd65cea51fe2aab3f3273df84329d133a", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "c949d93a325a28da2333dde5a9ab61986ad2c2b7226347db6a28303b9139865e"},
4 4 "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.3.2", "d50091e3c9492d73e17fc1e1619a9b09d6a5ef99160eb4d736926fd475a16ca3", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "471be5151874ae7931911057d1467d908955f93554f7a6cd1b7d804cac8cef53"},
5 + "castore": {:hex, :castore, "1.0.21", "0a0e8330dc267a40a3b7ad86d39302764bb71758172904e6a59d5ad6443ce307", [:mix], [], "hexpm", "e42e22723e25dbd46876d056a03f685513d6e98f6b5e555dc551321decd76c5c"},
5 6 "cbor": {:hex, :cbor, "1.0.2", "9b0af85af291a556e10a0ffd48ba9a21a75e711828fafd3af193d56d95f0907f", [:mix], [], "hexpm", "edbc9b4a16eb93a582437b9b249c340a75af03958e338fb43d8c1be9fc65b864"},
6 7 "cc_precompiler": {:hex, :cc_precompiler, "0.1.11", "8c844d0b9fb98a3edea067f94f616b3f6b29b959b6b3bf25fee94ffe34364768", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3427232caf0835f94680e5bcf082408a70b48ad68a5f5c0b02a3bea9f3a075b9"},
7 8 "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"},
@@ -45,7 +46,9 @@
45 46 "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
46 47 "nimble_totp": {:hex, :nimble_totp, "1.0.0", "79753bae6ce59fd7cacdb21501a1dbac249e53a51c4cd22b34fa8438ee067283", [:mix], [], "hexpm", "6ce5e4c068feecdb782e85b18237f86f66541523e6bad123e02ee1adbe48eda9"},
47 48 "oban": {:hex, :oban, "2.22.1", "9d2a38cec95070b31c1e274fae55f3925089f62159d8f3facabee0454d55b257", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af2508c156c5b0ec30b21b0883babf7e2716af35ed5d264095896103fe3cea37"},
49 + "octo_fetch": {:hex, :octo_fetch, "0.5.0", "f50701568b9fc752656367f82cc134d5fbefff37c5a0e8ddfcceb02ceee3f5fc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6226cc3c14ca948ee9f25fb0446322e5c288e215da9beba7899b6b5f4cd3ccb0"},
48 50 "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
51 + "peep": {:hex, :peep, "3.5.0", "9f6ead7b0f2c684494200c8fc02e7e62e8c459afe861b29bd859e4c96f402ed8", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5a73a99c6e60062415efeb7e536a663387146463a3d3df1417da31fd665ac210"},
49 52 "phoenix": {:hex, :phoenix, "1.8.7", "d8d755b4ff4b449f610223dd706b4ae64155cb720d3dc09c706c079ecea189e4", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "47352f72d6ab31009ef77516b1b3a14745be97b54061fd458031b9d8294869d5"},
50 53 "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"},
51 54 "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"},
@@ -57,6 +60,7 @@
57 60 "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"},
58 61 "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},
59 62 "postgrex": {:hex, :postgrex, "0.22.2", "4aec14df2a72722aee92492566edbeeb44e233ecb86b1915d03136297ef1385d", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8946382ddb06294f56026ac4278b3cc212bac8a2c82ed68b4087819ed1abc53b"},
63 + "prom_ex": {:hex, :prom_ex, "1.11.0", "1f6d67f2dead92224cb4f59beb3e4d319257c5728d9638b4a5e8ceb51a4f9c7e", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "76b074bc3730f0802978a7eb5c7091a65473eaaf07e99ec9e933138dcc327805"},
60 64 "protobuf": {:hex, :protobuf, "0.16.0", "d1878725105d49162977cf3408ccc3eac4f3532e26e5a9e250f2c624175d10f6", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "f0d0d3edd8768130f24cc2cfc41320637d32c80110e80d13f160fa699102c828"},
61 65 "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"},
62 66 "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"},
@@ -69,6 +73,7 @@
69 73 "tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"},
70 74 "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
71 75 "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
76 + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
72 77 "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
73 78 "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"},
74 79 "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},

Parents: 5af55d9