Serve Prometheus metrics on 9568

139e48c · Gabriel Morell · 2026-08-23 17:51

9 files +104 -0
Message
{commit_body(@commit)}

Files changed

modified config/config.exs
+14 −0
@@ -208,6 +208,20 @@ config :logger, :default_formatter,
208 208 # Use Jason for JSON parsing in Phoenix
209 209 config :phoenix, :json_library, Jason
210 210
211 +# Prometheus metrics, exposed on port 9568 by OhMyExOhIrcd.MetricsPlug and scraped
212 +# from the pod annotations in argo-01-bos/neiam-ircd/app.yml.
213 +#
214 +# `grafana: :disabled` is load-bearing, not boilerplate: PromEx would otherwise
215 +# upload its own dashboards on boot, and the dashboards for this fleet are
216 +# committed in deployment/grafana and pushed from there. `metrics_server:
217 +# :disabled` because that option is Cowboy-based — see OhMyExOhIrcd.MetricsPlug.
218 +config :oh_my_ex_oh_ircd, OhMyExOhIrcd.PromEx,
219 + disabled: false,
220 + manual_metrics_start_delay: :no_delay,
221 + drop_metrics_groups: [],
222 + grafana: :disabled,
223 + metrics_server: :disabled
224 +
211 225 # Import environment specific config. This must remain at the bottom
212 226 # of this file so it overrides the configuration defined above.
213 227 import_config "#{config_env()}.exs"
modified lib/oh_my_ex_oh_ircd/application.ex
+3 −0
@@ -11,6 +11,9 @@ defmodule OhMyExOhIrcd.Application do
11 11
12 12 children = [
13 13 OhMyExOhIrcdWeb.Telemetry,
14 + OhMyExOhIrcd.PromEx,
15 + # The metrics listener, on its own port — see OhMyExOhIrcd.MetricsPlug.
16 + {Bandit, plug: OhMyExOhIrcd.MetricsPlug, scheme: :http, port: 9568},
14 17 OhMyExOhIrcd.Repo,
15 18 # Runs pending migrations before anything that depends on the schema
16 19 # starts (no-op unless `:migrate_on_start` is enabled — see the module).
added lib/oh_my_ex_oh_ircd/metrics_plug.ex
+27 −0
@@ -0,0 +1,27 @@
1 +defmodule OhMyExOhIrcd.MetricsPlug do
2 + @moduledoc """
3 + Serves `/metrics` on port 9568, separately from the site.
4 +
5 + Not mounted in the endpoint on purpose. The Ingress matches `/` as a prefix,
6 + so anything in the Phoenix router is reachable from the internet — that is
7 + how the Rust services ended up serving `/metrics` publicly. A second listener
8 + on a port the Ingress does not name keeps it to in-cluster scrapers, and the
9 + scrape annotations on the pod point here rather than at 4000.
10 +
11 + This runs on Bandit, the same server the endpoint uses. PromEx ships its own
12 + `metrics_server` option which would do the same job, but it is built on
13 + Plug.Cowboy — using it would mean compiling and running Cowboy alongside
14 + Bandit for one route.
15 + """
16 +
17 + use Plug.Builder
18 +
19 + plug PromEx.Plug, prom_ex_module: OhMyExOhIrcd.PromEx
20 +
21 + # `PromEx.Plug` passes anything that is not /metrics straight through rather
22 + # than halting, so without this the listener would answer every other path
23 + # with 200 and an empty body.
24 + plug :not_found
25 +
26 + defp not_found(conn, _opts), do: send_resp(conn, 404, "")
27 +end
added lib/oh_my_ex_oh_ircd/prom_ex.ex
+49 −0
@@ -0,0 +1,49 @@
1 +defmodule OhMyExOhIrcd.PromEx do
2 + @moduledoc """
3 + Prometheus metrics for oh_my_ex_oh_ircd, served on port 9568 by `OhMyExOhIrcd.MetricsPlug`.
4 +
5 + ## Why the prefix is `phx` rather than `oh_my_ex_oh_ircd`
6 +
7 + PromEx names metrics `[otp_app, :prom_ex, plugin]` by default, which would
8 + give `oh_my_ex_oh_ircd_prom_ex_phoenix_http_requests_total` here and a *different*
9 + name in every other Phoenix app in this fleet — six near-identical apps that
10 + no single dashboard could describe.
11 +
12 + Every plugin below overrides that to `[:phx, plugin]`, so all six emit
13 + identical names and the Kubernetes `namespace` label is what tells them
14 + apart. That is the same shape the Rust services have, where one dashboard is
15 + cloned per app with nothing changed but the namespace. Two apps sharing a
16 + namespace would collide; none do.
17 +
18 + ## Grafana upload is off
19 +
20 + PromEx can push its own bundled dashboards to Grafana on boot. That would
21 + fight `deployment/grafana`, where git is the source of truth and `sync.sh`
22 + pushes dashboards the other way — an uploaded dashboard would either be
23 + reverted by the next sync or quietly win over the committed one. Disabled
24 + explicitly in `config/config.exs` rather than left to the default, because
25 + this is the kind of thing that gets turned on by a copied config snippet.
26 + """
27 +
28 + use PromEx, otp_app: :oh_my_ex_oh_ircd
29 +
30 + alias PromEx.Plugins
31 +
32 + @impl true
33 + def plugins do
34 + [
35 + # Uptime and build info, so a restart is visible without reading the
36 + # process metrics.
37 + {Plugins.Application, metric_prefix: [:phx, :application]},
38 + # The BEAM itself: run queue, memory by kind, process and port counts.
39 + # This is most of what is worth knowing about an Elixir app and none of
40 + # it is visible from the outside.
41 + {Plugins.Beam, metric_prefix: [:phx, :beam]},
42 + # `path` is the *route pattern* from the router, not the request path, so
43 + # the label stays bounded no matter how many distinct URLs are hit.
44 + {Plugins.Phoenix,
45 + router: OhMyExOhIrcdWeb.Router, endpoint: OhMyExOhIrcdWeb.Endpoint, metric_prefix: [:phx, :phoenix]},
46 + {Plugins.Ecto, repos: [OhMyExOhIrcd.Repo], metric_prefix: [:phx, :ecto]}
47 + ]
48 + end
49 +end
modified mix.exs
+6 −0
@@ -63,6 +63,12 @@ defmodule OhMyExOhIrcd.MixProject do
63 63 {:req, "~> 0.5"},
64 64 {:telemetry_metrics, "~> 1.0"},
65 65 {:telemetry_poller, "~> 1.0"},
66 + # Pinned to a minor, not just a major: the metric names these six apps
67 + # share are what makes one dashboard describe all of them, and a
68 + # plugin rename between minors would only show up as an empty panel.
69 + # 1.12 wants ecto >= 3.14, which angler's lock does not have — lift
70 + # this across the whole fleet at once or not at all.
71 + {:prom_ex, "~> 1.11.0"},
66 72 {:gettext, "~> 1.0"},
67 73 {:jason, "~> 1.2"},
68 74 {:dns_cluster, "~> 0.2.0"},
modified mix.lock
+5 −0
@@ -31,7 +31,9 @@
31 31 "mint": {:hex, :mint, "1.8.0", "b964eaf4416f2dee2ba88968d52239fca5621b0402b9c95f55a08eb9d74803e9", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "f3c572c11355eccf00f22275e9b42463bc17bd28db13be1e28f8e0bb4adbc849"},
32 32 "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
33 33 "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
34 + "octo_fetch": {:hex, :octo_fetch, "0.5.0", "f50701568b9fc752656367f82cc134d5fbefff37c5a0e8ddfcceb02ceee3f5fc", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "6226cc3c14ca948ee9f25fb0446322e5c288e215da9beba7899b6b5f4cd3ccb0"},
34 35 "parent": {:hex, :parent, "0.12.1", "495c4386f06de0df492e0a7a7199c10323a55e9e933b27222060dd86dccd6d62", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2ab589ef1f37bfcedbfb5ecfbab93354972fb7391201b8907a866dadd20b39d1"},
36 + "peep": {:hex, :peep, "3.5.0", "9f6ead7b0f2c684494200c8fc02e7e62e8c459afe861b29bd859e4c96f402ed8", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5a73a99c6e60062415efeb7e536a663387146463a3d3df1417da31fd665ac210"},
35 37 "phoenix": {:hex, :phoenix, "1.8.7", "d8d755b4ff4b449f610223dd706b4ae64155cb720d3dc09c706c079ecea189e4", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "47352f72d6ab31009ef77516b1b3a14745be97b54061fd458031b9d8294869d5"},
36 38 "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"},
37 39 "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"},
@@ -43,12 +45,15 @@
43 45 "plug": {:hex, :plug, "1.19.2", "e4950525b22c6789dfb38a3f95d47171ba159da3fc5a33be9643b43d5e8adb98", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b6fce20a56af5e60fa5dfecf3f907bb98ec981be43c79a3809a499bc3d133de0"},
44 46 "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},
45 47 "postgrex": {:hex, :postgrex, "0.22.2", "4aec14df2a72722aee92492566edbeeb44e233ecb86b1915d03136297ef1385d", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8946382ddb06294f56026ac4278b3cc212bac8a2c82ed68b4087819ed1abc53b"},
48 + "prom_ex": {:hex, :prom_ex, "1.11.0", "1f6d67f2dead92224cb4f59beb3e4d319257c5728d9638b4a5e8ceb51a4f9c7e", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "76b074bc3730f0802978a7eb5c7091a65473eaaf07e99ec9e933138dcc327805"},
46 49 "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"},
47 50 "site_encrypt": {:hex, :site_encrypt, "0.7.0", "7ab24eff86beafee0915f686608877cdc51506e1f3e4747ea6ae04ca1064f1e9", [:mix], [{:bandit, "~> 0.7 or ~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.10", [hex: :jose, repo: "hexpm", optional: false]}, {:mint, "~> 1.4", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:parent, "~> 0.11", [hex: :parent, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.5", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:x509, "~> 0.8", [hex: :x509, repo: "hexpm", optional: false]}], "hexpm", "4f755ac86e2572e3e6c393c9f9cc8550173b00364de359b31f1f0c142fda0aa8"},
51 + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
48 52 "swoosh": {:hex, :swoosh, "1.25.2", "cd3e53b0391439395492e5dce8c22288733f22603e21136162d03cd153669be9", [:mix], [{:bandit, ">= 1.0.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:idna, "~> 6.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: true]}, {:multipart, "~> 0.4", [hex: :multipart, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0aecf65b2845f13f4d440e0945715432bbde2d815e2302adf7df549cd9bdafed"},
49 53 "tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"},
50 54 "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"},
51 55 "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
56 + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"},
52 57 "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
53 58 "thousand_island": {:hex, :thousand_island, "1.4.3", "2158209580f633be38d43ec4e3ce0a01079592b9657afff9080d5d8ca149a3af", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e4ce09b0fd761a58594d02814d40f77daff460c48a7354a15ab353bb998ea0b"},
54 59 "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},

Parents: d510a92