GITGUD

neiam neiam /mutant-operator

An Experimental Operator

public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages

⭐

Log in to mark this repository.

Mutation: a MutantDeployment configured at runtime

f38ff4e · gmorell · 2026-11-09 06:17

9 files +818 -560
Message 
{commit_body(@commit)}

Files changed

modified Cargo.lock
+1 −0
@@ -1894,6 +1894,7 @@ dependencies = [
1894 1894 "axum",
1895 1895 "derive",
1896 1896 "futures",
1897 + "getrandom 0.3.2",
1897 1898 "k8s-openapi",
1898 1899 "kube",
1899 1900 "schemars",
modified Cargo.toml
+1 −0
@@ -19,6 +19,7 @@ tracing-subscriber = "0.3.19"
19 19 sea-orm = { version = "2.0.4", features = ["sqlx-postgres", "runtime-tokio-native-tls"] }
20 20 futures = "0.3.31"
21 21 axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio", "tower-log", "query"] }
22 +getrandom = "0.3"
22 23
23 24
24 25 [package.metadata.sqlx]
modified README.md
+100 −49
@@ -44,103 +44,154 @@ spec:
44 44 NGINX_EXPERIMENTAL: "true"
45 45 instances: 3
46 46 ```
47 +
47 48 Removing a mutation from the list deletes its Deployment. Deleting the
48 49 MutantDeployment deletes all of them.
49 50
50 −## Mutation resources
51 +## Mutations: variants configured at runtime
51 52
52 −Mutations can also be standalone `Mutation` resources, created at runtime
53 −without touching the MutantDeployment. Use them when the MutantDeployment is
54 −owned by Argo CD or another GitOps tool:
53 +A `Mutation` is the same thing as a MutantDeployment, a base Deployment
54 +plus variants of it, but it is configured at runtime through the HTTP API
55 +below instead of being declared in git. Use it when variants come and go
56 +while things run. It stays a Kubernetes object, so the operator keeps no
57 +state of its own.
55 58
56 59 ```yaml
57 60 apiVersion: neiam.org/v1
58 61 kind: Mutation
59 62 metadata:
60 − name: nginx-mutant-beta
63 + name: browser-snap
61 64 spec:
62 − mutantDeployment: nginx-mutant # same namespace
63 − label: beta
64 − env:
65 − FEATURE_X: "on"
66 − instances: 2
65 + deployment: # the base, as in a MutantDeployment
66 + apiVersion: apps/v1
67 + kind: Deployment
68 + metadata:
69 + name: browser-snap
70 + spec: { ... }
71 + variants:
72 + - label: radxa-8-hd
73 + env:
74 + SNAP_RES: "1280x800"
75 + instances: 3
76 + - label: lilygo47
77 + env:
78 + SNAP_RES: "960x540"
79 + instances: 1
67 80 ```
68 81
69 82 ```console
70 83 $ kubectl get mutations
71 −NAME TARGET LABEL INSTANCES READY STATE AGE
72 −nginx-mutant-beta nginx-mutant beta 2 2 Ready 1m
84 +NAME BASE READY SECRET AGE
85 +browser-snap browser-snap 2/2 variants ready browser-snap-secret 1m
73 86 ```
74 87
75 −- A MutantDeployment runs its inline `spec.mutations` plus every Mutation
76 − that targets it. Labels are unique per MutantDeployment. An inline
77 − mutation wins a clash, then the oldest Mutation. A Mutation that loses is
78 − not run and shows `State: Conflict`.
79 −- `status` reports the generated Deployment, its ready replicas, and
80 − `Progressing`, `Ready`, `Invalid` or `Conflict`.
81 −- The operator makes each Mutation a dependent (`ownerReferences`) of its
82 − MutantDeployment. Argo CD shows it under the MutantDeployment in the
83 − resource tree, and deleting the MutantDeployment deletes its Mutations.
88 +- Each variant becomes Deployment `<base name>-<label>`. Adding a variant
89 + starts one, removing it deletes it, and deleting the Mutation deletes
90 + them all.
91 +- Each Mutation gets a Secret, `<mutation-name>-secret`, holding a random
92 + 64-hex-character `token`. The operator creates it once and never rewrites
93 + it, and it is deleted with the Mutation. It is not injected into the
94 + variants' pods. Anything in the namespace can read it as an env var:
95 +
96 + ```yaml
97 + - name: BROWSER_SNAP_TOKEN
98 + valueFrom:
99 + secretKeyRef:
100 + name: browser-snap-secret
101 + key: token
102 + optional: true # exists once the operator has seen the Mutation
103 + ```
104 +
105 +- `status` reports the Secret, a summary, and each variant's Deployment,
106 + ready replicas and state: `Progressing`, `Ready`, `Invalid`, or
107 + `Conflict`. `Conflict` means the label repeats, or the Deployment name is
108 + already owned by another Mutation or MutantDeployment. The operator never
109 + takes over another owner's Deployment.
110 +- If the base is invalid, nothing is applied or pruned: what is running
111 + keeps running until the base is fixed.
84 112
85 113 ### Argo CD
86 114
87 −Mutations created through the API are not in git and carry no Argo CD
88 −tracking label, so Argo CD neither prunes nor reverts them. Inline
89 −mutations remain in git and are read-only through the API. For Argo CD to
90 −show Mutation health, add to `argocd-cm`:
115 +Mutations are meant to be created at runtime, so they do not go in git.
116 +The operator API refuses to change or delete a Mutation that Argo CD syncs
117 +(one with its `argocd.argoproj.io/tracking-id` annotation or
118 +`app.kubernetes.io/instance` label), because the next sync would undo it.
119 +To show Mutation health in Argo CD, add to `argocd-cm`:
91 120
92 121 ```yaml
93 122 resource.customizations.health.neiam.org_Mutation: |
94 123 hs = { status = "Progressing", message = "Waiting for the operator" }
95 − if obj.status ~= nil and obj.status.state ~= nil then
96 − hs.message = obj.status.message or obj.status.state
97 − if obj.status.state == "Ready" then hs.status = "Healthy"
98 − elseif obj.status.state == "Progressing" then hs.status = "Progressing"
99 − else hs.status = "Degraded" end
124 + if obj.status ~= nil and obj.status.variants ~= nil then
125 + hs.status = "Healthy"
126 + hs.message = obj.status.summary
127 + for _, v in ipairs(obj.status.variants) do
128 + if v.state == "Invalid" or v.state == "Conflict" then
129 + hs.status = "Degraded"; hs.message = v.label .. ": " .. (v.message or v.state); break
130 + elseif v.state == "Progressing" then
131 + hs.status = "Progressing"
132 + end
133 + end
100 134 end
101 135 return hs
102 136 ```
103 137
104 138 ## HTTP API
105 139
106 −The operator serves an HTTP API to list, create and destroy mutations.
107 −Creating one makes a Mutation resource named `<mutantdeployment>-<label>`.
108 −Destroying one deletes it.
140 +The operator serves an HTTP API to create, list and delete Mutations and
141 +their variants.
109 142
110 143 The API is off until `MUTANT_API_TOKEN` is set (`operator.yml` reads it from
111 144 the `mutantops-api` secret). Every `/api` request then needs
112 145 `Authorization: Bearer <token>`. `MUTANT_API_INSECURE=true` serves it without
113 146 auth. `MUTANT_API_ADDR` sets the listen address (default `0.0.0.0:8080`).
114 147
148 +```text
149 +GET /api/v1/namespaces/{ns}/mutations
150 +POST /api/v1/namespaces/{ns}/mutations {name, deployment, variants}
151 +GET /api/v1/namespaces/{ns}/mutations/{name}
152 +DELETE /api/v1/namespaces/{ns}/mutations/{name}
153 +GET /api/v1/namespaces/{ns}/mutations/{name}/variants
154 +POST /api/v1/namespaces/{ns}/mutations/{name}/variants {label, env, instances}
155 +GET /api/v1/namespaces/{ns}/mutations/{name}/variants/{label}
156 +DELETE /api/v1/namespaces/{ns}/mutations/{name}/variants/{label}
157 +```
158 +
159 +Creating answers 201, or 400 if invalid and 409 if the name or label is
160 +taken. Deleting answers 204, or 404 if absent. Changes to an Argo-synced
161 +Mutation answer 409. Concurrent variant edits are serialised on the
162 +Mutation's `resourceVersion`, so none is lost.
163 +
115 164 ```sh
116 165 kubectl -n mutants create secret generic mutantops-api --from-literal=token="$(openssl rand -hex 32)"
117 166 kubectl -n mutants rollout restart deploy/mutantops-operator
118 167 TOKEN=$(kubectl -n mutants get secret mutantops-api -o jsonpath='{.data.token}' | base64 -d)
119 168 API=http://mutantops-api.mutants.svc # or: kubectl -n mutants port-forward svc/mutantops-api 8080:80
120 −MD=$API/api/v1/namespaces/default/mutantdeployments/nginx-mutant
121 −
122 −# MutantDeployments in a namespace, with their running mutation labels
123 −curl -H "Authorization: Bearer $TOKEN" $API/api/v1/namespaces/default/mutantdeployments
124 −# list / get mutations: inline and resource, with live status
125 −curl -H "Authorization: Bearer $TOKEN" $MD/mutations
126 −curl -H "Authorization: Bearer $TOKEN" $MD/mutations/canary
127 −# create (201; 409 if the label is taken, 400 if invalid, 404 if no such MutantDeployment)
128 −curl -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \
129 − -d '{"label":"beta","env":{"FEATURE_X":"on"},"instances":2}' $MD/mutations
130 −# destroy (204; 404 if absent; 409 if inline, which is changed in git instead)
131 −curl -X DELETE -H "Authorization: Bearer $TOKEN" $MD/mutations/beta
169 +NS=$API/api/v1/namespaces/default
170 +AUTH="Authorization: Bearer $TOKEN"
171 +
172 +# create a Mutation from a base Deployment (deployment.json)
173 +jq -n --slurpfile d deployment.json '{name: "web", deployment: $d[0], variants: []}' |
174 + curl -H "$AUTH" -H 'Content-Type: application/json' -d @- $NS/mutations
175 +# add and remove variants
176 +curl -H "$AUTH" -H 'Content-Type: application/json' \
177 + -d '{"label":"beta","env":{"FEATURE_X":"on"},"instances":2}' $NS/mutations/web/variants
178 +curl -X DELETE -H "$AUTH" $NS/mutations/web/variants/beta
179 +# list
180 +curl -H "$AUTH" $NS/mutations
181 +curl -H "$AUTH" $NS/mutations/web/variants
132 182 ```
133 183
134 −A listed mutation looks like:
184 +A listed variant looks like:
135 185
136 186 ```json
137 187 {
138 188 "label": "beta",
139 189 "env": { "FEATURE_X": "on" },
140 190 "instances": 2,
141 − "source": { "kind": "resource", "name": "nginx-mutant-beta" },
142 − "status": { "state": "Ready", "message": null, "deployment": "nginx-beta", "replicas": 2, "readyReplicas": 2 }
191 + "readOnly": false,
192 + "status": { "label": "beta", "state": "Ready", "message": null, "deployment": "web-beta", "replicas": 2, "readyReplicas": 2 }
143 193 }
144 194 ```
145 195
146 −`source.kind` is `inline` for mutations declared in the MutantDeployment.
196 +A Mutation adds its `name`, `namespace`, base `deployment`, `variants`,
197 +`secret`, `summary`, `message` and `readOnly`.
modified crd.yml
+69 −48
@@ -42,9 +42,9 @@ spec:
42 42 shortNames:
43 43 - md
44 44 ---
45 −# A single mutation of a MutantDeployment, created at runtime (e.g. through
46 −# the operator's HTTP API) so the MutantDeployment itself can stay owned by
47 −# GitOps. Schema matches `Mutation` in src/crd.rs.
45 +# A Mutation: a base Deployment plus variants of it, like a
46 +# MutantDeployment, but configured at runtime through the operator's HTTP
47 +# API rather than declared in git. Schema matches `Mutation` in src/crd.rs.
48 48 apiVersion: apiextensions.k8s.io/v1
49 49 kind: CustomResourceDefinition
50 50 metadata:
@@ -65,21 +65,15 @@ spec:
65 65 subresources:
66 66 status: {}
67 67 additionalPrinterColumns:
68 − - name: Target
68 + - name: Base
69 69 type: string
70 − jsonPath: .spec.mutantDeployment
71 − - name: Label
72 − type: string
73 − jsonPath: .spec.label
74 − - name: Instances
75 − type: integer
76 − jsonPath: .spec.instances
70 + jsonPath: .spec.deployment.metadata.name
77 71 - name: Ready
78 − type: integer
79 − jsonPath: .status.readyReplicas
80 − - name: State
81 72 type: string
82 − jsonPath: .status.state
73 + jsonPath: .status.summary
74 + - name: Secret
75 + type: string
76 + jsonPath: .status.secret
83 77 - name: Age
84 78 type: date
85 79 jsonPath: .metadata.creationTimestamp
@@ -92,46 +86,73 @@ spec:
92 86 spec:
93 87 type: object
94 88 required:
95 − - mutantDeployment
96 − - label
97 − - instances
89 + - deployment
98 90 properties:
99 − mutantDeployment:
100 − description: Name of the MutantDeployment, in the same namespace, to mutate.
101 − type: string
102 − label:
103 − description: Suffix of the generated Deployment's name; unique per MutantDeployment.
104 − type: string
105 − pattern: '^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$'
106 − env:
91 + deployment:
92 + description: The base Deployment every variant is derived from.
107 93 type: object
108 − default: {}
109 − additionalProperties:
110 − type: string
111 − instances:
112 − type: integer
113 − format: int32
114 − minimum: 0
94 + x-kubernetes-preserve-unknown-fields: true
95 + variants:
96 + type: array
97 + default: []
98 + items:
99 + type: object
100 + required:
101 + - label
102 + - instances
103 + properties:
104 + label:
105 + description: Suffix of the variant's Deployment name; unique within the Mutation.
106 + type: string
107 + pattern: '^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$'
108 + env:
109 + type: object
110 + default: {}
111 + additionalProperties:
112 + type: string
113 + instances:
114 + type: integer
115 + format: int32
116 + minimum: 0
115 117 status:
116 118 type: object
117 − required:
118 − - state
119 119 properties:
120 − state:
121 − type: string
122 − enum: [Progressing, Ready, Invalid, Conflict]
123 − message:
120 + summary:
121 + description: e.g. "2/3 variants ready".
124 122 type: string
125 123 nullable: true
126 − deployment:
127 − description: The generated Deployment.
124 + secret:
125 + description: The Mutation's generated Secret, holding a random `token`.
128 126 type: string
129 127 nullable: true
130 − replicas:
131 − type: integer
132 − format: int32
133 − nullable: true
134 − readyReplicas:
135 − type: integer
136 − format: int32
128 + message:
129 + type: string
137 130 nullable: true
131 + variants:
132 + type: array
133 + items:
134 + type: object
135 + required:
136 + - label
137 + - state
138 + properties:
139 + label:
140 + type: string
141 + state:
142 + type: string
143 + enum: [Progressing, Ready, Invalid, Conflict]
144 + message:
145 + type: string
146 + nullable: true
147 + deployment:
148 + description: The generated Deployment.
149 + type: string
150 + nullable: true
151 + replicas:
152 + type: integer
153 + format: int32
154 + nullable: true
155 + readyReplicas:
156 + type: integer
157 + format: int32
158 + nullable: true
modified operator.yml
+18 −48
@@ -66,55 +66,25 @@ apiVersion: rbac.authorization.k8s.io/v1
66 66 kind: ClusterRole
67 67 metadata:
68 68 name: mutantops-operator
69 +# Only what the operator uses: generated Deployments, MutantDeployments
70 +# (read), Mutations and their status, and each Mutation's generated Secret
71 +# (get/create only; garbage collection deletes it with the Mutation).
69 72 rules:
70 −- apiGroups:
71 − - ""
72 − resources:
73 − - pods
74 − - services
75 − - services/finalizers
76 − - endpoints
77 − - persistentvolumeclaims
78 − - events
79 − - configmaps
80 − - secrets
81 − verbs:
82 − - create
83 − - delete
84 − - get
85 − - list
86 − - patch
87 − - update
88 − - watch
89 −- apiGroups:
90 − - apps
91 − resources:
92 − - deployments
93 − - daemonsets
94 − - replicasets
95 − - statefulsets
96 − verbs:
97 − - create
98 − - delete
99 − - get
100 − - list
101 − - patch
102 − - update
103 − - watch
104 −- apiGroups:
105 − - neiam.org
106 − resources:
107 − - mutantdeployments
108 − - mutations
109 − - mutations/status
110 − verbs:
111 − - create
112 − - delete
113 − - get
114 − - list
115 − - patch
116 − - update
117 − - watch
73 +- apiGroups: ["apps"]
74 + resources: ["deployments"]
75 + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
76 +- apiGroups: ["neiam.org"]
77 + resources: ["mutantdeployments"]
78 + verbs: ["get", "list", "watch"]
79 +- apiGroups: ["neiam.org"]
80 + resources: ["mutations"]
81 + verbs: ["get", "list", "watch", "create", "patch", "delete"]
82 +- apiGroups: ["neiam.org"]
83 + resources: ["mutations/status"]
84 + verbs: ["get", "patch", "update"]
85 +- apiGroups: [""]
86 + resources: ["secrets"]
87 + verbs: ["get", "create"]
118 88 ---
119 89 apiVersion: rbac.authorization.k8s.io/v1
120 90 kind: ClusterRoleBinding
modified src/api.rs
+202 −144
@@ -1,24 +1,25 @@
1 −//! HTTP API for listing, creating and deleting the mutations of a live
2 −//! MutantDeployment.
3 −//!
4 −//! Created mutations are standalone `Mutation` resources, never edits to the
5 −//! MutantDeployment, which can stay owned by Argo CD (or any GitOps tool)
6 −//! without the API and the sync fighting over it. Mutations declared inline
7 −//! in the MutantDeployment are listed too, but are read-only here: they
8 −//! change in git.
1 +//! HTTP API for Mutations: base Deployments whose variants are configured at
2 +//! runtime.
9 3 //!
10 4 //! ```text
11 5 //! GET /healthz
12 −//! GET /api/v1/namespaces/{ns}/mutantdeployments
13 −//! GET /api/v1/namespaces/{ns}/mutantdeployments/{name}/mutations
14 −//! POST /api/v1/namespaces/{ns}/mutantdeployments/{name}/mutations
15 −//! GET /api/v1/namespaces/{ns}/mutantdeployments/{name}/mutations/{label}
16 −//! DELETE /api/v1/namespaces/{ns}/mutantdeployments/{name}/mutations/{label}
6 +//! GET /api/v1/namespaces/{ns}/mutations
7 +//! POST /api/v1/namespaces/{ns}/mutations {name, deployment, variants}
8 +//! GET /api/v1/namespaces/{ns}/mutations/{name}
9 +//! DELETE /api/v1/namespaces/{ns}/mutations/{name}
10 +//! GET /api/v1/namespaces/{ns}/mutations/{name}/variants
11 +//! POST /api/v1/namespaces/{ns}/mutations/{name}/variants {label, env, instances}
12 +//! GET /api/v1/namespaces/{ns}/mutations/{name}/variants/{label}
13 +//! DELETE /api/v1/namespaces/{ns}/mutations/{name}/variants/{label}
17 14 //! ```
15 +//!
16 +//! A Mutation is a Kubernetes object, so the operator keeps no state of its
17 +//! own. Ones synced by Argo CD are listed with `readOnly: true` and cannot be
18 +//! changed here: the next sync would put them back.
18 19
19 20 use crate::crd::{
20 − applied_status, resolve, MutantDeployment, Mutation, MutationDef, MutationSpec, MutationStatus,
21 − Source, OWNER_LABEL, TARGET_LABEL,
21 + insert_variant, is_gitops_managed, remove_variant, validate_base, validate_label, validate_variants,
22 + Mutation, MutationSpec, Variant, VariantState, VariantStatus,
22 23 };
23 24 use axum::{
24 25 extract::{Path, Request, State},
@@ -30,14 +31,18 @@ use axum::{
30 31 };
31 32 use k8s_openapi::api::apps::v1::Deployment;
32 33 use kube::{
33 − api::{Api, DeleteParams, ListParams, PostParams},
34 − Client, Resource, ResourceExt,
34 + api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams},
35 + Client, ResourceExt,
35 36 };
36 −use serde::Serialize;
37 +use serde::{Deserialize, Serialize};
37 38 use serde_json::json;
38 −use std::{collections::BTreeMap, sync::Arc};
39 +use std::sync::Arc;
39 40 use tracing::{info, warn};
40 41
42 +/// Attempts at a read-modify-write of a Mutation's variants before giving
43 +/// up on conflicts.
44 +const MAX_ATTEMPTS: usize = 5;
45 +
41 46 #[derive(Clone)]
42 47 pub struct AppState {
43 48 client: Client,
@@ -53,14 +58,15 @@ impl AppState {
53 58
54 59 pub fn router(state: AppState) -> Router {
55 60 let api = Router::new()
56 − .route("/namespaces/{ns}/mutantdeployments", get(list_mutants))
61 + .route("/namespaces/{ns}/mutations", get(list_mutations).post(create_mutation))
62 + .route("/namespaces/{ns}/mutations/{name}", get(get_mutation).delete(delete_mutation))
57 63 .route(
58 − "/namespaces/{ns}/mutantdeployments/{name}/mutations",
59 − get(list_mutations).post(create_mutation),
64 + "/namespaces/{ns}/mutations/{name}/variants",
65 + get(list_variants).post(create_variant),
60 66 )
61 67 .route(
62 − "/namespaces/{ns}/mutantdeployments/{name}/mutations/{label}",
63 − get(get_mutation).delete(delete_mutation),
68 + "/namespaces/{ns}/mutations/{name}/variants/{label}",
69 + get(get_variant).delete(delete_variant),
64 70 )
65 71 .route_layer(middleware::from_fn_with_state(state.clone(), require_token));
66 72 Router::new()
@@ -132,168 +138,201 @@ impl IntoResponse for ApiError {
132 138
133 139 type ApiResult<T> = Result<T, ApiError>;
134 140
135 −/// A mutation as the API reports it: its definition, where it was declared,
136 −/// and its live state.
141 +/// A Mutation as the API reports it.
137 142 #[derive(Serialize, Debug)]
143 +#[serde(rename_all = "camelCase")]
138 144 struct MutationView {
145 + name: String,
146 + namespace: String,
147 + /// The base Deployment.
148 + deployment: Deployment,
149 + variants: Vec<VariantView>,
150 + /// The generated Secret, once the operator has made it.
151 + secret: Option<String>,
152 + summary: Option<String>,
153 + message: Option<String>,
154 + /// Synced by Argo CD: changes in git, not here.
155 + read_only: bool,
156 +}
157 +
158 +/// A variant: its definition and its last reported state.
159 +#[derive(Serialize, Debug)]
160 +#[serde(rename_all = "camelCase")]
161 +struct VariantView {
139 162 #[serde(flatten)]
140 − def: MutationDef,
141 − source: Source,
142 − status: MutationStatus,
163 + variant: Variant,
164 + status: VariantStatus,
165 + read_only: bool,
143 166 }
144 167
145 −#[derive(Serialize)]
146 −struct MutantSummary {
168 +#[derive(Deserialize)]
169 +struct CreateMutation {
147 170 name: String,
148 − deployment: String,
149 − mutations: Vec<String>,
171 + deployment: Deployment,
172 + #[serde(default)]
173 + variants: Vec<Variant>,
150 174 }
151 175
152 −/// Every mutation of `name`: the ones running, then Mutation resources that
153 −/// are not (invalid, or a duplicate label), with why.
154 −async fn views(client: &Client, ns: &str, name: &str) -> ApiResult<Vec<MutationView>> {
155 − let mutant = Api::<MutantDeployment>::namespaced(client.clone(), ns).get(name).await?;
156 − let resources = Api::<Mutation>::namespaced(client.clone(), ns)
157 − .list(&ListParams::default())
158 − .await?
159 − .items;
160 − // Status is read live from the Deployments rather than from Mutation
161 − // status, so inline mutations, which have no status, get one too.
162 − let deployments: BTreeMap<String, Deployment> = Api::<Deployment>::namespaced(client.clone(), ns)
163 − .list(&ListParams::default().labels(&format!("{OWNER_LABEL}={name}")))
164 − .await?
165 − .into_iter()
166 − .map(|d| (d.name_any(), d))
167 − .collect();
168 −
169 − let resolved = resolve(&mutant, &resources);
170 − let mut out: Vec<_> = resolved
171 − .active
172 − .into_iter()
173 − .map(|(def, source)| {
174 − let dname = mutant.deployment_name(&def.label);
175 − let status = applied_status(dname.clone(), def.instances, deployments.get(&dname));
176 − MutationView { def, source, status }
177 − })
178 − .collect();
179 − for (rname, state, message) in resolved.rejected {
180 − if let Some(m) = resources.iter().find(|m| m.name_any() == rname) {
181 − out.push(MutationView {
182 − def: m.spec.def.clone(),
183 − source: Source::Resource { name: rname },
184 − status: MutationStatus { state, message: Some(message), ..Default::default() },
185 − });
176 +impl From<Mutation> for MutationView {
177 + fn from(m: Mutation) -> Self {
178 + let read_only = is_gitops_managed(&m);
179 + let status = m.status.clone().unwrap_or_default();
180 + let variants = m
181 + .spec
182 + .variants
183 + .iter()
184 + .map(|v| VariantView {
185 + status: status.variants.iter().find(|s| s.label == v.label).cloned().unwrap_or_else(|| {
186 + // Added since the operator last reported.
187 + VariantStatus::rejected(&v.label, VariantState::Progressing, "not yet reconciled".into())
188 + }),
189 + variant: v.clone(),
190 + read_only,
191 + })
192 + .collect();
193 + MutationView {
194 + name: m.name_any(),
195 + namespace: m.namespace().unwrap_or_default(),
196 + deployment: m.spec.deployment,
197 + variants,
198 + secret: status.secret,
199 + summary: status.summary,
200 + message: status.message,
201 + read_only,
186 202 }
187 203 }
188 − Ok(out)
189 204 }
190 205
191 −async fn list_mutants(State(st): State<AppState>, Path(ns): Path<String>) -> ApiResult<Json<Vec<MutantSummary>>> {
192 − let mutants = Api::<MutantDeployment>::namespaced(st.client.clone(), &ns)
193 − .list(&ListParams::default())
194 − .await?;
195 − let resources = Api::<Mutation>::namespaced(st.client, &ns)
196 − .list(&ListParams::default())
197 − .await?
198 − .items;
199 − Ok(Json(
200 − mutants
201 − .into_iter()
202 − .map(|md| MutantSummary {
203 − mutations: resolve(&md, &resources).active.into_iter().map(|(d, _)| d.label).collect(),
204 − deployment: md.base_name(),
205 − name: md.name_any(),
206 − })
207 − .collect(),
208 − ))
206 +fn api(st: &AppState, ns: &str) -> Api<Mutation> {
207 + Api::namespaced(st.client.clone(), ns)
209 208 }
210 209
211 −async fn list_mutations(
212 − State(st): State<AppState>,
213 − Path((ns, name)): Path<(String, String)>,
214 −) -> ApiResult<Json<Vec<MutationView>>> {
215 − Ok(Json(views(&st.client, &ns, &name).await?))
210 +async fn list_mutations(State(st): State<AppState>, Path(ns): Path<String>) -> ApiResult<Json<Vec<MutationView>>> {
211 + let items = api(&st, &ns).list(&ListParams::default()).await?;
212 + Ok(Json(items.into_iter().map(Into::into).collect()))
216 213 }
217 214
218 215 async fn get_mutation(
219 216 State(st): State<AppState>,
220 − Path((ns, name, label)): Path<(String, String, String)>,
217 + Path((ns, name)): Path<(String, String)>,
221 218 ) -> ApiResult<Json<MutationView>> {
222 − // The running one: listed first, ahead of any rejected duplicate.
223 − views(&st.client, &ns, &name)
224 − .await?
225 − .into_iter()
226 − .find(|v| v.def.label == label)
227 − .map(Json)
228 − .ok_or_else(|| ApiError::NotFound(format!("mutation {label:?} not found")))
219 + Ok(Json(api(&st, &ns).get(&name).await?.into()))
229 220 }
230 221
231 −/// Creates a Mutation resource named `{name}-{label}`.
232 222 async fn create_mutation(
223 + State(st): State<AppState>,
224 + Path(ns): Path<String>,
225 + Json(req): Json<CreateMutation>,
226 +) -> ApiResult<(StatusCode, Json<MutationView>)> {
227 + validate_label(&req.name).map_err(ApiError::BadRequest)?;
228 + validate_base(&req.deployment).map_err(ApiError::BadRequest)?;
229 + validate_variants(&req.variants).map_err(ApiError::BadRequest)?;
230 + let mutation = Mutation::new(&req.name, MutationSpec { deployment: req.deployment, variants: req.variants });
231 + let created = api(&st, &ns).create(&PostParams::default(), &mutation).await?;
232 + info!(%ns, name = %req.name, "mutation created");
233 + Ok((StatusCode::CREATED, Json(created.into())))
234 +}
235 +
236 +async fn delete_mutation(
233 237 State(st): State<AppState>,
234 238 Path((ns, name)): Path<(String, String)>,
235 − Json(def): Json<MutationDef>,
236 −) -> ApiResult<(StatusCode, Json<Mutation>)> {
237 − def.validate().map_err(ApiError::BadRequest)?;
238 − let mutant = Api::<MutantDeployment>::namespaced(st.client.clone(), &ns).get(&name).await?;
239 − // Refused here rather than created only to sit in Conflict.
240 − if views(&st.client, &ns, &name).await?.iter().any(|v| v.def.label == def.label) {
241 − return Err(ApiError::Conflict(format!("mutation {:?} already exists", def.label)));
242 − }
239 +) -> ApiResult<StatusCode> {
240 + let api = api(&st, &ns);
241 + refuse_if_gitops(&api.get(&name).await?)?;
242 + api.delete(&name, &DeleteParams::default()).await?;
243 + info!(%ns, %name, "mutation deleted");
244 + Ok(StatusCode::NO_CONTENT)
245 +}
243 246
244 − let label = def.label.clone();
245 − let mut mutation = Mutation::new(
246 − &format!("{name}-{label}"),
247 − MutationSpec { mutant_deployment: name.clone(), def },
248 − );
249 − mutation.metadata.labels = Some(BTreeMap::from([(TARGET_LABEL.to_string(), name.clone())]));
250 − // Set up front so the Mutation is a dependent of its MutantDeployment
251 − // from the start; the controller also adds it to hand-made ones.
252 − mutation.metadata.owner_references = mutant.owner_ref(&()).map(|o| vec![o]);
247 +async fn list_variants(
248 + State(st): State<AppState>,
249 + Path((ns, name)): Path<(String, String)>,
250 +) -> ApiResult<Json<Vec<VariantView>>> {
251 + Ok(Json(MutationView::from(api(&st, &ns).get(&name).await?).variants))
252 +}
253 253
254 − let created = Api::<Mutation>::namespaced(st.client, &ns)
255 − .create(&PostParams::default(), &mutation)
256 − .await?;
257 − info!(%ns, %name, %label, "mutation created");
254 +async fn get_variant(
255 + State(st): State<AppState>,
256 + Path((ns, name, label)): Path<(String, String, String)>,
257 +) -> ApiResult<Json<VariantView>> {
258 + MutationView::from(api(&st, &ns).get(&name).await?)
259 + .variants
260 + .into_iter()
261 + .find(|v| v.variant.label == label)
262 + .map(Json)
263 + .ok_or_else(|| ApiError::NotFound(format!("variant {label:?} not found")))
264 +}
265 +
266 +async fn create_variant(
267 + State(st): State<AppState>,
268 + Path((ns, name)): Path<(String, String)>,
269 + Json(variant): Json<Variant>,
270 +) -> ApiResult<(StatusCode, Json<Variant>)> {
271 + variant.validate().map_err(ApiError::BadRequest)?;
272 + let created = variant.clone();
273 + update_variants(&api(&st, &ns), &name, |list| {
274 + insert_variant(list, variant.clone()).map_err(ApiError::Conflict)
275 + })
276 + .await?;
277 + info!(%ns, %name, label = %created.label, "variant created");
258 278 Ok((StatusCode::CREATED, Json(created)))
259 279 }
260 280
261 −/// Deletes every Mutation resource of `name` with this label: the running
262 −/// one and any duplicate waiting behind it, which would otherwise take its
263 −/// place. Inline mutations are refused.
264 −async fn delete_mutation(
281 +async fn delete_variant(
265 282 State(st): State<AppState>,
266 283 Path((ns, name, label)): Path<(String, String, String)>,
267 284 ) -> ApiResult<StatusCode> {
268 − let matching: Vec<_> = views(&st.client, &ns, &name)
269 − .await?
270 − .into_iter()
271 − .filter(|v| v.def.label == label)
272 − .collect();
273 − if matching.is_empty() {
274 − return Err(ApiError::NotFound(format!("mutation {label:?} not found")));
275 − }
276 − if matching.iter().any(|v| v.source == Source::Inline) {
285 + update_variants(&api(&st, &ns), &name, |list| {
286 + remove_variant(list, &label)
287 + .map(drop)
288 + .ok_or_else(|| ApiError::NotFound(format!("variant {label:?} not found")))
289 + })
290 + .await?;
291 + info!(%ns, %name, %label, "variant deleted");
292 + Ok(StatusCode::NO_CONTENT)
293 +}
294 +
295 +fn refuse_if_gitops(m: &Mutation) -> ApiResult<()> {
296 + if is_gitops_managed(m) {
277 297 return Err(ApiError::Conflict(format!(
278 − "mutation {label:?} is declared inline in MutantDeployment {name}; remove it from its spec.mutations (in git) instead"
298 + "mutation {} is synced by Argo CD; change it in git instead",
299 + m.name_any()
279 300 )));
280 301 }
281 − let api = Api::<Mutation>::namespaced(st.client, &ns);
282 − for v in matching {
283 − if let Source::Resource { name: rname } = v.source {
284 − match api.delete(&rname, &DeleteParams::default()).await {
285 − Ok(_) | Err(kube::Error::Api(kube::core::ErrorResponse { code: 404, .. })) => {}
286 − Err(e) => return Err(e.into()),
287 − }
302 + Ok(())
303 +}
304 +
305 +/// Read-modify-write of `spec.variants`, guarded by `resourceVersion` so a
306 +/// concurrent edit (another API call, or kubectl) is never overwritten: the
307 +/// API server answers 409 when the object changed since the read, and the
308 +/// edit is retried against the fresh copy.
309 +async fn update_variants<F>(api: &Api<Mutation>, name: &str, edit: F) -> ApiResult<()>
310 +where
311 + F: Fn(&mut Vec<Variant>) -> ApiResult<()>,
312 +{
313 + for _ in 0..MAX_ATTEMPTS {
314 + let current = api.get(name).await?;
315 + refuse_if_gitops(&current)?;
316 + let mut variants = current.spec.variants.clone();
317 + edit(&mut variants)?;
318 + let patch = json!({
319 + "metadata": { "resourceVersion": current.resource_version() },
320 + "spec": { "variants": variants },
321 + });
322 + match api.patch(name, &PatchParams::default(), &Patch::Merge(&patch)).await {
323 + Ok(_) => return Ok(()),
324 + Err(kube::Error::Api(e)) if e.code == 409 => continue,
325 + Err(e) => return Err(e.into()),
288 326 }
289 327 }
290 − info!(%ns, %name, %label, "mutation deleted");
291 − Ok(StatusCode::NO_CONTENT)
328 + Err(ApiError::Conflict(format!("{name} kept changing; gave up after {MAX_ATTEMPTS} attempts")))
292 329 }
293 330
294 331 #[cfg(test)]
295 332 mod tests {
296 333 use super::*;
334 + use crate::crd::MutationStatus;
335 + use std::collections::HashMap;
297 336
298 337 #[test]
299 338 fn auth() {
@@ -304,4 +343,23 @@ mod tests {
304 343 assert!(!authorized(Some("s3cret"), Some("s3creT")));
305 344 assert!(!authorized(Some("s3cret"), None));
306 345 }
346 +
347 + #[test]
348 + fn view_joins_variants_with_status() {
349 + let v = |label: &str| Variant { label: label.into(), env: HashMap::new(), instances: 1 };
350 + let mut m = Mutation::new(
351 + "browser-snap",
352 + MutationSpec { deployment: Deployment::default(), variants: vec![v("lilygo47"), v("new")] },
353 + );
354 + m.status = Some(MutationStatus {
355 + secret: Some("browser-snap-secret".into()),
356 + variants: vec![VariantStatus { ready_replicas: Some(1), ..VariantStatus::rejected("lilygo47", VariantState::Ready, String::new()) }],
357 + ..Default::default()
358 + });
359 + let view = MutationView::from(m);
360 + assert_eq!(view.secret.as_deref(), Some("browser-snap-secret"));
361 + assert!(!view.read_only);
362 + assert_eq!(view.variants[0].status.state, VariantState::Ready);
363 + assert_eq!(view.variants[1].status.message.as_deref(), Some("not yet reconciled"));
364 + }
307 365 }
modified src/controller.rs
+177 −89
@@ -1,18 +1,21 @@
1 −//! Reconciles each MutantDeployment, together with the Mutation resources
2 −//! that target it, into one Deployment per mutation; prunes Deployments
3 −//! whose mutation is gone; and reports each Mutation's state in its status.
1 +//! Reconciles MutantDeployments and Mutations alike: a base Deployment plus
2 +//! variants becomes one Deployment per variant, and Deployments whose
3 +//! variant is gone are pruned. A Mutation also gets a generated Secret and a
4 +//! status reporting each variant.
4 5
5 6 use crate::crd::{
6 − applied_status, build_deployment, resolve, MutantDeployment, Mutation, MutationStatus, Source,
7 + applied_status, build_deployment, build_secret, deployment_name, generate_token, secret_name, summarize,
8 + validate_base, MutantDeployment, Mutation, MutationStatus, Variant, VariantState, VariantStatus,
7 9 MUTATION_LABEL, OWNER_LABEL,
8 10 };
9 11 use futures::StreamExt;
10 12 use k8s_openapi::api::apps::v1::Deployment;
13 +use k8s_openapi::api::core::v1::Secret;
14 +use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference;
11 15 use kube::{
12 − api::{Api, DeleteParams, ListParams, Patch, PatchParams},
16 + api::{Api, DeleteParams, ListParams, Patch, PatchParams, PostParams},
13 17 runtime::{
14 18 controller::{Action, Controller},
15 − reflector::ObjectRef,
16 19 watcher,
17 20 },
18 21 Client, Resource, ResourceExt,
@@ -22,9 +25,7 @@ use std::{collections::HashSet, fmt, sync::Arc, time::Duration};
22 25 use tracing::{info, warn};
23 26
24 27 const FIELD_MANAGER: &str = "mutantops-operator";
25 −/// Periodic resync. Events drive reconciliation; this only catches what no
26 −/// event points at, such as a Mutation retargeted away from a
27 −/// MutantDeployment, whose old Deployment is then pruned.
28 +/// Periodic resync. Events drive reconciliation; this is a backstop.
28 29 const RESYNC: Duration = Duration::from_secs(300);
29 30
30 31 #[derive(Debug)]
@@ -54,113 +55,200 @@ struct Ctx {
54 55 client: Client,
55 56 }
56 57
57 −async fn reconcile(mutant: Arc<MutantDeployment>, ctx: Arc<Ctx>) -> Result<Action, MutantopsError> {
58 − let ns = mutant
59 − .namespace()
60 − .ok_or_else(|| MutantopsError::Invalid("MutantDeployment has no namespace".into()))?;
61 − let controller_ref = mutant
62 − .controller_owner_ref(&())
63 − .ok_or_else(|| MutantopsError::Invalid("MutantDeployment has no uid".into()))?;
64 − let deployments: Api<Deployment> = Api::namespaced(ctx.client.clone(), &ns);
65 − let mutations: Api<Mutation> = Api::namespaced(ctx.client.clone(), &ns);
66 − let apply = PatchParams::apply(FIELD_MANAGER).force();
67 −
68 − // Inline entries can only be edited in the MutantDeployment itself, and
69 − // have no status to report into, so a bad one is just logged.
70 − for def in &mutant.spec.mutations {
71 − if let Err(msg) = def.validate() {
72 − warn!(mutant = %mutant.name_any(), %ns, "skipping inline mutation: {msg}");
73 − }
74 − }
58 +/// Who the generated Deployments belong to.
59 +struct Owner {
60 + name: String,
61 + ns: String,
62 + controller_ref: OwnerReference,
63 +}
75 64
76 − let resources = mutations.list(&ListParams::default()).await?.items;
77 − // Each Mutation becomes a dependent of its MutantDeployment: Argo CD then
78 − // shows it in the MutantDeployment's resource tree, and deleting the
79 − // MutantDeployment garbage-collects it. Done before the status writes
80 − // below, which would invalidate the resourceVersion this is guarded by.
81 − let owner = mutant.owner_ref(&()).expect("uid checked above");
82 − for m in resources.iter().filter(|m| m.spec.mutant_deployment == mutant.name_any()) {
83 − if !m.owner_references().iter().any(|o| o.uid == owner.uid) {
84 − let mut refs = m.owner_references().to_vec();
85 − refs.push(owner.clone());
86 − let patch = json!({ "metadata": { "ownerReferences": refs, "resourceVersion": m.resource_version() } });
87 − mutations.patch(&m.name_any(), &PatchParams::default(), &Patch::Merge(&patch)).await?;
88 − }
65 +impl Owner {
66 + fn of<K: Resource<DynamicType = ()>>(obj: &K) -> Result<Self, MutantopsError> {
67 + Ok(Self {
68 + name: obj.name_any(),
69 + ns: obj
70 + .namespace()
71 + .ok_or_else(|| MutantopsError::Invalid(format!("{} has no namespace", obj.name_any())))?,
72 + controller_ref: obj
73 + .controller_owner_ref(&())
74 + .ok_or_else(|| MutantopsError::Invalid(format!("{} has no uid", obj.name_any())))?,
75 + })
89 76 }
77 +}
90 78
91 − let resolved = resolve(&mutant, &resources);
92 − let find = |name: &str| resources.iter().find(|m| m.name_any() == name);
79 +/// Applies one Deployment per variant of `base` and prunes the owner's
80 +/// Deployments whose variant is gone. Returns each variant's status.
81 +///
82 +/// A variant is skipped, and reported, when it is invalid, repeats an
83 +/// earlier label, or its Deployment name is already controlled by a
84 +/// different owner: two owners would otherwise take turns overwriting it.
85 +async fn apply_variants(
86 + client: &Client,
87 + owner: &Owner,
88 + base: &Deployment,
89 + variants: &[Variant],
90 +) -> Result<Vec<VariantStatus>, kube::Error> {
91 + let api: Api<Deployment> = Api::namespaced(client.clone(), &owner.ns);
92 + let apply = PatchParams::apply(FIELD_MANAGER).force();
93 + let uid = &owner.controller_ref.uid;
93 94
95 + let mut statuses = Vec::with_capacity(variants.len());
94 96 let mut wanted = HashSet::new();
95 − for (def, source) in &resolved.active {
96 − let name = mutant.deployment_name(&def.label);
97 − let deploy = build_deployment(&mutant, def, controller_ref.clone());
98 − let applied = deployments.patch(&name, &apply, &Patch::Apply(&deploy)).await?;
99 − if let Source::Resource { name: mname } = source {
100 − if let Some(m) = find(mname) {
101 − set_status(&mutations, m, applied_status(name.clone(), def.instances, Some(&applied))).await?;
97 + let mut labels = HashSet::new();
98 + for v in variants {
99 + if let Err(msg) = v.validate() {
100 + statuses.push(VariantStatus::rejected(&v.label, VariantState::Invalid, msg));
101 + continue;
102 + }
103 + if !labels.insert(v.label.as_str()) {
104 + let msg = "label repeats an earlier variant".to_string();
105 + statuses.push(VariantStatus::rejected(&v.label, VariantState::Conflict, msg));
106 + continue;
107 + }
108 + let name = deployment_name(base, &v.label);
109 + if let Some(existing) = api.get_opt(&name).await? {
110 + let other = existing.owner_references().iter().find(|o| o.controller == Some(true) && &o.uid != uid);
111 + if let Some(other) = other {
112 + let msg = format!("deployment {name} is already managed by {} {}", other.kind, other.name);
113 + statuses.push(VariantStatus::rejected(&v.label, VariantState::Conflict, msg));
114 + continue;
102 115 }
103 116 }
117 + let deploy = build_deployment(base, Some(owner.ns.clone()), &owner.name, v, owner.controller_ref.clone());
118 + let applied = api.patch(&name, &apply, &Patch::Apply(&deploy)).await?;
119 + statuses.push(applied_status(&v.label, name.clone(), v.instances, Some(&applied)));
104 120 wanted.insert(name);
105 121 }
106 − for (mname, state, message) in &resolved.rejected {
107 − if let Some(m) = find(mname) {
108 − let status = MutationStatus { state: *state, message: Some(message.clone()), ..Default::default() };
109 − set_status(&mutations, m, status).await?;
110 − }
111 − }
112 122
113 − // Prune: Deployments this MutantDeployment owns whose mutation is gone.
114 − // Checking the owner uid as well as the label means a Deployment that
115 − // merely carries the label is never deleted.
116 − let selector = format!("{OWNER_LABEL}={}", mutant.name_any());
117 − for existing in deployments.list(&ListParams::default().labels(&selector)).await? {
123 + // Prune. Checking the owner uid as well as the label means a Deployment
124 + // that merely carries the label, or belongs to a same-named owner of
125 + // the other kind, is never deleted.
126 + let selector = format!("{OWNER_LABEL}={}", owner.name);
127 + for existing in api.list(&ListParams::default().labels(&selector)).await? {
118 128 let name = existing.name_any();
119 − let owned = existing.owner_references().iter().any(|o| o.uid == controller_ref.uid);
129 + let owned = existing.owner_references().iter().any(|o| &o.uid == uid);
120 130 if owned && !wanted.contains(&name) {
121 − info!(%name, %ns, mutation = ?existing.labels().get(MUTATION_LABEL), "pruning removed mutation");
122 − deployments.delete(&name, &DeleteParams::background()).await?;
131 + info!(%name, ns = %owner.ns, variant = ?existing.labels().get(MUTATION_LABEL), "pruning removed variant");
132 + api.delete(&name, &DeleteParams::background()).await?;
123 133 }
124 134 }
135 + Ok(statuses)
136 +}
125 137
138 +async fn reconcile_mutant(md: Arc<MutantDeployment>, ctx: Arc<Ctx>) -> Result<Action, MutantopsError> {
139 + let owner = Owner::of(md.as_ref())?;
140 + // A MutantDeployment has no status to report into, so problems are logged.
141 + for s in apply_variants(&ctx.client, &owner, &md.spec.deployment, &md.spec.mutations).await? {
142 + if let Some(msg) = s.message {
143 + warn!(mutant = %owner.name, ns = %owner.ns, variant = %s.label, state = ?s.state, "{msg}");
144 + }
145 + }
126 146 Ok(Action::requeue(RESYNC))
127 147 }
128 148
129 −/// Writes `status` unless it is already current. Skipping no-op writes
130 −/// matters: every write is a watch event that reconciles again.
131 −async fn set_status(api: &Api<Mutation>, m: &Mutation, status: MutationStatus) -> Result<(), kube::Error> {
132 − if m.status.as_ref() == Some(&status) {
133 − return Ok(());
149 +async fn reconcile_mutation(m: Arc<Mutation>, ctx: Arc<Ctx>) -> Result<Action, MutantopsError> {
150 + if m.metadata.deletion_timestamp.is_some() {
151 + // Garbage collection removes its Deployments and Secret.
152 + return Ok(Action::await_change());
153 + }
154 + let owner = Owner::of(m.as_ref())?;
155 + let secrets: Api<Secret> = Api::namespaced(ctx.client.clone(), &owner.ns);
156 +
157 + let mut status = MutationStatus::default();
158 + match ensure_secret(&secrets, &m).await? {
159 + Ok(name) => status.secret = Some(name),
160 + Err(msg) => status.message = Some(msg),
161 + }
162 + match validate_base(&m.spec.deployment) {
163 + // Nothing is applied or pruned from a broken base: what is running
164 + // keeps running until the base is fixed.
165 + Err(msg) => status.message = Some(msg),
166 + Ok(()) => status.variants = apply_variants(&ctx.client, &owner, &m.spec.deployment, &m.spec.variants).await?,
167 + }
168 + status.summary = Some(summarize(&status.variants));
169 +
170 + if m.status.as_ref() != Some(&status) {
171 + // Skipped when unchanged: every write is a watch event, which would
172 + // reconcile again.
173 + let api: Api<Mutation> = Api::namespaced(ctx.client.clone(), &owner.ns);
174 + let patch = json!({ "status": status });
175 + api.patch_status(&owner.name, &PatchParams::default(), &Patch::Merge(&patch)).await?;
176 + }
177 + Ok(Action::requeue(RESYNC))
178 +}
179 +
180 +/// Creates the Mutation's Secret unless it exists, returning its name, or
181 +/// why there is none as a message for the Mutation's status.
182 +///
183 +/// An existing Secret is never rewritten, so the token stays stable. One
184 +/// that this Mutation does not own is left alone and reported: it was made
185 +/// by someone else, and its contents are not ours to vouch for.
186 +async fn ensure_secret(api: &Api<Secret>, m: &Mutation) -> Result<Result<String, String>, MutantopsError> {
187 + let name = secret_name(&m.name_any());
188 + let uid = m.uid();
189 + let owned = |s: &Secret| s.owner_references().iter().any(|o| Some(&o.uid) == uid.as_ref());
190 +
191 + if let Some(existing) = api.get_opt(&name).await? {
192 + return Ok(if owned(&existing) {
193 + Ok(name)
194 + } else {
195 + Err(format!("secret {name} exists but is not owned by this Mutation; not using it"))
196 + });
197 + }
198 +
199 + let owner = m
200 + .controller_owner_ref(&())
201 + .ok_or_else(|| MutantopsError::Invalid("Mutation has no uid".into()))?;
202 + let token = generate_token().map_err(|e| MutantopsError::Invalid(format!("no randomness for token: {e}")))?;
203 + match api.create(&PostParams::default(), &build_secret(m, owner, token)).await {
204 + Ok(_) => {
205 + info!(secret = %name, mutation = %m.name_any(), "created mutation secret");
206 + Ok(Ok(name))
207 + }
208 + // Lost a race with another reconcile; the next pass checks ownership.
209 + Err(kube::Error::Api(e)) if e.code == 409 => Ok(Ok(name)),
210 + // Invalid for this Mutation alone: report it there.
211 + Err(kube::Error::Api(e)) if e.code == 422 => {
212 + warn!(secret = %name, "cannot create mutation secret: {}", e.message);
213 + Ok(Err(format!("cannot create secret {name}: {}", e.message)))
214 + }
215 + Err(e) => Err(e.into()),
134 216 }
135 − let patch = json!({ "status": status });
136 − api.patch_status(&m.name_any(), &PatchParams::default(), &Patch::Merge(&patch)).await?;
137 − Ok(())
138 217 }
139 218
140 −fn error_policy(mutant: Arc<MutantDeployment>, error: &MutantopsError, _ctx: Arc<Ctx>) -> Action {
141 − warn!(mutant = %mutant.name_any(), "reconcile failed: {error}");
219 +fn error_policy<K: ResourceExt>(obj: Arc<K>, error: &MutantopsError, _ctx: Arc<Ctx>) -> Action {
220 + warn!(name = %obj.name_any(), ns = ?obj.namespace(), "reconcile failed: {error}");
142 221 Action::requeue(Duration::from_secs(10))
143 222 }
144 223
145 −/// Runs the controller until a shutdown signal.
224 +/// Runs both controllers until a shutdown signal.
146 225 pub async fn run(client: Client) {
147 − let mutants = Api::<MutantDeployment>::all(client.clone());
148 − Controller::new(mutants, watcher::Config::default())
149 − // Watch only the Deployments we generate: a hand edit or delete is
150 − // put back, and replica readiness flows into Mutation status.
151 − .owns(Api::<Deployment>::all(client.clone()), watcher::Config::default().labels(OWNER_LABEL))
152 − // A Mutation created, changed or deleted reconciles its target.
153 − .watches(Api::<Mutation>::all(client.clone()), watcher::Config::default(), |m| {
154 − let ns = m.namespace()?;
155 − Some(ObjectRef::new(&m.spec.mutant_deployment).within(&ns))
156 − })
226 + let ctx = Arc::new(Ctx { client: client.clone() });
227 + // Each watches only the Deployments it generates: a hand edit or delete
228 + // is put back, and replica readiness flows into status.
229 + let owned = || watcher::Config::default().labels(OWNER_LABEL);
230 +
231 + let mutants = Controller::new(Api::<MutantDeployment>::all(client.clone()), watcher::Config::default())
232 + .owns(Api::<Deployment>::all(client.clone()), owned())
157 233 .shutdown_on_signal()
158 − .run(reconcile, error_policy, Arc::new(Ctx { client }))
234 + .run(reconcile_mutant, error_policy, ctx.clone())
159 235 .for_each(|res| async move {
160 236 match res {
161 − Ok((obj, _)) => info!(name = %obj.name, ns = ?obj.namespace, "reconciled"),
162 − Err(err) => warn!("reconcile failed: {err}"),
237 + Ok((obj, _)) => info!(kind = "MutantDeployment", name = %obj.name, ns = ?obj.namespace, "reconciled"),
238 + Err(err) => warn!(kind = "MutantDeployment", "reconcile failed: {err}"),
163 239 }
164 − })
165 − .await;
240 + });
241 +
242 + let mutations = Controller::new(Api::<Mutation>::all(client.clone()), watcher::Config::default())
243 + .owns(Api::<Deployment>::all(client.clone()), owned())
244 + .shutdown_on_signal()
245 + .run(reconcile_mutation, error_policy, ctx)
246 + .for_each(|res| async move {
247 + match res {
248 + Ok((obj, _)) => info!(kind = "Mutation", name = %obj.name, ns = ?obj.namespace, "reconciled"),
249 + Err(err) => warn!(kind = "Mutation", "reconcile failed: {err}"),
250 + }
251 + });
252 +
253 + futures::join!(mutants, mutations);
166 254 }
modified src/crd.rs
+250 −182
@@ -1,31 +1,33 @@
1 1 //! The `MutantDeployment` and `Mutation` resources, and the pure logic that
2 2 //! turns them into concrete `Deployment`s.
3 3 //!
4 −//! A mutation can be declared two ways:
5 −//! - inline, in `MutantDeployment.spec.mutations` (lives in git with the
6 −//! MutantDeployment, so GitOps tools own it);
7 −//! - as a standalone `Mutation` resource naming its MutantDeployment in
8 −//! `spec.mutantDeployment` (created at runtime, e.g. through the HTTP API,
9 −//! without touching the GitOps-managed MutantDeployment).
4 +//! Both are a base Deployment plus variants of it, one Deployment per
5 +//! variant. They differ in who configures them:
6 +//! - a `MutantDeployment` is declared in git (its variants are
7 +//! `spec.mutations`), so GitOps tools own it;
8 +//! - a `Mutation` is configured at runtime through the operator's HTTP API
9 +//! (its variants are `spec.variants`), and gets a generated Secret.
10 10
11 11 use k8s_openapi::api::apps::v1::Deployment;
12 −use k8s_openapi::api::core::v1::EnvVar;
12 +use k8s_openapi::api::core::v1::{EnvVar, Secret};
13 13 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference};
14 +use k8s_openapi::ByteString;
14 15 use kube::{CustomResource, ResourceExt};
15 16 use schemars::JsonSchema;
16 17 use serde::{Deserialize, Serialize};
17 18 use std::collections::{BTreeMap, HashMap, HashSet};
18 19
19 −/// Set on every generated Deployment: the mutation it was built from.
20 +/// Set on every generated Deployment: the variant it was built from.
20 21 /// Kept unprefixed for compatibility with Deployments created before it
21 22 /// was namespaced.
22 23 pub const MUTATION_LABEL: &str = "mutation";
23 −/// Set on every generated Deployment: the MutantDeployment that owns it.
24 −/// Used to find Deployments to prune when a mutation is removed.
24 +/// Set on every generated Deployment: the name of the MutantDeployment or
25 +/// Mutation that owns it. Used to find Deployments to prune.
25 26 pub const OWNER_LABEL: &str = "mutantops.neiam.org/owner";
26 −/// Set on Mutation resources the API creates: their MutantDeployment, so
27 −/// they can be selected with `kubectl get mutations -l ...`.
28 −pub const TARGET_LABEL: &str = "mutantops.neiam.org/mutant-deployment";
27 +/// Set on each Mutation's generated Secret: the Mutation it belongs to.
28 +pub const SECRET_MUTATION_LABEL: &str = "mutantops.neiam.org/mutation";
29 +/// Key of the generated token in a Mutation's Secret.
30 +pub const SECRET_TOKEN_KEY: &str = "token";
29 31
30 32 #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
31 33 #[kube(group = "neiam.org", version = "v1", kind = "MutantDeployment", namespaced)]
@@ -33,10 +35,10 @@ pub struct MutantDeploymentSpec {
33 35 #[schemars(skip)]
34 36 pub deployment: Deployment,
35 37 #[serde(default)]
36 − pub mutations: Vec<MutationDef>,
38 + pub mutations: Vec<Variant>,
37 39 }
38 40
39 −#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, PartialEq, JsonSchema)]
41 +#[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)]
40 42 #[kube(
41 43 group = "neiam.org",
42 44 version = "v1",
@@ -44,17 +46,18 @@ pub struct MutantDeploymentSpec {
44 46 namespaced,
45 47 status = "MutationStatus"
46 48 )]
47 −#[serde(rename_all = "camelCase")]
48 49 pub struct MutationSpec {
49 − /// Name of the MutantDeployment, in the same namespace, to mutate.
50 − pub mutant_deployment: String,
51 − #[serde(flatten)]
52 − pub def: MutationDef,
50 + /// The base Deployment every variant is derived from.
51 + #[schemars(skip)]
52 + pub deployment: Deployment,
53 + #[serde(default)]
54 + pub variants: Vec<Variant>,
53 55 }
54 56
55 −/// One mutation, however it was declared.
57 +/// One variant of a base Deployment: its own name suffix, env overrides
58 +/// and replica count.
56 59 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, JsonSchema)]
57 −pub struct MutationDef {
60 +pub struct Variant {
58 61 pub label: String,
59 62 #[serde(default)]
60 63 pub env: HashMap<String, String>,
@@ -66,7 +69,24 @@ pub struct MutationDef {
66 69 pub struct MutationStatus {
67 70 // `None` serializes as null on purpose: status is written as a merge
68 71 // patch, where null clears a field left over from an earlier state.
69 − pub state: MutationState,
72 + /// e.g. "2/3 variants ready", for `kubectl get mutations`.
73 + #[serde(default)]
74 + pub summary: Option<String>,
75 + /// The Mutation's generated Secret, holding a random `token`.
76 + #[serde(default)]
77 + pub secret: Option<String>,
78 + /// A problem with the Mutation as a whole, such as its Secret.
79 + #[serde(default)]
80 + pub message: Option<String>,
81 + #[serde(default)]
82 + pub variants: Vec<VariantStatus>,
83 +}
84 +
85 +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, JsonSchema)]
86 +#[serde(rename_all = "camelCase")]
87 +pub struct VariantStatus {
88 + pub label: String,
89 + pub state: VariantState,
70 90 #[serde(default)]
71 91 pub message: Option<String>,
72 92 /// The generated Deployment.
@@ -79,21 +99,28 @@ pub struct MutationStatus {
79 99 }
80 100
81 101 #[derive(Deserialize, Serialize, Clone, Copy, Debug, Default, PartialEq, Eq, JsonSchema)]
82 −pub enum MutationState {
102 +pub enum VariantState {
83 103 /// Deployment applied; not all replicas ready yet.
84 104 #[default]
85 105 Progressing,
86 106 /// Every requested replica is ready.
87 107 Ready,
88 − /// Not applied: the spec is invalid.
108 + /// Not applied: the variant is invalid.
89 109 Invalid,
90 − /// Not applied: another mutation of the same MutantDeployment already
91 − /// uses this label.
110 + /// Not applied: its label repeats an earlier variant's, or its
111 + /// Deployment name is already taken by another MutantDeployment or
112 + /// Mutation.
92 113 Conflict,
93 114 }
94 115
95 −impl MutationDef {
96 − /// Checks the mutation can become a valid Deployment, so a bad entry is
116 +impl VariantStatus {
117 + pub fn rejected(label: &str, state: VariantState, message: String) -> Self {
118 + Self { label: label.into(), state, message: Some(message), deployment: None, replicas: None, ready_replicas: None }
119 + }
120 +}
121 +
122 +impl Variant {
123 + /// Checks the variant can become a valid Deployment, so a bad entry is
97 124 /// refused at the API instead of failing reconciliation later.
98 125 pub fn validate(&self) -> Result<(), String> {
99 126 validate_label(&self.label)?;
@@ -107,9 +134,33 @@ impl MutationDef {
107 134 }
108 135 }
109 136
110 −/// The label becomes both a name suffix and a label value, so it must be an
111 −/// RFC 1123 label: lowercase alphanumerics and '-', 1-63 chars, alphanumeric
112 −/// at both ends.
137 +/// Checks every variant, and that no two share a label.
138 +pub fn validate_variants(variants: &[Variant]) -> Result<(), String> {
139 + let mut seen = HashSet::new();
140 + for v in variants {
141 + v.validate()?;
142 + if !seen.insert(&v.label) {
143 + return Err(format!("variant label {:?} appears more than once", v.label));
144 + }
145 + }
146 + Ok(())
147 +}
148 +
149 +/// Checks a base Deployment can have variants built from it.
150 +pub fn validate_base(base: &Deployment) -> Result<(), String> {
151 + match base.metadata.name.as_deref() {
152 + None | Some("") => return Err("deployment.metadata.name is required: variants are named after it".into()),
153 + Some(_) => {}
154 + }
155 + if base.spec.as_ref().and_then(|s| s.template.spec.as_ref()).is_none() {
156 + return Err("deployment.spec.template.spec is required".into());
157 + }
158 + Ok(())
159 +}
160 +
161 +/// Labels become both a name suffix and a label value, and Mutation names
162 +/// a Secret name prefix, so both must be RFC 1123 labels: lowercase
163 +/// alphanumerics and '-', 1-63 chars, alphanumeric at both ends.
113 164 pub fn validate_label(label: &str) -> Result<(), String> {
114 165 let ok = !label.is_empty()
115 166 && label.len() <= 63
@@ -122,7 +173,7 @@ pub fn validate_label(label: &str) -> Result<(), String> {
122 173 Ok(())
123 174 } else {
124 175 Err(format!(
125 − "invalid label {label:?}: must be 1-63 lowercase alphanumerics or '-', \
176 + "invalid name {label:?}: must be 1-63 lowercase alphanumerics or '-', \
126 177 starting and ending alphanumeric"
127 178 ))
128 179 }
@@ -139,94 +190,40 @@ fn validate_env_name(name: &str) -> Result<(), String> {
139 190 }
140 191 }
141 192
142 −impl MutantDeployment {
143 − /// Name of the base Deployment the mutations are derived from.
144 − pub fn base_name(&self) -> String {
145 − self.spec
146 − .deployment
147 − .metadata
148 − .name
149 − .clone()
150 − .unwrap_or_else(|| "mutant".into())
151 − }
152 −
153 − /// Name of the Deployment generated for `label`.
154 − pub fn deployment_name(&self, label: &str) -> String {
155 − format!("{}-{}", self.base_name(), label)
156 − }
193 +/// Name of the base Deployment variants are derived from.
194 +pub fn base_name(base: &Deployment) -> String {
195 + base.metadata.name.clone().unwrap_or_else(|| "mutant".into())
157 196 }
158 197
159 −/// Where an active mutation was declared.
160 −#[derive(Serialize, Clone, Debug, PartialEq, Eq)]
161 −#[serde(tag = "kind", rename_all = "camelCase")]
162 −pub enum Source {
163 − /// In the MutantDeployment's `spec.mutations`.
164 − Inline,
165 − /// A `Mutation` resource of this name.
166 − Resource { name: String },
198 +/// Name of the Deployment generated for variant `label` of `base`.
199 +pub fn deployment_name(base: &Deployment, label: &str) -> String {
200 + format!("{}-{}", base_name(base), label)
167 201 }
168 202
169 −/// The outcome of combining a MutantDeployment's inline mutations with the
170 −/// Mutation resources that target it.
171 −#[derive(Debug, Default)]
172 −pub struct Resolved {
173 − /// Mutations to run, one per label.
174 − pub active: Vec<(MutationDef, Source)>,
175 − /// Mutation resources not run, by name, with why.
176 − pub rejected: Vec<(String, MutationState, String)>,
177 −}
178 −
179 −/// Decides which mutations run. Inline mutations come first, since they
180 −/// live in git with the MutantDeployment; Mutation resources follow, oldest
181 −/// first, so an existing mutation is never displaced by a newer one with the
182 −/// same label. Invalid inline entries are dropped (the caller logs them).
183 −pub fn resolve(mutant: &MutantDeployment, resources: &[Mutation]) -> Resolved {
184 − let mut out = Resolved::default();
185 − let mut labels = HashSet::new();
186 −
187 − for def in &mutant.spec.mutations {
188 − if def.validate().is_ok() && labels.insert(def.label.clone()) {
189 − out.active.push((def.clone(), Source::Inline));
190 − }
203 +/// Adds `variant` to `list`, refusing a duplicate label.
204 +pub fn insert_variant(list: &mut Vec<Variant>, variant: Variant) -> Result<(), String> {
205 + if list.iter().any(|v| v.label == variant.label) {
206 + return Err(format!("variant {:?} already exists", variant.label));
191 207 }
208 + list.push(variant);
209 + Ok(())
210 +}
192 211
193 − let mut targeting: Vec<_> = resources
194 − .iter()
195 − .filter(|m| m.spec.mutant_deployment == mutant.name_any() && m.metadata.deletion_timestamp.is_none())
196 − .collect();
197 − targeting.sort_by(|a, b| {
198 − (a.creation_timestamp(), a.name_any()).cmp(&(b.creation_timestamp(), b.name_any()))
199 − });
200 − for m in targeting {
201 − let def = &m.spec.def;
202 − if let Err(msg) = def.validate() {
203 − out.rejected.push((m.name_any(), MutationState::Invalid, msg));
204 − } else if !labels.insert(def.label.clone()) {
205 − out.rejected.push((
206 − m.name_any(),
207 − MutationState::Conflict,
208 − format!("label {:?} is already used by another mutation of {}", def.label, mutant.name_any()),
209 − ));
210 − } else {
211 − out.active.push((def.clone(), Source::Resource { name: m.name_any() }));
212 − }
213 − }
214 − out
212 +/// Removes the variant labelled `label`, returning it if it was present.
213 +pub fn remove_variant(list: &mut Vec<Variant>, label: &str) -> Option<Variant> {
214 + let idx = list.iter().position(|v| v.label == label)?;
215 + Some(list.remove(idx))
215 216 }
216 217
217 −/// Status of an applied mutation, from its Deployment's state.
218 −pub fn applied_status(name: String, desired: i32, deploy: Option<&Deployment>) -> MutationStatus {
219 − let status = deploy.and_then(|d| d.status.as_ref());
220 − let ready = status.and_then(|s| s.ready_replicas).unwrap_or(0);
218 +/// Status of an applied variant, from its Deployment's state.
219 +pub fn applied_status(label: &str, name: String, desired: i32, deploy: Option<&Deployment>) -> VariantStatus {
220 + let ready = deploy.and_then(|d| d.status.as_ref()).and_then(|s| s.ready_replicas).unwrap_or(0);
221 221 let observed_current = deploy.is_some_and(|d| {
222 222 d.status.as_ref().and_then(|s| s.observed_generation) >= d.metadata.generation
223 223 });
224 − MutationStatus {
225 − state: if observed_current && ready >= desired {
226 − MutationState::Ready
227 − } else {
228 − MutationState::Progressing
229 − },
224 + VariantStatus {
225 + label: label.into(),
226 + state: if observed_current && ready >= desired { VariantState::Ready } else { VariantState::Progressing },
230 227 message: None,
231 228 deployment: Some(name),
232 229 replicas: Some(desired),
@@ -234,23 +231,72 @@ pub fn applied_status(name: String, desired: i32, deploy: Option<&Deployment>) -
234 231 }
235 232 }
236 233
237 −/// Builds the Deployment for one mutation, ready for server-side apply.
234 +/// e.g. "2/3 variants ready".
235 +pub fn summarize(variants: &[VariantStatus]) -> String {
236 + let ready = variants.iter().filter(|v| v.state == VariantState::Ready).count();
237 + format!("{ready}/{} variants ready", variants.len())
238 +}
239 +
240 +/// Markers Argo CD puts on what it syncs, by tracking method: the
241 +/// annotation (Argo CD 3's default) and the label (earlier default).
242 +const ARGO_TRACKING_ANNOTATION: &str = "argocd.argoproj.io/tracking-id";
243 +const ARGO_TRACKING_LABEL: &str = "app.kubernetes.io/instance";
244 +
245 +/// Whether a GitOps tool owns this Mutation, so it changes in git and the
246 +/// API must not change or delete it (the sync would only put it back).
247 +pub fn is_gitops_managed(m: &Mutation) -> bool {
248 + m.annotations().contains_key(ARGO_TRACKING_ANNOTATION) || m.labels().contains_key(ARGO_TRACKING_LABEL)
249 +}
250 +
251 +/// Name of the Secret generated for the Mutation named `mutation`.
252 +pub fn secret_name(mutation: &str) -> String {
253 + format!("{mutation}-secret")
254 +}
255 +
256 +/// The Secret generated for `mutation`: a random `token`, owned by the
257 +/// Mutation so it is garbage-collected with it. Created once and never
258 +/// rewritten, so the token is stable for the Mutation's lifetime.
259 +pub fn build_secret(mutation: &Mutation, owner: OwnerReference, token: String) -> Secret {
260 + Secret {
261 + metadata: ObjectMeta {
262 + name: Some(secret_name(&mutation.name_any())),
263 + namespace: mutation.namespace(),
264 + labels: Some(BTreeMap::from([(SECRET_MUTATION_LABEL.to_string(), mutation.name_any())])),
265 + owner_references: Some(vec![owner]),
266 + ..Default::default()
267 + },
268 + type_: Some("Opaque".into()),
269 + data: Some(BTreeMap::from([(SECRET_TOKEN_KEY.to_string(), ByteString(token.into_bytes()))])),
270 + ..Default::default()
271 + }
272 +}
273 +
274 +/// 32 random bytes from the OS, hex-encoded.
275 +pub fn generate_token() -> Result<String, getrandom::Error> {
276 + let mut bytes = [0u8; 32];
277 + getrandom::fill(&mut bytes)?;
278 + Ok(bytes.iter().map(|b| format!("{b:02x}")).collect())
279 +}
280 +
281 +/// Builds the Deployment for one variant of `base`, ready for server-side
282 +/// apply, owned by `owner` (named `owner_name`) in namespace `ns`.
238 283 pub fn build_deployment(
239 − mutant: &MutantDeployment,
240 − mutation: &MutationDef,
284 + base: &Deployment,
285 + ns: Option<String>,
286 + owner_name: &str,
287 + variant: &Variant,
241 288 owner: OwnerReference,
242 289 ) -> Deployment {
243 − let base = &mutant.spec.deployment;
244 290 let mut labels: BTreeMap<String, String> = base.metadata.labels.clone().unwrap_or_default();
245 − labels.insert(MUTATION_LABEL.into(), mutation.label.clone());
246 − labels.insert(OWNER_LABEL.into(), mutant.name_any());
291 + labels.insert(MUTATION_LABEL.into(), variant.label.clone());
292 + labels.insert(OWNER_LABEL.into(), owner_name.into());
247 293
248 294 let mut deploy = Deployment {
249 295 // Only the user-authored parts of the base metadata are carried over;
250 296 // server-populated fields would make the apply fail.
251 297 metadata: ObjectMeta {
252 − name: Some(mutant.deployment_name(&mutation.label)),
253 − namespace: mutant.namespace(),
298 + name: Some(deployment_name(base, &variant.label)),
299 + namespace: ns,
254 300 labels: Some(labels),
255 301 annotations: base.metadata.annotations.clone(),
256 302 owner_references: Some(vec![owner]),
@@ -261,18 +307,18 @@ pub fn build_deployment(
261 307 };
262 308
263 309 if let Some(spec) = &mut deploy.spec {
264 − spec.replicas = Some(mutation.instances);
310 + spec.replicas = Some(variant.instances);
265 311 // Pod labels (not the selector, which is immutable on existing
266 − // Deployments) so a Service or query can single out one mutation.
312 + // Deployments) so a Service or query can single out one variant.
267 313 spec.template
268 314 .metadata
269 315 .get_or_insert_with(Default::default)
270 316 .labels
271 317 .get_or_insert_with(Default::default)
272 − .insert(MUTATION_LABEL.into(), mutation.label.clone());
318 + .insert(MUTATION_LABEL.into(), variant.label.clone());
273 319
274 320 if let Some(pod) = &mut spec.template.spec {
275 − let mut env: Vec<_> = mutation.env.iter().collect();
321 + let mut env: Vec<_> = variant.env.iter().collect();
276 322 env.sort(); // stable order, so re-applies don't churn the pod template
277 323 for container in &mut pod.containers {
278 324 let vars = container.env.get_or_insert_with(Vec::new);
@@ -295,10 +341,9 @@ pub fn build_deployment(
295 341 #[cfg(test)]
296 342 mod tests {
297 343 use super::*;
298 − use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
299 344
300 − fn mutant(inline: Vec<MutationDef>) -> MutantDeployment {
301 − let deployment: Deployment = serde_json::from_value(serde_json::json!({
345 + fn base() -> Deployment {
346 + serde_json::from_value(serde_json::json!({
302 347 "apiVersion": "apps/v1",
303 348 "kind": "Deployment",
304 349 "metadata": { "name": "nginx", "labels": { "app": "nginx" }, "resourceVersion": "9" },
@@ -314,36 +359,32 @@ mod tests {
314 359 }
315 360 }
316 361 }))
317 − .unwrap();
318 − let mut md = MutantDeployment::new("nginx-mutant", MutantDeploymentSpec { deployment, mutations: inline });
319 − md.metadata.namespace = Some("default".into());
320 − md
362 + .unwrap()
321 363 }
322 364
323 − fn def(label: &str) -> MutationDef {
324 − MutationDef {
365 + fn variant(label: &str) -> Variant {
366 + Variant {
325 367 label: label.into(),
326 368 env: HashMap::from([("MODE".into(), "canary".into()), ("EXTRA".into(), "1".into())]),
327 369 instances: 1,
328 370 }
329 371 }
330 372
331 − fn resource(name: &str, target: &str, label: &str, created_secs: i64) -> Mutation {
332 − let mut m = Mutation::new(name, MutationSpec { mutant_deployment: target.into(), def: def(label) });
333 − m.metadata.creation_timestamp = Some(Time(k8s_openapi::chrono::DateTime::from_timestamp(created_secs, 0).unwrap()));
373 + fn mutation(name: &str) -> Mutation {
374 + let mut m = Mutation::new(name, MutationSpec { deployment: base(), variants: vec![variant("a")] });
375 + m.metadata.namespace = Some("default".into());
334 376 m
335 377 }
336 378
337 379 #[test]
338 380 fn builds_deployment() {
339 − let md = mutant(vec![]);
340 − let d = build_deployment(&md, &def("canary"), OwnerReference::default());
381 + let d = build_deployment(&base(), Some("default".into()), "web", &variant("canary"), OwnerReference::default());
341 382 assert_eq!(d.metadata.name.as_deref(), Some("nginx-canary"));
342 383 assert_eq!(d.metadata.namespace.as_deref(), Some("default"));
343 384 assert_eq!(d.metadata.resource_version, None);
344 385 let labels = d.metadata.labels.unwrap();
345 386 assert_eq!(labels[MUTATION_LABEL], "canary");
346 − assert_eq!(labels[OWNER_LABEL], "nginx-mutant");
387 + assert_eq!(labels[OWNER_LABEL], "web");
347 388 assert_eq!(labels["app"], "nginx");
348 389
349 390 let spec = d.spec.unwrap();
@@ -356,49 +397,35 @@ mod tests {
356 397
357 398 #[test]
358 399 fn validates() {
359 − assert!(def("canary").validate().is_ok());
400 + assert!(variant("canary").validate().is_ok());
360 401 for bad in ["", "Canary", "-a", "a-", "a_b", &"a".repeat(64)] {
361 402 assert!(validate_label(bad).is_err(), "{bad:?}");
362 403 }
363 − let mut m = def("ok");
364 − m.instances = -1;
365 − assert!(m.validate().is_err());
366 − let mut m = def("ok");
367 − m.env.insert("1BAD".into(), "x".into());
368 − assert!(m.validate().is_err());
404 + let mut v = variant("ok");
405 + v.instances = -1;
406 + assert!(v.validate().is_err());
407 + let mut v = variant("ok");
408 + v.env.insert("1BAD".into(), "x".into());
409 + assert!(v.validate().is_err());
410 +
411 + assert!(validate_variants(&[variant("a"), variant("b")]).is_ok());
412 + assert!(validate_variants(&[variant("a"), variant("a")]).is_err());
413 +
414 + assert!(validate_base(&base()).is_ok());
415 + let mut nameless = base();
416 + nameless.metadata.name = None;
417 + assert!(validate_base(&nameless).is_err());
418 + assert!(validate_base(&Deployment::default()).is_err());
369 419 }
370 420
371 421 #[test]
372 − fn resolves_inline_first_then_oldest_resource() {
373 − let md = mutant(vec![def("canary")]);
374 − let mut bad = resource("bad", "nginx-mutant", "Nope", 1);
375 − bad.spec.def.label = "Nope".into();
376 − let resources = [
377 − resource("newer-beta", "nginx-mutant", "beta", 20),
378 − resource("older-beta", "nginx-mutant", "beta", 10),
379 − resource("dup-canary", "nginx-mutant", "canary", 5),
380 − resource("elsewhere", "other-mutant", "gamma", 1),
381 − bad,
382 − ];
383 − let r = resolve(&md, &resources);
384 −
385 − let active: Vec<_> = r.active.iter().map(|(d, s)| (d.label.as_str(), s.clone())).collect();
386 − assert_eq!(
387 − active,
388 − [
389 − ("canary", Source::Inline),
390 − ("beta", Source::Resource { name: "older-beta".into() }),
391 − ]
392 − );
393 − let rejected: Vec<_> = r.rejected.iter().map(|(n, s, _)| (n.as_str(), *s)).collect();
394 − assert_eq!(
395 − rejected,
396 − [
397 − ("bad", MutationState::Invalid),
398 − ("dup-canary", MutationState::Conflict),
399 − ("newer-beta", MutationState::Conflict),
400 − ]
401 − );
422 + fn inserts_and_removes() {
423 + let mut list = vec![variant("a")];
424 + assert!(insert_variant(&mut list, variant("a")).is_err());
425 + insert_variant(&mut list, variant("b")).unwrap();
426 + assert_eq!(remove_variant(&mut list, "a").map(|v| v.label), Some("a".into()));
427 + assert_eq!(remove_variant(&mut list, "a"), None);
428 + assert_eq!(list.len(), 1);
402 429 }
403 430
404 431 #[test]
@@ -408,20 +435,61 @@ mod tests {
408 435 "status": { "observedGeneration": 2, "readyReplicas": 2 }
409 436 }))
410 437 .unwrap();
411 − assert_eq!(applied_status("x".into(), 2, Some(&d)).state, MutationState::Ready);
412 − assert_eq!(applied_status("x".into(), 3, Some(&d)).state, MutationState::Progressing);
438 + assert_eq!(applied_status("x", "d".into(), 2, Some(&d)).state, VariantState::Ready);
439 + assert_eq!(applied_status("x", "d".into(), 3, Some(&d)).state, VariantState::Progressing);
413 440 d.metadata.generation = Some(3); // spec changed, not yet observed
414 − assert_eq!(applied_status("x".into(), 2, Some(&d)).state, MutationState::Progressing);
415 − assert_eq!(applied_status("x".into(), 0, None).state, MutationState::Progressing);
441 + assert_eq!(applied_status("x", "d".into(), 2, Some(&d)).state, VariantState::Progressing);
442 + assert_eq!(applied_status("x", "d".into(), 0, None).state, VariantState::Progressing);
443 +
444 + let ready = applied_status("a", "d".into(), 0, Some(&d));
445 + let rejected = VariantStatus::rejected("b", VariantState::Conflict, "taken".into());
446 + assert_eq!(summarize(&[ready, rejected]), "0/2 variants ready");
447 + }
448 +
449 + #[test]
450 + fn detects_gitops_managed() {
451 + let mut m = mutation("a");
452 + assert!(!is_gitops_managed(&m));
453 + m.annotations_mut().insert(ARGO_TRACKING_ANNOTATION.into(), "crib:neiam.org/Mutation:x/a".into());
454 + assert!(is_gitops_managed(&m));
455 + let mut m = mutation("b");
456 + m.labels_mut().insert(ARGO_TRACKING_LABEL.into(), "crib".into());
457 + assert!(is_gitops_managed(&m));
416 458 }
417 459
418 460 #[test]
419 − fn mutation_wire_format() {
461 + fn builds_secret() {
462 + let m = mutation("browser-snap");
463 + let s = build_secret(&m, OwnerReference { uid: "u1".into(), ..Default::default() }, "abc".into());
464 + assert_eq!(s.metadata.name.as_deref(), Some("browser-snap-secret"));
465 + assert_eq!(s.metadata.namespace.as_deref(), Some("default"));
466 + assert_eq!(s.metadata.owner_references.unwrap()[0].uid, "u1");
467 + assert_eq!(s.metadata.labels.unwrap()[SECRET_MUTATION_LABEL], "browser-snap");
468 + assert_eq!(s.data.unwrap()[SECRET_TOKEN_KEY].0, b"abc");
469 + }
470 +
471 + #[test]
472 + fn tokens_are_random_hex() {
473 + let (a, b) = (generate_token().unwrap(), generate_token().unwrap());
474 + assert_eq!(a.len(), 64);
475 + assert!(a.bytes().all(|c| c.is_ascii_hexdigit()));
476 + assert_ne!(a, b);
477 + }
478 +
479 + #[test]
480 + fn wire_format() {
420 481 let m: MutationSpec = serde_json::from_value(serde_json::json!({
421 − "mutantDeployment": "nginx-mutant", "label": "beta", "env": { "A": "1" }, "instances": 2
482 + "deployment": { "metadata": { "name": "browser-snap" } },
483 + "variants": [{ "label": "lilygo47", "env": { "SNAP_RES": "960x540" }, "instances": 1 }]
484 + }))
485 + .unwrap();
486 + assert_eq!(base_name(&m.deployment), "browser-snap");
487 + assert_eq!(m.variants[0].label, "lilygo47");
488 + // A MutantDeployment's variants keep their `mutations` key.
489 + let md: MutantDeploymentSpec = serde_json::from_value(serde_json::json!({
490 + "deployment": {}, "mutations": [{ "label": "a", "instances": 1 }]
422 491 }))
423 492 .unwrap();
424 − assert_eq!(m.mutant_deployment, "nginx-mutant");
425 − assert_eq!(m.def.label, "beta");
493 + assert_eq!(md.mutations[0].label, "a");
426 494 }
427 495 }

Parents: c8d9978