neiam
/fieri-forever
the flavortown lunch engine
public · Issues · Pulls · Labels · Forks · Compare · Actions success · Badges · Packages
⭐
Log in to mark this repository.
db-transfer: copy dev data into prod (just db-export / db-import)
904aa0c · Gabriel Morell · 2026-10-29 01:27
Files changed
modified
.gitignore
+1
−0
@@ -1,2 +1,3 @@
| 1 | 1 | /target |
| 2 | 2 | storage/ |
| 3 | +*.dump |
modified
Justfile
+10
−0
@@ -72,3 +72,13 @@ logs:
| 72 | 72 | # Mint an invite inside the running webservice container |
| 73 | 73 | invite-in-pod days="7": |
| 74 | 74 | {{compose}} exec webservice app-registration-token create --days {{days}} |
| 75 | + | |
| 76 | +# --- dev -> prod data (see scripts/db-transfer.sh) --- | |
| 77 | + | |
| 78 | +# Dump the dev DB, minus test accounts and dev-only rows. Usage: just db-export [file] | |
| 79 | +db-export file="": | |
| 80 | + scripts/db-transfer.sh export {{file}} | |
| 81 | + | |
| 82 | +# Replace prod's DB with a dump (backs prod up first, asks to confirm) | |
| 83 | +db-import file: | |
| 84 | + scripts/db-transfer.sh import {{file}} |
modified
README.md
+14
−0
@@ -250,6 +250,20 @@ runbook in its README.
| 250 | 250 | On SIGTERM the server stops accepting, drains for up to 5 s, then exits — |
| 251 | 251 | open live-vote streams don't end by themselves, and browsers reconnect. |
| 252 | 252 | |
| 253 | +### Copying dev data into prod | |
| 254 | + | |
| 255 | +```sh | |
| 256 | +just db-export fieri.dump # from the dev container; dev itself is untouched | |
| 257 | +just db-import fieri.dump # into the cluster, with your kubectl context | |
| 258 | +``` | |
| 259 | + | |
| 260 | +The export leaves out sessions, device tokens and invites, plus the | |
| 261 | +alice/bob/carol/dave test accounts and the "Boston office" test area | |
| 262 | +(`DROP_USERS` / `DROP_AREAS` change that; `""` keeps all). The import saves | |
| 263 | +prod's current data to `prod-backup-*.dump` first, asks you to type the | |
| 264 | +namespace, stops the app, swaps the database in and starts the app again. | |
| 265 | +A backup restores the same way: `just db-import prod-backup-….dump`. | |
| 266 | + | |
| 253 | 267 | ## Containers |
| 254 | 268 | |
| 255 | 269 | ```sh |
added
scripts/db-transfer.sh
+165
−0
@@ -0,0 +1,165 @@
| 1 | +#!/usr/bin/env bash | |
| 2 | +# Copy the dev database into prod, so prod starts with your areas, | |
| 3 | +# restaurants, ranks and history instead of re-entering them by hand. | |
| 4 | +# | |
| 5 | +# scripts/db-transfer.sh export [file] dev Postgres -> dump file | |
| 6 | +# scripts/db-transfer.sh import <file> dump file -> prod (REPLACES it) | |
| 7 | +# | |
| 8 | +# export works on a throwaway copy inside the dev container, so the dev | |
| 9 | +# database itself is never changed. In the copy it: | |
| 10 | +# - empties sessions, device tokens and invites (they belong to dev: | |
| 11 | +# cookies for localhost, tokens for this machine, links to dev URLs); | |
| 12 | +# - deletes DROP_USERS (default: the alice/bob/carol/dave test accounts, | |
| 13 | +# whose password is public) and DROP_AREAS (default: the "Boston office" | |
| 14 | +# test area). Restaurants, lunches and outings they created stay, with | |
| 15 | +# the creator left blank. Set either to "" to keep everything. | |
| 16 | +# | |
| 17 | +# import first saves a dump of whatever prod has now (prod-backup-*.dump, | |
| 18 | +# restorable with this same command), then stops the app, replaces the | |
| 19 | +# fieri_eternal database with the file, and starts the app again; its boot | |
| 20 | +# migrations run as usual. It asks before touching anything unless --yes. | |
| 21 | +# | |
| 22 | +# Environment: | |
| 23 | +# DEV_CONTAINER dev Postgres container (fieri-eternal-dev-pg) | |
| 24 | +# DEV_USER, DEV_DB (app_user, app) | |
| 25 | +# ENGINE podman or docker (podman) | |
| 26 | +# KUBE_CONTEXT kubectl context for prod (kubectl's current one) | |
| 27 | +# NAMESPACE prod namespace (fieri-eternal) | |
| 28 | +set -euo pipefail | |
| 29 | + | |
| 30 | +DEV_CONTAINER=${DEV_CONTAINER:-fieri-eternal-dev-pg} | |
| 31 | +DEV_USER=${DEV_USER:-app_user} | |
| 32 | +DEV_DB=${DEV_DB:-app} | |
| 33 | +ENGINE=${ENGINE:-podman} | |
| 34 | +NAMESPACE=${NAMESPACE:-fieri-eternal} | |
| 35 | +PROD_DB=fieri_eternal | |
| 36 | +DEPLOYMENT=fieri-eternal | |
| 37 | +CLUSTER=postgres | |
| 38 | +DROP_USERS=${DROP_USERS-alice bob carol dave} | |
| 39 | +DROP_AREAS=${DROP_AREAS-Boston office} | |
| 40 | + | |
| 41 | +die() { echo "error: $*" >&2; exit 1; } | |
| 42 | +say() { echo "==> $*" >&2; } | |
| 43 | +dev() { "$ENGINE" exec -i "$DEV_CONTAINER" "$@"; } | |
| 44 | +kc() { kubectl ${KUBE_CONTEXT:+--context "$KUBE_CONTEXT"} -n "$NAMESPACE" "$@"; } | |
| 45 | + | |
| 46 | +# Single-quoted SQL literal list: "a b" -> 'a','b' | |
| 47 | +sql_list() { | |
| 48 | + local out="" w | |
| 49 | + for w in "$@"; do out+="${out:+,}'${w//\'/\'\'}'"; done | |
| 50 | + echo "$out" | |
| 51 | +} | |
| 52 | + | |
| 53 | +cmd_export() { | |
| 54 | + local file=${1:-fieri-$(date +%Y%m%d-%H%M%S).dump} | |
| 55 | + local stage=fieri_export_stage | |
| 56 | + [[ -e $file ]] && die "$file already exists" | |
| 57 | + | |
| 58 | + say "copying $DEV_DB into a staging database in $DEV_CONTAINER" | |
| 59 | + dev dropdb -U "$DEV_USER" --if-exists "$stage" | |
| 60 | + dev createdb -U "$DEV_USER" "$stage" | |
| 61 | + # Expanded now: $stage is local, and gone by the time EXIT fires. | |
| 62 | + trap "dev dropdb -U '$DEV_USER' --if-exists '$stage' >/dev/null 2>&1 || true" EXIT | |
| 63 | + dev pg_dump -U "$DEV_USER" -Fc "$DEV_DB" | dev pg_restore -U "$DEV_USER" -d "$stage" --no-owner | |
| 64 | + | |
| 65 | + local users areas | |
| 66 | + # shellcheck disable=SC2086 # word-split usernames on purpose | |
| 67 | + users=$(sql_list $DROP_USERS) | |
| 68 | + areas=$( [[ -n $DROP_AREAS ]] && sql_list "$DROP_AREAS" || true) | |
| 69 | + say "pruning: dev-only rows${users:+; users $DROP_USERS}${areas:+; area \"$DROP_AREAS\"}" | |
| 70 | + dev psql -U "$DEV_USER" -d "$stage" -v ON_ERROR_STOP=1 -q <<SQL | |
| 71 | +BEGIN; | |
| 72 | +TRUNCATE user_sessions, api_tokens, registration_token_areas, registration_tokens; | |
| 73 | +${areas:+DELETE FROM areas WHERE name IN ($areas);} | |
| 74 | +${users:+ | |
| 75 | +CREATE TEMP TABLE gone AS SELECT id FROM users WHERE username IN ($users); | |
| 76 | +UPDATE areas SET created_by = NULL WHERE created_by IN (SELECT id FROM gone); | |
| 77 | +UPDATE restaurants SET added_by = NULL WHERE added_by IN (SELECT id FROM gone); | |
| 78 | +UPDATE lunches SET logged_by = NULL WHERE logged_by IN (SELECT id FROM gone); | |
| 79 | +UPDATE outings SET created_by = NULL WHERE created_by IN (SELECT id FROM gone); | |
| 80 | +UPDATE area_invitations SET invited_by = NULL WHERE invited_by IN (SELECT id FROM gone); | |
| 81 | +DELETE FROM users WHERE id IN (SELECT id FROM gone); | |
| 82 | +} | |
| 83 | +COMMIT; | |
| 84 | +SQL | |
| 85 | + | |
| 86 | + dev pg_dump -U "$DEV_USER" -Fc --no-owner --no-privileges "$stage" > "$file" | |
| 87 | + say "wrote $file. It holds:" | |
| 88 | + dev psql -U "$DEV_USER" -d "$stage" -At -F ': ' <<'SQL' >&2 | |
| 89 | +SELECT 'users', string_agg(username, ', ' ORDER BY username) FROM users | |
| 90 | +UNION ALL SELECT 'areas', string_agg(name, ', ' ORDER BY name) FROM areas | |
| 91 | +UNION ALL SELECT 'restaurants', count(*)::text FROM restaurants | |
| 92 | +UNION ALL SELECT 'ranks', count(*)::text FROM rankings | |
| 93 | +UNION ALL SELECT 'lunches', count(*)::text FROM lunches; | |
| 94 | +SQL | |
| 95 | +} | |
| 96 | + | |
| 97 | +# The StackGres primary: the pod labelled role=master (older StackGres) or | |
| 98 | +# role=primary, else the only pod there is. | |
| 99 | +primary_pod() { | |
| 100 | + local sel="stackgres.io/cluster-name=$CLUSTER" role pod | |
| 101 | + for role in master primary; do | |
| 102 | + pod=$(kc get pods -l "$sel,role=$role" -o name 2>/dev/null | head -1) | |
| 103 | + [[ -n $pod ]] && { echo "${pod#pod/}"; return; } | |
| 104 | + done | |
| 105 | + pod=$(kc get pods -l "$sel" -o name | head -1) | |
| 106 | + [[ -n $pod ]] || die "no Postgres pod for SGCluster $CLUSTER in $NAMESPACE" | |
| 107 | + echo "${pod#pod/}" | |
| 108 | +} | |
| 109 | + | |
| 110 | +cmd_import() { | |
| 111 | + local file=${1:-} yes=${2:-} | |
| 112 | + [[ -n $file ]] || die "usage: $0 import <file> [--yes]" | |
| 113 | + [[ -f $file ]] || die "$file not found" | |
| 114 | + pg_restore --list "$file" >/dev/null 2>&1 \ | |
| 115 | + || { dev pg_restore --list < "$file" >/dev/null || die "$file isn't a pg_dump custom-format file"; } | |
| 116 | + | |
| 117 | + local pod; pod=$(primary_pod) | |
| 118 | + local ctx; ctx=$(kubectl ${KUBE_CONTEXT:+--context "$KUBE_CONTEXT"} config current-context) | |
| 119 | + pg() { kc exec -i "$pod" -c patroni -- "$@"; } | |
| 120 | + | |
| 121 | + echo "This REPLACES database $PROD_DB in $ctx/$NAMESPACE (pod $pod) with $file." >&2 | |
| 122 | + if [[ $yes != --yes ]]; then | |
| 123 | + read -r -p "Type the namespace ($NAMESPACE) to go ahead: " answer | |
| 124 | + [[ $answer == "$NAMESPACE" ]] || die "not confirmed" | |
| 125 | + fi | |
| 126 | + | |
| 127 | + local exists backup="" | |
| 128 | + exists=$(pg psql -U postgres -Atc "SELECT 1 FROM pg_database WHERE datname='$PROD_DB'") | |
| 129 | + if [[ $exists == 1 ]]; then | |
| 130 | + backup=prod-backup-$(date +%Y%m%d-%H%M%S).dump | |
| 131 | + say "backing up prod to $backup" | |
| 132 | + pg pg_dump -U postgres -Fc "$PROD_DB" > "$backup" | |
| 133 | + fi | |
| 134 | + | |
| 135 | + local replicas | |
| 136 | + replicas=$(kc get deployment "$DEPLOYMENT" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 1) | |
| 137 | + [[ $replicas == 0 ]] && replicas=1 | |
| 138 | + say "stopping the app" | |
| 139 | + kc scale deployment "$DEPLOYMENT" --replicas=0 | |
| 140 | + kc wait --for=delete pod -l name="$DEPLOYMENT" --timeout=60s 2>/dev/null || true | |
| 141 | + # If anything below fails, still bring the app back up (on the backup's | |
| 142 | + # data, which is still there unless the restore got partway). | |
| 143 | + trap "say 'starting the app again'; kc scale deployment '$DEPLOYMENT' --replicas='$replicas' >/dev/null" EXIT | |
| 144 | + | |
| 145 | + say "replacing $PROD_DB" | |
| 146 | + pg psql -U postgres -v ON_ERROR_STOP=1 -q -o /dev/null <<SQL | |
| 147 | +SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$PROD_DB' AND pid <> pg_backend_pid(); | |
| 148 | +DROP DATABASE IF EXISTS $PROD_DB; | |
| 149 | +CREATE DATABASE $PROD_DB; | |
| 150 | +SQL | |
| 151 | + pg pg_restore -U postgres -d "$PROD_DB" --no-owner --no-privileges --exit-on-error < "$file" | |
| 152 | + | |
| 153 | + pg psql -U postgres -d "$PROD_DB" -At -F ': ' >&2 <<'SQL' | |
| 154 | +SELECT 'users', string_agg(username, ', ' ORDER BY username) FROM users | |
| 155 | +UNION ALL SELECT 'areas', string_agg(name, ', ' ORDER BY name) FROM areas | |
| 156 | +UNION ALL SELECT 'restaurants', count(*)::text FROM restaurants; | |
| 157 | +SQL | |
| 158 | + say "done${backup:+; the old prod data is in $backup}" | |
| 159 | +} | |
| 160 | + | |
| 161 | +case ${1:-} in | |
| 162 | + export) shift; cmd_export "$@" ;; | |
| 163 | + import) shift; cmd_import "$@" ;; | |
| 164 | + *) sed -n '2,/^set -e/p' "$0" | sed '$d; s/^# \{0,1\}//' >&2; exit 2 ;; | |
| 165 | +esac |
Parents: e6f490a