#!/usr/bin/env bash
# Deploy this working tree to production. Stream E (E4b); the launch's database replace, Stream K7.
# Run from the dev server:
#
#     deploy/production.sh                # sync + build + up + health check
#     deploy/production.sh --caddy        # also (re)build and reload the droplet's Caddy
#     deploy/production.sh --replace-db backups/launch/staging-<stamp>.dump [--dry-run] [--yes]
#                                         # the launch: production's database becomes staging's
#     deploy/production.sh --replace-db <dump> --force-replace-db
#                                         # a replace AFTER the launch: it discards every collection
#                                         # and every decision taken on live since the launch
#
# A deploy, in order: dumps the production database (always: it is small and a migration may
# ride), rsyncs the code and the read-only inputs over Tailscale, builds and recreates the
# containers there, waits for /api/health, prints the version it reports. It never copies
# .app.env or data/: production keeps its own secrets.
#
# --replace-db <dump> (plan W17; RUNBOOK 'Launch checklist'), every step fatal, in this order:
#   1. read the dump's alembic head and its sidecars (<dump>.counts, .head, .sources, written by
#      deploy/launch-dump.sh); refuse unless the dump, the sidecar and this code's head agree
#   2. dump production to backups/production/pre-replace-<stamp>.dump (the rollback), copied home
#   3. rsync and build the code; stop the app container
#   4. drop and recreate the public schema; pg_restore --no-owner --no-privileges --exit-on-error
#   5. alembic_version must equal the dump's head
#   6. every table's row count must equal the sidecar's (compared before anything writes a row)
#   7. revoke every session (app.cli sessions revoke --all) and expire every unused welcome or
#      reset link (account_tokens.expires_at = now() where unused)
#   8. print the per-source settings beside staging's; confirm (or --yes)
#   9. start, health, the version; the security-audit reminder
# A refusal after step 2 leaves the app stopped and prints the rollback command.
#
# The launch happens once. Step 9 writes backups/production/LAUNCHED here ("launched <stamp> from
# <dump>") and a copy in the droplet's backups/, and from then on --replace-db refuses at the top,
# before step 1 and before any ssh, naming that file and --force-replace-db. Rian's rule: a push to
# live never syncs collection data again. A second replace would discard every collection and every
# decision taken on live since the launch, so it is a typed flag, not a prompt. --dry-run reports
# the marker the same way and writes nothing. A --local rehearsal is never blocked: it replaces a
# scratch database here, not live.
#
# Rehearsal (dfp-devdb, never production): --local --db-container dfp-devdb --db dfp_prod_like
# --app-exec "<python -m app.cli with DATABASE_URL on that database>" runs the database half here,
# with no ssh, rsync, build, stop or start. --dry-run prints every command and runs none.
set -euo pipefail

HOST="${DFP_PROD_HOST:-deploy@bwlive}"        # Tailscale name; set DFP_PROD_HOST to override
REMOTE_DIR=/srv/apps/dutyfreeprofessor
PORT=3149
COMPOSE="docker compose -f docker-compose.yml -f docker-compose.production.yml"
CADDY=0
REPLACE_DB=""
LOCAL=0
DRY=0
YES=0
FORCE_REPLACE=0
CONTAINER=dutyfreeprofessor-db
DB=dfp
APP_EXEC=""
PG_READER="${DFP_PG_READER:-dfp-devdb}"       # a local container with pg_restore, to read a dump's head
while [ $# -gt 0 ]; do
  case "$1" in
    --caddy) CADDY=1; shift ;;
    --replace-db) REPLACE_DB="${2:-}"; [ -n "$REPLACE_DB" ] || { echo "--replace-db needs a dump file" >&2; exit 2; }; shift 2 ;;
    --local) LOCAL=1; shift ;;
    --dry-run) DRY=1; shift ;;
    --force-replace-db) FORCE_REPLACE=1; shift ;;
    --yes) YES=1; shift ;;
    --db-container) CONTAINER="${2:-}"; shift 2 ;;
    --db) DB="${2:-}"; shift 2 ;;
    --app-exec) APP_EXEC="${2:-}"; shift 2 ;;
    --seed-db) echo "--seed-db is removed: it swallowed restore errors. Use --replace-db (RUNBOOK 'Launch checklist')." >&2; exit 2 ;;
    -h|--help) sed -n '2,42p' "$0"; exit 0 ;;
    *) echo "unknown argument: $1" >&2; exit 2 ;;
  esac
done
if [ "$LOCAL" = 1 ] && [ -z "$REPLACE_DB" ]; then
  echo "--local rehearses --replace-db only; a code deploy has no local form" >&2; exit 2
fi
if [ "$LOCAL" = 1 ] && [ "$CONTAINER" = dutyfreeprofessor-db ]; then
  echo "REFUSED: --local against dutyfreeprofessor-db would replace staging; rehearse on dfp-devdb (--db-container dfp-devdb)" >&2; exit 2
fi
if [ -n "$REPLACE_DB" ] && [ "$CADDY" = 1 ]; then
  echo "--replace-db and --caddy are separate steps; run the Caddy deploy on its own" >&2; exit 2
fi
if [ "$FORCE_REPLACE" = 1 ] && [ -z "$REPLACE_DB" ]; then
  echo "--force-replace-db modifies --replace-db; on its own it does nothing" >&2; exit 2
fi

cd "$(dirname "$0")/.."
here=$(pwd)
LAUNCH_MARKER=backups/production/LAUNCHED   # written by step 9; --replace-db refuses while it exists
say() { printf '\n== %s\n' "$*"; }
remote() { ssh -o BatchMode=yes -o ConnectTimeout=15 "$HOST" "$@"; }
# A command on the host whose database is being replaced: this machine in a rehearsal, else the droplet.
on_target() { if [ "$LOCAL" = 1 ]; then bash -c "$1"; else remote "$1"; fi; }
# A step that changes something: printed always, run unless --dry-run.
run() { printf '  $ %s\n' "$1"; [ "$DRY" = 1 ] && return 0; on_target "$1"; }
refuse() { echo "REFUSED: $*" >&2; exit 1; }
# set -e stops on a failing command without a word; say which line, so no failure is ever silent.
trap 'echo "FAILED: line $LINENO of deploy/production.sh exited non-zero" >&2' ERR

if [ -n "$REPLACE_DB" ]; then
  # The launch happens once. Before step 1, before any ssh: a second replace discards everything
  # live has collected and decided since, so it is refused until a person types the flag.
  # A rehearsal (--local) replaces a scratch database here, not live, and is never blocked.
  if [ "$LOCAL" = 0 ] && [ "$FORCE_REPLACE" = 0 ] && [ -e "$LAUNCH_MARKER" ]; then
    {
      echo "REFUSED: $here/$LAUNCH_MARKER says the launch already happened:"
      sed 's/^/    /' "$LAUNCH_MARKER"
      echo "  A push to live carries code and migrations, never data again. A second --replace-db"
      echo "  discards every collection and every decision taken on live since the launch."
      echo "  If that is what you mean, with the rollback dump in hand: --force-replace-db."
    } >&2
    exit 1
  fi
  [ -f "$REPLACE_DB" ] || refuse "no dump at $REPLACE_DB"
  for side in counts head sources; do
    [ -s "$REPLACE_DB.$side" ] || refuse "$REPLACE_DB.$side is missing: make the dump with deploy/launch-dump.sh, which writes it"
  done
  [ "$LOCAL" = 1 ] && [ -z "$APP_EXEC" ] && refuse "--local needs --app-exec (the app CLI on the rehearsal database)"
  if [ -z "$APP_EXEC" ]; then
    APP_EXEC="cd $REMOTE_DIR && $COMPOSE run --rm --no-deps app python -m app.cli"
  fi
  DBX="docker exec -i $CONTAINER"
  STAMP=$(date -u +%Y-%m-%d-%H%M%S)   # to the second: a rerun must never overwrite the rollback it took before

  say "1. the dump, its sidecars and this code agree on the schema"
  dump_head=$(docker exec -i "$PG_READER" pg_restore -f - --data-only -t alembic_version < "$REPLACE_DB" \
    | awk '/^COPY public.alembic_version/ {on=1; next} /^\\\./ {on=0} on {print $1}') \
    || refuse "could not read $REPLACE_DB with the container $PG_READER (one with pg_restore; set DFP_PG_READER): is it a pg_dump -Fc file?"
  side_head=$(tr -d '[:space:]' < "$REPLACE_DB.head")
  code_head=$(cd main && ../.venv-dev/bin/alembic heads 2>/dev/null | awk '{print $1}' | head -1)
  echo "  dump $dump_head · sidecar $side_head · code $code_head"
  [ -n "$dump_head" ] || refuse "the dump carries no alembic_version rows"
  [ "$dump_head" = "$side_head" ] || refuse "the dump's head $dump_head differs from its sidecar's $side_head: the sidecar belongs to another dump"
  [ "$dump_head" = "$code_head" ] || refuse "the dump's head $dump_head differs from this code's head $code_head: deploy staging with this code and dump again"
  echo "  $(wc -l < "$REPLACE_DB.counts") tables, $(awk -F'\t' '{s += $2} END {print s}' "$REPLACE_DB.counts") rows in the sidecar"

  if [ "$LOCAL" = 0 ]; then
    say "preflight"
    if remote "pgrep -f 'app.cli (collect|verify)' >/dev/null 2>&1"; then
      refuse "a collection or verify is running on $HOST; freeze collections first"
    fi
    remote "test -f $REMOTE_DIR/.app.env" || refuse "production .app.env missing on $HOST"
    remote "grep -c REPLACE_WITH_ $REMOTE_DIR/.app.env || true" | grep -q '^0$' || refuse "production .app.env still has placeholders"
  fi

  say "2. the rollback: dump the database being replaced"
  if [ "$LOCAL" = 1 ]; then
    rollback_dir=backups/rehearsal
    rollback="$rollback_dir/pre-replace-$STAMP.dump"
    run "mkdir -p $rollback_dir && test ! -e $rollback && $DBX pg_dump -U dfp -Fc $DB > $rollback && test -s $rollback"
    target_dump="$here/$REPLACE_DB"
  else
    rollback="backups/production/pre-replace-$STAMP.dump"
    run "mkdir -p $REMOTE_DIR/backups && test ! -e $REMOTE_DIR/backups/pre-replace-$STAMP.dump && $DBX pg_dump -U dfp -Fc $DB > $REMOTE_DIR/backups/pre-replace-$STAMP.dump && test -s $REMOTE_DIR/backups/pre-replace-$STAMP.dump"
    printf '  $ rsync %s:%s/backups/pre-replace-%s.dump %s\n' "$HOST" "$REMOTE_DIR" "$STAMP" "$rollback"
    if [ "$DRY" = 0 ]; then
      mkdir -p backups/production
      rsync -az "$HOST:$REMOTE_DIR/backups/pre-replace-$STAMP.dump" "$rollback"
      test -s "$rollback" || refuse "the rollback dump did not arrive at $rollback"
    fi
    target_dump="$REMOTE_DIR/backups/replace.dump"
  fi
  # The restore runs where the database is: the droplet reads its own copy of the rollback dump.
  rollback_at="$rollback"
  where=""
  if [ "$LOCAL" = 0 ]; then rollback_at="$REMOTE_DIR/backups/pre-replace-$STAMP.dump"; where=" on $HOST"; fi
  rollback_line="restore the rollback$where: $DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -c 'drop schema public cascade; create schema public;' && $DBX pg_restore -U dfp -d $DB --no-owner --no-privileges --exit-on-error < $rollback_at, then redeploy the previous tag (RUNBOOK 'Launch checklist', rollback)"
  trap 'status=$?; [ $status -ne 0 ] && echo "ROLLBACK: $rollback_line" >&2; exit $status' EXIT

  if [ "$LOCAL" = 0 ]; then
    say "3. code, build, stop the app"
    printf '  $ rsync main/ import/ public/ uploads/ compose files -> %s\n' "$HOST"
    if [ "$DRY" = 0 ]; then
      rsync -az --delete --exclude node_modules --exclude __pycache__ --exclude .pytest_cache --exclude .mypy_cache \
        --exclude web/dist --exclude static --exclude '*.pyc' "$here/main/" "$HOST:$REMOTE_DIR/main/"
      rsync -az --delete "$here/import/" "$HOST:$REMOTE_DIR/import/"
      rsync -az --delete "$here/public/" "$HOST:$REMOTE_DIR/public/"
      rsync -az "$here/uploads/" "$HOST:$REMOTE_DIR/uploads/"
      rsync -az "$here/docker-compose.yml" "$here/docker-compose.production.yml" "$here/.env" "$HOST:$REMOTE_DIR/"
      rsync -az "$REPLACE_DB" "$HOST:$target_dump"
    fi
    run "cd $REMOTE_DIR && $COMPOSE build app"
    run "cd $REMOTE_DIR && $COMPOSE stop app"
  else
    say "3. (rehearsal: no code, build or app container)"
  fi

  say "4. replace the schema and restore, errors fatal"
  run "$DBX psql -U dfp -d $DB -q -v ON_ERROR_STOP=1 -c 'set client_min_messages = warning; drop schema public cascade; create schema public;'"
  run "$DBX pg_restore -U dfp -d $DB --no-owner --no-privileges --exit-on-error < $target_dump"

  if [ "$DRY" = 1 ]; then
    say "5 to 9 (dry run): the checks read the restored database, so they print only"
    echo "  $ $DBX psql -U dfp -d $DB -tA -c 'select version_num from alembic_version'   # must be $dump_head"
    echo "  $ (every table's count(*) in public) compared with $REPLACE_DB.counts"
    echo "  $ $APP_EXEC sessions revoke --all"
    echo "  $ $DBX psql -U dfp -d $DB -c \"update account_tokens set expires_at = now() where used_at is null and expires_at > now()\""
    echo "  $ (sources printed beside $REPLACE_DB.sources; confirm)"
    [ "$LOCAL" = 0 ] && echo "  $ cd $REMOTE_DIR && $COMPOSE up -d --remove-orphans; curl /api/health"
    [ "$LOCAL" = 0 ] && echo "  $ (write $LAUNCH_MARKER here and copy it to $HOST:$REMOTE_DIR/backups/LAUNCHED)"
    trap - EXIT
    say "dry run: nothing ran"
    exit 0
  fi

  say "5. the restored schema"
  restored_head=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -c 'select version_num from alembic_version'" | tr -d '[:space:]')
  echo "  alembic_version $restored_head"
  [ "$restored_head" = "$dump_head" ] || refuse "the restored head $restored_head is not the dump's $dump_head"

  say "6. row counts against the sidecar"
  tables=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -c \"select table_name from information_schema.tables where table_schema = 'public' and table_type = 'BASE TABLE' order by table_name\"")
  sql=""
  for t in $tables; do sql+="${sql:+ union all }select '$t', count(*) from public.\\\"$t\\\""; done
  here_counts=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -F \$'\\t' -c \"$sql\"" | sort)
  if ! diff_out=$(diff <(sort "$REPLACE_DB.counts") <(printf '%s\n' "$here_counts")); then
    echo "  table counts differ (< the sidecar, > the restored database):" >&2
    printf '%s\n' "$diff_out" | sed 's/^/    /' >&2
    refuse "the restored database does not hold what staging held at dump time"
  fi
  echo "  $(printf '%s\n' "$here_counts" | wc -l) tables, $(printf '%s\n' "$here_counts" | awk -F'\t' '{s += $2} END {print s}') rows: equal"

  say "7. every session revoked, every unused link expired"
  if [ "$LOCAL" = 1 ]; then
    (cd main && bash -c "$APP_EXEC sessions revoke --all")
  else
    remote "$APP_EXEC sessions revoke --all"
  fi
  expired=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -c \"with e as (update account_tokens set expires_at = now() where used_at is null and expires_at > now() returning 1) select count(*) from e\"")
  echo "  expired $(echo "$expired" | tr -d '[:space:]') unused welcome or reset link(s)"
  live=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -c \"select count(*) from sessions where revoked_at is null and expires_at > now()\"" | tr -d '[:space:]')
  [ "$live" = 0 ] || refuse "$live session(s) still live after the revoke"

  say "8. sources: this database (left) beside the staging list the dump was taken with (right)"
  here_sources=$(on_target "$DBX psql -U dfp -d $DB -v ON_ERROR_STOP=1 -tA -F \$'\\t' -c \"select slug, enabled, delay_seconds, identity_mode, coalesce(permission_record, '') from sources order by slug\"")
  paste <(printf '%s\n' "$here_sources" | awk -F'\t' '{printf "%-28s %s %5s %s\n", $1, $2, $3, $4}') \
        <(awk -F'\t' '{printf "%-28s %s %5s %s\n", $1, $2, $3, $4}' "$REPLACE_DB.sources") | sed 's/^/  /'
  if [ "$(printf '%s\n' "$here_sources")" != "$(cat "$REPLACE_DB.sources")" ]; then
    refuse "the sources differ from the staging list; read the two columns above"
  fi
  if [ "$YES" = 0 ]; then
    read -r -p "  These are the sources production will collect with. Type yes to start: " answer
    [ "$answer" = yes ] || refuse "not confirmed; the app stays stopped"
  fi

  trap - EXIT
  if [ "$LOCAL" = 1 ]; then
    say "done (rehearsal): replaced $CONTAINER/$DB from $REPLACE_DB; rollback dump $rollback"
    exit 0
  fi
  say "9. start and health"
  remote "cd $REMOTE_DIR && $COMPOSE up -d --remove-orphans"
  for i in $(seq 1 30); do
    if body=$(remote "curl -s --max-time 5 http://172.17.0.1:$PORT/api/health"); then echo "  $body"; break; fi
    sleep 2
    [ "$i" = 30 ] && { echo "health never answered" >&2; remote "cd $REMOTE_DIR && docker compose logs --tail 50 app"; echo "ROLLBACK: $rollback_line" >&2; exit 1; }
  done
  say "the launch marker: --replace-db refuses from now on"
  mkdir -p "$(dirname "$LAUNCH_MARKER")"
  printf 'launched %s from %s\n' "$STAMP" "$REPLACE_DB" > "$LAUNCH_MARKER"
  echo "  $here/$LAUNCH_MARKER"
  if rsync -az "$LAUNCH_MARKER" "$HOST:$REMOTE_DIR/backups/LAUNCHED"; then
    echo "  $HOST:$REMOTE_DIR/backups/LAUNCHED"
  else
    echo "  WARNING: the marker did not reach $HOST:$REMOTE_DIR/backups/LAUNCHED; copy it by hand" >&2
  fi

  say "done: production holds staging's database; rollback dump $rollback"
  echo "  Next: srv-gw security-audit on the dev server, then the checklist's next line (RUNBOOK 'Launch checklist')."
  exit 0
fi

say "preflight"
git status --porcelain | grep -q . && echo "note: working tree has uncommitted changes" || true
# A deploy recreates the app container and kills any collection or verify running there.
# Refuse while one is running; the window file in .logs/runs/ says when it ends.
if remote "pgrep -f 'app.cli (collect|verify)' >/dev/null 2>&1"; then
  echo "REFUSED: a collection or verify is running on $HOST (pgrep app.cli). A deploy would kill it." >&2
  echo "Wait for the open window in .logs/runs/window-*.md to say finished, then deploy." >&2
  exit 1
fi
remote "test -f $REMOTE_DIR/.app.env" || { echo "production .app.env missing on $HOST (stage it: RUNBOOK 'Production')" >&2; exit 1; }
remote "grep -c REPLACE_WITH_ $REMOTE_DIR/.app.env || true" | grep -q '^0$' || { echo "production .app.env still has placeholders" >&2; exit 1; }

if remote "docker ps --format '{{.Names}}' | grep -q '^dutyfreeprofessor-db$'"; then
  say "dump production database first"
  remote "mkdir -p $REMOTE_DIR/backups && docker exec dutyfreeprofessor-db pg_dump -U dfp -Fc dfp > $REMOTE_DIR/backups/dfp-\$(date -u +%Y-%m-%d-%H%M)-pre-deploy.dump"
fi

say "sync code and inputs"
rsync -az --delete \
  --exclude node_modules --exclude __pycache__ --exclude .pytest_cache --exclude .mypy_cache \
  --exclude web/dist --exclude static --exclude '*.pyc' \
  "$here/main/" "$HOST:$REMOTE_DIR/main/"
rsync -az --delete "$here/import/" "$HOST:$REMOTE_DIR/import/"
rsync -az --delete "$here/public/" "$HOST:$REMOTE_DIR/public/"
rsync -az "$here/uploads/" "$HOST:$REMOTE_DIR/uploads/"     # additive: production may hold newer uploads
rsync -az "$here/docker-compose.yml" "$here/docker-compose.production.yml" "$here/.env" "$HOST:$REMOTE_DIR/"

if [ "$CADDY" = 1 ]; then
  say "caddy"
  rsync -az "$here/deploy/caddy/" --exclude cloudflare.env.example "$HOST:/srv/caddy/"
  remote "test -f /srv/caddy/cloudflare.env" || { echo "/srv/caddy/cloudflare.env missing on $HOST" >&2; exit 1; }
  remote "cd /srv/caddy && docker compose up -d --build && docker exec caddy caddy reload --config /etc/caddy/Caddyfile"
fi

say "build and recreate"
remote "cd $REMOTE_DIR && $COMPOSE up -d --build --remove-orphans"

say "health"
for i in $(seq 1 30); do
  if body=$(remote "curl -s --max-time 5 http://172.17.0.1:$PORT/api/health"); then
    echo "$body"; break
  fi
  sleep 2
  [ "$i" = 30 ] && { echo "health never answered" >&2; remote "cd $REMOTE_DIR && docker compose logs --tail 50 app"; exit 1; }
done
say "done"