#!/bin/bash
# Traces the app container's memory through the sweep, and sheds load before the cgroup OOMs.
#
# Why: on 18 Sep eight collectors exhausted a 1 GB cgroup in six minutes and the kernel killed
# two. The cap is 4 GB now, but collectors GROW as they accumulate, so a ceiling is not a
# guarantee -- it is a deadline. The trace is also the evidence for a real question: how much
# memory a full sweep needs, and so whether crawls belong on this host at all.
set -u
D="$(cd "$(dirname "$0")" && pwd)"
TRACE="$D/memory-trace.tsv"
SHED_MIB=3400          # act before the kernel does
[ -f "$TRACE" ] || printf 'time\tapp_mib\tcollectors\tprices\n' > "$TRACE"

mib() { awk '{printf "%d", $1/1048576}' /sys/fs/cgroup/system.slice/docker-"$1".scope/memory.current 2>/dev/null; }
ID=$(docker inspect dutyfreeprofessor-app --format '{{.Id}}')

while :; do
  m=$(mib "$ID"); [ -z "$m" ] && break          # container gone: the sweep is over
  n=$(docker exec dutyfreeprofessor-app sh -c 'for p in /proc/[0-9]*; do c=$(tr "\0" "\n" < $p/cmdline 2>/dev/null | tail -1); case "$c" in avolta*|extime*) echo x;; esac; done' 2>/dev/null | wc -l)
  p=$(docker exec dutyfreeprofessor-db psql -U dfp -d dfp -tAc \
        "SELECT coalesce(sum(prices_written),0) FROM collection_runs WHERE status='running';" 2>/dev/null | tr -d '[:space:]')
  printf '%s\t%s\t%s\t%s\n' "$(date '+%F %T')" "$m" "$n" "${p:-?}" >> "$TRACE"

  if [ "$m" -gt "$SHED_MIB" ] && [ "$n" -gt 1 ]; then
    # Shed the heaviest collector: it is the one closest to taking the others with it. Its
    # source keeps its committed prices and a re-run supersedes the row it leaves behind.
    victim=$(docker exec dutyfreeprofessor-app sh -c 'for p in /proc/[0-9]*; do c=$(tr "\0" "\n" < $p/cmdline 2>/dev/null | tail -1); r=$(awk "/^VmRSS/{print \$2}" $p/status 2>/dev/null); case "$c" in avolta*|extime*) echo "$r ${p#/proc/} $c";; esac; done' 2>/dev/null | sort -rn | head -1)
    pid=$(echo "$victim" | awk '{print $2}'); slug=$(echo "$victim" | awk '{print $3}')
    if [ -n "$pid" ]; then
      docker exec dutyfreeprofessor-app python -c "import os,signal; os.kill($pid, signal.SIGTERM)" 2>/dev/null
      echo "$(date '+%F %T') SHED $slug (pid $pid) at ${m}MiB" >> "$D/guard.log"
    fi
  fi
  sleep 120
done
echo "$(date '+%F %T') guard ends: container gone" >> "$D/guard.log"