#!/usr/bin/env bash
# Issue 4295 context qualification on one RTX 4090: boot EXACTLY the production engine pod
# (argocd-apps apps/arbi-engines.yaml rendered by `helm template` through
# scripts/ci/fleet_prewarm.py render; image swapped to the released slim-207d3f7 digest),
# outside the cluster. Node paths map to a local copy of the verified production inputs;
# the fleet boot cache maps to a private empty directory (nothing is published to prod's tier).
# Holds the shared GPU lock for the container's whole life.
set -euo pipefail
ROOT=/home/arbi-admin/ctx4295
IMG=${IMG:?}
NAME=ctx4295-engine
PORT=18741
SPEC=$ROOT/engine-spec.json
docker inspect "$NAME" >/dev/null 2>&1 && { echo "container $NAME exists" >&2; exit 2; }
OUT=$ROOT/runs/$(date -u +%Y%m%dT%H%M%SZ)
SCRATCH=$OUT/scratch
mkdir -p "$SCRATCH" "$ROOT/fleet"
python3 "$ROOT/fleet_prewarm.py" docker-args "$SPEC" "$SCRATCH" "$PORT" \
  --map "/mnt/k8scache=$ROOT/k8s-root" --map "/mnt/arbi-engine-cache=$ROOT/fleet" > "$OUT/docker-args.txt"
mapfile -t ARGS < "$OUT/docker-args.txt"
RUN_OPTS=(); IMAGE_ARGS=(); seen=0
for a in "${ARGS[@]}"; do
  if [ $seen = 0 ] && [ "$a" = "--" ]; then seen=1; continue; fi
  if [ $seen = 0 ]; then RUN_OPTS+=("$a"); else IMAGE_ARGS+=("$a"); fi
done
INIT_MOUNTS=(); ROOTS=(); INIT_DIRS=()
while read -r a b; do
  if [ "$a" = mkdir ]; then INIT_DIRS+=("$b"); else ROOTS+=("$b"); INIT_MOUNTS+=(-v "$a:$b"); echo "$a" >> "$OUT/volumes.txt"; docker volume create "$a" >/dev/null; fi
done < <(python3 "$ROOT/fleet_prewarm.py" empty-dirs "$SPEC" "$SCRATCH")
mkdir -p "$ROOT/fleet/flat-weights"
docker run --rm --network none -e NVIDIA_VISIBLE_DEVICES=void --user 0:0 -e DIRS="${INIT_DIRS[*]}" -e OWNER=10001:10001 \
  "${INIT_MOUNTS[@]}" -v "$ROOT/fleet:/fleetroot" --entrypoint sh "$IMG" -c \
  'chmod 0777 "$@"; chown -R 10001:10001 /fleetroot; for d in $DIRS; do mkdir -p "$d" && chown "$OWNER" "$d" || exit 1; done' init "${ROOTS[@]}"
cat "$SCRATCH"/secret-*/tokens | head -1 | tr -d '\n' > "$ROOT/token"; chmod 600 "$ROOT/token"
source "$ROOT/gpu_lock.sh"
GPU_LOCK_WAIT_S=${GPU_LOCK_WAIT_S:-0} gpu_lock 0 || exit 1
echo "$OUT" > "$ROOT/current-run"
nvidia-smi --query-gpu=name,driver_version,memory.total,memory.used --format=csv > "$OUT/gpu-before.csv"
nvidia-smi --query-compute-apps=pid,used_memory --format=csv >> "$OUT/gpu-before.csv"
docker image inspect "$IMG" --format '{{json .RepoDigests}} {{.Id}}' > "$OUT/image.txt"
( while :; do
    printf '%s,%s\n' "$(date -u +%s)" "$(nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits -i 0)"
    sleep 1; done ) > "$OUT/gpu-samples.csv" & SAMPLER=$!
t0=$(date +%s); echo "$t0" > "$OUT/t0.txt"
docker run -d --name "$NAME" --gpus device=0 -p "127.0.0.1:$PORT:$PORT" "${RUN_OPTS[@]}" "$IMG" "${IMAGE_ARGS[@]}" > "$OUT/container-id.txt"
# Stay alive (lock held) until the operator stops the container: docker stop ctx4295-engine
docker wait "$NAME" > "$OUT/exit-code.txt" || true
docker logs "$NAME" > "$OUT/engine.log" 2>&1 || true
docker rm "$NAME" >/dev/null 2>&1 || true
kill "$SAMPLER" 2>/dev/null || true
while read -r v; do docker volume rm "$v" >/dev/null 2>&1 || true; done < "$OUT/volumes.txt"
rm -rf "$SCRATCH"
