#!/bin/bash
# Runs inside a clean Ubuntu 24.04 cloud-image userland. Follows the execution-003
# fresh-host recipe and the protocol-006 preparation commands as documented.
# No full-model simulation is started: the launcher is run in preflight mode only.
set -uo pipefail
export DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8
LOG=/var/log/felisyn-rehearsal
mkdir -p "$LOG"
STEPS=$LOG/steps.tsv
printf 'step\tstart_utc\tend_utc\tseconds\texit_code\texpected\n' > "$STEPS"

record() {  # record NAME START_EPOCH START_UTC CODE EXPECTED
  local end
  end=$(date -u +%s)
  printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$1" "$3" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$((end-$2))" "$4" "$5" >> "$STEPS"
  echo "$1 exit=$4 expected=$5 seconds=$((end-$2))"
}

step() {  # step NAME COMMAND... ; the rehearsal stops at the first unexpected failure
  local name=$1 start start_utc code
  shift
  start=$(date -u +%s); start_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  "$@" > "$LOG/$name.log" 2>&1
  code=$?
  record "$name" "$start" "$start_utc" "$code" 0
  if [ "$code" -ne 0 ]; then echo "STOPPED at $name"; package; exit "$code"; fi
}

expect_code() {  # expect_code NAME EXPECTED COMMAND... ; for checks designed to refuse this host
  local name=$1 expected=$2 start start_utc code
  shift 2
  start=$(date -u +%s); start_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  "$@" > "$LOG/$name.log" 2>&1
  code=$?
  record "$name" "$start" "$start_utc" "$code" "$expected"
  if [ "$expected" = nonzero ] && [ "$code" -ne 0 ]; then return 0; fi
  if [ "$code" = "$expected" ]; then return 0; fi
  echo "STOPPED at $name: unexpected exit $code"; package; exit 3
}

package() {  # copy receipts into one archive; called on success or at the first failure
  local out=/tmp/rehearsal-receipts
  rm -rf "$out"; mkdir -p "$out/protocol-006"
  cp -a "$LOG" "$out/logs"
  [ -d /opt/felisyn/evidence ] && cp -a /opt/felisyn/evidence "$out/build-evidence"
  # The NEST source export is reproducible from its pinned commit; keep only its recorded hash.
  find "$out" -name '*.tar' -delete
  if [ -z "${RUN:-}" ]; then RUN=$(ls -1dt /opt/felisyn/runs/*/ 2>/dev/null | head -n 1); RUN=${RUN%/}; fi
  if [ -n "${RUN:-}" ] && [ -d "$RUN" ]; then
    mkdir -p "$out/regression-run"
    find "$RUN" -maxdepth 1 -type f -exec cp -a {} "$out/regression-run/" \;
    raw="$RUN/framework/tests/full_model/models/LSV1M_tiny_stepcurrentmodule/LSV1M_pytest_____"
    [ -d "$raw" ] && cp -a "$raw" "$out/regression-run/raw-output"
  fi
  for f in runtime-check.json compatibility-check.json; do
    [ -f "/opt/felisyn/protocol-006/$f" ] && cp -a "/opt/felisyn/protocol-006/$f" "$out/protocol-006/"
  done
  tar -C /tmp -czf /tmp/rehearsal-receipts.tar.gz rehearsal-receipts
  sha256sum /tmp/rehearsal-receipts.tar.gz > /tmp/rehearsal-receipts.tar.gz.sha256
}

step userland-before bash -c 'cat /etc/os-release; echo; echo "installed packages: $(dpkg-query -W | wc -l)"; echo "nproc: $(nproc)"; grep -E "MemTotal|MemAvailable" /proc/meminfo'

# execution-003 README, "Fresh-host recipe", as root in the copied package directory.
cd /tmp/felisyn-reproduction
step bootstrap-host bash bootstrap-host.sh
step copy-locks cp requirements-resolved-candidate.txt requirements-transitive.lock.txt /tmp/
step build-python runuser -u felisyn -- bash /tmp/felisyn-reproduction/build-python.sh
step fetch-sources runuser -u felisyn -- bash /tmp/felisyn-reproduction/fetch-runtime-sources.sh
step build-runtime runuser -u felisyn -- bash /tmp/felisyn-reproduction/build-runtime.sh
step install-harness install -o felisyn -g felisyn -m 644 run_regression.py inspect_regression.py \
  plot_regression.py capture_environment.py /opt/felisyn/

# execution-003 README, "Re-run on the existing research host", as felisyn.
ENVSETUP='export PATH=/opt/felisyn/venv/bin:$PATH; export MPLBACKEND=Agg; source /opt/felisyn/venv/bin/nest_vars.sh'
step regression runuser -u felisyn -- bash -c "$ENVSETUP; cd /opt/felisyn; python /opt/felisyn/run_regression.py --checkout /opt/felisyn/sources/mozaik --output /opt/felisyn/runs"
RUN=$(tail -n 1 "$LOG/regression.log" | python3 -c 'import json,sys; print(json.load(sys.stdin)["run_directory"])')
step inspect-regression runuser -u felisyn -- bash -c "$ENVSETUP; python /opt/felisyn/inspect_regression.py --run '$RUN'"
step plot-regression runuser -u felisyn -- bash -c "$ENVSETUP; python /opt/felisyn/plot_regression.py --run '$RUN'"
step capture-environment runuser -u felisyn -- bash -c "$ENVSETUP; cd /opt/felisyn; python /opt/felisyn/capture_environment.py"

# protocol-006 README, "Environment and preflight", as felisyn. Preflight must refuse this host.
step install-protocol bash -c 'install -d -o felisyn -g felisyn /opt/felisyn/protocol-006 && cp -a /tmp/protocol-006/. /opt/felisyn/protocol-006/ && chown -R felisyn:felisyn /opt/felisyn/protocol-006'
P6='source /opt/felisyn/venv/bin/activate; source /opt/felisyn/venv/bin/nest_vars.sh; export MPLBACKEND=Agg; cd /opt/felisyn/protocol-006'
step protocol-unittest runuser -u felisyn -- bash -c "$P6; python -m unittest -v test_contract.py test_metrics.py"
step protocol-runtime runuser -u felisyn -- bash -c "$P6; python verify_runtime.py --sources /opt/felisyn/sources --output runtime-check.json"
step protocol-compatibility runuser -u felisyn -- bash -c "$P6; python check_compatibility.py --model /opt/felisyn/protocol-006/model --output compatibility-check.json"
step full-runs-dir runuser -u felisyn -- mkdir -p /opt/felisyn/full-runs
expect_code protocol-preflight 2 runuser -u felisyn -- bash -c "$P6; python launch.py --output /opt/felisyn/full-runs"

# The identity checks that launch.py --execute performs after its hardware gate, run directly.
step execute-identity-checks runuser -u felisyn -- bash -c "$P6; python - <<'EOF'
import hashlib, json, pathlib, subprocess
root = pathlib.Path('/opt/felisyn/protocol-006')
c = json.loads((root/'run-contract.json').read_text())
ok = True
for name, digest in c['model_files'].items():
    good = hashlib.sha256((root/'model'/name).read_bytes()).hexdigest() == digest
    ok &= good
    if not good: print('changed model asset', name)
print('model files checked:', len(c['model_files']))
for s in c['source_commits']:
    d = '/opt/felisyn/sources/' + s['directory']
    head = subprocess.check_output(['git', '-C', d, 'rev-parse', 'HEAD'], text=True).strip()
    dirty = subprocess.check_output(['git', '-C', d, 'status', '--porcelain', '--untracked-files=no'], text=True).strip()
    print(s['directory'], head, 'match' if head == s['commit'] else 'MISMATCH', 'clean' if not dirty else 'DIRTY')
    ok &= head == s['commit'] and not dirty
raise SystemExit(0 if ok else 1)
EOF"

# Open MPI slot behaviour on this host: 16 ranks must be refused on 4 cores; 4 ranks must start.
step cpu-topology bash -c 'lscpu; echo; nproc'
expect_code mpi-16-ranks-on-4-cores nonzero runuser -u felisyn -- bash -c "$ENVSETUP; mpirun -np 16 hostname"
step mpi-4-rank-smoke runuser -u felisyn -- bash -c "$ENVSETUP; mpirun -np 4 python -c 'from mpi4py import MPI; import nest; c = MPI.COMM_WORLD; print(c.Get_rank(), c.Get_size(), nest.__version__)'"

package
echo "REHEARSAL COMPLETE"
