#!/bin/bash
# Full-network feasibility probe on the existing host: unchanged model with the spontaneous
# recorder recipe, 16 virtual processes split as RANKS x THREADS, NoStimulation shortened to
# PROBE_MS. Samples host memory, swap and load every 15 s. Run as the research user, normally
# inside a systemd transient unit with a memory ceiling so the rest of the host stays responsive.
set -uo pipefail
RANKS=${1:?ranks}; THREADS=${2:?threads}; PROBE_MS=${3:?probe ms}
source /opt/felisyn/venv/bin/activate
source /opt/felisyn/venv/bin/nest_vars.sh
OUT=/opt/felisyn/feasibility-010/full-${RANKS}x${THREADS}-${PROBE_MS}ms
mkdir -p "$OUT/results"
export MPLBACKEND=Agg PYTHONHASHSEED=0 OMPI_MCA_mpi_yield_when_idle=1 OMP_NUM_THREADS=$THREADS
export OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1
export FELISYN_MODEL=/opt/felisyn/protocol-006/model FELISYN_OUT=$OUT/probe.json FELISYN_PROBE_MS=$PROBE_MS
( echo 'epoch,memavailable_kb,swapfree_kb,felisyn_rss_sum_kb,load1,pswpin,pswpout'
  while true; do
    printf '%s,%s,%s,%s,%s,%s,%s\n' "$(date -u +%s)" "$(awk '/^MemAvailable/{print $2}' /proc/meminfo)" \
      "$(awk '/^SwapFree/{print $2}' /proc/meminfo)" "$(ps -u felisyn -o rss= | awk '{s+=$1} END{print s+0}')" \
      "$(cut -d' ' -f1 /proc/loadavg)" "$(awk '/^pswpin/{print $2}' /proc/vmstat)" "$(awk '/^pswpout/{print $2}' /proc/vmstat)"
    sleep 15
  done ) > "$OUT/resources.csv" &
SAMPLER=$!
date -u +%Y-%m-%dT%H:%M:%SZ > "$OUT/start-utc.txt"
ARGS=(/opt/felisyn/feasibility-010/probe_feasibility.py nest "$THREADS" param_spont/defaults results_dir "'$OUT/results/'" trial 0 felisyn_feasibility_probe)
if [ "$RANKS" = 1 ]; then
  /usr/bin/time -v -o "$OUT/time.txt" python "${ARGS[@]}" > "$OUT/probe.log" 2>&1
else
  /usr/bin/time -v -o "$OUT/time.txt" mpirun --oversubscribe -np "$RANKS" python "${ARGS[@]}" > "$OUT/probe.log" 2>&1
fi
echo "exit=$?" > "$OUT/exit.txt"
date -u +%Y-%m-%dT%H:%M:%SZ > "$OUT/end-utc.txt"
kill "$SAMPLER"
