#!/bin/bash
# Protocol 025: the Mac queue. Fresh network instances pynn_seed 15, 16, 17 (reserve 18), each run at least twice with
# identical inputs and analysed only if two completed runs agree on every field of the export but the run name and the
# run time (compare_025.py). A lean successor of protocol 023 revision 2's queue, same rule and same run command.
#
#   queue_025.sh setup     make ~/felisyn-local/run-025 from run-023r2's model and run files, checked against its list
#   queue_025.sh run       the queue (start the run folder's copy, detached; started again after a stop it goes by its receipts)
#   queue_025.sh smoke     a rehearsal: built seed 6, the 6-step smoke sequence, twice, compared and archived, in run-025-smoke
#
# Per seed: attempts a1, a2, ... (run name m25-sK-aN, export runs/sK-aN.json.gz, receipts runs/sK-aN-*), each under
# memcap (cap 3.7 GiB, disk floor 3 GiB) and timeout 12h. After each completed run its Mozaik folder is archived and the
# export is compared with each earlier completed run of the seed until one agrees: then the seed is "agreed aI aJ" and
# aI's export is the one to analyse. A seed is invalid after four completed runs without an agreeing pair, after two
# counted failures (a non-zero exit, a timeout, an exit 0 without export, an interrupted attempt), after a memory kill
# (exit 99), or when it would need a tenth attempt name. A stop at the disk floor (exit 98) is not counted. One invalid
# slot gets the reserve; if three valid instances are no longer possible the queue exits 1.
# Exit codes: 0 three instances settled; 1 fewer than three possible; 2 refused (changed files, another job); 4 paused.
set -u
MODE=${1:-}
L=<home>/felisyn-local
SRC=$L/run-023r2
W=<workspace>/work/controller-025
case $MODE in smoke) B=$L/run-025-smoke;; *) B=$L/run-025;; esac
V=$L/venv; N=$L/runtime-nm
export PATH="/opt/homebrew/opt/coreutils/libexec/gnubin:$PATH"
MEMCAP=$L/tools/memcap.py
HEAVY='tools/memcap[.]py|run_controller(_mem)?[.]py nest '
R=$B/runs; A=$B/archive
CAP_GIB=3.7; FLOOR_GIB=3; START_KB=$((5 * 1024 * 1024))       # an attempt starts with at least 2 GiB above the floor
REDUCTION=(sheets.l4_cortex_exc.params.sx 2000.0 sheets.retina_lgn.params.size "(3.0,3.0)"
           sheets.l4_cortex_exc.AfferentConnection.or_map_stretch 8.375)
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*"; }
free_kb() { df -k / | awk 'NR==2{print $4}'; }

setup() {
    [ -e "$B" ] && { echo "refused: $B exists"; exit 2; }
    mkdir -p "$B/model-2mm/param_spont" "$B/rec" "$B/seq" "$B/tools" "$R" "$A"
    cp -p $SRC/model-2mm/model.py $SRC/model-2mm/or_map_new_16x16 $SRC/model-2mm/eye_path.pickle "$B/model-2mm/"
    cp -p $SRC/model-2mm/param_spont/* "$B/model-2mm/param_spont/"
    cp -p $SRC/rec/* "$B/rec/"
    cp -p $SRC/run_controller.py $SRC/run_controller_mem.py $SRC/controller_experiments.py $SRC/lgn_cache.py "$B/"
    cp -p $SRC/runtime-sha256.txt "$B/"
    cp -p $W/sequences/*.json "$B/seq/"
    cp -p $W/compare_025.py $W/archive_mozaik_025.py "$B/tools/"
    cp -p $W/queue_025.sh "$B/"                       # the run uses this copy, listed with the queue files
    # the model and run files must be protocol 023 revision 2's, by its own list; then this folder's list is written
    grep -E "  $SRC/(model-2mm|rec)/|  $SRC/(run_controller|run_controller_mem|controller_experiments|lgn_cache)[.]py$" $SRC/server-files-sha256.txt \
        | sed "s#  $SRC/#  $B/#" > "$B/model-files-sha256.txt"
    sha256sum -c --strict --quiet "$B/model-files-sha256.txt" || { echo "refused: the model files are not 023 revision 2's"; exit 2; }
    ( cd "$B" && sha256sum seq/*.json tools/*.py runtime-sha256.txt queue_025.sh > queue-files-sha256.txt )
    echo "set up $B: $(wc -l < "$B/model-files-sha256.txt") model and run files as in 023 revision 2, $(wc -l < "$B/queue-files-sha256.txt") queue files"
}

files_ok() {
    sha256sum -c --strict --quiet "$B/model-files-sha256.txt" > /dev/null 2>&1 && ( cd "$B" && sha256sum -c --strict --quiet queue-files-sha256.txt > /dev/null 2>&1 )
}

wait_clear() {   # no other heavy job, and enough disk; checks every 5 min, pauses after 6 h
    local waited=0
    while :; do
        local p; p=$(pgrep -f "$HEAVY" | grep -v "^$$\$" | tr '\n' ' ')
        local kb; kb=$(free_kb)
        if [ -z "$p" ] && [ "$kb" -ge "$START_KB" ]; then return 0; fi
        [ "$waited" -ge 21600 ] && { log "paused: after 6 h still waiting (heavy job PIDs: ${p:-none}; $kb kB free)"; exit 4; }
        [ $((waited % 1800)) = 0 ] && log "waiting: heavy job PIDs: ${p:-none}; $kb kB free (need $START_KB)"
        sleep 300; waited=$((waited + 300))
    done
}

archive() {   # archive SEED N
    local stem=s$1-a$2
    [ -e "$A/$stem-archive-removed.txt" ] && return 0
    ls -d "$B/model-2mm/FelisynController_m25-s$1-a${2}_____"* > /dev/null 2>&1 || return 0
    python -B "$MEMCAP" --cap-gib $CAP_GIB --min-free-gib 1 --out "$R/$stem-archive-mem" -- \
        timeout 30m python -B "$B/tools/archive_mozaik_025.py" "$B/model-2mm" "m25-s$1-a$2" "$A" "$stem" >> "$R/$stem-archive.log" 2>&1
    log "$stem: archive exit $?"
}

attempt() {   # attempt SEED N SEQUENCE: runs one attempt, writes runs/sK-aN-exit last
    local k=$1 n=$2 seq=$3 name=s$1-a$2 run=m25-s$1-a$2
    wait_clear
    files_ok || { log "refused: a frozen file changed"; exit 2; }
    date -u +%Y-%m-%dT%H:%M:%SZ > "$R/$name-start-utc.txt"
    export FELISYN_SEQUENCE=$seq FELISYN_EXPORT=$R/$name.json.gz
    env | grep -E '^(FELISYN_|PYTHON|MPLBACKEND=|OPENBLAS_|MallocNanoZone=)' | LC_ALL=C sort > "$R/$name-env.txt"
    echo "memcap --cap-gib $CAP_GIB --min-free-gib $FLOOR_GIB timeout -k 10m 12h python ../run_controller_mem.py nest 4 param_spont/defaults ${REDUCTION[*]} pynn_seed $k $run" > "$R/$name-command.txt"
    { sysctl vm.swapusage; df -k /; pmset -g therm; uptime; } > "$R/$name-machine-start.txt" 2>&1
    log "$name: start (free $(free_kb) kB)"
    python -B "$MEMCAP" --cap-gib $CAP_GIB --min-free-gib $FLOOR_GIB --out "$R/$name-mem" -- \
        timeout -k 10m 12h python ../run_controller_mem.py nest 4 param_spont/defaults "${REDUCTION[@]}" pynn_seed "$k" "$run" > "$R/$name.log" 2>&1
    local rc=$?
    unset FELISYN_SEQUENCE FELISYN_EXPORT
    { sysctl vm.swapusage; df -k /; pmset -g therm; uptime; } > "$R/$name-machine-end.txt" 2>&1
    date -u +%Y-%m-%dT%H:%M:%SZ > "$R/$name-end-utc.txt"
    if [ "$rc" = 0 ] && [ ! -f "$R/$name.json.gz" ]; then rc=97; fi           # exit 0 without an export
    if [ "$rc" != 0 ] && [ -e "$R/$name.json.gz" ]; then mv "$R/$name.json.gz" "$R/$name.partial-export"; fi
    [ "$rc" = 0 ] && sha256sum "$R/$name.json.gz" > "$R/$name-export-sha256.txt"
    echo "$rc" > "$R/$name-exit.txt"                 # before the archive: a stop during the archive loses no run
    log "$name: exit $rc"
    archive "$k" "$n"
}

seed() {   # seed K SEQUENCE: runs the paired rule for one seed; writes runs/sK-status.txt
    local k=$1 seq=$2 st=$R/s$1-status.txt
    [ -e "$st" ] && { log "s$k: already settled: $(cat "$st")"; return 0; }
    local completed=() failures=0 n rc
    for n in 1 2 3 4 5 6 7 8 9; do
        local name=s$k-a$n
        if [ -e "$R/$name-start-utc.txt" ] && [ ! -e "$R/$name-exit.txt" ]; then      # interrupted earlier: a counted failure
            wait_clear                                # an orphaned run of a killed queue must end first
            [ -e "$R/$name.json.gz" ] && mv "$R/$name.json.gz" "$R/$name.partial-export"
            echo interrupted > "$R/$name-exit.txt"; log "$name: was interrupted; counted"
        fi
        local fresh=0
        [ -e "$R/$name-exit.txt" ] || { attempt "$k" "$n" "$seq"; fresh=1; }
        rc=$(cat "$R/$name-exit.txt")
        archive "$k" "$n"                             # a no-op once the folder is archived and removed
        case $rc in
            0) local i agreed=
               for i in "${completed[@]:-}"; do
                   [ -z "$i" ] && continue
                   local cmp=$R/s$k-a$i-a$n-compare.json
                   if [ ! -e "$cmp" ]; then
                       python -B "$MEMCAP" --cap-gib $CAP_GIB --min-free-gib 1 --out "$R/s$k-a$i-a$n-compare-mem" -- \
                           timeout 30m python -B "$B/tools/compare_025.py" "$R/s$k-a$i.json.gz" "$R/$name.json.gz" "$cmp" >> "$R/s$k-compare.log" 2>&1
                       local c=$?
                       [ "$c" = 0 ] || [ "$c" = 3 ] || { log "s$k: comparison a$i a$n made no record (exit $c): the queue stops"; rm -f "$cmp"; exit 2; }
                   fi
                   python -c "import json,sys; sys.exit(0 if json.load(open('$cmp'))['agree'] is True else 1)"
                   local g=$?
                   if [ "$g" = 0 ]; then agreed=$i; break; fi
                   [ "$g" = 1 ] || { log "s$k: the comparison record a$i a$n cannot be read: the queue stops"; exit 2; }
                   log "s$k: a$i and a$n disagree"
               done
               if [ -n "$agreed" ]; then echo "agreed a$agreed a$n" > "$st"; log "s$k: agreed a$agreed a$n"; return 0; fi
               completed+=("$n")
               if [ "${#completed[@]}" -ge 4 ]; then echo "invalid: no agreeing pair in four completed runs" > "$st"; log "s$k: $(cat "$st")"; return 0; fi;;
            98) log "$name: stopped at the disk floor; not counted"
                [ "$fresh" = 1 ] && { log "paused after a disk-floor stop: free disk, then start run again"; rm -f "$R/queue.lock"; exit 4; };;
            99) echo "invalid: killed for memory (a$n)" > "$st"; log "s$k: $(cat "$st")"; return 0;;
            *) failures=$((failures + 1))
               if [ "$failures" -ge 2 ]; then echo "invalid: two counted failures" > "$st"; log "s$k: $(cat "$st")"; return 0; fi;;
        esac
    done
    echo "invalid: attempts used up" > "$st"; log "s$k: $(cat "$st")"
}

activate() {
    for v in $(env | sed -n 's/^\(FELISYN_[A-Za-z0-9_]*\)=.*/\1/p'); do unset "$v"; done
    unset PYTHONOPTIMIZE PYTHONHASHSEED PYTHONPATH OMP_NUM_THREADS OMP_WAIT_POLICY
    source "$V/bin/activate" && source "$V/bin/nest_vars.sh" && source "$N/bin/nest_vars.sh" || { echo "refused: cannot activate"; exit 2; }
    export MPLBACKEND=Agg PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=4 MallocNanoZone=0 FELISYN_STEP_FRAMES=30 FELISYN_GAP_FRAMES=21 FELISYN_WARMUP_MS=0 FELISYN_LEAD_MS=504
    cd "$B/model-2mm" || { echo "refused: $B/model-2mm is missing"; exit 2; }
}

case $MODE in
    setup) setup;;
    run)
        [ -d "$R" ] || { echo "refused: run setup first"; exit 2; }
        exec >> "$R/queue.log" 2>&1
        if [ -e "$R/queue.lock" ] && kill -0 "$(cat "$R/queue.lock")" 2>/dev/null; then log "refused: another queue is alive"; exit 2; fi
        echo $$ > "$R/queue.lock"
        activate
        log "queue start (PID $$): $(sw_vers -productVersion) $(sw_vers -buildVersion); free $(free_kb) kB; $(sysctl -n vm.swapusage)"
        for k in 15 16 17; do
            seed $k "$B/seq/sequence-025-s$k.json"
            [ "$(grep -l '^invalid' "$R"/s1[5-7]-status.txt 2>/dev/null | wc -l | tr -d ' ')" -ge 2 ] && break
        done
        bad=$(grep -l '^invalid' "$R"/s15-status.txt "$R"/s16-status.txt "$R"/s17-status.txt 2>/dev/null | wc -l | tr -d ' ')
        if [ "$bad" = 1 ]; then log "one slot is invalid: the reserve, seed 18, runs"; seed 18 "$B/seq/sequence-025-s18.json"; fi
        good=$(grep -l '^agreed' "$R"/s1[5-8]-status.txt 2>/dev/null | wc -l | tr -d ' ')
        rm -f "$R/queue.lock"
        if [ "$good" = 3 ]; then log "queue done: three instances settled with an agreeing pair"; exit 0; fi
        log "queue done: only $good valid instances; three are no longer possible"; exit 1;;
    smoke)
        [ -e "$B" ] && { echo "refused: $B exists"; exit 2; }
        setup > /dev/null; exec >> "$R/queue.log" 2>&1
        activate
        log "smoke start"
        seed 6 "$B/seq/smoke.json"
        log "smoke done: $(cat "$R/s6-status.txt")";;
    *) echo "usage: queue_025.sh setup|run|smoke"; exit 2;;
esac
