"""Protocol 023 revision 2: compare two runs of one instance (the paired rule). Standard library only, Python 3.9.

  python -B compare_runs_023.py CONTRACT.json K EXPORT_I EXPORT_J SERVER_RECORD_I SERVER_RECORD_J OUT.json

Run by run_queue_023.sh, under memcap with a 30 min timeout, from the run folder's copy of the frozen package, after
the server check of attempt J of pynn_seed K, for each earlier completed attempt I of that seed in attempt order, until
one agrees. EXPORT_I and EXPORT_J are runs/sK-aI.json.gz and runs/sK-aJ.json.gz (I < J), SERVER_RECORD_I and _J their
server check records (runs/sK-aN-server-check.json) and OUT runs/sK-aI-aJ-compare.json.

Spec: docs/specs/2026-11-08-protocol-023-r2-paired.md ("What agree means", "The paired rule", "Each check twice, and
each record whole"). Derived from the diagnosis's diag-023/tools/compare_prefix.py, whose validation became its tests.

The two exports agree when, read as JSON values, they are equal in every field but two: the run name (the last element
of argv) and run_seconds (the wall-clock time). So every spike train of every segment, the neuron tables, the stimulus
records, the resolved parameters and seeds, the NEST kernel's records and the rest of argv must be equal. The run name
is left out of the equality, but each export must carry its own attempt's, m23-sK-intact-aN, so that a fault in one
run's name is a disagreement (one more run), never an agreeing pair whose server checks differ (code review of 8
November). It reads spike times and records only, and computes no plaid quantity. A difference that holds a NaN or an
infinity is recorded with that value as text ("NaN", "Infinity", "-Infinity"), so the record stays strict JSON.

Exit codes: 0 the exports agree and their server checks are the same (both passed, or both refused with equal kinds
lists; the refusal texts are never compared, because a 'run' refusal quotes the run name); 4 the exports agree and
their server checks differ; 3 the exports do not agree. Any other exit means no comparison was made (no record), and
the queue stops (exit 2) until resume makes it again.

Before it compares, it stops, writing nothing: unless its running code is the code the contract was made from (the
contract's made_from entry; checked first, so the launcher's pre-flight with absent exports reaches "no export at" only
after it); unless K is a declared instance and the exports are regular files named sK-aI.json.gz and sK-aJ.json.gz with
I < J; and unless each server record is the frozen checker's server record of that export for this contract. If OUT
exists and is this tool's record of these two exports for this contract (the queue was stopped after the comparison),
it prints that record's verdict and exits with it; any other OUT is a stop. A read failure is never a verdict: an
export that cannot be parsed is read a second time, and recorded as unreadable (it agrees with no other export, exit 3)
only when it fails again while its bytes read cleanly with the SHA-256 its server check recorded. An export that
changes while it is compared is a stop.

The record is written whole: a temporary file OUT.tmp-<pid> in OUT's folder, flushed to disk, linked to OUT (which
fails if OUT exists, so it is written once), the folder flushed and the temporary file removed. A power loss leaves the
whole record or none; a leftover temporary file of this tool is removed when it runs again.
Simulation only.
"""
import glob
import gzip
import hashlib
import json
import os
import re
import stat
import sys
import time
import zlib
from pathlib import Path

SELF = 'controller/compare_runs_023.py'
CHECKER = 'controller/check_instance_023.py'
SEQUENCE = 'FullfieldDriftingSinusoidalGratingSequence'
LGN = ('X_ON', 'X_OFF')
AGREE, DISAGREE, CHECKS_DIFFER = 0, 3, 4           # the exit codes the queue reads
LEFT_OUT = ('the run name (argv[-1])', 'run_seconds')
RULE = ('equal as JSON values in every field but the run name (argv[-1]) and run_seconds, and each run name its own '
        "attempt's (m23-sK-intact-aN)")
EXPORT_NAME = re.compile(r'^s(\d+)-a([1-9])\.json\.gz$')
PARSE_FAULTS = (OSError, EOFError, ValueError, zlib.error, RecursionError)
USAGE = ('usage: python -B compare_runs_023.py CONTRACT.json K EXPORT_I EXPORT_J SERVER_RECORD_I SERVER_RECORD_J '
         'OUT.json')


def stop(message):
    """No comparison is made and nothing is written; the comparison can be made again correctly."""
    raise SystemExit('refused: ' + message)


def sha256(path):
    h = hashlib.sha256()
    with open(path, 'rb') as fh:
        for block in iter(lambda: fh.read(1 << 20), b''):
            h.update(block)
    return h.hexdigest()


def utc():
    return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())


# ---------------------------------------------------------------- the frozen code and the inputs

def made_from(contract, rel):
    """The SHA-256 the contract records for a file it was made from, by its path in the package (rel)."""
    entries = contract.get('made_from') if isinstance(contract.get('made_from'), dict) else {}
    hits = [h for key, h in entries.items() if key == rel or key.endswith('/' + rel)]
    if len(hits) != 1 or not isinstance(hits[0], str):
        stop('the contract records no single SHA-256 for ' + rel)
    return hits[0]


def load_contract(path):
    try:
        contract = json.loads(Path(path).read_text(encoding='utf-8'))
    except (OSError, ValueError) as exc:
        stop('the contract {} cannot be read ({}: {})'.format(path, type(exc).__name__, exc))
    if not isinstance(contract, dict):
        stop('the contract {} is not a JSON object'.format(path))
    return contract


def check_running(contract):
    """The running tool is the file the contract was made from."""
    if sha256(__file__) != made_from(contract, SELF):
        stop('the running {} is not the one the contract was made from'.format(SELF))


def export_file(path, k, which):
    """(path, attempt) of an export that must be a regular file named sK-aN.json.gz."""
    path = Path(path)
    m = EXPORT_NAME.match(path.name)
    if not m or int(m.group(1)) != k:
        stop('{} is not the export of attempt {} of pynn_seed {} (sK-aN.json.gz)'.format(path, which, k))
    try:
        st = os.lstat(path)
    except OSError as exc:
        stop('no export at {} ({})'.format(path, type(exc).__name__))
    if not stat.S_ISREG(st.st_mode):
        stop('{} is not a regular file'.format(path))
    return path, int(m.group(2))


def digest_of(path):
    try:
        return sha256(path)
    except OSError as exc:
        stop('{} cannot be read here ({}: {}); nothing is written'.format(path, type(exc).__name__, exc))


def server_record(path, k, export, digest, contract_sha, checker_sha):
    """The server check record of an export: the frozen checker's server record of that very file for this
    contract. Returns {'file', 'sha256', 'valid', 'kinds'}."""
    try:
        rec = json.loads(Path(path).read_text(encoding='utf-8'))
    except (OSError, ValueError) as exc:
        stop('the server check record {} cannot be read ({}: {})'.format(path, type(exc).__name__, exc))
    if not isinstance(rec, dict):
        stop('{} is not a server check record'.format(path))
    if (rec.get('where') != 'server' or rec.get('pynn_seed') != k or rec.get('export') != Path(export).name
            or rec.get('export_sha256') != digest):
        stop('{} is not the server check record of {} (SHA-256 {})'.format(path, Path(export).name, digest))
    if rec.get('contract_sha256') != contract_sha:
        stop('{} was made for another contract'.format(path))
    if rec.get('checker_sha256') != checker_sha:
        stop('{} was made by another checker than the frozen one'.format(path))
    valid, kinds = rec.get('valid'), rec.get('kinds')
    if (not isinstance(valid, bool) or not isinstance(kinds, list) or not all(isinstance(x, str) for x in kinds)
            or valid != (not kinds)):
        stop('{}: its verdict and kinds are not a check record\'s'.format(path))
    return {'file': Path(path).name, 'sha256': sha256(path), 'valid': valid, 'kinds': list(kinds)}


def same_checks(rec_i, rec_j):
    """The two server checks are the same: both passed, or both refused with equal kinds lists. The refusal texts
    are never compared: a 'run' refusal quotes the run name, so two agreeing runs refused for one reason differ in
    their texts. check_instance_023.same_checks is the same definition."""
    if rec_i['valid'] or rec_j['valid']:
        return rec_i['valid'] and rec_j['valid']
    return list(rec_i['kinds']) == list(rec_j['kinds'])


# ---------------------------------------------------------------- reading and comparing

def parse(path):
    with gzip.open(str(path), 'rt', encoding='utf-8') as fh:
        return json.load(fh)


def same_bytes(path, digest, what='compared'):
    try:
        now = sha256(path)
    except OSError as exc:
        stop('{} cannot be read here ({}: {}); nothing is written'.format(path, type(exc).__name__, exc))
    if now != digest:
        stop('{} changed while it was {}; nothing is written'.format(path, what))


def load(path, digest):
    """(export, None), or (None, why) when it cannot be parsed twice while its bytes read cleanly with the checked
    SHA-256; a read fault (the bytes cannot be read, or have changed) is a stop."""
    try:
        return parse(path), None
    except PARSE_FAULTS:
        pass
    same_bytes(path, digest)
    try:                                        # a second read: a fault of the content fails the same way twice
        return parse(path), None
    except PARSE_FAULTS as exc:
        same_bytes(path, digest)
        return None, '{}: {}'.format(type(exc).__name__, exc)


def stripped(ex):
    """The export without the two fields that may differ between agreeing runs."""
    out = dict(ex)
    out.pop('run_seconds', None)
    argv = out.get('argv')
    if isinstance(argv, list) and argv:
        out['argv'] = argv[:-1]
    return out


def run_name(k, n):
    """The declared run name of attempt N of pynn_seed K."""
    return 'm23-s{}-intact-a{}'.format(k, n)


def own_run_name(ex, run):
    """True when the export's run name (argv[-1]) is `run`, its own attempt's."""
    argv = ex.get('argv') if isinstance(ex, dict) else None
    return isinstance(argv, list) and bool(argv) and argv[-1] == run


def agree(a, b):
    """True when the two exports, read as JSON values, are equal but for argv[-1] and run_seconds."""
    if not isinstance(a, dict) or not isinstance(b, dict):
        return False
    if isinstance(a.get('argv'), list) != isinstance(b.get('argv'), list):
        return False
    if isinstance(a.get('argv'), list) and len(a['argv']) != len(b['argv']):
        return False
    return stripped(a) == stripped(b)


def first_diff(x, y):
    """The first time at which two trains differ: the smaller spike at the first differing position, or the next
    spike of the longer train; None if equal."""
    for s, t in zip(x, y):
        if s != t:
            try:
                return min(s, t)
            except TypeError:
                return s
    if len(x) == len(y):
        return None
    return x[len(y)] if len(x) > len(y) else y[len(x)]


def order_key(row):
    t = row[0]
    return (t is None or not isinstance(t, (int, float)), t if isinstance(t, (int, float)) else 0.0, int(row[1])
            if str(row[1]).isdigit() else 0, str(row[1]))


def segment_differences(idx, x, y, names):
    """One segment that differs: its other fields, and per spike train the number that differ, the first differing
    time and the cell ids (all of them in the LGN, the first 20 otherwise, by first differing time)."""
    sheet = x.get('sheet') if isinstance(x, dict) else None
    stim = x.get('stimulus', {}).get('name') if isinstance(x, dict) and isinstance(x.get('stimulus'), dict) else None
    entry = {'index': idx, 'sheet': sheet, 'stimulus': stim}
    if not isinstance(x, dict) or not isinstance(y, dict):
        entry['fields'] = ['the segment itself']
        return entry
    fields = sorted(key for key in set(x) | set(y) if key != 'spikes' and x.get(key) != y.get(key))
    if fields:
        entry['fields'] = fields
    sx, sy = x.get('spikes'), y.get('spikes')
    if sx != sy:
        if not isinstance(sx, dict) or not isinstance(sy, dict):
            entry['fields'] = sorted(set(entry.get('fields', [])) | {'spikes'})
            return entry
        rows = []
        for cell in set(sx) | set(sy):
            a, b = sx.get(cell), sy.get(cell)
            if a == b:
                continue
            if a is None or b is None:
                t = (b if a is None else a)
                rows.append((t[0] if isinstance(t, list) and t else None, cell))
            elif isinstance(a, list) and isinstance(b, list):
                rows.append((first_diff(a, b), cell))
            else:
                rows.append((None, cell))
        rows.sort(key=order_key)
        times = [t for t, _ in rows if isinstance(t, (int, float))]
        keep = rows if sheet in LGN else rows[:20]
        entry.update(differing_trains=len(rows), first_differing_time_ms=min(times) if times else None,
                     differing_cells=[c for _, c in keep],
                     cells_only_in={names[0]: sorted((c for c in sx if c not in sy), key=lambda c: (len(c), c)),
                                    names[1]: sorted((c for c in sy if c not in sx), key=lambda c: (len(c), c))})
        if sheet not in LGN:
            entry['differing_cells_note'] = 'the first 20 by first differing time'
    return entry


def table_differences(a, b):
    """Per sheet and field of the neuron tables: the first differing index and both lengths."""
    out = []
    if not isinstance(a, dict) or not isinstance(b, dict):
        return [{'sheet': None, 'field': 'the tables themselves'}]
    for sheet in sorted(set(a) | set(b)):
        ta, tb = a.get(sheet), b.get(sheet)
        if ta == tb:
            continue
        if not isinstance(ta, dict) or not isinstance(tb, dict):
            out.append({'sheet': sheet, 'field': 'the table itself'})
            continue
        for field in sorted(set(ta) | set(tb)):
            fa, fb = ta.get(field), tb.get(field)
            if fa == fb:
                continue
            row = {'sheet': sheet, 'field': field}
            if isinstance(fa, list) and isinstance(fb, list):
                row['first_differing_index'] = next((i for i, (u, v) in enumerate(zip(fa, fb)) if u != v),
                                                    min(len(fa), len(fb)))
                row['lengths'] = [len(fa), len(fb)]
            out.append(row)
    return out


def outside(ex, end_ms):
    """Every spike of a sequence segment outside [0, end_ms] ms."""
    out = []
    for seg in ex.get('segments') or []:
        if not isinstance(seg, dict) or not isinstance(seg.get('stimulus'), dict):
            continue
        if seg['stimulus'].get('name') != SEQUENCE or not isinstance(seg.get('spikes'), dict):
            continue
        for cell, train in seg['spikes'].items():
            for t in train if isinstance(train, list) else []:
                if isinstance(t, (int, float)) and not 0.0 <= t <= end_ms:
                    out.append({'sheet': seg.get('sheet'), 'id': cell, 'time_ms': t})
    return out


def finite(x):
    """x with every NaN and infinity as text, for a strict JSON record; and whether any was found."""
    if isinstance(x, float) and x != x:
        return 'NaN', True
    if isinstance(x, float) and x in (float('inf'), float('-inf')):
        return ('Infinity' if x > 0 else '-Infinity'), True
    if isinstance(x, dict):
        pairs = [(key, finite(v)) for key, v in x.items()]
        return {key: v for key, (v, _) in pairs}, any(f for _, (_, f) in pairs)
    if isinstance(x, (list, tuple)):
        items = [finite(v) for v in x]
        return [v for v, _ in items], any(f for _, f in items)
    return x, False


def differences(a, b, names, end_ms, runs=None):
    """What differs between two exports that do not agree, for the record (runs: their declared run names)."""
    sa, sb = stripped(a), stripped(b)
    fields = sorted(key for key in set(sa) | set(sb) if sa.get(key) != sb.get(key))
    out = {'fields': fields, 'values': {}}
    if runs is not None and not (own_run_name(a, runs[0]) and own_run_name(b, runs[1])):
        def last(ex):
            argv = ex.get('argv')
            return argv[-1] if isinstance(argv, list) and argv else None
        out['run_names'] = {names[0]: last(a), names[1]: last(b), 'declared': {names[0]: runs[0], names[1]: runs[1]}}
    for key in fields:
        if key not in ('segments', 'neurons'):
            out['values'][key] = {names[0]: sa.get(key), names[1]: sb.get(key)}
    if 'argv' in a and 'argv' in b and isinstance(a['argv'], list) and isinstance(b['argv'], list) \
            and len(a['argv']) != len(b['argv']):
        out['values']['argv'] = {names[0]: a['argv'], names[1]: b['argv']}
        out['fields'] = sorted(set(out['fields']) | {'argv'})
    seg_a, seg_b = a.get('segments'), b.get('segments')
    if seg_a != seg_b and isinstance(seg_a, list) and isinstance(seg_b, list):
        out['segments'] = [segment_differences(i, x, y, names) for i, (x, y) in enumerate(zip(seg_a, seg_b))
                           if x != y]
        if len(seg_a) != len(seg_b):
            out['segment_counts'] = {names[0]: len(seg_a), names[1]: len(seg_b)}

            def tail(segs):
                return [{'index': i, 'sheet': s.get('sheet') if isinstance(s, dict) else None,
                         'stimulus': (s.get('stimulus') or {}).get('name') if isinstance(s, dict) else None}
                        for i, s in enumerate(segs) if i >= min(len(seg_a), len(seg_b))]
            out['segments_only_in'] = {names[0]: tail(seg_a), names[1]: tail(seg_b)}
    if a.get('neurons') != b.get('neurons'):
        out['tables'] = table_differences(a.get('neurons'), b.get('neurons'))
    out['spikes_outside'] = {'window_ms': [0.0, end_ms], names[0]: outside(a, end_ms), names[1]: outside(b, end_ms)}
    return out


def compare(path_i, path_j, digests, names, end_ms, runs):
    """(verdict, unreadable names and reasons, differences or None) for two exports; runs: their declared run names.
    Non-finite numbers in the differences are given as text."""
    a, why_a = load(path_i, digests[0])
    b, why_b = load(path_j, digests[1])
    unreadable = {n: w for n, w in zip(names, (why_a, why_b)) if w is not None}
    if unreadable:
        return 'disagree', unreadable, None
    if agree(a, b) and own_run_name(a, runs[0]) and own_run_name(b, runs[1]):
        return 'agree', {}, None
    try:
        diff, odd = finite(differences(a, b, names, end_ms, runs))
    except (TypeError, ValueError, AttributeError, KeyError, RecursionError) as exc:
        return 'disagree', {}, {'not_computed': '{}: {}'.format(type(exc).__name__, exc)}
    if odd:
        diff['non_finite_as_text'] = True
    return 'disagree', {}, diff


# ---------------------------------------------------------------- the record

def leftovers(out):
    """Remove this tool's leftover temporary files of OUT (a power loss between the write and the link)."""
    for p in glob.glob(glob.escape(str(out)) + '.tmp-*'):
        try:
            os.unlink(p)
        except FileNotFoundError:
            pass


def write_whole(out, text):
    """Write OUT once and whole: a temporary file in its folder, flushed to disk, linked to OUT (which fails if OUT
    exists), the folder flushed, the temporary file removed."""
    out = Path(out)
    tmp = out.with_name('{}.tmp-{}'.format(out.name, os.getpid()))
    fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
    try:
        with os.fdopen(fd, 'w', encoding='utf-8') as fh:
            fh.write(text)
            fh.flush()
            os.fsync(fh.fileno())
        try:
            os.link(str(tmp), str(out))
        except FileExistsError:
            stop('{} exists; a comparison record is never overwritten'.format(out))
        dfd = os.open(str(out.parent), os.O_RDONLY)
        try:
            os.fsync(dfd)
        finally:
            os.close(dfd)
    finally:
        try:
            os.unlink(str(tmp))
        except FileNotFoundError:
            pass


def read_existing(out, expected):
    """OUT already holds this tool's record of these exports for this contract: its exit code; else a stop."""
    try:
        rec = json.loads(Path(out).read_text(encoding='utf-8'))
    except (OSError, ValueError):
        rec = None
    if not isinstance(rec, dict) or any(rec.get(key) != value for key, value in expected.items()) \
            or rec.get('exit') not in (AGREE, DISAGREE, CHECKS_DIFFER):
        stop("{} exists and is not this tool's record of these exports for this contract; a comparison record is "
             'never overwritten'.format(out))
    return rec


def main(args):
    if len(args) != 7:
        raise SystemExit(USAGE)
    contract_path, k, export_i, export_j, record_i, record_j, out = args
    contract = load_contract(contract_path)
    check_running(contract)                     # first: the pre-flight reaches "no export at" only after it
    if not re.fullmatch(r'\d+', k):
        stop('{!r} is not a pynn_seed'.format(k))
    k = int(k)
    instances = contract.get('instances') if isinstance(contract.get('instances'), dict) else {}
    if str(k) not in instances:
        stop('pynn_seed {} is not a declared instance of the contract'.format(k))
    path_i, i = export_file(export_i, k, 'I')
    path_j, j = export_file(export_j, k, 'J')
    if not i < j:
        stop('the exports are attempts a{} and a{}; give the earlier attempt first'.format(i, j))
    out = Path(out)
    leftovers(out)
    names = ('a{}'.format(i), 'a{}'.format(j))
    digests = (digest_of(path_i), digest_of(path_j))
    contract_sha, tool_sha = sha256(contract_path), sha256(__file__)
    checker_sha = made_from(contract, CHECKER)
    servers = (server_record(record_i, k, path_i, digests[0], contract_sha, checker_sha),
               server_record(record_j, k, path_j, digests[1], contract_sha, checker_sha))
    expected = {'tool_sha256': tool_sha, 'contract_sha256': contract_sha, 'pynn_seed': k, 'attempts': [i, j],
                'exports': {n: {'file': p.name, 'sha256': d} for n, p, d in zip(names, (path_i, path_j), digests)}}
    if out.exists() or out.is_symlink():
        rec = read_existing(out, expected)
        print('pynn_seed {}: {} and {} already compared: {} (exit {})'.format(k, names[0], names[1], rec['verdict'],
                                                                             rec['exit']))
        return rec['exit']
    try:
        end_ms = float(contract['sequence']['model_time_ms'])
    except (KeyError, TypeError, ValueError):
        stop("the contract gives no model time (sequence.model_time_ms)")
    verdict, unreadable, diff = compare(path_i, path_j, digests, names, end_ms, (run_name(k, i), run_name(k, j)))
    same_bytes(path_i, digests[0])
    same_bytes(path_j, digests[1])
    rec = {'scope': ('protocol 023 revision 2: the comparison of two runs of one instance (the paired rule); it reads '
                     'spike times and records only and computes no plaid quantity; simulation only'),
           'tool': SELF, **expected, 'rule': RULE, 'left_out': list(LEFT_OUT), 'verdict': verdict,
           'server_records': {n: s for n, s in zip(names, servers)}, 'server_checks': None}
    if verdict == 'agree':
        same = same_checks(*servers)
        rec['server_checks'] = 'same' if same else 'differ'
        rec['exit'] = AGREE if same else CHECKS_DIFFER
    else:
        rec['exit'] = DISAGREE
        rec['unreadable'] = sorted(unreadable)
        rec['unreadable_reasons'] = unreadable
        rec['differences'] = diff
    rec['utc'] = utc()
    write_whole(out, json.dumps(rec, indent=1, allow_nan=False) + '\n')
    print('pynn_seed {}: {} and {}: {}{} (exit {})'.format(
        k, names[0], names[1], verdict,
        '' if verdict != 'agree' else ', server checks ' + rec['server_checks'], rec['exit']))
    return rec['exit']


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))
