"""Protocol 022 revision 2 (plaids pilot): the analysis of the intact and plain-lesion runs against the contract.

  python3 analyze_plaids.py CONTRACT.json OUT_DIR intact=EXPORT.json.gz|FAILED lesion=EXPORT.json.gz|FAILED [--not-frozen]

NAME=FAILED records that a run failed twice (one rerun under the identical contract, named -a2, is allowed).
Spec: docs/specs/2026-10-26-protocol-022-plaids-r2.md. Simulation only: no wallet, key or order exists in this code.

It stops with a message and writes nothing unless it is the declared analysis, started as declared:
  - CONTRACT.json is a file named contract.json, OUT_DIR holds no results.json (a results file is never
    overwritten), and every export given exists;
  - a manifest.json next to the contract lists contract.json, controller/analyze_plaids.py,
    controller/plaid_stats.py, controller/frozen017.py and the contract's sequence file with their SHA-256, and the
    running files are those (--not-frozen skips this, for synthetic tests only);
  - protocol 017's contract, decoder and imported code (analyze_decisions, decisions, pipeline) match 017's own
    manifest, and the contract's protocol_017_contract_sha256 and protocol_017_manifest_sha256 are 017's.

Every other check writes results.json, scoped as the spec's decision rule says:
  - NOT INTERPRETABLE: a refusal on the contract (unreadable, or without sequence.file, sequence.sha256 and the two
    017 pins), on the sequence (not its SHA-256 in the contract, or not the declared design: the warm-up [45 deg,
    30 %] first, then the 15 conditions with their repeats; plaids with both contrasts above 0 and at most 50 %,
    different orientations and a sum of at most 100 %; no unknown keys) or on the intact export (an unreadable or
    partial one included); the intact run failed twice; a primary not estimable; the intact run fails its 30 % check.
  - The primary stands and only the connection readings are missing: the lesion run failed twice, a refusal on the
    lesion export, the lesion fails its 30 % check, or the LGN spike trains or neuron tables differ between the runs.
A refusal makes the exit non-zero after results.json is written.

check_run_plaids refuses an export unless it has one presentation of the declared steps (step_entry's rounding,
first_step 0), exactly one sequence segment per sheet, starting at 0 ms, ending at the presentation's duration and
holding a train of finite spike times for every neuron of the sheet's table and no other; step 30, gap 21 and frame
7, a 72-frame lead-in, 0.8 cpd and 2 Hz, no separate warm-up (InternalStimulus), the LGN cache recorded as disabled,
the command line of 017's engine and condition (intact: 017's intact-A; lesion: its five feedforward-only overrides)
with a run name matching ^m22p-(intact|lesion)-a[12]$, and the resolved weights and seeds of 017's contract entry.
An export that cannot be read (a truncated or corrupt gzip, invalid JSON, a missing key) is refused with the reason.

Counting reuses 017's frozen code unchanged: load_run, decoder_setup, counts_searchsorted in every window, and
counts_by_step on a fresh read of the export as the independent recount, which must be equal; the readout
z = sum(c * w) uses pipeline.decoder_weights (balanced, all layer 4 excitatory cells) and is checked against
population_vector_weighted's angle and strength, and the vectorised decision side against request. LGN spike trains
are compared with lgn_digests and lgn_identity. plaid_stats then gives every measure, interval, p-value, reading and
text; results.json holds them all, with the Python, NumPy and SciPy versions.
"""
import hashlib, json, math, platform, re, sys, zlib
sys.dont_write_bytecode = True                  # before frozen017 is imported: no __pycache__ beside frozen code
from pathlib import Path
import numpy as np
import scipy

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import frozen017  # noqa: E402
FROZEN = Path(frozen017.add_to_path())
import analyze_decisions as a  # noqa: E402  (frozen 017 code, imported unchanged)
import decisions as d  # noqa: E402
import pipeline as p  # noqa: E402
import plaid_stats as ps  # noqa: E402

RUNS = {'intact': 'intact-A', 'lesion': 'feedforward-only'}      # each run and its entry in 017's contract
RUN_NAME = re.compile(r'^m22p-(intact|lesion)-a[12]$')
STEP_FRAMES, GAP_FRAMES, FRAME_MS, LEAD_FRAMES = 30, 21, 7.0, 72
SPATIAL_FREQUENCY_CPD, TEMPORAL_FREQUENCY_HZ = 0.8, 2.0
SHEETS = {'V1_Exc_L4': 'L4E', 'V1_Inh_L4': 'L4I', 'V1_Exc_L2/3': 'L23E', 'V1_Inh_L2/3': 'L23I', 'X_ON': 'X_ON', 'X_OFF': 'X_OFF'}
WINDOWS = {'V1_Exc_L4': ('G', 'D', 'onset', 'sustained', 'offset', 'step')}       # other sheets: G and step
READOUT_WINDOWS = ('D', 'G', 'onset', 'sustained', 'offset')
CONTRACT_NAME = 'contract.json'
CONTRACT_PINS = ('protocol_017_contract_sha256', 'protocol_017_manifest_sha256')
FROZEN_CODE = (CONTRACT_NAME, 'controller/analyze_plaids.py', 'controller/plaid_stats.py', 'controller/frozen017.py')
FROZEN_017 = ('contract.json', 'decoder.json', 'controller/analyze_decisions.py', 'controller/decisions.py',
              'controller/pipeline.py')
STEP_KEYS = {'orientation_rad', 'contrast', 'orientation2_rad', 'contrast2', 'condition', 'block', 'quarter', 'analysed'}
DECLARED_GROUP_CELLS = {45: 915, 90: 721, 135: 1071}                 # instance A (spec); reported, not a refusal
READ_ERRORS = (OSError, EOFError, ValueError, KeyError, TypeError, AttributeError, IndexError, zlib.error,
               OverflowError, RecursionError)       # an infinite number or a deeply nested document, too
GEOMETRY_NOTE = ('From the neuron table alone: the cosine of each plaid pattern\'s relative phase at the cell\'s '
                 'receptive-field centre (x_um and y_um at 1,000 um per degree, imagen\'s SineGrating phase); where it '
                 'is above 0 the two components add in phase. Block A depends only on y, and not on its sign; block B '
                 'also depends on the directions of the x and y axes, which no check here confirms are imagen\'s.')


class Refusal(Exception):
    """An input is not the declared one: see the module docstring for what each refusal does to the outcome."""


def refuse(message):
    raise Refusal(message)


def stop(message):
    raise SystemExit(f'refused: {message}')


def sha256(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


# ---------------------------------------------------------------- the declared analysis and inputs

def check_frozen(base, allow_unfrozen=False):
    """In the frozen package the contract and this code must be the manifest's files (the sequence file follows in
    check_frozen_sequence, once the contract names it). Returns the record and the manifest's files (or None)."""
    manifest = base / 'manifest.json'
    if not manifest.exists():
        if not allow_unfrozen:
            stop('no manifest.json next to the contract (use --not-frozen only for synthetic tests)')
        print('no manifest.json next to the contract: inputs are not a frozen package', flush=True)
        return {'manifest': False}, None
    try:
        listed = {f['path']: f['sha256'] for f in json.loads(manifest.read_text())['files']}
    except (OSError, ValueError, KeyError, TypeError) as exc:
        stop(f'manifest.json cannot be read ({type(exc).__name__}: {exc})')
    for rel in FROZEN_CODE:
        if rel not in listed or not (base / rel).is_file() or sha256(base / rel) != listed[rel]:
            stop(f'{rel} differs from the frozen manifest')
    running = {'controller/analyze_plaids.py': Path(__file__), 'controller/plaid_stats.py': Path(ps.__file__),
               'controller/frozen017.py': Path(frozen017.__file__)}
    for rel, path in running.items():
        if sha256(path) != listed[rel]:
            stop(f'the running {rel} is not the frozen one')
    return {'manifest': True, 'manifest_sha256': sha256(manifest), 'files_checked': list(FROZEN_CODE)}, listed


def check_frozen_sequence(base, contract, frozen, listed):
    """In the frozen package the sequence file the contract names must be the manifest's."""
    if listed is None:
        return
    rel = contract['sequence']['file']
    if rel not in listed or not (base / rel).is_file() or sha256(base / rel) != listed[rel]:
        stop(f'the sequence file {rel} differs from the frozen manifest')
    frozen['files_checked'].append(rel)


def read_contract(path):
    """The contract, or a refusal on it: it must be a JSON object with sequence.file (a relative path),
    sequence.sha256 and the SHA-256 pins of protocol 017's contract and manifest."""
    try:
        contract = json.loads(Path(path).read_text())
    except (OSError, ValueError) as exc:
        refuse(f'the contract cannot be read ({type(exc).__name__}: {exc})')
    if not isinstance(contract, dict):
        refuse('the contract is not a JSON object')
    seq = contract.get('sequence')
    if not (isinstance(seq, dict) and isinstance(seq.get('file'), str) and isinstance(seq.get('sha256'), str)):
        refuse('the contract needs sequence.file and sequence.sha256')
    rel = Path(seq['file'])
    if rel.is_absolute() or '..' in rel.parts:
        refuse(f"the contract's sequence file {seq['file']!r} is not a path inside the package")
    for key in CONTRACT_PINS:
        if not isinstance(contract.get(key), str):
            refuse(f'the contract needs {key}')
    return contract


def check_017(contract):
    """Protocol 017's contract and decoder, after checking them and the imported 017 code against 017's manifest,
    and 017's contract and manifest against the contract's pins."""
    base = FROZEN.parent
    manifest = base / 'manifest.json'
    listed = {f['path']: f['sha256'] for f in json.loads(manifest.read_text())['files']}
    for rel in FROZEN_017:
        if sha256(base / rel) != listed.get(rel):
            stop(f'protocol 017 {rel} differs from its frozen manifest')
    for module, rel in ((a, 'controller/analyze_decisions.py'), (d, 'controller/decisions.py'), (p, 'controller/pipeline.py')):
        if Path(module.__file__).resolve() != (base / rel).resolve():
            stop(f'imported {module.__name__} is not protocol 017 {rel}')
    if contract['protocol_017_contract_sha256'] != listed['contract.json']:
        stop("the contract's protocol_017_contract_sha256 is not protocol 017's contract")
    manifest_sha = sha256(manifest)
    if contract['protocol_017_manifest_sha256'] != manifest_sha:
        stop("the contract's protocol_017_manifest_sha256 is not protocol 017's manifest")
    c017 = json.loads((base / 'contract.json').read_text())
    if sha256(base / 'decoder.json') != c017['decoder']['sha256']:
        stop("protocol 017's decoder.json does not match its contract")
    decoder = json.loads((base / 'decoder.json').read_text())
    if (decoder['sheets'] != ['V1_Exc_L4'] or decoder['mode'] != 'balanced' or decoder.get('centre_um')
            or list(decoder['window_ms']) != list(ps.WINDOWS_MS['D'])):
        stop(f"protocol 017's decoder {decoder} is not the declared readout")
    return c017, decoder, {'package': str(base), 'contract_sha256': listed['contract.json'], 'manifest_sha256': manifest_sha,
                           'decoder_sha256': listed['decoder.json'], 'files_checked': list(FROZEN_017)}


def step_entry(s):
    """A step of the sequence file as version 3 shows it (version 2's rounding; four numbers for a plaid)."""
    entry = [round(float(s['orientation_rad']), 6), round(float(s['contrast']), 3)]
    if 'orientation2_rad' in s or 'contrast2' in s:
        entry += [round(float(s['orientation2_rad']), 6), round(float(s['contrast2']), 3)]
    return entry


def step_problem(entry):
    if not all(isinstance(v, float) and math.isfinite(v) for v in entry):
        return 'numbers must be finite'
    if len(entry) == 2:
        return None if 0.0 <= entry[1] <= 100.0 else 'a single needs a contrast of 0 to 100 %'
    o1, c1, o2, c2 = entry
    if not (c1 > 0 and c2 > 0):
        return 'a plaid needs both contrasts above 0 %'
    if c1 + c2 > 100.0:
        return 'a plaid needs contrasts summing to at most 100 %'
    if c1 > 50.0 or c2 > 50.0:
        return 'a plaid needs each contrast at most 50 %'
    if ps.angdiff(o1, o2) < math.radians(0.001):
        return 'a plaid needs different orientations'
    return None


def check_sequence(seq):
    """The steps as shown and the declared condition of each analysed step, or a refusal on the sequence: the warm-up
    [45 deg, 30 %] first, then exactly the declared conditions and repeats."""
    steps = seq.get('steps') if isinstance(seq, dict) else None
    if not isinstance(steps, list) or len(steps) != 1 + ps.N_ANALYSED:
        refuse(f'the sequence must hold {1 + ps.N_ANALYSED} steps')
    shown = []
    for k, s in enumerate(steps):
        if not isinstance(s, dict):
            refuse(f'step {k} is not an object')
        unknown = sorted(set(s) - STEP_KEYS)
        if unknown:
            refuse(f'step {k} has unknown keys {unknown}')
        if ('orientation2_rad' in s) != ('contrast2' in s):
            refuse(f'step {k}: a plaid needs both orientation2_rad and contrast2')
        keys = ['orientation_rad', 'contrast'] + (['orientation2_rad', 'contrast2'] if 'contrast2' in s else [])
        if not all(isinstance(s.get(key), (int, float)) and not isinstance(s.get(key), bool) for key in keys):
            refuse(f'step {k}: {", ".join(keys)} must be numbers')
        entry = step_entry(s)
        problem = step_problem(entry)
        if problem:
            refuse(f'step {k} {entry}: {problem}')
        if 'analysed' in s and s['analysed'] is not (k > 0):
            refuse(f'step {k}: analysed must be {k > 0}')
        shown.append(entry)
    if ps.components_of(shown[0]) != ps.WARMUP:
        refuse(f'the first step {shown[0]} is not the declared warm-up [45 deg, 30 %]')
    cond = [ps.condition_of(x) for x in shown[1:]]
    bad = [k + 1 for k, j in enumerate(cond) if j is None]
    if bad:
        refuse(f'steps {bad} are not declared conditions')
    counts = np.bincount(cond, minlength=len(ps.CONDITIONS)).tolist()
    if counts != [c[2] for c in ps.CONDITIONS]:
        refuse(f'condition repeats {dict(zip(ps.LABELS, counts))} are not the declared ones')
    for k, (s, j) in enumerate(zip(steps[1:], cond), 1):
        if 'condition' in s and s['condition'] != ps.LABELS[j]:
            refuse(f"step {k} is labelled {s['condition']!r} but shows {ps.LABELS[j]!r}")
    return shown, np.array(cond)


def read_sequence(base, contract):
    """The declared sequence, or a refusal on it: its file must match the contract's SHA-256 and pass check_sequence."""
    seq_file = base / contract['sequence']['file']
    if not seq_file.is_file():
        refuse(f'the sequence file {seq_file.name} is not next to the contract')
    if sha256(seq_file) != contract['sequence']['sha256']:
        refuse(f'{seq_file.name} does not match its SHA-256 in the contract')
    try:
        seq = json.loads(seq_file.read_text())
    except ValueError as exc:
        refuse(f'{seq_file.name} cannot be read ({type(exc).__name__}: {exc})')
    return check_sequence(seq)


# ---------------------------------------------------------------- one run

def expected_argv(c017, name, run_name):
    e = c017['engine']
    return ['nest', str(e['threads']), e['parameter_file'], *e['reduction_overrides'], *c017['conditions'][RUNS[name]],
            run_name]


def load(name, path):
    """017's load_run, with an export it cannot read (a truncated or corrupt gzip, invalid JSON, segments without the
    keys it reads) turned into a refusal."""
    try:
        return a.load_run(path)
    except SystemExit as exc:                     # 017's loader refuses an export with no sequence segment
        refuse(f'{name}: {exc}')
    except READ_ERRORS as exc:
        refuse(f'{name}: the export cannot be read ({type(exc).__name__}: {exc})')


def finite_numbers(values, n):
    return (isinstance(values, list) and len(values) == n
            and all(isinstance(x, (int, float)) and not isinstance(x, bool) and math.isfinite(x) for x in values))


def check_tables(name, ex):
    """The neuron tables the analysis reads, or a refusal: distinct integer ids for every sheet, orientation_pref for
    the layer 4 sheets and x_um and y_um for layer 4 excitatory cells, one finite number per neuron."""
    nt = ex.get('neurons')
    if not isinstance(nt, dict):
        refuse(f'{name}: the export has no neuron tables')
    for sh in SHEETS:
        t = nt.get(sh)
        if not isinstance(t, dict):
            refuse(f'{name}: no neuron table for {sh}')
        ids = t.get('ids')
        if not (isinstance(ids, list) and ids and all(isinstance(i, int) and not isinstance(i, bool) for i in ids)
                and len(set(ids)) == len(ids)):
            refuse(f'{name}: the {sh} neuron ids are not a list of distinct integers')
        need = (('orientation_pref',) if sh in ('V1_Exc_L4', 'V1_Inh_L4') else ()) + (('x_um', 'y_um') if sh == 'V1_Exc_L4' else ())
        for key in need:
            if not finite_numbers(t.get(key), len(ids)):
                refuse(f'{name}: the {sh} table has no finite {key} for each of its {len(ids)} neurons')


def check_segments(name, ex, chunk):
    """Each sheet's sequence segment starts at 0 ms and holds a train of finite spike times for every neuron of the
    sheet's table and for no other, or a refusal (a missing train would otherwise count as silence)."""
    for sh in SHEETS:
        seg = chunk[sh]
        ids = {str(i) for i in ex['neurons'][sh]['ids']}
        if seg.get('t_start_ms') != 0:
            refuse(f"{name}: the {sh} sequence segment starts at {seg.get('t_start_ms')!r} ms, not 0")
        spikes = seg.get('spikes')
        if not isinstance(spikes, dict):
            refuse(f'{name}: the {sh} sequence segment has no spike trains')
        if set(spikes) != ids:
            refuse(f'{name}: the {sh} sequence segment has trains for {len(ids & set(spikes))} of the {len(ids)} neurons '
                   f'in its table and {len(set(spikes) - ids)} others')
        for i, train in spikes.items():
            try:
                t = np.asarray(train) if isinstance(train, list) else None
            except (TypeError, ValueError):            # ragged nesting
                t = None
            if t is None or t.ndim != 1 or (t.size and (t.dtype.kind not in 'if' or not np.isfinite(t).all())):
                refuse(f'{name}: the {sh} train of neuron {i} is not a list of finite spike times')


def check_run_plaids(name, ex, chunks, shown, c017):
    """The step period and layout of a run, or a refusal (see the module docstring)."""
    if any(s['stimulus']['name'] == 'InternalStimulus' for s in ex['segments']):
        refuse(f'{name}: a separate warm-up (InternalStimulus) was run; the declared design has none')
    argv = ex.get('argv') if isinstance(ex.get('argv'), list) else []
    run_name = argv[-1] if argv else None
    m = RUN_NAME.match(run_name) if isinstance(run_name, str) else None
    if not m or m.group(1) != name:
        refuse(f'{name}: run name {run_name!r} does not match ^m22p-{name}-a[12]$')
    if argv != expected_argv(c017, name, run_name):
        refuse(f'{name}: command line {argv} is not the declared {expected_argv(c017, name, run_name)}')
    if ex.get('resolved') != c017['resolved_weights_and_seeds'][RUNS[name]]:
        refuse(f"{name}: resolved weights or seeds differ from 017's {RUNS[name]}")
    cache = ex.get('lgn_cache')
    if not isinstance(cache, dict) or cache.get('enabled') is not False:
        refuse(f'{name}: the LGN cache is not recorded as disabled ({cache!r})')
    if [ex.get('step_frames'), ex.get('gap_frames')] != [STEP_FRAMES, GAP_FRAMES]:
        refuse(f"{name}: step and gap frames {ex.get('step_frames')}, {ex.get('gap_frames')} are not {STEP_FRAMES}, {GAP_FRAMES}")
    if len(chunks) != 1:
        refuse(f'{name}: {len(chunks)} presentations; the declared design has one')
    chunk = chunks[0]
    missing = [sh for sh in SHEETS if sh not in chunk]
    if missing:
        refuse(f'{name}: no sequence segment for {missing}')
    per_sheet = {sh: sum(1 for s in ex['segments'] if s.get('sheet') == sh and s['stimulus']['name'] == a.SEQUENCE_STIMULUS)
                 for sh in SHEETS}
    if any(n != 1 for n in per_sheet.values()):
        refuse(f'{name}: sequence segments per sheet {per_sheet}; the declared export has one per sheet')
    check_tables(name, ex)
    check_segments(name, ex, chunk)
    stims = [chunk[sh]['stimulus'] for sh in SHEETS]
    if any(st != stims[0] for st in stims):
        refuse(f"{name}: the sheets' stimulus records differ")
    st = stims[0]
    if [st.get('step_frames'), st.get('gap_frames'), st.get('frame_duration')] != [STEP_FRAMES, GAP_FRAMES, FRAME_MS]:
        refuse(f'{name}: step timing differs from the declared {STEP_FRAMES}, {GAP_FRAMES} and {FRAME_MS} ms frames')
    if st.get('lead_frames') != LEAD_FRAMES:
        refuse(f"{name}: lead-in of {st.get('lead_frames')} frames, not the declared {LEAD_FRAMES}")
    if st.get('first_step', 0) != 0:
        refuse(f"{name}: the presentation starts at step {st.get('first_step')}, not 0")
    if [st.get('spatial_frequency'), st.get('temporal_frequency')] != [SPATIAL_FREQUENCY_CPD, TEMPORAL_FREQUENCY_HZ]:
        refuse(f"{name}: gratings of {st.get('spatial_frequency')} cpd and {st.get('temporal_frequency')} Hz, not the "
               f'declared {SPATIAL_FREQUENCY_CPD} cpd and {TEMPORAL_FREQUENCY_HZ} Hz')
    try:
        presented = json.loads(st['sequence']) if isinstance(st.get('sequence'), str) else None
    except ValueError:
        presented = None
    if presented != shown:
        refuse(f'{name}: presented sequence differs from the declared sequence')
    expected = a.chunk_bounds(len(shown), 0, LEAD_FRAMES)            # no chunking: one presentation with the lead-in
    found = [(int(st.get('first_step', 0)), len(presented), st.get('lead_frames', 0.0))]
    if found != [(f, last - f, float(lead)) for f, last, lead in expected]:
        refuse(f'{name}: presentation {found} is not the declared {expected}')
    period = (STEP_FRAMES + GAP_FRAMES) * FRAME_MS
    layout = [(f, last - f, lead * FRAME_MS) for f, last, lead in expected]
    duration = layout[0][2] + len(shown) * period
    if st.get('duration_ms') != duration:
        refuse(f"{name}: presentation of {st.get('duration_ms')} ms, not the declared {duration}")
    for sh in SHEETS:
        t = chunk[sh].get('t_stop_ms')
        if not (isinstance(t, (int, float)) and not isinstance(t, bool) and abs(t - duration) < 1.0):
            refuse(f'{name}: the {sh} sequence segment ends at {t!r} ms, not at the presentation\'s {duration} ms')
    return period, layout


def windows_of(sheet):
    return WINDOWS.get(sheet, ('G', 'step'))


def count_all(ex, chunks, layout, period, method):
    return {(sh, w): method(chunks, layout, [(sh, i) for i in ex['neurons'][sh]['ids']], period, list(ps.WINDOWS_MS[w]))
            for sh in SHEETS for w in windows_of(sh)}


def readouts_of(counts, weights):
    """z per step and window, computed as 017's population_vector_weighted computes it."""
    return {w: np.array([np.sum(c * weights) for c in counts[('V1_Exc_L4', w)]]) for w in READOUT_WINDOWS}


def check_decoder(name, counts_d, weights, z):
    """z against 017's decoder (angle and strength) and the vectorised side against 017's request, every step."""
    side = ps.decision_side(z)
    for k, c in enumerate(counts_d):
        theta, strength = p.population_vector_weighted(c, weights)
        total = c[np.asarray(weights) != 0].sum()
        if z[k] == 0 or total == 0:
            mine = (None, 0.0)
        else:
            mine = (float(np.angle(z[k]) / 2 % math.pi), float(abs(z[k]) / total))
        if (theta, strength) != mine:
            refuse(f'{name}: step {k}: the readout differs from the frozen decoder ({mine} against {(theta, strength)})')
        if p.request(theta, strength)['side'] != {1: 'buy', -1: 'sell', 0: 'hold'}[int(side[k])]:
            refuse(f'{name}: step {k}: the vectorised decision side differs from the frozen request')


def summaries_of(ex, counts):
    """Per analysed step (the warm-up dropped): every real summary, and the cell groups with their phase geometry."""
    nt = ex['neurons']
    out, info = {}, {'cells': {}}
    masks = {}
    for sh in ('V1_Exc_L4', 'V1_Inh_L4'):
        prefs = np.asarray(nt[sh]['orientation_pref'], float)
        masks[sh] = {g: ps.group_mask(prefs, g) for g in ps.GROUPS_DEG}
        info['cells'][SHEETS[sh]] = {str(g): int(m.sum()) for g, m in masks[sh].items()}
    for w in windows_of('V1_Exc_L4'):
        if w == 'step':
            continue
        for g, m in masks['V1_Exc_L4'].items():
            out[f'{w}|L4E|{g}'] = counts[('V1_Exc_L4', w)][1:, m].mean(axis=1)
    for g, m in masks['V1_Inh_L4'].items():
        out[f'G|L4I|{g}'] = counts[('V1_Inh_L4', 'G')][1:, m].mean(axis=1)
    for sh, short in SHEETS.items():
        for w in ('G', 'step'):
            out[f'{w}|{short}|all'] = counts[(sh, w)][1:].mean(axis=1)
    out['D|L4E|sum'] = counts[('V1_Exc_L4', 'D')][1:].sum(axis=1)
    x = np.asarray(nt['V1_Exc_L4']['x_um'], float) / ps.UM_PER_DEGREE
    y = np.asarray(nt['V1_Exc_L4']['y_um'], float) / ps.UM_PER_DEGREE
    info['phase_quartiles'], info['phase_geometry'] = {}, {}
    for pattern, (o1, o2) in ps.PHASE_PATTERNS.items():
        for g in ps.GROUPS_DEG:
            cells = np.where(masks['V1_Exc_L4'][g])[0]
            cosv = ps.relative_phase_cos(x[cells], y[cells], o1, o2)
            info['phase_geometry'][f'{g} deg, block {pattern} pattern'] = {
                'cells': int(len(cells)), 'mean_cos': float(cosv.mean()) if len(cells) else float('nan'),
                'share_cos_above_0': float((cosv > 0).mean()) if len(cells) else float('nan')}
            if g not in (45, 135):
                continue
            q = ps.quartiles(cosv)
            info['phase_quartiles'][f'{g} deg, block {pattern} pattern'] = [
                {'cells': int((q == k).sum()), 'mean_cos': float(cosv[q == k].mean())} for k in range(4)]
            for k in range(4):
                out[f'G|L4E|{g}|{pattern}{k + 1}'] = counts[('V1_Exc_L4', 'G')][1:, cells[q == k]].mean(axis=1)
    return out, info


def lead_in_hz(chunk, ex, lead):
    out = {}
    for sh, short in SHEETS.items():
        trains = chunk[sh]['spikes']
        n = sum(int(np.count_nonzero(np.asarray(t, float) < lead)) for t in trains.values())
        out[short] = n / (len(ex['neurons'][sh]['ids']) * lead / 1000.0)
    return out


def analyse_run(name, path, shown, c017, decoder):
    """Check one export, count it twice (searchsorted, then a fresh read by step) and return its per-step data."""
    try:
        digest = sha256(path)
    except OSError as exc:                         # a file that exists but cannot be opened is a refusal, not a crash
        refuse(f'{name}: the export cannot be opened ({type(exc).__name__}: {exc})')
    ex, chunks = load(name, path)
    try:
        period, layout = check_run_plaids(name, ex, chunks, shown, c017)
    except (OverflowError, RecursionError, ValueError, TypeError) as exc:   # malformed values the layout checks met
        refuse(f'{name}: the export is malformed ({type(exc).__name__}: {exc})')
    ids, weights, use = a.decoder_setup(ex, decoder)
    if [i for _, i in ids] != ex['neurons']['V1_Exc_L4']['ids'] or not use.all():
        refuse(f'{name}: the decoder does not read every layer 4 excitatory cell in table order')
    counts = count_all(ex, chunks, layout, period, a.counts_searchsorted)
    z = readouts_of(counts, weights)
    check_decoder(name, counts[('V1_Exc_L4', 'D')], weights, z['D'])
    summaries, info = summaries_of(ex, counts)
    neurons = ex['neurons']
    digests = a.lgn_digests(chunks)
    lead = lead_in_hz(chunks[0], ex, layout[0][2])
    provenance = {'export': Path(path).name, 'export_sha256': digest, 'run_name': ex['argv'][-1], 'argv': ex['argv'],
                  'resolved': ex['resolved'], 'run_seconds': ex.get('run_seconds'), 'nest': ex.get('nest'),
                  'lgn_cache': ex.get('lgn_cache'), 'presentations': layout, 'step_period_ms': period, **info,
                  'cells_as_declared_for_instance_A': info['cells']['L4E'] == {str(g): n for g, n in DECLARED_GROUP_CELLS.items()}}
    del ex, chunks
    ex2, chunks2 = load(name, path)                # an independent recount on a fresh read
    if sha256(path) != digest:
        refuse(f'{name}: the export changed while it was analysed')
    for key, c in count_all(ex2, chunks2, layout, period, a.counts_by_step).items():
        if not np.array_equal(c, counts[key]):
            refuse(f'{name}: the recount of {key[0]} in window {key[1]} differs from the count')
    del ex2, chunks2
    provenance['recount_matches'] = True
    names = list(summaries)
    data = {'Y': np.column_stack([summaries[n] for n in names]), 'summaries': names,
            'Z': np.column_stack([z[w][1:] for w in READOUT_WINDOWS]), 'readouts': list(READOUT_WINDOWS)}
    per_step = {'z_D': [[float(v.real), float(v.imag)] for v in z['D'][1:]],
                **{f'G|L4E|{g}': summaries[f'G|L4E|{g}'].tolist() for g in ps.GROUPS_DEG}}
    return {'data': data, 'neurons': neurons, 'digests': digests, 'lead_in_hz': lead, 'provenance': provenance,
            'per_step': per_step}


# ---------------------------------------------------------------- results

def clean(x):
    """JSON-ready: numpy types to Python, non-finite numbers to None."""
    if isinstance(x, dict):
        return {str(k): clean(v) for k, v in x.items()}
    if isinstance(x, (list, tuple)):
        return [clean(v) for v in x]
    if isinstance(x, np.ndarray):
        return clean(x.tolist())
    if isinstance(x, (bool, np.bool_)):
        return bool(x)
    if isinstance(x, (int, np.integer)):
        return int(x)
    if isinstance(x, (float, np.floating)):
        return float(x) if math.isfinite(x) else None
    return x


def write(path, res):
    """results.json, created once: an existing file is never replaced."""
    text = json.dumps(clean(res), indent=1, allow_nan=False) + '\n'
    try:
        with open(path, 'x', encoding='utf-8') as fh:
            fh.write(text)
    except FileExistsError:
        stop(f'{path} exists: a results file is never overwritten (give a new OUT_DIR)')


def software():
    return {'python': sys.version, 'numpy': np.__version__, 'scipy': scipy.__version__, 'platform': platform.platform()}


def parse_runs(runs):
    given = dict(r.split('=', 1) for r in runs if '=' in r)
    if len(given) != len(runs) or set(given) != set(RUNS):
        stop(f'give exactly intact=EXPORT|FAILED and lesion=EXPORT|FAILED, got {list(runs)}')
    for name, value in given.items():
        if value != 'FAILED' and not Path(value).is_file():
            stop(f'{name}: no export at {value} (give the export, or FAILED for a run that failed twice)')
    return given


def lgn_condition(identity, digests):
    """Why the connection readings are missing if the LGN spike trains of the two runs are not identical, else None."""
    pair = identity.get('intact|lesion', {})
    bad = []
    for sh in a.LGN_SHEETS:
        r, n = pair.get(sh), len(digests['intact'].get(sh, {}))
        if not r or r['identical_fraction'] != 1.0 or r['neurons_compared'] != n or n == 0:
            bad.append(f"{sh} {r['identical_fraction'] if r else None} of {n}")
    return f"the LGN spike trains differ between the runs (identical fraction: {', '.join(bad)})" if bad else None


def attach_geometry(measures, provenance):
    """D5: the size of the fixed phase geometry, next to S10's quartiles and next to P1's group terms in S1."""
    geo = provenance['phase_geometry']
    measures.setdefault('S10', {})['geometry'] = {'groups': geo, 'note': GEOMETRY_NOTE}
    measures['S1']['XOS']['phase_geometry_block_A'] = {str(g): geo[f'{g} deg, block A pattern'] for g in (45, 135)}


def analyse(contract_path, runs, allow_unfrozen=False):
    contract_path = Path(contract_path)
    if contract_path.name != CONTRACT_NAME or not contract_path.is_file():
        stop(f'{contract_path} is not a file named {CONTRACT_NAME} (the frozen package names its contract so)')
    given = parse_runs(runs)
    base = contract_path.parent
    frozen, listed = check_frozen(base, allow_unfrozen)
    res = {'scope': 'protocol 022 revision 2, plaids pilot: results; simulation only',
           'contract_sha256': sha256(contract_path), 'frozen_check': frozen, 'software': software(),
           'declared': ps.declared(),
           'run_status': {n: ('failed twice' if v == 'FAILED' else 'analysed') for n, v in given.items()},
           'refusals': {}}
    try:
        contract = read_contract(contract_path)
        check_frozen_sequence(base, contract, frozen, listed)
        c017, decoder, res['protocol_017'] = check_017(contract)
        shown, cond = read_sequence(base, contract)
    except Refusal as exc:                          # a refusal on the contract or the sequence
        res['refusals']['contract or sequence'] = str(exc)
        res.update(ps.not_interpretable(f'a refusal fired: {exc}'))
        res['connection_readings'] = ps.connection_readings(None, None, res['outcome'])
        return res, False
    res['sequence_sha256'] = contract['sequence']['sha256']
    previous = np.array([sum(x[1::2]) for x in shown[:-1]])         # each analysed step's predecessor, summed contrast
    longest, run = 1, 1
    for x, y in zip(cond, cond[1:]):
        run = run + 1 if x == y else 1
        longest = max(longest, run)
    res['sequence'] = {'analysed_steps': len(cond), 'conditions': [ps.LABELS[j] for j in cond], 'longest_run': longest,
                       'mean_previous_contrast': {lab: float(previous[cond == j].mean()) for j, lab in enumerate(ps.LABELS)}}
    runs_data = {}
    for name in RUNS:
        if given[name] == 'FAILED':
            continue
        try:
            runs_data[name] = analyse_run(name, given[name], shown, c017, decoder)
        except Refusal as exc:
            res['refusals'][name] = str(exc)
            res['run_status'][name] = 'refused'
    conditions, missing = [], []                    # NOT INTERPRETABLE causes; reasons the connection readings are missing
    if given['intact'] == 'FAILED':
        conditions.append('the intact run failed twice')
    if 'intact' in res['refusals']:
        conditions.append(f"a refusal fired: {res['refusals']['intact']}")
    if given['lesion'] == 'FAILED':
        missing.append('the lesion run failed twice')
    if 'lesion' in res['refusals']:
        missing.append(f"a refusal fired: {res['refusals']['lesion']}")
    res['runs'] = {n: r['provenance'] for n, r in runs_data.items()}
    identity = None
    if len(runs_data) == 2:
        if runs_data['intact']['neurons'] != runs_data['lesion']['neurons']:
            missing.append('the neuron tables of the two exports differ')
        digests = {n: r['digests'] for n, r in runs_data.items()}
        identity = a.lgn_identity(digests)
        lgn = lgn_condition(identity, digests)
        if lgn:
            missing.append(lgn)
    res['lgn_identical_spike_trains'] = identity
    idx = ps.draw(len(cond), ps.BOOT, ps.SEED)
    res['bootstrap'] = {'replicates': int(idx.shape[0]), 'seed': ps.SEED, 'steps': int(idx.shape[1]),
                        'draw_sha256': hashlib.sha256(np.ascontiguousarray(idx, dtype=np.int64).tobytes()).hexdigest(),
                        'same draw': 'every summary and both runs'}
    stats = ps.evaluate({n: r['data'] for n, r in runs_data.items()}, cond, idx, previous,
                        {n: r['lead_in_hz'] for n, r in runs_data.items()})
    for n, r in runs_data.items():
        stats[n]['per_step'] = r['per_step']
        attach_geometry(stats[n], r['provenance'])
    res['measures'] = stats
    if 'intact' in stats:
        v = stats['intact']['validation_30']
        if not v['passes']:
            conditions.append('the intact run fails the 30 % validation check (' + '; '.join(v['failures']) + ')')
        res.update(ps.outcome(stats['intact']['primary'], conditions))
    else:
        res.update(ps.not_interpretable('; '.join(conditions)))
    if 'lesion' in stats and not stats['lesion']['validation_30']['passes']:
        missing.append('the lesion run fails its 30 % validation check (' + '; '.join(stats['lesion']['validation_30']['failures']) + ')')
    res['connection_readings'] = ps.connection_readings(stats.get('intact', {}).get('primary'),
                                                        stats.get('lesion', {}).get('primary'), res['outcome'], missing)
    return res, not res['refusals']


def main(contract_path, out_dir, *runs, allow_unfrozen=False):
    out = Path(out_dir) / 'results.json'
    if out.exists():
        stop(f'{out} exists: a results file is never overwritten (give a new OUT_DIR)')
    res, ok = analyse(contract_path, runs, allow_unfrozen)
    out.parent.mkdir(parents=True, exist_ok=True)
    write(out, res)
    print('outcome:', res['outcome'])
    print(res['text'])
    if 'measures' in res and 'intact' in res['measures']:
        for k, r in res['measures']['intact']['primary'].items():
            print(f"  {k}: {r['estimate']:+.3f}, 95 % {np.round(r['ci95'], 3).tolist()}, 90 % {np.round(r['ci90'], 3).tolist()}, "
                  f"p {r['p']:.4f}, estimable {r['estimable']}")
    print('connection readings:', res['connection_readings']['readout']['reading'])
    if not ok:
        raise SystemExit('refused: ' + '; '.join(res['refusals'].values()))
    return res


if __name__ == '__main__':
    args = [x for x in sys.argv[1:] if x != '--not-frozen']
    if len(args) < 3:
        raise SystemExit(__doc__)
    main(*args, allow_unfrozen='--not-frozen' in sys.argv[1:])
