"""Controller-015 milestone 2 (protocol 017): turn each model run into decisions and evaluate them.

Refuses to run unless every input is the declared one: if a manifest.json sits next to the contract (the
frozen package), the contract and the analysis code must match it; the sequence and decoder files must
match their SHA-256 in the contract; exactly the declared runs must be given; and each export must carry
the declared command line, the declared lead-in and no separate warm-up, the presented sequence and the
resolved weights and seeds of its condition. Then, per run: decode every step with the fixed decoder, write the four-stage decision log
(model output, decoded request, executor decision, simulated outcome), and replay it: re-read the written
log, recount the spikes from a fresh read of the export with an independent counting method, rebuild the
log and require an exact match. Finally evaluate the declared measures and readings (decisions.evaluate)
and report whether the LGN spike trains are identical across runs. Simulation only: no wallet, key,
exchange or real order exists anywhere in this code.

  python3 analyze_decisions.py CONTRACT.json OUT_DIR NAME=EXPORT.json.gz [NAME=EXPORT.json.gz ...]

Without a manifest.json next to the contract it refuses, unless --not-frozen is given (synthetic tests only).
"""
import gzip, hashlib, json, sys
from pathlib import Path
import numpy as np
import decisions as d
import pipeline as p

SEQUENCE_STIMULUS = 'FullfieldDriftingSinusoidalGratingSequence'
LGN_SHEETS = ('X_ON', 'X_OFF')
FROZEN_CODE = ('contract.json', 'controller/analyze_decisions.py', 'controller/decisions.py', 'controller/pipeline.py')


def sha256(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def refuse(message):
    raise SystemExit(f'refused: {message}')


def load_run(path):
    """The export and its sequence presentations in order: a list of {sheet: segment}, one per chunk."""
    ex = json.load(gzip.open(path, 'rt'))
    chunks = {}
    for s in ex['segments']:
        if s['stimulus']['name'] == SEQUENCE_STIMULUS:
            chunks.setdefault(int(s['stimulus'].get('first_step', 0)), {})[s['sheet']] = s
    if not chunks:
        refuse(f'{path}: no sequence segment')
    return ex, [chunks[k] for k in sorted(chunks)]


def chunk_bounds(n_steps, max_steps, lead_frames):
    """(first, last, lead) per presentation, as server/controller_experiments.py builds them."""
    n_chunks = -(-n_steps // max_steps) if max_steps else 1
    edges = [round(k * n_steps / n_chunks) for k in range(n_chunks + 1)]
    return [(edges[j], edges[j + 1], lead_frames if j == 0 else 0) for j in range(n_chunks)]


def expected_argv(contract, name):
    e = contract['engine']
    return ['nest', str(e['threads']), e['parameter_file'], *e['reduction_overrides'], *contract['conditions'][name],
            contract['run_name_prefix'] + name]


def check_frozen(base, allow_unfrozen=False):
    """In the frozen package, the contract and the analysis code must be the manifest's files."""
    manifest = base / 'manifest.json'
    if not manifest.exists():
        if not allow_unfrozen:
            refuse('no manifest.json next to the contract (use --not-frozen only for synthetic tests)')
        print('no manifest.json next to the contract: inputs are not a frozen package', flush=True)
        return {'manifest': False}
    listed = {f['path']: f['sha256'] for f in json.loads(manifest.read_text())['files']}
    for rel in FROZEN_CODE:
        if rel not in listed or sha256(base / rel) != listed[rel]:
            refuse(f'{rel} differs from the frozen manifest')
    if sha256(Path(__file__).resolve()) != listed['controller/analyze_decisions.py']:
        refuse('the running analyze_decisions.py is not the frozen one')
    return {'manifest': True, 'files_checked': list(FROZEN_CODE)}


def check_run(contract, name, ex, chunks, declared_steps):
    """Returns the step period and, per presentation, (first step, number of steps, lead-in ms)."""
    enc = contract['encoder']
    if enc.get('separate_warmup_ms', 0) == 0 and any(s['stimulus']['name'] == 'InternalStimulus' for s in ex['segments']):
        refuse(f'{name}: a separate warm-up was run, but the declared design has none')
    if ex.get('argv') != expected_argv(contract, name):
        refuse(f'{name}: command line {ex.get("argv")} is not the declared {expected_argv(contract, name)}')
    if ex.get('resolved') != contract['resolved_weights_and_seeds'][name]:
        refuse(f'{name}: resolved weights or seeds differ from the declared condition')
    stims = [next(iter(c.values()))['stimulus'] for c in chunks]
    shown = [pair for st in stims for pair in json.loads(st['sequence'])]
    want = [[round(float(s['orientation_rad']), 6), round(float(s['contrast']), 3)] for s in declared_steps]
    if shown != want:
        refuse(f'{name}: presented sequence differs from the declared sequence')
    expected = chunk_bounds(len(want), enc.get('chunk_max_steps', 0), round(enc['lead_ms'] / enc['frame_ms']))
    found = [(int(st.get('first_step', 0)), len(json.loads(st['sequence'])), st.get('lead_frames', 0.0)) for st in stims]
    if found != [(a, b - a, float(lead)) for a, b, lead in expected]:
        refuse(f'{name}: presentations {found} are not the declared chunks {expected}')
    for st in stims:
        if [st['step_frames'], st['gap_frames'], st['frame_duration']] != [enc[k] for k in ('step_frames', 'gap_frames', 'frame_ms')]:
            refuse(f'{name}: step timing differs from the declared timing')
    period = (stims[0]['step_frames'] + stims[0]['gap_frames']) * stims[0]['frame_duration']
    return period, [(first, n, lead * enc['frame_ms']) for first, n, lead in found]


def decoder_setup(ex, decoder):
    sheets = decoder['sheets']
    ids = [(sh, i) for sh in sheets for i in ex['neurons'][sh]['ids']]
    prefs = np.array([v for sh in sheets for v in ex['neurons'][sh]['orientation_pref']])
    use = np.ones(len(ids), bool)
    if decoder.get('centre_um'):
        x = np.array([v for sh in sheets for v in ex['neurons'][sh]['x_um']])
        y = np.array([v for sh in sheets for v in ex['neurons'][sh]['y_um']])
        use = np.hypot(x - x.mean(), y - y.mean()) <= decoder['centre_um']
    return ids, p.decoder_weights(prefs, decoder['mode'], use), use


def counts_searchsorted(chunks, layout, ids, period, window):
    counts = np.zeros((sum(n for _, n, _ in layout), len(ids)))
    for seq, (first, n, lead) in zip(chunks, layout):
        onsets = lead + np.arange(n) * period
        for j, (sh, i) in enumerate(ids):
            t = np.sort(np.asarray(seq[sh]['spikes'].get(str(i), []), float))
            counts[first:first + n, j] = np.searchsorted(t, onsets + window[1], 'left') - np.searchsorted(t, onsets + window[0], 'left')
    return counts


def counts_by_step(chunks, layout, ids, period, window):
    """Independent recount for the replay: assign each spike to its step, then test its offset."""
    counts = np.zeros((sum(n for _, n, _ in layout), len(ids)))
    for seq, (first, n, lead) in zip(chunks, layout):
        for j, (sh, i) in enumerate(ids):
            t = np.asarray(seq[sh]['spikes'].get(str(i), []), float) - lead
            k = np.floor(t / period).astype(int)
            off = t - k * period
            ok = (k >= 0) & (k < n) & (off >= window[0]) & (off < window[1])
            counts[first:first + n, j] = np.bincount(k[ok], minlength=n)[:n]
    return counts


def lgn_digests(chunks):
    """Per LGN neuron, one digest of its spike trains over all presentations, in order."""
    out = {}
    for sh in LGN_SHEETS:
        if all(sh in c for c in chunks):
            ids = set().union(*(c[sh]['spikes'] for c in chunks))
            out[sh] = {i: hashlib.sha256(b'|'.join(np.asarray(c[sh]['spikes'].get(i, []), float).tobytes() for c in chunks)).hexdigest()
                       for i in ids}
    return out


def lgn_identity(digests):
    """Fraction of LGN neurons whose sequence-segment spike trains are identical, for each pair of runs."""
    names = list(digests)
    out = {}
    for a in range(len(names)):
        for b in range(a + 1, len(names)):
            x, y = digests[names[a]], digests[names[b]]
            res = {}
            for sh in LGN_SHEETS:
                if sh in x and sh in y:
                    ids = set(x[sh]) & set(y[sh])
                    same = sum(x[sh][i] == y[sh][i] for i in ids)
                    res[sh] = {'neurons_compared': len(ids), 'identical_fraction': same / len(ids) if ids else None}
            out[f'{names[a]}|{names[b]}'] = res
    return out


def main(contract_path, out_dir, *runs, allow_unfrozen=False):
    contract_path = Path(contract_path)
    contract = json.loads(contract_path.read_text())
    base = contract_path.parent
    frozen = check_frozen(base, allow_unfrozen)
    for key in ('sequence', 'decoder'):
        f = base / contract[key]['file']
        if sha256(f) != contract[key]['sha256']:
            refuse(f'{f.name} does not match its SHA-256 in the contract')
    sequence = json.loads((base / contract['sequence']['file']).read_text())
    decoder = json.loads((base / contract['decoder']['file']).read_text())
    s, prices, steps = sequence['signals'], sequence['prices'], sequence['steps']
    given = dict(arg.split('=', 1) for arg in runs)
    if set(given) != set(contract['conditions']):
        refuse(f'runs {sorted(given)} are not exactly the declared {sorted(contract["conditions"])}')
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)
    window = decoder['window_ms']
    estimates, provenance, lgn = {}, {}, {}
    for name in contract['run_order']:
        path = given[name]
        ex, chunks = load_run(path)
        period, layout = check_run(contract, name, ex, chunks, steps)
        ids, weights, use = decoder_setup(ex, decoder)
        counts = counts_searchsorted(chunks, layout, ids, period, window)
        est = [p.population_vector_weighted(c, weights) for c in counts]
        log = p.run_decisions(steps, prices, est)
        log_path = out / f'decisions-{name}.json'
        log_path.write_text(json.dumps({'run': name, 'export_sha256': sha256(path), 'decoder': decoder, 'log': log}) + '\n')
        lgn[name] = lgn_digests(chunks)
        info = {'run_seconds': ex['run_seconds'], 'nest': ex['nest'], 'argv': ex['argv'], 'resolved': ex['resolved']}
        del ex, chunks
        # replay: the written log against a rebuild from a fresh read with an independent count
        written = json.loads(log_path.read_text())['log']
        ex2, chunks2 = load_run(path)
        ids2, weights2, _ = decoder_setup(ex2, decoder)
        recount = counts_by_step(chunks2, layout, ids2, period, window)
        if not np.array_equal(recount, counts):
            refuse(f'{name}: replay recount differs from the decoding count')
        rebuilt = json.loads(json.dumps(p.run_decisions(steps, prices, [p.population_vector_weighted(c, weights2) for c in recount])))
        if rebuilt != written:
            refuse(f'{name}: replayed decisions differ from the written log')
        sides = d.sides_from_signal(d.decision_signal(d.thetas(est)))
        if sides.tolist() != [{'buy': 1, 'sell': -1, 'hold': 0}[r['request']['side']] for r in log]:
            refuse(f'{name}: vectorized requests differ from the logged requests')
        del ex2, chunks2
        estimates[name] = est
        provenance[name] = {'export': Path(path).name, 'export_sha256': sha256(path), 'decision_log': log_path.name,
                            'decision_log_sha256': sha256(log_path), 'replay_matches': True, 'presentations': layout, 'step_period_ms': period,
                            **info, 'neurons_decoded': int(use.sum()), 'mean_count_per_neuron': float(counts[:, use].mean())}
        print(f'{name}: inputs as declared; replay matches; decoded {int(use.sum())} neurons, '
              f'{counts[:, use].mean():.2f} spikes per neuron per window', flush=True)
    res = d.evaluate(s, prices, estimates, lesion=tuple(contract['analysis']['lesion_pair']),
                     replicate=tuple(contract['analysis']['replication_pair']))
    res.update({'scope': 'controller-015 milestone 2 (protocol 017) results; simulation only',
                'contract_sha256': sha256(contract_path), 'sequence_sha256': contract['sequence']['sha256'],
                'decoder': decoder, 'decoder_sha256': contract['decoder']['sha256'], 'runs': provenance,
                'lgn_identical_spike_trains': lgn_identity(lgn), 'all_replays_match': True, 'frozen_check': frozen})
    (out / 'results.json').write_text(json.dumps(res, indent=1) + '\n')
    for name in estimates:
        m = res['policies'][name]
        print(f"{name:17s} decodes {m['decodes']}; accuracy {m['accuracy_vs_rule']:.3f} (error null {m['error_null']['accuracy_95']}); "
              f"c {m['history_c']:+.3f} {m['history_c_95']} -> {m['carry_over']}; gain {m['gain_b']:.2f}; error SD {m['error_sd_deg']:.1f} deg")
    print('readings:', res['readings'])


if __name__ == '__main__':
    args = [x for x in sys.argv[1:] if x != '--not-frozen']
    main(*args, allow_unfrozen='--not-frozen' in sys.argv[1:])
