"""Protocol 025: the decision map. Protocol 024's rule, unchanged, on a 5 x 5 grid of signal strengths, fresh runs.

  python3 -B decide_025.py OUT.json sK=EXPORT.json.gz [sK=EXPORT.json.gz ...] [--smoke]

The rule is protocol 024's, with nothing refitted: z is protocol 017's frozen readout (all layer 4 excitatory cells,
balanced weights, counts 40 to 300 ms after onset); z0 the mean z over the instance's 8 blank steps; S half the
difference of Im mean z for A alone at 30 % and B alone at 30 % (8 steps each; refused below 1,500);
v = -Im(z - z0) / S; action B if v > 0.32, A if v < -0.32, else hold.

Scored steps, 56 per instance, with the intended action (a, b are the strengths of A and B in percent):
  equal cells (7.5, 15, 22.5 and 30 on both)            hold       16 steps
  a - b >= 15  (5 cells: A15, A22.5, A22.5+B7.5, A30+B7.5, A30+B15)     A      20 steps
  b - a >= 15  (the mirror cells)                                       B      20 steps
Not scored: the calibration cells, and the 8 cells whose strengths differ by 7.5 (the transition zone, described).

Outcome over the three analysed instances (the seeds the queue settled with an agreeing pair, three of 15, 16, 17 and
the reserve 18):
  PASS               every instance takes the intended action in at least 51 of its 56 scored steps, and the three
                     together in at least 160 of 168;
  FAIL               otherwise;
  NOT INTERPRETABLE  a refusal: not exactly three instances, an export that is not the declared seed's run of its
                     declared sequence or not the earlier run of the queue's agreeing pair, a lead-in whose LGN
                     spike trains differ from the reference of protocol 023,
                     S below 1,500, or frozen code not at its recorded hashes.
Run from work/controller-025 of the workspace. Simulation only; an engineering rule, not a claim about cat cortex.
"""
import gzip, hashlib, json, os, sys
import numpy as np

sys.dont_write_bytecode = True
T = 0.32
MIN_SCALE = 1500.0
NEED_INSTANCE, OF_INSTANCE, N_INSTANCES = 51, 56, 3
NEED_POOLED, OF_POOLED = 160, 168
SEEDS = (15, 16, 17, 18)
SEQUENCES = {15: '3e21d2fba3ff7e28bc3dd3748cc2dc324b4d15331c70db298336fccc60596773',
             16: 'afffd036fb62ebd2cea420314cebc6be1275f0c1a421addeea29acaff65012f7',
             17: 'dabaa90b0ccf108629003ade5ff9bea248c2858c6cbc22e8ea787c940bdde8fe',
             18: 'e21b30717eb2dd6bf44e40b7de707e17f00943a440ceebd330ed932a965bd117'}
FROZEN = {'protocol-017/pipeline.py': '9801606903df981ab15966d2b0182a4ebb4bdea4e9ad24e427b3165f3abc51d4'}
LEADIN = ('work/controller-025/ref/leadin-reference-023.json', '126485e68db25f813f33f9c3a34c55f025c60bba3f2564f8cc64ac4c9a893645')
CAL = ('blank', 'A30', 'B30')
WINDOW_MS = (40.0, 300.0)


class Refusal(Exception):
    pass


def action(v, t=T):
    return 'B' if v > t else ('A' if v < -t else 'hold')


def role(a, b):
    """('calibration' | 'scored' | 'transition', intended action or None) of the cell with strengths a and b."""
    if (a, b) in ((0, 0), (30, 0), (0, 30)):
        return 'calibration', None
    if a == b:
        return 'scored', 'hold'
    if abs(a - b) >= 15:
        return 'scored', 'A' if a > b else 'B'
    return 'transition', None


def decide(cells, labels, z):
    """cells: (a, b) per analysed step; labels: its condition label; z: its complex readout."""
    z = np.asarray(z, complex)
    lab = np.asarray(labels)
    for c in CAL:
        if (lab == c).sum() != 8:
            raise Refusal(f'{c}: {int((lab == c).sum())} calibration steps, not 8')
    z0 = z[lab == 'blank'].mean()
    s = float((z[lab == 'A30'].mean().imag - z[lab == 'B30'].mean().imag) / 2)
    if not np.isfinite(s) or s < MIN_SCALE:
        raise Refusal(f'the response scale S is {s:.1f}, below {MIN_SCALE:.0f}')
    v = -(z - z0).imag / s
    acts = [action(x) for x in v]
    steps, by, hits, scored = [], {}, 0, 0
    for (a, b), l, x, act in zip(cells, labels, v, acts):
        r, intended = role(a, b)
        steps.append({'condition': l, 'a': a, 'b': b, 'role': r, 'intended': intended, 'v': float(x), 'action': act})
        c = by.setdefault(l, {'a': a, 'b': b, 'role': r, 'intended': intended, 'v': [], 'actions': {'A': 0, 'hold': 0, 'B': 0}})
        c['v'].append(float(x))
        c['actions'][act] += 1
        if r == 'scored':
            scored += 1
            hits += act == intended
    if scored != OF_INSTANCE or len(by) != 25:
        raise Refusal(f'{scored} scored steps in {len(by)} cells, not {OF_INSTANCE} in 25')
    for c in by.values():
        vs = c.pop('v')
        c.update(n=len(vs), v_mean=float(np.mean(vs)), v_sd=float(np.std(vs, ddof=1)), v_min=min(vs), v_max=max(vs))
        if c['role'] == 'scored':
            c['as_intended'] = c['actions'][c['intended']]
    relay = {}
    for l, c in by.items():                      # the relay rule on every two-signal cell, reported only
        if c['a'] and c['b']:
            la, lb = f"A{c['a']:g}", f"B{c['b']:g}"
            lin = by[la]['v_mean'] + by[lb]['v_mean']
            relay[l] = {'v_relay': lin, 'relay_action': action(lin), 'v_circuit_mean': c['v_mean'], 'intended': c['intended']}
    return {'z0': [float(z0.real), float(z0.imag)], 'S': s, 'threshold': T, 'as_intended': hits, 'of': scored,
            'cells': by, 'relay_rule': relay, 'steps': steps}


def outcome(per_instance):
    hits = {k: r['as_intended'] for k, r in per_instance.items()}
    pooled = sum(hits.values())
    ok = len(hits) == N_INSTANCES and all(h >= NEED_INSTANCE for h in hits.values()) and pooled >= NEED_POOLED
    return {'outcome': 'PASS' if ok else 'FAIL', 'as_intended': hits, 'pooled': pooled, 'of_pooled': OF_POOLED,
            'rule': f'every instance at least {NEED_INSTANCE} of {OF_INSTANCE}, and the three at least {NEED_POOLED} of {OF_POOLED}'}


def sha256(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for b in iter(lambda: f.read(1 << 20), b''):
            h.update(b)
    return h.hexdigest()


def leadin_problem(ex, reference):
    """None if every LGN train before the lead-in's end equals the reference's; else a short description."""
    lead = reference['lead_ms']
    for sh in ('X_ON', 'X_OFF'):
        seg = [s for s in ex['segments'] if s['sheet'] == sh and 'sequence' in s['stimulus']]
        if len(seg) != 1:
            return f'{len(seg)} sequence presentations of {sh}'
        got = {i: [float(t) for t in ts if float(t) < lead] for i, ts in seg[0]['spikes'].items()}
        ref = reference['trains'][sh]
        if got.keys() != ref.keys():
            return f'{sh}: the cells differ from the reference'
        same = sum(sorted(got[i]) == sorted(float(t) for t in ref[i]) for i in ref)
        if same != len(ref):
            return f'{sh}: {same} of {len(ref)} lead-in trains identical to the reference'
    return None


def read_steps(path, k, root, smoke=False):
    """Cells, labels and z of every analysed step of seed k's export, after checking what was run and shown."""
    sys.path.insert(0, root + '/outputs/felisyn-research/protocol-017/controller')
    import pipeline as p
    ex = json.load(gzip.open(path, 'rt'))
    if not smoke:
        seqdoc = json.load(open(f'{root}/work/controller-025/sequences/sequence-025-s{k}.json'))
        if sha256(f'{root}/work/controller-025/sequences/sequence-025-s{k}.json') != SEQUENCES[k]:
            raise Refusal(f's{k}: the sequence file is not the declared one')
        if str(ex['resolved'].get('pynn_seed')) != str(k) or 'pynn_seed' not in ex['argv'] or ex['argv'][ex['argv'].index('pynn_seed') + 1] != str(k):
            raise Refusal(f's{k}: the export is not a run of pynn_seed {k}')
        bad = leadin_problem(ex, json.load(open(f'{root}/{LEADIN[0]}')))
        if bad:
            raise Refusal(f's{k}: lead-in: {bad}')
    seg = [s for s in ex['segments'] if s['sheet'] == 'V1_Exc_L4' and 'sequence' in s['stimulus']]
    if len(seg) != 1:
        raise Refusal(f'{len(seg)} sequence presentations of layer 4, not 1')
    st = seg[0]['stimulus']
    frame = float(st['frame_duration'])
    period = (float(st['step_frames']) + float(st['gap_frames'])) * frame
    lead = float(st['lead_frames']) * frame
    if (float(st['step_frames']), float(st['gap_frames']), lead, frame) != (30.0, 21.0, 504.0, 7.0):
        raise Refusal('the timing is not 30 + 21 frames of 7 ms after a 504 ms lead-in')
    shown = json.loads(st['sequence']) if isinstance(st['sequence'], str) else st['sequence']
    a45, b135 = round(np.pi / 4, 6), round(3 * np.pi / 4, 6)
    cells = []
    for s in shown:
        s = [float(x) for x in s]
        if len(s) == 2:
            cells.append((s[1], 0.0) if s[0] == a45 else (0.0, s[1]) if s[0] == b135 else None)
        else:
            cells.append((s[1], s[3]) if (s[0], s[2]) == (a45, b135) else None)
    if None in cells:
        raise Refusal('a shown step is not a cell of the grid')
    if not smoke:
        want = [(float(x['a']), float(x['b'])) for x in seqdoc['steps']]
        if cells != want:
            raise Refusal(f's{k}: the shown steps are not the declared sequence')
    if float(seg[0].get('t_start_ms', 0.0)) != 0.0 or float(st['duration_ms']) + 1e-6 < lead + period * len(shown):
        raise Refusal('the presentation does not start at 0 or is shorter than its steps')
    ids = ex['neurons']['V1_Exc_L4']['ids']
    w = p.decoder_weights(np.asarray(ex['neurons']['V1_Exc_L4']['orientation_pref'], float), 'balanced')
    sp = seg[0]['spikes']
    trains = [np.sort(np.asarray(sp.get(str(i), []), float)) for i in ids]
    z = []
    for n in range(1, len(shown)):                                    # step 0 is the warm-up
        lo, hi = lead + period * n + WINDOW_MS[0], lead + period * n + WINDOW_MS[1]
        c = np.array([np.searchsorted(t, hi, 'left') - np.searchsorted(t, lo, 'left') for t in trains])
        z.append(complex(np.sum(c * w)))
    cells = cells[1:]
    labels = ['blank' if a == 0 and b == 0 else '+'.join(q for q in (f'A{a:g}' if a else '', f'B{b:g}' if b else '') if q) for a, b in cells]
    return cells, labels, z


def main(argv):
    smoke = '--smoke' in argv
    args = [a for a in argv if a != '--smoke']
    out, runs = args[0], dict(a.split('=', 1) for a in args[1:])
    root = __file__.rsplit('/work/', 1)[0]
    res = {'scope': 'protocol 025, the decision map; simulation only; an engineering rule', 'smoke_run': smoke,
           'threshold': T, 'min_scale': MIN_SCALE, 'tool_sha256': sha256(__file__), 'exports': {}, 'instances': {}, 'refusals': {}}
    try:
        if sha256(root + '/outputs/felisyn-research/protocol-017/controller/pipeline.py') != FROZEN['protocol-017/pipeline.py']:
            raise Refusal('the frozen readout code is not at its recorded hash')
        if not smoke:
            if sha256(f'{root}/{LEADIN[0]}') != LEADIN[1]:
                raise Refusal('the lead-in reference is not at its recorded hash')
            seeds = sorted(int(n[1:]) for n in runs)
            if len(seeds) != N_INSTANCES or not set(seeds) <= set(SEEDS):
                raise Refusal(f'the runs given are {sorted(runs)}, not three of the declared seeds')
        for name, path in runs.items():
            res['exports'][name] = sha256(path)
            if not smoke:                             # the export must be the earlier run of the queue's agreeing pair
                folder = os.path.dirname(os.path.abspath(path))
                st = open(f'{folder}/{name}-status.txt').read().split()
                if st[0] != 'agreed' or os.path.basename(path) != f'{name}-{st[1]}.json.gz':
                    raise Refusal(f"{name}: not the earlier run of the queue's agreeing pair")
                rec = json.load(open(f'{folder}/{name}-{st[1]}-{st[2]}-compare.json'))
                if rec['agree'] is not True or rec['sha256'][0] != res['exports'][name]:
                    raise Refusal(f'{name}: the comparison record does not cover this export')
                res.setdefault('pairs', {})[name] = {'status': ' '.join(st), 'pair_sha256': rec['sha256']}
            cells, labels, z = read_steps(path, int(name[1:]) if not smoke else None, root, smoke)
            if smoke:
                res['instances'][name] = {'steps': [{'condition': l, 'z': [v.real, v.imag]} for l, v in zip(labels, z)]}
            else:
                res['instances'][name] = decide(cells, labels, z)
    except Refusal as exc:
        res['refusals']['refusal'] = str(exc)
    except (KeyError, OSError, ValueError, EOFError, TypeError, IndexError) as exc:
        res['refusals']['refusal'] = f'{type(exc).__name__}: {exc}'
    if smoke:
        res['outcome'] = 'none: a rehearsal'
    elif res['refusals']:
        res['outcome'] = 'NOT INTERPRETABLE'
    else:
        res.update(outcome(res['instances']))
    with open(out, 'w') as f:
        json.dump(res, f, indent=1, sort_keys=True)
        f.write('\n')
    print(res['outcome'], {k: f"{r.get('as_intended')}/{r.get('of')}" for k, r in res['instances'].items()}, res['refusals'])


if __name__ == '__main__':
    main(sys.argv[1:])
