"""Protocol 026: decisions over three steps. Protocol 024's rule on the mean of three consecutive steps, fresh runs.

  python3 -B decide_026.py OUT.json sK=EXPORT.json.gz sK=EXPORT.json.gz sK=EXPORT.json.gz

Per step, nothing is refitted: z is protocol 017's frozen readout (all layer 4 excitatory cells, balanced weights,
counts 40 to 300 ms after onset); z0 the mean z over the instance's 8 single blank steps; S half the difference of
Im mean z for A alone at 30 % and B alone at 30 % (8 single steps each; refused below 1,500); v = -Im(z - z0) / S.
New here: a block is the same stimulus for three steps in a row, and the block's decision is read from the mean of
its three v: action B if the mean is above 0.32, A if below -0.32, else hold (protocol 024's threshold).

Scored blocks, 26 per instance, with the intended action (a, b are the strengths of A and B in percent):
  close   two signals, strengths differ by 7.5 (6 cells, 2 blocks each)        the stronger's action   12 blocks
  equal   7.5, 15, 22.5 and 30 on both (2 blocks each)                         hold                     8 blocks
  clear   A15, B15, A30+B15, A15+B30, A22.5+B7.5, A7.5+B22.5 (1 block each)    the stronger's action    6 blocks
Not scored: the two blocks of one signal alone at 7.5 (described), and the calibration steps.

Outcome over the three analysed instances (the seeds the queue settled with an agreeing pair, three of 18, 19, 20 and
the reserve 21):
  PASS               every instance takes the intended action in at least 24 of its 26 scored blocks, and the three
                     together in at least 75 of 78;
  FAIL               otherwise;
  NOT INTERPRETABLE  a refusal: not exactly three instances, an export that is not the declared seed's run of its
                     declared sequence or not the earlier run of the queue's agreeing pair, a lead-in whose LGN
                     spike trains differ from the reference of protocol 023, S below 1,500, or frozen code not at
                     its recorded hashes.
Reported with no decision role: every step's v and single-step action (blocks and calibration singles), the single-step decision of each block's
first, second and third step, the decision from the first two steps, and the mean v by position in the block.
Run from work/controller-026 of the workspace. Simulation only; an engineering rule, not a claim about cat cortex.
"""
import gzip, hashlib, json, os, sys, zlib
import numpy as np

sys.dont_write_bytecode = True
T = 0.32
MIN_SCALE = 1500.0
BLOCK = 3
NEED_INSTANCE, OF_INSTANCE, N_INSTANCES = 24, 26, 3
NEED_POOLED, OF_POOLED = 75, 78
SEEDS = (18, 19, 20, 21)
SEQUENCES = {18: 'ee01fb3284d1037660c2c8bea70aabac63b907700c441d9d29c617fd6c3e7c19',
             19: 'f3a5584ff53a97948367cc398075be8cd29b6b2f1f3c5a89a41a2046b525e484',
             20: '4e92ddf32f444602ce1f9d78af139488970449cb5b114cf31721e6442120ff1a',
             21: '380571d19dd1459b98019d50a8d6f8327c8cd5e87a8fbb87331984282f9a5d46'}
FROZEN = {'protocol-017/pipeline.py': '9801606903df981ab15966d2b0182a4ebb4bdea4e9ad24e427b3165f3abc51d4'}
LEADIN = ('work/controller-026/ref/leadin-reference-023.json', '126485e68db25f813f33f9c3a34c55f025c60bba3f2564f8cc64ac4c9a893645')
CAL = ('blank', 'A30', 'B30')
CLEAR = ((15, 0), (0, 15), (30, 15), (15, 30), (22.5, 7.5), (7.5, 22.5))
WINDOW_MS = (40.0, 300.0)
KINDS = ('close', 'equal', 'clear', 'weak')


class Refusal(Exception):
    pass


def action(v, t=T):
    return 'B' if v > t else ('A' if v < -t else 'hold')


def role(a, b):
    """(kind, intended action or None) of a block with strengths a and b; a Refusal for a cell that is no block cell."""
    if a == b and a > 0:
        return 'equal', 'hold'
    if a and b and abs(a - b) == 7.5:
        return 'close', 'A' if a > b else 'B'
    if (a, b) in CLEAR:
        return 'clear', 'A' if a > b else 'B'
    if (a, b) in ((7.5, 0), (0, 7.5)):
        return 'weak', None
    raise Refusal(f'the cell ({a:g}, {b:g}) is not a block cell of this protocol')


def decide(cells, labels, z, blocks):
    """cells: (a, b) per analysed step; labels: its condition; z: its complex readout; blocks: its block number, 0 for
    a calibration single."""
    z = np.asarray(z, complex)
    lab = np.asarray(labels)
    blk = np.asarray(blocks, int)
    for c in CAL:
        if ((lab == c) & (blk == 0)).sum() != 8 or (lab == c).sum() != 8:
            raise Refusal(f'{c}: not 8 single calibration steps')
    if (blk == 0).sum() != 24:
        raise Refusal('a single step is not a calibration step')
    z0 = z[lab == 'blank'].mean()
    s = float((z[lab == 'A30'].mean().imag - z[lab == 'B30'].mean().imag) / 2)
    if not np.isfinite(s) or s < MIN_SCALE:
        raise Refusal(f'the response scale S is {s:.1f}, below {MIN_SCALE:.0f}')
    v = -(z - z0).imag / s
    out, hits, scored = [], 0, 0
    tally = {k: {'blocks': 0, 'actions': {'A': 0, 'hold': 0, 'B': 0}, 'as_intended': 0, 'first_step': 0, 'second_step': 0,
                 'third_step': 0, 'first_two': 0, 'toward': [[], [], []]} for k in KINDS}
    for b in sorted(set(blk.tolist()) - {0}):
        idx = np.flatnonzero(blk == b)
        if len(idx) != BLOCK or idx[-1] - idx[0] != BLOCK - 1 or len({cells[i] for i in idx}) != 1:
            raise Refusal(f'block {b} is not three consecutive steps of one cell')
        a, bb = cells[idx[0]]
        kind, intended = role(a, bb)
        vs = [float(v[i]) for i in idx]
        m = float(np.mean(vs))
        rec = {'block': int(b), 'first_analysed_step': int(idx[0]) + 1, 'condition': labels[idx[0]], 'a': a, 'b': bb, 'kind': kind, 'intended': intended, 'v': vs,
               'v_mean': m, 'action': action(m), 'step_actions': [action(x) for x in vs], 'action_first_two': action(float(np.mean(vs[:2])))}
        t = tally[kind]
        t['blocks'] += 1
        t['actions'][rec['action']] += 1
        sign = -1.0 if a > bb else 1.0                       # v toward the stronger signal's side; plain v for equal cells
        for p in range(BLOCK):
            t['toward'][p].append(vs[p] * (sign if kind != 'equal' else 1.0))
        if intended is not None:
            scored += 1
            ok = rec['action'] == intended
            hits += ok
            t['as_intended'] += ok
            for p, name in enumerate(('first_step', 'second_step', 'third_step')):
                t[name] += rec['step_actions'][p] == intended
            t['first_two'] += rec['action_first_two'] == intended
        out.append(rec)
    want = {'close': 12, 'equal': 8, 'clear': 6, 'weak': 2}
    if {k: t['blocks'] for k, t in tally.items()} != want or scored != OF_INSTANCE:
        raise Refusal(f'the blocks are {[(k, t["blocks"]) for k, t in tally.items()]}, not {want}')
    for k, t in tally.items():
        t['mean_v_toward_by_position'] = [float(np.mean(x)) for x in t.pop('toward')]
        if k == 'weak':                                      # no intended action: the tallies of hits do not apply
            for name in ('as_intended', 'first_step', 'second_step', 'third_step', 'first_two'):
                t[name] = None
    singles = [{'analysed_step': int(i) + 1, 'condition': labels[i], 'v': float(v[i]), 'action': action(float(v[i]))}
               for i in np.flatnonzero(blk == 0)]
    cal = {c: {'v_mean': float(v[lab == c].mean()), 'v_min': float(v[lab == c].min()), 'v_max': float(v[lab == c].max())} for c in CAL}
    return {'z0': [float(z0.real), float(z0.imag)], 'S': s, 'threshold': T, 'as_intended': hits, 'of': scored,
            'by_kind': tally, 'calibration': cal, 'singles': singles, 'blocks': out}


def outcome(per_instance):
    hits = {k: r['as_intended'] for k, r in per_instance.items()}
    pooled = sum(hits.values())
    ok = len(hits) == N_INSTANCES and all(h >= NEED_INSTANCE for h in hits.values()) and pooled >= NEED_POOLED
    return {'outcome': 'PASS' if ok else 'FAIL', 'as_intended': hits, 'pooled': pooled, 'of_pooled': OF_POOLED,
            'rule': f'every instance at least {NEED_INSTANCE} of {OF_INSTANCE}, and the three at least {NEED_POOLED} of {OF_POOLED}'}


def sha256(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for b in iter(lambda: f.read(1 << 20), b''):
            h.update(b)
    return h.hexdigest()


def leadin_problem(ex, reference):
    """None if every LGN train before the lead-in's end equals the reference's; else a short description."""
    lead = reference['lead_ms']
    for sh in ('X_ON', 'X_OFF'):
        seg = [s for s in ex['segments'] if s['sheet'] == sh and 'sequence' in s['stimulus']]
        if len(seg) != 1:
            return f'{len(seg)} sequence presentations of {sh}'
        got = {i: [float(t) for t in ts if float(t) < lead] for i, ts in seg[0]['spikes'].items()}
        ref = reference['trains'][sh]
        if got.keys() != ref.keys():
            return f'{sh}: the cells differ from the reference'
        same = sum(sorted(got[i]) == sorted(float(t) for t in ref[i]) for i in ref)
        if same != len(ref):
            return f'{sh}: {same} of {len(ref)} lead-in trains identical to the reference'
    return None


def read_steps(path, k, root):
    """Cells, labels and z of every analysed step of seed k's export, after checking what was run and shown."""
    sys.path.insert(0, root + '/outputs/felisyn-research/protocol-017/controller')
    import pipeline as p
    ex = json.load(gzip.open(path, 'rt'))
    seqdoc = json.load(open(f'{root}/work/controller-026/sequences/sequence-026-s{k}.json'))
    if sha256(f'{root}/work/controller-026/sequences/sequence-026-s{k}.json') != SEQUENCES[k]:
        raise Refusal(f's{k}: the sequence file is not the declared one')
    if str(ex['resolved'].get('pynn_seed')) != str(k) or 'pynn_seed' not in ex['argv'] or ex['argv'][ex['argv'].index('pynn_seed') + 1] != str(k):
        raise Refusal(f's{k}: the export is not a run of pynn_seed {k}')
    bad = leadin_problem(ex, json.load(open(f'{root}/{LEADIN[0]}')))
    if bad:
        raise Refusal(f's{k}: lead-in: {bad}')
    seg = [s for s in ex['segments'] if s['sheet'] == 'V1_Exc_L4' and 'sequence' in s['stimulus']]
    if len(seg) != 1:
        raise Refusal(f'{len(seg)} sequence presentations of layer 4, not 1')
    st = seg[0]['stimulus']
    frame = float(st['frame_duration'])
    period = (float(st['step_frames']) + float(st['gap_frames'])) * frame
    lead = float(st['lead_frames']) * frame
    if (float(st['step_frames']), float(st['gap_frames']), lead, frame) != (30.0, 21.0, 504.0, 7.0):
        raise Refusal('the timing is not 30 + 21 frames of 7 ms after a 504 ms lead-in')
    shown = json.loads(st['sequence']) if isinstance(st['sequence'], str) else st['sequence']
    a45, b135 = round(np.pi / 4, 6), round(3 * np.pi / 4, 6)
    cells = []
    for s in shown:
        s = [float(x) for x in s]
        if len(s) == 2:
            cells.append((s[1], 0.0) if s[0] == a45 else (0.0, s[1]) if s[0] == b135 else None)
        else:
            cells.append((s[1], s[3]) if (s[0], s[2]) == (a45, b135) else None)
    if None in cells:
        raise Refusal('a shown step is not a cell of the grid')
    want = [(float(x['a']), float(x['b'])) for x in seqdoc['steps']]
    if cells != want:
        raise Refusal(f's{k}: the shown steps are not the declared sequence')
    if float(seg[0].get('t_start_ms', 0.0)) != 0.0 or float(st['duration_ms']) + 1e-6 < lead + period * len(shown):
        raise Refusal('the presentation does not start at 0 or is shorter than its steps')
    ids = ex['neurons']['V1_Exc_L4']['ids']
    w = p.decoder_weights(np.asarray(ex['neurons']['V1_Exc_L4']['orientation_pref'], float), 'balanced')
    sp = seg[0]['spikes']
    trains = [np.sort(np.asarray(sp.get(str(i), []), float)) for i in ids]
    z = []
    for n in range(1, len(shown)):                                    # step 0 is the warm-up
        lo, hi = lead + period * n + WINDOW_MS[0], lead + period * n + WINDOW_MS[1]
        c = np.array([np.searchsorted(t, hi, 'left') - np.searchsorted(t, lo, 'left') for t in trains])
        z.append(complex(np.sum(c * w)))
    cells = cells[1:]
    labels = ['blank' if a == 0 and b == 0 else '+'.join(q for q in (f'A{a:g}' if a else '', f'B{b:g}' if b else '') if q) for a, b in cells]
    blocks = [int(x['block']) for x in seqdoc['steps'][1:]]
    return cells, labels, z, blocks


def main(argv):
    out, runs = argv[0], dict(a.split('=', 1) for a in argv[1:])
    root = os.path.abspath(__file__).rsplit('/work/', 1)[0]
    res = {'scope': 'protocol 026, decisions over three steps; simulation only; an engineering rule',
           'threshold': T, 'min_scale': MIN_SCALE, 'tool_sha256': sha256(__file__), 'exports': {}, 'instances': {}, 'refusals': {}}
    try:
        if sha256(root + '/outputs/felisyn-research/protocol-017/controller/pipeline.py') != FROZEN['protocol-017/pipeline.py']:
            raise Refusal('the frozen readout code is not at its recorded hash')
        if sha256(f'{root}/{LEADIN[0]}') != LEADIN[1]:
            raise Refusal('the lead-in reference is not at its recorded hash')
        seeds = sorted(int(n[1:]) for n in runs)
        if len(seeds) != N_INSTANCES or not set(seeds) <= set(SEEDS):
            raise Refusal(f'the runs given are {sorted(runs)}, not three of the declared seeds')
        for name, path in runs.items():
            res['exports'][name] = sha256(path)
            folder = os.path.dirname(os.path.abspath(path))          # the export must be the earlier run of the queue's agreeing pair
            st = open(f'{folder}/{name}-status.txt').read().split()
            if st[0] != 'agreed' or os.path.basename(path) != f'{name}-{st[1]}.json.gz':
                raise Refusal(f"{name}: not the earlier run of the queue's agreeing pair")
            rec = json.load(open(f'{folder}/{name}-{st[1]}-{st[2]}-compare.json'))
            if rec['agree'] is not True or rec['sha256'][0] != res['exports'][name]:
                raise Refusal(f'{name}: the comparison record does not cover this export')
            res.setdefault('pairs', {})[name] = {'status': ' '.join(st), 'pair_sha256': rec['sha256']}
            res['instances'][name] = decide(*read_steps(path, int(name[1:]), root))
    except Refusal as exc:
        res['refusals']['refusal'] = str(exc)
    except (KeyError, OSError, ValueError, EOFError, TypeError, IndexError, ImportError, zlib.error) as exc:
        res['refusals']['refusal'] = f'{type(exc).__name__}: {exc}'
    if res['refusals']:
        res['outcome'] = 'NOT INTERPRETABLE'
    else:
        res.update(outcome(res['instances']))
    with open(out, 'w') as f:
        json.dump(res, f, indent=1, sort_keys=True)
        f.write('\n')
    print(res['outcome'], {k: f"{r.get('as_intended')}/{r.get('of')}" for k, r in res['instances'].items()}, res['refusals'])


if __name__ == '__main__':
    main(sys.argv[1:])
