"""Protocol 022's declared follow-up: the conductance readings (spec lines 245 to 250). Exploratory by declaration; they
never change the pilot's or protocol 023's outcome.

  (nm environment) python -B readings.py OUT.json INTACT.json.gz LESION.json.gz --v1 DIR --v2 DIR --v3 DIR

Each DIR is a validation's record folder (its *-result.json must say pass; V3's replay must be of this lesion export).
A missing DIR means that validation does not license its readings. --smoke skips the licence and the duration check
and marks the output as a smoke run (the stand-in in smoke/).

Window G = [20, 230) ms after each step's onset; groups: layer 4 excitatory cells within +-15 deg of 45 and 135 deg; the
block A plaids. Conductances per source class, rebuilt offline by conductance.py from the recorded spikes and the
exported lists: afferent (LGN ON + OFF), intracortical excitatory E (layer 4 excitatory + layer 2/3 excitatory),
inhibitory I (layer 4 inhibitory). G_Y(X) is a group's mean conductance of class Y over the steps of condition X.
  A_aff(g, p) = [G_aff(P) - G_aff(B)] / [(G_aff(T) - G_aff(B)) + (G_aff(M) - G_aff(B))], in both networks, needs V1 and V3;
  XOS on the afferent conductance (plaid_stats.xos, unchanged) and the intact spike XOS minus it, needs V1 and V3;
  MM1 m_I = mean over the combinations of [G_I(P) - G_I(T)] / [G_I(T) - G_I(B)], MM2 m_E likewise, need V2.
Rules as declared: a combination counts only if its denominator is at least 3 standard errors (the bootstrap SD of the
denominator); intervals from protocol 022's whole-step paired bootstrap (plaid_stats.Fit, draw: 10,000 replicates, seed
2202); MM1 and MM2 by Holm over two at 0.05, with margin 0.10.
STATED AMENDMENT, fixed before any pilot export was opened (README, "Amendment of 13 November"): the declared text speaks of "the 12 group-and-plaid
combinations" and "at least 9 of its 12". Twelve was the count of an earlier draft with six orthogonal plaids
(design-panel-r2.json: g in {G45, G135} and the 6 orthogonal plaids). The declared stimulus set has four block A plaids,
so eight combinations exist: {45, 135} x the four block A plaids. A reading here needs at least 6 of its 8, the same
three quarters. Another literal count, the three +-15 deg groups x four plaids = 12, is not taken: the 90 deg group has
no test grating in a 45/135 plaid, so MM1 and MM2 are undefined for it and could reach 8 of 12 at most. Under either
literal count MM1 and MM2 could never be made; the result says so beside every reading.
The afferent XOS carries protocol 022's own caution: where the mean cross drive lies outside +-0.10 it is not a
suppression index, and it is made only if each of its four own-single denominators is at least 3 standard errors."""
import gzip
import hashlib
import json
import math
import sys
sys.dont_write_bytecode = True
import numpy as np

WS = '<workspace>'
sys.path.insert(0, WS + '/outputs/felisyn-research/protocol-022/controller')
import plaid_stats as ps        # noqa: E402  (frozen, protocol 022)
import conductance as cd        # noqa: E402

CONN = WS + '/work/controller-022/plaids/pilot/connections'
SEQ = 'FullfieldDriftingSinusoidalGratingSequence'
TARGET = 'V1_Exc_L4'
GROUPS = (45, 135)
CLASSES = {'aff': ('V1AffConnectionOn', 'V1AffConnectionOff'), 'E': ('V1L4ExcL4ExcConnection', 'V1L23ExcL4ExcConnection'),
           'I': ('V1L4InhL4ExcConnection',)}
COMBINATIONS = tuple((g, p) for p in ps.BLOCK_A_PLAIDS for g in GROUPS)
NEEDED, SE_FLOOR, DELTA_M, ALPHA = 6, 3.0, 0.10, 0.05
LITERAL_NOTE = ('The declared text counts 12 combinations and needs 9; the declared stimulus set has 8 (two groups x four '
                'block A plaids), and this reading needs 6 of them (amendment fixed before any pilot export was opened). '
                'Under the literal count the reading would not be made.')


def sha256(path):
    h = hashlib.sha256()
    with open(path, 'rb') as fh:
        for b in iter(lambda: fh.read(1 << 20), b''):
            h.update(b)
    return h.hexdigest()


def clean(x):
    """JSON without NaN or Infinity (None instead)."""
    if isinstance(x, dict):
        return {k: clean(v) for k, v in x.items()}
    if isinstance(x, (list, tuple)):
        return [clean(v) for v in x]
    if isinstance(x, float) and not math.isfinite(x):
        return None
    return x


def roles(g, plaid):
    """The test single (g's own orientation at its contrast in the plaid) and the mask single (the other component)."""
    comps = dict(ps.COMPONENTS[plaid])
    other = [d for d in comps if d != g][0]
    return ps.single(g, comps[g]), ps.single(other, comps[other])


def summaries(export, classes, allow_short=False):
    """Per analysed step: each group's mean conductance per class in window G (nS) and its mean spike count per cell."""
    index = json.load(open(CONN + '/index.json'))
    proj = {p['name']: p for p in index['projections']}
    pops = index['populations']
    ex = json.load(gzip.open(export, 'rt'))
    seg = {}
    for s in ex['segments']:
        if s['stimulus'].get('name') == SEQ:
            if s['sheet'] in seg:
                raise SystemExit(f"{export}: more than one sequence presentation of {s['sheet']}")
            seg[s['sheet']] = s
    stim = seg[TARGET]['stimulus']
    frame = float(stim['frame_duration'])
    period = (float(stim['step_frames']) + float(stim['gap_frames'])) * frame
    lead = float(stim['lead_frames']) * frame
    steps = json.loads(stim['sequence']) if isinstance(stim['sequence'], str) else stim['sequence']
    if ps.components_of(steps[0]) != tuple((float(d), float(c)) for d, c in ps.WARMUP):
        raise SystemExit(f'{export}: the first step is not the declared warm-up')
    cond = [ps.condition_of(s) for s in steps[1:]]                     # the first step is the warm-up
    if None in cond or len(cond) != ps.N_ANALYSED or np.bincount(cond, minlength=ps.N_CONDITIONS).tolist() != [c[2] for c in ps.CONDITIONS]:
        raise SystemExit(f'{export}: the shown steps are not the declared sequence')
    if float(seg[TARGET].get('t_start_ms', 0.0)) != 0.0:
        raise SystemExit(f'{export}: the presentation does not start at 0')
    if not allow_short and float(stim['duration_ms']) + 1e-6 < lead + period * len(steps):
        raise SystemExit(f'{export}: the presentation is shorter than its {len(steps)} steps')
    a, b = ps.WINDOWS_MS['G']
    windows = [(lead + period * k + a, lead + period * k + b) for k in range(1, len(steps))]
    n_post = pops[TARGET]['size']
    prefs = np.asarray(ex['neurons'][TARGET]['orientation_pref'], float)
    ids = [int(i) for i in ex['neurons'][TARGET]['ids']]
    if ids != list(range(pops[TARGET]['first_id'], pops[TARGET]['first_id'] + n_post)):
        raise SystemExit(f'{export}: the neuron table is not the connection lists\' population')
    masks = {g: ps.group_mask(prefs, g) for g in GROUPS}
    neuron = index['neurons'][TARGET]['cells'][0]
    names, cols = [], []
    for cls in classes:
        total = np.zeros((n_post, len(windows)))
        for name in CLASSES[cls]:
            p = proj[name]
            sheet = p['source_sheet']
            first = pops[sheet]['first_id']
            spikes = {int(c) - first: [float(t) for t in v] for c, v in seg[sheet]['spikes'].items()}
            if spikes and not (0 <= min(spikes) and max(spikes) < pops[sheet]['size']):
                raise SystemExit(f'{export}: {sheet} holds a cell outside the lists\' population')
            if sha256(f"{CONN}/{p['file']}") != p['sha256']:
                raise SystemExit(f"{p['file']} is not the exported list")
            z = np.load(f"{CONN}/{p['file']}")
            if any(p['stp_min_max'][k][0] != p['stp_min_max'][k][1] for k in ('U', 'tau_rec', 'tau_psc', 'tau_fac')):
                raise SystemExit(f'{name}: a synapse parameter is not one value')
            stp = {k: p['stp_min_max'][k][0] for k in ('U', 'tau_rec', 'tau_psc', 'tau_fac')}
            tau = neuron['tau_syn_in'] if p['receptor_type'] == 'inhibitory' else neuron['tau_syn_ex']
            total += cd.window_means(z['pre'], z['post'], z['weight'].astype(float) * 1000.0, z['delay'], pops[sheet]['size'],
                                     n_post, spikes, stp, tau, windows)
        for g in GROUPS:
            names.append(f'{cls}|{g}'); cols.append(total[masks[g]].mean(axis=0))
    first = pops[TARGET]['first_id']
    rec = seg[TARGET]['spikes']
    for g in GROUPS:
        cells = [str(first + i) for i in np.nonzero(masks[g])[0]]
        col = []
        for lo, hi in windows:
            col.append(sum(np.searchsorted(np.sort(rec.get(c, ())), hi, 'left') - np.searchsorted(np.sort(rec.get(c, ())), lo, 'left') for c in cells) / len(cells))
        names.append(f'G|L4E|{g}'); cols.append(np.array(col))
    return names, np.array(cols).T, np.array(cond), {'export_sha256': sha256(export), 'group_cells': {g: int(m.sum()) for g, m in masks.items()},
                                                    'steps': len(steps), 'lead_ms': lead, 'period_ms': period}


def boot(names, Y, cond, idx):
    """The estimate and the replicates of every condition mean, by protocol 022's frozen Fit."""
    fit = ps.Fit(Y, np.zeros((len(cond), 1), complex), cond)
    M, Mz = fit.boot_means(idx)
    return ps.Means(fit.M, fit.Mz, names, ['none']), ps.Means(M, Mz, names, ['none'])


def combination_reading(e, b, numerator, denominator):
    """A reading over the eight combinations: each value = numerator / denominator, kept if the denominator is at least
    3 bootstrap standard errors; the mean over the kept ones, if at least six are kept."""
    parts, kept_e, kept_b = {}, [], []
    for g, plaid in COMBINATIONS:
        de, db = denominator(e, g, plaid), denominator(b, g, plaid)
        se = float(np.std(db, ddof=1))
        ok = bool(se > 0 and de >= SE_FLOOR * se) if se > 0 else bool(de > 0)
        with np.errstate(divide='ignore', invalid='ignore'):
            ve, vb = numerator(e, g, plaid) / de, numerator(b, g, plaid) / db
        parts[f'{g}|{plaid}'] = {'value': float(ve), 'denominator': float(de), 'denominator_se': se, 'kept': ok,
                                 'ci95': ps.interval(float(ve), vb)['ci95']}
        if ok:
            kept_e.append(ve); kept_b.append(vb)
    out = {'combinations': parts, 'kept': len(kept_e), 'of': len(COMBINATIONS), 'needed': NEEDED, 'note': LITERAL_NOTE}
    if len(kept_e) >= NEEDED:
        out['made'] = True
        out.update(ps.interval(float(np.mean(kept_e)), np.mean(np.stack(kept_b), axis=0)))
    else:
        out['made'] = False
        out['why'] = f'only {len(kept_e)} of 8 combinations have a denominator of at least 3 standard errors'
    return out


def a_aff(e, b):
    def num(m, g, p):
        return m.D(f'aff|{g}', p)

    def den(m, g, p):
        t, k = roles(g, p)
        return m.D(f'aff|{g}', t) + m.D(f'aff|{g}', k)
    return combination_reading(e, b, num, den)


def modulation(e, b, cls):
    def num(m, g, p):
        return m.R(f'{cls}|{g}', p) - m.R(f'{cls}|{g}', roles(g, p)[0])

    def den(m, g, p):
        return m.D(f'{cls}|{g}', roles(g, p)[0])
    return combination_reading(e, b, num, den)


def mm_verdicts(mm1, mm2):
    """Holm over the two made readings at 0.05; a direction is supported only past the margin of 0.10."""
    p = {k: (r['p'] if r.get('made') else 1.0) for k, r in (('MM1', mm1), ('MM2', mm2))}      # always a family of two
    h = ps.holm(p, ALPHA)
    out = {'holm': h, 'margin': DELTA_M}
    for k, r, sign, text in (('MM1', mm1, 1, 'the second grating raises the inhibition onto cells tuned to the first'),
                             ('MM2', mm2, -1, 'the second grating lowers the intracortical excitation onto cells tuned to the first')):
        if not r.get('made'):
            out[k] = 'not made'
        elif k in h['rejected'] and sign * r['estimate'] >= DELTA_M:
            out[k] = f'supported: {text} (Holm at 0.05, beyond the margin of 0.10)'
        elif k in h['rejected'] and -sign * r['estimate'] >= DELTA_M:
            out[k] = 'the opposite direction, beyond the margin (Holm at 0.05)'
        else:
            out[k] = 'not shown at this margin'
    return out


def afferent_xos(e, b):
    """XOS on the afferent conductance (plaid_stats.xos, unchanged), made only if each own-single denominator is at
    least 3 standard errors; with its mean cross drive and protocol 022's caution where that lies outside +-0.10."""
    xe, xb = ps.xos(e, 'aff|{g}'), ps.xos(b, 'aff|{g}')
    dens = {k: {'value': float(xe['own_single'][k]), 'se': float(np.std(xb['own_single'][k], ddof=1))} for k in xe['own_single']}
    ok = all(d['se'] > 0 and d['value'] >= SE_FLOOR * d['se'] for d in dens.values())
    out = {'made': ok, 'own_single_denominators': dens, 'cross_drive_mean': ps.interval(float(xe['cross_mean']), xb['cross_mean'])}
    if ok:
        out.update(ps.interval(float(xe['value']), xb['value']))
        out['parts'] = {k: float(v) for k, v in xe['parts'].items()}
        if abs(float(xe['cross_mean'])) > ps.CROSS_DRIVE_MAX:
            out['caution'] = ('the mean cross drive lies outside +-0.10: the orthogonal grating alone drives this conductance, '
                              'so this value is not a suppression index comparable with the spike XOS (protocol 022, P1 validity)')
    else:
        out['why'] = 'an own-single denominator is below 3 standard errors'
    return out, xe, xb


def licence(dirs, lesion):
    """Which validations license readings: each record folder's *-result.json must say pass, and V3's replay must be
    of this lesion export."""
    import glob
    import os
    out = {}
    for v in ('V1', 'V2', 'V3'):
        d = dirs.get(v)
        if not d:
            continue
        files = sorted(glob.glob(os.path.join(d, '*-result.json')))
        if len(files) != 1:
            raise SystemExit(f'{v}: {d} holds {len(files)} result files, not one')
        r = json.load(open(files[0]))
        if r.get('pass') is not True or r.get('valid_attempt') is not True:
            raise SystemExit(f'{v}: {files[0]} does not record a pass')
        out[v] = {'result': files[0], 'sha256': sha256(files[0])}
        if v == 'V3':
            c = json.load(open(os.path.join(d, 'p0a.criteria.json')))
            if c['export_sha256'] != sha256(lesion) or c['sources'] != 'aff':
                raise SystemExit('V3: its replay is not of this lesion export from the afferent lists')
            out[v]['export_sha256'] = c['export_sha256']
    return out


def compute(intact, lesion, licensed, idx=None, allow_short=False):
    idx = ps.draw() if idx is None else idx
    res = {'scope': 'protocol 022 follow-up: conductance readings; exploratory; simulation only; no claim about cat cortex',
           'licensed': sorted(licensed), 'combinations': [f'{g}|{p}' for g, p in COMBINATIONS],
           'bootstrap': {'replicates': int(idx.shape[0]), 'seed': ps.SEED}, 'runs': {}}
    aff_ok, mm_ok = {'V1', 'V3'} <= licensed, 'V2' in licensed
    tables = {}
    for run, path in (('intact', intact), ('lesion', lesion)):
        classes = [c for c in ('aff', 'E', 'I') if (c == 'aff' and aff_ok) or (c != 'aff' and mm_ok and run == 'intact')]
        if not classes:
            continue
        names, Y, cond, meta = summaries(path, classes, allow_short)
        e, b = boot(names, Y, cond, idx)
        r = {'meta': meta, 'condition_means': {n: {lab: float(e.R(n, lab)) for lab in ps.LABELS} for n in names}}
        if aff_ok:
            tables[run] = Y[:, [names.index(f'aff|{g}') for g in GROUPS]]
            r['A_aff'] = a_aff(e, b)
            r['XOS_afferent_conductance'], xa_e, xa_b = afferent_xos(e, b)
            if run == 'intact':                 # the declared reading: the intact XOS minus the same measure on the conductance
                xs_e, xs_b = ps.xos(e, 'G|L4E|{g}'), ps.xos(b, 'G|L4E|{g}')
                r['XOS_spikes'] = ps.interval(float(xs_e['value']), xs_b['value'])
                if r['XOS_afferent_conductance']['made']:
                    r['XOS_spikes_minus_afferent'] = ps.interval(float(xs_e['value'] - xa_e['value']), xs_b['value'] - xa_b['value'])
        if mm_ok and run == 'intact':
            r['MM1_m_I'], r['MM2_m_E'] = modulation(e, b, 'I'), modulation(e, b, 'E')
            r['MM_verdicts'] = mm_verdicts(r['MM1_m_I'], r['MM2_m_E'])
        res['runs'][run] = r
    if len(tables) == 2:
        diff = float(np.abs(tables['intact'] - tables['lesion']).max())
        res['afferent_tables'] = {'largest_difference_nS': diff, 'equal': diff == 0.0,
                                  'note': 'the two runs share every LGN spike and the same lists, so "in both networks" is one afferent table if equal'}
    return res


def main(argv):
    import os
    here = os.path.dirname(os.path.abspath(__file__))
    smoke = '--smoke' in argv
    argv = [a for a in argv if a != '--smoke']
    dirs = {}
    for v in ('V1', 'V2', 'V3'):
        flag = '--' + v.lower()
        if flag in argv:
            i = argv.index(flag); dirs[v] = argv[i + 1]; argv = argv[:i] + argv[i + 2:]
    out, intact, lesion = argv[0], argv[1], argv[2]
    lic = {} if smoke else licence(dirs, lesion)
    res = compute(intact, lesion, {'V1', 'V2', 'V3'} if smoke else set(lic), allow_short=smoke)
    res['smoke'] = smoke
    res['licence'] = lic
    res['tool_sha256'] = {f: sha256(os.path.join(here, f)) for f in ('readings.py', 'conductance.py')}
    res['inputs_sha256'] = {'plaid_stats.py': sha256(ps.__file__), 'connections/index.json': sha256(CONN + '/index.json')}
    json.dump(clean(res), open(out, 'w'), indent=1, sort_keys=True, allow_nan=False)
    for run, r in res['runs'].items():
        for k in ('A_aff', 'XOS_afferent_conductance', 'XOS_spikes', 'XOS_spikes_minus_afferent', 'MM1_m_I', 'MM2_m_E'):
            if k in r:
                v = r[k]
                print(run, k, 'not made' if v.get('made') is False else f"{v['estimate']:+.4f} [{v['ci95'][0]:+.4f}, {v['ci95'][1]:+.4f}]")
        if 'MM_verdicts' in r:
            print(run, {k: r['MM_verdicts'][k] for k in ('MM1', 'MM2')})
    if 'afferent_tables' in res:
        print('afferent tables equal in both runs:', res['afferent_tables']['equal'])


if __name__ == '__main__':
    main(sys.argv[1:])
