"""Protocol 018: compute the declared fluctuation measures for Hahn et al. 2017's cat data and run 011, and the
declared readings. Refuses to run unless the inputs match their SHA-256 in the contract.

  python3 compare.py CONTRACT.json OUT.json

Primary measure: fluctuations.within_segment_pairwise at 100 ms, the shared latent-rate fluctuation between
units pooled over all unit pairs, measured inside 1 s segments. Units: every recording site (data); 18 model
pools, one per layer at each point of a 3 x 3 grid 400 um apart (model).
Uncertainty: a bootstrap over segments (2000 resamples) for every estimate, and a counting-noise null with each
unit's spikes redistributed uniformly within its segments (200 shuffles).
Reading, per state group: the model's 95 % interval against the range spanned by the four cats' 95 % intervals.
Robustness: the same reading against data with near-simultaneous multi-site events removed (three or more sites
within +-0.2 ms, a sign of common-mode artifacts; data only, and removal can only lower the data's value), and for
every model variant.
"""
import hashlib, json, sys, tempfile, zipfile
from pathlib import Path
import numpy as np
import fluctuations as f
import hahn_data, model_data

BOOT, NULL_SHUFFLES, N_SEGMENTS, DRAWS, SEED = 2000, 200, 40, 2000, 18
PRIMARY_BIN_S = 0.1
CURVE_BINS_S = (0.01, 0.05, 0.1, 0.25)
SESSION_BINS_S = (0.1, 1.0)
GROUPS = {'desynchronized (1-2)': (1, 2), 'desynchronized I': (1,), 'desynchronized II': (2,), 'synchronized fast': (3,),
          'synchronized slow I': (4,), 'synchronized slow II': (5,), 'synchronized (3-5)': (3, 4, 5)}
PRIMARY_GROUP = 'desynchronized (1-2)'
COINCIDENCE_WINDOW_TICKS, COINCIDENCE_SHOULDER_TICKS, COINCIDENCE_MIN_SITES = 2, (10, 50), 3   # ticks of 0.1 ms
READINGS = {'below': 'less shared fluctuation than {group} cat cortex', 'within': 'within the range of {group} cat cortex',
            'above': 'more shared fluctuation than {group} cat cortex'}
PRIMARY_UNITS = 'pools: 3 x 3 grid 400 um apart, L4 and L2/3, all populations, 100 um'
# name: (populations, grid spacing um, use pools (True) or single neurons within 500 um (False), first segment used)
MODEL_VARIANTS = {
    PRIMARY_UNITS: (model_data.POPULATIONS, 400.0, True, 0),
    'pools, excitatory only': (model_data.EXCITATORY, 400.0, True, 0),
    'pools, inhibitory only': (model_data.INHIBITORY, 400.0, True, 0),
    'pools 800 um apart': (model_data.POPULATIONS, 800.0, True, 0),
    'single neurons within 500 um, all pairs': (model_data.POPULATIONS, None, False, 0),
    'pools, first 2 s dropped': (model_data.POPULATIONS, 400.0, True, 2),
}


def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def refuse(msg):
    raise SystemExit(f'refused: {msg}')


def interval(x):
    x = np.asarray(x, float)
    x = x[np.isfinite(x)]
    return [float(np.percentile(x, 2.5)), float(np.percentile(x, 97.5))] if len(x) else [float('nan'), float('nan')]


def by_unit(trains, starts, bin_s):
    return np.array([f.segment_counts(t, starts, bin_s) for t in trains])


def estimate(trains, starts, rng):
    """Primary measure with its bootstrap interval, its counting-noise null and the time-scale curve."""
    boot, shuffles = BOOT, NULL_SHUFFLES
    counts = by_unit(trains, starts, PRIMARY_BIN_S)
    null = [f.within_segment_pairwise(f.shuffle_within_segments(counts, rng)) for _ in range(shuffles)]
    return {'value': f.within_segment_pairwise(counts), 'ci95': interval(f.bootstrap_segments(f.within_segment_pairwise, counts, boot, rng)),
            'null_mean': float(np.mean(null)), 'null_95': interval(null),
            'curve': {str(b): f.within_segment_pairwise(by_unit(trains, starts, b)) for b in CURVE_BINS_S},
            'units': len(trains), 'segments': int(len(starts)), 'spikes': int(counts.sum())}


def span(intervals):
    """The range spanned by the cats' intervals; undefined (nan) if any bound is not finite."""
    bounds = [b for c in intervals for b in c]
    if not all(np.isfinite(bounds)):
        return [float('nan'), float('nan')]
    return [min(c[0] for c in intervals), max(c[1] for c in intervals)]


def position(ci, ref):
    if not all(np.isfinite(list(ci) + list(ref))):
        return 'undefined'
    return 'below' if ci[1] < ref[0] else ('above' if ci[0] > ref[1] else 'within')


def main(contract_path, out_path):
    contract_path = Path(contract_path)
    contract = json.loads(contract_path.read_text())
    base = contract_path.parent
    paths = {k: (base / v['path']).resolve() for k, v in contract['inputs'].items()}
    for k, p in paths.items():
        if sha(p) != contract['inputs'][k]['sha256']:
            refuse(f'{k} ({p.name}) does not match its SHA-256 in the contract')
    rng = np.random.default_rng(SEED)
    res = {'scope': 'protocol 018 results: shared population fluctuations of run 011 against Hahn et al. 2017 (cat area 17)',
           'contract_sha256': sha(contract_path),
           'parameters': {'boot': BOOT, 'null_shuffles': NULL_SHUFFLES, 'segments_per_draw': N_SEGMENTS, 'draws': DRAWS, 'seed': SEED,
                          'primary_bin_s': PRIMARY_BIN_S, 'primary_units': PRIMARY_UNITS},
           'data': {}, 'reference': {}, 'model': {}, 'readings': {}}
    with tempfile.TemporaryDirectory() as tmp:
        with zipfile.ZipFile(paths['hahn_zip']) as z:
            for i in range(1, 5):
                z.extract(f'cat{i}_data.txt', tmp)
        for i in range(1, 5):
            d = hahn_data.load_cat(Path(tmp) / f'cat{i}_data.txt')
            sites = [d['sites'][s] for s in sorted(d['sites'])]
            stable = [d['sites'][s] for s in d['stable_sites']]
            masks, ratio, flagged, events = f.cross_site_coincidences(sites, COINCIDENCE_WINDOW_TICKS, COINCIDENCE_SHOULDER_TICKS,
                                                                      COINCIDENCE_MIN_SITES)
            cleaned = [t[~m] for t, m in zip(sites, masks)]
            cat = {'sites': len(sites), 'stable_sites': len(stable), 'unstable_sites': d['unstable_sites'],
                   'mixed_windows': d['mixed_windows'], 'groups': {}, 'groups_coincidences_removed': {},
                   'cross_site_coincidences': {'window_ticks': COINCIDENCE_WINDOW_TICKS, 'shoulder_ticks': list(COINCIDENCE_SHOULDER_TICKS),
                                               'min_sites': COINCIDENCE_MIN_SITES, 'peak_to_shoulder': ratio, 'flagged_fraction': flagged,
                                               'events': events, 'spikes_removed': int(sum(m.sum() for m in masks)),
                                               'flagged_fraction_by_group': {}}}
            for g, states in GROUPS.items():
                starts = np.sort(np.concatenate([d['segments'][s] for s in states]))
                e = estimate(sites, starts, rng)
                counts = by_unit(sites, starts, PRIMARY_BIN_S)
                glob = [f.within_segment_pairwise(counts[:, rng.choice(len(starts), N_SEGMENTS, replace=False)]) for _ in range(DRAWS)]
                local = [f.within_segment_pairwise(counts[:, idx]) for idx in f.local_draw_indices(starts, N_SEGMENTS, DRAWS, rng)]
                e['secondary'] = {
                    'draws_40_global_95': interval(glob), 'draws_40_local_95': interval(local),
                    'pooled_within_excess_cv2_100ms': f.within_segment_excess_cv2(f.population_counts(sites, starts, PRIMARY_BIN_S)),
                    'session_excess_cv2_stable_sites': {str(b): f.excess_cv2(f.population_counts(stable, starts, b)) for b in SESSION_BINS_S}}
                e['_global'], e['_local'] = glob, local
                cat['groups'][g] = e
                cat['groups_coincidences_removed'][g] = estimate(cleaned, starts, rng)
                in_group = sum(int(f.population_counts([x[m]], starts, 1.0).sum()) for x, m in zip(sites, masks))
                cat['cross_site_coincidences']['flagged_fraction_by_group'][g] = in_group / max(e['spikes'], 1)
            res['data'][f'cat{i}'] = cat
    for g in GROUPS:
        cis = [res['data'][f'cat{i}']['groups'][g]['ci95'] for i in range(1, 5)]
        clean = [res['data'][f'cat{i}']['groups_coincidences_removed'][g]['ci95'] for i in range(1, 5)]
        res['reference'][g] = {'range_of_cat_intervals': span(cis),
                               'cat_values': [res['data'][f'cat{i}']['groups'][g]['value'] for i in range(1, 5)],
                               'range_coincidences_removed': span(clean)}
    model = model_data.load_model(paths['run011_spikes'], paths['positions'])
    for name, (pops, spacing, use_pools, first) in MODEL_VARIANTS.items():
        trains = model_data.layer_pools(model, pops, spacing)[1] if use_pools else model_data.select(model, pops, 500.0)
        starts = model_data.SEGMENT_STARTS_S[first:]
        e = estimate(trains, starts, rng)
        if name == PRIMARY_UNITS:
            e['secondary'] = {'pooled_within_excess_cv2_100ms': f.within_segment_excess_cv2(f.population_counts(trains, starts, PRIMARY_BIN_S)),
                              'session_excess_cv2': {str(b): f.excess_cv2(f.population_counts(trains, starts, b)) for b in SESSION_BINS_S}}
        res['model'][name] = e
    primary = res['model'][PRIMARY_UNITS]
    for g in GROUPS:
        ref = res['reference'][g]['range_of_cat_intervals']
        pos = position(primary['ci95'], ref)
        variants = {name: position(res['model'][name]['ci95'], ref) for name in MODEL_VARIANTS}
        defined = [v for v in variants.values() if v != 'undefined']
        cleaned_pos = position(primary['ci95'], res['reference'][g]['range_coincidences_removed'])
        pooled_global = np.concatenate([res['data'][f'cat{i}']['groups'][g]['_global'] for i in range(1, 5)])
        res['readings'][g] = {'model_value': primary['value'], 'model_ci95': primary['ci95'], 'reference': ref, 'position': pos,
                              'text': READINGS[pos].format(group=g) if pos in READINGS else 'undefined: an interval is not finite',
                              'variant_positions': variants, 'undefined_variants': [k for k, v in variants.items() if v == 'undefined'],
                              'robust_to_model_units': all(v == pos for v in defined),
                              'position_coincidences_removed': cleaned_pos, 'robust_to_coincidence_removal': cleaned_pos == pos,
                              'percentile_of_model_in_global_draws': float(np.mean(pooled_global < primary['value']) * 100)}
    for i in range(1, 5):
        for g in GROUPS:
            res['data'][f'cat{i}']['groups'][g].pop('_global'), res['data'][f'cat{i}']['groups'][g].pop('_local')
    res['primary_reading'] = res['readings'][PRIMARY_GROUP]
    Path(out_path).write_text(json.dumps(res, indent=1) + '\n')
    print(json.dumps({g: {k: r[k] for k in ('position', 'model_value', 'model_ci95', 'reference', 'robust_to_model_units',
                                            'robust_to_coincidence_removal')}
                      for g, r in res['readings'].items()}, indent=1))


if __name__ == '__main__':
    main(*sys.argv[1:3])
