"""Protocol-013 model side: spontaneous Vm synchrony and Vm autocorrelation of run 011.

Follows Yu & Ferster (2010, Neuron 68:1187) for pairs and Azouz & Gray (1999, J Neurosci 19:2209)
for single neurons, adapted to the model's 1 kHz voltage traces as declared in README.md. Reads
the four voltage exports, the spike export and positions.json; every input is checked against
the SHA-256 in contract.json.

Spike removal (the model paper's rule, primary): samples within 2.5 ms of a spike time are
replaced by linear interpolation between the samples just outside that window. Sensitivity for
tau ('hold', closer to Azouz & Gray, who replaced spike samples by the threshold voltage): the
same samples are replaced by the last sample before the window.

Pairs (Yu & Ferster): 40 non-overlapping 1 s blocks; mean removed per block; per block
  R(tau) = sum_t x(t + tau) y(t) / sqrt(sum x^2 sum y^2)   (Matlab xcorr 'coeff');
R averaged over blocks; peak = maximum within 10 ms of zero lag; width = full width at half of
that peak. The same after a high-pass of the whole trace (Yu & Ferster: "20-Hz Butterworth";
order 4 and zero phase are our assumptions). Coherence: 7 Slepian tapers (NW = 4) per 1 s block,
cross- and auto-spectra averaged over blocks and tapers, |S_xy| / sqrt(S_xx S_yy) (raw, not
Fisher-transformed); band means 1-10 Hz and 20-80 Hz.

Neurons (Azouz & Gray): 3-point median filter after spike removal; 80 non-overlapping 500 ms
epochs, mean removed; autocorrelation at lags 0-100 ms averaged over epochs; least-squares fit
y = a + b exp(-x / c) on lags 0-100 ms; tau = c. Fits that fail or end within 1% of a parameter
bound are counted as failures. Azouz & Gray do not state their autocorrelation normalisation:
the unbiased per-lag estimate is primary, the 'coeff' estimate a sensitivity check.

  python vm_synchrony.py --analysis RUN_DIRECTORY/analysis --positions positions.json --output OUT
"""
import argparse, gzip, hashlib, json, platform
from pathlib import Path
import numpy as np
import scipy
from scipy.signal import butter, filtfilt, medfilt
from scipy.signal.windows import dpss
from scipy.optimize import curve_fit

HERE = Path(__file__).resolve().parent
BLOCK, N_BLOCKS = 1000, 40                  # 1 s blocks at 1 ms sampling
NFFT = 2048                                 # >= 2 * BLOCK - 1, so circular correlation equals linear
PEAK_WINDOW, WIDTH_MAXLAG = 10, 500         # ms
HIGHPASS_HZ, HIGHPASS_ORDER = 20.0, 4
TAPER_NW, TAPER_K, MAX_FREQ_HZ = 4, 7, 100
LOW_BAND, HIGH_BAND = (1, 10), (20, 80)     # Hz, inclusive
EPOCH, N_EPOCHS, AC_MAXLAG = 500, 80, 100   # Azouz & Gray
FIT_BOUNDS = ([-1.0, 0.0, 0.1], [1.0, 2.0, 1000.0])
SPIKE_HALF_WINDOW_MS = 2.5
MAX_DISTANCE_UM = 500.0
DISTANCE_BINS = [0, 100, 200, 300, 400, 500]
SIMPLE_COMPLEX_MAX_UM = 200.0               # Yu & Ferster's simple-complex pair: tips close horizontally
BOOTSTRAP, BOOTSTRAP_CHUNK, SEED = 2000, 100, 13
POPULATIONS = {'V1_Exc_L4': 'L4E', 'V1_Inh_L4': 'L4I', 'V1_Exc_L2/3': 'L23E', 'V1_Inh_L2/3': 'L23I'}
GROUPS = [('L23E-L23E', 'V1_Exc_L2/3', 'V1_Exc_L2/3'), ('L23E-L23I', 'V1_Exc_L2/3', 'V1_Inh_L2/3'),
          ('L23I-L23I', 'V1_Inh_L2/3', 'V1_Inh_L2/3'), ('L4E-L23E', 'V1_Exc_L4', 'V1_Exc_L2/3')]
PRIMARY = 'L23E-L23E'
TAU_PRIMARY = ['V1_Exc_L4', 'V1_Exc_L2/3']  # intracellular samples are dominated by excitatory cells
COMBINED_READING = {
    ('below the data', 'below the data'): 'less synchronized than the recordings at slow and at fast time scales',
    ('below the data', 'within the data'): 'overall synchrony weaker than in the recordings; fast synchrony comparable',
    ('below the data', 'above the data'): 'overall synchrony weaker than in the recordings; fast synchrony stronger',
    ('within the data', 'below the data'): 'overall synchrony comparable; fast synchrony weaker than in the recordings',
    ('within the data', 'within the data'): 'comparable to the recordings at slow and at fast time scales',
    ('within the data', 'above the data'): 'overall synchrony comparable; fast synchrony stronger than in the recordings',
    ('above the data', 'below the data'): 'overall synchrony stronger than in the recordings; fast synchrony weaker',
    ('above the data', 'within the data'): 'overall synchrony stronger than in the recordings; fast synchrony comparable',
    ('above the data', 'above the data'): 'more synchronized than the recordings at slow and at fast time scales'}


# ---------------------------------------------------------------- signal steps (unit-tested)

def remove_spikes(v, first_sample_ms, spike_times_ms, sample_ms=1.0, half=SPIKE_HALF_WINDOW_MS, mode='interpolate'):
    v = np.array(v, dtype=float)
    t = first_sample_ms + sample_ms * np.arange(len(v))
    mask = np.zeros(len(v), bool)
    for s in spike_times_ms:
        lo, hi = np.searchsorted(t, s - half - 1e-9), np.searchsorted(t, s + half + 1e-9, side='right')
        mask[lo:hi] = True
    if not mask.any():
        return v
    edges = np.diff(np.concatenate([[0], mask.astype(int), [0]]))
    for a, b in zip(np.where(edges == 1)[0], np.where(edges == -1)[0]):     # samples a .. b-1 replaced
        left, right = a - 1, b
        if left < 0 and right >= len(v):
            raise ValueError('trace is entirely inside spike windows')
        if left < 0:
            v[a:b] = v[right]
        elif right >= len(v) or mode == 'hold':
            v[a:b] = v[left]
        else:
            v[a:b] = v[left] + (v[right] - v[left]) * (np.arange(a, b) - left) / (right - left)
    return v


def highpass(v, fs=1000.0):
    b, a = butter(HIGHPASS_ORDER, HIGHPASS_HZ / (fs / 2), btype='high')
    return filtfilt(b, a, v)


def normalized_blocks(v, n=N_BLOCKS, length=BLOCK):
    x = np.asarray(v[:n * length], dtype=float).reshape(n, length)
    x = x - x.mean(axis=1, keepdims=True)
    norm = np.sqrt((x ** 2).sum(axis=1, keepdims=True))
    if np.any(norm == 0):
        raise ValueError('a block has zero variance')
    return x / norm


def block_spectra(v):
    """FFT of each normalized block; the block-averaged 'coeff' cross-correlation of two traces is
    irfft(mean_b X_b conj(Y_b))."""
    return np.fft.rfft(normalized_blocks(v), n=NFFT, axis=1)


def xcorr_from_spectra(X, Y, maxlag=WIDTH_MAXLAG):
    c = np.fft.irfft((X * np.conj(Y)).mean(axis=0), n=NFFT)
    lags = np.arange(-maxlag, maxlag + 1)
    return lags, c[lags % NFFT]


def peak_and_width(lags, r, peak_window=PEAK_WINDOW):
    """Peak within +-peak_window of zero lag and the full width at half that peak; width is None when
    the peak is not positive or the half height is not crossed within the lag range."""
    centre = np.abs(lags) <= peak_window
    i = np.flatnonzero(centre)[np.argmax(r[centre])]
    peak = float(r[i])
    half = peak / 2
    if peak <= 0:
        return peak, float(lags[i]), None
    left = i
    while left > 0 and r[left - 1] >= half:
        left -= 1
    right = i
    while right < len(r) - 1 and r[right + 1] >= half:
        right += 1
    if left == 0 or right == len(r) - 1:
        return peak, float(lags[i]), None
    xl = lags[left - 1] + (half - r[left - 1]) / (r[left] - r[left - 1]) * (lags[left] - lags[left - 1])
    xr = lags[right] + (r[right] - half) / (r[right] - r[right + 1]) * (lags[right + 1] - lags[right])
    return peak, float(lags[i]), float(xr - xl)


def taper_spectra(v, fs=1000.0):
    """Tapered block spectra up to MAX_FREQ_HZ: (blocks * tapers) x frequencies, complex64."""
    x = np.asarray(v[:N_BLOCKS * BLOCK], dtype=float).reshape(N_BLOCKS, BLOCK)
    x = x - x.mean(axis=1, keepdims=True)
    tapers = dpss(BLOCK, TAPER_NW, TAPER_K)                     # K x BLOCK
    freqs = np.fft.rfftfreq(BLOCK, 1 / fs)
    keep = freqs <= MAX_FREQ_HZ
    spec = np.fft.rfft(x[:, None, :] * tapers[None, :, :], n=BLOCK, axis=2)[:, :, keep]
    return spec.reshape(-1, spec.shape[-1]).astype(np.complex64), freqs[keep]


def coherence(SX, SY):
    sxy = (SX * np.conj(SY)).mean(axis=0)
    sxx = (np.abs(SX) ** 2).mean(axis=0)
    syy = (np.abs(SY) ** 2).mean(axis=0)
    return np.abs(sxy) / np.sqrt(sxx * syy)


def band_mean(freqs, c, band):
    sel = (freqs >= band[0]) & (freqs <= band[1])
    return float(c[sel].mean())


def autocorrelation_tau(v_clean, unbiased=True):
    """Return (tau, autocorrelation, status). Primary: unbiased per-lag estimate (sum over the
    EPOCH - lag overlapping samples divided by their number), scaled to 1 at lag 0. Sensitivity:
    'coeff' (sum divided by the epoch energy). status: 'ok', 'fit_failed' or 'at_bound'."""
    v = medfilt(np.asarray(v_clean, dtype=float), 3)
    x = v[:N_EPOCHS * EPOCH].reshape(N_EPOCHS, EPOCH)
    x = x - x.mean(axis=1, keepdims=True)
    energy = (x ** 2).sum(axis=1)
    if np.any(energy == 0):
        return None, None, 'fit_failed'
    if unbiased:
        ac = np.array([((x[:, lag:] * x[:, :EPOCH - lag]).sum(axis=1) / (EPOCH - lag) / (energy / EPOCH)).mean()
                       for lag in range(AC_MAXLAG + 1)])
    else:
        ac = np.array([((x[:, lag:] * x[:, :EPOCH - lag]).sum(axis=1) / energy).mean() for lag in range(AC_MAXLAG + 1)])
    tau, status = fit_exponential(ac)
    return tau, ac, status


def fit_exponential(ac):
    """Least-squares fit of a + b exp(-lag / c) on lags 0..len(ac)-1 (ms); (c, 'ok'), or (None, status)
    when the fit fails or ends within 1% of a bound on b or c."""
    lags = np.arange(len(ac), dtype=float)
    try:
        (a, b, c), _ = curve_fit(lambda s, a, b, c: a + b * np.exp(-s / c), lags, ac, p0=(0.0, 1.0, 20.0),
                                 bounds=FIT_BOUNDS, maxfev=20000)
    except RuntimeError:
        return None, 'fit_failed'
    (lo_b, lo_c), (hi_b, hi_c) = FIT_BOUNDS[0][1:], FIT_BOUNDS[1][1:]
    if c <= lo_c * 1.01 or c >= hi_c * 0.99 or b <= lo_b + 0.01 * (hi_b - lo_b) or b >= hi_b * 0.99:
        return None, 'at_bound'
    return float(c), 'ok'


def pairs_within(xa, ya, xb, yb, same, max_um=MAX_DISTANCE_UM):
    d = np.hypot(xa[:, None] - xb[None, :], ya[:, None] - yb[None, :])
    ok = d <= max_um
    if same:
        ok &= np.triu(np.ones_like(ok, dtype=bool), k=1)
    i, j = np.nonzero(ok)
    return i, j, d[i, j]


def summarize(values, rng):
    v = np.asarray([x for x in values if x is not None and np.isfinite(x)], dtype=float)
    if len(v) == 0:
        return {'n': 0}
    boot = np.concatenate([np.median(rng.choice(v, size=(BOOTSTRAP_CHUNK, len(v)), replace=True), axis=1)
                           for _ in range(BOOTSTRAP // BOOTSTRAP_CHUNK)])
    return {'n': int(len(v)), 'median': float(np.median(v)), 'q25': float(np.percentile(v, 25)), 'q75': float(np.percentile(v, 75)),
            'p10': float(np.percentile(v, 10)), 'p90': float(np.percentile(v, 90)), 'min': float(v.min()), 'max': float(v.max()),
            'median_bootstrap_95': [float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5))]}


def outcome(model_median, bounds):
    """Declared reading against the digitized data: bounds = (lower bound of the data's 25th percentile,
    upper bound of its 75th percentile), which allow for the symbols hidden in the figure."""
    lo, hi = bounds
    return 'below the data' if model_median < lo else ('above the data' if model_median > hi else 'within the data')


def tau_outcome(model_median, mean=38.0, dispersion=26.0):
    if abs(model_median - mean) <= dispersion:
        return 'consistent (within one dispersion of the mean)'
    return 'shorter than the data' if model_median < mean else 'longer than the data'


def placement(values, found_range, data_median):
    """Fraction of model values inside the found data range, and the percentile of the data median
    within the model distribution."""
    v = np.asarray([x for x in values if x is not None and np.isfinite(x)], dtype=float)
    inside = float(np.mean((v >= found_range[0]) & (v <= found_range[1])))
    return {'fraction_inside_found_data_range': inside, 'found_data_range': list(found_range),
            'data_median_percentile_in_model': float(100.0 * np.mean(v < data_median))}


# ---------------------------------------------------------------- run on the retained data

def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
    ap.add_argument('--analysis', type=Path, required=True)
    ap.add_argument('--positions', type=Path, required=True)
    ap.add_argument('--output', type=Path, required=True)
    ap.add_argument('--contract', type=Path, default=HERE / 'contract.json')
    args = ap.parse_args(argv)
    contract = json.loads(args.contract.read_text())
    inputs = {}
    for name, expected in contract['model_inputs'].items():
        path = args.positions if name == 'positions.json' else args.analysis / name
        digest = sha(path)
        if digest != expected:
            raise SystemExit(f'SHA-256 mismatch for {name}: {digest}')
        inputs[name] = digest
    positions = json.loads(args.positions.read_text())['populations']
    spikes = {}
    with gzip.open(args.analysis / 'exported-spikes.json.gz', 'rt') as fh:
        for pop in json.load(fh):
            spikes[pop['population']] = {int(s['id']): np.asarray(s['times_ms'], dtype=float) for s in pop['spikes']}
    targets = contract['data_targets']
    rng = np.random.default_rng(SEED)
    results = {'scope': 'protocol-013 model side, run 011, spontaneous 40.32 s; descriptive', 'inputs': inputs,
               'provenance': {'script_sha256': sha(__file__), 'contract_sha256': sha(args.contract), 'python': platform.python_version(),
                              'numpy': np.__version__, 'scipy': scipy.__version__,
                              'script_matches_contract': contract.get('scripts', {}).get('vm_synchrony.py') == sha(__file__)},
               'populations': {}, 'tau': {}, 'groups': {}}
    traces, tau_pool = {}, {'primary': [], 'all': []}
    needed = {p for _, a, b in GROUPS for p in (a, b)}
    for population, short in POPULATIONS.items():
        npz = np.load(args.analysis / f'exported-analog-{short}-voltage_mV.npz')
        timing = json.loads(str(npz['timing']))
        if timing['sample_period_ms'] != 1.0:
            raise SystemExit('expected 1 ms sampling')
        ids = [int(i) for i in npz['ids']]
        if ids != positions[population]['ids']:
            raise SystemExit(f'{population}: ids differ between the voltage export and positions.json')
        missing = [i for i in ids if i not in spikes[population]]
        if missing:
            raise SystemExit(f'{population}: {len(missing)} analog-recorded neurons have no spike record')
        values = npz['values']
        clean = np.vstack([remove_spikes(v, timing['first_sample_ms'], spikes[population][i]) for i, v in zip(ids, values)])
        held = [remove_spikes(v, timing['first_sample_ms'], spikes[population][i], mode='hold') for i, v in zip(ids, values)]
        del values
        fits = [autocorrelation_tau(v) for v in clean]
        fits_coeff = [autocorrelation_tau(v, unbiased=False) for v in clean]
        fits_hold = [autocorrelation_tau(v) for v in held]
        del held
        taus = [f[0] for f in fits]
        tau_pool['all'] += taus
        if population in TAU_PRIMARY:
            tau_pool['primary'] += taus
        status = lambda fs: {s: int(sum(f[2] == s for f in fs)) for s in ('ok', 'fit_failed', 'at_bound')}
        results['populations'][population] = {
            'neurons': len(ids), 'timing': timing, 'spikes_removed': int(sum(len(spikes[population][i]) for i in ids)),
            'vm_sd_after_spike_removal_mV': summarize(clean[:, :N_BLOCKS * BLOCK].std(axis=1), rng),
            'tau_ms': summarize(taus, rng), 'tau_fit_status': status(fits),
            'tau_coeff_ms': summarize([f[0] for f in fits_coeff], rng), 'tau_coeff_fit_status': status(fits_coeff),
            'tau_hold_ms': summarize([f[0] for f in fits_hold], rng), 'tau_hold_fit_status': status(fits_hold),
            'tau_per_neuron_ms': taus}
        if population in needed:
            traces[population] = {'ids': ids, 'clean': clean, 'x': np.asarray(positions[population]['x_um']),
                                  'y': np.asarray(positions[population]['y_um'])}
        del clean
        print(population, 'tau median', results['populations'][population]['tau_ms'].get('median'), flush=True)
    primary_tau = summarize(tau_pool['primary'], rng)
    results['tau'] = {'primary_set': TAU_PRIMARY, 'primary_ms': primary_tau, 'all_neurons_ms': summarize(tau_pool['all'], rng),
                      'data': targets['azouz_gray_1999'],
                      'declared_outcome': tau_outcome(primary_tau['median']) if primary_tau.get('n') else None}

    spectra, freqs = {}, None
    for population, tr in traces.items():
        un = np.empty((len(tr['ids']), N_BLOCKS, NFFT // 2 + 1), np.complex64)
        hp = np.empty_like(un)
        tp = None
        for k, v in enumerate(tr['clean']):
            un[k] = block_spectra(v)
            hp[k] = block_spectra(highpass(v))
            s, freqs = taper_spectra(v)
            if tp is None:
                tp = np.empty((len(tr['ids']),) + s.shape, np.complex64)
            tp[k] = s
        spectra[population] = {'unfiltered': un, 'highpass': hp, 'tapers': tp}
        tr['clean'] = None
    for name, pa, pb in GROUPS:
        A, B = traces[pa], traces[pb]
        i, j, dist = pairs_within(A['x'], A['y'], B['x'], B['y'], same=(pa == pb))
        rows, coh_sum = [], np.zeros(len(freqs))
        for k in range(len(i)):
            lags, r = xcorr_from_spectra(spectra[pa]['unfiltered'][i[k]], spectra[pb]['unfiltered'][j[k]])
            peak, peak_lag, width = peak_and_width(lags, r)
            lags, rh = xcorr_from_spectra(spectra[pa]['highpass'][i[k]], spectra[pb]['highpass'][j[k]])
            hpeak, hlag, hwidth = peak_and_width(lags, rh)
            c = coherence(spectra[pa]['tapers'][i[k]], spectra[pb]['tapers'][j[k]])
            coh_sum += c
            rows.append({'id_a': A['ids'][i[k]], 'id_b': B['ids'][j[k]], 'distance_um': round(float(dist[k]), 2),
                         'peak': peak, 'peak_lag_ms': peak_lag, 'width_ms': width, 'highpass_peak': hpeak, 'highpass_width_ms': hwidth,
                         'coherence_low': band_mean(freqs, c, LOW_BAND), 'coherence_high': band_mean(freqs, c, HIGH_BAND)})
        group = {'pairs': len(rows), 'max_distance_um': MAX_DISTANCE_UM}
        for key in ['peak', 'width_ms', 'highpass_peak', 'highpass_width_ms', 'coherence_low', 'coherence_high', 'distance_um']:
            group[key] = summarize([r_[key] for r_ in rows], rng)
        group['width_not_reached'] = int(sum(r_['width_ms'] is None for r_ in rows))
        group['highpass_width_not_reached'] = int(sum(r_['highpass_width_ms'] is None for r_ in rows))
        group['by_distance'] = []
        for lo, hi in zip(DISTANCE_BINS[:-1], DISTANCE_BINS[1:]):
            sel = [r_ for r_ in rows if lo < r_['distance_um'] <= hi]
            group['by_distance'].append({'distance_um': [lo, hi], 'pairs': len(sel),
                                         'peak_median': float(np.median([r_['peak'] for r_ in sel])) if sel else None,
                                         'highpass_peak_median': float(np.median([r_['highpass_peak'] for r_ in sel])) if sel else None})
        group['mean_coherence_spectrum'] = {'freq_hz': freqs.tolist(), 'coherence': (coh_sum / max(len(rows), 1)).tolist()}
        if name == 'L4E-L23E':
            near = [r_ for r_ in rows if r_['distance_um'] <= SIMPLE_COMPLEX_MAX_UM]
            group['within_200_um'] = {'pairs': len(near), 'peak': summarize([r_['peak'] for r_ in near], rng),
                                      'data_simple_complex_pair': targets['yu_ferster_simple_complex_pair']}
        if name == PRIMARY:
            yf = targets['yu_ferster_2010']
            u = outcome(group['peak']['median'], targets['fig4A_blank_bounds_p25_low_p75_high'])
            h = outcome(group['highpass_peak']['median'], targets['fig4B_blank_bounds_p25_low_p75_high'])
            group['declared_outcomes'] = {'unfiltered_peak': u, 'highpass_peak': h, 'combined_reading': COMBINED_READING[(u, h)]}
            group['placement'] = {'unfiltered_peak': placement([r_['peak'] for r_ in rows], yf['fig4A_blank_found_range'], yf['fig4A_blank_median_found']),
                                  'highpass_peak': placement([r_['highpass_peak'] for r_ in rows], yf['fig4B_blank_found_range'], yf['fig4B_blank_median_found'])}
            group['data_example_pair'] = targets['yu_ferster_example_complex_pair']
            group['per_pair'] = rows
        results['groups'][name] = group
        print(name, len(rows), 'pairs; peak median', group['peak'].get('median'), '; high-pass', group['highpass_peak'].get('median'), flush=True)
    args.output.mkdir(parents=True, exist_ok=True)
    (args.output / 'vm-synchrony-results.json').write_text(json.dumps(results) + '\n')


if __name__ == '__main__':
    main()
