"""Digitize the 'Blank' (spontaneous) x-values of Yu & Ferster (2010) Figure 4A and 4B.

Yu J, Ferster D (2010) Membrane potential synchrony in primary visual cortex during sensory
stimulation. Neuron 68:1187-1201 (NIH author manuscript, PMC3031170). Figure 4A plots, for each
pair of layer 2/3 complex cells, the peak Vm cross-correlation during visual stimulation (y)
against the blank, spontaneous period (x); 4B does the same after a 20 Hz high-pass.

Method (fixed before any model statistic is computed):
  1. Check the image's SHA-256. The image is fetched from PMC by URL if absent; it is not redistributed.
  2. Calibrate each panel from its own axes: the x-axis and y-axis lines and their tick marks at
     0, 0.2, ..., 1 are found as dark pixel runs; value = linear fit of tick pixel positions.
  3. Upsample the plotting area 4x (bicubic) and classify pixels as red, green or black symbol colour.
  4. Filled symbols: match a disk template (radius measured on isolated filled symbols) against
     each colour mask; accept maxima whose disk coverage reaches a declared fraction.
  5. Open symbols: match an annulus template (ring radii measured on an isolated open symbol)
     against the union of the colour masks. Accept candidates greedily in order of coverage: a
     candidate needs a declared total coverage and a declared share of ring pixels not already
     explained by accepted symbols, and must lie at least a declared separation from them. This
     recovers rings partly hidden behind others; rings hidden completely cannot be recovered.
  6. Colour of an open symbol: the colour that dominates its unexplained ring pixels. Write one row
     per detection: panel, colour, filled or open, coverage, clustered or not, pixel and value
     coordinates. Colour and fill are descriptive only; the comparison uses the x (blank) values.
  7. A symbol that was not found is hidden behind found ones, so its centre lies within HIDDEN_MARGIN_PX
     of some detection (a symbol diameter plus the half pixel by which accepted rings are widened when
     marking explained pixels). Quantile bounds for all 44 pairs place every hidden symbol at the lowest
     detected x-value minus that margin, or at the highest plus it. Detections with another detection
     within one diameter are flagged as clustered (descriptive only).
  Candidate centres are local maxima over a window; ties on a coverage plateau are resolved in raster
  order, which can bias a centre by up to 0.75 px (about 0.005) towards the top left.

  python digitize_fig4.py --image IMAGE --output DIR
"""
import argparse, csv, hashlib, json, urllib.request
from pathlib import Path
import numpy as np
from PIL import Image

URL = 'https://cdn.ncbi.nlm.nih.gov/pmc/blobs/965e/3031170/4646cd63e023/nihms253487f4.jpg'
SHA256 = '46d7348c7374c164c082b363efc49acc093b35ea8c7db5351ef46702c360134d'
# Panel search boxes in the 745 x 1050 image (left, top, right, bottom); each contains one set of axes.
PANELS = {'A_unfiltered': (75, 80, 235, 245), 'B_highpass_20Hz': (270, 80, 430, 245)}
UP = 4                      # upsampling factor
RING = (4.2, 5.5)           # inner and outer ring radius in original pixels (isolated open symbol, panel A)
FILLED_RADIUS = 4.0         # disk template radius; isolated filled symbols measure 4.3 to 4.6 px
FILLED_COVERAGE = 0.95      # fraction of the disk template that must carry one symbol colour
FILLED_SEPARATION_PX = 4.5  # filled centres must be at least about one symbol radius apart
RING_COVERAGE = 0.55        # fraction of annulus pixels that must carry any symbol colour
RING_NOVEL = 0.30           # fraction of annulus pixels not already explained by accepted symbols
MIN_SEPARATION_PX = 2.5     # accepted centres must be at least this far apart (original pixels)
DIAMETER_PX = 2 * RING[1]   # outer symbol diameter (original pixels); defines the descriptive cluster flag
HIDDEN_MARGIN_PX = 2 * RING[1] + 0.5  # how far a hidden centre can lie from a detection (original pixels)
TICKS = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]


def load(image_path):
    p = Path(image_path)
    if not p.exists():
        req = urllib.request.Request(URL, headers={'User-Agent': 'Mozilla/5.0 (felisyn research; figure retrieval)'})
        p.write_bytes(urllib.request.urlopen(req, timeout=60).read())
    data = p.read_bytes()
    if hashlib.sha256(data).hexdigest() != SHA256:
        raise SystemExit(f'SHA-256 mismatch for {p}')
    return np.asarray(Image.open(p).convert('RGB')).astype(float)


def dark(rgb):
    return rgb.max(axis=2) < 110


def runs(mask_1d):
    """Start and end (inclusive) of True runs."""
    m = np.concatenate([[False], mask_1d, [False]]).astype(int)
    d = np.diff(m)
    return list(zip(np.where(d == 1)[0], np.where(d == -1)[0] - 1))


def calibrate(rgb, box):
    """Find the x-axis row, the y-axis column and the six tick positions on each axis."""
    l, t, r, b = box
    sub = dark(rgb[t:b, l:r])
    row = int(np.argmax(sub.sum(axis=1)))          # longest dark horizontal line: the x-axis
    col = int(np.argmax(sub.sum(axis=0)))          # longest dark vertical line: the y-axis
    xrun = max(runs(sub[row]), key=lambda s: s[1] - s[0])
    yrun = max(runs(sub[:, col]), key=lambda s: s[1] - s[0])
    below = sub[row + 2:row + 5].any(axis=0)       # tick marks hang below the x-axis
    left = sub[:, max(col - 4, 0):col - 1].any(axis=1)  # and stick out left of the y-axis
    xticks = [(a + c) / 2 for a, c in runs(below) if c - a <= 2 and xrun[0] - 1 <= a <= xrun[1] + 1]
    yticks = [(a + c) / 2 for a, c in runs(left) if c - a <= 2 and yrun[0] - 1 <= a <= yrun[1] + 1]
    if len(yticks) == 5:  # the y = 0 tick coincides with the axis corner; use the x-axis row if the spacing confirms it
        spacing = float(np.mean(np.diff(sorted(yticks))))
        if abs((row - max(yticks)) - spacing) <= 1.5:
            yticks = yticks + [float(row)]
    if len(xticks) != 6 or len(yticks) != 6:
        raise SystemExit(f'tick detection failed for box {box}: x {xticks}, y {yticks}')
    xfit = np.polyfit(np.array(xticks) + l, TICKS, 1)
    yfit = np.polyfit(np.array(sorted(yticks, reverse=True)) + t, TICKS, 1)
    return {'x_axis_row': row + t, 'y_axis_col': col + l, 'x_ticks_px': [x + l for x in xticks],
            'y_ticks_px': [y + t for y in sorted(yticks, reverse=True)], 'x_fit': xfit.tolist(), 'y_fit': yfit.tolist(),
            'x_fit_max_residual': float(np.max(np.abs(np.polyval(xfit, np.array(xticks) + l) - TICKS))),
            'y_fit_max_residual': float(np.max(np.abs(np.polyval(yfit, np.array(sorted(yticks, reverse=True)) + t) - TICKS)))}


def colour_masks(rgb):
    r, g, b = rgb[..., 0], rgb[..., 1], rgb[..., 2]
    return {'red': (r > 170) & (g < 130) & (b < 130),
            'green': (g > r + 35) & (g > b + 10) & (r < 140),
            'black': (rgb.max(axis=2) < 90)}


def annulus(inner, outer):
    r = int(np.ceil(outer))
    yy, xx = np.mgrid[-r:r + 1, -r:r + 1]
    d = np.hypot(yy, xx)
    return ((d >= inner) & (d <= outer)).astype(float), (d < inner * 0.6).astype(float)


def stamp(shape, y, x, template):
    """Window slices of an image and of a centred template, clipped at the image border."""
    r = template.shape[0] // 2
    y0, y1, x0, x1 = max(y - r, 0), min(y + r + 1, shape[0]), max(x - r, 0), min(x + r + 1, shape[1])
    return (slice(y0, y1), slice(x0, x1)), template[r - (y - y0):r + (y1 - y), r - (x - x0):r + (x1 - x)]


def detections(masks, up):
    from scipy.signal import fftconvolve
    from scipy.ndimage import maximum_filter
    ring, _ = annulus(RING[0] * up, RING[1] * up)
    ring_mark, _ = annulus((RING[0] - 0.5) * up, (RING[1] + 0.5) * up)   # pixels explained by an accepted open symbol
    disk_mark, _ = annulus(0, (FILLED_RADIUS + 1.0) * up)                # pixels explained by an accepted filled symbol
    r = int(np.ceil(FILLED_RADIUS * up))
    yy, xx = np.mgrid[-r:r + 1, -r:r + 1]
    disk = (np.hypot(yy, xx) <= FILLED_RADIUS * up).astype(float)
    union = masks['red'] | masks['green'] | masks['black']
    explained = np.zeros(union.shape, bool)
    found = []

    def far(y, x, sep_px=MIN_SEPARATION_PX):
        return all((y - fy) ** 2 + (x - fx) ** 2 >= (sep_px * up) ** 2 for fy, fx, *_ in found)

    for colour in ('red', 'green', 'black'):                      # filled symbols
        cover = fftconvolve(masks[colour].astype(float), disk, mode='same') / disk.sum()
        local = cover == maximum_filter(cover, size=int(FILLED_SEPARATION_PX * up) * 2 + 1)
        ys, xs = np.nonzero(local & (cover >= FILLED_COVERAGE))
        for i in np.argsort(-cover[ys, xs], kind='stable'):
            y, x = int(ys[i]), int(xs[i])
            if far(y, x, FILLED_SEPARATION_PX):
                found.append((y, x, float(cover[y, x]), colour, True))
                sl, tw = stamp(union.shape, y, x, disk_mark)
                explained[sl] |= tw > 0
    cover = fftconvolve(union.astype(float), ring, mode='same') / ring.sum()   # open symbols
    local = cover == maximum_filter(cover, size=3)
    ys, xs = np.nonzero(local & (cover >= RING_COVERAGE))
    for i in np.argsort(-cover[ys, xs], kind='stable'):
        y, x = int(ys[i]), int(xs[i])
        if not far(y, x):
            continue
        sl, tw = stamp(union.shape, y, x, ring)
        novel = union[sl] & ~explained[sl] & (tw > 0)
        if novel.sum() / ring.sum() < RING_NOVEL:
            continue
        counts = {c: int((masks[c][sl] & novel).sum()) for c in ('red', 'green', 'black')}
        found.append((y, x, float(cover[y, x]), max(counts, key=counts.get), False))
        sl2, tw2 = stamp(union.shape, y, x, ring_mark)
        explained[sl2] |= tw2 > 0
    return found


def quantile_bounds(found, total, low, high, qs=(10, 25, 50, 75, 90)):
    """Bounds on quantiles of all `total` values when only `found` are known and each missing value
    lies in [low, high]: lower bound with every missing value at low, upper bound with all at high."""
    k = total - len(found)
    lo = np.concatenate([np.full(k, low), found]); hi = np.concatenate([found, np.full(k, high)])
    out = {f'p{q}': [float(np.percentile(lo, q)), float(np.percentile(hi, q))] for q in qs}
    out['missing'] = int(k)
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
    ap.add_argument('--image', required=True)
    ap.add_argument('--output', required=True)
    a = ap.parse_args()
    rgb = load(a.image)
    out = Path(a.output); out.mkdir(parents=True, exist_ok=True)
    rows, summary = [], {'image_sha256': SHA256, 'source_url': URL, 'panels': {}}
    for panel, box in PANELS.items():
        cal = calibrate(rgb, box)
        l, t, r, b = box
        x0 = cal['y_axis_col'] + 2; y1 = cal['x_axis_row'] - 2          # plotting area inside the axes
        sub = rgb[t:y1, x0:r]
        big = np.asarray(Image.fromarray(sub.astype(np.uint8)).resize((sub.shape[1] * UP, sub.shape[0] * UP), Image.BICUBIC)).astype(float)
        masks = colour_masks(big)
        n_panel = 0
        for py, px, v, colour, filled in detections(masks, UP):
            ox, oy = x0 + (px + 0.5) / UP - 0.5, t + (py + 0.5) / UP - 0.5
            rows.append({'panel': panel, 'colour': colour, 'filled': filled, 'coverage': round(v, 3),
                         'x_px': round(ox, 2), 'y_px': round(oy, 2),
                         'blank_value': round(float(np.polyval(cal['x_fit'], ox)), 4),
                         'visual_value': round(float(np.polyval(cal['y_fit'], oy)), 4)})
            n_panel += 1
        mine = [r_ for r_ in rows if r_['panel'] == panel]
        for r_ in mine:
            r_['clustered'] = any(o is not r_ and np.hypot(float(o['x_px']) - float(r_['x_px']), float(o['y_px']) - float(r_['y_px'])) < DIAMETER_PX
                                  for o in mine)
        xs = np.array([r_['blank_value'] for r_ in mine])
        clustered = np.array([r_['blank_value'] for r_ in mine if r_['clustered']])
        diameter_value = DIAMETER_PX * abs(cal['x_fit'][0])
        margin_value = HIDDEN_MARGIN_PX * abs(cal['x_fit'][0])
        hidden_range = [float(xs.min() - margin_value), float(xs.max() + margin_value)]
        bounds = quantile_bounds(xs, 44, *hidden_range)
        summary['panels'][panel] = {'calibration': cal, 'symbols_found': n_panel, 'expected_pairs': 44,
                                    'clustered_symbols': int(len(clustered)), 'symbol_diameter_value': float(diameter_value),
                                    'hidden_margin_value': float(margin_value),
                                    'hidden_symbol_range': hidden_range, 'bounds_with_missing_points': bounds,
                                    'blank_median': float(np.median(xs)), 'blank_q25': float(np.percentile(xs, 25)),
                                    'blank_q75': float(np.percentile(xs, 75)), 'blank_p10': float(np.percentile(xs, 10)),
                                    'blank_min': float(xs.min()), 'blank_max': float(xs.max()),
                                    'value_per_pixel': float(cal['x_fit'][0])}
    with (out / 'yu-ferster-fig4-digitized.csv').open('w', newline='') as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0])); w.writeheader(); w.writerows(rows)
    import PIL, scipy
    summary['versions'] = {'python': '.'.join(map(str, __import__('sys').version_info[:3])), 'numpy': np.__version__,
                           'scipy': scipy.__version__, 'pillow': PIL.__version__}
    summary['parameters'] = {'upsample': UP, 'diameter_px': DIAMETER_PX, 'hidden_margin_px': HIDDEN_MARGIN_PX, 'ring_radii_px': RING, 'filled_radius_px': FILLED_RADIUS, 'filled_coverage': FILLED_COVERAGE,
                             'ring_coverage': RING_COVERAGE, 'ring_novel': RING_NOVEL, 'min_separation_px': MIN_SEPARATION_PX}
    (out / 'yu-ferster-fig4-summary.json').write_text(json.dumps(summary, indent=1) + '\n')
    for p, s in summary['panels'].items():
        print(p, 'found', s['symbols_found'], 'median', round(s['blank_median'], 3), 'IQR', round(s['blank_q25'], 3), round(s['blank_q75'], 3),
              'range', round(s['blank_min'], 3), round(s['blank_max'], 3), 'tick residual', round(s['calibration']['x_fit_max_residual'], 4))
        print('   bounds with', s['bounds_with_missing_points']['missing'], 'missing:',
              {q: [round(v, 3) for v in b] for q, b in s['bounds_with_missing_points'].items() if q != 'missing'})


if __name__ == '__main__':
    main()
