#!/usr/bin/env python3
"""Backtest of the Nasdaq "Initial Balance" (IB) breakout as stated on Raj Shamani's
Figuring Out FO414 (26:48-29:12), for the umar-audit reel.

Rules live in projects/umar-audit/ib_rules.json (edit there, not here).
Data: Dukascopy USATECHIDXUSD BID+ASK 1-minute candles, built by tools/ib_data.py
(`fetch` then `build`) into projects/umar-audit/data_raw/usatech_session_1m.pkl.

Execution model (see BACKTEST.md for the reasoning):
  * IB = high/low of the BID 1-minute bars 09:30-09:59 New York time (DST-aware).
  * Entry = resting stop order. Long fires when BID trades above the IB high and fills
    at the ASK (level + that minute's spread, or the minute's open if it gapped through);
    short fires when BID trades below the IB low and fills at the BID.
  * Stop/target are checked inside each minute with the executable side (long exits on
    the BID, short exits on the ASK). Stop and target in the same minute -> stop first.
    On the entry minute a stop touch is a loss; a target counts only if the minute closed
    beyond it.
  * Flat at the NYSE close from the exchange calendar (13:00 on half-days).
  * Slippage on every stop/market fill, a round-trip commission, both in index points.

Usage:
  .venv/bin/python tools/ib_backtest.py run          # full grid -> json + BACKTEST.md tables
  .venv/bin/python tools/ib_backtest.py spot --n 5   # print 5 random days bar by bar
  .venv/bin/python tools/ib_backtest.py crosscheck   # Yahoo NQ=F / QQQ 1m vs Dukascopy, recent days
  .venv/bin/python tools/ib_backtest.py plots        # PNG checks in data_raw/
"""
from __future__ import annotations

import argparse
import json
import math
import random
import sys
from dataclasses import dataclass
from pathlib import Path

import numpy as np
import pandas as pd

ROOT = Path(__file__).resolve().parents[1]
PROJ = ROOT / "projects/umar-audit"
RAW = PROJ / "data_raw"
RULES = PROJ / "ib_rules.json"
SESSION_PKL = RAW / "usatech_session_1m.pkl"
GAPS_JSON = RAW / "usatech_gaps.json"
OUT_JSON = ROOT / "engine/public/projects/umar-audit/data/ib_results.json"
TRADES_CSV = RAW / "ib_trades_headline.csv"
MD = PROJ / "BACKTEST.md"
NY = "America/New_York"


# ------------------------------------------------------------------ data ---
@dataclass
class Day:
    day: pd.Timestamp
    t: np.ndarray        # minute labels (NY), datetime64
    hhmm: np.ndarray     # int, e.g. 930
    bo: np.ndarray; bh: np.ndarray; bl: np.ndarray; bc: np.ndarray
    ao: np.ndarray; ah: np.ndarray; al: np.ndarray; ac: np.ndarray


HD_PKL = RAW / "histdata_session_1m.pkl"


def load_days(no_spread: bool = False, source: str = "dukascopy") -> list[Day]:
    """source 'dukascopy' (BID+ASK) or 'histdata' (BID only, so always no spread)."""
    df = pd.read_pickle(SESSION_PKL if source == "dukascopy" else HD_PKL)
    if source == "histdata":
        for k in "ohlc":
            df[f"{k}_b"] = df[k]
        no_spread = True
    out = []
    for d, g in df.groupby("day", sort=True):
        g = g.sort_index()
        idx = g.index
        hhmm = (idx.hour * 100 + idx.minute).to_numpy()
        b = [g[f"{k}_b"].to_numpy(float) for k in "ohlc"]
        a = b if no_spread else [g[f"{k}_a"].to_numpy(float) for k in "ohlc"]
        out.append(Day(pd.Timestamp(d), idx.to_numpy(), hhmm, *b, *a))
    return out


def hhmm_int(s: str) -> int:
    h, m = s.split(":")
    return int(h) * 100 + int(m)


# ------------------------------------------------------------ simulation ---
def first_true(mask: np.ndarray, start: int) -> int:
    """Index of the first True at or after `start`, or -1."""
    if start >= len(mask):
        return -1
    sub = mask[start:]
    i = int(np.argmax(sub))
    return start + i if sub[i] else -1


def manage(D: Day, direction: int, i: int, fill: float, stop: float, k: float, c: dict) -> dict:
    """Follow one position from entry minute i to stop / target / session close."""
    slip = c["slip"]
    if direction == 1:
        risk = fill - stop
        target = fill + k * risk
        stop_hit = D.bl <= stop          # sell stop on the BID
        tgt_hit = D.bh >= target         # sell limit on the BID
    else:
        risk = stop - fill
        target = fill - k * risk
        stop_hit = D.ah >= stop          # buy stop on the ASK
        tgt_hit = D.al <= target         # buy limit on the ASK
    n = len(D.t)
    j, reason = -1, "eod"
    # entry minute: stop touch -> loss (conservative); target only if the minute closed beyond it.
    # c["optimistic"] (sensitivity bound only): stop on the entry minute only if it CLOSED beyond
    # the stop, and a same-minute stop/target tie goes to the target.
    opt = c.get("optimistic", False)
    stop_close_beyond = (D.bc[i] <= stop) if direction == 1 else (D.ac[i] >= stop)
    if (stop_close_beyond if opt else stop_hit[i]):
        j, reason = i, "stop"
    elif (direction == 1 and D.bc[i] >= target) or (direction == -1 and D.ac[i] <= target):
        j, reason = i, "target"
    else:
        js, jt = first_true(stop_hit, i + 1), first_true(tgt_hit, i + 1)
        if js >= 0 and (jt < 0 or js < jt or (js == jt and not opt)):   # same minute -> stop first
            j, reason = js, "stop"
        elif jt >= 0:
            j, reason = jt, "target"
    if reason == "stop":
        if direction == 1:
            px = min(stop, D.bo[j]) if j > i else stop
            px -= slip
        else:
            px = max(stop, D.ao[j]) if j > i else stop
            px += slip
    elif reason == "target":
        px = target                           # limit fill at the level (no price improvement)
    else:
        j = n - 1
        px = D.bc[j] - slip if direction == 1 else D.ac[j] + slip
    pts = (px - fill) * direction - c["comm"]
    return {"exit_i": j, "exit_px": px, "reason": reason, "risk": risk, "target": target,
            "pts": pts, "R": pts / risk, "entry_minute_exit": j == i}


def simulate_day(D: Day, v: dict, rules: dict, c: dict) -> tuple[list[dict], dict]:
    """Return (trades, flags) for one session and one variant."""
    flags = {}
    start = hhmm_int(rules["ib_start"])
    m0 = (start // 100) * 60 + start % 100 + rules["ib_minutes"]
    ib_end = (m0 // 60) * 100 + m0 % 60
    ib = (D.hhmm >= start) & (D.hhmm < ib_end)
    if ib.sum() < rules["ib_minutes"] * 0.9 or D.hhmm[0] != start:
        flags["bad_ib"] = 1
        return [], flags
    first_after = int(np.argmax(D.hhmm >= ib_end))
    if D.hhmm[first_after] < ib_end:
        flags["no_after_ib"] = 1
        return [], flags
    last_entry = len(D.t) - 1
    if rules.get("last_entry_time"):
        le = hhmm_int(rules["last_entry_time"])
        last_entry = int(np.max(np.where(D.hhmm < le)[0]))
    hi, lo = D.bh[ib].max(), D.bl[ib].min()
    op = D.bo[0]
    ib_close = D.bc[ib][-1]
    buf = rules["entry"]["buffer_points"]
    L_long, L_short = hi + buf, lo - buf
    long_trig = D.bh > L_long
    short_trig = D.bl < L_short
    long_trig[:first_after] = False
    short_trig[:first_after] = False
    long_trig[last_entry + 1:] = False
    short_trig[last_entry + 1:] = False

    ibdir = int(np.sign(ib_close - op))
    allowed = {1, -1}
    if v["filter"] == "ib_direction":
        allowed = {ibdir} if ibdir != 0 else set()

    def entry_fill(direction: int, i: int) -> float:
        if direction == 1:
            spread = max(D.ao[i] - D.bo[i], D.ac[i] - D.bc[i])
            return max(L_long, D.bo[i]) + spread + c["slip"]
        return min(L_short, D.bo[i]) - c["slip"]

    def stop_level(direction: int) -> float:
        if v["stop"] == "open":
            return op
        return lo if direction == 1 else hi

    trades = []
    i_long = first_true(long_trig, first_after) if 1 in allowed else -1
    i_short = first_true(short_trig, first_after) if -1 in allowed else -1
    if i_long < 0 and i_short < 0:
        return [], flags
    if i_long >= 0 and i_long == i_short:
        flags["both_sides_same_minute"] = 1     # path unknowable -> no trade
        return [], flags
    if i_short < 0 or (0 <= i_long < i_short):
        first = (1, i_long)
    else:
        first = (-1, i_short)

    def take(direction: int, i: int) -> dict | None:
        fill = entry_fill(direction, i)
        stop = stop_level(direction)
        risk = (fill - stop) * direction
        if risk <= 0 or risk < rules["min_risk_frac_of_ib_range"] * (hi - lo):
            flags["skipped_tiny_risk"] = flags.get("skipped_tiny_risk", 0) + 1
            return None
        r = manage(D, direction, i, fill, stop, v["target_r"], c)
        r.update({"day": str(D.day.date()), "dir": direction, "entry_i": i, "entry_px": fill, "stop": stop,
                  "ib_hi": hi, "ib_lo": lo, "open": op, "ib_dir": ibdir,
                  "entry_t": pd.Timestamp(D.t[i]).strftime("%H:%M"),
                  "exit_t": pd.Timestamp(D.t[r["exit_i"]]).strftime("%H:%M")})
        return r

    t1 = take(*first)
    if t1 is None:
        return [], flags
    trades.append(t1)
    if v["mode"] == "reverse_after_stop" and t1["reason"] == "stop":
        other = -first[0]
        trig = short_trig if other == -1 else long_trig
        i2 = first_true(trig, t1["exit_i"])      # may fire in the stop-out minute itself
        if i2 >= 0:
            t2 = take(other, i2)
            if t2 is not None:
                t2["second"] = True
                trades.append(t2)
    return trades, flags


def premise_stats(days: list[Day], rules: dict) -> dict:
    """Test the claims around the rule: IB direction continuation and 'point of no return'."""
    start = hhmm_int(rules["ib_start"])
    m0 = (start // 100) * 60 + start % 100 + rules["ib_minutes"]
    ib_end = (m0 // 60) * 100 + m0 % 60
    n = cont = sameday = pnr = brk_any = brk_both = 0
    for D in days:
        ib = (D.hhmm >= start) & (D.hhmm < ib_end)
        if ib.sum() < rules["ib_minutes"] * 0.9 or D.hhmm[0] != start:
            continue
        after = D.hhmm >= ib_end
        if not after.any():
            continue
        op, ibc = D.bo[0], D.bc[ib][-1]
        hi, lo = D.bh[ib].max(), D.bl[ib].min()
        close = D.bc[-1]
        d = np.sign(ibc - op)
        ah, al = D.bh[after].max(), D.bl[after].min()
        brk_any += (ah > hi) or (al < lo)
        brk_both += (ah > hi) and (al < lo)
        if d == 0:
            continue
        n += 1
        cont += np.sign(close - ibc) == d
        sameday += np.sign(close - op) == d
        pnr += (al >= lo) if d > 0 else (ah <= hi)
    total = sum(1 for _ in days)
    return {
        "days_with_directional_ib": n,
        "rest_of_day_continues_ib_direction_pct": round(100 * cont / n, 1),
        "day_close_vs_open_matches_ib_direction_pct": round(100 * sameday / n, 1),
        "point_of_no_return_holds_pct": round(100 * pnr / n, 1),
        "_point_of_no_return_def": "bullish IB: the IB low is never broken after 10:00; bearish IB: the IB high is never broken",
        "days_ib_broken_either_side_pct": round(100 * brk_any / total, 1),
        "days_ib_broken_both_sides_pct": round(100 * brk_both / total, 1),
    }


def compare_sources(days_a: list[Day], days_b: list[Day], v: dict, rules: dict) -> dict:
    """Day-by-day agreement of the trade decision on two data sources (no costs)."""
    c = costs(rules, True)
    B = {str(D.day.date()): D for D in days_b}
    n = side = outcome = 0
    hi_diff, lo_diff = [], []
    for D in days_a:
        E = B.get(str(D.day.date()))
        if E is None:
            continue
        ta, _ = simulate_day(D, v, rules, c)
        tb, _ = simulate_day(E, v, rules, c)
        n += 1
        sa = ta[0]["dir"] if ta else 0
        sb = tb[0]["dir"] if tb else 0
        side += sa == sb
        oa = ta[0]["reason"] if ta else "none"
        ob = tb[0]["reason"] if tb else "none"
        outcome += (sa == sb) and (oa == ob)
        ib = D.hhmm < 1000
        ibb = E.hhmm < 1000
        if ib.any() and ibb.any():
            hi_diff.append(D.bh[ib].max() - E.bh[ibb].max())
            lo_diff.append(D.bl[ib].min() - E.bl[ibb].min())
    return {"days_compared": n, "same_first_breakout_side_pct": round(100 * side / n, 1),
            "same_side_and_same_exit_type_pct": round(100 * outcome / n, 1),
            "median_abs_ib_high_diff_points": round(float(np.median(np.abs(hi_diff))), 2),
            "median_abs_ib_low_diff_points": round(float(np.median(np.abs(lo_diff))), 2)}


# ------------------------------------------------------------- statistics ---
def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
    if n == 0:
        return (float("nan"), float("nan"))
    p = k / n
    den = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / den
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / den
    return centre - half, centre + half


def summarize(tr: pd.DataFrame, n_days: int, rules: dict, flags: dict) -> dict:
    n = len(tr)
    if n == 0:
        return {"trades": 0}
    R = tr["R"].to_numpy()
    wins = R > 0
    cum = np.cumsum(R)
    dd_r = float(np.max(np.maximum.accumulate(np.concatenate([[0], cum]))[1:] - cum))
    eq = rules["start_equity"] * np.cumprod(1 + rules["risk_per_trade_pct"] / 100 * R)
    peak = np.maximum.accumulate(np.concatenate([[rules["start_equity"]], eq]))[1:]
    dd_pct = float(np.max(1 - eq / peak) * 100)
    gp, gl = R[R > 0].sum(), -R[R < 0].sum()
    avg_win = float(R[wins].mean()) if wins.any() else 0.0
    avg_loss = float(-R[~wins].mean()) if (~wins).any() else 0.0
    lo, hi = wilson(int(wins.sum()), n)
    se = R.std(ddof=1) / math.sqrt(n) if n > 1 else float("nan")
    yrs = tr.groupby(tr["day"].str[:4])
    years = []
    for y, g in yrs:
        r = g["R"].to_numpy()
        years.append({"year": int(y), "trades": int(len(r)), "win_rate": round(100 * (r > 0).mean(), 1),
                      "target_hit_rate": round(100 * (g["reason"] == "target").mean(), 1),
                      "total_R": round(float(r.sum()), 1), "avg_R": round(float(r.mean()), 3)})
    weeks = n_days / 5.0
    streak = best_streak = 0
    for w in wins:
        streak = 0 if w else streak + 1
        best_streak = max(best_streak, streak)
    return {
        "trades": n,
        "sessions": n_days,
        "trades_per_week": round(n / weeks, 2),
        "win_rate_pct": round(100 * wins.mean(), 1),
        "win_rate_95ci_pct": [round(100 * lo, 1), round(100 * hi, 1)],
        "target_hit_rate_pct": round(100 * (tr["reason"] == "target").mean(), 1),
        "stop_rate_pct": round(100 * (tr["reason"] == "stop").mean(), 1),
        "eod_exit_rate_pct": round(100 * (tr["reason"] == "eod").mean(), 1),
        "avg_R": round(float(R.mean()), 3),
        "avg_R_95ci": [round(float(R.mean() - 1.96 * se), 3), round(float(R.mean() + 1.96 * se), 3)],
        "expectancy_R": round(float(R.mean()), 3),
        "avg_win_R": round(avg_win, 3),
        "avg_loss_R": round(avg_loss, 3),
        "breakeven_win_rate_pct": round(100 * avg_loss / (avg_win + avg_loss), 1) if (avg_win + avg_loss) else None,
        "profit_factor": round(float(gp / gl), 3) if gl > 0 else None,
        "total_R": round(float(R.sum()), 1),
        "max_drawdown_R": round(dd_r, 1),
        "longest_losing_streak": int(best_streak),
        "final_equity_1pct": round(float(eq[-1]), 0),
        "max_drawdown_pct_1pct_risk": round(dd_pct, 1),
        "median_risk_points": round(float(tr["risk"].median()), 1),
        "entry_minute_exits": int(tr["entry_minute_exit"].sum()),
        "years": years,
        "years_at_or_above_60pct_win": [y["year"] for y in years if y["win_rate"] >= 60],
        "flags": flags,
    }


def downsample(arr: np.ndarray, dates: list[str], k: int = 300) -> list[list]:
    """<= k points [trade_no, value, 'YYYY-MM-DD'], always keeping the first and last trade."""
    n = len(arr)
    if n == 0:
        return []
    idx = np.unique(np.linspace(0, n - 1, min(k, n)).round().astype(int))
    return [[int(i + 1), round(float(arr[i]), 3), dates[i]] for i in idx]


def variant_key(v: dict) -> str:
    return f"stop={v['stop']}|target={v['target_r']}R|mode={v['mode']}|filter={v['filter']}"


def grid(rules: dict) -> list[dict]:
    out = []
    g = rules["grid"]
    for s in g["stop"]:
        for k in g["target_r"]:
            for m in g["mode"]:
                for f in g["filter"]:
                    if f == "ib_direction" and m != "first_break":
                        continue  # only one side is ever allowed, so 'reverse' adds nothing
                    out.append({"stop": s, "target_r": k, "mode": m, "filter": f})
    return out


def costs(rules: dict, zero: bool, optimistic: bool = False) -> dict:
    if zero:
        return {"slip": 0.0, "comm": 0.0, "optimistic": optimistic}
    return {"slip": rules["costs"]["slippage_points_per_market_fill"],
            "comm": rules["costs"]["commission_points_round_trip"], "optimistic": optimistic}


def run_variant(days: list[Day], v: dict, rules: dict, c: dict) -> tuple[pd.DataFrame, dict]:
    rows, fl = [], {}
    for D in days:
        t, f = simulate_day(D, v, rules, c)
        rows += t
        for k, val in f.items():
            fl[k] = fl.get(k, 0) + val
    return pd.DataFrame(rows), fl


# -------------------------------------------------------------------- run ---
def cmd_run(a) -> None:
    rules = json.loads(RULES.read_text())
    days = load_days()
    info = json.loads(GAPS_JSON.read_text()) if GAPS_JSON.exists() else {}
    variants = grid(rules)
    res, trades_by = {}, {}
    for v in variants:
        key = variant_key(v)
        tr, fl = run_variant(days, v, rules, costs(rules, False))
        s = summarize(tr, len(days), rules, fl)
        R = tr["R"].to_numpy() if len(tr) else np.array([])
        eq = rules["start_equity"] * np.cumprod(1 + rules["risk_per_trade_pct"] / 100 * R)
        dates = tr["day"].tolist() if len(tr) else []
        s["equity_R"] = downsample(np.cumsum(R), dates)
        s["equity_usd"] = downsample(eq, dates)
        s["variant"] = v
        res[key] = s
        trades_by[key] = tr
        print(f"{key:70s} n={s['trades']:5d} win={s['win_rate_pct']:5.1f}% tgt={s['target_hit_rate_pct']:5.1f}% "
              f"avgR={s['avg_R']:+.3f} PF={s['profit_factor']} DD_R={s['max_drawdown_R']} "
              f"$={s['final_equity_1pct']:.0f} yrs>=60%={s['years_at_or_above_60pct_win']}")
    # cost sensitivity: frictionless (no spread, no slippage, no commission) for the headline set
    days0 = load_days(no_spread=True)
    zero = {}
    for key in rules["headline_variants"]:
        v = res[key]["variant"]
        tr, fl = run_variant(days0, v, rules, costs(rules, True))
        s = summarize(tr, len(days0), rules, fl)
        zero[key] = {k: s[k] for k in ("trades", "win_rate_pct", "target_hit_rate_pct", "avg_R", "profit_factor",
                                        "total_R", "years_at_or_above_60pct_win", "years")}
        print(f"[no costs] {key:60s} n={s['trades']} win={s['win_rate_pct']}% avgR={s['avg_R']:+.3f} PF={s['profit_factor']}")
    # intrabar sensitivity: most generous ordering inside a minute (upper bound), full costs
    opt = {}
    for key in rules["headline_variants"]:
        v = res[key]["variant"]
        tr, fl = run_variant(days, v, rules, costs(rules, False, optimistic=True))
        s = summarize(tr, len(days), rules, fl)
        opt[key] = {k: s[k] for k in ("trades", "win_rate_pct", "target_hit_rate_pct", "avg_R", "profit_factor",
                                       "total_R", "years_at_or_above_60pct_win")}
        print(f"[optimistic intrabar] {key:55s} n={s['trades']} win={s['win_rate_pct']}% avgR={s['avg_R']:+.3f} PF={s['profit_factor']}")
    # second source: HistData NSXUSD (BID only -> no costs), same days as Dukascopy where both exist
    second = {}
    if HD_PKL.exists():
        hd = load_days(source="histdata")
        dk_days = {str(D.day.date()) for D in days}
        hd = [D for D in hd if str(D.day.date()) in dk_days]
        for key in rules["headline_variants"]:
            v = res[key]["variant"]
            tr, fl = run_variant(hd, v, rules, costs(rules, True))
            s = summarize(tr, len(hd), rules, fl)
            second[key] = {k: s[k] for k in ("trades", "win_rate_pct", "target_hit_rate_pct", "avg_R", "profit_factor",
                                              "total_R", "years_at_or_above_60pct_win")}
            print(f"[histdata no costs] {key:55s} n={s['trades']} win={s['win_rate_pct']}% avgR={s['avg_R']:+.3f} PF={s['profit_factor']}")
        second["_day_level_agreement"] = compare_sources(days0, hd, res[rules["headline_variants"][0]]["variant"], rules)
        second["_premise_histdata"] = premise_stats(hd, rules)
        print("day-level agreement:", second["_day_level_agreement"])
    prem = premise_stats(days, rules)
    print(json.dumps(prem, indent=1))
    # spread stats during the session
    df = pd.read_pickle(SESSION_PKL)
    sp = (df["c_a"] - df["c_b"])
    spread = {str(y): round(float(g.median()), 2) for y, g in sp.groupby(df.index.year)}
    level = {str(y): round(float(g.median()), 0) for y, g in df["c_b"].groupby(df.index.year)}

    head = rules["headline_variants"][0]
    hd = trades_by[head]
    hd.to_csv(TRADES_CSV, index=False)
    for key in rules["headline_variants"][1:]:
        slug = key.replace("|", "_").replace("=", "-")
        trades_by[key].to_csv(RAW / f"ib_trades_{slug}.csv", index=False)

    pairs = [(k, y["year"], y["win_rate"], y["trades"]) for k, s in res.items() for y in s["years"]]
    best = max(res.items(), key=lambda kv: kv[1]["win_rate_pct"])
    best_ev = max(res.items(), key=lambda kv: kv[1]["avg_R"])
    grid_summary = {
        "variants_tested": len(res),
        "best_win_rate_variant": best[0], "best_win_rate_pct": best[1]["win_rate_pct"],
        "best_avg_R_variant": best_ev[0], "best_avg_R": best_ev[1]["avg_R"],
        "variant_years_at_or_above_60pct": [{"variant": k, "year": y, "win_rate": w, "trades": n}
                                             for k, y, w, n in pairs if w >= 60],
        "variant_years_total": len(pairs),
        "variants_with_positive_avg_R": [k for k, s in res.items() if s["avg_R"] > 0],
    }
    print(json.dumps(grid_summary, indent=1))
    out = {
        "title": "Nasdaq 'Initial Balance' breakout - backtest of the rules stated on Figuring Out FO414 (26:48-29:12)",
        "claim": {"win_rate_pct": [60, 70], "trades_per_week": [2, 3], "ib_continuation_pct": [80, 90],
                  "source": "Raj Shamani, Figuring Out FO414, 26:48-29:12 (https://www.youtube.com/watch?v=odBTJ4P8yxA&t=1608s)"},
        "data": {"source": "Dukascopy Bank historical datafeed (datafeed.dukascopy.com)",
                 "symbol": "USATECHIDXUSD (US Tech 100 / NASDAQ-100 CFD)", "bars": "1-minute BID and ASK",
                 "first_day": str(days[0].day.date()), "last_day": str(days[-1].day.date()),
                 "period_label": period_label(days),
                 "sessions": len(days), "sessions_by_year": sessions_by_year(days),
                 "interim": info.get("ask_modelled_days", 0) > 50 or info.get("years_filter", "all") != "all",
                 "years_filter": info.get("years_filter", "all"),
                 "ask_real_days": len(days) - info.get("ask_modelled_days", 0),
                 "ask_modelled_days": info.get("ask_modelled_days", 0),
                 "ask_modelled_note": info.get("_ask_modelled_note", ""),
                 "sessions_flagged": len([g for g in info.get("gaps", []) if "missing" not in g["issue"]]),
                 "flagged_sessions": [g for g in info.get("gaps", []) if "missing" not in g["issue"]],
                 "median_spread_points_by_year": spread, "median_price_by_year": level},
        "rules": {k: rules[k] for k in ("ib_start", "ib_minutes", "entry", "costs", "min_risk_frac_of_ib_range",
                                        "same_minute_rule", "risk_per_trade_pct", "start_equity")},
        "breakeven_win_rate_at_2R_pct": 33.3,
        "headline_variants": rules["headline_variants"],
        "grid_summary": grid_summary,
        "premise": prem,
        "no_cost_check": zero,
        "optimistic_intrabar_check": opt,
        "second_source_check": second,
        "variants": res,
    }
    OUT_JSON.parent.mkdir(parents=True, exist_ok=True)
    OUT_JSON.write_text(json.dumps(out, separators=(",", ":")))
    print(f"wrote {OUT_JSON.relative_to(ROOT)} ({OUT_JSON.stat().st_size/1024:.0f} KB)")
    write_md_tables(out)


def sessions_by_year(days: list[Day]) -> dict:
    out: dict = {}
    for D in days:
        out[str(D.day.year)] = out.get(str(D.day.year), 0) + 1
    return out


def period_label(days: list[Day]) -> str:
    """e.g. '2015-2017 + 2023-2026 (2026 = Jan-Sep)'."""
    ys = sorted({D.day.year for D in days})
    runs, start = [], ys[0]
    for a_, b_ in zip(ys, ys[1:] + [None]):
        if b_ != a_ + 1:
            runs.append(f"{start}" if start == a_ else f"{start}-{a_}")
            start = b_
    last = days[-1].day
    tail = f" ({last.year} = Jan-{last.strftime('%b')})" if last.month < 12 else ""
    return " + ".join(runs) + tail


# ---------------------------------------------------------------- markdown ---
def label(key: str) -> str:
    v = dict(p.split("=") for p in key.split("|"))
    s = "stop @ IB other end" if v["stop"] == "ib_other_end" else "stop @ 09:30 open"
    m = "1st break only" if v["mode"] == "first_break" else "+ reverse after stop"
    f = "" if v["filter"] == "none" else ", IB-candle direction only"
    return f"{s}, {v['target']}, {m}{f}"


def write_md_tables(out: dict) -> None:
    V = out["variants"]
    L = []
    L.append(f"_Generated by `tools/ib_backtest.py run` from {out['data']['first_day']} to {out['data']['last_day']}, "
             f"{out['data']['sessions']} NYSE sessions. Win = trade closed with net R > 0 after costs._\n")
    L.append("### Headline variants (target 2R)\n")
    L.append("| Variant | Trades | Trades/wk | Win rate (95% CI) | 2R target hit | Avg R | Profit factor | Total R | Max DD (R) | $10k @1% risk → | Max DD @1% | Years ≥60% win |")
    L.append("|---|---|---|---|---|---|---|---|---|---|---|---|")
    for k in out["headline_variants"]:
        s = V[k]
        L.append(f"| {label(k)} | {s['trades']} | {s['trades_per_week']} | {s['win_rate_pct']}% "
                 f"({s['win_rate_95ci_pct'][0]}–{s['win_rate_95ci_pct'][1]}) | {s['target_hit_rate_pct']}% | "
                 f"{s['avg_R']:+.3f} | {s['profit_factor']} | {s['total_R']:+.1f} | {s['max_drawdown_R']} | "
                 f"${s['final_equity_1pct']:,.0f} | {s['max_drawdown_pct_1pct_risk']}% | "
                 f"{len(s['years_at_or_above_60pct_win'])} of {len(s['years'])} |")
    g = out["grid_summary"]
    L.append(f"\nAcross all {g['variants_tested']} variants: highest overall win rate {g['best_win_rate_pct']}% "
             f"({label(g['best_win_rate_variant'])}); best average R {g['best_avg_R']:+.3f} ({label(g['best_avg_R_variant'])}); "
             f"variant-years with a win rate of 60% or more: {len(g['variant_years_at_or_above_60pct'])} of {g['variant_years_total']}"
             + (" (" + "; ".join(f"{label(x['variant'])} in {x['year']}: {x['win_rate']}% of {x['trades']}" for x in g['variant_years_at_or_above_60pct']) + ")" if g['variant_years_at_or_above_60pct'] else "")
             + f"; variants with positive average R after costs: {len(g['variants_with_positive_avg_R'])}.")
    L.append("\n### Full grid (all targets)\n")
    L.append("| Variant | Trades | Win rate | Target hit | Stop | EOD exit | Avg win R | Avg loss R | Break-even win rate | Avg R | PF | Total R | Max DD (R) | Max DD @1% |")
    L.append("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")
    for k, s in V.items():
        L.append(f"| {label(k)} | {s['trades']} | {s['win_rate_pct']}% | {s['target_hit_rate_pct']}% | {s['stop_rate_pct']}% | "
                 f"{s['eod_exit_rate_pct']}% | {s['avg_win_R']} | {s['avg_loss_R']} | {s['breakeven_win_rate_pct']}% | "
                 f"{s['avg_R']:+.3f} | {s['profit_factor']} | {s['total_R']:+.1f} | {s['max_drawdown_R']} | {s['max_drawdown_pct_1pct_risk']}% |")
    L.append("\n### Yearly win rate, headline variants (2R)\n")
    yrs = sorted({y["year"] for k in out["headline_variants"] for y in V[k]["years"]})
    L.append("| Year | " + " | ".join(label(k).replace(", 2R", "") for k in out["headline_variants"]) + " |")
    L.append("|---|" + "---|" * len(out["headline_variants"]))
    for y in yrs:
        cells = []
        for k in out["headline_variants"]:
            row = next((r for r in V[k]["years"] if r["year"] == y), None)
            cells.append(f"{row['win_rate']}% ({row['trades']}, {row['total_R']:+.0f}R)" if row else "–")
        last = out["data"]["last_day"]
        ytag = f"{y} (to {last[5:]})" if str(y) == last[:4] and last[5:7] != "12" else str(y)
        L.append(f"| {ytag} | " + " | ".join(cells) + " |")
    L.append("\nCell = win rate (trades, total R).\n")
    L.append("### Same headline variants with zero costs (no spread, slippage or commission)\n")
    L.append("| Variant | Trades | Win rate | Target hit | Avg R | PF | Total R | Years ≥60% |")
    L.append("|---|---|---|---|---|---|---|---|")
    for k, s in out["no_cost_check"].items():
        L.append(f"| {label(k)} | {s['trades']} | {s['win_rate_pct']}% | {s['target_hit_rate_pct']}% | {s['avg_R']:+.3f} | "
                 f"{s['profit_factor']} | {s['total_R']:+.1f} | {len(s['years_at_or_above_60pct_win'])} |")
    k0 = out["headline_variants"][0]
    L.append(f"\nPer year, {label(k0)}, after costs vs zero costs:\n")
    L.append("| Year | Trades | Win rate (costs) | Total R (costs) | Win rate (no costs) | Total R (no costs) |")
    L.append("|---|---|---|---|---|---|")
    zy = {y["year"]: y for y in out["no_cost_check"][k0]["years"]}
    for y in V[k0]["years"]:
        z = zy.get(y["year"], {})
        L.append(f"| {y['year']} | {y['trades']} | {y['win_rate']}% | {y['total_R']:+.1f} | {z.get('win_rate', '–')}% | {z.get('total_R', 0):+.1f} |")
    L.append("\n### Same headline variants with the most generous intrabar assumption (full costs)\n")
    L.append("Stop on the entry minute only if that minute closed beyond the stop; a stop/target tie in one minute goes to the target. This is an upper bound, not the headline.\n")
    L.append("| Variant | Trades | Win rate | Target hit | Avg R | PF | Total R | Years ≥60% |")
    L.append("|---|---|---|---|---|---|---|---|")
    for k, s in out["optimistic_intrabar_check"].items():
        L.append(f"| {label(k)} | {s['trades']} | {s['win_rate_pct']}% | {s['target_hit_rate_pct']}% | {s['avg_R']:+.3f} | "
                 f"{s['profit_factor']} | {s['total_R']:+.1f} | {len(s['years_at_or_above_60pct_win'])} |")
    sc = out.get("second_source_check") or {}
    if sc:
        L.append("\n### Second data source: HistData.com NSXUSD 1-minute (BID only, so no costs), same sessions\n")
        L.append("| Variant | Trades | Win rate | Target hit | Avg R | PF | Total R | Years ≥60% |")
        L.append("|---|---|---|---|---|---|---|---|")
        for k, s in sc.items():
            if k.startswith("_"):
                continue
            L.append(f"| {label(k)} | {s['trades']} | {s['win_rate_pct']}% | {s['target_hit_rate_pct']}% | {s['avg_R']:+.3f} | "
                     f"{s['profit_factor']} | {s['total_R']:+.1f} | {len(s['years_at_or_above_60pct_win'])} |")
        ag = sc["_day_level_agreement"]
        L.append(f"\nDay-level agreement with Dukascopy (strict variant, no costs): {ag['days_compared']} sessions, same first-breakout side "
                 f"{ag['same_first_breakout_side_pct']}%, same side and same exit type {ag['same_side_and_same_exit_type_pct']}%, "
                 f"median |IB high diff| {ag['median_abs_ib_high_diff_points']} pts, |IB low diff| {ag['median_abs_ib_low_diff_points']} pts.")
        ph = sc["_premise_histdata"]
        L.append(f"HistData premise check: rest-of-day continues IB direction {ph['rest_of_day_continues_ib_direction_pct']}%, "
                 f"point of no return held {ph['point_of_no_return_holds_pct']}%.")
    p = out["premise"]
    L.append("\n### The premise behind the rule\n")
    L.append(f"- Sessions with a directional IB candle: {p['days_with_directional_ib']}")
    L.append(f"- Rest of the day (10:00 → close) moved in the IB candle's direction: **{p['rest_of_day_continues_ib_direction_pct']}%** (claim: 80–90%)")
    L.append(f"- Day closed on the IB candle's side of the 09:30 open: {p['day_close_vs_open_matches_ib_direction_pct']}%")
    L.append(f"- 'Point of no return' held (bullish IB: IB low never broken after 10:00; bearish: IB high never broken): {p['point_of_no_return_holds_pct']}% (claim: 70–80%)")
    L.append(f"- IB broken on at least one side after 10:00: {p['days_ib_broken_either_side_pct']}% of sessions; on both sides: {p['days_ib_broken_both_sides_pct']}%")
    L.append("\n### Data\n")
    d = out["data"]
    L.append(f"- {d['source']}, {d['symbol']}, {d['bars']}. Period {d['period_label']} ({d['first_day']} → {d['last_day']}), "
             f"{d['sessions']} sessions; by year: " + ", ".join(f"{y}: {n}" for y, n in d["sessions_by_year"].items()) + ".")
    L.append(f"- Real ASK file on {d['ask_real_days']} sessions; modelled ASK on {d['ask_modelled_days']}.")
    if d["flagged_sessions"]:
        L.append(f"- {d['sessions_flagged']} sessions where the CFD feed went flat for part of the session (no ticks; "
                 f"the stale last price is used): " + ", ".join(f"{x['day']} ({x['issue']})" for x in d["flagged_sessions"]) + ".")
    L.append("- Median session spread (points) by year: " + ", ".join(f"{y}: {v}" for y, v in d["median_spread_points_by_year"].items()))
    L.append("- Median price by year: " + ", ".join(f"{y}: {v:,.0f}" for y, v in d["median_price_by_year"].items()))
    block = "\n".join(L)
    txt = MD.read_text() if MD.exists() else "<!-- SUMMARY:START -->\n<!-- SUMMARY:END -->\n<!-- RESULTS:START -->\n<!-- RESULTS:END -->\n"
    txt = replace_block(txt, "RESULTS", block)
    txt = replace_block(txt, "SUMMARY", summary_block(out))
    MD.write_text(txt)
    print(f"updated tables in {MD.relative_to(ROOT)}")


def replace_block(txt: str, name: str, body: str) -> str:
    a, b = f"<!-- {name}:START -->", f"<!-- {name}:END -->"
    if a not in txt:
        return txt
    pre, rest = txt.split(a, 1)
    _, post = rest.split(b, 1)
    return pre + a + "\n" + body + "\n" + b + post


def summary_block(out: dict) -> str:
    d, V, g, p = out["data"], out["variants"], out["grid_summary"], out["premise"]
    L = []
    if d["interim"]:
        L.append(f"> **INTERIM RUN.** Period: {d['period_label']}, {d['sessions']} NYSE sessions "
                 f"(years with complete data at the time of the run; filter `{d['years_filter']}`). "
                 f"Real Dukascopy ASK file on {d['ask_real_days']} sessions; on the other {d['ask_modelled_days']} the ASK "
                 f"is BID + the median session spread of the 4 nearest days with a real ASK file (the spread is close to "
                 f"constant through the session). The full 2015-2026 run with real BID/ASK replaces this when the download finishes; "
                 f"rules unchanged.\n")
    else:
        L.append(f"Period: {d['period_label']}, {d['sessions']} NYSE sessions, real Dukascopy BID and ASK 1-minute data"
                 + (f" (ASK modelled from nearby days' spreads on {d['ask_modelled_days']} sessions with no ASK file)" if d['ask_modelled_days'] else "")
                 + ".\n")
    L.append("**Claim:** 60–70% win rate \"over the years\", 1:2 risk-reward, 2–3 trades a week; the market continues in the IB "
             "direction 80–90% of the time.\n")
    L.append("**What the data shows (target 2R, after costs):**\n")
    for k in out["headline_variants"]:
        s = V[k]
        ny = len(s["years"])
        L.append(f"- {label(k)}: {s['trades']} trades ({s['trades_per_week']}/week), win rate **{s['win_rate_pct']}%** "
                 f"(95% CI {s['win_rate_95ci_pct'][0]}–{s['win_rate_95ci_pct'][1]}), full 2R target hit {s['target_hit_rate_pct']}%, "
                 f"avg {s['avg_R']:+.3f}R per trade, profit factor {s['profit_factor']}, total {s['total_R']:+.1f}R; "
                 f"$10,000 at 1% risk → ${s['final_equity_1pct']:,.0f} (max drawdown {s['max_drawdown_pct_1pct_risk']}%); "
                 f"years at 60%+ win rate: **{len(s['years_at_or_above_60pct_win'])} of {ny}**; "
                 f"years with a profit: {sum(1 for y in s['years'] if y['total_R'] > 0)} of {ny} "
                 f"({', '.join(f'{y['year']} {y['total_R']:+.0f}R' for y in s['years'])}).")
    vy = g["variant_years_at_or_above_60pct"]
    L.append(f"\nAcross all {g['variants_tested']} variants (1R/2R/3R targets, both stops, with/without reversal or IB-direction filter): "
             f"highest overall win rate {g['best_win_rate_pct']}% ({label(g['best_win_rate_variant'])}); "
             f"{len(vy)} of {g['variant_years_total']} variant-years reached 60%"
             + (" (" + "; ".join(f"{label(x['variant'])}, {x['year']}: {x['win_rate']}%" for x in vy) + ")" if vy else "")
             + f"; {len(g['variants_with_positive_avg_R'])} variants had a positive average R after costs"
             + (" (" + "; ".join(f"{label(k)}: {V[k]['avg_R']:+.3f}R" for k in g['variants_with_positive_avg_R']) + ")" if g['variants_with_positive_avg_R'] else "")
             + ".")
    k0 = out["headline_variants"][0]
    z, s0 = out["no_cost_check"][k0], V[k0]
    L.append(f"\nCosts: with zero spread, slippage and commission the strict variant ({label(k0)}) has a {z['win_rate_pct']}% win "
             f"rate, {z['avg_R']:+.3f}R per trade, profit factor {z['profit_factor']} (after costs: {s0['win_rate_pct']}%, "
             f"{s0['avg_R']:+.3f}R, {s0['profit_factor']}). So whether it makes or loses a little money depends on costs, "
             f"but the win rate is far below 60% either way. Costs weigh most in 2015-2017, when the index was near "
             f"4,000-6,000 and a ~1-point CFD spread was a bigger share of the stop distance.")
    L.append(f"\nPremise: after a directional first 30 minutes, the rest of the day (10:00 → close) moved the same way on "
             f"{p['rest_of_day_continues_ib_direction_pct']}% of sessions (claim: 80–90%); counting the first 30 minutes' own "
             f"move, the day closed on the IB candle's side of the open on {p['day_close_vs_open_matches_ib_direction_pct']}%. "
             f"The 'point of no return' (the far end of the IB never revisited after 10:00) held on "
             f"{p['point_of_no_return_holds_pct']}% of sessions (claim: 70–80%); measured on the strict variant's trades, the "
             f"stop at the far end of the IB survived to the close on {round(100 - V[out['headline_variants'][0]]['stop_rate_pct'], 1)}% "
             f"of trades. Of those trades {V[out['headline_variants'][0]]['eod_exit_rate_pct']}% ended at the close with neither stop nor "
             f"target hit, which is why the average winning trade is only {V[out['headline_variants'][0]]['avg_win_R']}R, not 2R. "
             f"The IB was broken on at least one side on "
             f"{p['days_ib_broken_either_side_pct']}% of sessions, i.e. about 5 signals a week, not 2–3.")
    L.append("\nBreak-even win rate for a clean 1:2 bet is 33.3%. Win = trade closed with net R > 0; most of those are "
             "end-of-day exits smaller than 2R, so the full-target hit rate is shown separately.")
    return "\n".join(L)


# ------------------------------------------------------------- spot check ---
def cmd_spot(a) -> None:
    rules = json.loads(RULES.read_text())
    days = load_days()
    rng = random.Random(a.seed)
    pick = sorted(rng.sample(range(len(days)), a.n), key=lambda i: days[i].day) if not a.days else \
        [i for i, D in enumerate(days) if str(D.day.date()) in a.days.split(",")]
    v = dict(p.split("=") for p in rules["headline_variants"][0].split("|"))
    v["target_r"] = float(v.pop("target").rstrip("R"))
    c = costs(rules, False)
    calendar_check(days)
    for i in pick:
        D = days[i]
        print("=" * 100)
        print(f"{D.day.date()} ({D.day.day_name()})  first bar {pd.Timestamp(D.t[0])}  last bar {pd.Timestamp(D.t[-1])}  bars={len(D.t)}")
        ib = D.hhmm < 1000
        print(f"IB bars 09:30-09:59 (BID): open={D.bo[0]:.2f} high={D.bh[ib].max():.2f} low={D.bl[ib].min():.2f} "
              f"close(09:59)={D.bc[ib][-1]:.2f}  range={D.bh[ib].max()-D.bl[ib].min():.2f}  spread@10:00={D.ao[ib.sum()]-D.bo[ib.sum()]:.2f}")
        print("   IB bars (first 3, last 2), BID o/h/l/c:")
        for j in [0, 1, 2, int(ib.sum()) - 2, int(ib.sum()) - 1]:
            print(f"   {pd.Timestamp(D.t[j]).strftime('%H:%M %Z')}  {D.bo[j]:.2f} {D.bh[j]:.2f} {D.bl[j]:.2f} {D.bc[j]:.2f}")
        for vv in [v, {**v, "stop": "open"}]:
            tr, fl = simulate_day(D, vv, rules, c)
            print(f"  variant stop={vv['stop']}: flags={fl}")
            for t in tr:
                print(f"   {'LONG ' if t['dir']==1 else 'SHORT'} entry {t['entry_t']} @ {t['entry_px']:.2f}  stop {t['stop']:.2f}  "
                      f"target {t['target']:.2f}  risk {t['risk']:.2f}pts -> {t['reason'].upper()} {t['exit_t']} @ {t['exit_px']:.2f}  R={t['R']:+.3f}")
        tr, _ = simulate_day(D, v, rules, c)
        if tr:
            t = tr[0]
            lo_i, hi_i = max(0, t["entry_i"] - 3), min(len(D.t), t["exit_i"] + 3)
            show = list(range(lo_i, min(hi_i, t["entry_i"] + 6)))
            if t["exit_i"] - 3 > t["entry_i"] + 6:
                show += list(range(t["exit_i"] - 3, hi_i))
            print("   bars around entry/exit (BID o/h/l/c | ASK o/h/l/c):")
            for j in show:
                tag = " <- entry" if j == t["entry_i"] else (" <- exit" if j == t["exit_i"] else "")
                print(f"   {pd.Timestamp(D.t[j]).strftime('%H:%M')}  {D.bo[j]:.2f} {D.bh[j]:.2f} {D.bl[j]:.2f} {D.bc[j]:.2f} | "
                      f"{D.ao[j]:.2f} {D.ah[j]:.2f} {D.al[j]:.2f} {D.ac[j]:.2f}{tag}")


def calendar_check(days: list[Day]) -> None:
    """DST switches and half-days: first/last bar and UTC offset per session."""
    import exchange_calendars as xc
    print("=" * 100)
    print("CALENDAR CHECK: sessions around each US DST switch and every early close")
    by = {D.day: D for D in days}
    idx = pd.DatetimeIndex(sorted(by))
    picks = []
    for y in sorted({d.year for d in idx}):
        for when in (f"{y}-03-08", f"{y}-11-01"):   # US DST starts 2nd Sun of Mar, ends 1st Sun of Nov
            w = pd.Timestamp(when)
            sw = w + pd.Timedelta(days=(6 - w.weekday()) % 7)  # that Sunday
            before = idx[idx < sw][-1:] if (idx < sw).any() else []
            after = idx[idx > sw][:1]
            picks += list(before) + list(after)
    cal = xc.get_calendar("XNYS", start="2015-01-02", end="2026-12-31")
    early = [pd.Timestamp(d.date()) for d, c in cal.schedule["close"].items()
             if c.tz_convert(NY).hour < 16 and pd.Timestamp(d.date()) in by]
    for d in sorted(set(picks) | set(early)):
        D = by[d]
        f, l = pd.Timestamp(D.t[0]), pd.Timestamp(D.t[-1])
        tag = "EARLY CLOSE" if d in early else "DST switch"
        print(f"  {d.date()} {tag:11s} first bar {f.strftime('%H:%M %z')}  last bar {l.strftime('%H:%M %z')}  bars={len(D.t)}")


# ------------------------------------------------------------ cross-check ---
def yahoo_1m(ticker: str) -> pd.DataFrame:
    """Yahoo 1-minute bars for the last ~30 days (8-day chunks), cached as csv.gz."""
    import yfinance as yf
    path = RAW / "yahoo" / f"{ticker.replace('=', '_')}_1m.csv.gz"
    if path.exists():
        df = pd.read_csv(path, index_col=0, parse_dates=True)
        df.index = pd.to_datetime(df.index, utc=True).tz_convert(NY)
        return df
    frames = []
    end = pd.Timestamp.now(tz="UTC").normalize() + pd.Timedelta(days=1)
    for k in range(4):
        e = end - pd.Timedelta(days=7 * k)
        s = e - pd.Timedelta(days=7)
        d = yf.download(ticker, start=s.strftime("%Y-%m-%d"), end=e.strftime("%Y-%m-%d"), interval="1m",
                        prepost=True, progress=False, auto_adjust=False)
        if len(d):
            if isinstance(d.columns, pd.MultiIndex):
                d.columns = d.columns.get_level_values(0)
            frames.append(d[["Open", "High", "Low", "Close", "Volume"]])
    df = pd.concat(frames).sort_index()
    df = df[~df.index.duplicated()]
    path.parent.mkdir(parents=True, exist_ok=True)
    df.to_csv(path)
    df.index = pd.to_datetime(df.index, utc=True).tz_convert(NY)
    return df


def ib_decision(o, h, l, c, hhmm, close_hhmm=1600) -> dict:
    """Same strict rule (stop = other IB end, 2R, first break, no costs) on a plain OHLC series."""
    ib = (hhmm >= 930) & (hhmm < 1000)
    if ib.sum() < 25:
        return {}
    hi, lo = h[ib].max(), l[ib].min()
    after = np.where((hhmm >= 1000) & (hhmm < close_hhmm))[0]
    res = {"ib_hi": hi, "ib_lo": lo, "ib_dir": int(np.sign(c[ib][-1] - o[ib][0]))}
    for j in after:
        up, dn = h[j] > hi, l[j] < lo
        if up and dn:
            res.update(side="both-same-minute")
            return res
        if up or dn:
            d = 1 if up else -1
            entry = hi if up else lo
            stop = lo if up else hi
            tgt = entry + 2 * (entry - stop)
            res.update(side="LONG" if up else "SHORT", entry_t=f"{hhmm[j]//100:02d}:{hhmm[j]%100:02d}")
            for k in after[after > j]:
                sh = (l[k] <= stop) if up else (h[k] >= stop)
                th = (h[k] >= tgt) if up else (l[k] <= tgt)
                if sh:
                    res["outcome"] = "stop"
                    return res
                if th:
                    res["outcome"] = "target"
                    return res
            res["outcome"] = "eod " + ("+" if (c[after[-1]] - entry) * d > 0 else "-")
            return res
    res["side"] = "none"
    return res


def cmd_crosscheck(a) -> None:
    df = pd.read_pickle(SESSION_PKL)
    rows = []
    for tk in ["NQ=F", "QQQ"]:
        y = yahoo_1m(tk)
        for day, g in y.groupby(y.index.date):
            dk = df[df["day"] == pd.Timestamp(day)]
            if dk.empty:
                continue
            gg = g[(g.index.hour * 100 + g.index.minute >= 930) & (g.index.hour * 100 + g.index.minute < 1600)]
            if len(gg) < 300:
                continue
            hh = (gg.index.hour * 100 + gg.index.minute).to_numpy()
            yd = ib_decision(*(gg[k].to_numpy(float) for k in ["Open", "High", "Low", "Close"]), hh)
            kh = (dk.index.hour * 100 + dk.index.minute).to_numpy()
            dd = ib_decision(*(dk[f"{k}_b"].to_numpy(float) for k in "ohlc"), kh, close_hhmm=kh.max() + 1)
            if not yd or not dd:
                continue
            rows.append({"day": str(day), "src": tk,
                         "yahoo": f"{yd['side']} {yd.get('entry_t','')} {yd.get('outcome','')}".strip(),
                         "duka": f"{dd['side']} {dd.get('entry_t','')} {dd.get('outcome','')}".strip(),
                         "ib_range_ratio": round((yd['ib_hi'] - yd['ib_lo']) / (dd['ib_hi'] - dd['ib_lo']), 3),
                         "same_ib_dir": yd["ib_dir"] == dd["ib_dir"],
                         "same_side": yd["side"] == dd["side"],
                         "same_outcome": yd.get("outcome", "").split(" ")[0] == dd.get("outcome", "").split(" ")[0]})
    r = pd.DataFrame(rows)
    pd.set_option("display.width", 200)
    print(r.to_string(index=False))
    for tk, g in r.groupby("src"):
        print(f"{tk}: {len(g)} days, same IB direction {g.same_ib_dir.mean():.0%}, same breakout side {g.same_side.mean():.0%}, "
              f"same outcome {g.same_outcome.mean():.0%}")
    r.to_csv(RAW / "crosscheck_yahoo_vs_dukascopy.csv", index=False)


# ------------------------------------------------------------------ plots ---
def cmd_plots(a) -> None:
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    out = json.loads(OUT_JSON.read_text())
    V = out["variants"]
    cols = ["#1f77b4", "#d62728", "#2ca02c", "#9467bd", "#ff7f0e", "#8c564b"]
    fig, axes = plt.subplots(1, 2, figsize=(15, 6))
    for k, col in zip(out["headline_variants"], cols):
        e, u = V[k]["equity_R"], V[k]["equity_usd"]
        axes[0].plot(pd.to_datetime([p[2] for p in e]), [p[1] for p in e], color=col,
                     label=f"{label(k)}  ({V[k]['total_R']:+.0f}R, {V[k]['trades']} trades)")
        axes[1].plot(pd.to_datetime([p[2] for p in u]), [p[1] for p in u], color=col,
                     label=f"{label(k)}  (${V[k]['final_equity_1pct']:,.0f})")
    axes[0].axhline(0, color="k", lw=0.8)
    axes[1].axhline(out["rules"]["start_equity"], color="k", lw=0.8)
    axes[0].set_title("Cumulative R, trade by trade (target 2R, after costs)")
    axes[1].set_title("$10,000 risking 1% per trade (compounded)")
    axes[1].set_yscale("log")
    for ax in axes:
        ax.set_xlabel("trade date")
        ax.grid(alpha=0.3)
        ax.legend(fontsize=7)
    d = out["data"]
    gap = "  |  flat stretches = years not in this run" if d.get("interim") else ""
    fig.suptitle(f"Nasdaq IB breakout, Dukascopy USATECHIDXUSD 1-min, {d['period_label']} ({d['sessions']} sessions)"
                 f"{'  |  INTERIM' if d.get('interim') else ''}{gap}")
    fig.tight_layout()
    p1 = RAW / "check_equity_curves.png"
    fig.savefig(p1, dpi=110)
    plt.close(fig)

    fig, ax = plt.subplots(figsize=(13, 6))
    keys = out["headline_variants"]
    yrs = sorted({y["year"] for k in keys for y in V[k]["years"]})
    w = 0.8 / len(keys)
    ax.axhspan(60, 70, color="gold", alpha=0.35, label="claimed 60-70% win rate")
    ax.axhline(33.3, color="k", ls="--", lw=1, label="break-even at 1:2 (33.3%, before costs)")
    for n, (k, col) in enumerate(zip(keys, cols)):
        wr = [next((r["win_rate"] for r in V[k]["years"] if r["year"] == y), np.nan) for y in yrs]
        ax.bar(np.arange(len(yrs)) + (n - len(keys) / 2 + 0.5) * w, wr, w, color=col, label=label(k))
    last = out["data"]["last_day"]
    ax.set_xticks(range(len(yrs)), [f"{y}\n(Jan-{pd.Timestamp(last).strftime('%b')})" if str(y) == last[:4] and last[5:7] != "12"
                                    else str(y) for y in yrs])
    ax.set_ylim(0, 100)
    ax.set_ylabel("win rate % (net R > 0)")
    ax.set_title(f"Yearly win rate vs the 60-70% claim (target 2R, after costs) - {out['data']['period_label']}"
                 f"{' - INTERIM' if out['data'].get('interim') else ''}")
    ax.legend(fontsize=7, ncol=2, loc="upper left")
    ax.grid(axis="y", alpha=0.3)
    fig.tight_layout()
    p2 = RAW / "check_yearly_winrate.png"
    fig.savefig(p2, dpi=110)
    plt.close(fig)
    print(p1, p2)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("cmd", choices=["run", "spot", "crosscheck", "plots"])
    ap.add_argument("--n", type=int, default=5)
    ap.add_argument("--seed", type=int, default=7)
    ap.add_argument("--days", default="")
    a = ap.parse_args()
    {"run": cmd_run, "spot": cmd_spot, "crosscheck": cmd_crosscheck, "plots": cmd_plots}[a.cmd](a)


if __name__ == "__main__":
    sys.exit(main())