"""
Chart-data integrity validator for TradingView exports
======================================================

Catches *broken or deficient* TV chart-data CSV exports BEFORE they distort
backtests, parity checks, or winner-metric refreshes.

Why this exists
---------------
On 2026-06-23 the same MLP winners produced **34× different** backtest P&L
(4H: 176% vs 5,987%) across TV exports taken minutes apart — because the
exported price history differed (a truncated / partially-loaded chart). Score
parity did NOT catch it (the recompute matched the stale score column). The
only reliable signal was the data itself: a deficient export has too few bars /
starts too late / has gaps. This tool checks the data directly.

Five preconditions (per CLAUDE.md Primary Directive):
  1. Objective metric : number of files failing integrity checks (target 0).
  2. Measurement      : the per-file structural checks below.
  3. Action space     : exit non-zero (blocks downstream use) + optional ntfy.
  4. Stopping         : one pass per invocation (run after each export, or wire
                        to a launchd WatchPaths agent — see --help epilog).
  5. Guardrail        : callers (compare_tv_trades, refresh_winner_metrics) can
                        import `validate_file` as a preflight and refuse to run
                        on ERROR-level data.

Checks (per data CSV)
---------------------
  ERROR (hard fail — data is unusable / will produce wrong backtests):
    - not a chart-data file (looks like a trade-list export)
    - missing OHLC/time columns
    - NaN / <=0 prices; high<low; high<max(open,close); low>min(open,close)
    - non-monotonic or duplicate timestamps
    - TRUNCATED history: first bar later than SCORE_START (IS window not covered)
      or row count << expected for the date span & timeframe
  WARN (review — may be benign):
    - first bar later than TRAIN_START (indicator warm-up may be short)
    - last bar far behind "now" (stale export)
    - large time gaps between bars
    - extreme single-bar moves (possible price glitch; legit crashes also land here)

Usage
-----
    # validate every chart CSV in data/mlp/ (default)
    python3 tools/validate_chart_data.py

    # validate specific files
    python3 tools/validate_chart_data.py "data/mlp/COINBASE_BTCUSD, 240.csv"

    # treat warnings as failures; push an ntfy alert on any failure
    python3 tools/validate_chart_data.py --strict --notify

Exit code: 0 = all clean (warnings allowed unless --strict); 1 = ≥1 file failed.
"""

import argparse
import glob
import sys
from pathlib import Path
from typing import Optional

import numpy as np
import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from config import SCORE_START, TRAIN_START  # noqa: E402

# Timeframe → bar interval. Mirrors tools/compare_tv_trades.py.
TF_DURATIONS = {
    "4H": pd.Timedelta(hours=4), "6H": pd.Timedelta(hours=6),
    "8H": pd.Timedelta(hours=8), "12H": pd.Timedelta(hours=12),
    "1D": pd.Timedelta(days=1),
}
_MINUTES_TO_TF = {"240": "4H", "360": "6H", "480": "8H", "720": "12H"}

# Assets whose price history predates SCORE_START. For these, a first bar AFTER
# SCORE_START means a truncated export (hard error). Younger assets (SOL launched
# 2020, LINK listed 2019) legitimately start later → that's a warning, not an error.
ASSETS_FULL_HISTORY = ("COINBASE_BTCUSD", "COINBASE_ETHUSD")

# Thresholds
EXTREME_MOVE = 0.50          # |close pct change| above this is flagged (review)
GAP_FACTOR = 1.5             # diff > GAP_FACTOR × bar interval is a gap
MIN_ROW_COVERAGE = 0.90      # actual rows must be ≥ this × expected for the span
STALE_EXPORT_DAYS = 30       # last bar older than this (vs now) → WARN


def detect_asset(path: str) -> str:
    """Asset prefix from a TV export filename, e.g. 'COINBASE_BTCUSD, 240.csv'."""
    stem = Path(path).stem
    return stem.split(",")[0].strip().split("-")[0].strip()


def detect_timeframe(path: str) -> Optional[str]:
    name = Path(path).stem.upper()
    for tf in TF_DURATIONS:
        if name.endswith(f"-{tf}") or name.endswith(f"_{tf}") or name.endswith(f", {tf}"):
            return tf
    for mins, tf in _MINUTES_TO_TF.items():
        if name.endswith(f", {mins}"):
            return tf
    return None


def validate_file(path: str) -> tuple[list[str], list[str], dict]:
    """Validate one chart-data CSV.

    Returns (errors, warnings, stats). Empty errors → usable data.
    Importable as a preflight: `errs, warns, _ = validate_file(p)`.
    """
    errors: list[str] = []
    warnings: list[str] = []
    stats: dict = {}

    p = Path(path)
    if not p.exists():
        return [f"file not found: {path}"], [], stats

    try:
        df = pd.read_csv(path)
    except Exception as exc:  # noqa: BLE001
        return [f"could not read CSV: {exc}"], [], stats
    df.columns = df.columns.str.lower().str.strip()

    # Wrong file type: a trade-list export, not chart data.
    if "trade number" in df.columns or {"type", "signal"}.issubset(df.columns):
        return ["looks like a trade-list export, not chart data"], [], stats

    required = {"time", "open", "high", "low", "close"}
    missing = required - set(df.columns)
    if missing:
        errors.append(f"missing required columns: {sorted(missing)}")
        return errors, warnings, stats

    n = len(df)
    stats["rows"] = n
    if n == 0:
        return ["empty file (0 rows)"], [], stats

    # --- Timestamps ---
    t = pd.to_datetime(df["time"], errors="coerce", utc=True)
    if t.isna().any():
        errors.append(f"{int(t.isna().sum())} unparseable timestamps")
    t = t.dt.tz_localize(None)
    if t.dropna().duplicated().any():
        errors.append(f"{int(t.duplicated().sum())} duplicate timestamps")
    if not t.dropna().is_monotonic_increasing:
        errors.append("timestamps are not strictly increasing")
    first, last = t.min(), t.max()
    stats["first"], stats["last"] = first, last

    # --- OHLC sanity ---
    for col in ("open", "high", "low", "close"):
        v = pd.to_numeric(df[col], errors="coerce")
        if v.isna().any():
            errors.append(f"{col}: {int(v.isna().sum())} NaN/non-numeric values")
        if (v <= 0).any():
            errors.append(f"{col}: {int((v <= 0).sum())} values <= 0")
    o, h, l, c = (pd.to_numeric(df[x], errors="coerce") for x in ("open", "high", "low", "close"))
    if (h < l).any():
        errors.append(f"{int((h < l).sum())} bars with high < low")
    if (h < o).any() or (h < c).any():
        errors.append(f"{int(((h < o) | (h < c)).sum())} bars with high < open/close")
    if (l > o).any() or (l > c).any():
        errors.append(f"{int(((l > o) | (l > c)).sum())} bars with low > open/close")

    # --- Coverage / truncation (the check that catches the 34× data drift) ---
    score_start = pd.Timestamp(SCORE_START)
    train_start = pd.Timestamp(TRAIN_START)
    asset = detect_asset(path)
    if pd.notna(first):
        if first > score_start and asset in ASSETS_FULL_HISTORY:
            errors.append(
                f"TRUNCATED history: first bar {first.date()} is after SCORE_START "
                f"{score_start.date()} — {asset} has older history; the in-sample "
                f"window is not fully covered (likely a partially-loaded chart export)"
            )
        elif first > score_start:
            warnings.append(
                f"first bar {first.date()} is after SCORE_START {score_start.date()} "
                f"— in-sample window not fully covered (expected if {asset} is younger "
                f"than SCORE_START; a truncated export otherwise)"
            )
        elif first > train_start:
            warnings.append(
                f"first bar {first.date()} is after TRAIN_START {train_start.date()} "
                f"— indicator warm-up may be short"
            )
    now = pd.Timestamp.now("UTC").replace(tzinfo=None)
    if pd.notna(last) and (now - last).days > STALE_EXPORT_DAYS:
        warnings.append(f"last bar {last.date()} is {(now - last).days}d behind now (stale export?)")

    # --- Timeframe-aware gap + row-count checks ---
    tf = detect_timeframe(path)
    stats["tf"] = tf
    if tf and pd.notna(first) and pd.notna(last):
        interval = TF_DURATIONS[tf]
        diffs = t.dropna().diff().dropna()
        gaps = diffs[diffs > interval * GAP_FACTOR]
        if len(gaps):
            warnings.append(
                f"{len(gaps)} time gaps > {GAP_FACTOR}×{tf} (largest {gaps.max()})"
            )
        expected = int((last - first) / interval) + 1
        stats["expected_rows"] = expected
        if n < expected * MIN_ROW_COVERAGE:
            errors.append(
                f"TRUNCATED/sparse: {n} rows but ~{expected} expected for "
                f"{first.date()}→{last.date()} at {tf} ({n / expected:.0%} coverage)"
            )

    # --- Extreme single-bar moves (glitch review) ---
    ret = c.pct_change().abs()
    extreme = ret[ret > EXTREME_MOVE]
    if len(extreme):
        worst = ret.idxmax()
        warnings.append(
            f"{len(extreme)} bars move >{EXTREME_MOVE:.0%} in one bar "
            f"(worst {ret.max():.0%} at {t.iloc[worst].date() if pd.notna(t.iloc[worst]) else '?'}) "
            f"— verify not a price glitch"
        )

    return errors, warnings, stats


def main():
    ap = argparse.ArgumentParser(
        description="Validate TradingView chart-data CSV exports for integrity.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=(
            "Wire as an on-change guard (externally-created files) via launchd WatchPaths:\n"
            "  ~/Library/LaunchAgents/com.james.tradingbot-data-validator.plist with\n"
            "  WatchPaths=[<repo>/data/mlp] running:\n"
            "    python3 tools/validate_chart_data.py --notify   (alerts on ERRORs only)\n"
            "  Logs → projects/logs/tradingbot-data-validator.log.\n"
        ),
    )
    ap.add_argument("files", nargs="*", help="chart CSVs (default: data/mlp chart files)")
    ap.add_argument("--strict", action="store_true", help="treat warnings as failures")
    ap.add_argument("--quiet", action="store_true", help="only print files with problems")
    ap.add_argument("--notify", action="store_true", help="send an ntfy alert on failure")
    args = ap.parse_args()

    files = args.files
    if not files:
        root = Path(__file__).resolve().parent.parent
        files = sorted(
            f for f in glob.glob(str(root / "data" / "mlp" / "*.csv"))
            if not Path(f).name.startswith("MLPScores_")
        )
    if not files:
        print("No chart-data files found to validate.")
        return 0

    n_err = n_warn = 0
    failed_files = []
    for f in files:
        errors, warnings, stats = validate_file(f)
        ok = not errors and (not warnings or not args.strict)
        if ok and args.quiet:
            continue
        icon = "✅" if not errors and not warnings else ("❌" if errors else "⚠️ ")
        span = ""
        if stats.get("first") is not None:
            span = f"  [{stats.get('rows','?')} rows, {pd.Timestamp(stats['first']).date()}→{pd.Timestamp(stats['last']).date()}]"
        print(f"{icon} {Path(f).name}{span}")
        for e in errors:
            print(f"     ERROR: {e}")
        for w in warnings:
            print(f"     warn:  {w}")
        if errors:
            n_err += 1
            failed_files.append(Path(f).name)
        elif warnings:
            n_warn += 1
            if args.strict:
                failed_files.append(Path(f).name)

    print(f"\n{len(files)} file(s): {n_err} with errors, {n_warn} with warnings only.")
    failed = bool(n_err) or (args.strict and bool(n_warn))
    if failed and args.notify:
        try:
            from tools.ntfy import notify
            notify(
                f"Chart-data validation FAILED: {', '.join(failed_files)}",
                title="TradingBot data integrity",
                priority="high",
                tags=["warning"],
            )
        except Exception as exc:  # noqa: BLE001
            print(f"(ntfy failed: {exc})")
    return 1 if failed else 0


if __name__ == "__main__":
    sys.exit(main())