"""
Auto-Optimizer Loop
Iteratively runs the GPU random search optimizer and tracks the best result across runs.
Parameter ranges are never modified automatically — edit the params JSON file manually.
"""

import os
import json
import subprocess
import pandas as pd
import sys
import argparse
import time
import math
import threading
import io
import sqlite3
import uuid
from concurrent.futures import ProcessPoolExecutor, as_completed

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import strategies.strategy_activation_scores as strategy_module
from config import TRAIN_START, SCORE_START, TRAIN_END, OOS_START, RESULTS_DIR, WINNERS_DIR, SWEEPS_DIR, REPORTS_DIR, OUTPUT_DIR, MVRV_BEAR_THRESHOLD as _MVRV_BEAR_DEFAULT, MVRV_BULL_THRESHOLD as _MVRV_BULL_DEFAULT, composite_score as _composite_score, SUBPERIOD_SPLIT, WEIGHT_COLS, WFO_FOLDS, WFO_MIN_OOS_TRADES, WFO_MIN_VALID_FOLDS, WFO_TOP_N

# --- Configuration ---
STRATEGY_FILE = "strategy_activation_scores.py"
OPTIMIZATION_METRIC = "Composite"

# TradingView export with pre-calculated indicators (overridden by --data CLI arg)
DATA_FILE = "data/mlp/COINBASE_BTCUSD, 1D.csv"
ASSET = "COINBASE_BTCUSD"
TIMEFRAME = "1D"
PARAMS_FILE = f"strategies/params/params_strategy_activation_scores_{ASSET}_{TIMEFRAME}.json"

SEARCH_ITERATIONS = 50000000  # Increased to 50 Million
MAX_ITERATIONS = 3
SEARCH_MODE = "random"  # overridden by --search arg in main()
REGIME = "all"                          # overridden by --regime arg in main()
REGIME_BEAR_THRESHOLD = _MVRV_BEAR_DEFAULT  # overridden by --regime-bear-threshold arg in main()
REGIME_BULL_THRESHOLD = _MVRV_BULL_DEFAULT  # overridden by --regime-bull-threshold arg in main()
SWEEP_DB_FILE = "results/sweep_database.db"
DB_SCHEMA_VER = 3  # v3: SCORE_START moved to 2017-01-01 (was 2018-01-01); i_w_mvrv_cont unlocked (zscore col present)
GPU_RATE = 140000  # 140k samples/s an average seen recently for Random searches

class Logger(object):
    def __init__(self, filename="optimization_log.txt"):
        self.terminal = sys.stdout
        self.log = open(filename, "w", encoding="utf-8")

    def write(self, message):
        self.terminal.write(message)
        self.log.write(message)
        self.log.flush()

    def flush(self):
        self.terminal.flush()
        self.log.flush()


def write_winner_readable(winner_row, strategy_name, results_dir, asset="", timeframe="1D"):
    """
    Writes two human-readable companion files alongside the winner CSV:
      - optimization_winner_<name>_cheatsheet.txt : TradingView UI groups + values
      - optimization_winner_<name>_pine_snippet.pine : input.float() defaults to paste
    """
    # Map param name -> (TV UI label, TV group, Pine type)
    # Tuples: (param_key, tv_label, tv_group, pine_type, default)
    # default=None  → value comes from winner_row (optimized param)
    # default=<val> → fixed Pine default; not optimized but must be set in TradingView
    PARAM_META = [
        ("i_long_entry_activation_threshold",             "Long Entry Activation Threshold",          "Neural Activation Thresholds", "float", None),
        ("i_long_exit_activation_threshold",              "Long Exit Activation Threshold",           "Neural Activation Thresholds", "float", None),
        ("i_long_exit_activation_confirmation_threshold", "Long Exit Activation Confirmation Threshold", "Neural Activation Thresholds", "float", None),
        ("i_use_long_exit_confirmation",                  "Use Long Exit Confirmation",               "Neural Activation Thresholds", "bool",  None),
        ("i_use_long_entry_confirmation",                 "Use Long Entry Confirmation",              "Neural Activation Thresholds", "bool",  None),
        ("i_trailing_stop_threshold",                     "Trailing Stop % (0 = disabled)",           "Neural Activation Thresholds", "float", None),
        ("i_w_stoch",                                     "Stochastic Weight",                        "Neural Activation Weights",    "float", None),
        ("i_w_macd_pred",                                 "MACD Prediction Weight",                   "Neural Activation Weights",    "float", None),
        ("i_w_osc",                                       "OSC Weight",                               "Neural Activation Weights",    "float", None),
        ("i_w_macd_bullish",                              "MACD Bullish Weight",                      "Neural Activation Weights",    "float", None),
        ("i_w_m3_momentum",                               "M3 Momentum Weight",                       "Neural Activation Weights",    "float", None),
        ("i_m3_momentum_period",                          "M3 Momentum Period",                       "Neural Activation Weights",    "int",   None),
        ("i_w_m2_tiny",                                   "M2 Tiny Momentum Weight",                  "Neural Activation Weights",    "float", None),
        ("i_w_rsid_osc",                                  "RSID OSC Weight",                          "Neural Activation Weights",    "float", None),
        ("i_w_stoch_div_osc",                             "Stoch Div OSC Weight",                     "Neural Activation Weights",    "float", None),
        ("i_w_vwap_div_osc",                              "VWAP Div OSC Weight",                      "Neural Activation Weights",    "float", None),
        ("i_w_stoch_peaking",                             "Stoch Peaking Weight",                     "Neural Activation Weights",    "float", None),
        ("i_w_stoch_bottoming",                           "Stoch Bottoming Weight",                   "Neural Activation Weights",    "float", None),
        ("i_w_m3_div_osc",                                "M3 Div OSC Weight",                        "Neural Activation Weights",    "float", None),
        ("i_w_m2_div_osc",                                "M2 Div OSC Weight",                        "Neural Activation Weights",    "float", None),
        ("i_w_m2_div_osc_noOffset",                       "M2 Div OSC (No Offset) Weight",            "Neural Activation Weights",    "float", None),
        ("i_w_bearish_engulfing",                         "Bearish Engulfing Weight",                 "Candlestick Patterns",         "float", None),
        ("i_cs_body_quality_ratio",                       "Body Quality Ratio",                       "Candlestick Patterns",         "float", None),
        ("i_cs_confidence_scaling_factor",                "Confidence Scaling Factor",                "Candlestick Patterns",         "float", None),
        ("i_w_bullish_hammer",                            "Bullish Hammer Weight",                    "Candlestick Patterns",         "float", None),
        ("i_cs_hammer_min_lower_shadow",                  "Hammer Min Lower Shadow Ratio",            "Candlestick Patterns",         "float", 0.60),
        ("i_cs_hammer_max_body_ratio",                    "Hammer Max Body Ratio",                    "Candlestick Patterns",         "float", 0.35),
        ("i_cs_hammer_max_upper_shadow",                  "Hammer Max Upper Shadow Ratio",            "Candlestick Patterns",         "float", 0.15),
        ("i_w_bullish_engulfing",                         "Bullish Engulfing Weight",                 "Candlestick Patterns",         "float", None),
        ("i_cs_bull_eng_scaling_factor",                  "Bullish Engulfing Scaling Factor",         "Candlestick Patterns",         "float", 2.5),
        ("i_w_shooting_star",                             "Shooting Star Weight",                     "Candlestick Patterns",         "float", None),
        ("i_cs_star_min_upper_shadow",                    "Shooting Star Min Upper Shadow",           "Candlestick Patterns",         "float", 0.60),
        ("i_cs_star_max_body_ratio",                      "Shooting Star Max Body Ratio",             "Candlestick Patterns",         "float", 0.35),
        ("i_cs_star_max_lower_shadow",                    "Shooting Star Max Lower Shadow",           "Candlestick Patterns",         "float", 0.15),
        ("i_w_btc_spx_corr",                              "BTC/SPX Correlation Weight",               "Macro Signals",                "float", None),
        ("i_w_dxy",                                       "DXY Weight",                               "Macro Signals",                "float", None),
        ("i_w_vix",                                       "VIX Weight",                               "Macro Signals",                "float", None),
        ("i_w_btc_dom",                                   "BTC Dominance Weight",                     "Macro Signals",                "float", None),
        ("i_w_us10y",                                     "US 10Y Yield Weight",                      "Macro Signals",                "float", None),
        ("i_w_spy",                                       "SPY Weight",                               "Macro Signals",                "float", None),
        ("i_w_gold",                                      "Gold Weight",                              "Macro Signals",                "float", None),
        ("i_w_mvrv",                                      "MVRV Z-Score Weight (discrete)",           "On-Chain Regime Signals",      "float", None),
        ("i_w_mvrv_cont",                                 "MVRV Z-Score Weight (continuous)",         "On-Chain Regime Signals",      "float", None),
        ("i_w_nupl",                                      "Realized Price NUPL Weight",               "On-Chain Regime Signals",      "float", None),
        ("i_w_fed_net_liq",                               "Fed Net Liquidity Sign Weight",            "On-Chain Regime Signals",      "float", None),
        ("i_w_gc_position",                               "Gaussian Channel Position Weight",          "On-Chain Regime Signals",      "float", None),
        ("i_w_us2y",                                      "US 2Y Yield Weight",                        "Rates & Macro",                "float", None),
        ("i_w_yield_curve",                               "Yield Curve (10Y-2Y) Weight",               "Rates & Macro",                "float", None),
        ("i_w_qqq_spy_ratio",                             "QQQ/SPY Ratio Weight",                      "Rates & Macro",                "float", None),
        ("i_regime_window",                               "Regime Filter Window (0=off)",             "MVRV Macro Regime",            "int",   None),
        ("i_regime_entry_min_score",                      "Regime Filter Min Score",                  "MVRV Macro Regime",            "float", None),
        ("i_mvrv_suppress_bear",                          "Suppress Entries in Bear Regime",          "MVRV Macro Regime",            "bool",  None),
        ("i_w_rsid_reg_bull",                             "RSI Div Regular Bullish Weight",           "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_reg_bear",                             "RSI Div Regular Bearish Weight",           "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_hid_bull",                             "RSI Div Hidden Bullish Weight",            "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_hid_bear",                             "RSI Div Hidden Bearish Weight",            "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_rt_bull",                              "RSI Div Real-Time Bullish Weight",         "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_rt_bear",                              "RSI Div Real-Time Bearish Weight",         "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_slow_bull",                            "RSI Div Slowing Bullish Weight",           "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_slow_bear",                            "RSI Div Slowing Bearish Weight",           "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_delayed_peak",                         "RSI Div Delayed Peak Weight",              "RSI Divergence Signals",       "float", None),
        ("i_w_rsid_delayed_dip",                          "RSI Div Delayed Dip Weight",               "RSI Divergence Signals",       "float", None),
        ("i_div_window",                                  "RSI Div Window (bars signal stays active)","RSI Divergence Signals",       "int",   None),
        ("i_w_oi_roc",                                    "Perp OI ROC Weight",                       "Derivatives & Market Structure","float", None),
        ("i_w_usdt_d",                                    "USDT Dominance ROC Weight",                "Derivatives & Market Structure","float", None),
        ("i_w_basis",                                     "Spot-Perp Basis Weight",                   "Derivatives & Market Structure","float", None),
        ("i_w_fear_greed",                                "Fear & Greed Index Weight (disabled)",     "Sentiment & Sub-TF Signals",   "float", None),
        ("i_w_btc_gold",                                  "BTC/Gold Ratio Weight",                    "Sentiment & Sub-TF Signals",   "float", None),
        ("i_w_rsi_subtf",                                 "Sub-TF RSI Weight (half chart period)",    "Sentiment & Sub-TF Signals",   "float", None),
    ]

    calmar    = winner_row.get("Calmar Ratio",  "N/A")
    sortino   = winner_row.get("Sortino Ratio", "N/A")
    sharpe    = winner_row.get("Sharpe Ratio",  "N/A")
    pnl       = winner_row.get("Total P&L %",   "N/A")
    dd        = winner_row.get("Max Drawdown %","N/A")
    trades    = winner_row.get("Total Trades",  "N/A")
    composite = winner_row.get("Composite",     "N/A")
    import datetime as _dt
    date_str = _dt.date.today().isoformat()

    composite_str = f"{float(composite):.4f}" if composite != "N/A" else "N/A"
    wfo_score = winner_row.get("WFO_Score", None)
    wfo_str = f"{float(wfo_score):.4f}" if wfo_score not in (None, "N/A") and float(wfo_score) > 0 else "N/A"
    selection_label = "WFO-selected" if wfo_str != "N/A" else "IS-Composite-selected"
    asset_tf = f"{asset}_{timeframe}" if asset else timeframe
    header = (
        f"{'='*72}\n"
        f"  OPTIMIZED PARAMETERS — {asset_tf} — {date_str}  [{selection_label}]\n"
        f"  Composite: {composite_str}  |  Calmar: {float(calmar):.4f}  |  Sortino: {float(sortino):.4f}  |  Sharpe: {float(sharpe):.4f}\n"
        f"  WFO Score: {wfo_str}  (mean OOS Calmar across 5 temporal folds)\n"
        f"  P&L: {float(pnl):,.1f}%  |  Max Drawdown: {float(dd):.2f}%  |  Trades: {int(float(trades))}\n"
        f"  NOTE: Metrics above cover the SCORING WINDOW ({SCORE_START} → {TRAIN_END}) only.\n"
        f"{'='*72}\n"
    )

    def _fmt(val, pine_type, default):
        """Format a param value; fall back to default if val is missing."""
        if (val == "N/A" or val is None) and default is not None:
            val = default
        try:
            if pine_type == "bool":
                return "true" if str(val).lower() in ("true", "1") else "false"
            elif pine_type == "int":
                return str(int(float(val)))
            else:
                return f"{float(val):g}"
        except Exception:
            return str(val)

    # Params already handled by PARAM_META — used to detect unlisted extras below
    _param_meta_keys = {key for key, *_ in PARAM_META}

    # --- Cheat-sheet (.txt) ---
    lines = [header]
    current_group = None
    for key, label, group, pine_type, default in PARAM_META:
        if group != current_group:
            lines.append(f"\n--- {group} ---")
            current_group = group
        val = winner_row.get(key, "N/A")
        suffix = "  (fixed Pine default)" if (val == "N/A" or val is None) and default is not None else ""
        val_str = _fmt(val, pine_type, default)
        lines.append(f"  {label:<48} {val_str:<12}  ({key}){suffix}")

    # Fallback: any i_w_* params in winner_row not covered by PARAM_META
    # (catches newly-added signals before PARAM_META is updated)
    # Use .keys() so this works whether winner_row is a dict or a pandas Series
    # (iterating a Series yields values, not index labels)
    unlisted = [k for k in winner_row.keys() if isinstance(k, str) and k.startswith('i_w_') and k not in _param_meta_keys]
    if unlisted:
        lines.append(f"\n--- ⚠ Unlisted params (add to PARAM_META in write_winner_readable) ---")
        for key in sorted(unlisted):
            val_str = _fmt(winner_row.get(key, "N/A"), "float", None)
            lines.append(f"  {key:<48} {val_str:<12}  ({key})")

    cheatsheet_path = os.path.join(results_dir, f"optimization_winner_{strategy_name}_{asset_tf}_cheatsheet.txt")
    with open(cheatsheet_path, "w") as f:
        f.write("\n".join(lines) + "\n")

    # --- Pine snippet (.pine) ---
    pine_lines = [
        f"// === Optimized Defaults — {asset_tf} — {date_str} ===",
        f"// Calmar {float(calmar):.4f} | Sortino {float(sortino):.4f} | P&L {float(pnl):,.0f}% | DD {float(dd):.2f}% | Trades {int(float(trades))}",
        f"// Metrics above: scoring window {SCORE_START} → {TRAIN_END} only (TradingView shows full history from {TRAIN_START})",
        f"// Replace the first argument of each input.float/bool/int() call below:",
        "",
    ]
    current_group = None
    for key, label, group, pine_type, default in PARAM_META:
        if group != current_group:
            pine_lines.append(f"// --- {group} ---")
            current_group = group
        val = winner_row.get(key, "N/A")
        val_str = _fmt(val, pine_type, default)
        comment = "  // fixed Pine default" if (val == "N/A" or val is None) and default is not None else ""
        pine_lines.append(f"{key:<48} = input.{pine_type}({val_str}, ...){comment}")

    # Fallback: unlisted i_w_* extras (same safety net as cheatsheet)
    if unlisted:
        pine_lines.append("// --- ⚠ Unlisted params (add to PARAM_META) ---")
        for key in sorted(unlisted):
            val_str = _fmt(winner_row.get(key, "N/A"), "float", None)
            pine_lines.append(f"{key:<48} = input.float({val_str}, ...)  // ⚠ add to PARAM_META")

    pine_path = os.path.join(results_dir, f"optimization_winner_{strategy_name}_{asset_tf}_pine_snippet.pine")
    with open(pine_path, "w") as f:
        f.write("\n".join(pine_lines) + "\n")

    print(f"   >> [READABLE] Cheat-sheet: {cheatsheet_path}")
    print(f"   >> [READABLE] Pine snippet: {pine_path}")


def sanitize_params(params):
    """Ensures parameter ranges are valid (start < stop, step > 0)."""
    # Parameters that MUST be fixed (values list) to prevent GPU/CPU mismatch
    locked_params = ['i_m3_momentum_period']

    for key, config in params.items():
        if key in locked_params and 'start' in config:
            # Convert back to fixed values if AI tried to make them ranges
            val = config.get('start', 0)
            params[key] = {'values': [val]}
            continue

        if 'start' in config and 'stop' in config and 'step' in config:
            try:
                start = float(config['start'])
                stop = float(config['stop'])
                step = float(config['step'])
                
                # 1. Fix inverted ranges
                if start > stop:
                    start, stop = stop, start
                
                # 2. Fix zero/negative step
                if step <= 0:
                    step = abs(step) if step != 0 else 0.1
                    
                # 3. Ensure step isn't larger than the range (which would yield only 1 value)
                if step > (stop - start) and start != stop:
                    step = (stop - start) / 5.0  # Force at least 5 steps
                
                # 4. Ensure step isn't too small (preventing massive search spaces)
                min_step = (stop - start) / 1000.0
                if step < min_step:
                    step = min_step

                config['start'], config['stop'], config['step'] = start, stop, step
            except (ValueError, TypeError):
                continue
    return params

def ensure_winner_in_search_space(params, winner_row):
    """
    Ensures that the parameters from the winner_row are included in the search space
    defined by params. Modifies params in-place.
    """
    if winner_row is None:
        return

    print("   >> [SAFEGUARD] Extending search space to include global best parameters (preventing drift)...")
    
    for key, config in params.items():
        if key in winner_row:
            try:
                best_val = float(winner_row[key])
                if 'start' in config and 'stop' in config:
                    if best_val < config['start']:
                        config['start'] = best_val
                    if best_val > config['stop']:
                        config['stop'] = best_val
            except (ValueError, TypeError):
                continue

def init_sweep_db():
    """Create the sweep results database and table if they don't exist."""
    os.makedirs(RESULTS_DIR, exist_ok=True)
    os.makedirs(WINNERS_DIR, exist_ok=True)
    os.makedirs(SWEEPS_DIR, exist_ok=True)
    os.makedirs(REPORTS_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    conn = sqlite3.connect(SWEEP_DB_FILE)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS sweep_results (
            id              INTEGER PRIMARY KEY AUTOINCREMENT,
            asset           TEXT    NOT NULL,
            timeframe       TEXT    NOT NULL,
            schema_ver      INTEGER NOT NULL DEFAULT 1,
            recorded_at     TEXT    NOT NULL,
            run_id          TEXT,
            train_start     TEXT,
            train_end       TEXT,
            score_start     TEXT,
            calmar_ratio    REAL,
            sortino_ratio   REAL,
            sharpe_ratio    REAL,
            pnl_dd_ratio        REAL,
            pnl_dd_percentile   REAL,
            total_pnl_pct       REAL,
            max_drawdown        REAL,
            total_trades        INTEGER,
            pct_in_market       REAL,
            i_long_entry_activation_threshold              REAL,
            i_long_exit_activation_threshold               REAL,
            i_long_exit_activation_confirmation_threshold  REAL,
            i_use_long_exit_confirmation                   INTEGER,
            i_use_long_entry_confirmation                  INTEGER,
            i_trailing_stop_threshold                      REAL,
            i_w_stoch                REAL,
            i_w_macd_pred            REAL,
            i_w_osc                  REAL,
            i_w_macd_bullish         REAL,
            i_w_m3_momentum          REAL,
            i_m3_momentum_period     REAL,
            i_w_m2_tiny              REAL,
            i_w_rsid_osc             REAL,
            i_w_stoch_div_osc        REAL,
            i_w_vwap_div_osc         REAL,
            i_w_stoch_peaking        REAL,
            i_w_stoch_bottoming      REAL,
            i_w_m3_div_osc           REAL,
            i_w_m2_div_osc           REAL,
            i_w_m2_div_osc_noOffset  REAL,
            i_w_bearish_engulfing           REAL,
            i_w_bullish_hammer              REAL,
            i_w_bullish_engulfing           REAL,
            i_w_shooting_star               REAL,
            i_cs_body_quality_ratio         REAL,
            i_cs_confidence_scaling_factor  REAL,
            i_cs_bull_eng_scaling_factor    REAL,
            i_w_btc_spx_corr  REAL,
            i_w_dxy           REAL,
            i_w_vix           REAL,
            i_w_btc_dom       REAL,
            i_w_us10y         REAL,
            i_w_spy           REAL,
            i_w_gold          REAL,
            i_w_mvrv          REAL,
            i_w_mvrv_cont     REAL,
            i_w_nupl          REAL,
            i_w_fed_net_liq   REAL,
            i_w_gc_position   REAL,
            i_w_us2y             REAL,
            i_w_yield_curve      REAL,
            i_w_qqq_spy_ratio    REAL,
            i_w_rsid_reg_bull    REAL,
            i_w_rsid_reg_bear    REAL,
            i_w_rsid_hid_bull    REAL,
            i_w_rsid_hid_bear    REAL,
            i_w_rsid_rt_bull     REAL,
            i_w_rsid_rt_bear     REAL,
            i_w_rsid_slow_bull   REAL,
            i_w_rsid_slow_bear   REAL,
            i_w_rsid_delayed_peak REAL,
            i_w_rsid_delayed_dip  REAL,
            i_w_oi_roc           REAL,
            i_w_usdt_d           REAL,
            i_w_basis            REAL,
            i_regime_window         REAL,
            i_regime_entry_min_score REAL,
            i_mvrv_suppress_bear    REAL,
            iteration_number        INTEGER,
            gpu_score               REAL,
            composite_score         REAL,
            subperiod_consistent    INTEGER   -- NULL=unchecked, 0=failed, 1=passed
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_asset_tf   ON sweep_results (asset, timeframe)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_pnl_dd     ON sweep_results (pnl_dd_ratio)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_run_id     ON sweep_results (run_id)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_schema_ver ON sweep_results (schema_ver)")

    # Schema migration: add any columns present in _PARAM_COLS / _METRIC_TO_COL that are
    # missing from the existing table (happens when new signals are added after the DB exists).
    existing_cols = {r[1] for r in conn.execute("PRAGMA table_info(sweep_results)").fetchall()}
    for col in list(_PARAM_COLS) + list(_METRIC_TO_COL.values()):
        if col not in existing_cols:
            conn.execute(f"ALTER TABLE sweep_results ADD COLUMN {col} REAL")
            print(f"   >> [DB] Migrated: added column '{col}' to sweep_results")
    for col, col_type in [("iteration_number", "INTEGER"), ("gpu_score", "REAL"),
                          ("subperiod_consistent", "INTEGER"), ("search_strategy", "TEXT")]:
        if col not in existing_cols:
            conn.execute(f"ALTER TABLE sweep_results ADD COLUMN {col} {col_type}")
            print(f"   >> [DB] Migrated: added column '{col}' to sweep_results")

    conn.commit()
    conn.close()


# Mapping from result-dict keys to DB column names
_METRIC_TO_COL = {
    "P&L/DD Ratio":   "pnl_dd_ratio",
    "Total P&L %":    "total_pnl_pct",
    "Max Drawdown %": "max_drawdown",
    "Total Trades":   "total_trades",
    "% In Market":    "pct_in_market",
    "Calmar Ratio":   "calmar_ratio",
    "Sortino Ratio":  "sortino_ratio",
    "Sharpe Ratio":   "sharpe_ratio",
    "Composite":      "composite_score",
    "WFO_Score":      "wfo_score",
}

# All param column names that exist in the DB schema.
# Weight params (i_w_*) come from config.WEIGHT_COLS — the single source of truth.
# To add a new signal: add it to config.WEIGHT_COLS. The DB migration runs automatically.
_NON_WEIGHT_PARAM_COLS = {
    "i_long_entry_activation_threshold",
    "i_long_exit_activation_threshold",
    "i_long_exit_activation_confirmation_threshold",
    "i_use_long_exit_confirmation",
    "i_use_long_entry_confirmation",
    "i_trailing_stop_threshold",
    "i_m3_momentum_period",
    "i_cs_body_quality_ratio",
    "i_cs_confidence_scaling_factor",
    "i_cs_bull_eng_scaling_factor",
    "i_regime_window",
    "i_regime_entry_min_score",
    "i_mvrv_suppress_bear",
    "i_div_window",
}
_PARAM_COLS = set(WEIGHT_COLS) | _NON_WEIGHT_PARAM_COLS


def write_to_sweep_db(verified_results, asset, timeframe, run_id, iteration_number=None, search_strategy=None):
    """
    Writes all CPU-verified results with at least 1 trade and Sortino > -5 to the sweep database.
    Filters on trades + Sortino rather than P&L/DD so high-drawdown regime runs (e.g. sideways
    training verified on full IS) are still captured. P&L/DD is stored as-is and can be 0 when
    drawdown exceeds the 40% cap — that is an analysis detail, not a write gate.
    Called after each iteration so no data is thrown away.
    """
    import datetime as _dt

    positive = [r for r in verified_results
                if r.get("Total Trades", 0) > 0 and r.get("Calmar Ratio", -10.0) > 0]
    if not positive:
        return

    # Compute percentile rank within this iteration's positive results.
    # When optimizing by Composite, rank by composite so mine_sweep_db.py top-quartile
    # filtering reflects the optimization objective. Fall back to Calmar otherwise.
    rank_key = "Composite" if OPTIMIZATION_METRIC == "Composite" else "Calmar Ratio"
    scores = sorted(r.get(rank_key, 0.0) for r in positive)
    n = max(len(scores) - 1, 1)
    for r in positive:
        s = r.get(rank_key, 0.0)
        idx = scores.index(s)
        r["_pnl_dd_percentile"] = 100.0 * idx / n

    now = _dt.datetime.now().isoformat(timespec="seconds")
    rows_to_insert = []
    for r in positive:
        row = {
            "asset": asset,
            "timeframe": timeframe,
            "schema_ver": DB_SCHEMA_VER,
            "recorded_at": now,
            "run_id": run_id,
            "train_start": TRAIN_START,
            "train_end": TRAIN_END,
            "score_start": SCORE_START,
            "pnl_dd_percentile": r.get("_pnl_dd_percentile"),
            "iteration_number": iteration_number,
            "search_strategy": search_strategy,
            "gpu_score": r.get("GPU_Score"),
            "subperiod_consistent": (1 if r.get("subperiod_consistent") else 0)
                                     if "subperiod_consistent" in r else None,
        }
        for src, col in _METRIC_TO_COL.items():
            val = r.get(src)
            row[col] = float(val) if val is not None else None
        for col in _PARAM_COLS:
            val = r.get(col)
            row[col] = float(val) if val is not None else None
        rows_to_insert.append(row)

    cols = list(rows_to_insert[0].keys())
    placeholders = ", ".join("?" * len(cols))
    col_names = ", ".join(cols)
    sql = f"INSERT INTO sweep_results ({col_names}) VALUES ({placeholders})"
    values = [tuple(row[c] for c in cols) for row in rows_to_insert]

    try:
        conn = sqlite3.connect(SWEEP_DB_FILE)
        conn.executemany(sql, values)
        conn.commit()
        conn.close()
        print(f"   >> [DB] Saved {len(rows_to_insert)} verified results to sweep database.")
    except Exception as e:
        print(f"   >> [DB] Warning: could not write to sweep database: {e}")


def _verify_one(args):
    """
    Worker function for parallel CPU verification.
    Must be at module level so pickle can serialise it for ProcessPoolExecutor.
    On Linux (fork), df_data_bytes is inherited from the parent — no actual copy.
    Runs two extra calculate_metrics calls (subperiod halves) to check IS consistency.
    """
    params_dict, df_data_bytes, score_start, optimization_metric = args
    import io as _io
    import pandas as _pd
    import strategies.strategy_activation_scores as _strat
    from config import SUBPERIOD_SPLIT as _SPLIT
    try:
        df_data = _pd.read_pickle(_io.BytesIO(df_data_bytes))
        df_res = _strat.generate_signals(df_data, **params_dict)
        metrics = _strat.calculate_metrics(df_res, score_start=score_start)

        # Subperiod consistency: require Calmar > 0 in both IS halves.
        # P1: score_start → 2020-12-31   P2: 2021-01-01 → TRAIN_END
        split_ts = _pd.to_datetime(_SPLIT)
        m1 = _strat.calculate_metrics(
            df_res[df_res['time'] <= split_ts].copy(), score_start=score_start)
        m2 = _strat.calculate_metrics(
            df_res[df_res['time'] >  split_ts].copy(), score_start=_SPLIT)
        metrics['subperiod_consistent'] = (
            m1.get('Calmar Ratio', -10.0) >= 0 and m1.get('Sortino Ratio', -10.0) >= 0 and
            m2.get('Calmar Ratio', -10.0) >= 0 and m2.get('Sortino Ratio', -10.0) >= 0
        )

        result = params_dict.copy()
        result.update(metrics)
        result['verified'] = True
        return result
    except Exception:
        return None


def verify_top_results(sweep_file, data_file, top_n=5000, run_id=None, iteration_number=None, search_strategy=None):
    """
    Reads the sweep results, picks the top N by the optimization metric, and verifies them on CPU.
    Returns the best VERIFIED result row.
    """
    print(f"   >> [VERIFY] Verifying Top {top_n} GPU results on CPU (sorted by {OPTIMIZATION_METRIC})...")
    
    if not os.path.exists(sweep_file):
        print(f"   >> [VERIFY] Sweep file not found: {sweep_file}")
        if os.path.exists(RESULTS_DIR):
            files = os.listdir(RESULTS_DIR)
            print(f"   >> [VERIFY] Files found in {RESULTS_DIR}: {files}")
        return None

    try:
        df_sweep = pd.read_csv(sweep_file)
        if df_sweep.empty: return None
            
        # Sweep CSV has GPU-computed columns (Calmar Ratio, etc.) but not CPU-only
        # metrics like P&L/DD Ratio. Sort by best available GPU column for pre-selection.
        gpu_sort_col = OPTIMIZATION_METRIC if OPTIMIZATION_METRIC in df_sweep.columns else "Calmar Ratio"
        df_sweep = df_sweep.sort_values(by=gpu_sort_col, ascending=False).head(top_n)
        
        df_data = pd.read_csv(data_file)
        df_data.columns = df_data.columns.str.lower()
        if 'time' in df_data.columns:
            df_data['time'] = pd.to_datetime(df_data['time'], utc=True).dt.tz_localize(None)
            mask = (df_data['time'] >= TRAIN_START) & (df_data['time'] <= TRAIN_END)
            df_data = df_data.loc[mask].copy()

        # Compute IS years from the scoring window for composite metric floor.
        is_years = 8.0  # safe default
        try:
            df_score_window = df_data[df_data['time'] >= pd.to_datetime(SCORE_START)]
            if len(df_score_window) > 1:
                t0 = df_score_window['time'].iloc[0]
                t1 = df_score_window['time'].iloc[-1]
                is_years = max(0.5, (t1 - t0).days / 365.25)
        except Exception:
            pass

        # Serialise df_data once; on Linux fork the bytes are inherited (effectively free).
        _buf = io.BytesIO()
        df_data.to_pickle(_buf)
        df_data_bytes = _buf.getvalue()

        candidate_params = [row.to_dict() for _, row in df_sweep.iterrows()]
        worker_args = [(p, df_data_bytes, SCORE_START, OPTIMIZATION_METRIC) for p in candidate_params]

        verified_results = []
        cpu_count = os.cpu_count() or 4
        # Cap at 6 workers to avoid OOM when verifying 5000 candidates after a large GPU run.
        # Each worker forks the parent + imports Numba + holds a copy of df_data in memory;
        # 16 workers was causing SIGKILL (broken pipe) after 130M-sample GPU searches.
        max_workers = max(1, min(6, cpu_count - 4))
        print(f"   >> [VERIFY] Launching {max_workers} parallel workers for {len(candidate_params)} candidates...")

        with ProcessPoolExecutor(max_workers=max_workers) as executor:
            futures = {executor.submit(_verify_one, a): a[0] for a in worker_args}
            for fut in as_completed(futures):
                result = fut.result()
                if result is not None:
                    original_params = futures[fut]
                    result['GPU_Score'] = original_params.get("Calmar Ratio", 0)
                    # Compute composite score for every verified result using IS window length.
                    # GPU kernel optimises raw Calmar; composite is applied here at CPU step.
                    calmar   = result.get("Calmar Ratio", 0.0)
                    sortino  = result.get("Sortino Ratio", 0.0)
                    trades   = int(result.get("Total Trades", 0))
                    pnl      = result.get("Total P&L %", 0.0)
                    composite = _composite_score(calmar, sortino, trades, is_years, pnl)
                    # Zero out composite if strategy failed in either IS subperiod
                    if not result.get('subperiod_consistent', True):
                        composite = 0.0
                    result['Composite'] = composite
                    result['CPU_Score'] = result.get(OPTIMIZATION_METRIC, 0)
                    verified_results.append(result)

        if not verified_results: return None, None

        # --- WFO rescore: evaluate top N IS candidates across temporal OOS folds ---
        # Sort by IS Composite, take top WFO_TOP_N, run 4-fold WFO in parallel.
        # WFO score = mean(OOS Calmar per fold) across folds with >= WFO_MIN_OOS_TRADES.
        # Only positive-Composite IS candidates are eligible for WFO.
        eligible = [r for r in verified_results if r.get('Composite', 0) > 0]
        wfo_winner = None
        if eligible:
            # Pre-filter by IS Calmar (not Composite) — Composite's trade-count bonus
            # inflates high-frequency solutions that aren't regime-selective.
            # WFO measures per-fold Calmar, so IS Calmar is a better proxy.
            top_for_wfo = sorted(eligible, key=lambda x: x.get('Calmar Ratio', 0), reverse=True)[:WFO_TOP_N]
            print(f"   >> [WFO] Rescoring top {len(top_for_wfo)}/{len(eligible)} eligible candidates "
                  f"(sorted by IS Calmar) across {len(WFO_FOLDS)} folds...")
            _wfo_t0 = time.time()
            wfo_args = [
                (p, df_data_bytes, WFO_FOLDS, WFO_MIN_OOS_TRADES, WFO_MIN_VALID_FOLDS)
                for p in top_for_wfo
            ]
            with ProcessPoolExecutor(max_workers=max_workers) as executor:
                wfo_futures = {executor.submit(_wfo_score_one, a): i for i, a in enumerate(wfo_args)}
                for fut in as_completed(wfo_futures):
                    idx = wfo_futures[fut]
                    wfo_score, fold_calmars, wfo_min, wfo_neg = fut.result()
                    top_for_wfo[idx]['WFO_Score'] = wfo_score
                    top_for_wfo[idx]['WFO_Fold_Calmars'] = fold_calmars
                    top_for_wfo[idx]['WFO_Min_Fold'] = wfo_min
                    top_for_wfo[idx]['WFO_Neg_Folds'] = wfo_neg

            _wfo_elapsed = time.time() - _wfo_t0
            print(f"   >> [WFO] Completed in {_wfo_elapsed:.1f}s ({_wfo_elapsed/len(top_for_wfo)*1000:.0f}ms/candidate)")

            # Propagate WFO fields back to the full verified_results list.
            wfo_lookup = {id(r): r for r in top_for_wfo}
            for r in verified_results:
                if id(r) in wfo_lookup:
                    r['WFO_Score'] = wfo_lookup[id(r)].get('WFO_Score', 0.0)
                    r['WFO_Min_Fold'] = wfo_lookup[id(r)].get('WFO_Min_Fold', -99.0)
                    r['WFO_Neg_Folds'] = wfo_lookup[id(r)].get('WFO_Neg_Folds', 0)

            # WFO winner = best worst-fold Calmar (most regime-robust).
            # Ties broken by mean WFO score, then IS Composite.
            # This ensures the winner must perform in the 2022 bear and 2023 sideways
            # folds — the historical analogues of the current OOS correction regime.
            wfo_winner = max(top_for_wfo, key=lambda x: (x.get('WFO_Min_Fold', -99.0), x.get('WFO_Score', 0.0), x.get('Composite', 0.0)))
            n_positive_wfo = sum(1 for r in top_for_wfo if r.get('WFO_Score', 0) > 0)
            fold_labels = ["2021", "2022", "2023", "2024H1", "PoliRun"]
            best_folds_str = "  |  ".join(
                f"{fold_labels[i]}={c:.3f}" for i, c in enumerate(wfo_winner.get('WFO_Fold_Calmars', []))
            )
            print(f"   >> [WFO] {n_positive_wfo}/{len(top_for_wfo)} candidates pass WFO  |  "
                  f"WFO Winner: IS={wfo_winner.get('Composite', 0):.4f}  WFO_Mean={wfo_winner.get('WFO_Score', 0):.4f}  "
                  f"WFO_Min={wfo_winner.get('WFO_Min_Fold', -99):.4f}  NegFolds={wfo_winner.get('WFO_Neg_Folds', '?')}")
            print(f"   >> [WFO] Fold Calmars: {best_folds_str}")

        write_to_sweep_db(verified_results, ASSET, TIMEFRAME, run_id, iteration_number=iteration_number, search_strategy=search_strategy)

        df_verified = pd.DataFrame(verified_results)
        best_verified = df_verified.sort_values(by=OPTIMIZATION_METRIC, ascending=False).iloc[0]

        print(f"   >> [VERIFY] IS Winner: GPU={best_verified['GPU_Score']:.4f} -> CPU={best_verified['CPU_Score']:.4f}")
        return best_verified, wfo_winner

    except Exception as e:
        print(f"   >> [VERIFY] Error: {e}")
        return None, None

def _wfo_score_one(args):
    """
    Worker: evaluate one candidate across all WFO OOS folds.

    For each fold (is_end, oos_start, oos_end):
      - Generate signals on all data up to oos_end (IS warms up crossunder state).
      - Score only the OOS slice [oos_start, oos_end].
      - Record OOS Calmar if OOS trade count >= min_oos_trades.

    Returns (wfo_score, fold_calmars) where wfo_score = mean(fold_calmars) when
    at least min_valid_folds qualify, otherwise (0.0, fold_calmars).

    Must be at module level so pickle can serialise it for ProcessPoolExecutor.
    """
    params_dict, df_full_bytes, folds, min_oos_trades, min_valid_folds = args
    import io as _io
    import pandas as _pd
    import numpy as _np
    import strategies.strategy_activation_scores as _strat

    try:
        df_full = _pd.read_pickle(_io.BytesIO(df_full_bytes))
        fold_calmars = []

        for (is_end, oos_start, oos_end) in folds:
            # Include IS data for crossunder warmup; score only OOS period.
            df_window = df_full[df_full['time'] <= _pd.to_datetime(oos_end)].copy()
            df_signals = _strat.generate_signals(df_window, **params_dict)
            # Slice to OOS period (positions carry over from IS naturally).
            df_oos = df_signals[df_signals['time'] <= _pd.to_datetime(oos_end)].copy()
            metrics = _strat.calculate_metrics(df_oos, score_start=oos_start)
            oos_trades = int(metrics.get('Total Trades', 0))
            if oos_trades >= min_oos_trades:
                fold_calmars.append(metrics.get('Calmar Ratio', -99.0))

        if len(fold_calmars) < min_valid_folds:
            return 0.0, fold_calmars, -99.0, len(fold_calmars)
        wfo_min = float(min(fold_calmars))
        wfo_neg = sum(1 for c in fold_calmars if c < 0)
        return float(_np.mean(fold_calmars)), fold_calmars, wfo_min, wfo_neg

    except Exception:
        return 0.0, [], -99.0, 0


def sanitize_winner_file(strategy_name, data_file):
    """
    Checks if the winner file exists and if its recorded metrics match a CPU backtest.
    If there's a discrepancy, it updates the file with the correct CPU metrics.
    """
    winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
    if not os.path.exists(winner_file):
        return

    print(f"   >> [SANITY CHECK] Verifying existing winner file...")
    try:
        df = pd.read_csv(winner_file)
        if df.empty: return
        
        row = df.iloc[0]
        recorded_score = row.get(OPTIMIZATION_METRIC, 0)
        
        df_data = pd.read_csv(data_file)
        df_data.columns = df_data.columns.str.lower()
        if 'time' in df_data.columns:
            df_data['time'] = pd.to_datetime(df_data['time'], utc=True).dt.tz_localize(None)
            mask = (df_data['time'] >= TRAIN_START) & (df_data['time'] <= TRAIN_END)
            df_data = df_data.loc[mask].copy()

        params = row.to_dict()
        df_res = strategy_module.generate_signals(df_data.copy(), **params)
        metrics = strategy_module.calculate_metrics(df_res, score_start=SCORE_START)
        # For Composite metric, compute it from Calmar + Trades + is_years.
        if OPTIMIZATION_METRIC == "Composite":
            try:
                df_score_w = df_data[df_data['time'] >= pd.to_datetime(SCORE_START)]
                is_yrs = max(0.5, (df_score_w['time'].iloc[-1] - df_score_w['time'].iloc[0]).days / 365.25)
            except Exception:
                is_yrs = 8.0
            metrics['Composite'] = _composite_score(
                metrics.get('Calmar Ratio', 0.0), metrics.get('Sortino Ratio', 0.0),
                int(metrics.get('Total Trades', 0)), is_yrs, metrics.get('Total P&L %', 0.0))
        cpu_score = metrics.get(OPTIMIZATION_METRIC, 0)

        if abs(recorded_score - cpu_score) > 1e-6:
            print(f"   >> [FIX] Winner file score changed (Recorded: {recorded_score:.4f} → CPU: {cpu_score:.4f}). Updating...")
            for k, v in metrics.items():
                df.at[0, k] = v
            df.to_csv(winner_file, index=False)
            row = df.iloc[0]  # refresh row with updated metrics
        else:
            print(f"   >> [SANITY CHECK] Winner file is valid ({OPTIMIZATION_METRIC}: {cpu_score:.4f}).")

        # Always regenerate cheatsheet/pine snippet from the current CSV row so they
        # stay in sync even if a previous run wrote the CSV without updating them.
        write_winner_readable(row, strategy_name, WINNERS_DIR, asset=ASSET, timeframe=TIMEFRAME)

        return cpu_score

    except Exception as e:
        print(f"   >> [SANITY CHECK] Failed to verify/fix winner file: {e}")

    return None

def write_performance_report(strategy_name, data_file):
    """
    Runs the current winner on the FULL data (training + OOS) and writes a
    side-by-side in-sample vs out-of-sample performance report to
    results/performance_report_{ASSET}_{TIMEFRAME}.md.

    In-sample  : SCORE_START → TRAIN_END
    Out-of-sample : OOS_START → end of data
    """
    import datetime as _dt
    winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
    if not os.path.exists(winner_file):
        return

    try:
        row = pd.read_csv(winner_file).iloc[0]
        params = row.to_dict()

        df_full = pd.read_csv(data_file)
        df_full.columns = df_full.columns.str.lower()
        if 'time' in df_full.columns:
            df_full['time'] = pd.to_datetime(df_full['time'], utc=True).dt.tz_localize(None)

        df_signals = strategy_module.generate_signals(df_full.copy(), **params)

        # In-sample: signals on full data, score restricted to SCORE_START–TRAIN_END
        train_end_ts = pd.to_datetime(TRAIN_END)
        df_is = df_signals[df_signals['time'] <= train_end_ts].copy()
        is_metrics = strategy_module.calculate_metrics(df_is, score_start=SCORE_START)

        # Out-of-sample: signals on full data, score from OOS_START onwards
        oos_metrics = strategy_module.calculate_metrics(df_signals, score_start=OOS_START)

        oos_last = df_full['time'].iloc[-1].strftime('%Y-%m-%d') if 'time' in df_full.columns else "?"

        def fmt(m):
            pnl   = m.get('Total P&L %', 0)
            dd    = m.get('Max Drawdown %', 0)
            sort  = m.get('Sortino Ratio', 0)
            sharp = m.get('Sharpe Ratio', 0)
            calm  = m.get('Calmar Ratio', 0)
            tr    = int(m.get('Total Trades', 0))
            mkt   = m.get('% In Market', 0)
            pdd   = m.get('P&L/DD Ratio', 0)
            return pnl, dd, sort, sharp, calm, tr, mkt, pdd

        ip, idd, iso, ish, ica, itr, iim, ipdd = fmt(is_metrics)
        op, odd, oso, osh, oca, otr, oim, opdd = fmt(oos_metrics)

        # OOS quality verdict
        if itr == 0:
            verdict = "⚠ No in-sample trades — cannot evaluate"
        elif otr == 0:
            verdict = "⚠ No OOS trades yet"
        elif otr < 10:
            direction = f"+{op:.1f}%" if op >= 0 else f"{op:.1f}%"
            verdict = f"⚠ FEW OOS TRADES ({otr}) — P&L={direction} (need ≥10 for Sortino to be reliable)"
        elif oso <= -5.0:
            verdict = f"⚠ FEW OOS TRADES ({otr}) — Sortino not reliable"
        elif oso < 0:
            verdict = f"❌ POOR — negative OOS Sortino ({oso:.4f})"
        else:
            ratio = oso / iso if iso > 0 else 0
            if ratio >= 0.8:
                verdict = f"✅ EXCELLENT — OOS Sortino is {ratio:.0%} of in-sample"
            elif ratio >= 0.5:
                verdict = f"✅ ACCEPTABLE — OOS Sortino is {ratio:.0%} of in-sample"
            elif ratio >= 0.25:
                verdict = f"⚠ WEAK — OOS Sortino is {ratio:.0%} of in-sample"
            else:
                verdict = f"❌ POOR — OOS Sortino is {ratio:.0%} of in-sample"

        lines = [
            f"# Performance Report — {ASSET} {TIMEFRAME}",
            f"Generated: {_dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "",
            f"| Window | Period | P&L | Max DD | Sortino | Sharpe | Calmar | Trades | In Market | P&L/DD |",
            f"|--------|--------|-----|--------|---------|--------|--------|--------|-----------|--------|",
            f"| **In-Sample** | {SCORE_START} → {TRAIN_END} | {ip:,.1f}% | {idd:.2f}% | {iso:.4f} | {ish:.4f} | {ica:.4f} | {itr} | {iim:.1f}% | {ipdd:.1f} |",
            f"| **Out-of-Sample** | {OOS_START} → {oos_last} | {op:,.1f}% | {odd:.2f}% | {oso:.4f} | {osh:.4f} | {oca:.4f} | {otr} | {oim:.1f}% | {opdd:.1f} |",
            "",
            f"**Verdict:** {verdict}",
            "",
            "---",
            f"*In-sample metrics use the scoring window only (crossunder signals generated on full history).*",
            f"*Generated by auto_optimize_loop.py*",
        ]

        out_path = os.path.join(REPORTS_DIR, f"performance_report_{ASSET}_{TIMEFRAME}.md")
        with open(out_path, "w") as f:
            f.write("\n".join(lines) + "\n")
        print(f"   >> [OOS] Performance report → {out_path}")
        print(f"   >> [OOS] In-sample  Sortino={iso:.4f}  P&L={ip:,.1f}%  Trades={itr}")
        print(f"   >> [OOS] Out-sample Sortino={oso:.4f}  P&L={op:,.1f}%  Trades={otr}  ({verdict.split(' — ')[0]})")

    except Exception as e:
        print(f"   >> [OOS] Failed to generate performance report: {e}")


_NICE_STEPS = [
    0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.25, 0.5,
    1, 2, 5, 10, 20, 25, 50, 100, 200, 500,
]

def _nice_step(raw):
    """Rounds a raw step to the nearest clean value from _NICE_STEPS."""
    return min(_NICE_STEPS, key=lambda s: abs(math.log(max(s, 1e-12)) - math.log(max(raw, 1e-12))))


def analyze_params_coverage(params_file, total_samples):
    """
    Reports parameter coverage, then auto-adjusts any params whose step is
    so fine (>50 values) or so coarse (<3 values) that random search will be
    ineffective.  Target: ~20 values per param.  Saves the file if changed.
    """
    TARGET_VALUES = 20
    MAX_VALUES    = 50   # above this → coarsen step
    MIN_VALUES    = 3    # below this (with range > 0) → finer step

    try:
        with open(params_file, 'r') as f:
            params = json.load(f)
    except Exception:
        return

    total_space = calculate_search_space(params_file)
    coverage_pct = (total_samples / total_space * 100) if total_space > 0 else 100.0

    print(f"\n{'='*60}")
    print(f"  PARAMETER COVERAGE ANALYSIS")
    print(f"{'='*60}")
    print(f"  Search Space : {format_quantity(total_space)}")
    print(f"  Total Samples: {format_quantity(total_samples)}")
    print(f"  Coverage     : {coverage_pct:.6f}%")

    adjustments = []
    for key, config in params.items():
        if 'start' not in config or 'stop' not in config or 'step' not in config:
            continue
        start = float(config['start'])
        stop  = float(config['stop'])
        step  = float(config['step'])
        rng   = stop - start
        if step <= 0 or rng <= 0:
            continue

        num_values = max(1, round(rng / step))

        if num_values > MAX_VALUES or num_values < MIN_VALUES:
            raw_new   = rng / TARGET_VALUES
            new_step  = _nice_step(raw_new)
            new_count = max(1, round(rng / new_step))
            config['step'] = new_step
            adjustments.append((key, num_values, step, new_count, new_step))

    if adjustments:
        print(f"\n  Auto-adjusted {len(adjustments)} param(s) to ~{TARGET_VALUES} values each:")
        for key, old_n, old_s, new_n, new_s in adjustments:
            tag = "FINE" if old_n > MAX_VALUES else "COARSE"
            print(f"    [{tag}] {key:<42} {old_n:>4}→{new_n:<4} values  step {old_s}→{new_s}")
        with open(params_file, 'w') as f:
            json.dump(params, f, indent=4)
        print(f"  Saved adjusted params to {params_file}")
    else:
        print(f"\n  All params already in [{MIN_VALUES}–{MAX_VALUES}] values range — no adjustment needed.")

    # With 25+ free parameters, coverage % is always near 0 by definition —
    # what matters is that each param has a healthy number of discrete values
    # so random samples land at meaningful positions on every axis.
    with open(params_file, 'r') as f:
        adj_params = json.load(f)
    per_param_counts = []
    for key, cfg in adj_params.items():
        if 'start' in cfg and 'stop' in cfg and 'step' in cfg:
            rng  = float(cfg['stop']) - float(cfg['start'])
            step = float(cfg['step'])
            if step > 0 and rng > 0:
                per_param_counts.append(max(1, round(rng / step)))
    if per_param_counts:
        print(f"\n  Per-param value counts (min/median/max): "
              f"{min(per_param_counts)} / {sorted(per_param_counts)[len(per_param_counts)//2]} / {max(per_param_counts)}")
        cramped = [c for c in per_param_counts if c < MIN_VALUES]
        bloated = [c for c in per_param_counts if c > MAX_VALUES]
        if cramped or bloated:
            print(f"  ⚠  {len(bloated)} over-fine, {len(cramped)} over-coarse after adjustment (check manually).")
        else:
            print(f"  ✓  All range-based params have {MIN_VALUES}–{MAX_VALUES} values. Random search is well-calibrated.")

    print(f"{'='*60}\n")


# Mapping: i_w_* param key → list of candidate CSV column names to look for.
# None = computed on-the-fly from OHLC — always safe, never warn.
_WEIGHT_TO_CSV_COLS = {
    'i_w_stoch':               ['stoch_norm'],
    'i_w_macd_pred':           ['macd_pred_norm'],
    'i_w_osc':                 ['osc_norm'],
    'i_w_macd_bullish':        ['macd_bullish_norm'],
    'i_w_m3_momentum':         ['m3_momentum_norm'],
    'i_w_m2_tiny':             ['m2_tiny_norm'],
    'i_w_rsid_osc':            ['rsid_norm'],
    'i_w_stoch_div_osc':       ['stoch_div_norm'],
    'i_w_vwap_div_osc':        ['vwap_div_norm'],
    'i_w_stoch_peaking':       ['stoch_peak_norm'],
    'i_w_stoch_bottoming':     ['stoch_bot_norm'],
    'i_w_m3_div_osc':          ['m3_div_norm'],
    'i_w_bearish_engulfing':   ['bearish_engulfing_score'],
    'i_w_m2_div_osc_noOffset': ['m2_nooff_norm'],
    'i_w_m2_div_osc':          ['m2_div_norm'],
    'i_w_btc_spx_corr':        ['btc_spx_corr_30'],
    'i_w_dxy':                 ['dxy_roc_norm'],
    'i_w_vix':                 ['vix_pctrank_inv'],
    'i_w_btc_dom':             ['btc_dom_roc_sign'],
    'i_w_us10y':               ['us10y_roc_inv_sign'],
    'i_w_spy':                 ['spy_above_200ema'],
    'i_w_gold':                ['gold_roc_pctrank'],
    'i_w_mvrv':                ['mvrv_zscore_value'],
    'i_w_mvrv_cont':           ['mvrv_zscore_cont', 'zscore'],
    'i_w_nupl':                ['nupl_norm'],
    'i_w_fed_net_liq':         ['fed_net_liq_sign'],
    'i_w_gc_position':         ['gc_position'],
    # Candlestick patterns: calculated from OHLC, always present — never warn
    'i_w_bullish_hammer':      None,
    'i_w_bullish_engulfing':   None,
    'i_w_shooting_star':       None,
}


def _check_pine_validation_sentinel():
    """
    Warn if the Pine strategy file has been modified since it was last confirmed
    to compile in TradingView. Gives a 10-second countdown so the user can Ctrl+C
    to fix the Pine file first, preventing a wasted optimization run.
    """
    import time as _time
    pine_file = "strategies/strategy_activation_scores.pine"
    sentinel_file = ".pine_tv_validated"

    if not os.path.exists(pine_file):
        return  # nothing to check

    pine_mtime = os.path.getmtime(pine_file)

    if not os.path.exists(sentinel_file):
        print("\n" + "=" * 70)
        print("⚠️  PINE VALIDATION WARNING")
        print("=" * 70)
        print(f"  The Pine strategy file has NEVER been confirmed to compile in")
        print(f"  TradingView. If it has compile errors, this optimization run")
        print(f"  will produce weights that can't be used.")
        print(f"")
        print(f"  To dismiss this warning permanently, paste the Pine file into")
        print(f"  TradingView, confirm it compiles, then run:")
        print(f"      python3 tools/check_pine.py --mark-valid")
        print("=" * 70)
        _countdown(10)
        return

    sentinel_mtime = os.path.getmtime(sentinel_file)
    if pine_mtime > sentinel_mtime:
        from datetime import datetime as _dt
        pine_dt = _dt.fromtimestamp(pine_mtime).strftime("%Y-%m-%d %H:%M:%S")
        sent_dt = _dt.fromtimestamp(sentinel_mtime).strftime("%Y-%m-%d %H:%M:%S")
        print("\n" + "=" * 70)
        print("⚠️  PINE VALIDATION WARNING")
        print("=" * 70)
        print(f"  The Pine strategy file was modified AFTER the last TV validation.")
        print(f"  Pine modified:   {pine_dt}")
        print(f"  Last validated:  {sent_dt}")
        print(f"")
        print(f"  If the edits introduced compile errors, this optimization run")
        print(f"  will produce weights that can't be used.")
        print(f"")
        print(f"  After verifying in TradingView, run:")
        print(f"      python3 tools/check_pine.py --mark-valid")
        print("=" * 70)
        _countdown(10)


def _countdown(seconds):
    """Print a countdown, allowing Ctrl+C to abort."""
    import time as _time
    import sys as _sys
    try:
        for i in range(seconds, 0, -1):
            _sys.stdout.write(f"\r  Continuing in {i}s... (Ctrl+C to cancel)  ")
            _sys.stdout.flush()
            _time.sleep(1)
        _sys.stdout.write("\r  Continuing...                                \n\n")
        _sys.stdout.flush()
    except KeyboardInterrupt:
        print("\n\n  Run cancelled by user.")
        _sys.exit(0)


def validate_params_sync(params_file):
    """
    Pre-run sync check: ensure every i_w_* param in the params JSON is also in
    config.WEIGHT_COLS. If any are missing, they will be silently skipped by the
    sweep dashboard and SHAP — wasting a long optimization run.

    This is a hard stop (non-interactive) because a missing entry in WEIGHT_COLS
    means results will be incomplete regardless of how long the run takes.
    Fix: add the missing param(s) to WEIGHT_COLS in config.py, then restart.
    """
    try:
        with open(params_file) as f:
            params = json.load(f)
    except Exception as e:
        print(f"   >> [WARN] Could not read params file for sync check: {e}")
        return

    weight_cols_set = set(WEIGHT_COLS)
    missing = [k for k in params if k.startswith('i_w_') and k not in weight_cols_set]

    if not missing:
        return

    print()
    print("=" * 70)
    print("   >> [ERROR] PARAMS/CONFIG OUT OF SYNC — aborting before wasting run time.")
    print()
    print("   The following i_w_* params are in the params JSON but NOT in")
    print("   config.WEIGHT_COLS. Results for these signals will be silently")
    print("   missing from the sweep dashboard and SHAP importance reports.")
    print()
    for k in sorted(missing):
        print(f"      {k}")
    print()
    print("   Fix: add the param(s) above to WEIGHT_COLS in config.py, then restart.")
    print("   The DB migration runs automatically — no manual ALTER TABLE needed.")
    print("=" * 70)
    print()
    sys.exit(1)


def _validate_all_params_sync():
    """
    Hard stop if ANY of the 20 active params files is missing a key from WEIGHT_COLS
    or i_div_window. Catches the case where config.py was updated but sync_params.py
    wasn't run, which would cause SHAP/dashboard to report MISSING FROM SOME PARAMS
    and silently treat those weights as zero in affected runs.

    Fix: python3 tools/sync_params.py --apply
    """
    import glob as _glob
    pattern = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
                           "strategies", "params", "params_strategy_activation_scores_*_*.json")
    files = sorted(_glob.glob(pattern))
    if not files:
        return  # nothing to check
    expected = set(WEIGHT_COLS) | {"i_div_window"}
    bad_files = []
    for path in files:
        try:
            with open(path) as f:
                p = json.load(f)
        except Exception:
            continue
        missing = [k for k in expected if k not in p]
        if missing:
            bad_files.append((os.path.basename(path), missing))
    if not bad_files:
        return
    print()
    print("=" * 70)
    print("   >> [ERROR] PARAMS FILES OUT OF SYNC — aborting before wasting run time.")
    print()
    print("   The following active params files are missing keys that are in")
    print("   config.WEIGHT_COLS (or i_div_window). SHAP will show MISSING FROM")
    print("   SOME PARAMS warnings and those weights will be silently zero.")
    print()
    for fname, missing in bad_files:
        print(f"      {fname}:")
        for k in missing:
            print(f"        {k}")
    print()
    print("   Fix: python3 tools/sync_params.py --apply")
    print("=" * 70)
    print()
    sys.exit(1)


def validate_data_columns(params_file, data_file):
    """
    Check that every i_w_* param in the params file has a corresponding column
    in the data CSV. If any are missing, the optimizer silently treats them as
    all-zero (wasted search budget). Warn the user and prompt to continue or quit.

    Normal action on warning: quit, re-export TV chart data with the missing
    columns included in data/, then restart the optimization run.
    """
    try:
        with open(params_file) as f:
            params = json.load(f)
        df_head = pd.read_csv(data_file, nrows=0)
        csv_cols = set(c.lower().strip() for c in df_head.columns)
    except Exception as e:
        print(f"   >> [WARN] Could not validate data columns: {e}")
        return

    missing = []
    for key in params:
        if not key.startswith('i_w_'):
            continue
        candidates = _WEIGHT_TO_CSV_COLS.get(key)
        if candidates is None:
            continue  # computed from OHLC — always fine
        # Skip params locked to a single value — they can't waste search budget
        pdef = params[key]
        if isinstance(pdef, dict):
            vals = pdef.get('values')
            if vals is not None and len(vals) == 1:
                continue  # locked; missing column is intentional
        if not any(c.lower() in csv_cols for c in candidates):
            missing.append((key, candidates))

    if not missing:
        return

    print()
    print("=" * 70)
    print("   >> [WARNING] Missing CSV columns for weight params in params file.")
    print()
    print("   The following params are in the search space but have NO matching")
    print("   column in the data CSV. The optimizer will treat them as all-zero,")
    print("   wasting search budget on params that have no effect.")
    print()
    for key, cols in missing:
        print(f"      {key}")
        print(f"        expected column(s): {', '.join(cols)}")
    print()
    print("   Normal action: quit now, re-export the TV chart data to include")
    print("   the missing indicator columns, save to data/, then restart.")
    print("=" * 70)
    print()
    if sys.stdin.isatty():
        answer = input("   Continue anyway? [y/N]: ").strip().lower()
        if answer != 'y':
            print("   Aborting. Update your TV data export and restart.")
            sys.exit(0)
        print("   Continuing with missing columns treated as zero.")
    else:
        print("   [NON-INTERACTIVE] Continuing automatically (batch mode). Missing columns treated as zero.")
    print()


def calculate_search_space(params_file):
    """Calculates the approximate size of the search space from the JSON file."""
    try:
        with open(params_file, 'r') as f:
            params = json.load(f)
        total = 1
        for key, config in params.items():
            if 'start' in config and 'stop' in config and 'step' in config:
                start, stop, step = config['start'], config['stop'], config['step']
                if step <= 0: continue
                count = max(1, int((stop - start) / step))
                total *= count
            elif 'values' in config:
                total *= len(config['values'])
        return total
    except Exception:
        return 0

def format_quantity(n):
    """Formats numbers with suffixes (million, billion, etc)."""
    if n >= 1_000_000_000_000_000_000_000_000:
        return f"{n/1_000_000_000_000_000_000_000_000:.2f} septillion"
    elif n >= 1_000_000_000_000_000_000_000:
        return f"{n/1_000_000_000_000_000_000_000:.2f} sextillion"
    elif n >= 1_000_000_000_000_000_000:
        return f"{n/1_000_000_000_000_000_000:.2f} quintillion"
    elif n >= 1_000_000_000_000_000:
        return f"{n/1_000_000_000_000_000:.2f} quadrillion"
    elif n >= 1_000_000_000_000:
        return f"{n/1_000_000_000_000:.2f} trillion"
    elif n >= 1_000_000_000:
        return f"{n/1_000_000_000:.2f} billion"
    elif n >= 1_000_000:
        return f"{n/1_000_000:.0f} million"
    else:
        return f"{n:,}"

def format_duration(seconds):
    """Formats seconds into a friendly string (e.g. '6m 20s')."""
    if seconds < 60:
        return f"~{seconds:.0f}s"
    m = int(seconds // 60)
    s = int(seconds % 60)
    if m < 60:
        return f"~{m}m {s}s"
    h = int(m // 60)
    m = m % 60
    return f"~{h}h {m}m"

def run_optimization(iteration_samples, baseline_score=0.0):
    """
    Runs the existing optimize_strategy.py script using Popen for better process control.
    Includes a timeout and forceful termination to prevent hangs.
    """
    total_space = calculate_search_space(PARAMS_FILE)
    effective_iterations = min(total_space, iteration_samples) if total_space > 0 else iteration_samples
    search_method = "Grid" if total_space > 0 and total_space <= effective_iterations else "Random"

    est_time_str = format_duration(effective_iterations / GPU_RATE) if search_method == "Random" else "N/A (Grid)"
    
    print(f"   >> Running {search_method} search (Space: {format_quantity(total_space)} | Sample: {format_quantity(effective_iterations)})...")
    print(f"   >> Estimated Time: {est_time_str} (GPU)")
    
    cmd = [
        "python3", "strategies/optimize_strategy.py",
        "--data", DATA_FILE,
        "--strategy_file", STRATEGY_FILE,
        "--params_file", PARAMS_FILE,
        "--asset", ASSET,
        "--timeframe", TIMEFRAME,
        "--random_search", str(iteration_samples),
        "--gpu",
        "--metric", OPTIMIZATION_METRIC,
        f"--baseline-score={baseline_score}",
        "--search", SEARCH_MODE,
        "--regime", REGIME,
        "--regime-bear-threshold", str(REGIME_BEAR_THRESHOLD),
        "--regime-bull-threshold", str(REGIME_BULL_THRESHOLD),
    ]
    
    print(f"   >> Executing: {' '.join(cmd)}")
    env = os.environ.copy()
    if os.path.exists("/usr/lib/wsl/lib"):
        current_ld = env.get("LD_LIBRARY_PATH", "")
        if "/usr/lib/wsl/lib" not in current_ld:
            env["LD_LIBRARY_PATH"] = f"{current_ld}:/usr/lib/wsl/lib" if current_ld else "/usr/lib/wsl/lib"
            print("   >> [WSL CONFIG] Injected /usr/lib/wsl/lib into LD_LIBRARY_PATH for GPU access.")

    process = subprocess.Popen(cmd, text=True, env=env, preexec_fn=os.setsid,
                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1)

    def _stream(proc):
        for line in proc.stdout:
            print(line, end='', flush=True)

    stream_thread = threading.Thread(target=_stream, args=(process,), daemon=True)
    stream_thread.start()

    timeout_seconds = 30 * 60

    try:
        return_code = process.wait(timeout=timeout_seconds)
        stream_thread.join()

        if return_code != 0:
            print(f"   >> [ERROR] Optimization script exited with code {return_code}.")
            sys.exit(1)

    except subprocess.TimeoutExpired:
        print(f"\n   >> [ERROR] Optimization process timed out after {timeout_seconds / 60} minutes.")
        print("   >> Forcefully terminating process group...")

        import signal
        os.killpg(os.getpgid(process.pid), signal.SIGTERM)
        time.sleep(2)
        os.killpg(os.getpgid(process.pid), signal.SIGKILL)
        stream_thread.join(timeout=5)

        print("   >> Process terminated. The loop will continue, but the underlying issue persists.")

    return ""

def get_run_best_result():
    """Reads the best result from the CURRENT run (optimization_run_best_*.csv)."""
    strategy_name = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")

    run_best_file = os.path.join(WINNERS_DIR, f"optimization_run_best_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")

    if os.path.exists(run_best_file):
        df = pd.read_csv(run_best_file)
        return df.iloc[0]
    else:
        print(f"   >> [INFO] 'Run Best' file not found (Run likely produced no trades/improvement). Falling back to Global Winner.")
        winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
        if os.path.exists(winner_file):
            return pd.read_csv(winner_file).iloc[0]
        return None

def main():
    parser = argparse.ArgumentParser(
        description="Auto-Optimize Strategy (iterative GPU random/Optuna search)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Arguments:
  --data PATH       Path to OHLCV data CSV. Asset and TF extracted from filename.
  --hours N         Total runtime budget. Divided evenly across ~15-min iterations.
  --search MODE     'random' (default) or 'optuna' (DB-history biased sampling + importance report).
                    Optuna biased sampling works from iteration 1 (requires prior DB rows).
                    Importance report (results/reports/optuna_importance_*.md) requires ~3h+ to appear
                    because it needs ≥30 CPU-verified results from a single iteration.
  --regime MODE     Filter IS bars by MVRV regime: all (default) / bull / bear / sideways.
  --params PATH     Override the params JSON file (default: derived from --data).
  --no-dashboard    Skip sweep dashboard regeneration after each iteration.
  --reset-db [ASSET TF]  Delete DB rows then exit. No args = wipe all.

Examples:
  # First run on a new combo — always start with random
  python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 2

  # After ≥1 random run — switch to Optuna for DB-biased sampling
  # Use ≥4h if you want the importance report (results/reports/) to appear
  python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4 --search optuna

  # Optuna with LLM-guided range narrowing (only ≥6h runs)
  python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-1D.csv --hours 8 --search optuna --llm

  # Bull-regime only (filter IS training bars to MVRV bull phase)
  python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4 --regime bull

  # Wipe one combo's DB rows and restart
  python3 tools/auto_optimize_loop.py --reset-db COINBASE_BTCUSD 4H
  python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4

  # Usually called via batch runners rather than directly:
  python3 run_btc.py --hours 10 --tfs 4H 6H 8H 12H 1D
  python3 run_btc.py --hours 10 --search optuna
  python3 run_marathon.py --hours 96 --hours-per-combo 1
"""
    )
    parser.add_argument("--data", type=str, default=None, help="Path to the OHLCV data CSV (e.g. results/COINBASE_BTCUSD-1D.csv). Timeframe is extracted from the filename.")
    parser.add_argument("--hours", type=float, help="Target runtime in hours (scales sample size automatically)")
    parser.add_argument("--reset-db", nargs="*", metavar="ASSET_TF",
                        help="Delete sweep DB rows then exit. "
                             "No args: wipe all. Two args (ASSET TIMEFRAME): wipe that combo. "
                             "E.g.: --reset-db COINBASE_BTCUSD 4H")
    parser.add_argument("--no-dashboard", dest="no_dashboard", action="store_true",
                        help="Skip regenerating the sweep dashboard (mine_sweep_db.py) after each iteration")
    parser.add_argument("--tighten", dest="tighten", action="store_true",
                        help="Auto-apply q5-q95 range tightening (tighten_params.py --apply) after each iteration. "
                             "Off by default — only enable after several runs have built up sufficient DB coverage.")
    parser.add_argument("--search", type=str, default="random", choices=["random", "optuna"],
                        help="Search strategy passed to optimize_strategy.py: 'random' (default) or "
                             "'optuna' (biased sampling from DB history + importance analysis). "
                             "Optuna mode requires prior DB results to be effective.")
    parser.add_argument("--params", type=str, default=None,
                        help="Override the params JSON file (default: derived from --data filename). "
                             "Used by run_phased.py to pass phase-specific param files.")
    parser.add_argument("--regime", type=str, default="all", choices=["all", "bull", "bear", "sideways"],
                        help="Pre-filter IS training bars by MVRV regime before optimisation. "
                             "'bull': zscore >= bull_threshold; 'bear': zscore < bear_threshold; "
                             "'sideways': bear_threshold <= zscore < bull_threshold; 'all': no filter (default).")
    parser.add_argument("--regime-bear-threshold", type=float, default=_MVRV_BEAR_DEFAULT,
                        help=f"zscore percentile below which bars are Bear (default: {_MVRV_BEAR_DEFAULT}). "
                             "Historically: 2022 bear median=6.6, 2018 bear median=9.9.")
    parser.add_argument("--regime-bull-threshold", type=float, default=_MVRV_BULL_DEFAULT,
                        help=f"zscore percentile at or above which bars are Bull (default: {_MVRV_BULL_DEFAULT}). "
                             "Aligns with onset of confirmed bull runs.")
    args = parser.parse_args()

    # Handle --reset-db immediately (before any optimization setup)
    if args.reset_db is not None:
        init_sweep_db()
        conn = sqlite3.connect(SWEEP_DB_FILE)
        if len(args.reset_db) == 0:
            n = conn.execute("SELECT COUNT(*) FROM sweep_results").fetchone()[0]
            conn.execute("DELETE FROM sweep_results")
            conn.commit()
            print(f"Wiped entire sweep database ({n} rows deleted).")
        elif len(args.reset_db) == 2:
            asset_arg, tf_arg = args.reset_db[0], args.reset_db[1].upper()
            n = conn.execute(
                "SELECT COUNT(*) FROM sweep_results WHERE asset=? AND timeframe=?",
                (asset_arg, tf_arg)).fetchone()[0]
            conn.execute("DELETE FROM sweep_results WHERE asset=? AND timeframe=?",
                         (asset_arg, tf_arg))
            conn.commit()
            print(f"Deleted {n} rows for {asset_arg} {tf_arg}.")
        else:
            print("Usage: --reset-db  OR  --reset-db ASSET TIMEFRAME")
        conn.close()
        sys.exit(0)

    global DATA_FILE, PARAMS_FILE, ASSET, TIMEFRAME, MAX_ITERATIONS, SEARCH_MODE, REGIME, REGIME_BEAR_THRESHOLD, REGIME_BULL_THRESHOLD
    SEARCH_MODE = args.search
    REGIME = args.regime
    REGIME_BEAR_THRESHOLD = args.regime_bear_threshold
    REGIME_BULL_THRESHOLD = args.regime_bull_threshold
    if args.data:
        DATA_FILE = args.data
    # Derive ASSET and TIMEFRAME from data filename.
    # Supports two naming formats:
    #   mlp format:  "COINBASE_BTCUSD, 240.csv"  -> ASSET=COINBASE_BTCUSD, TF=4H
    #   legacy format: "COINBASE_BTCUSD-1D.csv"  -> ASSET=COINBASE_BTCUSD, TF=1D
    _MIN_TO_TF = {"240": "4H", "360": "6H", "480": "8H", "720": "12H", "1D": "1D"}
    stem = os.path.splitext(os.path.basename(DATA_FILE))[0]
    if ', ' in stem:
        _asset_part, _period = stem.rsplit(', ', 1)
        ASSET, TIMEFRAME = _asset_part, _MIN_TO_TF.get(_period, _period.upper())
    else:
        parts = stem.rsplit('-', 1)
        if len(parts) == 2:
            ASSET, TIMEFRAME = parts[0], parts[1].upper()
        else:
            ASSET, TIMEFRAME = stem, "1D"
    PARAMS_FILE = f"strategies/params/params_strategy_activation_scores_{ASSET}_{TIMEFRAME}.json"
    if args.params:
        PARAMS_FILE = args.params
        print(f"   >> [CONFIG] Params override: {PARAMS_FILE}")
    if not os.path.exists(PARAMS_FILE):
        # Fall back to timeframe-generic template, then to 1D generic template
        tf_template  = f"strategies/params/params_strategy_activation_scores_{TIMEFRAME}.json"
        gen_template = "strategies/params/params_strategy_activation_scores_1D.json"
        template = tf_template if os.path.exists(tf_template) else (gen_template if os.path.exists(gen_template) else None)
        if template:
            import shutil
            shutil.copy(template, PARAMS_FILE)
            print(f"   >> [CONFIG] Created asset-specific params {PARAMS_FILE} from template {template}.")
        else:
            print(f"   >> [ERROR] Params file not found: {PARAMS_FILE} (and no template found)")
            sys.exit(1)
    print(f"   >> [CONFIG] Data: {DATA_FILE} | Asset: {ASSET} | Timeframe: {TIMEFRAME} | Params: {PARAMS_FILE}")

    sys.stdout = Logger(os.path.join(OUTPUT_DIR, f"optimization_log_{ASSET}_{TIMEFRAME}.txt"))

    global SEARCH_ITERATIONS
    if args.hours:
        total_seconds = args.hours * 3600
        # Target ~15 minutes per iteration; scale iteration count with total runtime
        TARGET_SECS_PER_ITER = 15 * 60
        MAX_ITERATIONS = max(3, round(total_seconds / TARGET_SECS_PER_ITER))
        seconds_per_iter = total_seconds / MAX_ITERATIONS
        scaled_iterations = int(seconds_per_iter * GPU_RATE)

        SEARCH_ITERATIONS = max(1_000_000, scaled_iterations)

        print(f"--- Time-Based Scaling Enabled ---")
        print(f"Target Runtime : {args.hours} hours ({total_seconds:.0f}s)")
        print(f"Iterations     : {MAX_ITERATIONS}  (~{seconds_per_iter/60:.0f} min each)")
        print(f"Samples/iter   : {SEARCH_ITERATIONS:,}  (Total: {SEARCH_ITERATIONS * MAX_ITERATIONS:,})")

    # Regime filtering is handled inside optimize_strategy.py via --regime / --regime-bear-threshold / --regime-bull-threshold.
    if REGIME != "all":
        print(f"   >> [REGIME] Training on '{REGIME}' bars only "
              f"(bear_thr={REGIME_BEAR_THRESHOLD}, bull_thr={REGIME_BULL_THRESHOLD}). "
              f"Filter applied inside optimizer subprocess.")

    _check_pine_validation_sentinel()
    analyze_params_coverage(PARAMS_FILE, SEARCH_ITERATIONS * MAX_ITERATIONS)
    validate_params_sync(PARAMS_FILE)      # hard stop if config.WEIGHT_COLS is out of sync
    _validate_all_params_sync()            # hard stop if ANY active params file is missing a key
    validate_data_columns(PARAMS_FILE, DATA_FILE)

    print(f"--- Starting Auto-Optimization Loop ({MAX_ITERATIONS} Iterations) ---")
    print(f"Target: {STRATEGY_FILE}")
    print(f"Params: {PARAMS_FILE}")
    print(f"Data Source: {DATA_FILE}")
    
    init_sweep_db()
    run_id = str(uuid.uuid4())
    try:
        _db_rows_before = sqlite3.connect(SWEEP_DB_FILE).execute(
            "SELECT COUNT(*) FROM sweep_results WHERE asset=? AND timeframe=?",
            (ASSET, TIMEFRAME)).fetchone()[0]
    except Exception:
        _db_rows_before = 0

    strategy_name_clean = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")
    verified_best = sanitize_winner_file(strategy_name_clean, DATA_FILE)

    # Guard: winner file exists but DB is empty for this combo — stale baseline risk.
    _winner_file_check = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name_clean}_{ASSET}_{TIMEFRAME}.csv")
    if os.path.exists(_winner_file_check) and _db_rows_before == 0:
        print("\n" + "=" * 70)
        print("⚠️  STALE WINNER / EMPTY DB WARNING")
        print("=" * 70)
        print(f"  A winner file exists for {ASSET} {TIMEFRAME} but the sweep DB has")
        print(f"  0 rows for this combo. This usually means:")
        print(f"    a) The DB was reset/deleted after the winner was found, OR")
        print(f"    b) The scoring formula or simulation changed since the winner was saved.")
        print(f"")
        print(f"  Risk: the winner's score will be used as the GPU pre-filter baseline.")
        print(f"  If it's inflated relative to the current simulation, nothing new will")
        print(f"  ever beat it and no new results will be saved — silently.")
        print(f"")
        print(f"  Recommended action (if scoring or sim changed):")
        print(f"      python3 tools/reset_results.py --apply   # wipe winner + start fresh")
        print(f"  Or if the winner is still valid, continue and it will self-correct.")
        print("=" * 70)
        _countdown(10)

    history = []
    global_best_score = -float('inf')
    global_best_row = None   # full winner row; used to restore file if GPU subprocess overwrites it
    global_best_calmar = 0.0  # Calmar of best CPU-verified winner; used as GPU baseline score
    global_wfo_best_score = -float('inf')  # best WFO_Min_Fold seen across all iterations
    global_wfo_best_row = None             # params row of the cross-iteration WFO winner
    global_wfo_promoted = False            # True once a WFO winner (WFO_Min>=0) has been written to main file
    previous_score = -float('inf')
    start_loop_time = time.time()

    # Use the CPU-recalculated score (from sanitize_winner_file) as the baseline.
    # This ensures the baseline reflects the current data and commission settings,
    # not a stale score that was computed on a different TV export or without commission.
    if verified_best is not None:
        global_best_score = verified_best
        print(f"   >> [INIT] Baseline {OPTIMIZATION_METRIC} (CPU-verified): {global_best_score:.4f}")
    else:
        strategy_name = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")
        historical_winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
        if os.path.exists(historical_winner_file):
            try:
                hist_df = pd.read_csv(historical_winner_file)
                global_best_score = hist_df.iloc[0].get(OPTIMIZATION_METRIC, -float('inf'))
                print(f"   >> [INIT] Loaded historical best {OPTIMIZATION_METRIC}: {global_best_score:.4f} (unverified)")
            except Exception:
                pass

    # Load the initial winner row so we can restore it if the GPU subprocess overwrites it
    # without finding a CPU-verified improvement.
    _init_winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name_clean}_{ASSET}_{TIMEFRAME}.csv")
    if os.path.exists(_init_winner_file):
        try:
            global_best_row = pd.read_csv(_init_winner_file).iloc[0]
            global_best_calmar = float(global_best_row.get("Calmar Ratio", 0.0))
            # Initialize WFO baseline from saved winner's WFO_Min_Fold so we don't
            # downgrade a good prior winner just because -0.X > -inf on iteration 1.
            _init_wfo_min = global_best_row.get("WFO_Min_Fold", None)
            if _init_wfo_min is not None and not (isinstance(_init_wfo_min, float) and _init_wfo_min != _init_wfo_min):
                global_wfo_best_score = float(_init_wfo_min)
                global_wfo_promoted = True   # loaded winner IS the promoted WFO result — guard against downgrade from first iteration
                print(f"   >> [INIT] Loaded existing winner WFO_Min_Fold={global_wfo_best_score:.4f} as baseline (WFO guard active)")
        except Exception:
            pass

    for i in range(1, MAX_ITERATIONS + 1):
        print(f"\n=== Iteration {i}/{MAX_ITERATIONS} ===")
        
        if args.hours:
            elapsed_total = time.time() - start_loop_time
            remaining_total = max(0, (args.hours * 3600) - elapsed_total)
            print(f"   >> Global Time Remaining: {format_duration(remaining_total)}")
        
        strategy_name = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")
        run_best_file = os.path.join(WINNERS_DIR, f"optimization_run_best_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
        if os.path.exists(run_best_file):
            os.remove(run_best_file)
        
        iter_start = time.time()
        # GPU optimizes Calmar internally. Pass the Calmar of the current CPU-verified
        # winner so the GPU only retains candidates that beat it. P&L/DD units can't
        # be used directly here since they differ in scale from Calmar.
        opt_output = run_optimization(SEARCH_ITERATIONS, global_best_calmar)
        iter_elapsed = time.time() - iter_start
        rate = (min(calculate_search_space(PARAMS_FILE), SEARCH_ITERATIONS) if calculate_search_space(PARAMS_FILE) > 0 else SEARCH_ITERATIONS) / iter_elapsed if iter_elapsed > 0 else 0
        print(f"   >> Actual Time: {format_duration(iter_elapsed)} | Rate: {rate/1000:,.0f}k samples/s")
        
        if rate < 200000 and opt_output:
            print("   >> [WARNING] Low sampling rate detected. Dumping optimizer logs for debugging:")
            print(opt_output[-2000:])
        
        strategy_name = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")
        sweep_file = os.path.join(SWEEPS_DIR, f"optimization_sweep_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
        
        best_row, wfo_winner_this_iter = verify_top_results(sweep_file, DATA_FILE, top_n=5000, run_id=run_id, iteration_number=i, search_strategy=args.search)
        is_verified = False

        # Track global WFO winner across iterations using WFO_Min_Fold (worst-fold
        # Calmar) — consistent with per-iteration winner selection logic.
        # When a new WFO min-fold global best is found, write to BOTH the dedicated
        # _wfo.csv AND the main winner cheatsheet so the cheatsheet always reflects
        # the best regime-robust result found, not just IS-optimal iterations.
        if wfo_winner_this_iter is not None:
            wfo_min_this = wfo_winner_this_iter.get('WFO_Min_Fold', -99.0)
            wfo_mean_this = wfo_winner_this_iter.get('WFO_Score', 0.0)
            if wfo_min_this > global_wfo_best_score:
                global_wfo_best_score = wfo_min_this
                global_wfo_best_row = wfo_winner_this_iter
                wfo_winner_file = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}_wfo.csv")
                pd.DataFrame([wfo_winner_this_iter]).to_csv(wfo_winner_file, index=False)
                print(f"   >> [WFO] ★ New global WFO winner (WFO_Min={wfo_min_this:.4f}  WFO_Mean={wfo_mean_this:.4f}  IS={wfo_winner_this_iter.get('Composite', 0):.4f})")
                # Only promote to main winner file when all folds are profitable (WFO_Min >= 0).
                # A negative WFO_Min means the worst fold still loses money — not worth
                # overwriting a better IS winner for a marginal improvement in the worst fold.
                if wfo_min_this >= 0:
                    winner_csv_path = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
                    pd.DataFrame([wfo_winner_this_iter]).to_csv(winner_csv_path, index=False)
                    write_winner_readable(wfo_winner_this_iter, strategy_name, WINNERS_DIR, asset=ASSET, timeframe=TIMEFRAME)
                    global_best_row = wfo_winner_this_iter  # keep restore pointer in sync
                    global_wfo_promoted = True
                    print(f"   >> [WFO] ★ Promoted to main winner (all folds positive)")
                else:
                    print(f"   >> [WFO] ★ Saved to _wfo.csv only (WFO_Min<0, keeping main winner intact)")

        if best_row is None:
            best_row = get_run_best_result()
            if best_row is not None:
                print("   >> [INFO] Sweep file missing. Verifying 'Run Best' result on CPU...")
                try:
                    df_data_fallback = pd.read_csv(DATA_FILE)
                    df_data_fallback.columns = df_data_fallback.columns.str.lower()
                    if 'time' in df_data_fallback.columns:
                        df_data_fallback['time'] = pd.to_datetime(df_data_fallback['time'], utc=True).dt.tz_localize(None)
                        mask = (df_data_fallback['time'] >= TRAIN_START) & (df_data_fallback['time'] <= TRAIN_END)
                        df_data_fallback = df_data_fallback.loc[mask].copy()
                    params = best_row.to_dict()
                    df_res = strategy_module.generate_signals(df_data_fallback.copy(), **params)
                    metrics = strategy_module.calculate_metrics(df_res, score_start=SCORE_START)
                    
                    print(f"   >> [VERIFY] Run Best: {OPTIMIZATION_METRIC} {best_row.get(OPTIMIZATION_METRIC,0):.4f} -> {metrics.get(OPTIMIZATION_METRIC, 0):.4f}")
                    
                    for key, value in metrics.items():
                        best_row[key] = value
                    is_verified = True 
                except Exception as e:
                    print(f"   >> [FALLBACK] CPU calculation failed: {e}")
        else:
            is_verified = True
            
        if best_row is None:
            print("   >> [WARNING] No valid results found for this run.")
            break
            
        score = best_row.get(OPTIMIZATION_METRIC, 0)
        trades = int(best_row.get('Total Trades', 0))
        pnl = best_row.get('Total P&L %', 0)
        dd = best_row.get('Max Drawdown %', 0)
        sortino = best_row.get('Sortino Ratio', 0.0)
        sharpe = best_row.get('Sharpe Ratio', 0.0)
        calmar = best_row.get('Calmar Ratio', 0.0)

        pct_in_market = best_row.get('% In Market', 0.0)
        if OPTIMIZATION_METRIC == "Composite":
            print(f"   >> Best Composite: {score:.4f} | Calmar: {calmar:.4f} | Sortino: {sortino:.4f} | P&L: {pnl:,.0f}% | DD: {dd:.2f}% | Trades: {trades} | In Market: {pct_in_market:.1f}%")
        else:
            print(f"   >> Best {OPTIMIZATION_METRIC}: {score:.4f} | Sortino: {sortino:.4f} | Sharpe: {sharpe:.4f} | P&L: {pnl:,.0f}% | DD: {dd:.2f}% | Trades: {trades} | In Market: {pct_in_market:.1f}%")

        is_new_global_best = score > global_best_score and score > 0
        is_stagnant = (score == previous_score) and (i > 1)
        _skip_write = False  # set True inside is_new_global_best block when WFO guard fires

        # Prefer WFO winner as primary saved result — it was selected for robustness
        # across distinct market regimes, not just IS Composite maximization.
        # Fall back to IS Composite winner only when WFO produced no valid result.
        wfo_score_this_iter = wfo_winner_this_iter.get('WFO_Score', 0.0) if wfo_winner_this_iter is not None else 0.0
        winner_row = wfo_winner_this_iter if wfo_score_this_iter > 0 else best_row
        winner_source = "WFO" if wfo_score_this_iter > 0 else "IS-Composite"

        winner_csv_path = os.path.join(WINNERS_DIR, f"optimization_winner_{strategy_name}_{ASSET}_{TIMEFRAME}.csv")
        if is_new_global_best:
            if winner_row is not None:
                # Guard: don't overwrite a better WFO winner with an inferior one.
                # When a WFO winner has already been promoted, only write to the main
                # file if this iteration's WFO winner is at least as good as the global
                # WFO best (the WFO block above would have already written it in that
                # case — so this is a harmless double write). Skip the write if the
                # current WFO winner is worse, preserving the superior result on disk.
                _wfo_min_this_iter = (wfo_winner_this_iter.get('WFO_Min_Fold', -99.0)
                                      if winner_source == "WFO" and wfo_winner_this_iter is not None
                                      else None)
                _skip_write = (global_wfo_promoted and winner_source == "WFO"
                               and _wfo_min_this_iter is not None
                               and _wfo_min_this_iter < global_wfo_best_score)
                if _skip_write:
                    print(f"   >> [WFO] IS global best improved but WFO_Min={_wfo_min_this_iter:.4f} < "
                          f"existing promoted best {global_wfo_best_score:.4f}. Restoring superior WFO winner.")
                    # GPU subprocess may have already written its Calmar-only result (no Composite
                    # column) to the winner CSV. Explicitly restore the WFO winner to undo any
                    # GPU overwrite — otherwise the file is left in an inconsistent state.
                    if global_best_row is not None:
                        pd.DataFrame([global_best_row]).to_csv(winner_csv_path, index=False)
                        write_winner_readable(global_best_row, strategy_name, WINNERS_DIR, asset=ASSET, timeframe=TIMEFRAME)
                else:
                    pd.DataFrame([winner_row]).to_csv(winner_csv_path, index=False)
                    write_winner_readable(winner_row, strategy_name, WINNERS_DIR, asset=ASSET, timeframe=TIMEFRAME)
                if not _skip_write:
                    # Keep the floor monotonically non-decreasing so that future GPU iterations
                    # cannot accept candidates that would have been rejected by a prior winner's
                    # Calmar, even when a new winner has higher Composite but lower Calmar.
                    # Only update when the winner was actually written — skip_write means the
                    # inferior WFO candidate's Calmar must not inflate the GPU pre-filter.
                    global_best_calmar = max(global_best_calmar, float(winner_row.get("Calmar Ratio", 0.0)))
                # Only preserve global_best_row over the IS winner if a WFO winner has
                # already been promoted to the main file this run. Without a promoted WFO
                # winner, global_best_row must follow the IS winner so the "else" restore
                # branch doesn't overwrite a newly saved IS winner with a stale old row.
                if not global_wfo_promoted:
                    global_best_row = winner_row

            if is_verified:
                if _skip_write:
                    print(f"   >> [INFO] IS {OPTIMIZATION_METRIC} improved ({global_best_score:.4f} → {score:.4f}) but WFO guard kept existing winner on disk (WFO_Min={_wfo_min_this_iter:.4f} < {global_wfo_best_score:.4f})")
                else:
                    print(f"   >> [SUCCESS] New VERIFIED global best {OPTIMIZATION_METRIC} found (Old: {global_best_score:.4f}, New: {score:.4f}) [{winner_source} winner saved]")
                global_best_score = score
            else:
                print(f"   >> [CAUTION] New best found ({score:.4f}) but NOT VERIFIED. Will NOT update global baseline to avoid corruption.")
        else:
            # The GPU subprocess may have overwritten the winner CSV with a Calmar-only result
            # that doesn't pass CPU verification. Restore the authoritative winner here.
            # global_best_row is always the cross-iteration best WFO winner (maintained by
            # the WFO tracking block above), so this restores the most robust result found.
            if global_best_row is not None:
                pd.DataFrame([global_best_row]).to_csv(winner_csv_path, index=False)
                write_winner_readable(global_best_row, strategy_name, WINNERS_DIR, asset=ASSET, timeframe=TIMEFRAME)
            if is_stagnant:
                print(f"   >> [STAGNATION] Result identical to previous run.")
        
        if trades == 0 and opt_output:
            print("   >> [WARNING] Zero trades detected. Optimizer Output (tail):")
            print(opt_output[-1000:])
            
        # --- Per-iteration summary (compact one-liner) ---
        # Distinguish: did the winner file actually change, or just the IS Composite score?
        _winner_file_updated = is_new_global_best and not _skip_write
        best_marker = " ★ NEW BEST" if _winner_file_updated else (" ↑ IS improved (WFO guard held)" if is_new_global_best else "")
        wfo_score_display = best_row.get('WFO_Score', None) if best_row is not None else None
        wfo_suffix = f"  WFO={wfo_score_display:.4f}" if wfo_score_display is not None else ""
        if OPTIMIZATION_METRIC == "Composite":
            print(f"\n  Iteration Summary {i}/{MAX_ITERATIONS}: Composite={score:.4f}  Calmar={calmar:.4f}  Sortino={sortino:.4f}  P&L={pnl:,.1f}%  DD={dd:.2f}%  Trades={trades}  Best={global_best_score:.4f}{wfo_suffix}{best_marker}\n")
        else:
            print(f"\n  Iteration Summary {i}/{MAX_ITERATIONS}: {OPTIMIZATION_METRIC}={score:.4f}  Sortino={sortino:.4f}  P&L={pnl:,.1f}%  DD={dd:.2f}%  Trades={trades}  Best={global_best_score:.4f}{wfo_suffix}{best_marker}\n")

        if not args.no_dashboard:
            subprocess.run([sys.executable, "mine_sweep_db.py"], check=False)
            # Auto-apply high-conviction sign locks and range tightenings to the
            # params JSON so the next iteration samples from the reduced space.
            subprocess.run([sys.executable, "tools/lock_params.py", "--apply"], check=False)
            if args.tighten:
                subprocess.run([sys.executable, "tools/tighten_params.py", "--apply"], check=False)
            subprocess.run([sys.executable, "tools/oos_dashboard.py"], check=False)
            subprocess.run([sys.executable, "tools/shap_feature_importance.py"], check=False)

        history.append(f"Iteration Summary {i}: {OPTIMIZATION_METRIC}={score}, Trades={trades}")
        previous_score = score
        
        if i == MAX_ITERATIONS:
            print(f"   >> [INFO] Final iteration {i} complete.")
            break

        with open(PARAMS_FILE, 'r') as f:
            current_params = json.load(f)
            
        sanitized_params = sanitize_params(current_params)
        if sanitized_params != current_params:
            print("   >> [INIT] Sanitized initial parameter ranges (fixed steps/bounds).")
            current_params = sanitized_params
            with open(PARAMS_FILE, 'w') as f:
                json.dump(current_params, f, indent=4)
            
        if i > 1:
            time.sleep(5)
            
        time.sleep(5)

    total_elapsed = time.time() - start_loop_time
    try:
        _db_rows_after = sqlite3.connect(SWEEP_DB_FILE).execute(
            "SELECT COUNT(*) FROM sweep_results WHERE asset=? AND timeframe=?",
            (ASSET, TIMEFRAME)).fetchone()[0]
        _db_total = sqlite3.connect(SWEEP_DB_FILE).execute(
            "SELECT COUNT(*) FROM sweep_results").fetchone()[0]
    except Exception:
        _db_rows_after = _db_rows_before
        _db_total = 0
    _db_added = _db_rows_after - _db_rows_before

    print("\n" + "="*60)
    print("  OPTIMIZATION RUN COMPLETE")
    print("="*60)
    for entry in history:
        print(f"  {entry}")
    print(f"\n  Global Best IS {OPTIMIZATION_METRIC}: {global_best_score:.4f}")
    if global_wfo_best_row is not None:
        print(f"  Global Best WFO Score   : {global_wfo_best_score:.4f}"
              f"  (IS={global_wfo_best_row.get('Composite', 0):.4f})")
    print(f"  Total Elapsed Time: {format_duration(total_elapsed)}")
    print(f"  DB Rows Added      : +{_db_added} {ASSET} {TIMEFRAME}  (total DB: {_db_total:,})")
    print("="*60)
    print("\nFinal results are in the 'results/' directory.")

    if not args.no_dashboard:
        subprocess.run([sys.executable, "mine_sweep_db.py"], check=False)

    strategy_name_final = STRATEGY_FILE.replace(".py", "").replace("strategy_", "")
    write_performance_report(strategy_name_final, DATA_FILE)

    if not args.no_dashboard:
        subprocess.run([sys.executable, "tools/oos_dashboard.py"], check=False)

    if not args.no_dashboard:
        # Run SHAP across ALL assets/TFs in the DB so the report always reflects
        # the full picture, not just the combo that just finished.
        subprocess.run(
            [sys.executable, "tools/shap_feature_importance.py"],
            check=False,
        )

    # Phone notification via ntfy (https://ntfy.sh) — subscribe to jlo_alerts in the app
    try:
        msg = (
            f"✅ {ASSET} {TIMEFRAME} optimization done! "
            f"Best P&L/DD={global_best_score:.2f} "
            f"({format_duration(total_elapsed)})"
        )
        subprocess.run(
            ["curl", "-s", "-d", msg, "ntfy.sh/jlo_alerts"],
            timeout=10, check=False
        )
    except Exception:
        pass  # never block on notification failure

if __name__ == "__main__":
    main()