"""Non-promoting, deterministic incumbent accounting and archived-model audit."""
from __future__ import annotations

import argparse
import collections
import hashlib
import json
from pathlib import Path
import sys
import time
import platform

import numpy as np
import pandas as pd

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
from config import OOS_START, SCORE_START, TRAIN_END, WFO_FOLDS
from strategies.mlp_accounting import net_returns, window_metrics
from strategies.strategy_mlp_scores import FEATURE_COLS, generate_signals, _score_to_signals
from tools.run_mlp_score_structure_experiment import _data_path, _load_winner, _winner_path

DEFAULT_OUTPUT = REPO / 'results/mlp_measurement_audit_2026_09_21/baselines'


def digest(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def benchmark(frame, kind):
    """Decide at close; first economic exposure and entry fee occur next bar."""
    result = frame[['time', 'close']].copy()
    eligible = result.time >= pd.Timestamp(SCORE_START)
    if kind == 'cash':
        position = pd.Series(False, index=result.index)
    elif kind == 'buy_hold':
        position = eligible
    elif kind == 'trend_200_calendar_days':
        price = result.set_index('time').close.resample('D').last()
        decision = (price > price.rolling('200D', min_periods=200).mean()).shift(1, fill_value=False)
        position = eligible & result.time.dt.normalize().map(decision).fillna(False)
    else:
        raise ValueError(kind)
    previous = position.shift(1, fill_value=False)
    result['position'] = position.astype(int)
    result['execute_entry'] = position & ~previous
    result['execute_exit'] = ~position & previous
    return result


def ledger(signals):
    """Attribute disjoint paid return intervals to their entry signal."""
    returns = net_returns(signals).to_numpy()
    equity_before = np.r_[1., np.cumprod(1 + returns)[:-1]]
    rows = []
    active = None
    for i in range(len(signals)):
        if bool(signals.execute_exit.iloc[i]) and active is not None:
            rows.append((active, i, False))
            active = None
        if bool(signals.execute_entry.iloc[i]):
            if active is not None:
                raise ValueError('Overlapping entries')
            active = i
    if active is not None:
        rows.append((active, len(signals)-1, True))
    output = []
    for number, (entry, end, opened) in enumerate(rows, 1):
        first = entry + 1
        value = float(np.prod(1 + returns[first:end+1]) - 1)
        contribution = float(equity_before[first] * value) if first < len(signals) else 0.
        output.append({'trade': number, 'entry_index': entry, 'first_return_index': first,
                       'last_return_index': end, 'entry_time': str(signals.time.iloc[entry]),
                       'end_time': str(signals.time.iloc[end]), 'open': opened,
                       'unfilled_terminal_entry': first > end,
                       'net_return_pct': value*100, 'initial_equity_pnl': contribution})
    return pd.DataFrame(output)


def overridden_metrics(signals, values, start, end):
    """Measure a diagnostic return path without changing its calendar."""
    synthetic = signals[['time']].copy()
    synthetic['close'] = np.cumprod(1 + np.asarray(values))
    synthetic['position'] = 1
    synthetic['execute_entry'] = False
    synthetic['execute_exit'] = False
    # Prepend a zero-return observation so first-row returns are represented.
    first = synthetic.iloc[[0]].copy()
    first['time'] -= signals.time.diff().median()
    first['close'] = 1.
    combined = pd.concat([first, synthetic], ignore_index=True)
    result = window_metrics(combined, start, end)
    actual = window_metrics(signals, start, end)
    for key in ['entries', 'exits', 'exposure_pct', 'carry_in', 'open_at_end']:
        result[key] = actual[key]
    return result


def windows(signals, created):
    last = signals.time.iloc[-1]
    yield 'IS', SCORE_START, TRAIN_END
    yield 'full', SCORE_START, last
    yield 'OOS_previously_inspected', OOS_START, last
    yield 'since_creation', created, last
    for year in range(pd.Timestamp(SCORE_START).year, last.year+1):
        yield f'annual_{year}', max(pd.Timestamp(SCORE_START), pd.Timestamp(year, 1, 1)), min(last, pd.Timestamp(year+1, 1, 1)-pd.Timedelta(nanoseconds=1))
    for end in pd.date_range(SCORE_START, last, freq='ME'):
        end = end + pd.Timedelta(days=1)-pd.Timedelta(nanoseconds=1)
        if end > last:
            continue
        for months in [12, 24]:
            start = (end.normalize()+pd.Timedelta(days=1))-pd.DateOffset(months=months)
            if start >= pd.Timestamp(SCORE_START):
                yield f'rolling_{months}m_{end.date()}', start, end


def measure(signals, name, start, end):
    mask = (signals.time >= pd.Timestamp(start)) & (signals.time <= pd.Timestamp(end) if not isinstance(end, str) else signals.time < pd.Timestamp(end)+pd.Timedelta(days=1))
    base = {'window': name, 'start': str(start), 'end': str(end), 'observed_bars': int(mask.sum())}
    if not mask.any():
        return {**base, 'status': 'no_coverage'}
    return {**base, 'status': 'observed', **window_metrics(signals, start, end)}


def run(output=DEFAULT_OUTPUT):
    started = time.monotonic()
    output = Path(output)
    if output.exists() and any(output.iterdir()):
        raise ValueError('Use a new empty output directory; completed evidence is immutable')
    output.mkdir(parents=True, exist_ok=True)
    shortlist = [str(REPO/'strategies/params/mlp'/f'mlp_weights_COINBASE_BTCUSD_6H_{family}_seed{seed}.json')
                 for family in ['grouped_v1', 'random_sparse_v1', 'hybrid_grouped_v1'] for seed in [9090, 9191, 9292]]
    contract = {'archived_candidates_predeclared': shortlist, 'entry_gate': SCORE_START,
                'fees_per_side': .005, 'terminal': 'marked to market, no synthetic exit',
                'tail_policy': 'drop exactly final export row on every timeframe',
                'windows': 'continuous signals; carry-in return retained',
                'benchmark': 'close > trailing 200-calendar-day SMA; 200-day warmup; close decision, next-bar returns',
                'regime': 'prior close vs prior trailing200D SMA; masked noncontiguous attribution, not standalone tradable returns',
                'archive_limit': 'nine saved candidates plus incumbent; saved-threshold replay and equal additional128trial tuning; historical selection budgets unequal'}
    contract['benchmark'] = 'previous completed daily close vs200-calendar-day daily SMA,200 daily observations; same fees'
    contract['equal_additional_tuning'] = {'count_per_model':128,'seed':20260921,'selection_window':[SCORE_START,TRAIN_END],
        'rank':'calendar Calmar with20entries; if none eligible retain highest with ineligible flag',
        'thresholds':'same seeded quantiles of each model IS scores; entry/exit/confirmation uniform quantiles0.05..0.95',
        'trail_choices':[0,5,10,20,30],'confirmation':'both booleans sampled','runtime_cap_seconds':600,
        'limitation':'equal additional budget cannot equalize historical fitting or selection; later windows previously inspected'}
    (output/'protocol.json').write_text(json.dumps(contract, indent=2)+'\n')
    metrics, trades, concentration, regimes, archived, reconcile, inputs, tuning = [], [], [], [], [], [], [], []
    for tf in ['4H', '6H', '8H', '12H', '1D']:
        path = _data_path('COINBASE_BTCUSD', tf)
        data = pd.read_csv(path)
        data.columns = data.columns.str.lower().str.strip()
        data.time = pd.to_datetime(data.time, unit='s' if pd.api.types.is_numeric_dtype(data.time) else None, utc=True).dt.tz_localize(None)
        data = data.iloc[:-1].copy().reset_index(drop=True)
        if data.time.isna().any() or not data.time.is_monotonic_increasing or data.time.duplicated().any():
            raise ValueError('Invalid times')
        params = _load_winner('COINBASE_BTCUSD', tf)
        params['_pine_time_start'] = SCORE_START
        artifact = json.loads(Path(params['mlp_weights_file']).read_text())
        cols = artifact['feature_cols']
        if cols != FEATURE_COLS or set(cols)-set(data.columns):
            raise ValueError('Unexpected feature schema/order')
        if not np.isfinite(data[['close','high']]).all().all():
            raise ValueError('Invalid prices')
        if (data[['close','high']]<=0).any().any():
            raise ValueError('Nonpositive prices')
        for col in set(cols):
            values=data[col].to_numpy(dtype=float)
            finite=np.isfinite(values)
            if not finite.any() or not finite[np.flatnonzero(finite)[0]:].all():
                raise ValueError(f'Interior missing/nonfinite feature {tf}/{col}')
        created = pd.Timestamp(artifact['training']['created']).tz_convert(None)
        signals = generate_signals(data, **params)
        inputs.append({'tf': tf, 'data': str(path), 'data_sha256': digest(path),
                       'winner_sha256': digest(_winner_path('COINBASE_BTCUSD', tf)),
                       'weights': params['mlp_weights_file'], 'weights_sha256': digest(params['mlp_weights_file']),
                       'created': str(created), 'training': artifact['training'],
                       'missing_counts_preserved_and_zero_filled': data[list(dict.fromkeys(cols))].isna().sum().to_dict(),
                       'duplicate_features': {k:v for k,v in collections.Counter(cols).items() if v>1},
                       'first': str(data.time.iloc[0]), 'last': str(data.time.iloc[-1]), 'excluded_tail_rows':1})
        for kind, series in [('incumbent',signals)]+[(k,benchmark(data,k)) for k in ['cash','buy_hold','trend_200_calendar_days']]:
            for name,start,end in windows(signals,created):
                metrics.append({'tf':tf,'model':kind,**measure(series,name,start,end)})
        led = ledger(signals)
        if not np.isclose(led.initial_equity_pnl.sum(),np.prod(1+net_returns(signals))-1,rtol=1e-10,atol=1e-10):
            raise AssertionError('Trade contribution ledger does not reconstruct equity')
        trades.extend(led.assign(tf=tf).to_dict('records'))
        winning = led.loc[led.initial_equity_pnl>0].sort_values('initial_equity_pnl',ascending=False)
        for count in [1,3]:
            values = net_returns(signals).copy()
            for row in winning.head(count).itertuples():
                values.iloc[row.first_return_index:row.last_return_index+1] = 0.
            concentration.append({'tf':tf,'removed_winners':count,'share_of_positive_equity_pnl':float(winning.head(count).initial_equity_pnl.sum()/winning.initial_equity_pnl.sum()) if len(winning) else None,
                                  'diagnostic':'largest absolute initial-equity P&L contributions; zero entire winning-trade net return intervals; no reentry; original counts/exposure retained',
                                  **overridden_metrics(signals,values,SCORE_START,signals.time.iloc[-1])})
        average = data.set_index('time').close.rolling('200D',min_periods=1).mean().to_numpy()
        labels = pd.Series(np.where(data.close>average,'above_sma','below_sma')).shift(1).fillna('warmup')
        labels.loc[data.time<data.time.iloc[0]+pd.Timedelta(days=200)]='warmup'
        for label in ['above_sma','below_sma','warmup']:
            mask = labels.eq(label)&(data.time>=pd.Timestamp(SCORE_START))
            values = net_returns(signals).where(mask,0.)
            durations=data.time.diff().dt.total_seconds().fillna(data.time.diff().dt.total_seconds().median())
            seconds=float(durations.loc[mask].sum())
            exposure=float((signals.position.shift(1,fill_value=0).loc[mask]*durations.loc[mask]).sum()/seconds*100) if seconds else None
            regimes.append({'tf':tf,'regime':label,'attributed_bars':int(mask.sum()),
                            'entry_events_in_regime':int((signals.execute_entry&mask).sum()),
                            'exits_in_regime':int((signals.execute_exit&mask).sum()),
                            'attributed_years':seconds/(365.25*86400),'exposure_within_regime_pct':exposure,
                            'interpretation':'noncontiguous masked contribution on full calendar; not regime strategy',
                            **overridden_metrics(signals,values,SCORE_START,signals.time.iloc[-1])})
        tvpath=REPO/'data/mlp'/f'MLPScores_COINBASE_BTCUSD_2026-06-25_{tf.lower()}.csv'
        tv=pd.read_csv(tvpath)
        exits=tv[tv.Type.str.startswith('Exit')]
        rec=measure(signals,'IS',SCORE_START,TRAIN_END)
        reconcile.append({'tf':tf,'saved_pnl_pct':params['Total P&L %'],'saved_dd_pct':params['Max Drawdown %'],
                          'saved_trades':params['Total Trades'],'recomputed_pnl_pct':rec['pnl_pct'],'recomputed_dd_pct':rec['dd_pct'],
                          'recomputed_entries':rec['entries'],'tv_file':str(tvpath),'tv_sha256':digest(tvpath),
                          'tv_final_cumulative_pct':exits.iloc[-1]['Cumulative PnL %'],'tv_exit_rows':len(exits),
                          'tv_last_exit':exits.iloc[-1]['Date and time'],'tv_matching_current_state':'unverified; old June25 export'})
        if tf=='6H':
            for candidate in [params['mlp_weights_file']]+shortlist:
                candidate_params=params.copy()
                model=json.loads(Path(candidate).read_text())
                if model['feature_cols']!=cols:
                    raise ValueError('Archive schema mismatch')
                if candidate in shortlist:
                    candidate_params.update(model['training']['recommended_thresholds'])
                candidate_params['mlp_weights_file']=candidate
                replay=generate_signals(data,**candidate_params)
                for name,start,end in [('IS',SCORE_START,TRAIN_END),('OOS_previously_inspected',OOS_START,data.time.iloc[-1])]+[(f'historical_fold_{i}',s,e) for i,(_,s,e) in enumerate(WFO_FOLDS)]:
                    archived.append({'variant':'saved_thresholds','candidate':Path(candidate).name,'weights_sha256':digest(candidate),'thresholds':json.dumps({k:v for k,v in candidate_params.items() if k.startswith('i_')},sort_keys=True),**measure(replay,name,start,end)})
                rng=np.random.default_rng(20260921)
                scores=replay.loc[(replay.time>=pd.Timestamp(SCORE_START))&(replay.time<pd.Timestamp(TRAIN_END)+pd.Timedelta(days=1)),'activation_score']
                choices=[]
                for trial in range(128):
                    if time.monotonic()-started>600:
                        raise RuntimeError('Predeclared600second runtime limit reached; partial output is not complete')
                    selected=candidate_params.copy()
                    for key,q in zip(['i_long_entry_activation_threshold','i_long_exit_activation_threshold','i_long_exit_activation_confirmation_threshold'],rng.uniform(.05,.95,3)):
                        selected[key]=float(scores.quantile(q))
                    selected['i_trailing_stop_threshold']=int(rng.choice([0,5,10,20,30]))
                    selected['i_use_long_entry_confirmation']=bool(rng.integers(2))
                    selected['i_use_long_exit_confirmation']=bool(rng.integers(2))
                    trial_signals=_score_to_signals(replay.copy(),selected,replay.stoch_peak_norm.to_numpy(),replay.close.to_numpy(),replay.high.to_numpy())
                    result=window_metrics(trial_signals,SCORE_START,TRAIN_END)
                    eligible=result['entries']>=20
                    rank=result['calmar'] if result['calmar'] is not None else -float('inf')
                    choices.append((eligible,rank,-trial,selected))
                    tuning.append({'candidate':Path(candidate).name,'trial':trial,'eligible':eligible,'params':json.dumps({k:v for k,v in selected.items() if k.startswith('i_')},sort_keys=True),**result})
                eligible,_,trial,selected=max(choices,key=lambda x:x[:3])
                (output/f'selection_{Path(candidate).stem}.json').write_text(json.dumps({'candidate':candidate,'eligible':eligible,'trial':-trial,'params':selected},indent=2)+'\n')
                tuned=_score_to_signals(replay.copy(),selected,replay.stoch_peak_norm.to_numpy(),replay.close.to_numpy(),replay.high.to_numpy())
                for name,start,end in [('IS',SCORE_START,TRAIN_END),('OOS_previously_inspected',OOS_START,data.time.iloc[-1])]+[(f'historical_fold_{i}',s,e) for i,(_,s,e) in enumerate(WFO_FOLDS)]:
                    archived.append({'variant':'equal_extra_tuning','candidate':Path(candidate).name,'eligible_selection':eligible,'thresholds':json.dumps({k:v for k,v in selected.items() if k.startswith('i_')},sort_keys=True),**measure(tuned,name,start,end)})
    for name,rows in [('metrics',metrics),('trade_ledger',trades),('concentration_diagnostics',concentration),('regime_attribution',regimes),('archived_candidates',archived),('archive_tuning_trials',tuning),('reconciliation',reconcile)]:
        pd.DataFrame(rows).to_csv(output/f'{name}.csv',index=False)
    legacy=DEFAULT_OUTPUT.parent/'legacy_replay_variants.csv'
    manifest={'contract':contract,'inputs':inputs,'legacy_variants':{'path':str(legacy),'sha256':digest(legacy)} if legacy.exists() else None,
              'environment':{'python':sys.version,'numpy':np.__version__,'pandas':pd.__version__,'platform':platform.platform()},
              'source_sha256':{p:digest(REPO/p) for p in ['tools/run_mlp_incumbent_audit.py','strategies/mlp_accounting.py','strategies/strategy_mlp_scores.py','strategies/strategy_activation_scores.py','config.py']},
              'outputs_sha256':{p.name:digest(p) for p in sorted(output.iterdir()) if p.is_file()}}
    (output/'manifest.json').write_text(json.dumps(manifest,indent=2,default=str)+'\n')
    return pd.DataFrame(reconcile)


if __name__=='__main__':
    parser=argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output',type=Path,default=DEFAULT_OUTPUT)
    args=parser.parse_args()
    print(run(args.output).to_string(index=False))