ګ9j֌dZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZej"j%dej"j'ej"j'ej"j)eddlmZddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z$m%Z&m'Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.dZ/dZ0da1da2d a3d tdd tfd a4d a5da6da7da8e$a9e&a:dZ;dZZ?dhdZ@dZAdZBdZCdddddddd d!d"d# ZDhd$ZEeFe*eEzZGdid%ZHd&ZIdjd'ZJd(ZKd)ZLd*ZMgd+ZNd,ZOd-ZPid.d/gd0d1gd2d3gd4d5gd6d7gd8d9gd:d;gdd?gd@dAgdBdCgdDdEgdFdGgdHdIgdJdKgdLdMgdNdOgdPgdQgdRgdSgdTgdUgdVdWgdXgdYgdZgdddd[ ZQd\ZRd]ZSd^ZTd_ZUd`ZVdaZWdbZXdcZYdkddZZdeZ[dfZ\e]dgk(re\yy)lu Auto-Optimizer Loop Iteratively runs the GPU random search optimizer and tracks the best result across runs. Parameter ranges are never modified automatically — edit the params JSON file manually. N)ProcessPoolExecutor as_completed) TRAIN_START SCORE_START TRAIN_END OOS_START RESULTS_DIR WINNERS_DIR SWEEPS_DIR REPORTS_DIR OUTPUT_DIRMVRV_BEAR_THRESHOLDMVRV_BULL_THRESHOLDcomposite_scoreSUBPERIOD_SPLIT WEIGHT_COLS WFO_FOLDSWFO_MIN_OOS_TRADESWFO_MIN_VALID_FOLDS WFO_TOP_Nzstrategy_activation_scores.py Compositez data/mlp/COINBASE_BTCUSD, 1D.csvCOINBASE_BTCUSD1D4strategies/params/params_strategy_activation_scores__.jsonirandomallzresults/sweep_database.dbi"c eZdZddZdZdZy)LoggercTtj|_t|dd|_y)Nwzutf-8)encoding)sysstdoutterminalopenlog)selffilenames C/Users/jameslopez/projects/TradingBot25/tools/auto_optimize_loop.py__init__zLogger.__init__.s  #8c|jj||jj||jjyN)r'writer)flush)r*messages r,r1z Logger.write2s2 G$ w r.cl|jj|jjyr0)r'r2r))r*s r,r2z Logger.flush7s   r.N)zoptimization_log.txt)__name__ __module__ __qualname__r-r1r2r.r,r!r!-s9 r.r!c gd}|jdd}|jdd}|jdd}|jdd} |jdd} |jdd} |jd d} d d l} | jjj }| dk7r t | d nd}|jd d }|dvrt |d kDr t |d nd}|dk7rdnd}|r|d|n|}dd|d|d|d|dt |d dt |d dt |d d|dt | ddt | ddt t | d td!td"dd#}d$}|Dchc]^}}|}}}|g}d }|D]l\}}}}}||k7r|jd%|d&|}|j|d}|dk(s||d'nd(} ||||}!|jd)|d*d+|!d,d-|d.| n|jD"cgc],}"t|"ts|"jd/s&|"|vs+|".}#}"|#rX|jd0t|#D]9}||j|dd1d }!|jd)|d*d+|!d,d-|d.;tj j#|d2|d|d3}$t%|$d45}%|%j'd#j#|d#zd d d d5|d|d6d7t |d d8t |d d9t | d:d;t | ddt(d.d?d(g}&d }|D]j\}}}}}||k7r|&jd@|d&|}|j|d}||||}!|dk(s||dAnd(}'|&j|d*dB|dC|!dD|'l|#rS|&jdEt|#D]4}||j|dd1d }!|&j|d*dF|!dG6tj j#|d2|d|dH}(t%|(d45}%|%j'd#j#|&d#zd d d t+dI|$t+dJ|(y cc}}wcc}"w#1swYxYw#1swY>xYw)Kz Writes two human-readable companion files alongside the winner CSV: - optimization_winner__cheatsheet.txt : TradingView UI groups + values - optimization_winner__pine_snippet.pine : input.float() defaults to paste )E)!i_long_entry_activation_thresholdzLong Entry Activation ThresholdNeural Activation ThresholdsfloatN) i_long_exit_activation_thresholdzLong Exit Activation Thresholdr;r<N)-i_long_exit_activation_confirmation_thresholdz+Long Exit Activation Confirmation Thresholdr;r<N)i_use_long_exit_confirmationzUse Long Exit Confirmationr;boolN)i_use_long_entry_confirmationzUse Long Entry Confirmationr;r@N)i_trailing_stop_thresholdzTrailing Stop % (0 = disabled)r;r<N) i_w_stochzStochastic WeightNeural Activation Weightsr<N) i_w_macd_predzMACD Prediction WeightrDr<N)i_w_oscz OSC WeightrDr<N)i_w_macd_bullishzMACD Bullish WeightrDr<N)i_w_m3_momentumzM3 Momentum WeightrDr<N)i_m3_momentum_periodzM3 Momentum PeriodrDintN) i_w_m2_tinyzM2 Tiny Momentum WeightrDr<N) i_w_rsid_osczRSID OSC WeightrDr<N)i_w_stoch_div_osczStoch Div OSC WeightrDr<N)i_w_vwap_div_osczVWAP Div OSC WeightrDr<N)i_w_stoch_peakingzStoch Peaking WeightrDr<N)i_w_stoch_bottomingzStoch Bottoming WeightrDr<N)i_w_m3_div_osczM3 Div OSC WeightrDr<N)i_w_m2_div_osczM2 Div OSC WeightrDr<N)i_w_m2_div_osc_noOffsetzM2 Div OSC (No Offset) WeightrDr<N)i_w_bearish_engulfingzBearish Engulfing WeightCandlestick Patternsr<N)i_cs_body_quality_ratiozBody Quality RatiorUr<N)i_cs_confidence_scaling_factorzConfidence Scaling FactorrUr<N)i_w_bullish_hammerzBullish Hammer WeightrUr<N)i_cs_hammer_min_lower_shadowzHammer Min Lower Shadow RatiorUr<333333?)i_cs_hammer_max_body_ratiozHammer Max Body RatiorUr<ffffff?)i_cs_hammer_max_upper_shadowzHammer Max Upper Shadow RatiorUr<333333?)i_w_bullish_engulfingzBullish Engulfing WeightrUr<N)i_cs_bull_eng_scaling_factorz Bullish Engulfing Scaling FactorrUr<g@)i_w_shooting_starzShooting Star WeightrUr<N)i_cs_star_min_upper_shadowzShooting Star Min Upper ShadowrUr<rZ)i_cs_star_max_body_ratiozShooting Star Max Body RatiorUr<r\)i_cs_star_max_lower_shadowzShooting Star Max Lower ShadowrUr<r^)i_w_btc_spx_corrzBTC/SPX Correlation Weight Macro Signalsr<N)i_w_dxyz DXY Weightrfr<N)i_w_vixz VIX Weightrfr<N) i_w_btc_domzBTC Dominance Weightrfr<N) i_w_us10yzUS 10Y Yield Weightrfr<N)i_w_spyz SPY Weightrfr<N)i_w_goldz Gold Weightrfr<N)i_w_mvrvzMVRV Z-Score Weight (discrete)On-Chain Regime Signalsr<N) i_w_mvrv_contz MVRV Z-Score Weight (continuous)rnr<N)i_w_nuplzRealized Price NUPL Weightrnr<N)i_w_fed_net_liqzFed Net Liquidity Sign Weightrnr<N)i_w_gc_positionz Gaussian Channel Position Weightrnr<N)i_w_us2yzUS 2Y Yield Weight Rates & Macror<N)i_w_yield_curvezYield Curve (10Y-2Y) Weightrtr<N)i_w_qqq_spy_ratiozQQQ/SPY Ratio Weightrtr<N)i_regime_windowzRegime Filter Window (0=off)MVRV Macro RegimerJN)i_regime_entry_min_scorezRegime Filter Min Scorerxr<N)i_mvrv_suppress_bearzSuppress Entries in Bear Regimerxr@N)i_w_rsid_reg_bullzRSI Div Regular Bullish WeightRSI Divergence Signalsr<N)i_w_rsid_reg_bearzRSI Div Regular Bearish Weightr|r<N)i_w_rsid_hid_bullzRSI Div Hidden Bullish Weightr|r<N)i_w_rsid_hid_bearzRSI Div Hidden Bearish Weightr|r<N)i_w_rsid_rt_bullz RSI Div Real-Time Bullish Weightr|r<N)i_w_rsid_rt_bearz RSI Div Real-Time Bearish Weightr|r<N)i_w_rsid_slow_bullzRSI Div Slowing Bullish Weightr|r<N)i_w_rsid_slow_bearzRSI Div Slowing Bearish Weightr|r<N)i_w_rsid_delayed_peakzRSI Div Delayed Peak Weightr|r<N)i_w_rsid_delayed_dipzRSI Div Delayed Dip Weightr|r<N) i_div_windowz)RSI Div Window (bars signal stays active)r|rJN) i_w_oi_roczPerp OI ROC WeightDerivatives & Market Structurer<N) i_w_usdt_dzUSDT Dominance ROC Weightrr<N) i_w_basiszSpot-Perp Basis Weightrr<N)i_w_fear_greedz$Fear & Greed Index Weight (disabled)Sentiment & Sub-TF Signalsr<N) i_w_btc_goldzBTC/Gold Ratio Weightrr<N) i_w_rsi_subtfz%Sub-TF RSI Weight (half chart period)rr<N Calmar RatioN/A Sortino Ratio Sharpe Ratio Total P&L %Max Drawdown % Total TradesrrN.4f WFO_Score)Nrz WFO-selectedzIS-Composite-selectedrzH========================================================================u OPTIMIZED PARAMETERS —  — z [z] Composite: z | Calmar: z | Sortino: z | Sharpe: z WFO Score: z3 (mean OOS Calmar across 5 temporal folds) P&L: ,.1fz% | Max Drawdown: .2fz% | Trades: z1 NOTE: Metrics above cover the SCORING WINDOW ( → z) only.  c|dk(s|||} |dk(rt|jdvrdSdS|dk(rttt|St|dS#t$rt|cYSwxYw)z=Format a param value; fall back to default if val is missing.rr@)true1rfalserJg)strlowerrJr< Exception)val pine_typedefaults r,_fmtz#write_winner_readable.._fmts 5LCKW-@C F"!$S!1]!BvOOe#3uSz?++*Q( s8O s!!A A !A  A A76A7z --- z ---z (fixed Pine default) z<48 z<12 ()i_w_uI --- ⚠ Unlisted params (add to PARAM_META in write_winner_readable) ---r<optimization_winner_z_cheatsheet.txtr#u// === Optimized Defaults —  ===z // Calmar z | Sortino z | P&L ,.0fz% | DD z % | Trades z!// Metrics above: scoring window z+ only (TradingView shows full history from zH// Replace the first argument of each input.float/bool/int() call below:z// --- z // fixed Pine defaultz = input.(z, ...)u2// --- ⚠ Unlisted params (add to PARAM_META) ---z = input.float(u , ...) // ⚠ add to PARAM_METAz_pine_snippet.pinez >> [READABLE] Cheat-sheet: z >> [READABLE] Pine snippet: )getdatetimedatetoday isoformatr<rJrrappendkeys isinstancer startswithsortedospathjoinr(r1rprint)) winner_row strategy_name results_dirasset timeframe PARAM_METAcalmarsortinosharpepnlddtrades composite_dtdate_str composite_str wfo_scorewfo_strselection_labelasset_tfheaderrkeyr_param_meta_keyslines current_grouplabelgrouprrrsuffixval_strkunlistedcheatsheet_pathf pine_linescomment pine_paths) r,write_winner_readabler<sFJP~6F6G~6F}6C/6B~6F{6Ixx~~))+H1:e1CuY',M{D1I+4M+IeT]N^abNby!#&hmG(/5(8n>UO).%)%IH (&&.ZuXJc/ARS%mE&M#3FnUZ[bUcdgThhuv{}CwDEHvIIy!*T""6uRyo^TWX]^dXeTfSgh;;F-uYKW` ("    ,66:a:6HEM1;-UE9g M ! LL6%- .!MnnS%(.1UlckwOb)hjsIw/ r%Agc]#cU!F8DE2<&??,z,a 1c0Bq||TZG[`aiy`y,Hz ac(#C:>>#u5wEG LL2c#Ya}CuA> ?$ggll;2F}oUVW_V``o0pqO os #q  % 4'( $ ) %zF U6]3'{5>#2FgeTWjY]M^^efklnfopsettADEJKQERAS@T U +K=i[HsttAAB C R JM1;-UE9g M !   wd3 4!MnnS%(sIw/03u  QXQd+jlSIYyk7)6'ST2<NO(#C:>>#u5wEG   S  Aab c$ [,@qQYPZZl*mnI i   *%,-  *?*; <= +I; 78w7${ $ #:  s6 R>#S9S SS#$S 5$S SScdg}|jD]\}}||vrd|vr|jdd}d|gi||<)d|vs.d|vs3d|vs8 t|d}t|d}t|d}||kDr||}}|dkr|dk7r t|nd}|||z kDr ||k7r||z dz }||z d z }||kr|}|||c|d<|d<|d<|S#tt f$rYwxYw) z 0).rIstartrvaluesstopstep皙?g@g@@)itemsrr<abs ValueError TypeError) params locked_paramsrconfigrrrrmin_steps r,sanitize_paramsrs>,,M||~ V - Gv$5**Wa(C#cU+F3K  f 6!1f6F fWo.VF^,VF^,4<"&4E19(, 3t9sD4%<(Ud] 5LC/D!5LF2(?#DBGt?w=&B M *  s A>CC! C!c|ytd|jD];\}}||vs t||}d|vrd|vr||dkr||d<||dkDr||d<=y#ttf$rYPwxYw)z Ensures that the parameters from the winner_row are included in the search space defined by params. Modifies params in-place. Nz` >> [SAFEGUARD] Extending search space to include global best parameters (preventing drift)...rr)rrr<rr)rrrrbest_vals r,ensure_winner_in_search_spacers  lm||~ V *   C1f$6)9&/1*2w&.0)1v& *  s0AA0/A0ctjtdtjtdtjtdtjt dtjt dtjt}|jd|jd|jd|jd|jd|jdjDchc]}|d  }}ttttjzD]+}||vs|jd |d t!d |d -dD]0\}}||vs |jd |d|t!d |d 2|j#|j%ycc}w)z@Create the sweep results database and table if they don't exist.T)exist_oka CREATE TABLE IF NOT EXISTS sweep_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, asset TEXT NOT NULL, timeframe TEXT NOT NULL, schema_ver INTEGER NOT NULL DEFAULT 1, recorded_at TEXT NOT NULL, run_id TEXT, train_start TEXT, train_end TEXT, score_start TEXT, calmar_ratio REAL, sortino_ratio REAL, sharpe_ratio REAL, pnl_dd_ratio REAL, pnl_dd_percentile REAL, total_pnl_pct REAL, max_drawdown REAL, total_trades INTEGER, pct_in_market REAL, i_long_entry_activation_threshold REAL, i_long_exit_activation_threshold REAL, i_long_exit_activation_confirmation_threshold REAL, i_use_long_exit_confirmation INTEGER, i_use_long_entry_confirmation INTEGER, i_trailing_stop_threshold REAL, i_w_stoch REAL, i_w_macd_pred REAL, i_w_osc REAL, i_w_macd_bullish REAL, i_w_m3_momentum REAL, i_m3_momentum_period REAL, i_w_m2_tiny REAL, i_w_rsid_osc REAL, i_w_stoch_div_osc REAL, i_w_vwap_div_osc REAL, i_w_stoch_peaking REAL, i_w_stoch_bottoming REAL, i_w_m3_div_osc REAL, i_w_m2_div_osc REAL, i_w_m2_div_osc_noOffset REAL, i_w_bearish_engulfing REAL, i_w_bullish_hammer REAL, i_w_bullish_engulfing REAL, i_w_shooting_star REAL, i_cs_body_quality_ratio REAL, i_cs_confidence_scaling_factor REAL, i_cs_bull_eng_scaling_factor REAL, i_w_btc_spx_corr REAL, i_w_dxy REAL, i_w_vix REAL, i_w_btc_dom REAL, i_w_us10y REAL, i_w_spy REAL, i_w_gold REAL, i_w_mvrv REAL, i_w_mvrv_cont REAL, i_w_nupl REAL, i_w_fed_net_liq REAL, i_w_gc_position REAL, i_w_us2y REAL, i_w_yield_curve REAL, i_w_qqq_spy_ratio REAL, i_w_rsid_reg_bull REAL, i_w_rsid_reg_bear REAL, i_w_rsid_hid_bull REAL, i_w_rsid_hid_bear REAL, i_w_rsid_rt_bull REAL, i_w_rsid_rt_bear REAL, i_w_rsid_slow_bull REAL, i_w_rsid_slow_bear REAL, i_w_rsid_delayed_peak REAL, i_w_rsid_delayed_dip REAL, i_w_oi_roc REAL, i_w_usdt_d REAL, i_w_basis REAL, i_regime_window REAL, i_regime_entry_min_score REAL, i_mvrv_suppress_bear REAL, iteration_number INTEGER, gpu_score REAL, composite_score REAL, subperiod_consistent INTEGER -- NULL=unchecked, 0=failed, 1=passed ) zMCREATE INDEX IF NOT EXISTS idx_asset_tf ON sweep_results (asset, timeframe)zICREATE INDEX IF NOT EXISTS idx_pnl_dd ON sweep_results (pnl_dd_ratio)zCCREATE INDEX IF NOT EXISTS idx_run_id ON sweep_results (run_id)zGCREATE INDEX IF NOT EXISTS idx_schema_ver ON sweep_results (schema_ver)z PRAGMA table_info(sweep_results)z%ALTER TABLE sweep_results ADD COLUMN z REALz# >> [DB] Migrated: added column 'z' to sweep_results))iteration_numberINTEGER) gpu_scoreREAL)subperiod_consistentr)search_strategyTEXTrN)rmakedirsr r r r r sqlite3connect SWEEP_DB_FILEexecutefetchalllist _PARAM_COLS_METRIC_TO_COLrrcommitclose)connr existing_colscolcol_types r, init_sweep_dbr2sKK d+KK d+KK T*KK d+KK T* ??= )DLLTT j LL`aLL\]LLVWLLZ[$(<<0R#S#\#\#^_#^aQqT#^M_K 4(=(=(?#@@ m # LL@UK L 7urrwrIrzrVryrBr`r?rArWr=r:r>cddl}|Dcgc].}|jdddkDr|jdddkDr|0}}|sytdk(rdndtfd|D} t t | dz d} |D]0}|jd } | j | } d | z| z |d <2|jjjd } g}|D]}||t| |ttt|jd |||jdd|vr|jdrdndndd tjD](\}}|j|}| t|nd|<*t D]%}|j|}| t|nd|<'|j#t%|dj'}dj)dt |z}dj)|}d|d|d}|Dcgc]t+fd|D}} t-j.t0}|j3|||j5|j7t9dt |dycc}wcc}w#t:$r}t9d|Yd}~yd}~wwxYw)u Writes all CPU-verified results with at least 1 trade and Sortino > -5 to the sweep database. Filters on trades + Sortino rather than P&L/DD so high-drawdown regime runs (e.g. sideways training verified on full IS) are still captured. P&L/DD is stored as-is and can be 0 when drawdown exceeds the 40% cap — that is an analysis detail, not a write gate. Called after each iteration so no data is thrown away. rNrr$rc3BK|]}|jdyw)Nr).0rrank_keys r, z$write_to_sweep_db..s;(QAEE(C((rr Y@_pnl_dd_percentileseconds)timespec GPU_Scorer) rr schema_ver recorded_atrun_id train_start train_end score_startpnl_dd_percentilerrrr, ?zINSERT INTO sweep_results (z ) VALUES (rc3(K|] }| ywr0r8)r"crows r,r$z$write_to_sweep_db..s)DqCFDsz >> [DB] Saved z$ verified results to sweep database.z7 >> [DB] Warning: could not write to sweep database: )rrOPTIMIZATION_METRICrmaxlenindexnowr DB_SCHEMA_VERrrrr rr<r rrrrtuplerrr executemanyr r rr)verified_resultsrrr-rrrrpositivescoresnsidxr;rows_to_insertsrcrrcols placeholders col_namessqlr6rr er#s ` @r,write_to_sweep_dbrLs+V+a55+a/AEE.%4PST4T+HV  2[@{nH ;(; ;F CK!OQA  EE(C ll1o"'#+/  ,,    & & & :CN "'&"&!"';!< 0.{+(>!(C+,%%0F*GQQIM  ',,.HC%%*C%(_uSz$CH/C%%*C%(_uSz$CH c"/2 q!&&( )D99S3t9_-L $I ' {*\N! LC5C D^ce)D))^F DM}- f%   !#n"5!66Z[\mV^E M GsKLLMs$3JJ/A#J J>&J99J>c|\}}}}ddl}ddl}ddlm}ddlm} |j |j|} |j| fi|} |j| |} |j|} |j| | d| kj|} |j| | d| kDj|}| jdddk\xrC| jdddk\xr,|jdddk\xr|jdddk\| d <|j}|j| d |d <|S#t$rYywxYw) u3 Worker function for parallel CPU verification. Must be at module level so pickle can serialise it for ProcessPoolExecutor. On Linux (fork), df_data_bytes is inherited from the parent — no actual copy. Runs two extra calculate_metrics calls (subperiod halves) to check IS consistency. rN)rr0timerrrrTverified)iopandas%strategies.strategy_activation_scoresstrategy_activation_scoresrr read_pickleBytesIOgenerate_signalscalculate_metrics to_datetimecopyrupdater)args params_dict df_data_bytesr0optimization_metric_io_pd_strat_SPLITdf_datadf_resmetricssplit_tsm1m2results r, _verify_onerksDH@K -@:0//#++m"<=(((@K@**6{*K??6*  % % 6&>X- . 3 3 5;&P  % % 6&>X- . 3 3 56&K FF>5 )Q . W266/53QUV3V W FF>5 )Q . W3566/53QUV3V &' !!# g!z sD/E EEc8td|dtdtjj |s`td|tjj t r.tj t }tdt d|y tj|}|jryt|jvrtnd}|j|d j|}tj|} | jjj| _ d | jvrltj| d d j j#d| d <| d t$k\| d t&kz} | j(| j+} d} | | d tjt,k\} t/| dkDr@| d j0d} | d j0d}t3d|| z j4dz } t9j:}| j=||j?}|jADcgc]\}}|jC}}}|Dcgc]}||t,tf}}g}tjDxsd}t3dtGd|dz }td|dt/|dtI|5}|Dcic]}|jKtL||d}}tO|D]}|jQ}|||}|jSdd|d<|jSdd}|jSdd} tU|jSdd}!|jSdd}"tW|| |!| |"}#|jSdd sd}#|#|d <|jStd|d!<|jY| ddd|sy"|D$cgc]}$|$jSd ddkDs|$}%}$d}&|%rt[|%d#d $dt\}'td%t/|'d&t/|%d't/t^d(taj`}(|'Dcgc]}||t^tbtdf})}tI|5}tg|)D*cic]\}*}|jKth||*}+}*}tO|+D]<}|+|},|jQ\}-}.}/}0|-|'|,d)<|.|'|,d*<|/|'|,d+<|0|'|,d,<> dddtaj`|(z }1td-|1d.d/|1t/|'z d0zd1d2|'D$cic]}$tk|$|$}2}$|D]s}$tk|$|2vs|2tk|$jSd)d|$d)<|2tk|$jSd+d3|$d+<|2tk|$jSd,d|$d,<ut3|'d45}&tmd6|'D}3gd78d8jo8fd9tg|&jSd*gD}4td:|3d&t/|'d;|&jSd dd<d=|&jSd)dd<d>|&jSd+d?d<d@|&jSd,dA tdB|4tq|trtt|||Ctjv|}5|5jtd j0d}6tdD|6dd<dE|6d!d<|6|&fS#t6$rY> [VERIFY] Verifying Top z GPU results on CPU (sorted by )...z% >> [VERIFY] Sweep file not found: z >> [VERIFY] Files found in : NrF)by ascendingrOTutc @rr?v@z >> [VERIFY] Launching z parallel workers for z candidates...) max_workersr*r rrrrr CPU_ScoreNNc&|jddS)Nrrr!xs r,z$verify_top_results..s~q9Qr.)rreversez >> [WFO] Rescoring top /z2 eligible candidates (sorted by IS Calmar) across z folds...rWFO_Fold_Calmars WFO_Min_Fold WFO_Neg_Foldsz >> [WFO] Completed in .1fzs (.0fz ms/candidate)Xcl|jdd|jdd|jddfS)Nrrrr rr!r}s r,rz$verify_top_results..sE~u9UWXW\W\]hjmWnpqpupuwBDGqH9Ir.rc3LK|]}|jdddkDsdyw)rrrNr!)r"rs r,r$z%verify_top_results..s$ UKq155a;PST;TKs$$)2021202220232024H1PoliRunz | c3:K|]\}}|d|dyw)=z.3fNr8)r"ir5 fold_labelss r,r$z%verify_top_results..s,*8i1;q>"!Ac7+8isz >> [WFO] z( candidates pass WFO | WFO Winner: IS=r WFO_Mean=z WFO_Min=iz NegFolds=r3z >> [WFO] Fold Calmars: )rrz >> [VERIFY] IS Winner: GPU=z -> CPU=z >> [VERIFY] Error: )>* % 5j\BC 77>>+ &JJ{+E 2;-r%I JB;;z* >>$/BXEUEU.U*[i ''<5'INNuU++i(!//--335 W__ $ nnWV_$GJJVVW[\GFOFO{2wv)7STDkk$',,.G %gfo 9T&TUO?#a'$V,11!4$V,11"5sR"WNNV$;< zz|$  8@8I8I8KL8KfaCKKM8KLUefUePQ=+7JKUe fLLN'a !SIM23  )+6LSQaMbLccqrs [ 9XFQRkx{A6!F%zz/3?G"6::na#@AF%zz-=C 0&(TW XI!::&K$^48?K$_5 5> 99;0L -l3-?s)>PQ)RAo& &[/IJJ UK UUNGK$\\*8A*..QcegBh8i*N L 0#k2B1CD$$.NN;$B3#G{S]SaSabmopSqruRvw'^^NC@E[Q[Q_Q_`oqtQuPvx y .~.>? @*E9fWgzI Jll#34 #//3FRW/X]]^_`  .}[/I#.NhWdepWqruVvwxj((M   MfS: 96N f>=9<  &qc*+s!^ :C6^ 1A0]!A^ )]!^ ]'A^ ;]1!],!"]1B?]1 ^ ^ ]>.]>2A)^ ^5^ ^!^3A ^A^ ^^ /F!^  ]^ ]^ ,]11];6^ ^^ ^ _)^<<_c|\}}}}}ddl}ddl}ddl}ddlm}  |j |j |} g} |D]\} } }| | d|j|kj}| j|fi|}||d|j|kj}| j|| }t|jdd}||k\s| j|jddt| |krd| dt| fStt!| }t#d | D}t|j%| | ||fS#t&$r dgddfcYSwxYw) a$ Worker: evaluate one candidate across all WFO OOS folds. For each fold (is_end, oos_start, oos_end): - Generate signals on all data up to oos_end (IS warms up crossunder state). - Score only the OOS slice [oos_start, oos_end]. - Record OOS Calmar if OOS trade count >= min_oos_trades. Returns (wfo_score, fold_calmars) where wfo_score = mean(fold_calmars) when at least min_valid_folds qualify, otherwise (0.0, fold_calmars). Must be at module level so pickle can serialise it for ProcessPoolExecutor. rNrOrNrrrr c3,K|] }|dks dyw)rrNr8r"r5s r,r$z!_wfo_score_one..s7AQas )rQrRnumpyrSrTrUrVrYrZrWrXrJrrr9r<rrmeanr)r\r] df_full_bytesfoldsmin_oos_tradesmin_valid_foldsr`ra_nprbdf_fullris_end oos_startoos_end df_window df_signalsdf_oosrf oos_tradesrrs r,rrs~JNFK~:!//#++m"<= ,1 (VY3??73K KLQQSI000JkJJ 6 2coog6N NOTTVF..v9.MGW[[;> >L)*777SXXl+,lGWLL !Bq  !sCE"?E"AE""E43E4c tjjtd|dtdt d}tjj |sytd tj|}|jry|jd}|jtd}tj|}|jjj!|_d|jvrltj"|dd j$j'd|d<|dt(k\|dt*kz}|j,|j/}|j1}t3j4|j/fi|} t3j6| t8 } td k(r ||dtj"t8k\} t;d | djd | djdz j<dz } tA| jdd| jddtC| jdd| | jdd| d <| jtd} tE|| z dkDr`td|dd| dd| jGD]\}}||jHd|f<|jK|d|jd}ntdtd| ddtM||ttt | S#t>$rd} Y%wxYw#t>$r}td |Yd}~yd}~wwxYw)!z Checks if the winner file exists and if its recorded metrics match a CPU backtest. If there's a discrepancy, it updates the file with the correct CPU metrics. rr.csvNz6 >> [SANITY CHECK] Verifying existing winner file...rrOTrqrNrrurtrvrsrr rrrgư>z1 >> [FIX] Winner file score changed (Recorded: ru → CPU: z). Updating...Fr:z+ >> [SANITY CHECK] Winner file is valid (rnz).rrz7 >> [SANITY CHECK] Failed to verify/fix winner file: )'rrrr rrrrrrrrrr7rrrrYrrrrrrZrstrategy_modulerWrXrr8rrrrJrratto_csvr)rr winner_filedfr6recorded_scorerdrrrerf df_score_wis_yrs cpu_scorervrKs r,sanitize_winner_filers '',,{.B=/QRSXRYYZ[dZeei,jkK 77>>+ & BD-M [[ % 88Vggaj!4a8++i(!//--335 W__ $ nnWV_$GJJVVW[\GFOFO{2wv)7STDkk$',,.G 11',,.KFK!33F T + - $WV_{8S%ST S:f#5#:#:2#>FASAXAXYZA[#["a"adj"jk$4 NC0'++os2SGKK23VW[[X[=\$^GK KK 3Q7 ~ ) *T 1 EnUXEYYcdmnqcrsAB C 1ad ( IIkI /''!*C ?@S?TTVW`adVeegh i c=+UV_`)  , M GsKLL MsE#!MD5M;AL0DM0 L?;M>L??M M# MM#cddl}tjjtd|dt dt d}tjj|sy tj|jd}|j}tj|}|jjj|_ d|jvr6tj|ddj j#d|d<t%j&|j)fi|}tjt*}||d|kj)} t%j,| t. } t%j,|t0 } d|jvr!|djd j3d nd } d } | | \}}}}}}}}| | \}}}}}}}}|dk(rd}n||dk(rd}nt|dkr|dk\rd|ddn|dd}d|d|d}nS|dkrd|d}nG|dkrd|dd}n:|dkDr||z nd} | dk\rd| dd}n!| d k\rd!| dd}n| d"k\rd#| dd}nd$| dd}d%t d&t d'|jj5j3d(d)d*d+d,t.d-t*d.|d/d0|d1d0|dd.|dd.|dd.|d.|dd0|dd2d3t0d-| d.|d/d0|d1d0|dd.|dd.|dd.|d.|dd0|dd2d)d4|d)d5d6d7g }!tjjt6d8t dt d9}"t9|"d:5}#|#j;d;j|!d;zdddt=d<|"t=d=|dd>|d/d?|t=d@|dd>|d/d?|dA|j?dBdd y#1swYZxYw#t@$r}$t=dC|$Yd}$~$yd}$~$wwxYw)Du# Runs the current winner on the FULL data (training + OOS) and writes a side-by-side in-sample vs out-of-sample performance report to results/performance_report_{ASSET}_{TIMEFRAME}.md. In-sample : SCORE_START → TRAIN_END Out-of-sample : OOS_START → end of data rNrrrrOTrqrNrtz%Y-%m-%dr3cH|jdd}|jdd}|jdd}|jdd}|jdd}t|jdd}|jdd}|jd d}||||||||fS) Nrrrrrrrrr)rrJ) mrrsortsharpcalmtrmktpdds r,fmtz%write_performance_report..fmtPsEE-+CEE*A.BEE/1-DEE.!,EEE.!,Dna01BEE-+CEE.!,CD%r3; ;r.u+⚠ No in-sample trades — cannot evaluateu⚠ No OOS trades yet +r%u⚠ FEW OOS TRADES (u ) — P&L=u( (need ≥10 for Sortino to be reliable)gu) — Sortino not reliableu#❌ POOR — negative OOS Sortino (rrg?u!✅ EXCELLENT — OOS Sortino is z.0%z of in-sampleruu"✅ ACCEPTABLE — OOS Sortino is ?u⚠ WEAK — OOS Sortino is u❌ POOR — OOS Sortino is u# Performance Report — rz Generated: %Y-%m-%d %H:%M:%Srz\| Window | Period | P&L | Max DD | Sortino | Sharpe | Calmar | Trades | In Market | P&L/DD |z\|--------|--------|-----|--------|---------|--------|--------|--------|-----------|--------|z| **In-Sample** | rz | rz% | rz |z| **Out-of-Sample** | z **Verdict:** z---z_*In-sample metrics use the scoring window only (crossunder signals generated on full history).*z$*Generated by auto_optimize_loop.py*performance_report_z.mdr#ru# >> [OOS] Performance report → z >> [OOS] In-sample Sortino= P&L= % Trades=z >> [OOS] Out-sample Sortino=rrz3 >> [OOS] Failed to generate performance report: )!rrrrr rrrrrrrrrrrYrrrrWrZrrXrrstrftimer;r r(r1rsplitr)%rrrrr6rrr train_end_tsdf_is is_metrics oos_metricsoos_lastripiddisoishicaitriimipddopoddosooshocaotroimopddverdict directionratiorout_pathrrKs% r,write_performance_reportr$-s'',,{.B=/QRSXRYYZ[dZeei,jkK 77>>+ &RIkk+&++A.++i(!//--335 W__ $ nnWV_$GJJVVW[\GFO$55gllnOO ~~i0 :f-=>CCE$66u+V &77 PYZ DJgooD]76?''+44Z@cf <25Z.Cc3S$14[1A.Cc3S$ !8CG AX-G 2X)+q!Bs81 ClI,SEI;FnoG D[,SE1KLG 1W;C9AFG!$qC#IaE|=eC[ V#>uSkW$8s =Q8s =Q(wa { ;#,,**,556IJK L jj  U9+SD cRUYVZ[^_bZccfgjknfoorsvwzr{{~@CDDGHKLOGPPTUYZ]T^^` a$YKuXJc"T$sSViW[\_`c[ddghklogppstwx{s||AD@EEHILMPHQQUVZ[^U__a b G9 %  m2  77<< /B5'9+UX-YZ (C A GGDIIe$t+ ,! 3H:>? /Cyr$izRUQVWX /Cyr$izRUQVVYZaZgZghoZpqrZsYttuvw ! I CA3GHHIs2K;O $O;AO OO P)O<<P)gMbP?gMb`?g{Gzt?g{Gz?g{Gz?g?rg?rrurr2dic,ttfdS)z>Rounds a raw step to the nearest clean value from _NICE_STEPS.c ttjt|dtjtdz S)Ng-q=)rmathr)r8)rCraws r,rz_nice_step..s1#dhhs1e}.EQTUXZ_Q`Ha.a*br.r)r _NICE_STEPS)r/s`r, _nice_stepr1s { b ccr.cxd}d}d} t|d5}tj|}dddt |}|dkDr||z dznd}t d d t d t d t d t |t d t |t d|ddg} jD]\} } d| vsd| vsd| vrt| d} t| d} t| d}| | z }|dks|dkrMtdt||z }||kDs||ksp||z }t|}tdt||z }|| d<| j| ||||f| rt dt| d|d| D]1\} }}}}||kDrdnd}t d|d| dd|dd|d d!|d| 3t|d"5}tj||d#$dddt d%|nt d&|d'|d(t|d5}tj|}dddg}jD]r\} }d|vs d|vsd|vst|dt|dz }t|d}|dkDsF|dkDsL|jtdt||z t|rt d)t|d*t!|t|d+zd*t||Dcgc] }||ks | }}|Dcgc] }||kDs | }}|s|r%t d,t|d-t|d.nt d/|d'|d0t d d y#1swYTxYw#t$rYywxYw#1swYxYw#1swYnxYwcc}wcc}w)1z Reports parameter coverage, then auto-adjusts any params whose step is so fine (>50 values) or so coarse (<3 values) that random search will be ineffective. Target: ~20 values per param. Saves the file if changed. r'r)rrNrr*r&r<============================================================z PARAMETER COVERAGE ANALYSISz Search Space : z Total Samples: z Coverage : z.6frrrrrz Auto-adjusted z param(s) to ~z values each:FINECOARSEz [z] z<42rz>4u→z<4z values step r#rwindentz Saved adjusted params to z All params already in [u–u(] values range — no adjustment needed.z, Per-param value counts (min/median/max): z / r%u ⚠ z over-fine, z/ over-coarse after adjustment (check manually).u# ✓ All range-based params have z* values. Random search is well-calibrated.)r(jsonloadrcalculate_search_spacerformat_quantityrr<r8roundr1rr9dumprr) params_file total_samples TARGET_VALUES MAX_VALUES MIN_VALUESrr total_space coverage_pct adjustmentsrrrrrrng num_valuesraw_newnew_step new_countold_nold_snew_nnew_stag adj_paramsper_param_countscfgr5crampedbloateds r,analyze_params_coveragerUs MJJ +s #qYYq\F$ )5K:E/MK/#5uL Bvh- )+ VH ok:; <= om<= >? l3/q 12K||~ V & F&$8F&

fm# V%c'l(;;CV%DaxC!G ''AuS4Z/@(AB '  =%&'s62B+CCHXDY[\D\+]*^^abefvbwaxz {.A.!j.1.A.A.!j.1.A g GCL>c'l^Crs t 7 |3zlR|} ~ VHB-I$ # L$ # BAsW N M<N N#N% N2N2! N7,N7<NN NNN"%N/rC stoch_normrEmacd_pred_normrFosc_normrGmacd_bullish_normrHm3_momentum_normrK m2_tiny_normrL rsid_normrMstoch_div_normrN vwap_div_normrOstoch_peak_normrPstoch_bot_normrQ m3_div_normrTbearish_engulfing_scorerS m2_nooff_normrR m2_div_normrebtc_spx_corr_30rg dxy_roc_normvix_pctrank_invbtc_dom_roc_signus10y_roc_inv_signspy_above_200emagold_roc_pctrankmvrv_zscore_valuemvrv_zscore_contzscore nupl_normfed_net_liq_sign gc_position) rhrirjrkrlrmrorprqrrrXr_racddl}d}d}tjj|sytjj |}tjj|st dt dt dt dt d t d t t d t d t d t dt dytjj |}||kDrddlm}|j|jd}|j|jd}t dt dt dt dt d|t d|t t dt d t t dt d t dt dyy)z Warn if the Pine strategy file has been modified since it was last confirmed to compile in TradingView. Gives a 10-second countdown so the user can Ctrl+C to fix the Pine file first, preventing a wasted optimization run. rNz*strategies/strategy_activation_scores.pinez.pine_tv_validatedG ======================================================================u⚠️ PINE VALIDATION WARNINGF======================================================================z? The Pine strategy file has NEVER been confirmed to compile inz> TradingView. If it has compile errors, this optimization runz* will produce weights that can't be used.z? To dismiss this warning permanently, paste the Pine file intoz- TradingView, confirm it compiles, then run:z. python3 tools/check_pine.py --mark-validr)rrzC The Pine strategy file was modified AFTER the last TV validation.z Pine modified: z Last validated: z? If the edits introduced compile errors, this optimization runz& After verifying in TradingView, run:) rOrrrgetmtimer _countdownr fromtimestampr )_time pine_file sentinel_file pine_mtimesentinel_mtimerpine_dtsent_dts r,_check_pine_validation_sentinelrs >) $!!),J 77>>- ( o /0 h OQ NP :< c  OQ =? >@ h2WW%%m4NN",##J/889LM##N3<<=PQ o /0 h SU #G9-. #G9-. c  OQ :< c  68 >@ h2##r.cddl}ddl} t|ddD]L}|jj d|d|jj |j dN|jj d|jj y#t$rtd|jdYywxYw) z,Print a countdown, allowing Ctrl+C to abort.rNrtz Continuing in zs... (Ctrl+C to cancel) rz2 Continuing... z Run cancelled by user.) rOr%ranger&r1r2sleepKeyboardInterruptrexit)r(rx_sysrs r,rvrvDs w2&A KK   21#5NO P KK    KKN' QR   ,- ! sBB%CCc t|5}tj|}dddt t }Dcgc]}|jds||vs|}}|syt t dt dt t dt dt dt t|D]}t d |t t d t d t dt tjd y#1swYxYw#t$r}t d|Yd}~yd}~wwxYwcc}w) u Pre-run sync check: ensure every i_w_* param in the params JSON is also in config.WEIGHT_COLS. If any are missing, they will be silently skipped by the sweep dashboard and SHAP — wasting a long optimization run. This is a hard stop (non-interactive) because a missing entry in WEIGHT_COLS means results will be incomplete regardless of how long the run takes. Fix: add the missing param(s) to WEIGHT_COLS in config.py, then restart. Nz8 >> [WARN] Could not read params file for sync check: rrtuM >> [ERROR] PARAMS/CONFIG OUT OF SYNC — aborting before wasting run time.z? The following i_w_* params are in the params JSON but NOT inzA config.WEIGHT_COLS. Results for these signals will be silentlyz@ missing from the sweep dashboard and SHAP importance reports. zI Fix: add the param(s) above to WEIGHT_COLS in config.py, then restart.uH The DB migration runs automatically — no manual ALTER TABLE needed.r) r(r8r9rrsetrrrr%r)r>rrrKweight_cols_setrmissings r,validate_params_syncrTs + !YYq\F +&O V&QALL$8Qo=Uq&GV  G (O YZ G KL MN LM G G_ qcl G UV TU (O GHHQK7   HLM Ws> D-D!D-EEE!D*&D-- E6E  Ec (ddl}tjjtjj tjj tjj t ddd}t|j|}|syttdhz}g}|D]s} t|5}tj|}ddd|Dcgc] }|vs| } }| sD|jtjj|| fu|syt!t!dt!dt!t!d t!d t!d t!|D])\} } t!d | d | D]}t!d|+t!t!dt!dt!t#j$dy#1swYxYw#t$rYUwxYwcc}w)aq Hard stop if ANY of the 20 active params files is missing a key from WEIGHT_COLS or i_div_window. Catches the case where config.py was updated but sync_params.py wasn't run, which would cause SHAP/dashboard to report MISSING FROM SOME PARAMS and silently treat those weights as zero in affected runs. Fix: python3 tools/sync_params.py --apply rN strategiesrz*params_strategy_activation_scores_*_*.jsonrrtuL >> [ERROR] PARAMS FILES OUT OF SYNC — aborting before wasting run time.zA The following active params files are missing keys that are inzD config.WEIGHT_COLS (or i_div_window). SHAP will show MISSING FROMz@ SOME PARAMS warnings and those weights will be silently zero.r:z z, Fix: python3 tools/sync_params.py --applyr)globrrrdirnameabspath__file__rrrr(r8r9rrbasenamerr%r) _globpatternrexpected bad_filesrrrrrfnames r,_validate_all_params_syncr}sggll277??277??277??8;T+UV'3_aG 5::g& 'E ;>"22HI dqIIaL'5h!1*1h5    bgg..t4g> ?  G (O XY G MN PQ LM G#w ugQ A HQC. !$ G 89 (O GHHQK3   5s6: G?G2G?' H1H2G< 7G?? H  H c t|5}tj|}dddtj|d}t d|j D g}D]}|jdstj|}|-||} t| tr"| jd} | t| dk(rdt fd |Dry|j!||f|syttd td ttd td tdt|D]0\}} td|tddj#| 2ttdtdtd tt$j&j)rbt+dj-j/} | dk7r tdt%j0dtdtytdty#1swY'xYw#t$r}td|Yd}~yd}~wwxYw)a Check that every i_w_* param in the params file has a corresponding column in the data CSV. If any are missing, the optimizer silently treats them as all-zero (wasted search budget). Warn the user and prompt to continue or quit. Normal action on warning: quit, re-export TV chart data with the missing columns included in data/, then restart the optimization run. Nr)nrowsc3XK|]"}|jj$ywr0)rstriprs r,r$z(validate_data_columns..sB/Qqwwy(/s(*z. >> [WARN] Could not validate data columns: rrrc3BK|]}|jvywr0)r)r"r5csv_colss r,r$z(validate_data_columns..s=*Q1779(*r%rtzE >> [WARNING] Missing CSV columns for weight params in params file.zD The following params are in the search space but have NO matchingzE column in the data CSV. The optimizer will treat them as all-zero,z7 wasting search budget on params that have no effect.rz expected column(s): r2zB Normal action: quit now, re-export the TV chart data to includez> the missing indicator columns, save to data/, then restart.z Continue anyway? [y/N]: yz4 Aborting. Update your TV data export and restart.z3 Continuing with missing columns treated as zero.z\ [NON-INTERACTIVE] Continuing automatically (batch mode). Missing columns treated as zero.)r(r8r9rrrrrrr_WEIGHT_TO_CSV_COLSrrdictr9anyrrr%stdinisattyinputrrr)r>rrrdf_headrKrr candidatespdefvalsrGanswerrs @r,validate_data_columnsrs + !YYq\F++iq1B'//BB G~~f% (,,S1   c{ dD !88H%DCIN=*== NNC, -  G (O QR G PQ QR CD G T sen ,TYYt_,=>? G NO JK (O G yy45;;=CCE S= H I HHQK CD G lm Gg   >qcBCs- II ;I II I7I22I7cv t|d5}tj|}dddd}jD]]\}}d|vr>d|vr:d|vr6|d|d|d}}}|dkr't dt ||z |z } || z}Hd|vsM|t |dz}_|S#1swY}xYw#t$rYywxYw) zGCalculates the approximate size of the search space from the JSON file.rNrrrrrr)r(r8r9rr8rJr9r) r>rrtotalrrrrrcounts r,r:r:s +s #qYYq\F$!<<>KC& Vv%5&F:J$*7OVF^VF^Tt19hAsD5LD#89:V#VH-..* $ # s. B,B A&B, B, B)%B,, B87B8c|dk\r |dz ddS|dk\r |dz ddS|dk\r |dz ddS|dk\r |dz dd S|d k\r |d z dd S|d k\r |d z dd S|dk\r |dz ddS|dS)z6Formats numbers with suffixes (million, billion, etc).lB3v^!<rz septillionl@=7M.cz sextillionlNZoz quintillionlI5z quadrillionlJ)z trillioniʚ;z billion@Brz million,r8)rBs r,r;r;s --55c:+FF + +11#6kBB ' '--c2,?? # #))#.l;;  %%c*)44 m M/#&h// iI+c"(++Ar.c|dkrd|ddSt|dz}t|dz}|dkr d|d|dSt|dz}|dz}d|d|dS)z7Formats seconds into a friendly string (e.g. '6m 20s').<~rrCzm zh r)rJ)r(rrChs r,format_durationrs{|73-q!! GrMA GbLA2v1#Rs!} AG A BA qcA3a=r.c tt}|dkDr t||n|}|dkDr||krdnd}|dk(rt|tz nd}t d|dt |dt |dt d |d d d d tdtdtdtdtdt|ddtd|dtdtdttdtt g}t ddj#|t$j&j)}t$j*j-dr-|j/dd}d|vr|r|dnd|d<t d t1j2|d!|t$j4t0j6t0j8d"#} d$} t;j<| | fd!%} | j?d&} | jA| '} | j#| dk7r$t d(| d)tCjDd"y#t0jF$rt d*| d+z d,t d-dd.l$}t%jJt%jL| jN|jPtSjTd/t%jJt%jL| jN|jV| j#d0't d1YywxYw)2z Runs the existing optimize_strategy.py script using Popen for better process control. Includes a timeout and forceful termination to prevent hangs. rGridRandomz N/A (Grid)z >> Running z search (Space: z | Sample: rmz >> Estimated Time: z (GPU)python3zstrategies/optimize_strategy.py--dataz--strategy_filez --params_filez--assetz --timeframez--random_searchz--gpuz--metricz--baseline-score=--search--regime--regime-bear-threshold--regime-bull-thresholdz >> Executing: rz/usr/lib/wsl/libLD_LIBRARY_PATHrz:/usr/lib/wsl/libzQ >> [WSL CONFIG] Injected /usr/lib/wsl/lib into LD_LIBRARY_PATH for GPU access.Tr)textenv preexec_fnr&stderrbufsizecB|jD]}t|ddy)NrT)endr2)r&r)proclines r,_streamz!run_optimization.._streamGsKKD $Bd + r.)targetr\daemoni)timeoutz3 >> [ERROR] Optimization script exited with code .z4 >> [ERROR] Optimization process timed out after rz minutes.z- >> Forcefully terminating process group...Nr%r&zT >> Process terminated. The loop will continue, but the underlying issue persists.),r: PARAMS_FILErrGPU_RATErr; DATA_FILE STRATEGY_FILErrrr7 SEARCH_MODEREGIMEREGIME_BEAR_THRESHOLDREGIME_BULL_THRESHOLDrrenvironrZrrr subprocessPopensetsidPIPESTDOUT threadingThreadrwaitr%rTimeoutExpiredsignalkillpggetpgidpidSIGTERMrOrSIGKILL)iteration_samplesbaseline_scorerCeffective_iterations search_method est_time_strcmdr current_ldprocessr stream_threadtimeout_seconds return_coders r,run_optimizationrs )5KBMPQ/3{,=>Wh)Ao+AU2UF[cMGTX`G`?#7(#BCfrL N=/)9/+:V9WWbcrtHdIcJJN OP "<. 78 4)=5y301' N+,KF!3'<#=!3'<#= C" chhsm_ -. **// C ww~~()WW.3 Z /IS |3D%EYkC! " e fs3299&0ooj>O>OYZ\G,$$G7*TRMOfll?l;  !  G }TUV W HHQK   $ $ f EoXZFZE[[def => "**W[[)6>>: 1  "**W[[)6>>:1% de  fsA H**C L  L c Ftjddjdd}tjj t d|dt dtd}tjj|r$tj|}|jdStdtjj t d |dt dtd}tjj|r"tj|jdSy ) zIReads the best result from the CURRENT run (optimization_run_best_*.csv)..pyr strategy_optimization_run_best_rrrzr >> [INFO] 'Run Best' file not found (Run likely produced no trades/improvement). Falling back to Global Winner.rN) rreplacerrrr rrrrrrr)r run_best_filerrs r,get_run_best_resultrfs!))%4<<["MMGGLL0F}oUVW\V]]^_h^iim.noM ww~~m$ [[ 'wwqz C Eggll;2F}oUVW\V]]^_h^iim0no 77>>+ &;;{+003 3r.c0tjdtjd}|jdtdd|jdt d |jd d d d |jdddd|jdddd|jdtdddgd|jdtdd|jdtdgd d!|jd"t t d#t d$|jd%t td&td'|j}|jattjt}t|jd(k(rS|jd)j!d(}|jd*|j#t%d+|d,nt|jd-k(r|jd(|jd.j'}}|jd/||fj!d(}|jd0||f|j#t%d1|d2|d3|d4n t%d5|j)t+j,d(|j.a|j2a|j6a|j:a|j>r |j>a d6d7d8d9d:d;}tBjDjGtBjDjIt@d(}d<|vr9|jKdtNd?tPd@a)|jTr|jTa)t%dAtRtBjDjWtRsd>tPd@} dB} tBjDjW| r| n"tBjDjW| r| nd} | r1d(dl,}|j[| tRt%dCtRdD| d4n(t%dEtRdFt+j,d.t%dGt@dHtNdItPdJtRt]tBjDj_t`dKtNd?tPdLt*_1|jdr|jddMz}dN}tgdOti||z a5|tjz }tm|tnz}tgdP|a8t%dQt%dR|jddS|dTdUt%dVtjdW|dXz dTdYt%dZtpd[d\tptjzd[d]t4dk7r!t%d^t4d_t8d`t<datstutRtptjztwtRtyt{tRt@t%dbtjdct%ddt|t%detRt%dft@tt tj} tjtjd/tNtPfj!d(}t|jdgdhjdidh}t|t@}tBjDj_tdj|d?tNd?tPdk}tBjDjW|r|d(k(rt%dlt%dmt%dnt%dotNd3tPdpt%dqt%drt%dst%t%dtt%dut%dvt%t%dwt%dxt%dyt%dntdzg}t d{ }d}d|}t d{ }d}d}}t d{ }tj} ||}t%d~td|dnt|jdgdhjdidh}!tBjDj_tdj|!d?tNd?tPdk}"tBjDjW|"rZ tj|"}#|#jd(jMtt d{ }t%dtd|ddtBjDj_tdj|d?tNd?tPdk}$tBjDjW|$r tj|$jd(}t |jMdd|}|jMdd}%|%2t|%t r|%|%k7st |%}d}t%d|ddtd.tjd.zD ]}&t%d|&dtjd|jdrJtj| z }'tgd(|jddMz|'z }(t%dt|(t|jdgdhjdidh}!tBjDj_td|!d?tNd?tPdk})tBjDjW|)rtCj|)tj}*ttp|}+tj|*z },|,d(kDr8ttRd(kDrtttRtpntp|,z nd(}-t%dt|,d|-dz dd|-dkr|+rt%dt%|+ddt|jdgdhjdidh}!tBjDj_td|!d?tNd?tPdk}.t|.t@d||&|j.\}/}0d}}1|0L|0jMdd}2|0jMdd|}3|2|kDr"|2}|0}tBjDj_tdj|!d?tNd?tPd}4tj|0gj|4d}t%d|2dd|3dd|0jMdd(dd]|2d(k\rtBjDj_tdj|!d?tNd?tPdk}5tj|0gj|5d}t|0|!ttNtP|0}d}t%dn t%d|/t}/|/{t%d tjt@}6|6jjj|6_Xd|6jvrltj|6ddjjd|6d<|6dtk\|6dtkz}7|6j|7j[}6|/j}8tj|6j[fi|8}9tj|9t}:t%dtd3|/jMtd(dd|:jMtd(d|:jD] \};}<|<|/|;< d}1nd}1|/t%dn|/jMtd(}>tm|/jMdd(}?|/jMdd(}@|/jMdd(}A|/jMdd|}B|/jMdd|}C|/jMdd|}D|/jMdd|}Etdk(r(t%d|>ddDddBdd@ddAdd|?dEddn.t%dtd|>ddBddCdd@ddAdd|?dEdd|>|kDxr|>d(kD}F|>|k(xr|&d.kD}Gd}}H|0|0jMdd|nd|}I|Id(kDr|0n|/}JId(kDrdnd}KtBjDj_tdj|!d?tNd?tPdk}5FrWJKdk(r|0|0jMddnd}L|xrKdk(xr LduxrL|k}H|HrZt%dLdd|dd|tj|gj|5d}t||!ttNtPnCtjJgj|5d}t|J|!ttNtPHs%tg|t JjMdd|}|sJ}|1rGHr$t%dtd|dd|>ddLdd|dd] nt%dtd|dd|>ddKd̝ |>}nct%d|>ddΝnR|Ctj|gj|5d}t||!ttNtPGr t%dϫ|?d(k(r|+rt%dЫt%|+ddFxrH }M|MrdnFrdndh}N|/|/jMddnd}O|OdOdndh}Ptdk(r6t%d|&dtjd|>ddDddBdd@dڛdAdd|?d|dPNdޝn8t%d|&dtjdtd|>ddBdd@dڛdAdd|?d|dPNdޝ|jstjt*jdgd}tjt*jddgd}|jr(tjt*jddgd}tjt*jdgd}tjt*jdgd}|jd|&dtd|>d|?|>}|&tjk(rt%d|&dnttRd5}Qtj|Q}RdddtR}S|S|Rk7r>t%dS}RttRd5}QtjR|Qdddd|&d.kDrtjdtjd tj| z }T tjtjd/tNtPfj!d(}Utjtjd)j!d(}VU|z }Wt%dt%dt%d|D]}Xt%d|Xt%dtd|d|$t%d|dd|jMdd(dd]t%dtTt%dWd3tNd3tPdVd[d] t%dt%d|js'tjt*jdgd}t|jdgdhjdidh}Yt|Yt@|js'tjt*jdgd}|js'tjt*jdgd} dtNd3tPd|ddtTd] }Ztjddd|Zdgdzd}y#t$rd(}YwxYw#t$rYJwxYw#t$rY wxYw#t$r}=t%d|=Yd}=~=|d}=~=wwxYw#1swYNxYw#1swY xYw#t$r|}Ud(}VYYwxYw#t$rYywxYw(Nz;Auto-Optimize Strategy (iterative GPU random/Optuna search)u Arguments: --data PATH Path to OHLCV data CSV. Asset and TF extracted from filename. --hours N Total runtime budget. Divided evenly across ~15-min iterations. --search MODE 'random' (default) or 'optuna' (DB-history biased sampling + importance report). Optuna biased sampling works from iteration 1 (requires prior DB rows). Importance report (results/reports/optuna_importance_*.md) requires ~3h+ to appear because it needs ≥30 CPU-verified results from a single iteration. --regime MODE Filter IS bars by MVRV regime: all (default) / bull / bear / sideways. --params PATH Override the params JSON file (default: derived from --data). --no-dashboard Skip sweep dashboard regeneration after each iteration. --reset-db [ASSET TF] Delete DB rows then exit. No args = wipe all. Examples: # First run on a new combo — always start with random python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 2 # After ≥1 random run — switch to Optuna for DB-biased sampling # Use ≥4h if you want the importance report (results/reports/) to appear python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4 --search optuna # Optuna with LLM-guided range narrowing (only ≥6h runs) python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-1D.csv --hours 8 --search optuna --llm # Bull-regime only (filter IS training bars to MVRV bull phase) python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4 --regime bull # Wipe one combo's DB rows and restart python3 tools/auto_optimize_loop.py --reset-db COINBASE_BTCUSD 4H python3 tools/auto_optimize_loop.py --data data/COINBASE_BTCUSD-4H.csv --hours 4 # Usually called via batch runners rather than directly: python3 run_btc.py --hours 10 --tfs 4H 6H 8H 12H 1D python3 run_btc.py --hours 10 --search optuna python3 run_marathon.py --hours 96 --hours-per-combo 1 ) descriptionformatter_classepilogrzkPath to the OHLCV data CSV (e.g. results/COINBASE_BTCUSD-1D.csv). Timeframe is extracted from the filename.)typerhelpz--hoursz:Target runtime in hours (scales sample size automatically))rrz --reset-db*ASSET_TFzDelete sweep DB rows then exit. No args: wipe all. Two args (ASSET TIMEFRAME): wipe that combo. E.g.: --reset-db COINBASE_BTCUSD 4H)nargsmetavarrz--no-dashboard no_dashboard store_truezMSkip regenerating the sweep dashboard (mine_sweep_db.py) after each iteration)destactionrz --tightentightenuAuto-apply q5-q95 range tightening (tighten_params.py --apply) after each iteration. Off by default — only enable after several runs have built up sufficient DB coverage.rroptunazSearch strategy passed to optimize_strategy.py: 'random' (default) or 'optuna' (biased sampling from DB history + importance analysis). Optuna mode requires prior DB results to be effective.)rrchoicesrz--paramszOverride the params JSON file (default: derived from --data filename). Used by run_phased.py to pass phase-specific param files.rr)rbullbearsidewayszPre-filter IS training bars by MVRV regime before optimisation. 'bull': zscore >= bull_threshold; 'bear': zscore < bear_threshold; 'sideways': bear_threshold <= zscore < bull_threshold; 'all': no filter (default).rz6zscore percentile below which bars are Bear (default: z<). Historically: 2022 bear median=6.6, 2018 bear median=9.9.rz> [CONFIG] Params override: z;strategies/params/params_strategy_activation_scores_1D.jsonz- >> [CONFIG] Created asset-specific params z from template z% >> [ERROR] Params file not found: z (and no template found)z >> [CONFIG] Data: z | Asset: z | Timeframe: z | Params: optimization_log_z.txtiirrz"--- Time-Based Scaling Enabled ---zTarget Runtime : z hours (rzs)zIterations : z (~rz min each)zSamples/iter : rz (Total: rz >> [REGIME] Training on 'z' bars only (bear_thr=z , bull_thr=z.). Filter applied inside optimizer subprocess.z%--- Starting Auto-Optimization Loop (z Iterations) ---zTarget: zParams: z Data Source: rrrrrrsu'⚠️ STALE WINNER / EMPTY DB WARNINGrtz A winner file exists for z but the sweep DB hasz, 0 rows for this combo. This usually means:z> a) The DB was reset/deleted after the winner was found, ORzL b) The scoring formula or simulation changed since the winner was saved.zG Risk: the winner's score will be used as the GPU pre-filter baseline.zG If it's inflated relative to the current simulation, nothing new willu= ever beat it and no new results will be saved — silently.z1 Recommended action (if scoring or sim changed):zJ python3 tools/reset_results.py --apply # wipe winner + start freshzE Or if the winner is still valid, continue and it will self-correct.rinfr Fz >> [INIT] Baseline z (CPU-verified): rz$ >> [INIT] Loaded historical best rnz (unverified)rrTz1 >> [INIT] Loaded existing winner WFO_Min_Fold=z as baseline (WFO guard active)z === Iteration rrz >> Global Time Remaining: rz >> Actual Time: z | Rate: rrz k samples/si@ zQ >> [WARNING] Low sampling rate detected. Dumping optimizer logs for debugging:i0optimization_sweep_)rr-rrrrz_wfo.csvru/ >> [WFO] ★ New global WFO winner (WFO_Min=rz IS=rru< >> [WFO] ★ Promoted to main winner (all folds positive)uN >> [WFO] ★ Saved to _wfo.csv only (WFO_Min<0, keeping main winner intact)zF >> [INFO] Sweep file missing. Verifying 'Run Best' result on CPU...rOrqrNz >> [VERIFY] Run Best: z -> z) >> [FALLBACK] CPU calculation failed: z4 >> [WARNING] No valid results found for this run.rrrrrrz >> Best Composite: z | Calmar: z | Sortino: z | P&L: z% | DD: rz % | Trades: z | In Market: rrz >> Best z | Sharpe: WFOz IS-Compositez0 >> [WFO] IS global best improved but WFO_Min=z < existing promoted best z . Restoring superior WFO winner.z >> [INFO] IS z improved (rz6) but WFO guard kept existing winner on disk (WFO_Min=z < z) >> [SUCCESS] New VERIFIED global best z found (Old: z, New: z) [z winner saved]z >> [CAUTION] New best found (zH) but NOT VERIFIED. Will NOT update global baseline to avoid corruption.z4 >> [STAGNATION] Result identical to previous run.z> >> [WARNING] Zero trades detected. Optimizer Output (tail):iu ★ NEW BESTu! ↑ IS improved (WFO guard held)z WFO=z Iteration Summary z : Composite=z Calmar=z Sortino=rrz% DD=rz Best=rrzmine_sweep_db.py)checkztools/lock_params.pyz--applyztools/tighten_params.pyztools/oos_dashboard.pyz tools/shap_feature_importance.pyzIteration Summary z , Trades=z >> [INFO] Final iteration z complete.rzE >> [INIT] Sanitized initial parameter ranges (fixed steps/bounds).r#rwr6r&z= ============================================================z OPTIMIZATION RUN COMPLETEr3rz Global Best IS z Global Best WFO Score : z (IS=z Total Elapsed Time: z DB Rows Added : +z (total DB: z/ Final results are in the 'results/' directory.u✅ z optimization done! Best P&L/DD=z (curlz-sz-dzntfy.sh/jlo_alerts)rr)sargparseArgumentParserRawDescriptionHelpFormatter add_argumentrr<_MVRV_BEAR_DEFAULT_MVRV_BULL_DEFAULT parse_argsreset_dbrrrrr9rfetchoner rupperr r%rsearchrregimerregime_bear_thresholdrregime_bull_thresholdrdatarrrsplitextrrsplitrrrrrrshutilrZr!rr r&hoursr8r<MAX_ITERATIONSrJrSEARCH_ITERATIONSrrUrrrruuiduuid4rrrr rvrOr7rrrrrrremoverr:rr rrrrrrrrYrrrrrrrrWrXrrrrrun executabler rr(r8r9rr=rr$)[parserr\r rB asset_argtf_arg _MIN_TO_TFstem _asset_part_periodparts tf_template gen_templatetemplater0 total_secondsTARGET_SECS_PER_ITERseconds_per_iterscaled_iterationsr-_db_rows_beforestrategy_name_clean verified_best_winner_file_checkhistoryglobal_best_scoreglobal_best_rowglobal_best_calmarglobal_wfo_best_scoreglobal_wfo_best_rowglobal_wfo_promotedprevious_scorestart_loop_timerhistorical_winner_filehist_df_init_winner_file _init_wfo_minr elapsed_totalremaining_totalr iter_start opt_output iter_elapsedraterbest_rowwfo_winner_this_iter is_verified wfo_min_this wfo_mean_thiswfo_winner_filewinner_csv_pathdf_data_fallbackrrrerfrvaluerKscorerrrrrrris_new_global_best is_stagnant _skip_writewfo_score_this_iterr winner_source_wfo_min_this_iter_winner_file_updated best_markerwfo_score_display wfo_suffixrcurrent_paramssanitized_params total_elapsed_db_rows_after _db_total _db_addedentrystrategy_name_finalmsgs[ r,mainr|vsc  $ $Q <<#'FP sD@mn  4pq  CCD (~lln  )Lwx  hS[H\VW  dYZ  eEhrs 1GYUVhUijYYZ 1GY[\n[opIIJ    D }} }- t}}  " ABKKMaPA LL4 5 KKM 1!OD E  1 $ $ a 0$--2B2H2H2JvI RF#%%-XZ3A LLR#V, . KKM HQCz)AfXQ? @ E F   ++K [[F 66 66 yyII d4TRJ 77  BGG,,Y7 8 ;D t|#{{43 W& w (Py C# u:?$Qxq)9 E9#T E9HqQZP[[`aK {{kk  0 >? 77>>+ &Mi[X]^ T "$''.."=;TVT[T[TbTbcoTpLvz   KK+ . A+o^f]gghi j 9+F^_ ` HHQK !)Jug^I;Vabman op Z3DUG1YKW[1\]^CJ zz T) &Qm6J&J KL(>9 08 ;< +<= 24 !$**XmC5HKL !.!16Fr6I#5NjYZ !"3A!6jARUcAcde@ffghi ,VH501=R$%K):^)KL%+y1 1.1AAQ RS H]O $% H[M "# M) %&O  F!//-8@@ N I  ( 1. (//r:BB;PRS()STU <> NP \^ c  WY WY MO c  AC Z\ UW h2Gu O"5\MEl]NiikO  ) &':&;;LM^_bLcde%--eR8@@bQ !#k=QR_Q``abgahhijsittx;y!z 77>>0 1 ++&<=$+LLO$7$78KeTYl]$[!<=P3'O!P ,//EM(*]E2RWdhuWu(-m(<%&*#IJ_`cIdeDEF1nq( )  1^$4D9: :: IIK/9M!!djj4&7=%HIO 1//2R1ST U%--eR8@@bQ  [4J=/YZ[`Zaabclbmmq2rs 77>>- ( IIm $YY[ &&79KL yy{Z/ bnqrbrOefqOruvOv*;79JK}NR^^xy #OL$A#B)DQUIVZK[[fgh &=Z e f *UV$ %%--eR8@@bQ WW\\*0CM?RSTYSZZ[\e[ffj.kl );J Y]fl@ASWS^S^*_&&  +/33NEJL044[#FM33(4%&:#"$'',,{>RS`Raabchbiijktjuu}<~" 234;;OSX;YG UXGYYdersvdww|~R~V~VWbde~fgj}kklmn 1$&(ggll;BVWdVeefglfmmnoxnyy}@~&OLL"6!78??W\?])*> {bgs|}&:O*.'XZjl  *,H#^_K'){{9'=$/?/G/G/K/K/Q/Q/S$,!1!9!9935>>BRSYBZ`d3e3h3h3t3tuy3z(0 0 8K GL\]cLdhqLqr+;+?+?+E+J+J+L(%--/F,==>N>S>S>U`Y_`F-??T_`G56I5J!HLLYlmnLopsKttxzAzEzEFY[\z]^aybcd&-mmo U(- '6"&KK   H I  0!4X\\.!45ll=!, \\*A .,,4nc2nc2 ]C8 + - *5+[ LY`adXeemnqrvmwwACDG@HHTU[T\\jkxy|j}}~ @ K 34BuSkgVY]Zeflmpeqqyz}CzDDLMOPSLTT`ag`hhvwDEHvIIJK L"%66D519.;QU  MaLl266{CHru-@1-D)( !4q!8n '',,{6J=/YZ[`Zaabclbmmq4rs %*7%)?DXDd';&>&>~u&U+/# 3 N}7M N#5T#A N#58M#MLM_`cLde44I#3NNnpq '2 o%67>>V[>\-o}kafr{|LL*.55oU5S)*m[X]irs" *--?z~~VdfiGjAk)l& +&0O,-@,AM^_bLcchinorhstjk}~AjBBEF[\_E``abcEFYEZZghyz}g~FGLMPFQQTUbTccqrs$)!8s DLMN * o./66e6T%o}kY^jstLN Q;: R S *UV$ % 2E+o)=ok}DgDF ?G?SHLLd;Y]9J9Vv/45\^ + - *1#Q~.>l5QT+U^_efi^jjtu|~AuBBHILMQHRRXY[\_X``jkqjrryzKLOyPQ[P\]h\iikl m *1#Q~.>bAT@UUVW\]`Vaaklstwkxx~@CDHIIOPRSVOWWabhaiipqBCFpGHRGST_S``bc d  NNCNN,>?u M NNCNN,BINV[ \||0I9U]bc NNCNN,DEU S NNCNN,NOW\ ]+A3b1D0EQugYW]V^_`   1!J? @  +s #q!YYq\N$+>: ~ - Y Z-Nk3'1 .!A6( q5 JJqM 1 u*xIIK/1M 7?? N I  ( 1.OOM2:: 022:(*Q@ 0I - '( &M 5'l  34B7H6M NO& ,-B3,G*..{A>sC1F G "?=#A"B CD $YKqq =QZ[\P]]^ _` &M <=   (:;5I'//r:BB;PRS0)<   (@AO     ^^? @  5'9+&,S12 ./q 2  T4&: ;e k f  "   d!KEaSIJJK|$ #(' ( ^    s"AA_AA_0BA`E,A`SA`5TAaV A=Aa^AAa#_ A_-_,A_-_0 A_=_<A_=` A` ` A` ` A`2`A`-`-A`2`5A`? aAa a Aa aAa a# Aa/a.Aa/__main__)rrr{)rNNN)r )^__doc__rr8rrRrr%rrOr.rrQrr4concurrent.futuresrrrinsertrrrrSrTrrrrrrr r r r r rr#rr$rrrrrrrrrr7rrrrr3r2rrrrrr<robjectr!rrrrr _NON_WEIGHT_PARAM_COLSrr rLrkrrrr$r0r1rUrrrvrrrr:r;rrrr|r5r8r.r,rs9      @277??277??277??83L#MNO?YYYYY0 ! /   DUG1YKW\]  **+   V u9p&P,pj%%$$%$%$'!   +!77 BMJ"JPd*!Z9v`IF d Oh  ~  01  |  34   23  /  }  01 0  12  01    9: 0     12! "/# $!22 23 45 23 23 34 2H= +} 23 -###? F0f  &R,^= @$& GR I V zFr.