
    9j]                     F   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
 dZdZdZdZdZg d	ZeD  ci c]  } | j#                  d
      d   |  c} Zg dZddddddZeaddZd Zd Zd Zd Zd Zd Zd Zd Zd Zd dZ d Z!e"dk(  r e!        yyc c} w )!u  
run_marathon.py — Autonomous multi-cycle optimizer for unattended runs.

Alternates random → optuna search cycles across all 20 combos, applying
sign-stability locks between cycles and sending ntfy progress notifications
at each milestone.

Cycle pattern:
  odd  cycles → random search  (broad exploration)
  even cycles → optuna search  (exploit DB history)
  After every random cycle → lock_params --apply --cross-asset
  After every cycle → mine_sweep_db + write results/marathon_progress.md

Notifications:
  - ntfy per-combo: brief verdict ping after each combo completes
  - ntfy per-cycle: full summary (tiers, locks, OOS hits, ETA)
  - results/marathon_progress.md updated after each cycle

Usage:
    python3 run_marathon.py                          # 96h, 1h/combo, all 20 combos
    python3 run_marathon.py --hours 48               # 48h total
    python3 run_marathon.py --hours-per-combo 2      # 2h per combo per cycle
    python3 run_marathon.py --assets COINBASE_ETHUSD BINANCE_SOLUSD  # alts only
    python3 run_marathon.py --start-mode optuna      # start with optuna (if random already done)
    python3 run_marathon.py --dry-run                # print plan only, don't run
    N)datetime	timedelta
jlo_alertszresults/sweep_database.dbzresults/oos_dashboard.mdzresults/marathon_progress.md   )COINBASE_BTCUSDCOINBASE_ETHUSDBINANCE_SOLUSDBINANCE_LINKUSD_)4H6H8H12H1D240360480720r   c                     dddd|xs d d| dt          g}	 t        j                  |dd	
       y # t        $ r Y y w xY w)Ncurlz-sz-HzTitle: Marathonz-dzntfy.sh/
   T)timeoutcapture_output)_ntfy_topic
subprocessrun	Exception)messagetitlecmds      7/Users/jameslopez/projects/TradingBot25/run_marathon.pyntfyr$   =   sT    4)<*(=>H[M24CsBt< s   2 	>>c                 4    | dk  ry| dk  ry| dk  ry| dk  ryy	)
N2      🔴 SPARSE      🟡 EMERGINGi     🟢 USABLEi     🟢🟢 RICH	   🔵 DEEP )ns    r#   
tier_labelr/   H   s)    2v3w3w4x    c                  T   t         j                  j                  t              si S 	 t	        j
                  t              } | j                  dt         d      j                         }| j                          |D ci c]  \  }}}||f| c}}}S c c}}}w # t        $ r i cY S w xY w)z8Return {(asset, tf): n_rows} for current schema version.zFSELECT asset, timeframe, COUNT(*) FROM sweep_results WHERE schema_ver=z GROUP BY asset, timeframe)ospathexistsDB_FILEsqlite3connectexecuteDB_SCHEMA_VERfetchallcloser   )connrowsatr.   s        r#   get_db_countsr@   P   s    77>>'"		w'||  -.HJ
 (* 	 	

)-.gaAA	... 	s$   AB =BB B B'&B'c                 &   dddddd}|D ]3  }|D ],  }| j                  ||fd      }|t        |      xx   dz  cc<   . 5 |j                         D cg c]   \  }}|s	| d|j                         d    " }	}}dj	                  |	      S c c}}w )Nr   )r,   r+   r*   r)   r'         ×r    | )getr/   itemssplitjoin)
countsassetstfsbucketsr>   r?   r.   kvpartss
             r#   tier_summary_strrP   `   s    !VWhijGA

Aq61%AJqM"a'"   29FAAs"QWWYr]O$EF::e Gs   
BBc            
         g } t         j                  j                  t              s| S 	 t	        t              5 }|D ]  }d|vsd|v sd|v r|j                         j                  d      j                  d      D cg c]  }|j                          }}t        |      dk\  sh| j                  |d   |d   |d   |d   |d	   d
        	 ddd       | S c c}w # 1 sw Y   | S xY w# t        $ r Y | S w xY w)z3Return list of OOS row dicts from oos_dashboard.md.|z---Assetr   r   rB         r   )assettfis_sortoos_pnlverdictN)
r2   r3   r4   OOS_FILEopenstriprG   lenappendr   )r=   flineprO   s        r#   read_oos_tablerc   l   s    D77>>(#(^qd?etmw$,0JJL,>,>s,C,I,I#,NO,Nq,NOu:#KK#(8#(8#(8#(8#(9!    K P	  K  KsG   C( AC;CC#(CC( CC% C( %C( (	C54C5c                     | j                  d      d   }t               D ]+  }|d   | k(  s	|d   |k(  s|d   |k(  s|d   |d   |d   fc S  y)	zCLook up the OOS verdict for a specific combo from oos_dashboard.md.r   r   rV   rW   rZ   rY   rX   )   —re   re   )rG   rc   )rV   rW   shortrows       r#   get_oos_verdictrh      sa    KKR ELE!S\U%:D	Ry>3y>3y>AA   r0   c                      g } t               D ]2  }d|d   v s| j                  |d    d|d    d|d    d|d           4 | S )z&Return list of EXCELLENT/ACCEPT lines.   ✅rZ   rV    rW   : rY   )rc   r_   )hitsrg   s     r#   get_oos_highlightsrn      s\    DC	N"KK3w<.#d)Bs9~6FaIGWXY   Kr0   c                  ^   t         j                  dddg} 	 t        j                  | ddd      }|j                  |j
                  z   }t        t        j                  d|            }t        t        j                  d|            }|||fS # t        $ r}d	d	t        |      fcY d }~S d }~ww xY w)
Nztools/lock_params.pyz--applyz--cross-assetT<   )r   textr   zAPPLY\s+LockzSKIP\s+Lockr   )sys
executabler   r   stdoutstderrr^   refindallr   str)r"   resultoutputappliedskippedes         r#   apply_locksr~      s    >>19o
NCDtRP.bjj&9:bjj89'' !SV|s   A4B 	B,B'!B,'B,c                  t    	 t        j                  t        j                  dgdd       y # t        $ r Y y w xY w)Nzmine_sweep_db.pyTx   )r   r   )r   r   rr   rs   r   r-   r0   r#   update_dashboardr      s6    (:;&*C	9 s   (+ 	77c                 R	   t        j                         }| }t               }t               }|D ci c]  }|d   |d   f| }}g }|j	                  d       |j	                  d       |j	                  d|d   dd       |j	                  d	|dd       |j	                  d
|d    d|d    d|d   dd|d   dd	       |j	                  d|d   dd       |j	                  dt
         d       |j	                  d       t        j                         |d   z
  dz  }|j	                  d       |j	                  d       |j	                  d|d    d|d    d|d     d!|d   |d   k  rd"nd#        |j	                  d$|dd%|d&   dd'       |j	                  d(|d)    d*|d+    d,       |j	                  d-|d.           |j	                  d       |j	                  d/       |j	                  d       |j	                  d0       |j	                  d1       |d2   D ]H  }	|d3   D ]>  }
|j                  |	|
fd4      }|j	                  d5|	 d6|
 d6|d7d6t        |       d8	       @ J |j	                  d       |j	                  d9       |j	                  d       |j	                  d:       |j	                  d;       |d2   D ]\  }	|d3   D ]R  }
|	|
f}||v r0||   }|j	                  d5|	 d6|
 d6|d<    d6|d=    d6|d>    d8       ;|j	                  d5|	 d6|
 d?       T ^ |j	                  d       |d@   r|j	                  dA       |j	                  d       |d@   D ]  }|j	                  dB|dC    d|d    dD|dE    dF|dG   ddH|dI    d*|dJ    d,       |j	                  d       |dK   r|j	                  dL|dK           |j	                  d       |dM   s~|j	                  dN       |j	                  d;       |dM   D ]9  }|j	                  d5|d    dO|d    d6|d<    d6|d=    d6|d>    dP|dQ   d7d8       ; |j	                  d        t        t        dRdST      5 }|j                  dUj                  |      dUz          dVdVdV       yVc c}w # 1 sw Y   yVxY w)Wz7Write results/marathon_progress.md from run_state dict.rV   rW   z# Marathon Progress z**Started:** 
start_time%Y-%m-%d %H:%M  z**Updated:** z
**Plan:** 
max_cycles    cycles × n_combos    combos × hours_per_combo.1fh = ~total_hours.0fzh  z	**ETA:** etaz**ntfy topic:** `z`  
wall_start  z	## Statusz
- Cycle **current_cycle/z** `[current_modeu   ]` — zIN PROGRESSCOMPLETEz- Elapsed: zh / hourszh budgetz- Runs: total_ok OK / 
total_fail failedz- Locks applied (total): cumulative_locksz## DB Tier Progressz| Asset | TF | Rows | Tier |z|---|---|---|---|rJ   rK   r   z| rD   ,z |z## OOS Dashboard (latest)z,| Asset | TF | IS Sort | OOS P&L | Verdict |z|---|---|---|---|---|rX   rY   rZ   u    | — | — | not run yet |cycle_historyz## Cycle Historyz
### Cycle num [modeu   ]  — 	elapsed_hu   h — okfaillocksz- **Locks applied:** combosz4| Combo | IS Sortino | OOS P&L | Verdict | DB rows |-z | +db_rowswutf-8encoding
N)r   nowr@   rc   r_   r   timerE   r/   r\   PROGRESS_FILEwriterH   )	run_stater   srI   oos_rowsr
oos_lookuplinesr   r>   r?   r.   keychcrr`   s                   r#   write_progress_mdr      s9   CAF!H4<=Hq1W:qw'*HJ=E	LL&'	LL	LL=< @CD	LL=^ 4B78	LL:ao.k!J-TUVgThilSmmrst  vC  tD  EH  sI  IL  M  N	LL9QuXn5R89	LL$[M56	LL q.$6I	LL	LL	LL:a011\?2C5>IZH[[b%&%71\?%JMPZ[] ^	LL;yoT!G*S1AJK	LL8AjM?&<0AIJ	LL,Q/A-B,CDE	LL 
LL&'	LL	LL/0	LL$%x[5A

Aq61%ALL2aSA3c!Ac*Q-CD   
LL 
LL,-	LL	LL?@	LL()x[5Aa&Cj sOr!Cs#a	l^3q|nCPQR[P\~]_`ar!Cs*FGH   
LL 	'(RO$BLL:bi[!L/1B"RZL Q  ";4F2d8*F2f:,V]_ `LL'{4R[MBCLL(|ST45X,BLL2bk]!BtH:SIs$&yM?#bm_DIWXHYY["] ^ ' R  % 
mS7	3q			% 4'( 
4	3I >H 
4	3s   R+$RR&c                    d|  dt         j                  ||       d}|  d| d| d}	t        j                  dd|d	|d
t	        |      g}
|dk7  r|
d|dt	        |      gz  }
|r|
dgz  }
t        d|	 d|ddd       |j                  dt        j                         dd|	 d       |j                          t               j                  | |fd      }t        j                         }	 t        j                  |
|dz  dz        }t        j                         |z
  }|j                  dk(  }t               j                  | |fd      }||z
  }|rd"nd#}|j                  d$t        j                         dd%|	 d&|dz  d d'| d(| d)       |j                          |||fS # t        j                  $ r5 t        j                         |z
  }d}|j                  d|dz  d d!       Y w xY w)*Nz	data/mlp/, z.csvr   r   ]ztools/auto_optimize_loop.pyz--dataz--search--hoursall--regime--regime-threshold	--tightenu     →   (z.2fh)T)flushz
[z%H:%M:%Sz] START r   r   r   g      ?)r   Fz  TIMEOUT after rp   r   zm
OKFAIL[z] END r   zm  [z]  +z rows
)
_TF_TO_MINrE   rr   rs   rx   printr   r   r   r   r@   r   r   r   
returncodeTimeoutExpired)rV   rW   r   search_modelog_fhregimeregime_thresholdtighten	data_pathlabelr"   	db_beforestartry   elapsedr   db_afterdb_addedstatuss                      r#   	run_combor     s   E7"Z^^B%;$<DAI'2$bQ/E>>8Y
Kc%j"C 
F$8#>N:OPP}	F5'U3Kr
*$7
LL3x||~h/xwbAB
LLN##UBK3IIIKE=UT\C-?@))+%!# ##UBK3H9$HF
LL1X\\^H-VE7"WRZ<LDQWPXX\]e\ffmno
LLNw   $$ =))+%'
3'7s;<=s   *AF! !AG)(G)c                  D   t        j                  dt         j                  t              } | j	                  dt
        dd       | j	                  dt
        dd	       | j	                  d
t        d d       | j	                  ddd d       | j	                  ddd d       | j	                  ddddgd       | j	                  dt        dt         d       | j	                  ddd       | j	                  dt        d g d!d"#       | j	                  d$t
        d%d&       | j	                  d'dd(       | j                         }|j                  a|j                  rJ|j                  D cg c]4  }t        j                  |j                         |j                               6 c}nt         }|j"                  r(|j"                  D cg c]  }|j                          c}nt$        }|D cg c]  }|D ]  }||f  }}}t'        |      }||j(                  z  }	|j*                  xs, t-        d)t/        j0                  |j2                  |	z              }
|	|j2                  kD  r#|j2                  |z  |_        ||j(                  z  }	g }|j4                  }t7        |
      D ]  }|j9                  |       |dk(  rdnd} |
|	z  }t;        j<                         t?        |*      z   }tA        d+d,        tA        d-       tA        d.d/jC                  |              tA        d0d/jC                  |              tA        d1|        tA        d2|
 d3d4jC                  |       d       tA        d5|j(                  d6d7       tA        d8|	d6d7       tA        d9|j2                  d:d;|d:d<       tA        d=|d>       tA        d?t                tA        d@tD         dA       |jF                  d k7  r&tA        dB|jF                   dC|jH                   d       tA        d, d+       |jJ                  ratM        |d)      D ]Q  \  }}tA        dD| dE|
 dF| d3| dG|j(                  d6dH|	d:d<       |dk(  rtA        dI       tA        dJtD                S y tO        jN                         }i dKt;        j<                         dL|
dM|dN|j(                  dO|dP|j2                  dQ|dR|dS|dT|dUd)dV|dW   dXdWdYdWdZdWd[g }tQ        jR                  d\d]^       d_t;        j<                         d`da}tU        |dbdcd      }|jW                  det;        j<                          d+       |jW                  df|
 dgd4jC                  |       dh       tY        de|
 di| dG|j(                  d6dj|d:dkd4jC                  |       dl|dmdntD         dop       dW}dW}dW}tM        |d)      D ]l  \  }}tO        jN                         }||dU<   ||dV<   tA        d+d,        tA        dq| dE|
 dr|j                          dst;        j<                         d>       tA        d,        |jW                  d+d, dt| dE|
 du| dvt;        j<                          d+       dW}dW}g }tM        |d)      D ]  \  }\  }}tO        jN                         |z
  dwz  } | |j(                  z   |j2                  dxz  kD  rGdy| d6dz|d)z
   d{|d)z
   dE| d|| d}}!tA        d~|!        tY        |!dp       |j[                            y ||z
  ||
|z
  z  z   }"t;        j<                         t?        |"|j(                  z  *      z   }#tA        d| dE| dv| d| d| dE|
 d|#dm       t]        |||j(                  |||jF                  |jH                  |j^                        \  }$}%}&ta        ||      \  }'}(})|j9                  |||)|(|'|&|$d       |$r|d)z  }|d)z  }n
|d)z  }|d)z  }||dX<   ||dY<   d|'v rdnd|'v rdnd}*tY        d| dE|
 dv|jc                  d      d    d| du| d|) d|( d|' d|&dd| dE| d|* d|jc                  d      d    d| dp        tO        jN                         |z
  dwz  }+dW},|dk(  rRtA        d       te               \  }-}.}/|-},||-z  }||dZ<   |jW                  d|- d|. d+|/ d+       tA        d|- d|. d       tA        d       tg                |d[   j9                  |||+|||,|d       tA        dtD         d       ti        |       tk               }0tm        |0||      }1to               }2d| dE|
 du| d|+d6d	d| d| dd|1 d}g}3|,r|3j9                  d|, d| d       |2r&|3j9                  ddjC                  |2d d       z          ||
k  rCt;        j<                         t?        |	*      z   }4|3j9                  d|d)z    du||    d|4dmd}       tY        d+jC                  |3      d| dE|
 du| dp       |jW                  dt| d| d| d|, d|+d6d       o tO        jN                         |z
  dwz  }5|
d)z   |dU<   ti        |       tk               }0tm        |0||      }1to               }2d|
 d|5d6dd| d| dd|1 d}d| d}g}6|2r#|6j9                  ddjC                  |2      z          tY        d+jC                  |6      dp       tA        d+d,        tA        d|5d6d| d| d       tA        d| d|1        |2rtA        ddjC                  |2              tA        d, d+       |jW                  d|
 d|5d6d| d| d| dŝ       |j[                          y c c}w c c}w c c}}w )Nz!Autonomous multi-cycle optimizer.)descriptionformatter_classepilogr   g      X@z.Total wall-clock budget in hours (default: 96))typedefaulthelpz--hours-per-combog      ?z(Hours per combo per cycle (default: 1.0)z--cyclesz-Max cycles to run (default: auto from budget)z--assets*ASSET)nargsr   metavarz--tfsTFz--start-moderandomoptunaz)Search mode for cycle 1 (default: random))r   choicesr   z--ntfy-topiczntfy topic (default: ))r   r   z	--dry-run
store_truez#Print plan and exit without running)actionr   r   r   )r   bullbearzRegime filter for all combos: 'bull'/'bear' trains on regime-filtered IS data only. Use to build separate bull/bear param sets (Phase 4a). Default: 'all' (no filter).)r   r   r   r   r         .@z:MVRV zscore threshold for bull/bear split (default: 15.0).r   zSAuto-apply q5-q95 range tightening after each iteration (tighten_params.py --apply)rB   )r   r   zA=================================================================z  MARATHON OPTIMIZERz  Assets:          r   z  Timeframes:      z  Combos:          z  Cycles:          r   u    → z  Per combo/cycle: r   hz  Per cycle:       z  Total budget:    r   zh  (using ~r   z  ETA:             r   z  ntfy topic:      z  Progress file:   z (written after each cycle)z  Regime filter:   z (zscore threshold=z  Cycle r   rl   r   zh = u)       → lock_params --apply --cross-assetu       → mine_sweep_db + write r   r   r   r   r   r   r   rJ   rK   r   r   r   r   r   r   r   r   rz   T)exist_okzoutput/marathon_z%Y%m%d_%H%M%Sz.logr   r   r   zMarathon started: zCycles: z	  Modes: z

r   r   z	h.
Plan: z. ETA z%b %d %H:%Mz:.
Progress: github.com/jamesjlopez/TradingBot25/blob/main/u   🚀 Marathon started)r!   z  CYCLE z  [z]  started z
CYCLE r   z] r   gRQ?zBudget reached after zh. Completed z full cycles + z
 in cycle .z
  [BUDGET] u   ⏱ Marathon budget reachedz
  [r   z  |  cycle z	  |  ETA )r   r   r   )rV   rW   rX   rY   rZ   r   r   rj   u   ⚠u   ❌r   r   r   z]
IS Sortino: z  OOS: r   z
+r   z DB rows  (z combos done)rk   z donez+
  [LOCKS] Applying sign-stability locks...z
[LOCKS] Applied=z	 Skipped=z    Applied z
 lock(s), z already OK.z)  [DASHBOARD] Updating sweep dashboard...)r   r   r   r   r   r   r   z  [PROGRESS] Writing z...zCycle z
] done in zh.zCombos: r   z failed.zTiers: zLocks: z applied (total: z).u	   ✅ OOS: rD   r   zNext: cycle z], ETA u   📊 Cycle z] donez SUMMARY: ok=z fail=z locks=z	 elapsed=zh
u   🏁 Marathon complete: z cycles in zRuns: zFinal tiers: zLocks applied: u   ✅ EXCELLENT: u   🏁 Marathon completeu     MARATHON COMPLETE  —  u   h  —  r   z	  Locks: z  |  Tiers: z
MARATHON COMPLETE: z	 cycles, zh, z	 failed, z locks
)8argparseArgumentParserRawDescriptionHelpFormatter__doc__add_argumentfloatint
NTFY_TOPICrx   
parse_args
ntfy_topicr   rJ   _ASSET_ALIASESrE   upper
ALL_ASSETSrK   
TIMEFRAMESr^   r   cyclesmaxmathfloorr   
start_moderanger_   r   r   r   r   rH   r   r   r   dry_run	enumerater   r2   makedirsr\   r   r$   r;   r   r   rh   rG   r~   r   r   r@   rP   rn   )7parserargsr>   rJ   rW   rK   r?   r   r.   hours_per_cycler   modesr   r   r   r   imr   r   log_pathr   r   r   r   	cycle_numr   cycle_startcycle_ok
cycle_failcycle_combosrV   elapsed_totalmsg	remaining	eta_finalr   r   r   rZ   rY   rX   iconcycle_elapsed_hlocks_applied	n_applied	n_skippedlock_outputrI   t_stroos_hits	msg_linesnext_etatotal_elapsed	final_msgs7                                                          r#   mainr'  %  s   $$7 <<F
 	tM  O
+%G  I

dL  N

#tWM
#tTJ
8XBVH  J

4ZLB  D
LB  D

eE\r  s ,5$Y  [
Lr  tD //KPTP[P[L1n  AGGI6LakF15488,8Rbhhj8,*C$2fcq!fcffF2[A$...OPAtzz$**2N'O PJ ##zzA~d222EOOD:T8+x  .K
,,.9;7
7C	Bvh-	 "			& 12
34			#/
01	s
#$	
|3w||E/B.C1
EF	 4 4S9
;<	4A
67	

3/{;s:K2
NO	N3
45	}
-.	.I
JK{{e#DKK=0CDDYDYCZZ[\]	VHB-||eQ'DAqHQCqBqcQC{4CWCWX[B\\`apqt`uuwxyH}AC2=/BC	 (
 	 J(,,.* 	! 	4//	
 	+ 	$** 	# 	& 	# 	* 	! 	%( 	! 	! 	A  	"!I& KK4(!(,,.!?tDHHcG4F
LL%hlln%5R89
LL8J<ye1D0ETJK
ZLA3k$BVBVWZA[[`almp`q re$%VC+< =CCP/	S &	 HJ"+E1"5	;iik%.	/"%0	.!6(m1ZLK4E4E4G3HT\T`T`TbcqSrstr&)Aj\K=PRS[S_S_SaRbbdef
'2NA{r!YY[:5=Mt333djj46GG.}S.A B$$-aK=!uAaS
S\R]]^`cU+,S =>AZ)%; <<I TEYEY9Y)ZZIE!AaS5'2$k)Aj\QZ[depZqrs$-eR9M9M{\b59[[SWShSh6:ll%D!B )8r(B%GWgb"w7#2!  QQa
q&.Ij!&0Il# "W,5EW<L5RWDI;a
|2ekk#.>r.B-C1RD;- X&iwwir' CQ<{1#Qqc@ aC 0 45Qrd%@	G 3T  99;4< ("@B0;-Iy+#,M9,,<I()LL-i[	)B{m[]^_L:i[MN 	9; 	/"))k(J""+
 	 	%m_C89)$ $o&vvs;(* YKqB{m:oVYEZZ\]xjzl(;eWA
	
 w}o5FGWFXXZ[\[5::hrl+CCDz!||~	(HHH|IaK=5;K:LGT\]hSiijklTYYy!;yk:,bQ\P]]c)dex	{-z
| T+_Ioc5J#O 	PU #6\ YY[:-5M!+aIoi F4E!#H #:,k-9LBO

&H5
wa 
*+1-	I *UZZ-AAB9	%=>	Bvh-	&}S&9(6R\Q]]d
ef	I&'|E7
;<

8,-./	VHB-
LL(ImC=PPS:VJ<y9I8J(T U
LLN} M,2s   *9nn+n__main__)N)r   r   F)#r   r   rr   r2   r   r   r6   r  rv   r   r   r   r5   r[   r   r9   r   rG   r   r  r   r   r$   r/   r@   rP   rc   rh   rn   r~   r   r   r   r'  __name__)r>   s   0r#   <module>r*     s   6  
 	     	 (
+*.
 0::z!!''#,r"A%z:,
eEN

 0	K)b!H~B zF e ;s   B