
     =jդ              	         U d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z
ddlZddlmZ ddlZddlZddlmZ ddlZddlZddlZ ee      j1                         j2                  j2                  Zej6                  j9                  d ee             ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z+m,Z,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3 d	Z4e4a5g d
Z6edz  dz  Z7edz  dz  Z8edz  dz  Z9da:de;d<   da<de;d<   g dZ=g dZ>dZ?dZ@dZAd6dZBd7dZCd6dZDd8dZEd9d:dZFd;dZGd<dZHd=dZId>d ZJd?d!ZK	 	 	 d@	 	 	 	 	 	 	 	 	 dAd#ZLdBd$ZMdCd%ZNdDd&ZO eP       ZQdEd'ZRdFd(ZSdGd)ZTdHd*ZUdId+ZV	 dJ	 	 	 dKd,ZWdLd-ZX	 dM	 	 	 	 	 	 	 dNd.ZYdOd/ZZdPd0Z[	 	 	 	 	 	 	 	 dQd1Z\dQd2Z]dde2d"d"ddddf		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dRd3Z^dSd4Z_e`d5k(  r e_        yy)Ta  
Deeper threshold sweep for BTC MLP presets.

This intentionally keeps MLP weights fixed and explores threshold/config
parameters around the current winners and artifact recommendations. It reuses
strategy_mlp_scores' score-to-signal block and canonical calculate_metrics so
candidate metrics stay comparable to optimizer winners.

Examples:
    .venv/bin/python tools/run_mlp_deep_sweep.py --timeframes 6H 8H 12H 1D --samples 50000
    .venv/bin/python tools/run_mlp_deep_sweep.py --timeframes 6H --samples 10000 --promote
    )annotationsN)Path)Any)
	OOS_STARTSCORE_STARTSUBPERIOD_SPLIT	TRAIN_ENDTRAIN_START	WFO_FOLDSWFO_MIN_OOS_TRADESWFO_MIN_VALID_FOLDScomposite_scoreget_min_trades)FEATURE_COLS_prepare_features_score_to_signalscalculate_metricsload_mlp_artifactmlp_forward)DEFAULT_WORKER_FRACTIONresolve_worker_countCOINBASE_BTCUSD4H6H8H12H1Dresultswinnerssweepsreportsz
str | None
_WORKER_TFzIdict[str, tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]] | None_WORKER_CACHEmlp_weights_file!i_long_entry_activation_threshold i_long_exit_activation_threshold-i_long_exit_activation_confirmation_thresholdi_trailing_stop_thresholdi_use_long_exit_confirmationi_use_long_entry_confirmationi_regime_windowi_regime_entry_min_scorei_mvrv_suppress_beari_exit_score_windowi_entry_score_window)Total TradesTotal P&L %Sharpe RatioSortino RatioCalmar RatioP&L/DD RatioMax Drawdown %% In MarketOOS_Total TradesOOS_Total P&L %OOS_Sortino RatioOOS_Bear_Pct      ?   g     @U@c                *    t         dt         d|  dz  S )N(optimization_winner_strategy_mlp_scores__.csv)
WINNER_DIRASSETtfs    C/Users/jameslopez/projects/TradingBot25/tools/run_mlp_deep_sweep.pywinner_pathrI   i   s    B5'2$dSSS    c                    dt          d|  dS )Nz"strategies/params/mlp/mlp_weights_rB   z.json)rE   rF   s    rH   artifact_pathrL   m   s    /wat5AArJ   c                    dddddd}|j                  | |       }t        dz  dz  t         d	| d
z  }|j                         r|S t        dz  t         d|  d
z  S )N240360480720r   r   datamlpz, rC   -)getREPOrE   exists)rG   _TF_MINUTESperiodmlp_paths       rH   	data_pathr[   q   sm    e5dSK__R$Ff}u$%6($'??H&=eWAbT...rJ   c                x    t        | t              r | j                         j                         dv S t	        |       S )N>   1yestrue)
isinstancestrstriplowerbool)vs    rH   as_boolrf   {   s.    !Swwy $8887NrJ   c                j    	 t        j                  |       r|S t        |       S # t        $ r |cY S w xY wN)pdisnafloat	Exception)re   defaults     rH   as_floatrn      s4    771:NQx s   $ 
$ 22c                :   t        |       }|j                         sy t        t        j                  |j                                     }|sy t        |d         }t        |j                  dd            |d<   t        |j                  dd            |d<   d|d<   |S )Nr   r/   Fr,   r?   r+   )	rI   rW   listcsv
DictReaderopendictrf   rU   )rG   pathrowsrows       rH   load_winnerrx      s    r?D;;=tyy{+,D
tAw-C")#''2H%*P"QC+2377;Z\a3b+cC'(*+C&'JrJ   c                   t        |       }|d   j                  di       j                  di       }| |j                  dd      |j                  dd      |j                  dd      |j                  dd      d	d
ddd
d	d	d}|S )Nmetatrainingrecommended_thresholdsr'           r(   r)   r*   r?   Fr        @r%   )r   rU   )weights_pathartrecparamss       rH   recommended_from_artifactr      s    
L
)C
f+//*b
)
-
-.F
KC(-0WW5XZ]-^,/GG4VX[,\9<;S:
 &)WW-H#%N()).$+ %  !F  MrJ   c                z    t        t        | |      ||z
        } t        | |z
  |z        }t        |||z  z   d      S )N   )minmaxround)valuestartstopstepns        rH   snapr      sB    E5!4$;/Euu}$%AT!1%%rJ   c                    | d   t        | d         t        | d         t        | d         t        | d         t        t        | j                  dd                  fS )Nr&   r'   r(   r)   r*   r/   F)rn   rd   rf   rU   ps    rH   	param_keyr      sf    	
678567BCD./0WQUU159:; rJ   c                   t        | d         t        | d         t        | d         t        | d         t        dt        | j                  dd                  ddd	d
t	        | j                  dd            t        dt        t        | j                  dd                        t        dt        t        | j                  dd                        dS )Nr&   r'   r(   r)   r}   r*   r?   Fr   r~   r/   r0   r1   r%   )ra   rn   r   rU   rf   intr   s    rH   normalise_paramr      s    "4 56-5a8[6\-],4Q7Y5Z,[9A=>:
 &)hquu=XZ]7^._%`()).$+ '.De(L M"1c(1559NPQ3R*S&TU #As8AEE:PRS4T+U'V W rJ   Fc	                
  !" t         j                  j                  |      }	t        |       }
t	        |       }|xs g }g }|
r|j                  t        |
             t        j                  j                  |      r#|j                  t        t        |                   |D ]%  }|j                  t        t        |                   ' | dk(  rVt        dk(  rMt	        d      }t        j                  j                  |      r#|j                  t        t        |                   t               "g !d"!"fd}|rdnd\  }}|rdnd\  }}||n|}||n|}||n|}||n|}|rd	nd
}t        j                  d|d      }|D ]k  }t        |      }t        |j!                  dd            }t#        |t        j$                  t        j&                  ||z
                       |d<    ||d       m t)        d||z
  dz        }t)        d||z
  dz        }t+        |dz        }t)        d||z
        }|rlt)        d|t-        |      z        }t/        |      D ]E  \  }}t1        |      D ]0  }t        |      }t3        |	j5                  t        |d         |      ||d      |d<   t3        |	j5                  t        |d         |      ||d      |d<   t3        |	j5                  t        |d         |      ||d      |d<   t#        |	j7                  t        j                  d|d                  |d<   t9        |	j7                  ddg            |d<   t+        |	j7                  g d            |d<   t+        |	j7                  g d            |d<    ||d|        3 H t;        |D ch c]  }|d   	 c}      } t1        |      D ]5  }t=        |	j7                  |             t#        |	j7                  t        j                  ||d                  t#        |	j7                  t        j                  ||d                  t#        |	j7                  t        j                  ||d                  t#        |	j7                  t        j                  d|d                  ddddt9        |	j7                  ddg            t+        |	j7                  g d            t+        |	j7                  g d            d } ||d!       8 !S c c}w )#Nr   r   r   c                    t        |       }t        |      }|v ry j                  |       ||d<   j                  |       y )Nsource)r   r   addappend)r   r   ppkeyoutseens       rH   r   z generate_candidates.<locals>.add   s?    Qm$;8

2rJ   )g     g     @)     rg     r@)g     pg     p@)r   g      y@g      Y@g      I@r}   g      @r*   seedg      @g      $@g?r   r?   r'   r(   r)   FTr/   )r?   r?   r?               r0   r1   local_r&   r~   r%   broad)r   dict[str, Any]r   ra   returnNone)nprandomdefault_rngrx   rL   r   r   osru   rW   r   rE   setarangert   rn   rU   rk   argminabsr   r   len	enumerateranger   normalchoicerd   sortedra   )#rG   samplesr   extra_weightswideentry_loentry_hiexit_loexit_hirngcurrentown_artifactseedsr   alt_12hr   _entry_lo_def_entry_hi_def_exit_lo_def_exit_hi_deftrail_hiallowed_trails	raw_traillocal_sigma_entrylocal_sigma_exitlocal_nbroad_nper_seedirB   r   weight_choicesr   r   s#                                    @@rH   generate_candidatesr      s    ))


%C"oG $L!'RM"$E_W-.	ww~~l#_%>|%LMN%_%>|%LMN & 
Tze00  &77>>'"LL)B7)KLM!$D "C 7;? M=6:?L<#/x]H#/x]H"/w\G"/w\G.25HIIc8S1MGQUU#>DE	).}RYYrvvm^gNgGh=i/j)k
%&Av	 
 sX%8C$?@tg&73%>?'D.!G!Ww&'Gq'SZ/0e$DAq8_G9=JJx*M(NOQbch:56 9=JJx*L(MNP`aWd945 FJJJx*Y(Z[]mnWdFAB 27szz"))CQY[^B_7`1a-.,0UDM1J,K()+.szz:O/P+Q'(,/

;P0Q,R()As|$% % %* EBEqQ12EBCN7^ #CJJ~$> ?16szz"))HV^`cBd7e1f05cjj7T[]aAb6c0d=B3::biiX_ahjnNoCp=q).szz"))CSV:W/X)Y,--2 (/$(UDM)B$C#&szz2G'H#I$'

3H(I$J
 	Aw " J% Cs   &S?c                   t        j                  t        |             }|j                  j                  j                         j                  j                         |_        t        j                  |d   d      j                  j                  d       |d<   |j                  |d   t        k\     j                         j                  d      }t        |      }|j                  d      xs g }|D cg c]  }|t         vs||j                  vs| }}|r/t#        dt$        j&                  j)                  |       d| d      t+        ||xs d       }|g d	   j                         }d
D ]  }	|	|j                  v s||	   ||	<    t-        ||d         |d<   ||d d df   |d   j/                  t0        j2                        |d   j/                  t0        j2                        fS c c}w )NtimeT)utc)dropfeature_colszStale artifact z8: expects features not in current FEATURE_COLS or data: z. Retrain to skip.)r   closehigh)mvrv_regimelayersactivation_score	   r   r   )ri   read_csvr[   columnsra   rc   rb   to_datetimedttz_localizelocr
   copyreset_indexr   rU   _FEATURE_COLS
ValueErrorr   ru   basenamer   r   to_numpyr   float64)
rG   r   fullr   art_colscmissing_from_currentXscoredcols
             rH   load_scored_framer   4  s   ;;y}%D<<##))+//557DL>>$v,D9<<HHNDL88DLK/0557CCCND
L
)Cww~&,"H'/dx!1M3IaW[WcWcNcAxdbgg..|<= >--A,BBTV
 	
 	$ 0D1A+,113F$,,s)F3K   "-QH!>F1QT7DM222::>V@U@UVXV`V`@aaa es   1G'?G'G'c           	        t        j                  t        |d         t        |d         t        |d         gt         j                        }t        j                  t        j
                  | j                  t         j                        d d d f   |d d d f   z
        d      }t        t        j                  ||k              S )Nr'   r(   r)   )dtyper?   )axis)	r   arrayrn   r   r   r   r   r   sum)scoresr   margin
thresholdsdists        rH   fragile_countr   K  s    V?@AV>?@VKLM	

 jjJ 66"&&4QW=
4QR7@SST[\]DrvvdVm$%%rJ   c                   | | d   t        j                  |      k\  | d   t        j                  |      k  z     }t        |      dk  ryt        d|d   j                  d   |d   j                  d   z
  j
                  dz        S )Nr   r         ?r   g     v@)ri   	Timestampr   r   ilocdays)dfscore_startendsls       rH   is_yearsr  X  s{    	RZ2<<44Fr||TWGX9XY	ZB
2w{sRZ__R(2f:??1+==CCfLMMrJ   c                .   |d   }||vr	 t        | |      ||<   ||   t        u rnt        |      }|j                  i dddd	d
ddddddd	dd	dd	dd	dddddddddddddg ddd	ddd	d	d	d	dd|j                  dd      d
       |S ||   \  }}}}	t        |      }
t        |
d<   t        |j                         |
|||	      }||d   t        j                  t              k     j                         }t!        |t               }t#        |t        t              }t%        |j                  dd	      |j                  dd	      t'        |j                  dd            ||j                  dd	            }t        j                  t(              }t!        ||d   |k     j                         t               }t!        ||d   |kD     j                         t(               }g }t*        D ]  \  }}}||d   t        j                  |      k     j                         }t!        ||t,        !      }t'        |j                  dd            t,        k\  sh|j/                  t1        |j                  dd"                    t3        |      t4        k\  rXt1        t7        j8                  |            }t1        t7        j:                  |            }t'        t=        d# |D                    }nd	}d"}t3        t*              }|d   j?                         t        j                  t@              k\  rRt!        |t@        d$!      }t'        |j                  dd            dk(  rDd%D ]  }|j                  |      d&k(  sd	||<    n"tB        D ci c]  }|t6        jD                   }}t6        jD                  x}x} }!d'|jF                  v r||d   t        j                  t@              k\     }"t3        |"      dkD  rzt3        |"      }#t1        |"d'   d(k(  j=                         |#z  d)z        }t1        |"d'   dk(  j=                         |#z  d)z        } t1        |"d'   d$k(  j=                         |#z  d)z        }!tH        D ci c]  }|||   
 }}|j                  |       |j                  i d|dtK        |      d|j                  dd&      d*|j                  dd&      d+|j                  dd      d|j                  dd&      d,|j                  dd&      d-|j                  dd      d|d|d|dtM        jN                  |D $cg c]  }$tQ        |$d.       c}$      d|j                  dt6        jD                        d/|j                  dt6        jD                        d0|j                  dt6        jD                        d1|j                  dt6        jD                        d2|j                  dt6        jD                        || |!tS        |d3   |d4      tS        |d3   |d5      |j                  dd6      d7       |S # t        $ rD}t        dt        j                  j                  |       d| d       t        ||<   Y d }~fd }~ww xY wc c}w c c}w c c}$w )8Nr&   z	  [SKIP] : Tflushr2   r   r3   r}   r4        8r5   r6   r7   r8   r9   	CompositeMin_Trades_FloorSub1_CalmarSub2_Calmar	WFO_ScoreWFO_Min_FoldWFO_Neg_Foldsc   WFO_Fold_Calmarsr:   r   stale)
r;   r<   OOS_Calmar RatioOOS_Max Drawdown %r=   OOS_Neutral_PctOOS_Bull_PctFragile_0.05Fragile_1.0r   _pine_time_startr   )r  )r  
min_trades     Xc              3  ,   K   | ]  }|d k  s	d  yw)r   r?   N ).0r   s     rH   	<genexpr>zevaluate.<locals>.<genexpr>  s     ;\QU!\s   
r?   )r6   r5   r4         $r   r   d   Sub1_SortinoSub1_TradesSub2_SortinoSub2_Trades   r;   r<   r  r  r   g?r>   	candidate)r=   r  r  r  r  r   )*r   r   printr   ru   r   _STALE_SENTINELrt   updaterU   r   r   r   ri   r  r	   r   r  r   r   r   r   r   r   rk   r   r   r   meanr   r   r   r   METRIC_COLSnanr   
PARAM_COLSr   jsondumpsr   r   )%rG   r   cacheweightsexcresultr   
stoch_peak	close_arrhigh_arrsignal_paramssigs
train_sigsmetricsyearscompsplitsub1sub2fold_calmarsrB   	oos_startoos_endfold_windowfold_metrics	wfo_scorewfo_minwfo_negoos_kkoos_bear_pctoos_neutral_pctoos_bull_pctoos_sigsr   r   s%                                        rH   evaluaterU  b  s   '(Ge	-.r7;E'N W~(f 	A~q 	A- 	Anf 	A'	A1?	AIWY\	A(#	A/<c	ACNPS	A +A	A 0=f	A GTU[	A $V		A .<V		A FUVX		A
 +B	A
 1CA	A
 [^-3.13[^(+RPR"(**Xw"?	A 		B .3Gn+FJ	8LM(3M$%V[[]M:yRZ[Dd6lbll9&==>CCEJ
DGZi8ENC(OS)GKK*+M3'D LL)EZ
6(:e(CDIIKYdeDZ
6(:U(BCHHJXghDL!*9g F!3r||G7L!LMRRT()Xjk|237IIl&6&6~u&M NO	 "+
 <//"'',/0	|,-c;\;;<	i.F|R\\)44)J sww~q)*a/G772;%'!CG H #..+Qq"&&y+. 57FF:L:?\$VY(??@x=1HA#Xm%<%B$G$G$IA$MPS$STL#Xm%<%B$G$G$IA$MPS$STO#Xm%<%B$G$G$IA$MPS$STL4>?JqalJF?
MM'
MM	
	
u 5	
 488NE:	
 DHH_e<		

 488NA6	
 488NE:	
 DHH_e<	
 488NA6	
 	
 G	
 W	
 

+NAE!QK+N O	
  ?	
 sww}bff=	
  "&&!A	
   ?!	
" !#''*:BFF"C#	
$ ).()*5G*H&RVW(4F)GQTUjj;7/	
6 MQ  	-Ibgg..w783%@M,E'N	-| / @ ,Os)   X8 Z3Z=Z8	Z9Z  Zc                    | a i a|ay rh   )r#   r$   rE   )rG   assets     rH   init_workerrX    s    JMErJ   c                Z    t         t        t        d      t        t         | t              S )Nz sweep worker was not initialized)r#   r$   RuntimeErrorrU  )r   s    rH   evaluate_workerr[    s(    ]2=>>J66rJ   c                :   t        t        | j                  dd            dk(        }t        | j                  dd            }t        | j                  dd            }t        t        | j                  dd      d            }t        t        | j                  dd	            dkD        t        t        | j                  d
d	            dkD        z   }t        | j                  dd	      d	      }t        |dkD        t        |dkD        |||| t        t        | j                  dd                   t        | j                  dd            t        | j                  dd            |t        | j                  dd	            |t        | j                  dd            fS )Nr  r   r  r}   r7   r8   r  r  r%  r  r<   r  r  r  r   r  r6   r3   )r   rn   rU   r   )rw   
fragile_ok	compositepnl_dddrawdownsub_okoos_sortinos          rH   
rank_tuplerc    s]   Xcggna89Q>?Jc23Icggnc23F8CGG$4f=vFGH#''-781<=HSWWUbdiMjDknoDo@ppF377#6>FKIMFQJ			Xcggor23	44/0c*+/0,- rJ   c                    t        |       }t        t        z   D ci c]  }||j                  |d       }}t	        j
                  |g      j                  |d       y c c}w )N Findex)rI   r3  r1  rU   ri   	DataFrameto_csv)rG   rw   ru   rP  ordereds        rH   write_winnerrk    sY    r?D*4{*BC*BQq#''!R. *BGCLL'""4u"5 Ds   Ac           
     &   || j                  d      dk(  ryt        | j                  dd            dk  ryt        | j                  dd            dk7  ryt        | j                  d	d
            dk  st        | j                  dd
            dk  ryt        t        | j                  dd                  dk7  ryt        t        |j                  dd            t        |xs i j                  dd                  }t        t        |j                  dd
            t        |xs i j                  dd
                  }t        | j                  dd            |k  ryt        | j                  dd
            |k  ryt        t        | j                  d      d            }t        t        t        |xs i j                  d      d      t        |j                  d      d                  }t        | j                  d      d      }|t        k\  xr |t        k\  xr	 |t
        k  }|rjt        | j                  d      d
      }	t        t        |j                  d      d
      t        |xs i j                  d      d
            }
|	|
t        z
  k  ryy)Nr   r   Fr7   r}   r   r  r  r  r%  r  r  r6   r:   r=   r<   T)rU   rn   r   r   OOS_MIN_TRADESOOS_BEAR_PCT_SKIPOOS_SORTINO_DELTA)bestr   stored_winner
ref_pnl_dd
ref_calmarbest_oos_tradesref_oos_tradesrQ  oos_gate_appliesbest_oos_sortinoref_oos_sortinos              rH   
promotablery    sU   $((8,	9-.!3,-2./14-Y^A_8`de8e
8DHH_b123q8 ^S12-%2**>3?@J ^U34-%2**>5ABJ -.;/0:= (488,>#?EFO#-%2**+=>D/0#6 N DHH^4c:L>) 	-n,	-,, 
 #DHH-@$A5IW[[!45u=m)r../BCUK
 o0AAArJ   c                    | dk  rg S t        t        d|      |       }t        | |      \  }}g }d}t        |      D ]&  }||z   ||k  rdndz   }|j	                  ||f       |}( |S )zIReturn balanced, contiguous ranges covering every candidate exactly once.r   r?   )r   r   divmodr   r   )	totalworkersshard_countbaseextrarangesr   ordinalr  s	            rH   partition_candidate_rangesr  6  s{    z	c!Wou-K,KD%FE%dl7U?a:ucl# & MrJ   c           
         i }g }t        |d      D ]M  \  }}|j                  t        | ||             |s%|dz  dk(  s.t        |  d|ddt	        |      dd       O |S )	Nr?     r   : evaluated ,/Tr  )r   r   rU  r-  r   )rG   
candidatesshow_progressr6  r   idxr   s          rH   evaluate_candidates_sequentialr  E  sr    
 QSEG Q/VxFE23S4Z1_RDSG1S_Q,?@M 0 NrJ   c                    | j                  | j                  dz         }|j                  d      5 }t        j                  ||t        j
                         d d d        |j                  |        y # 1 sw Y   xY w)Nz.tmpwb)protocol)with_suffixsuffixrs   pickledumpHIGHEST_PROTOCOLreplace)ru   payload	temp_pathhandles       rH   _write_pickle_atomicr  S  s[      v!56I		GVf.E.EF 
d 
	s   'A11A:c                   |j                  d      5 }t        j                  |      }d d d        j                  d      dk7  s|j                  d      | k7  rt	        d      |j                  d      }t        |t              r1|j                  dd      |j                  d	d      z
  t        |      k7  rt	        d
      t        |d         a	t        | |d      }t        |dt        | |d   |d	   |d   |d       y # 1 sw Y   xY w)Nrbversionr?   rG   zinvalid sweep shard metadatar  r  r   r   z#invalid sweep shard candidate rangerW  F)r  r  )r  rW  rG   r  r   r  r   )rs   r  loadrU   r   r`   rp   r   ra   rE   r  r  )rG   
input_pathoutput_pathr  r  r  r   s          rH   run_internal_shardr  Z  s    		&++f% 
{{9"gkk$&72&=788\*Jj$'7;;ua+@7;;wXYCZ+Z^abl^m+m>?? !E,R5QGy)W%5>	
 
	s   C>>Dc                   t        t        |      |      }t        |      dk  rt        | |      dfS t        |  dt        |       dd       t	        j
                  dt         d|  d      5 }t        |      }g }g }	 t        |      D ]  \  }\  }	}
|d	|d
dz  }|d	|d
dz  }|d	|d
dz  }t        |dt        | ||	|
||	|
 d       |j                  d      }t        j                  t        j                  t        t        t               j#                               dt        dd| dt        |      dt        |      gt$        |t        j&                        }|j)                  ||	|
||f       |j)                  |||f        |D ]R  \  }}}|j+                         }|j-                          |dk7  s-|j/                  dd      dd  }t1        d| d|        	 g }d}|D ]  \  }}	}
}}|	|k7  s|j;                         st1        d      |j                  d      5 }t=        j>                  |      }d d d        dt        | ||	|
f}jA                  d      |jA                  d       |jA                  d!      |jA                  d"      |jA                  d#      |jA                  d$      f}|jA                  d%      }||k7  s!tC        |tD              rt        |      |
|	z
  k7  rt1        d&      |jG                  |       |
} |t        |      k7  st        |      t        |      k7  rt1        d'      |t        |      fcd d d        S # t2        $ rZ |D ]S  \  }}}|j5                          |j7                          |j+                          |j8                  rD|j-                          U  w xY w# 1 sw Y   |xY w# 1 sw Y   y xY w)(Nr?   z: retrying with z subprocess shardsTr  z
mlp-sweep-rT   )prefixzshard-04dz
.input.pklz.output.pklz.log)r  rW  rG   r  r   r  r  r  --asset--_internal-shard--_shard-tf--_shard-input--_shard-output)cwdstdoutstderrr   zutf-8r  )encodingerrorsi`zsweep shard failed (z): z"sweep shard coverage is incompleter  r  rW  rG   r  r   r  r   zinvalid sweep shard outputz5sweep shard merge did not preserve candidate coverage)$r  r   r  r-  tempfileTemporaryDirectoryrE   r   r   r  rs   
subprocessPopensys
executablera   __file__resolverV   STDOUTr   waitr   	read_textrZ  rl   poll	terminateclosedrW   r  r  rU   r`   rp   extend)rG   r  r}  r  temp_dir	temp_rootspecs	processesr  r   r  r  r  log_path
log_handleprocess
returncodetailrB   mergedexpected_startr  outputexpectedactualshard_resultss                             rH   evaluate_candidates_subprocessr  t  s   
 (JAF
6{a-b*=q@@	RD V-?
@M		$	$jqA,F	G8N	>@	4	)26):%%&6'#j)II
'F73-{*KK$}D'AA$#$!& #*!&"&0s&; &]]40
$**DN2245!+%(J)K( %%,,!$ guc;IJ  ':x!@AI *;L 2;-X$\\^
  "?#--wy-QRWRXYD&)=j\TF'STT 2; (*38/GUCa&k.@.@.B"#GHH!!$'6V, (5"guc:H

9%

7#

4 

9%

7#

5!F #JJy1M!M4)HCP]L^behmbmLm"#?@@MM-( N% 49& S_,Fs:0NVWWs6{"c 
H	Gb  	*3&Q<<>)%%'LLN!(($$& +4 	 (' 
H	GsL   /N> D6M7(MAN>!N17D
N>AN.N..N>1N;6N>>Oc                v   |dk(  r+t        |  dt        |      ddd       t        | |      dfS t        |  dt        |      dd| dd       t        dt	        d	t        |      t        d|d
z        z              }t        j                  d      }g }	 t        j                  j                  ||t        | t        f      5 }t        |j                  t        ||      d      D ]?  \  }}|j                  |       |dz  dk(  s t        |  d|ddt        |      dd       A 	 d d d        ||fS # 1 sw Y   xY w# t         t"        f$ rf}	t        |  d|	 dd       	 t%        | ||      cY d }	~	S # t&        $ r/}
t        |  d|
 dd       t)        | |d      cY d }
~
cY d }	~	S d }
~
ww xY wd }	~	ww xY w)Nr?   z: evaluating r  z candidates sequentiallyTr  z candidates with z worker processes   r   spawn)max_workers
mp_contextinitializerinitargs)	chunksizer  r   r  r  z: worker pool unavailable ()z#: subprocess sharding unavailable (z(); falling back to sequential evaluation)r-  r   r  r   r   mpget_context
concurrentfuturesProcessPoolExecutorrX  rE   r   mapr[  r   OSErrorPermissionErrorr  rl   evaluate_candidates)rG   r  r}  
chunk_sizer  r   poolr  r9  r8  	shard_excs              rH   r  r    s   !|M#j/!!44LMUYZ-b*=q@@	RDc*oa00A'J[
\dhiQCZC7Q;4G!GHIJ(JG:33!#%[	 4 

 (/:Yc)dfghVv&:?RDSG1S_Q4GHPTU  i
" G#
 
 _% :/uA6dC	:1"j'JJ 	:RD;I;Fnow{|&r:q999	:	:sm   -E AD7"D7+E 7E <E F8F3&E82F88	F0F+ F0!F3%F8+F00F33F8c                T   t        | ||||	|
|||	      }t        |       }|rt        |      nd }|r"t        |      }d|d<   |j	                  d|       t        |t        |      |      }t        | ||      \  }}|j                  t        d       |d   }t        d |D        d       }|^|\t        |j                  dd	            }t        |j                  dd	            }|dkD  r!||d
z  k  rt        d|  d|dd|ddd       d[d}|D cg c]  } ||      s| }}|r|d   n|}t        |||      }|s>|r;|8t        t        |j                  d      d            }t        t!        t        |xs i j                  d      d      t        |j                  d      d                  }t        |j                  d      d      }|t"        k\  r|t"        k\  r|t$        k  rt        |j                  d      d	      } t!        t        |j                  d      d	      t        |xs i j                  d      d	            }!| |!t&        z
  k  r*t        d|  d| dd|!ddt&         d| d| d|ddd       |xr |xr |d u }"t(        j+                  dd       t(        d t,         d!|  d"z  }#t/        j0                  |d |       j3                  |#d#$       |r|s|"rt5        | |       i d%| d&t        |      d'|d(t7        |#j9                  t:                    d)t=        |xr |xs |"      d*t=        |      d+|r|j                  d      nt>        j@                  d,|r|j                  d-      nt>        j@                  d.|r|j                  d/      nt>        j@                  d0|r|j                  d      nt>        j@                  d1|j                  d      d2|j                  d3      d4|j                  d5      d6|j                  d7      d8|j                  d9      d:|j                  d;      d<|j                  d=      i d>|j                  d?      d@|j                  dA      dB|j                  dC      dD|j                  d      dE|j                  d-      dF|j                  dG      dH|j                  dI      dJ|j                  dK      dL|j                  dM      dN|j                  d/      dO|j                  dP      dQ|j                  d      dR|j                  dS      dT|j                  d      dU|j                  dV      dW|j                  dX      dY|j                  dP      |j                  d      |j                  d-      dZS c c}w )\N)r   r   r   r   r   r   r   r   T)r   reversec              3  J   K   | ]  }|j                  d       dk(  s|  yw)r   r   N)rU   )r#  rs     rH   r$  zrun_tf.<locals>.<genexpr>  s      H1155?i+GQs   ##r6   r%  r   z	  [WARN] z: cur_eval Calmar=z.3fz! is <50% of stored winner Calmar=ue    — winner artifact was likely overwritten during training; using stored metrics as promotion floor.r  c                |   | j                  d      dk7  xr t        | j                  dd            dkD  xr t        | j                  dd            dk(  xrg t        | j                  dd	            dkD  xrG t        | j                  d
d	            dkD  xr' t        t        | j                  dd                  dk(  S )Nr   r   r7   r}   r   r  r  r  r%  r  r  )rU   rn   r   )rw   s    rH   _gate_passeszrun_tf.<locals>._gate_passes  s    GGH* A56:A45:A 67!;A 67!;	A
 HSWW_b9:;q@	
rJ   r:   r}   r=   r<   z  [OOS BLOCK] z: new OOS_Sortino=z < ref=u    − z (trades: best=z, ref=z, bear=z.0fz%)parentsexist_ok'optimization_sweep_strategy_mlp_scores_rB   rC   Frf  	timeframe	evaluatedr}  
sweep_filepromotedry  current_calmarcurrent_pnl_ddr7   current_wfo_minr  current_oos_sortinobest_sourcebest_weightsr&   
best_entryr'   	best_exitr(   	best_confr)   
best_trailr*   best_suppress_bearr/   best_tradesr2   best_pnlr3   best_drawdownr8   best_calmarbest_pnl_ddbest_compositer  best_sub1_calmarr  best_sub2_calmarr  best_wfo_scorer  best_wfo_minbest_wfo_negr  rt  best_oos_pnlr;   rw  best_fragile_0_05r  best_fragile_1_0r  overall_wfo_neg)overall_calmaroverall_pnl_dd)rw   r   r   rd   )!r   rx   rt   r   insertr   r   r  sortrc  nextrn   rU   r-  ry  r   r   rm  rn  ro  	SWEEP_DIRmkdirrE   ri   rh  ri  rk  ra   relative_torV   rd   r   r2  )$rG   r   r   top_kpromoter   r}  worker_fractionforce_promoter   r   r   r   r   r  r   rq  resolved_workersr   actual_workersbest_overallcur_eval
stored_caleval_calr  r  gate_passingrp  can_promote
best_oos_t	ref_oos_tbear_pctbest_oos_sorref_oos_sorforce_writingr   s$                                       rH   run_tfr$    sR    %R$D.6-4gGJ "oG &-DM$M!'*%!W%+GS_oV1"jBRSG^LLZL.1:LHH$OH  9m//FG
hll>5AB>hc)99B41(3 @$S) *EFMQ
  ':w!,q/AwL:*<?DT8];K <H,@$((+=">DE
m)r../ABCHX\\"45s;
 	 DHH^4c:'I,GHWhLh#DHH-@$A5IL&9:EB--2223FGOK k,===$RD(:<:L M&s+51B0C D%%/LykRUVXZ 	 "GlGx47GMOOD4O0
?wat4P
PCLL%!((E(:KMR&R&S_& 	>& 	c#//$/0	&
 	D'1kCmD& 	d;'& 	((,,~6& 	((,,~6& 	88<<7& 	Hx||,?@RTRXRX& 	txx)& 	!34& 	dhhBC& 	TXX@A& 	TXXMN&  	dhh:;!&" 	dhh'=>#&$ 	txx/%&& 	DHH]+'&( 	"23)&* 	txx/+&, 	txx/-&. 	$((;//&0 	DHH]31&2 	DHH]33&4 	$((;/5&6 	07&8 	19&: 	488$67;&< 	!23=&> 	DHH%89?&@ 	TXXn5A&B 	DHH]3C&F 	<++O<G&H '**>:&**>:K& &K ;s   V%V%c                 j   t        j                  d      } | j                  dt        d       | j                  ddt        t               | j                  d	t
        d
d       | j                  dt
        d       | j                  dt
        d       | j                  ddg d       | j                  dt
        dd       | j                  dt        t        d       | j                  ddd       | j                  ddd        | j                  d!dd"       | j                  d#t        d d$       | j                  d%t        d d&       | j                  d't        d d(       | j                  d)t        d d*       | j                  d+dt         j                         | j                  d,d t         j                         | j                  d-d t         j                         | j                  d.d t         j                         | j                         }|j                  a|j                  rs|j                  r|j                  r|j                  s| j!                  d/       t#        |j                  t%        |j                        t%        |j                               y t&        j)                  d0d01       g }t+        |j,                        D ]  \  }}t/        d2| d3|j0                  d4d5d06       |j3                  t5        ||j0                  |j6                  |d7z  z   |j8                  |j:                  xs |j<                  |j>                  |j@                  |jB                  |j<                  |jD                  |jF                  |jH                  |jJ                  |jL                  8              t&        d9z  }tO        jP                  |      jS                  |d:;       t/        d<|jU                  tV                      t/        tO        jP                  |      jY                  d:;             y )=NzRun deeper MLP threshold sweeps)descriptionr  zAAsset identifier, e.g. COINBASE_ETHUSD (default: COINBASE_BTCUSD))rm   helpz--timeframes+)nargsrm   choicesz	--samplesiP  z$Approximate candidates per timeframe)typerm   r'  z--seedi
'5)r+  rm   z--top-ki  z--extra-weights*z`Additional MLP artifact JSONs to include as candidate weight files for every selected timeframe.)r)  rm   r'  z	--workersr   zWorker processes for candidate evaluation. Use 0 for auto, currently 50%% of logical CPUs capped by candidate count; use 1 for sequential evaluation.z--worker-fractionz+Logical CPU fraction used when --workers=0.z	--promote
store_truezWrite improved winner CSVs)actionr'  z--force-promotez~Write best gate-passing candidate even when no prior winner exists (use for first-ever run on a new asset). Implies --promote.z--wideuN   Double the threshold search ranges: entry ±600, exit/conf ±750, trail 0-100.z
--entry-loz%Override entry threshold lower bound.z
--entry-hiz%Override entry threshold upper bound.z	--exit-loz$Override exit threshold lower bound.z	--exit-hiz$Override exit threshold upper bound.r  r  r  r  z2internal shard mode requires tf, input, and outputTr  z
=== r
  r  z requested samples ===r  i  )r  r   r   r   r   r   zmlp_deep_sweep_summary.csvFrf  z
Summary written: )-argparseArgumentParseradd_argument_DEFAULT_ASSETTFSr   rk   r   SUPPRESS
parse_argsrW  rE   _internal_shard	_shard_tf_shard_input_shard_outputerrorr  r   
REPORT_DIRr  r   
timeframesr-  r   r   r$  r   r  r  r  r   r}  r  r   r   r   r   r   ri   rh  ri  r  rV   	to_string)parserargs	summariesoffsetrG   summary_paths         rH   mainrC  q  s   $$1RSF
	>`  b
c3L
#uCij
sH=
	T:
)b  A
#q@  A +%AXJ  L
L?[\
),[  \ m  o
5$Elm
5$Elm
5$Ekl
5$Ekl
+LxGXGXY
t(:K:KL
($X=N=NO
)4h>O>OPD JJE~~T%6%6d>P>PLLMN4>>40A0A+BDI[I[D\]TD1I0
rd"T\\!,,BC4PDLL$))ftm2K $

DLL,ND<N<N $ 2 2DLL$BVBV.2.@.@tyy)-(,dllL 	M 1  <<LLL""<u"=	 8 8 >?
@A	",,y
!
+
+%
+
89rJ   __main__)rG   ra   r   r   )rG   ra   r   ra   )re   r   r   rd   )r}   )re   r   rm   rk   r   rk   )rG   ra   r   dict[str, Any] | None)r   ra   r   r   )
r   rk   r   rk   r   rk   r   rk   r   rk   )r   r   r   tuple[Any, ...])r   r   r   r   )NFNNNN)rG   ra   r   r   r   r   r   list[str] | Noner   rd   r   float | Noner   rH  r   rH  r   rH  r   list[dict[str, Any]])rG   ra   r   ra   r   z7tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray])r   z	pd.Seriesr   r   r   rk   r   r   )r  zpd.DataFramer  ra   r  ra   r   rk   )rG   ra   r   r   r6  zBdict[str, tuple[pd.DataFrame, np.ndarray, np.ndarray, np.ndarray]]r   r   )rG   ra   rW  ra   r   r   )r   r   r   r   )rw   r   r   rF  )rG   ra   rw   r   r   r   rh   )rp  r   r   rE  rq  rE  r   rd   )r|  r   r}  r   r   zlist[tuple[int, int]])T)rG   ra   r  rI  r  rd   r   rI  )ru   r   r  r   r   r   )rG   ra   r  r   r  r   r   r   )rG   ra   r  rI  r}  r   r   z tuple[list[dict[str, Any]], int])rG   ra   r   r   r   r   r  r   r  rd   r   rG  r}  r   r  rk   r  rd   r   rd   r   rH  r   rH  r   rH  r   rH  r   r   )r   r   )a__doc__
__future__r   r/  concurrent.futuresr  rq   r   r4  multiprocessingr  r  pathlibr   r  r  typingr   numpyr   pandasri   r  r  r  parentrV   ru   r  ra   configr   r   r   r	   r
   r   r   r   r   r   strategies.strategy_mlp_scoresr   r   r   r   r   r   r   tools.worker_utilsr   r   r2  rE   r3  rD   r  r;  r#   __annotations__r$   r3  r1  ro  rm  rn  rI   rL   r[   rf   rn   rx   r   r   r   r   r   r   r   r  objectr.  rU  rX  r[  rc  rk  ry  r  r  r  r  r  r  r$  rC  __name__r"  rJ   rH   <module>rY     s   #   
 	          
H~&&-- 3t9     M #%I	)
9x'	I	)

J [_X _
"   TB/,&& puPTNR_"._AM_!-_?K_Wk_Db.
&N (m`706 7;636?C6r$ $  	4[#[#$[# [# &	[#|> .2!$;!&U$(4#'~*~<?~!~ ~ /3~ "	~ 5A	~
 !~
 3?~
 KY~B7:t zF rJ   