ƫ9ju PddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZej$j'ej$j)ej$j+ej$j-edddlmZddlmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z%m&Z'dZ(dZ)dZ*dZ+d Z,d Z-d Z.d Z/d dddddZ0dZ1e,fdZ2e-fdZ3dZ4dZ5dZ6 ddl7Z7ddl8m9Z9dZ;dZdZ?dZ@eAd k(re@yy#e:$rdZ7dZ9Y)wxYw)!Nz..)library_activation_scores) RESULTS_DIR WINNERS_DIR SWEEPS_DIR REPORTS_DIR SCORE_START TRAIN_ENDMVRV_BEAR_THRESHOLDMVRV_BULL_THRESHOLDcomposite_scoreget_min_tradeszresults/sweep_database.dbzresults/optuna_studiesg?皙?i@KLi = calmar_ratio sortino_ratio sharpe_ratio pnl_dd_ratior ) Calmar Ratio Sortino Ratio Sharpe Ratio P&L/DD Ratio Compositec||dk(r|Sd|jvrtd|d|S|d}|dk(r||k}n|dk(r||k\}n ||k\||kz}t|}|j|j j d}td |d |d |d t|d |dt||z dzdd |S)zFilter df to only bars matching the requested regime using the 'zscore' column. Returns filtered copy. If 'zscore' column is absent, warns and returns df unchanged.allzscorez[WARNING] --regime=zK requested but 'zscore' column not found in data; no regime filter applied.bearbullTdropz[regime=z ] bear_thr=z bull_thr=z: kept /z IS bars (dz.1fz%))columnsprintlenloccopy reset_index)dfregimebear_thresholdbull_thresholdzmaskn_befores G/Users/jameslopez/projects/TradingBot25/strategies/optimize_strategy.py_apply_regime_maskr13s rzz! #F8+vwx  8 A >! 6 N"^#N(:;2wH     ( (d ( 3B HVHK'7z.AQRb'!H:ZB0@0DS/I MN Ic\tjjtst j S t jt}t jd|d|d|||t|f}|j|S#t$rt j cYSwxYw)zEReturn a DataFrame of top_k rows for this asset/TF from the sweep DB.zOSELECT * FROM sweep_results WHERE asset=? AND timeframe=? AND schema_ver=? AND z >0 ORDER BY z DESC LIMIT ?)params) ospathexists SWEEP_DB_FILEpd DataFramesqlite3connectread_sql DB_SCHEMA_VERclose Exception)asset timeframedb_coltop_kconnr)s r0_load_db_top_resultsrFIs 77>>- (||~ }- [[BBHJx} . %M5A    ||~sAB B+*B+c$|jst|dkriStdtt||z}|j |}i}t ||D]\}}|j |} | | jjr8| jjjt} t| dkrttj| ddg\} } ||| k\|| kz} t| dkr| ||<|S)a: Compute biased sampling ranges from the top fraction of historical results. Returns dict: param_name -> (biased_vals_array | None). biased_vals_array contains the subset of the param grid that falls in the p5-p95 range of the top performers. None means no meaningful signal (use full range).  _)emptyr%maxintheadzipgetisnardropnavaluesastypefloatnp percentile)df_top param_namesparam_vals_arraystop_fracn_top top_subsetbiasednamevalscol col_cleanp5p95 biased_valss r0_compute_biased_rangesrh[s||s6{R' CF h./ 0EU#J F+'89 dnnT" ;#((*..* JJL''..u5 y>A  -- Ar73CDBJ43;78 { a  "t : Mr2c  ddlm}m}i}|j D]\}}d|vr7|d}t |dkDs||D cgc] } t | c} ||<Atj|d|d|d}t |dkDsp|t |dt |dt |d ||<|S#t$ricYSwxYwcc} w) z$ 12tyy7Ltxx499 <=DHHw'::A>$((C 3$78F4L(   MM&,,33U%,4 -$8 6{R V L] P   s#3H="6I = I I  IIc ^ ddl}|jjd}|jj||}t jtdt jjtd |d |d }d |d |dtjjjdd|dt|jdddg} t!|j#dD]B\} \} } dt%dt'| dzz} | j)d| d| d| dd | d Dt+|d5}|j-dj| dzdddt d|y#t$r}t d|Yd}~yd}~wwxYw#1swYktSc!U38_-- r$tD6c#YauB?@? h   % 4'(  .xj 9:)  8<=$  s#=E?$F#? F FF #F,)cudac ,g}tr\|jdtjddd|jdtjjdd gd}t j |ddd }|jd k(rg|jjjd Dcgc] }t|c}\}}}}|jd |d|d||z dzdddj|Scc}w#YxYw)z"Retrieves CPU, RAM, and GPU stats.zCPU: N)intervalz3.0f%zRAM: )z nvidia-smizD--query-gpu=temperature.gpu,utilization.gpu,memory.used,memory.totalz--format=csv,noheader,nounitsTr)capture_outputtexttimeoutr,zGPU: zC z % | VRAM: r".0fz | ) psutilr cpu_percentvirtual_memorypercent subprocessrun returncodestdoutstripsplitrOr)statscmdprocxtemputilmem_used mem_totals r0get_hardware_statsrs  E  uV//>tDAFG uV224<@c:tj}||k\}|s|tz tkry|a||z }|dkDr||z nd}|dkDr||z |z nd} tt j t | } |dd} d||z dzddt|d t|d | d | d |d d|d dt} tjjd| d|rtjjdtjjy)Nr)secondsz,.0fz it/sz Progress: r"z5.1fz% (r!z ) | Rate: z | ETA: z | Best GPU Calmar: rz (vs z) |  z r) time_last_progress_timePROGRESS_INTERVALstrr timedeltarOrrsysrrflush) iterationtotal start_time best_scorebaseline_scoreris_finalelapsedrate remainingetarate_strlines r0print_progressrs{ ))+CE!H 226GGJG")A+9w 1D.2Qh"d*AI h  Y8 9CtE"H  %+D15H5S4TTUVijoVpUqq{}E|FFNORNSSghrsvgww|}KLO|PPTUgUiTj kDJJr$t_% JJr2c 'hijtjd}|jdtdd|jdtdd|jd td |jd td d|jddd|jdtdgdd|jdt dd|jdtdddgd|jd tdd!|jd"tdd#|jd$td%gd&d'|jd(t t d)t d*|jd+t td,td*|j}d-d.d/d0d1d2}tjjtjj|jd3}d4|vr8|jd4d5\}}||j||j!}}nJ|jd6d5}t#|d7k(r|d3n|}t#|d7k(r|d5j!nd1}|j$s||_|j&s||_|j$d8|j&|_tjj+tjj-t.} tjj1| tjj|j2} tjj5| s:|j2} tjj5| st7d9| yt8j:j=d:| } t8j:j?| } | j@jC| t7d;|jd<d=tEjF|j} | jHjjKjjM| _$tjjtjj|j2d3}|jOd>d?}|jPxsd@|dA}tS|dB5}tUjV|}dddggcijjYD]\}}ij[|dC|vrjj[|dC0dD|vs5t]j^|dD|dE|dF}jj[|Dcgc]}ta|dGc}xs|dDgdHivr5dIivr1ijcdH}j|d3gk(rijcdI}dJgj|<tjdtfdKtjdthdKtjdtjdKtjj1thdL|d8|j(dM}|jl}|}t7dN|dOdP|jndQd5}jD]}|t#|z}|jpd3kDxr||jpkD}|r |jpn|}|jrdk(} | rdRn|rdSndT}!t7dU|!dV|dWdX|jndYd=tjj1tjdZ|d8|j(dM}"tjj5|"rtjt|"d}#d}$t d[ }%twjv}&g}'d\}(|jxr]t{| d]rPt|})d3}*t7d^d=| j}+d_|+jHvrxtEj|+d_d`jjd|+d_<|+d_dak\|+d_tkz},|+j|,jjdb}+t|+|j|j|j}+| j|+}-t#|+}.iD/cgc]}/|/jdcs|/}0}/iD/cgc] }/dd|/vs|/ }1}/iD/cgc]}/|/jdes|/}2}/jDcgc]}t]j|}3}tijD4cic]\}4}t#|d5k(s|4|d3}5}4}tj|jndf}6i}7t}8| rt|j$|j&|6}9|9jsbt|9i|3}7tdg|7jD}:t7dht#|9di|:djt#idktdlzdmdn d=n t7dod= |*|krt|)||*z };t]j|;t#|0ft\jp}g}?tti|3D]\}@\}}| r|7j|nd}A|At#Ad7k\rt |;tz}B|;|Bz }CAt\jjd3t#|A|Bq}D|t\jjd3t#||Cq}Et]j|;t\jp}F|D|Fd|B|E|F|Bdt\jj|Fn.|t\jjd3t#||;q}F|?j[F||0vrF|dd|2jc|f<| j|-|<|=|>}Gt|;D]}HG|H\}I}J}K}L}M|Mdrk\r*JdskDr%dtIduz zdvkDrdtIduz zdw|.z zdtz duz}N|NJz }Ondx}O|jndyvrO}Pn|jndzk(rL}PnK}Pt]jPs{P|%kDrP}%tt#iDQcic]}Qi|Q|?|QH}R}Qi|RIJKLOMd{}$P|kDr^P}|$}#tEj|#gjtjj1thd||d8|j(dMd}~t#|'|(ks P|'d3d3kDs8tt#iDQcic]}Qi|Q|?|QH}S}Q|SjKLOMdt#|'|(krtj|'P|*HzSftj|'P|*HzSf|*|;z }*| r|*|8k\r|'r|8tz }8|'DTcgc]}T|Td7 }U}TtEj|U}V|Vj|jn|Vj|6|V|6<t|Vi|3}W|Wr7W}7td|7jD}Xt7d|*dWd|Xdd=t|*||&|||*|krny| j}Zd_|ZjHvrhtEjZd_d`jjd|Zd_<|Zd_dak\|Zd_tkz},|Zj|,j}ZtZ|j|j|j}Z|rijfdt|DnifdtjjD}[t|[D]\}H}\Zj}]| j|]fi|\}^| j|^}_|_j|jnd}P|P|%kDrP}%i\_}$|P|kDrP}|$}#tHd5z||&|||Hd5z|k\snt7d|'r|jndk(rdn |jnht|'DTcgc]}T|Td7 c}Thfdd}`tEj|`j|"d}~t7dt#|`d|"|$retEj|$gjtjj1thd||d8|j(dMd}~t7d|#r||kDrt7d|jndnt7d|jnd| rX|'rUt7dd=|'DTcgc]}T|Td7 }a}TtEj|a}btj|jndf}c|bj|jn|bj|c|b|c<t|j$|j&|cd}dtEj|bjd|dgdj}etijD4cic]\}4}t#|d5k(s|4|d3}f}4}tei|c|f}g|g-tg|j$|j&|jnn t7dtjjtjjtrtjtjd3y#1swY{xYwcc}wcc}/wcc}/wcc}/wcc}wcc}}4wcc}Qwcc}Qwcc}Tw#ttf$r}Yt7dYYd}Y~YRd}Y~YwwxYwcc}Twcc}Twcc}}4w)NzOptimize Strategy Parameters) descriptionz--dataTzPath to data CSV)typerequiredhelpz--strategy_filez Name of the strategy Python filez --params_filez+Path to JSON file defining parameter ranges)rrz--random_searchi'z$Number of random combinations to try)rdefaultrz--gpu store_truezEnable GPU acceleration)actionrz--metricr)rrrrrzThe metric to optimize for. 'Composite' = Calmar*(1+log_bonus); GPU pre-filters by raw Calmar, composite applied at CPU verification step.)rrchoicesrz--baseline-scorerz1The baseline score for the chosen metric to beat.z--searchrandomrzvSearch strategy: 'random' (pure uniform) or 'optuna' (biased sampling from DB history + post-run importance analysis).z --timeframez\Timeframe suffix for output files (e.g. '1D'). Auto-derived from --data filename if not set.z--assetzcAsset name for output files (e.g. 'COINBASE_BTCUSD'). Auto-derived from --data filename if not set.z--regimer)rrrsidewayszPre-filter IS training bars by MVRV regime. 'bull': zscore >= bull_threshold; 'bear': zscore < bear_threshold; 'sideways': bear_threshold <= zscore < bull_threshold; 'all': no filter (default).z--regime-bear-thresholdzDzscore percentile below which bars are classified as Bear (default: z).z--regime-bull-thresholdzJzscore percentile at or above which bars are classified as Bull (default: 4H6H8H12H1D)240360480720rrz, rl-rLrz"Error: Strategy file not found at strategy_modulezLoading data from z...)r strategy_zstrategies/params_z.jsonrrUrmrnror~i_regime_windowi_regime_entry_min_scoreg@roptimization_winner_z.csvz5Attempting to beat baseline Calmar (GPU pre-filter): rz [CPU optimising for: ]z Optuna-BiasedRandomGridz Starting z Search with rz iterations (optimizing for z)...optimization_sweep_infrexecute_gpu_batchz2Pre-calculating indicators and uploading to GPU...r)utcz 2015-01-01ri_w_ thresholdi_use_rc3&K|] }|d ywNrlr.0rxs r0 zmain..sR*@QAMq*@z [Optuna] Loaded z DB rows; biased ranges for r!z params (r"rz% exploitation).uJ [Optuna] No DB history yet — running as random search this iteration.)dtype)sizer{rg?gY@g& .>gv@g$)rrr)z Total P&L %zMax Drawdown %rrr Total Tradesoptimization_run_best_F)index)rrrr$c3&K|] }|d ywrrrs r0r zmain..s'[3IaQ]3Ir!z( [Optuna] Refreshed biased ranges at z samples (z params directed).z Optimization stopped: c 3K|]:}ttDcgc]}tj|c}<ycc}wwN)dictrQrchoice)rrrxr[param_values_lists r0r zmain..s?ygxbcSIZ.[IZAv}}Q/?IZ.[\]gx.[sAAAc3HK|]}tt|ywr))r*rQ)rcr[s r0r zmain..sMNXrWmnOSTWXcefTgOhrWs"z Optimization Complete.rc(|jdS)Nr)rR)r gpu_sort_keys r0zmain../sTUTYTYZfhiTjr2)keyreversezTop z results saved to zBest result of this run saved.z/ [SUCCESS] GPU found improvement over baseline z" (pending CPU verification + WFO).z+ [INFO] No improvement found over baseline .z5 [Optuna] Running parameter importance analysis...i)rDi) ignore_indexuH [Optuna] Not enough data for importance analysis (need ≥30 trials).)zargparseArgumentParser add_argumentrrOrWr r parse_argsr5r6splitextbasenamedatarsplitrRupperr%rArBasset_tfdirnameabspath__file__r strategy_filer7r$ importlibrspec_from_file_locationmodule_from_specloader exec_moduler9read_csvr#lowerrreplace params_filerjsonloadrsrrXrtrr&rrrrrmetric random_searchsearchremovergpuhasattrGPU_BATCH_SIZEr' to_datetimedt tz_localizer r&r(r1r*regime_bear_thresholdregime_bull_thresholdprepare_gpu_data startswitharrayrQMETRIC_TO_DB_COLOPTUNA_REFRESHrFrMrhsumrUEXPLOIT_FRACTIONrzerosfloat64rrrandintshufflerrangeisfiniter:to_csvupdateheapqheappush heappushpoprKeyboardInterruptr@ itertoolsproductgenerate_signalscalculate_metricssortedconcatrPdrop_duplicatesrrrrrstderrrr?exit)kparserargs _MIN_TO_TFstem asset_partperiod derived_asset derived_tfparts script_dir strategy_pathspecstrategy_signalsr)strategy_basename strategy_namerLrrurarwrbrrw_idxrs_idxwinner_filenameinitial_best_scorer total_combosrx use_random iterations use_optuna search_labelsweep_filename best_resultrun_best_resultrun_best_scorertop_results_heapMAX_SWEEP_RESULTS BATCH_SIZEtotal_processeddf_gpur.gpu_static_dataTp weight_namesthreshold_names config_namesr\rrrC biased_rangesnext_refresh_at df_db_topn_biasedcurrent_batch_sizeweights thresholdsconfigsbatch_values_colsidxbiased_v n_exploit n_explore exploit_col explore_col col_values gpu_resultsipnlddsharpesortinotradesannualised_pctcalmar current_scorejcurrent_paramsres_dictitem heap_recordsdf_heap new_biased n_updatedr df_cpu_base combinationskwargsdf_cpusignalsmetrics sorted_sweep run_recordsdf_run db_col_local df_db_hist df_combinedlocked_params_localrr0r[r,sk @@@r0mainrs8  $ $1O PF sT@RS )dIkl c8ef )UIop  ;TU  nO]dpq *J}~  hS[H\PQ  CEcd  TAfg  eEhrs 1GZcdwcxxz{} 1GZij}i~ABC    D d4TRJ 77  BGG,,TYY7 8 ;D t|![[q1 F$. vv||~0Vz  C#$'J!Oa ),UqU1X^^%d ::" >>#zzl!DNN#34DM!:;JGGLLRWW-=-=d>P>P-QRM 77>>- (** ww~~m, 7 G H  >> 1 12C] SD ~~66t<KK,- tyyk -T: TYY B%%'++113BJ(()9)9$:L:L)MNqQ%--k2>M""S(:;L:MU&SK k3 1 !  &("K"%++- f4 v   $ $VH%5 6  99VG_fVnfVnMD  $ $4%@4aeAqk4%@%UVG_DU V .K',F+,U""#45 V $ + &&'ABF)0 f %KK d+KK d+KK T*ggll;2F}oUVW[WdWdVeei0jkO,,#J ABTUXAYYpquq|q|p}}~ AL A ##a'ML4;M;M,MJ'1##|J(J'1OJTZL Il^=A>Z[_[f[fZggk ltxyWW\\*0CM?RSTXTaTaSbbf.ghN ww~~n%ryy'@KOEl]NJ xxG,.AB#  B$O V^^ #^^F6NEHHTTUYZF6N6Nl2vf~7RSDZZ%**,88d8CF#FDKK9S9SUYUoUop*;;FC K#.G;a!,,v2F; G&1Fk[A5E1kF#.I;a!,,x2H; I8IJ8IRXXd^8IJ-0>O-P`-PTQTWXYTZ^_T_AaD-P `"%%dkk>B ( ,TZZPI?? 6y+O` a R-*>*>*@RR+C N+;;WX`Waabcfgrcsbtu*3.s33CELPRbjno^ 2!J.%(Z/5Q%R"(($6L8I#JRTR\R\]XX'93;O&PXZXbXbc (($6#:"**M$&!)23{DU3V)W%C$:D}006$H+H 0B$'(:=M(M$N $6$B &.ryy/@/@CMXa/@/b&c &*299+<+>rww||K[qrrAABCGCPCPBQQUZV@W_d>e+,/@@MTdefTghiTjDjUZ[^_j[kUl#mUlPQKN4Ea4H4K$KUl#m RY9?QW)YZ/03DD!NN+;m__`M`bj=kl!--.>P_bcPcem@noS3V #55/_"DIY#~5O8H#I8HDG8HL#I ll<8G&-kk$++w{{6?R&SGFO!7N_!`J!(2 $''[=3G3G3I'[$[  I/Z[I\]""+,>@GKM J Tfgw"J.Dggi [(( ("$..V1D$"O"R"R"^"^_c"dK '<7KTRbbd 36{DU3Vf3V41aZ]^_Z`deZeq!A$w3Vf0 m\CV    $UDJJ T \ ]JJJJ  HHQK} &AlHFIJ`R*t$n$J"9- 2 ,QC0 1 1 2HE*=gsAM&.AM3 AM87AM8 AM= AM=AN,AN6AN#AN :AN  H#AN!1CAN!ANB AN!#AN!9AN A>AN! ANBAN!B% AO F6 AOJAOJ&AOM&AM0NAN!N!AO N0AOOAO __main__)Bpandasr9numpyrXrnr5rrerimportlib.utilrDr6rMrrrrjr;r6rrArr@rB strategiesrlibrwrrrrrr r r r _composite_scorer _get_min_tradesr8OPTUNA_STUDIES_DIRr>ra OPTUNA_TOP_KOPTUNA_TOP_FRACr_rUr^r1rFrhryrrrnumbarrrrrrrrr__name__rr2r0rsR    RWW__X-F MNO7OOO1 -  %%$$' ,:F$M\:07t;: $B  zFa  F Ds1 D D%$D%