From 660dbaa102caba239782108757b948af29713692 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Sat, 14 Jun 2025 17:22:24 +0200 Subject: [PATCH] fix(qav3): add training hard requirements over pivots MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit also add debugging, enabled for now in the model Signed-off-by: Jérôme Benoit --- .../freqaimodels/QuickAdapterRegressorV3.py | 90 +++++++++++++++---- .../user_data/strategies/QuickAdapterV3.py | 11 ++- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py index d5f6bd6..c5ac4b9 100644 --- a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py +++ b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py @@ -19,6 +19,7 @@ from pathlib import Path from freqtrade.freqai.base_models.BaseRegressionModel import BaseRegressionModel from freqtrade.freqai.data_kitchen import FreqaiDataKitchen +debug = True TEST_SIZE = 0.1 @@ -301,15 +302,19 @@ class QuickAdapterRegressorV3(BaseRegressionModel): optuna_train_params = self.get_optuna_params(dk.pair, "train") if optuna_train_params: - train_window = optuna_train_params.get("train_period_candles") - X = X.iloc[-train_window:] - y = y.iloc[-train_window:] - train_weights = train_weights[-train_window:] - - test_window = optuna_train_params.get("test_period_candles") - X_test = X_test.iloc[-test_window:] - y_test = y_test.iloc[-test_window:] - test_weights = test_weights[-test_window:] + value: float = optuna_train_params.get("value") + if isinstance(value, float) and np.isfinite(value): + train_window = optuna_train_params.get("train_period_candles") + if isinstance(train_window, int) and train_window > 0: + X = X.iloc[-train_window:] + y = y.iloc[-train_window:] + train_weights = train_weights[-train_window:] + + test_window = optuna_train_params.get("test_period_candles") + if isinstance(test_window, int) and test_window > 0: + X_test = X_test.iloc[-test_window:] + y_test = y_test.iloc[-test_window:] + test_weights = test_weights[-test_window:] eval_set, eval_weights = self.eval_set_and_weights(X_test, y_test, test_weights) @@ -1079,30 +1084,79 @@ def train_objective( candles_step: int, model_training_parameters: dict, ) -> float: - min_test_window: int = fit_live_predictions_candles * 4 - if len(X_test) < min_test_window: - logger.warning(f"Insufficient test data: {len(X_test)} < {min_test_window}") - return np.inf - max_test_window: int = len(X_test) + test_ok = True + test_length = len(X_test) + if debug: + n_test_minima: int = sp.signal.find_peaks(-y_test[EXTREMA_COLUMN])[0].size + n_test_maxima: int = sp.signal.find_peaks(y_test[EXTREMA_COLUMN])[0].size + n_test_extrema: int = n_test_minima + n_test_maxima + min_test_extrema: int = int( + round((test_length / fit_live_predictions_candles) * 2) + ) + logger.info( + f"{test_length=}, {n_test_minima=}, {n_test_maxima=}, {n_test_extrema=}, {min_test_extrema=}" + ) + min_test_window: int = fit_live_predictions_candles + if test_length < min_test_window: + logger.warning(f"Insufficient test data: {test_length} < {min_test_window}") + test_ok = False + max_test_window: int = test_length test_window: int = trial.suggest_int( "test_period_candles", min_test_window, max_test_window, step=candles_step ) X_test = X_test.iloc[-test_window:] y_test = y_test.iloc[-test_window:] + n_test_minima = sp.signal.find_peaks(-y_test[EXTREMA_COLUMN])[0].size + n_test_maxima = sp.signal.find_peaks(y_test[EXTREMA_COLUMN])[0].size + n_test_extrema = n_test_minima + n_test_maxima + min_test_extrema: int = int(round((test_window / fit_live_predictions_candles) * 2)) + if n_test_extrema < min_test_extrema: + if debug: + logger.warning( + f"Insufficient extrema in test data with {test_window}: {n_test_extrema} < {min_test_extrema}" + ) + test_ok = False test_weights = test_weights[-test_window:] + train_ok = True + train_length = len(X) + if debug: + n_train_minima: int = sp.signal.find_peaks(-y[EXTREMA_COLUMN])[0].size + n_train_maxima: int = sp.signal.find_peaks(y[EXTREMA_COLUMN])[0].size + n_train_extrema: int = n_train_minima + n_train_maxima + min_train_extrema: int = int( + round((train_length / fit_live_predictions_candles) * 2) + ) + logger.info( + f"{train_length=}, {n_train_minima=}, {n_train_maxima=}, {n_train_extrema=}, {min_train_extrema=}" + ) min_train_window: int = min_test_window * int(round(1 / test_size - 1)) - if len(X) < min_train_window: - logger.warning(f"Insufficient train data: {len(X)} < {min_train_window}") - return np.inf - max_train_window: int = len(X) + if train_length < min_train_window: + logger.warning(f"Insufficient train data: {train_length} < {min_train_window}") + train_ok = False + max_train_window: int = train_length train_window: int = trial.suggest_int( "train_period_candles", min_train_window, max_train_window, step=candles_step ) X = X.iloc[-train_window:] y = y.iloc[-train_window:] + n_train_minima = sp.signal.find_peaks(-y[EXTREMA_COLUMN])[0].size + n_train_maxima = sp.signal.find_peaks(y[EXTREMA_COLUMN])[0].size + n_train_extrema = n_train_minima + n_train_maxima + min_train_extrema: int = int( + round((train_window / fit_live_predictions_candles) * 2) + ) + if n_train_extrema < min_train_extrema: + if debug: + logger.warning( + f"Insufficient extrema in train data with {train_window} : {n_train_extrema} < {min_train_extrema}" + ) + train_ok = False train_weights = train_weights[-train_window:] + if not test_ok or not train_ok: + return np.inf + model = fit_regressor( regressor=regressor, X=X, diff --git a/quickadapter/user_data/strategies/QuickAdapterV3.py b/quickadapter/user_data/strategies/QuickAdapterV3.py index 60db753..bb2db50 100644 --- a/quickadapter/user_data/strategies/QuickAdapterV3.py +++ b/quickadapter/user_data/strategies/QuickAdapterV3.py @@ -14,6 +14,7 @@ from technical.pivots_points import pivots_points from freqtrade.persistence import Trade import numpy as np import pandas_ta as pta +import scipy as sp from Utils import ( alligator, @@ -34,6 +35,8 @@ from Utils import ( zlema, ) +debug = False + logger = logging.getLogger(__name__) EXTREMA_COLUMN = "&s-extrema" @@ -375,7 +378,7 @@ class QuickAdapterV3(IStrategy): ) def set_label_natr_ratio(self, pair: str, label_natr_ratio: float): - if isinstance(label_natr_ratio, float) and not np.isnan(label_natr_ratio): + if isinstance(label_natr_ratio, float) and np.isfinite(label_natr_ratio): self._label_params[pair]["label_natr_ratio"] = label_natr_ratio def get_entry_natr_ratio(self, pair: str) -> float: @@ -432,6 +435,12 @@ class QuickAdapterV3(IStrategy): dataframe[EXTREMA_COLUMN], self.freqai_info.get("extrema_smoothing_window", 5), ) + if debug: + logger.info(f"{dataframe[EXTREMA_COLUMN].to_numpy()=}") + n_minima = sp.signal.find_peaks(-dataframe[EXTREMA_COLUMN])[0].size + n_maxima = sp.signal.find_peaks(dataframe[EXTREMA_COLUMN])[0].size + n_extrema = n_minima + n_maxima + logger.info(f"{n_extrema=}") return dataframe def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: -- 2.43.0