fix(qav3): keep optuna HPO values on a per pair basis

Signed-off-by: Jérôme Benoit <jerome.benoit@piment-noir.org>

diff --git a/quickadapter/user_data/config-template.json b/quickadapter/user_data/config-template.json
index 13effa223cb2ad93ada0f506781600ea4b65c467..d73e690553482226762d632e55172060a6c48bcd 100644 (file)
--- a/quickadapter/user_data/config-template.json
+++ b/quickadapter/user_data/config-template.json
@@ -129,7 +129,9 @@
       "DI_value_param3": 0,
       "DI_cutoff": 2,
       "&s-minima_sort_threshold": -2,
-      "&s-maxima_sort_threshold": 2
+      "&s-maxima_sort_threshold": 2,
+      "label_period_candles": 100,
+      "rmse": 0
     },
     "feature_parameters": {
       "include_corr_pairlist": [

diff --git a/quickadapter/user_data/freqaimodels/LightGBMRegressorQuickAdapterV35.py b/quickadapter/user_data/freqaimodels/LightGBMRegressorQuickAdapterV35.py
index e299b4baa81e289a1851b0e935b4961afac084e1..78b666c7519731303d984e794ce392d345eb1775 100644 (file)
--- a/quickadapter/user_data/freqaimodels/LightGBMRegressorQuickAdapterV35.py
+++ b/quickadapter/user_data/freqaimodels/LightGBMRegressorQuickAdapterV35.py
@@ -46,7 +46,7 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
             and self.__optuna_config.get("enabled", False)
             and self.data_split_parameters.get("test_size", TEST_SIZE) > 0
         )
-        self.__optuna_hp = {}
+        self.__optuna_hp: dict[str, dict] = {}
 
     def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
         """
@@ -69,7 +69,7 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
 
         start = time.time()
         if self.__optuna_hyperopt:
-            study_name = str(dk.pair)
+            study_name = dk.pair
             storage = self.get_optuna_storage(dk)
             pruner = optuna.pruners.HyperbandPruner()
             study = optuna.create_study(
@@ -103,28 +103,30 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
                 gc_after_trial=True,
             )
 
-            self.__optuna_hp = study.best_params
+            if dk.pair not in self.__optuna_hp:
+                self.__optuna_hp[dk.pair] = {}
+
+            self.__optuna_hp[dk.pair]["rmse"] = study.best_value
+            self.__optuna_hp[dk.pair].update(study.best_params)
             # log params
-            for key, value in self.__optuna_hp.items():
+            for key, value in self.__optuna_hp[dk.pair].items():
                 logger.info(f"Optuna hyperopt | {key:>20s} : {value}")
-            logger.info(
-                f"Optuna hyperopt | {'best objective value':>20s} : {study.best_value}"
-            )
 
-            train_window = self.__optuna_hp.get("train_period_candles")
+            train_window = self.__optuna_hp[dk.pair].get("train_period_candles")
             X = X.tail(train_window)
             y = y.tail(train_window)
             train_weights = train_weights[-train_window:]
 
-            test_window = self.__optuna_hp.get("test_period_candles")
+            test_window = self.__optuna_hp[dk.pair].get("test_period_candles")
             X_test = X_test.tail(test_window)
             y_test = y_test.tail(test_window)
             test_weights = test_weights[-test_window:]
 
+            # FIXME: find a better way to propagate optuna computed params to strategy
             if dk.pair not in self.freqai_info["feature_parameters"]:
                 self.freqai_info["feature_parameters"][dk.pair] = {}
             self.freqai_info["feature_parameters"][dk.pair]["label_period_candles"] = (
-                self.__optuna_hp.get("label_period_candles")
+                self.__optuna_hp[dk.pair].get("label_period_candles")
             )
 
         eval_set, eval_weights = self.eval_set_and_weights(X_test, y_test, test_weights)
@@ -171,7 +173,7 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
                 self.__optuna_hyperopt
                 and pair in self.freqai_info["feature_parameters"]
             ):
-                label_period_candles = self.__optuna_hp.get(
+                label_period_candles = self.__optuna_hp.get(pair, {}).get(
                     "label_period_candles", self.ft_params["label_period_candles"]
                 )
             else:
@@ -214,6 +216,15 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
         dk.data["extra_returns_per_train"]["DI_value_param3"] = f[2]
         dk.data["extra_returns_per_train"]["DI_cutoff"] = cutoff
 
+        dk.data["extra_returns_per_train"]["label_period_candles"] = (
+            self.__optuna_hp.get(pair, {}).get(
+                "label_period_candles", self.ft_params["label_period_candles"]
+            )
+        )
+        dk.data["extra_returns_per_train"]["rmse"] = self.__optuna_hp.get(pair, {}).get(
+            "rmse", 0
+        )
+
     def eval_set_and_weights(self, X_test, y_test, test_weights):
         if self.data_split_parameters.get("test_size", TEST_SIZE) == 0:
             eval_set = None
@@ -228,13 +239,11 @@ class LightGBMRegressorQuickAdapterV35(BaseRegressionModel):
         storage_dir = str(dk.full_path)
         storage_type = self.__optuna_config.get("storage_type", "sqlite")
         if storage_type == "sqlite":
-            storage = (
-                f"sqlite:///{storage_dir}/optuna-{sanitize_path(str(dk.pair))}.sqlite"
-            )
+            storage = f"sqlite:///{storage_dir}/optuna-{sanitize_path(dk.pair)}.sqlite"
         elif storage_type == "file":
             storage = optuna.storages.JournalStorage(
                 optuna.storages.journal.JournalFileBackend(
-                    f"{storage_dir}/optuna-{sanitize_path(str(dk.pair))}.log"
+                    f"{storage_dir}/optuna-{sanitize_path(dk.pair)}.log"
                 )
             )
         return storage

diff --git a/quickadapter/user_data/freqaimodels/XGBoostRegressorQuickAdapterV35.py b/quickadapter/user_data/freqaimodels/XGBoostRegressorQuickAdapterV35.py
index b5de25057da5c090ab8f0e5f5ed688c9809a0474..2f82bd2fea0272f41114dd8bbdf762c2a037b9e2 100644 (file)
--- a/quickadapter/user_data/freqaimodels/XGBoostRegressorQuickAdapterV35.py
+++ b/quickadapter/user_data/freqaimodels/XGBoostRegressorQuickAdapterV35.py
@@ -46,7 +46,7 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
             and self.__optuna_config.get("enabled", False)
             and self.data_split_parameters.get("test_size", TEST_SIZE) > 0
         )
-        self.__optuna_hp = {}
+        self.__optuna_hp: dict[str, dict] = {}
 
     def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
         """
@@ -73,7 +73,7 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
 
         start = time.time()
         if self.__optuna_hyperopt:
-            study_name = str(dk.pair)
+            study_name = dk.pair
             storage = self.get_optuna_storage(dk)
             pruner = optuna.pruners.HyperbandPruner()
             study = optuna.create_study(
@@ -107,28 +107,30 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
                 gc_after_trial=True,
             )
 
-            self.__optuna_hp = study.best_params
+            if dk.pair not in self.__optuna_hp:
+                self.__optuna_hp[dk.pair] = {}
+
+            self.__optuna_hp[dk.pair]["rmse"] = study.best_value
+            self.__optuna_hp[dk.pair].update(study.best_params)
             # log params
-            for key, value in self.__optuna_hp.items():
+            for key, value in self.__optuna_hp[dk.pair].items():
                 logger.info(f"Optuna hyperopt | {key:>20s} : {value}")
-            logger.info(
-                f"Optuna hyperopt | {'best objective value':>20s} : {study.best_value}"
-            )
 
-            train_window = self.__optuna_hp.get("train_period_candles")
+            train_window = self.__optuna_hp[dk.pair].get("train_period_candles")
             X = X.tail(train_window)
             y = y.tail(train_window)
             train_weights = train_weights[-train_window:]
 
-            test_window = self.__optuna_hp.get("test_period_candles")
+            test_window = self.__optuna_hp[dk.pair].get("test_period_candles")
             X_test = X_test.tail(test_window)
             y_test = y_test.tail(test_window)
             test_weights = test_weights[-test_window:]
 
+            # FIXME: find a better way to propagate optuna computed params to strategy
             if dk.pair not in self.freqai_info["feature_parameters"]:
                 self.freqai_info["feature_parameters"][dk.pair] = {}
             self.freqai_info["feature_parameters"][dk.pair]["label_period_candles"] = (
-                self.__optuna_hp.get("label_period_candles")
+                self.__optuna_hp[dk.pair].get("label_period_candles")
             )
 
         eval_set, eval_weights = self.eval_set_and_weights(X_test, y_test, test_weights)
@@ -174,7 +176,7 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
                 self.__optuna_hyperopt
                 and pair in self.freqai_info["feature_parameters"]
             ):
-                label_period_candles = self.__optuna_hp.get(
+                label_period_candles = self.__optuna_hp.get(pair, {}).get(
                     "label_period_candles", self.ft_params["label_period_candles"]
                 )
             else:
@@ -217,6 +219,15 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
         dk.data["extra_returns_per_train"]["DI_value_param3"] = f[2]
         dk.data["extra_returns_per_train"]["DI_cutoff"] = cutoff
 
+        dk.data["extra_returns_per_train"]["label_period_candles"] = (
+            self.__optuna_hp.get(pair, {}).get(
+                "label_period_candles", self.ft_params["label_period_candles"]
+            )
+        )
+        dk.data["extra_returns_per_train"]["rmse"] = self.__optuna_hp.get(pair, {}).get(
+            "rmse", 0
+        )
+
     def eval_set_and_weights(self, X_test, y_test, test_weights):
         if self.data_split_parameters.get("test_size", TEST_SIZE) == 0:
             eval_set = None
@@ -231,13 +242,11 @@ class XGBoostRegressorQuickAdapterV35(BaseRegressionModel):
         storage_dir = str(dk.full_path)
         storage_type = self.__optuna_config.get("storage_type", "sqlite")
         if storage_type == "sqlite":
-            storage = (
-                f"sqlite:///{storage_dir}/optuna-{sanitize_path(str(dk.pair))}.sqlite"
-            )
+            storage = f"sqlite:///{storage_dir}/optuna-{sanitize_path(dk.pair)}.sqlite"
         elif storage_type == "file":
             storage = optuna.storages.JournalStorage(
                 optuna.storages.journal.JournalFileBackend(
-                    f"{storage_dir}/optuna-{sanitize_path(str(dk.pair))}.log"
+                    f"{storage_dir}/optuna-{sanitize_path(dk.pair)}.log"
                 )
             )
         return storage

diff --git a/quickadapter/user_data/strategies/QuickAdapterV3.py b/quickadapter/user_data/strategies/QuickAdapterV3.py
index fbe6d12131cf1962806f5832e7787038ed2a43a6..19f13f2a5cab4f33b4aeb2950d5ce30951eea4ea 100644 (file)
--- a/quickadapter/user_data/strategies/QuickAdapterV3.py
+++ b/quickadapter/user_data/strategies/QuickAdapterV3.py
@@ -68,7 +68,7 @@ class QuickAdapterV3(IStrategy):
     plot_config = {
         "main_plot": {},
         "subplots": {
-            # "rmse": {"rmse": {"color": "#c28ce3", "type": "line"}},
+            "rmse": {"rmse": {"color": "#c28ce3", "type": "line"}},
             "extrema": {
                 "&s-extrema": {"color": "#f53580", "type": "line"},
                 "&s-minima_sort_threshold": {"color": "#4ae747", "type": "line"},