From bda5bfd388ac11492f36a4f70d182766a8fb3893 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Sun, 26 Jul 2026 21:58:41 +0200 Subject: [PATCH] feat(quickadapter): make HPO trial seed variation configurable (#119) Add freqai.optuna_hyperopt.vary_model_seed_by_trial (default true) to make the per-trial model-seed variation explicit. - true (default) preserves the historical behavior: each HPO trial adds trial.number to the regressor model seed. false uses the same model seed for every trial and the final fit. - Independent from freqai.optuna_hyperopt.seed (Optuna samplers + label-candle shuffling); wired through the canonical defaults, config template, startup logging, and README. - Seed NGBoost's base DecisionTreeRegressor as well: NGBoost.random_state only seeds subsampling/validation split, not the cloned base learner, so the base tree needs its own random_state for reproducibility. BREAKING CHANGE: the optuna_hyperopt boolean options (enabled, continuous, warm_start, space_reduction, vary_model_seed_by_trial) are now validated at the option layer and raise ValueError on non-boolean values instead of relying on Python truthiness; a config passing a non-boolean (e.g. 1 or "false") for these keys must use real booleans. --- README.md | 3 +- quickadapter/user_data/config-template.json | 3 +- .../freqaimodels/QuickAdapterRegressorV3.py | 34 +++++++++++++++++-- quickadapter/user_data/strategies/Utils.py | 4 ++- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1a2415a..5cc51c9 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,8 @@ docker compose up -d --build | freqai.optuna_hyperopt.space_reduction | false | bool | Enable/disable `hp` search space reduction based on previous best parameters. | | freqai.optuna_hyperopt.space_fraction | 0.4 | float [0,1] | Fraction of the `hp` search space to use with `space_reduction`. Lower values create narrower search ranges around the best parameters. | | freqai.optuna_hyperopt.min_resource | 3 | int >= 1 | Minimum resource per [HyperbandPruner](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.pruners.HyperbandPruner.html) rung. | -| freqai.optuna_hyperopt.seed | 1 | int >= 0 | HPO RNG seed. | +| freqai.optuna_hyperopt.seed | 1 | int >= 0 | HPO RNG seed used by the Optuna samplers and label-candle shuffling. | +| freqai.optuna_hyperopt.vary_model_seed_by_trial | true | bool | Add `trial.number` to each regressor's configured model seed (or its default seed of `1`) during HPO. `true` samples model randomness across trials and preserves the historical behavior; `false` evaluates every trial and the final fit with the same model seed. This does not change `freqai.optuna_hyperopt.seed`. | ## ReforceXY diff --git a/quickadapter/user_data/config-template.json b/quickadapter/user_data/config-template.json index 8a87d49..7bdff38 100644 --- a/quickadapter/user_data/config-template.json +++ b/quickadapter/user_data/config-template.json @@ -171,7 +171,8 @@ "n_trials": 100, "timeout": 7200, "label_candles_step": 1, - "storage": "file" + "storage": "file", + "vary_model_seed_by_trial": true }, "extra_returns_per_train": { "DI_value_param1": 0, diff --git a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py index afa2b87..b96ef9d 100644 --- a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py +++ b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py @@ -427,6 +427,15 @@ class QuickAdapterRegressorV3(BaseRegressionModel): OPTUNA_SPACE_REDUCTION_DEFAULT: Final[bool] = False OPTUNA_SPACE_FRACTION_DEFAULT: Final[float] = 0.4 OPTUNA_SEED_DEFAULT: Final[int] = 1 + OPTUNA_VARY_MODEL_SEED_BY_TRIAL_DEFAULT: Final[bool] = True + + _OPTUNA_BOOL_OPTIONS: Final[tuple[str, ...]] = ( + "enabled", + "continuous", + "warm_start", + "space_reduction", + "vary_model_seed_by_trial", + ) _DATA_SPLIT_METHODS: Final[tuple[str, ...]] = ( "train_test_split", @@ -1298,12 +1307,22 @@ class QuickAdapterRegressorV3(BaseRegressionModel): "space_fraction": QuickAdapterRegressorV3.OPTUNA_SPACE_FRACTION_DEFAULT, "min_resource": QuickAdapterRegressorV3.OPTUNA_MIN_RESOURCE_DEFAULT, "seed": QuickAdapterRegressorV3.OPTUNA_SEED_DEFAULT, + "vary_model_seed_by_trial": ( + QuickAdapterRegressorV3.OPTUNA_VARY_MODEL_SEED_BY_TRIAL_DEFAULT + ), } optuna_hyperopt = self.config.get("freqai", {}).get("optuna_hyperopt", {}) - return { + optuna_config = { **optuna_default_config, **optuna_hyperopt, } + for option in QuickAdapterRegressorV3._OPTUNA_BOOL_OPTIONS: + if not isinstance(optuna_config[option], bool): + raise ValueError( + f"freqai.optuna_hyperopt.{option} must be a boolean " + f"(got {type(optuna_config[option]).__name__})" + ) + return optuna_config @property def _min_label_period_candles(self) -> int: @@ -1515,6 +1534,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) logger.info(f" min_resource: {optuna_config.get('min_resource')}") logger.info(f" seed: {optuna_config.get('seed')}") + logger.info( + " vary_model_seed_by_trial: " + f"{optuna_config.get('vary_model_seed_by_trial')}" + ) logger.info(f" label_sampler: {optuna_config.get('label_sampler')}") logger.info( @@ -2800,8 +2823,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): model_training_parameters, self._optuna_config["space_reduction"], self._optuna_config["space_fraction"], - dk.data_path, - init_model, + model_path=dk.data_path, + init_model=init_model, + vary_model_seed_by_trial=self._optuna_config[ + "vary_model_seed_by_trial" + ], ), direction=optuna.study.StudyDirection.MINIMIZE, ) @@ -5028,6 +5054,7 @@ def hp_objective( space_fraction: float, model_path: Optional[Path] = None, init_model: Any = None, + vary_model_seed_by_trial: bool = True, ) -> float: study_model_parameters = get_optuna_study_model_parameters( trial, @@ -5054,6 +5081,7 @@ def hp_objective( init_model=init_model, model_path=model_path, trial=trial, + vary_model_seed_by_trial=vary_model_seed_by_trial, ) y_pred = model.predict(X_validation) diff --git a/quickadapter/user_data/strategies/Utils.py b/quickadapter/user_data/strategies/Utils.py index cd3822e..bf3be75 100644 --- a/quickadapter/user_data/strategies/Utils.py +++ b/quickadapter/user_data/strategies/Utils.py @@ -3882,6 +3882,7 @@ def fit_regressor( callbacks: list[RegressorCallback] | None = None, model_path: Path | None = None, trial: optuna.trial.Trial | None = None, + vary_model_seed_by_trial: bool = True, ) -> Any: fit_callbacks = list(callbacks) if callbacks else [] @@ -3902,7 +3903,7 @@ def fit_regressor( f"supported values are {', '.join(REGRESSORS)}" ) model_training_parameters.setdefault(spec.seed_param, 1) - if trial is not None: + if trial is not None and vary_model_seed_by_trial: model_training_parameters[spec.seed_param] = ( model_training_parameters[spec.seed_param] + trial.number ) @@ -4052,6 +4053,7 @@ def fit_regressor( max_depth=model_training_parameters.pop("max_depth", None), min_samples_split=model_training_parameters.pop("min_samples_split", 2), min_samples_leaf=model_training_parameters.pop("min_samples_leaf", 1), + random_state=model_training_parameters["random_state"], ), **model_training_parameters, ) -- 2.53.0