From: Jérôme Benoit Date: Tue, 28 Jul 2026 16:26:06 +0000 (+0200) Subject: feat(quickadapter): make label study schema reset configurable (#147) X-Git-Url: https://git.piment-noir.org/?a=commitdiff_plain;h=4cb5b218a1f8bc35f7238ca7fadf8de14fb111c8;p=freqai-strategies.git feat(quickadapter): make label study schema reset configurable (#147) * feat(quickadapter): make label study schema reset configurable Add a boolean reset_label_study_on_schema_mismatch option that preserves the current reset behavior by default and lets operators retain incompatible label studies with a warning. Avoid rewriting incompatible selection metadata when preservation is selected, and fail closed when study inspection or deletion fails. Refs #87 * docs(quickadapter): clarify reset_label_study_on_schema_mismatch fail-closed behavior The `true` value keeps the historical destructive reset but no longer matches main byte-for-byte on error paths: a study inspection error (either value) or a deletion error (under `true`) now fails closed and aborts the optimization instead of silently recreating or reusing the study. Reword the README tunable description accordingly and align terminology with the neighboring entries. * docs(quickadapter): tighten reset_label_study_on_schema_mismatch abort wording Say the fail-closed paths abort study creation, matching the actual mechanism: optuna_create_study returns None before the study is created, and the caller then skips the optimization cycle. --- diff --git a/README.md b/README.md index fa58f24..d4b9a03 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,7 @@ docker compose up -d --build | freqai.optuna_hyperopt.space_fraction | 0.4 | float [0,1] | Fraction of the `hp` search space to use with `space_reduction`. Lower values create narrower search ranges around the best parameters. | | freqai.optuna_hyperopt.min_resource | 3 | int >= 1 | Minimum resource per [HyperbandPruner](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.pruners.HyperbandPruner.html) rung. | | freqai.optuna_hyperopt.seed | 1 | int >= 0 | HPO RNG seed used by the Optuna samplers and label-candle shuffling. | +| freqai.optuna_hyperopt.reset_label_study_on_schema_mismatch | true | bool | Reset a persisted `label` study when its selection schema is missing, invalid, or incompatible. `true` keeps the historical destructive reset, deleting the study before recreating it; `false` preserves its trials and stored metadata, permits caller-managed reuse in memory, and does not persist selected params until the schema is reconciled. Both fail closed: an inspection error, or (under `true`) a deletion error, aborts study creation. Has no effect when `continuous=true` or outside live/dry-run modes, where studies are always reset. | | freqai.optuna_hyperopt.vary_model_seed_by_trial | true | bool | Add `trial.number` to each regressor's configured model seed (or its default seed of `1`) during HPO. `true` samples model randomness across trials and preserves the historical behavior; `false` evaluates every trial and the final fit with the same model seed. This does not change `freqai.optuna_hyperopt.seed`. | ## ReforceXY diff --git a/quickadapter/user_data/config-template.json b/quickadapter/user_data/config-template.json index 7bdff38..0fcbc2b 100644 --- a/quickadapter/user_data/config-template.json +++ b/quickadapter/user_data/config-template.json @@ -172,6 +172,7 @@ "timeout": 7200, "label_candles_step": 1, "storage": "file", + "reset_label_study_on_schema_mismatch": true, "vary_model_seed_by_trial": true }, "extra_returns_per_train": { diff --git a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py index 61f0f71..49f5c35 100644 --- a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py +++ b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py @@ -437,6 +437,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): OPTUNA_SPACE_REDUCTION_DEFAULT: Final[bool] = False OPTUNA_SPACE_FRACTION_DEFAULT: Final[float] = 0.4 OPTUNA_SEED_DEFAULT: Final[int] = 1 + OPTUNA_RESET_LABEL_STUDY_ON_SCHEMA_MISMATCH_DEFAULT: Final[bool] = True OPTUNA_VARY_MODEL_SEED_BY_TRIAL_DEFAULT: Final[bool] = True _OPTUNA_BOOL_OPTIONS: Final[tuple[str, ...]] = ( @@ -444,6 +445,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): "continuous", "warm_start", "space_reduction", + "reset_label_study_on_schema_mismatch", "vary_model_seed_by_trial", ) @@ -1305,6 +1307,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): "space_fraction": QuickAdapterRegressorV3.OPTUNA_SPACE_FRACTION_DEFAULT, "min_resource": QuickAdapterRegressorV3.OPTUNA_MIN_RESOURCE_DEFAULT, "seed": QuickAdapterRegressorV3.OPTUNA_SEED_DEFAULT, + "reset_label_study_on_schema_mismatch": ( + QuickAdapterRegressorV3.OPTUNA_RESET_LABEL_STUDY_ON_SCHEMA_MISMATCH_DEFAULT + ), "vary_model_seed_by_trial": ( QuickAdapterRegressorV3.OPTUNA_VARY_MODEL_SEED_BY_TRIAL_DEFAULT ), @@ -1533,6 +1538,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) logger.info(f" min_resource: {optuna_config.get('min_resource')}") logger.info(f" seed: {optuna_config.get('seed')}") + logger.info( + " reset_label_study_on_schema_mismatch: " + f"{optuna_config.get('reset_label_study_on_schema_mismatch')}" + ) logger.info( " vary_model_seed_by_trial: " f"{optuna_config.get('vary_model_seed_by_trial')}" @@ -4585,7 +4594,17 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt best params found has invalid optimization target value(s)" ) if self.live: - self.optuna_save_best_params(pair, namespace) + if ( + namespace == _OPTUNA_NAMESPACES.label + and study.user_attrs.get("selection_metadata") + != self._optuna_label_selection_metadata() + ): + logger.warning( + f"[{pair}] Optuna {namespace} best params not persisted: " + "the preserved study selection schema is incompatible" + ) + else: + self.optuna_save_best_params(pair, namespace) return study @staticmethod @@ -4838,18 +4857,28 @@ class QuickAdapterRegressorV3(BaseRegressionModel): # cutoff's in-memory best, which stays causal as it predates the current # cutoff. continuous = self._optuna_config.get("continuous") or not self.live + label_schema_mismatch_preserved = False if continuous: QuickAdapterRegressorV3.optuna_delete_study( pair, namespace, study_name, storage ) elif namespace == _OPTUNA_NAMESPACES.label: - existing_study = QuickAdapterRegressorV3.optuna_load_study( - study_name, storage - ) - if existing_study is not None: - existing_selection_metadata = existing_study.user_attrs.get( - "selection_metadata" + try: + existing_study = QuickAdapterRegressorV3.optuna_load_study( + study_name, storage ) + existing_selection_metadata = ( + existing_study.user_attrs.get("selection_metadata") + if existing_study is not None + else None + ) + except Exception as e: + logger.error( + f"[{pair}] Optuna {namespace} study {study_name} inspection failed: {e!r}", + exc_info=True, + ) + return None + if existing_study is not None: existing_schema_version = ( existing_selection_metadata.get("schema_version") if isinstance(existing_selection_metadata, dict) @@ -4866,14 +4895,22 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if existing_schema_version is None else f"v{existing_schema_version!r}" ) + reset_study = self._optuna_config[ + "reset_label_study_on_schema_mismatch" + ] logger.warning( f"[{pair}] Optuna {namespace} study {study_name}: " f"selection schema {version_repr} incompatible " - f"with v{target_version}; resetting study" - ) - QuickAdapterRegressorV3.optuna_delete_study( - pair, namespace, study_name, storage + f"with v{target_version}; " + f"{'resetting' if reset_study else 'preserving'} study" ) + if reset_study: + if not QuickAdapterRegressorV3.optuna_delete_study( + pair, namespace, study_name, storage + ): + return None + else: + label_schema_mismatch_preserved = True samplers, sampler = self.optuna_samplers_by_namespace(namespace) if sampler not in samplers: @@ -4892,7 +4929,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): storage=storage, load_if_exists=not continuous, ) - if namespace == _OPTUNA_NAMESPACES.label: + if ( + namespace == _OPTUNA_NAMESPACES.label + and not label_schema_mismatch_preserved + ): new_selection_metadata = self._optuna_label_selection_metadata() existing_selection_metadata = study.user_attrs.get("selection_metadata") if existing_selection_metadata != new_selection_metadata: @@ -4985,9 +5025,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): namespace: OptunaNamespace, study_name: str, storage: optuna.storages.BaseStorage, - ) -> None: + ) -> bool: try: optuna.delete_study(study_name=study_name, storage=storage) + return True except KeyError as e: # A missing study is a benign no-op: non-live runs use a fresh # InMemoryStorage and the first live/dry-run optimization per pair @@ -4996,11 +5037,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): logger.debug( f"[{pair}] Optuna {namespace} study {study_name} absent; nothing to delete: {e!r}" ) + return True except Exception as e: logger.warning( f"[{pair}] Optuna {namespace} study {study_name} deletion failed: {e!r}", exc_info=True, ) + return False @staticmethod def optuna_load_study( @@ -5008,7 +5051,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) -> Optional[optuna.study.Study]: try: study = optuna.load_study(study_name=study_name, storage=storage) - except Exception: + except KeyError: study = None return study