From 5aec5259184b4f7ec199c904a4f3fa4da76f5e2c Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Tue, 28 Jul 2026 19:41:09 +0200 Subject: [PATCH] fix(quickadapter): make epsilon fill causal (#148) * fix(quickadapter): make epsilon fill causal Derive each causal row epsilon baseline from pivot weights available with its label while preserving the global non-causal formulation. * fix(quickadapter): resolve undefined as_dict in set_freqai_targets Replace the unimported as_dict() call with the dominant self.freqai_info.get("feature_parameters", {}) idiom, matching the sibling get_causal_mode call site and the other feature_parameters lookups in the file. Fixes the F821/NameError that aborted set_freqai_targets on every fit, and removes the idiom divergence. Reformat the README configuration tunables table via prettier (table re-padding only; no cell content changed). --- README.md | 220 ++++++++-------- .../user_data/strategies/QuickAdapterV3.py | 6 + quickadapter/user_data/strategies/Utils.py | 242 +++++++++++++++--- 3 files changed, 320 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index d4b9a03..255fb1b 100644 --- a/README.md +++ b/README.md @@ -37,116 +37,116 @@ docker compose up -d --build ### Configuration tunables -| Path | Default | Type / Range | Description | -| -------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| _Protections_ | | | | -| custom_protections.trade_duration_candles | 72 | int >= 1 | Estimated trade duration in candles. Scales protections stop duration candles and trade limit. | -| custom_protections.lookback_period_fraction | 0.5 | float (0,1] | Fraction of `fit_live_predictions_candles` used to calculate `lookback_period_candles` for _MaxDrawdown_ and _StoplossGuard_ protections. | -| custom_protections.cooldown.enabled | true | bool | Enable/disable _CooldownPeriod_ protection. | -| custom_protections.cooldown.stop_duration_candles | 4 | int >= 1 | Number of candles to wait before allowing new trades after a trade is closed. | -| custom_protections.drawdown.enabled | true | bool | Enable/disable _MaxDrawdown_ protection. | -| custom_protections.drawdown.max_allowed_drawdown | 0.2 | float (0,1) | Maximum allowed drawdown. | -| custom_protections.stoploss.enabled | true | bool | Enable/disable _StoplossGuard_ protection. | -| _Leverage_ | | | | -| leverage | `proposed_leverage` | float [1.0, max_leverage] | Leverage. Fallback to `proposed_leverage` for the pair. | -| _Exit pricing_ | | | | -| exit_pricing.trade_price_target_method | `moving_average` | enum {`moving_average`,`quantile_interpolation`,`weighted_average`} | Trade NATR computation method. | -| exit_pricing.thresholds_calibration.decline_quantile | 0.5 | float (0,1) | PnL decline quantile threshold. | -| _Reversal confirmation_ | | | | -| reversal_confirmation.lookback_period_candles | 0 | int >= 0 | Prior confirming candles; 0 = none. | -| reversal_confirmation.decay_fraction | 0.5 | float (0,1] | Geometric per-candle volatility adjusted reversal threshold relaxation factor. | -| reversal_confirmation.min_natr_multiplier_fraction | 0.0095 | float [0,1] | Lower bound fraction (< upper bound) for volatility adjusted reversal threshold. | -| reversal_confirmation.max_natr_multiplier_fraction | 0.0125 | float [0,1] | Upper bound fraction (> lower bound) for volatility adjusted reversal threshold. | -| _Regressor model_ | | | | -| freqai.regressor | `xgboost` | enum {`xgboost`,`lightgbm`,`histgradientboostingregressor`,`ngboost`,`catboost`} | Machine learning regressor algorithm. | -| _Model training parameters_ | | | | -| freqai.model_training_parameters.gpu_vram_gb | 80 | enum {8,10,12,16,24,32,40,48,64,80} | Available GPU VRAM (GB) for CatBoost, not total. Constrains `depth`, `border_count`, and `max_ctr_complexity` ranges. | -| _Data split parameters_ | | | | -| freqai.data_split_parameters.method | `train_test_split` | enum {`train_test_split`,`timeseries_split`} | Data splitting strategy. `train_test_split` for sequential split, `timeseries_split` for chronological split with configurable gap. | -| freqai.data_split_parameters.test_size | 0.1 | float (0,1) \| int >= 1 \| None | Outer holdout size. The same parameter reserves the chronological tail of the remaining training rows as inner validation for HPO and early stopping; a fractional value is relative to those remaining rows, not the original window. The holdout is predicted once and reported as weighted `holdout_rmse` in the original label scale; it therefore measures the pre-refit selection model, not the refitted deployed model. `None` (sklearn dynamic sizing) applies only to `timeseries_split`; `train_test_split` requires a float or int; inner validation then falls back to `0.1`. | -| freqai.data_split_parameters.n_splits | 5 | int >= 2 | Controls train/test proportions for `timeseries_split` (higher = larger train set). | -| freqai.data_split_parameters.gap | 0 | int >= 0 | Samples to exclude between train/test for `timeseries_split`. When `0` and `causal_mode=true` (default), auto-set from `label_horizon_candles`; when `0` and `causal_mode=false`, auto-set from `label_period_candles`. Under `causal_mode=true`, an explicit `gap= 1 \| None | Maximum training set size for `timeseries_split`. When set, creates a sliding window instead of expanding train set. None = no limit. | -| _Label smoothing_ | | | | -| freqai.label_smoothing.method | `gaussian` | enum {`none`,`gaussian`,`kaiser`,`kaiser_bessel_derived`,`triang`,`smm`,`sma`,`savgol`,`gaussian_filter1d`} | Label smoothing method (`kaiser_bessel_derived` uses an even-length Kaiser-Bessel-derived zero-phase kernel; `smm`=median, `sma`=mean, `savgol`=Savitzky–Golay). | -| freqai.label_smoothing.window_candles | 5 | int >= 3 | Smoothing window length (candles). | -| freqai.label_smoothing.beta | 8.0 | float > 0 | Shape parameter for `kaiser` and `kaiser_bessel_derived` kernels. | -| freqai.label_smoothing.polyorder | 3 | int >= 0 | Polynomial order for `savgol` smoothing. | -| freqai.label_smoothing.mode | `mirror` | `savgol`: enum {`mirror`,`constant`,`nearest`,`wrap`,`interp`}; `gaussian_filter1d`: enum {`mirror`,`constant`,`nearest`,`wrap`}; ignored otherwise | Boundary mode for `savgol` and `gaussian_filter1d`. | -| freqai.label_smoothing.sigma | 1.0 | float > 0 | Gaussian `sigma` for `gaussian_filter1d` smoothing. | -| _Label weighting_ | | | | -| freqai.label_weighting.strategy | `none` | enum {`none`,`uniform`,`amplitude`,`amplitude_threshold_ratio`,`volume_rate`,`speed`,`efficiency_ratio`,`volume_weighted_efficiency_ratio`,`combined`} | Label weighting metric: none (`none`), uniform unit weight on every detected pivot (`uniform`), swing amplitude (`amplitude`), swing amplitude / median volatility-threshold ratio (`amplitude_threshold_ratio`), swing volume per candle (`volume_rate`), swing speed (`speed`), swing efficiency ratio (`efficiency_ratio`), swing volume-weighted efficiency ratio (`volume_weighted_efficiency_ratio`), or combined metrics aggregation (`combined`). Switching between `none` and any other strategy requires deleting trained models to realign training emphasis. | -| freqai.label_weighting.metric_coefficients | {} | dict[str, float] | Per-metric coefficients for `combined` strategy. Keys: `amplitude`, `amplitude_threshold_ratio`, `volume_rate`, `speed`, `efficiency_ratio`, `volume_weighted_efficiency_ratio`. | -| freqai.label_weighting.aggregation | `arithmetic_mean` | enum {`arithmetic_mean`,`geometric_mean`,`harmonic_mean`,`quadratic_mean`,`weighted_median`,`softmax`} | Metric aggregation method for `combined` strategy. `arithmetic_mean`=(Σ(w·m)/Σ(w)), `geometric_mean`=(∏(m^w))^(1/Σw), `harmonic_mean`=Σ(w)/(Σ(w/m)), `quadratic_mean`=(Σ(w·m²)/Σ(w))^(1/2), `weighted_median`=Q₀.₅(m,w), `softmax`=Σ(m·s_i) where s_i=w_i·exp(m_i/T)/Σ(w_j·exp(m_j/T)). | -| freqai.label_weighting.softmax_temperature | 1.0 | float > 0 | Temperature T for `softmax` aggregation, controls distribution sharpness. | -| freqai.label_weighting.fill_method | `zero` | enum {`zero`,`epsilon`,`gaussian`,`epsilon_gaussian`} | Off-pivot weighting scheme. `zero` hard-zeros off-pivot rows; `epsilon` applies the epsilon floor `fill_epsilon * (pivot_weights)`; `gaussian` applies per-pivot Gaussian bumps; `epsilon_gaussian` sums the `epsilon` floor and the `gaussian` bumps. Pivot rows take the max of their raw weight and the off-pivot field at their index (no-op for `zero`). Switching away from `zero` may require retuning tree-leaf regularization (`min_child_weight`, `lambda`) and resetting any prior Optuna study. Changing this parameter requires deleting trained models. | -| freqai.label_weighting.fill_epsilon | 0.000001 | float [0,1] | Off-pivot fraction of the pivot baseline. Ignored when `fill_method` not in {`epsilon`,`epsilon_gaussian`}. | -| freqai.label_weighting.fill_epsilon_baseline | `mean` | enum {`mean`,`median`} | Pivot baseline statistic. `mean` tracks central tendency; `median` is robust against pivot-weight skew. Ignored when `fill_method` not in {`epsilon`,`epsilon_gaussian`}. | -| freqai.label_weighting.fill_sigma_candles | 25.0 | float >= 0.5 | Gaussian standard deviation in candles for the per-pivot bumps. Acts as the upper bound on per-pivot sigma when `fill_bandwidth == "knn"`. Lower bound 0.5 prevents severe underflow in the Gaussian tail. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`}. | -| freqai.label_weighting.fill_sigma_min_candles | 0.5 | float >= 0.5 | Lower bound on per-pivot sigma in candles when `fill_bandwidth == "knn"`. Clipped to `fill_sigma_candles` when larger. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | -| freqai.label_weighting.fill_bandwidth | `fixed` | enum {`fixed`,`knn`} | Per-pivot Gaussian bandwidth selector. `fixed` applies a constant `fill_sigma_candles` to every pivot (legacy behavior). `knn` adapts each pivot's sigma to local pivot density via `sigma_p = clip(fill_bandwidth_alpha * d_k(p), fill_sigma_min_candles, fill_sigma_candles)` where `d_k(p)` is the index distance to the `k`-th nearest pivot neighbor (Loftsgaarden & Quesenberry 1965; Silverman 1986, §5.2). Mitigates the crushing of weaker pivots by stronger neighbors in dense clusters. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`}. | -| freqai.label_weighting.fill_bandwidth_neighbors | 1 | int >= 1 | `k` for the k-nearest-neighbor bandwidth selector. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | -| freqai.label_weighting.fill_bandwidth_alpha | 0.5 | float > 0 | Multiplicative factor on the k-th neighbor distance. Smaller values produce sharper, more separated Gaussians; larger values approach the `fixed` behavior. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | -| freqai.label_weighting.support_policy | `fallback` | enum {`fallback`,`raise`} | Policy when active label weighting fails support checks (after row filtering and causal split guards). `raise` aborts the fit with `ValueError`; `fallback` logs a `WARNING` and uses sanitized base sample weights for that fit. Eval (test/val) weights bypass this policy and always fall back on composition errors. | -| freqai.label_weighting.min_pivot_equivalent_count | 3 | int >= 1 | Minimum number of surviving pivot-equivalent label weights required after filtering. Pivot-equivalent rows are weights at least 10% of the surviving maximum label weight. | -| freqai.label_weighting.min_positive_label_weight_fraction | 0.01 | float [0,1] | Minimum fraction of filtered training rows with finite positive label weights. | -| freqai.label_weighting.min_effective_sample_size | 3.0 | float >= 1 | Minimum Kish effective sample size of the final composed training weights. | -| _Label pipeline_ | | | | -| freqai.label_pipeline.standardization | `none` | enum {`none`,`zscore`,`robust`,`mmad`,`power_yj`} | Standardization method applied to labels before normalization. `none`=w, `zscore`=(w-μ)/σ, `robust`=(w-median)/(Q₃-Q₁), `mmad`=(w-median)/(MAD·k), `power_yj`=YJ(w). | -| freqai.label_pipeline.robust_quantiles | [0.25, 0.75] | list[float] where 0 <= Q1 < Q3 <= 1 | Quantile range for robust standardization, Q1 and Q3. | -| freqai.label_pipeline.mmad_scaling_factor | 1.4826 | float > 0 | Scaling factor for MMAD standardization. | -| freqai.label_pipeline.normalization | `maxabs` | enum {`maxabs`,`minmax`,`sigmoid`,`none`} | Normalization method applied to labels. `maxabs`=w/max(\|w\|), `minmax`=low+(w-min)/(max-min)·(high-low), `sigmoid`=2·σ(scale·w)-1, `none`=w. | -| freqai.label_pipeline.minmax_range | [-1.0, 1.0] | list[float] | Target range for `minmax` normalization, min and max. | -| freqai.label_pipeline.sigmoid_scale | 1.0 | float > 0 | Scale parameter for `sigmoid` normalization, controls steepness. | -| freqai.label_pipeline.gamma | 1.0 | float (0,10] | Contrast exponent applied to labels after normalization: >1 emphasizes extrema, values between 0 and 1 soften. | -| _Feature parameters_ | | | | -| freqai.feature_parameters.label_period_candles | min/max midpoint | int >= 1 | Zigzag labeling NATR period. | -| freqai.feature_parameters.label_horizon_candles | `label_period_candles` | int >= 1 | Conservative fixed purge used by causal train/test guards and as the default `timeseries_split` gap. Zigzag labels additionally expose their exact row-wise confirmation time; centered smoothing composes the maximum availability time across each kernel support. When unset, falls back to `label_period_candles`. | -| freqai.feature_parameters.causal_mode | true | bool | Causal split guard toggle. When `true` (default): rejects `data_split_parameters.shuffle=true`, `shuffle_after_split=true`, `reverse_train_test_order=true`; for `timeseries_split` auto-sets `gap=label_horizon_candles` when unset/`0` (rejects explicit `gap= 1 | Minimum labeling NATR period used for reversals labeling HPO. | -| freqai.feature_parameters.max_label_period_candles | 24 | int >= 1 | Maximum labeling NATR period used for reversals labeling HPO. | -| freqai.feature_parameters.label_natr_multiplier | min/max midpoint | float > 0 | Zigzag labeling NATR multiplier. | -| freqai.feature_parameters.min_label_natr_multiplier | 9.0 | float > 0 | Minimum labeling NATR multiplier used for reversals labeling HPO. | -| freqai.feature_parameters.max_label_natr_multiplier | 12.0 | float > 0 | Maximum labeling NATR multiplier used for reversals labeling HPO. | -| freqai.feature_parameters.label_frequency_candles | `auto` | int >= 2 \| `auto` | Reversals labeling frequency. `auto` = max(2, 2 \* number of whitelisted pairs). | -| freqai.feature_parameters.label_weights | [1/7,1/7,1/7,1/7,1/7,1/7,1/7] | list[float] | Per-objective weights for trial selection methods. Objectives: (1) number of detected reversals, (2) median swing amplitude, (3) median (swing amplitude / median volatility-threshold ratio), (4) median swing volume per candle, (5) median swing speed, (6) median swing efficiency ratio, (7) median swing volume-weighted efficiency ratio. | -| freqai.feature_parameters.label_p_order | None | float \| None | Lp exponent for parameterized metrics. Used by `minkowski` distance (default 2.0) and `power_mean` aggregation (default 1.0). Ignored by other metrics. | -| freqai.feature_parameters.label_method | `compromise_programming` | enum {`compromise_programming`,`topsis`,`kmeans`,`kmeans2`,`kmedoids`,`knn`,`medoid`} | HPO `label` Pareto front trial selection method. | -| freqai.feature_parameters.label_distance_metric | `euclidean` | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`,`harmonic_mean`,`geometric_mean`,`arithmetic_mean`,`quadratic_mean`,`cubic_mean`,`power_mean`,`weighted_sum`} | Distance metric for `compromise_programming` and `topsis` methods. Invalid values warn and fall back to `euclidean`. | -| freqai.feature_parameters.label_cluster_metric | `euclidean` | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`} | Distance metric for `kmeans`, `kmeans2`, and `kmedoids` methods. Invalid values warn and fall back to `euclidean`. | -| freqai.feature_parameters.label_cluster_selection_method | `topsis` | enum {`compromise_programming`,`topsis`} | Cluster selection method for clustering-based label methods. | -| freqai.feature_parameters.label_cluster_trial_selection_method | `topsis` | enum {`compromise_programming`,`topsis`} | Best cluster trial selection method for clustering-based label methods. | -| freqai.feature_parameters.label_density_metric | method-dependent | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`} | Distance metric for `knn` and `medoid` methods. Invalid values warn and fall back to the method's natural default (`minkowski` for `knn`, `euclidean` for `medoid`). | -| freqai.feature_parameters.label_density_aggregation | `power_mean` | enum {`power_mean`,`quantile`,`min`,`max`} | Aggregation method for KNN neighbor distances. | -| freqai.feature_parameters.label_density_n_neighbors | 5 | int >= 1 | Number of neighbors for KNN. | -| freqai.feature_parameters.label_density_aggregation_param | aggregation-dependent | float \| None | Tunable for KNN neighbor distance aggregation: Lp exponent (`power_mean`) or quantile value (`quantile`). | -| freqai.feature_parameters.scaler | `minmax` | enum {`minmax`,`maxabs`,`standard`,`robust`} | Feature scaling method. `minmax`=MinMaxScaler, `maxabs`=MaxAbsScaler, `standard`=StandardScaler, `robust`=RobustScaler. Changing this parameter requires deleting trained models. | -| freqai.feature_parameters.range | [-1.0, 1.0] | list[float] | Target range for `minmax` scaler, min and max. Changing this parameter requires deleting trained models. | -| _Label prediction_ | | | | -| freqai.label_prediction.method | `thresholding` | enum {`none`,`thresholding`} | Prediction method. `none` disables threshold computation, `thresholding` enables adaptive threshold calculation. | -| freqai.label_prediction.selection_method | `rank_extrema` | enum {`rank_extrema`,`rank_peaks`,`partition`} | Extrema selection method. `rank_extrema` ranks extrema values, `rank_peaks` ranks detected peak values, `partition` uses sign-based partitioning. | -| freqai.label_prediction.threshold_method | `mean` | enum {`mean`,`isodata`,`li`,`minimum`,`otsu`,`triangle`,`yen`,`median`,`soft_extremum`} | Thresholding method for prediction thresholds. | -| freqai.label_prediction.soft_extremum_alpha | 12.0 | float >= 0 | Alpha for `soft_extremum` threshold method. | -| freqai.label_prediction.outlier_quantile | 0.999 | float (0,1) | Quantile threshold for predictions outlier filtering. | -| freqai.label_prediction.keep_fraction | 0.0075 | float (0,1] | Fraction of extrema used for thresholds. 1 uses all, lower values keep only most significant. Applies to `rank_extrema` and `rank_peaks`; ignored for `partition`. | -| _Optuna / HPO_ | | | | -| freqai.optuna_hyperopt.enabled | false | bool | Enables regressor and dynamic label HPO. | -| freqai.optuna_hyperopt.sampler | `tpe` | enum {`tpe`,`auto`} | HPO sampler algorithm for `hp` namespace. `tpe` uses [TPESampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.TPESampler.html) with multivariate, group, and constant_liar (when multiple workers), `auto` uses [AutoSampler](https://hub.optuna.org/samplers/auto_sampler). | -| freqai.optuna_hyperopt.label_sampler | `auto` | enum {`auto`,`tpe`,`nsgaii`,`nsgaiii`} | HPO sampler algorithm for multi-objective `label` namespace. `nsgaii` uses [NSGAIISampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.NSGAIISampler.html), `nsgaiii` uses [NSGAIIISampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.NSGAIIISampler.html). | -| freqai.optuna_hyperopt.storage | `file` | enum {`file`,`sqlite`} | HPO storage backend. | -| freqai.optuna_hyperopt.continuous | true | bool | Continuous HPO. Forced for both namespaces in backtest and hyperopt, resetting the study on each optimization. | -| freqai.optuna_hyperopt.warm_start | true | bool | Warm start HPO with previous best value(s). Persisted values are loaded and saved only in live and dry-run modes; non-live runs reuse only values produced earlier in the same run. | -| freqai.optuna_hyperopt.n_startup_trials | 15 | int >= 0 | HPO startup trials. | -| freqai.optuna_hyperopt.n_trials | 50 | int >= 1 | Maximum HPO trials. | -| freqai.optuna_hyperopt.n_jobs | CPU threads / 4 | int >= 1 | Parallel HPO workers. | -| freqai.optuna_hyperopt.timeout | 7200 | int >= 0 | HPO wall-clock timeout in seconds. | -| freqai.optuna_hyperopt.label_candles_step | 1 | int >= 1 | Step for Zigzag NATR period `label` search space. | -| freqai.optuna_hyperopt.space_reduction | false | bool | Enable/disable `hp` search space reduction based on previous best parameters. | -| freqai.optuna_hyperopt.space_fraction | 0.4 | float [0,1] | Fraction of the `hp` search space to use with `space_reduction`. Lower values create narrower search ranges around the best parameters. | -| freqai.optuna_hyperopt.min_resource | 3 | int >= 1 | Minimum resource per [HyperbandPruner](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.pruners.HyperbandPruner.html) rung. | -| freqai.optuna_hyperopt.seed | 1 | int >= 0 | HPO RNG seed used by the Optuna samplers and label-candle shuffling. | -| freqai.optuna_hyperopt.reset_label_study_on_schema_mismatch | true | bool | Reset a persisted `label` study when its selection schema is missing, invalid, or incompatible. `true` keeps the historical destructive reset, deleting the study before recreating it; `false` preserves its trials and stored metadata, permits caller-managed reuse in memory, and does not persist selected params until the schema is reconciled. Both fail closed: an inspection error, or (under `true`) a deletion error, aborts study creation. Has no effect when `continuous=true` or outside live/dry-run modes, where studies are always reset. | -| freqai.optuna_hyperopt.vary_model_seed_by_trial | true | bool | Add `trial.number` to each regressor's configured model seed (or its default seed of `1`) during HPO. `true` samples model randomness across trials and preserves the historical behavior; `false` evaluates every trial and the final fit with the same model seed. This does not change `freqai.optuna_hyperopt.seed`. | +| Path | Default | Type / Range | Description | +| -------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| _Protections_ | | | | +| custom_protections.trade_duration_candles | 72 | int >= 1 | Estimated trade duration in candles. Scales protections stop duration candles and trade limit. | +| custom_protections.lookback_period_fraction | 0.5 | float (0,1] | Fraction of `fit_live_predictions_candles` used to calculate `lookback_period_candles` for _MaxDrawdown_ and _StoplossGuard_ protections. | +| custom_protections.cooldown.enabled | true | bool | Enable/disable _CooldownPeriod_ protection. | +| custom_protections.cooldown.stop_duration_candles | 4 | int >= 1 | Number of candles to wait before allowing new trades after a trade is closed. | +| custom_protections.drawdown.enabled | true | bool | Enable/disable _MaxDrawdown_ protection. | +| custom_protections.drawdown.max_allowed_drawdown | 0.2 | float (0,1) | Maximum allowed drawdown. | +| custom_protections.stoploss.enabled | true | bool | Enable/disable _StoplossGuard_ protection. | +| _Leverage_ | | | | +| leverage | `proposed_leverage` | float [1.0, max_leverage] | Leverage. Fallback to `proposed_leverage` for the pair. | +| _Exit pricing_ | | | | +| exit_pricing.trade_price_target_method | `moving_average` | enum {`moving_average`,`quantile_interpolation`,`weighted_average`} | Trade NATR computation method. | +| exit_pricing.thresholds_calibration.decline_quantile | 0.5 | float (0,1) | PnL decline quantile threshold. | +| _Reversal confirmation_ | | | | +| reversal_confirmation.lookback_period_candles | 0 | int >= 0 | Prior confirming candles; 0 = none. | +| reversal_confirmation.decay_fraction | 0.5 | float (0,1] | Geometric per-candle volatility adjusted reversal threshold relaxation factor. | +| reversal_confirmation.min_natr_multiplier_fraction | 0.0095 | float [0,1] | Lower bound fraction (< upper bound) for volatility adjusted reversal threshold. | +| reversal_confirmation.max_natr_multiplier_fraction | 0.0125 | float [0,1] | Upper bound fraction (> lower bound) for volatility adjusted reversal threshold. | +| _Regressor model_ | | | | +| freqai.regressor | `xgboost` | enum {`xgboost`,`lightgbm`,`histgradientboostingregressor`,`ngboost`,`catboost`} | Machine learning regressor algorithm. | +| _Model training parameters_ | | | | +| freqai.model_training_parameters.gpu_vram_gb | 80 | enum {8,10,12,16,24,32,40,48,64,80} | Available GPU VRAM (GB) for CatBoost, not total. Constrains `depth`, `border_count`, and `max_ctr_complexity` ranges. | +| _Data split parameters_ | | | | +| freqai.data_split_parameters.method | `train_test_split` | enum {`train_test_split`,`timeseries_split`} | Data splitting strategy. `train_test_split` for sequential split, `timeseries_split` for chronological split with configurable gap. | +| freqai.data_split_parameters.test_size | 0.1 | float (0,1) \| int >= 1 \| None | Outer holdout size. The same parameter reserves the chronological tail of the remaining training rows as inner validation for HPO and early stopping; a fractional value is relative to those remaining rows, not the original window. The holdout is predicted once and reported as weighted `holdout_rmse` in the original label scale; it therefore measures the pre-refit selection model, not the refitted deployed model. `None` (sklearn dynamic sizing) applies only to `timeseries_split`; `train_test_split` requires a float or int; inner validation then falls back to `0.1`. | +| freqai.data_split_parameters.n_splits | 5 | int >= 2 | Controls train/test proportions for `timeseries_split` (higher = larger train set). | +| freqai.data_split_parameters.gap | 0 | int >= 0 | Samples to exclude between train/test for `timeseries_split`. When `0` and `causal_mode=true` (default), auto-set from `label_horizon_candles`; when `0` and `causal_mode=false`, auto-set from `label_period_candles`. Under `causal_mode=true`, an explicit `gap= 1 \| None | Maximum training set size for `timeseries_split`. When set, creates a sliding window instead of expanding train set. None = no limit. | +| _Label smoothing_ | | | | +| freqai.label_smoothing.method | `gaussian` | enum {`none`,`gaussian`,`kaiser`,`kaiser_bessel_derived`,`triang`,`smm`,`sma`,`savgol`,`gaussian_filter1d`} | Label smoothing method (`kaiser_bessel_derived` uses an even-length Kaiser-Bessel-derived zero-phase kernel; `smm`=median, `sma`=mean, `savgol`=Savitzky–Golay). | +| freqai.label_smoothing.window_candles | 5 | int >= 3 | Smoothing window length (candles). | +| freqai.label_smoothing.beta | 8.0 | float > 0 | Shape parameter for `kaiser` and `kaiser_bessel_derived` kernels. | +| freqai.label_smoothing.polyorder | 3 | int >= 0 | Polynomial order for `savgol` smoothing. | +| freqai.label_smoothing.mode | `mirror` | `savgol`: enum {`mirror`,`constant`,`nearest`,`wrap`,`interp`}; `gaussian_filter1d`: enum {`mirror`,`constant`,`nearest`,`wrap`}; ignored otherwise | Boundary mode for `savgol` and `gaussian_filter1d`. | +| freqai.label_smoothing.sigma | 1.0 | float > 0 | Gaussian `sigma` for `gaussian_filter1d` smoothing. | +| _Label weighting_ | | | | +| freqai.label_weighting.strategy | `none` | enum {`none`,`uniform`,`amplitude`,`amplitude_threshold_ratio`,`volume_rate`,`speed`,`efficiency_ratio`,`volume_weighted_efficiency_ratio`,`combined`} | Label weighting metric: none (`none`), uniform unit weight on every detected pivot (`uniform`), swing amplitude (`amplitude`), swing amplitude / median volatility-threshold ratio (`amplitude_threshold_ratio`), swing volume per candle (`volume_rate`), swing speed (`speed`), swing efficiency ratio (`efficiency_ratio`), swing volume-weighted efficiency ratio (`volume_weighted_efficiency_ratio`), or combined metrics aggregation (`combined`). Switching between `none` and any other strategy requires deleting trained models to realign training emphasis. | +| freqai.label_weighting.metric_coefficients | {} | dict[str, float] | Per-metric coefficients for `combined` strategy. Keys: `amplitude`, `amplitude_threshold_ratio`, `volume_rate`, `speed`, `efficiency_ratio`, `volume_weighted_efficiency_ratio`. | +| freqai.label_weighting.aggregation | `arithmetic_mean` | enum {`arithmetic_mean`,`geometric_mean`,`harmonic_mean`,`quadratic_mean`,`weighted_median`,`softmax`} | Metric aggregation method for `combined` strategy. `arithmetic_mean`=(Σ(w·m)/Σ(w)), `geometric_mean`=(∏(m^w))^(1/Σw), `harmonic_mean`=Σ(w)/(Σ(w/m)), `quadratic_mean`=(Σ(w·m²)/Σ(w))^(1/2), `weighted_median`=Q₀.₅(m,w), `softmax`=Σ(m·s_i) where s_i=w_i·exp(m_i/T)/Σ(w_j·exp(m_j/T)). | +| freqai.label_weighting.softmax_temperature | 1.0 | float > 0 | Temperature T for `softmax` aggregation, controls distribution sharpness. | +| freqai.label_weighting.fill_method | `zero` | enum {`zero`,`epsilon`,`gaussian`,`epsilon_gaussian`} | Off-pivot weighting scheme. `zero` hard-zeros off-pivot rows; `epsilon` applies the epsilon floor `fill_epsilon * (pivot_weights)`; in causal mode, each row's baseline uses only pivot weights available with that row's label. `gaussian` applies per-pivot Gaussian bumps; `epsilon_gaussian` sums the `epsilon` floor and the `gaussian` bumps. Pivot rows take the max of their raw weight and the off-pivot field at their index (no-op for `zero`). Switching away from `zero` may require retuning tree-leaf regularization (`min_child_weight`, `lambda`) and resetting any prior Optuna study. Changing this parameter requires deleting trained models. | +| freqai.label_weighting.fill_epsilon | 0.000001 | float [0,1] | Off-pivot fraction of the pivot baseline. Ignored when `fill_method` not in {`epsilon`,`epsilon_gaussian`}. | +| freqai.label_weighting.fill_epsilon_baseline | `mean` | enum {`mean`,`median`} | Pivot baseline statistic. `mean` tracks central tendency; `median` is robust against pivot-weight skew. Ignored when `fill_method` not in {`epsilon`,`epsilon_gaussian`}. | +| freqai.label_weighting.fill_sigma_candles | 25.0 | float >= 0.5 | Gaussian standard deviation in candles for the per-pivot bumps. Acts as the upper bound on per-pivot sigma when `fill_bandwidth == "knn"`. Lower bound 0.5 prevents severe underflow in the Gaussian tail. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`}. | +| freqai.label_weighting.fill_sigma_min_candles | 0.5 | float >= 0.5 | Lower bound on per-pivot sigma in candles when `fill_bandwidth == "knn"`. Clipped to `fill_sigma_candles` when larger. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | +| freqai.label_weighting.fill_bandwidth | `fixed` | enum {`fixed`,`knn`} | Per-pivot Gaussian bandwidth selector. `fixed` applies a constant `fill_sigma_candles` to every pivot (legacy behavior). `knn` adapts each pivot's sigma to local pivot density via `sigma_p = clip(fill_bandwidth_alpha * d_k(p), fill_sigma_min_candles, fill_sigma_candles)` where `d_k(p)` is the index distance to the `k`-th nearest pivot neighbor (Loftsgaarden & Quesenberry 1965; Silverman 1986, §5.2). Mitigates the crushing of weaker pivots by stronger neighbors in dense clusters. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`}. | +| freqai.label_weighting.fill_bandwidth_neighbors | 1 | int >= 1 | `k` for the k-nearest-neighbor bandwidth selector. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | +| freqai.label_weighting.fill_bandwidth_alpha | 0.5 | float > 0 | Multiplicative factor on the k-th neighbor distance. Smaller values produce sharper, more separated Gaussians; larger values approach the `fixed` behavior. Ignored when `fill_method` not in {`gaussian`,`epsilon_gaussian`} or `fill_bandwidth != "knn"`. | +| freqai.label_weighting.support_policy | `fallback` | enum {`fallback`,`raise`} | Policy when active label weighting fails support checks (after row filtering and causal split guards). `raise` aborts the fit with `ValueError`; `fallback` logs a `WARNING` and uses sanitized base sample weights for that fit. Eval (test/val) weights bypass this policy and always fall back on composition errors. | +| freqai.label_weighting.min_pivot_equivalent_count | 3 | int >= 1 | Minimum number of surviving pivot-equivalent label weights required after filtering. Pivot-equivalent rows are weights at least 10% of the surviving maximum label weight. | +| freqai.label_weighting.min_positive_label_weight_fraction | 0.01 | float [0,1] | Minimum fraction of filtered training rows with finite positive label weights. | +| freqai.label_weighting.min_effective_sample_size | 3.0 | float >= 1 | Minimum Kish effective sample size of the final composed training weights. | +| _Label pipeline_ | | | | +| freqai.label_pipeline.standardization | `none` | enum {`none`,`zscore`,`robust`,`mmad`,`power_yj`} | Standardization method applied to labels before normalization. `none`=w, `zscore`=(w-μ)/σ, `robust`=(w-median)/(Q₃-Q₁), `mmad`=(w-median)/(MAD·k), `power_yj`=YJ(w). | +| freqai.label_pipeline.robust_quantiles | [0.25, 0.75] | list[float] where 0 <= Q1 < Q3 <= 1 | Quantile range for robust standardization, Q1 and Q3. | +| freqai.label_pipeline.mmad_scaling_factor | 1.4826 | float > 0 | Scaling factor for MMAD standardization. | +| freqai.label_pipeline.normalization | `maxabs` | enum {`maxabs`,`minmax`,`sigmoid`,`none`} | Normalization method applied to labels. `maxabs`=w/max(\|w\|), `minmax`=low+(w-min)/(max-min)·(high-low), `sigmoid`=2·σ(scale·w)-1, `none`=w. | +| freqai.label_pipeline.minmax_range | [-1.0, 1.0] | list[float] | Target range for `minmax` normalization, min and max. | +| freqai.label_pipeline.sigmoid_scale | 1.0 | float > 0 | Scale parameter for `sigmoid` normalization, controls steepness. | +| freqai.label_pipeline.gamma | 1.0 | float (0,10] | Contrast exponent applied to labels after normalization: >1 emphasizes extrema, values between 0 and 1 soften. | +| _Feature parameters_ | | | | +| freqai.feature_parameters.label_period_candles | min/max midpoint | int >= 1 | Zigzag labeling NATR period. | +| freqai.feature_parameters.label_horizon_candles | `label_period_candles` | int >= 1 | Conservative fixed purge used by causal train/test guards and as the default `timeseries_split` gap. Zigzag labels additionally expose their exact row-wise confirmation time; centered smoothing composes the maximum availability time across each kernel support. When unset, falls back to `label_period_candles`. | +| freqai.feature_parameters.causal_mode | true | bool | Causal split guard toggle. When `true` (default): rejects `data_split_parameters.shuffle=true`, `shuffle_after_split=true`, `reverse_train_test_order=true`; for `timeseries_split` auto-sets `gap=label_horizon_candles` when unset/`0` (rejects explicit `gap= 1 | Minimum labeling NATR period used for reversals labeling HPO. | +| freqai.feature_parameters.max_label_period_candles | 24 | int >= 1 | Maximum labeling NATR period used for reversals labeling HPO. | +| freqai.feature_parameters.label_natr_multiplier | min/max midpoint | float > 0 | Zigzag labeling NATR multiplier. | +| freqai.feature_parameters.min_label_natr_multiplier | 9.0 | float > 0 | Minimum labeling NATR multiplier used for reversals labeling HPO. | +| freqai.feature_parameters.max_label_natr_multiplier | 12.0 | float > 0 | Maximum labeling NATR multiplier used for reversals labeling HPO. | +| freqai.feature_parameters.label_frequency_candles | `auto` | int >= 2 \| `auto` | Reversals labeling frequency. `auto` = max(2, 2 \* number of whitelisted pairs). | +| freqai.feature_parameters.label_weights | [1/7,1/7,1/7,1/7,1/7,1/7,1/7] | list[float] | Per-objective weights for trial selection methods. Objectives: (1) number of detected reversals, (2) median swing amplitude, (3) median (swing amplitude / median volatility-threshold ratio), (4) median swing volume per candle, (5) median swing speed, (6) median swing efficiency ratio, (7) median swing volume-weighted efficiency ratio. | +| freqai.feature_parameters.label_p_order | None | float \| None | Lp exponent for parameterized metrics. Used by `minkowski` distance (default 2.0) and `power_mean` aggregation (default 1.0). Ignored by other metrics. | +| freqai.feature_parameters.label_method | `compromise_programming` | enum {`compromise_programming`,`topsis`,`kmeans`,`kmeans2`,`kmedoids`,`knn`,`medoid`} | HPO `label` Pareto front trial selection method. | +| freqai.feature_parameters.label_distance_metric | `euclidean` | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`,`harmonic_mean`,`geometric_mean`,`arithmetic_mean`,`quadratic_mean`,`cubic_mean`,`power_mean`,`weighted_sum`} | Distance metric for `compromise_programming` and `topsis` methods. Invalid values warn and fall back to `euclidean`. | +| freqai.feature_parameters.label_cluster_metric | `euclidean` | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`} | Distance metric for `kmeans`, `kmeans2`, and `kmedoids` methods. Invalid values warn and fall back to `euclidean`. | +| freqai.feature_parameters.label_cluster_selection_method | `topsis` | enum {`compromise_programming`,`topsis`} | Cluster selection method for clustering-based label methods. | +| freqai.feature_parameters.label_cluster_trial_selection_method | `topsis` | enum {`compromise_programming`,`topsis`} | Best cluster trial selection method for clustering-based label methods. | +| freqai.feature_parameters.label_density_metric | method-dependent | enum {`euclidean`,`minkowski`,`chebyshev`,`cityblock`,`sqeuclidean`,`seuclidean`,`mahalanobis`} | Distance metric for `knn` and `medoid` methods. Invalid values warn and fall back to the method's natural default (`minkowski` for `knn`, `euclidean` for `medoid`). | +| freqai.feature_parameters.label_density_aggregation | `power_mean` | enum {`power_mean`,`quantile`,`min`,`max`} | Aggregation method for KNN neighbor distances. | +| freqai.feature_parameters.label_density_n_neighbors | 5 | int >= 1 | Number of neighbors for KNN. | +| freqai.feature_parameters.label_density_aggregation_param | aggregation-dependent | float \| None | Tunable for KNN neighbor distance aggregation: Lp exponent (`power_mean`) or quantile value (`quantile`). | +| freqai.feature_parameters.scaler | `minmax` | enum {`minmax`,`maxabs`,`standard`,`robust`} | Feature scaling method. `minmax`=MinMaxScaler, `maxabs`=MaxAbsScaler, `standard`=StandardScaler, `robust`=RobustScaler. Changing this parameter requires deleting trained models. | +| freqai.feature_parameters.range | [-1.0, 1.0] | list[float] | Target range for `minmax` scaler, min and max. Changing this parameter requires deleting trained models. | +| _Label prediction_ | | | | +| freqai.label_prediction.method | `thresholding` | enum {`none`,`thresholding`} | Prediction method. `none` disables threshold computation, `thresholding` enables adaptive threshold calculation. | +| freqai.label_prediction.selection_method | `rank_extrema` | enum {`rank_extrema`,`rank_peaks`,`partition`} | Extrema selection method. `rank_extrema` ranks extrema values, `rank_peaks` ranks detected peak values, `partition` uses sign-based partitioning. | +| freqai.label_prediction.threshold_method | `mean` | enum {`mean`,`isodata`,`li`,`minimum`,`otsu`,`triangle`,`yen`,`median`,`soft_extremum`} | Thresholding method for prediction thresholds. | +| freqai.label_prediction.soft_extremum_alpha | 12.0 | float >= 0 | Alpha for `soft_extremum` threshold method. | +| freqai.label_prediction.outlier_quantile | 0.999 | float (0,1) | Quantile threshold for predictions outlier filtering. | +| freqai.label_prediction.keep_fraction | 0.0075 | float (0,1] | Fraction of extrema used for thresholds. 1 uses all, lower values keep only most significant. Applies to `rank_extrema` and `rank_peaks`; ignored for `partition`. | +| _Optuna / HPO_ | | | | +| freqai.optuna_hyperopt.enabled | false | bool | Enables regressor and dynamic label HPO. | +| freqai.optuna_hyperopt.sampler | `tpe` | enum {`tpe`,`auto`} | HPO sampler algorithm for `hp` namespace. `tpe` uses [TPESampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.TPESampler.html) with multivariate, group, and constant_liar (when multiple workers), `auto` uses [AutoSampler](https://hub.optuna.org/samplers/auto_sampler). | +| freqai.optuna_hyperopt.label_sampler | `auto` | enum {`auto`,`tpe`,`nsgaii`,`nsgaiii`} | HPO sampler algorithm for multi-objective `label` namespace. `nsgaii` uses [NSGAIISampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.NSGAIISampler.html), `nsgaiii` uses [NSGAIIISampler](https://optuna.readthedocs.io/en/stable/reference/samplers/generated/optuna.samplers.NSGAIIISampler.html). | +| freqai.optuna_hyperopt.storage | `file` | enum {`file`,`sqlite`} | HPO storage backend. | +| freqai.optuna_hyperopt.continuous | true | bool | Continuous HPO. Forced for both namespaces in backtest and hyperopt, resetting the study on each optimization. | +| freqai.optuna_hyperopt.warm_start | true | bool | Warm start HPO with previous best value(s). Persisted values are loaded and saved only in live and dry-run modes; non-live runs reuse only values produced earlier in the same run. | +| freqai.optuna_hyperopt.n_startup_trials | 15 | int >= 0 | HPO startup trials. | +| freqai.optuna_hyperopt.n_trials | 50 | int >= 1 | Maximum HPO trials. | +| freqai.optuna_hyperopt.n_jobs | CPU threads / 4 | int >= 1 | Parallel HPO workers. | +| freqai.optuna_hyperopt.timeout | 7200 | int >= 0 | HPO wall-clock timeout in seconds. | +| freqai.optuna_hyperopt.label_candles_step | 1 | int >= 1 | Step for Zigzag NATR period `label` search space. | +| freqai.optuna_hyperopt.space_reduction | false | bool | Enable/disable `hp` search space reduction based on previous best parameters. | +| freqai.optuna_hyperopt.space_fraction | 0.4 | float [0,1] | Fraction of the `hp` search space to use with `space_reduction`. Lower values create narrower search ranges around the best parameters. | +| freqai.optuna_hyperopt.min_resource | 3 | int >= 1 | Minimum resource per [HyperbandPruner](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.pruners.HyperbandPruner.html) rung. | +| freqai.optuna_hyperopt.seed | 1 | int >= 0 | HPO RNG seed used by the Optuna samplers and label-candle shuffling. | +| freqai.optuna_hyperopt.reset_label_study_on_schema_mismatch | true | bool | Reset a persisted `label` study when its selection schema is missing, invalid, or incompatible. `true` keeps the historical destructive reset, deleting the study before recreating it; `false` preserves its trials and stored metadata, permits caller-managed reuse in memory, and does not persist selected params until the schema is reconciled. Both fail closed: an inspection error, or (under `true`) a deletion error, aborts study creation. Has no effect when `continuous=true` or outside live/dry-run modes, where studies are always reset. | +| freqai.optuna_hyperopt.vary_model_seed_by_trial | true | bool | Add `trial.number` to each regressor's configured model seed (or its default seed of `1`) during HPO. `true` samples model randomness across trials and preserves the historical behavior; `false` evaluates every trial and the final fit with the same model seed. This does not change `freqai.optuna_hyperopt.seed`. | ## ReforceXY diff --git a/quickadapter/user_data/strategies/QuickAdapterV3.py b/quickadapter/user_data/strategies/QuickAdapterV3.py index c81dcec..db177c1 100644 --- a/quickadapter/user_data/strategies/QuickAdapterV3.py +++ b/quickadapter/user_data/strategies/QuickAdapterV3.py @@ -997,6 +997,9 @@ class QuickAdapterV3(IStrategy): label_weighting = self.label_weighting label_smoothing = self.label_smoothing series_length = len(dataframe) + causal_mode = get_causal_mode( + self.freqai_info.get("feature_parameters", {}), logger + ) for label_col in LABEL_COLUMNS: label_params = self.get_label_params(pair, label_col) @@ -1036,6 +1039,9 @@ class QuickAdapterV3(IStrategy): metrics=label_data.metrics, weighting_config=col_weighting_config, logger=logger, + known_at_lookahead=( + label_data.known_at_lookahead if causal_mode else None + ), ) if label_data.known_at_lookahead is not None: dataframe[ diff --git a/quickadapter/user_data/strategies/Utils.py b/quickadapter/user_data/strategies/Utils.py index dc04501..0a5e6fb 100644 --- a/quickadapter/user_data/strategies/Utils.py +++ b/quickadapter/user_data/strategies/Utils.py @@ -2200,6 +2200,34 @@ def _impute_weights( return weights +def _causal_impute_weights( + weights: NDArray[np.floating], + *, + availability: NDArray[np.integer], + default_weight: float = 1.0, +) -> NDArray[np.floating]: + """Impute each non-finite value from signals available at the same event.""" + values = weights.astype(float, copy=True) + if values.size == 0: + return values + finite_mask = np.isfinite(values) + running_median = ( + pd.Series(values) + .where(finite_mask) + .expanding() + .median() + .fillna(default_weight) + .to_numpy(dtype=float) + ) + event_ends = np.flatnonzero(np.r_[availability[1:] != availability[:-1], True]) + event_medians = np.repeat( + running_median[event_ends], + np.diff(np.r_[-1, event_ends]), + ) + values[~finite_mask] = event_medians[~finite_mask] + return values + + _GAUSSIAN_FILL_CHUNK_BUDGET: Final[int] = 50_000_000 _GAUSSIAN_FILL_DENSITY_WARN: Final[float] = 0.1 @@ -2455,6 +2483,8 @@ def _compute_combined_label_weights( metric_coefficients: dict[str, Any], aggregation: CombinedAggregation, softmax_temperature: float, + *, + impute: Callable[[NDArray[np.floating]], NDArray[np.floating]] = _impute_weights, ) -> NDArray[np.floating]: if len(metrics) == 0: return np.asarray([], dtype=float) @@ -2473,7 +2503,7 @@ def _compute_combined_label_weights( values_array = np.asarray(metric_values, dtype=float) if values_array.size == 0: continue - imputed_metrics.append(_impute_weights(values_array)) + imputed_metrics.append(impute(values_array)) coefficients_list.append(float(coefficient)) if len(imputed_metrics) == 0: @@ -2516,6 +2546,152 @@ def _compute_epsilon_floor( return float(eps) * b +def _compute_label_weight_values( + n_indices: int, + metrics: dict[str, list[float]], + label_weighting: dict[str, Any], + *, + impute: Callable[[NDArray[np.floating]], NDArray[np.floating]] = _impute_weights, +) -> NDArray[np.floating]: + strategy = label_weighting["strategy"] + if strategy == WEIGHT_STRATEGIES[1]: # "uniform" + weights = np.ones(n_indices, dtype=float) + elif strategy in metrics: + weights = np.asarray(metrics[strategy], dtype=float) + elif strategy == WEIGHT_STRATEGIES[8]: # "combined" + weights = _compute_combined_label_weights( + metrics=metrics, + metric_coefficients=label_weighting["metric_coefficients"], + aggregation=label_weighting["aggregation"], + softmax_temperature=label_weighting["softmax_temperature"], + impute=impute, + ) + else: + raise ValueError( + f"Invalid weighting strategy value {strategy!r}: " + f"supported values are {', '.join(WEIGHT_STRATEGIES)} or metric names {', '.join(metrics.keys())}" + ) + return impute(weights) + + +def _compute_causal_epsilon_fill( + n_values: int, + indices_array: NDArray[np.integer], + valid_mask: NDArray[np.bool_], + metrics: dict[str, list[float]], + label_weighting: dict[str, Any], + known_at_lookahead: pd.Series, +) -> NDArray[np.floating]: + """Per-row epsilon floor from pivot weights fixed at their availability.""" + if len(known_at_lookahead) != n_values: + raise ValueError( + "Invalid known_at_lookahead length " + f"{len(known_at_lookahead)}: must be {n_values}" + ) + positions, known_at_lookahead_values = _sanitize_known_at_lookahead( + known_at_lookahead + ) + if not valid_mask.any(): + return np.zeros(n_values, dtype=float) + + source_indices = np.flatnonzero(valid_mask) + pivot_order = np.argsort(indices_array[valid_mask], kind="stable") + source_indices = source_indices[pivot_order] + pivot_indices = indices_array[source_indices] + + known_at_positions = positions + known_at_lookahead_values + if label_weighting["strategy"] == WEIGHT_STRATEGIES[1]: # "uniform" + pivot_available_at = known_at_positions[pivot_indices] + else: + pivot_available_at = np.empty(pivot_indices.size, dtype=np.int64) + pivot_available_at[:-1] = known_at_positions[pivot_indices[1:]] + pivot_available_at[-1] = n_values + + resolved_mask = pivot_available_at < n_values + if not resolved_mask.any(): + return np.zeros(n_values, dtype=float) + + source_indices = source_indices[resolved_mask] + pivot_available_at = pivot_available_at[resolved_mask] + availability_order = np.argsort(pivot_available_at, kind="stable") + source_indices = source_indices[availability_order] + pivot_available_at = pivot_available_at[availability_order] + ordered_metrics = { + metric_name: np.asarray(metric_values)[source_indices].tolist() + for metric_name, metric_values in metrics.items() + } + pivot_values = _compute_label_weight_values( + source_indices.size, + ordered_metrics, + label_weighting, + impute=functools.partial( + _causal_impute_weights, + availability=pivot_available_at, + ), + ) + + baseline = label_weighting["fill_epsilon_baseline"] + if baseline == FILL_EPSILON_BASELINES[0]: # "mean" + running_baseline = ( + pd.Series(pivot_values).expanding().mean().to_numpy(dtype=float) + ) + elif baseline == FILL_EPSILON_BASELINES[1]: # "median" + running_baseline = ( + pd.Series(pivot_values).expanding().median().to_numpy(dtype=float) + ) + else: + raise ValueError( + f"Invalid fill_epsilon_baseline value {baseline!r}: " + f"supported values are {', '.join(FILL_EPSILON_BASELINES)}" + ) + + event_ends = np.flatnonzero( + np.r_[pivot_available_at[1:] != pivot_available_at[:-1], True] + ) + availability_events = pivot_available_at[event_ends] + event_floors = float(label_weighting["fill_epsilon"]) * running_baseline[event_ends] + + available_count = np.searchsorted( + availability_events, known_at_positions, side="right" + ) + fill_weights = np.zeros(n_values, dtype=float) + has_available_pivot = available_count > 0 + fill_weights[has_available_pivot] = event_floors[ + available_count[has_available_pivot] - 1 + ] + return fill_weights + + +def _compute_epsilon_fill( + n_values: int, + indices_array: NDArray[np.integer], + valid_mask: NDArray[np.bool_], + weights: NDArray[np.floating], + metrics: dict[str, list[float]], + label_weighting: dict[str, Any], + known_at_lookahead: pd.Series | None, +) -> NDArray[np.floating]: + if known_at_lookahead is not None: + return _compute_causal_epsilon_fill( + n_values, + indices_array, + valid_mask, + metrics, + label_weighting, + known_at_lookahead, + ) + return np.full( + n_values, + _compute_epsilon_floor( + weights, + valid_mask, + label_weighting["fill_epsilon"], + label_weighting["fill_epsilon_baseline"], + ), + dtype=float, + ) + + def _compute_gaussian_bumps( n_values: int, indices_array: NDArray[np.integer], @@ -2550,13 +2726,15 @@ def compute_label_weights( weighting_config: dict[str, Any], *, logger: Logger, + known_at_lookahead: pd.Series | None = None, ) -> NDArray[np.floating]: """Compute per-row label importance weights. Returns an array with positive values at pivot ``indices`` (scaled by - strategy) and off-pivot values controlled by ``fill_method``. Callers - must skip invocation when strategy is ``'none'``; this raises - ValueError otherwise. + strategy) and off-pivot values controlled by ``fill_method``. + ``known_at_lookahead`` enables a per-row causal epsilon baseline; ``None`` + preserves the global non-causal baseline. Callers must skip invocation when + strategy is ``'none'``; this raises ValueError otherwise. """ label_weighting = {**DEFAULTS_LABEL_WEIGHTING, **weighting_config} strategy = label_weighting["strategy"] @@ -2579,44 +2757,29 @@ def compute_label_weights( n_values, ) - weights: NDArray[np.floating] - - if strategy == WEIGHT_STRATEGIES[1]: # "uniform" - weights = np.ones(n_indices, dtype=float) - elif strategy in metrics: - weights = np.asarray(metrics[strategy], dtype=float) - elif strategy == WEIGHT_STRATEGIES[8]: # "combined" - weights = _compute_combined_label_weights( - metrics=metrics, - metric_coefficients=label_weighting["metric_coefficients"], - aggregation=label_weighting["aggregation"], - softmax_temperature=label_weighting["softmax_temperature"], - ) - else: - raise ValueError( - f"Invalid weighting strategy value {strategy!r}: " - f"supported values are {', '.join(WEIGHT_STRATEGIES)} or metric names {', '.join(metrics.keys())}" - ) - - weights = _impute_weights(weights) + weights = _compute_label_weight_values(n_indices, metrics, label_weighting) if weights.size == 0: return np.zeros(n_values, dtype=float) + if weights.size != n_indices: + raise ValueError( + f"Invalid indices_array/weights values: length mismatch, " + f"got {n_indices} indices but {weights.size} weights" + ) fill_method = label_weighting["fill_method"] if fill_method == FILL_METHODS[0]: # "zero" fill_weights = np.zeros(n_values, dtype=float) elif fill_method == FILL_METHODS[1]: # "epsilon" - fill_weights = np.full( + fill_weights = _compute_epsilon_fill( n_values, - _compute_epsilon_floor( - weights, - valid_mask, - label_weighting["fill_epsilon"], - label_weighting["fill_epsilon_baseline"], - ), - dtype=float, + indices_array, + valid_mask, + weights, + metrics, + label_weighting, + known_at_lookahead, ) elif fill_method == FILL_METHODS[2]: # "gaussian" fill_weights = _compute_gaussian_bumps( @@ -2628,11 +2791,14 @@ def compute_label_weights( ) np.add( fill_weights, - _compute_epsilon_floor( - weights, + _compute_epsilon_fill( + n_values, + indices_array, valid_mask, - label_weighting["fill_epsilon"], - label_weighting["fill_epsilon_baseline"], + weights, + metrics, + label_weighting, + known_at_lookahead, ), out=fill_weights, ) @@ -2665,8 +2831,8 @@ def weight_fill_radius(weighting_config: dict[str, Any]) -> int: (``gaussian``/``epsilon_gaussian``). ``fill_sigma_candles`` upper-bounds the per-pivot sigma (including ``knn``, which clips below it), so ``ceil(k*fill_sigma_candles)`` covers every pivot's material Gaussian - support. The additive epsilon floor is a global O(fill_epsilon) term, left - unconstrained (negligible). + support. The additive epsilon floor uses its separate per-row causal + baseline, so it needs no local fill radius. """ label_weighting = {**DEFAULTS_LABEL_WEIGHTING, **weighting_config} if label_weighting["fill_method"] not in ( -- 2.53.0