From 8b709e392ee1a3e354001de81334f654b8611e63 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=B4me=20Benoit?= Date: Fri, 1 May 2026 16:03:12 +0200 Subject: [PATCH] fix: validate epoch-ms range before converting int64 date columns Reject int64 values outside [2010, 2035] epoch-ms range to fail fast on corrupted data instead of silently producing wrong dates. Catches nanosecond/microsecond values that would pass the int64 dtype check but produce garbage timestamps if interpreted as milliseconds. --- ReforceXY/user_data/strategies/RLAgentStrategy.py | 14 ++++++++++++++ quickadapter/user_data/strategies/Utils.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/ReforceXY/user_data/strategies/RLAgentStrategy.py b/ReforceXY/user_data/strategies/RLAgentStrategy.py index 38b016a..7cbe108 100644 --- a/ReforceXY/user_data/strategies/RLAgentStrategy.py +++ b/ReforceXY/user_data/strategies/RLAgentStrategy.py @@ -19,9 +19,23 @@ logger = logging.getLogger(__name__) ACTION_COLUMN: Final = "&-action" +_EPOCH_MS_MIN = 1_262_304_000_000 # 2010-01-01T00:00:00Z +_EPOCH_MS_MAX = 2_051_222_400_000 # 2035-01-01T00:00:00Z + + def _ensure_datetime_series(series: pd.Series) -> pd.Series: """Ensure a date series is datetime64[ms, UTC], following freqtrade's data handler pattern.""" if pd.api.types.is_integer_dtype(series): + sample = series.dropna() + if sample.empty: + return pd.to_datetime(series, unit="ms", utc=True).dt.as_unit("ms") + probe = int(sample.iat[0]) + if not (_EPOCH_MS_MIN <= probe <= _EPOCH_MS_MAX): + raise ValueError( + f"Integer date column value {probe} is outside the expected epoch-ms " + f"range [{_EPOCH_MS_MIN}, {_EPOCH_MS_MAX}]. " + "Data is likely corrupted or uses a different unit." + ) return pd.to_datetime(series, unit="ms", utc=True).dt.as_unit("ms") return series.dt.as_unit("ms") diff --git a/quickadapter/user_data/strategies/Utils.py b/quickadapter/user_data/strategies/Utils.py index 9ade5af..3050883 100644 --- a/quickadapter/user_data/strategies/Utils.py +++ b/quickadapter/user_data/strategies/Utils.py @@ -644,9 +644,23 @@ def get_label_prediction_config( ) +_EPOCH_MS_MIN = 1_262_304_000_000 # 2010-01-01T00:00:00Z +_EPOCH_MS_MAX = 2_051_222_400_000 # 2035-01-01T00:00:00Z + + def ensure_datetime_series(series: pd.Series) -> pd.Series: """Ensure a date series is datetime64[ms, UTC], following freqtrade's data handler pattern.""" if pd.api.types.is_integer_dtype(series): + sample = series.dropna() + if sample.empty: + return pd.to_datetime(series, unit="ms", utc=True).dt.as_unit("ms") + probe = int(sample.iat[0]) + if not (_EPOCH_MS_MIN <= probe <= _EPOCH_MS_MAX): + raise ValueError( + f"Integer date column value {probe} is outside the expected epoch-ms " + f"range [{_EPOCH_MS_MIN}, {_EPOCH_MS_MAX}]. " + "Data is likely corrupted or uses a different unit." + ) return pd.to_datetime(series, unit="ms", utc=True).dt.as_unit("ms") return series.dt.as_unit("ms") -- 2.43.0