From: Jérôme Benoit Date: Sun, 30 Aug 2026 10:55:04 +0000 (+0200) Subject: chore: enforce Ruff across Python projects (#240) X-Git-Url: https://git.piment-noir.org/?a=commitdiff_plain;h=3b802543c5bb7311730870da8abf480c2dbf9791;p=freqai-strategies.git chore: enforce Ruff across Python projects (#240) * chore: enforce Ruff across Python projects * fix(ci): pin Ruff tooling with uvx * fix(ci): align Ruff tooling semantics * fix(devcontainer): upgrade persistent Ruff * fix(quickadapter): preserve validation contracts * fix(devcontainer): remove redundant Ruff upgrade --- diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..e96694d --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,32 @@ +name: Ruff + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: ruff-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Check and format + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + cache-dependency-glob: .github/workflows/ruff.yml + - name: Check repository + run: uvx ruff@latest check . + - name: Check repository formatting + run: uvx ruff@latest format --check . diff --git a/ReforceXY/reward_space_analysis/pyproject.toml b/ReforceXY/reward_space_analysis/pyproject.toml index 943d3c3..effc197 100644 --- a/ReforceXY/reward_space_analysis/pyproject.toml +++ b/ReforceXY/reward_space_analysis/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ dev = [ "pytest>=9.0.3", "pytest-cov>=7.0", - "ruff>=0.8", + "ruff>=0.16.5", ] [build-system] @@ -92,30 +92,8 @@ exclude_lines = [ ] [tool.ruff] -line-length = 100 +extend = "../../ruff.toml" target-version = "py311" -[tool.ruff.lint] -select = [ - "E", # pycodestyle errors - "W", # pycodestyle warnings - "F", # pyflakes - "I", # isort - "B", # flake8-bugbear - "C4", # flake8-comprehensions - "UP", # pyupgrade - "SIM", # flake8-simplify - "TCH", # flake8-type-checking - "PTH", # flake8-use-pathlib - "RUF", # ruff-specific rules -] -ignore = [ - "E501", # line too long -] - [tool.ruff.lint.isort] known-first-party = ["reward_space_analysis"] - -[tool.ruff.format] -quote-style = "double" -indent-style = "space" diff --git a/ReforceXY/reward_space_analysis/reward_space_analysis.py b/ReforceXY/reward_space_analysis/reward_space_analysis.py index 459a224..43a1d2d 100644 --- a/ReforceXY/reward_space_analysis/reward_space_analysis.py +++ b/ReforceXY/reward_space_analysis/reward_space_analysis.py @@ -76,7 +76,7 @@ DEFAULT_IDLE_DURATION_MULTIPLIER = 4 # When that diagnostic column is not available (e.g., reporting from partial datasets), # we fall back to the weaker heuristic |Σ shaping| < PBRS_INVARIANCE_TOL. PBRS_INVARIANCE_TOL: float = 1e-6 -# Default discount factor γ for potential-based reward shaping # noqa: RUF003 +# Default discount factor γ for potential-based reward shaping POTENTIAL_GAMMA_DEFAULT: float = 0.95 # Default risk/reward ratio (RR) @@ -159,7 +159,7 @@ DEFAULT_MODEL_REWARD_PARAMETERS: RewardParams = { "exit_factor_threshold": 1000.0, # === PBRS PARAMETERS === # Potential-based reward shaping core parameters - # Discount factor γ for potential term (0 ≤ γ ≤ 1) # noqa: RUF003 + # Discount factor γ for potential term (0 ≤ γ ≤ 1) "potential_gamma": POTENTIAL_GAMMA_DEFAULT, # Exit potential modes: canonical | non_canonical | progressive_release | spike_cancel | retain_previous "exit_potential_mode": "canonical", @@ -208,7 +208,7 @@ DEFAULT_MODEL_REWARD_PARAMETERS_HELP: dict[str, str] = { "check_invariants": "Enable runtime invariant checks", "exit_factor_threshold": "Warn if |exit_factor| exceeds", # PBRS parameters - "potential_gamma": "PBRS discount γ (0-1)", # noqa: RUF001 + "potential_gamma": "PBRS discount γ (0-1)", "exit_potential_mode": "Exit potential mode (canonical|non_canonical|progressive_release|spike_cancel|retain_previous)", "exit_potential_decay": "Decay for progressive_release (0-1)", "hold_potential_enabled": "Enable hold potential Φ", @@ -790,7 +790,7 @@ class RewardBreakdown: next_potential: float = 0.0 # PBRS helpers base_reward: float = 0.0 - pbrs_delta: float = 0.0 # Δ(s,a,s') = γ·Φ(s') - Φ(s) # noqa: RUF003 + pbrs_delta: float = 0.0 # Δ(s,a,s') = γ·Φ(s') - Φ(s) invariance_correction: float = 0.0 @@ -2931,7 +2931,7 @@ def _apply_transform_arctan(value: float) -> float: def _apply_transform_sigmoid(value: float) -> float: - """sigmoid: 2σ(x) - 1, σ(x) = 1/(1 + e^(-x)) in (-1, 1).""" # noqa: RUF002 + """sigmoid: 2σ(x) - 1, σ(x) = 1/(1 + e^(-x)) in (-1, 1).""" x = value try: if x >= 0: @@ -4017,7 +4017,7 @@ def write_complete_statistical_analysis( f.write("|--------|-------|-------------|\n") f.write(f"| Mean Base Reward | {mean_base:.6f} | Average reward before PBRS |\n") f.write(f"| Std Base Reward | {std_base:.6f} | Variability of base reward |\n") - f.write(f"| Mean PBRS Delta | {mean_pbrs:.6f} | Average γ·Φ(s') - Φ(s) |\n") # noqa: RUF001 + f.write(f"| Mean PBRS Delta | {mean_pbrs:.6f} | Average γ·Φ(s') - Φ(s) |\n") f.write(f"| Std PBRS Delta | {std_pbrs:.6f} | Variability of PBRS delta |\n") f.write( f"| Mean Invariance Correction | {mean_inv_corr:.6f} | Average reward_shaping - pbrs_delta |\n" diff --git a/ReforceXY/reward_space_analysis/tests/pbrs/test_pbrs.py b/ReforceXY/reward_space_analysis/tests/pbrs/test_pbrs.py index 2b81534..86c5b64 100644 --- a/ReforceXY/reward_space_analysis/tests/pbrs/test_pbrs.py +++ b/ReforceXY/reward_space_analysis/tests/pbrs/test_pbrs.py @@ -1117,7 +1117,7 @@ class TestPBRS(RewardSpaceTestBase): self.assertLessEqual(abs(shap), PBRS.MAX_ABS_SHAPING) # With bounded transforms and hold_potential_ratio=1: - # |Φ(s)| <= base_factor and |Δ| <= (1+γ)*base_factor # noqa: RUF003 + # |Φ(s)| <= base_factor and |Δ| <= (1+γ)*base_factor self.assertLessEqual(abs(float(shap)), (1.0 + gamma) * PARAMS.BASE_FACTOR) def test_report_cumulative_invariance_aggregation(self): diff --git a/ReforceXY/reward_space_analysis/tests/transforms/test_transforms.py b/ReforceXY/reward_space_analysis/tests/transforms/test_transforms.py index 6d72d1e..8c53a5d 100644 --- a/ReforceXY/reward_space_analysis/tests/transforms/test_transforms.py +++ b/ReforceXY/reward_space_analysis/tests/transforms/test_transforms.py @@ -35,7 +35,7 @@ class TestTransforms(RewardSpaceTestBase): ("asinh", [0.0], [0.0]), # More complex calculations tested separately # arctan transform: (2/π) · arctan(x) in (-1, 1) ("arctan", [0.0, 1.0], [0.0, 2.0 / math.pi * math.atan(1.0)]), - # sigmoid transform: 2σ(x) - 1, σ(x) = 1/(1 + e^(-x)) in (-1, 1) # noqa: RUF003 + # sigmoid transform: 2σ(x) - 1, σ(x) = 1/(1 + e^(-x)) in (-1, 1) ("sigmoid", [0.0], [0.0]), # More complex calculations tested separately # clip transform: clip(x, -1, 1) in [-1, 1] ("clip", [0.0, 0.5, 2.0, -2.0], [0.0, 0.5, 1.0, -1.0]), diff --git a/ReforceXY/reward_space_analysis/uv.lock b/ReforceXY/reward_space_analysis/uv.lock index ea5e288..6229cae 100644 --- a/ReforceXY/reward_space_analysis/uv.lock +++ b/ReforceXY/reward_space_analysis/uv.lock @@ -483,7 +483,7 @@ requires-dist = [ { name = "pandas" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.3" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=7.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.8" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.16.5" }, { name = "scikit-learn" }, { name = "scipy", specifier = ">=1.11" }, ] @@ -491,27 +491,27 @@ provides-extras = ["dev"] [[package]] name = "ruff" -version = "0.16.3" +version = "0.16.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/85/c8e12473c93018f92d19dd988a294202e1c27426c47ec4de53ffb847b8d8/ruff-0.16.5.tar.gz", hash = "sha256:1b88500f9ffbcab3dedb0082c9f9492e91ec3d618aac1236a3e0189938f7040b", size = 4912003, upload-time = "2026-08-27T16:34:18.258Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, - { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, - { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, - { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, - { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, - { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, - { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, - { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, - { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, - { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, - { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, - { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, - { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, - { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, - { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, - { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/77c90a970fe2dae17a723acbd011043ea97c98d7deacccefdc4ba74ec512/ruff-0.16.5-py3-none-linux_armv6l.whl", hash = "sha256:12e5f673e774c35fbb62f288809c7653b73445f8ecec6b6063fd6ea3521aa14b", size = 10011941, upload-time = "2026-08-27T16:33:41.287Z" }, + { url = "https://files.pythonhosted.org/packages/4b/46/6cf67cf6411885a1d6f7f6d801682f155536a85176d10b605e2ceffed8bd/ruff-0.16.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:eda58a5802de40e7ed5b32b64e0b32539338cc6fcd2c78f61e3ad6a0d79f51c3", size = 10204049, upload-time = "2026-08-27T16:33:44.056Z" }, + { url = "https://files.pythonhosted.org/packages/46/fd/c8720ca7a090abf0c2fef4abe8a5ef6e5127ed15196d8886ff75a2b370e2/ruff-0.16.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5ae9a7b9a8875131f40f8fe967cc86abf899779efd663cb7ce3d572d01da7eb", size = 9809037, upload-time = "2026-08-27T16:33:46.257Z" }, + { url = "https://files.pythonhosted.org/packages/43/45/a684caacdedaca180f52bacccc40bf0789d2c5a7c75f25324853e9eaedb5/ruff-0.16.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b719b0a1f4d59710d283ab2965f621684a108a9e41da622e3b23f0326cd0025", size = 9964129, upload-time = "2026-08-27T16:33:48.352Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/5d2bcdaca6b5b93d1b4dfc166cd2aebf7680143a1b38a28759df13a94d31/ruff-0.16.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2298f2780ed1be0c5cb1361e32ab7b1467f3cce7dabe101d2210a314f2fe42e9", size = 9821518, upload-time = "2026-08-27T16:33:50.57Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ff/011cce29accf9257d5974145b733fc653a37985ed6825413a3987cefbfe0/ruff-0.16.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:258f29035a2dd021e7861e631b227a5b3f14e50c1184c9a6a122c5f4576154d7", size = 10534835, upload-time = "2026-08-27T16:33:52.522Z" }, + { url = "https://files.pythonhosted.org/packages/d7/5a/f0cf109bada9bba0e96c90c21c9f9251803f57225c32d293327a03c710d6/ruff-0.16.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9a4f0432966834019c74d1b7e5c51224305d7713f3d7faf3e7451f1a3be3cde", size = 11252550, upload-time = "2026-08-27T16:33:54.521Z" }, + { url = "https://files.pythonhosted.org/packages/63/4d/1d481aaea2046c6a7ed7c291f9004c669cce3c087b6b376ed5b08271e3fe/ruff-0.16.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5eb3a8c3d0ade9cea42b591fd530368e8798380e30e0a308b85a5cf718f09ea", size = 10777949, upload-time = "2026-08-27T16:33:56.88Z" }, + { url = "https://files.pythonhosted.org/packages/ee/34/ee245ca55f64443233034b3d02b03236b19242004281247c079390b7facd/ruff-0.16.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0f69e191a13a3c9816f63163c88790cb12cd157bbbb384e9c44745702ab105", size = 10311656, upload-time = "2026-08-27T16:33:59.12Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/c33a333e341c0a2b96c715b52d89a606f5a34cd4ac493cd9b8d0187186b8/ruff-0.16.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0eeab41fbea2c42f98dfb9822cdccda9d24ba38d49f6dc945b5c236d48f0ef29", size = 10532125, upload-time = "2026-08-27T16:34:01.166Z" }, + { url = "https://files.pythonhosted.org/packages/30/e1/a64cef78b40192497bb98a27a8aa8f2c98ee9ee15bc97f7712d94ef32937/ruff-0.16.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f0768e9df4300713fff30733c87575f68b6f1d8de41184e505b7fdd9c0c95eaf", size = 10097648, upload-time = "2026-08-27T16:34:03.16Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4e/4cdc9ed3c3e109d2f71e62572a37457298d7bc7501ec3138babb7ed32bbd/ruff-0.16.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:95cc70cdc7aa80c338de356279d2adbeb2de0f520b9ecd8aba75b94e95e02f91", size = 9829344, upload-time = "2026-08-27T16:34:05.134Z" }, + { url = "https://files.pythonhosted.org/packages/39/4a/31ed35ce31729955fc583ee0d176d6e784c1290cb0b0a75cb2134c1ab72a/ruff-0.16.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d185c8398ded1bfd91c0c2cb258346307571eccc473a8490af8c3977399c384a", size = 10277117, upload-time = "2026-08-27T16:34:07.425Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a0/60356d86687b4b666d593df213f4dc3041750d024cb7bf2cfa81cfd65c2e/ruff-0.16.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fb8e3a3c4c6a784150a7ced53b015f4b253fc2bf97a610886419ead64b4756ef", size = 10711653, upload-time = "2026-08-27T16:34:09.712Z" }, + { url = "https://files.pythonhosted.org/packages/ed/20/656d67f5b25ca9bda4e02b1de25867b2954e1d19e03648060f167ad0f4cc/ruff-0.16.5-py3-none-win32.whl", hash = "sha256:288b0a5f080492fe5635db849f9e2e84aa3cce7b7f0e955997d416c507c76a26", size = 10034250, upload-time = "2026-08-27T16:34:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/5b/42/ee8e68a207b9127fcde6c3d7e197def432f346cb1af159e1fa14ca0d1cdc/ruff-0.16.5-py3-none-win_amd64.whl", hash = "sha256:ddc6385fb2137f616357ca03d6c74f4be987f80fed4008566b754f6032b8546f", size = 10516714, upload-time = "2026-08-27T16:34:13.963Z" }, + { url = "https://files.pythonhosted.org/packages/73/e3/7df5a396e445b9ba49ce9a9437439a4d80042c61c0ade199abf8d16de1ac/ruff-0.16.5-py3-none-win_arm64.whl", hash = "sha256:a64abe90968719b851bb7cedffaa8753fbdbdadab483089682db623f3edc587e", size = 10391564, upload-time = "2026-08-27T16:34:16.064Z" }, ] [[package]] diff --git a/ReforceXY/user_data/freqaimodels/ReforceXY.py b/ReforceXY/user_data/freqaimodels/ReforceXY.py index c1414ae..7cf9257 100644 --- a/ReforceXY/user_data/freqaimodels/ReforceXY.py +++ b/ReforceXY/user_data/freqaimodels/ReforceXY.py @@ -9,23 +9,16 @@ import stat import time import warnings from collections import defaultdict, deque -from collections.abc import Iterator, Mapping -from contextlib import contextmanager +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager, suppress from datetime import datetime, timezone from pathlib import Path from typing import ( Any, - Callable, ClassVar, - Dict, Final, - List, Literal, NamedTuple, - Optional, - Tuple, - Type, - Union, assert_never, cast, ) @@ -103,9 +96,7 @@ def _dedupe_historic_predictions_on_date_pred(frame: pd.DataFrame) -> pd.DataFra block = work[content] numeric = block.apply(pd.to_numeric, errors="coerce") is_numeric = numeric.notna() - informative = (block.notna() & is_numeric & numeric.ne(0)) | ( - block.notna() & ~is_numeric - ) + informative = (block.notna() & is_numeric & numeric.ne(0)) | (block.notna() & ~is_numeric) work = work.assign( _dp=date_pred.to_numpy(), _score=informative.sum(axis=1).to_numpy(), @@ -116,9 +107,7 @@ def _dedupe_historic_predictions_on_date_pred(frame: pd.DataFrame) -> pd.DataFra ["_dp", "_score", "_nonnull", "_order"], kind="stable" ) kept = contested.drop_duplicates("_dp", keep="last") - return kept.drop(columns=["_dp", "_score", "_nonnull", "_order"]).reset_index( - drop=True - ) + return kept.drop(columns=["_dp", "_score", "_nonnull", "_order"]).reset_index(drop=True) def _install_date_pred_dedup_patch() -> None: @@ -141,13 +130,9 @@ def _install_date_pred_dedup_patch() -> None: self, pair: str, pred_df: pd.DataFrame, dataframe: pd.DataFrame ) -> None: original_set_initial(self, pair, pred_df, dataframe) - frame = _dedupe_historic_predictions_on_date_pred( - self.historic_predictions[pair] - ) + frame = _dedupe_historic_predictions_on_date_pred(self.historic_predictions[pair]) self.historic_predictions[pair] = frame - self.model_return_values[pair] = frame.tail(len(dataframe.index)).reset_index( - drop=True - ) + self.model_return_values[pair] = frame.tail(len(dataframe.index)).reset_index(drop=True) def append_model_predictions( self, @@ -158,13 +143,9 @@ def _install_date_pred_dedup_patch() -> None: strat_df: pd.DataFrame, ) -> None: original_append(self, pair, predictions, do_preds, dk, strat_df) - frame = _dedupe_historic_predictions_on_date_pred( - self.historic_predictions[pair] - ) + frame = _dedupe_historic_predictions_on_date_pred(self.historic_predictions[pair]) self.historic_predictions[pair] = frame - self.model_return_values[pair] = frame.tail(len(strat_df.index)).reset_index( - drop=True - ) + self.model_return_values[pair] = frame.tail(len(strat_df.index)).reset_index(drop=True) def attach_return_values_to_return_dataframe( self, pair: str, dataframe: pd.DataFrame @@ -186,7 +167,7 @@ _install_date_pred_dedup_patch() ModelType = Literal["PPO", "RecurrentPPO", "MaskablePPO", "DQN", "QRDQN"] ScheduleTypeKnown = Literal["linear", "constant"] -ScheduleType = Union[ScheduleTypeKnown, Literal["unknown"]] +ScheduleType = ScheduleTypeKnown | Literal["unknown"] ExitPotentialMode = Literal[ "canonical", "non_canonical", @@ -198,7 +179,7 @@ TransformFunction = Literal["tanh", "softsign", "arctan", "sigmoid", "asinh", "c ExitAttenuationMode = Literal["legacy", "sqrt", "linear", "power", "half_life"] ActivationFunction = Literal["relu", "tanh", "elu", "leaky_relu"] OptimizerClassOptuna = Literal["adamw", "rmsprop"] -OptimizerClass = Union[OptimizerClassOptuna, Literal["adam"]] +OptimizerClass = OptimizerClassOptuna | Literal["adam"] NetArchSize = Literal["small", "medium", "large", "extra_large"] StorageBackend = Literal["sqlite", "file"] SamplerType = Literal["tpe", "auto"] @@ -311,7 +292,7 @@ class ReforceXY(BaseReinforcementLearningModel): DEFAULT_EFFICIENCY_MIN_RANGE_EPSILON: Final[float] = 1e-6 DEFAULT_EFFICIENCY_MIN_RANGE_FRACTION: Final[float] = 0.01 - _MODEL_TYPES: Final[Tuple[ModelType, ...]] = ( + _MODEL_TYPES: Final[tuple[ModelType, ...]] = ( "PPO", "RecurrentPPO", "MaskablePPO", @@ -319,12 +300,12 @@ class ReforceXY(BaseReinforcementLearningModel): "QRDQN", ) _MODEL_TYPES_SET: Final[frozenset[ModelType]] = frozenset(_MODEL_TYPES) - _SCHEDULE_TYPES_KNOWN: Final[Tuple[ScheduleTypeKnown, ...]] = ("linear", "constant") - _SCHEDULE_TYPES: Final[Tuple[ScheduleType, ...]] = ( + _SCHEDULE_TYPES_KNOWN: Final[tuple[ScheduleTypeKnown, ...]] = ("linear", "constant") + _SCHEDULE_TYPES: Final[tuple[ScheduleType, ...]] = ( *_SCHEDULE_TYPES_KNOWN, "unknown", ) - _EXIT_POTENTIAL_MODES: Final[Tuple[ExitPotentialMode, ...]] = ( + _EXIT_POTENTIAL_MODES: Final[tuple[ExitPotentialMode, ...]] = ( "canonical", "non_canonical", "progressive_release", @@ -334,7 +315,7 @@ class ReforceXY(BaseReinforcementLearningModel): _EXIT_POTENTIAL_MODES_SET: Final[frozenset[ExitPotentialMode]] = frozenset( _EXIT_POTENTIAL_MODES ) - _TRANSFORM_FUNCTIONS: Final[Tuple[TransformFunction, ...]] = ( + _TRANSFORM_FUNCTIONS: Final[tuple[TransformFunction, ...]] = ( "tanh", "softsign", "arctan", @@ -342,44 +323,40 @@ class ReforceXY(BaseReinforcementLearningModel): "asinh", "clip", ) - _TRANSFORM_FUNCTIONS_SET: Final[frozenset[TransformFunction]] = frozenset( - _TRANSFORM_FUNCTIONS - ) - _EXIT_ATTENUATION_MODES: Final[Tuple[ExitAttenuationMode, ...]] = ( + _TRANSFORM_FUNCTIONS_SET: Final[frozenset[TransformFunction]] = frozenset(_TRANSFORM_FUNCTIONS) + _EXIT_ATTENUATION_MODES: Final[tuple[ExitAttenuationMode, ...]] = ( "legacy", "sqrt", "linear", "power", "half_life", ) - _ACTIVATION_FUNCTIONS: Final[Tuple[ActivationFunction, ...]] = ( + _ACTIVATION_FUNCTIONS: Final[tuple[ActivationFunction, ...]] = ( "relu", "tanh", "elu", "leaky_relu", ) - _OPTIMIZER_CLASSES_OPTUNA: Final[Tuple[OptimizerClassOptuna, ...]] = ( + _OPTIMIZER_CLASSES_OPTUNA: Final[tuple[OptimizerClassOptuna, ...]] = ( "adamw", "rmsprop", ) - _OPTIMIZER_CLASSES: Final[Tuple[OptimizerClass, ...]] = ( + _OPTIMIZER_CLASSES: Final[tuple[OptimizerClass, ...]] = ( *_OPTIMIZER_CLASSES_OPTUNA, "adam", ) - _NET_ARCH_SIZES: Final[Tuple[NetArchSize, ...]] = ( + _NET_ARCH_SIZES: Final[tuple[NetArchSize, ...]] = ( "small", "medium", "large", "extra_large", ) - _STORAGE_BACKENDS: Final[Tuple[StorageBackend, ...]] = ("sqlite", "file") + _STORAGE_BACKENDS: Final[tuple[StorageBackend, ...]] = ("sqlite", "file") _SAMPLERS: Final[_Samplers] = _Samplers() _JOURNAL_TAIL_PROBE_BYTES: Final[int] = 64 * 1024 _JOURNAL_OP_CODE_KEY: Final[str] = "op_code" _JOURNAL_OPERATION_CODES: Final[frozenset[int]] = frozenset(range(10)) - _JOURNAL_RECOVERABLE_ERRORS: Final[ - type[Exception] | tuple[type[Exception], ...] - ] = ( + _JOURNAL_RECOVERABLE_ERRORS: Final[type[Exception] | tuple[type[Exception], ...]] = ( KeyError, AssertionError, TypeError, @@ -389,53 +366,43 @@ class ReforceXY(BaseReinforcementLearningModel): _QUARANTINE_TAG: Final[str] = "corrupt" _QUARANTINE_TIE_BREAK_LIMIT: Final[int] = 99 _BEST_PARAMS_LOCK_FILENAME: Final[str] = ".hyperopt-best-params.lock" - _PPO_N_STEPS: Final[Tuple[int, ...]] = (512, 1024, 2048, 4096) + _PPO_N_STEPS: Final[tuple[int, ...]] = (512, 1024, 2048, 4096) _PPO_N_STEPS_MIN: Final[int] = min(_PPO_N_STEPS) _PPO_N_STEPS_MAX: Final[int] = max(_PPO_N_STEPS) _HYPEROPT_EVAL_FREQ_REDUCTION_FACTOR: Final[float] = 4.0 - _action_masks_cache: ClassVar[Dict[Tuple[bool, float], NDArray[np.bool_]]] = {} + _action_masks_cache: ClassVar[dict[tuple[bool, float], NDArray[np.bool_]]] = {} def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.pairs: List[str] = self.config.get("exchange", {}).get("pair_whitelist") + self.pairs: list[str] = self.config.get("exchange", {}).get("pair_whitelist") if not self.pairs: raise ValueError( "Config [global]: missing 'pair_whitelist' in exchange section " "or StaticPairList method not defined in pairlists configuration" ) - self.action_masking: bool = ( - self.model_type == ReforceXY._MODEL_TYPES[2] - ) # "MaskablePPO" + self.action_masking: bool = self.model_type == ReforceXY._MODEL_TYPES[2] # "MaskablePPO" self.rl_config.setdefault("action_masking", self.action_masking) self.inference_masking: bool = self.rl_config.get("inference_masking", True) - self.recurrent: bool = ( - self.model_type == ReforceXY._MODEL_TYPES[1] - ) # "RecurrentPPO" + self.recurrent: bool = self.model_type == ReforceXY._MODEL_TYPES[1] # "RecurrentPPO" self.lr_schedule: bool = self.rl_config.get("lr_schedule", False) self.cr_schedule: bool = self.rl_config.get("cr_schedule", False) self.n_envs: int = self.rl_config.get("n_envs", 1) self.n_eval_envs: int = self.rl_config.get("n_eval_envs", 1) self.multiprocessing: bool = self.rl_config.get("multiprocessing", False) - self.eval_multiprocessing: bool = self.rl_config.get( - "eval_multiprocessing", False - ) + self.eval_multiprocessing: bool = self.rl_config.get("eval_multiprocessing", False) self.frame_stacking: int = self.rl_config.get("frame_stacking", 0) self.n_eval_steps: int = self.rl_config.get("n_eval_steps", 10_000) self.n_eval_episodes: int = self.rl_config.get("n_eval_episodes", 5) - self.max_no_improvement_evals: int = self.rl_config.get( - "max_no_improvement_evals", 0 - ) + self.max_no_improvement_evals: int = self.rl_config.get("max_no_improvement_evals", 0) self.min_evals: int = self.rl_config.get("min_evals", 0) self.rl_config.setdefault("tensorboard_throttle", 1) self.plot_new_best: bool = self.rl_config.get("plot_new_best", False) self.check_envs: bool = self.rl_config.get("check_envs", True) - self.progressbar_callback: Optional[ProgressBarCallback] = None + self.progressbar_callback: ProgressBarCallback | None = None # Optuna hyperopt - self.rl_config_optuna: Dict[str, Any] = self.freqai_info.get( - "rl_config_optuna", {} - ) + self.rl_config_optuna: dict[str, Any] = self.freqai_info.get("rl_config_optuna", {}) self.hyperopt: bool = ( self.freqai_info.get("enabled", False) and self.rl_config_optuna.get("enabled", False) @@ -443,19 +410,15 @@ class ReforceXY(BaseReinforcementLearningModel): ) self.optuna_timeout_hours: float = self.rl_config_optuna.get("timeout_hours", 0) self.optuna_n_trials: int = self.rl_config_optuna.get("n_trials", 100) - self.optuna_n_startup_trials: int = self.rl_config_optuna.get( - "n_startup_trials", 15 - ) - self.optuna_purge_period: int = int( - self.rl_config_optuna.get("purge_period", 0) - ) - self.optuna_eval_callback: Optional[MaskableTrialEvalCallback] = None - self._model_params_cache: Optional[Dict[str, Any]] = None - self._lstm_states_cache: Dict[ + self.optuna_n_startup_trials: int = self.rl_config_optuna.get("n_startup_trials", 15) + self.optuna_purge_period: int = int(self.rl_config_optuna.get("purge_period", 0)) + self.optuna_eval_callback: MaskableTrialEvalCallback | None = None + self._model_params_cache: dict[str, Any] | None = None + self._lstm_states_cache: dict[ str, - Tuple[ + tuple[ int, - Optional[Tuple[NDArray[np.float32], NDArray[np.float32]]], + tuple[NDArray[np.float32], NDArray[np.float32]] | None, NDArray[np.bool_], ], ] = {} @@ -467,7 +430,7 @@ class ReforceXY(BaseReinforcementLearningModel): Configure GPU memory fraction limit from model_training_parameters. Called after config validation, before any CUDA operations. """ - gpu_memory_fraction: Optional[float] = self.model_training_parameters.get( + gpu_memory_fraction: float | None = self.model_training_parameters.get( "gpu_memory_fraction" ) if gpu_memory_fraction is None: @@ -550,9 +513,7 @@ class ReforceXY(BaseReinforcementLearningModel): function will set them to proper values and warn them """ if not isinstance(self.n_envs, int) or self.n_envs < 1: - logger.warning( - "Config [global]: n_envs=%r invalid; defaulting to 1", self.n_envs - ) + logger.warning("Config [global]: n_envs=%r invalid; defaulting to 1", self.n_envs) self.n_envs = 1 if not isinstance(self.n_eval_envs, int) or self.n_eval_envs < 1: logger.warning( @@ -600,19 +561,13 @@ class ReforceXY(BaseReinforcementLearningModel): self.n_eval_episodes, ) self.n_eval_episodes = 5 - if ( - not isinstance(self.optuna_purge_period, int) - or self.optuna_purge_period < 0 - ): + if not isinstance(self.optuna_purge_period, int) or self.optuna_purge_period < 0: logger.warning( "Config [global]: purge_period=%r invalid; defaulting to 0", self.optuna_purge_period, ) self.optuna_purge_period = 0 - if ( - self.rl_config_optuna.get("continuous", False) - and self.optuna_purge_period > 0 - ): + if self.rl_config_optuna.get("continuous", False) and self.optuna_purge_period > 0: logger.warning( "Config [global]: purge_period has no effect when continuous=True; defaulting to 0" ) @@ -643,8 +598,8 @@ class ReforceXY(BaseReinforcementLearningModel): ) def pack_env_dict( - self, pair: str, model_params: Optional[Dict[str, Any]] = None - ) -> Dict[str, Any]: + self, pair: str, model_params: dict[str, Any] | None = None + ) -> dict[str, Any]: env_info = super().pack_env_dict(pair) config = env_info.setdefault("config", {}) @@ -652,15 +607,13 @@ class ReforceXY(BaseReinforcementLearningModel): rl_cfg = freqai_cfg.setdefault("rl_config", {}) model_reward_parameters = rl_cfg.setdefault("model_reward_parameters", {}) - gamma: Optional[float] = None + gamma: float | None = None if model_params and isinstance(model_params.get("gamma"), (int, float)): gamma = float(model_params.get("gamma")) elif self.hyperopt: best_trial_params = self.load_best_trial_params(pair) - if best_trial_params and isinstance( - best_trial_params.get("gamma"), (int, float) - ): + if best_trial_params and isinstance(best_trial_params.get("gamma"), (int, float)): gamma = float(best_trial_params.get("gamma")) if ( @@ -678,15 +631,13 @@ class ReforceXY(BaseReinforcementLearningModel): if gamma is not None: model_reward_parameters["potential_gamma"] = gamma else: - logger.warning( - "Env [%s]: no valid discount gamma resolved for environment", pair - ) + logger.warning("Env [%s]: no valid discount gamma resolved for environment", pair) return env_info def set_train_and_eval_environments( self, - data_dictionary: Dict[str, DataFrame], + data_dictionary: dict[str, DataFrame], prices_train: DataFrame, prices_test: DataFrame, dk: FreqaiDataKitchen, @@ -744,14 +695,14 @@ class ReforceXY(BaseReinforcementLearningModel): env_info=env_dict, ) - def get_model_params(self) -> Dict[str, Any]: + def get_model_params(self) -> dict[str, Any]: """ Get model parameters """ if self._model_params_cache is not None: return copy.deepcopy(self._model_params_cache) - model_params: Dict[str, Any] = copy.deepcopy(self.model_training_parameters) + model_params: dict[str, Any] = copy.deepcopy(self.model_training_parameters) model_params.setdefault("seed", 42) model_params.setdefault("gamma", 0.95) @@ -761,7 +712,7 @@ class ReforceXY(BaseReinforcementLearningModel): if isinstance(lr, (int, float)): lr = float(lr) model_params["learning_rate"] = get_schedule( - cast(ScheduleTypeKnown, ReforceXY._SCHEDULE_TYPES[0]), lr + cast("ScheduleTypeKnown", ReforceXY._SCHEDULE_TYPES[0]), lr ) logger.info( "Config [global]: learning rate linear schedule enabled, initial=%.6f", @@ -769,16 +720,12 @@ class ReforceXY(BaseReinforcementLearningModel): ) # "PPO" - if ( - not self.hyperopt - and ReforceXY._MODEL_TYPES[0] in self.model_type - and self.cr_schedule - ): + if not self.hyperopt and ReforceXY._MODEL_TYPES[0] in self.model_type and self.cr_schedule: cr = model_params.get("clip_range", 0.2) if isinstance(cr, (int, float)): cr = float(cr) model_params["clip_range"] = get_schedule( - cast(ScheduleTypeKnown, ReforceXY._SCHEDULE_TYPES[0]), cr + cast("ScheduleTypeKnown", ReforceXY._SCHEDULE_TYPES[0]), cr ) logger.info( "Config [global]: clip range linear schedule enabled, initial=%.2f", @@ -797,12 +744,10 @@ class ReforceXY(BaseReinforcementLearningModel): if not model_params.get("policy_kwargs"): model_params["policy_kwargs"] = {} - default_net_arch: List[int] = [128, 128] - net_arch: Union[ - List[int], - Dict[str, List[int]], - NetArchSize, - ] = model_params.get("policy_kwargs", {}).get("net_arch", default_net_arch) + default_net_arch: list[int] = [128, 128] + net_arch: list[int] | dict[str, list[int]] | NetArchSize = model_params.get( + "policy_kwargs", {} + ).get("net_arch", default_net_arch) # "PPO" if ReforceXY._MODEL_TYPES[0] in self.model_type: @@ -810,7 +755,7 @@ class ReforceXY(BaseReinforcementLearningModel): if net_arch in ReforceXY._NET_ARCH_SIZES: model_params["policy_kwargs"]["net_arch"] = get_net_arch( self.model_type, - cast(NetArchSize, net_arch), + cast("NetArchSize", net_arch), ) else: logger.warning( @@ -830,14 +775,10 @@ class ReforceXY(BaseReinforcementLearningModel): } elif isinstance(net_arch, dict): pi = ( - net_arch.get("pi") - if isinstance(net_arch.get("pi"), list) - else default_net_arch + net_arch.get("pi") if isinstance(net_arch.get("pi"), list) else default_net_arch ) vf = ( - net_arch.get("vf") - if isinstance(net_arch.get("vf"), list) - else default_net_arch + net_arch.get("vf") if isinstance(net_arch.get("vf"), list) else default_net_arch ) model_params["policy_kwargs"]["net_arch"] = {"pi": pi, "vf": vf} else: @@ -856,7 +797,7 @@ class ReforceXY(BaseReinforcementLearningModel): if net_arch in ReforceXY._NET_ARCH_SIZES: model_params["policy_kwargs"]["net_arch"] = get_net_arch( self.model_type, - cast(NetArchSize, net_arch), + cast("NetArchSize", net_arch), ) else: logger.warning( @@ -898,7 +839,7 @@ class ReforceXY(BaseReinforcementLearningModel): total_timesteps: int, hyperopt: bool = False, hyperopt_reduction_factor: float = _HYPEROPT_EVAL_FREQ_REDUCTION_FACTOR, - model_params: Optional[Dict[str, Any]] = None, + model_params: dict[str, Any] | None = None, ) -> int: """Calculate evaluation frequency. @@ -921,7 +862,7 @@ class ReforceXY(BaseReinforcementLearningModel): # "PPO" if ReforceXY._MODEL_TYPES[0] in self.model_type: - eval_freq: Optional[int] = None + eval_freq: int | None = None if model_params: n_steps = model_params.get("n_steps") if isinstance(n_steps, int) and n_steps > 0: @@ -939,7 +880,7 @@ class ReforceXY(BaseReinforcementLearningModel): eval_freq = max(1, (self.n_eval_steps + n_envs - 1) // n_envs) if hyperopt and hyperopt_reduction_factor > 1.0: - eval_freq = max(1, int(round(eval_freq / hyperopt_reduction_factor))) + eval_freq = max(1, round(eval_freq / hyperopt_reduction_factor)) return min(eval_freq, max_n_calls) @@ -948,12 +889,12 @@ class ReforceXY(BaseReinforcementLearningModel): eval_env: BaseEnvironment, eval_freq: int, data_path: str, - trial: Optional[Trial] = None, - ) -> List[BaseCallback]: + trial: Trial | None = None, + ) -> list[BaseCallback]: """ Get the model specific callbacks """ - callbacks: List[BaseCallback] = [] + callbacks: list[BaseCallback] = [] no_improvement_callback = None rollout_plot_callback = None verbose = self.get_model_params().get("verbose", 0) @@ -1010,9 +951,7 @@ class ReforceXY(BaseReinforcementLearningModel): callbacks.append(self.optuna_eval_callback) return callbacks - def fit( - self, data_dictionary: Dict[str, Any], dk: FreqaiDataKitchen, **kwargs - ) -> Any: + def fit(self, data_dictionary: dict[str, Any], dk: FreqaiDataKitchen, **kwargs) -> Any: """ Model fitting method :param data_dictionary: dict = common data dictionary containing all train/test features/labels/weights. @@ -1023,15 +962,11 @@ class ReforceXY(BaseReinforcementLearningModel): train_df = data_dictionary.get("train_features") train_timesteps = len(train_df) if train_timesteps <= 0: - raise ValueError( - f"Training [{dk.pair}]: train_features dataframe has zero length" - ) + raise ValueError(f"Training [{dk.pair}]: train_features dataframe has zero length") test_df = data_dictionary.get("test_features") eval_timesteps = len(test_df) train_cycles = max(1, int(self.rl_config.get("train_cycles", 25))) - total_timesteps = ReforceXY._ceil_to_multiple( - train_timesteps * train_cycles, self.n_envs - ) + total_timesteps = ReforceXY._ceil_to_multiple(train_timesteps * train_cycles, self.n_envs) train_days = steps_to_days(train_timesteps, self.config.get("timeframe")) eval_days = steps_to_days(eval_timesteps, self.config.get("timeframe")) total_days = steps_to_days(total_timesteps, self.config.get("timeframe")) @@ -1095,9 +1030,7 @@ class ReforceXY(BaseReinforcementLearningModel): ) if n_steps > 0: rollout = n_steps * self.n_envs - aligned_total_timesteps = ReforceXY._ceil_to_multiple( - total_timesteps, rollout - ) + aligned_total_timesteps = ReforceXY._ceil_to_multiple(total_timesteps, rollout) if aligned_total_timesteps != total_timesteps: total_timesteps = aligned_total_timesteps logger.info( @@ -1109,9 +1042,7 @@ class ReforceXY(BaseReinforcementLearningModel): ) if self.activate_tensorboard: - tensorboard_log_path = Path( - self.full_path / "tensorboard" / Path(dk.data_path).name - ) + tensorboard_log_path = Path(self.full_path / "tensorboard" / Path(dk.data_path).name) else: tensorboard_log_path = None @@ -1119,9 +1050,7 @@ class ReforceXY(BaseReinforcementLearningModel): prices_train, prices_test = self.build_ohlc_price_dataframes( dk.data_dictionary, dk.pair, dk ) - self.set_train_and_eval_environments( - dk.data_dictionary, prices_train, prices_test, dk - ) + self.set_train_and_eval_environments(dk.data_dictionary, prices_train, prices_test, dk) model = self.get_init_model(dk.pair) if model is not None: @@ -1165,9 +1094,7 @@ class ReforceXY(BaseReinforcementLearningModel): if model_filepath.is_file(): logger.info("Model [%s]: found best model at %s", dk.pair, model_filepath) try: - best_model = self.MODELCLASS.load( - dk.data_path / f"{model_filename}_model" - ) + best_model = self.MODELCLASS.load(dk.data_path / f"{model_filename}_model") return best_model except Exception as e: logger.error( @@ -1201,9 +1128,7 @@ class ReforceXY(BaseReinforcementLearningModel): position, _, trade_duration = self.get_state_info(dk.pair) virtual_position = ReforceXY._normalize_position(position) virtual_trade_duration = trade_duration - np_dataframe: NDArray[np.float32] = dataframe.to_numpy( - dtype=np.float32, copy=False - ) + np_dataframe: NDArray[np.float32] = dataframe.to_numpy(dtype=np.float32, copy=False) n = np_dataframe.shape[0] window_size: int = self.CONV_WIDTH frame_stacking: int = self.frame_stacking @@ -1241,7 +1166,7 @@ class ReforceXY(BaseReinforcementLearningModel): frame_buffer: deque[np.float32] = deque( maxlen=frame_stacking if frame_stacking_enabled else None ) - zero_frame: Optional[NDArray[np.float32]] = None + zero_frame: NDArray[np.float32] | None = None model_id = id(model) lstm_states_cache_valid = ( self.live @@ -1265,16 +1190,14 @@ class ReforceXY(BaseReinforcementLearningModel): self.live, self.recurrent, ) - lstm_states: Optional[Tuple[NDArray[np.float32], NDArray[np.float32]]] = ( - None - ) + lstm_states: tuple[NDArray[np.float32], NDArray[np.float32]] | None = None episode_start = np.array([True], dtype=bool) def _predict(start_idx: int) -> int: nonlocal zero_frame, lstm_states, episode_start end_idx: int = start_idx + window_size np_observation = np_dataframe[start_idx:end_idx, :] - action_masks_param: Dict[str, Any] = {} + action_masks_param: dict[str, Any] = {} if add_state_info: if self.live: @@ -1356,9 +1279,7 @@ class ReforceXY(BaseReinforcementLearningModel): dk.pair, observations.shape, ) - action, _ = model.predict( - observations, deterministic=True, **action_masks_param - ) + action, _ = model.predict(observations, deterministic=True, **action_masks_param) action = int(action.item()) logger.debug( "Predict [%s]: predicted action=%s (%d)", @@ -1369,7 +1290,7 @@ class ReforceXY(BaseReinforcementLearningModel): return action - predicted_actions: List[int] = [] + predicted_actions: list[int] = [] for start_idx in range(0, n - window_size + 1): action = _predict(start_idx) predicted_actions.append(action) @@ -1388,7 +1309,7 @@ class ReforceXY(BaseReinforcementLearningModel): if self.live and self.recurrent: self._lstm_states_cache[dk.pair] = (model_id, lstm_states, episode_start) - return DataFrame({label: actions_df["action"] for label in dk.label_list}) + return DataFrame(dict.fromkeys(dk.label_list, actions_df["action"])) @staticmethod def delete_study(study_name: str, storage: BaseStorage) -> None: @@ -1411,15 +1332,15 @@ class ReforceXY(BaseReinforcementLearningModel): def _optuna_retrain_counters_path(self) -> Path: return Path(self.full_path / "optuna-retrain-counters.json") - def _load_optuna_retrain_counters(self, pair: str) -> Dict[str, int]: + def _load_optuna_retrain_counters(self, pair: str) -> dict[str, int]: counters_path = self._optuna_retrain_counters_path() if not counters_path.is_file(): return {} try: with counters_path.open("r", encoding="utf-8") as read_file: - data: Dict[str, int] = json.load(read_file) + data: dict[str, int] = json.load(read_file) if isinstance(data, dict): - result: Dict[str, int] = {} + result: dict[str, int] = {} for key, value in data.items(): if isinstance(key, str) and isinstance(value, int): result[key] = value @@ -1434,9 +1355,7 @@ class ReforceXY(BaseReinforcementLearningModel): ) return {} - def _save_optuna_retrain_counters( - self, counters: Dict[str, int], pair: str - ) -> None: + def _save_optuna_retrain_counters(self, counters: dict[str, int], pair: str) -> None: counters_path = self._optuna_retrain_counters_path() try: with counters_path.open("w", encoding="utf-8") as write_file: @@ -1498,10 +1417,7 @@ class ReforceXY(BaseReinforcementLearningModel): op_code = record.get(ReforceXY._JOURNAL_OP_CODE_KEY) if op_code is None and fail_open_missing_op_code: return False - return ( - type(op_code) is not int - or op_code not in ReforceXY._JOURNAL_OPERATION_CODES - ) + return type(op_code) is not int or op_code not in ReforceXY._JOURNAL_OPERATION_CODES @staticmethod def _create_recovered_journal_storage( @@ -1532,9 +1448,7 @@ class ReforceXY(BaseReinforcementLearningModel): def _quarantine_journal(journal_path: Path, cause: Exception) -> Path | None: if not journal_path.exists(): return None - quarantine_path = ReforceXY._quarantine_path( - journal_path, datetime.now(timezone.utc) - ) + quarantine_path = ReforceXY._quarantine_path(journal_path, datetime.now(timezone.utc)) journal_path.rename(quarantine_path) logger.warning( "Optuna journal %s corrupt (%r); quarantined to %s; resuming with fresh journal", @@ -1591,7 +1505,7 @@ class ReforceXY(BaseReinforcementLearningModel): return storage @staticmethod - def study_has_best_trial(study: Optional[Study]) -> bool: + def study_has_best_trial(study: Study | None) -> bool: if study is None: return False try: @@ -1607,7 +1521,7 @@ class ReforceXY(BaseReinforcementLearningModel): f"Hyperopt [global]: unsupported sampler '{sampler_value}'. " f"Valid: {', '.join(ReforceXY._SAMPLERS)}" ) - sampler = cast(SamplerType, sampler_value) + sampler = cast("SamplerType", sampler_value) seed = self.rl_config_optuna.get("seed", 42) match sampler: case ReforceXY._SAMPLERS.tpe: @@ -1627,16 +1541,12 @@ class ReforceXY(BaseReinforcementLearningModel): "Hyperopt [global]: using AutoSampler (seed=%d)", seed, ) - return optunahub.load_module("samplers/auto_sampler").AutoSampler( - seed=seed - ) + return optunahub.load_module("samplers/auto_sampler").AutoSampler(seed=seed) case _: assert_never(sampler) @staticmethod - def create_pruner( - min_resource: int, max_resource: int, reduction_factor: int - ) -> BasePruner: + def create_pruner(min_resource: int, max_resource: int, reduction_factor: int) -> BasePruner: logger.info( "Hyperopt [global]: using HyperbandPruner (min_resource=%d, max_resource=%d, reduction_factor=%d)", min_resource, @@ -1654,15 +1564,12 @@ class ReforceXY(BaseReinforcementLearningModel): return ((value + multiple - 1) // multiple) * multiple @staticmethod - def _ppo_resources( - total_timesteps: int, n_envs: int, reduction_factor: int - ) -> Tuple[int, int]: + def _ppo_resources(total_timesteps: int, n_envs: int, reduction_factor: int) -> tuple[int, int]: min_n_steps = ReforceXY._PPO_N_STEPS_MIN max_n_steps = ReforceXY._PPO_N_STEPS_MAX min_resource = max( 2 * reduction_factor, - round(min_n_steps / ReforceXY._HYPEROPT_EVAL_FREQ_REDUCTION_FACTOR) - * n_envs, + round(min_n_steps / ReforceXY._HYPEROPT_EVAL_FREQ_REDUCTION_FACTOR) * n_envs, ) rollout = max_n_steps * n_envs return ( @@ -1670,9 +1577,7 @@ class ReforceXY(BaseReinforcementLearningModel): max(min_resource, ReforceXY._ceil_to_multiple(total_timesteps, rollout)), ) - def optimize( - self, dk: FreqaiDataKitchen, total_timesteps: int - ) -> Optional[Dict[str, Any]]: + def optimize(self, dk: FreqaiDataKitchen, total_timesteps: int) -> dict[str, Any] | None: """ Runs hyperparameter optimization using Optuna and returns the best hyperparameters found merged with the user defined parameters """ @@ -1683,8 +1588,7 @@ class ReforceXY(BaseReinforcementLearningModel): pair_purge_count = self._increment_optuna_retrain_counter(dk.pair) pair_purge_triggered = ( - self.optuna_purge_period > 0 - and pair_purge_count % self.optuna_purge_period == 0 + self.optuna_purge_period > 0 and pair_purge_count % self.optuna_purge_period == 0 ) if continuous or pair_purge_triggered: @@ -1721,9 +1625,7 @@ class ReforceXY(BaseReinforcementLearningModel): study: Study = create_study( study_name=study_name, sampler=self.create_sampler(), - pruner=ReforceXY.create_pruner( - min_resource, max_resource, reduction_factor - ), + pruner=ReforceXY.create_pruner(min_resource, max_resource, reduction_factor), direction=direction, storage=storage, load_if_exists=load_if_exists, @@ -1838,14 +1740,11 @@ class ReforceXY(BaseReinforcementLearningModel): ) def _best_trial_params_path(self, pair: str) -> Path: - return ( - self.full_path - / f"hyperopt-best-params-{ReforceXY._sanitize_pair(pair)}.json" - ) + return self.full_path / f"hyperopt-best-params-{ReforceXY._sanitize_pair(pair)}.json" def _resolve_legacy_best_trial_params( self, pair: str, best_trial_params_path: Path - ) -> Optional[Path]: + ) -> Path | None: base = pair.split("/")[0] legacy_path = self.full_path / f"hyperopt-best-params-{base}.json" if ( @@ -1854,9 +1753,7 @@ class ReforceXY(BaseReinforcementLearningModel): or not legacy_path.is_file() ): return None - base_pair_count = sum( - 1 for configured in self.pairs if configured.split("/")[0] == base - ) + base_pair_count = sum(1 for configured in self.pairs if configured.split("/")[0] == base) if base_pair_count == 1: return legacy_path logger.warning( @@ -1892,9 +1789,7 @@ class ReforceXY(BaseReinforcementLearningModel): return try: if not stat.S_ISREG(os.fstat(lock_fd).st_mode): - raise OSError( - f"Hyperopt best params lock {lock_path} must be a regular file" - ) + raise OSError(f"Hyperopt best params lock {lock_path} must be a regular file") fcntl.flock(lock_fd, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) yield finally: @@ -1904,14 +1799,13 @@ class ReforceXY(BaseReinforcementLearningModel): def _reject_best_trial_params_symlink(best_trial_params_path: Path) -> None: if best_trial_params_path.is_symlink(): raise OSError( - f"Hyperopt best params path {best_trial_params_path} " - "must not be a symlink" + f"Hyperopt best params path {best_trial_params_path} must not be a symlink" ) @staticmethod def _quarantine_corrupt_best_trial_params( best_trial_params_path: Path, pair: str, cause: Exception - ) -> Optional[Path]: + ) -> Path | None: if not best_trial_params_path.exists(): return None quarantine_path = ReforceXY._quarantine_path( @@ -1938,17 +1832,13 @@ class ReforceXY(BaseReinforcementLearningModel): ) return quarantine_path - def save_best_trial_params( - self, best_trial_params: Dict[str, Any], pair: str - ) -> None: + def save_best_trial_params(self, best_trial_params: dict[str, Any], pair: str) -> None: """ Save the best trial hyperparameters found during hyperparameter optimization """ best_trial_params_path = self._best_trial_params_path(pair) - logger.info( - "Hyperopt [%s]: saving best params to %s", pair, best_trial_params_path - ) - temporary_path: Optional[Path] = None + logger.info("Hyperopt [%s]: saving best params to %s", pair, best_trial_params_path) + temporary_path: Path | None = None try: with self._locked_best_trial_params(best_trial_params_path, exclusive=True): self._reject_best_trial_params_symlink(best_trial_params_path) @@ -1981,18 +1871,15 @@ class ReforceXY(BaseReinforcementLearningModel): os.fchown( write_file.fileno(), existing_metadata.st_uid - if temporary_metadata.st_uid - != existing_metadata.st_uid + if temporary_metadata.st_uid != existing_metadata.st_uid else -1, existing_metadata.st_gid - if temporary_metadata.st_gid - != existing_metadata.st_gid + if temporary_metadata.st_gid != existing_metadata.st_gid else -1, ) except PermissionError as chown_error: logger.debug( - "Hyperopt [%s]: best params ownership " - "preservation skipped: %r", + "Hyperopt [%s]: best params ownership preservation skipped: %r", pair, chown_error, ) @@ -2003,7 +1890,7 @@ class ReforceXY(BaseReinforcementLearningModel): json.dump(best_trial_params, write_file, indent=4) write_file.flush() os.fsync(write_file.fileno()) - os.replace(temporary_path, best_trial_params_path) + temporary_path.replace(best_trial_params_path) temporary_path = None except BaseException as error: if temporary_path is not None: @@ -2011,8 +1898,7 @@ class ReforceXY(BaseReinforcementLearningModel): temporary_path.unlink(missing_ok=True) except OSError as cleanup_error: logger.error( - "Hyperopt [%s]: best params temporary file %s cleanup " - "failed: %r", + "Hyperopt [%s]: best params temporary file %s cleanup failed: %r", pair, temporary_path.name, cleanup_error, @@ -2028,7 +1914,7 @@ class ReforceXY(BaseReinforcementLearningModel): ) raise - def load_best_trial_params(self, pair: str) -> Optional[Dict[str, Any]]: + def load_best_trial_params(self, pair: str) -> dict[str, Any] | None: """ Load the best trial hyperparameters found and saved during hyperparameter optimization """ @@ -2039,9 +1925,7 @@ class ReforceXY(BaseReinforcementLearningModel): with self._locked_best_trial_params(best_trial_params_path, exclusive=False): self._reject_best_trial_params_symlink(best_trial_params_path) if not best_trial_params_path.is_file(): - legacy_path = self._resolve_legacy_best_trial_params( - pair, best_trial_params_path - ) + legacy_path = self._resolve_legacy_best_trial_params(pair, best_trial_params_path) if legacy_path is None: return None best_trial_params_path = legacy_path @@ -2061,9 +1945,7 @@ class ReforceXY(BaseReinforcementLearningModel): if not best_trial_params_path.is_file(): return None try: - with best_trial_params_path.open( - "r", encoding="utf-8" - ) as read_file: + with best_trial_params_path.open("r", encoding="utf-8") as read_file: best_trial_params = json.load(read_file) except (json.JSONDecodeError, UnicodeDecodeError) as decode_error: quarantined = self._quarantine_corrupt_best_trial_params( @@ -2077,21 +1959,16 @@ class ReforceXY(BaseReinforcementLearningModel): def _get_train_and_eval_environments( self, dk: FreqaiDataKitchen, - train_df: Optional[DataFrame] = None, - test_df: Optional[DataFrame] = None, - prices_train: Optional[DataFrame] = None, - prices_test: Optional[DataFrame] = None, - seed: Optional[int] = None, - env_info: Optional[Dict[str, Any]] = None, - trial: Optional[Trial] = None, - model_params: Optional[Dict[str, Any]] = None, - ) -> Tuple[VecEnv, VecEnv]: - if ( - train_df is None - or test_df is None - or prices_train is None - or prices_test is None - ): + train_df: DataFrame | None = None, + test_df: DataFrame | None = None, + prices_train: DataFrame | None = None, + prices_test: DataFrame | None = None, + seed: int | None = None, + env_info: dict[str, Any] | None = None, + trial: Trial | None = None, + model_params: dict[str, Any] | None = None, + ) -> tuple[VecEnv, VecEnv]: + if train_df is None or test_df is None or prices_train is None or prices_test is None: train_df = dk.data_dictionary["train_features"] test_df = dk.data_dictionary["test_features"] prices_train, prices_test = self.build_ohlc_price_dataframes( @@ -2101,7 +1978,7 @@ class ReforceXY(BaseReinforcementLearningModel): if trial is not None: seed += trial.number set_random_seed(seed) - env_info: Dict[str, Any] = ( + env_info: dict[str, Any] = ( self.pack_env_dict(dk.pair, model_params) if env_info is None else env_info ) env_prefix = f"trial_{trial.number}_" if trial is not None else "" @@ -2149,7 +2026,7 @@ class ReforceXY(BaseReinforcementLearningModel): return train_env, eval_env - def get_optuna_params(self, trial: Trial) -> Dict[str, Any]: + def get_optuna_params(self, trial: Trial) -> dict[str, Any]: # "RecurrentPPO" if ReforceXY._MODEL_TYPES[1] in self.model_type: return sample_params_recurrentppo(trial) @@ -2167,9 +2044,7 @@ class ReforceXY(BaseReinforcementLearningModel): f"Hyperopt [{trial.study.study_name}]: model type '{self.model_type}' not supported" ) - def objective( - self, trial: Trial, dk: FreqaiDataKitchen, total_timesteps: int - ) -> float: + def objective(self, trial: Trial, dk: FreqaiDataKitchen, total_timesteps: int) -> float: """ Objective function for Optuna trials hyperparameter optimization """ @@ -2213,18 +2088,14 @@ class ReforceXY(BaseReinforcementLearningModel): # Ensure that the sampled parameters take precedence params = deepmerge(self.get_model_params(), params) params["seed"] = params.get("seed", 42) + trial.number - logger.info( - "Hyperopt [%s]: trial #%d params: %s", study_name, trial.number, params - ) + logger.info("Hyperopt [%s]: trial #%d params: %s", study_name, trial.number, params) # "PPO" if ReforceXY._MODEL_TYPES[0] in self.model_type: n_steps = params.get("n_steps", 0) if n_steps > 0: rollout = n_steps * self.n_envs - aligned_total_timesteps = ReforceXY._ceil_to_multiple( - total_timesteps, rollout - ) + aligned_total_timesteps = ReforceXY._ceil_to_multiple(total_timesteps, rollout) if aligned_total_timesteps != total_timesteps: total_timesteps = aligned_total_timesteps @@ -2252,9 +2123,7 @@ class ReforceXY(BaseReinforcementLearningModel): **params, ) - eval_freq = self.get_eval_freq( - total_timesteps, hyperopt=True, model_params=params - ) + eval_freq = self.get_eval_freq(total_timesteps, hyperopt=True, model_params=params) callbacks = self.get_callbacks(eval_env, eval_freq, str(dk.data_path), trial) try: model.learn(total_timesteps=total_timesteps, callback=callbacks) @@ -2310,9 +2179,7 @@ class ReforceXY(BaseReinforcementLearningModel): del model, train_env, eval_env if nan_encountered: - raise TrialPruned( - f"Hyperopt [{study_name}]: NaN encountered during training" - ) + raise TrialPruned(f"Hyperopt [{study_name}]: NaN encountered during training") if self.optuna_eval_callback.is_pruned: raise TrialPruned(f"Hyperopt [{study_name}]: pruned by eval callback") @@ -2336,13 +2203,13 @@ class ReforceXY(BaseReinforcementLearningModel): def make_env( - MyRLEnv: Type[BaseEnvironment], + MyRLEnv: type[BaseEnvironment], env_id: str, rank: int, seed: int, df: DataFrame, price: DataFrame, - env_info: Dict[str, Any], + env_info: dict[str, Any], ) -> Callable[[], BaseEnvironment]: """ Utility function for multiprocessed env. @@ -2364,7 +2231,7 @@ def make_env( return _init -MyRLEnv: Type[BaseEnvironment] +MyRLEnv: type[BaseEnvironment] class MyRLEnv(Base5ActionRLEnv): @@ -2376,7 +2243,7 @@ class MyRLEnv(Base5ActionRLEnv): self.action_masking: bool = self.rl_config.get("action_masking", False) # === INTERNAL STATE === - self._last_closed_position: Optional[Positions] = None + self._last_closed_position: Positions | None = None self._last_closed_trade_tick: int = 0 self._max_unrealized_profit: float = -np.inf self._min_unrealized_profit: float = np.inf @@ -2406,14 +2273,11 @@ class MyRLEnv(Base5ActionRLEnv): self.max_idle_duration_candles: int = int( model_reward_parameters.get( "max_idle_duration_candles", - ReforceXY.DEFAULT_IDLE_DURATION_MULTIPLIER - * self.max_trade_duration_candles, + ReforceXY.DEFAULT_IDLE_DURATION_MULTIPLIER * self.max_trade_duration_candles, ) ) # === PBRS COMMON PARAMETERS === - self._potential_gamma = float( - model_reward_parameters.get("potential_gamma", 0.95) - ) + self._potential_gamma = float(model_reward_parameters.get("potential_gamma", 0.95)) if np.isclose(self._potential_gamma, 0.0): logger.warning( "PBRS [%s]: potential_gamma=0 detected; PBRS delta will be -Φ(s) " @@ -2441,9 +2305,7 @@ class MyRLEnv(Base5ActionRLEnv): ReforceXY._EXIT_POTENTIAL_MODES[0], ", ".join(ReforceXY._EXIT_POTENTIAL_MODES), ) - self._exit_potential_mode = ReforceXY._EXIT_POTENTIAL_MODES[ - 0 - ] # "canonical" + self._exit_potential_mode = ReforceXY._EXIT_POTENTIAL_MODES[0] # "canonical" self._exit_potential_decay: float = float( model_reward_parameters.get( "exit_potential_decay", ReforceXY.DEFAULT_EXIT_POTENTIAL_DECAY @@ -2466,13 +2328,13 @@ class MyRLEnv(Base5ActionRLEnv): ) ) self._entry_additive_transform_pnl: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "entry_additive_transform_pnl", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" ) self._entry_additive_transform_duration: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "entry_additive_transform_duration", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" @@ -2494,13 +2356,13 @@ class MyRLEnv(Base5ActionRLEnv): ) ) self._hold_potential_transform_pnl: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "hold_potential_transform_pnl", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" ) self._hold_potential_transform_duration: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "hold_potential_transform_duration", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" @@ -2517,18 +2379,16 @@ class MyRLEnv(Base5ActionRLEnv): ) ) self._exit_additive_gain: float = float( - model_reward_parameters.get( - "exit_additive_gain", ReforceXY.DEFAULT_EXIT_ADDITIVE_GAIN - ) + model_reward_parameters.get("exit_additive_gain", ReforceXY.DEFAULT_EXIT_ADDITIVE_GAIN) ) self._exit_additive_transform_pnl: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "exit_additive_transform_pnl", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" ) self._exit_additive_transform_duration: TransformFunction = cast( - TransformFunction, + "TransformFunction", model_reward_parameters.get( "exit_additive_transform_duration", ReforceXY._TRANSFORM_FUNCTIONS[0] ), # "tanh" @@ -2545,11 +2405,10 @@ class MyRLEnv(Base5ActionRLEnv): self._entry_additive_enabled = False self._exit_additive_enabled = False # "non_canonical" - elif self._exit_potential_mode == ReforceXY._EXIT_POTENTIAL_MODES[1]: - if self._entry_additive_enabled or self._exit_additive_enabled: - logger.warning( - "PBRS [%s]: non-canonical mode, additive enabled", self.id - ) + elif self._exit_potential_mode == ReforceXY._EXIT_POTENTIAL_MODES[1] and ( + self._entry_additive_enabled or self._exit_additive_enabled + ): + logger.warning("PBRS [%s]: non-canonical mode, additive enabled", self.id) if MyRLEnv.is_unsupported_pbrs_config( self._hold_potential_enabled, getattr(self, "add_state_info", False) @@ -2591,9 +2450,7 @@ class MyRLEnv(Base5ActionRLEnv): def _get_entry_unrealized_profit(self, next_position: Positions) -> float: current_open = self.prices.iloc[self._current_tick].open - if not isinstance(current_open, (int, float, np.floating)) or not np.isfinite( - current_open - ): + if not isinstance(current_open, (int, float, np.floating)) or not np.isfinite(current_open): return 0.0 next_pnl = 0.0 @@ -2617,7 +2474,7 @@ class MyRLEnv(Base5ActionRLEnv): action: int, trade_duration: float, current_pnl: float, - ) -> Tuple[Positions, int, float]: + ) -> tuple[Positions, int, float]: """Compute next transition state tuple (next_position, next_duration, next_pnl). Parameters @@ -2681,7 +2538,7 @@ class MyRLEnv(Base5ActionRLEnv): gain: float, transform_pnl: TransformFunction, transform_duration: TransformFunction, - risk_reward_ratio: Optional[float] = None, + risk_reward_ratio: float | None = None, ) -> float: """Generic bounded bi-component signal combining PnL and duration.""" if not enabled: @@ -2848,10 +2705,7 @@ class MyRLEnv(Base5ActionRLEnv): """ mode = self._exit_potential_mode # "canonical" or "non_canonical" - if ( - mode == ReforceXY._EXIT_POTENTIAL_MODES[0] - or mode == ReforceXY._EXIT_POTENTIAL_MODES[1] - ): + if mode == ReforceXY._EXIT_POTENTIAL_MODES[0] or mode == ReforceXY._EXIT_POTENTIAL_MODES[1]: return 0.0 # "progressive_release" if mode == ReforceXY._EXIT_POTENTIAL_MODES[2]: @@ -2896,9 +2750,7 @@ class MyRLEnv(Base5ActionRLEnv): ) # "canonical" @staticmethod - def is_unsupported_pbrs_config( - hold_potential_enabled: bool, add_state_info: bool - ) -> bool: + def is_unsupported_pbrs_config(hold_potential_enabled: bool, add_state_info: bool) -> bool: """Return True if PBRS potential relies on hidden state. Case: hold_potential enabled while auxiliary state info (pnl, trade_duration) is excluded @@ -3102,9 +2954,7 @@ class MyRLEnv(Base5ActionRLEnv): if not self._hold_potential_enabled and not ( self._entry_additive_enabled or self._exit_additive_enabled ): - logger.debug( - "PBRS [%s]: all PBRS features disabled, returning zeros", self.id - ) + logger.debug("PBRS [%s]: all PBRS features disabled, returning zeros", self.id) self._last_prev_potential = float(prev_potential) self._last_next_potential = float(prev_potential) self._last_entry_additive = 0.0 @@ -3154,11 +3004,7 @@ class MyRLEnv(Base5ActionRLEnv): next_potential = 0.0 reward_shaping = 0.0 - if ( - is_entry - and self._entry_additive_enabled - and not self.is_pbrs_invariant_mode() - ): + if is_entry and self._entry_additive_enabled and not self.is_pbrs_invariant_mode(): entry_additive = self._compute_entry_additive( next_pnl, pnl_target, @@ -3169,11 +3015,9 @@ class MyRLEnv(Base5ActionRLEnv): elif is_exit: if ( - self._exit_potential_mode - == ReforceXY._EXIT_POTENTIAL_MODES[0] # "canonical" + self._exit_potential_mode == ReforceXY._EXIT_POTENTIAL_MODES[0] # "canonical" ) or ( - self._exit_potential_mode - == ReforceXY._EXIT_POTENTIAL_MODES[1] # "non_canonical" + self._exit_potential_mode == ReforceXY._EXIT_POTENTIAL_MODES[1] # "non_canonical" ): next_potential = 0.0 reward_shaping = -prev_potential @@ -3217,9 +3061,7 @@ class MyRLEnv(Base5ActionRLEnv): self.total_features = signal_features self.shape = (self.window_size, self.total_features) - self.observation_space = Box( - low=-np.inf, high=np.inf, shape=self.shape, dtype=np.float32 - ) + self.observation_space = Box(low=-np.inf, high=np.inf, shape=self.shape, dtype=np.float32) def _is_valid(self, action: int) -> bool: return ReforceXY.get_action_masks(self.can_short, self._position)[action] @@ -3229,7 +3071,7 @@ class MyRLEnv(Base5ActionRLEnv): df: DataFrame, prices: DataFrame, window_size: int, - reward_kwargs: Dict[str, Any], + reward_kwargs: dict[str, Any], starting_point=True, ) -> None: """ @@ -3238,12 +3080,12 @@ class MyRLEnv(Base5ActionRLEnv): super().reset_env(df, prices, window_size, reward_kwargs, starting_point) self._set_observation_space() - def reset(self, seed=None, **kwargs) -> Tuple[NDArray[np.float32], Dict[str, Any]]: + def reset(self, seed=None, **kwargs) -> tuple[NDArray[np.float32], dict[str, Any]]: """ Reset is called at the beginning of every episode """ observation, history = super().reset(seed, **kwargs) - self._last_closed_position: Optional[Positions] = None + self._last_closed_position: Positions | None = None self._last_closed_trade_tick: int = 0 self._max_unrealized_profit = -np.inf self._min_unrealized_profit = np.inf @@ -3283,9 +3125,7 @@ class MyRLEnv(Base5ActionRLEnv): model_reward_parameters.get("exit_plateau", ReforceXY.DEFAULT_EXIT_PLATEAU) ) exit_plateau_grace = float( - model_reward_parameters.get( - "exit_plateau_grace", ReforceXY.DEFAULT_EXIT_PLATEAU_GRACE - ) + model_reward_parameters.get("exit_plateau_grace", ReforceXY.DEFAULT_EXIT_PLATEAU_GRACE) ) if exit_plateau_grace < 0.0: logger.warning( @@ -3302,9 +3142,7 @@ class MyRLEnv(Base5ActionRLEnv): return 1.0 / math.sqrt(1.0 + dr) def _linear(dr: float, p: Mapping[str, Any]) -> float: - slope = float( - p.get("exit_linear_slope", ReforceXY.DEFAULT_EXIT_LINEAR_SLOPE) - ) + slope = float(p.get("exit_linear_slope", ReforceXY.DEFAULT_EXIT_LINEAR_SLOPE)) if slope < 0.0: logger.warning( "PBRS [%s]: exit_linear_slope=%.2f invalid; defaulting to 1.0", @@ -3318,10 +3156,7 @@ class MyRLEnv(Base5ActionRLEnv): tau = p.get("exit_power_tau") if isinstance(tau, (int, float)): tau = float(tau) - if 0.0 < tau <= 1.0: - alpha = -math.log(tau) / ReforceXY._LOG_2 - else: - alpha = 1.0 + alpha = -math.log(tau) / ReforceXY._LOG_2 if 0.0 < tau <= 1.0 else 1.0 else: alpha = 1.0 return 1.0 / math.pow(1.0 + dr, alpha) @@ -3332,7 +3167,7 @@ class MyRLEnv(Base5ActionRLEnv): return 1.0 return math.pow(2.0, -dr / hl) - strategies: Dict[str, Callable[[float, Mapping[str, Any]], float]] = { + strategies: dict[str, Callable[[float, Mapping[str, Any]], float]] = { ReforceXY._EXIT_ATTENUATION_MODES[0]: _legacy, ReforceXY._EXIT_ATTENUATION_MODES[1]: _sqrt, ReforceXY._EXIT_ATTENUATION_MODES[2]: _linear, @@ -3348,7 +3183,7 @@ class MyRLEnv(Base5ActionRLEnv): else: effective_dr = duration_ratio - strategy_fn = strategies.get(exit_attenuation_mode, None) + strategy_fn = strategies.get(exit_attenuation_mode) if strategy_fn is None: logger.warning( "PBRS [%s]: exit_attenuation_mode=%r invalid; defaulting to %r. Valid: %s", @@ -3360,9 +3195,7 @@ class MyRLEnv(Base5ActionRLEnv): strategy_fn = _linear try: - time_attenuation_coefficient = strategy_fn( - effective_dr, model_reward_parameters - ) + time_attenuation_coefficient = strategy_fn(effective_dr, model_reward_parameters) except Exception as e: logger.warning( "PBRS [%s]: exit_attenuation_mode=%r failed (%r); defaulting to %r (effective_dr=%.5f)", @@ -3373,9 +3206,7 @@ class MyRLEnv(Base5ActionRLEnv): effective_dr, exc_info=True, ) - time_attenuation_coefficient = _linear( - effective_dr, model_reward_parameters - ) + time_attenuation_coefficient = _linear(effective_dr, model_reward_parameters) return time_attenuation_coefficient @@ -3389,11 +3220,7 @@ class MyRLEnv(Base5ActionRLEnv): """ Compute exit factor: base_factor · time_attenuation_coefficient · pnl_target_coefficient · efficiency_coefficient. """ - if not ( - np.isfinite(base_factor) - and np.isfinite(pnl) - and np.isfinite(duration_ratio) - ): + if not (np.isfinite(base_factor) and np.isfinite(pnl) and np.isfinite(duration_ratio)): return 0.0 time_attenuation_coefficient = self._compute_time_attenuation_coefficient( @@ -3403,9 +3230,7 @@ class MyRLEnv(Base5ActionRLEnv): pnl_target_coefficient = self._compute_pnl_target_coefficient( pnl, self._pnl_target, model_reward_parameters ) - efficiency_coefficient = self._compute_efficiency_coefficient( - pnl, model_reward_parameters - ) + efficiency_coefficient = self._compute_efficiency_coefficient(pnl, model_reward_parameters) exit_factor = ( base_factor @@ -3417,9 +3242,7 @@ class MyRLEnv(Base5ActionRLEnv): check_invariants = model_reward_parameters.get( "check_invariants", ReforceXY.DEFAULT_CHECK_INVARIANTS ) - check_invariants = ( - check_invariants if isinstance(check_invariants, bool) else True - ) + check_invariants = check_invariants if isinstance(check_invariants, bool) else True if check_invariants: if not np.isfinite(exit_factor): logger.warning( @@ -3485,14 +3308,10 @@ class MyRLEnv(Base5ActionRLEnv): ) if pnl_ratio > 1.0: - pnl_target_coefficient = ( - 1.0 + win_reward_factor * base_pnl_target_coefficient - ) + pnl_target_coefficient = 1.0 + win_reward_factor * base_pnl_target_coefficient elif pnl_ratio < -(1.0 / self.rr): loss_penalty_factor = win_reward_factor * self.rr - pnl_target_coefficient = ( - 1.0 + loss_penalty_factor * base_pnl_target_coefficient - ) + pnl_target_coefficient = 1.0 + loss_penalty_factor * base_pnl_target_coefficient return pnl_target_coefficient @@ -3503,14 +3322,10 @@ class MyRLEnv(Base5ActionRLEnv): Compute exit efficiency coefficient (typically 0.5-1.5) based on exit timing quality. """ efficiency_weight = float( - model_reward_parameters.get( - "efficiency_weight", ReforceXY.DEFAULT_EFFICIENCY_WEIGHT - ) + model_reward_parameters.get("efficiency_weight", ReforceXY.DEFAULT_EFFICIENCY_WEIGHT) ) efficiency_center = float( - model_reward_parameters.get( - "efficiency_center", ReforceXY.DEFAULT_EFFICIENCY_CENTER - ) + model_reward_parameters.get("efficiency_center", ReforceXY.DEFAULT_EFFICIENCY_CENTER) ) efficiency_coefficient = 1.0 @@ -3570,7 +3385,7 @@ class MyRLEnv(Base5ActionRLEnv): - exit_additive: Optional exit bonus (breaks PBRS invariance) """ model_reward_parameters = self.rl_config.get("model_reward_parameters", {}) - base_reward: Optional[float] = None + base_reward: float | None = None self._last_invalid_penalty = 0.0 self._last_idle_penalty = 0.0 @@ -3581,9 +3396,7 @@ class MyRLEnv(Base5ActionRLEnv): if not self.action_masking and not self._is_valid(action): self.tensorboard_log("invalid", category="actions") base_reward = float( - model_reward_parameters.get( - "invalid_action", ReforceXY.DEFAULT_INVALID_ACTION - ) + model_reward_parameters.get("invalid_action", ReforceXY.DEFAULT_INVALID_ACTION) ) self._last_invalid_penalty = float(base_reward) @@ -3616,9 +3429,7 @@ class MyRLEnv(Base5ActionRLEnv): idle_duration = self.get_idle_duration() idle_duration_ratio = idle_duration / max(1, max_idle_duration) base_reward = ( - -idle_factor - * idle_penalty_ratio - * idle_duration_ratio**idle_penalty_power + -idle_factor * idle_penalty_ratio * idle_duration_ratio**idle_penalty_power ) self._last_idle_penalty = float(base_reward) @@ -3642,9 +3453,7 @@ class MyRLEnv(Base5ActionRLEnv): base_reward = 0.0 else: base_reward = ( - -hold_factor - * hold_penalty_ratio - * (duration_ratio - 1.0) ** hold_penalty_power + -hold_factor * hold_penalty_ratio * (duration_ratio - 1.0) ** hold_penalty_power ) self._last_hold_penalty = float(base_reward) @@ -3698,12 +3507,8 @@ class MyRLEnv(Base5ActionRLEnv): features_window_array = features_window.to_numpy(dtype=np.float32, copy=False) if features_window_array.shape[0] < self.window_size: pad_size = self.window_size - features_window_array.shape[0] - pad_array = np.zeros( - (pad_size, features_window_array.shape[1]), dtype=np.float32 - ) - features_window_array = np.concatenate( - [pad_array, features_window_array], axis=0 - ) + pad_array = np.zeros((pad_size, features_window_array.shape[1]), dtype=np.float32) + features_window_array = np.concatenate([pad_array, features_window_array], axis=0) if self.add_state_info: observations = np.concatenate( [ @@ -3748,7 +3553,7 @@ class MyRLEnv(Base5ActionRLEnv): self._max_unrealized_profit = -np.inf self._min_unrealized_profit = np.inf - def execute_trade(self, action: int) -> Optional[str]: + def execute_trade(self, action: int) -> str | None: """ Execute trade based on the given action """ @@ -3797,9 +3602,7 @@ class MyRLEnv(Base5ActionRLEnv): self._total_reward_shaping += reward_shaping_delta return reward + reward_shaping_delta - def step( - self, action: int - ) -> Tuple[NDArray[np.float32], float, bool, bool, Dict[str, Any]]: + def step(self, action: int) -> tuple[NDArray[np.float32], float, bool, bool, dict[str, Any]]: """ Take a step in the environment based on the provided action """ @@ -3861,9 +3664,7 @@ class MyRLEnv(Base5ActionRLEnv): "idle_duration": idle_duration, "idle_ratio": (idle_duration / max_idle_duration), "trade_duration": trade_duration, - "duration_ratio": ( - trade_duration / max(1, self.max_trade_duration_candles) - ), + "duration_ratio": (trade_duration / max(1, self.max_trade_duration_candles)), "trade_count": len(self.trade_history) // 2, } self._update_history(info) @@ -3875,9 +3676,7 @@ class MyRLEnv(Base5ActionRLEnv): info, ) - def append_trade_history( - self, trade_type: str, price: float, profit: float - ) -> None: + def append_trade_history(self, trade_type: str, price: float, profit: float) -> None: self.trade_history.append( { "tick": self._current_tick, @@ -3966,9 +3765,11 @@ class MyRLEnv(Base5ActionRLEnv): return self._max_unrealized_profit def _update_max_unrealized_profit(self, pnl: float) -> None: - if self._position in (Positions.Long, Positions.Short): - if pnl > self._max_unrealized_profit: - self._max_unrealized_profit = pnl + if ( + self._position in (Positions.Long, Positions.Short) + and pnl > self._max_unrealized_profit + ): + self._max_unrealized_profit = pnl def get_min_unrealized_profit(self) -> float: """ @@ -3983,9 +3784,11 @@ class MyRLEnv(Base5ActionRLEnv): return self._min_unrealized_profit def _update_min_unrealized_profit(self, pnl: float) -> None: - if self._position in (Positions.Long, Positions.Short): - if pnl < self._min_unrealized_profit: - self._min_unrealized_profit = pnl + if ( + self._position in (Positions.Long, Positions.Short) + and pnl < self._min_unrealized_profit + ): + self._min_unrealized_profit = pnl def get_most_recent_return(self) -> float: """ @@ -4124,9 +3927,7 @@ class MyRLEnv(Base5ActionRLEnv): if self.trade_history: _trade_history_df = DataFrame(self.trade_history) if "tick" in _trade_history_df.columns: - _rollout_history = merge( - _rollout_history, _trade_history_df, on="tick", how="left" - ) + _rollout_history = merge(_rollout_history, _trade_history_df, on="tick", how="left") try: history = merge( @@ -4185,12 +3986,7 @@ class MyRLEnv(Base5ActionRLEnv): ticks = history.get("tick") history_open = history.get("open") - if ( - ticks is None - or len(ticks) == 0 - or history_open is None - or len(history_open) == 0 - ): + if ticks is None or len(ticks) == 0 or history_open is None or len(history_open) == 0: return fig axs[0].plot(ticks, history_open, linewidth=1, color="orchid", zorder=1) @@ -4312,7 +4108,7 @@ class InfoMetricsCallback(TensorboardCallback): self.throttle = 1 if throttle < 1 else throttle def _safe_logger_record( - self, key: str, value: Any, exclude: Optional[Tuple[str, ...]] = None + self, key: str, value: Any, exclude: tuple[str, ...] | None = None ) -> None: try: self.logger.record(key, value, exclude=exclude) @@ -4343,9 +4139,9 @@ class InfoMetricsCallback(TensorboardCallback): @staticmethod def _build_train_freq( - train_freq: Optional[Union[TrainFreq, int, Tuple[int, ...], List[int]]], - ) -> Optional[int]: - train_freq_val: Optional[int] = None + train_freq: TrainFreq | int | tuple[int, ...] | list[int] | None, + ) -> int | None: + train_freq_val: int | None = None if isinstance(train_freq, TrainFreq) and hasattr(train_freq, "frequency"): if isinstance(train_freq.frequency, int): train_freq_val = train_freq.frequency @@ -4364,13 +4160,11 @@ class InfoMetricsCallback(TensorboardCallback): env = getattr(self, "training_env", None) while env is not None: if hasattr(env, "n_stack"): - try: - n_stack = int(getattr(env, "n_stack")) - except Exception: - pass + with suppress(Exception): + n_stack = int(env.n_stack) break env = getattr(env, "venv", None) - hparam_dict: Dict[str, Any] = { + hparam_dict: dict[str, Any] = { "algorithm": self.model.__class__.__name__, "n_envs": int(self.model.n_envs), "n_stack": n_stack, @@ -4406,9 +4200,7 @@ class InfoMetricsCallback(TensorboardCallback): ) if getattr(self.model, "target_kl", None) is not None: hparam_dict["target_kl"] = float(self.model.target_kl) - if ( - ReforceXY._MODEL_TYPES[1] in self.model.__class__.__name__ - ): # "RecurrentPPO" + if ReforceXY._MODEL_TYPES[1] in self.model.__class__.__name__: # "RecurrentPPO" policy = getattr(self.model, "policy", None) if policy is not None: lstm_actor = getattr(policy, "lstm_actor", None) @@ -4426,9 +4218,7 @@ class InfoMetricsCallback(TensorboardCallback): "gradient_steps": int(self.model.gradient_steps), "learning_starts": int(self.model.learning_starts), "target_update_interval": int(self.model.target_update_interval), - "exploration_initial_eps": float( - self.model.exploration_initial_eps - ), + "exploration_initial_eps": float(self.model.exploration_initial_eps), "exploration_final_eps": float(self.model.exploration_final_eps), "exploration_fraction": float(self.model.exploration_fraction), "exploration_rate": float(self.model.exploration_rate), @@ -4441,7 +4231,7 @@ class InfoMetricsCallback(TensorboardCallback): hparam_dict.update({"train_freq": train_freq}) if ReforceXY._MODEL_TYPES[4] in self.model.__class__.__name__: # "QRDQN" hparam_dict.update({"n_quantiles": int(self.model.n_quantiles)}) - metric_dict: Dict[str, float | int] = { + metric_dict: dict[str, float | int] = { "eval/mean_reward": 0.0, "eval/mean_reward_std": 0.0, "rollout/ep_rew_mean": 0.0, @@ -4487,9 +4277,7 @@ class InfoMetricsCallback(TensorboardCallback): logger_exclude = ("stdout", "log", "json", "csv") def _is_number(x: Any) -> bool: - return isinstance( - x, (int, float, np.integer, np.floating) - ) and not isinstance(x, bool) + return isinstance(x, (int, float, np.integer, np.floating)) and not isinstance(x, bool) def _is_finite_number(x: Any) -> bool: if not _is_number(x): @@ -4499,14 +4287,12 @@ class InfoMetricsCallback(TensorboardCallback): except Exception: return False - infos_list: List[Dict[str, Any]] | None = self.locals.get("infos") - aggregated_info: Dict[str, Any] = {} + infos_list: list[dict[str, Any]] | None = self.locals.get("infos") + aggregated_info: dict[str, Any] = {} if isinstance(infos_list, list) and infos_list: - numeric_acc: Dict[str, List[float]] = defaultdict(list) - non_numeric_counts: Dict[str, Dict[Any, int]] = defaultdict( - lambda: defaultdict(int) - ) + numeric_acc: dict[str, list[float]] = defaultdict(list) + non_numeric_counts: dict[str, dict[Any, int]] = defaultdict(lambda: defaultdict(int)) filtered_values: int = 0 for info_dict in infos_list: @@ -4527,10 +4313,8 @@ class InfoMetricsCallback(TensorboardCallback): continue aggregated_info[k] = np.mean(values) if len(values) > 1: - try: + with suppress(Exception): aggregated_info[f"{k}_std"] = np.std(values, ddof=1) - except Exception: - pass for key in ("reward", "pnl"): values = numeric_acc.get(key) @@ -4554,16 +4338,12 @@ class InfoMetricsCallback(TensorboardCallback): if not counts: continue if len(counts) == 1: - try: + with suppress(Exception): aggregated_info[f"{k}_mode"] = next(iter(counts.keys())) - except Exception: - pass else: aggregated_info[f"{k}_mode"] = "mixed" - self._safe_logger_record( - "info/n_envs", len(infos_list), exclude=logger_exclude - ) + self._safe_logger_record("info/n_envs", len(infos_list), exclude=logger_exclude) if filtered_values > 0: self._safe_logger_record( @@ -4578,10 +4358,8 @@ class InfoMetricsCallback(TensorboardCallback): except Exception: tensorboard_metrics_list = [] - aggregated_tensorboard_metrics: Dict[str, Dict[str, Any]] = defaultdict(dict) - aggregated_tensorboard_metric_counts: Dict[str, Dict[str, int]] = defaultdict( - dict - ) + aggregated_tensorboard_metrics: dict[str, dict[str, Any]] = defaultdict(dict) + aggregated_tensorboard_metric_counts: dict[str, dict[str, int]] = defaultdict(dict) for env_metrics in tensorboard_metrics_list or []: if not isinstance(env_metrics, dict): continue @@ -4608,10 +4386,8 @@ class InfoMetricsCallback(TensorboardCallback): if isinstance(infos_list, list) and infos_list: cat_keys = ("action", "position") - cat_counts: Dict[str, Dict[Any, int]] = { - k: defaultdict(int) for k in cat_keys - } - cat_totals: Dict[str, int] = {k: 0 for k in cat_keys} + cat_counts: dict[str, dict[Any, int]] = {k: defaultdict(int) for k in cat_keys} + cat_totals: dict[str, int] = dict.fromkeys(cat_keys, 0) for info_dict in infos_list: if not isinstance(info_dict, dict): continue @@ -4640,14 +4416,8 @@ class InfoMetricsCallback(TensorboardCallback): self._safe_logger_record( f"{category}/{metric}_sum", value, exclude=logger_exclude ) - count = aggregated_tensorboard_metric_counts.get(category, {}).get( - metric - ) - if ( - _is_finite_number(value) - and isinstance(count, int) - and count > 0 - ): + count = aggregated_tensorboard_metric_counts.get(category, {}).get(metric) + if _is_finite_number(value) and isinstance(count, int) and count > 0: self._safe_logger_record( f"{category}/{metric}_mean", float(value) / float(count), @@ -4662,9 +4432,7 @@ class InfoMetricsCallback(TensorboardCallback): else: progress_done = 0.0 progress_remaining = 1.0 - progress_done - self._safe_logger_record( - "train/progress_done", progress_done, exclude=logger_exclude - ) + self._safe_logger_record("train/progress_done", progress_done, exclude=logger_exclude) self._safe_logger_record( "train/progress_remaining", progress_remaining, exclude=logger_exclude ) @@ -4700,9 +4468,7 @@ class InfoMetricsCallback(TensorboardCallback): lr = getattr(self.model, "learning_rate", None) lr = _eval_schedule(lr) if _is_finite_number(lr): - self._safe_logger_record( - "train/learning_rate", float(lr), exclude=logger_exclude - ) + self._safe_logger_record("train/learning_rate", float(lr), exclude=logger_exclude) except Exception: pass @@ -4711,9 +4477,7 @@ class InfoMetricsCallback(TensorboardCallback): cr = getattr(self.model, "clip_range", None) cr = _eval_schedule(cr) if _is_finite_number(cr): - self._safe_logger_record( - "train/clip_range", float(cr), exclude=logger_exclude - ) + self._safe_logger_record("train/clip_range", float(cr), exclude=logger_exclude) except Exception: pass @@ -4773,9 +4537,9 @@ class MaskableTrialEvalCallback(MaskableEvalCallback): deterministic: bool = True, render: bool = False, use_masking: bool = True, - best_model_save_path: Optional[str] = None, - callback_on_new_best: Optional[BaseCallback] = None, - callback_after_eval: Optional[BaseCallback] = None, + best_model_save_path: str | None = None, + callback_on_new_best: BaseCallback | None = None, + callback_after_eval: BaseCallback | None = None, verbose: int = 0, **kwargs, ): @@ -4865,9 +4629,7 @@ class MaskableTrialEvalCallback(MaskableEvalCallback): try: logger_exclude = ("stdout", "log", "json", "csv") self.logger.record("eval/idx", self.eval_idx, exclude=logger_exclude) - self.logger.record( - "eval/num_timesteps", self.num_timesteps, exclude=logger_exclude - ) + self.logger.record("eval/num_timesteps", self.num_timesteps, exclude=logger_exclude) self.logger.record( "eval/last_mean_reward", last_mean_reward, exclude=logger_exclude ) @@ -4932,7 +4694,7 @@ class SimpleLinearSchedule: :param initial_value: (float or str) The initial value for the schedule """ - def __init__(self, initial_value: Union[float, str]) -> None: + def __init__(self, initial_value: float | str) -> None: # Force conversion to float self.initial_value = float(initial_value) @@ -4943,15 +4705,11 @@ class SimpleLinearSchedule: return f"SimpleLinearSchedule(initial_value={self.initial_value})" -def deepmerge(dst: Dict[str, Any], src: Dict[str, Any]) -> Dict[str, Any]: +def deepmerge(dst: dict[str, Any], src: dict[str, Any]) -> dict[str, Any]: """Recursively merge two dicts without mutating inputs""" dst_copy = copy.deepcopy(dst) for k, v in src.items(): - if ( - k in dst_copy - and isinstance(dst_copy[k], Mapping) - and isinstance(v, Mapping) - ): + if k in dst_copy and isinstance(dst_copy[k], Mapping) and isinstance(v, Mapping): dst_copy[k] = deepmerge(dst_copy[k], v) else: dst_copy[k] = v @@ -4965,7 +4723,7 @@ def _compute_gradient_steps(tf: int, ss: int) -> int: def compute_gradient_steps(train_freq: Any, subsample_steps: Any) -> int: - tf: Optional[int] = None + tf: int | None = None if isinstance(train_freq, TrainFreq): tf = train_freq.frequency if isinstance(train_freq.frequency, int) else None if isinstance(train_freq, (tuple, list)) and train_freq: @@ -4973,7 +4731,7 @@ def compute_gradient_steps(train_freq: Any, subsample_steps: Any) -> int: elif isinstance(train_freq, int): tf = train_freq - ss: Optional[int] = subsample_steps if isinstance(subsample_steps, int) else None + ss: int | None = subsample_steps if isinstance(subsample_steps, int) else None if isinstance(tf, int) and isinstance(ss, int): return _compute_gradient_steps(tf, ss) @@ -4999,7 +4757,7 @@ def steps_to_days(steps: int, timeframe: str) -> float: def get_schedule_type( schedule: Any, -) -> Tuple[ScheduleType, float, float]: +) -> tuple[ScheduleType, float, float]: if isinstance(schedule, (int, float)): try: schedule = float(schedule) @@ -5040,9 +4798,7 @@ def get_schedule( return ConstantSchedule(initial_value) -def get_net_arch( - model_type: str, net_arch_type: NetArchSize -) -> Union[List[int], Dict[str, List[int]]]: +def get_net_arch(model_type: str, net_arch_type: NetArchSize) -> list[int] | dict[str, list[int]]: """ Get network architecture """ @@ -5075,7 +4831,7 @@ def get_net_arch( def get_activation_fn( activation_fn_name: ActivationFunction, -) -> Type[th.nn.Module]: +) -> type[th.nn.Module]: """ Get activation function """ @@ -5089,7 +4845,7 @@ def get_activation_fn( def get_optimizer_class( optimizer_class_name: OptimizerClass, -) -> Type[th.optim.Optimizer]: +) -> type[th.optim.Optimizer]: """ Get optimizer class """ @@ -5101,10 +4857,10 @@ def get_optimizer_class( def convert_optuna_params_to_model_params( - model_type: str, optuna_params: Dict[str, Any] -) -> Dict[str, Any]: - model_params: Dict[str, Any] = {} - policy_kwargs: Dict[str, Any] = {} + model_type: str, optuna_params: dict[str, Any] +) -> dict[str, Any]: + model_params: dict[str, Any] = {} + policy_kwargs: dict[str, Any] = {} lr = optuna_params.get("learning_rate") if lr is None: @@ -5127,9 +4883,7 @@ def convert_optuna_params_to_model_params( ] for param in required_ppo_params: if optuna_params.get(param) is None: - raise ValueError( - f"Hyperopt [{model_type}]: missing '{param}' in params" - ) + raise ValueError(f"Hyperopt [{model_type}]: missing '{param}' in params") cr = optuna_params.get("clip_range") cr = get_schedule( optuna_params.get("cr_schedule", ReforceXY._SCHEDULE_TYPES[1]), @@ -5153,14 +4907,10 @@ def convert_optuna_params_to_model_params( if optuna_params.get("target_kl") is not None: model_params["target_kl"] = float(optuna_params.get("target_kl")) if ReforceXY._MODEL_TYPES[1] in model_type: # "RecurrentPPO" - policy_kwargs["lstm_hidden_size"] = int( - optuna_params.get("lstm_hidden_size") - ) + policy_kwargs["lstm_hidden_size"] = int(optuna_params.get("lstm_hidden_size")) policy_kwargs["n_lstm_layers"] = int(optuna_params.get("n_lstm_layers")) if optuna_params.get("enable_critic_lstm") is not None: - policy_kwargs["enable_critic_lstm"] = bool( - optuna_params.get("enable_critic_lstm") - ) + policy_kwargs["enable_critic_lstm"] = bool(optuna_params.get("enable_critic_lstm")) elif ReforceXY._MODEL_TYPES[3] in model_type: # "DQN" required_dqn_params = [ "gamma", @@ -5176,9 +4926,7 @@ def convert_optuna_params_to_model_params( ] for param in required_dqn_params: if optuna_params.get(param) is None: - raise ValueError( - f"Hyperopt [{model_type}]: missing '{param}' in params" - ) + raise ValueError(f"Hyperopt [{model_type}]: missing '{param}' in params") train_freq = optuna_params.get("train_freq") subsample_steps = optuna_params.get("subsample_steps") gradient_steps = compute_gradient_steps(train_freq, subsample_steps) @@ -5191,24 +4939,15 @@ def convert_optuna_params_to_model_params( "buffer_size": int(optuna_params.get("buffer_size")), "train_freq": train_freq, "gradient_steps": gradient_steps, - "exploration_fraction": float( - optuna_params.get("exploration_fraction") - ), - "exploration_initial_eps": float( - optuna_params.get("exploration_initial_eps") - ), - "exploration_final_eps": float( - optuna_params.get("exploration_final_eps") - ), - "target_update_interval": int( - optuna_params.get("target_update_interval") - ), + "exploration_fraction": float(optuna_params.get("exploration_fraction")), + "exploration_initial_eps": float(optuna_params.get("exploration_initial_eps")), + "exploration_final_eps": float(optuna_params.get("exploration_final_eps")), + "target_update_interval": int(optuna_params.get("target_update_interval")), "learning_starts": int(optuna_params.get("learning_starts")), } ) if ( - ReforceXY._MODEL_TYPES[4] in model_type - and optuna_params.get("n_quantiles") is not None + ReforceXY._MODEL_TYPES[4] in model_type and optuna_params.get("n_quantiles") is not None ): # "QRDQN" policy_kwargs["n_quantiles"] = int(optuna_params["n_quantiles"]) else: @@ -5219,19 +4958,19 @@ def convert_optuna_params_to_model_params( if net_arch_value in ReforceXY._NET_ARCH_SIZES: policy_kwargs["net_arch"] = get_net_arch( model_type, - cast(NetArchSize, net_arch_value), + cast("NetArchSize", net_arch_value), ) if optuna_params.get("activation_fn"): activation_fn_value = str(optuna_params["activation_fn"]) if activation_fn_value in ReforceXY._ACTIVATION_FUNCTIONS: policy_kwargs["activation_fn"] = get_activation_fn( - cast(ActivationFunction, activation_fn_value) + cast("ActivationFunction", activation_fn_value) ) if optuna_params.get("optimizer_class"): optimizer_value = str(optuna_params["optimizer_class"]) if optimizer_value in ReforceXY._OPTIMIZER_CLASSES: policy_kwargs["optimizer_class"] = get_optimizer_class( - cast(OptimizerClass, optimizer_value) + cast("OptimizerClass", optimizer_value) ) if optuna_params.get("ortho_init") is not None: policy_kwargs["ortho_init"] = bool(optuna_params["ortho_init"]) @@ -5240,12 +4979,10 @@ def convert_optuna_params_to_model_params( return model_params -def get_common_ppo_optuna_params(trial: Trial) -> Dict[str, Any]: +def get_common_ppo_optuna_params(trial: Trial) -> dict[str, Any]: return { "n_steps": trial.suggest_categorical("n_steps", list(ReforceXY._PPO_N_STEPS)), - "batch_size": trial.suggest_categorical( - "batch_size", [64, 128, 256, 512, 1024] - ), + "batch_size": trial.suggest_categorical("batch_size", [64, 128, 256, 512, 1024]), "gamma": trial.suggest_categorical( "gamma", [0.93, 0.95, 0.97, 0.98, 0.99, 0.995, 0.997, 0.999, 0.9999] ), @@ -5266,9 +5003,7 @@ def get_common_ppo_optuna_params(trial: Trial) -> Dict[str, Any]: "target_kl", [None, 0.003, 0.01, 0.015, 0.02, 0.03, 0.04, 0.1] ), "ortho_init": trial.suggest_categorical("ortho_init", [True, False]), - "net_arch": trial.suggest_categorical( - "net_arch", list(ReforceXY._NET_ARCH_SIZES) - ), + "net_arch": trial.suggest_categorical("net_arch", list(ReforceXY._NET_ARCH_SIZES)), "activation_fn": trial.suggest_categorical( "activation_fn", list(ReforceXY._ACTIVATION_FUNCTIONS) ), @@ -5278,7 +5013,7 @@ def get_common_ppo_optuna_params(trial: Trial) -> Dict[str, Any]: } -def sample_params_ppo(trial: Trial) -> Dict[str, Any]: +def sample_params_ppo(trial: Trial) -> dict[str, Any]: """ Sampler for PPO hyperparams """ @@ -5287,7 +5022,7 @@ def sample_params_ppo(trial: Trial) -> Dict[str, Any]: ) -def sample_params_recurrentppo(trial: Trial) -> Dict[str, Any]: +def sample_params_recurrentppo(trial: Trial) -> dict[str, Any]: """ Sampler for RecurrentPPO hyperparams """ @@ -5295,21 +5030,15 @@ def sample_params_recurrentppo(trial: Trial) -> Dict[str, Any]: ppo_optuna_params.update( { "n_lstm_layers": trial.suggest_int("n_lstm_layers", 1, 2), - "lstm_hidden_size": trial.suggest_categorical( - "lstm_hidden_size", [64, 128, 256, 512] - ), - "enable_critic_lstm": trial.suggest_categorical( - "enable_critic_lstm", [True, False] - ), + "lstm_hidden_size": trial.suggest_categorical("lstm_hidden_size", [64, 128, 256, 512]), + "enable_critic_lstm": trial.suggest_categorical("enable_critic_lstm", [True, False]), } ) return convert_optuna_params_to_model_params("RecurrentPPO", ppo_optuna_params) -def get_common_dqn_optuna_params(trial: Trial) -> Dict[str, Any]: - exploration_final_eps = trial.suggest_float( - "exploration_final_eps", 0.01, 0.2, step=0.01 - ) +def get_common_dqn_optuna_params(trial: Trial) -> dict[str, Any]: + exploration_final_eps = trial.suggest_float("exploration_final_eps", 0.01, 0.2, step=0.01) exploration_initial_eps = trial.suggest_float( "exploration_initial_eps", exploration_final_eps, 1.0 ) @@ -5327,9 +5056,7 @@ def get_common_dqn_optuna_params(trial: Trial) -> Dict[str, Any]: "gamma": trial.suggest_categorical( "gamma", [0.93, 0.95, 0.97, 0.98, 0.99, 0.995, 0.997, 0.999, 0.9999] ), - "batch_size": trial.suggest_categorical( - "batch_size", [64, 128, 256, 512, 1024] - ), + "batch_size": trial.suggest_categorical("batch_size", [64, 128, 256, 512, 1024]), "learning_rate": trial.suggest_float("learning_rate", 1e-5, 3e-3, log=True), "lr_schedule": trial.suggest_categorical( "lr_schedule", list(ReforceXY._SCHEDULE_TYPES_KNOWN) @@ -5348,9 +5075,7 @@ def get_common_dqn_optuna_params(trial: Trial) -> Dict[str, Any]: "learning_starts": trial.suggest_categorical( "learning_starts", [500, 1000, 2000, 5000, 10000, 25000, 50000] ), - "net_arch": trial.suggest_categorical( - "net_arch", list(ReforceXY._NET_ARCH_SIZES) - ), + "net_arch": trial.suggest_categorical("net_arch", list(ReforceXY._NET_ARCH_SIZES)), "activation_fn": trial.suggest_categorical( "activation_fn", list(ReforceXY._ACTIVATION_FUNCTIONS) ), @@ -5360,7 +5085,7 @@ def get_common_dqn_optuna_params(trial: Trial) -> Dict[str, Any]: } -def sample_params_dqn(trial: Trial) -> Dict[str, Any]: +def sample_params_dqn(trial: Trial) -> dict[str, Any]: """ Sampler for DQN hyperparams """ @@ -5369,12 +5094,10 @@ def sample_params_dqn(trial: Trial) -> Dict[str, Any]: ) -def sample_params_qrdqn(trial: Trial) -> Dict[str, Any]: +def sample_params_qrdqn(trial: Trial) -> dict[str, Any]: """ Sampler for QRDQN hyperparams """ dqn_optuna_params = get_common_dqn_optuna_params(trial) dqn_optuna_params.update({"n_quantiles": trial.suggest_int("n_quantiles", 10, 250)}) - return convert_optuna_params_to_model_params( - ReforceXY._MODEL_TYPES[4], dqn_optuna_params - ) + return convert_optuna_params_to_model_params(ReforceXY._MODEL_TYPES[4], dqn_optuna_params) diff --git a/ReforceXY/user_data/strategies/RLAgentStrategy.py b/ReforceXY/user_data/strategies/RLAgentStrategy.py index 9cbe106..c6c9834 100644 --- a/ReforceXY/user_data/strategies/RLAgentStrategy.py +++ b/ReforceXY/user_data/strategies/RLAgentStrategy.py @@ -1,7 +1,7 @@ import datetime import logging from functools import reduce -from typing import Any, Final, Literal, Optional +from typing import Any, Final, Literal import numpy as np import pandas as pd @@ -112,16 +112,12 @@ class RLAgentStrategy(IStrategy): return dataframe - def populate_indicators( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: dataframe = self.freqai.start(dataframe, metadata, self) return dataframe - def populate_entry_trend( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: enter_long_conditions = [ dataframe.get("do_predict") == 1, dataframe.get(ACTION_COLUMN) == RLAgentStrategy._ACTION_ENTER_LONG, # 1, @@ -142,9 +138,7 @@ class RLAgentStrategy(IStrategy): return dataframe - def populate_exit_trend( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: exit_long_conditions = [ dataframe.get("do_predict") == 1, dataframe.get(ACTION_COLUMN) == RLAgentStrategy._ACTION_EXIT_LONG, # 2, @@ -155,9 +149,7 @@ class RLAgentStrategy(IStrategy): dataframe.get("do_predict") == 1, dataframe.get(ACTION_COLUMN) == RLAgentStrategy._ACTION_EXIT_SHORT, # 4, ] - dataframe.loc[ - reduce(lambda x, y: x & y, exit_short_conditions), "exit_short" - ] = 1 + dataframe.loc[reduce(lambda x, y: x & y, exit_short_conditions), "exit_short"] = 1 last_candle = dataframe.iloc[-1] if last_candle.get("do_predict") == 2: @@ -178,7 +170,7 @@ class RLAgentStrategy(IStrategy): current_rate: float, proposed_leverage: float, max_leverage: float, - entry_tag: Optional[str], + entry_tag: str | None, side: str, **kwargs: Any, ) -> float: diff --git a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py index a8d6bc7..f51987e 100644 --- a/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py +++ b/quickadapter/user_data/freqaimodels/QuickAdapterRegressorV3.py @@ -4,20 +4,18 @@ import logging import random import time import warnings +from collections.abc import Callable +from collections.abc import Set as AbstractSet from dataclasses import dataclass from datetime import datetime, timezone from functools import cached_property from pathlib import Path from typing import ( - AbstractSet, Any, - Callable, ClassVar, Final, Literal, NamedTuple, - Optional, - Union, assert_never, cast, ) @@ -36,20 +34,9 @@ from freqtrade.exceptions import DependencyException from freqtrade.freqai.base_models.BaseRegressionModel import BaseRegressionModel from freqtrade.freqai.data_drawer import FreqaiDataDrawer from freqtrade.freqai.data_kitchen import FreqaiDataKitchen -from numpy.typing import NDArray -from optuna.storages import JournalStorage -from optuna.storages.journal import JournalFileBackend -from optuna.study.study import ObjectiveFuncType -from sklearn.model_selection import TimeSeriesSplit, train_test_split -from sklearn.preprocessing import ( - MaxAbsScaler, - MinMaxScaler, - RobustScaler, - StandardScaler, -) + # Disabled: scikit-learn-extra 0.3.0 fails on Python 3.14 (__gxx_personality_v0). # from sklearn_extra.cluster import KMedoids - from LabelTransformer import ( CUSTOM_THRESHOLD_METHODS, EXTREMA_SELECTION_METHODS, @@ -65,9 +52,20 @@ from LabelTransformer import ( ThresholdMethod, get_label_column_config, ) - +from numpy.typing import NDArray +from optuna.storages import JournalStorage +from optuna.storages.journal import JournalFileBackend +from optuna.study.study import ObjectiveFuncType +from sklearn.model_selection import TimeSeriesSplit, train_test_split +from sklearn.preprocessing import ( + MaxAbsScaler, + MinMaxScaler, + RobustScaler, + StandardScaler, +) from Utils import ( - enum_error_message, + _OPTUNA_LABEL_SELECTION_SCHEMA_VERSION, + _OPTUNA_NAMESPACES, DEFAULT_MAX_LABEL_NATR_MULTIPLIER, DEFAULT_MAX_LABEL_PERIOD_CANDLES, DEFAULT_MIN_LABEL_NATR_MULTIPLIER, @@ -75,18 +73,17 @@ from Utils import ( DEFAULT_REGRESSOR, DEFAULTS_LABEL_PREDICTION, LABEL_COLUMNS, - LabelWeightSupportError, REGRESSORS, - Regressor, WEIGHT_STRATEGIES, - _OPTUNA_NAMESPACES, - _OPTUNA_LABEL_SELECTION_SCHEMA_VERSION, + LabelWeightSupportError, OptunaNamespace, + Regressor, + _optuna_quarantine_path, compose_sample_weights, ensure_datetime_series, - make_test_set_and_weights, - fit_regressor, + enum_error_message, finite_sample, + fit_regressor, format_dict, format_number, get_causal_mode, @@ -102,16 +99,16 @@ from Utils import ( label_known_at_lookahead_column_name, label_weight_column_name, label_weight_known_at_lookahead_column_name, + make_test_set_and_weights, migrate_config, - _optuna_quarantine_path, optuna_load_best_params, optuna_save_best_params, require_bool, require_numeric, - sanitize_and_renormalize, safe_distribution_fit, - summarize_label_weight_support, + sanitize_and_renormalize, soft_extremum, + summarize_label_weight_support, zigzag, ) @@ -211,11 +208,9 @@ DensityAggregation = Literal["power_mean", "quantile", "min", "max"] DistanceMethod = Literal["compromise_programming", "topsis"] ClusterMethod = Literal["kmeans", "kmeans2", "kmedoids"] DensityMethod = Literal["knn", "medoid"] -SelectionMethod = Union[DistanceMethod, ClusterMethod, DensityMethod] +SelectionMethod = DistanceMethod | ClusterMethod | DensityMethod ValidationMode = Literal["warn", "raise", "none"] -SplitFn = Callable[ - [pd.DataFrame, pd.DataFrame, "SampleWeightInputs", pd.DataFrame], dict[str, Any] -] +SplitFn = Callable[[pd.DataFrame, pd.DataFrame, "SampleWeightInputs", pd.DataFrame], dict[str, Any]] warnings.simplefilter(action="ignore", category=FutureWarning) logger = logging.getLogger(__name__) @@ -251,17 +246,13 @@ class SampleWeightInputs: def __post_init__(self) -> None: if self.base.ndim != 1: - raise ValueError( - f"SampleWeightInputs.base: must be 1-D (ndim={self.base.ndim})" - ) + raise ValueError(f"SampleWeightInputs.base: must be 1-D (ndim={self.base.ndim})") if self.label is not None and self.base.shape != self.label.shape: raise ValueError( f"SampleWeightInputs.label: shape {self.label.shape} " f"!= base shape {self.base.shape}" ) - missing = ( - self._REQUIRED_LABEL_WEIGHTING_KEYS - self.label_weighting_config.keys() - ) + missing = self._REQUIRED_LABEL_WEIGHTING_KEYS - self.label_weighting_config.keys() if missing: raise KeyError( f"SampleWeightInputs.label_weighting_config: missing required keys " @@ -359,13 +350,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): _OPTUNA_JOURNAL_TAIL_PROBE_BYTES: Final[int] = 65536 _OPTUNA_SAMPLERS: Final[_OptunaSamplers] = _OptunaSamplers() _OPTUNA_HPO_SAMPLERS: Final[_OptunaHpoSamplers] = _OptunaHpoSamplers() - _OPTUNA_HPO_SAMPLERS_SET: Final[frozenset[OptunaSampler]] = frozenset( - _OPTUNA_HPO_SAMPLERS - ) + _OPTUNA_HPO_SAMPLERS_SET: Final[frozenset[OptunaSampler]] = frozenset(_OPTUNA_HPO_SAMPLERS) _OPTUNA_LABEL_SAMPLERS: Final[_OptunaLabelSamplers] = _OptunaLabelSamplers() - _OPTUNA_LABEL_SAMPLERS_SET: Final[frozenset[OptunaSampler]] = frozenset( - _OPTUNA_LABEL_SAMPLERS - ) + _OPTUNA_LABEL_SAMPLERS_SET: Final[frozenset[OptunaSampler]] = frozenset(_OPTUNA_LABEL_SAMPLERS) _SCALER_TYPES: Final[tuple[ScalerType, ...]] = ( "minmax", @@ -387,9 +374,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) _METHOD_COMPROMISE_PROGRAMMING: Final[str] = _DISTANCE_METHODS[0] _METHOD_TOPSIS: Final[str] = _DISTANCE_METHODS[1] - _DISTANCE_METHODS_SET: Final[frozenset[DistanceMethod]] = frozenset( - _DISTANCE_METHODS - ) + _DISTANCE_METHODS_SET: Final[frozenset[DistanceMethod]] = frozenset(_DISTANCE_METHODS) _CLUSTER_METHODS: Final[tuple[ClusterMethod, ...]] = ( "kmeans", "kmeans2", @@ -419,9 +404,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): _SELECTION_KMEDOIDS: Final[str] = _SELECTION_METHODS[4] _SELECTION_KNN: Final[str] = _SELECTION_METHODS[5] _SELECTION_MEDOID: Final[str] = _SELECTION_METHODS[6] - _SELECTION_METHODS_SET: Final[frozenset[SelectionMethod]] = frozenset( - _SELECTION_METHODS - ) + _SELECTION_METHODS_SET: Final[frozenset[SelectionMethod]] = frozenset(_SELECTION_METHODS) _DISTANCE_METRICS: Final[tuple[str, ...]] = ( "euclidean", @@ -521,17 +504,15 @@ class QuickAdapterRegressorV3(BaseRegressionModel): LABEL_DISTANCE_METRIC_DEFAULT: Final[str] = _DISTANCE_METRICS[0] # "euclidean" LABEL_CLUSTER_METRIC_DEFAULT: Final[str] = _DISTANCE_METRICS[0] # "euclidean" - LABEL_CLUSTER_SELECTION_METHOD_DEFAULT: Final[DistanceMethod] = _DISTANCE_METHODS[ + LABEL_CLUSTER_SELECTION_METHOD_DEFAULT: Final[DistanceMethod] = _DISTANCE_METHODS[1] # "topsis" + LABEL_CLUSTER_TRIAL_SELECTION_METHOD_DEFAULT: Final[DistanceMethod] = _DISTANCE_METHODS[ 1 ] # "topsis" - LABEL_CLUSTER_TRIAL_SELECTION_METHOD_DEFAULT: Final[DistanceMethod] = ( - _DISTANCE_METHODS[1] # "topsis" - ) LABEL_DENSITY_N_NEIGHBORS_DEFAULT: Final[int] = 5 - LABEL_DENSITY_AGGREGATION_DEFAULT: Final[DensityAggregation] = ( - _DENSITY_AGGREGATIONS[0] # "power_mean" - ) + LABEL_DENSITY_AGGREGATION_DEFAULT: Final[DensityAggregation] = _DENSITY_AGGREGATIONS[ + 0 + ] # "power_mean" OPTUNA_N_JOBS_DEFAULT: Final[int] = 1 OPTUNA_N_STARTUP_TRIALS_DEFAULT: Final[int] = 15 @@ -555,7 +536,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): "vary_model_seed_by_trial", ) - _OPTUNA_INT_OPTION_BOUNDS: Final[dict[str, tuple[int, Optional[int]]]] = { + _OPTUNA_INT_OPTION_BOUNDS: Final[dict[str, tuple[int, int | None]]] = { "n_jobs": (1, None), "n_startup_trials": (0, None), "n_trials": (1, None), @@ -575,33 +556,28 @@ class QuickAdapterRegressorV3(BaseRegressionModel): TIMESERIES_GAP_DEFAULT: Final[int] = 0 TIMESERIES_MAX_TRAIN_SIZE_DEFAULT: Final[int | None] = None - _EXTREMA_SELECTION_METHODS_SET: Final[frozenset[ExtremaSelectionMethod]] = ( - frozenset(EXTREMA_SELECTION_METHODS) + _EXTREMA_SELECTION_METHODS_SET: Final[frozenset[ExtremaSelectionMethod]] = frozenset( + EXTREMA_SELECTION_METHODS ) _CUSTOM_THRESHOLD_METHODS_SET: Final[frozenset[CustomThresholdMethod]] = frozenset( CUSTOM_THRESHOLD_METHODS ) - _SKIMAGE_THRESHOLD_METHODS_SET: Final[frozenset[SkimageThresholdMethod]] = ( - frozenset(SKIMAGE_THRESHOLD_METHODS) - ) - _THRESHOLD_METHODS_SET: Final[frozenset[ThresholdMethod]] = frozenset( - THRESHOLD_METHODS - ) - _OPTUNA_NAMESPACES_SET: Final[frozenset[OptunaNamespace]] = frozenset( - _OPTUNA_NAMESPACES + _SKIMAGE_THRESHOLD_METHODS_SET: Final[frozenset[SkimageThresholdMethod]] = frozenset( + SKIMAGE_THRESHOLD_METHODS ) + _THRESHOLD_METHODS_SET: Final[frozenset[ThresholdMethod]] = frozenset(THRESHOLD_METHODS) + _OPTUNA_NAMESPACES_SET: Final[frozenset[OptunaNamespace]] = frozenset(_OPTUNA_NAMESPACES) @staticmethod def _coerce_int(value: Any, name: str, *, minimum: int) -> int: if isinstance(value, bool) or not isinstance(value, int) or value < minimum: raise ValueError( - f"Invalid data_split_parameters.{name} value {value!r}: " - f"must be int >= {minimum}" + f"Invalid data_split_parameters.{name} value {value!r}: must be int >= {minimum}" ) return value @staticmethod - def _coerce_optional_int(value: Any, name: str, *, minimum: int) -> Optional[int]: + def _coerce_optional_int(value: Any, name: str, *, minimum: int) -> int | None: if value is None: return None return QuickAdapterRegressorV3._coerce_int(value, name, minimum=minimum) @@ -612,22 +588,16 @@ class QuickAdapterRegressorV3(BaseRegressionModel): unfiltered_df: pd.DataFrame, ) -> None: if not unfiltered_df.index.is_unique: - raise ValueError( - "unfiltered_df.index must be unique for causal split guards" - ) + raise ValueError("unfiltered_df.index must be unique for causal split guards") if not filtered_dataframe.index.isin(unfiltered_df.index).all(): - raise ValueError( - "filtered_dataframe.index must be a subset of unfiltered_df.index" - ) + raise ValueError("filtered_dataframe.index must be a subset of unfiltered_df.index") @staticmethod def _row_positions( filtered_dataframe: pd.DataFrame, unfiltered_df: pd.DataFrame, ) -> pd.Series: - QuickAdapterRegressorV3._validate_index_alignment( - filtered_dataframe, unfiltered_df - ) + QuickAdapterRegressorV3._validate_index_alignment(filtered_dataframe, unfiltered_df) positions = pd.Series( np.arange(len(unfiltered_df), dtype=np.int64), index=unfiltered_df.index ) @@ -651,9 +621,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): silently (opt-in by emission). Returns ``None`` when none is usable; callers then fall back to the position-based purge. """ - QuickAdapterRegressorV3._validate_index_alignment( - filtered_dataframe, unfiltered_df - ) + QuickAdapterRegressorV3._validate_index_alignment(filtered_dataframe, unfiltered_df) series_list: list[pd.Series] = [] for label_col in LABEL_COLUMNS: for lookahead_col in ( @@ -764,9 +732,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): context, exc, ) - return compose_sample_weights( - base_weights, None, logger=logger, context=context - ) + return compose_sample_weights(base_weights, None, logger=logger, context=context) @staticmethod def _apply_support_policy( @@ -797,9 +763,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): context, reason_text, ) - return compose_sample_weights( - base_weights, None, logger=logger, context=context - ) + return compose_sample_weights(base_weights, None, logger=logger, context=context) case _: assert_never(policy) @@ -825,9 +789,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ``_apply_support_policy``. Returns the composed weights on success or the fallback weights from the policy on failure. """ - policy = cast( - LabelWeightSupportPolicy, label_weighting_config["support_policy"] - ) + policy = cast("LabelWeightSupportPolicy", label_weighting_config["support_policy"]) if label_weights is None: # Non-"none" label-weighting strategy with no available label # weights (``zigzag`` produced zero pivots): the support policy @@ -841,13 +803,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): context=context, policy=policy, reasons=[ - f"label_weighting.strategy={strategy!r} configured but " - f"no label weights available (no pivots detected)" + ( + f"label_weighting.strategy={strategy!r} configured but " + f"no label weights available (no pivots detected)" + ) ], ) - return compose_sample_weights( - base_weights, None, logger=logger, context=context - ) + return compose_sample_weights(base_weights, None, logger=logger, context=context) try: composed = compose_sample_weights( @@ -892,14 +854,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): surviving weights post-pipeline); ``pivot_equivalent_count`` and ``positive_label_weight_fraction`` derive from ``label_weights``. """ - policy = cast( - LabelWeightSupportPolicy, label_weighting_config["support_policy"] - ) + policy = cast("LabelWeightSupportPolicy", label_weighting_config["support_policy"]) summary = summarize_label_weight_support(label_weights, sample_weights) reasons: list[str] = [] - min_pivot_equivalent_count = label_weighting_config[ - "min_pivot_equivalent_count" - ] + min_pivot_equivalent_count = label_weighting_config["min_pivot_equivalent_count"] min_positive_label_weight_fraction = label_weighting_config[ "min_positive_label_weight_fraction" ] @@ -939,7 +897,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return sample_weights @staticmethod - def _get_selection_category(method: str) -> Optional[str]: + def _get_selection_category(method: str) -> str | None: for ( category, methods, @@ -949,7 +907,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return None @staticmethod - def _get_label_p_order_default(distance_metric: str) -> Optional[float]: + def _get_label_p_order_default(distance_metric: str) -> float | None: if distance_metric == QuickAdapterRegressorV3._METRIC_MINKOWSKI: return 2.0 elif distance_metric == QuickAdapterRegressorV3._METRIC_POWER_MEAN: @@ -957,7 +915,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return None @staticmethod - def _get_label_density_metric_default(method: DensityMethod) -> Optional[str]: + def _get_label_density_metric_default(method: DensityMethod) -> str | None: if method == QuickAdapterRegressorV3._DENSITY_MEDOID: return QuickAdapterRegressorV3._METRIC_EUCLIDEAN elif method == QuickAdapterRegressorV3._DENSITY_KNN: @@ -967,7 +925,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _get_label_density_aggregation_param_default( aggregation: DensityAggregation, - ) -> Optional[float]: + ) -> float | None: if aggregation == QuickAdapterRegressorV3._DENSITY_AGG_POWER_MEAN: return 1.0 elif aggregation == QuickAdapterRegressorV3._DENSITY_AGG_QUANTILE: @@ -976,13 +934,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _validate_scalar( - value: Optional[float], + value: float | None, *, ctx: str, mode: ValidationMode, - predicate: Optional[Callable[[float], bool]] = None, + predicate: Callable[[float], bool] | None = None, constraint: str = "", - ) -> Optional[float]: + ) -> float | None: if value is None: return None if mode == "none": @@ -1010,8 +968,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _validate_minkowski_p( - p: Optional[float], *, ctx: str, mode: ValidationMode = "raise" - ) -> Optional[float]: + p: float | None, *, ctx: str, mode: ValidationMode = "raise" + ) -> float | None: return QuickAdapterRegressorV3._validate_scalar( p, ctx=ctx, mode=mode, predicate=lambda v: v > 0, constraint="must be > 0" ) @@ -1019,8 +977,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _prepare_distance_kwargs( distance_metric: str, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, mode: ValidationMode = "none", metric_ctx: str = "distance_metric", p_ctx: str = "p", @@ -1035,9 +993,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): kwargs["w"] = weights if distance_metric == QuickAdapterRegressorV3._METRIC_MINKOWSKI: - validated_p = QuickAdapterRegressorV3._validate_minkowski_p( - p, ctx=p_ctx, mode=mode - ) + validated_p = QuickAdapterRegressorV3._validate_minkowski_p(p, ctx=p_ctx, mode=mode) if validated_p is not None: kwargs["p"] = validated_p @@ -1045,8 +1001,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _validate_quantile_q( - q: Optional[float], *, ctx: str, mode: ValidationMode = "raise" - ) -> Optional[float]: + q: float | None, *, ctx: str, mode: ValidationMode = "raise" + ) -> float | None: return QuickAdapterRegressorV3._validate_scalar( q, ctx=ctx, @@ -1057,14 +1013,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _validate_power_mean_p( - p: Optional[float], *, ctx: str, mode: ValidationMode = "raise" - ) -> Optional[float]: + p: float | None, *, ctx: str, mode: ValidationMode = "raise" + ) -> float | None: return QuickAdapterRegressorV3._validate_scalar(p, ctx=ctx, mode=mode) @staticmethod def _validate_metric_weights_support( metric: str, *, ctx: str, mode: ValidationMode = "warn" - ) -> Optional[str]: + ) -> str | None: if metric not in QuickAdapterRegressorV3._UNSUPPORTED_WEIGHTS_METRICS_SET: return metric @@ -1084,7 +1040,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): *, ctx: str, mode: ValidationMode = "raise", - ) -> Optional[NDArray[np.floating]]: + ) -> NDArray[np.floating] | None: uniform_weights = np.full(n_objectives, 1.0 / n_objectives) if weights is None: @@ -1109,10 +1065,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return np_weights / weights_sum if not isinstance(weights, (list, tuple, np.ndarray)): - msg = ( - f"Invalid {ctx} {type(weights).__name__!r}: " - f"must be a list, tuple, or array" - ) + msg = f"Invalid {ctx} {type(weights).__name__!r}: must be a list, tuple, or array" if mode == "raise": raise ValueError(msg) logger.warning(f"{msg}, using uniform weights") @@ -1162,8 +1115,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): *, ctx: str, mode: ValidationMode = "raise", - default: Optional[str] = None, - ) -> Optional[str]: + default: str | None = None, + ) -> str | None: if value in valid_set: return value @@ -1186,17 +1139,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): mode: ValidationMode = "warn", ) -> str: if aggregate_allowed: - valid_metrics = ( - QuickAdapterRegressorV3._LABEL_SELECTION_DISTANCE_METRICS_SET - ) + valid_metrics = QuickAdapterRegressorV3._LABEL_SELECTION_DISTANCE_METRICS_SET else: # Cluster/density paths route the metric to SciPy/sklearn APIs # (``pairwise_distances``, ``KMeans``, ``KMedoids``, ``NearestNeighbors``) # which reject aggregate metrics computed by reduction; restrict the # valid set to SciPy-compatible non-probability metrics. - valid_metrics = ( - QuickAdapterRegressorV3._CLUSTER_DENSITY_DISTANCE_METRICS_SET - ) + valid_metrics = QuickAdapterRegressorV3._CLUSTER_DENSITY_DISTANCE_METRICS_SET valid_options = tuple( candidate for candidate in QuickAdapterRegressorV3._DISTANCE_METRICS @@ -1210,23 +1159,21 @@ class QuickAdapterRegressorV3(BaseRegressionModel): mode=mode, default=default, ) - return cast(str, resolved_metric) + return cast("str", resolved_metric) @staticmethod def _prepare_knn_kwargs( distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, mode: ValidationMode = "warn", p_ctx: str = "label_density_p", ) -> dict[str, Any]: knn_kwargs: dict[str, Any] = {} if distance_metric == QuickAdapterRegressorV3._METRIC_MINKOWSKI: - validated_p = QuickAdapterRegressorV3._validate_minkowski_p( - p, ctx=p_ctx, mode=mode - ) + validated_p = QuickAdapterRegressorV3._validate_minkowski_p(p, ctx=p_ctx, mode=mode) if validated_p is not None: knn_kwargs["p"] = validated_p if weights is not None: @@ -1237,11 +1184,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _resolve_p_order( distance_metric: str, - label_p_order: Optional[float], + label_p_order: float | None, *, ctx: str, mode: ValidationMode = "raise", - ) -> Optional[float]: + ) -> float | None: p = ( label_p_order if label_p_order is not None @@ -1317,11 +1264,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) config["trial_selection_method"] = trial_selection_method elif category == "density": - density_method = cast(DensityMethod, label_method) - density_metric_default = ( - QuickAdapterRegressorV3._get_label_density_metric_default( - density_method - ) + density_method = cast("DensityMethod", label_method) + density_metric_default = QuickAdapterRegressorV3._get_label_density_metric_default( + density_method ) distance_metric = self.ft_params.get( "label_density_metric", @@ -1338,7 +1283,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if density_method == QuickAdapterRegressorV3._DENSITY_KNN: aggregation = cast( - DensityAggregation, + "DensityAggregation", self.ft_params.get( "label_density_aggregation", QuickAdapterRegressorV3.LABEL_DENSITY_AGGREGATION_DEFAULT, @@ -1397,10 +1342,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): continue suffix = QuickAdapterRegressorV3._CONFIG_KEY_TO_TUNABLE_SUFFIX.get(key, key) tunable_name = f"label_{category}_{suffix}" - if isinstance(value, float): - formatted_value = format_number(value) - else: - formatted_value = value + formatted_value = format_number(value) if isinstance(value, float) else value logger.info(f" {tunable_name}: {formatted_value}") def _optuna_label_selection_metadata(self) -> dict[str, Any]: @@ -1416,12 +1358,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) label_weights = self.ft_params.get("label_weights") label_p_order = self.ft_params.get("label_p_order") - if label_weights is not None and not all( - np.isfinite(float(w)) for w in label_weights - ): - raise ValueError( - f"label_weights contains non-finite values: {label_weights!r}" - ) + if label_weights is not None and not all(np.isfinite(float(w)) for w in label_weights): + raise ValueError(f"label_weights contains non-finite values: {label_weights!r}") if label_p_order is not None and not np.isfinite(float(label_p_order)): raise ValueError(f"label_p_order is non-finite: {label_p_order!r}") return { @@ -1430,9 +1368,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): "label_weights": ( [float(w) for w in label_weights] if label_weights is not None else None ), - "label_p_order": ( - float(label_p_order) if label_p_order is not None else None - ), + "label_p_order": (float(label_p_order) if label_p_order is not None else None), } @cached_property @@ -1554,9 +1490,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @cached_property def label_weighting(self) -> dict[str, Any]: - return get_label_weighting_config( - self.freqai_info.get("label_weighting"), logger - ) + return get_label_weighting_config(self.freqai_info.get("label_weighting"), logger) @cached_property def label_pipeline(self) -> dict[str, Any]: @@ -1564,9 +1498,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @cached_property def label_prediction(self) -> dict[str, Any]: - return get_label_prediction_config( - self.freqai_info.get("label_prediction"), logger - ) + return get_label_prediction_config(self.freqai_info.get("label_prediction"), logger) @cached_property def _label_defaults(self) -> tuple[int, float]: @@ -1590,9 +1522,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): half_label_frequency_candles = int(label_frequency_candles / 2) self._optuna_label_candle_pool_full_cache[cache_key] = [ max(1, label_frequency_candles + offset) - for offset in range( - -half_label_frequency_candles, half_label_frequency_candles + 1 - ) + for offset in range(-half_label_frequency_candles, half_label_frequency_candles + 1) ] return copy.deepcopy(self._optuna_label_candle_pool_full_cache[cache_key]) @@ -1614,13 +1544,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): raise ValueError( "Invalid freqai configuration: 'identifier' must be a non-empty string" ) - self._optuna_hyperopt: Optional[bool] = ( + self._optuna_hyperopt: bool | None = ( self.freqai_info.get("enabled", False) and self._optuna_config.get("enabled") - and self.data_split_parameters.get( - "test_size", QuickAdapterRegressorV3._TEST_SIZE - ) - != 0 + and self.data_split_parameters.get("test_size", QuickAdapterRegressorV3._TEST_SIZE) != 0 ) self._optuna_hp_value: dict[str, float] = {} self._holdout_rmse: dict[str, float] = {} @@ -1634,9 +1561,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self._optuna_label_candle: dict[str, int] = {} self._optuna_label_candles: dict[str, int] = {} self._optuna_label_incremented_pairs: list[str] = [] - default_label_period_candles, default_label_natr_multiplier = ( - self._label_defaults - ) + default_label_period_candles, default_label_natr_multiplier = self._label_defaults # ``self.live`` is unset until ``IFreqaiModel.start()``, so derive trade-mode # from the configured runmode here. trade_mode = self.config.get("runmode") in TRADE_MODES @@ -1647,9 +1572,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): -1 ] * QuickAdapterRegressorV3._OPTUNA_LABEL_N_OBJECTIVES self._optuna_hp_params[pair] = ( - self.optuna_load_best_params(pair, _OPTUNA_NAMESPACES.hp) - if trade_mode - else None + self.optuna_load_best_params(pair, _OPTUNA_NAMESPACES.hp) if trade_mode else None ) or {} configured_label_params = { "label_period_candles": self.ft_params.get( @@ -1664,9 +1587,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ), } self._optuna_label_params[pair] = ( - self.optuna_load_best_params(pair, _OPTUNA_NAMESPACES.label) - if trade_mode - else None + self.optuna_load_best_params(pair, _OPTUNA_NAMESPACES.label) if trade_mode else None ) or configured_label_params self.set_optuna_label_candle(pair) self._optuna_label_candles[pair] = 0 @@ -1698,9 +1619,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): logger.info(f" n_trials: {optuna_config.get('n_trials')}") logger.info(f" timeout: {optuna_config.get('timeout')}") logger.info(f" space_reduction: {optuna_config.get('space_reduction')}") - logger.info( - f" space_fraction: {format_number(optuna_config.get('space_fraction'))}" - ) + logger.info(f" space_fraction: {format_number(optuna_config.get('space_fraction'))}") logger.info(f" min_resource: {optuna_config.get('min_resource')}") logger.info(f" seed: {optuna_config.get('seed')}") logger.info( @@ -1708,14 +1627,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f"{optuna_config.get('reset_label_study_on_schema_mismatch')}" ) logger.info( - " vary_model_seed_by_trial: " - f"{optuna_config.get('vary_model_seed_by_trial')}" + f" vary_model_seed_by_trial: {optuna_config.get('vary_model_seed_by_trial')}" ) logger.info(f" label_sampler: {optuna_config.get('label_sampler')}") - logger.info( - f" label_candles_step: {optuna_config.get('label_candles_step')}" - ) + logger.info(f" label_candles_step: {optuna_config.get('label_candles_step')}") label_method = self.ft_params.get( "label_method", QuickAdapterRegressorV3.LABEL_METHOD_DEFAULT ) @@ -1729,25 +1645,19 @@ class QuickAdapterRegressorV3(BaseRegressionModel): formatted_label_weights = [format_number(w) for w in label_weights] logger.info(f" label_weights: [{', '.join(formatted_label_weights)}]") else: - logger.info( - " label_weights: [1.0, ...] * n_objectives, l1 normalized (default)" - ) + logger.info(" label_weights: [1.0, ...] * n_objectives, l1 normalized (default)") label_p_order_config = self.ft_params.get("label_p_order") if label_p_order_config is not None: - logger.info( - f" label_p_order: {format_number(float(label_p_order_config))}" - ) + logger.info(f" label_p_order: {format_number(float(label_p_order_config))}") else: distance_metric = label_config["distance_metric"] if distance_metric in { QuickAdapterRegressorV3._METRIC_MINKOWSKI, QuickAdapterRegressorV3._METRIC_POWER_MEAN, }: - label_p_order_default = ( - QuickAdapterRegressorV3._get_label_p_order_default( - distance_metric - ) + label_p_order_default = QuickAdapterRegressorV3._get_label_p_order_default( + distance_metric ) logger.info( f" label_p_order: {format_number(label_p_order_default)} (default for {distance_metric})" @@ -1773,9 +1683,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): logger.info( f" minmax_range: ({format_number(col_pipeline['minmax_range'][0])}, {format_number(col_pipeline['minmax_range'][1])})" ) - logger.info( - f" sigmoid_scale: {format_number(col_pipeline['sigmoid_scale'])}" - ) + logger.info(f" sigmoid_scale: {format_number(col_pipeline['sigmoid_scale'])}") logger.info(f" gamma: {format_number(col_pipeline['gamma'])}") col_prediction = get_label_column_config( @@ -1791,9 +1699,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): logger.info( f" soft_extremum_alpha: {format_number(col_prediction['soft_extremum_alpha'])}" ) - logger.info( - f" keep_fraction: {format_number(col_prediction['keep_fraction'])}" - ) + logger.info(f" keep_fraction: {format_number(col_prediction['keep_fraction'])}") if col_prediction["method"] == PREDICTION_METHODS[0]: # "none" logger.warning( f" Prediction method is 'none' for label [{label_col}]: " @@ -1801,9 +1707,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f"entry signals based on them will never trigger." ) - default_label_period_candles, default_label_natr_multiplier = ( - self._label_defaults - ) + default_label_period_candles, default_label_natr_multiplier = self._label_defaults label_period_candles = self.ft_params.get( "label_period_candles", default_label_period_candles ) @@ -1811,13 +1715,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self.ft_params.get("label_natr_multiplier", default_label_natr_multiplier) ) logger.info("Label Hyperparameters:") - logger.info( - f" fit_live_predictions_candles: {self._fit_live_predictions_candles}" - ) + logger.info(f" fit_live_predictions_candles: {self._fit_live_predictions_candles}") if self._optuna_hyperopt: - logger.info( - f" label_period_candles: {label_period_candles} (initial value)" - ) + logger.info(f" label_period_candles: {label_period_candles} (initial value)") logger.info( f" label_natr_multiplier: {format_number(label_natr_multiplier)} (initial value)" ) @@ -1844,15 +1744,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): else: logger.info("Label Parameters:") logger.info(f" label_period_candles: {label_period_candles}") - logger.info( - f" label_natr_multiplier: {format_number(label_natr_multiplier)}" - ) + logger.info(f" label_natr_multiplier: {format_number(label_natr_multiplier)}") logger.info(f" label_horizon_candles: {self._label_horizon_candles()}") scaler = self.ft_params.get("scaler", QuickAdapterRegressorV3.SCALER_DEFAULT) - feature_range = self.ft_params.get( - "range", QuickAdapterRegressorV3.RANGE_DEFAULT - ) + feature_range = self.ft_params.get("range", QuickAdapterRegressorV3.RANGE_DEFAULT) logger.info("Feature Parameters:") logger.info(f" scaler: {scaler}") logger.info( @@ -1868,9 +1764,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): raise ValueError(enum_error_message("namespace", namespace, tuple(stores))) return stores[namespace] - def get_optuna_params( - self, pair: str, namespace: OptunaNamespace - ) -> dict[str, Any]: + def get_optuna_params(self, pair: str, namespace: OptunaNamespace) -> dict[str, Any]: store = self._resolve_optuna_store( namespace, { @@ -1898,23 +1792,17 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) return store.get(pair, np.nan) - def set_optuna_value( - self, pair: str, namespace: OptunaNamespace, value: float - ) -> None: + def set_optuna_value(self, pair: str, namespace: OptunaNamespace, value: float) -> None: store = self._resolve_optuna_store( namespace, {_OPTUNA_NAMESPACES.hp: self._optuna_hp_value} ) store[pair] = value - def get_optuna_values( - self, pair: str, namespace: OptunaNamespace - ) -> list[float | int]: + def get_optuna_values(self, pair: str, namespace: OptunaNamespace) -> list[float | int]: store = self._resolve_optuna_store( namespace, {_OPTUNA_NAMESPACES.label: self._optuna_label_values} ) - return store.get( - pair, [np.nan] * QuickAdapterRegressorV3._OPTUNA_LABEL_N_OBJECTIVES - ) + return store.get(pair, [np.nan] * QuickAdapterRegressorV3._OPTUNA_LABEL_N_OBJECTIVES) def set_optuna_values( self, pair: str, namespace: OptunaNamespace, values: list[float | int] @@ -1939,9 +1827,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): def set_optuna_label_candle(self, pair: str) -> None: if len(self._optuna_label_candle_pool) == 0: - logger.warning( - f"[{pair}] Optuna label candle pool is empty, reinitializing" - ) + logger.warning(f"[{pair}] Optuna label candle pool is empty, reinitializing") logger.debug( f"[{pair}] Optuna label candle pool state: " f"pool={self._optuna_label_candle_pool}, " @@ -1975,9 +1861,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): - set(self._optuna_label_candle.values()) ) if len(optuna_label_available_candles) > 0: - self._optuna_label_candle_pool.extend( - sorted(optuna_label_available_candles) - ) + self._optuna_label_candle_pool.extend(sorted(optuna_label_available_candles)) self._optuna_label_shuffle_rng.shuffle(self._optuna_label_candle_pool) def define_data_pipeline(self, threads: int = -1) -> Pipeline: @@ -1991,9 +1875,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): mode="raise", ) - feature_range = self.ft_params.get( - "range", QuickAdapterRegressorV3.RANGE_DEFAULT - ) + feature_range = self.ft_params.get("range", QuickAdapterRegressorV3.RANGE_DEFAULT) if not isinstance(feature_range, (list, tuple)) or len(feature_range) != 2: raise ValueError( @@ -2023,9 +1905,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): scaler_obj = SKLearnWrapper(MinMaxScaler(feature_range=feature_range)) steps = [ - (name, scaler_obj) - if name in ("scaler", "post-pca-scaler") - else (name, transformer) + (name, scaler_obj) if name in ("scaler", "post-pca-scaler") else (name, transformer) for name, transformer in pipeline.steps ] @@ -2041,9 +1921,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ] ) - def train( - self, unfiltered_df: pd.DataFrame, pair: str, dk: FreqaiDataKitchen, **kwargs - ) -> Any: + def train(self, unfiltered_df: pd.DataFrame, pair: str, dk: FreqaiDataKitchen, **kwargs) -> Any: """Train a model with per-row sample weights. Dispatches on ``data_split_parameters.method``: @@ -2134,8 +2012,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): test_size = dsp["test_size"] if isinstance(test_size, bool) or not isinstance(test_size, (int, float)): raise ValueError( - f"Invalid data_split_parameters.test_size value {test_size!r}: " - f"must be int or float" + f"Invalid data_split_parameters.test_size value {test_size!r}: must be int or float" ) if test_size == 0 and feat_dict.get("reverse_train_test_order", False): raise ValueError( @@ -2170,9 +2047,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): features, labels, weights.base, weights.label, **sklearn_kwargs ) if causal_mode: - row_positions = QuickAdapterRegressorV3._row_positions( - features, unfiltered_df - ) + row_positions = QuickAdapterRegressorV3._row_positions(features, unfiltered_df) first_test_position = int(row_positions.loc[test_features.index].min()) label_horizon_candles = self._label_horizon_candles(dk.pair) train_positions = row_positions.loc[train_features.index] @@ -2184,9 +2059,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): features, unfiltered_df ) if known_at_lookahead is not None: - train_known_at_lookahead = known_at_lookahead.loc[ - train_features.index - ] + train_known_at_lookahead = known_at_lookahead.loc[train_features.index] train_known_at_position = train_positions.to_numpy( dtype=np.int64 ) + train_known_at_lookahead.to_numpy(dtype=np.int64) @@ -2218,11 +2091,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if feat_dict.get("shuffle_after_split", False): parent_seed = sklearn_kwargs.get("random_state") - shuffle_rng = ( - random.Random(parent_seed) - if parent_seed is not None - else random.Random() - ) + shuffle_rng = random.Random(parent_seed) if parent_seed is not None else random.Random() train_features, train_labels, train_base_weights, train_label_weights = ( QuickAdapterRegressorV3._shuffle_split_rows( train_features, @@ -2333,9 +2202,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) weight_col = label_weight_column_name(LABEL_COLUMNS[0]) if weight_col in unfiltered_df.columns: - label_weights = unfiltered_df.loc[ - features_filtered.index, weight_col - ].to_numpy(dtype=float) + label_weights = unfiltered_df.loc[features_filtered.index, weight_col].to_numpy( + dtype=float + ) logger.debug("label weight column active: %r", weight_col) else: label_weights = None @@ -2357,9 +2226,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): split_fn: SplitFn, **kwargs, ) -> Any: - logger.info( - f"-------------------- Starting training {pair} --------------------" - ) + logger.info(f"-------------------- Starting training {pair} --------------------") start_time = time.time() features_filtered, labels_filtered = dk.filter_features( unfiltered_df, @@ -2376,9 +2243,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f"{end_date} --------------------" ) dd = split_fn(features_filtered, labels_filtered, weights, unfiltered_df) - dd = self._add_validation_split( - dd, features_filtered, weights, unfiltered_df, pair - ) + dd = self._add_validation_split(dd, features_filtered, weights, unfiltered_df, pair) dd = self._add_refit_data( dd, features_filtered, @@ -2387,9 +2252,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): unfiltered_df, pair, ) - train_positions = features_filtered.index.get_indexer( - dd["train_features"].index - ) + train_positions = features_filtered.index.get_indexer(dd["train_features"].index) if (train_positions < 0).any(): raise ValueError( f"[{pair}] _train_common: unable to align training rows to " @@ -2422,18 +2285,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) -> NDArray[np.bool_]: """Return rows whose labels are known before ``cutoff_position``.""" row_positions = QuickAdapterRegressorV3._row_positions(features, unfiltered_df) - known_at_lookahead = QuickAdapterRegressorV3._known_at_lookahead( - features, unfiltered_df - ) + known_at_lookahead = QuickAdapterRegressorV3._known_at_lookahead(features, unfiltered_df) if known_at_lookahead is None: return ( - row_positions.to_numpy(dtype=np.int64) - + self._label_horizon_candles(pair) + row_positions.to_numpy(dtype=np.int64) + self._label_horizon_candles(pair) < cutoff_position ) return ( - row_positions.to_numpy(dtype=np.int64) - + known_at_lookahead.to_numpy(dtype=np.int64) + row_positions.to_numpy(dtype=np.int64) + known_at_lookahead.to_numpy(dtype=np.int64) < cutoff_position ) @@ -2456,12 +2315,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): """Reserve the chronological tail of the training set for selection.""" validation_size = self._get_validation_size() if validation_size == 0: - data_dictionary["validation_features"] = data_dictionary[ - "train_features" - ].iloc[:0] - data_dictionary["validation_labels"] = data_dictionary["train_labels"].iloc[ - :0 - ] + data_dictionary["validation_features"] = data_dictionary["train_features"].iloc[:0] + data_dictionary["validation_labels"] = data_dictionary["train_labels"].iloc[:0] data_dictionary["validation_weights"] = data_dictionary["train_weights"][:0] return data_dictionary if ( @@ -2502,9 +2357,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): row_positions = QuickAdapterRegressorV3._row_positions( data_dictionary["train_features"], unfiltered_df ) - first_validation_position = int( - row_positions.loc[validation_features.index].min() - ) + first_validation_position = int(row_positions.loc[validation_features.index].min()) train_positions = row_positions.loc[train_features.index] known_at_lookahead = QuickAdapterRegressorV3._known_at_lookahead( data_dictionary["train_features"], unfiltered_df @@ -2516,9 +2369,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): else: train_known_at_position = train_positions.to_numpy( dtype=np.int64 - ) + known_at_lookahead.loc[train_features.index].to_numpy( - dtype=np.int64 - ) + ) + known_at_lookahead.loc[train_features.index].to_numpy(dtype=np.int64) keep_mask = train_known_at_position < first_validation_position train_features = train_features.loc[keep_mask] train_labels = train_labels.loc[keep_mask] @@ -2532,12 +2383,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): len(unfiltered_df), ) data_dictionary["test_features"] = holdout_features.loc[holdout_mask] - data_dictionary["test_labels"] = data_dictionary["test_labels"].loc[ - holdout_mask - ] - data_dictionary["test_weights"] = data_dictionary["test_weights"][ - holdout_mask - ] + data_dictionary["test_labels"] = data_dictionary["test_labels"].loc[holdout_mask] + data_dictionary["test_weights"] = data_dictionary["test_weights"][holdout_mask] if data_dictionary["test_features"].empty: logger.warning( f"[{pair}] causal purge emptied the holdout (label horizon " @@ -2613,9 +2460,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): refit_features = features.loc[keep_mask] refit_labels = labels.loc[keep_mask] refit_base_weights = weights.base[keep_mask] - refit_label_weights = ( - None if weights.label is None else weights.label[keep_mask] - ) + refit_label_weights = None if weights.label is None else weights.label[keep_mask] if ( self.data_split_parameters.get( "method", QuickAdapterRegressorV3.DATA_SPLIT_METHOD_DEFAULT @@ -2719,12 +2564,12 @@ class QuickAdapterRegressorV3(BaseRegressionModel): # Label-only re-gate: base-only weights carry no pivot/fraction/ESS # support to recheck (settled pre-pipeline), so they skip this stage. if weight_inputs.label is not None: - post_pipeline_base_weights = pipeline_labels.pop( - base_weight_column - ).to_numpy(dtype=float) - post_pipeline_label_weights = pipeline_labels.pop( - label_weight_column - ).to_numpy(dtype=float) + post_pipeline_base_weights = pipeline_labels.pop(base_weight_column).to_numpy( + dtype=float + ) + post_pipeline_label_weights = pipeline_labels.pop(label_weight_column).to_numpy( + dtype=float + ) # Load-bearing: ``fit_transform`` captured ``label_list`` WITH the smuggled # columns; restore it to the real labels or the next validation/test # transform rebuilds y with the wrong column count (``ValueError``). @@ -2776,33 +2621,22 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f"transform (outlier removal); relax SVM/DBSCAN outlier " f"thresholds or increase test_size" ) - dd["validation_weights"] = ( - QuickAdapterRegressorV3._sanitize_pipeline_weights( - dd["validation_features"], - dd["validation_weights"], - pair=pair, - context="validation", - ) - ) - dd["validation_labels"], _, _ = dk.label_pipeline.transform( - dd["validation_labels"] + dd["validation_weights"] = QuickAdapterRegressorV3._sanitize_pipeline_weights( + dd["validation_features"], + dd["validation_weights"], + pair=pair, + context="validation", ) + dd["validation_labels"], _, _ = dk.label_pipeline.transform(dd["validation_labels"]) - if ( - self.data_split_parameters.get( - "test_size", QuickAdapterRegressorV3._TEST_SIZE - ) - != 0 - ): + if self.data_split_parameters.get("test_size", QuickAdapterRegressorV3._TEST_SIZE) != 0: if dd["test_labels"].shape[0] == 0: if dd.get("holdout_purged_empty"): return dd method = self.data_split_parameters.get( "method", QuickAdapterRegressorV3.DATA_SPLIT_METHOD_DEFAULT ) - if ( - method == QuickAdapterRegressorV3._DATA_SPLIT_TIMESERIES - ): # timeseries_split + if method == QuickAdapterRegressorV3._DATA_SPLIT_TIMESERIES: # timeseries_split n_splits = self.data_split_parameters.get( "n_splits", QuickAdapterRegressorV3.TIMESERIES_N_SPLITS_DEFAULT ) @@ -2889,9 +2723,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) raw_gap = self.data_split_parameters.get("gap", None) gap = QuickAdapterRegressorV3._coerce_int( - raw_gap - if raw_gap is not None - else QuickAdapterRegressorV3.TIMESERIES_GAP_DEFAULT, + raw_gap if raw_gap is not None else QuickAdapterRegressorV3.TIMESERIES_GAP_DEFAULT, "gap", minimum=0, ) @@ -2913,9 +2745,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ): test_size = int(len(filtered_dataframe) * test_size) elif not ( - not isinstance(test_size, bool) - and isinstance(test_size, int) - and test_size >= 1 + not isinstance(test_size, bool) and isinstance(test_size, int) and test_size >= 1 ): raise ValueError( f"Invalid data_split_parameters.test_size value {test_size!r}: " @@ -2970,9 +2800,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): test_labels = labels.iloc[test_idx] train_base_weights = weights.base[train_idx] test_base_weights = weights.base[test_idx] - train_label_weights = ( - None if weights.label is None else weights.label[train_idx] - ) + train_label_weights = None if weights.label is None else weights.label[train_idx] test_label_weights = None if weights.label is None else weights.label[test_idx] if causal_mode: @@ -3039,9 +2867,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): test_weights, ) - def fit( - self, data_dictionary: dict[str, Any], dk: FreqaiDataKitchen, **kwargs - ) -> Any: + def fit(self, data_dictionary: dict[str, Any], dk: FreqaiDataKitchen, **kwargs) -> Any: X = data_dictionary.get("train_features") y = data_dictionary.get("train_labels") train_weights = data_dictionary.get("train_weights") @@ -3079,9 +2905,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self._optuna_config["space_fraction"], model_path=dk.data_path, init_model=selection_init_model, - vary_model_seed_by_trial=self._optuna_config[ - "vary_model_seed_by_trial" - ], + vary_model_seed_by_trial=self._optuna_config["vary_model_seed_by_trial"], ), direction=optuna.study.StudyDirection.MINIMIZE, ) @@ -3118,9 +2942,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): index=y_test.index, ) holdout_labels, _, _ = dk.label_pipeline.inverse_transform(y_test.copy()) - holdout_predictions, _, _ = dk.label_pipeline.inverse_transform( - holdout_predictions - ) + holdout_predictions, _, _ = dk.label_pipeline.inverse_transform(holdout_predictions) self._holdout_rmse[dk.pair] = float( sklearn.metrics.root_mean_squared_error( holdout_labels, @@ -3143,10 +2965,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): data_dictionary["train_labels"] = data_dictionary.pop("refit_labels") data_dictionary["train_weights"] = data_dictionary.pop("refit_weights") refit_weight_inputs = data_dictionary.pop("refit_weight_inputs") - if ( - not self.freqai_info.get("fit_live_predictions_candles", 0) - or not self.live - ): + if not self.freqai_info.get("fit_live_predictions_candles", 0) or not self.live: dk.fit_labels() ( data_dictionary["train_features"], @@ -3185,7 +3004,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self, pair: str, namespace: OptunaNamespace, - callback: Callable[[], Optional[optuna.study.Study]], + callback: Callable[[], optuna.study.Study | None], ) -> None: if namespace not in {_OPTUNA_NAMESPACES.label}: raise ValueError( @@ -3204,11 +3023,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if optuna_label_remaining_candles <= 0: try: callback() - except Exception as e: - logger.error( - f"[{pair}] Optuna {namespace} callback execution failed: {e!r}", - exc_info=True, - ) + except Exception: + logger.exception(f"[{pair}] Optuna {namespace} callback execution failed") finally: self.set_optuna_label_candle(pair) self._optuna_label_candles[pair] = 0 @@ -3285,9 +3101,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ), ) di_sample = finite_sample( - [] - if di_values is None - else pd.to_numeric(di_values, errors="coerce"), + [] if di_values is None else pd.to_numeric(di_values, errors="coerce"), positive_only=True, ) f = safe_distribution_fit( @@ -3319,12 +3133,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): QuickAdapterRegressorV3._DI_CUTOFF_DEFAULT, ) cutoff = QuickAdapterRegressorV3._DI_CUTOFF_DEFAULT - dk.data["extra_returns_per_train"][f"{label_col}_minima_threshold"] = ( - min_pred - ) - dk.data["extra_returns_per_train"][f"{label_col}_maxima_threshold"] = ( - max_pred - ) + dk.data["extra_returns_per_train"][f"{label_col}_minima_threshold"] = min_pred + dk.data["extra_returns_per_train"][f"{label_col}_maxima_threshold"] = max_pred dk.data["extra_returns_per_train"]["DI_value_param1"] = f[0] dk.data["extra_returns_per_train"]["DI_value_param2"] = f[1] dk.data["extra_returns_per_train"]["DI_value_param3"] = f[2] @@ -3362,24 +3172,18 @@ class QuickAdapterRegressorV3(BaseRegressionModel): f[1], ) - dk.data["extra_returns_per_train"]["label_period_candles"] = ( - self.get_optuna_params(pair, _OPTUNA_NAMESPACES.label).get( - "label_period_candles" - ) - ) - dk.data["extra_returns_per_train"]["label_natr_multiplier"] = ( - self.get_optuna_params( - pair, - _OPTUNA_NAMESPACES.label, - ).get("label_natr_multiplier") - ) + dk.data["extra_returns_per_train"]["label_period_candles"] = self.get_optuna_params( + pair, _OPTUNA_NAMESPACES.label + ).get("label_period_candles") + dk.data["extra_returns_per_train"]["label_natr_multiplier"] = self.get_optuna_params( + pair, + _OPTUNA_NAMESPACES.label, + ).get("label_natr_multiplier") current_holdout_rmse = self._holdout_rmse.get(pair) if not self.live: holdout_series = dk.full_df.get("holdout_rmse") - history_dates = ensure_datetime_series( - self.dd.historic_predictions[pair]["date"] - ) + history_dates = ensure_datetime_series(self.dd.historic_predictions[pair]["date"]) full_dates = ensure_datetime_series(dk.full_df["date"]) current_dates = full_dates.loc[full_dates > history_dates.max()] if holdout_series is None: @@ -3397,14 +3201,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): elif pair not in self._session_fitted_pairs: historic = self.dd.historic_predictions.get(pair) if historic is not None and "holdout_rmse" in historic: - holdout_values = pd.to_numeric( - historic["holdout_rmse"], errors="coerce" - ).dropna() + holdout_values = pd.to_numeric(historic["holdout_rmse"], errors="coerce").dropna() if not holdout_values.empty: current_holdout_rmse = float(holdout_values.iloc[-1]) - holdout_rmse = QuickAdapterRegressorV3.optuna_validate_value( - current_holdout_rmse - ) + holdout_rmse = QuickAdapterRegressorV3.optuna_validate_value(current_holdout_rmse) dk.data["extra_returns_per_train"]["holdout_rmse"] = ( holdout_rmse if holdout_rmse is not None else np.inf ) @@ -3430,9 +3230,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if self.live or pair_dataframe.empty: return pair_dataframe - history_dates = ensure_datetime_series( - self.dd.historic_predictions[pair]["date"] - ) + history_dates = ensure_datetime_series(self.dd.historic_predictions[pair]["date"]) if history_dates.empty: logger.debug( "[%s] Label HPO skipped: no prior predictions to bound the current FreqAI prediction time", @@ -3457,7 +3255,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): dk: FreqaiDataKitchen, pair: str, fit_live_predictions_candles: int, - ) -> Optional[optuna.study.Study]: + ) -> optuna.study.Study | None: label_dataframe = self._label_hpo_dataframe_as_of_prediction_time(dk, pair) if label_dataframe.empty: return None @@ -3478,7 +3276,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) @staticmethod - def optuna_validate_value(value: Any) -> Optional[float]: + def optuna_validate_value(value: Any) -> float | None: return value if isinstance(value, (int, float)) and np.isfinite(value) else None def min_max_pred( @@ -3487,15 +3285,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): col_prediction_config: dict[str, Any], pred_df: pd.DataFrame, fit_live_predictions_candles: int, - label_period_candles: Optional[int], + label_period_candles: int | None, ) -> tuple[float, float]: if label_period_candles is None or label_period_candles <= 0: label_period_candles = int( self.ft_params.get("label_period_candles", self._label_defaults[0]) ) thresholds_candles = ( - max(2, int(fit_live_predictions_candles / label_period_candles)) - * label_period_candles + max(2, int(fit_live_predictions_candles / label_period_candles)) * label_period_candles ) pred_label = pred_df.get(label_col) @@ -3542,7 +3339,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def _calculate_n_kept_extrema(size: int, keep_fraction: float) -> int: - return max(1, int(round(size * keep_fraction))) if size > 0 else 0 + return max(1, round(size * keep_fraction)) if size > 0 else 0 @staticmethod def _get_ranked_peaks( @@ -3559,16 +3356,12 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) pred_label_minima = ( - pred_label.loc[ - pred_label.iloc[minima_indices].nsmallest(n_kept_minima).index - ] + pred_label.loc[pred_label.iloc[minima_indices].nsmallest(n_kept_minima).index] if n_kept_minima > 0 else pd.Series(dtype=float) ) pred_label_maxima = ( - pred_label.loc[ - pred_label.iloc[maxima_indices].nlargest(n_kept_maxima).index - ] + pred_label.loc[pred_label.iloc[maxima_indices].nlargest(n_kept_maxima).index] if n_kept_maxima > 0 else pd.Series(dtype=float) ) @@ -3586,22 +3379,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): n_maxima: int, keep_fraction: float = 1.0, ) -> tuple[pd.Series, pd.Series]: - n_kept_minima = QuickAdapterRegressorV3._calculate_n_kept_extrema( - n_minima, keep_fraction - ) - n_kept_maxima = QuickAdapterRegressorV3._calculate_n_kept_extrema( - n_maxima, keep_fraction - ) + n_kept_minima = QuickAdapterRegressorV3._calculate_n_kept_extrema(n_minima, keep_fraction) + n_kept_maxima = QuickAdapterRegressorV3._calculate_n_kept_extrema(n_maxima, keep_fraction) pred_label_minima = ( - pred_label.nsmallest(n_kept_minima) - if n_kept_minima > 0 - else pd.Series(dtype=float) + pred_label.nsmallest(n_kept_minima) if n_kept_minima > 0 else pd.Series(dtype=float) ) pred_label_maxima = ( - pred_label.nlargest(n_kept_maxima) - if n_kept_maxima > 0 - else pd.Series(dtype=float) + pred_label.nlargest(n_kept_maxima) if n_kept_maxima > 0 else pd.Series(dtype=float) ) logger.debug( @@ -3616,35 +3401,27 @@ class QuickAdapterRegressorV3(BaseRegressionModel): selection_method: ExtremaSelectionMethod, keep_fraction: float = 1.0, ) -> tuple[pd.Series, pd.Series]: - pred_label = ( - pd.to_numeric(pred_label, errors="coerce") - .where(np.isfinite, np.nan) - .dropna() - ) + pred_label = pd.to_numeric(pred_label, errors="coerce").where(np.isfinite, np.nan).dropna() if pred_label.empty: return pd.Series(dtype=float), pd.Series(dtype=float) if selection_method == EXTREMA_SELECTION_METHODS[0]: # "rank_extrema" - minima_indices, maxima_indices = ( - QuickAdapterRegressorV3._get_extrema_indices(pred_label) - ) - pred_label_minima, pred_label_maxima = ( - QuickAdapterRegressorV3._get_ranked_extrema( - pred_label, - minima_indices.size, - maxima_indices.size, - keep_fraction, - ) + minima_indices, maxima_indices = QuickAdapterRegressorV3._get_extrema_indices( + pred_label + ) + pred_label_minima, pred_label_maxima = QuickAdapterRegressorV3._get_ranked_extrema( + pred_label, + minima_indices.size, + maxima_indices.size, + keep_fraction, ) elif selection_method == EXTREMA_SELECTION_METHODS[1]: # "rank_peaks" - minima_indices, maxima_indices = ( - QuickAdapterRegressorV3._get_extrema_indices(pred_label) + minima_indices, maxima_indices = QuickAdapterRegressorV3._get_extrema_indices( + pred_label ) - pred_label_minima, pred_label_maxima = ( - QuickAdapterRegressorV3._get_ranked_peaks( - pred_label, minima_indices, maxima_indices, keep_fraction - ) + pred_label_minima, pred_label_maxima = QuickAdapterRegressorV3._get_ranked_peaks( + pred_label, minima_indices, maxima_indices, keep_fraction ) elif selection_method == EXTREMA_SELECTION_METHODS[2]: # "partition" @@ -3654,9 +3431,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): pred_label_minima = pred_label[pred_label < -eps] else: raise ValueError( - enum_error_message( - "selection_method", selection_method, EXTREMA_SELECTION_METHODS - ) + enum_error_message("selection_method", selection_method, EXTREMA_SELECTION_METHODS) ) return pred_label_minima, pred_label_maxima @@ -3682,15 +3457,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): # ±2.0 fallbacks are out-of-[-1, 1] normalized-range sentinels. @staticmethod def safe_min_pred(pred_label: pd.Series) -> float: - return QuickAdapterRegressorV3._safe_pred( - pred_label, lambda series: series.min(), -2.0 - ) + return QuickAdapterRegressorV3._safe_pred(pred_label, lambda series: series.min(), -2.0) @staticmethod def safe_max_pred(pred_label: pd.Series) -> float: - return QuickAdapterRegressorV3._safe_pred( - pred_label, lambda series: series.max(), 2.0 - ) + return QuickAdapterRegressorV3._safe_pred(pred_label, lambda series: series.max(), 2.0) @staticmethod def _resolve_min_max( @@ -3718,9 +3489,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) soft_minimum = soft_extremum(pred_label_minima, alpha=-alpha) soft_maximum = soft_extremum(pred_label_maxima, alpha=alpha) - return QuickAdapterRegressorV3._resolve_min_max( - soft_minimum, soft_maximum, pred_label - ) + return QuickAdapterRegressorV3._resolve_min_max(soft_minimum, soft_maximum, pred_label) @staticmethod def median_min_max( @@ -3732,15 +3501,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): pred_label, selection_method, keep_fraction ) - if pred_label_minima.empty: - min_val = np.nan - else: - min_val = np.nanmedian(pred_label_minima.to_numpy()) + min_val = np.nan if pred_label_minima.empty else np.nanmedian(pred_label_minima.to_numpy()) - if pred_label_maxima.empty: - max_val = np.nan - else: - max_val = np.nanmedian(pred_label_maxima.to_numpy()) + max_val = np.nan if pred_label_maxima.empty else np.nanmedian(pred_label_maxima.to_numpy()) return QuickAdapterRegressorV3._resolve_min_max(min_val, max_val, pred_label) @@ -3759,10 +3522,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): threshold_func = getattr(skimage.filters, f"threshold_{method}") except AttributeError: raise ValueError( - enum_error_message( - "skimage threshold method", method, SKIMAGE_THRESHOLD_METHODS - ) - ) + enum_error_message("skimage threshold method", method, SKIMAGE_THRESHOLD_METHODS) + ) from None min_func = QuickAdapterRegressorV3.apply_skimage_threshold max_func = QuickAdapterRegressorV3.apply_skimage_threshold @@ -3780,11 +3541,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if values.size == 0: return np.nan - if ( - values.size == 1 - or np.unique(values).size < 3 - or np.allclose(values, values[0]) - ): + if values.size == 1 or np.unique(values).size < 3 or np.allclose(values, values[0]): return np.nanmedian(values) try: return threshold_func(values) @@ -3800,7 +3557,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): matrix: NDArray[np.floating], reference_point: NDArray[np.floating], *, - weights: Optional[NDArray[np.floating]] = None, + weights: NDArray[np.floating] | None = None, standardized: bool = False, ) -> NDArray[np.floating]: if standardized: @@ -3815,11 +3572,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): weights = np.ones(matrix.shape[1]) return ( - np.sqrt( - np.nansum( - weights * (np.sqrt(matrix) - np.sqrt(reference_point)) ** 2, axis=1 - ) - ) + np.sqrt(np.nansum(weights * (np.sqrt(matrix) - np.sqrt(reference_point)) ** 2, axis=1)) / QuickAdapterRegressorV3._SQRT_2 ) @@ -3829,18 +3582,15 @@ class QuickAdapterRegressorV3(BaseRegressionModel): reference_point: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, mode: ValidationMode = "none", p_ctx: str = "p", ) -> NDArray[np.floating]: power = ( QuickAdapterRegressorV3._POWER_MEAN_MAP[distance_metric] if distance_metric in QuickAdapterRegressorV3._POWER_MEAN_METRICS_SET - else ( - QuickAdapterRegressorV3._validate_power_mean_p(p, ctx=p_ctx, mode=mode) - or 1.0 - ) + else (QuickAdapterRegressorV3._validate_power_mean_p(p, ctx=p_ctx, mode=mode) or 1.0) ) if weights is None: weights = np.ones(matrix.shape[1]) @@ -3867,10 +3617,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): distance_metric: str, *, weights: NDArray[np.floating], - p: Optional[float], + p: float | None, method: str, apply_abs: bool, - cdist_kwargs: Optional[dict[str, Any]] = None, + cdist_kwargs: dict[str, Any] | None = None, ) -> NDArray[np.floating]: if distance_metric in QuickAdapterRegressorV3._SCIPY_METRICS_SET: if cdist_kwargs is None: @@ -3897,9 +3647,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): normalized_matrix, reference_point, weights=weights, - standardized=( - distance_metric == QuickAdapterRegressorV3._METRIC_SHELLINGER - ), + standardized=(distance_metric == QuickAdapterRegressorV3._METRIC_SHELLINGER), ) if distance_metric in ( @@ -3936,8 +3684,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): normalized_matrix: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> NDArray[np.floating]: n_samples, n_objectives = normalized_matrix.shape @@ -3966,13 +3714,12 @@ class QuickAdapterRegressorV3(BaseRegressionModel): matrix: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> NDArray[np.floating]: if matrix.ndim != 2: raise ValueError( - f"Invalid matrix (shape={matrix.shape}, ndim={matrix.ndim}): " - f"must be 2-dimensional" + f"Invalid matrix (shape={matrix.shape}, ndim={matrix.ndim}): must be 2-dimensional" ) if matrix.shape[1] == 0: raise ValueError( @@ -3980,9 +3727,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) if not np.all(np.isfinite(matrix)): - raise ValueError( - "Invalid matrix: must contain only finite values (no NaN or inf)" - ) + raise ValueError("Invalid matrix: must contain only finite values (no NaN or inf)") n = matrix.shape[0] if n == 0: @@ -4016,8 +3761,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): normalized_matrix: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> NDArray[np.floating]: n_samples, n_objectives = normalized_matrix.shape @@ -4081,8 +3826,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ideal_point_2d: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> float: cdist_kwargs = QuickAdapterRegressorV3._prepare_distance_kwargs( distance_metric=distance_metric, @@ -4108,27 +3853,22 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ideal_point_2d: NDArray[np.floating], distance_metric: str, *, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> tuple[int, float]: if best_cluster_indices.size == 1: best_trial_index = best_cluster_indices[0] - best_trial_distance = ( - QuickAdapterRegressorV3._calculate_trial_distance_to_ideal( - normalized_matrix, - best_trial_index, - ideal_point_2d, - distance_metric, - weights=weights, - p=p, - ) + best_trial_distance = QuickAdapterRegressorV3._calculate_trial_distance_to_ideal( + normalized_matrix, + best_trial_index, + ideal_point_2d, + distance_metric, + weights=weights, + p=p, ) return best_trial_index, best_trial_distance - if ( - trial_selection_method - == QuickAdapterRegressorV3._METHOD_COMPROMISE_PROGRAMMING - ): + if trial_selection_method == QuickAdapterRegressorV3._METHOD_COMPROMISE_PROGRAMMING: scores = QuickAdapterRegressorV3._compromise_programming_scores( normalized_matrix[best_cluster_indices], distance_metric, @@ -4153,15 +3893,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): min_score_position = np.nanargmin(scores) best_trial_index = best_cluster_indices[min_score_position] - best_trial_distance = ( - QuickAdapterRegressorV3._calculate_trial_distance_to_ideal( - normalized_matrix, - best_trial_index, - ideal_point_2d, - distance_metric, - weights=weights, - p=p, - ) + best_trial_distance = QuickAdapterRegressorV3._calculate_trial_distance_to_ideal( + normalized_matrix, + best_trial_index, + ideal_point_2d, + distance_metric, + weights=weights, + p=p, ) return best_trial_index, best_trial_distance @@ -4173,8 +3911,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): distance_metric: str, selection_method: DistanceMethod, trial_selection_method: DistanceMethod, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, ) -> NDArray[np.floating]: n_samples, n_objectives = normalized_matrix.shape @@ -4192,9 +3930,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): QuickAdapterRegressorV3._CLUSTER_KMEANS2, }: if cluster_method == QuickAdapterRegressorV3._CLUSTER_KMEANS: - kmeans = sklearn.cluster.KMeans( - n_clusters=n_clusters, random_state=42, n_init=10 - ) + kmeans = sklearn.cluster.KMeans(n_clusters=n_clusters, random_state=42, n_init=10) cluster_labels = kmeans.fit_predict(normalized_matrix) cluster_centers = kmeans.cluster_centers_ else: # kmeans2 @@ -4202,16 +3938,11 @@ class QuickAdapterRegressorV3(BaseRegressionModel): normalized_matrix, n_clusters, rng=42, minit="++" ) - if ( - selection_method - == QuickAdapterRegressorV3._METHOD_COMPROMISE_PROGRAMMING - ): - cluster_center_scores = ( - QuickAdapterRegressorV3._compromise_programming_scores( - cluster_centers, - distance_metric, - p=p, - ) + if selection_method == QuickAdapterRegressorV3._METHOD_COMPROMISE_PROGRAMMING: + cluster_center_scores = QuickAdapterRegressorV3._compromise_programming_scores( + cluster_centers, + distance_metric, + p=p, ) elif selection_method == QuickAdapterRegressorV3._METHOD_TOPSIS: cluster_center_scores = QuickAdapterRegressorV3._topsis_scores( @@ -4238,16 +3969,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): trial_distances = np.full(n_samples, np.inf) if best_cluster_indices is not None and best_cluster_indices.size > 0: - best_trial_index, best_trial_distance = ( - self._select_best_trial_from_cluster( - normalized_matrix, - trial_selection_method, - best_cluster_indices, - ideal_point_2d, - distance_metric, - weights=weights, - p=p, - ) + best_trial_index, best_trial_distance = self._select_best_trial_from_cluster( + normalized_matrix, + trial_selection_method, + best_cluster_indices, + ideal_point_2d, + distance_metric, + weights=weights, + p=p, ) trial_distances[best_trial_index] = best_trial_distance return trial_distances @@ -4276,9 +4005,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): *, distance_metric: str, n_neighbors: int, - weights: Optional[NDArray[np.floating]] = None, - p: Optional[float] = None, - aggregation_param: Optional[float] = None, + weights: NDArray[np.floating] | None = None, + p: float | None = None, + aggregation_param: float | None = None, ) -> NDArray[np.floating]: n_samples, _ = normalized_matrix.shape @@ -4478,7 +4207,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): candidates = [ (float(distance), trial.number, trial) - for trial, distance in zip(trials, distances) + for trial, distance in zip(trials, distances, strict=False) if np.isfinite(distance) ] if not candidates: @@ -4507,15 +4236,15 @@ class QuickAdapterRegressorV3(BaseRegressionModel): lower_bound = min(min_n_clusters, upper_bound) if n_uniques <= 3: return min(n_uniques, upper_bound) - n_clusters = int(round((np.log2(n_uniques) + np.sqrt(n_uniques)) / 2.0)) + n_clusters = round((np.log2(n_uniques) + np.sqrt(n_uniques)) / 2.0) return min(max(lower_bound, n_clusters), upper_bound) def _calculate_distances( self, normalized_matrix: NDArray[np.floating], selection_method: SelectionMethod, - objective_indices: Optional[NDArray[np.intp]] = None, - original_n_objectives: Optional[int] = None, + objective_indices: NDArray[np.intp] | None = None, + original_n_objectives: int | None = None, ) -> NDArray[np.floating]: if normalized_matrix.ndim != 2: raise ValueError( @@ -4588,15 +4317,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): mode="raise", ) - if n_samples == 1: - if method in { - QuickAdapterRegressorV3._SELECTION_MEDOID, - QuickAdapterRegressorV3._SELECTION_KMEANS, - QuickAdapterRegressorV3._SELECTION_KMEANS2, - QuickAdapterRegressorV3._SELECTION_KMEDOIDS, - QuickAdapterRegressorV3._SELECTION_KNN, - }: - return np.array([0.0]) + if n_samples == 1 and method in { + QuickAdapterRegressorV3._SELECTION_MEDOID, + QuickAdapterRegressorV3._SELECTION_KMEANS, + QuickAdapterRegressorV3._SELECTION_KMEANS2, + QuickAdapterRegressorV3._SELECTION_KMEDOIDS, + QuickAdapterRegressorV3._SELECTION_KNN, + }: + return np.array([0.0]) if category == "distance": distance_metric = label_config["distance_metric"] @@ -4644,7 +4372,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) if category == "density": - density_method = cast(DensityMethod, method) + density_method = cast("DensityMethod", method) density_metric = label_config["distance_metric"] p = QuickAdapterRegressorV3._resolve_p_order( density_metric, @@ -4655,11 +4383,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if density_method == QuickAdapterRegressorV3._DENSITY_KNN: knn_n_neighbors = int(label_config["n_neighbors"]) - knn_aggregation = cast(DensityAggregation, label_config["aggregation"]) - if ( - knn_aggregation - not in QuickAdapterRegressorV3._DENSITY_AGGREGATIONS_SET - ): + knn_aggregation = cast("DensityAggregation", label_config["aggregation"]) + if knn_aggregation not in QuickAdapterRegressorV3._DENSITY_AGGREGATIONS_SET: raise ValueError( f"Invalid aggregation value in label_config {knn_aggregation!r}: " f"supported values are {', '.join(QuickAdapterRegressorV3._DENSITY_AGGREGATIONS)}" @@ -4693,7 +4418,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): def _get_multi_objective_study_best_trial( self, namespace: OptunaNamespace, study: optuna.study.Study - ) -> Optional[optuna.trial.FrozenTrial]: + ) -> optuna.trial.FrozenTrial | None: if namespace not in {_OPTUNA_NAMESPACES.label}: raise ValueError( enum_error_message("namespace", namespace, (_OPTUNA_NAMESPACES.label,)) @@ -4717,8 +4442,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): isinstance(trial.values, list) and len(trial.values) == n_objectives and all( - isinstance(value, (int, float)) - and (np.isfinite(value) or np.isinf(value)) + isinstance(value, (int, float)) and (np.isfinite(value) or np.isinf(value)) for value in trial.values ) ) @@ -4726,15 +4450,13 @@ class QuickAdapterRegressorV3(BaseRegressionModel): if not best_trials: return None - objective_values_matrix = np.array( - [trial.values for trial in best_trials], dtype=float - ) + objective_values_matrix = np.array([trial.values for trial in best_trials], dtype=float) normalized_matrix = QuickAdapterRegressorV3._normalize_objective_values( objective_values_matrix, study.directions ) original_n_objectives = normalized_matrix.shape[1] - non_constant_objective_indices = ( - QuickAdapterRegressorV3._non_constant_objective_indices(normalized_matrix) + non_constant_objective_indices = QuickAdapterRegressorV3._non_constant_objective_indices( + normalized_matrix ) if non_constant_objective_indices.size == 0: return QuickAdapterRegressorV3._select_lowest_number_trial(best_trials) @@ -4747,31 +4469,23 @@ class QuickAdapterRegressorV3(BaseRegressionModel): original_n_objectives=original_n_objectives, ) - return QuickAdapterRegressorV3._select_best_trial_by_distance( - best_trials, trial_distances - ) + return QuickAdapterRegressorV3._select_best_trial_by_distance(best_trials, trial_distances) def optuna_optimize( self, pair: str, namespace: OptunaNamespace, objective: ObjectiveFuncType, - direction: Optional[optuna.study.StudyDirection] = None, - directions: Optional[list[optuna.study.StudyDirection]] = None, - ) -> Optional[optuna.study.Study]: + direction: optuna.study.StudyDirection | None = None, + directions: list[optuna.study.StudyDirection] | None = None, + ) -> optuna.study.Study | None: if direction is not None and directions is not None: raise ValueError( "Cannot specify both 'direction' and 'directions'. Use one or the other" ) is_study_single_objective = direction is not None and directions is None - if ( - not is_study_single_objective - and isinstance(directions, list) - and len(directions) < 2 - ): - raise ValueError( - "Multi-objective study must have at least 2 objectives specified" - ) + if not is_study_single_objective and isinstance(directions, list) and len(directions) < 2: + raise ValueError("Multi-objective study must have at least 2 objectives specified") study = self.optuna_create_study( pair=pair, @@ -4786,9 +4500,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self.optuna_enqueue_previous_best_params(pair, namespace, study) objective_type = "single" if is_study_single_objective else "multi" - logger.info( - f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt started" - ) + logger.info(f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt started") start_time = time.time() try: study.optimize( @@ -4798,11 +4510,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): timeout=self._optuna_config["timeout"], gc_after_trial=True, ) - except Exception as e: + except Exception: time_spent = time.time() - start_time - logger.error( - f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt failed ({time_spent:.2f} secs): {e!r}", - exc_info=True, + logger.exception( + f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt failed ({time_spent:.2f} secs)" ) return @@ -4822,13 +4533,10 @@ class QuickAdapterRegressorV3(BaseRegressionModel): metric_log_msg = "" else: try: - best_trial = self._get_multi_objective_study_best_trial( - namespace, study - ) - except Exception as e: - logger.error( - f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt failed ({time_spent:.2f} secs): {e!r}", - exc_info=True, + best_trial = self._get_multi_objective_study_best_trial(namespace, study) + except Exception: + logger.exception( + f"[{pair}] Optuna {namespace} {objective_type} objective hyperopt failed ({time_spent:.2f} secs)" ) best_trial = None if not best_trial: @@ -4843,9 +4551,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): **self.get_optuna_params(pair, namespace), } label_config = self._resolve_label_method_config( - self.ft_params.get( - "label_method", QuickAdapterRegressorV3.LABEL_METHOD_DEFAULT - ) + self.ft_params.get("label_method", QuickAdapterRegressorV3.LABEL_METHOD_DEFAULT) ) metric_log_msg = f" ({format_dict(label_config, style='params')})" logger.info( @@ -4875,9 +4581,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return study @staticmethod - def _optuna_quarantine_journal( - journal_path: Path, pair: str, cause: Exception - ) -> Optional[Path]: + def _optuna_quarantine_journal(journal_path: Path, pair: str, cause: Exception) -> Path | None: """Atomically move a corrupt Optuna journal aside. Return the quarantine path on success, or ``None`` if @@ -4896,12 +4600,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) try: journal_path.rename(quarantine_path) - except OSError as rename_exc: - logger.error( - f"[{pair}] Optuna journal {journal_path.name} " - f"quarantine failed: {rename_exc!r}", - exc_info=True, - ) + except OSError: + logger.exception(f"[{pair}] Optuna journal {journal_path.name} quarantine failed") raise logger.warning( f"[{pair}] Optuna journal {journal_path.name} corrupt ({cause!r}); " @@ -4972,9 +4672,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): QuickAdapterRegressorV3._optuna_quarantine_journal( journal_path, pair, - ValueError( - "trailing journal record is truncated or malformed JSON" - ), + ValueError("trailing journal record is truncated or malformed JSON"), ) def _build_journal_storage() -> JournalStorage: @@ -4995,9 +4693,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): storage = optuna.storages.RDBStorage( url=f"sqlite:///{storage_dir}/{storage_filename}.sqlite", heartbeat_interval=60, - failed_trial_callback=optuna.storages.RetryFailedTrialCallback( - max_retry=3 - ), + failed_trial_callback=optuna.storages.RetryFailedTrialCallback(max_retry=3), ) else: raise ValueError( @@ -5009,18 +4705,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) return storage - def optuna_create_pruner( - self, is_single_objective: bool - ) -> optuna.pruners.BasePruner: + def optuna_create_pruner(self, is_single_objective: bool) -> optuna.pruners.BasePruner: if is_single_objective: - return optuna.pruners.HyperbandPruner( - min_resource=self._optuna_config["min_resource"] - ) + return optuna.pruners.HyperbandPruner(min_resource=self._optuna_config["min_resource"]) else: return optuna.pruners.NopPruner() def optuna_create_sampler( - self, sampler: Optional[OptunaSampler] = None + self, sampler: OptunaSampler | None = None ) -> optuna.samplers.BaseSampler: if sampler is None: sampler = self._optuna_config.get( @@ -5072,16 +4764,12 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self._optuna_config["label_sampler"], ) else: - raise ValueError( - enum_error_message("namespace", namespace, _OPTUNA_NAMESPACES) - ) + raise ValueError(enum_error_message("namespace", namespace, _OPTUNA_NAMESPACES)) @staticmethod def _optuna_label_selection_metadata_compatible(existing_marker: Any) -> bool: schema_version = ( - existing_marker.get("schema_version") - if isinstance(existing_marker, dict) - else None + existing_marker.get("schema_version") if isinstance(existing_marker, dict) else None ) return ( not isinstance(schema_version, bool) @@ -5089,9 +4777,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): and schema_version == _OPTUNA_LABEL_SELECTION_SCHEMA_VERSION ) - def _optuna_study_marker( - self, namespace: OptunaNamespace - ) -> Optional[_OptunaStudyMarker]: + def _optuna_study_marker(self, namespace: OptunaNamespace) -> _OptunaStudyMarker | None: if namespace == _OPTUNA_NAMESPACES.hp: identity = QuickAdapterRegressorV3._OPTUNA_HP_OBJECTIVE_IDENTITY # ``hp`` always resets on identity mismatch: a changed objective @@ -5109,12 +4795,8 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return _OptunaStudyMarker( user_attr_key="selection_metadata", build_marker=self._optuna_label_selection_metadata, - is_compatible=( - QuickAdapterRegressorV3._optuna_label_selection_metadata_compatible - ), - reset_on_mismatch=bool( - self._optuna_config["reset_label_study_on_schema_mismatch"] - ), + is_compatible=(QuickAdapterRegressorV3._optuna_label_selection_metadata_compatible), + reset_on_mismatch=bool(self._optuna_config["reset_label_study_on_schema_mismatch"]), ) return None @@ -5122,20 +4804,17 @@ class QuickAdapterRegressorV3(BaseRegressionModel): self, pair: str, namespace: OptunaNamespace, - direction: Optional[optuna.study.StudyDirection] = None, - directions: Optional[list[optuna.study.StudyDirection]] = None, - ) -> Optional[optuna.study.Study]: + direction: optuna.study.StudyDirection | None = None, + directions: list[optuna.study.StudyDirection] | None = None, + ) -> optuna.study.Study | None: if direction is not None and directions is not None: raise ValueError( "Cannot specify both 'direction' and 'directions'. Use one or the other" ) is_study_single_objective = direction is not None and directions is None - if not is_study_single_objective: - if directions is None or len(directions) < 2: - raise ValueError( - "Multi-objective study must have at least 2 objectives specified" - ) + if not is_study_single_objective and (directions is None or len(directions) < 2): + raise ValueError("Multi-objective study must have at least 2 objectives specified") identifier = self.freqai_info.get("identifier") study_name = f"{identifier}-{pair}-{namespace}" @@ -5143,10 +4822,9 @@ class QuickAdapterRegressorV3(BaseRegressionModel): try: storage = self.optuna_create_storage(pair) - except Exception as e: - logger.error( - f"[{pair}] Optuna {namespace} storage creation failed for study {study_name}: {e!r}", - exc_info=True, + except Exception: + logger.exception( + f"[{pair}] Optuna {namespace} storage creation failed for study {study_name}" ) return None @@ -5157,28 +4835,21 @@ class QuickAdapterRegressorV3(BaseRegressionModel): continuous = self._optuna_config.get("continuous") or not self.live study_marker_mismatch_preserved = False if continuous: - QuickAdapterRegressorV3.optuna_delete_study( - pair, namespace, study_name, storage - ) + QuickAdapterRegressorV3.optuna_delete_study(pair, namespace, study_name, storage) elif study_marker is not None: try: - existing_study = QuickAdapterRegressorV3.optuna_load_study( - study_name, storage - ) + existing_study = QuickAdapterRegressorV3.optuna_load_study(study_name, storage) existing_marker = ( existing_study.user_attrs.get(study_marker.user_attr_key) if existing_study is not None else None ) - except Exception as e: - logger.error( - f"[{pair}] Optuna {namespace} study {study_name} inspection failed: {e!r}", - exc_info=True, + except Exception: + logger.exception( + f"[{pair}] Optuna {namespace} study {study_name} inspection failed" ) return None - if existing_study is not None and not study_marker.is_compatible( - existing_marker - ): + if existing_study is not None and not study_marker.is_compatible(existing_marker): reset_study = study_marker.reset_on_mismatch logger.warning( f"[{pair}] Optuna {namespace} study {study_name}: " @@ -5197,9 +4868,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): samplers, sampler = self.optuna_samplers_by_namespace(namespace) if sampler not in samplers: raise ValueError( - enum_error_message( - f"optuna {namespace} sampler", sampler, tuple(samplers) - ) + enum_error_message(f"optuna {namespace} sampler", sampler, tuple(samplers)) ) try: @@ -5225,15 +4894,14 @@ class QuickAdapterRegressorV3(BaseRegressionModel): ) study.set_user_attr(study_marker.user_attr_key, target_marker) return study - except Exception as e: - logger.error( - f"[{pair}] Optuna {namespace} study creation failed for study {study_name}: {e!r}", - exc_info=True, + except Exception: + logger.exception( + f"[{pair}] Optuna {namespace} study creation failed for study {study_name}" ) return None def optuna_validate_params( - self, pair: str, namespace: OptunaNamespace, study: Optional[optuna.study.Study] + self, pair: str, namespace: OptunaNamespace, study: optuna.study.Study | None ) -> bool: if not study: return False @@ -5253,7 +4921,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return QuickAdapterRegressorV3.optuna_validate_value(best_value) is not None def optuna_enqueue_previous_best_params( - self, pair: str, namespace: OptunaNamespace, study: Optional[optuna.study.Study] + self, pair: str, namespace: OptunaNamespace, study: optuna.study.Study | None ) -> None: if not study: return @@ -5288,7 +4956,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): def optuna_load_best_params( self, pair: str, namespace: OptunaNamespace - ) -> Optional[dict[str, Any]]: + ) -> dict[str, Any] | None: expected = ( self._optuna_label_selection_metadata() if namespace == _OPTUNA_NAMESPACES.label @@ -5335,7 +5003,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): @staticmethod def optuna_load_study( study_name: str, storage: optuna.storages.BaseStorage - ) -> Optional[optuna.study.Study]: + ) -> optuna.study.Study | None: try: study = optuna.load_study(study_name=study_name, storage=storage) except KeyError: @@ -5343,7 +5011,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return study @staticmethod - def optuna_study_has_best_trial(study: Optional[optuna.study.Study]) -> bool: + def optuna_study_has_best_trial(study: optuna.study.Study | None) -> bool: if not study: return False try: @@ -5353,7 +5021,7 @@ class QuickAdapterRegressorV3(BaseRegressionModel): return False @staticmethod - def optuna_study_has_best_trials(study: Optional[optuna.study.Study]) -> bool: + def optuna_study_has_best_trials(study: optuna.study.Study | None) -> bool: if not study: return False try: @@ -5377,7 +5045,7 @@ def hp_objective( model_training_parameters: dict[str, Any], space_reduction: bool, space_fraction: float, - model_path: Optional[Path] = None, + model_path: Path | None = None, init_model: Any = None, vary_model_seed_by_trial: bool = True, ) -> float: @@ -5450,10 +5118,7 @@ def label_objective( ) df = df.iloc[ - -( - max(2, int(fit_live_predictions_candles / label_period_candles)) - * label_period_candles - ) : + -(max(2, int(fit_live_predictions_candles / label_period_candles)) * label_period_candles) : ] if df.empty: @@ -5494,9 +5159,7 @@ def label_objective( if not np.isfinite(median_speed): median_speed = 0.0 - median_efficiency_ratio = np.nanmedian( - np.asarray(pivots_efficiency_ratios, dtype=float) - ) + median_efficiency_ratio = np.nanmedian(np.asarray(pivots_efficiency_ratios, dtype=float)) if not np.isfinite(median_efficiency_ratio): median_efficiency_ratio = 0.0 diff --git a/quickadapter/user_data/strategies/LabelTransformer.py b/quickadapter/user_data/strategies/LabelTransformer.py index 2b2415c..a6d3527 100644 --- a/quickadapter/user_data/strategies/LabelTransformer.py +++ b/quickadapter/user_data/strategies/LabelTransformer.py @@ -12,6 +12,7 @@ from datasieve.transforms.base_transform import ( BaseTransform, ListOrNone, ) +from EnumErrors import enum_error_message from numpy.typing import ArrayLike, NDArray from sklearn.preprocessing import ( MaxAbsScaler, @@ -20,7 +21,6 @@ from sklearn.preprocessing import ( RobustScaler, StandardScaler, ) -from EnumErrors import enum_error_message logger = logging.getLogger(__name__) @@ -209,9 +209,7 @@ EXTREMA_SELECTION_METHODS: Final[tuple[ExtremaSelectionMethod, ...]] = ( "partition", ) -SkimageThresholdMethod = Literal[ - "mean", "isodata", "li", "minimum", "otsu", "triangle", "yen" -] +SkimageThresholdMethod = Literal["mean", "isodata", "li", "minimum", "otsu", "triangle", "yen"] SKIMAGE_THRESHOLD_METHODS: Final[tuple[SkimageThresholdMethod, ...]] = ( "mean", "isodata", @@ -286,9 +284,7 @@ class _ColumnState: @dataclass class _LabelTransformerConfig: - default: dict[str, Any] = field( - default_factory=lambda: DEFAULTS_LABEL_PIPELINE.copy() - ) + default: dict[str, Any] = field(default_factory=lambda: DEFAULTS_LABEL_PIPELINE.copy()) columns: dict[str, dict[str, Any]] = field(default_factory=dict) @classmethod @@ -482,9 +478,7 @@ class LabelTransformer(BaseTransform): inverse=inverse, ) - def _fit_standardization( - self, values: NDArray[np.floating], state: _ColumnState - ) -> None: + def _fit_standardization(self, values: NDArray[np.floating], state: _ColumnState) -> None: method = state.config["standardization"] if method == STANDARDIZATION_TYPES[0]: # none return @@ -500,45 +494,31 @@ class LabelTransformer(BaseTransform): if method == STANDARDIZATION_TYPES[3]: # mmad state.median = float(np.median(values)) mad = np.median(np.abs(values - state.median)) - state.mad = ( - float(mad) if np.isfinite(mad) and not np.isclose(mad, 0.0) else 1.0 - ) + state.mad = float(mad) if np.isfinite(mad) and not np.isclose(mad, 0.0) else 1.0 return if method == STANDARDIZATION_TYPES[4]: # power_yj - state.power_transformer = PowerTransformer( - method="yeo-johnson", standardize=True - ) + state.power_transformer = PowerTransformer(method="yeo-johnson", standardize=True) state.power_transformer.fit(values.reshape(-1, 1)) return - raise ValueError( - enum_error_message("standardization", method, STANDARDIZATION_TYPES) - ) + raise ValueError(enum_error_message("standardization", method, STANDARDIZATION_TYPES)) - def _fit_normalization( - self, values: NDArray[np.floating], state: _ColumnState - ) -> None: + def _fit_normalization(self, values: NDArray[np.floating], state: _ColumnState) -> None: method = state.config["normalization"] if method == NORMALIZATION_TYPES[0]: # maxabs state.maxabs_scaler = MaxAbsScaler() state.maxabs_scaler.fit(values.reshape(-1, 1)) return if method == NORMALIZATION_TYPES[1]: # minmax - state.minmax_scaler = MinMaxScaler( - feature_range=state.config["minmax_range"] - ) + state.minmax_scaler = MinMaxScaler(feature_range=state.config["minmax_range"]) state.minmax_scaler.fit(values.reshape(-1, 1)) return if method in (NORMALIZATION_TYPES[2], NORMALIZATION_TYPES[3]): # sigmoid, none return - raise ValueError( - enum_error_message("normalization", method, NORMALIZATION_TYPES) - ) + raise ValueError(enum_error_message("normalization", method, NORMALIZATION_TYPES)) - def _fit_column( - self, column_name: str, values: NDArray[np.floating] - ) -> _ColumnState: + def _fit_column(self, column_name: str, values: NDArray[np.floating]) -> _ColumnState: config = self._config.get_column_config(column_name) state = _ColumnState(config=config) diff --git a/quickadapter/user_data/strategies/QuickAdapterV3.py b/quickadapter/user_data/strategies/QuickAdapterV3.py index 13670ab..c36886f 100644 --- a/quickadapter/user_data/strategies/QuickAdapterV3.py +++ b/quickadapter/user_data/strategies/QuickAdapterV3.py @@ -2,15 +2,14 @@ import datetime import hashlib import logging import math +from collections.abc import Callable from functools import cached_property, lru_cache, reduce from pathlib import Path from typing import ( Any, - Callable, ClassVar, Final, Literal, - Optional, TypedDict, ) @@ -28,8 +27,8 @@ from freqtrade.strategy.interface import IStrategy from LabelTransformer import ( COMBINED_AGGREGATIONS, FILL_METHODS, - SMOOTHING_METHODS, SMOOTHING_METHOD_MODES, + SMOOTHING_METHODS, SMOOTHING_MODES, WEIGHT_STRATEGIES, get_label_column_config, @@ -62,8 +61,8 @@ from Utils import ( generate_label_data, get_callable_sha256, get_causal_mode, - get_distance, get_custom_protections_config, + get_distance, get_exit_pricing_config, get_fit_live_predictions_candles, get_label_defaults, @@ -95,10 +94,8 @@ InterpolationDirection = Literal["direct", "inverse"] OrderType = Literal["entry", "exit"] TradingMode = Literal["spot", "margin", "futures"] -DfSignature = tuple[int, Optional[datetime.datetime]] -CandleDeviationCacheKey = tuple[ - str, DfSignature, float, float, int, InterpolationDirection, float -] +DfSignature = tuple[int, datetime.datetime | None] +CandleDeviationCacheKey = tuple[str, DfSignature, float, float, int, InterpolationDirection, float] CandleThresholdCacheKey = tuple[str, DfSignature, str, int, float, float] @@ -139,9 +136,7 @@ class QuickAdapterV3(IStrategy): _TRADE_DIRECTIONS: Final[tuple[TradeDirection, ...]] = ("long", "short") _TRADE_LONG: Final[str] = _TRADE_DIRECTIONS[0] _TRADE_SHORT: Final[str] = _TRADE_DIRECTIONS[1] - _TRADE_DIRECTIONS_SET: Final[frozenset[TradeDirection]] = frozenset( - _TRADE_DIRECTIONS - ) + _TRADE_DIRECTIONS_SET: Final[frozenset[TradeDirection]] = frozenset(_TRADE_DIRECTIONS) _INTERPOLATION_DIRECTIONS: Final[tuple[InterpolationDirection, ...]] = ( "direct", "inverse", @@ -207,7 +202,7 @@ class QuickAdapterV3(IStrategy): _PARTIAL_EXIT_MIN_STAKE_MARGIN: Final[float] = 1e-3 # FreqAI is crashing if ``minimal_roi`` is a property - minimal_roi = {str(timeframe_minutes * 864): -1} + minimal_roi: ClassVar[dict[str, int]] = {str(timeframe_minutes * 864): -1} process_only_new_candles = True @@ -270,38 +265,32 @@ class QuickAdapterV3(IStrategy): return get_fit_live_predictions_candles(self.config.get("freqai"), logger) @staticmethod - def _is_unlimited_max_open_trades(max_open_trades: int | float) -> bool: + def _is_unlimited_max_open_trades(max_open_trades: float) -> bool: return max_open_trades == -1 or max_open_trades == math.inf @cached_property def protections(self) -> list[dict[str, Any]]: fit_live_predictions_candles = self._fit_live_predictions_candles - protections = get_custom_protections_config( - self.config.get("custom_protections"), logger - ) + protections = get_custom_protections_config(self.config.get("custom_protections"), logger) trade_duration_candles = protections["trade_duration_candles"] lookback_period_fraction = protections["lookback_period_fraction"] lookback_period_candles = max( - 1, int(round(fit_live_predictions_candles * lookback_period_fraction)) + 1, round(fit_live_predictions_candles * lookback_period_fraction) ) cooldown = protections["cooldown"] cooldown_stop_duration_candles = cooldown["stop_duration_candles"] - stoploss_stop_duration_candles = max( - cooldown_stop_duration_candles, trade_duration_candles - ) + stoploss_stop_duration_candles = max(cooldown_stop_duration_candles, trade_duration_candles) drawdown_stop_duration_candles = max( stoploss_stop_duration_candles, fit_live_predictions_candles, ) max_open_trades = self.config.get("max_open_trades", 0) - unlimited_max_open_trades = QuickAdapterV3._is_unlimited_max_open_trades( - max_open_trades - ) + unlimited_max_open_trades = QuickAdapterV3._is_unlimited_max_open_trades(max_open_trades) estimated_trade_limit = max( 2, - int(round(lookback_period_candles / max(1, trade_duration_candles))), + round(lookback_period_candles / max(1, trade_duration_candles)), ) if unlimited_max_open_trades: stoploss_trade_limit = estimated_trade_limit @@ -310,7 +299,7 @@ class QuickAdapterV3(IStrategy): max_open_trades = int(max_open_trades) stoploss_trade_limit = min( estimated_trade_limit, - max(2, int(round(max_open_trades * 0.75))), + max(2, round(max_open_trades * 0.75)), ) drawdown_trade_limit = 2 * max_open_trades @@ -371,15 +360,11 @@ class QuickAdapterV3(IStrategy): @cached_property def label_weighting(self) -> dict[str, Any]: - return get_label_weighting_config( - self.freqai_info.get("label_weighting"), logger - ) + return get_label_weighting_config(self.freqai_info.get("label_weighting"), logger) @cached_property def label_smoothing(self) -> dict[str, Any]: - return get_label_smoothing_config( - self.freqai_info.get("label_smoothing"), logger - ) + return get_label_smoothing_config(self.freqai_info.get("label_smoothing"), logger) @cached_property def exit_pricing(self) -> dict[str, str | float]: @@ -395,9 +380,7 @@ class QuickAdapterV3(IStrategy): @cached_property def reversal_confirmation(self) -> dict[str, int | float]: - return get_reversal_confirmation_config( - self.config.get("reversal_confirmation"), logger - ) + return get_reversal_confirmation_config(self.config.get("reversal_confirmation"), logger) @cached_property def _label_defaults(self) -> tuple[int, float]: @@ -418,9 +401,7 @@ class QuickAdapterV3(IStrategy): "Invalid freqai configuration: 'identifier' must be defined in freqai section" ) self.models_full_path = Path( - self.config.get("user_data_dir") - / "models" - / self.freqai_info.get("identifier") + self.config.get("user_data_dir") / "models" / self.freqai_info.get("identifier") ) feature_parameters = self.freqai_info.get("feature_parameters", {}) if get_causal_mode(feature_parameters, logger): @@ -437,9 +418,7 @@ class QuickAdapterV3(IStrategy): "label_smoothing.mode='wrap' is incompatible with " "feature_parameters.causal_mode=true" ) - default_label_period_candles, default_label_natr_multiplier = ( - self._label_defaults - ) + default_label_period_candles, default_label_natr_multiplier = self._label_defaults self._label_params: dict[str, dict[str, Any]] = {} load_persisted_label_params = self.is_trade_runmode for pair in self.pairs: @@ -465,10 +444,8 @@ class QuickAdapterV3(IStrategy): } ) self._candle_duration_secs = int(self.timeframe_minutes * 60) - self.last_candle_start_secs: dict[str, Optional[int]] = {} - self._max_take_profit_history_size = max( - 1, int(12 * 60 / self.timeframe_minutes) - ) + self.last_candle_start_secs: dict[str, int | None] = {} + self._max_take_profit_history_size = max(1, int(12 * 60 / self.timeframe_minutes)) self._candle_deviation_cache: dict[CandleDeviationCacheKey, float] = {} self._candle_threshold_cache: dict[CandleThresholdCacheKey, float] = {} self._cached_df_signature: dict[str, DfSignature] = {} @@ -504,12 +481,8 @@ class QuickAdapterV3(IStrategy): QuickAdapterV3._FILL_EPSILON, QuickAdapterV3._FILL_EPSILON_GAUSSIAN, ): - logger.info( - f" fill_epsilon: {format_number(col_weighting['fill_epsilon'])}" - ) - logger.info( - f" fill_epsilon_baseline: {col_weighting['fill_epsilon_baseline']}" - ) + logger.info(f" fill_epsilon: {format_number(col_weighting['fill_epsilon'])}") + logger.info(f" fill_epsilon_baseline: {col_weighting['fill_epsilon_baseline']}") if fill_method in ( QuickAdapterV3._FILL_GAUSSIAN, QuickAdapterV3._FILL_EPSILON_GAUSSIAN, @@ -552,10 +525,7 @@ class QuickAdapterV3(IStrategy): method = col_smoothing["method"] if col_weighting["strategy"] != QuickAdapterV3._WEIGHT_NONE and ( method == QuickAdapterV3._SMOOTHING_SMM - or ( - method == QuickAdapterV3._SMOOTHING_SAVGOL - and col_smoothing["polyorder"] >= 2 - ) + or (method == QuickAdapterV3._SMOOTHING_SAVGOL and col_smoothing["polyorder"] >= 2) ): logger.warning( f" Label [{label_col}]: smoothing method {method!r} can " @@ -613,12 +583,8 @@ class QuickAdapterV3(IStrategy): if self.protections: for protection in self.protections: method = protection.get("method", "Unknown") - protection_params = { - k: v for k, v in protection.items() if k != "method" - } - logger.info( - f" {method}: {format_dict(protection_params, style='dict')}" - ) + protection_params = {k: v for k, v in protection.items() if k != "method"} + logger.info(f" {method}: {format_dict(protection_params, style='dict')}") else: logger.info(" No protections enabled") @@ -674,9 +640,7 @@ class QuickAdapterV3(IStrategy): closes, length=period, ) - dataframe["%-linearreg_angle-period"] = ta.LINEARREG_ANGLE( - dataframe, timeperiod=period - ) + dataframe["%-linearreg_angle-period"] = ta.LINEARREG_ANGLE(dataframe, timeperiod=period) dataframe["%-atr-period"] = ta.ATR(dataframe, timeperiod=period) dataframe["%-natr-period"] = ta.NATR(dataframe, timeperiod=period) return dataframe @@ -711,9 +675,7 @@ class QuickAdapterV3(IStrategy): dataframe["%-raw_volume"] = volumes dataframe["%-obv"] = ta.OBV(dataframe) label_period_candles = self.get_label_period_candles(str(metadata.get("pair"))) - dataframe["%-atr_label_period_candles"] = ta.ATR( - dataframe, timeperiod=label_period_candles - ) + dataframe["%-atr_label_period_candles"] = ta.ATR(dataframe, timeperiod=label_period_candles) dataframe["%-natr_label_period_candles"] = ta.NATR( dataframe, timeperiod=label_period_candles ) @@ -725,9 +687,7 @@ class QuickAdapterV3(IStrategy): normalize=True, logger=logger, ) - dataframe["%-diff_to_psar"] = closes - ta.SAR( - dataframe, acceleration=0.02, maximum=0.2 - ) + dataframe["%-diff_to_psar"] = closes - ta.SAR(dataframe, acceleration=0.02, maximum=0.2) kc = pta.kc( highs, lows, @@ -795,15 +755,9 @@ class QuickAdapterV3(IStrategy): context="feature_engineering_expand_basic:vwap_width", logger=logger, ) - dataframe["%-dist_to_vwap_upperband"] = get_distance( - closes, dataframe["vwap_upperband"] - ) - dataframe["%-dist_to_vwap_middleband"] = get_distance( - closes, dataframe["vwap_middleband"] - ) - dataframe["%-dist_to_vwap_lowerband"] = get_distance( - closes, dataframe["vwap_lowerband"] - ) + dataframe["%-dist_to_vwap_upperband"] = get_distance(closes, dataframe["vwap_upperband"]) + dataframe["%-dist_to_vwap_middleband"] = get_distance(closes, dataframe["vwap_middleband"]) + dataframe["%-dist_to_vwap_lowerband"] = get_distance(closes, dataframe["vwap_lowerband"]) dataframe["%-body"] = closes - opens dataframe["%-tail"] = (np.minimum(opens, closes) - lows).clip(lower=0) dataframe["%-wick"] = (highs - np.maximum(opens, closes)).clip(lower=0) @@ -838,7 +792,7 @@ class QuickAdapterV3(IStrategy): def get_label_period_candles( self, pair: str, - dataframe: Optional[DataFrame] = None, + dataframe: DataFrame | None = None, candle_idx: int = -1, ) -> int: if dataframe is not None: @@ -860,10 +814,7 @@ class QuickAdapterV3(IStrategy): def set_label_period_candles(self, pair: str, label_period_candles: Any) -> None: if is_finite_number(label_period_candles) and int(label_period_candles) > 0: label_period_candles = int(label_period_candles) - if ( - self._label_params[pair].get("label_period_candles") - != label_period_candles - ): + if self._label_params[pair].get("label_period_candles") != label_period_candles: self._label_params[pair]["label_period_candles"] = label_period_candles self._invalidate_pair_caches(pair) @@ -879,7 +830,7 @@ class QuickAdapterV3(IStrategy): def get_label_natr_multiplier( self, pair: str, - dataframe: Optional[DataFrame] = None, + dataframe: DataFrame | None = None, candle_idx: int = -1, ) -> float: if dataframe is not None: @@ -898,25 +849,17 @@ class QuickAdapterV3(IStrategy): ) def set_label_natr_multiplier(self, pair: str, label_natr_multiplier: Any) -> None: - if ( - is_finite_number(label_natr_multiplier) - and float(label_natr_multiplier) > 0.0 - ): + if is_finite_number(label_natr_multiplier) and float(label_natr_multiplier) > 0.0: label_natr_multiplier = float(label_natr_multiplier) - if ( - self._label_params[pair].get("label_natr_multiplier") - != label_natr_multiplier - ): - self._label_params[pair]["label_natr_multiplier"] = ( - label_natr_multiplier - ) + if self._label_params[pair].get("label_natr_multiplier") != label_natr_multiplier: + self._label_params[pair]["label_natr_multiplier"] = label_natr_multiplier self._invalidate_pair_caches(pair) def get_label_natr_multiplier_fraction( self, pair: str, fraction: float, - dataframe: Optional[DataFrame] = None, + dataframe: DataFrame | None = None, candle_idx: int = -1, ) -> float: if not isinstance(fraction, float) or not (0.0 <= fraction <= 1.0): @@ -951,22 +894,18 @@ class QuickAdapterV3(IStrategy): except (KeyError, ValueError) as e: raise ValueError( f"Invalid pattern value {pattern!r}: failed to format with {e!r}" - ) + ) from e def set_freqai_targets( self, dataframe: DataFrame, metadata: dict[str, Any], **kwargs ) -> DataFrame: pair = str(metadata.get("pair")) - series_duration = datetime.timedelta( - minutes=len(dataframe) * self.timeframe_minutes - ) + series_duration = datetime.timedelta(minutes=len(dataframe) * self.timeframe_minutes) label_weighting = self.label_weighting label_smoothing = self.label_smoothing series_length = len(dataframe) - causal_mode = get_causal_mode( - self.freqai_info.get("feature_parameters", {}), logger - ) + causal_mode = get_causal_mode(self.freqai_info.get("feature_parameters", {}), logger) finite_gaussian_support = causal_mode for label_col in LABEL_COLUMNS: @@ -1008,40 +947,32 @@ class QuickAdapterV3(IStrategy): weighting_config=col_weighting_config, finite_gaussian_support=finite_gaussian_support, logger=logger, - known_at_lookahead=( - label_data.known_at_lookahead if causal_mode else None - ), + known_at_lookahead=(label_data.known_at_lookahead if causal_mode else None), ) if label_data.known_at_lookahead is not None: if causal_mode: - imputation_masks = ( - compute_label_weight_imputation_dependency_mask( - len(label_data.indices), - label_data.metrics, - col_weighting_config, - ) + imputation_masks = compute_label_weight_imputation_dependency_mask( + len(label_data.indices), + label_data.metrics, + col_weighting_config, ) imputation_dependency_mask = imputation_masks.dependency_mask - imputation_leading_stable_mask = ( - imputation_masks.leading_stable_mask - ) - imputation_stable_release_index = ( - imputation_masks.stable_release_index - ) + imputation_leading_stable_mask = imputation_masks.leading_stable_mask + imputation_stable_release_index = imputation_masks.stable_release_index else: imputation_dependency_mask = None imputation_leading_stable_mask = None imputation_stable_release_index = -1 - dataframe[ - label_weight_known_at_lookahead_column_name(label_col) - ] = compute_label_weight_known_at_lookahead( - known_at_lookahead=label_data.known_at_lookahead, - indices=label_data.indices, - fill_radius=weight_fill_radius(col_weighting_config), - weighting_config=col_weighting_config, - imputation_dependency_mask=imputation_dependency_mask, - imputation_leading_stable_mask=imputation_leading_stable_mask, - imputation_stable_release_index=imputation_stable_release_index, + dataframe[label_weight_known_at_lookahead_column_name(label_col)] = ( + compute_label_weight_known_at_lookahead( + known_at_lookahead=label_data.known_at_lookahead, + indices=label_data.indices, + fill_radius=weight_fill_radius(col_weighting_config), + weighting_config=col_weighting_config, + imputation_dependency_mask=imputation_dependency_mask, + imputation_leading_stable_mask=imputation_leading_stable_mask, + imputation_stable_release_index=imputation_stable_release_index, + ) ) if label_col == EXTREMA_COLUMN: @@ -1055,9 +986,7 @@ class QuickAdapterV3(IStrategy): dataframe[label_col] = smooth(dataframe[label_col], **col_smoothing_config) if is_weighting_active: - smoothed_label_weights = smooth( - dataframe[label_weight_col], **col_smoothing_config - ) + smoothed_label_weights = smooth(dataframe[label_weight_col], **col_smoothing_config) dataframe[label_weight_col] = smoothed_label_weights.where( np.isfinite(smoothed_label_weights) & smoothed_label_weights.gt(0), 0.0, @@ -1081,15 +1010,11 @@ class QuickAdapterV3(IStrategy): if label_col == EXTREMA_COLUMN: dataframe[EXTREMA_DIRECTION_SMOOTHED_COLUMN] = dataframe[label_col] if is_weighting_active: - dataframe[EXTREMA_WEIGHT_SMOOTHED_COLUMN] = dataframe[ - label_weight_col - ] + dataframe[EXTREMA_WEIGHT_SMOOTHED_COLUMN] = dataframe[label_weight_col] return dataframe - def populate_indicators( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_indicators(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: dataframe = self.freqai.start(dataframe, metadata, self) di_values = dataframe.get("DI_values") @@ -1105,13 +1030,9 @@ class QuickAdapterV3(IStrategy): label_natr_multiplier_series = dataframe.get("label_natr_multiplier") if self.is_trade_runmode: if label_period_candles_series is not None: - self.set_label_period_candles( - pair, label_period_candles_series.iloc[-1] - ) + self.set_label_period_candles(pair, label_period_candles_series.iloc[-1]) if label_natr_multiplier_series is not None: - self.set_label_natr_multiplier( - pair, label_natr_multiplier_series.iloc[-1] - ) + self.set_label_natr_multiplier(pair, label_natr_multiplier_series.iloc[-1]) if label_period_candles_series is None: dataframe["natr_label_period_candles"] = ta.NATR( @@ -1129,22 +1050,16 @@ class QuickAdapterV3(IStrategy): for period in periods.unique(): period_rows = periods == period period_natr = ta.NATR(dataframe, timeperiod=int(period)) - dataframe.loc[period_rows, "natr_label_period_candles"] = ( - period_natr.loc[period_rows] - ) + dataframe.loc[period_rows, "natr_label_period_candles"] = period_natr.loc[ + period_rows + ] - dataframe["minima_threshold"] = dataframe.get( - f"{EXTREMA_COLUMN}_minima_threshold", np.nan - ) - dataframe["maxima_threshold"] = dataframe.get( - f"{EXTREMA_COLUMN}_maxima_threshold", np.nan - ) + dataframe["minima_threshold"] = dataframe.get(f"{EXTREMA_COLUMN}_minima_threshold", np.nan) + dataframe["maxima_threshold"] = dataframe.get(f"{EXTREMA_COLUMN}_maxima_threshold", np.nan) return dataframe - def populate_entry_trend( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_entry_trend(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: enter_long_conditions = [ dataframe.get("do_predict") == 1, dataframe.get("DI_catch") == 1, @@ -1167,15 +1082,13 @@ class QuickAdapterV3(IStrategy): return dataframe - def populate_exit_trend( - self, dataframe: DataFrame, metadata: dict[str, Any] - ) -> DataFrame: + def populate_exit_trend(self, dataframe: DataFrame, metadata: dict[str, Any]) -> DataFrame: return dataframe def get_trade_entry_date(self, trade: Trade) -> datetime.datetime: return timeframe_to_prev_date(self.config.get("timeframe"), trade.open_date_utc) - def get_trade_duration_candles(self, df: DataFrame, trade: Trade) -> Optional[int]: + def get_trade_duration_candles(self, df: DataFrame, trade: Trade) -> int | None: entry_date = self.get_trade_entry_date(trade) dates = df.get("date") if dates is None or dates.empty: @@ -1183,13 +1096,10 @@ class QuickAdapterV3(IStrategy): current_date = dates.iloc[-1] if isna(current_date): return None - return int( - ((current_date - entry_date).total_seconds() / 60.0) - / self.timeframe_minutes - ) + return int(((current_date - entry_date).total_seconds() / 60.0) / self.timeframe_minutes) def get_trade_annotation_line_start_date( - self, dataframe: DataFrame, trade: Trade, offset_candles: Optional[int] = None + self, dataframe: DataFrame, trade: Trade, offset_candles: int | None = None ) -> datetime.datetime: if offset_candles is None: offset_candles = QuickAdapterV3._ANNOTATION_LINE_OFFSET_CANDLES @@ -1198,8 +1108,7 @@ class QuickAdapterV3(IStrategy): offset_candles_remaining = max( 0, - offset_candles - - (trade_duration_candles if trade_duration_candles is not None else 0), + offset_candles - (trade_duration_candles if trade_duration_candles is not None else 0), ) offset_timedelta = datetime.timedelta( @@ -1210,14 +1119,14 @@ class QuickAdapterV3(IStrategy): @staticmethod @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) - def is_trade_duration_valid(trade_duration: Optional[int | float]) -> bool: + def is_trade_duration_valid(trade_duration: float | None) -> bool: return isinstance(trade_duration, (int, float)) and not ( isna(trade_duration) or trade_duration <= 0 ) def _trade_natr_window( self, df: DataFrame, trade: Trade - ) -> Optional[tuple[Any, float, Optional[float]]]: + ) -> tuple[Any, float, float | None] | None: label_natr = df.get("natr_label_period_candles") if label_natr is None or label_natr.empty: return None @@ -1239,9 +1148,7 @@ class QuickAdapterV3(IStrategy): return None return trade_label_natr, entry_natr, current_natr - def get_trade_weighted_average_natr( - self, df: DataFrame, trade: Trade - ) -> Optional[float]: + def get_trade_weighted_average_natr(self, df: DataFrame, trade: Trade) -> float | None: window = self._trade_natr_window(df, trade) if window is None: return None @@ -1266,8 +1173,7 @@ class QuickAdapterV3(IStrategy): ) -> float: return ( min_weight - + (max_weight - min_weight) - * (abs(quantile - 0.5) * 2.0) ** weighting_exponent + + (max_weight - min_weight) * (abs(quantile - 0.5) * 2.0) ** weighting_exponent ) entry_weight = calculate_weight(entry_quantile) @@ -1283,18 +1189,14 @@ class QuickAdapterV3(IStrategy): logger=logger, ) - def get_trade_quantile_interpolation_natr( - self, df: DataFrame, trade: Trade - ) -> Optional[float]: + def get_trade_quantile_interpolation_natr(self, df: DataFrame, trade: Trade) -> float | None: window = self._trade_natr_window(df, trade) if window is None: return None trade_label_natr, entry_natr, current_natr = window if current_natr is None: return entry_natr - trade_volatility_quantile = calculate_quantile( - trade_label_natr.to_numpy(), entry_natr - ) + trade_volatility_quantile = calculate_quantile(trade_label_natr.to_numpy(), entry_natr) if isna(trade_volatility_quantile): trade_volatility_quantile = 0.5 return np.interp( @@ -1305,7 +1207,7 @@ class QuickAdapterV3(IStrategy): def get_trade_moving_average_natr( self, df: DataFrame, pair: str, trade_duration_candles: int - ) -> Optional[float]: + ) -> float | None: if not QuickAdapterV3.is_trade_duration_valid(trade_duration_candles): return None label_natr = df.get("natr_label_period_candles") @@ -1317,9 +1219,7 @@ class QuickAdapterV3(IStrategy): trade_kama_natr_values = np.asarray( zl_kama(label_natr, timeperiod=trade_duration_candles), dtype=float ) - trade_kama_natr_values = trade_kama_natr_values[ - np.isfinite(trade_kama_natr_values) - ] + trade_kama_natr_values = trade_kama_natr_values[np.isfinite(trade_kama_natr_values)] if trade_kama_natr_values.size > 0: return trade_kama_natr_values[-1] except Exception as e: @@ -1331,20 +1231,16 @@ class QuickAdapterV3(IStrategy): def get_trade_natr( self, df: DataFrame, trade: Trade, trade_duration_candles: int - ) -> Optional[float]: - trade_natr_methods: dict[str, Callable[[], Optional[float]]] = { + ) -> float | None: + trade_natr_methods: dict[str, Callable[[], float | None]] = { # 0 - "moving_average" TRADE_NATR_METHODS[0]: lambda: self.get_trade_moving_average_natr( df, trade.pair, trade_duration_candles ), # 1 - "quantile_interpolation" - TRADE_NATR_METHODS[1]: lambda: self.get_trade_quantile_interpolation_natr( - df, trade - ), + TRADE_NATR_METHODS[1]: lambda: self.get_trade_quantile_interpolation_natr(df, trade), # 2 - "weighted_average" - TRADE_NATR_METHODS[2]: lambda: self.get_trade_weighted_average_natr( - df, trade - ), + TRADE_NATR_METHODS[2]: lambda: self.get_trade_weighted_average_natr(df, trade), } trade_natr_method_fn = trade_natr_methods.get(self.trade_natr_method) if trade_natr_method_fn is None: @@ -1362,9 +1258,7 @@ class QuickAdapterV3(IStrategy): n_filled_take_profit_exits = sum( 1 for order in trade.select_filled_orders(trade.exit_side) - if (order.ft_order_tag or "").startswith( - QuickAdapterV3._TAKE_PROFIT_ORDER_TAG_PREFIX - ) + if (order.ft_order_tag or "").startswith(QuickAdapterV3._TAKE_PROFIT_ORDER_TAG_PREFIX) ) return min(n_filled_take_profit_exits, QuickAdapterV3._FINAL_EXIT_STAGE) @@ -1379,7 +1273,7 @@ class QuickAdapterV3(IStrategy): trade: Trade, current_rate: float, natr_multiplier_fraction: float, - ) -> Optional[float]: + ) -> float | None: if not (0.0 <= natr_multiplier_fraction <= 1.0): raise ValueError( f"Invalid natr_multiplier_fraction value {natr_multiplier_fraction!r}: must be in range [0, 1]" @@ -1393,11 +1287,9 @@ class QuickAdapterV3(IStrategy): return ( current_rate * (trade_natr / 100.0) - * self.get_label_natr_multiplier_fraction( - trade.pair, natr_multiplier_fraction, df - ) + * self.get_label_natr_multiplier_fraction(trade.pair, natr_multiplier_fraction, df) * QuickAdapterV3.get_stoploss_factor( - trade_duration_candles + int(round(trade.nr_of_successful_exits**1.5)) + trade_duration_candles + round(trade.nr_of_successful_exits**1.5) ) ) @@ -1408,7 +1300,7 @@ class QuickAdapterV3(IStrategy): def get_take_profit_distance( self, df: DataFrame, trade: Trade, natr_multiplier_fraction: float - ) -> Optional[float]: + ) -> float | None: if not (0.0 <= natr_multiplier_fraction <= 1.0): raise ValueError( f"Invalid natr_multiplier_fraction value {natr_multiplier_fraction!r}: must be in range [0, 1]" @@ -1422,9 +1314,7 @@ class QuickAdapterV3(IStrategy): return ( trade.open_rate * (trade_natr / 100.0) - * self.get_label_natr_multiplier_fraction( - trade.pair, natr_multiplier_fraction, df - ) + * self.get_label_natr_multiplier_fraction(trade.pair, natr_multiplier_fraction, df) * QuickAdapterV3.get_take_profit_factor(trade_duration_candles) ) @@ -1439,17 +1329,13 @@ class QuickAdapterV3(IStrategy): timestamp = int(current_time.timestamp()) candle_duration_secs = max(1, int(self._candle_duration_secs)) candle_start_secs = (timestamp // candle_duration_secs) * candle_duration_secs - key = hashlib.sha256( - f"{pair}\x00{get_callable_sha256(callback)}".encode() - ).hexdigest() + key = hashlib.sha256(f"{pair}\x00{get_callable_sha256(callback)}".encode()).hexdigest() if candle_start_secs != self.last_candle_start_secs.get(key): self.last_candle_start_secs[key] = candle_start_secs try: callback() - except Exception as e: - logger.error( - f"[{pair}] Callback execution failed: {e!r}", exc_info=True - ) + except Exception: + logger.exception(f"[{pair}] Callback execution failed") threshold_secs = 10 * candle_duration_secs keys_to_remove = [ @@ -1469,10 +1355,8 @@ class QuickAdapterV3(IStrategy): current_profit: float, after_fill: bool, **kwargs, - ) -> Optional[float]: - df, _ = self.dp.get_analyzed_dataframe( - pair=pair, timeframe=self.config.get("timeframe") - ) + ) -> float | None: + df, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.config.get("timeframe")) if df.empty: return None @@ -1492,24 +1376,20 @@ class QuickAdapterV3(IStrategy): ) @staticmethod - def can_take_profit( - trade: Trade, current_rate: float, take_profit_price: float - ) -> bool: + def can_take_profit(trade: Trade, current_rate: float, take_profit_price: float) -> bool: return (trade.is_short and current_rate <= take_profit_price) or ( not trade.is_short and current_rate >= take_profit_price ) def get_take_profit_target( self, df: DataFrame, trade: Trade, exit_stage: int - ) -> Optional[tuple[float, float]]: + ) -> tuple[float, float] | None: natr_multiplier_fraction = ( QuickAdapterV3.partial_exit_stages[exit_stage][0] if exit_stage in QuickAdapterV3.partial_exit_stages else QuickAdapterV3._FINAL_EXIT_STAGE_PARAMS[0] ) - take_profit_distance = self.get_take_profit_distance( - df, trade, natr_multiplier_fraction - ) + take_profit_distance = self.get_take_profit_distance(df, trade, natr_multiplier_fraction) if not is_finite_number(take_profit_distance) or take_profit_distance <= 0: return None @@ -1517,9 +1397,7 @@ class QuickAdapterV3(IStrategy): -take_profit_distance if trade.is_short else take_profit_distance ) if take_profit_price == trade.open_rate: - take_profit_price = math.nextafter( - trade.open_rate, 0.0 if trade.is_short else math.inf - ) + take_profit_price = math.nextafter(trade.open_rate, 0.0 if trade.is_short else math.inf) if not np.isfinite(take_profit_price) or take_profit_price <= 0: return None return float(take_profit_price), float(take_profit_distance) @@ -1541,9 +1419,7 @@ class QuickAdapterV3(IStrategy): isinstance(previous_take_profit_entry, (tuple, list)) and len(previous_take_profit_entry) == 2 ): - candidate_exit_stage, candidate_take_profit_price = ( - previous_take_profit_entry - ) + candidate_exit_stage, candidate_take_profit_price = previous_take_profit_entry if isinstance(candidate_take_profit_price, bool): candidate_take_profit_price = None else: @@ -1574,9 +1450,7 @@ class QuickAdapterV3(IStrategy): price_history.append((exit_stage, take_profit_price)) if len(price_history) > self._max_take_profit_history_size: - history["take_profit_price"] = price_history[ - -self._max_take_profit_history_size : - ] + history["take_profit_price"] = price_history[-self._max_take_profit_history_size :] trade.set_custom_data("history", history) @staticmethod @@ -1596,20 +1470,13 @@ class QuickAdapterV3(IStrategy): return None @staticmethod - def _is_candle_date_aligned( - candle_date: datetime.datetime | None, timeframe: str - ) -> bool: + def _is_candle_date_aligned(candle_date: datetime.datetime | None, timeframe: str) -> bool: normalized_candle_date = QuickAdapterV3._as_utc_candle_date(candle_date) - if ( - normalized_candle_date is None - or not isinstance(timeframe, str) - or not timeframe - ): + if normalized_candle_date is None or not isinstance(timeframe, str) or not timeframe: return False try: return ( - timeframe_to_prev_date(timeframe, normalized_candle_date) - == normalized_candle_date + timeframe_to_prev_date(timeframe, normalized_candle_date) == normalized_candle_date ) except (OverflowError, TypeError, ValueError): return False @@ -1672,9 +1539,7 @@ class QuickAdapterV3(IStrategy): or take_profit_distance <= 0 or not is_finite_number(retracement_fraction) or not 0 < retracement_fraction <= 1 - or not QuickAdapterV3._is_candle_date_aligned( - normalized_candle_date, timeframe - ) + or not QuickAdapterV3._is_candle_date_aligned(normalized_candle_date, timeframe) or not isinstance(timeframe, str) or not timeframe ): @@ -1682,12 +1547,10 @@ class QuickAdapterV3(IStrategy): retracement_distance = take_profit_distance * retracement_fraction if not np.isfinite(retracement_distance) or retracement_distance < 0: return None - retracement_distance = ( - QuickAdapterV3._normalize_final_take_profit_retracement_distance( - best_rate=float(current_rate), - retracement_distance=float(retracement_distance), - trade_direction=trade_direction, - ) + retracement_distance = QuickAdapterV3._normalize_final_take_profit_retracement_distance( + best_rate=float(current_rate), + retracement_distance=float(retracement_distance), + trade_direction=trade_direction, ) if retracement_distance is None: return None @@ -1703,11 +1566,7 @@ class QuickAdapterV3(IStrategy): "trigger_candle_date": None, "timeframe": timeframe, } - return ( - state - if QuickAdapterV3._is_valid_final_take_profit_boundary(state) - else None - ) + return state if QuickAdapterV3._is_valid_final_take_profit_boundary(state) else None @staticmethod def _normalize_final_take_profit_state( @@ -1722,22 +1581,14 @@ class QuickAdapterV3(IStrategy): ) -> tuple[_FinalTakeProfitState | None, bool]: if state is None: return None, False - minimum_candle_date_utc = QuickAdapterV3._as_utc_candle_date( - minimum_candle_date - ) - current_candle_date_utc = QuickAdapterV3._as_utc_candle_date( - current_candle_date - ) + minimum_candle_date_utc = QuickAdapterV3._as_utc_candle_date(minimum_candle_date) + current_candle_date_utc = QuickAdapterV3._as_utc_candle_date(current_candle_date) if ( minimum_candle_date_utc is None or current_candle_date_utc is None or minimum_candle_date_utc > current_candle_date_utc - or not QuickAdapterV3._is_candle_date_aligned( - minimum_candle_date_utc, timeframe - ) - or not QuickAdapterV3._is_candle_date_aligned( - current_candle_date_utc, timeframe - ) + or not QuickAdapterV3._is_candle_date_aligned(minimum_candle_date_utc, timeframe) + or not QuickAdapterV3._is_candle_date_aligned(current_candle_date_utc, timeframe) or not isinstance(state, dict) or type(state.get("version")) is not int or state.get("version") @@ -1761,9 +1612,7 @@ class QuickAdapterV3(IStrategy): ): return None, True state_version = state["version"] - last_candle_date = QuickAdapterV3._as_utc_candle_date( - state.get("last_candle_date") - ) + last_candle_date = QuickAdapterV3._as_utc_candle_date(state.get("last_candle_date")) boundary_candle_date = ( last_candle_date if state_version == 1 @@ -1782,9 +1631,7 @@ class QuickAdapterV3(IStrategy): if ( last_candle_date is None or boundary_candle_date is None - or not QuickAdapterV3._is_candle_date_aligned( - boundary_candle_date, timeframe - ) + or not QuickAdapterV3._is_candle_date_aligned(boundary_candle_date, timeframe) or not QuickAdapterV3._is_candle_date_aligned(last_candle_date, timeframe) or boundary_candle_date < minimum_candle_date_utc or boundary_candle_date > last_candle_date @@ -1793,9 +1640,7 @@ class QuickAdapterV3(IStrategy): or ( trigger_candle_date is not None and ( - not QuickAdapterV3._is_candle_date_aligned( - trigger_candle_date, timeframe - ) + not QuickAdapterV3._is_candle_date_aligned(trigger_candle_date, timeframe) or trigger_candle_date <= boundary_candle_date or trigger_candle_date != last_candle_date ) @@ -1820,12 +1665,10 @@ class QuickAdapterV3(IStrategy): or retracement_distance <= 0 ): return None, True - retracement_distance = ( - QuickAdapterV3._normalize_final_take_profit_retracement_distance( - best_rate=best_rate, - retracement_distance=retracement_distance, - trade_direction=trade_direction, - ) + retracement_distance = QuickAdapterV3._normalize_final_take_profit_retracement_distance( + best_rate=best_rate, + retracement_distance=retracement_distance, + trade_direction=trade_direction, ) if retracement_distance is None: return None, True @@ -1839,9 +1682,7 @@ class QuickAdapterV3(IStrategy): "boundary_candle_date": boundary_candle_date.isoformat(), "last_candle_date": last_candle_date.isoformat(), "trigger_candle_date": ( - trigger_candle_date.isoformat() - if trigger_candle_date is not None - else None + trigger_candle_date.isoformat() if trigger_candle_date is not None else None ), "timeframe": timeframe, } @@ -1880,9 +1721,7 @@ class QuickAdapterV3(IStrategy): ) -> tuple[float, bool, bool]: boundary = QuickAdapterV3._final_take_profit_boundary(state) current_candle_date = QuickAdapterV3._as_utc_candle_date(candle_date) - previous_candle_date = QuickAdapterV3._as_utc_candle_date( - state["last_candle_date"] - ) + previous_candle_date = QuickAdapterV3._as_utc_candle_date(state["last_candle_date"]) if ( not is_finite_number(current_rate) or current_rate <= 0 @@ -1932,14 +1771,14 @@ class QuickAdapterV3(IStrategy): current_time: datetime.datetime, current_rate: float, current_profit: float, - min_stake: Optional[float], + min_stake: float | None, max_stake: float, current_entry_rate: float, current_exit_rate: float, current_entry_profit: float, current_exit_profit: float, **kwargs, - ) -> Optional[float] | tuple[Optional[float], Optional[str]]: + ) -> float | tuple[float | None, str | None] | None: pair = trade.pair if trade.has_open_orders: return None @@ -1948,22 +1787,16 @@ class QuickAdapterV3(IStrategy): if trade_exit_stage not in QuickAdapterV3.partial_exit_stages: return None - df, _ = self.dp.get_analyzed_dataframe( - pair=pair, timeframe=self.config.get("timeframe") - ) + df, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.config.get("timeframe")) if df.empty: return None - trade_take_profit_target = self.get_take_profit_target( - df, trade, trade_exit_stage - ) + trade_take_profit_target = self.get_take_profit_target(df, trade, trade_exit_stage) if trade_take_profit_target is None: return None trade_take_profit_price, _ = trade_take_profit_target - self.safe_append_trade_take_profit_price( - trade, trade_take_profit_price, trade_exit_stage - ) + self.safe_append_trade_take_profit_price(trade, trade_take_profit_price, trade_exit_stage) trade_partial_exit = QuickAdapterV3.can_take_profit( trade, current_exit_rate, trade_take_profit_price @@ -1978,9 +1811,7 @@ class QuickAdapterV3(IStrategy): ), ) if trade_partial_exit: - trade_stake_percent = QuickAdapterV3.partial_exit_stages[trade_exit_stage][ - 1 - ] + trade_stake_percent = QuickAdapterV3.partial_exit_stages[trade_exit_stage][1] trade_partial_stake_amount = trade_stake_percent * trade.stake_amount if min_stake is not None and min_stake > 0: current_position_value = trade.amount * current_exit_rate @@ -1994,14 +1825,10 @@ class QuickAdapterV3(IStrategy): current_exit_rate / current_entry_rate, 1.0 / (1.0 - abs(self.stoploss)), ) - min_remaining_position_value *= ( - 1.0 + QuickAdapterV3._PARTIAL_EXIT_MIN_STAKE_MARGIN - ) + min_remaining_position_value *= 1.0 + QuickAdapterV3._PARTIAL_EXIT_MIN_STAKE_MARGIN if current_position_value <= min_remaining_position_value: return None - remaining_position_value = current_position_value * ( - 1 - trade_stake_percent - ) + remaining_position_value = current_position_value * (1 - trade_stake_percent) if remaining_position_value < min_remaining_position_value: initial_trade_partial_stake_amount = trade_partial_stake_amount trade_partial_stake_amount = trade.stake_amount * ( @@ -2026,9 +1853,9 @@ class QuickAdapterV3(IStrategy): @staticmethod def weighted_close(series: Series, weight: float = 2.0) -> float: - return float( - series.get("high") + series.get("low") + weight * series.get("close") - ) / (2.0 + weight) + return float(series.get("high") + series.get("low") + weight * series.get("close")) / ( + 2.0 + weight + ) @staticmethod def _normalize_candle_idx(length: int, idx: int) -> int: @@ -2043,9 +1870,7 @@ class QuickAdapterV3(IStrategy): idx = length + idx return min(max(0, idx), length - 1) - def _invalidate_pair_caches( - self, pair: str, df_signature: Optional[DfSignature] = None - ) -> None: + def _invalidate_pair_caches(self, pair: str, df_signature: DfSignature | None = None) -> None: if df_signature is None or self._cached_df_signature.get(pair) != df_signature: self._candle_deviation_cache = { k: v for k, v in self._candle_deviation_cache.items() if k[0] != pair @@ -2085,9 +1910,7 @@ class QuickAdapterV3(IStrategy): if label_natr_series is None or label_natr_series.empty: return np.nan - candle_idx = QuickAdapterV3._normalize_candle_idx( - len(label_natr_series), candle_idx - ) + candle_idx = QuickAdapterV3._normalize_candle_idx(len(label_natr_series), candle_idx) label_natr_values = label_natr_series.iloc[: candle_idx + 1].to_numpy() if label_natr_values.size == 0: @@ -2124,9 +1947,7 @@ class QuickAdapterV3(IStrategy): ) candle_deviation = ( candle_label_natr_value / 100.0 - ) * self.get_label_natr_multiplier_fraction( - pair, natr_multiplier_fraction, df, candle_idx - ) + ) * self.get_label_natr_multiplier_fraction(pair, natr_multiplier_fraction, df, candle_idx) self._candle_deviation_cache[cache_key] = candle_deviation return self._candle_deviation_cache[cache_key] @@ -2157,9 +1978,7 @@ class QuickAdapterV3(IStrategy): min_natr_multiplier_fraction=min_natr_multiplier_fraction, max_natr_multiplier_fraction=max_natr_multiplier_fraction, candle_idx=candle_idx, - interpolation_direction=QuickAdapterV3._INTERPOLATION_DIRECTIONS[ - 0 - ], # "direct" + interpolation_direction=QuickAdapterV3._INTERPOLATION_DIRECTIONS[0], # "direct" ) if isna(current_deviation) or current_deviation <= 0: return np.nan @@ -2176,22 +1995,16 @@ class QuickAdapterV3(IStrategy): if side == QuickAdapterV3._TRADE_LONG: base_price = ( - QuickAdapterV3.weighted_close(candle) - if is_candle_bearish - else candle_close + QuickAdapterV3.weighted_close(candle) if is_candle_bearish else candle_close ) candle_threshold = base_price * (1 + current_deviation) elif side == QuickAdapterV3._TRADE_SHORT: base_price = ( - QuickAdapterV3.weighted_close(candle) - if is_candle_bullish - else candle_close + QuickAdapterV3.weighted_close(candle) if is_candle_bullish else candle_close ) candle_threshold = base_price * (1 - current_deviation) else: - raise ValueError( - enum_error_message("side", side, QuickAdapterV3._TRADE_DIRECTIONS) - ) + raise ValueError(enum_error_message("side", side, QuickAdapterV3._TRADE_DIRECTIONS)) self._candle_threshold_cache[cache_key] = candle_threshold return self._candle_threshold_cache[cache_key] @@ -2244,13 +2057,9 @@ class QuickAdapterV3(IStrategy): trade_direction = side max_lookback_period_candles = max(0, len(df) - 1) - lookback_period_candles = min( - lookback_period_candles, max_lookback_period_candles - ) + lookback_period_candles = min(lookback_period_candles, max_lookback_period_candles) if not isinstance(decay_fraction, (int, float)): - logger.debug( - f"[{pair}] Denied {trade_direction} {order}: invalid decay_fraction type" - ) + logger.debug(f"[{pair}] Denied {trade_direction} {order}: invalid decay_fraction type") return False if not (0.0 < decay_fraction <= 1.0): logger.debug( @@ -2307,9 +2116,7 @@ class QuickAdapterV3(IStrategy): max_natr_multiplier_fraction=decayed_max_natr_multiplier_fraction, candle_idx=-(k + 1), ) - if not isinstance(threshold_k, (int, float)) or not np.isfinite( - threshold_k - ): + if not isinstance(threshold_k, (int, float)) or not np.isfinite(threshold_k): return unmeasurable_history_ok if (side == QuickAdapterV3._TRADE_LONG and not (close_k > threshold_k)) or ( @@ -2344,10 +2151,8 @@ class QuickAdapterV3(IStrategy): current_rate: float, current_profit: float, **kwargs, - ) -> Optional[str]: - df, _ = self.dp.get_analyzed_dataframe( - pair=pair, timeframe=self.config.get("timeframe") - ) + ) -> str | None: + df, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.config.get("timeframe")) if df.empty: return None @@ -2447,12 +2252,10 @@ class QuickAdapterV3(IStrategy): ) if final_take_profit_state is not None: - boundary, trade_exit, state_changed = ( - QuickAdapterV3._advance_final_take_profit_state( - final_take_profit_state, - current_rate=current_rate, - candle_date=last_candle_date, - ) + boundary, trade_exit, state_changed = QuickAdapterV3._advance_final_take_profit_state( + final_take_profit_state, + current_rate=current_rate, + candle_date=last_candle_date, ) if state_normalized or state_changed: trade.set_custom_data( @@ -2472,24 +2275,17 @@ class QuickAdapterV3(IStrategy): ) if trade_exit: return ( - f"{QuickAdapterV3._TAKE_PROFIT_ORDER_TAG_PREFIX}" - f"{trade.trade_direction}_final" + f"{QuickAdapterV3._TAKE_PROFIT_ORDER_TAG_PREFIX}{trade.trade_direction}_final" ) return None - trade_take_profit_target = self.get_take_profit_target( - df, trade, trade_exit_stage - ) + trade_take_profit_target = self.get_take_profit_target(df, trade, trade_exit_stage) if trade_take_profit_target is None: return None trade_take_profit_price, trade_take_profit_distance = trade_take_profit_target - self.safe_append_trade_take_profit_price( - trade, trade_take_profit_price, trade_exit_stage - ) - if not QuickAdapterV3.can_take_profit( - trade, current_rate, trade_take_profit_price - ): + self.safe_append_trade_take_profit_price(trade, trade_take_profit_price, trade_exit_stage) + if not QuickAdapterV3.can_take_profit(trade, current_rate, trade_take_profit_price): self.throttle_callback( pair=pair, current_time=current_time, @@ -2537,7 +2333,7 @@ class QuickAdapterV3(IStrategy): rate: float, time_in_force: str, current_time: datetime.datetime, - entry_tag: Optional[str], + entry_tag: str | None, side: str, **kwargs, ) -> bool: @@ -2557,33 +2353,27 @@ class QuickAdapterV3(IStrategy): max_open_trades_per_side = self.max_open_trades_per_side if max_open_trades_per_side >= 0: open_trades = Trade.get_open_trades() - trades_per_side = sum( - 1 for trade in open_trades if trade.trade_direction == side - ) + trades_per_side = sum(1 for trade in open_trades if trade.trade_direction == side) if trades_per_side >= max_open_trades_per_side: return False - df, _ = self.dp.get_analyzed_dataframe( - pair=pair, timeframe=self.config.get("timeframe") - ) + df, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.config.get("timeframe")) if df.empty: - logger.info( - f"[{pair}] Denied {side} {QuickAdapterV3._ORDER_ENTRY}: dataframe is empty" - ) + logger.info(f"[{pair}] Denied {side} {QuickAdapterV3._ORDER_ENTRY}: dataframe is empty") return False - if self.reversal_confirmed( - df, - pair, - side, - QuickAdapterV3._ORDER_ENTRY, - rate, - self.reversal_confirmation["lookback_period_candles"], - self.reversal_confirmation["decay_fraction"], - self.reversal_confirmation["min_natr_multiplier_fraction"], - self.reversal_confirmation["max_natr_multiplier_fraction"], - ): - return True - return False + return bool( + self.reversal_confirmed( + df, + pair, + side, + QuickAdapterV3._ORDER_ENTRY, + rate, + self.reversal_confirmation["lookback_period_candles"], + self.reversal_confirmation["decay_fraction"], + self.reversal_confirmation["min_natr_multiplier_fraction"], + self.reversal_confirmation["max_natr_multiplier_fraction"], + ) + ) def is_short_allowed(self) -> bool: trading_mode = self.config.get("trading_mode") @@ -2596,13 +2386,11 @@ class QuickAdapterV3(IStrategy): return False else: raise ValueError( - enum_error_message( - "trading_mode", trading_mode, QuickAdapterV3._TRADING_MODES - ) + enum_error_message("trading_mode", trading_mode, QuickAdapterV3._TRADING_MODES) ) @cached_property - def _configured_leverage(self) -> Optional[float]: + def _configured_leverage(self) -> float | None: leverage = self.config.get("leverage") if leverage is None: return None @@ -2614,9 +2402,7 @@ class QuickAdapterV3(IStrategy): return None leverage = float(leverage) if leverage < 1.0: - logger.warning( - f"Invalid leverage value {leverage}: must be >= 1.0, clamping to 1.0" - ) + logger.warning(f"Invalid leverage value {leverage}: must be >= 1.0, clamping to 1.0") return leverage def leverage( @@ -2626,7 +2412,7 @@ class QuickAdapterV3(IStrategy): current_rate: float, proposed_leverage: float, max_leverage: float, - entry_tag: Optional[str], + entry_tag: str | None, side: str, **kwargs: Any, ) -> float: @@ -2657,8 +2443,8 @@ class QuickAdapterV3(IStrategy): if trade.open_date_utc > end_date: continue - trade_annotation_line_start_date = ( - self.get_trade_annotation_line_start_date(dataframe, trade) + trade_annotation_line_start_date = self.get_trade_annotation_line_start_date( + dataframe, trade ) trade_exit_stage = QuickAdapterV3.get_trade_exit_stage(trade) @@ -2693,16 +2479,14 @@ class QuickAdapterV3(IStrategy): ) final_take_profit_state = None if annotation_candle_date is not None: - final_take_profit_state, _ = ( - QuickAdapterV3._normalize_final_take_profit_state( - raw_final_take_profit_state, - exit_stage=final_exit_stage, - trade_direction=trade.trade_direction, - open_rate=trade.open_rate, - timeframe=self.timeframe, - minimum_candle_date=self.get_trade_entry_date(trade), - current_candle_date=annotation_candle_date, - ) + final_take_profit_state, _ = QuickAdapterV3._normalize_final_take_profit_state( + raw_final_take_profit_state, + exit_stage=final_exit_stage, + trade_direction=trade.trade_direction, + open_rate=trade.open_rate, + timeframe=self.timeframe, + minimum_candle_date=self.get_trade_entry_date(trade), + current_candle_date=annotation_candle_date, ) if final_take_profit_state is not None: @@ -2712,10 +2496,8 @@ class QuickAdapterV3(IStrategy): if boundary_candle_date is not None: trail_start = max(boundary_candle_date, start_date) if trail_start <= end_date: - final_take_profit_price = ( - QuickAdapterV3._final_take_profit_boundary( - final_take_profit_state - ) + final_take_profit_price = QuickAdapterV3._final_take_profit_boundary( + final_take_profit_state ) annotations.append( { @@ -2757,7 +2539,7 @@ class QuickAdapterV3(IStrategy): def optuna_load_best_params( self, pair: str, namespace: OptunaNamespace - ) -> Optional[dict[str, Any]]: + ) -> dict[str, Any] | None: # Strategy consumes only output tunables (``label_period_candles``, # ``label_horizon_candles``, ``label_natr_multiplier``); # selection-metadata drift on cached label ``best_params`` is diff --git a/quickadapter/user_data/strategies/Utils.py b/quickadapter/user_data/strategies/Utils.py index f5398af..d0e5578 100644 --- a/quickadapter/user_data/strategies/Utils.py +++ b/quickadapter/user_data/strategies/Utils.py @@ -8,7 +8,7 @@ import math import os import re import stat -from collections.abc import Iterator, Sequence +from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timezone @@ -20,7 +20,6 @@ from threading import Lock from typing import ( TYPE_CHECKING, Any, - Callable, Final, Literal, NamedTuple, @@ -36,8 +35,8 @@ import optuna import pandas as pd import scipy as sp import talib.abstract as ta -from freqtrade.misc import pair_to_filename from EnumErrors import enum_error_message +from freqtrade.misc import pair_to_filename from LabelTransformer import ( COMBINED_AGGREGATIONS, COMBINED_METRICS, @@ -249,9 +248,7 @@ def safe_divide( numerator_arr = np.asarray(numerator, dtype=float) denominator_arr = np.asarray(denominator, dtype=float) valid_mask = ( - np.isfinite(numerator_arr) - & np.isfinite(denominator_arr) - & (denominator_arr != 0.0) + np.isfinite(numerator_arr) & np.isfinite(denominator_arr) & (denominator_arr != 0.0) ) with np.errstate(divide="ignore", invalid="ignore"): result = np.divide( @@ -414,9 +411,7 @@ class _RangeValidator: return False if self.min_bound is not None and value[0] < self.min_bound: return False - if self.max_bound is not None and value[1] > self.max_bound: - return False - return True + return not (self.max_bound is not None and value[1] > self.max_bound) def message(self, param: str) -> str: if self.min_bound is not None and self.max_bound is not None: @@ -444,13 +439,7 @@ class _BoolValidator: return "must be a boolean" -_Validator = ( - _EnumValidator - | _NumericValidator - | _RangeValidator - | _DictValidator - | _BoolValidator -) +_Validator = _EnumValidator | _NumericValidator | _RangeValidator | _DictValidator | _BoolValidator @dataclass(frozen=True, slots=True) @@ -482,9 +471,7 @@ def _validate_params( f"Invalid {config_name} {param} keys {sorted(invalid_keys)!r}, " f"valid keys: {', '.join(spec.validator.valid_keys)}" ) - value = { - k: v for k, v in value.items() if k in spec.validator.valid_keys - } + value = {k: v for k, v in value.items() if k in spec.validator.valid_keys} if spec.output_type is not None: if spec.output_type is tuple and isinstance(value, (list, tuple)): value = (value[0], value[1]) @@ -515,40 +502,34 @@ def require_numeric( ) accepted_types = (int,) if require_int else (int, float) if type(value) not in accepted_types or not validator(value): - raise ValueError( - f"Invalid {context}.{name} value {value!r}: {validator.message(name)}" - ) + raise ValueError(f"Invalid {context}.{name} value {value!r}: {validator.message(name)}") return value def require_bool(value: Any, name: str, *, context: str) -> bool: validator = _BoolValidator() if not validator(value): - raise ValueError( - f"Invalid {context}.{name} value {value!r}: {validator.message(name)}" - ) + raise ValueError(f"Invalid {context}.{name} value {value!r}: {validator.message(name)}") return value def validate_range( - min_val: float | int, - max_val: float | int, + min_val: float, + max_val: float, logger: Logger, *, name: str, - default_min: float | int, - default_max: float | int, + default_min: float, + default_max: float, allow_equal: bool = False, non_negative: bool = True, finite_only: bool = True, - max_value: float | int | None = None, + max_value: float | None = None, ) -> tuple[float | int, float | int]: min_name = f"min_{name}" max_name = f"max_{name}" - if not isinstance(default_min, (int, float)) or not isinstance( - default_max, (int, float) - ): + if not isinstance(default_min, (int, float)) or not isinstance(default_max, (int, float)): raise ValueError( f"Invalid {name}: defaults must be numeric, " f"got min={type(default_min).__name__!r}, max={type(default_max).__name__!r}" @@ -564,9 +545,7 @@ def validate_range( f"got min={default_min!r}, max={default_max!r}" ) - def _validate_component( - value: float | int | None, name: str, default_value: float | int - ) -> float | int: + def _validate_component(value: float | None, name: str, default_value: float) -> float | int: constraints = [] if finite_only: constraints.append("finite") @@ -593,9 +572,7 @@ def validate_range( sanitized_max = _validate_component(max_val, max_name, default_max) ordering_ok = ( - (sanitized_min < sanitized_max) - if not allow_equal - else (sanitized_min <= sanitized_max) + (sanitized_min < sanitized_max) if not allow_equal else (sanitized_min <= sanitized_max) ) if not ordering_ok: logger.warning( @@ -617,20 +594,12 @@ _WEIGHTING_SPECS: Final[dict[str, _ParamSpec]] = { "strategy": _ParamSpec(_EnumValidator(WEIGHT_STRATEGIES)), "metric_coefficients": _ParamSpec(_DictValidator(COMBINED_METRICS)), "aggregation": _ParamSpec(_EnumValidator(COMBINED_AGGREGATIONS)), - "softmax_temperature": _ParamSpec( - _NumericValidator(min_value=0, min_exclusive=True) - ), + "softmax_temperature": _ParamSpec(_NumericValidator(min_value=0, min_exclusive=True)), "fill_method": _ParamSpec(_EnumValidator(FILL_METHODS)), - "fill_epsilon": _ParamSpec( - _NumericValidator(min_value=0.0, max_value=1.0), output_type=float - ), + "fill_epsilon": _ParamSpec(_NumericValidator(min_value=0.0, max_value=1.0), output_type=float), "fill_epsilon_baseline": _ParamSpec(_EnumValidator(FILL_EPSILON_BASELINES)), - "fill_sigma_candles": _ParamSpec( - _NumericValidator(min_value=0.5), output_type=float - ), - "fill_sigma_min_candles": _ParamSpec( - _NumericValidator(min_value=0.5), output_type=float - ), + "fill_sigma_candles": _ParamSpec(_NumericValidator(min_value=0.5), output_type=float), + "fill_sigma_min_candles": _ParamSpec(_NumericValidator(min_value=0.5), output_type=float), "fill_bandwidth": _ParamSpec(_EnumValidator(FILL_BANDWIDTHS)), "fill_bandwidth_neighbors": _ParamSpec( _NumericValidator(min_value=1, require_int=True), output_type=int @@ -645,42 +614,26 @@ _WEIGHTING_SPECS: Final[dict[str, _ParamSpec]] = { "min_positive_label_weight_fraction": _ParamSpec( _NumericValidator(min_value=0.0, max_value=1.0), output_type=float ), - "min_effective_sample_size": _ParamSpec( - _NumericValidator(min_value=1), output_type=float - ), + "min_effective_sample_size": _ParamSpec(_NumericValidator(min_value=1), output_type=float), } _PIPELINE_SPECS: Final[dict[str, _ParamSpec]] = { "standardization": _ParamSpec(_EnumValidator(STANDARDIZATION_TYPES)), - "robust_quantiles": _ParamSpec( - _RangeValidator(min_bound=0, max_bound=1), output_type=tuple - ), - "mmad_scaling_factor": _ParamSpec( - _NumericValidator(min_value=0, min_exclusive=True) - ), + "robust_quantiles": _ParamSpec(_RangeValidator(min_bound=0, max_bound=1), output_type=tuple), + "mmad_scaling_factor": _ParamSpec(_NumericValidator(min_value=0, min_exclusive=True)), "normalization": _ParamSpec(_EnumValidator(NORMALIZATION_TYPES)), "minmax_range": _ParamSpec(_RangeValidator(), output_type=tuple), "sigmoid_scale": _ParamSpec(_NumericValidator(min_value=0, min_exclusive=True)), - "gamma": _ParamSpec( - _NumericValidator(min_value=0, max_value=10, min_exclusive=True) - ), + "gamma": _ParamSpec(_NumericValidator(min_value=0, max_value=10, min_exclusive=True)), } _SMOOTHING_SPECS: Final[dict[str, _ParamSpec]] = { "method": _ParamSpec(_EnumValidator(SMOOTHING_METHODS)), - "window_candles": _ParamSpec( - _NumericValidator(min_value=1, require_int=True), output_type=int - ), - "beta": _ParamSpec( - _NumericValidator(min_value=0, min_exclusive=True), output_type=float - ), - "polyorder": _ParamSpec( - _NumericValidator(min_value=0, require_int=True), output_type=int - ), + "window_candles": _ParamSpec(_NumericValidator(min_value=1, require_int=True), output_type=int), + "beta": _ParamSpec(_NumericValidator(min_value=0, min_exclusive=True), output_type=float), + "polyorder": _ParamSpec(_NumericValidator(min_value=0, require_int=True), output_type=int), "mode": _ParamSpec(_EnumValidator(SMOOTHING_MODES)), - "sigma": _ParamSpec( - _NumericValidator(min_value=0, min_exclusive=True), output_type=float - ), + "sigma": _ParamSpec(_NumericValidator(min_value=0, min_exclusive=True), output_type=float), } _PREDICTION_SPECS: Final[dict[str, _ParamSpec]] = { @@ -688,14 +641,10 @@ _PREDICTION_SPECS: Final[dict[str, _ParamSpec]] = { "selection_method": _ParamSpec(_EnumValidator(EXTREMA_SELECTION_METHODS)), "threshold_method": _ParamSpec(_EnumValidator(THRESHOLD_METHODS)), "outlier_quantile": _ParamSpec( - _NumericValidator( - min_value=0, max_value=1, min_exclusive=True, max_exclusive=True - ), + _NumericValidator(min_value=0, max_value=1, min_exclusive=True, max_exclusive=True), output_type=float, ), - "soft_extremum_alpha": _ParamSpec( - _NumericValidator(min_value=0), output_type=float - ), + "soft_extremum_alpha": _ParamSpec(_NumericValidator(min_value=0), output_type=float), "keep_fraction": _ParamSpec( _NumericValidator(min_value=0, max_value=1, min_exclusive=True), output_type=float, @@ -824,9 +773,7 @@ def _adapt_label_generator( f"supported; declare an explicit (dataframe, params) or " f"(dataframe, params, logger) signature" ) - if any( - p.kind == inspect.Parameter.KEYWORD_ONLY and p.name == "logger" for p in params - ): + if any(p.kind == inspect.Parameter.KEYWORD_ONLY and p.name == "logger" for p in params): raise ValueError( f"Invalid label generator {generator!r}: keyword-only " f"``logger`` is not supported; declare ``logger`` as the " @@ -862,7 +809,7 @@ def _adapt_label_generator( f"parameter is named {positional[2].name!r}, expected " f"``logger``" ) - return cast(LabelGenerator, generator) + return cast("LabelGenerator", generator) @functools.wraps(generator) def adapted( @@ -999,9 +946,7 @@ def get_smoothing_kernel_half_width( sigma = max(float(config.get("sigma", DEFAULTS_LABEL_SMOOTHING["sigma"])), 0.0) return int(4.0 * sigma + 0.5) if method == SMOOTHING_METHODS[7]: # "savgol" - polyorder = max( - int(config.get("polyorder", DEFAULTS_LABEL_SMOOTHING["polyorder"])), 0 - ) + polyorder = max(int(config.get("polyorder", DEFAULTS_LABEL_SMOOTHING["polyorder"])), 0) effective_window, _, _ = get_savgol_params(raw_window, polyorder, "mirror") elif method == SMOOTHING_METHODS[3]: # "kaiser_bessel_derived" effective_window = get_even_window(raw_window) @@ -1046,9 +991,7 @@ def compose_label_lookahead( if known_at_lookahead.empty: return known_at_lookahead.copy() n = len(known_at_lookahead) - positions, known_at_lookahead_values = _sanitize_known_at_lookahead( - known_at_lookahead - ) + positions, known_at_lookahead_values = _sanitize_known_at_lookahead(known_at_lookahead) if kernel_half_width <= 0: return pd.Series( known_at_lookahead_values, @@ -1073,9 +1016,7 @@ def compose_label_lookahead( ) -TradeNatrMethod = Literal[ - "moving_average", "quantile_interpolation", "weighted_average" -] +TradeNatrMethod = Literal["moving_average", "quantile_interpolation", "weighted_average"] TRADE_NATR_METHODS: Final[tuple[TradeNatrMethod, ...]] = ( "moving_average", "quantile_interpolation", @@ -1099,9 +1040,7 @@ def as_dict(value: Any) -> dict[str, Any]: def as_config_section(value: Any, name: str, logger: Logger) -> dict[str, Any]: if value is not None and not isinstance(value, dict): - logger.warning( - f"Invalid {name} value {value!r}: must be a mapping, using defaults" - ) + logger.warning(f"Invalid {name} value {value!r}: must be a mapping, using defaults") return as_dict(value) @@ -1272,9 +1211,11 @@ CONFIG_DEPRECATIONS: Final[tuple[ConfigDeprecation, ...]] = ( "freqai.feature_parameters.causal_mode", None, lambda value: value is False, - "feature_parameters.causal_mode=false is deprecated: " - "causal split guards disabled; label lookahead leakage possible. " - "Default causal_mode=true; causal_mode=false for acausal baselines only.", + ( + "feature_parameters.causal_mode=false is deprecated: " + "causal split guards disabled; label lookahead leakage possible. " + "Default causal_mode=true; causal_mode=false for acausal baselines only." + ), ), ) @@ -1352,9 +1293,7 @@ def _get_label_config( ) default_config = {} - validated_default = validate_fn( - default_config, logger, f"{config_name}.default" - ) + validated_default = validate_fn(default_config, logger, f"{config_name}.default") columns_config = config.get("columns", {}) if not isinstance(columns_config, dict): @@ -1374,9 +1313,7 @@ def _get_label_config( for key, value in col_config.items(): if key in defaults_dict: temp = {key: value} - validated = validate_fn( - temp, logger, f"{config_name}.columns[{col_pattern!r}]" - ) + validated = validate_fn(temp, logger, f"{config_name}.columns[{col_pattern!r}]") validated_col[key] = validated[key] else: logger.warning( @@ -1441,20 +1378,15 @@ def get_label_kind_config( ) -> dict[str, Any]: if kind not in _LABEL_KIND_REGISTRY: raise ValueError( - f"Unknown label kind {kind!r}: supported values are " - f"{', '.join(_LABEL_KIND_REGISTRY)}" + f"Unknown label kind {kind!r}: supported values are {', '.join(_LABEL_KIND_REGISTRY)}" ) config = as_config_section(config, kind, logger) _, defaults, cross_field_validator = _LABEL_KIND_REGISTRY[kind] - validated = _get_label_config( - config, logger, kind, _label_kind_validator(kind), defaults - ) + validated = _get_label_config(config, logger, kind, _label_kind_validator(kind), defaults) if cross_field_validator is not None: for label_col in LABEL_COLUMNS: cross_field_validator( - get_label_column_config( - label_col, validated["default"], validated["columns"] - ), + get_label_column_config(label_col, validated["default"], validated["columns"]), f"{kind} for label {label_col!r}", ) return validated @@ -1494,9 +1426,7 @@ DEFAULTS_EXIT_PRICING: Final[dict[str, Any]] = { } _EXIT_PRICING_SPECS: Final[dict[str, _ParamSpec]] = { - "trade_natr_method": _ParamSpec( - _EnumValidator(TRADE_NATR_METHODS), output_type=str - ), + "trade_natr_method": _ParamSpec(_EnumValidator(TRADE_NATR_METHODS), output_type=str), "final_take_profit_retracement_fraction": _ParamSpec( _NumericValidator(min_value=0, max_value=1, min_exclusive=True), output_type=float, @@ -1553,9 +1483,7 @@ _COOLDOWN_PROTECTION_SPECS: Final[dict[str, _ParamSpec]] = { _DRAWDOWN_PROTECTION_SPECS: Final[dict[str, _ParamSpec]] = { "enabled": _ParamSpec(_BoolValidator()), "max_allowed_drawdown": _ParamSpec( - _NumericValidator( - min_value=0, max_value=1, min_exclusive=True, max_exclusive=True - ), + _NumericValidator(min_value=0, max_value=1, min_exclusive=True, max_exclusive=True), output_type=float, ), } @@ -1575,27 +1503,21 @@ def get_custom_protections_config(config: Any, logger: Logger) -> dict[str, Any] DEFAULTS_CUSTOM_PROTECTIONS, ) validated["cooldown"] = _validate_params( - as_config_section( - config.get("cooldown"), "custom_protections.cooldown", logger - ), + as_config_section(config.get("cooldown"), "custom_protections.cooldown", logger), logger, "custom_protections.cooldown", _COOLDOWN_PROTECTION_SPECS, DEFAULTS_COOLDOWN_PROTECTION, ) validated["drawdown"] = _validate_params( - as_config_section( - config.get("drawdown"), "custom_protections.drawdown", logger - ), + as_config_section(config.get("drawdown"), "custom_protections.drawdown", logger), logger, "custom_protections.drawdown", _DRAWDOWN_PROTECTION_SPECS, DEFAULTS_DRAWDOWN_PROTECTION, ) validated["stoploss"] = _validate_params( - as_config_section( - config.get("stoploss"), "custom_protections.stoploss", logger - ), + as_config_section(config.get("stoploss"), "custom_protections.stoploss", logger), logger, "custom_protections.stoploss", _STOPLOSS_PROTECTION_SPECS, @@ -1642,19 +1564,14 @@ _REVERSAL_CONFIRMATION_SCALAR_SPECS: Final[dict[str, _ParamSpec]] = { } -def get_reversal_confirmation_config( - config: Any, logger: Logger -) -> dict[str, int | float]: +def get_reversal_confirmation_config(config: Any, logger: Logger) -> dict[str, int | float]: config = as_config_section(config, "reversal_confirmation", logger) validated = _validate_params( config, logger, "reversal_confirmation", _REVERSAL_CONFIRMATION_SCALAR_SPECS, - { - key: DEFAULTS_REVERSAL_CONFIRMATION[key] - for key in _REVERSAL_CONFIRMATION_SCALAR_SPECS - }, + {key: DEFAULTS_REVERSAL_CONFIRMATION[key] for key in _REVERSAL_CONFIRMATION_SCALAR_SPECS}, ) min_natr_multiplier_fraction, max_natr_multiplier_fraction = validate_range( @@ -1687,20 +1604,14 @@ def get_reversal_confirmation_config( def get_causal_mode(config: dict[str, Any], logger: Logger) -> bool: causal_mode = config.get("causal_mode", True) if not isinstance(causal_mode, bool): - logger.warning( - f"Invalid causal_mode value {causal_mode!r}: must be bool, using True" - ) + logger.warning(f"Invalid causal_mode value {causal_mode!r}: must be bool, using True") return True return causal_mode def get_label_horizon_candles(config: dict[str, Any], logger: Logger) -> int: def _is_positive_int(value: Any) -> bool: - return ( - not isinstance(value, bool) - and isinstance(value, (int, np.integer)) - and value >= 1 - ) + return not isinstance(value, bool) and isinstance(value, (int, np.integer)) and value >= 1 fallback = config.get("label_period_candles", 1) if not _is_positive_int(fallback): @@ -1779,9 +1690,7 @@ def sanitize_and_renormalize( f"{context}: drop_mask shape {drop_mask.shape} != arr shape {arr.shape}" ) if not np.issubdtype(drop_mask.dtype, np.bool_): - raise ValueError( - f"{context}: drop_mask dtype {drop_mask.dtype} is not boolean" - ) + raise ValueError(f"{context}: drop_mask dtype {drop_mask.dtype} is not boolean") safe = np.where(drop_mask, 0.0, safe) total = safe.sum() rescale_overflow = False @@ -1793,16 +1702,14 @@ def sanitize_and_renormalize( if logger is not None: if rescale_overflow: logger.warning( - "%s: rescale factor non-finite (n=%d, total=%r); " - "falling back to uniform weights", + "%s: rescale factor non-finite (n=%d, total=%r); falling back to uniform weights", context, n, total, ) else: logger.warning( - "%s: weights collapsed (total=%r, n=%d); falling back " - "to uniform weights", + "%s: weights collapsed (total=%r, n=%d); falling back to uniform weights", context, total, n, @@ -1815,8 +1722,7 @@ def sanitize_and_renormalize( return masked * (n / total) if logger is not None: logger.warning( - "%s: drop_mask covers all rows in fallback; ignoring " - "mask to preserve mean=1", + "%s: drop_mask covers all rows in fallback; ignoring mask to preserve mean=1", context, ) return fallback @@ -1952,9 +1858,7 @@ def compose_sample_weights( """ base_weights = np.asarray(base_weights, dtype=float) if label_weights is None: - return sanitize_and_renormalize( - base_weights, logger=logger, context=f"{context}:base_only" - ) + return sanitize_and_renormalize(base_weights, logger=logger, context=f"{context}:base_only") n = base_weights.shape[0] arr = np.asarray(label_weights, dtype=float) if arr.shape != (n,): @@ -2087,9 +1991,7 @@ def get_gaussian_std(window: int) -> float: @lru_cache(maxsize=_CACHE_MAXSIZE_SMALL) -def get_savgol_params( - window: int, polyorder: int, mode: SmoothingMode -) -> tuple[int, int, str]: +def get_savgol_params(window: int, polyorder: int, mode: SmoothingMode) -> tuple[int, int, str]: if window <= polyorder: window = polyorder + 1 window = get_odd_window(window) @@ -2171,8 +2073,7 @@ def smooth( if n == 0: return series - if window_candles < 3: - window_candles = 3 + window_candles = max(window_candles, 3) if n < window_candles: return series if beta <= 0 or not np.isfinite(beta): @@ -2302,9 +2203,7 @@ _GAUSSIAN_FILL_DENSITY_WARN: Final[float] = 0.1 _WEIGHT_FILL_RADIUS_SIGMA_MULTIPLIER: Final[float] = 4.0 _ZIGZAG_CONFIRMATION_ALPHA: Final[float] = 0.05 # With all slopes successful, the one-sided Binomial(0.5) p-value is 2**-m. -_ZIGZAG_MIN_CONFIRMATION_SLOPES: Final[int] = math.ceil( - -math.log2(_ZIGZAG_CONFIRMATION_ALPHA) -) +_ZIGZAG_MIN_CONFIRMATION_SLOPES: Final[int] = math.ceil(-math.log2(_ZIGZAG_CONFIRMATION_ALPHA)) def _compute_pivot_kth_neighbor_distances( @@ -2329,12 +2228,8 @@ def _compute_pivot_kth_neighbor_distances( while left < right: left_count = (left + right) // 2 right_count = k - left_count - left_distance = ( - position - sorted_positions[i - left_count] if left_count else 0.0 - ) - right_distance = ( - sorted_positions[i + right_count] - position if right_count else 0.0 - ) + left_distance = position - sorted_positions[i - left_count] if left_count else 0.0 + right_distance = sorted_positions[i + right_count] - position if right_count else 0.0 if left_distance < right_distance: left = left_count + 1 else: @@ -2342,11 +2237,7 @@ def _compute_pivot_kth_neighbor_distances( distances[i] = min( max( (position - sorted_positions[i - left_count] if left_count else 0.0), - ( - sorted_positions[i + k - left_count] - position - if k - left_count - else 0.0 - ), + (sorted_positions[i + k - left_count] - position if k - left_count else 0.0), ) for left_count in (left - 1, left) if min_left <= left_count <= max_left @@ -2380,16 +2271,13 @@ def _compute_pivot_sigmas( if bandwidth == FILL_BANDWIDTHS[0] or M <= 1: # "fixed" or trivial return np.full(M, float(sigma_candles), dtype=float) if bandwidth != FILL_BANDWIDTHS[1]: # "knn" - raise ValueError( - enum_error_message("fill_bandwidth", bandwidth, FILL_BANDWIDTHS) - ) + raise ValueError(enum_error_message("fill_bandwidth", bandwidth, FILL_BANDWIDTHS)) d_k = _compute_pivot_kth_neighbor_distances(pivot_indices, neighbors) sigmas = float(alpha) * d_k sigma_max = float(sigma_candles) sigma_min = float(sigma_min_candles) - if sigma_min > sigma_max: - sigma_min = sigma_max + sigma_min = min(sigma_min, sigma_max) return np.clip(sigmas, sigma_min, sigma_max) @@ -2424,15 +2312,11 @@ def _gaussian_fill_weights( while preserving the upper bound ``Out[i] <= max_p w_p``. """ if sigma_candles < 0.5: - raise ValueError( - f"Invalid sigma_candles value {sigma_candles!r}: must be >= 0.5" - ) + raise ValueError(f"Invalid sigma_candles value {sigma_candles!r}: must be >= 0.5") if pivot_indices.size == 0: return np.zeros(n_values, dtype=float) if np.any(pivot_weights < 0.0): - raise ValueError( - f"Invalid pivot_weights min={float(pivot_weights.min())!r}: must be >= 0" - ) + raise ValueError(f"Invalid pivot_weights min={float(pivot_weights.min())!r}: must be >= 0") pivot_indices_array = pivot_indices.astype(float) pivot_weights_array = pivot_weights.astype(float) pivot_sigmas = _compute_pivot_sigmas( @@ -2444,11 +2328,7 @@ def _gaussian_fill_weights( sigma_min_candles=sigma_min_candles, ) M = pivot_indices_array.size - if ( - logger is not None - and n_values > 0 - and M / n_values > _GAUSSIAN_FILL_DENSITY_WARN - ): + if logger is not None and n_values > 0 and M / n_values > _GAUSSIAN_FILL_DENSITY_WARN: logger.warning( "gaussian_fill: pivot density M/N=%.3f > %.2f (M=%d, N=%d); " "consider tightening zigzag detection", @@ -2486,13 +2366,12 @@ def _gaussian_fill_weights( return out out = np.zeros(n_values, dtype=float) - support_radius = math.ceil( - _WEIGHT_FILL_RADIUS_SIGMA_MULTIPLIER * float(sigma_candles) - ) + support_radius = math.ceil(_WEIGHT_FILL_RADIUS_SIGMA_MULTIPLIER * float(sigma_candles)) for pivot, pivot_weight, pivot_sigma in zip( pivot_indices_array, pivot_weights_array, pivot_sigmas, + strict=False, ): if pivot_weight == 0.0: continue @@ -2567,21 +2446,13 @@ def _aggregate_metrics( softmax_temperature: float, ) -> NDArray[np.floating]: if aggregation == COMBINED_AGGREGATIONS[0]: # "arithmetic_mean" - return np.asarray( - sp.stats.pmean(stacked_metrics.T, p=1.0, weights=coefficients, axis=1) - ) + return np.asarray(sp.stats.pmean(stacked_metrics.T, p=1.0, weights=coefficients, axis=1)) elif aggregation == COMBINED_AGGREGATIONS[1]: # "geometric_mean" - return np.asarray( - sp.stats.pmean(stacked_metrics.T, p=0.0, weights=coefficients, axis=1) - ) + return np.asarray(sp.stats.pmean(stacked_metrics.T, p=0.0, weights=coefficients, axis=1)) elif aggregation == COMBINED_AGGREGATIONS[2]: # "harmonic_mean" - return np.asarray( - sp.stats.pmean(stacked_metrics.T, p=-1.0, weights=coefficients, axis=1) - ) + return np.asarray(sp.stats.pmean(stacked_metrics.T, p=-1.0, weights=coefficients, axis=1)) elif aggregation == COMBINED_AGGREGATIONS[3]: # "quadratic_mean" - return np.asarray( - sp.stats.pmean(stacked_metrics.T, p=2.0, weights=coefficients, axis=1) - ) + return np.asarray(sp.stats.pmean(stacked_metrics.T, p=2.0, weights=coefficients, axis=1)) elif aggregation == COMBINED_AGGREGATIONS[4]: # "weighted_median" return np.array( [ @@ -2600,19 +2471,13 @@ def _aggregate_metrics( scaled_metrics = stacked_metrics / softmax_temperature softmax_weights = sp.special.softmax(scaled_metrics, axis=0) combined_weights = softmax_weights * coefficients[:, np.newaxis] - combined_weights = combined_weights / np.sum( - combined_weights, axis=0, keepdims=True - ) + combined_weights = combined_weights / np.sum(combined_weights, axis=0, keepdims=True) return np.sum(stacked_metrics * combined_weights, axis=0) else: - raise ValueError( - enum_error_message("aggregation", aggregation, COMBINED_AGGREGATIONS) - ) + raise ValueError(enum_error_message("aggregation", aggregation, COMBINED_AGGREGATIONS)) -def _invalid_weight_strategy_message( - strategy: str, metrics: dict[str, list[float]] -) -> str: +def _invalid_weight_strategy_message(strategy: str, metrics: dict[str, list[float]]) -> str: return ( f"Invalid weighting strategy value {strategy!r}: " f"supported values are {', '.join(WEIGHT_STRATEGIES)} or metric names {', '.join(metrics.keys())}" @@ -2632,7 +2497,7 @@ def _select_combined_metrics( """ coefficients = _parse_metric_coefficients(metric_coefficients) if len(coefficients) == 0: - coefficients = {k: 1.0 for k in metrics.keys()} + coefficients = dict.fromkeys(metrics, 1.0) selected: list[tuple[str, NDArray[np.floating], float]] = [] for metric_name, metric_values in metrics.items(): @@ -2716,8 +2581,7 @@ def _compute_combined_label_weight_pipeline( ) if expected_length is not None and combined_weights.shape != (expected_length,): raise ValueError( - f"Invalid combined weights shape {combined_weights.shape}: " - f"must be ({expected_length},)" + f"Invalid combined weights shape {combined_weights.shape}: must be ({expected_length},)" ) return _CombinedWeightPipeline(selected, combined_weights) @@ -2826,8 +2690,7 @@ def compute_label_weight_imputation_dependency_mask( return _empty_masks() if values.shape != (n_indices,): raise ValueError( - f"Invalid metric {strategy!r} shape {values.shape}: " - f"must be ({n_indices},)" + f"Invalid metric {strategy!r} shape {values.shape}: must be ({n_indices},)" ) dependency = _nonfinite_imputation_dependency_mask(values) leading_stable = np.zeros(n_indices, dtype=bool) @@ -2874,11 +2737,7 @@ def compute_label_weight_imputation_dependency_mask( leading_stable = np.zeros(n_indices, dtype=bool) release_index = -1 - if ( - every_component_has_finite - and first_finite_indices - and min(first_finite_indices) >= 1 - ): + if every_component_has_finite and first_finite_indices and min(first_finite_indices) >= 1: stable_length = min(first_finite_indices) if bool(np.all(combined_weights[:stable_length] == 0.0)): leading_stable[:stable_length] = True @@ -2907,9 +2766,7 @@ def _compute_epsilon_floor( b = float(np.nanmedian(pivot_values)) else: raise ValueError( - enum_error_message( - "fill_epsilon_baseline", baseline, FILL_EPSILON_BASELINES - ) + enum_error_message("fill_epsilon_baseline", baseline, FILL_EPSILON_BASELINES) ) if not np.isfinite(b): b = 0.0 @@ -2952,12 +2809,9 @@ def _compute_causal_epsilon_fill( """Per-row epsilon floor from pivot weights fixed at their availability.""" if len(known_at_lookahead) != n_values: raise ValueError( - "Invalid known_at_lookahead length " - f"{len(known_at_lookahead)}: must be {n_values}" + f"Invalid known_at_lookahead length {len(known_at_lookahead)}: must be {n_values}" ) - positions, known_at_lookahead_values = _sanitize_known_at_lookahead( - known_at_lookahead - ) + positions, known_at_lookahead_values = _sanitize_known_at_lookahead(known_at_lookahead) if not valid_mask.any(): return np.zeros(n_values, dtype=float) @@ -2999,32 +2853,22 @@ def _compute_causal_epsilon_fill( baseline = label_weighting["fill_epsilon_baseline"] if baseline == FILL_EPSILON_BASELINES[0]: # "mean" - running_baseline = ( - pd.Series(pivot_values).expanding().mean().to_numpy(dtype=float) - ) + running_baseline = pd.Series(pivot_values).expanding().mean().to_numpy(dtype=float) elif baseline == FILL_EPSILON_BASELINES[1]: # "median" - running_baseline = ( - pd.Series(pivot_values).expanding().median().to_numpy(dtype=float) - ) + running_baseline = pd.Series(pivot_values).expanding().median().to_numpy(dtype=float) else: raise ValueError( - enum_error_message( - "fill_epsilon_baseline", baseline, FILL_EPSILON_BASELINES - ) + enum_error_message("fill_epsilon_baseline", baseline, FILL_EPSILON_BASELINES) ) event_ends = _segment_ends(pivot_available_at) availability_events = pivot_available_at[event_ends] event_floors = float(label_weighting["fill_epsilon"]) * running_baseline[event_ends] - available_count = np.searchsorted( - availability_events, known_at_positions, side="right" - ) + available_count = np.searchsorted(availability_events, known_at_positions, side="right") fill_weights = np.zeros(n_values, dtype=float) has_available_pivot = available_count > 0 - fill_weights[has_available_pivot] = event_floors[ - available_count[has_available_pivot] - 1 - ] + fill_weights[has_available_pivot] = event_floors[available_count[has_available_pivot] - 1] return fill_weights @@ -3214,8 +3058,7 @@ def weight_fill_radius(weighting_config: dict[str, Any]) -> int: ): return 0 return math.ceil( - _WEIGHT_FILL_RADIUS_SIGMA_MULTIPLIER - * float(label_weighting["fill_sigma_candles"]) + _WEIGHT_FILL_RADIUS_SIGMA_MULTIPLIER * float(label_weighting["fill_sigma_candles"]) ) @@ -3259,7 +3102,7 @@ def _compute_knn_pivot_sigma_availability( sigma_max = float(sigma_candles) alpha_value = float(alpha) for i, (pivot_position, kth_distance) in enumerate( - zip(pivot_positions, kth_distances) + zip(pivot_positions, kth_distances, strict=False) ): raw_sigma = alpha_value * kth_distance if raw_sigma >= sigma_max: @@ -3331,33 +3174,24 @@ def _compute_knn_pivot_sigma_availability( # atomically. The last replayed pivot's internal confirmation # is hidden by that watermark, but its successor must still be # at least ``pivot_spacing`` candles later. - first_future_pivot_position = ( - int(pivot_positions[bound]) + pivot_spacing - ) + first_future_pivot_position = int(pivot_positions[bound]) + pivot_spacing else: first_future_pivot_position = int(pivot_confirmations[bound]) + 1 has_future = first_future_pivot_position <= last_future_pivot_position confirmed_rank = min(neighbors, confirmed_neighbors) confirmed_closer = max(0, min(prefix_end, closer_right) - closer_left - 1) confirmed_within = max(0, min(prefix_end, within_right) - within_left - 1) - last_future_closer_position = min( - last_future_pivot_position, future_closer_end - ) + last_future_closer_position = min(last_future_pivot_position, future_closer_end) future_closer = ( 0 if last_future_closer_position < first_future_pivot_position - else (last_future_closer_position - first_future_pivot_position) - // pivot_spacing + else (last_future_closer_position - first_future_pivot_position) // pivot_spacing + 1 ) - all_future_within = ( - not has_future or last_future_pivot_position <= future_within_end - ) + all_future_within = not has_future or last_future_pivot_position <= future_within_end if raw_sigma >= sigma_max: - prefix_matches = ( - confirmed_neighbors == 0 or confirmed_closer < confirmed_rank - ) + prefix_matches = confirmed_neighbors == 0 or confirmed_closer < confirmed_rank suffix_matches = ( future_closer == 0 if confirmed_neighbors == 0 @@ -3433,9 +3267,7 @@ def compute_label_weight_known_at_lookahead( dropped pivot) where the run is a contiguous prefix. """ n = len(known_at_lookahead) - positions, known_at_lookahead_values = _sanitize_known_at_lookahead( - known_at_lookahead - ) + positions, known_at_lookahead_values = _sanitize_known_at_lookahead(known_at_lookahead) if n == 0: return pd.Series(positions, index=known_at_lookahead.index, dtype=np.int64) known_at_positions = positions + known_at_lookahead_values @@ -3448,9 +3280,7 @@ def compute_label_weight_known_at_lookahead( return np.zeros(raw_idx.size, dtype=bool) arr = np.asarray(mask) if arr.shape != raw_idx.shape: - raise ValueError( - f"Invalid {name} shape {arr.shape}: must be {raw_idx.shape}" - ) + raise ValueError(f"Invalid {name} shape {arr.shape}: must be {raw_idx.shape}") if arr.dtype != np.bool_: raise ValueError(f"Invalid {name} dtype {arr.dtype}: must be bool") return arr @@ -3511,9 +3341,7 @@ def compute_label_weight_known_at_lookahead( # Prefix max (not weight_availability[stable_release_index]) stays # leak-free if availability is non-monotone, at worst deferring # later; guarded to identity order (contiguous prefix run). - release = int( - np.max(weight_availability[: imputation_stable_release_index + 1]) - ) + release = int(np.max(weight_availability[: imputation_stable_release_index + 1])) avail_pivot[leading_stable_mask] = release base[idx] = np.maximum(base[idx], avail_pivot) if fill_radius > 0: @@ -3529,6 +3357,7 @@ def compute_label_weight_known_at_lookahead( band_weight_availability.tolist(), dependency_mask.tolist(), leading_stable_mask.tolist(), + strict=False, ): # A leading-run pivot imputes to 0.0 (zero bump): its own row is # released above; it spreads no band. @@ -3563,7 +3392,7 @@ def get_callable_sha256(fn: Callable[..., Any]) -> str: code = getattr(fn.__func__, "__code__", None) if code is None and hasattr(fn, "__func__"): code = getattr(fn.__func__, "__code__", None) - if code is None and hasattr(fn, "__call__"): + if code is None and hasattr(fn, "__call__"): # noqa: B004 - Check attribute visibility. code = getattr(fn.__call__, "__code__", None) if code is None: raise ValueError( @@ -3577,7 +3406,7 @@ _SCIENTIFIC_THRESHOLD_LOW = 1e-6 @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) -def format_number(value: int | float, significant_digits: int = 5) -> str: +def format_number(value: float, significant_digits: int = 5) -> str: if not isinstance(value, (int, float, np.integer, np.floating)): return str(value) if isinstance(value, (np.integer, np.floating)): @@ -3592,9 +3421,7 @@ def format_number(value: int | float, significant_digits: int = 5) -> str: abs_value = abs(value) - if abs_value >= _SCIENTIFIC_THRESHOLD_HIGH or ( - 0 < abs_value <= _SCIENTIFIC_THRESHOLD_LOW - ): + if abs_value >= _SCIENTIFIC_THRESHOLD_HIGH or (0 < abs_value <= _SCIENTIFIC_THRESHOLD_LOW): return f"{value:.{significant_digits - 1}e}" if abs_value == 0: @@ -3620,7 +3447,7 @@ _MAX_DEPTH = 2 class _FormatContext: - __slots__ = ("quote_strings", "sig_digits", "seen") + __slots__ = ("quote_strings", "seen", "sig_digits") def __init__(self, quote_strings: bool, sig_digits: int): self.quote_strings = quote_strings @@ -3671,10 +3498,7 @@ def _(value: np.bool_, ctx: _FormatContext, depth: int) -> str: @_format_value.register(str) def _(value: str, ctx: _FormatContext, depth: int) -> str: escaped = ( - value.replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") + value.replace("\\", "\\\\").replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t") ) if len(escaped) > _MAX_STR_LEN: escaped = escaped[:_MAX_STR_LEN] + "..." @@ -3711,8 +3535,7 @@ def _(value: dict, ctx: _FormatContext, depth: int) -> str: ctx.seen.add(obj_id) sep = ": " if ctx.quote_strings else "=" items = [ - f"{k}{sep}{_format_value(v, ctx, depth + 1)}" - for k, v in list(value.items())[:_MAX_ITEMS] + f"{k}{sep}{_format_value(v, ctx, depth + 1)}" for k, v in list(value.items())[:_MAX_ITEMS] ] if len(value) > _MAX_ITEMS: items.append(f"...+{len(value) - _MAX_ITEMS}") @@ -3793,9 +3616,7 @@ def top_log_return( if period < 1: raise ValueError(f"Invalid period value {period!r}: must be >= 1") - previous_close_top = ( - dataframe.get("close").rolling(period, min_periods=period).max().shift(1) - ) + previous_close_top = dataframe.get("close").rolling(period, min_periods=period).max().shift(1) return safe_log_ratio( dataframe.get("close"), @@ -3846,12 +3667,8 @@ def price_retracement_percent( if period < 1: raise ValueError(f"Invalid period value {period!r}: must be >= 1") - previous_close_low = ( - dataframe.get("close").rolling(period, min_periods=period).min().shift(1) - ) - previous_close_high = ( - dataframe.get("close").rolling(period, min_periods=period).max().shift(1) - ) + previous_close_low = dataframe.get("close").rolling(period, min_periods=period).min().shift(1) + previous_close_high = dataframe.get("close").rolling(period, min_periods=period).max().shift(1) denominator = safe_log_ratio( previous_close_high, previous_close_low, @@ -3896,14 +3713,10 @@ def calculate_zero_lag(series: pd.Series, period: int) -> pd.Series: @lru_cache(maxsize=_CACHE_MAXSIZE_SMALL) def get_ma_fn( mamode: str, -) -> Callable[ - [pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating] -]: +) -> Callable[[pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating]]: mamodes: dict[ str, - Callable[ - [pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating] - ], + Callable[[pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating]], ] = { "sma": ta.SMA, "ema": ta.EMA, @@ -3920,9 +3733,7 @@ def get_ma_fn( @lru_cache(maxsize=_CACHE_MAXSIZE_SMALL) def get_zl_ma_fn( mamode: str, -) -> Callable[ - [pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating] -]: +) -> Callable[[pd.Series | NDArray[np.floating], int], pd.Series | NDArray[np.floating]]: ma_fn = get_ma_fn(mamode) return lambda series, timeperiod: ma_fn( calculate_zero_lag(series, timeperiod), timeperiod=timeperiod @@ -3988,9 +3799,7 @@ def frama(df: pd.DataFrame, period: int = 16, zero_lag: bool = False) -> pd.Seri for i in range(period, n): window_highs = highs.iloc[i - period : i] window_lows = lows.iloc[i - period : i] - fd.iloc[i] = _fractal_dimension( - window_highs.to_numpy(), window_lows.to_numpy(), period - ) + fd.iloc[i] = _fractal_dimension(window_highs.to_numpy(), window_lows.to_numpy(), period) alpha = np.exp(-4.6 * (fd - 1)).clip(0.01, 1) @@ -3999,9 +3808,7 @@ def frama(df: pd.DataFrame, period: int = 16, zero_lag: bool = False) -> pd.Seri for i in range(period, n): if pd.isna(frama.iloc[i - 1]) or pd.isna(alpha.iloc[i]): continue - frama.iloc[i] = ( - alpha.iloc[i] * closes.iloc[i] + (1 - alpha.iloc[i]) * frama.iloc[i - 1] - ) + frama.iloc[i] = alpha.iloc[i] * closes.iloc[i] + (1 - alpha.iloc[i]) * frama.iloc[i - 1] return frama @@ -4126,12 +3933,10 @@ def find_fractals(df: pd.DataFrame, period: int = 2) -> tuple[list[int], list[in for i in range(period, n - period): is_high_fractal = all( - highs[i] > highs[i - j] and highs[i] > highs[i + j] - for j in range(1, period + 1) + highs[i] > highs[i - j] and highs[i] > highs[i + j] for j in range(1, period + 1) ) is_low_fractal = all( - lows[i] < lows[i - j] and lows[i] < lows[i + j] - for j in range(1, period + 1) + lows[i] < lows[i - j] and lows[i] < lows[i + j] for j in range(1, period + 1) ) if is_high_fractal: @@ -4226,9 +4031,7 @@ def _zigzag( natr = ta.NATR(df, timeperiod=natr_period) / 100.0 finite_natr_positions = np.flatnonzero(np.isfinite(natr.to_numpy(dtype=float))) - natr_warmup_end_pos = ( - int(finite_natr_positions[0]) if finite_natr_positions.size > 0 else n - ) + natr_warmup_end_pos = int(finite_natr_positions[0]) if finite_natr_positions.size > 0 else n natr_values = natr.bfill().to_numpy() indices: list[int] = df.index.tolist() @@ -4252,9 +4055,7 @@ def _zigzag( invalid_price_count, ) with np.errstate(divide="ignore", invalid="ignore"): - closes_log = np.where( - np.isfinite(closes) & (closes > 0.0), np.log(closes), np.nan - ) + closes_log = np.where(np.isfinite(closes) & (closes > 0.0), np.log(closes), np.nan) highs_log = np.where(np.isfinite(highs) & (highs > 0.0), np.log(highs), np.nan) lows_log = np.where(np.isfinite(lows) & (lows > 0.0), np.log(lows), np.nan) volumes = df.get("volume").to_numpy() @@ -4368,10 +4169,7 @@ def _zigzag( current_pos=current_pos, ) - if np.isfinite(duration) and duration > 0: - speed = amplitude / duration - else: - speed = np.nan + speed = amplitude / duration if np.isfinite(duration) and duration > 0 else np.nan return ( amplitude, @@ -4410,9 +4208,7 @@ def _zigzag( start_pos = min(previous_pos, current_pos) end_pos = max(previous_pos, current_pos) + 1 - avg_volume_per_candle = np.nansum(volumes[start_pos:end_pos]) / ( - end_pos - start_pos - ) + avg_volume_per_candle = np.nansum(volumes[start_pos:end_pos]) / (end_pos - start_pos) median_volume = np.nanmedian(volumes[start_pos:end_pos]) if ( np.isfinite(avg_volume_per_candle) @@ -4468,9 +4264,7 @@ def _zigzag( if not np.isfinite(total_volume) or np.isclose(total_volume, 0.0): return np.nan - vw_close_diffs = np.diff(closes_log[start_pos:end_pos]) * ( - volumes_slice / total_volume - ) + vw_close_diffs = np.diff(closes_log[start_pos:end_pos]) * (volumes_slice / total_volume) vw_path_length = np.nansum(np.abs(vw_close_diffs)) vw_net_move = abs(np.nansum(vw_close_diffs)) @@ -4501,9 +4295,7 @@ def _zigzag( latest_confirmation_pos, ) latest_confirmation_pos = confirmed_at_pos - known_at_positions[last_resolved_pos + 1 : resolve_through_pos + 1] = ( - confirmed_at_pos - ) + known_at_positions[last_resolved_pos + 1 : resolve_through_pos + 1] = confirmed_at_pos last_resolved_pos = max(last_resolved_pos, resolve_through_pos) if pivots_indices and indices[pos] == pivots_indices[-1]: return @@ -4532,11 +4324,9 @@ def _zigzag( previous_pos=last_pivot_pos, current_pos=pos, ) - volume_weighted_efficiency_ratio = ( - calculate_pivot_volume_weighted_efficiency_ratio( - previous_pos=last_pivot_pos, - current_pos=pos, - ) + volume_weighted_efficiency_ratio = calculate_pivot_volume_weighted_efficiency_ratio( + previous_pos=last_pivot_pos, + current_pos=pos, ) pivots_amplitudes[-1] = amplitude @@ -4544,9 +4334,7 @@ def _zigzag( pivots_volume_rates[-1] = volume_rate pivots_speeds[-1] = speed pivots_efficiency_ratios[-1] = efficiency_ratio - pivots_volume_weighted_efficiency_ratios[-1] = ( - volume_weighted_efficiency_ratio - ) + pivots_volume_weighted_efficiency_ratios[-1] = volume_weighted_efficiency_ratio pivots_indices.append(indices[pos]) pivots_values_log.append(value_log) @@ -4627,14 +4415,9 @@ def _zigzag( slopes_ok_threshold = calculate_slopes_ok_threshold(candidate_pivot_pos) n_slopes_ok = sum(slopes_ok) - binomtest = sp.stats.binomtest( - k=n_slopes_ok, n=n_slopes, p=0.5, alternative="greater" - ) + binomtest = sp.stats.binomtest(k=n_slopes_ok, n=n_slopes, p=0.5, alternative="greater") - return ( - binomtest.pvalue <= alpha - and (n_slopes_ok / n_slopes) >= slopes_ok_threshold - ) + return binomtest.pvalue <= alpha and (n_slopes_ok / n_slopes) >= slopes_ok_threshold start_pos = 0 initial_high_pos = start_pos @@ -4713,15 +4496,12 @@ def _zigzag( for i in range(last_pivot_pos + 1, n): if state == TrendDirection.UP: - if ( - np.isnan(candidate_pivot_value_log) - or highs_log[i] > highs_log[candidate_pivot_pos] - ): + if np.isnan(candidate_pivot_value_log) or highs_log[i] > highs_log[candidate_pivot_pos]: update_candidate_pivot(i, highs_log[i]) move_down = abs(lows_log[i] - candidate_pivot_value_log) - if move_down >= np.log1p( - thresholds[candidate_pivot_pos] - ) and is_pivot_confirmed(i, candidate_pivot_pos, TrendDirection.DOWN): + if move_down >= np.log1p(thresholds[candidate_pivot_pos]) and is_pivot_confirmed( + i, candidate_pivot_pos, TrendDirection.DOWN + ): add_pivot( candidate_pivot_pos, highs_log[candidate_pivot_pos], @@ -4732,15 +4512,12 @@ def _zigzag( state = TrendDirection.DOWN elif state == TrendDirection.DOWN: - if ( - np.isnan(candidate_pivot_value_log) - or lows_log[i] < lows_log[candidate_pivot_pos] - ): + if np.isnan(candidate_pivot_value_log) or lows_log[i] < lows_log[candidate_pivot_pos]: update_candidate_pivot(i, lows_log[i]) move_up = abs(highs_log[i] - candidate_pivot_value_log) - if move_up >= np.log1p( - thresholds[candidate_pivot_pos] - ) and is_pivot_confirmed(i, candidate_pivot_pos, TrendDirection.UP): + if move_up >= np.log1p(thresholds[candidate_pivot_pos]) and is_pivot_confirmed( + i, candidate_pivot_pos, TrendDirection.UP + ): add_pivot( candidate_pivot_pos, lows_log[candidate_pivot_pos], @@ -4754,17 +4531,13 @@ def _zigzag( indices=pivots_indices, values_log=pivots_values_log, directions=pivots_directions, - amplitudes=( - minmax_scale(pivots_amplitudes) if normalize else pivots_amplitudes - ), + amplitudes=(minmax_scale(pivots_amplitudes) if normalize else pivots_amplitudes), amplitude_threshold_ratios=( minmax_scale(pivots_amplitude_threshold_ratios) if normalize else pivots_amplitude_threshold_ratios ), - volume_rates=( - minmax_scale(pivots_volume_rates) if normalize else pivots_volume_rates - ), + volume_rates=(minmax_scale(pivots_volume_rates) if normalize else pivots_volume_rates), speeds=minmax_scale(pivots_speeds) if normalize else pivots_speeds, efficiency_ratios=pivots_efficiency_ratios, volume_weighted_efficiency_ratios=pivots_volume_weighted_efficiency_ratios, @@ -4790,9 +4563,7 @@ def zigzag( ).as_tuple() -Regressor = Literal[ - "xgboost", "lightgbm", "histgradientboostingregressor", "ngboost", "catboost" -] +Regressor = Literal["xgboost", "lightgbm", "histgradientboostingregressor", "ngboost", "catboost"] class RegressorSpec(NamedTuple): @@ -4860,13 +4631,9 @@ REGRESSORS: Final[tuple[Regressor, ...]] = tuple(spec.name for spec in _REGRESSO DEFAULT_REGRESSOR: Final[Regressor] = _REGRESSOR_SPECS.xgboost.name if set(_REGRESSOR_SPEC_BY_NAME) != set(get_args(Regressor)): - raise RuntimeError( - "_REGRESSOR_SPECS must define a spec for every Regressor literal member" - ) + raise RuntimeError("_REGRESSOR_SPECS must define a spec for every Regressor literal member") if any(spec.iteration_param not in spec.iteration_aliases for spec in _REGRESSOR_SPECS): - raise RuntimeError( - "each RegressorSpec.iteration_param must be listed in its iteration_aliases" - ) + raise RuntimeError("each RegressorSpec.iteration_param must be listed in its iteration_aliases") RegressorCallback = Callable[..., Any] | XGBoostTrainingCallback @@ -4913,9 +4680,7 @@ def get_ngboost_dist(dist_name: str) -> type: } if dist_name not in dist_map: - raise ValueError( - enum_error_message("dist_name", dist_name, tuple(dist_map.keys())) - ) + raise ValueError(enum_error_message("dist_name", dist_name, tuple(dist_map.keys()))) return dist_map[dist_name] @@ -4932,19 +4697,13 @@ def get_refit_model_training_parameters( if regressor == _REGRESSOR_SPECS.xgboost.name: fitted_iterations = int(model.get_booster().num_boosted_rounds()) initial_iterations = ( - int(init_model.get_booster().num_boosted_rounds()) - if init_model is not None - else 0 + int(init_model.get_booster().num_boosted_rounds()) if init_model is not None else 0 ) elif regressor == _REGRESSOR_SPECS.lightgbm.name: best_iteration = getattr(model, "best_iteration_", 0) or 0 - fitted_iterations = int( - best_iteration if best_iteration > 0 else model.n_estimators_ - ) + fitted_iterations = int(best_iteration if best_iteration > 0 else model.n_estimators_) initial_iterations = ( - int(init_model.booster_.current_iteration()) - if init_model is not None - else 0 + int(init_model.booster_.current_iteration()) if init_model is not None else 0 ) elif regressor == _REGRESSOR_SPECS.histgradientboostingregressor.name: fitted_iterations = int(model.n_iter_) @@ -5028,9 +4787,7 @@ def fit_regressor( from xgboost import XGBRegressor from xgboost.callback import EarlyStopping - early_stopping_rounds = _pop_early_stopping_rounds( - model_training_parameters, has_eval_set - ) + early_stopping_rounds = _pop_early_stopping_rounds(model_training_parameters, has_eval_set) if early_stopping_rounds is not None: fit_callbacks.append( @@ -5064,9 +4821,7 @@ def fit_regressor( elif regressor == _REGRESSOR_SPECS.lightgbm.name: from lightgbm import LGBMRegressor, early_stopping - early_stopping_rounds = _pop_early_stopping_rounds( - model_training_parameters, has_eval_set - ) + early_stopping_rounds = _pop_early_stopping_rounds(model_training_parameters, has_eval_set) if early_stopping_rounds is not None: fit_callbacks.append( @@ -5079,9 +4834,7 @@ def fit_regressor( if trial is not None and has_eval_set: fit_callbacks.append( - optuna.integration.LightGBMPruningCallback( - trial, "rmse", valid_name="valid_0" - ) + optuna.integration.LightGBMPruningCallback(trial, "rmse", valid_name="valid_0") ) model = LGBMRegressor(objective="regression", **model_training_parameters) @@ -5103,16 +4856,12 @@ def fit_regressor( model_training_parameters.pop("n_jobs", None) model_training_parameters.pop("l2_regularization_zero", None) - early_stopping_rounds = model_training_parameters.pop( - "early_stopping_rounds", None - ) + early_stopping_rounds = model_training_parameters.pop("early_stopping_rounds", None) if "n_iter_no_change" not in model_training_parameters: if early_stopping_rounds is not None: model_training_parameters["n_iter_no_change"] = early_stopping_rounds else: - model_training_parameters["n_iter_no_change"] = ( - _EARLY_STOPPING_ROUNDS_DEFAULT - ) + model_training_parameters["n_iter_no_change"] = _EARLY_STOPPING_ROUNDS_DEFAULT _apply_verbosity_alias(model_training_parameters) @@ -5147,9 +4896,7 @@ def fit_regressor( model_training_parameters.pop("n_jobs", None) - early_stopping_rounds = _pop_early_stopping_rounds( - model_training_parameters, has_eval_set - ) + early_stopping_rounds = _pop_early_stopping_rounds(model_training_parameters, has_eval_set) dist = model_training_parameters.pop("dist", "lognormal") @@ -5192,13 +4939,9 @@ def fit_regressor( if trial is not None: trial_path = model_path / f"hp_trial_{trial.number}" trial_path.mkdir(parents=True, exist_ok=True) - model_training_parameters["train_dir"] = str( - trial_path / "catboost_info" - ) + model_training_parameters["train_dir"] = str(trial_path / "catboost_info") else: - model_training_parameters["train_dir"] = str( - model_path / "catboost_info" - ) + model_training_parameters["train_dir"] = str(model_path / "catboost_info") task_type = model_training_parameters.get("task_type", "CPU") loss_function = model_training_parameters.get("loss_function", "RMSE") @@ -5217,9 +4960,7 @@ def fit_regressor( model_training_parameters.setdefault("thread_count", n_jobs) model_training_parameters.setdefault("max_ctr_complexity", 2) - early_stopping_rounds = _pop_early_stopping_rounds( - model_training_parameters, has_eval_set - ) + early_stopping_rounds = _pop_early_stopping_rounds(model_training_parameters, has_eval_set) _apply_verbosity_alias(model_training_parameters) @@ -5244,9 +4985,7 @@ def fit_regressor( early_stopping_rounds=early_stopping_rounds if early_stopping_rounds is not None and has_eval_set else None, - use_best_model=True - if early_stopping_rounds is not None and has_eval_set - else False, + use_best_model=bool(early_stopping_rounds is not None and has_eval_set), callbacks=fit_callbacks if fit_callbacks else None, init_model=init_model, ) @@ -5283,7 +5022,7 @@ def _build_int_range( ) -> tuple[int, int]: lo, hi = math.ceil(frange[0]), math.floor(frange[1]) if lo > hi: - lo = hi = max(min_val, int(round((frange[0] + frange[1]) / 2))) + lo = hi = max(min_val, round((frange[0] + frange[1]) / 2)) return max(min_val, lo), max(min_val, hi) @@ -5314,9 +5053,7 @@ _OPTUNA_BEST_PARAMS_QUARANTINE_TAG: Final[str] = "corrupt" _OPTUNA_BEST_PARAMS_QUARANTINE_TIE_BREAK_LIMIT: Final[int] = 8 -def _optuna_best_params_path( - base_path: Path, pair: str, namespace: OptunaNamespace -) -> Path: +def _optuna_best_params_path(base_path: Path, pair: str, namespace: OptunaNamespace) -> Path: return base_path / f"optuna-{namespace}-best-params-{pair_to_filename(pair)}.json" @@ -5398,13 +5135,9 @@ def _validate_optuna_label_best_params( schema_version = best_params.get("schema_version") if schema_version is None: if logger is not None: - logger.warning( - f"[{pair}] Ignoring Optuna label best params: missing schema_version" - ) + logger.warning(f"[{pair}] Ignoring Optuna label best params: missing schema_version") return None - if isinstance(schema_version, bool) or not isinstance( - schema_version, (int, np.integer) - ): + if isinstance(schema_version, bool) or not isinstance(schema_version, (int, np.integer)): if logger is not None: logger.warning( f"[{pair}] Ignoring Optuna label best params: invalid " @@ -5424,8 +5157,7 @@ def _validate_optuna_label_best_params( if not isinstance(selection_metadata, dict): if logger is not None: logger.warning( - f"[{pair}] Ignoring Optuna label best params: missing or invalid " - f"selection_metadata" + f"[{pair}] Ignoring Optuna label best params: missing or invalid selection_metadata" ) return None selection_schema_version = selection_metadata.get("schema_version") @@ -5536,12 +5268,10 @@ def _quarantine_corrupt_optuna_best_params( ) try: best_params_path.rename(quarantine_path) - except OSError as quarantine_error: + except OSError: if logger is not None: - logger.error( - f"[{pair}] Optuna {namespace} best params " - f"{best_params_path.name} quarantine failed: {quarantine_error!r}", - exc_info=True, + logger.exception( + f"[{pair}] Optuna {namespace} best params {best_params_path.name} quarantine failed" ) raise if logger is not None: @@ -5590,9 +5320,7 @@ def _locked_optuna_best_params( def _reject_optuna_best_params_symlink(best_params_path: Path) -> None: """Fail closed when the live best-params path is a symlink.""" if best_params_path.is_symlink(): - raise OSError( - f"Optuna best params path {best_params_path} must not be a symlink" - ) + raise OSError(f"Optuna best params path {best_params_path} must not be a symlink") def optuna_load_best_params( @@ -5742,23 +5470,18 @@ def optuna_save_best_params( json.dump(best_params, write_file, indent=4) write_file.flush() os.fsync(write_file.fileno()) - os.replace(temporary_path, best_params_path) + temporary_path.replace(best_params_path) temporary_path = None except BaseException as error: if temporary_path is not None: try: temporary_path.unlink(missing_ok=True) - except OSError as cleanup_error: - logger.error( - f"[{pair}] Optuna {namespace} best params temporary file " - f"{temporary_path.name} cleanup failed: {cleanup_error!r}", - exc_info=True, + except OSError: + logger.exception( + f"[{pair}] Optuna {namespace} best params temporary file {temporary_path.name} cleanup failed" ) if isinstance(error, Exception): - logger.error( - f"[{pair}] Optuna {namespace} failed to save best params: {error!r}", - exc_info=True, - ) + logger.exception(f"[{pair}] Optuna {namespace} failed to save best params") raise @@ -5772,12 +5495,8 @@ def get_optuna_study_model_parameters( ) -> dict[str, Any]: if regressor not in set(REGRESSORS): raise ValueError(enum_error_message("regressor", regressor, REGRESSORS)) - if not isinstance(space_fraction, (int, float)) or not ( - 0.0 <= space_fraction <= 1.0 - ): - raise ValueError( - f"Invalid space_fraction: must be in range [0, 1], got {space_fraction!r}" - ) + if not isinstance(space_fraction, (int, float)) or not (0.0 <= space_fraction <= 1.0): + raise ValueError(f"Invalid space_fraction: must be in range [0, 1], got {space_fraction!r}") def _build_ranges( default_ranges: dict[str, tuple[float, float]], @@ -5789,17 +5508,11 @@ def get_optuna_study_model_parameters( center_value = model_training_best_parameters.get(param) if center_value is None: # Use geometric mean for log-scaled params - if ( - param in log_scaled_params - and default_min > 0 - and default_max > 0 - ): + if param in log_scaled_params and default_min > 0 and default_max > 0: center_value = math.sqrt(default_min * default_max) else: center_value = midpoint(default_min, default_max) - if not isinstance(center_value, (int, float)) or not np.isfinite( - center_value - ): + if not isinstance(center_value, (int, float)) or not np.isfinite(center_value): continue if param in log_scaled_params: if center_value <= 0: @@ -5855,9 +5568,7 @@ def get_optuna_study_model_parameters( ranges = _build_ranges(default_ranges, log_scaled_params) booster = trial.suggest_categorical("booster", ["gbtree", "dart"]) - grow_policy = trial.suggest_categorical( - "grow_policy", ["depthwise", "lossguide"] - ) + grow_policy = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"]) params: dict[str, Any] = { # Boosting/Training @@ -5920,9 +5631,7 @@ def get_optuna_study_model_parameters( "reg_lambda": trial.suggest_float( "reg_lambda", ranges["reg_lambda"][0], ranges["reg_lambda"][1], log=True ), - "gamma": trial.suggest_float( - "gamma", ranges["gamma"][0], ranges["gamma"][1], log=True - ), + "gamma": trial.suggest_float("gamma", ranges["gamma"][0], ranges["gamma"][1], log=True), # Binning "max_bin": _optuna_suggest_int_from_range( trial, "max_bin", ranges["max_bin"], min_val=2 @@ -6043,9 +5752,7 @@ def get_optuna_study_model_parameters( params["drop_rate"] = trial.suggest_float("drop_rate", 0.0, 0.5) params["skip_drop"] = trial.suggest_float("skip_drop", 0.0, 0.7) params["max_drop"] = trial.suggest_int("max_drop", 10, 100) - params["uniform_drop"] = trial.suggest_categorical( - "uniform_drop", [False, True] - ) + params["uniform_drop"] = trial.suggest_categorical("uniform_drop", [False, True]) return params @@ -6081,9 +5788,7 @@ def get_optuna_study_model_parameters( ranges = _build_ranges(default_ranges, log_scaled_params) - l2_regularization_zero = trial.suggest_categorical( - "l2_regularization_zero", [False, True] - ) + l2_regularization_zero = trial.suggest_categorical("l2_regularization_zero", [False, True]) if l2_regularization_zero: l2_regularization = 0.0 else: @@ -6094,9 +5799,7 @@ def get_optuna_study_model_parameters( log=True, ) - max_depth = trial.suggest_categorical( - "max_depth", [None, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15] - ) + max_depth = trial.suggest_categorical("max_depth", [None, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15]) max_leaf_nodes_range = ranges["max_leaf_nodes"] if isinstance(max_depth, int) and max_depth > 0: @@ -6226,9 +5929,7 @@ def get_optuna_study_model_parameters( loss_function = model_training_parameters.get("loss_function", "RMSE") if task_type == "GPU": - gpu_vram_gb = model_training_parameters.get( - "gpu_vram_gb", _CATBOOST_GPU_VRAM_DEFAULT - ) + gpu_vram_gb = model_training_parameters.get("gpu_vram_gb", _CATBOOST_GPU_VRAM_DEFAULT) matched_vram_gb = max( (v for v in _CATBOOST_GPU_VRAM_PARAM_RANGES if v <= gpu_vram_gb), default=min(_CATBOOST_GPU_VRAM_PARAM_RANGES.keys()), @@ -6289,17 +5990,13 @@ def get_optuna_study_model_parameters( ranges = _build_ranges(default_ranges, log_scaled_params) - boosting_type = trial.suggest_categorical( - "boosting_type", boosting_type_options - ) + boosting_type = trial.suggest_categorical("boosting_type", boosting_type_options) bootstrap_type = trial.suggest_categorical("bootstrap_type", bootstrap_options) grow_policy = trial.suggest_categorical( "grow_policy", ["SymmetricTree", "Depthwise", "Lossguide"] ) if boosting_type == "Ordered" and grow_policy != "SymmetricTree": - raise optuna.TrialPruned( - "Ordered boosting is not supported for nonsymmetric trees" - ) + raise optuna.TrialPruned("Ordered boosting is not supported for nonsymmetric trees") params: dict[str, Any] = { # Boosting/Training @@ -6314,9 +6011,7 @@ def get_optuna_study_model_parameters( log=True, ), # Tree structure - "depth": _optuna_suggest_int_from_range( - trial, "depth", ranges["depth"], min_val=1 - ), + "depth": _optuna_suggest_int_from_range(trial, "depth", ranges["depth"], min_val=1), "min_data_in_leaf": _optuna_suggest_int_from_range( trial, "min_data_in_leaf", ranges["min_data_in_leaf"], min_val=1 ), @@ -6386,9 +6081,7 @@ def get_optuna_study_model_parameters( @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) def largest_divisor_to_step(integer: int, step: int) -> int | None: if not isinstance(integer, int) or integer <= 0: - raise ValueError( - f"Invalid integer value {integer!r}: must be a positive integer" - ) + raise ValueError(f"Invalid integer value {integer!r}: must be a positive integer") if not isinstance(step, int) or step <= 0: raise ValueError(f"Invalid step value {step!r}: must be a positive integer") @@ -6396,7 +6089,7 @@ def largest_divisor_to_step(integer: int, step: int) -> int | None: return integer best_divisor: int | None = None - max_divisor = int(math.isqrt(integer)) + max_divisor = math.isqrt(integer) for i in range(1, max_divisor + 1): if integer % i != 0: continue @@ -6485,7 +6178,7 @@ def get_min_max_label_period_candles( return low, high, candles_step -def _validate_step_args(value: float | int, step: int) -> None: +def _validate_step_args(value: float, step: int) -> None: if not isinstance(value, (int, float)): raise ValueError(f"Invalid value {value!r}: must be an integer or float") if not isinstance(step, int) or step <= 0: @@ -6493,7 +6186,7 @@ def _validate_step_args(value: float | int, step: int) -> None: @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) -def round_to_step(value: float | int, step: int) -> int: +def round_to_step(value: float, step: int) -> int: """ Round a value to the nearest multiple of a given step. :param value: The value to round. @@ -6516,7 +6209,7 @@ def round_to_step(value: float | int, step: int) -> int: def _step_round( - value: float | int, + value: float, step: int, int_op: Callable[[int, int], int], float_op: Callable[[float], int], @@ -6530,12 +6223,12 @@ def _step_round( @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) -def ceil_to_step(value: float | int, step: int) -> int: +def ceil_to_step(value: float, step: int) -> int: return _step_round(value, step, lambda v, s: -(-v // s), math.ceil) @lru_cache(maxsize=_CACHE_MAXSIZE_LARGE) -def floor_to_step(value: float | int, step: int) -> int: +def floor_to_step(value: float, step: int) -> int: return _step_round(value, step, lambda v, s: v // s, math.floor) @@ -6568,9 +6261,7 @@ def get_label_defaults( default_label_natr_multiplier = float( midpoint(min_label_natr_multiplier, max_label_natr_multiplier) ) - feature_parameters.setdefault( - "label_natr_multiplier", default_label_natr_multiplier - ) + feature_parameters.setdefault("label_natr_multiplier", default_label_natr_multiplier) min_label_period_candles = feature_parameters.get( "min_label_period_candles", default_min_label_period_candles @@ -6589,8 +6280,8 @@ def get_label_defaults( non_negative=True, finite_only=True, ) - default_label_period_candles = int( - round(midpoint(min_label_period_candles, max_label_period_candles)) + default_label_period_candles = round( + midpoint(min_label_period_candles, max_label_period_candles) ) return default_label_period_candles, default_label_natr_multiplier diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..e2cf39b --- /dev/null +++ b/ruff.toml @@ -0,0 +1,30 @@ +required-version = ">=0.16.5" +line-length = 100 +target-version = "py310" + +[lint] +allowed-confusables = ["γ", "σ"] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # Pyflakes + "I", # isort + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "UP", # pyupgrade + "SIM", # flake8-simplify + "TC", # flake8-type-checking + "PTH", # flake8-use-pathlib + "RUF", # Ruff-specific rules +] +ignore = [ + "E111", # Conflicts with the formatter + "E114", # Conflicts with the formatter + "E117", # Conflicts with the formatter + "E501", # The formatter applies pragmatic line wrapping + "W191", # Conflicts with the formatter +] + +[format] +quote-style = "double" +indent-style = "space"