fix(qa): tolerate tool version drift in snapshot comparison

Compare snapshots with basedpyrightVersion normalized out: a tool upgrade that changes no diagnostics no longer fails the check. The recorded version stays in snapshots and --write output as provenance, is still validated nonempty on both sides, still appears in the failure diff, and both versions are reported in the success message. Diagnostics, file inventory and schema stay byte-exact, so changed output still fails closed. Covers scripts/test_check_basedpyright.py (stdlib unittest): version-only drift matches, diagnostic change mismatches, empty version and missing key rejected.

diff --git a/scripts/check_basedpyright.py b/scripts/check_basedpyright.py
index b1138f08b3445b886f62f93980b13d6af47408dc..9b7232c8cf8f952b678a1c4128bdcd39cb448c86 100755 (executable)
--- a/scripts/check_basedpyright.py
+++ b/scripts/check_basedpyright.py
@@ -535,6 +535,19 @@ def _canonical_bytes(snapshot: Mapping[str, object]) -> bytes:
         raise QualityCheckError(f"Snapshot cannot be serialized: {error}") from error
 
 
+def _comparison_key(snapshot: Mapping[str, object]) -> bytes:
+    """Canonical comparison bytes with the tool version normalized out.
+
+    The recorded ``basedpyrightVersion`` stays in stored snapshots and in
+    ``--write`` output as provenance, but a tool upgrade that changes no
+    diagnostics must not fail the check: only the diagnostics (plus the
+    file inventory) form the acceptance contract.
+    """
+    comparable = dict(snapshot)
+    comparable["basedpyrightVersion"] = ""
+    return _canonical_bytes(comparable)
+
+
 def _validate_environment(project: Project) -> Path:
     marker = os.environ.get(QA_PROJECT_ENV)
     if marker != project.qa_marker:
@@ -675,11 +688,13 @@ def _check_project(project_name: str, *, write: bool) -> int:
 
     stored_snapshot = _read_snapshot(baseline)
     stored_bytes = _canonical_bytes(stored_snapshot)
-    if stored_bytes == current_bytes:
+    if _comparison_key(stored_snapshot) == _comparison_key(current_snapshot):
         print(
             f"BasedPyright snapshot matches for {project_name}: "
             f"{len(current_snapshot['diagnostics'])} diagnostics across "
-            f"{current_snapshot['filesAnalyzed']} analyzed files"
+            f"{current_snapshot['filesAnalyzed']} analyzed files "
+            f"(snapshot tool {stored_snapshot['basedpyrightVersion']}, "
+            f"current tool {current_snapshot['basedpyrightVersion']})"
         )
         return 0
 

diff --git a/scripts/test_check_basedpyright.py b/scripts/test_check_basedpyright.py
new file mode 100644 (file)
index 0000000..dce0176
--- /dev/null
+++ b/scripts/test_check_basedpyright.py
@@ -0,0 +1,52 @@
+"""Unit tests for the snapshot comparison logic in check_basedpyright."""
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from check_basedpyright import (
+    SCHEMA_VERSION,
+    QualityCheckError,
+    _comparison_key,
+    _validate_snapshot,
+)
+
+SOURCE = "scripts/check_basedpyright.py"
+
+
+def _snapshot(*, version, message="boom"):
+    return {
+        "schemaVersion": SCHEMA_VERSION,
+        "basedpyrightVersion": version,
+        "filesAnalyzed": 1,
+        "sourceFiles": [SOURCE],
+        "diagnostics": [{"file": SOURCE, "severity": "error", "message": message}],
+    }
+
+
+class ComparisonKeyTest(unittest.TestCase):
+    def test_version_drift_only_matches(self):
+        stored = _validate_snapshot(_snapshot(version="1.40.0"))
+        current = _validate_snapshot(_snapshot(version="1.40.1"))
+        self.assertEqual(_comparison_key(stored), _comparison_key(current))
+
+    def test_diagnostic_change_mismatches(self):
+        stored = _validate_snapshot(_snapshot(version="1.40.1"))
+        current = _validate_snapshot(_snapshot(version="1.40.1", message="bam"))
+        self.assertNotEqual(_comparison_key(stored), _comparison_key(current))
+
+    def test_empty_version_rejected(self):
+        with self.assertRaises(QualityCheckError):
+            _validate_snapshot(_snapshot(version=""))
+
+    def test_missing_key_rejected(self):
+        snapshot = _snapshot(version="1.40.1")
+        del snapshot["diagnostics"]
+        with self.assertRaises(QualityCheckError):
+            _validate_snapshot(snapshot)
+
+
+if __name__ == "__main__":
+    unittest.main()