test(storage): verify authentic historical SQLite upgrades

crates/test-suite/Cargo.toml

@@ -22,6 +22,8 @@ rusqlite = { version = "0.40.1", default-features = false, features = ["bundled"
 directories = "6.0.0"
 
 [dev-dependencies]
+blake3 = "1.8.2"
+rusqlite_migration = "2.6.0"
 rmcp = { version = "=3.1.2", default-features = false, features = ["client", "transport-async-rw"] }
 tempfile = "3.23.0"
 tokio = { version = "1.49.0", features = ["test-util"] }

crates/test-suite/fixtures/storage-migrations/.gitignore

@@ -0,0 +1,3 @@
+# Immutable release compatibility databases are deliberate repository fixtures.
+!v*.sqlite
+__pycache__/

crates/test-suite/fixtures/storage-migrations/compatibility.json

@@ -0,0 +1,92 @@
+{
+  "format": 1,
+  "latest_migration": 13,
+  "latest_schema_version": 12,
+  "released_through": 13,
+  "release_anchors": [
+    {
+      "version": "0.1.15",
+      "metadata_blake3": "566d221816e9581d049090e6e09fd36113fdca50d328030901f5ca279e402e40"
+    },
+    {
+      "version": "0.1.27",
+      "metadata_blake3": "4f0571003515799a3e98e1e95e6a815986d6dd01890b2a32de7c3f56e1d7c6a6"
+    },
+    {
+      "version": "0.1.28",
+      "metadata_blake3": "b79e25532d894e41c4aa3f5924d86c88be91d35b11fed226354890569f37fdf6"
+    }
+  ],
+  "allowed_schema_differences": [
+    "token_savings column order: historical ALTER TABLE appends accounting columns; column definitions and key/index order remain checked",
+    "meta column order: runtime database_identity precedes scope columns in v0.1.27; all declarations, defaults, checks and key/index order remain checked"
+  ],
+  "migrations": [
+    {
+      "version": 1,
+      "name": "SCHEMA_SQL",
+      "blake3": "254cecad5b78fcc2a1084211d2ec150781ea5e53d2f2d40b4f3eb47c496c8197"
+    },
+    {
+      "version": 2,
+      "name": "LOOKUP_INDEXES_SQL",
+      "blake3": "45a00433b655b0de0e0596a661b23bf375aba79bf353b4d44a1d4877cce6cb93"
+    },
+    {
+      "version": 3,
+      "name": "REPOSITORY_OWNERSHIP_SQL",
+      "blake3": "8eedf01529e145cd1a5118866e1e5bd5af165655c9158fadd120714eb015eac1"
+    },
+    {
+      "version": 4,
+      "name": "IMPORT_CANDIDATES_SQL",
+      "blake3": "ee65e3784988fbcb3fcd68b9b4c25b1745d9b10ddcd2cc2e1c3ab55229e7ada0"
+    },
+    {
+      "version": 5,
+      "name": "PATH_PROJECTION_SQL",
+      "blake3": "12037b370f53c81ed9f77048b9598b6ed66c71cfebe222f55fbc3fe85724154f"
+    },
+    {
+      "version": 6,
+      "name": "CACHE_ACCESS_SQL",
+      "blake3": "5432de233212cd91f8f4c4e149b37a2dc8f7fce19cd9b6160a29bf1627734d7e"
+    },
+    {
+      "version": 7,
+      "name": "STRUCTURAL_SEARCH_SQL",
+      "blake3": "7408b57c754126115dc1e493dc3a96ebad2548ebce4e4094c52ffca754b90b09"
+    },
+    {
+      "version": 8,
+      "name": "RETRIEVAL_RECEIPTS_SQL",
+      "blake3": "277e65897e23bfda2f4a15d4c65b62acd0c6b40f8f1d3d4439f0acb6cce77762"
+    },
+    {
+      "version": 9,
+      "name": "READ_DELTA_BASES_SQL",
+      "blake3": "4adaf5c2021944a94af8cf27763dd1a9286392508deaa0981793ab82206491d9"
+    },
+    {
+      "version": 10,
+      "name": "RECEIPT_EXACT_ONLY_SQL",
+      "blake3": "1611038f81218f60e58322171136b31377b59d36516b99d9cc3f20c210648f95"
+    },
+    {
+      "version": 11,
+      "name": "QUERY_COVERAGE_RECEIPTS_SQL",
+      "blake3": "c1bc327e7dcd16a616304ad0efc49ea108fd3882c788a706418ab426dd35e0d1"
+    },
+    {
+      "version": 12,
+      "name": "INDEX_DERIVATION_IDENTITY_SQL",
+      "blake3": "b2e297d542a2ba608bd0e68fbb75b04abc629ce5a48ed39548d260c8a0415c00"
+    },
+    {
+      "version": 13,
+      "name": "INDEX_SCOPE_METADATA_SQL",
+      "blake3": "b62393fcf80b7ec434194f70edbc0fade4d8b5ee047c71ee1644a2824ebaaf83"
+    }
+  ],
+  "release_fixtures_reviewed_through": 13
+}

crates/test-suite/fixtures/storage-migrations/generate.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Explicitly refresh immutable release anchors using published Linux binaries.
+
+Run --prepare to download/verify without executing Rust. Run without arguments
+to create the tiny repository and databases. Tests never run this generator,
+download artifacts, or need historical binaries. Review fixture/manifest diffs,
+then explicitly update release_anchors[*].metadata_blake3 in compatibility.json.
+Generator-only Python requirement: blake3==1.0.11.
+"""
+
+import hashlib
+import json
+import os
+import pathlib
+import resource
+import shutil
+import sqlite3
+import struct
+import subprocess
+import sys
+import tarfile
+import tempfile
+import tomllib
+import urllib.request
+
+from blake3 import blake3  # Generator-only dependency: blake3==1.0.11.
+
+HERE = pathlib.Path(__file__).resolve().parent
+WORKSPACE = HERE.parents[3]
+CACHE = pathlib.Path(tempfile.gettempdir()) / "leantoken-release-migration-generators"
+ARCHIVE = "leantoken-x86_64-unknown-linux-gnu.tar.xz"
+RELEASES = {
+    "0.1.15": "6934ba9ff0c93e565d556f6f35750f8fe5a91faa",
+    "0.1.27": "910ed8a34fbdd60e1c25c916e9ded12c8bf439e3",
+    "0.1.28": "62a0ee259508272fa0f0947efcba0e80bbcfc141",
+}
+SOURCE = 'use std::fmt;\npub fn legacy_widget() { println!("migración"); }\n'
+
+
+def sha256(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def git_bytes(revision, path):
+    return subprocess.check_output(["git", "show", f"{revision}:{path}"], cwd=WORKSPACE)
+
+
+def prepare(version, commit):
+    destination = CACHE / version
+    destination.mkdir(parents=True, exist_ok=True)
+    url = f"https://github.com/morluto/leantoken/releases/download/v{version}/{ARCHIVE}"
+    checksum = urllib.request.urlopen(url + ".sha256").read().decode().split()[0]
+    archive = destination / ARCHIVE
+    if not archive.exists():
+        with urllib.request.urlopen(url) as source, archive.open("wb") as output:
+            shutil.copyfileobj(source, output)
+    assert sha256(archive.read_bytes()) == checksum, "published archive checksum mismatch"
+    with tarfile.open(archive, "r:xz") as bundle:
+        members = [item for item in bundle.getmembers() if pathlib.PurePosixPath(item.name).name == "leantoken"]
+        assert len(members) == 1
+        binary = destination / "leantoken"
+        binary.write_bytes(bundle.extractfile(members[0]).read())
+        binary.chmod(0o755)
+    resolved = subprocess.check_output(["git", "rev-parse", f"v{version}^{{}}"], cwd=WORKSPACE).decode().strip()
+    assert resolved == commit, "release tag moved; compatibility decision required"
+    schema_path = "src/storage.rs" if version == "0.1.15" else "src/storage/schema.rs"
+    lock = git_bytes(commit, "Cargo.lock")
+    packages = {package["name"]: package["version"] for package in tomllib.loads(lock.decode())["package"]}
+    cli_path = "src/cli.rs" if version == "0.1.15" else "src/cli/mod.rs"
+    supports_max_workers = b"max_index_workers" in git_bytes(commit, cli_path)
+    metadata = {
+        "version": version,
+        "tag": f"v{version}",
+        "commit": commit,
+        "archive_url": url,
+        "archive_sha256": checksum,
+        "binary_sha256": sha256(binary.read_bytes()),
+        "schema_source": schema_path,
+        "schema_source_sha256": sha256(git_bytes(commit, schema_path)),
+        "cargo_lock_sha256": sha256(lock),
+        "rusqlite": packages["rusqlite"],
+        "libsqlite3_sys": packages["libsqlite3-sys"],
+        "rusqlite_migration": packages["rusqlite_migration"],
+        "max_index_workers_supported": supports_max_workers,
+        "generator_target": "x86_64-unknown-linux-gnu",
+        "fixture": f"v{version}.sqlite",
+        "source": SOURCE,
+        "command": ["leantoken", "--root", "<temporary>/repo", "--database", "<fixture>.sqlite", "--json", "index"],
+        "seed_scenario": "one Rust source with import, function, reference, UTF-8 content; indexed by the released executable",
+        "supported_until": "explicit compatibility removal decision with changelog entry; retain previous released format before schema release",
+    }
+    (destination / "prepared.json").write_text(json.dumps(metadata, indent=2) + "\n")
+    return binary, metadata
+
+
+def generate(version, binary, metadata):
+    root = pathlib.Path(tempfile.mkdtemp(prefix=f"leantoken-release-{version}-"))
+    try:
+        repository = root / "repo"
+        (repository / "src").mkdir(parents=True)
+        (repository / "src/lib.rs").write_text(SOURCE)
+        database = root / "index.sqlite"
+        command = [str(binary), "--root", str(repository), "--database", str(database), "--json", "index"]
+        if metadata["max_index_workers_supported"]:
+            command += ["--max-index-workers", "1"]
+            metadata["command"] += ["--max-index-workers", "1"]
+        environment = dict(os.environ)
+        for variable in ("HOME", "USERPROFILE", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "APPDATA", "LOCALAPPDATA", "TMP", "TMPDIR", "TEMP"):
+            isolated = root / "environment" / variable.lower()
+            isolated.mkdir(parents=True)
+            environment[variable] = str(isolated)
+        (root / "generation.json").write_text(json.dumps(metadata, indent=2) + "\n")
+        result = subprocess.run(command, check=True, capture_output=True, text=True, timeout=60, env=environment, preexec_fn=bound_output_files)
+        (root / "generator.stdout").write_text(result.stdout)
+        (root / "generator.stderr").write_text(result.stderr)
+        metadata["generator_stdout"] = json.loads(result.stdout)
+        assert database.exists(), result
+        payload = database.read_bytes()
+        wal_path = pathlib.Path(str(database) + "-wal")
+        wal = wal_path.read_bytes() if wal_path.exists() else b""
+        assert len(payload) <= 1024 * 1024, "release fixture exceeded 1 MiB"
+        assert len(wal) <= 4 * 1024 * 1024, "release WAL exceeded 4 MiB"
+        # A completed released process may leave a committed WAL. Preserve both
+        # durable byte streams verbatim instead of checkpointing with Python's
+        # different SQLite build. SHM is a derived, non-durable WAL index.
+        metadata["sqlite_writer_version_number"] = writer_version(payload, wal)
+        metadata["wal_snapshot_policy"] = "released main plus committed WAL bytes, no checkpoint or Python writes; SHM is derived"
+        metadata["wal_fixture"] = f"v{version}.released-wal" if wal else None
+        metadata["wal_bytes"] = len(wal)
+        metadata["wal_sha256"] = sha256(wal) if wal else None
+        metadata["wal_blake3"] = blake3(wal).hexdigest() if wal else None
+        # Read-only inspection includes committed WAL. Save original bytes first:
+        # connection close may remove SHM/WAL bookkeeping, never our artifact.
+        with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
+            metadata["migration_version"] = connection.execute("PRAGMA user_version").fetchone()[0]
+            metadata["schema_version"] = connection.execute("SELECT schema_version FROM meta").fetchone()[0]
+            schema = connection.execute("SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name").fetchall()
+            metadata["schema_sha256"] = sha256(json.dumps(schema, ensure_ascii=False, separators=(",", ":")).encode())
+            assert connection.execute("PRAGMA integrity_check").fetchall() == [("ok",)]
+            assert connection.execute("SELECT count(*) FROM files").fetchone()[0] == 1
+        metadata["fixture_sha256"] = sha256(payload)
+        metadata["fixture_blake3"] = blake3(payload).hexdigest()
+        metadata["fixture_bytes"] = len(payload)
+        (HERE / metadata["fixture"]).write_bytes(payload)
+        if wal:
+            (HERE / metadata["wal_fixture"]).write_bytes(wal)
+        (HERE / f"v{version}.json").write_text(json.dumps(metadata, indent=2, ensure_ascii=False) + "\n")
+    except Exception as failure:
+        for stream in ("stdout", "stderr"):
+            output = getattr(failure, stream, None)
+            if output:
+                (root / f"generator.{stream}").write_text(output if isinstance(output, str) else output.decode(errors="replace"))
+        print(f"Release generation failed; disposable main/WAL/source/log evidence preserved: {root}", file=sys.stderr)
+        raise
+    else:
+        shutil.rmtree(root)
+
+
+def bound_output_files():
+    # The fixture has one small source file; a4MiB/file hard limit also bounds failed
+    # main/WAL generation. This generator already requires Linux release assets.
+    resource.setrlimit(resource.RLIMIT_FSIZE, (4 * 1024 * 1024, 4 * 1024 * 1024))
+
+
+def writer_version(database, wal):
+    """Read SQLite's writer identity from page1 of the committed WAL snapshot."""
+    page_one = database[:100]
+    if wal:
+        assert len(wal) >= 32, "truncated WAL header"
+        page_size = struct.unpack(">I", wal[8:12])[0]
+        assert 512 <= page_size <= 65536
+        frame_size = 24 + page_size
+        assert (len(wal) - 32) % frame_size == 0, "truncated WAL frame"
+        frames = [wal[offset:offset + frame_size] for offset in range(32, len(wal), frame_size)]
+        committed = [index for index, frame in enumerate(frames) if struct.unpack(">I", frame[4:8])[0] != 0]
+        if committed:
+            for frame in frames[:committed[-1] + 1]:
+                if struct.unpack(">I", frame[:4])[0] == 1:
+                    page_one = frame[24:124]
+    assert page_one[:16] == b"SQLite format 3\0", "missing committed database header"
+    return struct.unpack(">I", page_one[96:100])[0]
+
+
+if __name__ == "__main__":
+    for version, commit in RELEASES.items():
+        binary, metadata = prepare(version, commit)
+        if "--prepare" not in sys.argv:
+            generate(version, binary, metadata)
+        print(f"{version}: {'prepared' if '--prepare' in sys.argv else 'generated'} {metadata['fixture']}")

crates/test-suite/fixtures/storage-migrations/v0.1.15.json

@@ -0,0 +1,57 @@
+{
+  "version": "0.1.15",
+  "tag": "v0.1.15",
+  "commit": "6934ba9ff0c93e565d556f6f35750f8fe5a91faa",
+  "archive_url": "https://github.com/morluto/leantoken/releases/download/v0.1.15/leantoken-x86_64-unknown-linux-gnu.tar.xz",
+  "archive_sha256": "609d0c482d787114995f960f3b4c5a7f1d707c07a121cfbe8b48963d216eebad",
+  "binary_sha256": "37e523734fff730ee4e9590f635704f29c76625644342e91595df00df7369e45",
+  "schema_source": "src/storage.rs",
+  "schema_source_sha256": "159ac431d70e2cab4a4d090b59d8ea2d221ce25d548e46ab21b9ef344305ce93",
+  "cargo_lock_sha256": "81b86188a449ce3af7fefc1e57ac93918232af7ccc1b9ed41b04fdc67afe2197",
+  "rusqlite": "0.40.1",
+  "libsqlite3_sys": "0.38.1",
+  "rusqlite_migration": "2.6.0",
+  "max_index_workers_supported": true,
+  "generator_target": "x86_64-unknown-linux-gnu",
+  "fixture": "v0.1.15.sqlite",
+  "source": "use std::fmt;\npub fn legacy_widget() { println!(\"migración\"); }\n",
+  "command": [
+    "leantoken",
+    "--root",
+    "<temporary>/repo",
+    "--database",
+    "<fixture>.sqlite",
+    "--json",
+    "index",
+    "--max-index-workers",
+    "1"
+  ],
+  "seed_scenario": "one Rust source with import, function, reference, UTF-8 content; indexed by the released executable",
+  "supported_until": "explicit compatibility removal decision with changelog entry; retain previous released format before schema release",
+  "generator_stdout": {
+    "repository_generation": 1,
+    "files_seen": 1,
+    "files_indexed": 1,
+    "files_unchanged": 0,
+    "files_removed": 0,
+    "files_skipped": 0,
+    "warnings": [],
+    "skip_reasons": {
+      "binary": 0,
+      "oversized_during_read": 0,
+      "failed": 0
+    }
+  },
+  "sqlite_writer_version_number": 3053002,
+  "wal_snapshot_policy": "released main plus committed WAL bytes, no checkpoint or Python writes; SHM is derived",
+  "wal_fixture": "v0.1.15.released-wal",
+  "wal_bytes": 341992,
+  "wal_sha256": "cd371222a86029513eab2531e1e969df9faed4e4f40d3a75b8b2b9d919864ba4",
+  "wal_blake3": "76b25a2d9bda6e2fc9afc7a91072215cc3f95d00ed30facdafc50bb15c97a474",
+  "migration_version": 7,
+  "schema_version": 6,
+  "schema_sha256": "73195c674ae62ccd0d2da9a6dd3a9ce2c9863ea7155fe7f52e61fe0f36d5ea99",
+  "fixture_sha256": "da63a52bdf62b3fc863995d79f497584be3ba97c1de4c55d1d301afbafab6cfc",
+  "fixture_blake3": "e880b964b41b340a350f70f1ae3a4ca0cdf1426078a16c66ba5f40dbced90ca2",
+  "fixture_bytes": 4096
+}