back compat test

.github/workflows/compat-gen-upload.yml

@@ -0,0 +1,32 @@
+name: Compat Fixture Upload
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to generate fixtures for (e.g. 0.62.0)"
+        required: true
+
+jobs:
+  upload-fixtures:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
+          aws-region: us-east-1
+
+      - name: Generate and upload fixtures
+        run: >
+          python3 vortex-test/compat-gen/scripts/upload.py
+          --version "${{ inputs.version }}"

.github/workflows/compat-test-weekly.yml

@@ -0,0 +1,24 @@
+name: Compat Test
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Monday 6am UTC
+  workflow_dispatch: { }
+
+env:
+  FIXTURES_URL: https://vortex-compat-fixtures.s3.amazonaws.com
+
+jobs:
+  compat-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Run compat tests
+        run: |
+          cargo run -p vortex-compat --release --bin compat-validate -- \
+            --fixtures-url "$FIXTURES_URL"

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "vortex-jni",
     "vortex-python",
     "vortex-tui",
+    "vortex-test/compat-gen",
     "vortex-test/e2e",
     "vortex-test/e2e-cuda",
     "xtask",
@@ -281,6 +282,7 @@ vortex-zstd = { version = "0.1.0", path = "./encodings/zstd", default-features =
 
 # No version constraints for unpublished crates.
 vortex-bench = { path = "./vortex-bench", default-features = false }
+vortex-compat = { path = "./vortex-test/compat-gen" }
 vortex-cuda = { path = "./vortex-cuda", default-features = false }
 vortex-cuda-macros = { path = "./vortex-cuda/macros" }
 vortex-duckdb = { path = "./vortex-duckdb", default-features = false }

vortex-test/compat-gen/Cargo.toml

@@ -0,0 +1,54 @@
+[package]
+name = "vortex-compat"
+authors = { workspace = true }
+description = "Backward-compatibility fixture generation and testing for Vortex"
+edition = { workspace = true }
+homepage = { workspace = true }
+include = { workspace = true }
+keywords = { workspace = true }
+license = { workspace = true }
+publish = false
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[lints]
+workspace = true
+
+[[bin]]
+name = "compat-gen"
+path = "src/main.rs"
+
+[[bin]]
+name = "compat-validate"
+path = "src/validate_main.rs"
+
+[dependencies]
+# Vortex crates
+vortex = { workspace = true, features = ["files", "tokio"] }
+vortex-array = { workspace = true, features = ["_test-harness"] }
+vortex-buffer = { workspace = true }
+vortex-error = { workspace = true }
+vortex-session = { workspace = true }
+vortex-utils = { workspace = true }
+
+# TPC-H generation
+arrow-array = { workspace = true }
+tpchgen = { workspace = true }
+tpchgen-arrow = { workspace = true }
+
+# ClickBench parquet reading
+parquet = { workspace = true }
+
+# Async runtime
+futures = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+
+# HTTP fetching (for ClickBench fixture + compat-test S3 downloads)
+reqwest = { workspace = true }
+
+# CLI + serialization
+chrono = { workspace = true, features = ["serde"] }
+clap = { workspace = true, features = ["derive"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }