fixup: run with multiple valgrind versions, add bench parameters

Author: not-matthias

.github/workflows/codspeed.yml

@@ -15,12 +15,15 @@ jobs:
         # IMPORTANT: The binary has to match the architecture of the runner!
         benchmark:
           - testdata/take_strings-aarch64
+        valgrind: 
+          - /usr/bin/valgrind
+          - ../vg-in-place
     steps:
       - uses: actions/checkout@v4
         with:
           lfs: true
 
-      # Skip installing package docs to avoid wasting time when installing valgrind
+      # Skip installing package docs to avoid wasting time when installing build dependencies
       # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
       - name: Skip installing package docs
         if: runner.os == 'Linux'
@@ -32,7 +35,6 @@ jobs:
           EOF
 
       # Build and install Valgrind
-      #
       - name: Update apt-get cache
         run: sudo apt-get update
 
@@ -67,11 +69,19 @@ jobs:
           ./vg-in-place --tool=callgrind --help > /dev/null || { echo "callgrind tool not accessible!"; exit 1; }
           echo "Valgrind build successful and callgrind tool is accessible"
 
+      # Install upstream Valgrind for comparison
+      - name: Install upstream Valgrind
+        run: sudo apt-get install -y valgrind
+
       # Setup benchmarks and run them
-      #
       - name: Install uv
         uses: astral-sh/setup-uv@v5
 
+      # Ensure that the runner doesn't install it's own valgrind by linking vg-in-place to /usr/local/bin/valgrind, 
+      # which makes the version check work correctly.
+      - name: Link vg-in-place to /usr/local/bin/valgrind
+        run: sudo ln -sf "$PWD/vg-in-place" /usr/local/bin/valgrind
+
       - name: Run the benchmarks
         uses: CodSpeedHQ/action@main
         env:
@@ -81,4 +91,4 @@ jobs:
         with:
           working-directory: bench
           mode: walltime
-          run: uv run bench.py --binary-path ${{ matrix.benchmark }} --valgrind-path ${{ github.workspace }}/vg-in-place
+          run: uv run bench.py --binary-path ${{ matrix.benchmark }} --valgrind-path ${{ matrix.valgrind }}

bench/bench.py

@@ -82,20 +82,32 @@ def runner(request):
     return request.config._valgrind_runner
 
 
+def pytest_generate_tests(metafunc):
+    """Parametrize tests with benchmark_id."""
+    if "benchmark_context" in metafunc.fixturenames:
+        runner = getattr(metafunc.config, "_valgrind_runner", None)
+        if not runner:
+            metafunc.parametrize("benchmark_context", ["unknown"])
+            return
+
+        benchmark_id = f"{runner.binary_path.stem}, {runner.valgrind_version}"
+        metafunc.parametrize("benchmark_context", [benchmark_id], ids=[benchmark_id])
+
+
 @pytest.mark.benchmark
-def test_baseline(runner):
+def test_baseline(runner, benchmark_context):
     if runner:
         runner.run_valgrind("--read-inline-info=no")
 
 
 @pytest.mark.benchmark
-def test_with_inline_info(runner):
+def test_with_inline_info(runner, benchmark_context):
     if runner:
         runner.run_valgrind("--read-inline-info=yes")
 
 
 @pytest.mark.benchmark
-def test_full_inline(runner):
+def test_full_inline(runner, benchmark_context):
     if runner:
         args = [
             "--trace-children=yes",
@@ -113,7 +125,7 @@ def test_full_inline(runner):
 
 
 @pytest.mark.benchmark
-def test_full(runner):
+def test_full(runner, benchmark_context):
     if runner:
         args = [
             "--trace-children=yes",