Skip to content

Commit 09fe459

Browse files
not-matthiasnot-matthias
committed
feat: add codspeed benchmarks for valgrind
1 parent c9593ee commit 09fe459

File tree

9 files changed

+390
-0
lines changed

9 files changed

+390
-0
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bench/testdata/* filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ jobs:
2222
steps:
2323
- uses: actions/checkout@v4
2424

25+
# Skip installing package docs to avoid wasting time when installing valgrind
26+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
27+
- name: Skip installing package docs
28+
if: runner.os == 'Linux'
29+
run: |
30+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
31+
path-exclude /usr/share/doc/*
32+
path-exclude /usr/share/man/*
33+
path-exclude /usr/share/info/*
34+
EOF
35+
2536
- name: Update apt-get cache
2637
run: sudo apt-get update
2738

‎.github/workflows/codspeed.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: CodSpeed Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
benchmarks:
12+
runs-on: codspeed-macro
13+
timeout-minutes: 20
14+
strategy:
15+
matrix:
16+
# IMPORTANT: The binary has to match the architecture of the runner!
17+
cmd:
18+
- testdata/take_strings-x86_64
19+
- echo Hello, World!
20+
- ls fib.py
21+
- tar czf /dev/null fib.py
22+
- python3 fib.py
23+
valgrind:
24+
- "3.26.0"
25+
- "3.25.1"
26+
- "local"
27+
steps:
28+
- uses: actions/checkout@v4
29+
with:
30+
lfs: true
31+
- uses: extractions/setup-just@v3
32+
33+
# Skip installing package docs to avoid wasting time when installing build dependencies
34+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
35+
- name: Skip installing package docs
36+
if: runner.os == 'Linux'
37+
run: |
38+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
39+
path-exclude /usr/share/doc/*
40+
path-exclude /usr/share/man/*
41+
path-exclude /usr/share/info/*
42+
EOF
43+
44+
- name: Cache Valgrind build
45+
uses: actions/cache@v4
46+
id: valgrind-cache
47+
with:
48+
path: /tmp/valgrind-upstream
49+
key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-build
50+
51+
# Build and install Valgrind
52+
- name: Update apt-get cache
53+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
54+
run: |
55+
sudo apt-get update
56+
57+
# Remove existing Valgrind installation
58+
sudo apt-get remove -y valgrind || true
59+
60+
- name: Install build dependencies
61+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
62+
run: |
63+
sudo apt-get install -y \
64+
build-essential \
65+
automake \
66+
autoconf \
67+
gdb \
68+
docbook \
69+
docbook-xsl \
70+
docbook-xml \
71+
xsltproc
72+
73+
- name: Build Valgrind (${{ matrix.valgrind }})
74+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
75+
run: just build ${{ matrix.valgrind }}
76+
77+
- name: Install Valgrind (${{ matrix.valgrind }})
78+
run: |
79+
just install ${{ matrix.valgrind }}
80+
81+
# Ensure libc6-dev is installed for Valgrind to work properly
82+
sudo apt-get update
83+
sudo apt-get install -y libc6-dev
84+
85+
- name: Verify Valgrind build
86+
run: /usr/local/bin/valgrind --version
87+
88+
# Setup benchmarks and run them
89+
- name: Install uv
90+
uses: astral-sh/setup-uv@v5
91+
92+
- name: Run the benchmarks
93+
uses: CodSpeedHQ/action@main
94+
env:
95+
CODSPEED_PERF_ENABLED: false
96+
with:
97+
working-directory: bench
98+
mode: walltime
99+
run: ./bench.py --cmd "${{ matrix.cmd }}" --valgrind-path /usr/local/bin/valgrind

‎Justfile‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Builds a specific valgrind version
2+
# Usage:
3+
# - just build 3.24.0: Downloads the specified version from sourceware.org, builds and installs it
4+
# - just build local: Builds the local Valgrind source in the current directory
5+
build version:
6+
#!/usr/bin/env bash
7+
set -euo pipefail
8+
9+
if [ "{{ version }}" = "local" ]; then
10+
just build-in "."
11+
else
12+
just build-upstream "{{ version }}"
13+
fi
14+
15+
build-in dir:
16+
#!/usr/bin/env bash
17+
set -euo pipefail
18+
cd "{{ dir }}"
19+
20+
# Check if we need to run autogen.sh (for git checkouts)
21+
if [ -f "autogen.sh" ] && [ ! -f "configure" ]; then
22+
./autogen.sh
23+
fi
24+
25+
./configure
26+
make include/vgversion.h
27+
make -j$(nproc) -C VEX
28+
make -j$(nproc) -C coregrind
29+
make -j$(nproc) -C callgrind
30+
31+
# Download, build and install upstream Valgrind from sourceware.org
32+
build-upstream version:
33+
#!/usr/bin/env bash
34+
set -euo pipefail
35+
36+
# Download and extract upstream Valgrind
37+
mkdir -p /tmp/valgrind-upstream
38+
rm -rf /tmp/valgrind-upstream/valgrind-{{ version }}*
39+
wget -q -O /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
40+
https://sourceware.org/pub/valgrind/valgrind-{{ version }}.tar.bz2
41+
tar -xjf /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
42+
-C /tmp/valgrind-upstream
43+
44+
# Build and install using build-in
45+
just build-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
46+
47+
install version:
48+
#!/usr/bin/env bash
49+
set -euo pipefail
50+
51+
if [ "{{ version }}" = "local" ]; then
52+
just install-in "."
53+
else
54+
just install-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
55+
fi
56+
57+
install-in dir:
58+
#!/usr/bin/env bash
59+
set -euo pipefail
60+
cd "{{ dir }}"
61+
sudo make install

‎bench/bench.py‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
#!/usr/bin/env -S uv run --script
2+
# /// script
3+
# requires-python = ">=3.9"
4+
# dependencies = [
5+
# "pytest>=8.4.2",
6+
# "pytest-codspeed>=4.2.0",
7+
# ]
8+
# ///
9+
10+
import argparse
11+
import os
12+
import shlex
13+
import subprocess
14+
import sys
15+
from pathlib import Path
16+
import time
17+
18+
import pytest
19+
20+
21+
class ValgrindRunner:
22+
"""Run Valgrind with different configurations."""
23+
24+
def __init__(
25+
self,
26+
cmd: str,
27+
valgrind_path: str = "valgrind",
28+
):
29+
"""Initialize valgrind runner.
30+
31+
Args:
32+
cmd: Command to profile (can be a path or arbitrary shell command)
33+
valgrind_path: Path to valgrind executable
34+
output_dir: Directory for callgrind output files
35+
"""
36+
self.cmd = cmd
37+
self.valgrind_path = valgrind_path
38+
39+
# Verify valgrind is available
40+
result = subprocess.run(
41+
[self.valgrind_path, "--version"],
42+
capture_output=True,
43+
text=True,
44+
)
45+
if result.returncode != 0:
46+
raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}")
47+
self.valgrind_version = result.stdout.strip()
48+
49+
def run_valgrind(self, *args: str) -> None:
50+
"""Execute valgrind with given arguments.
51+
52+
Args:
53+
*args: Valgrind arguments
54+
"""
55+
56+
cmd = [
57+
self.valgrind_path,
58+
"--tool=callgrind",
59+
"--log-file=/dev/null",
60+
*args,
61+
*shlex.split(self.cmd),
62+
]
63+
64+
result = subprocess.run(
65+
cmd,
66+
capture_output=True,
67+
text=True,
68+
)
69+
if result.returncode != 0:
70+
raise RuntimeError(
71+
f"Valgrind execution failed with code {result.returncode}\n"
72+
f"Stdout:\n{result.stdout}\n"
73+
f"Stderr:\n{result.stderr}"
74+
)
75+
76+
77+
@pytest.fixture
78+
def runner(request):
79+
"""Fixture to provide runner instance to tests."""
80+
return request.config._valgrind_runner
81+
82+
83+
def pytest_generate_tests(metafunc):
84+
"""Parametrize tests with valgrind configurations."""
85+
if "valgrind_args" in metafunc.fixturenames:
86+
runner = getattr(metafunc.config, "_valgrind_runner", None)
87+
if not runner:
88+
return
89+
90+
# Define valgrind configurations
91+
configs = [
92+
(["--read-inline-info=no"], "no-inline"),
93+
(["--read-inline-info=yes"], "inline"),
94+
(
95+
[
96+
"--trace-children=yes",
97+
"--cache-sim=yes",
98+
"--I1=32768,8,64",
99+
"--D1=32768,8,64",
100+
"--LL=8388608,16,64",
101+
"--collect-systime=nsec",
102+
"--compress-strings=no",
103+
"--combine-dumps=yes",
104+
"--dump-line=no",
105+
"--read-inline-info=yes",
106+
],
107+
"full-with-inline",
108+
),
109+
(
110+
[
111+
"--trace-children=yes",
112+
"--cache-sim=yes",
113+
"--I1=32768,8,64",
114+
"--D1=32768,8,64",
115+
"--LL=8388608,16,64",
116+
"--collect-systime=nsec",
117+
"--compress-strings=no",
118+
"--combine-dumps=yes",
119+
"--dump-line=no",
120+
],
121+
"full-no-inline",
122+
),
123+
]
124+
125+
# Create test IDs with format: valgrind-version, command, config-name
126+
test_ids = [
127+
f"{runner.valgrind_version}, {runner.cmd}, {config_name}"
128+
for _, config_name in configs
129+
]
130+
131+
# Parametrize with just the args
132+
metafunc.parametrize(
133+
"valgrind_args",
134+
[args for args, _ in configs],
135+
ids=test_ids,
136+
)
137+
138+
139+
@pytest.mark.benchmark
140+
def test_valgrind(runner, valgrind_args):
141+
if runner:
142+
runner.run_valgrind(*valgrind_args)
143+
144+
145+
def main():
146+
parser = argparse.ArgumentParser(
147+
description="Benchmark Valgrind with pytest-codspeed",
148+
formatter_class=argparse.RawDescriptionHelpFormatter,
149+
epilog="""
150+
Examples:
151+
# Run with a binary path
152+
uv run bench.py --cmd /path/to/binary
153+
154+
# Run with an arbitrary command
155+
uv run bench.py --cmd 'echo "hello world"'
156+
157+
# Run with custom valgrind installation
158+
uv run bench.py --cmd /usr/bin/ls --valgrind-path /usr/local/bin/valgrind
159+
""",
160+
)
161+
162+
parser.add_argument(
163+
"--cmd",
164+
type=str,
165+
required=True,
166+
help="Command to profile (can be a path to a binary or any arbitrary command)",
167+
)
168+
parser.add_argument(
169+
"--valgrind-path",
170+
type=str,
171+
default="valgrind",
172+
help="Path to valgrind executable (default: valgrind)",
173+
)
174+
args = parser.parse_args()
175+
176+
# Create runner instance
177+
runner = ValgrindRunner(
178+
cmd=args.cmd,
179+
valgrind_path=args.valgrind_path,
180+
)
181+
print(f"Valgrind version: {runner.valgrind_version}")
182+
print(f"Command: {args.cmd}")
183+
184+
# Plugin to pass runner to tests
185+
class RunnerPlugin:
186+
def pytest_configure(self, config):
187+
config._valgrind_runner = runner
188+
189+
exit_code = pytest.main(
190+
[__file__, "-v", "--codspeed", "--codspeed-warmup-time=0", "--codspeed-max-time=3"],
191+
plugins=[RunnerPlugin()],
192+
)
193+
if exit_code != 0 and exit_code != 5:
194+
print(f"Benchmark execution returned exit code: {exit_code}")
195+
196+
197+
if __name__ == "__main__":
198+
main()

‎bench/fib.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env python3
2+
"""Simple recursive Fibonacci benchmark."""
3+
4+
5+
def fib(n):
6+
"""Calculate Fibonacci number recursively."""
7+
if n <= 1:
8+
return n
9+
return fib(n - 1) + fib(n - 2)
10+
11+
12+
fib(3)

‎bench/pytest.ini‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[pytest]
2+
norecursedirs = testdata __pycache__ .pytest_cache *.egg-info

0 commit comments

Comments
 (0)