Skip to content

Add --drop-exec option to filter warmup samples #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 29, 2025
+187 −3
Merged

Add --drop-exec option to filter warmup samples #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1ff5cf
Add --drop-exec option to filter warmup samples
aemerson Oct 22, 2025
f44eded
Fix flake error
aemerson Oct 25, 2025
e30ba55
Remove accidental uv.lock commit
aemerson Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 59 additions & 3 deletions lnt/tests/test_suite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,23 @@ def run_test(self, opts):
self._fatal("--single-result must be given a single test name, "
"not a directory name")

# Parse and validate --drop-exec option
if opts.drop_exec is not None:
# Check for incompatible options
if opts.only_compile:
self._fatal("--drop-exec cannot be used with --only-compile")
if opts.build:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think --build means that we're necessarily skipping the tests, right? Can't you have both --build and --exec, in which case it would make sense to have --build and --drop-exec at the same time?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think --build and --exec is redundant as a concept because that's the default behavior. These options are only useful as a stopping point.

That said there's an issue which I don't think the current implementation is self consistent with. --exec (née --test-prebuilt) is supposed to skip previous steps, which is fine. But --build doesn't skip the --configure phase (and doing so seems like bad UX).

self._fatal("--drop-exec cannot be used with --build")
if opts.exec_multisample <= 1:
self._fatal("--drop-exec requires --exec-multisample > 1")

# opts.drop_exec is already an integer from click
if opts.drop_exec < 1:
self._fatal("--drop-exec must be at least 1")

if opts.drop_exec >= opts.exec_multisample:
self._fatal("--drop-exec would drop all %d samples" % opts.exec_multisample)

opts.cppflags = ' '.join(opts.cppflags)
opts.cflags = ' '.join(opts.cflags)
opts.cxxflags = ' '.join(opts.cxxflags)
Expand Down Expand Up @@ -428,6 +445,9 @@ def run_test(self, opts):
reports.append(run_report)
json_reports.append(json_data)

# Filter execution samples if --drop-exec was specified
reports, json_reports = self._filter_exec_samples(reports, json_reports)

report = self._create_merged_report(reports)

# Write the report out so it can be read by the submission tool.
Expand Down Expand Up @@ -537,8 +557,14 @@ def _run_interleaved_builds(self, opts):
build_dir = build_info['build_dir']
logger.info("Writing report for build: %s" % build_dir)

# Filter execution samples if --drop-exec was specified
build_reports, build_json_reports = self._filter_exec_samples(
reports_by_build[build_idx],
json_by_build[build_idx]
)

# Merge reports for this build
merged_report = self._create_merged_report(reports_by_build[build_idx])
merged_report = self._create_merged_report(build_reports)

# Write JSON report to build directory
report_path = os.path.join(build_dir, 'report.json')
Expand All @@ -548,13 +574,13 @@ def _run_interleaved_builds(self, opts):

# Write xUnit XML to build directory
xml_path = os.path.join(build_dir, 'test-results.xunit.xml')
str_template = _lit_json_to_xunit_xml(json_by_build[build_idx])
str_template = _lit_json_to_xunit_xml(build_json_reports)
with open(xml_path, 'w') as fd:
fd.write(str_template)

# Write CSV to build directory
csv_path = os.path.join(build_dir, 'test-results.csv')
str_template = _lit_json_to_csv(json_by_build[build_idx])
str_template = _lit_json_to_csv(build_json_reports)
with open(csv_path, 'w') as fd:
fd.write(str_template)

Expand Down Expand Up @@ -607,6 +633,32 @@ def _create_merged_report(self, reports):
test_samples = sum([r.tests for r in reports], [])
return lnt.testing.Report(machine, run, test_samples)

def _filter_exec_samples(self, reports, json_reports):
"""Filter out execution samples based on --drop-exec option.

Returns filtered (reports, json_reports) tuples.
"""
drop_exec_count = self.opts.drop_exec

if drop_exec_count is None or drop_exec_count == 0 or len(reports) == 0:
return reports, json_reports

# Drop the first N samples
filtered_reports = reports[drop_exec_count:]
filtered_json_reports = json_reports[drop_exec_count:]

# Log what we're dropping
if drop_exec_count == 1:
logger.info("Dropping first execution sample (iteration 0)")
else:
logger.info("Dropping first %d execution samples (iterations 0-%d)" %
(drop_exec_count, drop_exec_count - 1))

logger.info("Kept %d of %d execution samples after --drop-exec filtering" %
(len(filtered_reports), len(reports)))

return filtered_reports, filtered_json_reports

def _test_suite_dir(self):
return self.opts.test_suite_root

Expand Down Expand Up @@ -1341,6 +1393,10 @@ def diagnose(self):
@click.option("--compile-multisample", "compile_multisample",
help="Accumulate compile test data from multiple runs",
type=int, default=1, metavar="N")
@click.option("--drop-exec", "drop_exec",
help="Drop the first N execution samples to mitigate warmup effects. "
"Used with --exec-multisample.",
type=int, default=None, metavar="N")
@click.option("-d", "--diagnose", "diagnose",
help="Produce a diagnostic report for a particular "
"test, this will not run all the tests. Must be"
Expand Down
128 changes: 128 additions & 0 deletions tests/runtest/test_suite-drop-exec.shtest
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
# Check --drop-exec feature for filtering execution samples
# This test verifies that --drop-exec correctly drops first N execution samples

# Test 1: --drop-exec without --exec-multisample should fail
# RUN: not lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-ERR1 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --drop-exec 1 \
# RUN: > %t.err1.log 2> %t.err1.err
# RUN: filecheck --check-prefix CHECK-ERR1 < %t.err1.err %s
# CHECK-ERR1: --drop-exec requires --exec-multisample > 1

# Test 2: --drop-exec with --only-compile should fail
# RUN: not lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-ERR2 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --only-compile \
# RUN: --drop-exec 1 \
# RUN: > %t.err2.log 2> %t.err2.err
# RUN: filecheck --check-prefix CHECK-ERR2 < %t.err2.err %s
# CHECK-ERR2: --drop-exec cannot be used with --only-compile

# Test 3: --drop-exec with --build should fail
# RUN: not lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-ERR3 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --build \
# RUN: --drop-exec 1 \
# RUN: > %t.err3.log 2> %t.err3.err
# RUN: filecheck --check-prefix CHECK-ERR3 < %t.err3.err %s
# CHECK-ERR3: --drop-exec cannot be used with --build

# Test 4: --drop-exec dropping all samples should fail
# RUN: not lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-ERR4 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --exec-multisample 2 \
# RUN: --drop-exec 2 \
# RUN: > %t.err4.log 2> %t.err4.err
# RUN: filecheck --check-prefix CHECK-ERR4 < %t.err4.err %s
# CHECK-ERR4: --drop-exec would drop all 2 samples

# Test 5: --drop-exec 1 with 3 samples should work
# RUN: rm -rf %t.SANDBOX-DROP1
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-DROP1 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --exec-multisample 3 \
# RUN: --drop-exec 1 \
# RUN: --output %t.drop1.json \
# RUN: > %t.drop1.log 2> %t.drop1.err
# RUN: filecheck --check-prefix CHECK-DROP1 < %t.drop1.err %s
# CHECK-DROP1: Dropping first execution sample (iteration 0)
# CHECK-DROP1: Kept 2 of 3 execution samples after --drop-exec filtering

# Test 6: --drop-exec 2 with 5 samples should work
# RUN: rm -rf %t.SANDBOX-DROP2
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-DROP2 \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --exec-multisample 5 \
# RUN: --drop-exec 2 \
# RUN: --output %t.drop2.json \
# RUN: > %t.drop2.log 2> %t.drop2.err
# RUN: filecheck --check-prefix CHECK-DROP2 < %t.drop2.err %s
# CHECK-DROP2: Dropping first 2 execution samples (iterations 0-1)
# CHECK-DROP2: Kept 3 of 5 execution samples after --drop-exec filtering

# Test 7: --drop-exec with test-prebuilt mode
# First build
# RUN: rm -rf %t.SANDBOX-BUILD-PREBUILT
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-BUILD-PREBUILT \
# RUN: --no-timestamp \
# RUN: --test-suite %S/Inputs/test-suite-cmake \
# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --build \
# RUN: > %t.build-prebuilt.log 2> %t.build-prebuilt.err

# Now test prebuilt with --drop-exec
# RUN: rm -rf %t.SANDBOX-PREBUILT-TEST
# RUN: lnt runtest test-suite \
# RUN: --sandbox %t.SANDBOX-PREBUILT-TEST \
# RUN: --no-timestamp \
# RUN: --exec \
# RUN: --build-dir %t.SANDBOX-BUILD-PREBUILT/build \
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
# RUN: --exec-multisample 3 \
# RUN: --drop-exec 1 \
# RUN: --output %t.prebuilt-drop.json \
# RUN: > %t.prebuilt-drop.log 2> %t.prebuilt-drop.err
# RUN: filecheck --check-prefix CHECK-PREBUILT-DROP < %t.prebuilt-drop.err %s
# CHECK-PREBUILT-DROP: Dropping first execution sample (iteration 0)
# CHECK-PREBUILT-DROP: Kept 2 of 3 execution samples after --drop-exec filtering
Loading