Skip to content

Commit 518f3b8

Browse files
lslusarczyklslusarczyk
committed
[Benchmarks] add RR benchmarks with test
1 parent c238ace commit 518f3b8

File tree

4 files changed

+252
-4
lines changed

4 files changed

+252
-4
lines changed

‎devops/scripts/benchmarks/benches/compute.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,56 @@ def benchmarks(self) -> list[Benchmark]:
269269
)
270270
)
271271

272+
record_and_reply_params = product(list(PROFILERS), [0, 1], [0, 1])
273+
for profiler_type, emulate, instantiate in record_and_reply_params:
274+
benches += [
275+
RecordAndReplay(
276+
self,
277+
RUNTIMES.LEVEL_ZERO,
278+
profiler_type,
279+
nForksInLvl=2,
280+
nLvls=4,
281+
nCmdSetsInLvl=10,
282+
nInstantiations=10,
283+
nAppendKern=10,
284+
nAppendCopy=1,
285+
mRec=1,
286+
mInst=instantiate,
287+
mDest=0,
288+
emulate=emulate,
289+
),
290+
RecordAndReplay(
291+
self,
292+
RUNTIMES.LEVEL_ZERO,
293+
profiler_type,
294+
nForksInLvl=1,
295+
nLvls=1,
296+
nCmdSetsInLvl=10,
297+
nInstantiations=10,
298+
nAppendKern=10,
299+
nAppendCopy=10,
300+
mRec=1,
301+
mInst=instantiate,
302+
mDest=0,
303+
emulate=emulate,
304+
),
305+
RecordAndReplay(
306+
self,
307+
RUNTIMES.LEVEL_ZERO,
308+
profiler_type,
309+
nForksInLvl=1,
310+
nLvls=4,
311+
nCmdSetsInLvl=1,
312+
nInstantiations=0,
313+
nAppendKern=1,
314+
nAppendCopy=0,
315+
mRec=1,
316+
mInst=instantiate,
317+
mDest=0,
318+
emulate=emulate,
319+
),
320+
]
321+
272322
# Add UR-specific benchmarks
273323
benches += [
274324
# TODO: multithread_benchmark_ur fails with segfault
@@ -647,6 +697,46 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
647697
]
648698

649699

700+
class RecordAndReplay(ComputeBenchmark):
701+
def __init__(self, bench, runtime: RUNTIMES, profiler_type, **kwargs):
702+
self.rr_params = kwargs
703+
self.iterations_regular = 1000
704+
self.iterations_trace = 10
705+
super().__init__(
706+
bench,
707+
f"record_and_replay_benchmark_{runtime.value}",
708+
"RecordGraph",
709+
runtime,
710+
profiler_type,
711+
)
712+
713+
def name(self):
714+
ret = [self.profiler_type.value]
715+
for k, v in self.rr_params.items():
716+
if k[0] == "n": # numeric parameter
717+
ret.append(f"{k[1:]} {v}")
718+
elif k[0] == "m":
719+
if v != 0: # measure parameter
720+
ret.append(f"{k[1:]}")
721+
else: # boolean parameter
722+
if v != 0:
723+
ret.append(k)
724+
ret.sort()
725+
return f"{self.bench_name} {self.test} " + ", ".join(ret)
726+
727+
def display_name(self) -> str:
728+
return self.name()
729+
730+
def description(self) -> str:
731+
return f"{self.runtime.value} Graphs record and reply"
732+
733+
def get_tags(self):
734+
return ["L0"]
735+
736+
def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]:
737+
return [f"--{k}={v}" for k, v in self.rr_params.items()]
738+
739+
650740
class QueueInOrderMemcpy(ComputeBenchmark):
651741
def __init__(self, bench, isCopyOnly, source, destination, size, profiler_type):
652742
self.isCopyOnly = isCopyOnly

‎devops/scripts/benchmarks/git_project.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
33
# See LICENSE.TXT
44
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5-
5+
import os
66
from pathlib import Path
77
import shutil
88

@@ -167,6 +167,11 @@ def _setup_repo(self) -> bool:
167167
Returns:
168168
bool: True if the repository was cloned or updated, False if it was already up-to-date.
169169
"""
170+
if os.environ.get("LLVM_BENCHMARKS_UNIT_TESTING") == "1":
171+
log.debug(
172+
f"Skipping git operations during unit testing of {self._name} (LLVM_BENCHMARKS_UNIT_TESTING=1)."
173+
)
174+
return False
170175
if not self.src_dir.exists():
171176
self._git_clone()
172177
return True

‎devops/scripts/benchmarks/main.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,13 @@ def process_results(
137137
stddev_threshold_override
138138
if stddev_threshold_override is not None
139139
else options.stddev_threshold
140-
) * mean_value
140+
)
141+
threshold_scaled = threshold * mean_value
141142

142-
if stddev > threshold:
143-
log.warning(f"stddev {stddev} above the threshold {threshold} for {label}")
143+
if stddev > threshold_scaled:
144+
log.warning(
145+
f"stddev {stddev} above the threshold {threshold_scaled} ({threshold} times {mean_value}) for {label}"
146+
)
144147
valid_results = False
145148

146149
rlist.sort(key=lambda res: res.value)
@@ -228,6 +231,9 @@ def main(directory, additional_env_vars, compare_names, filter):
228231
benchmark for benchmark in s.benchmarks() if benchmark.enabled()
229232
]
230233
if filter:
234+
log.debug(
235+
f"Filtering {len(suite_benchmarks)} benchmarks in {s.name()} suite for {filter.pattern}"
236+
)
231237
suite_benchmarks = [
232238
benchmark
233239
for benchmark in suite_benchmarks
@@ -713,6 +719,7 @@ def validate_and_parse_env_args(env_args):
713719
options.dry_run = args.dry_run
714720
options.umf = args.umf
715721
options.iterations_stddev = args.iterations_stddev
722+
options.stddev_threshold = args.stddev_threshold
716723
options.build_igc = args.build_igc
717724
options.current_run_name = args.relative_perf
718725
options.cudnn_directory = args.cudnn_directory

‎devops/scripts/benchmarks/tests/test_integration.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import os
2+
import shutil
3+
import unittest
4+
import logging
5+
6+
import subprocess
7+
import json
8+
from collections import namedtuple
9+
10+
DataJson = namedtuple("DataJson", ["runs", "metadata", "tags", "names"])
11+
DataJsonRun = namedtuple("DataJsonRun", ["name", "results"])
12+
DataJsonResult = namedtuple(
13+
"DataJsonResult", ["name", "label", "suite", "value", "unit"]
14+
)
15+
16+
17+
class App:
18+
def __init__(self):
19+
self.TMP_DIR = os.path.dirname(__file__)
20+
self.OUTPUT_DIR = os.path.join(self.TMP_DIR, "tmp-output")
21+
self.RESULTS_DIR = os.path.join(self.TMP_DIR, "tmp-results")
22+
self.WORKDIR_DIR = os.path.join(self.TMP_DIR, "tmp-workdir")
23+
24+
def prepare_dirs(self):
25+
for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]:
26+
os.makedirs(d)
27+
28+
# when UT does not want to build compute-benchmarks from scratch, it can provide prebuilt path
29+
cb_targetpath = os.environ.get("COMPUTE_BENCHMARKS_BUILD_PATH")
30+
if cb_targetpath and os.path.isdir(cb_targetpath):
31+
cb_build_dir = os.path.join(self.WORKDIR_DIR, "compute-benchmarks-build")
32+
os.symlink(cb_targetpath, cb_build_dir)
33+
with open(
34+
os.path.join(self.WORKDIR_DIR, "BENCH_WORKDIR_VERSION"), "w"
35+
) as f:
36+
f.write("2.0") # TODO: take from main.INTERNAL_WORKDIR_VERSION
37+
38+
def remove_dirs(self):
39+
for d in [self.RESULTS_DIR, self.OUTPUT_DIR, self.WORKDIR_DIR]:
40+
if os.path.exists(d):
41+
shutil.rmtree(d)
42+
43+
def run_main(self, *args):
44+
45+
# TODO: not yet tested: "--detect-version", "sycl,compute_runtime"
46+
47+
return subprocess.run(
48+
[
49+
"./devops/scripts/benchmarks/main.py",
50+
self.WORKDIR_DIR,
51+
"--sycl",
52+
os.environ.get("ONEAPI_ROOT"),
53+
"--adapter",
54+
"opencl",
55+
"--save",
56+
"testplik",
57+
"--output-html",
58+
"remote",
59+
"--results-dir",
60+
self.RESULTS_DIR,
61+
"--output-dir",
62+
self.OUTPUT_DIR,
63+
"--preset",
64+
"Minimal",
65+
"--timestamp-override",
66+
"20240102_030405",
67+
"--stddev-threshold",
68+
"999999999.9",
69+
"--exit-on-failure",
70+
*args,
71+
]
72+
)
73+
74+
def get_output(self):
75+
output_file = os.path.join(self.OUTPUT_DIR, "data.json")
76+
with open(output_file) as f:
77+
out = json.load(f)
78+
return DataJson(
79+
runs=[
80+
DataJsonRun(
81+
name=run["name"],
82+
results=[
83+
DataJsonResult(
84+
name=r["name"],
85+
label=r["label"],
86+
suite=r["suite"],
87+
value=r["value"],
88+
unit=r["unit"],
89+
)
90+
for r in run["results"]
91+
],
92+
)
93+
for run in out["benchmarkRuns"]
94+
],
95+
metadata=out["benchmarkMetadata"],
96+
tags=out["benchmarkTags"],
97+
names=out["defaultCompareNames"],
98+
)
99+
100+
101+
# add "--verbose" for debug logs
102+
103+
104+
class TestE2E(unittest.TestCase):
105+
def setUp(self):
106+
# Load test data
107+
self.app = App()
108+
self.app.remove_dirs()
109+
self.app.prepare_dirs()
110+
111+
# clean directory with input, output
112+
113+
def tearDown(self):
114+
self.app.remove_dirs()
115+
116+
def test_record_and_replay(self):
117+
caseName = "RecordGraph AppendCopy 1, AppendKern 10, CmdSetsInLvl 10, ForksInLvl 2, Instantiations 10, Lvls 4, Rec, timer"
118+
run_result = self.app.run_main("--filter", caseName)
119+
self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly")
120+
121+
out = self.app.get_output()
122+
123+
testName = "record_and_replay_benchmark_l0 " + caseName
124+
self.assertIn(testName, [r.name for r in out.runs[0].results])
125+
126+
metadata = out.metadata[testName]
127+
self.assertEqual(metadata["type"], "benchmark")
128+
self.assertEqual(set(metadata["tags"]), {"L0"})
129+
130+
def test_submit_kernel(self):
131+
caseName = "SubmitKernel out of order with measure completion KernelExecTime=20"
132+
run_result = self.app.run_main("--filter", caseName + "$")
133+
self.assertEqual(run_result.returncode, 0, "Subprocess did not exit cleanly")
134+
135+
out = self.app.get_output()
136+
137+
testName = "api_overhead_benchmark_l0 " + caseName
138+
self.assertIn(testName, [r.name for r in out.runs[0].results])
139+
140+
metadata = out.metadata[testName]
141+
self.assertEqual(metadata["type"], "benchmark")
142+
self.assertEqual(set(metadata["tags"]), {"L0", "latency", "micro", "submit"})
143+
144+
145+
if __name__ == "__main__":
146+
unittest.main()

0 commit comments

Comments
 (0)