From a87fac51b39b73e9c20f4380493d19e84fa9818b Mon Sep 17 00:00:00 2001 From: Petrik Date: Tue, 18 Mar 2025 20:15:39 +0100 Subject: [PATCH] Warn if frameworks exceed the maximum number of test mutations > The number of test mutations will be limited to 10. https://github.com/TechEmpower/FrameworkBenchmarks/issues/8420 This change shows a warning message for frameworks if the maximum is reached. --- toolset/utils/metadata.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/toolset/utils/metadata.py b/toolset/utils/metadata.py index 6e440fc23ab..6f7388828b8 100644 --- a/toolset/utils/metadata.py +++ b/toolset/utils/metadata.py @@ -191,6 +191,18 @@ def parse_config(self, config, directory): % config['framework'], color=Fore.YELLOW) + # Check that each framework does not have more than the maximum number of tests + maximum_tests = 10 + non_broken_tests_filter = lambda test: (not hasattr(test, "tags")) or ("broken" not in test.tags) + non_broken_tests_to_run = list(filter(non_broken_tests_filter, tests_to_run)) + if len(non_broken_tests_to_run) > maximum_tests: + message = [ + "Framework %s defines %s tests in benchmark_config.json (max is %s)." + % (config['framework'], len(non_broken_tests_to_run), maximum_tests), + "Contact maintainers and remove deprecated or discarded ones to make room." + ] + log("\n".join(message), color=Fore.YELLOW) + # Check that each test configuration is acceptable # Throw exceptions if a field is missing, or how to improve the field for test_name, test_keys in test.items():