From df305e130088d04ed186324916ba550a60746aa1 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Tue, 27 May 2025 10:57:10 +0200 Subject: [PATCH 1/4] Check for incompatible types. Warn for unimplemented types --- problemtools/verifyproblem.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 3aec3a11..af9a86cd 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -828,6 +828,20 @@ def check(self, context: Context) -> bool: return self._check_res self._check_res = True + INCOMPATIBLE_TYPES = [ + (metadata.ProblemType.PASS_FAIL, metadata.ProblemType.SCORING), + (metadata.ProblemType.SUBMIT_ANSWER, metadata.ProblemType.MULTI_PASS), + (metadata.ProblemType.SUBMIT_ANSWER, metadata.ProblemType.INTERACTIVE), + ] + for t1, t2 in INCOMPATIBLE_TYPES: + if t1 in self._metadata.type and t2 in self._metadata.type: + self.error(f'Problem has incompatible types: {t1}, {t2}') + + if metadata.ProblemType.MULTI_PASS in self._metadata.type: + self.warning('The type multi-pass is not yet supported.') + if metadata.ProblemType.SUBMIT_ANSWER in self._metadata.type: + self.warning('The type submit-answer is not yet supported.') + # Check rights_owner if self._metadata.license == metadata.License.PUBLIC_DOMAIN: if self._metadata.rights_owner: From 8dec935ec6890f144b6e3abfa9c8cad3f26baf1e Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Tue, 27 May 2025 10:58:01 +0200 Subject: [PATCH 2/4] Check format of interaction samples. #277 Don't warn about empty sample when it contains interactions. --- problemtools/verifyproblem.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index af9a86cd..a73dd0f6 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -616,7 +616,8 @@ def check(self, context: Context) -> bool: if os.path.basename(self._datadir) != 'sample': self.error(f'Testcase group {self._datadir} exists, but does not contain any testcases') else: - self.warning(f'Sample testcase group {self._datadir} exists, but does not contain any testcases') + if not (self._problem.metadata.is_interactive() and glob.glob(os.path.join(self._datadir, '*.interaction'))): + self.warning(f'Sample testcase group {self._datadir} exists, but does not contain any testcases') # Check whether a <= b according to a natural sorting where numeric components # are compactified, so that e.g. "a" < "a1" < "a2" < "a10" = "a010" < "a10a". @@ -749,6 +750,15 @@ def check(self, context: Context) -> bool: self.warn_directory('problem statements', 'statement_directory') + for ifilename in glob.glob(os.path.join(self.problem.probdir, 'data/sample/*.interaction')): + if not self.problem.metadata.is_interactive(): + self.error(f'Problem is not interactive, but there is an interaction sample {ifilename}') + with open(ifilename, 'r') as interaction: + for i, line in enumerate(interaction): + if len(line) == 0 or (line[0] != '<' and line[0] != '>'): + self.error(f'Interaction {ifilename}: line {i + 1} does not start with < or >') + break + if not self.statements: if self.problem.format is FormatVersion.LEGACY: allowed_statements = ', '.join( From 266caa3501a1b945a554ee0ad45920b67f783361 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Tue, 27 May 2025 11:33:52 +0200 Subject: [PATCH 3/4] Add type methods for all types. Add convenience methods on Problem for easier access --- problemtools/metadata.py | 6 +++++ problemtools/verifyproblem.py | 45 +++++++++++++++++++++++------------ tests/test_metadata.py | 17 +++++++++++++ tests/test_verify_hello.py | 5 ++++ 4 files changed, 58 insertions(+), 15 deletions(-) diff --git a/problemtools/metadata.py b/problemtools/metadata.py index 726cc746..25535327 100644 --- a/problemtools/metadata.py +++ b/problemtools/metadata.py @@ -222,6 +222,12 @@ def is_scoring(self) -> bool: def is_interactive(self) -> bool: return ProblemType.INTERACTIVE in self.type + def is_multi_pass(self) -> bool: + return ProblemType.MULTI_PASS in self.type + + def is_submit_answer(self) -> bool: + return ProblemType.SUBMIT_ANSWER in self.type + @classmethod def from_legacy(cls: Type[Self], legacy: MetadataLegacy, names_from_statements: dict[str, str]) -> Self: metadata = legacy.model_dump() diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index a73dd0f6..9ce66967 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -282,7 +282,7 @@ def check(self, context: Context) -> bool: self.warning( f'Answer file ({anssize:.1f} Mb) is within 50% of output limit ({outputlim} Mb), you might want to increase output limit' ) - if not self._problem.metadata.is_interactive(): + if not self._problem.is_interactive(): val_res = self._problem.output_validators.validate(self, self.ansfile) if val_res.verdict != 'AC': if self.is_in_sample_group(): @@ -342,7 +342,7 @@ def run_submission(self, sub, runner: Runner, context: Context) -> Result: def run_submission_real(self, sub, context: Context, timelim: int, timelim_low: int, timelim_high: int) -> Result: # This may be called off-main thread. - if self._problem.metadata.is_interactive(): + if self._problem.is_interactive(): res_high = self._problem.output_validators.validate_interactive(self, sub, timelim_high, self._problem.submissions) else: outfile = os.path.join(self._problem.tmpdir, f'output-{self.counter}') @@ -456,7 +456,7 @@ def __init__(self, problem: Problem, datadir: str | None = None, parent: TestCas if problem_on_reject == 'grade': self.config['on_reject'] = 'continue' - if self._problem.metadata.is_pass_fail(): + if self._problem.is_pass_fail(): for key in TestCaseGroup._SCORING_ONLY_KEYS: if key not in self.config: self.config[key] = None @@ -543,7 +543,7 @@ def check(self, context: Context) -> bool: if field not in TestCaseGroup._DEFAULT_CONFIG.keys(): self.warning(f"Unknown key '{field}' in '{os.path.join(self._datadir, 'testdata.yaml')}'") - if not self._problem.metadata.is_scoring(): + if not self._problem.is_scoring(): for key in TestCaseGroup._SCORING_ONLY_KEYS: if self.config.get(key) is not None: self.error(f"Key '{key}' is only applicable for scoring problems, this is a pass-fail problem") @@ -551,7 +551,7 @@ def check(self, context: Context) -> bool: if self.config['on_reject'] not in ['break', 'continue']: self.error(f"Invalid value '{self.config['on_reject']}' for on_reject policy") - if self._problem.metadata.is_scoring(): + if self._problem.is_scoring(): # Check grading try: score_range = self.config['range'] @@ -616,7 +616,7 @@ def check(self, context: Context) -> bool: if os.path.basename(self._datadir) != 'sample': self.error(f'Testcase group {self._datadir} exists, but does not contain any testcases') else: - if not (self._problem.metadata.is_interactive() and glob.glob(os.path.join(self._datadir, '*.interaction'))): + if not (self._problem.is_interactive() and glob.glob(os.path.join(self._datadir, '*.interaction'))): self.warning(f'Sample testcase group {self._datadir} exists, but does not contain any testcases') # Check whether a <= b according to a natural sorting where numeric components @@ -715,7 +715,7 @@ def aggregate_results(self, sub, sub_results: list[SubmissionResult], shadow_res if sub_results: res.testcase = sub_results[-1].testcase res.additional_info = sub_results[-1].additional_info - if self._problem.metadata.is_scoring(): + if self._problem.is_scoring(): res.score = score min_score, max_score = self.get_score_range() if score is not None and not (min_score <= score <= max_score) and not self._seen_oob_scores: @@ -751,7 +751,7 @@ def check(self, context: Context) -> bool: self.warn_directory('problem statements', 'statement_directory') for ifilename in glob.glob(os.path.join(self.problem.probdir, 'data/sample/*.interaction')): - if not self.problem.metadata.is_interactive(): + if not self.problem.is_interactive(): self.error(f'Problem is not interactive, but there is an interaction sample {ifilename}') with open(ifilename, 'r') as interaction: for i, line in enumerate(interaction): @@ -847,9 +847,9 @@ def check(self, context: Context) -> bool: if t1 in self._metadata.type and t2 in self._metadata.type: self.error(f'Problem has incompatible types: {t1}, {t2}') - if metadata.ProblemType.MULTI_PASS in self._metadata.type: + if self.problem.is_multi_pass(): self.warning('The type multi-pass is not yet supported.') - if metadata.ProblemType.SUBMIT_ANSWER in self._metadata.type: + if self.problem.is_submit_answer(): self.warning('The type submit-answer is not yet supported.') # Check rights_owner @@ -867,10 +867,10 @@ def check(self, context: Context) -> bool: if self._metadata.uuid is None: self.error_in_2023_07(f'Missing uuid from problem.yaml. Add "uuid: {uuid.uuid4()}" to problem.yaml.') - if self._metadata.legacy_grading.show_test_data_groups and self._metadata.is_pass_fail(): + if self._metadata.legacy_grading.show_test_data_groups and self.problem.is_pass_fail(): self.error('Showing test data groups is only supported for scoring problems, this is a pass-fail problem') if ( - not self._metadata.is_pass_fail() + not self.problem.is_pass_fail() and self.problem.testdata.has_custom_groups() and 'show_test_data_groups' not in self._origdata.get('grading', {}) and self.problem.format is FormatVersion.LEGACY @@ -880,7 +880,7 @@ def check(self, context: Context) -> bool: ) if self._metadata.legacy_grading.on_reject is not None: - if self._metadata.is_pass_fail() and self._metadata.legacy_grading.on_reject == 'grade': + if self.problem.is_pass_fail() and self._metadata.legacy_grading.on_reject == 'grade': self.error("Invalid on_reject policy 'grade' for problem type 'pass-fail'") for deprecated_grading_key in ['accept_score', 'reject_score', 'range', 'on_reject']: @@ -1126,7 +1126,7 @@ def check(self, context: Context) -> bool: return self._check_res self._check_res = True - if self.problem.metadata.is_pass_fail() and len(self._graders) > 0: + if self.problem.is_pass_fail() and len(self._graders) > 0: self.error('There are grader programs but the problem is pass-fail') for grader in self._graders: @@ -1636,7 +1636,7 @@ def full_score_finite(self) -> bool: def fully_accepted(self, result: SubmissionResult) -> bool: min_score, max_score = self.problem.testdata.get_score_range() best_score = min_score if self.problem.metadata.legacy_grading.objective == 'min' else max_score - return result.verdict == 'AC' and (not self.problem.metadata.is_scoring() or result.score == best_score) + return result.verdict == 'AC' and (not self.problem.is_scoring() or result.score == best_score) def start_background_work(self, context: Context) -> None: # Send off an early background compile job for each submission and @@ -1752,6 +1752,21 @@ def _set_timelim(self, timelim: float) -> None: # Should only be called by Subm assert self._timelim is None, 'Attempted to set timelim twice' self._timelim = timelim + def is_pass_fail(self) -> bool: + return self.metadata.is_pass_fail() + + def is_scoring(self) -> bool: + return self.metadata.is_scoring() + + def is_interactive(self) -> bool: + return self.metadata.is_interactive() + + def is_multi_pass(self) -> bool: + return self.metadata.is_multi_pass() + + def is_submit_answer(self) -> bool: + return self.metadata.is_submit_answer() + def load(self) -> None: """Parses the problem package statically, loading up information with very little verification. diff --git a/tests/test_metadata.py b/tests/test_metadata.py index df2d3b12..9652c906 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -86,6 +86,21 @@ def test_parse_multi_source(minimal_2023_conf): assert m.source[2].url is None +def test_parse_complex_type(minimal_2023_conf): + c = minimal_2023_conf + c['type'] = ['scoring', 'multi-pass', 'interactive'] + m = metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'}) + assert len(m.type) == 3 + assert metadata.ProblemType.SCORING in m.type + assert metadata.ProblemType.MULTI_PASS in m.type + assert metadata.ProblemType.INTERACTIVE in m.type + assert not m.is_pass_fail() + assert m.is_scoring() + assert m.is_interactive() + assert m.is_multi_pass() + assert not m.is_submit_answer() + + def test_load_hello(): m, _ = metadata.load_metadata(Path(__file__).parent / 'hello') assert m.name['en'] == 'Hello World!' @@ -99,3 +114,5 @@ def test_load_hello(): assert m.is_pass_fail() assert not m.is_scoring() assert not m.is_interactive() + assert not m.is_multi_pass() + assert not m.is_submit_answer() diff --git a/tests/test_verify_hello.py b/tests/test_verify_hello.py index 9bf5cd43..f51de740 100644 --- a/tests/test_verify_hello.py +++ b/tests/test_verify_hello.py @@ -16,3 +16,8 @@ def test_load_hello(): # pytest and fork don't go along very well, so just run aspects that work without run assert p.config.check(context) assert p.attachments.check(context) + assert p.is_pass_fail() + assert not p.is_scoring() + assert not p.is_interactive() + assert not p.is_multi_pass() + assert not p.is_submit_answer() From b5cb10c3c351d004be86b42a85ee577694c6823b Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Tue, 27 May 2025 11:47:25 +0200 Subject: [PATCH 4/4] Improve warning for non-standard output validator languages #258 --- problemtools/verifyproblem.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 9ce66967..1b419785 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -1226,11 +1226,13 @@ def check(self, context: Context) -> bool: self.warn_directory('output validators', 'output_validator_directory') - recommended_output_validator_languages = {'c', 'cpp', 'python3'} + safe_output_validator_languages = {'c', 'cpp', 'python3'} for v in self._validators: - if isinstance(v, run.SourceCode) and v.language.lang_id not in recommended_output_validator_languages: - self.warning('output validator language %s is not recommended' % v.language.name) + if isinstance(v, run.SourceCode) and v.language.lang_id not in safe_output_validator_languages: + self.error_in_2023_07( + f'Output validator in {v.language.name}. Only {safe_output_validator_languages} are standardized. Check carefully if your CCS supports more (Kattis does not).' + ) if self.problem.metadata.legacy_validation == 'default' and self._validators: self.error('There are validator programs but problem.yaml has validation = "default"')