Improved checks for types, interaction samples, and output validator language

problemtools/metadata.py

@@ -222,6 +222,12 @@ def is_scoring(self) -> bool:
     def is_interactive(self) -> bool:
         return ProblemType.INTERACTIVE in self.type
 
+    def is_multi_pass(self) -> bool:
+        return ProblemType.MULTI_PASS in self.type
+
+    def is_submit_answer(self) -> bool:
+        return ProblemType.SUBMIT_ANSWER in self.type
+
     @classmethod
     def from_legacy(cls: Type[Self], legacy: MetadataLegacy, names_from_statements: dict[str, str]) -> Self:
         metadata = legacy.model_dump()

problemtools/verifyproblem.py

@@ -282,7 +282,7 @@ def check(self, context: Context) -> bool:
             self.warning(
                 f'Answer file ({anssize:.1f} Mb) is within 50% of output limit ({outputlim} Mb), you might want to increase output limit'
             )
-        if not self._problem.metadata.is_interactive():
+        if not self._problem.is_interactive():
             val_res = self._problem.output_validators.validate(self, self.ansfile)
             if val_res.verdict != 'AC':
                 if self.is_in_sample_group():
@@ -342,7 +342,7 @@ def run_submission(self, sub, runner: Runner, context: Context) -> Result:
 
     def run_submission_real(self, sub, context: Context, timelim: int, timelim_low: int, timelim_high: int) -> Result:
         # This may be called off-main thread.
-        if self._problem.metadata.is_interactive():
+        if self._problem.is_interactive():
             res_high = self._problem.output_validators.validate_interactive(self, sub, timelim_high, self._problem.submissions)
         else:
             outfile = os.path.join(self._problem.tmpdir, f'output-{self.counter}')
@@ -456,7 +456,7 @@ def __init__(self, problem: Problem, datadir: str | None = None, parent: TestCas
         if problem_on_reject == 'grade':
             self.config['on_reject'] = 'continue'
 
-        if self._problem.metadata.is_pass_fail():
+        if self._problem.is_pass_fail():
             for key in TestCaseGroup._SCORING_ONLY_KEYS:
                 if key not in self.config:
                     self.config[key] = None
@@ -543,15 +543,15 @@ def check(self, context: Context) -> bool:
             if field not in TestCaseGroup._DEFAULT_CONFIG.keys():
                 self.warning(f"Unknown key '{field}' in '{os.path.join(self._datadir, 'testdata.yaml')}'")
 
-        if not self._problem.metadata.is_scoring():
+        if not self._problem.is_scoring():
             for key in TestCaseGroup._SCORING_ONLY_KEYS:
                 if self.config.get(key) is not None:
                     self.error(f"Key '{key}' is only applicable for scoring problems, this is a pass-fail problem")
 
         if self.config['on_reject'] not in ['break', 'continue']:
             self.error(f"Invalid value '{self.config['on_reject']}' for on_reject policy")
 
-        if self._problem.metadata.is_scoring():
+        if self._problem.is_scoring():
             # Check grading
             try:
                 score_range = self.config['range']
@@ -616,7 +616,8 @@ def check(self, context: Context) -> bool:
             if os.path.basename(self._datadir) != 'sample':
                 self.error(f'Testcase group {self._datadir} exists, but does not contain any testcases')
             else:
-                self.warning(f'Sample testcase group {self._datadir} exists, but does not contain any testcases')
+                if not (self._problem.is_interactive() and glob.glob(os.path.join(self._datadir, '*.interaction'))):
+                    self.warning(f'Sample testcase group {self._datadir} exists, but does not contain any testcases')
 
         # Check whether a <= b according to a natural sorting where numeric components
         # are compactified, so that e.g. "a" < "a1" < "a2" < "a10" = "a010" < "a10a".
@@ -714,7 +715,7 @@ def aggregate_results(self, sub, sub_results: list[SubmissionResult], shadow_res
             if sub_results:
                 res.testcase = sub_results[-1].testcase
                 res.additional_info = sub_results[-1].additional_info
-            if self._problem.metadata.is_scoring():
+            if self._problem.is_scoring():
                 res.score = score
                 min_score, max_score = self.get_score_range()
                 if score is not None and not (min_score <= score <= max_score) and not self._seen_oob_scores:
@@ -749,6 +750,15 @@ def check(self, context: Context) -> bool:
 
         self.warn_directory('problem statements', 'statement_directory')
 
+        for ifilename in glob.glob(os.path.join(self.problem.probdir, 'data/sample/*.interaction')):
+            if not self.problem.is_interactive():
+                self.error(f'Problem is not interactive, but there is an interaction sample {ifilename}')
+            with open(ifilename, 'r') as interaction:
+                for i, line in enumerate(interaction):
+                    if len(line) == 0 or (line[0] != '<' and line[0] != '>'):
+                        self.error(f'Interaction {ifilename}: line {i + 1} does not start with < or >')
+                        break
+
         if not self.statements:
             if self.problem.format is FormatVersion.LEGACY:
                 allowed_statements = ', '.join(
@@ -828,6 +838,20 @@ def check(self, context: Context) -> bool:
             return self._check_res
         self._check_res = True
 
+        INCOMPATIBLE_TYPES = [
+            (metadata.ProblemType.PASS_FAIL, metadata.ProblemType.SCORING),
+            (metadata.ProblemType.SUBMIT_ANSWER, metadata.ProblemType.MULTI_PASS),
+            (metadata.ProblemType.SUBMIT_ANSWER, metadata.ProblemType.INTERACTIVE),
+        ]
+        for t1, t2 in INCOMPATIBLE_TYPES:
+            if t1 in self._metadata.type and t2 in self._metadata.type:
+                self.error(f'Problem has incompatible types: {t1}, {t2}')
+
+        if self.problem.is_multi_pass():
+            self.warning('The type multi-pass is not yet supported.')
+        if self.problem.is_submit_answer():
+            self.warning('The type submit-answer is not yet supported.')
+
         # Check rights_owner
         if self._metadata.license == metadata.License.PUBLIC_DOMAIN:
             if self._metadata.rights_owner:
@@ -843,10 +867,10 @@ def check(self, context: Context) -> bool:
         if self._metadata.uuid is None:
             self.error_in_2023_07(f'Missing uuid from problem.yaml. Add "uuid: {uuid.uuid4()}" to problem.yaml.')
 
-        if self._metadata.legacy_grading.show_test_data_groups and self._metadata.is_pass_fail():
+        if self._metadata.legacy_grading.show_test_data_groups and self.problem.is_pass_fail():
             self.error('Showing test data groups is only supported for scoring problems, this is a pass-fail problem')
         if (
-            not self._metadata.is_pass_fail()
+            not self.problem.is_pass_fail()
             and self.problem.testdata.has_custom_groups()
             and 'show_test_data_groups' not in self._origdata.get('grading', {})
             and self.problem.format is FormatVersion.LEGACY
@@ -856,7 +880,7 @@ def check(self, context: Context) -> bool:
             )
 
         if self._metadata.legacy_grading.on_reject is not None:
-            if self._metadata.is_pass_fail() and self._metadata.legacy_grading.on_reject == 'grade':
+            if self.problem.is_pass_fail() and self._metadata.legacy_grading.on_reject == 'grade':
                 self.error("Invalid on_reject policy 'grade' for problem type 'pass-fail'")
 
         for deprecated_grading_key in ['accept_score', 'reject_score', 'range', 'on_reject']:
@@ -1102,7 +1126,7 @@ def check(self, context: Context) -> bool:
             return self._check_res
         self._check_res = True
 
-        if self.problem.metadata.is_pass_fail() and len(self._graders) > 0:
+        if self.problem.is_pass_fail() and len(self._graders) > 0:
             self.error('There are grader programs but the problem is pass-fail')
 
         for grader in self._graders:
@@ -1202,11 +1226,13 @@ def check(self, context: Context) -> bool:
 
         self.warn_directory('output validators', 'output_validator_directory')
 
-        recommended_output_validator_languages = {'c', 'cpp', 'python3'}
+        safe_output_validator_languages = {'c', 'cpp', 'python3'}
 
         for v in self._validators:
-            if isinstance(v, run.SourceCode) and v.language.lang_id not in recommended_output_validator_languages:
-                self.warning('output validator language %s is not recommended' % v.language.name)
+            if isinstance(v, run.SourceCode) and v.language.lang_id not in safe_output_validator_languages:
+                self.error_in_2023_07(
+                    f'Output validator in {v.language.name}. Only {safe_output_validator_languages} are standardized. Check carefully if your CCS supports more (Kattis does not).'
+                )
 
         if self.problem.metadata.legacy_validation == 'default' and self._validators:
             self.error('There are validator programs but problem.yaml has validation = "default"')
@@ -1612,7 +1638,7 @@ def full_score_finite(self) -> bool:
     def fully_accepted(self, result: SubmissionResult) -> bool:
         min_score, max_score = self.problem.testdata.get_score_range()
         best_score = min_score if self.problem.metadata.legacy_grading.objective == 'min' else max_score
-        return result.verdict == 'AC' and (not self.problem.metadata.is_scoring() or result.score == best_score)
+        return result.verdict == 'AC' and (not self.problem.is_scoring() or result.score == best_score)
 
     def start_background_work(self, context: Context) -> None:
         # Send off an early background compile job for each submission and
@@ -1728,6 +1754,21 @@ def _set_timelim(self, timelim: float) -> None:  # Should only be called by Subm
         assert self._timelim is None, 'Attempted to set timelim twice'
         self._timelim = timelim
 
+    def is_pass_fail(self) -> bool:
+        return self.metadata.is_pass_fail()
+
+    def is_scoring(self) -> bool:
+        return self.metadata.is_scoring()
+
+    def is_interactive(self) -> bool:
+        return self.metadata.is_interactive()
+
+    def is_multi_pass(self) -> bool:
+        return self.metadata.is_multi_pass()
+
+    def is_submit_answer(self) -> bool:
+        return self.metadata.is_submit_answer()
+
     def load(self) -> None:
         """Parses the problem package statically, loading up information with very little verification.
 

tests/test_metadata.py

@@ -86,6 +86,21 @@ def test_parse_multi_source(minimal_2023_conf):
     assert m.source[2].url is None
 
 
+def test_parse_complex_type(minimal_2023_conf):
+    c = minimal_2023_conf
+    c['type'] = ['scoring', 'multi-pass', 'interactive']
+    m = metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'})
+    assert len(m.type) == 3
+    assert metadata.ProblemType.SCORING in m.type
+    assert metadata.ProblemType.MULTI_PASS in m.type
+    assert metadata.ProblemType.INTERACTIVE in m.type
+    assert not m.is_pass_fail()
+    assert m.is_scoring()
+    assert m.is_interactive()
+    assert m.is_multi_pass()
+    assert not m.is_submit_answer()
+
+
 def test_load_hello():
     m, _ = metadata.load_metadata(Path(__file__).parent / 'hello')
     assert m.name['en'] == 'Hello World!'
@@ -99,3 +114,5 @@ def test_load_hello():
     assert m.is_pass_fail()
     assert not m.is_scoring()
     assert not m.is_interactive()
+    assert not m.is_multi_pass()
+    assert not m.is_submit_answer()

tests/test_verify_hello.py

@@ -16,3 +16,8 @@ def test_load_hello():
         # pytest and fork don't go along very well, so just run aspects that work without run
         assert p.config.check(context)
         assert p.attachments.check(context)
+        assert p.is_pass_fail()
+        assert not p.is_scoring()
+        assert not p.is_interactive()
+        assert not p.is_multi_pass()
+        assert not p.is_submit_answer()