Test coverage of default_validator

.github/workflows/python-app.yml

@@ -45,6 +45,9 @@ jobs:
     - name: Run mypy
       run: |
         venv/bin/mypy --non-interactive --config-file mypy.ini -p problemtools
+    - name: Run unit tests for default_validator
+      run: |
+        support/default_validator/test_validator.py
 
   packages: # Use a separate job to test debian packaging to speed things up (no need to test this for every python version above)
     runs-on: ubuntu-latest

support/default_validator/test_validator.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import unittest
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+class TestDefaultValidator(unittest.TestCase):
+    VALIDATOR_PATH = Path(__file__).parent / 'default_validator'
+    TESTS_DIR = Path(__file__).parent / 'tests'
+
+    @classmethod
+    def setUpClass(cls):
+        # Compile the validator before running tests.
+        try:
+            subprocess.run(['make', 'default_validator'], cwd=Path(__file__).parent, check=True, capture_output=True, text=True)
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f'Failed to compile default_validator: {e.stderr}') from e
+
+        if not cls.VALIDATOR_PATH.is_file():
+            raise FileNotFoundError(f'Validator executable not found at {cls.VALIDATOR_PATH} after compilation.')
+
+    def _run_test_case(self, test_dir: Path):
+        judge_ans = test_dir / 'judge.ans'
+        user_out = test_dir / 'user.out'
+        args_file = test_dir / 'args.txt'
+        expected_exit_code_file = test_dir / 'expected_exit_code.txt'
+        expected_message_file = test_dir / 'expected_message.txt'
+
+        self.assertTrue(judge_ans.is_file(), f'judge.ans not found in {test_dir}')
+        self.assertTrue(user_out.is_file(), f'user.out not found in {test_dir}')
+        self.assertTrue(expected_exit_code_file.is_file(), f'expected_exit_code.txt not found in {test_dir}')
+
+        args = []
+        if args_file.is_file():
+            args = args_file.read_text().split()
+
+        expected_exit_code = int(expected_exit_code_file.read_text().strip())
+
+        with tempfile.TemporaryDirectory() as feedback_dir, open(user_out, 'r') as user_out_f:
+            # The validator expects judge_in, judge_ans, feedback_dir
+            # judge_in is not used by the validator for comparison, so we can pass a dummy file.
+            with tempfile.NamedTemporaryFile() as dummy_judge_in:
+                cmd = [str(self.VALIDATOR_PATH), str(dummy_judge_in.name), str(judge_ans), feedback_dir, *args]
+
+                result = subprocess.run(cmd, stdin=user_out_f, capture_output=True, text=True)
+
+                self.assertEqual(result.returncode, expected_exit_code, f'Wrong exit code. Stderr: {result.stderr}')
+
+                if expected_message_file.is_file():
+                    judgemessage_path = Path(feedback_dir) / 'judgemessage.txt'
+                    self.assertTrue(judgemessage_path.is_file(), 'judgemessage.txt was not created')
+                    actual_message = judgemessage_path.read_text().strip()
+                    expected_message = expected_message_file.read_text().strip()
+                    self.assertEqual(actual_message, expected_message)
+
+
+# --- Dynamic Test Creation ---
+def create_test_method(test_dir):
+    """Creates a test method for a given test case directory."""
+
+    def test_method(self):
+        self._run_test_case(test_dir)
+
+    return test_method
+
+
+if TestDefaultValidator.TESTS_DIR.is_dir():
+    test_cases = [d for d in TestDefaultValidator.TESTS_DIR.iterdir() if d.is_dir() and d.name.startswith('test_')]
+    for test_case_dir in test_cases:
+        test_name = test_case_dir.name
+        test_method = create_test_method(test_case_dir)
+        setattr(TestDefaultValidator, test_name, test_method)
+# --- End of Dynamic Test Creation ---
+
+
+if __name__ == '__main__':
+    unittest.main()

support/default_validator/tests/README.md

@@ -0,0 +1,47 @@
+# Default Validator Tests
+
+This directory contains test cases for the `default_validator`. Each test case resides in its own subdirectory, named `test_<description>` (e.g., `test_simple_ac` for a simple test that should be Accepted, or `test_float_wa` for a float comparison that should result in Wrong Answer).
+
+## Structure of a Test Case
+
+Each test case directory should contain the following files:
+
+*   `judge.ans`: The reference answer file. This is what the `default_validator` will compare against.
+*   `user.out`: The user's output file that the `default_validator` will evaluate.
+*   `args.txt`: (Optional) A plain text file containing command-line arguments to be passed to the `default_validator`. Each argument should be space-separated. For example: `case_sensitive float_absolute_tolerance 0.001`.
+*   `expected_exit_code.txt`: Contains the expected exit code of the `default_validator` for this test case (e.g., `42` for Accepted, `43` for Wrong Answer).
+*   `expected_message.txt`: (Optional) Contains the exact error message expected from the `default_validator` if the test case results in a Wrong Answer. This file should only exist if a message is expected.
+
+## Adding a New Test Case
+
+To add a new test case:
+
+1.  Create a new directory within `support/default_validator/tests/` (e.g., `tests/test_my_new_feature_ac`).
+2.  Inside this new directory, create `judge.ans` and `user.out` files with the desired content for your test.
+3.  If your test requires specific command-line arguments for the `default_validator` (e.g., `case_sensitive`, `float_tolerance`), create an `args.txt` file in the directory with these arguments.
+4.  Use the `run_and_generate_expected.py` script to automatically generate `expected_exit_code.txt` and optionally `expected_message.txt`:
+    ```bash
+    cd support/default_validator/
+    ./run_and_generate_expected.py tests/test_my_new_feature_ac
+    ```
+    This script will run the `default_validator` with your provided inputs and arguments, capture its exit code and any feedback message, and write them to the respective `expected_*.txt` files.
+
+## Updating an Existing Test Case
+
+If you modify `judge.ans`, `user.out`, or `args.txt` for an existing test case, you need to update its expected output:
+
+```bash
+cd support/default_validator/tests/
+./run_and_generate_expected.py test_case_to_update
+```
+
+This will regenerate the `expected_exit_code.txt` and `expected_message.txt` files based on your changes. Check with `git diff` and commit if the changes are what you expected.
+
+## Running the Tests
+
+To run all the test cases for `default_validator`:
+
+```bash
+cd support/default_validator/
+./test_validator.py
+```

support/default_validator/tests/run_and_generate_expected.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+#
+# A script to run the default_validator on a test case and generate the expected output files.
+#
+# Usage: ./run_and_generate_expected.py <path_to_test_directory>
+#
+# The test directory must contain:
+# - judge.ans: The correct output.
+# - user.out: The output to validate.
+# - args.txt: (Optional) arguments to pass to the validator.
+#
+# The script will create:
+# - expected_exit_code.txt: The exit code of the validator.
+# - expected_message.txt: The message from the validator, if any.
+#
+
+import argparse
+import subprocess
+import tempfile
+from pathlib import Path
+import sys
+
+
+def main():
+    """Main function"""
+    parser = argparse.ArgumentParser(description='Run default_validator and generate expected output files.')
+    parser.add_argument('test_dir', type=Path, help='Path to the test directory.')
+    args = parser.parse_args()
+
+    test_dir: Path = args.test_dir
+
+    if not test_dir.is_dir():
+        print(f'Error: Test directory not found at {test_dir}', file=sys.stderr)
+        sys.exit(1)
+
+    validator_path = Path(__file__).parent.parent / 'default_validator'
+    if not validator_path.is_file():
+        print('Compiling default_validator...', file=sys.stderr)
+        try:
+            subprocess.run(
+                ['make', 'default_validator'],
+                cwd=validator_path.parent,
+                check=True,
+                capture_output=True,
+                text=True,
+                encoding='utf-8',
+            )
+        except subprocess.CalledProcessError as e:
+            print(f'Failed to compile default_validator: {e.stderr}', file=sys.stderr)
+            sys.exit(1)
+        except Exception as e:
+            print(f'An unexpected error occurred during compilation: {e}', file=sys.stderr)
+            sys.exit(1)
+
+        if not validator_path.is_file():
+            print(f'Validator executable not found at {validator_path} after compilation.', file=sys.stderr)
+            sys.exit(1)
+
+    judge_ans = test_dir / 'judge.ans'
+    user_out = test_dir / 'user.out'
+    args_file = test_dir / 'args.txt'
+
+    if not judge_ans.is_file():
+        print(f'Error: judge.ans not found in {test_dir}', file=sys.stderr)
+        sys.exit(1)
+    if not user_out.is_file():
+        print(f'Error: user.out not found in {test_dir}', file=sys.stderr)
+        sys.exit(1)
+
+    validator_args = []
+    if args_file.is_file():
+        args_text = args_file.read_text(encoding='utf-8').strip()
+        if args_text:
+            validator_args = args_text.split()
+
+    with tempfile.TemporaryDirectory() as feedback_dir:
+        # The validator expects judge_in, judge_ans, feedback_dir
+        # judge_in is not used by the validator for comparison, so we can pass a dummy file.
+        with tempfile.NamedTemporaryFile() as dummy_judge_in, open(user_out, 'r', encoding='utf-8') as user_out_f:
+            cmd = [str(validator_path), str(dummy_judge_in.name), str(judge_ans), feedback_dir, *validator_args]
+
+            result = subprocess.run(cmd, stdin=user_out_f, capture_output=True, text=True, encoding='utf-8')
+
+            # Write expected_exit_code.txt
+            (test_dir / 'expected_exit_code.txt').write_text(str(result.returncode) + '\n', encoding='utf-8')
+            print(f'Wrote exit code {result.returncode} to expected_exit_code.txt')
+
+            # Write expected_message.txt if a message was generated
+            judgemessage_path = Path(feedback_dir) / 'judgemessage.txt'
+            if judgemessage_path.is_file():
+                message = judgemessage_path.read_text(encoding='utf-8')
+                if message:
+                    (test_dir / 'expected_message.txt').write_text(message, encoding='utf-8')
+                    print('Wrote message to expected_message.txt')
+            else:
+                # If there's no message, we should remove any existing expected_message.txt
+                expected_message_file = test_dir / 'expected_message.txt'
+                if expected_message_file.is_file():
+                    expected_message_file.unlink()
+                    print('Removed existing expected_message.txt as no message was generated.')
+
+
+if __name__ == '__main__':
+    main()

support/default_validator/tests/test_case_sensitive_ac/args.txt

@@ -0,0 +1 @@
+case_sensitive

support/default_validator/tests/test_case_sensitive_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_case_sensitive_ac/judge.ans

@@ -0,0 +1 @@
+Hello

support/default_validator/tests/test_case_sensitive_ac/user.out

@@ -0,0 +1 @@
+Hello

support/default_validator/tests/test_case_sensitive_wa/args.txt

@@ -0,0 +1 @@
+case_sensitive

support/default_validator/tests/test_case_sensitive_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_case_sensitive_wa/expected_message.txt

@@ -0,0 +1,4 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+String tokens mismatch
+Judge: "Hello"
+User: "hello"

support/default_validator/tests/test_case_sensitive_wa/judge.ans

@@ -0,0 +1 @@
+Hello

support/default_validator/tests/test_case_sensitive_wa/user.out

@@ -0,0 +1 @@
+hello

support/default_validator/tests/test_combined_flags_ac/args.txt

@@ -0,0 +1 @@
+case_sensitive space_change_sensitive float_tolerance 0.5

support/default_validator/tests/test_combined_flags_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_combined_flags_ac/judge.ans

@@ -0,0 +1 @@
+PI is 3.14

support/default_validator/tests/test_combined_flags_ac/user.out

@@ -0,0 +1 @@
+PI is 3.14

support/default_validator/tests/test_combined_flags_wa/args.txt

@@ -0,0 +1 @@
+case_sensitive space_change_sensitive float_tolerance 0.5

support/default_validator/tests/test_combined_flags_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_combined_flags_wa/expected_message.txt

@@ -0,0 +1,6 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+Too large difference.
+ Judge: 1.0
+ User: 2.0
+ Difference: -1.000000e+00
+ (abs tol 5.000000e-01 rel tol 5.000000e-01)

support/default_validator/tests/test_combined_flags_wa/judge.ans

@@ -0,0 +1 @@
+PI is 1.0

support/default_validator/tests/test_combined_flags_wa/user.out

@@ -0,0 +1 @@
+PI is 2.0

support/default_validator/tests/test_float_abs_tol_ac/args.txt

@@ -0,0 +1 @@
+float_absolute_tolerance 0.5

support/default_validator/tests/test_float_abs_tol_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_float_abs_tol_ac/judge.ans

@@ -0,0 +1 @@
+1

support/default_validator/tests/test_float_abs_tol_ac/user.out

@@ -0,0 +1 @@
+1.4

support/default_validator/tests/test_float_abs_tol_wa/args.txt

@@ -0,0 +1 @@
+float_absolute_tolerance 0.1

support/default_validator/tests/test_float_abs_tol_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_float_abs_tol_wa/expected_message.txt

@@ -0,0 +1,6 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+Too large difference.
+ Judge: 1
+ User: 1.4
+ Difference: -4.000000e-01
+ (abs tol 1.000000e-01 rel tol -1.000000e+00)

support/default_validator/tests/test_float_abs_tol_wa/judge.ans

@@ -0,0 +1 @@
+1

support/default_validator/tests/test_float_abs_tol_wa/user.out

@@ -0,0 +1 @@
+1.4

support/default_validator/tests/test_float_inf_judge_wa/args.txt

@@ -0,0 +1 @@
+float_tolerance 0.5

support/default_validator/tests/test_float_inf_judge_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_float_inf_judge_wa/expected_message.txt

@@ -0,0 +1,4 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+String tokens mismatch
+Judge: "inf"
+User: "0"

support/default_validator/tests/test_float_inf_judge_wa/judge.ans

@@ -0,0 +1 @@
+inf

support/default_validator/tests/test_float_inf_judge_wa/user.out

@@ -0,0 +1 @@
+0

support/default_validator/tests/test_float_rel_tol_ac/args.txt

@@ -0,0 +1 @@
+float_relative_tolerance 0.1

support/default_validator/tests/test_float_rel_tol_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_float_rel_tol_ac/judge.ans

@@ -0,0 +1 @@
+100

support/default_validator/tests/test_float_rel_tol_ac/user.out

@@ -0,0 +1 @@
+105

support/default_validator/tests/test_float_rel_tol_wa/args.txt

@@ -0,0 +1 @@
+float_relative_tolerance 0.01

support/default_validator/tests/test_float_rel_tol_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_float_rel_tol_wa/expected_message.txt

@@ -0,0 +1,6 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+Too large difference.
+ Judge: 100
+ User: 105
+ Difference: -5.000000e+00
+ (abs tol -1.000000e+00 rel tol 1.000000e-02)

support/default_validator/tests/test_float_rel_tol_wa/judge.ans

@@ -0,0 +1 @@
+100

support/default_validator/tests/test_float_rel_tol_wa/user.out

@@ -0,0 +1 @@
+105

support/default_validator/tests/test_float_tolerance_ac/args.txt

@@ -0,0 +1 @@
+float_tolerance 0.5

support/default_validator/tests/test_float_tolerance_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_float_tolerance_ac/judge.ans

@@ -0,0 +1 @@
+1

support/default_validator/tests/test_float_tolerance_ac/user.out

@@ -0,0 +1 @@
+1.1

support/default_validator/tests/test_float_tolerance_wa/args.txt

@@ -0,0 +1 @@
+float_tolerance 0.5

support/default_validator/tests/test_float_tolerance_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_float_tolerance_wa/expected_message.txt

@@ -0,0 +1,6 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+Too large difference.
+ Judge: 1
+ User: 1.50001
+ Difference: -5.000100e-01
+ (abs tol 5.000000e-01 rel tol 5.000000e-01)

support/default_validator/tests/test_float_tolerance_wa/judge.ans

@@ -0,0 +1 @@
+1

support/default_validator/tests/test_float_tolerance_wa/user.out

@@ -0,0 +1 @@
+1.50001

support/default_validator/tests/test_simple_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_simple_ac/judge.ans

@@ -0,0 +1 @@
+hello world

support/default_validator/tests/test_simple_ac/user.out

@@ -0,0 +1 @@
+hello world

support/default_validator/tests/test_simple_wa/expected_exit_code.txt

@@ -0,0 +1 @@
+43

support/default_validator/tests/test_simple_wa/expected_message.txt

@@ -0,0 +1,4 @@
+Wrong answer on line 1 of output (corresponding to line 1 in answer file)
+String tokens mismatch
+Judge: "world"
+User: "there"

support/default_validator/tests/test_simple_wa/judge.ans

@@ -0,0 +1 @@
+hello world

support/default_validator/tests/test_simple_wa/user.out

@@ -0,0 +1 @@
+hello there

support/default_validator/tests/test_space_sensitive_ac/args.txt

@@ -0,0 +1 @@
+space_change_sensitive

support/default_validator/tests/test_space_sensitive_ac/expected_exit_code.txt

@@ -0,0 +1 @@
+42

support/default_validator/tests/test_space_sensitive_ac/judge.ans

@@ -0,0 +1,2 @@
+hello
+ world

support/default_validator/tests/test_space_sensitive_ac/user.out

@@ -0,0 +1,2 @@
+hello
+ world

support/default_validator/tests/test_space_sensitive_wa/args.txt

@@ -0,0 +1 @@
+space_change_sensitive