From 2b45cb065a8c23dfe45fd3f4eee49e947463a0bb Mon Sep 17 00:00:00 2001
From: ee7 <45465154+ee7@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:21:19 +0200
Subject: [PATCH 1/4] runner: make a run-time error a top-level `error`

This is just a start - it should probably be a top-level `fail` and a
test-level `error`.
---
 src/runner.nim | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/runner.nim b/src/runner.nim
index a17b9ca..eebcd53 100644
--- a/src/runner.nim
+++ b/src/runner.nim
@@ -193,14 +193,21 @@ proc simplifyPathsInMessageValues(j: var JsonNode) =
       simplifyPaths test["message"].str
 
 proc writeOutput*(resultsFileName, runtimeOutput: string) =
-  var testResults = parseFile resultsFileName
-  let submissionOutput = runtimeOutput.extractSubmissionOutput
-
-  for index, test in submissionOutput.tests:
-    testResults["tests"][index]["output"] = test.output.newJString()
-
-  simplifyPathsInMessageValues testResults
-  resultsFileName.writeFile $testResults
+  let resultsContents = readFile resultsFileName
+  if resultsContents.len > 0:
+    try:
+      var testResults = parseJson resultsContents
+      let submissionOutput = runtimeOutput.extractSubmissionOutput
+
+      for index, test in submissionOutput.tests:
+        testResults["tests"][index]["output"] = test.output.newJString()
+
+      simplifyPathsInMessageValues testResults
+      resultsFileName.writeFile $testResults
+    except JsonParsingError:
+      writeTopLevelErrorJson(resultsFileName, runtimeOutput)
+  else:
+    writeTopLevelErrorJson(resultsFileName, runtimeOutput)
 
 proc run*(paths: Paths): tuple[output: string, exitCode: int] =
   ## Compiles and runs the file in `paths.tmpTest`. Returns its exit code and

From 7f08dfcd59fb89b53f7e782893433d2ec915d58d Mon Sep 17 00:00:00 2001
From: ee7 <45465154+ee7@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:22:56 +0200
Subject: [PATCH 2/4] tests: add test for hitting run-time call depth limit

Again, this should probably be a top-level `fail` and a test-level
`error`.
---
 .../runtime_error_recursion_limit/expected_results.json     | 6 ++++++
 tests/error/runtime_error_recursion_limit/identity.nim      | 2 ++
 tests/error/runtime_error_recursion_limit/test_identity.nim | 6 ++++++
 3 files changed, 14 insertions(+)
 create mode 100644 tests/error/runtime_error_recursion_limit/expected_results.json
 create mode 100644 tests/error/runtime_error_recursion_limit/identity.nim
 create mode 100644 tests/error/runtime_error_recursion_limit/test_identity.nim

diff --git a/tests/error/runtime_error_recursion_limit/expected_results.json b/tests/error/runtime_error_recursion_limit/expected_results.json
new file mode 100644
index 0000000..99f5eda
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit/expected_results.json
@@ -0,0 +1,6 @@
+{
+  "version": 2,
+  "status": "error",
+  "message": "Traceback (most recent call last)\n/nim/lib/pure/unittest.nim(654) test_identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\n(1874 calls omitted) ...\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim identity\nError: call depth limit reached in a debug build (2000 function calls). You can change it with -d:nimCallDepthLimit=<int> but really try to avoid deep recursions instead.\n",
+  "tests": []
+}
diff --git a/tests/error/runtime_error_recursion_limit/identity.nim b/tests/error/runtime_error_recursion_limit/identity.nim
new file mode 100644
index 0000000..929550a
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit/identity.nim
@@ -0,0 +1,2 @@
+func identity*(n: int): int =
+  identity(n)
diff --git a/tests/error/runtime_error_recursion_limit/test_identity.nim b/tests/error/runtime_error_recursion_limit/test_identity.nim
new file mode 100644
index 0000000..f0ebd38
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit/test_identity.nim
@@ -0,0 +1,6 @@
+import std/unittest
+import identity
+
+suite "Identity Function":
+  test "identity function of 1":
+    check identity(1) == 1

From cf21fc21901bd2ce2c10395c1795873e10fadd36 Mon Sep 17 00:00:00 2001
From: ee7 <45465154+ee7@users.noreply.github.com>
Date: Mon, 5 Jul 2021 23:07:06 +0200
Subject: [PATCH 3/4] tests: use top-level `fail` for run-time error

---
 .../expected_results.json                            | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/error/runtime_error_recursion_limit/expected_results.json b/tests/error/runtime_error_recursion_limit/expected_results.json
index 99f5eda..70089b4 100644
--- a/tests/error/runtime_error_recursion_limit/expected_results.json
+++ b/tests/error/runtime_error_recursion_limit/expected_results.json
@@ -1,6 +1,12 @@
 {
   "version": 2,
-  "status": "error",
-  "message": "Traceback (most recent call last)\n/nim/lib/pure/unittest.nim(654) test_identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\n(1874 calls omitted) ...\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim identity\nError: call depth limit reached in a debug build (2000 function calls). You can change it with -d:nimCallDepthLimit=<int> but really try to avoid deep recursions instead.\n",
-  "tests": []
+  "status": "fail",
+  "tests": [
+    {
+      "name": "identity function of 1",
+      "status": "error",
+      "message": "Traceback (most recent call last)\n/nim/lib/pure/unittest.nim(654) test_identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\n(1874 calls omitted) ...\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim(2) identity\nidentity.nim identity\nError: call depth limit reached in a debug build (2000 function calls). You can change it with -d:nimCallDepthLimit=<int> but really try to avoid deep recursions instead.\n",
+      "output": ""
+    }
+  ]
 }

From 3cb1dc8e09f0ea863bbc74d10811dda68efb53e3 Mon Sep 17 00:00:00 2001
From: ee7 <45465154+ee7@users.noreply.github.com>
Date: Wed, 7 Jul 2021 12:00:23 +0200
Subject: [PATCH 4/4] tests: add test for run-time error in the non-first case

---
 .../expected_results.json                       | 17 +++++++++++++++++
 .../identity.nim                                |  5 +++++
 .../test_identity.nim                           | 12 ++++++++++++
 3 files changed, 34 insertions(+)
 create mode 100644 tests/error/runtime_error_recursion_limit_middle_test/expected_results.json
 create mode 100644 tests/error/runtime_error_recursion_limit_middle_test/identity.nim
 create mode 100644 tests/error/runtime_error_recursion_limit_middle_test/test_identity.nim

diff --git a/tests/error/runtime_error_recursion_limit_middle_test/expected_results.json b/tests/error/runtime_error_recursion_limit_middle_test/expected_results.json
new file mode 100644
index 0000000..a9e6d8c
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit_middle_test/expected_results.json
@@ -0,0 +1,17 @@
+{
+  "version": 2,
+  "status": "fail",
+  "tests": [
+    {
+      "name": "identity function of 1",
+      "status": "pass",
+      "output": ""
+    },
+    {
+      "name": "identity function of 2",
+      "status": "error",
+      "message": "Traceback (most recent call last)\n/nim/lib/pure/unittest.nim(654) test_identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\n(1874 calls omitted) ...\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim(3) identity\nidentity.nim identity\nError: call depth limit reached in a debug build (2000 function calls). You can change it with -d:nimCallDepthLimit=<int> but really try to avoid deep recursions instead.\n",
+      "output": ""
+    }
+  ]
+}
diff --git a/tests/error/runtime_error_recursion_limit_middle_test/identity.nim b/tests/error/runtime_error_recursion_limit_middle_test/identity.nim
new file mode 100644
index 0000000..a14510b
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit_middle_test/identity.nim
@@ -0,0 +1,5 @@
+func identity*(n: int): int =
+  if n == 2:
+    identity(n)
+  else:
+    n
diff --git a/tests/error/runtime_error_recursion_limit_middle_test/test_identity.nim b/tests/error/runtime_error_recursion_limit_middle_test/test_identity.nim
new file mode 100644
index 0000000..d6c95b4
--- /dev/null
+++ b/tests/error/runtime_error_recursion_limit_middle_test/test_identity.nim
@@ -0,0 +1,12 @@
+import std/unittest
+import identity
+
+suite "Identity Function":
+  test "identity function of 1":
+    check identity(1) == 1
+
+  test "identity function of 2":
+    check identity(2) == 2
+
+  test "identity function of 3":
+    check identity(3) == 3