Upgrade test runner to version 2

Author: keiravillekode

bin/run-tests.sh

@@ -27,7 +27,7 @@ for test_dir in tests/*; do
       -e 's/\s*Elapsed time: [0-9]+ ms, on [0-9]+ job.\s*//g' \
       -e 's/\s*[0-9]+(\.[0-9]+)? ms\s*//g' \
       -e 's/\s*[0-9]+ (v?lines|bytes)\s*//g' \
-      -e 's#v_[0-9]+/tsession[^/]+#tsession#g' \
+      -e 's#v_[0-9]+/tsession[^/ "]+#tsession#g' \
       "${results_file_path}"
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"

bin/run.sh

@@ -1,16 +1,16 @@
 #!/usr/bin/env sh
 
 # Synopsis:
-# Run the test runner on a solution.
-
+# Run the Exercism V test runner on a solution.
+#
 # Arguments:
-# $1: exercise slug
-# $2: path to solution folder
-# $3: path to output directory
-
+#   $1: exercise slug
+#   $2: path to solution folder
+#   $3: path to output directory
+#
 # Output:
-# Writes the test results to a results.json file in the passed-in output directory.
-# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
+# Writes a v2 results.json to the output directory, per
+# https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 
 # Example:
 # ./bin/run.sh two-fer path/to/solution/folder/ path/to/output/directory/
@@ -33,26 +33,99 @@ echo "${slug}: testing..."
 
 cd "${solution_dir}" > /dev/null
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
 test_output=$(v -stats test run_test.v 2>&1)
 exit_code=$?
 
 cd - > /dev/null
 
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
+# Strip the solution directory prefix from paths so output is portable,
+# and remove ANSI color codes (V emits them even when captured)
+test_output=$(printf '%s' "${test_output}" | sed -e "s#${solution_dir}/\{0,1\}##g" -e 's/\x1b\[[0-9;]*m//g')
+
+# ---------- Error case (compile failure) ----------
+if [ ${exit_code} -ne 0 ] && echo "${test_output}" | grep -q "error:"; then
+    jq -n --arg output "${test_output}" \
+        '{version: 2, status: "error", message: $output}' > "${results_file}"
+    echo "${slug}: done"
+    exit 0
+fi
+
+# ---------- Extract test_code from the test file (in source order) ----------
+# Walks run_test.v line by line. When a "fn test_xxx() {" line is found,
+# collects subsequent lines (unindented) as the test body until "}".
+test_codes_json=$(jq -Rs '
+    split("\n") |
+    reduce .[] as $line (
+        {inside_test: false, name: "", body: "", result: []};
+        if .inside_test then
+            if ($line | test("^}")) then
+                .result += [{name: .name, test_code: .body}] |
+                .inside_test = false | .name = "" | .body = ""
+            else
+                .body += (if .body != "" then "\n" else "" end)
+                       + ($line | ltrimstr("\t"))
+            end
+        elif ($line | test("^fn test_")) then
+            .inside_test = true |
+            .name = ($line | capture("fn (?<n>test_[a-zA-Z0-9_]+)") | .n)
+        else . end
+    ) | .result
+' "${solution_dir}/run_test.v")
+
+# ---------- Parse V test output into per-test results ----------
+# V prints one "OK" or "FAIL" line per test with the pattern:
+#     OK    [1/9]...  | main.test_xxx()
+#     FAIL  [1/9]...  | main.test_xxx()
+# Lines between results (file:line, assert expression) are accumulated
+# as the failure message for the next FAIL.
+test_results_json=$(printf '%s' "${test_output}" | jq -Rs '
+    def test_name: capture("main\\.(?<n>[^(]+)") | .n;
+    split("\n") |
+    reduce .[] as $line (
+        {pending: "", tests: []};
+        if ($line | test("OK.*\\| *main\\.")) then
+            .tests += [{
+                name: ($line | test_name),
+                status: "pass"
+            }] | .pending = ""
+        elif ($line | test("FAIL.*\\| *main\\.")) then
+            .tests += [{
+                name: ($line | test_name),
+                status: "fail",
+                message: .pending
+            }] | .pending = ""
+        elif ($line | test("^---- Testing|^running tests in:|Summary for|^--------|^Failed command|compilation |generated  |V  source")) then
+            .
+        else
+            .pending += (if .pending != "" then "\n" + $line else $line end)
+        end
+    ) | .tests
+')
+
+# ---------- Merge in source file order ----------
+# Use test_codes order (= source file order) as the authority.
+# For each test_code entry, find the matching result by name.
+tests_json=$(jq -n \
+    --argjson codes "${test_codes_json}" \
+    --argjson results "${test_results_json}" \
+    '[ $codes[] | . as $c |
+        ($results[] | select(.name == $c.name)) // {status: "error", message: "Test did not run."}
+        | . + $c
+    ]')
+
+# ---------- Determine overall status ----------
 if [ ${exit_code} -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
+    overall="pass"
 else
-    echo "${test_output}" | grep -q "error:"
-    if [ $? -eq 0 ]; then
-        status="error"
-    else
-        status="fail"
-    fi
-
-    jq -n --arg output "${test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' > ${results_file}
+    overall="fail"
 fi
 
+# ---------- Assemble results and truncate output fields to 500 chars ----------
+jq -n --arg status "${overall}" --argjson tests "${tests_json}" '
+    def trunc: if length > 500 then .[:481] + " [output truncated]" else . end;
+    {version: 2, status: $status, tests: ($tests | map(
+        if .output then .output |= trunc else . end
+    ))}
+' > "${results_file}"
+
 echo "${slug}: done"

tests/example-all-fail/expected_results.json

@@ -1,5 +1,60 @@
 {
-  "version": 1,
+  "version": 2,
   "status": "fail",
-  "message": "---- Testing... ----------------------------------------------------------------\nrunning tests in: /opt/test-runner/tests/example-all-fail/run_test.v\n/opt/test-runner/tests/example-all-fail/run_test.v:4: fn test_not_divisible_by_4\n    assert !is_leap_year(2015)\n\n      FAIL  [1/9]1 assert  | main.test_not_divisible_by_4()\n/opt/test-runner/tests/example-all-fail/run_test.v:8: fn test_divisible_by_2_not_by_4\n    assert !is_leap_year(1970)\n\n      FAIL  [2/9]1 assert  | main.test_divisible_by_2_not_by_4()\n/opt/test-runner/tests/example-all-fail/run_test.v:12: fn test_divisible_by_4_not_by_100\n    assert is_leap_year(1996)\n\n      FAIL  [3/9]1 assert  | main.test_divisible_by_4_not_by_100()\n/opt/test-runner/tests/example-all-fail/run_test.v:16: fn test_divisible_by_4_and_5\n    assert is_leap_year(1960)\n\n      FAIL  [4/9]1 assert  | main.test_divisible_by_4_and_5()\n/opt/test-runner/tests/example-all-fail/run_test.v:20: fn test_divisible_by_100_not_by_400\n    assert !is_leap_year(2100)\n\n      FAIL  [5/9]1 assert  | main.test_divisible_by_100_not_by_400()\n/opt/test-runner/tests/example-all-fail/run_test.v:24: fn test_divisible_by_100_not_by_3\n    assert !is_leap_year(1900)\n\n      FAIL  [6/9]1 assert  | main.test_divisible_by_100_not_by_3()\n/opt/test-runner/tests/example-all-fail/run_test.v:28: fn test_divisible_by_400\n    assert is_leap_year(2000)\n\n      FAIL  [7/9]1 assert  | main.test_divisible_by_400()\n/opt/test-runner/tests/example-all-fail/run_test.v:32: fn test_divisible_by_100_not_by_125\n    assert is_leap_year(2400)\n\n      FAIL  [8/9]1 assert  | main.test_divisible_by_100_not_by_125()\n/opt/test-runner/tests/example-all-fail/run_test.v:36: fn test_divisible_by_200_not_by_400\n    assert !is_leap_year(1800)\n\n      FAIL  [9/9]1 assert  | main.test_divisible_by_200_not_by_400()\n     Summary for running V tests in \"run_test.v\": 9 failed, 9 total. Elapsed time:.\n        V  source  code size:,,   267 types,    12 modules,   128 files\ngenerated  target  code size:,\ncompilation took:, compilation speed:/s\n\n--------------------------------------------------------------------------------\nFailed command 1:    '/opt/vlang/v'  -stats  -o '/tmp/tsession/0_0/run_test' '/opt/test-runner/tests/example-all-fail/run_test.v'\nSummary for all V _test.v files: 1 failed, 1 total.Comptime:. Runtime:."
+  "tests": [
+    {
+      "name": "test_not_divisible_by_4",
+      "status": "fail",
+      "message": "run_test.v:4: fn test_not_divisible_by_4\n    assert !is_leap_year(2015)\n",
+      "test_code": "assert !is_leap_year(2015)"
+    },
+    {
+      "name": "test_divisible_by_2_not_by_4",
+      "status": "fail",
+      "message": "run_test.v:8: fn test_divisible_by_2_not_by_4\n    assert !is_leap_year(1970)\n",
+      "test_code": "assert !is_leap_year(1970)"
+    },
+    {
+      "name": "test_divisible_by_4_not_by_100",
+      "status": "fail",
+      "message": "run_test.v:12: fn test_divisible_by_4_not_by_100\n    assert is_leap_year(1996)\n",
+      "test_code": "assert is_leap_year(1996)"
+    },
+    {
+      "name": "test_divisible_by_4_and_5",
+      "status": "fail",
+      "message": "run_test.v:16: fn test_divisible_by_4_and_5\n    assert is_leap_year(1960)\n",
+      "test_code": "assert is_leap_year(1960)"
+    },
+    {
+      "name": "test_divisible_by_100_not_by_400",
+      "status": "fail",
+      "message": "run_test.v:20: fn test_divisible_by_100_not_by_400\n    assert !is_leap_year(2100)\n",
+      "test_code": "assert !is_leap_year(2100)"
+    },
+    {
+      "name": "test_divisible_by_100_not_by_3",
+      "status": "fail",
+      "message": "run_test.v:24: fn test_divisible_by_100_not_by_3\n    assert !is_leap_year(1900)\n",
+      "test_code": "assert !is_leap_year(1900)"
+    },
+    {
+      "name": "test_divisible_by_400",
+      "status": "fail",
+      "message": "run_test.v:28: fn test_divisible_by_400\n    assert is_leap_year(2000)\n",
+      "test_code": "assert is_leap_year(2000)"
+    },
+    {
+      "name": "test_divisible_by_100_not_by_125",
+      "status": "fail",
+      "message": "run_test.v:32: fn test_divisible_by_100_not_by_125\n    assert is_leap_year(2400)\n",
+      "test_code": "assert is_leap_year(2400)"
+    },
+    {
+      "name": "test_divisible_by_200_not_by_400",
+      "status": "fail",
+      "message": "run_test.v:36: fn test_divisible_by_200_not_by_400\n    assert !is_leap_year(1800)\n",
+      "test_code": "assert !is_leap_year(1800)"
+    }
+  ]
 }

tests/example-anagram-success/anagram.v

@@ -0,0 +1,18 @@
+module main
+
+fn sorted_lower(s string) []rune {
+	mut r := s.to_lower().runes()
+	r.sort()
+	return r
+}
+
+fn find_anagrams(subject string, candidates []string) []string {
+	target := sorted_lower(subject)
+	mut result := []string{}
+	for c in candidates {
+		if c.to_lower() != subject.to_lower() && sorted_lower(c) == target {
+			result << c
+		}
+	}
+	return result
+}

tests/example-anagram-success/expected_results.json

@@ -0,0 +1,26 @@
+{
+  "version": 2,
+  "status": "pass",
+  "tests": [
+    {
+      "name": "test_no_matches",
+      "status": "pass",
+      "test_code": "candidates := ['hello', 'world', 'zombies', 'pants']\nexpected := []string{}\nassert find_anagrams('diaper', candidates) == expected"
+    },
+    {
+      "name": "test_detects_three_anagrams",
+      "status": "pass",
+      "test_code": "candidates := ['gallery', 'ballerina', 'regally', 'clergy', 'largely', 'leading']\nexpected := ['gallery', 'regally', 'largely']\nassert find_anagrams('allergy', candidates) == expected"
+    },
+    {
+      "name": "test_detects_anagrams_case_insensitively",
+      "status": "pass",
+      "test_code": "candidates := ['cashregister', 'Carthorse', 'radishes']\nexpected := ['Carthorse']\nassert find_anagrams('Orchestra', candidates) == expected"
+    },
+    {
+      "name": "test_different_characters_may_have_the_same_bytes",
+      "status": "pass",
+      "test_code": "candidates := ['€a']\nexpected := []string{}\nassert find_anagrams('a⬂', candidates) == expected"
+    }
+  ]
+}

tests/example-anagram-success/run_test.v

@@ -0,0 +1,25 @@
+module main
+
+fn test_no_matches() {
+	candidates := ['hello', 'world', 'zombies', 'pants']
+	expected := []string{}
+	assert find_anagrams('diaper', candidates) == expected
+}
+
+fn test_detects_three_anagrams() {
+	candidates := ['gallery', 'ballerina', 'regally', 'clergy', 'largely', 'leading']
+	expected := ['gallery', 'regally', 'largely']
+	assert find_anagrams('allergy', candidates) == expected
+}
+
+fn test_detects_anagrams_case_insensitively() {
+	candidates := ['cashregister', 'Carthorse', 'radishes']
+	expected := ['Carthorse']
+	assert find_anagrams('Orchestra', candidates) == expected
+}
+
+fn test_different_characters_may_have_the_same_bytes() {
+	candidates := ['€a']
+	expected := []string{}
+	assert find_anagrams('a⬂', candidates) == expected
+}