|
1 | | -#!/usr/bin/env sh |
| 1 | +#!/usr/bin/env bash |
2 | 2 |
|
3 | 3 | # Synopsis: |
4 | | -# Run the test runner on a solution. |
| 4 | +# Run the Exercism Zig test runner on a solution. |
5 | 5 |
|
6 | 6 | # Arguments: |
7 | 7 | # $1: exercise slug |
8 | 8 | # $2: path to solution folder |
9 | 9 | # $3: path to output directory |
10 | 10 |
|
11 | 11 | # Output: |
12 | | -# Writes the test results to a results.json file in the passed-in output directory. |
13 | | -# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md |
| 12 | +# Writes a v2 results.json to the output directory, per |
| 13 | +# https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md |
14 | 14 |
|
15 | 15 | # Example: |
16 | 16 | # ./bin/run.sh two-fer path/to/solution/folder/ path/to/output/directory/ |
17 | 17 |
|
18 | | -# If any required arguments is missing, print the usage and exit |
19 | | -if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then |
| 18 | +# Print usage and exit non-zero. Called when required args are missing. |
| 19 | +usage() { |
20 | 20 | echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/" |
21 | 21 | exit 1 |
22 | | -fi |
| 22 | +} |
23 | 23 |
|
24 | | -slug="$1" |
25 | | -test_file="$(echo "test_${slug}.zig" | tr '-' '_')" |
26 | | -solution_dir=$(realpath "${2%/}") |
27 | | -output_dir=$(realpath "${3%/}") |
28 | | -results_file="${output_dir}/results.json" |
| 24 | +# Parse positional args, set globals consumed by later stages. |
| 25 | +# Sets: slug, test_file, solution_dir, output_dir, results_file. |
| 26 | +parse_args() { |
| 27 | + [[ -z "$1" || -z "$2" || -z "$3" ]] && usage |
| 28 | + slug="$1" |
| 29 | + test_file="$(echo "test_${slug}.zig" | tr '-' '_')" |
| 30 | + solution_dir=$(realpath "${2%/}") |
| 31 | + output_dir=$(realpath "${3%/}") |
| 32 | + results_file="${output_dir}/results.json" |
| 33 | +} |
29 | 34 |
|
30 | | -# Create the output directory if it doesn't exist |
31 | | -mkdir -p "${output_dir}" |
| 35 | +# Compile and run the solution's test file, strip the solution-dir prefix |
| 36 | +# and the "test command failed" trailer from the output so it is portable. |
| 37 | +# Prints the sanitized output to stdout and returns zig's exit code. |
| 38 | +run_zig_test() { |
| 39 | + local raw_output zig_exit |
| 40 | + raw_output=$(cd "${solution_dir}" && zig test -target x86_64-linux-musl "${test_file}" 2>&1) |
| 41 | + zig_exit=$? |
| 42 | + printf '%s' "${raw_output}" \ |
| 43 | + | sed -e "s#${solution_dir}/\{0,1\}##g" \ |
| 44 | + -e '/error: the following test command failed/,$d' |
| 45 | + return "${zig_exit}" |
| 46 | +} |
32 | 47 |
|
33 | | -echo "${slug}: testing..." |
| 48 | +# Emit a top-level error report (compile failure) and exit successfully — |
| 49 | +# the runner completed its job even though the solution did not build. |
| 50 | +emit_compile_error() { |
| 51 | + jq -n --arg message "${test_output}" \ |
| 52 | + '{version: 2, status: "error", message: $message}' > "${results_file}" |
| 53 | + echo "${slug}: done" |
| 54 | + exit 0 |
| 55 | +} |
34 | 56 |
|
35 | | -start_dir="$(pwd)" |
36 | | -cd "${solution_dir}" || exit 1 |
| 57 | +# Parse Zig's per-test output into a JSON array of test records. |
| 58 | +# Zig test output has one line per test, optionally followed by user |
| 59 | +# stdout and a status line: |
| 60 | +# 1/5 test_file.test.TEST NAME...OK (no user output) |
| 61 | +# 2/5 test_file.test.TEST NAME...Hello, World! (user output) |
| 62 | +# OK (status on next line) |
| 63 | +# 3/5 test_file.test.TEST NAME...FAIL (reason) (immediate fail) |
| 64 | +# Failed tests are followed by stack trace lines until the next test line |
| 65 | +# or the summary ("N passed; N skipped; N failed."). Parsing is done in |
| 66 | +# two passes: `segment` groups the raw lines into per-test blocks, then |
| 67 | +# `classify` turns each block into a single JSON test record. |
| 68 | +build_tests_json() { |
| 69 | + printf '%s' "${test_output}" | jq -Rs ' |
| 70 | + def test_name: capture("test\\.(?<n>.+)\\.\\.\\.") | .n; |
| 71 | + def user_output: capture("\\.\\.\\.((?<o>.+)$)") | .o // ""; |
| 72 | + def is_test_line: test("[0-9]+/[0-9]+ .*\\.test\\..*\\.\\.\\."); |
| 73 | + def is_summary: startswith("All ") or test("[0-9]+ passed;"); |
| 74 | + def is_ok: . == "OK" or endswith("OK"); |
| 75 | + def is_fail: startswith("FAIL") or test("\\.\\.\\.(FAIL|expected )"); |
37 | 76 |
|
38 | | -# Run the tests for the provided implementation file and redirect stdout and |
39 | | -# stderr to capture it |
40 | | -test_output=$(zig test -target x86_64-linux-musl "${test_file}" 2>&1) |
41 | | -exit_code=$? |
| 77 | + # Per-outcome record constructors. These set the leading fields; |
| 78 | + # classify extends records via `.field = ...` in the same order, |
| 79 | + # producing {name, status, message, output}. |
| 80 | + def pass_record(name): {name: name, status: "pass"}; |
| 81 | + def fail_record(name): {name: name, status: "fail", message: ""}; |
42 | 82 |
|
43 | | -cd "${start_dir}" || exit 1 |
| 83 | + # Pass 1 — segment the flat output into per-test line groups. |
| 84 | + # A new group opens on a test-header line; the current group |
| 85 | + # closes on the next test-header line or on the summary line. |
| 86 | + def segment: |
| 87 | + split("\n") |
| 88 | + | reduce .[] as $line ( |
| 89 | + {current: null, groups: []}; |
| 90 | + if ($line | is_test_line) then |
| 91 | + (if .current then .groups += [.current] else . end) |
| 92 | + | .current = [$line] |
| 93 | + elif ($line | is_summary) then |
| 94 | + (if .current then .groups += [.current] else . end) |
| 95 | + | .current = null |
| 96 | + elif .current then |
| 97 | + .current += [$line] |
| 98 | + else . end |
| 99 | + ) |
| 100 | + | if .current then .groups += [.current] else .groups end; |
44 | 101 |
|
45 | | -# Write the results.json file based on the exit code of the command that was |
46 | | -# just executed that tested the implementation file |
47 | | -if [ ${exit_code} -eq 0 ]; then |
48 | | - jq -n '{version: 1, status: "pass"}' > "${results_file}" |
49 | | -else |
50 | | - # Sanitize the output |
51 | | - sanitized_test_output=$(printf '%s' "${test_output}" | sed -n -e '/error: the following test command failed/q;p') |
| 102 | + # Pass 2 — turn one line group into one test record. |
| 103 | + # Three shapes handled: |
| 104 | + # (a) header ends with OK → immediate pass, no output |
| 105 | + # (b) header has FAIL/expected → immediate fail, message is |
| 106 | + # the stack trace in $rest |
| 107 | + # (c) otherwise → pending: user output on the |
| 108 | + # header, status line lives |
| 109 | + # somewhere in $rest |
| 110 | + def classify: |
| 111 | + .[0] as $header |
| 112 | + | ($header | test_name) as $name |
| 113 | + | ($header | user_output) as $header_output |
| 114 | + | .[1:] as $rest |
| 115 | + | if ($header | endswith("OK")) then |
| 116 | + pass_record($name) |
| 117 | + elif ($header | is_fail) then |
| 118 | + fail_record($name) |
| 119 | + | .message = ($rest | join("\n") | sub("\n+$"; "")) |
| 120 | + else |
| 121 | + ($rest | map(is_ok or is_fail) | index(true)) as $i |
| 122 | + | $rest[:$i] as $extra_output |
| 123 | + | $rest[$i:] as $from_status |
| 124 | + | ($header_output |
| 125 | + + (if ($extra_output | length) > 0 |
| 126 | + then "\n" + ($extra_output | join("\n")) |
| 127 | + else "" end)) as $output |
| 128 | + | if $from_status[0] | is_ok then |
| 129 | + pass_record($name) | .output = $output |
| 130 | + else |
| 131 | + fail_record($name) |
| 132 | + | .message = ($from_status | join("\n") | sub("\n+$"; "")) |
| 133 | + | .output = $output |
| 134 | + end |
| 135 | + end; |
52 | 136 |
|
53 | | - # Try to distinguish between failing tests and errors |
54 | | - if echo "${sanitized_test_output}" | grep "error:"; then |
55 | | - status="error" |
| 137 | + segment |
| 138 | + | map(classify) |
| 139 | + | map( |
| 140 | + if .message == "" or .message == null then del(.message) else . end |
| 141 | + | if .output == "" or .output == null then del(.output) else . end |
| 142 | + ) |
| 143 | + ' |
| 144 | +} |
| 145 | + |
| 146 | +# Write the final results.json. Truncates each test's "output" field to |
| 147 | +# 500 chars to bound report size. |
| 148 | +assemble_report() { |
| 149 | + local overall="$1" |
| 150 | + local tests_json="$2" |
| 151 | + jq -n --arg status "${overall}" --argjson tests "${tests_json}" ' |
| 152 | + def trunc: if length > 500 then .[:481] + " [output truncated]" else . end; |
| 153 | + {version: 2, status: $status, tests: ($tests | map( |
| 154 | + if .output then .output |= trunc else . end |
| 155 | + ))} |
| 156 | + ' > "${results_file}" |
| 157 | +} |
| 158 | + |
| 159 | +main() { |
| 160 | + parse_args "$@" |
| 161 | + mkdir -p "${output_dir}" |
| 162 | + echo "${slug}: testing..." |
| 163 | + |
| 164 | + local any_failed=0 |
| 165 | + test_output=$(run_zig_test) || any_failed=1 |
| 166 | + if (( any_failed )) && [[ "${test_output}" = *error:* ]]; then |
| 167 | + emit_compile_error |
| 168 | + fi |
| 169 | + |
| 170 | + local tests_json overall |
| 171 | + tests_json=$(build_tests_json) |
| 172 | + if (( any_failed == 0 )); then |
| 173 | + overall="pass" |
56 | 174 | else |
57 | | - status="fail" |
58 | | - fi |
| 175 | + overall="fail" |
| 176 | + fi |
| 177 | + assemble_report "${overall}" "${tests_json}" |
59 | 178 |
|
60 | | - jq -n --arg output "${sanitized_test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' > "${results_file}" |
61 | | -fi |
| 179 | + echo "${slug}: done" |
| 180 | +} |
62 | 181 |
|
63 | | -echo "${slug}: done" |
| 182 | +main "$@" |
0 commit comments