chore: Add github step summaries for e2e test suites

Added a workflow step to generate markdown summaries of test results
directly in the continuous integration interface. Providing this
information in the summary view helped developers identify failing
tests immediately without needing to dig through raw execution logs.

Signed-off-by: Chmouel Boudjnah <chmouel@redhat.com>

Author: chmouel

.github/workflows/e2e.yaml

@@ -386,6 +386,13 @@ jobs:
         run: |
           ./hack/gh-workflow-ci.sh run_e2e_tests
 
+      - name: Generate GitHub Step Summary
+        if: ${{ always() }}
+        env:
+          TEST_PROVIDER: ${{ matrix.provider }}
+        run: |
+          ./hack/gh-workflow-ci.sh generate_github_summary
+
       - name: Collect logs
         if: ${{ always() }}
         env:

hack/generate_github_summary.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Parse gotestsum JSON output and generate a GitHub Step Summary in markdown."""
+
+import collections
+import json
+import sys
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <json-file> <provider-name>", file=sys.stderr)
+        sys.exit(1)
+
+    json_file = sys.argv[1]
+    target = sys.argv[2]
+
+    results = {}  # test_name -> {action, elapsed}
+    test_output = collections.defaultdict(list)
+
+    with open(json_file) as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            test = entry.get("Test")
+            if not test:
+                continue
+            action = entry.get("Action", "")
+            if action == "output":
+                test_output[test].append(entry.get("Output", ""))
+            elif action in ("pass", "fail", "skip"):
+                results[test] = {
+                    "action": action,
+                    "elapsed": entry.get("Elapsed", 0),
+                }
+
+    passed = sorted([t for t, r in results.items() if r["action"] == "pass"])
+    failed = sorted([t for t, r in results.items() if r["action"] == "fail"])
+    skipped = sorted([t for t, r in results.items() if r["action"] == "skip"])
+
+    # Filter to top-level tests only (no subtests) for counts
+    top_passed = [t for t in passed if "/" not in t]
+    top_failed = [t for t in failed if "/" not in t]
+    top_skipped = [t for t in skipped if "/" not in t]
+
+    status = ":x: FAILED" if top_failed else ":white_check_mark: PASSED"
+
+    lines = []
+    lines.append(f"## E2E Tests: {target} {status}")
+    lines.append("")
+    lines.append(
+        f":white_check_mark: **{len(top_passed)}** passed"
+        f" | :x: **{len(top_failed)}** failed"
+        f" | :fast_forward: **{len(top_skipped)}** skipped"
+    )
+    lines.append("")
+
+    if top_failed:
+        lines.append("### Failed Tests")
+        lines.append("")
+        for t in top_failed:
+            elapsed = results[t]["elapsed"]
+            lines.append(f"<details><summary>:x: {t} ({elapsed:.1f}s)</summary>")
+            lines.append("")
+            lines.append("```")
+            output = "".join(test_output.get(t, []))
+            # Include subtest output too
+            for sub, r in sorted(results.items()):
+                if sub.startswith(t + "/") and r["action"] == "fail":
+                    output += "".join(test_output.get(sub, []))
+            lines.append(output.rstrip())
+            lines.append("```")
+            lines.append("")
+            lines.append("</details>")
+            lines.append("")
+
+    if top_passed:
+        lines.append("<details><summary>Passed Tests</summary>")
+        lines.append("")
+        for t in top_passed:
+            elapsed = results[t]["elapsed"]
+            lines.append(f"- :white_check_mark: {t} ({elapsed:.1f}s)")
+        lines.append("")
+        lines.append("</details>")
+        lines.append("")
+
+    if top_skipped:
+        lines.append("<details><summary>Skipped Tests</summary>")
+        lines.append("")
+        for t in top_skipped:
+            lines.append(f"- :fast_forward: {t}")
+        lines.append("")
+        lines.append("</details>")
+
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

hack/gh-workflow-ci.sh

@@ -115,8 +115,8 @@ get_tests() {
   # which runs the workflow YAML from main (old target names).
   local github_chunk_size github_remainder
   if [[ ${#github_tests[@]} -gt 0 ]]; then
-    github_chunk_size=$(( ${#github_tests[@]} / 2 ))
-    github_remainder=$(( ${#github_tests[@]} % 2 ))
+    github_chunk_size=$((${#github_tests[@]} / 2))
+    github_remainder=$((${#github_tests[@]} % 2))
   fi
 
   case "${target}" in
@@ -287,6 +287,23 @@ collect_logs() {
   detect_panic
 }
 
+generate_github_summary() {
+  local raw_output="/tmp/logs/e2e-test-output.json"
+  local target="${TEST_PROVIDER:-unknown}"
+
+  if [[ -z "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    echo "GITHUB_STEP_SUMMARY not set, skipping summary generation"
+    return 0
+  fi
+
+  if [[ ! -f "${raw_output}" ]]; then
+    echo "No test output found at ${raw_output}, skipping summary generation"
+    return 0
+  fi
+
+  python3 ./hack/generate_github_summary.py "${raw_output}" "${target}" >>"${GITHUB_STEP_SUMMARY}"
+}
+
 detect_panic() {
   # shellcheck disable=SC2016
   (find /tmp/logs/ -type f -regex '.*/pipelines-as-code.*/[0-9]\.log$' | xargs -r sed -n '/stderr F panic:.*/,$p' | head -n 80) >/tmp/panic.log
@@ -328,6 +345,10 @@ help() {
     Will output logs using snazzy formatting when available or otherwise through a simple
     python formatter. This makes debugging easier from the GitHub Actions interface.
 
+  generate_github_summary
+    Parse gotestsum JSON output and write a markdown summary to GITHUB_STEP_SUMMARY.
+    Required env vars: TEST_PROVIDER, GITHUB_STEP_SUMMARY
+
   print_tests
     Print the list of tests that would be run for each provider target.
 
@@ -350,6 +371,9 @@ collect_logs)
 output_logs)
   output_logs
   ;;
+generate_github_summary)
+  generate_github_summary
+  ;;
 print_tests)
   set +x
   for target in github_public github_ghe_1 github_ghe_2 github_ghe_3 gitlab_bitbucket gitea_1 gitea_2 gitea_3 concurrency flaky; do