[RHDHBUGS-2954]: CQA check failures should fail the build (#2055)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: themr0c

.github/workflows/content-quality-assessment.yml

@@ -121,7 +121,7 @@ jobs:
           cd pr-content
           git remote add base https://github.com/${{ github.event.pull_request.base.repo.full_name }}.git
           git fetch base ${{ github.event.pull_request.base.ref }}
-          OUTPUT=$(node build/scripts/cqa/index.js --all 2>&1 || true)
+          OUTPUT=$(node build/scripts/cqa/index.js --all 2>&1)
           echo "output<<EOF" >> $GITHUB_OUTPUT
           echo "$OUTPUT" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT

build/scripts/build-orchestrator.js

@@ -383,6 +383,31 @@ async function traceUrlToSource(url, repoRoot) {
   return output.trim().split('\n').map(f => f.replace(repoRoot + '/', ''));
 }
 
+// ── CQA content quality assessment ─────────────────────────────────────────
+
+async function runCqa(repoRoot, verbose) {
+  const cqaScript = join(__dirname, 'cqa', 'index.js');
+  const { code, duration, output } = await spawnCapture('node', [cqaScript, '--all'], {
+    cwd: repoRoot, verbose, groupName: 'CQA',
+  });
+
+  // Parse summary line from output: "Checks: 19 total, 19 pass, 0 fail"
+  let checksTotal = 0, checksPass = 0, checksFail = 0;
+  const summaryMatch = output.match(/Checks:\s+(\d+)\s+total,\s+(\d+)\s+pass,\s+(\d+)\s+fail/);
+  if (summaryMatch) {
+    checksTotal = Number.parseInt(summaryMatch[1], 10);
+    checksPass = Number.parseInt(summaryMatch[2], 10);
+    checksFail = Number.parseInt(summaryMatch[3], 10);
+  }
+
+  return {
+    status: code === 0 ? 'passed' : 'failed',
+    duration,
+    output,
+    stats: { total: checksTotal, pass: checksPass, fail: checksFail },
+  };
+}
+
 // ── Index HTML generation ────────────────────────────────────────────────────
 
 function getReleaseNotesLink(branch) {
@@ -497,7 +522,16 @@ function printLycheeSummary(lycheeResult) {
   console.log(lastLines.map(l => '  ' + l).join('\n'));
 }
 
-function printSummary(results, lycheeResult, patterns, totalDuration) {
+function printCqaSummary(cqaResult) {
+  console.log('\n=== CQA (Content Quality Assessment) ===');
+  const s = cqaResult.stats || {};
+  console.log(`Checks: ${s.total} total, ${s.pass} pass, ${s.fail} fail`);
+  if (cqaResult.status === 'failed') {
+    console.log('CQA validation failed — run `node build/scripts/cqa/index.js --all` for details');
+  }
+}
+
+function printSummary(results, lycheeResult, cqaResult, patterns, totalDuration) {
   const passed = results.filter(r => r.status === 'passed').length;
   const failed = results.filter(r => r.status === 'failed').length;
 
@@ -511,11 +545,15 @@ function printSummary(results, lycheeResult, patterns, totalDuration) {
   if (lycheeResult) {
     printLycheeSummary(lycheeResult);
   }
+
+  if (cqaResult) {
+    printCqaSummary(cqaResult);
+  }
 }
 
 // ── JSON report ──────────────────────────────────────────────────────────────
 
-function writeReport(branch, results, lycheeResult, concurrency, totalDuration, repoRoot) {
+function writeReport(branch, results, lycheeResult, cqaResult, concurrency, totalDuration, repoRoot) {
   const passed = results.filter(r => r.status === 'passed').length;
   const failed = results.filter(r => r.status === 'failed').length;
 
@@ -541,6 +579,10 @@ function writeReport(branch, results, lycheeResult, concurrency, totalDuration,
       stats: lycheeResult.stats || {},
       errors: lycheeResult.errors || [],
     } : null,
+    cqa: cqaResult ? {
+      status: cqaResult.status,
+      stats: cqaResult.stats || {},
+    } : null,
   };
 
   const reportPath = join(repoRoot, 'build-report.json');
@@ -616,16 +658,22 @@ async function main() {
     lycheeResult.errors = classifyErrors(lycheeResult.output, patterns);
   }
 
+  // Run CQA content quality assessment
+  console.log('\nRunning CQA content quality assessment...');
+  const cqaResult = await runCqa(repoRoot, args.verbose);
+
   const totalDuration = Math.round((Date.now() - totalStart) / 1000);
 
   // Print summary
-  printSummary(buildResults, lycheeResult, patterns, totalDuration);
+  printSummary(buildResults, lycheeResult, cqaResult, patterns, totalDuration);
 
   // Write JSON report
-  writeReport(args.branch, buildResults, lycheeResult, args.jobs, totalDuration, repoRoot);
+  writeReport(args.branch, buildResults, lycheeResult, cqaResult, args.jobs, totalDuration, repoRoot);
 
-  // Exit with error if any builds or lychee failed
-  const hasFailed = buildResults.some(r => r.status === 'failed') || lycheeResult.status === 'failed';
+  // Exit with error if any builds, lychee, or CQA failed
+  const hasFailed = buildResults.some(r => r.status === 'failed')
+    || lycheeResult.status === 'failed'
+    || cqaResult.status === 'failed';
   process.exit(hasFailed ? 1 : 0);
 }
 

build/scripts/cqa/index.js

@@ -139,7 +139,7 @@ async function runAllChecks(checks, titles, fixMode) {
 
   // Output in alphabetical order by check ID
   const sortedChecks = [...checks].sort((a, b) => a.id.localeCompare(b.id));
-  printAllChecksReport(sortedChecks, results, fixMode);
+  return printAllChecksReport(sortedChecks, results, fixMode);
 }
 
 async function collectAllResults(checks, titles, fixMode) {
@@ -236,6 +236,7 @@ function printAllChecksReport(sortedChecks, results, fixMode) {
   }
 
   printFinalSummary({ checksPass, checksFail, totalAutofixable, totalManual, totalDelegated, totalFixed, fixMode });
+  return { checksPass, checksFail };
 }
 
 function printFinalSummary({ checksPass, checksFail, totalAutofixable, totalManual, totalDelegated, totalFixed, fixMode }) {
@@ -384,7 +385,8 @@ async function main() {
   }
 
   if (allMode) {
-    await runAllChecks(checks, titles, fixMode);
+    const { checksFail } = await runAllChecks(checks, titles, fixMode);
+    process.exit(checksFail > 0 ? 1 : 0);
   } else {
     runPerTitleMode(checks, titles, fixMode);
   }