PFM-TASK-6308 refactor: enhance coverage evaluation to support build status checks and add pending status for parallel jobs

Author: senthanal

.github/workflows/fe-code-quality.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Unit Tests
         id: unit-tests
         continue-on-error: true # Allow the step to complete even if it fails
-        uses: collaborationFactory/github-actions/.github/actions/run-many@improvement/PFM-TASK-6308-Create-reusable-GHA-for-code-coverage-gate
+        uses: collaborationFactory/github-actions/.github/actions/run-many@master
         with:
           target: ${{ matrix.target }}
           jobIndex: ${{ matrix.jobIndex }}

tools/scripts/run-many/coverage-evaluator.spec.ts

@@ -72,7 +72,7 @@ describe('coverage-evaluator', () => {
       const result = evaluateCoverage(['project-a'], { global: {}, projects: {} });
 
       expect(result).toBe(0);
-      expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a');
+      expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a (null thresholds)');
 
       // Should write coverage report with skipped project showing individual metrics
       expect(fs.writeFileSync).toHaveBeenCalledWith(

tools/scripts/run-many/coverage-evaluator.ts

@@ -19,7 +19,7 @@ interface ProjectCoverageResult {
     functions: number;
     branches: number;
   } | null;
-  status: 'PASSED' | 'FAILED' | 'SKIPPED';
+  status: 'PASSED' | 'FAILED' | 'SKIPPED' | 'PENDING';
 }
 
 /**
@@ -182,10 +182,91 @@ function shouldSkipProject(thresholds: CoverageThreshold | null): boolean {
   return thresholds === null || (thresholds && Object.keys(thresholds).length === 0);
 }
 
+/**
+ * Evaluates coverage for build status only (no PR comment generation)
+ * Used by jobs that shouldn't generate coverage reports but need to fail if coverage is insufficient
+ */
+export function evaluateCoverageForBuildStatus(projects: string[], thresholds: ThresholdConfig): number {
+  if (!process.env.COVERAGE_THRESHOLDS) {
+    core.info('No coverage thresholds defined for build status check, skipping evaluation');
+    return 0; // No thresholds defined, 0 failures
+  }
+
+  let failedProjectsCount = 0;
+
+  core.info(`Evaluating coverage for build status only: ${projects.length} projects: ${projects.join(', ')}`);
+
+  for (const project of projects) {
+    const projectThresholds = getProjectThresholds(project, thresholds);
+
+    // Skip projects with null thresholds or explicitly empty thresholds
+    if (shouldSkipProject(projectThresholds)) {
+      const reason = projectThresholds === null ? 'null thresholds' : 'no thresholds defined';
+      core.info(`Coverage evaluation skipped for build status for ${project} (${reason})`);
+      continue;
+    }
+
+    // Try to find coverage file in various locations
+    const coveragePath = findCoverageSummaryFile(project);
+    let summary: CoverageSummary | null = null;
+
+    if (coveragePath) {
+      summary = extractCoverageData(coveragePath, project);
+    }
+
+    // If we didn't find project-specific coverage, try global coverage file
+    if (!summary) {
+      summary = tryReadFromGlobalCoverage(project);
+    }
+
+    if (!summary) {
+      core.warning(`No coverage data found for ${project} - marking as build failure`);
+      failedProjectsCount++;
+      continue;
+    }
+
+    // Log the actual coverage data found
+    core.info(`Build status coverage data for ${project}: lines=${summary.lines.pct}%, statements=${summary.statements.pct}%, functions=${summary.functions.pct}%, branches=${summary.branches.pct}%`);
+
+    let projectPassed = true;
+    const failedMetrics: string[] = [];
+
+    // Check each metric if threshold is defined
+    if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) {
+      projectPassed = false;
+      failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`);
+    }
+
+    if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) {
+      projectPassed = false;
+      failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`);
+    }
+
+    if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) {
+      projectPassed = false;
+      failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`);
+    }
+
+    if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) {
+      projectPassed = false;
+      failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`);
+    }
+
+    if (!projectPassed) {
+      core.error(`Project ${project} failed coverage thresholds for build status: ${failedMetrics.join(', ')}`);
+      failedProjectsCount++;
+    } else {
+      core.info(`Project ${project} passed all coverage thresholds for build status`);
+    }
+  }
+
+  return failedProjectsCount;
+}
+
 /**
  * Evaluates coverage for all projects against their thresholds
  */
-export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): number {
+export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig, jobProjects?: string[]): number {
   if (!process.env.COVERAGE_THRESHOLDS) {
     core.info('No coverage thresholds defined, skipping evaluation');
     return 0; // No thresholds defined, 0 failures
@@ -195,6 +276,9 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig
   const coverageResults: ProjectCoverageResult[] = [];
 
   core.info(`Evaluating coverage for ${projects.length} projects: ${projects.join(', ')}`);
+  if (jobProjects) {
+    core.info(`Projects actually processed by this job: ${jobProjects.join(', ')}`);
+  }
 
   // Debug: List coverage directory contents
   debugCoverageDirectory();
@@ -230,22 +314,27 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig
     }
 
     if (!summary) {
-      core.warning(`No coverage data found for ${project} in any location`);
-
-      // Try to list what files exist for this project specifically
-      const projectSpecificDir = path.resolve(process.cwd(), `coverage/${project}`);
-      if (fs.existsSync(projectSpecificDir)) {
-        const files = fs.readdirSync(projectSpecificDir);
-        core.info(`Files in coverage/${project}/: ${files.join(', ')}`);
+      // Check if this project was processed by this job or another job
+      const processedByThisJob = !jobProjects || jobProjects.includes(project);
+
+      if (processedByThisJob) {
+        core.warning(`No coverage data found for ${project} in any location`);
+        coverageResults.push({
+          project,
+          thresholds: projectThresholds,
+          actual: null,
+          status: 'FAILED' // Mark as failed if no coverage report is found for this job's projects
+        });
+        failedProjectsCount++;
+      } else {
+        core.info(`Project ${project} will be processed by another job - showing as pending`);
+        coverageResults.push({
+          project,
+          thresholds: projectThresholds,
+          actual: null,
+          status: 'PENDING'
+        });
       }
-
-      coverageResults.push({
-        project,
-        thresholds: projectThresholds,
-        actual: null,
-        status: 'FAILED' // Mark as failed if no coverage report is found
-      });
-      failedProjectsCount++;
       continue;
     }
 
@@ -329,6 +418,30 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
 
         comment += `| ${projectCell} | ${metric} | N/A | N/A | ⏩ SKIPPED |\n`;
       });
+    } else if (result.status === 'PENDING') {
+      // Show individual metrics for projects being processed by other jobs
+      const metrics = ['lines', 'statements', 'functions', 'branches'];
+      let hasAnyThreshold = false;
+      let firstRow = true;
+
+      metrics.forEach((metric) => {
+        // Skip metrics that don't have a threshold
+        if (!result.thresholds || !result.thresholds[metric]) return;
+
+        hasAnyThreshold = true;
+        const threshold = result.thresholds[metric];
+
+        // Only include project name in the first row for this project
+        const projectCell = firstRow ? result.project : '';
+        firstRow = false;
+
+        comment += `| ${projectCell} | ${metric} | ${threshold}% | Pending | ⏳ PENDING |\n`;
+      });
+
+      // Fallback if no specific thresholds are defined
+      if (!hasAnyThreshold) {
+        comment += `| ${result.project} | All | Defined | Pending | ⏳ PENDING |\n`;
+      }
     } else if (result.actual === null) {
       // Show individual thresholds when coverage data is missing
       const metrics = ['lines', 'statements', 'functions', 'branches'];
@@ -375,18 +488,27 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
     }
   });
 
-  // Add overall status with failed project count
-  const overallStatus = failedProjectsCount === 0 ? '✅ PASSED' :
-    failedProjectsCount === 1 ? '⚠️ WARNING (1 project failing)' :
-      `❌ FAILED (${failedProjectsCount} projects failing)`;
+  // Add overall status with failed project count (excluding pending projects)
+  const actualFailedCount = results.filter(r => r.status === 'FAILED').length;
+  const overallStatus = actualFailedCount === 0 ? '✅ PASSED' :
+    actualFailedCount === 1 ? '⚠️ WARNING (1 project failing)' :
+      `❌ FAILED (${actualFailedCount} projects failing)`;
   comment += `\n### Overall Status: ${overallStatus}\n`;
 
-  if (failedProjectsCount === 1) {
+  if (actualFailedCount === 1) {
     comment += '\n> Note: The build will continue, but this project should be fixed before merging.\n';
-  } else if (failedProjectsCount > 1) {
+  } else if (actualFailedCount > 1) {
     comment += '\n> Note: Multiple projects fail coverage thresholds. This PR will be blocked until fixed.\n';
   }
 
+  // Add explanation of status symbols
+  comment += '\n---\n';
+  comment += '**Status Legend:**\n';
+  comment += '- ✅ **PASSED**: Coverage meets or exceeds threshold\n';
+  comment += '- ❌ **FAILED**: Coverage is below threshold\n';
+  comment += '- ⏩ **SKIPPED**: Project excluded from coverage requirements\n';
+  comment += '- ⏳ **PENDING**: Project is being processed by another parallel job\n';
+
   // Add link to detailed HTML reports
   if (artifactUrl) {
     comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`;

tools/scripts/run-many/run-many.ts

@@ -4,7 +4,7 @@ import * as core from '@actions/core';
 import * as fs from 'fs';
 import * as path from 'path';
 import { getCoverageThresholds } from './threshold-handler';
-import { evaluateCoverage, generateEmptyCoverageReport, generateTestFailureReport, generatePlaceholderCoverageReport } from './coverage-evaluator';
+import { evaluateCoverage, evaluateCoverageForBuildStatus, generateEmptyCoverageReport, generateTestFailureReport, generatePlaceholderCoverageReport } from './coverage-evaluator';
 import { Utils } from '../artifacts/utils';
 
 function getE2ECommand(command: string, base: string): string {
@@ -115,15 +115,34 @@ function main() {
       core.info(JSON.stringify(thresholds, null, 2));
 
       if (commandSucceeded) {
-        // Command succeeded, evaluate actual coverage
-        const failedProjectsCount = evaluateCoverage(projects, thresholds);
-
-        if (failedProjectsCount > 1) {
-          core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`);
-          // Don't exit immediately - we set the failed status but continue running
-        } else if (failedProjectsCount === 1) {
-          core.warning('One project failed to meet coverage thresholds - this should be fixed before merging');
-          // Continue running, with a warning
+        if (isFirstJob) {
+          // First job: Generate comprehensive coverage report for ALL affected projects (including skipped ones)
+          // but only use build failure logic for projects this job actually processed
+          const allAffectedProjects = Utils.getAllProjects(true, base, target);
+          core.info(`First job: Generating comprehensive coverage report for all ${allAffectedProjects.length} affected projects: ${allAffectedProjects.join(', ')}`);
+          core.info(`First job: Actually processed ${projects.length} projects: ${projects.join(', ')}`);
+
+          // Generate comprehensive report (this will show skipped projects and projects processed by other jobs)
+          evaluateCoverage(allAffectedProjects, thresholds, projects);
+
+          // But only fail the build based on projects this job actually processed
+          const processedFailures = evaluateCoverageForBuildStatus(projects, thresholds);
+
+          if (processedFailures > 1) {
+            core.setFailed(`Multiple projects (${processedFailures}) in this job failed to meet coverage thresholds`);
+          } else if (processedFailures === 1) {
+            core.warning('One project in this job failed to meet coverage thresholds - this should be fixed before merging');
+          }
+        } else {
+          // Other jobs: Only evaluate their assigned projects for build status
+          core.info(`Job ${jobIndex}: Evaluating coverage for ${projects.length} assigned projects: ${projects.join(', ')}`);
+          const failedProjectsCount = evaluateCoverage(projects, thresholds);
+
+          if (failedProjectsCount > 1) {
+            core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`);
+          } else if (failedProjectsCount === 1) {
+            core.warning('One project failed to meet coverage thresholds - this should be fixed before merging');
+          }
         }
       } else {
         // Command failed, generate a failure report for first job only