From a49564a5ed5f90ace41dcc68ed56b93c1bcbee58 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 11:07:38 +0200 Subject: [PATCH 01/27] PFM-TASK-6308 Initial plan discussion --- .../Reuasble GHA Code Coverage Gate.vtt | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 .claudesync/reusable-gha-code-coverage-gate/Reuasble GHA Code Coverage Gate.vtt diff --git a/.claudesync/reusable-gha-code-coverage-gate/Reuasble GHA Code Coverage Gate.vtt b/.claudesync/reusable-gha-code-coverage-gate/Reuasble GHA Code Coverage Gate.vtt new file mode 100644 index 00000000..4c102c22 --- /dev/null +++ b/.claudesync/reusable-gha-code-coverage-gate/Reuasble GHA Code Coverage Gate.vtt @@ -0,0 +1,327 @@ +WEBVTT + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/7-0 +00:00:12.155 --> 00:00:12.875 +Oh, there you go. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/9-0 +00:00:18.215 --> 00:00:19.175 +You're sharing something. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/11-0 +00:00:22.775 --> 00:00:23.575 +Yeah, I do share. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/15-0 +00:00:25.405 --> 00:00:27.085 +So the sharing started. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/17-0 +00:00:28.225 --> 00:00:29.065 +Yeah. Yeah, so. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/34-0 +00:00:30.645 --> 00:00:35.939 +OK, +so now as we want to create a reusable + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/34-1 +00:00:35.939 --> 00:00:40.125 +GitHub coverage here new workflow. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/69-0 +00:00:42.895 --> 00:00:45.882 +So inside the test, +where do we start with? + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/69-1 +00:00:45.882 --> 00:00:50.431 +We have to create a first. +We have to create a new workflow in the + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/69-2 +00:00:50.431 --> 00:00:54.775 +GitHub action repo or we want to start +with the C Plus AP Repo. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/88-0 +00:00:57.695 --> 00:01:04.423 +We should start with the get up actions +repo because that's where all the + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/83-0 +00:01:01.685 --> 00:01:01.885 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/88-1 +00:01:04.423 --> 00:01:07.695 +reusable git up actions are defined. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/104-0 +00:01:09.215 --> 00:01:15.423 +And all the Fe reports users the get up +actions feasible get up actions from the + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/104-1 +00:01:15.423 --> 00:01:17.415 +get Up actions repository. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/120-0 +00:01:18.965 --> 00:01:24.934 +So we need to start from the get up +actions depository and that we already + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/120-1 +00:01:24.934 --> 00:01:26.685 +have the core quality. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/127-0 +00:01:28.245 --> 00:01:30.725 +Core flow and which is on workflow call. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/132-0 +00:01:30.675 --> 00:01:32.755 +Yep, Yep. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/147-0 +00:01:32.565 --> 00:01:38.536 +And we need similar to the code quality +workflow but with the coverage and + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/147-1 +00:01:38.536 --> 00:01:40.845 +coverage get get added to it. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/150-0 +00:01:43.385 --> 00:01:44.305 +And the governor? + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/178-0 +00:01:44.675 --> 00:01:49.986 +The secret for the workflow for now, +we will be calling same thing like as the + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/178-1 +00:01:49.986 --> 00:01:55.297 +same code quality we will be calling +something like runmini master that we are + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/178-2 +00:01:55.297 --> 00:01:57.515 +already calling in the unit test. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/192-0 +00:01:59.085 --> 00:02:05.780 +Something similar to that inside that we +will be also adding another param like + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/192-1 +00:02:05.780 --> 00:02:06.365 +secret. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/207-0 +00:02:07.965 --> 00:02:13.537 +Where we will be having the threshold +configuration for each library for that + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/207-1 +00:02:13.537 --> 00:02:13.965 +reple. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/257-0 +00:02:16.115 --> 00:02:22.043 +I think we need to switch. +I mean we need to switch between normal + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/257-1 +00:02:22.043 --> 00:02:26.289 +code quality and one with the code +quality kit. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/257-2 +00:02:26.289 --> 00:02:33.366 +So I think we could do that with this +secret which you mentioned GitHub secret. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/235-0 +00:02:27.045 --> 00:02:27.245 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/247-0 +00:02:31.595 --> 00:02:31.715 +And. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/257-3 +00:02:33.366 --> 00:02:40.355 +So if the GitHub secret as well you and +if it is defined then we can take the. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/262-0 +00:02:40.445 --> 00:02:40.645 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/263-0 +00:02:41.365 --> 00:02:43.405 +Code coverage workflow. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/280-0 +00:02:44.585 --> 00:02:49.643 +If not, then we we take the code quality, +the existing one. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/278-0 +00:02:48.835 --> 00:02:49.995 +Yeah. OK. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/280-1 +00:02:49.643 --> 00:02:52.425 +So this we could split the logic. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/287-0 +00:02:54.325 --> 00:02:58.365 +Mm hmm. +Then do we need another workflow for that? + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/299-0 +00:03:00.845 --> 00:03:03.702 +Yeah, +we need to create a new workflow file. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/299-1 +00:03:03.702 --> 00:03:04.845 +We can call it as. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/331-0 +00:03:06.485 --> 00:03:12.733 +Code quality kit. Yeah, +code quality kit and in the code quality + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/308-0 +00:03:06.885 --> 00:03:08.325 +Average K dot it. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/317-0 +00:03:11.325 --> 00:03:11.525 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/331-1 +00:03:12.733 --> 00:03:19.365 +git we can add this coverage flag to the +existing GIST unit test so. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/337-0 +00:03:21.215 --> 00:03:22.935 +And this will run for all. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/343-0 +00:03:24.525 --> 00:03:26.365 +All apps and lips the affected. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/374-0 +00:03:28.885 --> 00:03:35.357 +MX apps and lips and produces the +coverage and some of the coverage + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/374-1 +00:03:35.357 --> 00:03:43.066 +threshold need to be elevated against +this git up secret and this git up secret. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/374-2 +00:03:43.066 --> 00:03:45.445 +It should be an adjacent. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/386-0 +00:03:47.285 --> 00:03:52.405 +Object and this should provide the app +name or Lib name. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/382-0 +00:03:49.295 --> 00:03:49.495 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/388-0 +00:03:53.405 --> 00:03:54.005 +With that. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/405-0 +00:03:56.045 --> 00:03:58.595 +Coverage, +line coverage and statement coverage and + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/405-1 +00:03:58.595 --> 00:04:01.245 +functions coverage and franchise coverage +threshold. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/417-0 +00:04:05.945 --> 00:04:10.905 +And then the new quality gate, +our workflow this needs to be elevated. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/422-0 +00:04:12.445 --> 00:04:13.325 +And the pipeline. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/439-0 +00:04:15.045 --> 00:04:19.355 +Should be green if it satisfies the +threshold. If not, + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/439-1 +00:04:19.355 --> 00:04:24.605 +the pipeline should fail, +so they should be our target to achieve. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/458-0 +00:04:28.225 --> 00:04:32.612 +So for this we need to create an A master +plan. How to implement it? + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/446-0 +00:04:28.375 --> 00:04:29.495 +Mm hmm. OK. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/458-1 +00:04:32.612 --> 00:04:36.745 +Probably we could use the claw rate to +generate the master plan. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/454-0 +00:04:34.525 --> 00:04:34.725 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/465-0 +00:04:38.765 --> 00:04:41.165 +And let the cloud to guide us in the +process. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/468-0 +00:04:39.665 --> 00:04:44.025 +Yeah, makes sense. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/489-0 +00:04:46.345 --> 00:04:54.544 +OK. And maybe this is sloven is here, +I will share him share with the road map + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/489-1 +00:04:54.544 --> 00:04:57.865 +that we have and we'll get some. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/486-0 +00:04:55.905 --> 00:04:56.105 +Mm hmm. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/502-0 +00:05:00.925 --> 00:05:05.765 +Yeah. So maybe I stopped the recording. +You can use the transcript already for. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/500-0 +00:05:03.725 --> 00:05:03.925 +Yep. + +d2af6f38-3883-44ee-8f52-3ec1c2e894d4/503-0 +00:05:07.685 --> 00:05:07.805 +Yep. \ No newline at end of file From a184bb705bd6c3bc73d0ba9137a306c04f7cffce Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 11:38:21 +0200 Subject: [PATCH 02/27] PFM-TASK-6308 Update .gitignore and project configuration for GHA workflow --- ...sable GHA PR Workflow for Coverage Gate(Unit Test).md | 0 .../reusable-gha-pr-unit-test-coverage-gate.project.json | 9 +++++++-- .gitignore | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) rename {tools => .claudesync/reusable-gha-code-coverage-gate}/Reusable GHA PR Workflow for Coverage Gate(Unit Test).md (100%) diff --git a/tools/Reusable GHA PR Workflow for Coverage Gate(Unit Test).md b/.claudesync/reusable-gha-code-coverage-gate/Reusable GHA PR Workflow for Coverage Gate(Unit Test).md similarity index 100% rename from tools/Reusable GHA PR Workflow for Coverage Gate(Unit Test).md rename to .claudesync/reusable-gha-code-coverage-gate/Reusable GHA PR Workflow for Coverage Gate(Unit Test).md diff --git a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json index fcb65b44..a0aeb898 100644 --- a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json +++ b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json @@ -6,9 +6,14 @@ ".github", "jest.config.ts", "package.json", - "tsconfig.json" + "tsconfig.json", + ".claudesync" ], "excludes": [], "use_ignore_files": true, - "push_roots": [] + "push_roots": [ + "tools", + ".github", + ".claudesync" + ] } diff --git a/.gitignore b/.gitignore index 24ae7c7c..c3c58d76 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ .claudeignore .claudesync/*.project_id.json .claudesync/active_project.json + +#claudesync +.github From 312b55cc0921a23abd961db536bc95dd0ea7ffd8 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 15:02:50 +0200 Subject: [PATCH 03/27] PFM-TASK-6308 update master plan --- .../coverage-gate-master-plan.md | 391 ++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 .claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md diff --git a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md new file mode 100644 index 00000000..fef38d11 --- /dev/null +++ b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md @@ -0,0 +1,391 @@ +### Key Benefits + +- Enforces coverage standards across the codebase +- Provides immediate feedback to developers about coverage issues +- Uses existing tooling and workflow to minimize disruption +- Allows customization of thresholds per project or globally +- Can skip evaluation for projects where coverage isn't relevant + +This implementation will help maintain or improve code quality by ensuring adequate test coverage across the codebase while providing flexibility for different project requirements.# Master Plan: GitHub Actions Coverage Gate Implementation + +## Overview + +This master plan outlines the implementation of a Jest test coverage quality gate for GitHub Actions workflows. The gate will enforce configurable coverage thresholds on affected projects in the PR pipeline, blocking PRs that don't meet standards. + +## 1. JSON Structure for Coverage Thresholds + +The coverage thresholds will be stored as a GitHub repository secret (`COVERAGE_THRESHOLDS`) with the following JSON structure: + +```json +{ + "global": { + "lines": 80, + "statements": 80, + "functions": 75, + "branches": 70 + }, + "projects": { + "cf-platform": { + "lines": 85, + "statements": 85, + "functions": 80, + "branches": 75 + }, + "cf-components": null, // Skip coverage evaluation for this project + "cf-utils": { + "lines": 70, + "statements": 70, + "functions": 65, + "branches": 60 + } + } +} +``` + +Key features: +- `global`: Default thresholds applied to any project without specific settings +- `projects`: Project-specific thresholds that override global defaults +- `null` values: Explicitly skip coverage evaluation for specific projects + +## 2. Implementation in run-many.ts + +We'll modify the existing `tools/scripts/run-many/run-many.ts` script to: +1. Check for the presence of `COVERAGE_THRESHOLDS` environment variable +2. If present, parse it as JSON and extract thresholds +3. Add coverage collection to the test command +4. Evaluate coverage results against thresholds +5. Generate a coverage report for PR commenting +6. Fail the build if any project doesn't meet its thresholds + +### Key Functions to Implement: + +#### 2.1 Parse Coverage Thresholds + +```typescript +function getCoverageThresholds(): any { + if (!process.env.COVERAGE_THRESHOLDS) { + return { global: {}, projects: {} }; + } + + try { + return JSON.parse(process.env.COVERAGE_THRESHOLDS); + } catch (error) { + core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`); + return { global: {}, projects: {} }; + } +} + +function getProjectThresholds(project: string, thresholds: any): any { + // If project explicitly set to null, return null to skip + if (thresholds.projects && thresholds.projects[project] === null) { + return null; + } + + // If project has specific thresholds, use those + if (thresholds.projects && thresholds.projects[project]) { + return thresholds.projects[project]; + } + + // Otherwise, use global thresholds if available + if (thresholds.global) { + return thresholds.global; + } + + // If no thresholds defined, return null + return null; +} +``` + +#### 2.2 Evaluate Coverage Against Thresholds + +```typescript +function evaluateCoverage(projects: string[], thresholds: any): boolean { + if (!process.env.COVERAGE_THRESHOLDS) { + return true; // No thresholds defined, pass by default + } + + let allProjectsPassed = true; + const coverageResults = []; + + for (const project of projects) { + const projectThresholds = getProjectThresholds(project, thresholds); + + // Skip projects with null thresholds + if (projectThresholds === null) { + core.info(`Coverage evaluation skipped for ${project}`); + coverageResults.push({ + project, + thresholds: null, + actual: null, + status: 'SKIPPED' + }); + continue; + } + + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); + + if (!fs.existsSync(coveragePath)) { + core.warning(`No coverage report found for ${project} at ${coveragePath}`); + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: null, + status: 'FAILED' // Mark as failed if no coverage report is found + }); + allProjectsPassed = false; + continue; + } + + const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); + const summary = coverageData.total; // Use the summary from Jest coverage report + + const projectPassed = + summary.lines.pct >= projectThresholds.lines && + summary.statements.pct >= projectThresholds.statements && + summary.functions.pct >= projectThresholds.functions && + summary.branches.pct >= projectThresholds.branches; + + if (!projectPassed) { + allProjectsPassed = false; + } + + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: { + lines: summary.lines.pct, + statements: summary.statements.pct, + functions: summary.functions.pct, + branches: summary.branches.pct + }, + status: projectPassed ? 'PASSED' : 'FAILED' + }); + } + + // Post results to PR comment + postCoverageComment(coverageResults); + + return allProjectsPassed; +} +``` + +#### 2.3 Generate PR Comment with Tabular Results + +```typescript +function formatCoverageComment(results: any[]): string { + let comment = '## Test Coverage Results\n\n'; + comment += '| Project | Metric | Threshold | Actual | Status |\n'; + comment += '|---------|--------|-----------|--------|--------|\n'; + + results.forEach(result => { + if (result.status === 'SKIPPED') { + comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; + } else if (result.actual === null) { + comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; + } else { + const metrics = ['lines', 'statements', 'functions', 'branches']; + metrics.forEach((metric, index) => { + const threshold = result.thresholds[metric]; + const actual = result.actual[metric].toFixed(2); + const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED'; + + // Only include project name in the first row for this project + const projectCell = index === 0 ? result.project : ''; + + comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; + }); + } + }); + + // Add overall status + const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; + comment += `\n### Overall Status: ${overallStatus}\n`; + + return comment; +} + +function postCoverageComment(results: any[]): void { + const comment = formatCoverageComment(results); + + // Write to a file that will be used by thollander/actions-comment-pull-request action + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Coverage results saved for PR comment'); +} +``` + +#### 2.4 Modified Main Function + +```typescript +function main() { + const target = process.argv[2]; + const jobIndex = Number(process.argv[3]); + const jobCount = Number(process.argv[4]); + let base = process.argv[5]; + + // in case base is not a SHA1 commit hash add origin + if (!/\b[0-9a-f]{5,40}\b/.test(base)) base = 'origin/' + base; + if(base.includes('0000000000000000')){ + base = execSync(`git rev-parse --abbrev-ref origin/HEAD `).toString().trim(); + } + const ref = process.argv[6]; + + core.info(`Inputs:\n target ${target},\n jobIndex: ${jobIndex},\n jobCount ${jobCount},\n base ${base},\n ref ${ref}`) + + const projectsString = getAffectedProjects(target, jobIndex, jobCount, base, ref); + const projects = projectsString ? projectsString.split(',') : []; + + // Check if coverage gate is enabled + const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + + // Modified command construction + const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; + let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + + // Add coverage flag if enabled and target is test + if (coverageEnabled && target === 'test') { + cmd += ' --coverage'; + } + + if (target.includes('e2e')) { + cmd = getE2ECommand(cmd, base); + } + + if (projects.length > 0) { + runCommand(cmd); + + // Evaluate coverage if enabled and target is test + if (coverageEnabled && target === 'test') { + const thresholds = getCoverageThresholds(); + const passed = evaluateCoverage(projects, thresholds); + + if (!passed) { + core.setFailed('One or more projects failed to meet coverage thresholds'); + process.exit(1); + } + } + } else { + core.info('No affected projects :)'); + } +} +``` + +## 3. Sample PR Comment Output + +The PR comment will look like this: + +``` +## Test Coverage Results + +| Project | Metric | Threshold | Actual | Status | +|---------|--------|-----------|--------|--------| +| cf-platform | lines | 85% | 87.42% | ✅ PASSED | +| | statements | 85% | 86.10% | ✅ PASSED | +| | functions | 80% | 82.23% | ✅ PASSED | +| | branches | 75% | 77.18% | ✅ PASSED | +| cf-components | All | N/A | N/A | ⏩ SKIPPED | +| cf-utils | lines | 70% | 65.30% | ❌ FAILED | +| | statements | 70% | 68.45% | ❌ FAILED | +| | functions | 65% | 62.60% | ❌ FAILED | +| | branches | 60% | 55.20% | ❌ FAILED | +| some-lib | lines | 80% | 83.50% | ✅ PASSED | +| | statements | 80% | 81.75% | ✅ PASSED | +| | functions | 75% | 78.30% | ✅ PASSED | +| | branches | 70% | 72.40% | ✅ PASSED | + +### Overall Status: ❌ FAILED +``` + +## 4. Implementation Steps + +1. Update the `tools/scripts/run-many/run-many.ts` file: + - Add functions to parse coverage thresholds + - Implement thresholds evaluation logic + - Add coverage report generation + +2. Update the GitHub workflow file to: + - Pass the COVERAGE_THRESHOLDS secret as an environment variable to the run-many.ts script + - Add a step to post the coverage report as a PR comment using thollander/actions-comment-pull-request@v3 + + ```yaml + # Example step to add to the workflow file + - name: Comment PR with Coverage Report + if: github.event_name == 'pull_request' && always() + uses: thollander/actions-comment-pull-request@v3 + with: + message_path: 'coverage-report.txt' + comment_tag: 'coverage-report' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ``` + +3. Update the Jest configuration if needed to ensure coverage reports are generated in the expected format and location. + +## 5. Testing Approach + +1. Local testing: + - Create mock coverage reports + - Test threshold evaluation logic + - Verify comment formatting + +2. PR testing: + - Create a test PR with varying levels of coverage + - Verify the workflow correctly evaluates thresholds + - Check that PR comments are formatted properly + - Confirm build fails when thresholds are not met + +## 6. Estimated Development Timeline + +1. Day 1: Implement and test coverage threshold parsing and evaluation +2. Day 2: Implement PR comment generation and test output formatting +3. Day 3: Integration testing and documentation + +## 7. Future Enhancements + +1. Add trending data to show coverage improvement or regression +2. Support for different threshold levels for different branches +3. Visual charts or graphs in PR comments +4. Integration with SonarQube or other code quality tools + +## 8. Full Implementation Example + +Here's a brief example of the needed changes to GitHub workflow file: + +```yaml +# Partial .github/workflows/fe-code-quality.yml +name: Frontend Code Quality + +on: + workflow_call: + inputs: + # Existing inputs... + secrets: + # Existing secrets... + COVERAGE_THRESHOLDS: + required: false + +jobs: + # Existing jobs... + + unit-test: + runs-on: ubuntu-latest + if: ${{ !inputs.skip_unit_tests }} + needs: [setup] + steps: + # Existing setup steps... + + - name: Run Unit Tests + run: node tools/scripts/run-many/run-many.js test ${{ matrix.index }} ${{ strategy.job-total }} ${{ needs.setup.outputs.base }} ${{ github.head_ref || github.ref_name }} + env: + COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} + + - name: Comment PR with Coverage Report + if: github.event_name == 'pull_request' && always() && env.COVERAGE_THRESHOLDS != '' + uses: thollander/actions-comment-pull-request@v3 + with: + message_path: 'coverage-report.txt' + comment_tag: 'coverage-report' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +The complete implementation will modify the existing `run-many.ts` script to add coverage gate functionality while providing clear feedback via PR comments using the thollander/actions-comment-pull-request action. From 8765566c3157dd1820c449b282d02ff480a821f1 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 15:17:12 +0200 Subject: [PATCH 04/27] PFM-TASK-6308 update master plan --- .../coverage-gate-master-plan.md | 107 +++++++++++++----- 1 file changed, 81 insertions(+), 26 deletions(-) diff --git a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md index fef38d11..6ae9663c 100644 --- a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md +++ b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md @@ -66,7 +66,7 @@ function getCoverageThresholds(): any { if (!process.env.COVERAGE_THRESHOLDS) { return { global: {}, projects: {} }; } - + try { return JSON.parse(process.env.COVERAGE_THRESHOLDS); } catch (error) { @@ -80,17 +80,17 @@ function getProjectThresholds(project: string, thresholds: any): any { if (thresholds.projects && thresholds.projects[project] === null) { return null; } - + // If project has specific thresholds, use those if (thresholds.projects && thresholds.projects[project]) { return thresholds.projects[project]; } - + // Otherwise, use global thresholds if available if (thresholds.global) { return thresholds.global; } - + // If no thresholds defined, return null return null; } @@ -103,13 +103,13 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { if (!process.env.COVERAGE_THRESHOLDS) { return true; // No thresholds defined, pass by default } - + let allProjectsPassed = true; const coverageResults = []; - + for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); - + // Skip projects with null thresholds if (projectThresholds === null) { core.info(`Coverage evaluation skipped for ${project}`); @@ -121,9 +121,9 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { }); continue; } - + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); - + if (!fs.existsSync(coveragePath)) { core.warning(`No coverage report found for ${project} at ${coveragePath}`); coverageResults.push({ @@ -135,20 +135,20 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { allProjectsPassed = false; continue; } - + const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); const summary = coverageData.total; // Use the summary from Jest coverage report - - const projectPassed = + + const projectPassed = summary.lines.pct >= projectThresholds.lines && summary.statements.pct >= projectThresholds.statements && summary.functions.pct >= projectThresholds.functions && summary.branches.pct >= projectThresholds.branches; - + if (!projectPassed) { allProjectsPassed = false; } - + coverageResults.push({ project, thresholds: projectThresholds, @@ -161,18 +161,18 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { status: projectPassed ? 'PASSED' : 'FAILED' }); } - + // Post results to PR comment postCoverageComment(coverageResults); - + return allProjectsPassed; } ``` -#### 2.3 Generate PR Comment with Tabular Results +#### 2.3 Generate Coverage Reports and PR Comment ```typescript -function formatCoverageComment(results: any[]): string { +function formatCoverageComment(results: any[], artifactUrl: string): string { let comment = '## Test Coverage Results\n\n'; comment += '| Project | Metric | Threshold | Actual | Status |\n'; comment += '|---------|--------|-----------|--------|--------|\n'; @@ -201,11 +201,19 @@ function formatCoverageComment(results: any[]): string { const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; comment += `\n### Overall Status: ${overallStatus}\n`; + // Add link to detailed HTML reports + if (artifactUrl) { + comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`; + } + return comment; } function postCoverageComment(results: any[]): void { - const comment = formatCoverageComment(results); + // The actual artifact URL will be provided by GitHub Actions in the workflow + const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || ''; + + const comment = formatCoverageComment(results, artifactUrl); // Write to a file that will be used by thollander/actions-comment-pull-request action const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); @@ -245,7 +253,8 @@ function main() { // Add coverage flag if enabled and target is test if (coverageEnabled && target === 'test') { - cmd += ' --coverage'; + // Add coverage reporters for HTML, JSON, and JUnit output + cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,junit --reporters=default,jest-junit'; } if (target.includes('e2e')) { @@ -295,23 +304,58 @@ The PR comment will look like this: | | branches | 70% | 72.40% | ✅ PASSED | ### Overall Status: ❌ FAILED + +📊 [View Detailed HTML Coverage Reports](https://github.com/collaborationFactory/cplace-frontend/actions/runs/12345678) ``` +The comment includes: +1. A tabular view of all projects with their thresholds and actual coverage metrics +2. Statuses (PASSED, FAILED, SKIPPED) for each metric and project +3. An overall status summary +4. A link to the detailed HTML coverage reports uploaded as GitHub artifacts + ## 4. Implementation Steps 1. Update the `tools/scripts/run-many/run-many.ts` file: - Add functions to parse coverage thresholds - Implement thresholds evaluation logic - Add coverage report generation + - Modify test command to generate HTML and JUnit reports + +2. Update the Jest configuration to output multiple formats: + ```javascript + // In jest.config.js or similar + module.exports = { + // Other configuration... + coverageReporters: ['json', 'lcov', 'text', 'clover', 'html', 'junit'], + reporters: ['default', 'jest-junit'], + // Make sure HTML reports go to a consistent location + coverageDirectory: 'coverage' + }; + ``` -2. Update the GitHub workflow file to: +3. Update the GitHub workflow file to: - Pass the COVERAGE_THRESHOLDS secret as an environment variable to the run-many.ts script + - Add steps to upload coverage reports as artifacts - Add a step to post the coverage report as a PR comment using thollander/actions-comment-pull-request@v3 ```yaml - # Example step to add to the workflow file + # Example steps to add to the workflow file + - name: Upload Coverage Reports + if: always() && env.COVERAGE_THRESHOLDS != '' + uses: actions/upload-artifact@v3 + with: + name: coverage-reports + path: coverage/ + retention-days: 7 + + - name: Set Artifact URL + if: github.event_name == 'pull_request' && always() && env.COVERAGE_THRESHOLDS != '' + run: | + echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + - name: Comment PR with Coverage Report - if: github.event_name == 'pull_request' && always() + if: github.event_name == 'pull_request' && always() && env.COVERAGE_THRESHOLDS != '' uses: thollander/actions-comment-pull-request@v3 with: message_path: 'coverage-report.txt' @@ -319,8 +363,6 @@ The PR comment will look like this: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` -3. Update the Jest configuration if needed to ensure coverage reports are generated in the expected format and location. - ## 5. Testing Approach 1. Local testing: @@ -379,6 +421,19 @@ jobs: env: COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} + - name: Upload Coverage Reports + if: always() && env.COVERAGE_THRESHOLDS != '' + uses: actions/upload-artifact@v3 + with: + name: coverage-reports + path: coverage/ + retention-days: 7 + + - name: Set Artifact URL + if: github.event_name == 'pull_request' && always() && env.COVERAGE_THRESHOLDS != '' + run: | + echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + - name: Comment PR with Coverage Report if: github.event_name == 'pull_request' && always() && env.COVERAGE_THRESHOLDS != '' uses: thollander/actions-comment-pull-request@v3 @@ -388,4 +443,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` -The complete implementation will modify the existing `run-many.ts` script to add coverage gate functionality while providing clear feedback via PR comments using the thollander/actions-comment-pull-request action. +The complete implementation will modify the existing `run-many.ts` script to add coverage gate functionality while providing HTML reports as GitHub artifacts and clear feedback via PR comments using the thollander/actions-comment-pull-request action. From dea33af6805d56a486975bfc068676ebbf1f9513 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 15:47:21 +0200 Subject: [PATCH 05/27] PFM-TASK-6308 update master plan --- .../reusable-gha-pr-unit-test-coverage-gate.project.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json index a0aeb898..065dd951 100644 --- a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json +++ b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json @@ -4,10 +4,11 @@ "includes": [ "tools", ".github", + ".claudesync", "jest.config.ts", "package.json", - "tsconfig.json", - ".claudesync" + "tsconfig.json" + ], "excludes": [], "use_ignore_files": true, From 54393c6e29d6ade062f866a5adcf92ad2b0aec68 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 15:53:55 +0200 Subject: [PATCH 06/27] PFM-TASK-6308 update master plan --- ...able-gha-pr-unit-test-coverage-gate.project.json | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json index 065dd951..8cfa4611 100644 --- a/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json +++ b/.claudesync/reusable-gha-pr-unit-test-coverage-gate.project.json @@ -2,19 +2,16 @@ "project_name": "reusable-gha-pr-unit-test-coverage-gate", "project_description": "Reusable github actions for unit test coverage gate with PR workflow", "includes": [ + "**/*" + ], + "excludes": [], + "use_ignore_files": false, + "push_roots": [ "tools", ".github", ".claudesync", "jest.config.ts", "package.json", "tsconfig.json" - - ], - "excludes": [], - "use_ignore_files": true, - "push_roots": [ - "tools", - ".github", - ".claudesync" ] } From a9f03927c7ca0bde4295c70ecd1f43dac6615ef9 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 15:58:47 +0200 Subject: [PATCH 07/27] PFM-TASK-6308 update master plan --- .../coverage-gate-master-plan.md | 50 ++++++------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md index 6ae9663c..1710d9f5 100644 --- a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md +++ b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md @@ -103,13 +103,13 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { if (!process.env.COVERAGE_THRESHOLDS) { return true; // No thresholds defined, pass by default } - + let allProjectsPassed = true; const coverageResults = []; - + for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); - + // Skip projects with null thresholds if (projectThresholds === null) { core.info(`Coverage evaluation skipped for ${project}`); @@ -121,9 +121,9 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { }); continue; } - + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); - + if (!fs.existsSync(coveragePath)) { core.warning(`No coverage report found for ${project} at ${coveragePath}`); coverageResults.push({ @@ -135,20 +135,20 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { allProjectsPassed = false; continue; } - + const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); const summary = coverageData.total; // Use the summary from Jest coverage report - - const projectPassed = + + const projectPassed = summary.lines.pct >= projectThresholds.lines && summary.statements.pct >= projectThresholds.statements && summary.functions.pct >= projectThresholds.functions && summary.branches.pct >= projectThresholds.branches; - + if (!projectPassed) { allProjectsPassed = false; } - + coverageResults.push({ project, thresholds: projectThresholds, @@ -161,15 +161,15 @@ function evaluateCoverage(projects: string[], thresholds: any): boolean { status: projectPassed ? 'PASSED' : 'FAILED' }); } - + // Post results to PR comment postCoverageComment(coverageResults); - + return allProjectsPassed; } ``` -#### 2.3 Generate Coverage Reports and PR Comment +#### 2.3 Generate PR Comment with Tabular Results ```typescript function formatCoverageComment(results: any[], artifactUrl: string): string { @@ -304,16 +304,8 @@ The PR comment will look like this: | | branches | 70% | 72.40% | ✅ PASSED | ### Overall Status: ❌ FAILED - -📊 [View Detailed HTML Coverage Reports](https://github.com/collaborationFactory/cplace-frontend/actions/runs/12345678) ``` -The comment includes: -1. A tabular view of all projects with their thresholds and actual coverage metrics -2. Statuses (PASSED, FAILED, SKIPPED) for each metric and project -3. An overall status summary -4. A link to the detailed HTML coverage reports uploaded as GitHub artifacts - ## 4. Implementation Steps 1. Update the `tools/scripts/run-many/run-many.ts` file: @@ -322,19 +314,7 @@ The comment includes: - Add coverage report generation - Modify test command to generate HTML and JUnit reports -2. Update the Jest configuration to output multiple formats: - ```javascript - // In jest.config.js or similar - module.exports = { - // Other configuration... - coverageReporters: ['json', 'lcov', 'text', 'clover', 'html', 'junit'], - reporters: ['default', 'jest-junit'], - // Make sure HTML reports go to a consistent location - coverageDirectory: 'coverage' - }; - ``` - -3. Update the GitHub workflow file to: +2. Update the GitHub workflow file to: - Pass the COVERAGE_THRESHOLDS secret as an environment variable to the run-many.ts script - Add steps to upload coverage reports as artifacts - Add a step to post the coverage report as a PR comment using thollander/actions-comment-pull-request@v3 @@ -363,6 +343,8 @@ The comment includes: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ``` +3. Update the Jest configuration if needed to ensure coverage reports are generated in the expected format and location. + ## 5. Testing Approach 1. Local testing: From 66251599ad60d22ada1a45e031d4cc870d876a98 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 20 May 2025 21:59:38 +0200 Subject: [PATCH 08/27] PFM-TASK-6308 add RFC template for feature proposals --- .../RFC-template.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .claudesync/reusable-gha-code-coverage-gate/RFC-template.md diff --git a/.claudesync/reusable-gha-code-coverage-gate/RFC-template.md b/.claudesync/reusable-gha-code-coverage-gate/RFC-template.md new file mode 100644 index 00000000..caeb18b8 --- /dev/null +++ b/.claudesync/reusable-gha-code-coverage-gate/RFC-template.md @@ -0,0 +1,25 @@ +# RFC Title + +## Summary +A brief overview of the proposed feature or change. + +## Motivation / Why +Explain the problem or limitation this RFC addresses. + +## Proposed Solution / Design +Detailed description of the new API, syntax, or feature. Include examples if applicable. + +## Technical Details +In-depth explanation of implementation, internals, and integration with existing Angular features. + +## Benefits +List advantages such as improved developer experience, performance, or alignment with modern standards. + +## Drawbacks / Alternatives +Potential downsides or alternative solutions considered. + +## Community Feedback +Summary or invitation for community feedback to refine the proposal. + +## Conclusion +Final thoughts and next steps. From a299e95922fbaa65063a180ab218e29d646b06b8 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 09:59:33 +0200 Subject: [PATCH 09/27] PFM-TASK-6308 implement coverage gate with threshold evaluation and reporting --- .../coverage-gate-implementation-plan.md | 499 ++++++++++++++++++ 1 file changed, 499 insertions(+) create mode 100644 .claudesync/reusable-gha-code-coverage-gate/coverage-gate-implementation-plan.md diff --git a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-implementation-plan.md b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-implementation-plan.md new file mode 100644 index 00000000..150599ae --- /dev/null +++ b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-implementation-plan.md @@ -0,0 +1,499 @@ +# Coverage Gate Implementation Plan + +## 1. Project Structure + +We'll modify/create these files: +- Modify: `.github/workflows/fe-code-quality.yml` - Enhance existing workflow +- Create: `tools/scripts/run-many/coverage-evaluator.ts` - Module for evaluating coverage reports +- Create: `tools/scripts/run-many/threshold-handler.ts` - Module to parse and handle thresholds + +## 2. Detailed Implementation Steps + +### 2.1. Create Coverage Threshold Handler Module + +First, create `tools/scripts/run-many/threshold-handler.ts`: + +```typescript +import * as core from '@actions/core'; + +export interface CoverageThreshold { + lines?: number; + statements?: number; + functions?: number; + branches?: number; +} + +export interface ThresholdConfig { + global: CoverageThreshold; + projects: Record; +} + +/** + * Parses the COVERAGE_THRESHOLDS environment variable + */ +export function getCoverageThresholds(): ThresholdConfig { + if (!process.env.COVERAGE_THRESHOLDS) { + return { global: {}, projects: {} }; + } + + try { + return JSON.parse(process.env.COVERAGE_THRESHOLDS); + } catch (error) { + core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`); + return { global: {}, projects: {} }; + } +} + +/** + * Gets thresholds for a specific project + */ +export function getProjectThresholds(project: string, thresholds: ThresholdConfig): CoverageThreshold | null { + // If project explicitly set to null, return null to skip + if (thresholds.projects && thresholds.projects[project] === null) { + return null; + } + + // If project has specific thresholds, use those + if (thresholds.projects && thresholds.projects[project]) { + return thresholds.projects[project]; + } + + // Otherwise, use global thresholds if available + if (thresholds.global) { + return thresholds.global; + } + + // If no thresholds defined, return null + return null; +} +``` + +### 2.2. Create Coverage Evaluator Module + +Next, create `tools/scripts/run-many/coverage-evaluator.ts`: + +```typescript +import * as fs from 'fs'; +import * as path from 'path'; +import * as core from '@actions/core'; +import { CoverageThreshold, getProjectThresholds, ThresholdConfig } from './threshold-handler'; + +interface CoverageSummary { + lines: { pct: number }; + statements: { pct: number }; + functions: { pct: number }; + branches: { pct: number }; +} + +interface ProjectCoverageResult { + project: string; + thresholds: CoverageThreshold | null; + actual: { + lines: number; + statements: number; + functions: number; + branches: number; + } | null; + status: 'PASSED' | 'FAILED' | 'SKIPPED'; +} + +/** + * Evaluates coverage for all projects against their thresholds + */ +export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): boolean { + if (!process.env.COVERAGE_THRESHOLDS) { + return true; // No thresholds defined, pass by default + } + + let allProjectsPassed = true; + const coverageResults: ProjectCoverageResult[] = []; + + for (const project of projects) { + const projectThresholds = getProjectThresholds(project, thresholds); + + // Skip projects with null thresholds + if (projectThresholds === null) { + core.info(`Coverage evaluation skipped for ${project}`); + coverageResults.push({ + project, + thresholds: null, + actual: null, + status: 'SKIPPED' + }); + continue; + } + + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); + + if (!fs.existsSync(coveragePath)) { + core.warning(`No coverage report found for ${project} at ${coveragePath}`); + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: null, + status: 'FAILED' // Mark as failed if no coverage report is found + }); + allProjectsPassed = false; + continue; + } + + try { + const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); + const summary = coverageData.total as CoverageSummary; + + const projectPassed = + (!projectThresholds.lines || summary.lines.pct >= projectThresholds.lines) && + (!projectThresholds.statements || summary.statements.pct >= projectThresholds.statements) && + (!projectThresholds.functions || summary.functions.pct >= projectThresholds.functions) && + (!projectThresholds.branches || summary.branches.pct >= projectThresholds.branches); + + if (!projectPassed) { + allProjectsPassed = false; + } + + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: { + lines: summary.lines.pct, + statements: summary.statements.pct, + functions: summary.functions.pct, + branches: summary.branches.pct + }, + status: projectPassed ? 'PASSED' : 'FAILED' + }); + } catch (error) { + core.error(`Error processing coverage for ${project}: ${error.message}`); + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: null, + status: 'FAILED' + }); + allProjectsPassed = false; + } + } + + // Post results to PR comment + postCoverageComment(coverageResults); + + return allProjectsPassed; +} + +/** + * Formats the coverage results into a markdown table + */ +function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string): string { + let comment = '## Test Coverage Results\n\n'; + comment += '| Project | Metric | Threshold | Actual | Status |\n'; + comment += '|---------|--------|-----------|--------|--------|\n'; + + results.forEach(result => { + if (result.status === 'SKIPPED') { + comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; + } else if (result.actual === null) { + comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; + } else { + const metrics = ['lines', 'statements', 'functions', 'branches']; + metrics.forEach((metric, index) => { + // Skip metrics that don't have a threshold + if (!result.thresholds[metric]) return; + + const threshold = result.thresholds[metric]; + const actual = result.actual[metric].toFixed(2); + const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED'; + + // Only include project name in the first row for this project + const projectCell = index === 0 ? result.project : ''; + + comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; + }); + } + }); + + // Add overall status + const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; + comment += `\n### Overall Status: ${overallStatus}\n`; + + // Add link to detailed HTML reports + if (artifactUrl) { + comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`; + } + + return comment; +} + +/** + * Writes the coverage results to a file for PR comment + */ +function postCoverageComment(results: ProjectCoverageResult[]): void { + // The actual artifact URL will be provided by GitHub Actions in the workflow + const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || ''; + + const comment = formatCoverageComment(results, artifactUrl); + + // Write to a file that will be used by thollander/actions-comment-pull-request action + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Coverage results saved for PR comment'); +} +``` + +### 2.3. Modify run-many.ts Script + +Update `tools/scripts/run-many/run-many.ts` to incorporate the coverage gate functionality: + +```typescript +import { getAffectedProjects } from './affected-projects'; +import { execSync } from 'child_process'; +import * as core from '@actions/core'; +import * as path from 'path'; +import { getCoverageThresholds } from './threshold-handler'; +import { evaluateCoverage } from './coverage-evaluator'; + +function getE2ECommand(command: string, base: string): string { + command = command.concat(` -c ci --base=${base} --verbose`); + return command; +} + +function runCommand(command: string): void { + core.info(`Running > ${command}`); + + try { + const output = execSync(command, { stdio: 'pipe', maxBuffer: 1024 * 1024 * 1024, encoding: 'utf-8' }); // 1GB + core.info(output.toString()) + } catch (error) { + if (error.signal === 'SIGTERM') { + core.error('Timed out'); + } else if (error.code === 'ENOBUFS') { + core.error('Buffer exceeded'); + } + core.info(error.stdout.toString()); + core.error(error.stderr.toString()); + core.error(`Error message: ${error.message}`); + core.error(`Error name: ${error.name}`); + core.error(`Stacktrace:\n${error.stack}`); + core.setFailed(error); + } +} + +function main() { + const target = process.argv[2]; + const jobIndex = Number(process.argv[3]); + const jobCount = Number(process.argv[4]); + let base = process.argv[5]; + + // in case base is not a SHA1 commit hash add origin + if (!/\b[0-9a-f]{5,40}\b/.test(base)) base = 'origin/' + base; + if(base.includes('0000000000000000')){ + base = execSync(`git rev-parse --abbrev-ref origin/HEAD `).toString().trim(); + } + const ref = process.argv[6]; + + core.info(`Inputs:\n target ${target},\n jobIndex: ${jobIndex},\n jobCount ${jobCount},\n base ${base},\n ref ${ref}`) + + const projectsString = getAffectedProjects(target, jobIndex, jobCount, base, ref); + const projects = projectsString ? projectsString.split(',') : []; + + // Check if coverage gate is enabled + const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + + // Modified command construction + const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; + let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + + // Add coverage flag if enabled and target is test + if (coverageEnabled && target === 'test') { + // Add coverage reporters for HTML, JSON, and JUnit output + cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; + } + + if (target.includes('e2e')) { + cmd = getE2ECommand(cmd, base); + } + + if (projects.length > 0) { + runCommand(cmd); + + // Evaluate coverage if enabled and target is test + if (coverageEnabled && target === 'test') { + const thresholds = getCoverageThresholds(); + const passed = evaluateCoverage(projects, thresholds); + + if (!passed) { + core.setFailed('One or more projects failed to meet coverage thresholds'); + process.exit(1); + } + } + } else { + core.info('No affected projects :)'); + } +} + +main(); +``` + +### 2.4. Modify Existing Workflow File for Code Quality + +Update the existing `.github/workflows/fe-code-quality.yml`: + +```yaml +name: Frontend Code Quality Workflow + +on: + workflow_call: + secrets: + COVERAGE_THRESHOLDS: + required: false + +jobs: + code-quality: + runs-on: ubuntu-latest + strategy: + matrix: + target: [ 'test' ] + jobIndex: [ 1, 2, 3, 4 ] + env: + jobCount: 4 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - uses: actions/setup-node@v3 + with: + node-version: 18.19.1 + - name: Cache Node Modules + id: npm-cache + uses: actions/cache@v4 + with: + path: '**/node_modules' + key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }} + + - name: Formatter + run: npx nx format:check --base=origin/${{ github.event.pull_request.base.ref }} + + - name: Linter + run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} + + - name: Unit Tests + uses: collaborationFactory/github-actions/.github/actions/run-many@master + with: + target: ${{ matrix.target }} + jobIndex: ${{ matrix.jobIndex }} + jobCount: ${{ env.jobCount }} + base: ${{ github.event.pull_request.base.ref }} + ref: ${{ github.event.pull_request.head.ref }} + env: + COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} + + - name: Upload Coverage Reports + if: always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: coverage/ + retention-days: 7 + + - name: Set Artifact URL + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + run: | + echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Comment PR with Coverage Report + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + uses: thollander/actions-comment-pull-request@v3 + with: + file_path: 'coverage-report.txt' + comment_tag: 'coverage-report' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +## 3. Usage Instructions + +### 3.1. Setting Up Coverage Thresholds + +Project maintainers will need to create a GitHub repository secret named `COVERAGE_THRESHOLDS` with JSON content like: + +```json +{ + "global": { + "lines": 80, + "statements": 80, + "functions": 75, + "branches": 70 + }, + "projects": { + "cf-platform": { + "lines": 85, + "statements": 85, + "functions": 80, + "branches": 75 + }, + "cf-components": null, + "cf-utils": { + "lines": 70, + "statements": 70, + "functions": 65, + "branches": 60 + } + } +} +``` + +### 3.2. Enabling the Coverage Gate + +To enable the coverage gate in a project: + +1. Configure the `COVERAGE_THRESHOLDS` secret in your repository settings +2. Make sure your Jest configuration is compatible with the coverage reporters +3. No changes to workflows needed - the existing code quality workflow will automatically use coverage gates when the secret is set + +## 4. Testing Plan + +### 4.1. Unit Tests + +Create unit tests for: +- Threshold parsing logic +- Coverage evaluation logic +- PR comment formatting + +### 4.2. Integration Tests + +1. Test with valid thresholds and passing coverage +2. Test with valid thresholds and failing coverage +3. Test with empty or invalid threshold configurations +4. Test with missing coverage reports + +### 4.3. End-to-End Tests + +Create a sample PR with: +- Projects that pass the thresholds +- Projects that fail the thresholds +- Projects exempt from coverage evaluation + +## 5. Implementation Timeline + +- Day 1: Implement threshold handling and coverage evaluation modules +- Day 2: Update run-many.ts and modify workflow file +- Day 3: Testing and bug fixes +- Day 4: Documentation and final review + +## 6. Future Improvements + +- Add trend analysis to show coverage improvement/regression over time +- Support different thresholds for different branches +- Add visual charts to the PR comments +- Integration with code quality tools like SonarQube + +## 7. Key Benefits + +- Enforces coverage standards across the codebase +- Provides immediate feedback to developers about coverage issues +- Uses existing tooling and workflow to minimize disruption +- Allows customization of thresholds per project or globally +- Can skip evaluation for projects where coverage isn't relevant + +This implementation will help maintain or improve code quality by ensuring adequate test coverage across the codebase while providing flexibility for different project requirements. From 4bb95b2d55ea6d313c11b2264e0e7faca937b460 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 10:13:48 +0200 Subject: [PATCH 10/27] PFM-TASK-6308 create reusable GitHub Action for code coverage evaluation and reporting --- .github/workflows/fe-code-quality.yml | 31 +++- tools/scripts/run-many/coverage-evaluator.ts | 165 +++++++++++++++++++ tools/scripts/run-many/run-many.ts | 31 +++- tools/scripts/run-many/threshold-handler.ts | 63 +++++++ 4 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 tools/scripts/run-many/coverage-evaluator.ts create mode 100644 tools/scripts/run-many/threshold-handler.ts diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 3b049fd3..11f2cc60 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -1,6 +1,10 @@ name: Frontend Code Quality Workflow -on: workflow_call +on: + workflow_call: + secrets: + COVERAGE_THRESHOLDS: + required: false jobs: code-quality: @@ -8,7 +12,7 @@ jobs: strategy: matrix: target: [ 'test' ] - jobIndex: [ 1, 2, 3,4 ] + jobIndex: [ 1, 2, 3, 4 ] env: jobCount: 4 steps: @@ -41,3 +45,26 @@ jobs: jobCount: ${{ env.jobCount }} base: ${{ github.event.pull_request.base.ref }} ref: ${{ github.event.pull_request.head.ref }} + env: + COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} + + - name: Upload Coverage Reports + if: always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + uses: actions/upload-artifact@v4 + with: + name: coverage-reports + path: coverage/ + retention-days: 7 + + - name: Set Artifact URL + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + run: | + echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Comment PR with Coverage Report + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + uses: thollander/actions-comment-pull-request@v3 + with: + file_path: 'coverage-report.txt' + comment_tag: 'coverage-report' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts new file mode 100644 index 00000000..45de5900 --- /dev/null +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -0,0 +1,165 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as core from '@actions/core'; +import { CoverageThreshold, getProjectThresholds, ThresholdConfig } from './threshold-handler'; + +interface CoverageSummary { + lines: { pct: number }; + statements: { pct: number }; + functions: { pct: number }; + branches: { pct: number }; +} + +interface ProjectCoverageResult { + project: string; + thresholds: CoverageThreshold | null; + actual: { + lines: number; + statements: number; + functions: number; + branches: number; + } | null; + status: 'PASSED' | 'FAILED' | 'SKIPPED'; +} + +/** + * Evaluates coverage for all projects against their thresholds + */ +export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): boolean { + if (!process.env.COVERAGE_THRESHOLDS) { + return true; // No thresholds defined, pass by default + } + + let allProjectsPassed = true; + const coverageResults: ProjectCoverageResult[] = []; + + for (const project of projects) { + const projectThresholds = getProjectThresholds(project, thresholds); + + // Skip projects with null thresholds + if (projectThresholds === null) { + core.info(`Coverage evaluation skipped for ${project}`); + coverageResults.push({ + project, + thresholds: null, + actual: null, + status: 'SKIPPED' + }); + continue; + } + + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); + + if (!fs.existsSync(coveragePath)) { + core.warning(`No coverage report found for ${project} at ${coveragePath}`); + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: null, + status: 'FAILED' // Mark as failed if no coverage report is found + }); + allProjectsPassed = false; + continue; + } + + try { + const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); + const summary = coverageData.total as CoverageSummary; + + const projectPassed = + (!projectThresholds.lines || summary.lines.pct >= projectThresholds.lines) && + (!projectThresholds.statements || summary.statements.pct >= projectThresholds.statements) && + (!projectThresholds.functions || summary.functions.pct >= projectThresholds.functions) && + (!projectThresholds.branches || summary.branches.pct >= projectThresholds.branches); + + if (!projectPassed) { + allProjectsPassed = false; + } + + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: { + lines: summary.lines.pct, + statements: summary.statements.pct, + functions: summary.functions.pct, + branches: summary.branches.pct + }, + status: projectPassed ? 'PASSED' : 'FAILED' + }); + } catch (error) { + core.error(`Error processing coverage for ${project}: ${error.message}`); + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: null, + status: 'FAILED' + }); + allProjectsPassed = false; + } + } + + // Post results to PR comment + postCoverageComment(coverageResults); + + return allProjectsPassed; +} + +/** + * Formats the coverage results into a markdown table + */ +function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string): string { + let comment = '## Test Coverage Results\n\n'; + comment += '| Project | Metric | Threshold | Actual | Status |\n'; + comment += '|---------|--------|-----------|--------|--------|\n'; + + results.forEach(result => { + if (result.status === 'SKIPPED') { + comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; + } else if (result.actual === null) { + comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; + } else { + const metrics = ['lines', 'statements', 'functions', 'branches']; + metrics.forEach((metric, index) => { + // Skip metrics that don't have a threshold + if (!result.thresholds[metric]) return; + + const threshold = result.thresholds[metric]; + const actual = result.actual[metric].toFixed(2); + const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED'; + + // Only include project name in the first row for this project + const projectCell = index === 0 ? result.project : ''; + + comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; + }); + } + }); + + // Add overall status + const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; + comment += `\n### Overall Status: ${overallStatus}\n`; + + // Add link to detailed HTML reports + if (artifactUrl) { + comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`; + } + + return comment; +} + +/** + * Writes the coverage results to a file for PR comment + */ +function postCoverageComment(results: ProjectCoverageResult[]): void { + // The actual artifact URL will be provided by GitHub Actions in the workflow + const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || ''; + + const comment = formatCoverageComment(results, artifactUrl); + + // Write to a file that will be used by thollander/actions-comment-pull-request action + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Coverage results saved for PR comment'); +} \ No newline at end of file diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index 08524a5f..c8f0bb5e 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -1,6 +1,9 @@ import { getAffectedProjects } from './affected-projects'; import { execSync } from 'child_process'; import * as core from '@actions/core'; +import * as path from 'path'; +import { getCoverageThresholds } from './threshold-handler'; +import { evaluateCoverage } from './coverage-evaluator'; function getE2ECommand(command: string, base: string): string { command = command.concat(` -c ci --base=${base} --verbose`); @@ -11,7 +14,7 @@ function runCommand(command: string): void { core.info(`Running > ${command}`); try { - const output = execSync(command, { stdio: 'pipe', maxBuffer: 1024 * 1024 * 1024, encoding: 'utf-8' }); // 10MB + const output = execSync(command, { stdio: 'pipe', maxBuffer: 1024 * 1024 * 1024, encoding: 'utf-8' }); // 1GB core.info(output.toString()) } catch (error) { if (error.signal === 'SIGTERM') { @@ -43,17 +46,39 @@ function main() { core.info(`Inputs:\n target ${target},\n jobIndex: ${jobIndex},\n jobCount ${jobCount},\n base ${base},\n ref ${ref}`) - const projects = getAffectedProjects(target, jobIndex, jobCount, base, ref); + const projectsString = getAffectedProjects(target, jobIndex, jobCount, base, ref); + const projects = projectsString ? projectsString.split(',') : []; - const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projects}"`; + // Check if coverage gate is enabled + const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + + // Modified command construction + const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + // Add coverage flag if enabled and target is test + if (coverageEnabled && target === 'test') { + // Add coverage reporters for HTML, JSON, and JUnit output + cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; + } + if (target.includes('e2e')) { cmd = getE2ECommand(cmd, base); } if (projects.length > 0) { runCommand(cmd); + + // Evaluate coverage if enabled and target is test + if (coverageEnabled && target === 'test') { + const thresholds = getCoverageThresholds(); + const passed = evaluateCoverage(projects, thresholds); + + if (!passed) { + core.setFailed('One or more projects failed to meet coverage thresholds'); + process.exit(1); + } + } } else { core.info('No affected projects :)'); } diff --git a/tools/scripts/run-many/threshold-handler.ts b/tools/scripts/run-many/threshold-handler.ts new file mode 100644 index 00000000..56b3f580 --- /dev/null +++ b/tools/scripts/run-many/threshold-handler.ts @@ -0,0 +1,63 @@ +import * as core from '@actions/core'; + +/** + * Interface representing coverage thresholds for a project + */ +export interface CoverageThreshold { + lines?: number; + statements?: number; + functions?: number; + branches?: number; +} + +/** + * Interface representing the complete threshold configuration + * with global defaults and project-specific overrides + */ +export interface ThresholdConfig { + global: CoverageThreshold; + projects: Record; +} + +/** + * Parses the COVERAGE_THRESHOLDS environment variable + * @returns The parsed threshold configuration or empty defaults if not provided + */ +export function getCoverageThresholds(): ThresholdConfig { + if (!process.env.COVERAGE_THRESHOLDS) { + return { global: {}, projects: {} }; + } + + try { + return JSON.parse(process.env.COVERAGE_THRESHOLDS); + } catch (error) { + core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`); + return { global: {}, projects: {} }; + } +} + +/** + * Gets thresholds for a specific project + * @param project The project name + * @param thresholds The threshold configuration + * @returns Project-specific thresholds, global thresholds, or null if none defined or explicitly skipped + */ +export function getProjectThresholds(project: string, thresholds: ThresholdConfig): CoverageThreshold | null { + // If project explicitly set to null, return null to skip + if (thresholds.projects && thresholds.projects[project] === null) { + return null; + } + + // If project has specific thresholds, use those + if (thresholds.projects && thresholds.projects[project]) { + return thresholds.projects[project]; + } + + // Otherwise, use global thresholds if available + if (thresholds.global) { + return thresholds.global; + } + + // If no thresholds defined, return null + return null; +} \ No newline at end of file From c6b4573d9587d288dbd4a46a774a1e346b3a2e44 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 10:24:57 +0200 Subject: [PATCH 11/27] PFM-TASK-6308 create reusable GitHub Action for code coverage evaluation and reporting --- .github/workflows/fe-code-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 11f2cc60..a7040f7d 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -38,7 +38,7 @@ jobs: run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} - name: Unit Tests - uses: collaborationFactory/github-actions/.github/actions/run-many@master + uses: collaborationFactory/github-actions/.github/actions/run-many@improvement/PFM-TASK-6308-Create-reusable-GHA-for-code-coverage-gate with: target: ${{ matrix.target }} jobIndex: ${{ matrix.jobIndex }} From 035baf1ce917cb60bdcfb33dbc9eb8bca54100f2 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 11:24:38 +0200 Subject: [PATCH 12/27] PFM-TASK-6308 add input for enabling coverage gate in GitHub Action workflow --- .github/workflows/fe-code-quality.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index a7040f7d..9c7a6379 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -2,6 +2,11 @@ name: Frontend Code Quality Workflow on: workflow_call: + inputs: + ENABLE_COVERAGE_GATE: + required: false + type: boolean + default: false secrets: COVERAGE_THRESHOLDS: required: false @@ -15,6 +20,7 @@ jobs: jobIndex: [ 1, 2, 3, 4 ] env: jobCount: 4 + HAS_COVERAGE_THRESHOLDS: ${{ inputs.ENABLE_COVERAGE_GATE }} steps: - uses: actions/checkout@v4 with: @@ -38,7 +44,7 @@ jobs: run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} - name: Unit Tests - uses: collaborationFactory/github-actions/.github/actions/run-many@improvement/PFM-TASK-6308-Create-reusable-GHA-for-code-coverage-gate + uses: collaborationFactory/github-actions/.github/actions/run-many@master with: target: ${{ matrix.target }} jobIndex: ${{ matrix.jobIndex }} @@ -49,7 +55,7 @@ jobs: COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} - name: Upload Coverage Reports - if: always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: actions/upload-artifact@v4 with: name: coverage-reports @@ -57,12 +63,12 @@ jobs: retention-days: 7 - name: Set Artifact URL - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' run: | echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV - name: Comment PR with Coverage Report - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: thollander/actions-comment-pull-request@v3 with: file_path: 'coverage-report.txt' From 61072045fd2977579fd7cc0a3b10ca9d1d22265d Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 11:41:10 +0200 Subject: [PATCH 13/27] PFM-TASK-6308 refactor: update coverage gate logic and improve GitHub Action configuration --- .github/workflows/fe-code-quality.yml | 8 +++++--- tools/scripts/run-many/run-many.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 9c7a6379..f7f1e673 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -70,7 +70,9 @@ jobs: - name: Comment PR with Coverage Report if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: thollander/actions-comment-pull-request@v3 - with: - file_path: 'coverage-report.txt' - comment_tag: 'coverage-report' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + file-path: 'coverage-report.txt' + comment-tag: 'coverage-report' + mode: upsert diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index c8f0bb5e..b32e3a91 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -50,7 +50,7 @@ function main() { const projects = projectsString ? projectsString.split(',') : []; // Check if coverage gate is enabled - const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + const coverageEnabled = process.env.ENABLE_COVERAGE_GATE; // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; From 5e2406c67aac819e419b1bfe94b809784a0ad0a8 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 12:03:26 +0200 Subject: [PATCH 14/27] PFM-TASK-6308 refactor: enhance coverage evaluation logic and improve GitHub Action configuration --- .github/workflows/fe-code-quality.yml | 13 +++---- tools/scripts/run-many/coverage-evaluator.ts | 36 ++++++++++++++++---- tools/scripts/run-many/run-many.ts | 11 +++++- tools/scripts/run-many/threshold-handler.ts | 28 ++++++++------- 4 files changed, 59 insertions(+), 29 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index f7f1e673..6ae01166 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -2,11 +2,6 @@ name: Frontend Code Quality Workflow on: workflow_call: - inputs: - ENABLE_COVERAGE_GATE: - required: false - type: boolean - default: false secrets: COVERAGE_THRESHOLDS: required: false @@ -20,7 +15,6 @@ jobs: jobIndex: [ 1, 2, 3, 4 ] env: jobCount: 4 - HAS_COVERAGE_THRESHOLDS: ${{ inputs.ENABLE_COVERAGE_GATE }} steps: - uses: actions/checkout@v4 with: @@ -30,6 +24,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 18.19.1 + - name: Cache Node Modules id: npm-cache uses: actions/cache@v4 @@ -55,7 +50,7 @@ jobs: COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} - name: Upload Coverage Reports - if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' + if: always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' uses: actions/upload-artifact@v4 with: name: coverage-reports @@ -63,12 +58,12 @@ jobs: retention-days: 7 - name: Set Artifact URL - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' run: | echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV - name: Comment PR with Coverage Report - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' uses: thollander/actions-comment-pull-request@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index 45de5900..2502d970 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -27,12 +27,15 @@ interface ProjectCoverageResult { */ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): boolean { if (!process.env.COVERAGE_THRESHOLDS) { + core.info('No coverage thresholds defined, skipping evaluation'); return true; // No thresholds defined, pass by default } let allProjectsPassed = true; const coverageResults: ProjectCoverageResult[] = []; + core.info(`Evaluating coverage for ${projects.length} projects`); + for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); @@ -66,14 +69,35 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); const summary = coverageData.total as CoverageSummary; - const projectPassed = - (!projectThresholds.lines || summary.lines.pct >= projectThresholds.lines) && - (!projectThresholds.statements || summary.statements.pct >= projectThresholds.statements) && - (!projectThresholds.functions || summary.functions.pct >= projectThresholds.functions) && - (!projectThresholds.branches || summary.branches.pct >= projectThresholds.branches); + let projectPassed = true; + const failedMetrics: string[] = []; + + // Check each metric if threshold is defined + if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) { + projectPassed = false; + failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`); + } + + if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) { + projectPassed = false; + failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`); + } + + if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) { + projectPassed = false; + failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`); + } + + if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) { + projectPassed = false; + failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`); + } if (!projectPassed) { + core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`); allProjectsPassed = false; + } else { + core.info(`Project ${project} passed all coverage thresholds`); } coverageResults.push({ @@ -162,4 +186,4 @@ function postCoverageComment(results: ProjectCoverageResult[]): void { fs.writeFileSync(gitHubCommentsFile, comment); core.info('Coverage results saved for PR comment'); -} \ No newline at end of file +} diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index b32e3a91..70e098d1 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -50,7 +50,11 @@ function main() { const projects = projectsString ? projectsString.split(',') : []; // Check if coverage gate is enabled - const coverageEnabled = process.env.ENABLE_COVERAGE_GATE; + const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + + if (coverageEnabled && target === 'test') { + core.info('Coverage gate is enabled'); + } // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; @@ -72,6 +76,11 @@ function main() { // Evaluate coverage if enabled and target is test if (coverageEnabled && target === 'test') { const thresholds = getCoverageThresholds(); + + // Log the current coverage thresholds for debugging + core.info('Coverage threshold configuration:'); + core.info(JSON.stringify(thresholds, null, 2)); + const passed = evaluateCoverage(projects, thresholds); if (!passed) { diff --git a/tools/scripts/run-many/threshold-handler.ts b/tools/scripts/run-many/threshold-handler.ts index 56b3f580..c6926ae1 100644 --- a/tools/scripts/run-many/threshold-handler.ts +++ b/tools/scripts/run-many/threshold-handler.ts @@ -1,8 +1,5 @@ import * as core from '@actions/core'; -/** - * Interface representing coverage thresholds for a project - */ export interface CoverageThreshold { lines?: number; statements?: number; @@ -10,10 +7,6 @@ export interface CoverageThreshold { branches?: number; } -/** - * Interface representing the complete threshold configuration - * with global defaults and project-specific overrides - */ export interface ThresholdConfig { global: CoverageThreshold; projects: Record; @@ -21,15 +14,23 @@ export interface ThresholdConfig { /** * Parses the COVERAGE_THRESHOLDS environment variable - * @returns The parsed threshold configuration or empty defaults if not provided */ export function getCoverageThresholds(): ThresholdConfig { if (!process.env.COVERAGE_THRESHOLDS) { + core.info('No coverage thresholds defined, using empty configuration'); return { global: {}, projects: {} }; } try { - return JSON.parse(process.env.COVERAGE_THRESHOLDS); + const thresholdConfig = JSON.parse(process.env.COVERAGE_THRESHOLDS); + core.info(`Successfully parsed coverage thresholds`); + + // Validate structure + if (!thresholdConfig.global) { + core.warning('No global thresholds defined in configuration'); + } + + return thresholdConfig; } catch (error) { core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`); return { global: {}, projects: {} }; @@ -38,26 +39,27 @@ export function getCoverageThresholds(): ThresholdConfig { /** * Gets thresholds for a specific project - * @param project The project name - * @param thresholds The threshold configuration - * @returns Project-specific thresholds, global thresholds, or null if none defined or explicitly skipped */ export function getProjectThresholds(project: string, thresholds: ThresholdConfig): CoverageThreshold | null { // If project explicitly set to null, return null to skip if (thresholds.projects && thresholds.projects[project] === null) { + core.info(`Project ${project} is set to null in config, skipping coverage evaluation`); return null; } // If project has specific thresholds, use those if (thresholds.projects && thresholds.projects[project]) { + core.info(`Using specific thresholds for project ${project}`); return thresholds.projects[project]; } // Otherwise, use global thresholds if available if (thresholds.global) { + core.info(`Using global thresholds for project ${project}`); return thresholds.global; } // If no thresholds defined, return null + core.warning(`No thresholds defined for project ${project} and no global thresholds available`); return null; -} \ No newline at end of file +} From d5eddf027e23e2dbf6c118578f510bf600ea50fc Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 12:21:27 +0200 Subject: [PATCH 15/27] PFM-TASK-6308 refactor: update GitHub Action to use environment variable for coverage thresholds --- .github/workflows/fe-code-quality.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 6ae01166..ce773ab9 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -15,6 +15,7 @@ jobs: jobIndex: [ 1, 2, 3, 4 ] env: jobCount: 4 + HAS_COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS != '' }} steps: - uses: actions/checkout@v4 with: @@ -39,7 +40,7 @@ jobs: run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} - name: Unit Tests - uses: collaborationFactory/github-actions/.github/actions/run-many@master + uses: collaborationFactory/github-actions/.github/actions/run-many@improvement/PFM-TASK-6308-Create-reusable-GHA-for-code-coverage-gate with: target: ${{ matrix.target }} jobIndex: ${{ matrix.jobIndex }} @@ -50,7 +51,7 @@ jobs: COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} - name: Upload Coverage Reports - if: always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: actions/upload-artifact@v4 with: name: coverage-reports @@ -58,12 +59,12 @@ jobs: retention-days: 7 - name: Set Artifact URL - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' run: | echo "COVERAGE_ARTIFACT_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV - name: Comment PR with Coverage Report - if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && secrets.COVERAGE_THRESHOLDS != '' + if: github.event_name == 'pull_request' && always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: thollander/actions-comment-pull-request@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 2e8203f1b547cff73544df96d567b8ce7ed83c3a Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 12:43:50 +0200 Subject: [PATCH 16/27] PFM-TASK-6308 chore: add jest-junit for test result reporting in GitHub Action --- package-lock.json | 37 +++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 38 insertions(+) diff --git a/package-lock.json b/package-lock.json index 7cd6ea3c..a7160147 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@types/jest": "29.5.14", "@types/node": "^18.19.1", "jest": "29.7.0", + "jest-junit": "^16.0.0", "luxon": "^3.3.0", "prettier": "2.8.1", "ts-jest": "29.1.1", @@ -4417,6 +4418,22 @@ "fsevents": "^2.3.2" } }, + "node_modules/jest-junit": { + "version": "16.0.0", + "resolved": "https://cplace.jfrog.io/artifactory/api/npm/cplace-npm/jest-junit/-/jest-junit-16.0.0.tgz", + "integrity": "sha512-A94mmw6NfJab4Fg/BlvVOUXzXgF0XIH6EmTgJ5NDPp4xoKq0Kr7sErb+4Xs9nZvu58pJojz5RFGpqnZYJTrRfQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mkdirp": "^1.0.4", + "strip-ansi": "^6.0.1", + "uuid": "^8.3.2", + "xml": "^1.0.1" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/jest-leak-detector": { "version": "29.7.0", "resolved": "https://cplace.jfrog.io/artifactory/api/npm/cplace-npm/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", @@ -5542,6 +5559,19 @@ "node": "*" } }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://cplace.jfrog.io/artifactory/api/npm/cplace-npm/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -6524,6 +6554,13 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/xml": { + "version": "1.0.1", + "resolved": "https://cplace.jfrog.io/artifactory/api/npm/cplace-npm/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", + "dev": true, + "license": "MIT" + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://cplace.jfrog.io/artifactory/api/npm/cplace-npm/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index c43ae5e4..6551443c 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@types/jest": "29.5.14", "@types/node": "^18.19.1", "jest": "29.7.0", + "jest-junit": "^16.0.0", "luxon": "^3.3.0", "prettier": "2.8.1", "ts-jest": "29.1.1", From bbb5e22693df6747ddf5b3d5a6a112306806f1bf Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 13:11:24 +0200 Subject: [PATCH 17/27] PFM-TASK-6308 refactor: enhance coverage evaluation and reporting for affected projects --- tools/scripts/run-many/affected-projects.ts | 14 +++- tools/scripts/run-many/coverage-evaluator.ts | 70 ++++++++++++-------- tools/scripts/run-many/run-many.ts | 34 +++++++--- tools/scripts/run-many/threshold-handler.ts | 4 +- 4 files changed, 85 insertions(+), 37 deletions(-) diff --git a/tools/scripts/run-many/affected-projects.ts b/tools/scripts/run-many/affected-projects.ts index 58e1d747..f4877edc 100644 --- a/tools/scripts/run-many/affected-projects.ts +++ b/tools/scripts/run-many/affected-projects.ts @@ -21,7 +21,7 @@ export function getAffectedProjects( jobCount: number, base: string, ref: string -) { +): string { let allAffectedProjects = []; if (target === 'e2e' && ref === '') { allAffectedProjects = Utils.getAllProjects(false, null, target); @@ -32,5 +32,17 @@ export function getAffectedProjects( const projects = distributeProjectsEvenly(allAffectedProjects, jobCount); console.log(`Affected Projects:`); console.table(projects); + + // Handle case when no projects are assigned to this job index + if (jobIndex - 1 >= projects.length || !projects[jobIndex - 1] || projects[jobIndex - 1].length === 0) { + return ''; + } + return projects[jobIndex - 1].join(','); } + +// Check if there are any affected projects +export function hasAffectedProjects(base: string, target?: string): boolean { + const projects = Utils.getAllProjects(true, base, target); + return projects.length > 0; +} diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index 2502d970..e5990863 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -30,15 +30,15 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig core.info('No coverage thresholds defined, skipping evaluation'); return true; // No thresholds defined, pass by default } - + let allProjectsPassed = true; const coverageResults: ProjectCoverageResult[] = []; - + core.info(`Evaluating coverage for ${projects.length} projects`); - + for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); - + // Skip projects with null thresholds if (projectThresholds === null) { core.info(`Coverage evaluation skipped for ${project}`); @@ -50,9 +50,9 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig }); continue; } - + const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); - + if (!fs.existsSync(coveragePath)) { core.warning(`No coverage report found for ${project} at ${coveragePath}`); coverageResults.push({ @@ -64,42 +64,42 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig allProjectsPassed = false; continue; } - + try { const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); const summary = coverageData.total as CoverageSummary; - + let projectPassed = true; const failedMetrics: string[] = []; - + // Check each metric if threshold is defined if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) { projectPassed = false; failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`); } - + if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) { projectPassed = false; failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`); } - + if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) { projectPassed = false; failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`); } - + if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) { projectPassed = false; failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`); } - + if (!projectPassed) { core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`); allProjectsPassed = false; } else { core.info(`Project ${project} passed all coverage thresholds`); } - + coverageResults.push({ project, thresholds: projectThresholds, @@ -122,10 +122,10 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig allProjectsPassed = false; } } - + // Post results to PR comment postCoverageComment(coverageResults); - + return allProjectsPassed; } @@ -134,9 +134,15 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig */ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string): string { let comment = '## Test Coverage Results\n\n'; + + if (results.length === 0) { + comment += 'No projects were evaluated for coverage.\n'; + return comment; + } + comment += '| Project | Metric | Threshold | Actual | Status |\n'; comment += '|---------|--------|-----------|--------|--------|\n'; - + results.forEach(result => { if (result.status === 'SKIPPED') { comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; @@ -147,28 +153,28 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st metrics.forEach((metric, index) => { // Skip metrics that don't have a threshold if (!result.thresholds[metric]) return; - + const threshold = result.thresholds[metric]; const actual = result.actual[metric].toFixed(2); const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED'; - + // Only include project name in the first row for this project const projectCell = index === 0 ? result.project : ''; - + comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; }); } }); - + // Add overall status const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; comment += `\n### Overall Status: ${overallStatus}\n`; - + // Add link to detailed HTML reports if (artifactUrl) { comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`; } - + return comment; } @@ -178,12 +184,24 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st function postCoverageComment(results: ProjectCoverageResult[]): void { // The actual artifact URL will be provided by GitHub Actions in the workflow const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || ''; - + const comment = formatCoverageComment(results, artifactUrl); - + // Write to a file that will be used by thollander/actions-comment-pull-request action const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); fs.writeFileSync(gitHubCommentsFile, comment); - + core.info('Coverage results saved for PR comment'); } + +/** + * Generates a coverage report when no projects are affected + */ +export function generateEmptyCoverageReport(): void { + const comment = '## Test Coverage Results\n\n⏩ No projects were affected by this change that require coverage evaluation.\n'; + + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Empty coverage report generated (no affected projects)'); +} diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index 70e098d1..41905b63 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -1,9 +1,10 @@ import { getAffectedProjects } from './affected-projects'; import { execSync } from 'child_process'; import * as core from '@actions/core'; +import * as fs from 'fs'; import * as path from 'path'; import { getCoverageThresholds } from './threshold-handler'; -import { evaluateCoverage } from './coverage-evaluator'; +import { evaluateCoverage, generateEmptyCoverageReport } from './coverage-evaluator'; function getE2ECommand(command: string, base: string): string { command = command.concat(` -c ci --base=${base} --verbose`); @@ -31,6 +32,12 @@ function runCommand(command: string): void { } } +function ensureDirectoryExists(dirPath: string): void { + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }); + } +} + function main() { const target = process.argv[2]; const jobIndex = Number(process.argv[3]); @@ -46,15 +53,16 @@ function main() { core.info(`Inputs:\n target ${target},\n jobIndex: ${jobIndex},\n jobCount ${jobCount},\n base ${base},\n ref ${ref}`) - const projectsString = getAffectedProjects(target, jobIndex, jobCount, base, ref); - const projects = projectsString ? projectsString.split(',') : []; - // Check if coverage gate is enabled const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; - if (coverageEnabled && target === 'test') { - core.info('Coverage gate is enabled'); - } + // Get the affected projects + const projectsString = getAffectedProjects(target, jobIndex, jobCount, base, ref); + const projects = projectsString ? projectsString.split(',') : []; + + // Check if there are any affected projects (for first job only, to avoid duplicate reports) + const areAffectedProjects = projects.length > 0; + const isFirstJob = jobIndex === 1; // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; @@ -62,6 +70,7 @@ function main() { // Add coverage flag if enabled and target is test if (coverageEnabled && target === 'test') { + core.info('Coverage gate is enabled'); // Add coverage reporters for HTML, JSON, and JUnit output cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; } @@ -70,7 +79,7 @@ function main() { cmd = getE2ECommand(cmd, base); } - if (projects.length > 0) { + if (areAffectedProjects) { runCommand(cmd); // Evaluate coverage if enabled and target is test @@ -90,6 +99,15 @@ function main() { } } else { core.info('No affected projects :)'); + + // Generate empty coverage report for first job only when coverage is enabled + if (coverageEnabled && target === 'test' && isFirstJob) { + // Ensure coverage directory exists for artifact upload + ensureDirectoryExists(path.resolve(process.cwd(), 'coverage')); + + // Generate empty report + generateEmptyCoverageReport(); + } } } diff --git a/tools/scripts/run-many/threshold-handler.ts b/tools/scripts/run-many/threshold-handler.ts index c6926ae1..fa757b63 100644 --- a/tools/scripts/run-many/threshold-handler.ts +++ b/tools/scripts/run-many/threshold-handler.ts @@ -24,12 +24,12 @@ export function getCoverageThresholds(): ThresholdConfig { try { const thresholdConfig = JSON.parse(process.env.COVERAGE_THRESHOLDS); core.info(`Successfully parsed coverage thresholds`); - + // Validate structure if (!thresholdConfig.global) { core.warning('No global thresholds defined in configuration'); } - + return thresholdConfig; } catch (error) { core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`); From 6408f9890781660d986d4234c3c3269f3561257f Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 15:16:13 +0200 Subject: [PATCH 18/27] PFM-TASK-6308 test: add comprehensive tests for coverage evaluation and threshold handling --- .../run-many/coverage-evaluator.spec.ts | 335 ++++++++++++ tools/scripts/run-many/run-many.spec.ts | 492 ++++++++++++++++++ .../run-many/threshold-handler.spec.ts | 198 +++++++ 3 files changed, 1025 insertions(+) create mode 100644 tools/scripts/run-many/coverage-evaluator.spec.ts create mode 100644 tools/scripts/run-many/run-many.spec.ts create mode 100644 tools/scripts/run-many/threshold-handler.spec.ts diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts new file mode 100644 index 00000000..bb8ee77d --- /dev/null +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -0,0 +1,335 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import * as core from '@actions/core'; +import { evaluateCoverage, generateEmptyCoverageReport } from './coverage-evaluator'; +import { getProjectThresholds } from './threshold-handler'; + +// Mock dependencies +jest.mock('fs', () => ({ + existsSync: jest.fn(), + readFileSync: jest.fn(), + writeFileSync: jest.fn(), +})); +jest.mock('path', () => ({ + resolve: jest.fn((_, p) => p), +})); +jest.mock('@actions/core', () => ({ + info: jest.fn(), + warning: jest.fn(), + error: jest.fn(), +})); +jest.mock('./threshold-handler', () => ({ + getProjectThresholds: jest.fn(), +})); + +describe('coverage-evaluator', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80, statements: 80, functions: 75, branches: 70 }, + projects: {} + }); + }); + + afterEach(() => { + process.env = originalEnv; + }); + + describe('evaluateCoverage', () => { + it('should pass when COVERAGE_THRESHOLDS is not set', () => { + delete process.env.COVERAGE_THRESHOLDS; + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(true); + expect(core.info).toHaveBeenCalledWith('No coverage thresholds defined, skipping evaluation'); + }); + + it('should skip projects with null thresholds', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds.mockReturnValue(null); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(true); + expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a'); + expect(fs.writeFileSync).toHaveBeenCalled(); + + // Verify that the comment indicates the project was skipped + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('⏩ SKIPPED'); + }); + + it('should fail when coverage report is missing', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds.mockReturnValue({ lines: 80 }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(false); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(false); + expect(core.warning).toHaveBeenCalledWith('No coverage report found for project-a at coverage/project-a/coverage-summary.json'); + + // Verify that the comment indicates the project failed due to missing report + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('❌ FAILED'); + expect(comment).toContain('No Data'); + }); + + it('should fail when coverage is below thresholds', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds.mockReturnValue({ + lines: 80, + statements: 80, + functions: 75, + branches: 70 + }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync.mockReturnValue(JSON.stringify({ + total: { + lines: { pct: 75 }, + statements: { pct: 75 }, + functions: { pct: 70 }, + branches: { pct: 65 } + } + })); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(false); + expect(core.error).toHaveBeenCalledWith(expect.stringContaining('Project project-a failed coverage thresholds')); + + // Verify that the comment shows the failed metrics with correct values + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('| project-a | lines | 80% | 75.00% | ❌ FAILED |'); + expect(comment).toContain('| | statements | 80% | 75.00% | ❌ FAILED |'); + expect(comment).toContain('| | functions | 75% | 70.00% | ❌ FAILED |'); + expect(comment).toContain('| | branches | 70% | 65.00% | ❌ FAILED |'); + expect(comment).toContain('### Overall Status: ❌ FAILED'); + }); + + it('should pass when coverage meets thresholds', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds.mockReturnValue({ + lines: 80, + statements: 80, + functions: 75, + branches: 70 + }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync.mockReturnValue(JSON.stringify({ + total: { + lines: { pct: 85 }, + statements: { pct: 85 }, + functions: { pct: 80 }, + branches: { pct: 75 } + } + })); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(true); + expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); + + // Verify that the comment shows passing status with correct values + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); + expect(comment).toContain('| | statements | 80% | 85.00% | ✅ PASSED |'); + expect(comment).toContain('| | functions | 75% | 80.00% | ✅ PASSED |'); + expect(comment).toContain('| | branches | 70% | 75.00% | ✅ PASSED |'); + expect(comment).toContain('### Overall Status: ✅ PASSED'); + }); + + it('should handle errors in coverage processing', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds.mockReturnValue({ + lines: 80, + statements: 80, + functions: 75, + branches: 70 + }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync.mockImplementation(() => { + throw new Error('Test error'); + }); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(false); + expect(core.error).toHaveBeenCalledWith('Error processing coverage for project-a: Test error'); + + // Verify that the comment shows an error status + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('❌ FAILED'); + expect(comment).toContain('No Data'); + }); + + it('should pass even when some metrics are missing from thresholds', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + // Only include lines and functions thresholds + mockGetProjectThresholds.mockReturnValue({ + lines: 80, + functions: 75 + }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync.mockReturnValue(JSON.stringify({ + total: { + lines: { pct: 85 }, + statements: { pct: 85 }, + functions: { pct: 80 }, + branches: { pct: 75 } + } + })); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + expect(result).toBe(true); + + // Verify that only the defined thresholds are in the comment + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); + expect(comment).toContain('| | functions | 75% | 80.00% | ✅ PASSED |'); + expect(comment).not.toContain('| | statements |'); + expect(comment).not.toContain('| | branches |'); + }); + + it('should treat empty threshold objects as valid thresholds', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + // Return an empty object instead of null + mockGetProjectThresholds.mockReturnValue({}); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync.mockReturnValue(JSON.stringify({ + total: { + lines: { pct: 85 }, + statements: { pct: 85 }, + functions: { pct: 80 }, + branches: { pct: 75 } + } + })); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + // Should pass because no specific thresholds were set + expect(result).toBe(true); + expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); + }); + + it('should evaluate multiple projects correctly and generate summary', () => { + // First project passes + // Second project is skipped + // Third project fails + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds + .mockReturnValueOnce({ + lines: 80, + statements: 80, + functions: 75, + branches: 70 + }) + .mockReturnValueOnce(null) // Skip project-b + .mockReturnValueOnce({ + lines: 70, + statements: 70, + functions: 65, + branches: 60 + }); + + const mockExistsSync = fs.existsSync as jest.Mock; + mockExistsSync.mockReturnValue(true); + + const mockReadFileSync = fs.readFileSync as jest.Mock; + mockReadFileSync + .mockReturnValueOnce(JSON.stringify({ + total: { + lines: { pct: 85 }, + statements: { pct: 85 }, + functions: { pct: 80 }, + branches: { pct: 75 } + } + })) + .mockReturnValueOnce(JSON.stringify({ + total: { + lines: { pct: 65 }, + statements: { pct: 65 }, + functions: { pct: 60 }, + branches: { pct: 55 } + } + })); + + process.env.COVERAGE_ARTIFACT_URL = 'https://example.com/artifact'; + + const result = evaluateCoverage(['project-a', 'project-b', 'project-c'], { + global: { lines: 80, statements: 80, functions: 75, branches: 70 }, + projects: {} + }); + + expect(result).toBe(false); + + // Verify that the comment shows the correct status for each project + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + const comment = writeFileSyncMock.mock.calls[0][1]; + + // Project A passes + expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); + + // Project B is skipped + expect(comment).toContain('| project-b | All | N/A | N/A | ⏩ SKIPPED |'); + + // Project C fails + expect(comment).toContain('| project-c | lines | 70% | 65.00% | ❌ FAILED |'); + + // Overall status is failed + expect(comment).toContain('### Overall Status: ❌ FAILED'); + + // Artifact URL is included + expect(comment).toContain('📊 [View Detailed HTML Coverage Reports](https://example.com/artifact)'); + }); + }); + + describe('generateEmptyCoverageReport', () => { + it('should generate an empty report when no projects are affected', () => { + generateEmptyCoverageReport(); + + const writeFileSyncMock = fs.writeFileSync as jest.Mock; + expect(writeFileSyncMock).toHaveBeenCalledTimes(1); + + const [filePath, content] = writeFileSyncMock.mock.calls[0]; + expect(filePath).toBe('coverage-report.txt'); + expect(content).toContain('## Test Coverage Results'); + expect(content).toContain('No projects were affected by this change that require coverage evaluation'); + + expect(core.info).toHaveBeenCalledWith('Empty coverage report generated (no affected projects)'); + }); + }); +}); diff --git a/tools/scripts/run-many/run-many.spec.ts b/tools/scripts/run-many/run-many.spec.ts new file mode 100644 index 00000000..e07d09c4 --- /dev/null +++ b/tools/scripts/run-many/run-many.spec.ts @@ -0,0 +1,492 @@ +import { execSync } from 'child_process'; +import * as core from '@actions/core'; +import * as fs from 'fs'; +import * as path from 'path'; +import { getAffectedProjects } from './affected-projects'; +import { getCoverageThresholds } from './threshold-handler'; +import { evaluateCoverage, generateEmptyCoverageReport } from './coverage-evaluator'; + +// Define interfaces for custom error types we'll use +interface CommandError extends Error { + stdout?: { toString(): string }; + stderr?: { toString(): string }; + signal?: string; + status?: number; + code?: string; +} + +// Mock dependencies +jest.mock('child_process', () => ({ + execSync: jest.fn(), +})); +jest.mock('@actions/core', () => ({ + info: jest.fn(), + warning: jest.fn(), + error: jest.fn(), + setFailed: jest.fn(), +})); +jest.mock('./affected-projects', () => ({ + getAffectedProjects: jest.fn(), +})); +jest.mock('./threshold-handler', () => ({ + getCoverageThresholds: jest.fn(), +})); +jest.mock('./coverage-evaluator', () => ({ + evaluateCoverage: jest.fn(), + generateEmptyCoverageReport: jest.fn(), +})); +jest.mock('fs', () => ({ + existsSync: jest.fn(), + mkdirSync: jest.fn(), +})); +jest.mock('path', () => ({ + resolve: jest.fn((base, path) => `${base}/${path}`), +})); + +// Import the module under test +// Note: In a real test you would import directly, but for the sake of this exercise +// we'll simulate the behavior of the module +import * as runManyModule from './run-many'; + +describe('run-many', () => { + const originalEnv = process.env; + const originalExit = process.exit; + const originalArgv = process.argv; + let mockExit; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + mockExit = jest.fn(); + process.exit = mockExit as any; + process.argv = ['node', 'run-many.js', 'test', '1', '4', 'main', '']; + + // Reset fs.existsSync mock behavior + (fs.existsSync as jest.Mock).mockReturnValue(true); + }); + + afterEach(() => { + process.env = originalEnv; + process.exit = originalExit; + process.argv = originalArgv; + }); + + describe('runCommand function', () => { + it('should execute command successfully and log output', () => { + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('command output'); + + // Directly call the runCommand function + runCommand('test command'); + + // Verify it was called with the right args + expect(mockExecSync).toHaveBeenCalledWith('test command', { + stdio: 'pipe', + maxBuffer: 1024 * 1024 * 1024, + encoding: 'utf-8' + }); + expect(core.info).toHaveBeenCalledWith('Running > test command'); + expect(core.info).toHaveBeenCalledWith('command output'); + }); + + it('should handle command errors', () => { + const mockExecSync = execSync as jest.Mock; + const error: CommandError = new Error('Command failed'); + error.stdout = { toString: () => 'stdout output' }; + error.stderr = { toString: () => 'stderr output' }; + mockExecSync.mockImplementation(() => { + throw error; + }); + + // Directly call the runCommand function + runCommand('test command'); + + expect(core.info).toHaveBeenCalledWith('stdout output'); + expect(core.error).toHaveBeenCalledWith('stderr output'); + expect(core.setFailed).toHaveBeenCalledWith(error); + }); + + it('should handle timeout errors', () => { + const mockExecSync = execSync as jest.Mock; + const error: CommandError = new Error('Command timed out'); + error.signal = 'SIGTERM'; + error.stdout = { toString: () => 'stdout output' }; + error.stderr = { toString: () => 'stderr output' }; + mockExecSync.mockImplementation(() => { + throw error; + }); + + // Directly call the runCommand function + runCommand('test command'); + + expect(core.error).toHaveBeenCalledWith('Timed out'); + expect(core.info).toHaveBeenCalledWith('stdout output'); + expect(core.error).toHaveBeenCalledWith('stderr output'); + expect(core.setFailed).toHaveBeenCalledWith(error); + }); + + it('should handle buffer exceeded errors', () => { + const mockExecSync = execSync as jest.Mock; + const error: CommandError = new Error('Buffer exceeded'); + error.code = 'ENOBUFS'; + error.stdout = { toString: () => 'stdout output' }; + error.stderr = { toString: () => 'stderr output' }; + mockExecSync.mockImplementation(() => { + throw error; + }); + + // Directly call the runCommand function + runCommand('test command'); + + expect(core.error).toHaveBeenCalledWith('Buffer exceeded'); + expect(core.info).toHaveBeenCalledWith('stdout output'); + expect(core.error).toHaveBeenCalledWith('stderr output'); + expect(core.setFailed).toHaveBeenCalledWith(error); + }); + }); + + describe('main function', () => { + it('should run without coverage when COVERAGE_THRESHOLDS is not set', () => { + delete process.env.COVERAGE_THRESHOLDS; + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a,project-b'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + // Run the main function + main(); + + // Should run nx run-many without coverage flags + expect(mockExecSync).toHaveBeenCalledWith( + 'npx nx run-many --targets=test --projects="project-a,project-b" --parallel=false --prod', + expect.any(Object) + ); + + // Coverage evaluation functions should not be called + expect(getCoverageThresholds).not.toHaveBeenCalled(); + expect(evaluateCoverage).not.toHaveBeenCalled(); + }); + + it('should add coverage flags when running tests with COVERAGE_THRESHOLDS set', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a,project-b'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + const mockGetCoverageThresholds = getCoverageThresholds as jest.Mock; + mockGetCoverageThresholds.mockReturnValue({ + global: { lines: 80 } + }); + + const mockEvaluateCoverage = evaluateCoverage as jest.Mock; + mockEvaluateCoverage.mockReturnValue(true); + + // Run the main function + main(); + + // Should run nx run-many with coverage flags + expect(mockExecSync).toHaveBeenCalledWith( + 'npx nx run-many --targets=test --projects="project-a,project-b" --parallel=false --prod ' + + '--coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit', + expect.any(Object) + ); + + // Coverage evaluation should be performed + expect(mockGetCoverageThresholds).toHaveBeenCalled(); + expect(mockEvaluateCoverage).toHaveBeenCalledWith(['project-a', 'project-b'], expect.any(Object)); + }); + + it('should fail the build when coverage thresholds are not met', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a,project-b'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + const mockGetCoverageThresholds = getCoverageThresholds as jest.Mock; + mockGetCoverageThresholds.mockReturnValue({ + global: { lines: 80 } + }); + + const mockEvaluateCoverage = evaluateCoverage as jest.Mock; + mockEvaluateCoverage.mockReturnValue(false); // Coverage thresholds not met + + // Run the main function + main(); + + // Should run nx run-many with coverage flags + expect(mockExecSync).toHaveBeenCalledWith( + expect.stringContaining('--coverage'), + expect.any(Object) + ); + + // Should fail the build + expect(core.setFailed).toHaveBeenCalledWith('One or more projects failed to meet coverage thresholds'); + expect(mockExit).toHaveBeenCalledWith(1); + }); + + it('should add origin/ prefix to base branch that is not a SHA hash', () => { + process.argv[5] = 'develop'; + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + // Run the main function + main(); + + // Should call getAffectedProjects with origin/develop + expect(mockGetAffectedProjects).toHaveBeenCalledWith( + 'test', 1, 4, 'origin/develop', '' + ); + }); + + it('should not add origin/ prefix to SHA hash', () => { + process.argv[5] = 'abcdef1234567890abcdef1234567890abcdef12'; + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + // Run the main function + main(); + + // Should call getAffectedProjects with the unchanged SHA + expect(mockGetAffectedProjects).toHaveBeenCalledWith( + 'test', 1, 4, 'abcdef1234567890abcdef1234567890abcdef12', '' + ); + }); + + it('should handle 0000000 base by using git origin/HEAD', () => { + process.argv[5] = '0000000000000000'; + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a'); + + const mockExecSync = execSync as jest.Mock; + // First call returns the git head + mockExecSync.mockReturnValueOnce('origin/main'); + // Second call is for the run-many command + mockExecSync.mockReturnValueOnce('exec output'); + + // Run the main function + main(); + + // Should call git rev-parse to get the HEAD + expect(mockExecSync).toHaveBeenCalledWith('git rev-parse --abbrev-ref origin/HEAD '); + + // Should call getAffectedProjects with origin/main + expect(mockGetAffectedProjects).toHaveBeenCalledWith( + 'test', 1, 4, 'origin/main', '' + ); + }); + + it('should generate empty coverage report when no projects are affected', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue(''); // No affected projects + + const mockExecSync = execSync as jest.Mock; + + const mockGenerateEmptyCoverageReport = generateEmptyCoverageReport as jest.Mock; + + // Run the main function + main(); + + // Should log message about no affected projects + expect(core.info).toHaveBeenCalledWith('No affected projects :)'); + + // Should not run nx run-many + expect(mockExecSync).not.toHaveBeenCalled(); + + // Should generate empty coverage report + expect(mockGenerateEmptyCoverageReport).toHaveBeenCalled(); + + // Should ensure coverage directory exists + expect(fs.existsSync).toHaveBeenCalledWith(expect.stringContaining('coverage')); + }); + + it('should only generate empty report for first job', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + process.argv[3] = '2'; // Not the first job + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue(''); // No affected projects + + const mockExecSync = execSync as jest.Mock; + + const mockGenerateEmptyCoverageReport = generateEmptyCoverageReport as jest.Mock; + + // Run the main function + main(); + + // Should log message about no affected projects + expect(core.info).toHaveBeenCalledWith('No affected projects :)'); + + // Should not run nx run-many + expect(mockExecSync).not.toHaveBeenCalled(); + + // Should NOT generate empty coverage report (not first job) + expect(mockGenerateEmptyCoverageReport).not.toHaveBeenCalled(); + }); + + it('should create coverage directory if it does not exist', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue(''); // No affected projects + + // Mock fs.existsSync to return false for coverage directory + (fs.existsSync as jest.Mock).mockReturnValue(false); + + // Run the main function + main(); + + // Should attempt to create the coverage directory + expect(fs.mkdirSync).toHaveBeenCalledWith( + expect.stringContaining('coverage'), + { recursive: true } + ); + }); + + it('should add e2e specific flags when target includes e2e', () => { + process.argv[2] = 'e2e'; + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a,project-b'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + // Run the main function + main(); + + // Should run nx run-many with e2e specific flags + expect(mockExecSync).toHaveBeenCalledWith( + expect.stringContaining('-c ci --base=origin/main --verbose'), + expect.any(Object) + ); + }); + }); + + // Helper function to simulate runCommand + function runCommand(command: string): void { + try { + const mockExecSync = execSync as jest.Mock; + core.info(`Running > ${command}`); + const output = mockExecSync(command, { + stdio: 'pipe', + maxBuffer: 1024 * 1024 * 1024, + encoding: 'utf-8' + }); + core.info(output.toString()); + } catch (error) { + if (error.signal === 'SIGTERM') { + core.error('Timed out'); + } else if (error.code === 'ENOBUFS') { + core.error('Buffer exceeded'); + } + core.info(error.stdout?.toString() || ''); + core.error(error.stderr?.toString() || ''); + core.error(`Error message: ${error.message}`); + core.setFailed(error); + } + } + + // Helper function to simulate main + function main(): void { + const target = process.argv[2]; + const jobIndex = Number(process.argv[3]); + const jobCount = Number(process.argv[4]); + let base = process.argv[5]; + + // in case base is not a SHA1 commit hash add origin + if (!/\b[0-9a-f]{5,40}\b/.test(base)) base = 'origin/' + base; + if (base.includes('0000000000000000')) { + base = (execSync as jest.Mock)('git rev-parse --abbrev-ref origin/HEAD ').toString().trim(); + } + const ref = process.argv[6]; + + core.info(`Inputs:\n target ${target},\n jobIndex: ${jobIndex},\n jobCount ${jobCount},\n base ${base},\n ref ${ref}`); + + // Check if coverage gate is enabled + const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; + if (coverageEnabled && target === 'test') { + core.info('Coverage gate is enabled'); + } + + // Get the affected projects + const projectsString = (getAffectedProjects as jest.Mock)(target, jobIndex, jobCount, base, ref); + const projects = projectsString ? projectsString.split(',') : []; + + // Check if there are any affected projects (for first job only, to avoid duplicate reports) + const areAffectedProjects = projects.length > 0; + const isFirstJob = jobIndex === 1; + + // Modified command construction + const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; + let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + + // Add coverage flag if enabled and target is test + if (coverageEnabled && target === 'test') { + // Add coverage reporters for HTML, JSON, and JUnit output + cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; + } + + if (target.includes('e2e')) { + cmd += ` -c ci --base=${base} --verbose`; + } + + if (areAffectedProjects) { + // Run the command + runCommand(cmd); + + // Evaluate coverage if enabled and target is test + if (coverageEnabled && target === 'test') { + const thresholds = (getCoverageThresholds as jest.Mock)(); + const passed = (evaluateCoverage as jest.Mock)(projects, thresholds); + + if (!passed) { + core.setFailed('One or more projects failed to meet coverage thresholds'); + process.exit(1); + } + } + } else { + core.info('No affected projects :)'); + + // Generate empty coverage report for first job only when coverage is enabled + if (coverageEnabled && target === 'test' && isFirstJob) { + // Ensure coverage directory exists for artifact upload + const coverageDir = path.resolve(process.cwd(), 'coverage'); + if (!fs.existsSync(coverageDir)) { + fs.mkdirSync(coverageDir, { recursive: true }); + } + + // Generate empty report + (generateEmptyCoverageReport as jest.Mock)(); + } + } + } +}); diff --git a/tools/scripts/run-many/threshold-handler.spec.ts b/tools/scripts/run-many/threshold-handler.spec.ts new file mode 100644 index 00000000..1fc723b3 --- /dev/null +++ b/tools/scripts/run-many/threshold-handler.spec.ts @@ -0,0 +1,198 @@ +import * as core from '@actions/core'; +import { getCoverageThresholds, getProjectThresholds, CoverageThreshold, ThresholdConfig } from './threshold-handler'; + +// Mock @actions/core +jest.mock('@actions/core', () => ({ + error: jest.fn(), + info: jest.fn(), + warning: jest.fn(), +})); + +describe('threshold-handler', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + describe('getCoverageThresholds', () => { + it('should return empty thresholds when COVERAGE_THRESHOLDS is not set', () => { + delete process.env.COVERAGE_THRESHOLDS; + + const result = getCoverageThresholds(); + + expect(result).toEqual({ global: {}, projects: {} }); + expect(core.info).toHaveBeenCalledWith('No coverage thresholds defined, using empty configuration'); + }); + + it('should parse valid JSON from COVERAGE_THRESHOLDS', () => { + const thresholdConfig: ThresholdConfig = { + global: { lines: 80, statements: 80, functions: 75, branches: 70 }, + projects: { + 'project-a': { lines: 90, statements: 90, functions: 85, branches: 80 }, + 'project-b': null + } + }; + + process.env.COVERAGE_THRESHOLDS = JSON.stringify(thresholdConfig); + + const result = getCoverageThresholds(); + + expect(result).toEqual(thresholdConfig); + expect(core.info).toHaveBeenCalledWith('Successfully parsed coverage thresholds'); + }); + + it('should handle invalid JSON in COVERAGE_THRESHOLDS', () => { + process.env.COVERAGE_THRESHOLDS = 'invalid-json'; + + const result = getCoverageThresholds(); + + expect(result).toEqual({ global: {}, projects: {} }); + expect(core.error).toHaveBeenCalledWith(expect.stringContaining('Error parsing COVERAGE_THRESHOLDS')); + }); + + it('should log warning when global thresholds are missing', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + projects: { + 'project-a': { lines: 90 } + } + }); + + const result = getCoverageThresholds(); + + expect(result).toEqual({ + projects: { + 'project-a': { lines: 90 } + } + }); + expect(core.warning).toHaveBeenCalledWith('No global thresholds defined in configuration'); + }); + + it('should handle thresholds with missing properties', () => { + // Only specify lines and branches + const thresholdConfig = { + global: { lines: 80, branches: 70 }, + projects: { + 'project-a': { functions: 85 } + } + }; + + process.env.COVERAGE_THRESHOLDS = JSON.stringify(thresholdConfig); + + const result = getCoverageThresholds(); + + expect(result).toEqual(thresholdConfig); + }); + }); + + describe('getProjectThresholds', () => { + it('should return null for projects explicitly set to null', () => { + const thresholds: ThresholdConfig = { + global: { lines: 80 }, + projects: { 'project-a': null } + }; + + const result = getProjectThresholds('project-a', thresholds); + + expect(result).toBeNull(); + expect(core.info).toHaveBeenCalledWith('Project project-a is set to null in config, skipping coverage evaluation'); + }); + + it('should return project-specific thresholds when available', () => { + const projectThresholds: CoverageThreshold = { + lines: 90, + statements: 90, + functions: 85, + branches: 80 + }; + + const thresholds: ThresholdConfig = { + global: { lines: 80, statements: 80, functions: 75, branches: 70 }, + projects: { + 'project-a': projectThresholds + } + }; + + const result = getProjectThresholds('project-a', thresholds); + + expect(result).toEqual(projectThresholds); + expect(core.info).toHaveBeenCalledWith('Using specific thresholds for project project-a'); + }); + + it('should fall back to global thresholds when project-specific ones don\'t exist', () => { + const globalThresholds: CoverageThreshold = { + lines: 80, + statements: 80, + functions: 75, + branches: 70 + }; + + const thresholds: ThresholdConfig = { + global: globalThresholds, + projects: { + 'project-a': { lines: 90, statements: 90, functions: 85, branches: 80 } + } + }; + + const result = getProjectThresholds('project-b', thresholds); + + expect(result).toEqual(globalThresholds); + expect(core.info).toHaveBeenCalledWith('Using global thresholds for project project-b'); + }); + + it('should return empty object when global exists but is empty', () => { + const thresholds: ThresholdConfig = { + global: {}, // Empty but exists + projects: {} + }; + + const result = getProjectThresholds('project-a', thresholds); + + // Since global exists but is empty, it will return the empty object + expect(result).toEqual({}); + expect(core.info).toHaveBeenCalledWith('Using global thresholds for project project-a'); + }); + + it('should return null when global is undefined', () => { + // Create a config where global is undefined + const thresholds = { + projects: {} + } as ThresholdConfig; + + const result = getProjectThresholds('project-a', thresholds); + + expect(result).toBeNull(); + expect(core.warning).toHaveBeenCalledWith('No thresholds defined for project project-a and no global thresholds available'); + }); + + it('should handle partial project thresholds', () => { + const thresholds: ThresholdConfig = { + global: { lines: 80, statements: 80, functions: 75, branches: 70 }, + projects: { + 'project-a': { lines: 90 } // Only line threshold specified + } + }; + + const result = getProjectThresholds('project-a', thresholds); + + expect(result).toEqual({ lines: 90 }); + expect(core.info).toHaveBeenCalledWith('Using specific thresholds for project project-a'); + }); + + it('should handle missing projects object', () => { + const thresholds = { + global: { lines: 80, statements: 80, functions: 75, branches: 70 } + } as ThresholdConfig; + + const result = getProjectThresholds('project-a', thresholds); + + expect(result).toEqual({ lines: 80, statements: 80, functions: 75, branches: 70 }); + expect(core.info).toHaveBeenCalledWith('Using global thresholds for project project-a'); + }); + }); +}); From dd056ea4ac5d914b15faa318a124b1bdfb1ea20e Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Wed, 21 May 2025 15:54:01 +0200 Subject: [PATCH 19/27] PFM-TASK-6308 refactor: update coverage evaluation to return failure counts and improve reporting --- .github/workflows/fe-code-quality.yml | 30 +++++++++ .../run-many/coverage-evaluator.spec.ts | 64 +++++++++++++------ tools/scripts/run-many/coverage-evaluator.ts | 34 ++++++---- tools/scripts/run-many/run-many.spec.ts | 54 +++++++++++++--- tools/scripts/run-many/run-many.ts | 13 ++-- 5 files changed, 147 insertions(+), 48 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index ce773ab9..2e7db0dc 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -13,6 +13,7 @@ jobs: matrix: target: [ 'test' ] jobIndex: [ 1, 2, 3, 4 ] + fail-fast: false # Ensure all jobs run even if one fails env: jobCount: 4 HAS_COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS != '' }} @@ -40,6 +41,8 @@ jobs: run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} - name: Unit Tests + id: unit-tests + continue-on-error: true # Allow the step to complete even if it fails uses: collaborationFactory/github-actions/.github/actions/run-many@improvement/PFM-TASK-6308-Create-reusable-GHA-for-code-coverage-gate with: target: ${{ matrix.target }} @@ -50,6 +53,18 @@ jobs: env: COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS }} + # Use a step output to indicate if there are multiple failing projects + - name: Check for multiple failing projects + id: check-failures + run: | + if [[ "${{ steps.unit-tests.outcome }}" == "failure" ]]; then + echo "MULTIPLE_FAILURES=true" >> $GITHUB_ENV + echo "multiple_failures=true" >> $GITHUB_OUTPUT + else + echo "MULTIPLE_FAILURES=false" >> $GITHUB_ENV + echo "multiple_failures=false" >> $GITHUB_OUTPUT + fi + - name: Upload Coverage Reports if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: actions/upload-artifact@v4 @@ -72,3 +87,18 @@ jobs: file-path: 'coverage-report.txt' comment-tag: 'coverage-report' mode: upsert + + # Final job to check overall status and fail if multiple projects have coverage failures + check-coverage-status: + needs: code-quality + if: always() && needs.code-quality.result != 'cancelled' + runs-on: ubuntu-latest + steps: + - name: Check for multiple failing projects + run: | + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "::error::Multiple projects failed coverage thresholds - fix these issues before merging" + exit 1 + else + echo "All tests completed - no multiple coverage failures detected" + fi diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index bb8ee77d..70b3fe1f 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -39,22 +39,22 @@ describe('coverage-evaluator', () => { }); describe('evaluateCoverage', () => { - it('should pass when COVERAGE_THRESHOLDS is not set', () => { + it('should return zero failures when COVERAGE_THRESHOLDS is not set', () => { delete process.env.COVERAGE_THRESHOLDS; const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(true); + expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('No coverage thresholds defined, skipping evaluation'); }); - it('should skip projects with null thresholds', () => { + it('should skip projects with null thresholds and not count them as failures', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue(null); const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(true); + expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a'); expect(fs.writeFileSync).toHaveBeenCalled(); @@ -64,7 +64,7 @@ describe('coverage-evaluator', () => { expect(comment).toContain('⏩ SKIPPED'); }); - it('should fail when coverage report is missing', () => { + it('should count as one failure when coverage report is missing', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue({ lines: 80 }); @@ -73,7 +73,7 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(false); + expect(result).toBe(1); expect(core.warning).toHaveBeenCalledWith('No coverage report found for project-a at coverage/project-a/coverage-summary.json'); // Verify that the comment indicates the project failed due to missing report @@ -81,9 +81,10 @@ describe('coverage-evaluator', () => { const comment = writeFileSyncMock.mock.calls[0][1]; expect(comment).toContain('❌ FAILED'); expect(comment).toContain('No Data'); + expect(comment).toContain('⚠️ WARNING (1 project failing)'); }); - it('should fail when coverage is below thresholds', () => { + it('should count one failure when coverage is below thresholds', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue({ lines: 80, @@ -107,7 +108,7 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(false); + expect(result).toBe(1); expect(core.error).toHaveBeenCalledWith(expect.stringContaining('Project project-a failed coverage thresholds')); // Verify that the comment shows the failed metrics with correct values @@ -117,10 +118,11 @@ describe('coverage-evaluator', () => { expect(comment).toContain('| | statements | 80% | 75.00% | ❌ FAILED |'); expect(comment).toContain('| | functions | 75% | 70.00% | ❌ FAILED |'); expect(comment).toContain('| | branches | 70% | 65.00% | ❌ FAILED |'); - expect(comment).toContain('### Overall Status: ❌ FAILED'); + expect(comment).toContain('### Overall Status: ⚠️ WARNING (1 project failing)'); + expect(comment).toContain('Note: The build will continue, but this project should be fixed before merging.'); }); - it('should pass when coverage meets thresholds', () => { + it('should return zero failures when coverage meets thresholds', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue({ lines: 80, @@ -144,7 +146,7 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(true); + expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); // Verify that the comment shows passing status with correct values @@ -157,7 +159,7 @@ describe('coverage-evaluator', () => { expect(comment).toContain('### Overall Status: ✅ PASSED'); }); - it('should handle errors in coverage processing', () => { + it('should count one failure for errors in coverage processing', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue({ lines: 80, @@ -176,7 +178,7 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(false); + expect(result).toBe(1); expect(core.error).toHaveBeenCalledWith('Error processing coverage for project-a: Test error'); // Verify that the comment shows an error status @@ -184,6 +186,7 @@ describe('coverage-evaluator', () => { const comment = writeFileSyncMock.mock.calls[0][1]; expect(comment).toContain('❌ FAILED'); expect(comment).toContain('No Data'); + expect(comment).toContain('### Overall Status: ⚠️ WARNING (1 project failing)'); }); it('should pass even when some metrics are missing from thresholds', () => { @@ -209,7 +212,7 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); - expect(result).toBe(true); + expect(result).toBe(0); // Verify that only the defined thresholds are in the comment const writeFileSyncMock = fs.writeFileSync as jest.Mock; @@ -241,14 +244,15 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); // Should pass because no specific thresholds were set - expect(result).toBe(true); + expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); }); - it('should evaluate multiple projects correctly and generate summary', () => { + it('should count multiple failures correctly', () => { // First project passes // Second project is skipped // Third project fails + // Fourth project fails because no coverage data const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds .mockReturnValueOnce({ @@ -263,10 +267,22 @@ describe('coverage-evaluator', () => { statements: 70, functions: 65, branches: 60 + }) + .mockReturnValueOnce({ + lines: 90, + statements: 90, + functions: 90, + branches: 90 }); const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Make existsSync return true for project-a and project-c, but false for project-d + mockExistsSync.mockImplementation((path) => { + if (path.includes('project-d')) { + return false; // No coverage file for project-d + } + return true; + }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync @@ -286,15 +302,17 @@ describe('coverage-evaluator', () => { branches: { pct: 55 } } })); + // No need for third mock since we're making project-d file not exist process.env.COVERAGE_ARTIFACT_URL = 'https://example.com/artifact'; - const result = evaluateCoverage(['project-a', 'project-b', 'project-c'], { + const result = evaluateCoverage(['project-a', 'project-b', 'project-c', 'project-d'], { global: { lines: 80, statements: 80, functions: 75, branches: 70 }, projects: {} }); - expect(result).toBe(false); + // Two projects failed + expect(result).toBe(2); // Verify that the comment shows the correct status for each project const writeFileSyncMock = fs.writeFileSync as jest.Mock; @@ -309,8 +327,12 @@ describe('coverage-evaluator', () => { // Project C fails expect(comment).toContain('| project-c | lines | 70% | 65.00% | ❌ FAILED |'); - // Overall status is failed - expect(comment).toContain('### Overall Status: ❌ FAILED'); + // Project D has no coverage data + expect(comment).toContain('| project-d | All | Defined | No Data | ❌ FAILED |'); + + // Overall status is failed with multiple projects + expect(comment).toContain('### Overall Status: ❌ FAILED (2 projects failing)'); + expect(comment).toContain('Note: Multiple projects fail coverage thresholds. This PR will be blocked until fixed.'); // Artifact URL is included expect(comment).toContain('📊 [View Detailed HTML Coverage Reports](https://example.com/artifact)'); diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index e5990863..188b802d 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -25,13 +25,13 @@ interface ProjectCoverageResult { /** * Evaluates coverage for all projects against their thresholds */ -export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): boolean { +export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): number { if (!process.env.COVERAGE_THRESHOLDS) { core.info('No coverage thresholds defined, skipping evaluation'); - return true; // No thresholds defined, pass by default + return 0; // No thresholds defined, 0 failures } - let allProjectsPassed = true; + let failedProjectsCount = 0; const coverageResults: ProjectCoverageResult[] = []; core.info(`Evaluating coverage for ${projects.length} projects`); @@ -61,7 +61,7 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig actual: null, status: 'FAILED' // Mark as failed if no coverage report is found }); - allProjectsPassed = false; + failedProjectsCount++; continue; } @@ -95,7 +95,7 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig if (!projectPassed) { core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`); - allProjectsPassed = false; + failedProjectsCount++; } else { core.info(`Project ${project} passed all coverage thresholds`); } @@ -119,20 +119,20 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig actual: null, status: 'FAILED' }); - allProjectsPassed = false; + failedProjectsCount++; } } // Post results to PR comment - postCoverageComment(coverageResults); + postCoverageComment(coverageResults, failedProjectsCount); - return allProjectsPassed; + return failedProjectsCount; } /** * Formats the coverage results into a markdown table */ -function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string): string { +function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string, failedProjectsCount: number): string { let comment = '## Test Coverage Results\n\n'; if (results.length === 0) { @@ -166,10 +166,18 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st } }); - // Add overall status - const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED'; + // Add overall status with failed project count + const overallStatus = failedProjectsCount === 0 ? '✅ PASSED' : + failedProjectsCount === 1 ? '⚠️ WARNING (1 project failing)' : + `❌ FAILED (${failedProjectsCount} projects failing)`; comment += `\n### Overall Status: ${overallStatus}\n`; + if (failedProjectsCount === 1) { + comment += '\n> Note: The build will continue, but this project should be fixed before merging.\n'; + } else if (failedProjectsCount > 1) { + comment += '\n> Note: Multiple projects fail coverage thresholds. This PR will be blocked until fixed.\n'; + } + // Add link to detailed HTML reports if (artifactUrl) { comment += `\n📊 [View Detailed HTML Coverage Reports](${artifactUrl})\n`; @@ -181,11 +189,11 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st /** * Writes the coverage results to a file for PR comment */ -function postCoverageComment(results: ProjectCoverageResult[]): void { +function postCoverageComment(results: ProjectCoverageResult[], failedProjectsCount: number): void { // The actual artifact URL will be provided by GitHub Actions in the workflow const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || ''; - const comment = formatCoverageComment(results, artifactUrl); + const comment = formatCoverageComment(results, artifactUrl, failedProjectsCount); // Write to a file that will be used by thollander/actions-comment-pull-request action const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); diff --git a/tools/scripts/run-many/run-many.spec.ts b/tools/scripts/run-many/run-many.spec.ts index e07d09c4..2a549a34 100644 --- a/tools/scripts/run-many/run-many.spec.ts +++ b/tools/scripts/run-many/run-many.spec.ts @@ -186,7 +186,7 @@ describe('run-many', () => { }); const mockEvaluateCoverage = evaluateCoverage as jest.Mock; - mockEvaluateCoverage.mockReturnValue(true); + mockEvaluateCoverage.mockReturnValue(0); // No failures // Run the main function main(); @@ -203,7 +203,7 @@ describe('run-many', () => { expect(mockEvaluateCoverage).toHaveBeenCalledWith(['project-a', 'project-b'], expect.any(Object)); }); - it('should fail the build when coverage thresholds are not met', () => { + it('should allow tests to continue with one project failing coverage thresholds', () => { process.env.COVERAGE_THRESHOLDS = JSON.stringify({ global: { lines: 80 } }); @@ -220,7 +220,41 @@ describe('run-many', () => { }); const mockEvaluateCoverage = evaluateCoverage as jest.Mock; - mockEvaluateCoverage.mockReturnValue(false); // Coverage thresholds not met + mockEvaluateCoverage.mockReturnValue(1); // One project failed + + // Run the main function + main(); + + // Should run nx run-many with coverage flags + expect(mockExecSync).toHaveBeenCalledWith( + expect.stringContaining('--coverage'), + expect.any(Object) + ); + + // Should show warning but not fail the build + expect(core.warning).toHaveBeenCalledWith('One project failed to meet coverage thresholds - this should be fixed before merging'); + expect(core.setFailed).not.toHaveBeenCalled(); + expect(mockExit).not.toHaveBeenCalled(); + }); + + it('should fail the build when multiple projects fail coverage thresholds', () => { + process.env.COVERAGE_THRESHOLDS = JSON.stringify({ + global: { lines: 80 } + }); + + const mockGetAffectedProjects = getAffectedProjects as jest.Mock; + mockGetAffectedProjects.mockReturnValue('project-a,project-b,project-c'); + + const mockExecSync = execSync as jest.Mock; + mockExecSync.mockReturnValue('exec output'); + + const mockGetCoverageThresholds = getCoverageThresholds as jest.Mock; + mockGetCoverageThresholds.mockReturnValue({ + global: { lines: 80 } + }); + + const mockEvaluateCoverage = evaluateCoverage as jest.Mock; + mockEvaluateCoverage.mockReturnValue(2); // Two projects failed // Run the main function main(); @@ -232,8 +266,9 @@ describe('run-many', () => { ); // Should fail the build - expect(core.setFailed).toHaveBeenCalledWith('One or more projects failed to meet coverage thresholds'); - expect(mockExit).toHaveBeenCalledWith(1); + expect(core.setFailed).toHaveBeenCalledWith('Multiple projects (2) failed to meet coverage thresholds'); + // Notice we're not exiting, just setting the status to failed + expect(mockExit).not.toHaveBeenCalled(); }); it('should add origin/ prefix to base branch that is not a SHA hash', () => { @@ -466,11 +501,12 @@ describe('run-many', () => { // Evaluate coverage if enabled and target is test if (coverageEnabled && target === 'test') { const thresholds = (getCoverageThresholds as jest.Mock)(); - const passed = (evaluateCoverage as jest.Mock)(projects, thresholds); + const failedProjectsCount = (evaluateCoverage as jest.Mock)(projects, thresholds); - if (!passed) { - core.setFailed('One or more projects failed to meet coverage thresholds'); - process.exit(1); + if (failedProjectsCount > 1) { + core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); + } else if (failedProjectsCount === 1) { + core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); } } } else { diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index 41905b63..332448fb 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -90,11 +90,14 @@ function main() { core.info('Coverage threshold configuration:'); core.info(JSON.stringify(thresholds, null, 2)); - const passed = evaluateCoverage(projects, thresholds); - - if (!passed) { - core.setFailed('One or more projects failed to meet coverage thresholds'); - process.exit(1); + const failedProjectsCount = evaluateCoverage(projects, thresholds); + + if (failedProjectsCount > 1) { + core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); + // Don't exit immediately - we set the failed status but continue running + } else if (failedProjectsCount === 1) { + core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); + // Continue running, with a warning } } } else { From 1ca22e4e714bcd05744b773c9a5654c0adb16a78 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Mon, 26 May 2025 17:50:15 +0200 Subject: [PATCH 20/27] PFM-TASK-6308 refactor: enhance coverage reporting to show individual thresholds even when data is missing --- .../run-many/coverage-evaluator.spec.ts | 21 +++++++++++------- tools/scripts/run-many/coverage-evaluator.ts | 22 ++++++++++++++++++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index 70b3fe1f..09564804 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -66,7 +66,7 @@ describe('coverage-evaluator', () => { it('should count as one failure when coverage report is missing', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; - mockGetProjectThresholds.mockReturnValue({ lines: 80 }); + mockGetProjectThresholds.mockReturnValue({ lines: 80, statements: 75 }); const mockExistsSync = fs.existsSync as jest.Mock; mockExistsSync.mockReturnValue(false); @@ -76,11 +76,11 @@ describe('coverage-evaluator', () => { expect(result).toBe(1); expect(core.warning).toHaveBeenCalledWith('No coverage report found for project-a at coverage/project-a/coverage-summary.json'); - // Verify that the comment indicates the project failed due to missing report + // Verify that the comment shows individual thresholds with "No Data" const writeFileSyncMock = fs.writeFileSync as jest.Mock; const comment = writeFileSyncMock.mock.calls[0][1]; - expect(comment).toContain('❌ FAILED'); - expect(comment).toContain('No Data'); + expect(comment).toContain('| project-a | lines | 80% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | statements | 75% | No Data | ❌ FAILED |'); expect(comment).toContain('⚠️ WARNING (1 project failing)'); }); @@ -181,11 +181,13 @@ describe('coverage-evaluator', () => { expect(result).toBe(1); expect(core.error).toHaveBeenCalledWith('Error processing coverage for project-a: Test error'); - // Verify that the comment shows an error status + // Verify that the comment shows individual thresholds with "No Data" const writeFileSyncMock = fs.writeFileSync as jest.Mock; const comment = writeFileSyncMock.mock.calls[0][1]; - expect(comment).toContain('❌ FAILED'); - expect(comment).toContain('No Data'); + expect(comment).toContain('| project-a | lines | 80% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | statements | 80% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | functions | 75% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | branches | 70% | No Data | ❌ FAILED |'); expect(comment).toContain('### Overall Status: ⚠️ WARNING (1 project failing)'); }); @@ -328,7 +330,10 @@ describe('coverage-evaluator', () => { expect(comment).toContain('| project-c | lines | 70% | 65.00% | ❌ FAILED |'); // Project D has no coverage data - expect(comment).toContain('| project-d | All | Defined | No Data | ❌ FAILED |'); + expect(comment).toContain('| project-d | lines | 90% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | statements | 90% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | functions | 90% | No Data | ❌ FAILED |'); + expect(comment).toContain('| | branches | 90% | No Data | ❌ FAILED |'); // Overall status is failed with multiple projects expect(comment).toContain('### Overall Status: ❌ FAILED (2 projects failing)'); diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index 188b802d..70e13e64 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -147,7 +147,27 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st if (result.status === 'SKIPPED') { comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; } else if (result.actual === null) { - comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; + // Show individual thresholds even when coverage data is missing + const metrics = ['lines', 'statements', 'functions', 'branches']; + let hasAnyThreshold = false; + + metrics.forEach((metric, index) => { + // Skip metrics that don't have a threshold + if (!result.thresholds[metric]) return; + + hasAnyThreshold = true; + const threshold = result.thresholds[metric]; + + // Only include project name in the first row for this project + const projectCell = index === 0 ? result.project : ''; + + comment += `| ${projectCell} | ${metric} | ${threshold}% | No Data | ❌ FAILED |\n`; + }); + + // Fallback if no specific thresholds are defined + if (!hasAnyThreshold) { + comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; + } } else { const metrics = ['lines', 'statements', 'functions', 'branches']; metrics.forEach((metric, index) => { From 248695f1eebeadf1f19637bcc0f078ab7a8d13f7 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 06:44:18 +0200 Subject: [PATCH 21/27] PFM-TASK-6308 refactor: specify coverage directory for improved reporting in run-many script --- tools/scripts/run-many/run-many.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index 332448fb..d6182b88 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -72,7 +72,7 @@ function main() { if (coverageEnabled && target === 'test') { core.info('Coverage gate is enabled'); // Add coverage reporters for HTML, JSON, and JUnit output - cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; + cmd += ' --coverage --coverageDirectory=./coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; } if (target.includes('e2e')) { From 5758e1757898ee0a62cf5cbf35e13adbce5349b7 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 06:54:36 +0200 Subject: [PATCH 22/27] PFM-TASK-6308 refactor: optimize job configuration for code coverage in CI workflow --- .github/workflows/fe-code-quality.yml | 4 ++-- tools/scripts/run-many/run-many.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 2e7db0dc..c7ba9994 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -12,10 +12,10 @@ jobs: strategy: matrix: target: [ 'test' ] - jobIndex: [ 1, 2, 3, 4 ] + jobIndex: [ 1 ] fail-fast: false # Ensure all jobs run even if one fails env: - jobCount: 4 + jobCount: 1 HAS_COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS != '' }} steps: - uses: actions/checkout@v4 diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index d6182b88..88874d56 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -66,7 +66,7 @@ function main() { // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; - let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + let cmd = `${runManyProjectsCmd} --parallel=true --prod`; // Add coverage flag if enabled and target is test if (coverageEnabled && target === 'test') { From 6773c6149ffa9e8a89712e8de32f9891142cdda2 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 07:24:24 +0200 Subject: [PATCH 23/27] PFM-TASK-6308 refactor: enhance coverage evaluation logic and add debugging features --- .github/workflows/fe-code-quality.yml | 5 +- .../run-many/coverage-evaluator.spec.ts | 168 ++++++--- tools/scripts/run-many/coverage-evaluator.ts | 319 ++++++++++++++---- tools/scripts/run-many/run-many.ts | 8 +- 4 files changed, 394 insertions(+), 106 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index c7ba9994..a0489ebb 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -70,7 +70,10 @@ jobs: uses: actions/upload-artifact@v4 with: name: coverage-reports - path: coverage/ + path: | + coverage/ + coverage-report.txt + coverage-summary-debug.json retention-days: 7 - name: Set Artifact URL diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index 09564804..87963bab 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -9,9 +9,12 @@ jest.mock('fs', () => ({ existsSync: jest.fn(), readFileSync: jest.fn(), writeFileSync: jest.fn(), + readdirSync: jest.fn(), + statSync: jest.fn(), })); jest.mock('path', () => ({ - resolve: jest.fn((_, p) => p), + resolve: jest.fn((...args) => args.join('/')), + join: jest.fn((...args) => args.join('/')), })); jest.mock('@actions/core', () => ({ info: jest.fn(), @@ -32,6 +35,14 @@ describe('coverage-evaluator', () => { global: { lines: 80, statements: 80, functions: 75, branches: 70 }, projects: {} }); + + // Setup default mocks for file system operations + (fs.existsSync as jest.Mock).mockReturnValue(false); + (fs.readdirSync as jest.Mock).mockReturnValue([]); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => false }); + (fs.writeFileSync as jest.Mock).mockReturnValue(undefined); + (fs.readFileSync as jest.Mock).mockReturnValue('{}'); + (path.resolve as jest.Mock).mockImplementation((...args) => args.join('/')); }); afterEach(() => { @@ -46,39 +57,51 @@ describe('coverage-evaluator', () => { expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('No coverage thresholds defined, skipping evaluation'); + expect(fs.writeFileSync).not.toHaveBeenCalled(); }); it('should skip projects with null thresholds and not count them as failures', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue(null); + // Mock coverage directory exists but is empty + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + return filePath.includes('coverage') && !filePath.includes('coverage-summary.json'); + }); + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a'); - expect(fs.writeFileSync).toHaveBeenCalled(); - // Verify that the comment indicates the project was skipped - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; - expect(comment).toContain('⏩ SKIPPED'); + // Should write coverage report with skipped project + expect(fs.writeFileSync).toHaveBeenCalledWith( + expect.stringContaining('coverage-report.txt'), + expect.stringContaining('⏩ SKIPPED') + ); }); it('should count as one failure when coverage report is missing', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; mockGetProjectThresholds.mockReturnValue({ lines: 80, statements: 75 }); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(false); + // Mock coverage directory exists but no coverage files + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + return filePath.includes('coverage') && !filePath.includes('coverage-summary.json'); + }); + (fs.readdirSync as jest.Mock).mockReturnValue([]); // Empty directory const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); expect(result).toBe(1); - expect(core.warning).toHaveBeenCalledWith('No coverage report found for project-a at coverage/project-a/coverage-summary.json'); + expect(core.warning).toHaveBeenCalledWith('No coverage data found for project-a in any location'); // Verify that the comment shows individual thresholds with "No Data" - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; expect(comment).toContain('| project-a | lines | 80% | No Data | ❌ FAILED |'); expect(comment).toContain('| | statements | 75% | No Data | ❌ FAILED |'); expect(comment).toContain('⚠️ WARNING (1 project failing)'); @@ -93,8 +116,15 @@ describe('coverage-evaluator', () => { branches: 70 }); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Mock coverage directory and files exist + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + return filePath.includes('coverage-summary.json'); // Coverage file exists + }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync.mockReturnValue(JSON.stringify({ @@ -112,8 +142,11 @@ describe('coverage-evaluator', () => { expect(core.error).toHaveBeenCalledWith(expect.stringContaining('Project project-a failed coverage thresholds')); // Verify that the comment shows the failed metrics with correct values - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; expect(comment).toContain('| project-a | lines | 80% | 75.00% | ❌ FAILED |'); expect(comment).toContain('| | statements | 80% | 75.00% | ❌ FAILED |'); expect(comment).toContain('| | functions | 75% | 70.00% | ❌ FAILED |'); @@ -131,8 +164,15 @@ describe('coverage-evaluator', () => { branches: 70 }); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Mock coverage directory and files exist + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + return filePath.includes('coverage-summary.json'); // Coverage file exists + }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync.mockReturnValue(JSON.stringify({ @@ -150,8 +190,11 @@ describe('coverage-evaluator', () => { expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); // Verify that the comment shows passing status with correct values - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); expect(comment).toContain('| | statements | 80% | 85.00% | ✅ PASSED |'); expect(comment).toContain('| | functions | 75% | 80.00% | ✅ PASSED |'); @@ -168,8 +211,15 @@ describe('coverage-evaluator', () => { branches: 70 }); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Mock coverage directory and files exist + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + return filePath.includes('coverage-summary.json'); // Coverage file exists + }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync.mockImplementation(() => { @@ -179,11 +229,14 @@ describe('coverage-evaluator', () => { const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); expect(result).toBe(1); - expect(core.error).toHaveBeenCalledWith('Error processing coverage for project-a: Test error'); + expect(core.error).toHaveBeenCalledWith('Error parsing coverage file for project-a: Test error'); // Verify that the comment shows individual thresholds with "No Data" - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; expect(comment).toContain('| project-a | lines | 80% | No Data | ❌ FAILED |'); expect(comment).toContain('| | statements | 80% | No Data | ❌ FAILED |'); expect(comment).toContain('| | functions | 75% | No Data | ❌ FAILED |'); @@ -199,8 +252,15 @@ describe('coverage-evaluator', () => { functions: 75 }); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Mock coverage directory and files exist + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + return filePath.includes('coverage-summary.json'); // Coverage file exists + }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync.mockReturnValue(JSON.stringify({ @@ -217,8 +277,11 @@ describe('coverage-evaluator', () => { expect(result).toBe(0); // Verify that only the defined thresholds are in the comment - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); expect(comment).toContain('| | functions | 75% | 80.00% | ✅ PASSED |'); expect(comment).not.toContain('| | statements |'); @@ -230,8 +293,15 @@ describe('coverage-evaluator', () => { // Return an empty object instead of null mockGetProjectThresholds.mockReturnValue({}); - const mockExistsSync = fs.existsSync as jest.Mock; - mockExistsSync.mockReturnValue(true); + // Mock coverage directory and files exist + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + return filePath.includes('coverage-summary.json'); // Coverage file exists + }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync.mockReturnValue(JSON.stringify({ @@ -277,15 +347,22 @@ describe('coverage-evaluator', () => { branches: 90 }); - const mockExistsSync = fs.existsSync as jest.Mock; - // Make existsSync return true for project-a and project-c, but false for project-d - mockExistsSync.mockImplementation((path) => { - if (path.includes('project-d')) { - return false; // No coverage file for project-d + // Mock coverage directory exists + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { + return true; // Coverage directory exists + } + // Coverage files exist for project-a and project-c + if (filePath.includes('project-a') || filePath.includes('project-c')) { + return filePath.includes('coverage-summary.json'); } - return true; + // No coverage files for project-d + return false; }); + (fs.readdirSync as jest.Mock).mockReturnValue(['project-a', 'project-c']); + (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); + const mockReadFileSync = fs.readFileSync as jest.Mock; mockReadFileSync .mockReturnValueOnce(JSON.stringify({ @@ -304,7 +381,6 @@ describe('coverage-evaluator', () => { branches: { pct: 55 } } })); - // No need for third mock since we're making project-d file not exist process.env.COVERAGE_ARTIFACT_URL = 'https://example.com/artifact'; @@ -317,8 +393,11 @@ describe('coverage-evaluator', () => { expect(result).toBe(2); // Verify that the comment shows the correct status for each project - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - const comment = writeFileSyncMock.mock.calls[0][1]; + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; // Project A passes expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); @@ -348,11 +427,14 @@ describe('coverage-evaluator', () => { it('should generate an empty report when no projects are affected', () => { generateEmptyCoverageReport(); - const writeFileSyncMock = fs.writeFileSync as jest.Mock; - expect(writeFileSyncMock).toHaveBeenCalledTimes(1); + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + expect(writeFileCalls.length).toBeGreaterThan(0); + + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); - const [filePath, content] = writeFileSyncMock.mock.calls[0]; - expect(filePath).toBe('coverage-report.txt'); + const [filePath, content] = coverageReportCall; + expect(filePath).toContain('coverage-report.txt'); expect(content).toContain('## Test Coverage Results'); expect(content).toContain('No projects were affected by this change that require coverage evaluation'); diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index 70e13e64..22d18eb7 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -22,6 +22,159 @@ interface ProjectCoverageResult { status: 'PASSED' | 'FAILED' | 'SKIPPED'; } +/** + * Tries to find coverage summary file in common locations + */ +function findCoverageSummaryFile(project: string): string | null { + const possiblePaths = [ + // Standard Nx project structure + `coverage/${project}/coverage-summary.json`, + // Alternative coverage directory structure + `coverage/coverage-summary.json`, + // Project-specific coverage in different patterns + `coverage/apps/${project}/coverage-summary.json`, + `coverage/libs/${project}/coverage-summary.json`, + // Jest default location for the project + `coverage/lcov-report/coverage-summary.json`, + // Project-specific coverage in project directory + `apps/${project}/coverage/coverage-summary.json`, + `libs/${project}/coverage/coverage-summary.json`, + // Nx workspace structure variations + `dist/coverage/${project}/coverage-summary.json`, + `tmp/coverage/${project}/coverage-summary.json` + ]; + + for (const possiblePath of possiblePaths) { + const fullPath = path.resolve(process.cwd(), possiblePath); + if (fs.existsSync(fullPath)) { + core.info(`Found coverage file for ${project} at: ${fullPath}`); + return fullPath; + } + } + + return null; +} + +/** + * Tries to read coverage from a global coverage file that might contain multiple projects + */ +function tryReadFromGlobalCoverage(project: string): CoverageSummary | null { + const globalCoveragePath = path.resolve(process.cwd(), 'coverage/coverage-summary.json'); + + if (!fs.existsSync(globalCoveragePath)) { + return null; + } + + try { + const globalCoverageData = JSON.parse(fs.readFileSync(globalCoveragePath, 'utf8')); + + // Look for project-specific data in the global file + const projectPatterns = [ + project, + `apps/${project}`, + `libs/${project}`, + `src/app/${project}`, + `projects/${project}` + ]; + + for (const pattern of projectPatterns) { + if (globalCoverageData[pattern]) { + core.info(`Found coverage data for ${project} under key '${pattern}' in global coverage file`); + return globalCoverageData[pattern]; + } + } + + // Try to find any key that contains the project name + const keys = Object.keys(globalCoverageData); + for (const key of keys) { + if (key.includes(project) && globalCoverageData[key] && + globalCoverageData[key].lines && globalCoverageData[key].statements) { + core.info(`Found coverage data for ${project} under key '${key}' in global coverage file`); + return globalCoverageData[key]; + } + } + + return null; + } catch (error) { + core.warning(`Error reading global coverage file: ${error.message}`); + return null; + } +} + +/** + * Lists all files in coverage directory for debugging + */ +function debugCoverageDirectory(): void { + const coverageDir = path.resolve(process.cwd(), 'coverage'); + + if (!fs.existsSync(coverageDir)) { + core.warning('Coverage directory does not exist'); + return; + } + + core.info('Coverage directory contents:'); + try { + const listDirectory = (dir: string, depth = 0): void => { + if (depth > 3) return; // Prevent infinite recursion + + const items = fs.readdirSync(dir); + items.forEach(item => { + const fullPath = path.join(dir, item); + const stats = fs.statSync(fullPath); + const indent = ' '.repeat(depth); + + if (stats.isDirectory()) { + core.info(`${indent}📁 ${item}/`); + listDirectory(fullPath, depth + 1); + } else { + core.info(`${indent}📄 ${item}`); + } + }); + }; + + listDirectory(coverageDir); + } catch (error) { + core.warning(`Error listing coverage directory: ${error.message}`); + } +} + +/** + * Tries to extract coverage data from a coverage file, handling different formats + */ +function extractCoverageData(filePath: string, project: string): CoverageSummary | null { + try { + const coverageData = JSON.parse(fs.readFileSync(filePath, 'utf8')); + + // Try different possible structures + if (coverageData.total) { + // Standard jest coverage format + return coverageData.total as CoverageSummary; + } else if (coverageData[project]) { + // Project-specific coverage in the same file + return coverageData[project] as CoverageSummary; + } else if (coverageData.lines && coverageData.statements && coverageData.functions && coverageData.branches) { + // Direct coverage data + return coverageData as CoverageSummary; + } else { + // Try to find any object with coverage data + const keys = Object.keys(coverageData); + for (const key of keys) { + const data = coverageData[key]; + if (data && data.lines && data.statements && data.functions && data.branches) { + core.info(`Found coverage data under key '${key}' for project ${project}`); + return data as CoverageSummary; + } + } + } + + core.warning(`Unrecognized coverage file format for ${project}. Keys found: ${Object.keys(coverageData).join(', ')}`); + return null; + } catch (error) { + core.error(`Error parsing coverage file for ${project}: ${error.message}`); + return null; + } +} + /** * Evaluates coverage for all projects against their thresholds */ @@ -34,7 +187,10 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig let failedProjectsCount = 0; const coverageResults: ProjectCoverageResult[] = []; - core.info(`Evaluating coverage for ${projects.length} projects`); + core.info(`Evaluating coverage for ${projects.length} projects: ${projects.join(', ')}`); + + // Debug: List coverage directory contents + debugCoverageDirectory(); for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); @@ -51,10 +207,30 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig continue; } - const coveragePath = path.resolve(process.cwd(), `coverage/${project}/coverage-summary.json`); + // Try to find coverage file in various locations + const coveragePath = findCoverageSummaryFile(project); + let summary: CoverageSummary | null = null; + + if (coveragePath) { + summary = extractCoverageData(coveragePath, project); + } + + // If we didn't find project-specific coverage, try global coverage file + if (!summary) { + core.info(`Trying to find coverage data for ${project} in global coverage file`); + summary = tryReadFromGlobalCoverage(project); + } + + if (!summary) { + core.warning(`No coverage data found for ${project} in any location`); + + // Try to list what files exist for this project specifically + const projectSpecificDir = path.resolve(process.cwd(), `coverage/${project}`); + if (fs.existsSync(projectSpecificDir)) { + const files = fs.readdirSync(projectSpecificDir); + core.info(`Files in coverage/${project}/: ${files.join(', ')}`); + } - if (!fs.existsSync(coveragePath)) { - core.warning(`No coverage report found for ${project} at ${coveragePath}`); coverageResults.push({ project, thresholds: projectThresholds, @@ -65,62 +241,51 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig continue; } - try { - const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8')); - const summary = coverageData.total as CoverageSummary; + // Log the actual coverage data found + core.info(`Coverage data for ${project}: lines=${summary.lines.pct}%, statements=${summary.statements.pct}%, functions=${summary.functions.pct}%, branches=${summary.branches.pct}%`); - let projectPassed = true; - const failedMetrics: string[] = []; + let projectPassed = true; + const failedMetrics: string[] = []; - // Check each metric if threshold is defined - if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) { - projectPassed = false; - failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`); - } - - if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) { - projectPassed = false; - failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`); - } + // Check each metric if threshold is defined + if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) { + projectPassed = false; + failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`); + } - if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) { - projectPassed = false; - failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`); - } + if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) { + projectPassed = false; + failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`); + } - if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) { - projectPassed = false; - failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`); - } + if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) { + projectPassed = false; + failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`); + } - if (!projectPassed) { - core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`); - failedProjectsCount++; - } else { - core.info(`Project ${project} passed all coverage thresholds`); - } + if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) { + projectPassed = false; + failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`); + } - coverageResults.push({ - project, - thresholds: projectThresholds, - actual: { - lines: summary.lines.pct, - statements: summary.statements.pct, - functions: summary.functions.pct, - branches: summary.branches.pct - }, - status: projectPassed ? 'PASSED' : 'FAILED' - }); - } catch (error) { - core.error(`Error processing coverage for ${project}: ${error.message}`); - coverageResults.push({ - project, - thresholds: projectThresholds, - actual: null, - status: 'FAILED' - }); + if (!projectPassed) { + core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`); failedProjectsCount++; + } else { + core.info(`Project ${project} passed all coverage thresholds`); } + + coverageResults.push({ + project, + thresholds: projectThresholds, + actual: { + lines: summary.lines.pct, + statements: summary.statements.pct, + functions: summary.functions.pct, + branches: summary.branches.pct + }, + status: projectPassed ? 'PASSED' : 'FAILED' + }); } // Post results to PR comment @@ -147,19 +312,21 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st if (result.status === 'SKIPPED') { comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; } else if (result.actual === null) { - // Show individual thresholds even when coverage data is missing + // Show individual thresholds when coverage data is missing const metrics = ['lines', 'statements', 'functions', 'branches']; let hasAnyThreshold = false; + let firstRow = true; - metrics.forEach((metric, index) => { + metrics.forEach((metric) => { // Skip metrics that don't have a threshold - if (!result.thresholds[metric]) return; + if (!result.thresholds || !result.thresholds[metric]) return; hasAnyThreshold = true; const threshold = result.thresholds[metric]; // Only include project name in the first row for this project - const projectCell = index === 0 ? result.project : ''; + const projectCell = firstRow ? result.project : ''; + firstRow = false; comment += `| ${projectCell} | ${metric} | ${threshold}% | No Data | ❌ FAILED |\n`; }); @@ -169,20 +336,29 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`; } } else { + // Show actual coverage data with results const metrics = ['lines', 'statements', 'functions', 'branches']; - metrics.forEach((metric, index) => { + let firstRow = true; + + metrics.forEach((metric) => { // Skip metrics that don't have a threshold - if (!result.thresholds[metric]) return; + if (!result.thresholds || !result.thresholds[metric]) return; const threshold = result.thresholds[metric]; const actual = result.actual[metric].toFixed(2); - const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED'; + const status = parseFloat(actual) >= threshold ? '✅ PASSED' : '❌ FAILED'; // Only include project name in the first row for this project - const projectCell = index === 0 ? result.project : ''; + const projectCell = firstRow ? result.project : ''; + firstRow = false; comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; }); + + // Handle case where project has actual data but no thresholds defined + if (Object.keys(result.thresholds || {}).length === 0) { + comment += `| ${result.project} | All | No thresholds | ${result.actual.lines.toFixed(2)}% (lines) | ⏩ SKIPPED |\n`; + } } }); @@ -220,6 +396,29 @@ function postCoverageComment(results: ProjectCoverageResult[], failedProjectsCou fs.writeFileSync(gitHubCommentsFile, comment); core.info('Coverage results saved for PR comment'); + + // Also create a simple summary for debugging + createCoverageSummaryFile(results); +} + +/** + * Creates a simple coverage summary file for debugging and artifacts + */ +function createCoverageSummaryFile(results: ProjectCoverageResult[]): void { + const summaryData = { + timestamp: new Date().toISOString(), + results: results.map(result => ({ + project: result.project, + status: result.status, + thresholds: result.thresholds, + actual: result.actual + })) + }; + + const summaryFile = path.resolve(process.cwd(), 'coverage-summary-debug.json'); + fs.writeFileSync(summaryFile, JSON.stringify(summaryData, null, 2)); + + core.info(`Coverage summary debug file created at: ${summaryFile}`); } /** diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index 88874d56..d9e215b2 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -66,13 +66,17 @@ function main() { // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; - let cmd = `${runManyProjectsCmd} --parallel=true --prod`; + + // Disable parallel execution when coverage is enabled to avoid conflicts + const parallelFlag = (coverageEnabled && target === 'test') ? '--parallel=false' : '--parallel=true'; + let cmd = `${runManyProjectsCmd} ${parallelFlag} --prod`; // Add coverage flag if enabled and target is test if (coverageEnabled && target === 'test') { core.info('Coverage gate is enabled'); // Add coverage reporters for HTML, JSON, and JUnit output - cmd += ' --coverage --coverageDirectory=./coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; + // Note: Using individual project coverage directories + cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; } if (target.includes('e2e')) { From 51346e24b13cbfcac28c14b479895a24bf2eec0d Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 08:00:26 +0200 Subject: [PATCH 24/27] PFM-TASK-6308 refactor: expand job matrix for code coverage to improve parallel execution --- .github/workflows/fe-code-quality.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index a0489ebb..69334996 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -12,10 +12,10 @@ jobs: strategy: matrix: target: [ 'test' ] - jobIndex: [ 1 ] + jobIndex: [ 1, 2, 3, 4 ] fail-fast: false # Ensure all jobs run even if one fails env: - jobCount: 1 + jobCount: 4 HAS_COVERAGE_THRESHOLDS: ${{ secrets.COVERAGE_THRESHOLDS != '' }} steps: - uses: actions/checkout@v4 From bf564609c277ae41d5b546c5dec9bf4795986f85 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 12:37:21 +0200 Subject: [PATCH 25/27] PFM-TASK-6308 refactor: implement test failure report generation and placeholder coverage report --- .github/workflows/fe-code-quality.yml | 25 ++++++ .../run-many/coverage-evaluator.spec.ts | 44 +++++++++++ tools/scripts/run-many/coverage-evaluator.ts | 35 +++++++++ tools/scripts/run-many/run-many.spec.ts | 76 +++++++++++-------- tools/scripts/run-many/run-many.ts | 67 ++++++++++++---- 5 files changed, 198 insertions(+), 49 deletions(-) diff --git a/.github/workflows/fe-code-quality.yml b/.github/workflows/fe-code-quality.yml index 69334996..71ebad20 100644 --- a/.github/workflows/fe-code-quality.yml +++ b/.github/workflows/fe-code-quality.yml @@ -40,6 +40,15 @@ jobs: - name: Linter run: npx nx affected --target=lint --parallel --configuration=dev --base=origin/${{ github.event.pull_request.base.ref }} + # Ensure coverage-report.txt exists before running tests (for job 1 only) + - name: Initialize Coverage Report + if: matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' + run: | + mkdir -p coverage + echo "## Test Coverage Results" > coverage-report.txt + echo "" >> coverage-report.txt + echo "⏳ Coverage evaluation is in progress..." >> coverage-report.txt + - name: Unit Tests id: unit-tests continue-on-error: true # Allow the step to complete even if it fails @@ -65,6 +74,22 @@ jobs: echo "multiple_failures=false" >> $GITHUB_OUTPUT fi + # Ensure coverage-report.txt still exists after tests (fallback) + - name: Ensure Coverage Report Exists + if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' + run: | + if [ ! -f coverage-report.txt ]; then + echo "## Test Coverage Results" > coverage-report.txt + echo "" >> coverage-report.txt + if [[ "${{ steps.unit-tests.outcome }}" == "failure" ]]; then + echo "❌ **Test execution failed**" >> coverage-report.txt + echo "" >> coverage-report.txt + echo "> Please check the test logs above for detailed error information." >> coverage-report.txt + else + echo "⏩ No coverage data available for this change." >> coverage-report.txt + fi + fi + - name: Upload Coverage Reports if: always() && matrix.jobIndex == 1 && env.HAS_COVERAGE_THRESHOLDS == 'true' uses: actions/upload-artifact@v4 diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index 87963bab..79121dc0 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -441,4 +441,48 @@ describe('coverage-evaluator', () => { expect(core.info).toHaveBeenCalledWith('Empty coverage report generated (no affected projects)'); }); }); + + describe('generateTestFailureReport', () => { + it('should generate a test failure report with project names', () => { + const { generateTestFailureReport } = require('./coverage-evaluator'); + + generateTestFailureReport(['project-a', 'project-b']); + + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + expect(writeFileCalls.length).toBeGreaterThan(0); + + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const [filePath, content] = coverageReportCall; + expect(filePath).toContain('coverage-report.txt'); + expect(content).toContain('## Test Coverage Results'); + expect(content).toContain('Tests failed to execute'); + expect(content).toContain('project-a, project-b'); + expect(content).toContain('Overall Status: ❌ FAILED (Test execution failed)'); + + expect(core.info).toHaveBeenCalledWith('Test failure report generated for PR comment'); + }); + + it('should generate a test failure report with no specific projects', () => { + const { generateTestFailureReport } = require('./coverage-evaluator'); + + generateTestFailureReport([]); + + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + expect(writeFileCalls.length).toBeGreaterThan(0); + + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const [filePath, content] = coverageReportCall; + expect(filePath).toContain('coverage-report.txt'); + expect(content).toContain('## Test Coverage Results'); + expect(content).toContain('Tests failed to execute'); + expect(content).toContain('affected projects'); + expect(content).toContain('Overall Status: ❌ FAILED (Test execution failed)'); + + expect(core.info).toHaveBeenCalledWith('Test failure report generated for PR comment'); + }); + }); }); diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index 22d18eb7..aa500abe 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -432,3 +432,38 @@ export function generateEmptyCoverageReport(): void { core.info('Empty coverage report generated (no affected projects)'); } + +/** + * Generates a placeholder coverage report when job 1 has no projects but other jobs do + */ +export function generatePlaceholderCoverageReport(): void { + const comment = '## Test Coverage Results\n\n⏳ Coverage evaluation is in progress on other parallel jobs...\n\n' + + '> This report will be updated once all test jobs complete.\n'; + + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Placeholder coverage report generated (projects running in other jobs)'); +} + +/** + * Generates a coverage report when tests fail to execute + */ +export function generateTestFailureReport(projects: string[]): void { + const projectsList = projects.length > 0 ? projects.join(', ') : 'affected projects'; + const comment = `## Test Coverage Results + +❌ **Tests failed to execute** for: ${projectsList} + +The unit tests failed to run, likely due to compilation errors or configuration issues. Coverage evaluation cannot be performed until the tests can execute successfully. + +### Overall Status: ❌ FAILED (Test execution failed) + +> Note: Fix the test execution issues before coverage can be evaluated. +`; + + const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt'); + fs.writeFileSync(gitHubCommentsFile, comment); + + core.info('Test failure report generated for PR comment'); +} diff --git a/tools/scripts/run-many/run-many.spec.ts b/tools/scripts/run-many/run-many.spec.ts index 2a549a34..827c34bf 100644 --- a/tools/scripts/run-many/run-many.spec.ts +++ b/tools/scripts/run-many/run-many.spec.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { getAffectedProjects } from './affected-projects'; import { getCoverageThresholds } from './threshold-handler'; -import { evaluateCoverage, generateEmptyCoverageReport } from './coverage-evaluator'; +import { evaluateCoverage, generateEmptyCoverageReport, generateTestFailureReport } from './coverage-evaluator'; // Define interfaces for custom error types we'll use interface CommandError extends Error { @@ -34,6 +34,7 @@ jest.mock('./threshold-handler', () => ({ jest.mock('./coverage-evaluator', () => ({ evaluateCoverage: jest.fn(), generateEmptyCoverageReport: jest.fn(), + generateTestFailureReport: jest.fn(), })); jest.mock('fs', () => ({ existsSync: jest.fn(), @@ -43,16 +44,11 @@ jest.mock('path', () => ({ resolve: jest.fn((base, path) => `${base}/${path}`), })); -// Import the module under test -// Note: In a real test you would import directly, but for the sake of this exercise -// we'll simulate the behavior of the module -import * as runManyModule from './run-many'; - describe('run-many', () => { const originalEnv = process.env; const originalExit = process.exit; const originalArgv = process.argv; - let mockExit; + let mockExit: jest.Mock; beforeEach(() => { jest.clearAllMocks(); @@ -77,7 +73,7 @@ describe('run-many', () => { mockExecSync.mockReturnValue('command output'); // Directly call the runCommand function - runCommand('test command'); + const result = runCommand('test command'); // Verify it was called with the right args expect(mockExecSync).toHaveBeenCalledWith('test command', { @@ -87,6 +83,7 @@ describe('run-many', () => { }); expect(core.info).toHaveBeenCalledWith('Running > test command'); expect(core.info).toHaveBeenCalledWith('command output'); + expect(result).toBe(true); }); it('should handle command errors', () => { @@ -99,11 +96,12 @@ describe('run-many', () => { }); // Directly call the runCommand function - runCommand('test command'); + const result = runCommand('test command'); expect(core.info).toHaveBeenCalledWith('stdout output'); expect(core.error).toHaveBeenCalledWith('stderr output'); expect(core.setFailed).toHaveBeenCalledWith(error); + expect(result).toBe(false); }); it('should handle timeout errors', () => { @@ -117,12 +115,13 @@ describe('run-many', () => { }); // Directly call the runCommand function - runCommand('test command'); + const result = runCommand('test command'); expect(core.error).toHaveBeenCalledWith('Timed out'); expect(core.info).toHaveBeenCalledWith('stdout output'); expect(core.error).toHaveBeenCalledWith('stderr output'); expect(core.setFailed).toHaveBeenCalledWith(error); + expect(result).toBe(false); }); it('should handle buffer exceeded errors', () => { @@ -136,12 +135,13 @@ describe('run-many', () => { }); // Directly call the runCommand function - runCommand('test command'); + const result = runCommand('test command'); expect(core.error).toHaveBeenCalledWith('Buffer exceeded'); expect(core.info).toHaveBeenCalledWith('stdout output'); expect(core.error).toHaveBeenCalledWith('stderr output'); expect(core.setFailed).toHaveBeenCalledWith(error); + expect(result).toBe(false); }); }); @@ -160,7 +160,7 @@ describe('run-many', () => { // Should run nx run-many without coverage flags expect(mockExecSync).toHaveBeenCalledWith( - 'npx nx run-many --targets=test --projects="project-a,project-b" --parallel=false --prod', + 'npx nx run-many --targets=test --projects="project-a,project-b" --parallel=true --prod', expect.any(Object) ); @@ -347,7 +347,7 @@ describe('run-many', () => { main(); // Should log message about no affected projects - expect(core.info).toHaveBeenCalledWith('No affected projects :)'); + expect(core.info).toHaveBeenCalledWith('No affected projects in this job'); // Should not run nx run-many expect(mockExecSync).not.toHaveBeenCalled(); @@ -376,7 +376,7 @@ describe('run-many', () => { main(); // Should log message about no affected projects - expect(core.info).toHaveBeenCalledWith('No affected projects :)'); + expect(core.info).toHaveBeenCalledWith('No affected projects in this job'); // Should not run nx run-many expect(mockExecSync).not.toHaveBeenCalled(); @@ -426,8 +426,8 @@ describe('run-many', () => { }); }); - // Helper function to simulate runCommand - function runCommand(command: string): void { + // Helper functions to simulate the actual module behavior + function runCommand(command: string): boolean { try { const mockExecSync = execSync as jest.Mock; core.info(`Running > ${command}`); @@ -437,6 +437,7 @@ describe('run-many', () => { encoding: 'utf-8' }); core.info(output.toString()); + return true; // Command succeeded } catch (error) { if (error.signal === 'SIGTERM') { core.error('Timed out'); @@ -447,10 +448,10 @@ describe('run-many', () => { core.error(error.stderr?.toString() || ''); core.error(`Error message: ${error.message}`); core.setFailed(error); + return false; // Command failed } } - // Helper function to simulate main function main(): void { const target = process.argv[2]; const jobIndex = Number(process.argv[3]); @@ -468,25 +469,23 @@ describe('run-many', () => { // Check if coverage gate is enabled const coverageEnabled = !!process.env.COVERAGE_THRESHOLDS; - if (coverageEnabled && target === 'test') { - core.info('Coverage gate is enabled'); - } // Get the affected projects const projectsString = (getAffectedProjects as jest.Mock)(target, jobIndex, jobCount, base, ref); const projects = projectsString ? projectsString.split(',') : []; - // Check if there are any affected projects (for first job only, to avoid duplicate reports) + // Check if there are any affected projects const areAffectedProjects = projects.length > 0; const isFirstJob = jobIndex === 1; // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; - let cmd = `${runManyProjectsCmd} --parallel=false --prod`; + const parallelFlag = (coverageEnabled && target === 'test') ? '--parallel=false' : '--parallel=true'; + let cmd = `${runManyProjectsCmd} ${parallelFlag} --prod`; // Add coverage flag if enabled and target is test if (coverageEnabled && target === 'test') { - // Add coverage reporters for HTML, JSON, and JUnit output + core.info('Coverage gate is enabled'); cmd += ' --coverage --coverageReporters=json,lcov,text,clover,html,json-summary --reporters=default,jest-junit'; } @@ -496,23 +495,34 @@ describe('run-many', () => { if (areAffectedProjects) { // Run the command - runCommand(cmd); + const commandSucceeded = runCommand(cmd); - // Evaluate coverage if enabled and target is test + // Always evaluate coverage or generate report if enabled and target is test if (coverageEnabled && target === 'test') { const thresholds = (getCoverageThresholds as jest.Mock)(); - const failedProjectsCount = (evaluateCoverage as jest.Mock)(projects, thresholds); - - if (failedProjectsCount > 1) { - core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); - } else if (failedProjectsCount === 1) { - core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); + core.info('Coverage threshold configuration:'); + core.info(JSON.stringify(thresholds, null, 2)); + + if (commandSucceeded) { + // Command succeeded, evaluate actual coverage + const failedProjectsCount = (evaluateCoverage as jest.Mock)(projects, thresholds); + + if (failedProjectsCount > 1) { + core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); + } else if (failedProjectsCount === 1) { + core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); + } + } else { + // Command failed, generate a failure report for first job only + if (isFirstJob) { + (generateTestFailureReport as jest.Mock)(projects); + } } } } else { - core.info('No affected projects :)'); + core.info('No affected projects in this job'); - // Generate empty coverage report for first job only when coverage is enabled + // For the first job, generate an appropriate coverage report when coverage is enabled if (coverageEnabled && target === 'test' && isFirstJob) { // Ensure coverage directory exists for artifact upload const coverageDir = path.resolve(process.cwd(), 'coverage'); diff --git a/tools/scripts/run-many/run-many.ts b/tools/scripts/run-many/run-many.ts index d9e215b2..2cc1a3f4 100644 --- a/tools/scripts/run-many/run-many.ts +++ b/tools/scripts/run-many/run-many.ts @@ -4,19 +4,21 @@ import * as core from '@actions/core'; import * as fs from 'fs'; import * as path from 'path'; import { getCoverageThresholds } from './threshold-handler'; -import { evaluateCoverage, generateEmptyCoverageReport } from './coverage-evaluator'; +import { evaluateCoverage, generateEmptyCoverageReport, generateTestFailureReport, generatePlaceholderCoverageReport } from './coverage-evaluator'; +import { Utils } from '../artifacts/utils'; function getE2ECommand(command: string, base: string): string { command = command.concat(` -c ci --base=${base} --verbose`); return command; } -function runCommand(command: string): void { +function runCommand(command: string): boolean { core.info(`Running > ${command}`); try { const output = execSync(command, { stdio: 'pipe', maxBuffer: 1024 * 1024 * 1024, encoding: 'utf-8' }); // 1GB core.info(output.toString()) + return true; // Command succeeded } catch (error) { if (error.signal === 'SIGTERM') { core.error('Timed out'); @@ -29,6 +31,7 @@ function runCommand(command: string): void { core.error(`Error name: ${error.name}`); core.error(`Stacktrace:\n${error.stack}`); core.setFailed(error); + return false; // Command failed } } @@ -64,6 +67,23 @@ function main() { const areAffectedProjects = projects.length > 0; const isFirstJob = jobIndex === 1; + // For the first job with coverage enabled, ensure a coverage report is always created + // This handles cases where job 1 has no projects but other jobs do + if (coverageEnabled && target === 'test' && isFirstJob && !areAffectedProjects) { + // Ensure coverage directory exists for artifact upload + ensureDirectoryExists(path.resolve(process.cwd(), 'coverage')); + + // Check if there are any affected projects across all jobs + const allAffectedProjects = Utils.getAllProjects(true, base, target); + if (allAffectedProjects.length === 0) { + // No projects affected at all + generateEmptyCoverageReport(); + } else { + // Other jobs will have projects, so create a placeholder that indicates processing + generatePlaceholderCoverageReport(); + } + } + // Modified command construction const runManyProjectsCmd = `npx nx run-many --targets=${target} --projects="${projectsString}"`; @@ -84,9 +104,9 @@ function main() { } if (areAffectedProjects) { - runCommand(cmd); + const commandSucceeded = runCommand(cmd); - // Evaluate coverage if enabled and target is test + // Always evaluate coverage or generate report if enabled and target is test if (coverageEnabled && target === 'test') { const thresholds = getCoverageThresholds(); @@ -94,26 +114,41 @@ function main() { core.info('Coverage threshold configuration:'); core.info(JSON.stringify(thresholds, null, 2)); - const failedProjectsCount = evaluateCoverage(projects, thresholds); - - if (failedProjectsCount > 1) { - core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); - // Don't exit immediately - we set the failed status but continue running - } else if (failedProjectsCount === 1) { - core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); - // Continue running, with a warning + if (commandSucceeded) { + // Command succeeded, evaluate actual coverage + const failedProjectsCount = evaluateCoverage(projects, thresholds); + + if (failedProjectsCount > 1) { + core.setFailed(`Multiple projects (${failedProjectsCount}) failed to meet coverage thresholds`); + // Don't exit immediately - we set the failed status but continue running + } else if (failedProjectsCount === 1) { + core.warning('One project failed to meet coverage thresholds - this should be fixed before merging'); + // Continue running, with a warning + } + } else { + // Command failed, generate a failure report for first job only + if (isFirstJob) { + generateTestFailureReport(projects); + } } } } else { - core.info('No affected projects :)'); + core.info('No affected projects in this job'); - // Generate empty coverage report for first job only when coverage is enabled + // For the first job, generate an appropriate coverage report when coverage is enabled if (coverageEnabled && target === 'test' && isFirstJob) { // Ensure coverage directory exists for artifact upload ensureDirectoryExists(path.resolve(process.cwd(), 'coverage')); - // Generate empty report - generateEmptyCoverageReport(); + // Check if there are any affected projects across all jobs + const allAffectedProjects = Utils.getAllProjects(true, base, target); + if (allAffectedProjects.length === 0) { + // No projects affected at all + generateEmptyCoverageReport(); + } else { + // Other jobs will handle the projects, let the workflow's fallback handle the report + core.info('Other jobs will process affected projects - coverage report will be generated by workflow fallback if needed'); + } } } } From 778fe6bbe1f21d43195d19d20c955d5e13005992 Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 13:47:16 +0200 Subject: [PATCH 26/27] PFM-TASK-6308 refactor: enhance coverage report for skipped projects to include individual metrics --- .../coverage-gate-master-plan.md | 11 ++++++++-- .../run-many/coverage-evaluator.spec.ts | 20 ++++++++++++++++--- tools/scripts/run-many/coverage-evaluator.ts | 12 ++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md index 1710d9f5..1dcae7c1 100644 --- a/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md +++ b/.claudesync/reusable-gha-code-coverage-gate/coverage-gate-master-plan.md @@ -293,7 +293,10 @@ The PR comment will look like this: | | statements | 85% | 86.10% | ✅ PASSED | | | functions | 80% | 82.23% | ✅ PASSED | | | branches | 75% | 77.18% | ✅ PASSED | -| cf-components | All | N/A | N/A | ⏩ SKIPPED | +| cf-components | lines | N/A | N/A | ⏩ SKIPPED | +| | statements | N/A | N/A | ⏩ SKIPPED | +| | functions | N/A | N/A | ⏩ SKIPPED | +| | branches | N/A | N/A | ⏩ SKIPPED | | cf-utils | lines | 70% | 65.30% | ❌ FAILED | | | statements | 70% | 68.45% | ❌ FAILED | | | functions | 65% | 62.60% | ❌ FAILED | @@ -303,7 +306,11 @@ The PR comment will look like this: | | functions | 75% | 78.30% | ✅ PASSED | | | branches | 70% | 72.40% | ✅ PASSED | -### Overall Status: ❌ FAILED +### Overall Status: ❌ FAILED (1 project failing) + +> Note: The build will continue, but this project should be fixed before merging. + +📊 [View Detailed HTML Coverage Reports](https://github.com/example/repo/actions/runs/12345) ``` ## 4. Implementation Steps diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index 79121dc0..7cbf1d4a 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -74,11 +74,22 @@ describe('coverage-evaluator', () => { expect(result).toBe(0); expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a'); - // Should write coverage report with skipped project + // Should write coverage report with skipped project showing individual metrics expect(fs.writeFileSync).toHaveBeenCalledWith( expect.stringContaining('coverage-report.txt'), expect.stringContaining('⏩ SKIPPED') ); + + // Verify that the comment shows individual metrics for skipped project + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; + expect(comment).toContain('| project-a | lines | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | statements | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | functions | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | branches | N/A | N/A | ⏩ SKIPPED |'); }); it('should count as one failure when coverage report is missing', () => { @@ -402,8 +413,11 @@ describe('coverage-evaluator', () => { // Project A passes expect(comment).toContain('| project-a | lines | 80% | 85.00% | ✅ PASSED |'); - // Project B is skipped - expect(comment).toContain('| project-b | All | N/A | N/A | ⏩ SKIPPED |'); + // Project B is skipped (now shows individual metrics) + expect(comment).toContain('| project-b | lines | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | statements | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | functions | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | branches | N/A | N/A | ⏩ SKIPPED |'); // Project C fails expect(comment).toContain('| project-c | lines | 70% | 65.00% | ❌ FAILED |'); diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index aa500abe..fed0ad51 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -310,7 +310,17 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st results.forEach(result => { if (result.status === 'SKIPPED') { - comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`; + // Show individual metrics for skipped projects to maintain consistent table format + const metrics = ['lines', 'statements', 'functions', 'branches']; + let firstRow = true; + + metrics.forEach((metric) => { + // Only include project name in the first row for this project + const projectCell = firstRow ? result.project : ''; + firstRow = false; + + comment += `| ${projectCell} | ${metric} | N/A | N/A | ⏩ SKIPPED |\n`; + }); } else if (result.actual === null) { // Show individual thresholds when coverage data is missing const metrics = ['lines', 'statements', 'functions', 'branches']; From 049389a4ddf9e4872cf61f5263e0928e88c5fddf Mon Sep 17 00:00:00 2001 From: Senthanal Sirpi Manohar Date: Tue, 27 May 2025 16:04:04 +0200 Subject: [PATCH 27/27] PFM-TASK-6308 refactor: improve project skipping logic in coverage evaluation --- .../run-many/coverage-evaluator.spec.ts | 79 +++++++++++++++++-- tools/scripts/run-many/coverage-evaluator.ts | 21 ++--- tools/scripts/run-many/threshold-handler.ts | 15 ++++ 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/tools/scripts/run-many/coverage-evaluator.spec.ts b/tools/scripts/run-many/coverage-evaluator.spec.ts index 7cbf1d4a..fc467e7a 100644 --- a/tools/scripts/run-many/coverage-evaluator.spec.ts +++ b/tools/scripts/run-many/coverage-evaluator.spec.ts @@ -299,19 +299,52 @@ describe('coverage-evaluator', () => { expect(comment).not.toContain('| | branches |'); }); - it('should treat empty threshold objects as valid thresholds', () => { + it('should skip projects with empty threshold objects', () => { const mockGetProjectThresholds = getProjectThresholds as jest.Mock; // Return an empty object instead of null mockGetProjectThresholds.mockReturnValue({}); - // Mock coverage directory and files exist + // Mock coverage directory exists but is empty + (fs.existsSync as jest.Mock).mockImplementation((filePath) => { + return filePath.includes('coverage') && !filePath.includes('coverage-summary.json'); + }); + + const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + + // Should skip because no specific thresholds were set (empty object) + expect(result).toBe(0); + expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a (no thresholds defined)'); + + // Verify that the comment shows skipped status + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; + expect(comment).toContain('| project-a | lines | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | statements | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | functions | N/A | N/A | ⏩ SKIPPED |'); + expect(comment).toContain('| | branches | N/A | N/A | ⏩ SKIPPED |'); + }); + + it('should correctly handle mix of skipped, passed, and failed projects', () => { + const mockGetProjectThresholds = getProjectThresholds as jest.Mock; + mockGetProjectThresholds + .mockReturnValueOnce(null) // project-a: explicitly null (skip) + .mockReturnValueOnce({}) // project-b: empty thresholds (skip) + .mockReturnValueOnce({ lines: 80, statements: 80 }) // project-c: has thresholds + .mockReturnValueOnce({ lines: 90, statements: 90 }); // project-d: has thresholds + + // Mock coverage directory and files exist for projects that aren't skipped (fs.existsSync as jest.Mock).mockImplementation((filePath) => { if (filePath.includes('coverage') && !filePath.includes('coverage-summary.json')) { return true; // Coverage directory exists } - return filePath.includes('coverage-summary.json'); // Coverage file exists + // Coverage files exist for project-c only + return filePath.includes('project-c') && filePath.includes('coverage-summary.json'); }); - (fs.readdirSync as jest.Mock).mockReturnValue(['project-a']); + + (fs.readdirSync as jest.Mock).mockReturnValue(['project-c']); (fs.statSync as jest.Mock).mockReturnValue({ isDirectory: () => true }); const mockReadFileSync = fs.readFileSync as jest.Mock; @@ -324,11 +357,41 @@ describe('coverage-evaluator', () => { } })); - const result = evaluateCoverage(['project-a'], { global: {}, projects: {} }); + const result = evaluateCoverage(['project-a', 'project-b', 'project-c', 'project-d'], { + global: { lines: 80, statements: 80 }, + projects: {} + }); - // Should pass because no specific thresholds were set - expect(result).toBe(0); - expect(core.info).toHaveBeenCalledWith('Project project-a passed all coverage thresholds'); + // One project failed (project-d - no coverage data), two were skipped, one passed + expect(result).toBe(1); + + // Verify logs + expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-a (null thresholds)'); + expect(core.info).toHaveBeenCalledWith('Coverage evaluation skipped for project-b (no thresholds defined)'); + expect(core.info).toHaveBeenCalledWith('Project project-c passed all coverage thresholds'); + expect(core.warning).toHaveBeenCalledWith('No coverage data found for project-d in any location'); + + // Verify that the comment shows correct status for each project + const writeFileCalls = (fs.writeFileSync as jest.Mock).mock.calls; + const coverageReportCall = writeFileCalls.find(call => call[0].includes('coverage-report.txt')); + expect(coverageReportCall).toBeDefined(); + + const comment = coverageReportCall[1]; + + // project-a: skipped (null) + expect(comment).toContain('| project-a | lines | N/A | N/A | ⏩ SKIPPED |'); + + // project-b: skipped (empty) + expect(comment).toContain('| project-b | lines | N/A | N/A | ⏩ SKIPPED |'); + + // project-c: passed + expect(comment).toContain('| project-c | lines | 80% | 85.00% | ✅ PASSED |'); + + // project-d: failed (no coverage data) + expect(comment).toContain('| project-d | lines | 90% | No Data | ❌ FAILED |'); + + // Overall status should show 1 project failing (warning level) + expect(comment).toContain('### Overall Status: ⚠️ WARNING (1 project failing)'); }); it('should count multiple failures correctly', () => { diff --git a/tools/scripts/run-many/coverage-evaluator.ts b/tools/scripts/run-many/coverage-evaluator.ts index fed0ad51..4ec18f1d 100644 --- a/tools/scripts/run-many/coverage-evaluator.ts +++ b/tools/scripts/run-many/coverage-evaluator.ts @@ -175,6 +175,13 @@ function extractCoverageData(filePath: string, project: string): CoverageSummary } } +/** + * Determines if a project should be skipped based on its thresholds + */ +function shouldSkipProject(thresholds: CoverageThreshold | null): boolean { + return thresholds === null || (thresholds && Object.keys(thresholds).length === 0); +} + /** * Evaluates coverage for all projects against their thresholds */ @@ -195,12 +202,13 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig for (const project of projects) { const projectThresholds = getProjectThresholds(project, thresholds); - // Skip projects with null thresholds - if (projectThresholds === null) { - core.info(`Coverage evaluation skipped for ${project}`); + // Skip projects with null thresholds or explicitly empty thresholds + if (shouldSkipProject(projectThresholds)) { + const reason = projectThresholds === null ? 'null thresholds' : 'no thresholds defined'; + core.info(`Coverage evaluation skipped for ${project} (${reason})`); coverageResults.push({ project, - thresholds: null, + thresholds: projectThresholds, actual: null, status: 'SKIPPED' }); @@ -364,11 +372,6 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st comment += `| ${projectCell} | ${metric} | ${threshold}% | ${actual}% | ${status} |\n`; }); - - // Handle case where project has actual data but no thresholds defined - if (Object.keys(result.thresholds || {}).length === 0) { - comment += `| ${result.project} | All | No thresholds | ${result.actual.lines.toFixed(2)}% (lines) | ⏩ SKIPPED |\n`; - } } }); diff --git a/tools/scripts/run-many/threshold-handler.ts b/tools/scripts/run-many/threshold-handler.ts index fa757b63..96c8de5e 100644 --- a/tools/scripts/run-many/threshold-handler.ts +++ b/tools/scripts/run-many/threshold-handler.ts @@ -14,6 +14,21 @@ export interface ThresholdConfig { /** * Parses the COVERAGE_THRESHOLDS environment variable + * + * Configuration format: + * { + * "global": { "lines": 80, "statements": 80, "functions": 75, "branches": 70 }, + * "projects": { + * "project-a": { "lines": 90 }, // Override global thresholds + * "project-b": null, // Explicitly skip this project + * "project-c": {} // Skip this project (empty thresholds) + * } + * } + * + * Projects are skipped when: + * - Explicitly set to null in the projects configuration + * - Have empty threshold objects ({}) + * - No thresholds defined anywhere (no global, no project-specific) */ export function getCoverageThresholds(): ThresholdConfig { if (!process.env.COVERAGE_THRESHOLDS) {