From 77ed87df1076efa555ceb53d2e32dc11153c5942 Mon Sep 17 00:00:00 2001 From: yashgoyal0110 Date: Tue, 25 Aug 2026 13:37:45 +0530 Subject: [PATCH 1/2] visual-report-status-from-vitest --- .github/workflows/ci-test.yml | 2 +- .gitignore | 1 + visual-report.js | 346 ++++++++++++++++++++++++++++------ 3 files changed, 294 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 2b974185dc..7c7aa0947f 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -52,7 +52,7 @@ jobs: - name: Build and test (Ubuntu) id: test if: matrix.os == 'ubuntu-latest' - run: npm test -- --project=unit-tests + run: npm test -- --project=unit-tests --reporter=default --reporter=json --outputFile.json=./test/unit/visual/test-results.json continue-on-error: true env: CI: true diff --git a/.gitignore b/.gitignore index 501c81eaaf..9ec34200c1 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ preview/ __screenshots__/ actual-screenshots/ visual-report.html +test/unit/visual/test-results.json todo.md diff --git a/visual-report.js b/visual-report.js index 0f579dd676..f1242457b3 100644 --- a/visual-report.js +++ b/visual-report.js @@ -2,12 +2,139 @@ import fs from 'fs'; import path from 'path'; const SLASH_REGEX = /\//g; +// `notRun` means vitest never reported on the test at all, e.g. because it +// belongs to a project this CI job did not run. +const PASSED = 'passed'; +const FAILED = 'failed'; +const SKIPPED = 'skipped'; +const NOT_RUN = 'notRun'; +// Screenshot-only: the test failed before capturing this screenshot. +const NOT_CAPTURED = 'notCaptured'; + +// The assertion `visualTest()` registers inside each test's describe block. +const VISUAL_TEST_ASSERTION = 'matches expected screenshots'; + +const VITEST_STATUS = { + passed: PASSED, + failed: FAILED, + skipped: SKIPPED, + pending: SKIPPED, + todo: SKIPPED, + disabled: SKIPPED +}; + +const STATUS_LABEL = { + [PASSED]: 'PASS', + [FAILED]: 'FAIL', + [SKIPPED]: 'SKIPPED', + [NOT_RUN]: 'NOT RUN', + [NOT_CAPTURED]: 'NOT CAPTURED' +}; + +const STATUS_CLASS = { + [PASSED]: 'status-pass', + [FAILED]: 'status-fail', + [SKIPPED]: 'status-skip', + [NOT_RUN]: 'status-not-run', + [NOT_CAPTURED]: 'status-not-run' +}; + +// Mirrors `escapeName()` in test/unit/visual/visualTest.js, which turns suite +// names into the directory names under screenshots/. +function escapeName(name) { + return name.replace(SLASH_REGEX, '%2F'); +} + +function percent(value, total) { + if (total <= 0) return '0'; + const exact = (value / total) * 100; + const rounded = Math.round(exact); + // Don't let rounding read as a clean 0% or 100% when it isn't one. + if ((rounded === 0 && value > 0) || (rounded === 100 && value < total)) { + return exact.toFixed(1); + } + return String(rounded); +} + +function escapeHTML(text) { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +// Failure messages embed base64 data URLs of the images, which would balloon +// the report, so keep just the headline. +function summarizeFailure(messages) { + if (!messages || messages.length === 0) return null; + const firstLine = String(messages[0]).split('\n')[0].trim(); + if (!firstLine) return null; + return firstLine.length > 300 ? `${firstLine.slice(0, 300)}…` : firstLine; +} + +/** + * Read vitest's json reporter output, keyed by the screenshot directory each + * test corresponds to. Returns null if there are no usable results. + */ +function loadTestResults(resultsFile) { + if (!fs.existsSync(resultsFile)) { + console.warn( + `No vitest results found at ${resultsFile}. Falling back to inferring ` + + 'test results from the screenshots on disk, which cannot tell a failed ' + + 'test apart from one that never ran.' + ); + return null; + } + + let report; + try { + report = JSON.parse(fs.readFileSync(resultsFile, 'utf8')); + } catch (error) { + console.error(`Failed to read vitest results: ${resultsFile}`, error); + return null; + } + + const results = new Map(); + for (const fileResult of report.testResults || []) { + for (const assertion of fileResult.assertionResults || []) { + if (assertion.title !== VISUAL_TEST_ASSERTION) continue; + + const ancestors = assertion.ancestorTitles || []; + if (ancestors.length === 0) continue; + + // `visualTest()` saves screenshots under the escaped suite path. + const name = ancestors.map(escapeName).join('/'); + const status = VITEST_STATUS[assertion.status] || SKIPPED; + + // A test is reported once per project it runs in; a failure anywhere wins. + const existing = results.get(name); + if (existing && (existing.status === FAILED || status !== FAILED)) { + continue; + } + + results.set(name, { + status, + failure: summarizeFailure(assertion.failureMessages) + }); + } + } + + console.log( + `Loaded ${results.size} visual test result(s) from ${resultsFile}` + ); + return results; +} + async function generateVisualReport() { const expectedDir = path.join(process.cwd(), 'test/unit/visual/screenshots'); const actualDir = path.join( process.cwd(), 'test/unit/visual/actual-screenshots' ); + const resultsFile = + process.env.VISUAL_TEST_RESULTS || + path.join(process.cwd(), 'test/unit/visual/test-results.json'); const outputFile = path.join( process.cwd(), 'test/unit/visual/visual-report.html' @@ -19,6 +146,8 @@ async function generateVisualReport() { fs.mkdirSync(outputDir, { recursive: true }); } + const testResults = loadTestResults(resultsFile); + // Function to read image file and convert to data URL function imageToDataURL(filePath) { try { @@ -83,15 +212,28 @@ async function generateVisualReport() { } const testDir = path.dirname(fullPath); + // The test names vitest reports always use forward slashes. + const testName = testDir.split(path.sep).join('/'); + + // When vitest results exist they are the source of truth: a test they + // never mention never ran. Otherwise fall back to inferring from disk. + const result = testResults ? testResults.get(testName) : null; + const reportedStatus = testResults + ? result + ? result.status + : NOT_RUN + : null; const test = { - name: testDir, + name: testName, numScreenshots: metadata.numScreenshots || 0, + status: reportedStatus, + failure: result ? result.failure : null, screenshots: [] }; // Create flattened name for lookup - const flattenedName = testDir.replace(SLASH_REGEX, '-'); + const flattenedName = testName.replace(SLASH_REGEX, '-'); // Collect all screenshots for this test for (let i = 0; i < test.numScreenshots; i++) { @@ -111,17 +253,46 @@ async function generateVisualReport() { const hasActual = actualPath && fs.existsSync(actualPath); const hasDiff = fs.existsSync(diffPath); + let status; + if (reportedStatus === null) { + // No vitest results: the old disk-only heuristic. + status = hasExpected && hasActual && !hasDiff ? PASSED : FAILED; + } else if ( + reportedStatus === SKIPPED || + reportedStatus === NOT_RUN || + reportedStatus === PASSED + ) { + status = reportedStatus; + } else if (hasDiff) { + status = FAILED; + } else if (hasExpected && hasActual) { + // Another screenshot in the same test is what failed. + status = PASSED; + } else if (!hasActual) { + // The test bailed out before getting this far. + status = NOT_CAPTURED; + } else { + status = FAILED; + } + const screenshot = { index: i, expectedImage: hasExpected ? imageToDataURL(expectedPath) : null, actualImage: hasActual ? imageToDataURL(actualPath) : null, diffImage: hasDiff ? imageToDataURL(diffPath) : null, - passed: hasExpected && hasActual && !hasDiff + status, + passed: status === PASSED }; test.screenshots.push(screenshot); } + if (test.status === null) { + test.status = test.screenshots.every(s => s.status === PASSED) + ? PASSED + : FAILED; + } + // Don't add tests with no screenshots if (test.screenshots.length > 0) { testCases.push(test); @@ -141,20 +312,42 @@ async function generateVisualReport() { ); } - // Count passed/failed tests and screenshots + // Count tests and screenshots per status const totalTests = testCases.length; - let passedTests = 0; + const tests = { passed: 0, failed: 0, skipped: 0, notRun: 0 }; + const screenshots = { + passed: 0, + failed: 0, + notCaptured: 0, + skipped: 0, + notRun: 0 + }; let totalScreenshots = 0; - let passedScreenshots = 0; for (const test of testCases) { - const testPassed = test.screenshots.every(screenshot => screenshot.passed); - if (testPassed) passedTests++; + tests[test.status]++; totalScreenshots += test.screenshots.length; - passedScreenshots += test.screenshots.filter(s => s.passed).length; + for (const screenshot of test.screenshots) { + screenshots[screenshot.status]++; + } } + // Percentages only make sense against the tests that actually ran. + const executedTests = tests.passed + tests.failed; + const executedScreenshots = + screenshots.passed + screenshots.failed + screenshots.notCaptured; + + const fallbackNotice = testResults + ? '' + : `
+ No vitest results were found at ${escapeHTML(path.relative(process.cwd(), resultsFile))}, + so statuses below were inferred from the screenshots on disk. Tests that + were skipped or never ran are indistinguishable from failures in this mode. + Run the tests with --reporter=json --outputFile.json=test/unit/visual/test-results.json + to get accurate results. +
`; + // Generate HTML const html = ` @@ -223,6 +416,25 @@ async function generateVisualReport() { color: #a94442; } + .status-skip { + background-color: #fff3cd; + color: #856404; + } + + .status-not-run { + background-color: #e9ecef; + color: #555; + } + + .failure-message { + padding: 10px 15px; + background-color: #f2dede; + color: #a94442; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + border-bottom: 1px solid #ddd; + } + .screenshots { padding: 20px; } @@ -306,40 +518,70 @@ async function generateVisualReport() { border-radius: 4px; margin-top: 5px; } + + .warning-notice { + padding: 10px 15px; + background-color: #fff3cd; + color: #856404; + border-radius: 4px; + margin-bottom: 20px; + } + + code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + }

p5.js Visual Test Results

- - - + + + + +
+ ${fallbackNotice} +

Summary

Total Tests: ${totalTests}
- Passed Tests: ${passedTests} (${totalTests > 0 ? Math.round((passedTests / totalTests) * 100) : 0}%)
- Failed Tests: ${totalTests - passedTests} (${totalTests > 0 ? Math.round(((totalTests - passedTests) / totalTests) * 100) : 0}%)
+ Tests Run: ${executedTests}
+ Passed Tests: ${tests.passed} (${percent(tests.passed, executedTests)}% of tests run)
+ Failed Tests: ${tests.failed} (${percent(tests.failed, executedTests)}% of tests run)
+ Skipped Tests: ${tests.skipped}
+ Tests Not Run: ${tests.notRun}
+
Total Screenshots: ${totalScreenshots}
- Passed Screenshots: ${passedScreenshots} (${totalScreenshots > 0 ? Math.round((passedScreenshots / totalScreenshots) * 100) : 0}%)
+ Passed Screenshots: ${screenshots.passed} (${percent(screenshots.passed, executedScreenshots)}% of screenshots compared)
+ Failed Screenshots: ${screenshots.failed} (${percent(screenshots.failed, executedScreenshots)}% of screenshots compared)
+ Screenshots Not Captured: ${screenshots.notCaptured}
+ Skipped Screenshots: ${screenshots.skipped}
+ Screenshots Not Run: ${screenshots.notRun}
+
Report Generated: ${new Date().toLocaleString()}

${testCases - .map(test => { - const passed = test.screenshots.every(s => s.passed); - return ` -
+ .map( + test => ` +
- ${test.name} - ${passed ? 'PASS' : 'FAIL'} + ${escapeHTML(test.name)} + ${STATUS_LABEL[test.status]}
+ ${ + test.failure + ? `
${escapeHTML(test.failure)}
` + : '' + }
${test.screenshots .map( @@ -347,8 +589,8 @@ async function generateVisualReport() {
Screenshot #${screenshot.index + 1} - - ${screenshot.passed ? 'PASS' : 'FAIL'} + + ${STATUS_LABEL[screenshot.status]}
@@ -385,8 +627,8 @@ async function generateVisualReport() { .join('')}
- `; - }) + ` + ) .join('')}
@@ -395,33 +637,17 @@ async function generateVisualReport() { const buttons = document.querySelectorAll('.toggle-btn'); const testGroups = document.querySelectorAll('.test-group'); - document.getElementById('show-all').addEventListener('click', function() { - testGroups.forEach(el => { - el.style.display = 'block'; - }); - setActiveButton(this); - }); - - document.getElementById('show-failed').addEventListener('click', function() { - testGroups.forEach(el => { - el.style.display = el.classList.contains('test-failed') ? 'block' : 'none'; - }); - setActiveButton(this); - }); - - document.getElementById('show-passed').addEventListener('click', function() { - testGroups.forEach(el => { - el.style.display = el.classList.contains('test-passed') ? 'block' : 'none'; + buttons.forEach(button => { + button.addEventListener('click', function() { + const filter = this.dataset.filter; + testGroups.forEach(el => { + el.style.display = + filter === 'all' || el.dataset.status === filter ? 'block' : 'none'; + }); + buttons.forEach(other => other.classList.remove('active')); + this.classList.add('active'); }); - setActiveButton(this); }); - - function setActiveButton(activeButton) { - buttons.forEach(button => { - button.classList.remove('active'); - }); - activeButton.classList.add('active'); - } @@ -430,14 +656,26 @@ async function generateVisualReport() { // Write HTML to file fs.writeFileSync(outputFile, html); console.log(`Visual test report generated: ${outputFile}`); + console.log( + `${tests.passed} passed, ${tests.failed} failed, ${tests.skipped} skipped, ` + + `${tests.notRun} not run (of ${totalTests} total)` + ); return { totalTests, - passedTests, - failedTests: totalTests - passedTests, + executedTests, + passedTests: tests.passed, + failedTests: tests.failed, + skippedTests: tests.skipped, + notRunTests: tests.notRun, totalScreenshots, - passedScreenshots, - failedScreenshots: totalScreenshots - passedScreenshots, + executedScreenshots, + passedScreenshots: screenshots.passed, + failedScreenshots: screenshots.failed, + notCapturedScreenshots: screenshots.notCaptured, + skippedScreenshots: screenshots.skipped, + notRunScreenshots: screenshots.notRun, + usedTestResults: testResults !== null, reportPath: outputFile }; } From 964d68be418e3efaf4e8331cc924fc1a13f35314 Mon Sep 17 00:00:00 2001 From: yashgoyal0110 Date: Wed, 26 Aug 2026 12:37:20 +0530 Subject: [PATCH 2/2] Address review: move json reporter to vitest config, refine report output --- .github/workflows/ci-test.yml | 2 +- visual-report.js | 37 ++++++++++++++++++++++------------- vitest.config.js | 6 ++++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 7c7aa0947f..2b974185dc 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -52,7 +52,7 @@ jobs: - name: Build and test (Ubuntu) id: test if: matrix.os == 'ubuntu-latest' - run: npm test -- --project=unit-tests --reporter=default --reporter=json --outputFile.json=./test/unit/visual/test-results.json + run: npm test -- --project=unit-tests continue-on-error: true env: CI: true diff --git a/visual-report.js b/visual-report.js index f1242457b3..445b35a361 100644 --- a/visual-report.js +++ b/visual-report.js @@ -266,7 +266,7 @@ async function generateVisualReport() { } else if (hasDiff) { status = FAILED; } else if (hasExpected && hasActual) { - // Another screenshot in the same test is what failed. + // This screenshot matched; the test failed for some other reason. status = PASSED; } else if (!hasActual) { // The test bailed out before getting this far. @@ -279,7 +279,12 @@ async function generateVisualReport() { index: i, expectedImage: hasExpected ? imageToDataURL(expectedPath) : null, actualImage: hasActual ? imageToDataURL(actualPath) : null, - diffImage: hasDiff ? imageToDataURL(diffPath) : null, + // A leftover diff from an earlier run says nothing about a test + // vitest reported as passing, and a missing actual only means + // something when the test actually tried to produce one. + diffImage: + hasDiff && status === FAILED ? imageToDataURL(diffPath) : null, + showMissingActual: status === FAILED || status === NOT_CAPTURED, status, passed: status === PASSED }; @@ -335,8 +340,7 @@ async function generateVisualReport() { // Percentages only make sense against the tests that actually ran. const executedTests = tests.passed + tests.failed; - const executedScreenshots = - screenshots.passed + screenshots.failed + screenshots.notCaptured; + const executedScreenshots = screenshots.passed + screenshots.failed; const fallbackNotice = testResults ? '' @@ -344,8 +348,7 @@ async function generateVisualReport() { No vitest results were found at ${escapeHTML(path.relative(process.cwd(), resultsFile))}, so statuses below were inferred from the screenshots on disk. Tests that were skipped or never ran are indistinguishable from failures in this mode. - Run the tests with --reporter=json --outputFile.json=test/unit/visual/test-results.json - to get accurate results. + Run npm test to generate the results, then regenerate this report.
`; // Generate HTML @@ -602,14 +605,20 @@ async function generateVisualReport() { : `
No expected image found
` }
-
-
Actual
- ${ - screenshot.actualImage - ? `Actual Result` - : `
No actual image found
` - } -
+ ${ + screenshot.actualImage || screenshot.showMissingActual + ? ` +
+
Actual
+ ${ + screenshot.actualImage + ? `Actual Result` + : `
No actual image found
` + } +
+ ` + : '' + } ${ screenshot.diffImage ? ` diff --git a/vitest.config.js b/vitest.config.js index 8052bfa0fe..adce6ec546 100644 --- a/vitest.config.js +++ b/vitest.config.js @@ -10,6 +10,12 @@ const plugins = [ export default defineConfig({ test: { + // Emit machine-readable results alongside the terminal output so + // visual-report.js can tell a failed test apart from one that never ran. + reporters: ['default', 'json'], + outputFile: { + json: './test/unit/visual/test-results.json' + }, projects: [ { plugins,