Visual regression testing with Argos (#27)

In preparation of upgrading a major version of Docusaurus and other dependencies, it would be nice if we can easily identify any visual regressions without manually checking each page with each dependency change. This will upload screenshots to Argos for comparison against a reference (the main branch). If a pull request results in visual differences, it will notify us. Idea from https://docusaurus.io/blog/upgrading-frontend-dependencies-with-confidence-using-visual-regression-testing
JapanFinance · Nov 2, 2023 · dce9910 · dce9910
1 parent 4ea9620
commit dce9910
Show file tree

Hide file tree

Showing 9 changed files with 748 additions and 22 deletions.
diff --git a/.github/workflows/build-pullrequest.yml b/.github/workflows/build-pullrequest.yml
@@ -20,5 +20,15 @@ jobs:
 
       - name: Install dependencies
         run: yarn install --frozen-lockfile
-      - name: Test build website
-        run: yarn build
+
+      - name: Install Playwright browsers
+        run: yarn playwright install --with-deps chromium
+
+      - name: Build the website
+        run: yarn docusaurus build
+
+      - name: Take screenshots with Playwright
+        run: yarn playwright test
+
+      - name: Upload screenshots to Argos
+        run: yarn argos upload ./screenshots
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -31,8 +31,17 @@ jobs:
       - name: Install dependencies
         run: yarn install --frozen-lockfile
 
-      - name: Build website
-        run: yarn build
+      - name: Install Playwright browsers
+        run: yarn playwright install --with-deps chromium
+
+      - name: Build the website
+        run: yarn docusaurus build
+
+      - name: Take screenshots with Playwright
+        run: yarn playwright test
+
+      - name: Upload screenshots to Argos
+        run: yarn argos upload ./screenshots
 
       # Official GH Action does not work when deploying from sync job due to different starting git commit
       - name: Deploy gh-pages branch

diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,6 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# screenshots for visual regression testing
+/screenshots
diff --git a/package.json b/package.json
@@ -26,8 +26,12 @@
     "react-dom": "^17.0.2"
   },
   "devDependencies": {
+    "@argos-ci/cli": "^1.0.0",
+    "@argos-ci/playwright": "^1.0.1",
     "@docusaurus/module-type-aliases": "2.4.3",
+    "@playwright/test": "^1.39.0",
     "@tsconfig/docusaurus": "^2.0.2",
+    "cheerio": "^1.0.0-rc.12",
     "typescript": "^5.2.2"
   },
   "resolutions": {

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -0,0 +1,19 @@
+import {devices} from '@playwright/test';
+import type {PlaywrightTestConfig} from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  webServer: {
+    port: 3000,
+    command: 'yarn docusaurus serve',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+};
+
+export default config;
diff --git a/screenshot.css b/screenshot.css
@@ -0,0 +1,11 @@
+/* Iframes can load lazily */
+iframe,
+/* Gifs load lazily and are animated */
+img[src$='.gif'],
+
+/* Different docs last-update dates can alter layout */
+.theme-last-updated,
+/* Mermaid diagrams are rendered client-side and produce layout shifts */
+.docusaurus-mermaid-container {
+  display: none;
+}
diff --git a/screenshot.spec.ts b/screenshot.spec.ts
@@ -0,0 +1,33 @@
+import * as fs from 'fs';
+import {test} from '@playwright/test';
+import {argosScreenshot} from '@argos-ci/playwright';
+import {extractSitemapPathnames, pathnameToArgosName} from './utils';
+
+// Constants
+const siteUrl = 'http://localhost:3000';
+const sitemapPath = './build/sitemap.xml';
+const stylesheetPath = './screenshot.css';
+const stylesheet = fs.readFileSync(stylesheetPath).toString();
+
+// Wait for hydration, requires Docusaurus v2.4.3+
+// Docusaurus adds a <html data-has-hydrated="true"> once hydrated
+// See https://github.com/facebook/docusaurus/pull/9256
+function waitForDocusaurusHydration() {
+  return document.documentElement.dataset.hasHydrated === 'true';
+}
+
+function screenshotPathname(pathname: string) {
+  test(`pathname ${pathname}`, async ({page}) => {
+    const url = siteUrl + pathname;
+    await page.goto(url);
+    await page.waitForFunction(waitForDocusaurusHydration);
+    await page.addStyleTag({content: stylesheet});
+    await argosScreenshot(page, pathnameToArgosName(pathname));
+  });
+}
+
+test.describe('Docusaurus site screenshots', () => {
+  const pathnames = extractSitemapPathnames(sitemapPath);
+  console.log('Pathnames to screenshot:', pathnames);
+  pathnames.forEach(screenshotPathname);
+});
diff --git a/utils.ts b/utils.ts
@@ -0,0 +1,19 @@
+import * as cheerio from 'cheerio';
+import * as fs from 'fs';
+
+// Extract a list of pathnames, given a fs path to a sitemap.xml file
+// Docusaurus generates a build/sitemap.xml file for you!
+export function extractSitemapPathnames(sitemapPath: string): string[] {
+  const sitemap = fs.readFileSync(sitemapPath).toString();
+  const $ = cheerio.load(sitemap, {xmlMode: true});
+  const urls: string[] = [];
+  $('loc').each(function handleLoc() {
+    urls.push($(this).text());
+  });
+  return urls.map((url) => new URL(url).pathname);
+}
+
+// Converts a pathname to a decent screenshot name
+export function pathnameToArgosName(pathname: string): string {
+  return pathname.replace(/^\/|\/$/g, '') || 'index';
+}