Improve stability of component tests for Safari (#9985)

* Improve stability of component tests for Safari * revert tweak * print source page when connection fails * evaluate for title too
2023-03-15 22:48:01 +01:00 · 2023-03-15 22:48:01 +01:00 · 7bc087bb5a
commit 7bc087bb5a
parent 462bbc6ef8
10 changed files with 156 additions and 21 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -88,6 +88,8 @@ jobs:
          timeout_minutes: 20
          max_attempts: 3
          command: npm run test:e2e
+      - name: Safari Component Tests
+        run: npm run test:component
      # Disable Firefox tests due to https://github.com/puppeteer/puppeteer/issues/8923
      # - name: Setup Firefox
      #   uses: browser-actions/setup-firefox@latest
--- a/e2e/browser-runner/lit.test.js
+++ b/e2e/browser-runner/lit.test.js
@ -78,7 +78,14 @@ describe('Lit Component testing', () => {
        expect(window.TEST_COMMAND).toBe('serve')
    })

-    it('should allow to manual mock dependencies', async () => {
+    it('should allow to manual mock dependencies', async function () {
+        /**
+         * this fails in Safari as the click on the button is not recognised
+         */
+        if (browser.capabilities.browserName.toLowerCase() === 'safari') {
+            return this.skip()
+        }
+
        render(
            html`<simple-greeting name="WebdriverIO" />`,
            document.body
--- a/e2e/browser-runner/wdio.conf.js
+++ b/e2e/browser-runner/wdio.conf.js
@ -3,17 +3,22 @@ import url from 'node:url'
 import path from 'node:path'
 import { loadEnv } from 'vite'

+const isMac = os.platform() === 'darwin'
+const isWindows = os.platform() === 'win32'
+
 /**
 * WebdriverIO is using this example to test its component testing features
 * and we have experienced issues with Vue when running in Windows,
 * see https://github.com/testing-library/vue-testing-library/issues/292
 * Please ignore and remove this in your project!
 */
-if (process.env.CI && process.env.WDIO_PRESET === 'vue' && os.platform() === 'win32') {
+if (process.env.CI && process.env.WDIO_PRESET === 'vue' && (isWindows || isMac)) {
    process.exit(0)
 }

 const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
+const browserName = isMac ? 'safari' : 'chrome'
+const driver = isMac ? 'safaridriver' : 'chromedriver'
 export const config = {
    /**
     * specify test files
@ -26,10 +31,7 @@ export const config = {
    /**
     * capabilities
     */
-    capabilities: [{
-        browserName: 'chrome',
-        acceptInsecureCerts: true
-    }],
+    capabilities: [{ browserName }],

    /**
     * test configurations
@ -38,7 +40,7 @@ export const config = {
    framework: 'mocha',
    outputDir: path.join(__dirname, 'logs', process.env.WDIO_PRESET),
    reporters: ['spec', 'dot', 'junit'],
-    services: ['chromedriver'],
+    services: [driver],
    runner: ['browser', {
        preset: process.env.WDIO_PRESET,
        rootDir: path.resolve(__dirname, '..'),
@ -54,7 +56,8 @@ export const config = {
        },
        coverage: {
            enabled: true,
-            functions: 100
+            // we skip some tests on Mac, therefor lower coverage treshold
+            functions: isMac ? 66 : 100
        }
    }],

--- a/e2e/package-lock.json
+++ b/e2e/package-lock.json
@ -26,7 +26,8 @@
        "expect": "^29.1.2",
        "graphql-request": "^5.1.0",
        "mocha": "^10.2.0",
-        "wdio-chromedriver-service": "^8.0.1"
+        "wdio-chromedriver-service": "^8.0.1",
+        "wdio-safaridriver-service": "^2.1.0"
      }
    },
    "node_modules/@adobe/css-tools": {
@ -1416,12 +1417,27 @@
      "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
      "dev": true
    },
+    "node_modules/@types/split2": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/@types/split2/-/split2-3.2.1.tgz",
+      "integrity": "sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/stack-utils": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz",
      "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
      "dev": true
    },
+    "node_modules/@types/tcp-port-used": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/tcp-port-used/-/tcp-port-used-1.0.1.tgz",
+      "integrity": "sha512-6pwWTx8oUtWvsiZUCrhrK/53MzKVLnuNSSaZILPy3uMes9QnTrLMar9BDlJArbMOjDcjb3QXFk6Rz8qmmuySZw==",
+      "dev": true
+    },
    "node_modules/@types/testing-library__jest-dom": {
      "version": "5.14.5",
      "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",
@ -5730,6 +5746,12 @@
        "node": "*"
      }
    },
+    "node_modules/safaridriver": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.0.4.tgz",
+      "integrity": "sha512-EgK9Vc2Bk4WaECq1E7ti9GyeQ6JKKQNs40vNOE/b5Ul5dDEt429G0Kj5Nzs4FRS5ZIVa7nBck1TyHuinOYdz2Q==",
+      "dev": true
+    },
    "node_modules/safe-buffer": {
      "version": "5.2.1",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -6280,6 +6302,36 @@
        }
      }
    },
+    "node_modules/wdio-safaridriver-service": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/wdio-safaridriver-service/-/wdio-safaridriver-service-2.1.0.tgz",
+      "integrity": "sha512-Zg0aZn8KJo+MwvdOTje2cja4OcrkpoCBhQSzVlhZ3SszMnwhi4VQ1p5TdHB9mWZpuVtsLHWpmx/G7ceJWendVw==",
+      "dev": true,
+      "dependencies": {
+        "@types/split2": "^3.2.1",
+        "@types/tcp-port-used": "^1.0.1",
+        "@wdio/logger": "^8.1.0",
+        "fs-extra": "^11.1.0",
+        "safaridriver": "^0.0.4",
+        "split2": "^4.1.0",
+        "tcp-port-used": "^1.0.2"
+      },
+      "engines": {
+        "node": "^16.13 || >=18"
+      },
+      "peerDependencies": {
+        "@wdio/types": "^7.0.0 || ^8.0.0-alpha.219",
+        "webdriverio": "^7.0.0 || ^8.0.0-alpha.219"
+      },
+      "peerDependenciesMeta": {
+        "@wdio/types": {
+          "optional": true
+        },
+        "webdriverio": {
+          "optional": false
+        }
+      }
+    },
    "node_modules/webdriver": {
      "version": "8.5.7",
      "resolved": "https://registry.npmjs.org/webdriver/-/webdriver-8.5.7.tgz",
@ -7706,12 +7758,27 @@
      "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
      "dev": true
    },
+    "@types/split2": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/@types/split2/-/split2-3.2.1.tgz",
+      "integrity": "sha512-7uz3yU+LooBq4yNOzlZD9PU9/1Eu0rTD1MjQ6apOVEoHsPrMUrFw7W8XrvWtesm2vK67SBK9AyJcOXtMpl9bgQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/stack-utils": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz",
      "integrity": "sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==",
      "dev": true
    },
+    "@types/tcp-port-used": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@types/tcp-port-used/-/tcp-port-used-1.0.1.tgz",
+      "integrity": "sha512-6pwWTx8oUtWvsiZUCrhrK/53MzKVLnuNSSaZILPy3uMes9QnTrLMar9BDlJArbMOjDcjb3QXFk6Rz8qmmuySZw==",
+      "dev": true
+    },
    "@types/testing-library__jest-dom": {
      "version": "5.14.5",
      "resolved": "https://registry.npmjs.org/@types/testing-library__jest-dom/-/testing-library__jest-dom-5.14.5.tgz",
@ -10985,6 +11052,12 @@
        }
      }
    },
+    "safaridriver": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/safaridriver/-/safaridriver-0.0.4.tgz",
+      "integrity": "sha512-EgK9Vc2Bk4WaECq1E7ti9GyeQ6JKKQNs40vNOE/b5Ul5dDEt429G0Kj5Nzs4FRS5ZIVa7nBck1TyHuinOYdz2Q==",
+      "dev": true
+    },
    "safe-buffer": {
      "version": "5.2.1",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -11383,6 +11456,21 @@
        "tcp-port-used": "^1.0.2"
      }
    },
+    "wdio-safaridriver-service": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/wdio-safaridriver-service/-/wdio-safaridriver-service-2.1.0.tgz",
+      "integrity": "sha512-Zg0aZn8KJo+MwvdOTje2cja4OcrkpoCBhQSzVlhZ3SszMnwhi4VQ1p5TdHB9mWZpuVtsLHWpmx/G7ceJWendVw==",
+      "dev": true,
+      "requires": {
+        "@types/split2": "^3.2.1",
+        "@types/tcp-port-used": "^1.0.1",
+        "@wdio/logger": "^8.1.0",
+        "fs-extra": "^11.1.0",
+        "safaridriver": "^0.0.4",
+        "split2": "^4.1.0",
+        "tcp-port-used": "^1.0.2"
+      }
+    },
    "webdriver": {
      "version": "8.5.7",
      "resolved": "https://registry.npmjs.org/webdriver/-/webdriver-8.5.7.tgz",
--- a/e2e/package.json
+++ b/e2e/package.json
@ -41,7 +41,8 @@
    "expect": "^29.1.2",
    "graphql-request": "^5.1.0",
    "mocha": "^10.2.0",
-    "wdio-chromedriver-service": "^8.0.1"
+    "wdio-chromedriver-service": "^8.0.1",
+    "wdio-safaridriver-service": "^2.1.0"
  },
  "dependencies": {
    "lit": "^2.4.0",
--- a/packages/wdio-browser-runner/src/browser/mock.ts
+++ b/packages/wdio-browser-runner/src/browser/mock.ts
@ -17,9 +17,4 @@ export const dirname = () => ''
 export const resolve = () => ''
 export const sep = '/'
 export const type = 'browser'
-
-// mock jest-matcher-utils exports
-export const printDiffOrStringify = () => {}
-export const printReceived = () => {}
-export const printExpected = () => {}
 export default () => {}
--- a/packages/wdio-browser-runner/src/vite/constants.ts
+++ b/packages/wdio-browser-runner/src/vite/constants.ts
@ -2,6 +2,7 @@ import topLevelAwait from 'vite-plugin-top-level-await'
 import { esbuildCommonjs } from '@originjs/vite-plugin-commonjs'
 import type { InlineConfig } from 'vite'

+import { codeFrameFix } from './plugins/esbuild.js'
 import type { FrameworkPreset } from '../types.js'

 export const PRESET_DEPENDENCIES: Record<FrameworkPreset, [string, string, any] | undefined> = {
@ -41,7 +42,7 @@ export const DEFAULT_VITE_CONFIG: Partial<InlineConfig> = {
            'expect', 'serialize-error', 'minimatch', 'css-shorthand-properties',
            'lodash.merge', 'lodash.zip', 'lodash.clonedeep', 'lodash.pickby', 'lodash.flattendeep',
            'aria-query', 'grapheme-splitter', 'css-value', 'rgb2hex', 'p-iteration', 'fast-safe-stringify',
-            'deepmerge-ts', 'jest-util'
+            'deepmerge-ts', 'jest-util', 'jest-matcher-utils'
        ],
        esbuildOptions: {
            logLevel: 'silent',
@ -51,7 +52,8 @@ export const DEFAULT_VITE_CONFIG: Partial<InlineConfig> = {
            },
            // Enable esbuild polyfill plugins
            plugins: [
-                esbuildCommonjs(['@testing-library/vue'])
+                esbuildCommonjs(['@testing-library/vue']),
+                codeFrameFix()
            ],
        },
    }
--- a/packages/wdio-browser-runner/src/vite/plugins/esbuild.ts
+++ b/packages/wdio-browser-runner/src/vite/plugins/esbuild.ts
@ -0,0 +1,34 @@
+import fs from 'node:fs/promises'
+import type { Plugin, PluginBuild } from 'esbuild'
+
+export function codeFrameFix () {
+    return <Plugin>{
+        name: 'wdio:codeFrameFix',
+        setup (build: PluginBuild) {
+            build.onLoad(
+                { filter: /@babel\/code-frame/, namespace: 'file' },
+
+                /**
+                 * mock @babel/code-frame as it fails in Safari due
+                 * to usage of chalk
+                 */
+                async ({ path: id }: { path: string }) => {
+                    const code = await fs.readFile(id).then(
+                        (buf) => buf.toString(),
+                        () => undefined)
+
+                    if (!code) {
+                        return
+                    }
+
+                    return {
+                        contents: code.replace('require("@babel/highlight");', /*js*/`{
+                            shouldHighlight: false,
+                            reset: () => {}
+                        }`)
+                    }
+                }
+            )
+        }
+    }
+}
--- a/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts
+++ b/packages/wdio-browser-runner/src/vite/plugins/testrunner.ts
@ -41,8 +41,7 @@ const resolvedVirtualModuleId = '\0' + virtualModuleId
 * functionality
 */
 const MODULES_TO_MOCK = [
-    'import-meta-resolve', 'puppeteer-core', 'archiver', 'glob', 'devtools', 'ws',
-    'jest-matcher-utils', 'decamelize'
+    'import-meta-resolve', 'puppeteer-core', 'archiver', 'glob', 'devtools', 'ws', 'decamelize'
 ]

 const POLYFILLS = [
--- a/packages/wdio-runner/src/browser.ts
+++ b/packages/wdio-runner/src/browser.ts
@ -141,8 +141,12 @@ export default class BrowserFramework implements Omit<TestFramework, 'init'> {
                                viteError = [{ message: errorElems.map((elem) => elem.innerText).join('\n') }]
                            }
                        }
-                        const loadError = typeof window.__wdioErrors__ === 'undefined'
-                            ?  [{ message: `Failed to load test page (title = ${document.title})` }]
+                        const loadError = (
+                            typeof window.__wdioErrors__ === 'undefined' &&
+                            document.title !== 'WebdriverIO Browser Test' &&
+                            !document.querySelector('mocha-framework')
+                        )
+                            ?  [{ message: `Failed to load test page (title = "${document.title}", source: ${document.documentElement.innerHTML})` }]
                            : null
                        const errors = viteError || window.__wdioErrors__ || loadError
                        return { failures, errors, hasViteError: Boolean(viteError) }