import query-selector-shadow-dom so it can be used in browser

refs #9126

@types/query-selector-shadow-dom.d.ts

@@ -24,7 +24,3 @@ declare const querySelectorShadowDom: {
 declare module "query-selector-shadow-dom/plugins/puppeteer/index.js" {
     export = querySelectorShadowDom
 }
-
-declare module "query-selector-shadow-dom/plugins/webdriverio" {
-    export = querySelectorShadowDom
-}

examples/wdio/browser-runner/lit.test.js

@@ -1,5 +1,4 @@
-import expect from 'expect'
-import { waitFor } from '@testing-library/dom'
+import { expect, $ } from '@wdio/globals'
 import './components/LitComponent.ts'
 
 describe('Lit Component testing', () => {
@@ -8,8 +7,7 @@ describe('Lit Component testing', () => {
         elem.setAttribute('name', 'WebdriverIO')
         document.body.appendChild(elem)
 
-        await waitFor(() => {
-            expect(elem.shadowRoot.textContent).toBe('Hello, WebdriverIO!')
-        })
+        const innerElem = await $(elem).$('>>> p')
+        expect(await innerElem.getText()).toBe('Hello, WebdriverIO!')
     })
 })

examples/wdio/package.json

@@ -1,11 +1,11 @@
 {
   "scripts": {
     "test:browser": "run-s test:browser:*",
-    "test:browser:react": "WDIO_PRESET=react node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec react.test.tsx",
-    "test:browser:preact": "WDIO_PRESET=preact node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec pr-eact.test.tsx",
-    "test:browser:vue": "WDIO_PRESET=vue node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec vue.test.js",
-    "test:browser:svelte": "WDIO_PRESET=svelte node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec svelte.test.js",
-    "test:browser:lit": "WDIO_PRESET=lit node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec lit.test.js",
+    "test:browser:react": "WDIO_PRESET=react node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec ./browser-runner/react.test.tsx",
+    "test:browser:preact": "WDIO_PRESET=preact node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec ./browser-runner/pr-eact.test.tsx",
+    "test:browser:vue": "WDIO_PRESET=vue node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec ./browser-runner/vue.test.js",
+    "test:browser:svelte": "WDIO_PRESET=svelte node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec ./browser-runner/svelte.test.js",
+    "test:browser:lit": "WDIO_PRESET=lit node ../../packages/wdio-cli/bin/wdio.js ./browser-runner/wdio.conf.js --spec ./browser-runner/lit.test.js",
     "test:mocha": "node ../../packages/wdio-cli/bin/wdio.js ./mocha/wdio.conf.js",
     "test:jasmine": "node ../../packages/wdio-cli/bin/wdio.js ./jasmine/wdio.conf.js",
     "test:cucumber": "node ../../packages/wdio-cli/bin/wdio.js ./cucumber/wdio.conf.js",

packages/wdio-browser-runner/src/plugins/testrunner.ts

@@ -40,8 +40,6 @@ const resolvedVirtualModuleId = '\0' + virtualModuleId
 
 const MODULES_TO_MOCK = [
     'node:module', 'node:events', 'node:path', 'node:url', 'puppeteer-core', 'archiver',
-    'query-selector-shadow-dom/plugins/puppeteer/index.js',
-    'query-selector-shadow-dom/plugins/webdriverio/index.js',
     'glob', 'devtools', 'ws'
 ]
 

packages/webdriverio/src/commands/browser/savePDF.ts

@@ -57,7 +57,7 @@ export default async function savePDF (
     }
 
     const absoluteFilepath = getAbsoluteFilepath(filepath)
-    assertDirectoryExists(absoluteFilepath)
+    await assertDirectoryExists(absoluteFilepath)
 
     const pdf = await this.printPage(options?.orientation, options?.scale, options?.background,
         options?.width, options?.height, options?.top, options?.bottom, options?.left, options?.right,

packages/webdriverio/src/commands/browser/saveRecordingScreen.ts

@@ -38,7 +38,7 @@ export default async function saveRecordingScreen (
     }
 
     const absoluteFilepath = getAbsoluteFilepath(filepath)
-    assertDirectoryExists(absoluteFilepath)
+    await assertDirectoryExists(absoluteFilepath)
 
     const videoBuffer = await this.stopRecordingScreen()
     const video = Buffer.from(videoBuffer, 'base64')

packages/webdriverio/src/commands/browser/saveScreenshot.ts

@@ -32,7 +32,7 @@ export default async function saveScreenshot (
     }
 
     const absoluteFilepath = getAbsoluteFilepath(filepath)
-    assertDirectoryExists(absoluteFilepath)
+    await assertDirectoryExists(absoluteFilepath)
 
     const screenBuffer = await this.takeScreenshot()
     const screenshot = Buffer.from(screenBuffer, 'base64')

packages/webdriverio/src/commands/element/saveScreenshot.ts

@@ -1,4 +1,4 @@
-import fs from 'node:fs'
+import fs from 'node:fs/promises'
 import { getAbsoluteFilepath, assertDirectoryExists } from '../../utils/index.js'
 
 /**
@@ -31,11 +31,11 @@ export default async function saveScreenshot (
     }
 
     const absoluteFilepath = getAbsoluteFilepath(filepath)
-    assertDirectoryExists(absoluteFilepath)
+    await assertDirectoryExists(absoluteFilepath)
 
     const screenBuffer = await this.takeElementScreenshot(this.elementId)
     const screenshot = Buffer.from(screenBuffer, 'base64')
-    fs.writeFileSync(absoluteFilepath, screenshot)
+    await fs.writeFile(absoluteFilepath, screenshot)
 
     return screenshot
 }

packages/webdriverio/src/utils/index.ts

@@ -1,4 +1,4 @@
-import fs from 'node:fs'
+import fs from 'node:fs/promises'
 import http from 'node:http'
 import path from 'node:path'
 import { URL } from 'node:url'
@@ -8,8 +8,6 @@ import rgb2hex from 'rgb2hex'
 import GraphemeSplitter from 'grapheme-splitter'
 import logger from '@wdio/logger'
 import isPlainObject from 'is-plain-obj'
-// @ts-expect-error
-import { locatorStrategy } from 'query-selector-shadow-dom/plugins/webdriverio/index.js'
 import { SUPPORTED_BROWSER } from 'devtools'
 import { UNICODE_CHARACTERS } from '@wdio/utils'
 import type { ElementReference } from '@wdio/protocols'
@@ -17,6 +15,7 @@ import type { Options, Capabilities } from '@wdio/types'
 
 import * as browserCommands from '../commands/browser.js'
 import * as elementCommands from '../commands/element.js'
+import querySelectorAllDeep from './thirdParty/querySelectorShadowDom.js'
 import { ELEMENT_KEY, DRIVER_DEFAULT_ENDPOINT, DEEP_SELECTOR, Key } from '../constants.js'
 import { findStrategy } from './findStrategy.js'
 import type { ElementArray, ElementFunction, Selector, ParsedCSSValue, CustomLocatorReturnValue } from '../types'
@@ -236,11 +235,11 @@ export async function findElement(
     if (!this.isDevTools && typeof selector === 'string' && selector.startsWith(DEEP_SELECTOR)) {
         const notFoundError = new Error(`shadow selector "${selector.slice(DEEP_SELECTOR.length)}" did not return an HTMLElement`)
         let elem: ElementReference | ElementReference[] = await browserObject.execute(
-            locatorStrategy,
-            ...[
-                selector.slice(DEEP_SELECTOR.length),
-                (this as WebdriverIO.Element).elementId ? this : undefined
-            ].filter(Boolean)
+            `return (${querySelectorAllDeep}).apply(null, arguments)`,
+            false,
+            selector.slice(DEEP_SELECTOR.length),
+            // hard conversion from element id to Element is done by browser driver
+            ((this as WebdriverIO.Element).elementId ? this : undefined) as any as Element | Document
         )
         elem = Array.isArray(elem) ? elem[0] : elem
         return getElementFromResponse(elem) ? elem : notFoundError
@@ -312,11 +311,11 @@ export async function findElements(
      */
     if (!this.isDevTools && typeof selector === 'string' && selector.startsWith(DEEP_SELECTOR)) {
         const elems: ElementReference | ElementReference[] = await browserObject.execute(
-            locatorStrategy,
-            ...[
-                selector.slice(DEEP_SELECTOR.length),
-                (this as WebdriverIO.Element).elementId ? this : undefined
-            ].filter(Boolean)
+            `return (${querySelectorAllDeep}).apply(null, arguments)`,
+            true,
+            selector.slice(DEEP_SELECTOR.length),
+            // hard conversion from element id to Element is done by browser driver
+            ((this as WebdriverIO.Element).elementId ? this : undefined) as any as Element | Document
         )
         const elemArray = Array.isArray(elems) ? elems : [elems]
         return elemArray.filter((elem) => elem && getElementFromResponse(elem))
@@ -431,8 +430,9 @@ export function getAbsoluteFilepath(filepath: string) {
 /**
  * check if directory exists
  */
-export function assertDirectoryExists(filepath: string) {
-    if (!fs.existsSync(path.dirname(filepath))) {
+export async function assertDirectoryExists(filepath: string) {
+    const exist = await fs.access(path.dirname(filepath)).then(() => true, () => false)
+    if (!exist) {
         throw new Error(`directory (${path.dirname(filepath)}) doesn't exist`)
     }
 }

packages/webdriverio/src/utils/thirdParty/querySelectorShadowDom.ts

@@ -0,0 +1,316 @@
+/* eslint-disable brace-style */
+/* eslint-disable camelcase */
+/* istanbul ignore file */
+/**
+ * original source: https://github.com/Georgegriff/query-selector-shadow-dom
+ * Copyright by https://github.com/Georgegriff
+ */
+export default function querySelectorAllDeep (findMany: boolean, s: string, r: Element | Document) {
+    function normalizeSelector (sel: string) {
+        // save unmatched text, if any
+        function saveUnmatched() {
+            if (unmatched) {
+                // whitespace needed after combinator?
+                if (tokens.length > 0 && /^[~+>]$/.test(tokens[tokens.length - 1])) {
+                    tokens.push(' ')
+                }
+
+                // save unmatched text
+                tokens.push(unmatched)
+            }
+        }
+
+        var tokens: string[] = [],
+            match,
+            unmatched: string,
+            regex: RegExp,
+            state = [0],
+            next_match_idx = 0,
+            prev_match_idx,
+            not_escaped_pattern = /(?:[^\\]|(?:^|[^\\])(?:\\\\)+)$/,
+            whitespace_pattern = /^\s+$/,
+            state_patterns = [
+                /\s+|\/\*|["'>~+[(]/g, // general
+                /\s+|\/\*|["'[\]()]/g, // [..] set
+                /\s+|\/\*|["'[\]()]/g, // (..) set
+                null, // string literal (placeholder)
+                /\*\//g, // comment
+            ] as RegExp[]
+        sel = sel.trim()
+
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            unmatched = ''
+
+            regex = state_patterns[state[state.length - 1]]
+
+            regex.lastIndex = next_match_idx
+            match = regex.exec(sel)
+
+            // matched text to process?
+            if (match) {
+                prev_match_idx = next_match_idx
+                next_match_idx = regex.lastIndex
+
+                // collect the previous string chunk not matched before this token
+                if (prev_match_idx < next_match_idx - match[0].length) {
+                    unmatched = sel.substring(
+                        prev_match_idx,
+                        next_match_idx - match[0].length
+                    )
+                }
+
+                // general, [ ] pair, ( ) pair?
+                if (state[state.length - 1] < 3) {
+                    saveUnmatched()
+
+                    // starting a [ ] pair?
+                    if (match[0] === '[') {
+                        state.push(1)
+                    }
+                    // starting a ( ) pair?
+                    else if (match[0] === '(') {
+                        state.push(2)
+                    }
+                    // starting a string literal?
+                    else if (/^["']$/.test(match[0])) {
+                        state.push(3)
+                        state_patterns[3] = new RegExp(match[0], 'g')
+                    }
+                    // starting a comment?
+                    else if (match[0] === '/*') {
+                        state.push(4)
+                    }
+                    // ending a [ ] or ( ) pair?
+                    else if (/^[\])]$/.test(match[0]) && state.length > 0) {
+                        state.pop()
+                    }
+                    // handling whitespace or a combinator?
+                    else if (/^(?:\s+|[~+>])$/.test(match[0])) {
+                        // need to insert whitespace before?
+                        if (
+                            tokens.length > 0 &&
+                            !whitespace_pattern.test(tokens[tokens.length - 1]) &&
+                            state[state.length - 1] === 0
+                        ) {
+                            // add normalized whitespace
+                            tokens.push(' ')
+                        }
+
+                        // case-insensitive attribute selector CSS L4
+                        if (
+                            state[state.length - 1] === 1 &&
+                            tokens.length === 5 &&
+                            tokens[2].charAt(tokens[2].length - 1) === '='
+                        ) {
+                            tokens[4] = ' ' + tokens[4]
+                        }
+
+                        // whitespace token we can skip?
+                        if (whitespace_pattern.test(match[0])) {
+                            continue
+                        }
+                    }
+
+                    // save matched text
+                    tokens.push(match[0])
+                }
+                // otherwise, string literal or comment
+                else {
+                    // save unmatched text
+                    tokens[tokens.length - 1] += unmatched
+
+                    // unescaped terminator to string literal or comment?
+                    if (not_escaped_pattern.test(tokens[tokens.length - 1])) {
+                        // comment terminator?
+                        if (state[state.length - 1] === 4) {
+                            // ok to drop comment?
+                            if (
+                                tokens.length < 2 ||
+                                whitespace_pattern.test(tokens[tokens.length - 2])
+                            ) {
+                                tokens.pop()
+                            }
+                            // otherwise, turn comment into whitespace
+                            else {
+                                tokens[tokens.length - 1] = ' '
+                            }
+
+                            // handled already
+                            match[0] = ''
+                        }
+
+                        state.pop()
+                    }
+
+                    // append matched text to existing token
+                    tokens[tokens.length - 1] += match[0]
+                }
+            }
+            // otherwise, end of processing (no more matches)
+            else {
+                unmatched = sel.substr(next_match_idx)
+                saveUnmatched()
+
+                break
+            }
+        }
+
+        return tokens.join('').trim()
+    }
+
+    function _querySelectorDeep(selector: string, root: Element | Document, allElements = null) {
+        selector = normalizeSelector(selector)
+        let lightElement = root.querySelector(selector)
+
+        // @ts-expect-error createShadowRoot does not exist in head
+        if (document.head.createShadowRoot || document.head.attachShadow) {
+            // no need to do any special if selector matches something specific in light-dom
+            if (!findMany && lightElement) {
+                return lightElement
+            }
+
+            // split on commas because those are a logical divide in the operation
+            const selectionsToMake = splitByCharacterUnlessQuoted(selector, ',')
+
+            return selectionsToMake.reduce((acc: any, minimalSelector: any) => {
+                // if not finding many just reduce the first match
+                if (!findMany && acc) {
+                    return acc
+                }
+                // do best to support complex selectors and split the query
+                const splitSelector = splitByCharacterUnlessQuoted(minimalSelector
+                    //remove white space at start of selector
+                    .replace(/^\s+/g, '')
+                    .replace(/\s*([>+~]+)\s*/g, '$1'), ' ')
+                    // filter out entry white selectors
+                    .filter((entry: unknown) => !!entry)
+                    // convert "a > b" to ["a", "b"]
+                    .map((entry: string) => splitByCharacterUnlessQuoted(entry, '>'))
+
+                const possibleElementsIndex = splitSelector.length - 1
+                const lastSplitPart = splitSelector[possibleElementsIndex][splitSelector[possibleElementsIndex].length - 1]
+                const possibleElements = collectAllElementsDeep(lastSplitPart, root, allElements)
+                const findElements = findMatchingElement(splitSelector, possibleElementsIndex, root)
+                if (findMany) {
+                    acc = acc.concat(possibleElements.filter(findElements))
+                    return acc
+                }
+                acc = possibleElements.find(findElements)
+                return acc || null
+            }, findMany ? [] : null)
+        }
+
+        return !findMany
+            ? lightElement
+            : root.querySelectorAll(selector)
+    }
+
+    function findMatchingElement(splitSelector: string, possibleElementsIndex: number, root: Document | Element) {
+        return (element: Element) => {
+            let position = possibleElementsIndex
+            let parent = element
+            let foundElement = false
+            while (parent && !isDocumentNode(parent)) {
+                let foundMatch = true
+                if (splitSelector[position].length === 1) {
+                    foundMatch = parent.matches(splitSelector[position])
+                } else {
+                    // selector is in the format "a > b"
+                    // make sure a few parents match in order
+                    const reversedParts = ([]).concat(splitSelector[position] as any).reverse()
+                    let newParent = parent
+                    for (const part of reversedParts) {
+                        if (!newParent || !newParent.matches(part)) {
+                            foundMatch = false
+                            break
+                        }
+                        newParent = findParentOrHost(newParent, root)
+                    }
+                }
+
+                if (foundMatch && position === 0) {
+                    foundElement = true
+                    break
+                }
+                if (foundMatch) {
+                    position--
+                }
+                parent = findParentOrHost(parent, root)
+            }
+            return foundElement
+        }
+    }
+
+    function splitByCharacterUnlessQuoted(selector: string, character: string) {
+        return selector.match(/\\?.|^$/g)!.reduce((p, c) => {
+            if (c === '"' && !p.sQuote) {
+                p.quote ^= 1
+                p.a[p.a.length - 1] += c
+            } else if (c === '\'' && !p.quote) {
+                p.sQuote ^= 1
+                p.a[p.a.length - 1] += c
+
+            } else if (!p.quote && !p.sQuote && c === character) {
+                p.a.push('')
+            } else {
+                p.a[p.a.length - 1] += c
+            }
+            return p
+        }, { a: [''] } as any).a
+    }
+
+    /**
+     * Checks if the node is a document node or not.
+     * @param {Node} node
+     * @returns {node is Document | DocumentFragment}
+     */
+    function isDocumentNode(node: Element) {
+        return node.nodeType === Node.DOCUMENT_FRAGMENT_NODE || node.nodeType === Node.DOCUMENT_NODE
+    }
+
+    function findParentOrHost(element: Element, root: Element | Document) {
+        const parentNode = element.parentNode
+        // @ts-expect-error
+        return (parentNode && parentNode.host && parentNode.nodeType === 11)
+            // @ts-expect-error
+            ? parentNode.host
+            : parentNode === root
+                ? null
+                : parentNode
+    }
+
+    /**
+     * Finds all elements on the page, inclusive of those within shadow roots.
+     * @param {string=} selector Simple selector to filter the elements by. e.g. 'a', 'div.main'
+     * @return {!Array<string>} List of anchor hrefs.
+     * @author ebidel@ (Eric Bidelman)
+     * License Apache-2.0
+     */
+    function collectAllElementsDeep(selector = null, root: Document | Element, cachedElements = null) {
+        let allElements: Element[] = []
+
+        if (cachedElements) {
+            allElements = cachedElements
+        } else {
+            const findAllElements = function(nodes: NodeListOf<Element>) {
+                for (let i = 0; i < nodes.length; i++) {
+                    const el = nodes[i]
+                    allElements.push(el)
+                    // If the element has a shadow root, dig deeper.
+                    if (el.shadowRoot) {
+                        findAllElements(el.shadowRoot.querySelectorAll('*'))
+                    }
+                }
+            }
+            if ((root as Element).shadowRoot) {
+                findAllElements((root as Element).shadowRoot?.querySelectorAll('*')!)
+            }
+            findAllElements(root.querySelectorAll('*'))
+        }
+
+        return selector ? allElements.filter(el => el.matches(selector)) : allElements
+    }
+
+    return _querySelectorDeep(s, r)
+}

packages/webdriverio/tests/commands/element/saveScreenshot.test.ts

@@ -1,4 +1,4 @@
-import fs from 'node:fs'
+import fs from 'node:fs/promises'
 import path from 'node:path'
 import { expect, describe, it, vi, beforeEach, afterEach, SpyInstance } from 'vitest'
 
@@ -8,18 +8,18 @@ import { remote } from '../../../src/index.js'
 import * as utils from '../../../src/utils/index.js'
 
 vi.mock('got')
-vi.mock('fs')
+vi.mock('fs/promises')
 vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger')))
 
 describe('saveScreenshot', () => {
     let getAbsoluteFilepathSpy: SpyInstance
     let assertDirectoryExistsSpy: SpyInstance
-    let writeFileSyncSpy: SpyInstance
+    let writeFileSyncSpy = vi.spyOn(fs, 'writeFile')
 
     beforeEach(() => {
         getAbsoluteFilepathSpy = vi.spyOn(utils, 'getAbsoluteFilepath')
         assertDirectoryExistsSpy = vi.spyOn(utils, 'assertDirectoryExists')
-        writeFileSyncSpy = vi.spyOn(fs, 'writeFileSync')
+        vi.spyOn(fs, 'access').mockResolvedValue()
     })
 
     afterEach(() => {

packages/webdriverio/tests/utils.test.ts

@@ -1,4 +1,3 @@
-import fs from 'node:fs'
 import http from 'node:http'
 import path from 'node:path'
 import { describe, expect, it, vi, beforeEach } from 'vitest'
@@ -265,8 +264,10 @@ describe('utils', () => {
             expect(scope.findElement).not.toBeCalled()
             expect(scope.findElementFromElement).not.toBeCalled()
             expect(scope.execute).toBeCalledWith(
-                expect.any(Function),
-                '.foobar'
+                expect.any(String),
+                false,
+                '.foobar',
+                undefined
             )
             expect(elem[ELEMENT_KEY]).toBe('foobar')
         })
@@ -278,7 +279,8 @@ describe('utils', () => {
             expect(scope.findElement).not.toBeCalled()
             expect(scope.findElementFromElement).not.toBeCalled()
             expect(scope.execute).toBeCalledWith(
-                expect.any(Function),
+                expect.any(String),
+                false,
                 '.foobar',
                 scope
             )
@@ -379,8 +381,10 @@ describe('utils', () => {
             expect(scope.findElements).not.toBeCalled()
             expect(scope.findElementsFromElement).not.toBeCalled()
             expect(scope.execute).toBeCalledWith(
-                expect.any(Function),
-                '.foobar'
+                expect.any(String),
+                true,
+                '.foobar',
+                undefined
             )
             expect(elem).toHaveLength(1)
             expect(elem[0][ELEMENT_KEY]).toBe('foobar')
@@ -393,7 +397,8 @@ describe('utils', () => {
             expect(scope.findElements).not.toBeCalled()
             expect(scope.findElementsFromElement).not.toBeCalled()
             expect(scope.execute).toBeCalledWith(
-                expect.any(Function),
+                expect.any(String),
+                true,
                 '.foobar',
                 scope
             )
@@ -489,16 +494,11 @@ describe('utils', () => {
     })
 
     describe('assertDirectoryExists', () => {
-        beforeEach(async () => {
-            const fsOrig = await vi.importActual('fs') as typeof fs
-            vi.mocked(fs.existsSync).mockImplementation(fsOrig.existsSync.bind(fsOrig))
+        it('should fail if not existing directory', async () => {
+            await expect(() => assertDirectoryExists('/i/dont/exist.png')).rejects.toThrowError(new Error('directory (/i/dont) doesn\'t exist'))
         })
-
-        it('should fail if not existing directory', () => {
-            expect(() => assertDirectoryExists('/i/dont/exist.png')).toThrowError(new Error('directory (/i/dont) doesn\'t exist'))
-        })
-        it('should not fail if directory exists', () => {
-            expect(() => assertDirectoryExists('.')).not.toThrow()
+        it('should not fail if directory exists', async () => {
+            expect(await assertDirectoryExists('.')).toBe(undefined)
         })
     })