(jasmine): Bring back Jasmine assertions (#10533)

* (jasmine): bring back jasmine assertions

* add unit and smoke tests

* add smoke tests for different matcher types

* add smoke test for expectationResultHandler

* fix unit test

* fix type generation in windows

* fix import

__mocks__/jasmine.ts

@@ -1,26 +1,37 @@
 import { vi } from 'vitest'
 
+export const beforeAllHook = vi.fn()
+export const executeHook = vi.fn()
+export const jasmine = {
+    addReporter: vi.fn(),
+    specFilter: vi.fn(),
+    configure: vi.fn(),
+    getEnv: vi.fn().mockImplementation(() => jasmine),
+    Spec: {
+        prototype: {
+            addExpectationResult: vi.fn(),
+            execute: executeHook
+        }
+    },
+    Suite: {
+        prototype: {
+            beforeAll: beforeAllHook
+        }
+    },
+    matchers: {
+        toBe: vi.fn().mockReturnValue({
+            compare: vi.fn(),
+            negativeCompare: vi.fn()
+        })
+    },
+    addAsyncMatchers: vi.fn(),
+    beforeAll: vi.fn((cb) => cb())
+}
 export default class JasmineMock {
     args: any[]
-    beforeAllHook = vi.fn()
-    executeHook = vi.fn()
-    jasmine = {
-        addReporter: vi.fn(),
-        specFilter: vi.fn(),
-        configure: vi.fn(),
-        getEnv: vi.fn().mockImplementation(() => this.jasmine),
-        Spec: {
-            prototype: {
-                addExpectationResult: vi.fn(),
-                execute: this.executeHook
-            }
-        },
-        Suite: {
-            prototype: {
-                beforeAll: this.beforeAllHook
-            }
-        }
-    }
+    beforeAllHook = beforeAllHook
+    executeHook = executeHook
+    jasmine = jasmine
     env = {
         beforeAll: vi.fn(),
         beforeEach: vi.fn(),

packages/wdio-browser-runner/package.json

@@ -44,7 +44,7 @@
     "@wdio/utils": "8.11.0",
     "ast-types": "^0.14.2",
     "deepmerge-ts": "^5.0.0",
-    "expect-webdriverio": "^4.1.2",
+    "expect-webdriverio": "^4.2.5",
     "fast-safe-stringify": "^2.1.1",
     "get-port": "^7.0.0",
     "import-meta-resolve": "^3.0.0",

packages/wdio-globals/package.json

@@ -42,7 +42,7 @@
     "access": "public"
   },
   "optionalDependencies": {
-    "expect-webdriverio": "^4.0.1",
+    "expect-webdriverio": "^4.2.5",
     "webdriverio": "8.11.1"
   }
 }

packages/wdio-jasmine-framework/package.json

@@ -37,7 +37,7 @@
     "@wdio/logger": "8.11.0",
     "@wdio/types": "8.10.4",
     "@wdio/utils": "8.11.0",
-    "expect-webdriverio": "^4.0.1",
+    "expect-webdriverio": "^4.2.5",
     "jasmine": "^5.0.0"
   },
   "devDependencies": {

packages/wdio-jasmine-framework/src/index.ts

@@ -4,11 +4,12 @@ import type { EventEmitter } from 'node:events'
 import Jasmine from 'jasmine'
 import logger from '@wdio/logger'
 import { wrapGlobalTestMethod, executeHooksWithArgs } from '@wdio/utils'
-import { expect } from 'expect-webdriverio'
+import { matchers, getConfig } from 'expect-webdriverio'
 import { _setGlobal } from '@wdio/globals'
 import type { Options, Services, Capabilities } from '@wdio/types'
 
 import JasmineReporter from './reporter.js'
+import { jestResultToJasmine } from './utils.js'
 import type {
     JasmineOpts as jasmineNodeOpts, ResultHandlerPayload, FrameworkMessage, FormattedMessage
 } from './types.js'
@@ -63,7 +64,8 @@ class JasmineAdapter {
         this._reporter = new JasmineReporter(reporter, {
             cid: this._cid,
             specs: this._specs,
-            cleanStack: this._jasmineOpts.cleanStack
+            cleanStack: this._jasmineOpts.cleanStack,
+            jasmineOpts: this._jasmineOpts
         })
         this._hasTests = true
         this._jrunner.exitOnCompletion = false
@@ -193,6 +195,17 @@ class JasmineAdapter {
             executeMock.apply(this, args)
         }
 
+        /**
+         * set up WebdriverIO matchers with Jasmine
+         */
+        const expect = jasmineEnv.expectAsync
+        const matchers = this.#setupMatchers(jasmine)
+        jasmineEnv.beforeAll(() => jasmineEnv.addAsyncMatchers(matchers))
+        _setGlobal('expect', expect, this._config.injectGlobals)
+
+        /**
+         * load environment
+         */
         await this._loadFiles()
 
         /**
@@ -283,8 +296,8 @@ class JasmineAdapter {
             hookName,
             this._config[hookName],
             [this.prepareMessage(hookName)]
-        ).catch((e) => {
-            log.info(`Error in ${hookName} hook: ${e.stack.slice(7)}`)
+        ).catch((e: Error) => {
+            log.info(`Error in ${hookName} hook: ${e.stack?.slice(7)}`)
         })
     }
 
@@ -375,6 +388,34 @@ class JasmineAdapter {
             return origHandler.call(this, passed, data)
         }
     }
+
+    #setupMatchers (jasmine: jasmine.Jasmine): jasmine.CustomAsyncMatcherFactories {
+        // @ts-expect-error not exported in jasmine
+        const jasmineMatchers: jasmine.CustomMatcherFactories = jasmine.matchers
+        const syncMatchers: jasmine.CustomAsyncMatcherFactories = Object.entries(jasmineMatchers).reduce((prev, [name, fn]) => {
+            prev[name] = (util) => ({
+                compare: async <T>(actual: T, expected: T, ...args: any[]) => fn(util).compare(actual, expected, ...args),
+                negativeCompare: async <T>(actual: T, expected: T, ...args: unknown[]) => fn(util).negativeCompare!(actual, expected, ...args)
+            })
+            return prev
+        }, {} as jasmine.CustomAsyncMatcherFactories)
+        const wdioMatchers: jasmine.CustomAsyncMatcherFactories = Object.entries(matchers as Record<string, any>).reduce((prev, [name, fn]) => {
+            prev[name] = () => ({
+                async compare (...args: unknown[]) {
+                    const context = getConfig()
+                    const result = fn.apply({ ...context, isNot: false }, args)
+                    return jestResultToJasmine(result, false)
+                },
+                async negativeCompare (...args: unknown[]) {
+                    const context = getConfig()
+                    const result = fn.apply({ ...context, isNot: true }, args)
+                    return jestResultToJasmine(result, true)
+                }
+            })
+            return prev
+        }, {} as jasmine.CustomAsyncMatcherFactories)
+        return { ...wdioMatchers, ...syncMatchers }
+    }
 }
 
 const adapterFactory: { init?: Function } = {}

packages/wdio-jasmine-framework/src/reporter.ts

@@ -1,7 +1,7 @@
 import logger from '@wdio/logger'
 import type { EventEmitter } from 'node:events'
 
-import type { ReporterOptions, ParentSuite, TestEvent, SuiteEvent } from './types.js'
+import type { ReporterOptions, ParentSuite, TestEvent, SuiteEvent, JasmineOpts } from './types.js'
 
 const log = logger('@wdio/jasmine-framework')
 
@@ -12,6 +12,7 @@ export default class JasmineReporter {
 
     private _cid: string
     private _specs: string[]
+    private _jasmineOpts: JasmineOpts
     private _shouldCleanStack: boolean
     private _parent: ParentSuite[] = []
     private _failedCount = 0
@@ -24,6 +25,7 @@ export default class JasmineReporter {
     ) {
         this._cid = params.cid
         this._specs = params.specs
+        this._jasmineOpts = params.jasmineOpts
         this._shouldCleanStack = typeof params.cleanStack === 'boolean' ? params.cleanStack : true
     }
 
@@ -91,6 +93,29 @@ export default class JasmineReporter {
             newTest.status = 'pending'
         }
 
+        /**
+         * mark failing tests due to missing assertion
+         */
+        if (
+            test.status === 'failed' &&
+            this._jasmineOpts.failSpecWithNoExpectations &&
+            test.failedExpectations.length === 0 &&
+            test.passedExpectations.length === 0
+        ) {
+            test.failedExpectations.push({
+                matcherName: 'toHaveAssertion',
+                expected: ' >1 assertions',
+                actual: '0 assertions',
+                passed: false,
+                message: (
+                    'No assertions found in test! This test is failing because no assertions were found ' +
+                    'but "jasmineOpts" had a "failSpecWithNoExpectations" flag set to "true".\n' +
+                    '\nRead more on this Jasmine option at: https://jasmine.github.io/api/5.0/Configuration#failSpecWithNoExpectations'
+                ),
+                stack: ''
+            })
+        }
+
         if (test.failedExpectations && test.failedExpectations.length) {
             let errors = test.failedExpectations
             if (this._shouldCleanStack) {

packages/wdio-jasmine-framework/src/types.ts

@@ -1,7 +1,8 @@
 export interface ReporterOptions {
     cid: string
     specs: string[]
-    cleanStack?: boolean
+    cleanStack?: boolean,
+    jasmineOpts: JasmineOpts
 }
 
 export interface ParentSuite {

packages/wdio-jasmine-framework/src/utils.ts

@@ -0,0 +1,18 @@
+interface JestExpectationResult {
+    pass: boolean
+    message: () => string
+}
+
+const buildJasmineFromJestResult = (result: JestExpectationResult, isNot: boolean) => {
+    return {
+        pass: result.pass !== isNot,
+        message: result.message()
+    }
+}
+
+export const jestResultToJasmine = (result: JestExpectationResult, isNot: boolean) => {
+    if (result instanceof Promise) {
+        return result.then(jestStyleResult => buildJasmineFromJestResult(jestStyleResult, isNot))
+    }
+    return buildJasmineFromJestResult(result, isNot)
+}

packages/wdio-jasmine-framework/tests/adapter.test.ts

@@ -2,12 +2,17 @@ import path from 'node:path'
 import { expect, test, vi, it, describe, afterEach } from 'vitest'
 import logger from '@wdio/logger'
 import { wrapGlobalTestMethod, executeHooksWithArgs } from '@wdio/utils'
+import { jasmine } from 'jasmine'
 import type { EventEmitter } from 'node:events'
 
 import JasmineAdapterFactory, { JasmineAdapter } from '../src/index.js'
 
 vi.mock('jasmine')
-vi.mock('expect-webdriverio')
+vi.mock('expect-webdriverio', () => ({
+    matchers: {
+        toHaveTitle: vi.fn()
+    }
+}))
 vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger')))
 vi.mock('@wdio/utils', () => import(path.join(process.cwd(), '__mocks__', '@wdio/utils')))
 
@@ -70,7 +75,7 @@ test('should properly set up jasmine', async () => {
     expect(result).toBe(0)
     expect(vi.mocked(adapter['_jrunner']!.addSpecFile).mock.calls[0][0]).toEqual('/foo/bar.test.js')
     // @ts-ignore outdated types
-    expect(vi.mocked(adapter['_jrunner']!.jasmine.addReporter).mock.calls).toHaveLength(1)
+    expect(vi.mocked(adapter['_jrunner']!.jasmine.addReporter).mock.calls).toHaveLength(2)
     expect(vi.mocked(executeHooksWithArgs).mock.calls).toHaveLength(1)
 
     // @ts-expect-error
@@ -87,6 +92,12 @@ test('should properly set up jasmine', async () => {
     expect(adapter['_jrunner']!.configureDefaultReporter.name).toBe('noop')
     // @ts-ignore outdated types
     adapter['_jrunner']!.configureDefaultReporter()
+
+    expect(jasmine.addAsyncMatchers).toBeCalledTimes(1)
+    expect(jasmine.addAsyncMatchers).toBeCalledWith({
+        toBe: expect.any(Function),
+        toHaveTitle: expect.any(Function)
+    })
 })
 
 test('should propery wrap interfaces', async () => {
@@ -518,4 +529,5 @@ describe('hasTests', () => {
 afterEach(() => {
     vi.mocked(wrapGlobalTestMethod).mockClear()
     vi.mocked(executeHooksWithArgs).mockClear()
+    vi.mocked(jasmine.addAsyncMatchers).mockClear()
 })

packages/wdio-jasmine-framework/tests/reporter.test.ts

@@ -14,7 +14,8 @@ let runnerReporter: EventEmitter
 const PARAMS = {
     cid: '0-2',
     capabilities: { browserName: 'foobar' },
-    specs: ['/foo/bar.test.js']
+    specs: ['/foo/bar.test.js'],
+    jasmineOpts: {}
 }
 
 beforeEach(() => {

packages/wdio-runner/package.json

@@ -36,7 +36,7 @@
     "@wdio/types": "8.10.4",
     "@wdio/utils": "8.11.0",
     "deepmerge-ts": "^5.0.0",
-    "expect-webdriverio": "^4.0.1",
+    "expect-webdriverio": "^4.2.5",
     "gaze": "^1.1.2",
     "webdriver": "8.11.1",
     "webdriverio": "8.11.1"

packages/wdio-runner/src/index.ts

@@ -284,7 +284,14 @@ export default class Runner extends EventEmitter {
             this._browser = await initialiseInstance(config, caps, this._isMultiremote)
             _setGlobal('browser', this._browser, config.injectGlobals)
             _setGlobal('driver', this._browser, config.injectGlobals)
-            _setGlobal('expect', expect, config.injectGlobals)
+
+            /**
+             * for Jasmine we extend the Jasmine matchers instead of injecting the assertion
+             * library ourselves
+             */
+            if (config.framework !== 'jasmine') {
+                _setGlobal('expect', expect, config.injectGlobals)
+            }
 
             /**
              * re-assign previously registered custom commands to the actual instance

scripts/protocols.ts

@@ -34,7 +34,7 @@ export type Protocols = Record<ProtocolKeys, Protocol>
 const bidiTypesPath = path.resolve(__dirname, '..', 'packages', 'wdio-protocols', 'src', 'protocols', 'webdriverBidi.ts')
 const hasBidiTypesGenerated = await fs.access(bidiTypesPath).then(() => true, () => false)
 const webdriverBidi = hasBidiTypesGenerated
-    ? (await import(bidiTypesPath)).default
+    ? (await import(url.pathToFileURL(bidiTypesPath).pathname)).default
     : {}
 
 export const PROTOCOLS: Protocols = {

tests/helpers/config.js

@@ -1,7 +1,9 @@
 import path from 'node:path'
 import url from 'node:url'
+import fs from 'node:fs/promises'
 
 const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
+let expectationResults = ''
 
 export const config = {
     /**
@@ -32,7 +34,11 @@ export const config = {
         defaultTimeoutInterval: 1000 * 60 * 3,
         grep: 'SKIPPED_GREP',
         invertGrep: true,
-        require: ['ts-node/esm']
+        require: ['ts-node/esm'],
+        expectationResultHandler: (actual, assertion) => {
+            expectationResults += `expect(${typeof assertion.expected}).${assertion.matcherName}(${typeof assertion.actual})\n`
+            return fs.writeFile(path.resolve(__dirname, 'expectationResults.log'), expectationResults)
+        }
     },
 
     cucumberOpts: {

tests/jasmine/jasmineWithNoExpectations.js

@@ -0,0 +1,6 @@
+describe('a Jasmine test', () => {
+    it('should fail without any assertion', () => {
+        console.log('no assertion here')
+        // expect(1).toBe(2)
+    })
+})

tests/jasmine/test.js

@@ -6,6 +6,12 @@ describe('Jasmine smoke test', () => {
         expect(test).toBe(123)
     })
 
+    it('should support sync Jest and Jasmine matchers', () => {
+        expect(1).toBe(1) // available in both
+        expect({ foo: 'bar' }).toEqual({ foo: 'bar' }) // Jest matcher
+        expect(false).toBeFalse() // Jasmine matcher
+    })
+
     it('should return async value', async () => {
         await browser.isEventuallyDisplayedScenario()
         await expect(browser).toHaveTitle('Mock Page Title')

tests/smoke.runner.js

@@ -109,6 +109,8 @@ const cjsTestrunner = async () => {
  * Jasmine wdio testrunner tests
  */
 const jasmineTestrunner = async () => {
+    const logFile = path.resolve(__dirname, 'helpers', 'expectationResults.log')
+    await fs.rm(logFile, { force: true })
     const { skippedSpecs } = await launch('jasmineTestrunner', baseConfig, {
         autoCompileOpts: { autoCompile: false },
         specs: [
@@ -126,6 +128,21 @@ const jasmineTestrunner = async () => {
     }
 
     assert.strictEqual(skippedSpecs, 1)
+    assert.equal(
+        (await fs.readFile(logFile, 'utf-8')).toString(),
+        [
+            'expect(number).toBe(number)',
+            'expect(number).toBe(number)',
+            'expect(object).toEqual(object)',
+            'expect(object).toBeFalse(boolean)',
+            'expect(string).toHaveTitle(object)',
+            'expect(object).toBeDisplayed(object)',
+            'expect(object).toBeDisplayed(object)',
+            'expect(number).toBe(number)',
+            'expect(number).toBe(number)',
+            ''
+        ].join('\n')
+    )
 }
 
 /**
@@ -171,7 +188,7 @@ const jasmineTimeout = async () => {
         'spec was not failing due to timeout error'
     )
     assert.ok(
-        specLogs.includes('Error: expect(received).toBe(expected) // Object.is equality'),
+        specLogs.includes('at listOnTimeout (node:internal'),
         'spec was not failing a sync assertion error'
     )
     assert.ok(
@@ -211,7 +228,7 @@ const jasmineAfterAll = async () => {
         'spec did not execute the expected describe'
     )
     assert.ok(
-        specLogs.includes('Error: expect(received).toBe(expected)'),
+        specLogs.includes('actual expected') && specLogs.includes('truefalse'),
         'spec did not fail with the expected check'
     )
     assert.ok(
@@ -224,6 +241,35 @@ const jasmineAfterAll = async () => {
     )
 }
 
+/**
+ * Jasmine verify failSpecWithNoExpectations support
+ */
+const jasmineFailSpecWithNoExpectations = async () => {
+    const logFile = path.join(__dirname, 'jasmineWithNoExpectations.spec.log')
+    await launch('jasmineAfterAll', baseConfig, {
+        autoCompileOpts: { autoCompile: false },
+        specs: [path.resolve(__dirname, 'jasmine', 'jasmineWithNoExpectations.js')],
+        reporters: [
+            ['spec', {
+                outputDir: __dirname,
+                stdout: false,
+                logFile
+            }]
+        ],
+        framework: 'jasmine',
+        jasmineOpts: {
+            failSpecWithNoExpectations: true
+        }
+    }).catch((err) => err) // error expected
+
+    // eslint-disable-next-line no-control-regex
+    const specLogs = (await fs.readFile(logFile)).toString().replace(/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, '')
+    assert.ok(
+        specLogs.includes('No assertions found in test'),
+        'spec did not fail with the expected check'
+    )
+}
+
 /**
  * Cucumber wdio testrunner tests
  */
@@ -585,6 +631,7 @@ const nonGlobalTestrunner = async () => {
         jasmineReporter,
         jasmineTimeout,
         jasmineAfterAll,
+        jasmineFailSpecWithNoExpectations,
         retryFail,
         retryPass,
         customReporterString,