diff --git a/packages/browserstack-service/src/accessibility-handler.ts b/packages/browserstack-service/src/accessibility-handler.ts index f46b53e..22f61d5 100644 --- a/packages/browserstack-service/src/accessibility-handler.ts +++ b/packages/browserstack-service/src/accessibility-handler.ts @@ -67,7 +67,9 @@ import { validateCapsWithAppA11y, getAppA11yResults, executeAccessibilityScript, - isFalse + isFalse, + getHookType, + frameworkSupportsHook } from './util.js' import accessibilityScripts from './scripts/accessibility-scripts.js' import PerformanceTester from './instrumentation/performance/performance-tester.js' @@ -87,6 +89,8 @@ class _AccessibilityHandler { private _autoScanning: boolean = true private _testIdentifier: string | null = null private _testMetadata: TestMetadata = {} + /* Set while a supported hook is executing; scans fired in this window are stamped with it. */ + private _currentHookRunUuid: string | null = null private static _a11yScanSessionMap: A11yScanSessionMap = {} private _sessionId: string | null = null private listener = Listener.getInstance() @@ -418,6 +422,49 @@ class _AccessibilityHandler { } } + /** + * Hook scans. A driver command executed inside a test hook (before/after, beforeEach/afterEach, + * cucumber hooks) should fire an accessibility scan carrying the hook's run UUID so the backend + * (SeleniumHub appAllyHandler -> app-accessibility) reconciles it onto the wrapping test case + * instead of collapsing into a NULL row. `hookRunUuid` is the SAME uuid the SDK reports to + * TestHub as HookRunStarted (InsightsHandler.getCurrentHook) = the hook's BTCER uuid. + * Additive: when hookRunUuid is absent, in-test scan behaviour is unchanged. + */ + async beforeHook (test: Frameworks.Test | undefined, context: unknown, hookRunUuid?: string | null) { + try { + if (!this._accessibility || !this.shouldRunTestHooks(this._browser, this._accessibility)) { + return + } + if (!frameworkSupportsHook('before', this._framework)) { + return + } + + this._currentHookRunUuid = hookRunUuid || null + + if (this._framework === 'mocha' && this._sessionId) { + let shouldScan = this._autoScanning + const hookType = (test && typeof test.title === 'string') ? getHookType(test.title) : 'unknown' + const wrappedTest = (context as { currentTest?: Frameworks.Test } | undefined)?.currentTest + if ((hookType === 'BEFORE_EACH' || hookType === 'AFTER_EACH') && wrappedTest) { + let suiteTitle: unknown = wrappedTest.parent + if (suiteTitle && typeof suiteTitle === 'object') { + suiteTitle = (suiteTitle as { title?: string }).title + } + // @ts-expect-error fix type + shouldScan = this._autoScanning && shouldScanTestForAccessibility(suiteTitle as string | undefined, wrappedTest.title, this._accessibilityOptions) + } + AccessibilityHandler._a11yScanSessionMap[this._sessionId] = shouldScan + } + } catch (error) { + BStackLogger.error(`Exception in accessibility automation beforeHook: ${error}`) + } + } + + async afterHook () { + // Hook finished: subsequent (test-body) scans must not be stamped as hook scans. + this._currentHookRunUuid = null + } + /* * private methods */ @@ -433,7 +480,7 @@ class _AccessibilityHandler { ) ) { BStackLogger.debug(`Performing scan for ${command.class} ${command.name}`) - await performA11yScan(this.isAppAutomate, this._browser, true, true, command.name) + await performA11yScan(this.isAppAutomate, this._browser, true, true, command.name, undefined, this._currentHookRunUuid) } else if (skipScanForBidiWindowCommand) { BStackLogger.debug(`SDK-5047: skipping accessibility scan for BiDi window/context command '${command.name}' to avoid racing the WebdriverIO ContextManager during session-start window churn`) } diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 5b7399e..4b9dfcb 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -18,6 +18,7 @@ import PerformanceTester from '../../instrumentation/performance/performance-tes import * as PERFORMANCE_SDK_EVENTS from '../../instrumentation/performance/constants.js' import type { FetchDriverExecuteParamsEventRequest, FetchDriverExecuteParamsEventResponse } from '../../grpc/index.js' import { GrpcClient } from '../grpcClient.js' +import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' export default class AccessibilityModule extends BaseModule { @@ -34,6 +35,10 @@ export default class AccessibilityModule extends BaseModule { LOG_DISABLED_SHOWN: Map testMetadata: Record = {} currentTestName: string | null = null + // The run uuid of the hook currently executing (set at hook PRE, cleared at hook POST). + // Any scan fired while this is set is stamped with it as thHookRunUuid so the backend + // (SeleniumHub appAllyScan → hook_run_uuid) can reconcile the scan onto the wrapping test. + currentHookRunUuid: string | null = null constructor(accessibilityConfig: Accessibility, isNonBstackA11y: boolean) { super() @@ -42,6 +47,13 @@ export default class AccessibilityModule extends BaseModule { AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) + // Track the hook window for every hook state the framework supports. PRE captures the + // hook's run uuid + opens the scan gate; POST clears the uuid so later test-body scans + // are not mis-stamped as hook scans. + for (const hookFrameworkState of [TestFrameworkState.BEFORE_ALL, TestFrameworkState.BEFORE_EACH, TestFrameworkState.AFTER_EACH, TestFrameworkState.AFTER_ALL]) { + TestFramework.registerObserver(hookFrameworkState, HookState.PRE, this.onHookStart.bind(this)) + TestFramework.registerObserver(hookFrameworkState, HookState.POST, this.onHookEnd.bind(this)) + } this.accessibility = Boolean(accessibilityConfig) const accessibilityOptions = (BrowserstackCLI.getInstance().options as Record)?.accessibilityOptions as { [key: string]: string | boolean | undefined } this.autoScanning = Boolean(accessibilityOptions?.autoScanning ?? true) @@ -52,6 +64,39 @@ export default class AccessibilityModule extends BaseModule { this.isNonBstackA11y = isNonBstackA11y } + async onHookStart(args: Record) { + try { + const testInstance: TestFrameworkInstance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() + if (!testInstance || !autoInstance) { + return + } + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + // KEY_HOOK_ID is stamped on the instance at hook PRE (wdioMochaTestFramework.trackEvent) + // and is the SAME uuid reported to TestHub as the hook run, so a scan tagged with it can + // be self-joined onto the wrapping test in BTCER. + const hookRunUuid = TestFramework.getState(testInstance, TestFrameworkConstants.KEY_HOOK_ID) as string | undefined + this.currentHookRunUuid = hookRunUuid || null + + if (!this.accessibility) { + return + } + // Open the scan gate for the hook window so DOM-changing commands issued inside + // before/beforeEach/afterEach/after hooks trigger scans (web per-command path). The + // following onBeforeTest re-computes the per-test gate, so this only affects the hook. + if (this.autoScanning && sessionId !== undefined && sessionId !== null) { + this.accessibilityMap.set(sessionId, true) + } + } catch (error) { + this.logger.error(`Exception in accessibility onHookStart: ${error}`) + } + } + + async onHookEnd() { + // Hook finished: subsequent (test-body) scans must not be stamped with the hook uuid. + this.currentHookRunUuid = null + } + async onBeforeExecute() { try { const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() @@ -116,7 +161,8 @@ export default class AccessibilityModule extends BaseModule { if (!this.accessibility && !this.isAppAccessibility){ return } - return await this.performScanCli(browser) + // If invoked from inside a hook, currentHookRunUuid stamps the scan for the hook. + return await this.performScanCli(browser, undefined, this.currentHookRunUuid) } (browser as WebdriverIO.Browser).startA11yScanning = async () => { @@ -174,7 +220,7 @@ export default class AccessibilityModule extends BaseModule { !this.shouldPatchExecuteScript(args.length ? args[0] as string : null) ) { try { - await this.performScanCli(browser, command.name) + await this.performScanCli(browser, command.name, this.currentHookRunUuid) this.logger.debug(`Accessibility scan performed after ${command.name} command`) } catch (scanError) { this.logger.debug(`Error performing accessibility scan after ${command.name}: ${scanError}`) @@ -244,7 +290,7 @@ export default class AccessibilityModule extends BaseModule { if (!this.accessibility && !this.isAppAccessibility){ return } - const results = await this.performScanCli(browser) + const results = await this.performScanCli(browser, undefined, this.currentHookRunUuid) if (results){ const testIdentifier = String(testInstance.getContext().getId()) this.testMetadata[testIdentifier] = { @@ -387,7 +433,8 @@ export default class AccessibilityModule extends BaseModule { private async performScanCli( browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser, - commandName?: string + commandName?: string, + hookRunUuid?: string | null ): Promise | undefined> { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.A11Y_EVENTS.PERFORM_SCAN, @@ -400,7 +447,7 @@ export default class AccessibilityModule extends BaseModule { if (this.isAppAccessibility) { const testName=this.currentTestName || undefined const results: unknown = await (browser as WebdriverIO.Browser).execute( - formatString(this.scriptInstance.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName))) as string, + formatString(this.scriptInstance.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName, hookRunUuid))) as string, {} ) BStackLogger.debug(util.format(results as string)) diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index 8139506..a2c80a1 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -203,6 +203,12 @@ class _InsightsHandler { } } + /* Exposes the current hook run so other handlers (e.g. AccessibilityHandler) can + stamp the same hook UUID we report to TestHub as HookRunStarted. */ + getCurrentHook(): CurrentRunInfo { + return this._currentHook + } + async sendScenarioObjectSkipped(scenario: Scenario, feature: Feature, uri: string) { const testMetaData: TestMeta = { uuid: uuidv4(), diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index c5f21d1..3c81888 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -410,6 +410,9 @@ export default class BrowserstackService implements Services.ServiceInstance { } await this._insightsHandler?.beforeHook(test, context) + // Reuse the exact hook UUID InsightsHandler just reported to TestHub (HookRunStarted) + // so a11y hook scans carry a hook_run_uuid that matches the hook's BTCER row. + await this._accessibilityHandler?.beforeHook(test as Frameworks.Test, context, this._insightsHandler?.getCurrentHook()?.uuid) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterHook' }) @@ -440,6 +443,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } await this._insightsHandler?.afterHook(test, result) + await this._accessibilityHandler?.afterHook() } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeTest' }) diff --git a/packages/browserstack-service/src/util.ts b/packages/browserstack-service/src/util.ts index 136eebd..7a81048 100644 --- a/packages/browserstack-service/src/util.ts +++ b/packages/browserstack-service/src/util.ts @@ -571,9 +571,12 @@ export const formatString = (template: (string | null), ...values: (string | nul } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export const _getParamsForAppAccessibility = ( commandName?: string, testName?: string ): { thTestRunUuid: any, thBuildUuid: any, thJwtToken: any, authHeader: any, scanTimestamp: number, method: string | undefined, testName: string | undefined } => { +export const _getParamsForAppAccessibility = ( commandName?: string, testName?: string, hookRunUuid?: string | null ): { thTestRunUuid: any, thHookRunUuid: any, thBuildUuid: any, thJwtToken: any, authHeader: any, scanTimestamp: number, method: string | undefined, testName: string | undefined } => { return { 'thTestRunUuid': process.env.TEST_ANALYTICS_ID, + // Present only when the scan fires inside a hook (dropped by JSON.stringify when undefined, + // so in-test scans are unchanged). SeleniumHub appAllyHandler relays this as `hook_run_uuid`. + 'thHookRunUuid': hookRunUuid || undefined, 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT, 'authHeader': process.env.BSTACK_A11Y_JWT, @@ -584,7 +587,7 @@ export const _getParamsForAppAccessibility = ( commandName?: string, testName?: } /* eslint-disable @typescript-eslint/no-explicit-any */ -export const performA11yScan = async (isAppAutomate: boolean, browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser, isBrowserStackSession?: boolean, isAccessibility?: boolean | string, commandName?: string, testName?: string,) : Promise<{ [key: string]: any; } | undefined> => { +export const performA11yScan = async (isAppAutomate: boolean, browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser, isBrowserStackSession?: boolean, isAccessibility?: boolean | string, commandName?: string, testName?: string, hookRunUuid?: string | null,) : Promise<{ [key: string]: any; } | undefined> => { if (!isAccessibilityAutomationSession(isAccessibility)) { BStackLogger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.') @@ -593,7 +596,7 @@ export const performA11yScan = async (isAppAutomate: boolean, browser: Webdriver try { if (isAppAccessibilityAutomationSession(isAccessibility, isAppAutomate)) { - const results: unknown = await (browser as WebdriverIO.Browser).execute(formatString(AccessibilityScripts.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName))) as string, {}) + const results: unknown = await (browser as WebdriverIO.Browser).execute(formatString(AccessibilityScripts.performScan, JSON.stringify(_getParamsForAppAccessibility(commandName, testName, hookRunUuid))) as string, {}) BStackLogger.debug(util.format(results as string)) return ( results as { [key: string]: any; } | undefined ) } diff --git a/packages/browserstack-service/tests/accessibility-handler.test.ts b/packages/browserstack-service/tests/accessibility-handler.test.ts index ddaf472..417003e 100644 --- a/packages/browserstack-service/tests/accessibility-handler.test.ts +++ b/packages/browserstack-service/tests/accessibility-handler.test.ts @@ -433,6 +433,82 @@ describe('afterScenario', () => { }) }) +describe('beforeHook / afterHook (hook scans)', () => { + beforeEach(() => { + accessibilityHandler = new AccessibilityHandler(browser, caps, options, false, config, 'mocha', true, false, accessibilityOpts) + vi.spyOn(utils, 'isBrowserstackSession').mockReturnValue(true) + vi.spyOn(utils, 'isAccessibilityAutomationSession').mockReturnValue(true) + accessibilityHandler['_sessionId'] = 'session123' + }) + + it('stamps the hook run uuid inside a supported mocha per-test hook', async () => { + vi.spyOn(utils, 'shouldScanTestForAccessibility').mockReturnValue(true) + await accessibilityHandler.beforeHook( + { title: '"before each" hook', parent: 'suite' } as any, + { currentTest: { parent: 'suite', title: 'test' } }, + 'hook-uuid-1' + ) + expect(accessibilityHandler['_currentHookRunUuid']).toBe('hook-uuid-1') + }) + + it('clears the hook run uuid on afterHook so test-body scans are not stamped', async () => { + await accessibilityHandler.beforeHook( + { title: '"after each" hook', parent: 'suite' } as any, + { currentTest: { parent: 'suite', title: 'test' } }, + 'hook-uuid-2' + ) + expect(accessibilityHandler['_currentHookRunUuid']).toBe('hook-uuid-2') + await accessibilityHandler.afterHook({ title: '"after each" hook' } as any, {}, { passed: true } as any, 'hook-uuid-2') + expect(accessibilityHandler['_currentHookRunUuid']).toBeNull() + }) + + it('does not stamp when the framework does not support hooks', async () => { + const jasmineHandler = new AccessibilityHandler(browser, caps, options, false, config, 'jasmine', true, false, accessibilityOpts) + jasmineHandler['_sessionId'] = 'session123' + await jasmineHandler.beforeHook({ title: '"before each" hook' } as any, {}, 'hook-uuid-3') + expect(jasmineHandler['_currentHookRunUuid']).toBeNull() + }) + + it('actually PASSES the hook uuid to performA11yScan for a scan fired inside a hook', async () => { + vi.spyOn(utils, 'shouldScanTestForAccessibility').mockReturnValue(true) + const scanSpy = vi.spyOn(utils, 'performA11yScan').mockResolvedValue(undefined) + // enter a hook window -> sets the scan gate + _currentHookRunUuid + await accessibilityHandler.beforeHook( + { title: '"before each" hook', parent: 'suite' } as any, + { currentTest: { parent: 'suite', title: 'test' } }, + 'hook-uuid-42' + ) + // simulate a wrapped DOM-changing command running inside the hook + const orig = vi.fn().mockResolvedValue('ok') + await accessibilityHandler['commandWrapper']({ name: 'click', class: 'Element' } as any, undefined as any, orig, 'arg') + expect(scanSpy).toHaveBeenCalled() + const lastCall = scanSpy.mock.calls[scanSpy.mock.calls.length - 1] + // performA11yScan(isAppAutomate, browser, isBS, isA11y, commandName, testName, hookRunUuid) + expect(lastCall[lastCall.length - 1]).toBe('hook-uuid-42') + }) + + it('does NOT stamp a hook uuid on a test-body scan (afterHook cleared it)', async () => { + vi.spyOn(utils, 'shouldScanTestForAccessibility').mockReturnValue(true) + const scanSpy = vi.spyOn(utils, 'performA11yScan').mockResolvedValue(undefined) + await accessibilityHandler.beforeHook( + { title: '"before each" hook', parent: 'suite' } as any, + { currentTest: { parent: 'suite', title: 'test' } }, + 'hook-uuid-77' + ) + await accessibilityHandler.afterHook() // hook ends -> uuid cleared, gate remains on + const orig = vi.fn().mockResolvedValue('ok') + await accessibilityHandler['commandWrapper']({ name: 'click', class: 'Element' } as any, undefined as any, orig, 'arg') + expect(scanSpy).toHaveBeenCalled() + const lastCall = scanSpy.mock.calls[scanSpy.mock.calls.length - 1] + expect(lastCall[lastCall.length - 1]).toBeNull() + }) + + it('_getParamsForAppAccessibility puts the hook uuid on the scan payload as thHookRunUuid', () => { + expect(utils._getParamsForAppAccessibility('click', 'testName', 'hook-uuid-9').thHookRunUuid).toBe('hook-uuid-9') + expect(utils._getParamsForAppAccessibility('click', 'testName').thHookRunUuid).toBeUndefined() + }) +}) + describe('beforeTest', () => { let executeAsyncSpy: any let executeSpy: any diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 00c4443..561abba 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -51,7 +51,7 @@ vi.mock('../../../src/cli/grpcClient.js', () => ({ })) import AccessibilityModule from '../../../src/cli/modules/accessibilityModule.js' -import { validateCapsWithA11y, validateCapsWithAppA11y } from '../../../src/util.js' +import { validateCapsWithA11y, validateCapsWithAppA11y, _getParamsForAppAccessibility } from '../../../src/util.js' import accessibilityScripts from '../../../src/scripts/accessibility-scripts.js' import TestFramework from '../../../src/cli/frameworks/testFramework.js' import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' @@ -443,4 +443,71 @@ describe('AccessibilityModule', () => { ) }) }) + + // APPA11Y-5542: scans fired inside test hooks must carry the hook's run uuid + // (thHookRunUuid) so the backend (SeleniumHub appAllyScan -> hook_run_uuid) can + // reconcile them onto the wrapping test. onHookStart captures the uuid + opens the + // scan gate for the hook window; onHookEnd clears it so test-body scans are unstamped. + describe('onHookStart / onHookEnd (hook scans)', () => { + it('registers hook observers for every hook lifecycle state (PRE + POST)', () => { + for (const state of [ + TestFrameworkState.BEFORE_ALL, + TestFrameworkState.BEFORE_EACH, + TestFrameworkState.AFTER_EACH, + TestFrameworkState.AFTER_ALL + ]) { + expect(TestFramework.registerObserver).toHaveBeenCalledWith(state, HookState.PRE, expect.any(Function)) + expect(TestFramework.registerObserver).toHaveBeenCalledWith(state, HookState.POST, expect.any(Function)) + } + }) + + it('captures the hook run uuid and opens the scan gate at hook start', async () => { + vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + vi.mocked(AutomationFramework.getState).mockImplementation((instance: any, key: string) => + (key.includes('session_id') ? 12345 : {}) as any) + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(true) + }) + + it('does not open the scan gate when accessibility is disabled', async () => { + accessibilityModule.accessibility = false + vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBeUndefined() + }) + + it('clears the hook run uuid at hook end so later test-body scans are unstamped', async () => { + accessibilityModule.currentHookRunUuid = 'hook-uuid-123' + + await accessibilityModule.onHookEnd() + + expect(accessibilityModule.currentHookRunUuid).toBeNull() + }) + + it('threads the hook run uuid into the app scan payload (thHookRunUuid)', async () => { + accessibilityModule.accessibility = true + accessibilityModule.isAppAccessibility = true + mockBrowser.execute.mockResolvedValue({ scanned: true }) + + await (accessibilityModule as any).performScanCli(mockBrowser, 'click', 'hook-uuid-99') + + expect(_getParamsForAppAccessibility).toHaveBeenCalledWith('click', undefined, 'hook-uuid-99') + }) + + it('passes no hook uuid for an ordinary (non-hook) app scan', async () => { + accessibilityModule.accessibility = true + accessibilityModule.isAppAccessibility = true + mockBrowser.execute.mockResolvedValue({ scanned: true }) + + await (accessibilityModule as any).performScanCli(mockBrowser, 'click') + + expect(_getParamsForAppAccessibility).toHaveBeenCalledWith('click', undefined, undefined) + }) + }) }) \ No newline at end of file