diff --git a/package-lock.json b/package-lock.json index ab7ef4c..eb7394e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9456,7 +9456,7 @@ "undici": "^6.24.0", "uuid": "^11.1.0", "winston-transport": "^4.5.0", - "yauzl": "^3.0.0" + "yauzl": "^3.4.0" }, "devDependencies": { "@bufbuild/buf": "^1.55.1", diff --git a/packages/browserstack-service/package.json b/packages/browserstack-service/package.json index 2651bd0..66fc361 100644 --- a/packages/browserstack-service/package.json +++ b/packages/browserstack-service/package.json @@ -69,7 +69,7 @@ "undici": "^6.24.0", "uuid": "^11.1.0", "winston-transport": "^4.5.0", - "yauzl": "^3.0.0", + "yauzl": "^3.4.0", "@wdio/logger": "^9.0.0", "@wdio/reporter": "^9.0.0", "@wdio/types": "^9.0.0" diff --git a/packages/browserstack-service/src/@types/bstack-service-types.d.ts b/packages/browserstack-service/src/@types/bstack-service-types.d.ts index ff43c67..1ae813f 100644 --- a/packages/browserstack-service/src/@types/bstack-service-types.d.ts +++ b/packages/browserstack-service/src/@types/bstack-service-types.d.ts @@ -1,17 +1,16 @@ +// The WebdriverIO.Browser accessibility augmentations +// (getAccessibilityResultsSummary, getAccessibilityResults, performScan, +// startA11yScanning, stopA11yScanning) now live in src/index.ts inside +// `declare global { namespace WebdriverIO { ... } }`, so that tsc emits them +// into build/index.d.ts for consumers. They were moved out of this ambient +// file to avoid duplicate-declaration conflicts. The setCustomTags augmentation +// stays here (it is not part of the a11y type-shipping fix). declare namespace WebdriverIO { interface Browser { - getAccessibilityResultsSummary: () => Promise>, - getAccessibilityResults: () => Promise>>, - performScan: () => Promise | undefined>, - startA11yScanning: () => Promise, - stopA11yScanning: () => Promise + setCustomTags: (key: string, value: string) => Promise } interface MultiRemoteBrowser { - getAccessibilityResultsSummary: () => Promise>, - getAccessibilityResults: () => Promise>>, - performScan: () => Promise | undefined>, - startA11yScanning: () => Promise, - stopA11yScanning: () => Promise + setCustomTags: (key: string, value: string) => Promise } } diff --git a/packages/browserstack-service/src/accessibility-handler.ts b/packages/browserstack-service/src/accessibility-handler.ts index 76ae832..ff1ef90 100644 --- a/packages/browserstack-service/src/accessibility-handler.ts +++ b/packages/browserstack-service/src/accessibility-handler.ts @@ -1,4 +1,3 @@ -/// import util from 'node:util' import type { Capabilities, Frameworks, Options } from '@wdio/types' diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index e321352..594fb63 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -21,6 +21,7 @@ export const TestFrameworkConstants = { KEY_TEST_FAILURE_REASON : 'test_failure_reason', KEY_TEST_LOGS : 'test_logs', KEY_TEST_META : 'test_meta', + KEY_CUSTOM_TAGS : 'custom_metadata', KEY_TEST_DEFERRED : 'test_deferred', KEY_SESSION_NAME : 'test_session_name', KEY_AUTOMATE_SESSION_NAME : 'automate_session_name', diff --git a/packages/browserstack-service/src/cli/grpcClient.ts b/packages/browserstack-service/src/cli/grpcClient.ts index ddbb7f5..bbd325b 100644 --- a/packages/browserstack-service/src/cli/grpcClient.ts +++ b/packages/browserstack-service/src/cli/grpcClient.ts @@ -300,8 +300,17 @@ export class GrpcClient { } const { platformIndex, testFrameworkName, testFrameworkVersion, testFrameworkState, testHookState, testUuid, automationSessions, capabilities, executionContext } = data const sessions = automationSessions.map((automationSession) => { + // LTS: pass through optional product field so testHubModule's + // ltsActive override (sets product='loadTesting' on the + // AutomationSession) actually reaches the binary via the + // gRPC TestSessionEvent payload. Without this passthrough + // the binary's makeIntegrations falls back to 'automate' + // and test_run.origin lands as Automate instead of + // LoadTesting. Mirror of py-sdk a245a814 + + // browserstack-binary's existing s?.product read. return AutomationSessionConstructor.create({ provider: automationSession.provider, + product: automationSession.product, frameworkName: automationSession.frameworkName, frameworkVersion: automationSession.frameworkVersion, frameworkSessionId: automationSession.frameworkSessionId, diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 6c4b6eb..d7b005c 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -19,6 +19,7 @@ import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' +import CustomTagsModule from './modules/customTagsModule.js' import { isTurboScale, processAccessibilityResponse, shouldAddServiceVersion } from '../util.js' import ObservabilityModule from './modules/observabilityModule.js' import type { BrowserstackConfig, BrowserstackOptions, LaunchResponse } from '../types.js' @@ -166,6 +167,10 @@ export class BrowserstackCLI { this.modules[TestHubModule.MODULE_NAME] = new TestHubModule(startBinResponse.testhub) + // Custom-tag (multi Test-Case-ID) tagging rides the per-test event_json + // to TestHub, so it is gated on the testhub pipeline being active. + this.modules[CustomTagsModule.MODULE_NAME] = new CustomTagsModule() + if (startBinResponse.accessibility?.success){ process.env[BROWSERSTACK_ACCESSIBILITY] = 'true' const options = this.options as BrowserstackConfig & BrowserstackOptions diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 5fa5267..7ed1b58 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -1,4 +1,3 @@ -/// import BaseModule from './baseModule.js' import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' diff --git a/packages/browserstack-service/src/cli/modules/customTagsModule.ts b/packages/browserstack-service/src/cli/modules/customTagsModule.ts new file mode 100644 index 0000000..34b47f7 --- /dev/null +++ b/packages/browserstack-service/src/cli/modules/customTagsModule.ts @@ -0,0 +1,122 @@ +/// +import BaseModule from './baseModule.js' +import { BStackLogger } from '../cliLogger.js' +import TestFramework from '../frameworks/testFramework.js' +import AutomationFramework from '../frameworks/automationFramework.js' +import type AutomationFrameworkInstance from '../instances/automationFrameworkInstance.js' +import type TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import { AutomationFrameworkState } from '../states/automationFrameworkState.js' +import { HookState } from '../states/hookState.js' +import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' +import { CLIUtils } from '../cliUtils.js' +import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' +import { mergeIntoTags, parseCommaSeparatedValues } from '../../customTags.js' +import type { CustomMetadata } from '../../customTags.js' + +/** + * CustomTagsModule — CLI/gRPC path registration for `browser.setCustomTags`. + * + * Mirrors AccessibilityModule: registers the browser method in onBeforeExecute() + * (observer-bound to AutomationFrameworkState.CREATE / HookState.POST), instantiated + * from BrowserstackCLI.loadModules() whenever the binary is up. + * + * The per-test custom_metadata is merged directly into the tracked + * TestFrameworkInstance map under KEY_CUSTOM_TAGS ('custom_metadata'). That whole + * map is serialized verbatim into event_json (testHubModule.sendTestFrameworkEvent), + * so the binary reads it as event.custom_metadata. The instance is created fresh + * per test, so per-instance merge gives test-scoped union/dedupe; the binary is a + * strict pass-through and does NOT consolidate across events. + */ +export default class CustomTagsModule extends BaseModule { + + logger = BStackLogger + name: string + static MODULE_NAME = 'CustomTagsModule' + + constructor() { + super() + this.name = 'CustomTagsModule' + AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + } + + getModuleName() { + return CustomTagsModule.MODULE_NAME + } + + async onBeforeExecute() { + try { + const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() + if (!autoInstance) { + this.logger.debug('CustomTagsModule: No tracked automation instance found!') + return + } + + const browser = AutomationFramework.getDriver(autoInstance) as WebdriverIO.Browser + if (!browser) { + this.logger.debug('CustomTagsModule: No browser instance found for setCustomTags registration') + return + } + + (browser as WebdriverIO.Browser).setCustomTags = async (key: string, value: string): Promise => { + try { + if (!key || !value) { + this.logger.warn('setCustomTags: key and value are required; ignoring call') + return + } + + const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() + if (!testInstance) { + this.logger.warn('setCustomTags called outside of a resolvable test context; ignoring') + return + } + + const values = parseCommaSeparatedValues(value) + if (values.length === 0) { + this.logger.warn(`setCustomTags: no usable values parsed from "${value}"; ignoring call`) + return + } + + // Merge into the per-test instance map (test-scoped accumulator). + const existing = (TestFramework.getState(testInstance, TestFrameworkConstants.KEY_CUSTOM_TAGS) as CustomMetadata) || {} + mergeIntoTags(existing, key, values) + testInstance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_CUSTOM_TAGS]: existing + }) + + // Hook-level: if a hook is currently active, also stamp custom_metadata + // onto the active hook so binary reads hook.custom_metadata. + this.applyToActiveHook(testInstance, key, values) + + this.logger.debug(`setCustomTags: merged key=${key} values=${JSON.stringify(values)}`) + } catch (error) { + this.logger.warn(`setCustomTags: error while recording custom tags: ${error}`) + } + } + } catch (error) { + this.logger.error(`Error in CustomTagsModule.onBeforeExecute: ${error}`) + } + } + + /** + * If the test framework is currently inside a hook, merge custom_metadata onto + * the last-started hook record so the binary receives hook.custom_metadata. + * Best-effort: silently no-ops when no active hook is resolvable. + */ + private applyToActiveHook(testInstance: TestFrameworkInstance, key: string, values: string[]) { + try { + const currentState = testInstance.getCurrentTestState().toString() + if (!CLIUtils.matchHookRegex(currentState)) { + return + } + const hook = WdioMochaTestFramework.lastActiveHook(testInstance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) + if (!hook) { + return + } + const hookTags = (hook[TestFrameworkConstants.KEY_CUSTOM_TAGS] as CustomMetadata) || {} + mergeIntoTags(hookTags, key, values) + hook[TestFrameworkConstants.KEY_CUSTOM_TAGS] = hookTags + } catch (error) { + this.logger.debug(`setCustomTags: could not apply to active hook: ${error}`) + } + } +} diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index eefe4b3..24915e8 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -15,6 +15,7 @@ import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' import type AutomationFrameworkInstance from '../instances/automationFrameworkInstance.js' import AutomationFramework from '../frameworks/automationFramework.js' import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js' +import { isLoadTestingSession, getLtsSessionId } from '../../util.js' /** * TestHub Module for BrowserStack @@ -182,22 +183,49 @@ export default class TestHubModule extends BaseModule { } this.logger.debug(`sendTestSessionEvent: instance iteration ${JSON.stringify(autoInstances)}`) + // LTS: BLU runner pods route through a local Selenium hub + // so KEY_IS_BROWSERSTACK_HUB resolves to false — without + // this gate the AutomationSession lands with provider= + // unknown_grid and TestHub's o11y classifier sets + // test_run.origin=UnknownGrid. Mirrors py-sdk 3af3bba6 + // (force provider='browserstack' + product='loadTesting' + // under LTS) and 0efca1ae (override frameworkSessionId + // with the LTS pod-iteration env id). + // Hoisted above the loop — env vars are stable for the + // process lifetime, no need to re-read per iteration. + const ltsActive = isLoadTestingSession() + const ltsSessionId = ltsActive ? getLtsSessionId() : '' // Process automation instances for (const autoInstance of autoInstances) { - const sessionProvider = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_IS_BROWSERSTACK_HUB) as boolean + const sessionProvider = ltsActive ? 'browserstack' - : 'unknown_grid' + : (AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_IS_BROWSERSTACK_HUB) as boolean + ? 'browserstack' + : 'unknown_grid') + + // Null-safe: under LTS the local-Selenium AutomationFrameworkInstance + // may not have KEY_FRAMEWORK_SESSION_ID populated yet (the binary's + // hub assigns sessionId later than KEY_IS_BROWSERSTACK_HUB). Without + // the guard, AutomationFramework.getState returns undefined and the + // chained .toString() throws TypeError before the (ltsActive && + // ltsSessionId) ternary on line 215 has a chance to fall through to + // ltsSessionId. Default to '' so the ternary path stays untouched + // and the non-LTS fall-through is still safe. + const driverFrameworkSessionId = ( + AutomationFramework.getState( + autoInstance, + AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID, + )?.toString() ?? '' + ) const automationSession: AutomationSession = { provider: sessionProvider, ref: autoInstance.getRef(), hubUrl: this.config.hubUrl as string, - frameworkSessionId: AutomationFramework.getState( - autoInstance, - AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID, - ).toString(), + frameworkSessionId: (ltsActive && ltsSessionId) ? ltsSessionId : driverFrameworkSessionId, frameworkName: autoInstance.frameworkName, - frameworkVersion: autoInstance.frameworkVersion + frameworkVersion: autoInstance.frameworkVersion, + ...(ltsActive ? { product: 'loadTesting' } : {}) } this.logger.debug(`sendTestSessionEvent: automationSession: ${JSON.stringify(automationSession)}`) diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index 9486a1a..759b322 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -125,6 +125,13 @@ export const BROWSERSTACK_ACCESSIBILITY = 'BROWSERSTACK_ACCESSIBILITY' // Whether session is a test reporting session export const BROWSERSTACK_OBSERVABILITY = 'BROWSERSTACK_OBSERVABILITY' +// Load Testing Service (LTS) env vars: the pod-iteration session id (its +// presence is the single source of truth for "this run is an LTS pod +// iteration") and an optional opt-in flag for local repro / CI runs that +// don't have a real session id. +export const BROWSERSTACK_LTS_SESSION_ID = 'BROWSERSTACK_LTS_SESSION_ID' +export const BROWSERSTACK_LTS = 'BROWSERSTACK_LTS' + // New Test Reporting and Analytics environment variables export const BROWSERSTACK_TEST_REPORTING = 'BROWSERSTACK_TEST_REPORTING' export const BROWSERSTACK_TEST_REPORTING_DEBUG = 'BROWSERSTACK_TEST_REPORTING_DEBUG' diff --git a/packages/browserstack-service/src/custom-tags-handler.ts b/packages/browserstack-service/src/custom-tags-handler.ts new file mode 100644 index 0000000..68fe0ab --- /dev/null +++ b/packages/browserstack-service/src/custom-tags-handler.ts @@ -0,0 +1,91 @@ +/// +import InsightsHandler from './insights-handler.js' +import { CustomTagAccumulator } from './customTags.js' +import type { CustomMetadata } from './customTags.js' +import { o11yClassErrorHandler } from './util.js' +import { BStackLogger } from './bstackLogger.js' + +/** + * Legacy/Listener-path handler for `browser.setCustomTags`. + * + * Registered from service.ts in the `!BrowserstackCLI.getInstance().isRunning()` + * branch (alongside AccessibilityHandler.before() / InsightsHandler.before()). + * Deliberately NOT folded into InsightsHandler.before() — that method registers + * no browser methods and is SKIPPED when the CLI is running. + * + * Accumulates (key, value) pairs per active test, keyed on + * InsightsHandler.currentTest.uuid (set in beforeTest / setTestData). The + * accumulator is static so InsightsHandler.getRunData can drain it into the + * TestData.custom_metadata field at TestRunFinished (flushed in afterTest). + */ +class _CustomTagsHandler { + // Static so the flush sink (InsightsHandler.getRunData) can read it without a + // handler reference. Keyed on the per-test uuid. + public static accumulator: CustomTagAccumulator = new CustomTagAccumulator() + + constructor( + private _browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser, + private _caps: WebdriverIO.Capabilities | Record, + private _framework?: string + ) {} + + before() { + const register = (browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser) => { + (browser as WebdriverIO.Browser).setCustomTags = async (key: string, value: string): Promise => { + try { + if (this._framework !== 'mocha') { + BStackLogger.warn('setCustomTags is only supported for the mocha framework; ignoring call') + return + } + if (!key || !value) { + BStackLogger.warn('setCustomTags: key and value are required; ignoring call') + return + } + const testUuid = InsightsHandler.currentTest.uuid + if (!testUuid) { + BStackLogger.warn('setCustomTags called outside of a resolvable test context; ignoring') + return + } + const added = _CustomTagsHandler.accumulator.add(testUuid, key, value) + if (!added) { + BStackLogger.warn(`setCustomTags: no usable values parsed from "${value}"; ignoring call`) + return + } + BStackLogger.debug(`setCustomTags: recorded key=${key} value="${value}" for test=${testUuid}`) + } catch (error) { + BStackLogger.warn(`setCustomTags: error while recording custom tags: ${error}`) + } + } + } + + // Fan out to every MultiRemote instance (mirror service._executeCommand), + // plus the top-level browser object the user calls directly. + register(this._browser) + if ((this._browser as WebdriverIO.Browser).isMultiremote) { + const multiRemoteBrowser = this._browser as unknown as WebdriverIO.MultiRemoteBrowser + Object.keys(this._caps).forEach((browserName) => { + try { + const instance = multiRemoteBrowser.getInstance(browserName) + if (instance) { + register(instance) + } + } catch (error) { + BStackLogger.debug(`setCustomTags: could not register on multiremote instance ${browserName}: ${error}`) + } + }) + } + } + + /** Drain the accumulated custom_metadata for a test uuid; clears after read. */ + static drain(testUuid: string | undefined): CustomMetadata | undefined { + const data = _CustomTagsHandler.accumulator.get(testUuid) + _CustomTagsHandler.accumulator.clear(testUuid) + return data + } +} + +// https://github.com/microsoft/TypeScript/issues/6543 +const CustomTagsHandler: typeof _CustomTagsHandler = o11yClassErrorHandler(_CustomTagsHandler) +type CustomTagsHandler = _CustomTagsHandler + +export default CustomTagsHandler diff --git a/packages/browserstack-service/src/customTags.ts b/packages/browserstack-service/src/customTags.ts new file mode 100644 index 0000000..199251d --- /dev/null +++ b/packages/browserstack-service/src/customTags.ts @@ -0,0 +1,133 @@ +/** + * Helpers for one-to-many custom-tag (Test-Case-ID) tagging. + * + * Both runtime paths (CLI/gRPC module + legacy listener handler) funnel through + * the same tokenizer + merge engine so they produce an identical `custom_metadata` + * payload shape: + * { [key]: { field_type: 'multi_dropdown', values: [...] } } + * + * The binary is a strict pass-through for per-test/per-hook `custom_metadata` — it + * does NOT merge or dedupe across events — so the union+dedupe (set semantics) + * MUST be completed SDK-side before serialization. Value parsing mirrors the + * node-agent quote-aware tokenizer (helper.js `parseCommaSeparatedValues`). + */ + +export interface CustomMetadataEntry { + field_type: 'multi_dropdown' + values: string[] +} + +export type CustomMetadata = Record + +/** + * Quote-aware comma split. Mirrors node-agent `helper.js` tokenizer + * `/\s*"([^"]*)"\s*|[^,]+/g` — a quoted body is preserved verbatim (so an + * embedded comma is kept), otherwise the raw token is trimmed; empties dropped. + * '"checkout,cart", header' -> ['checkout,cart', 'header'] + * 'TC-1, TC-2' -> ['TC-1', 'TC-2'] + * Backward compatible: unquoted input tokenizes identically to a naive + * split(',').map(trim).filter(Boolean). + */ +export function parseCommaSeparatedValues(value: string | string[] | undefined | null): string[] { + if (!value) { + return [] + } + + if (Array.isArray(value)) { + return value + } + + const str = value.toString() + if (str.trim() === '') { + return [] + } + + const tokenizer = /\s*"([^"]*)"\s*|[^,]+/g + const out: string[] = [] + let match: RegExpExecArray | null + while ((match = tokenizer.exec(str)) !== null) { + // Group 1 = quoted body (preserved verbatim); otherwise trim the raw match. + const token = match[1] !== undefined ? match[1] : match[0].trim() + if (token !== '') { + out.push(token) + } + } + return out +} + +/** + * Union + dedupe, existing-first ordering. Mirrors node-agent + * `CustomTagManager.mergeValues` — repeated calls accumulate, they do not override. + */ +export function mergeValues(existingValues: string[] | undefined | null, incomingValues: string[] | undefined | null): string[] { + return Array.from(new Set([...(existingValues || []), ...(incomingValues || [])])) +} + +/** + * Create-or-merge a single tag key inside a `custom_metadata` container. + * Always produces `{ field_type: 'multi_dropdown', values: [...] }`; merges values + * via mergeValues. The container is mutated in place and returned for chaining. + * Mirrors node-agent `CustomTagManager.mergeIntoTags`. + */ +export function mergeIntoTags(tagsContainer: CustomMetadata, key: string, incomingValues: string[]): CustomMetadata { + if (!tagsContainer[key]) { + tagsContainer[key] = { + field_type: 'multi_dropdown', + values: [...(incomingValues || [])] + } + } else { + // Defensive: tolerate malformed pre-existing entries. + if (!tagsContainer[key].field_type) { + tagsContainer[key].field_type = 'multi_dropdown' + } + if (!Array.isArray(tagsContainer[key].values)) { + tagsContainer[key].values = [] + } + tagsContainer[key].values = mergeValues(tagsContainer[key].values, incomingValues) + } + + return tagsContainer +} + +/** + * Per-test-uuid accumulator. Both paths key on `InsightsHandler.currentTest.uuid` + * so repeated `setCustomTags` calls within a single test union into one entry, and + * the flush sink reads the final consolidated map for that test. + */ +export class CustomTagAccumulator { + private store: Map = new Map() + + /** + * Add (key, value) for a test uuid. value is quote-aware comma-split; the + * resulting values union+dedupe into the test's custom_metadata for that key. + * Returns false (no-op) when uuid/key/value resolve to nothing. + */ + add(testUuid: string | undefined, key: string, value: string): boolean { + if (!testUuid || !key) { + return false + } + const values = parseCommaSeparatedValues(value) + if (values.length === 0) { + return false + } + const container = this.store.get(testUuid) || {} + mergeIntoTags(container, key, values) + this.store.set(testUuid, container) + return true + } + + /** Final consolidated custom_metadata for a test uuid, or undefined if none. */ + get(testUuid: string | undefined): CustomMetadata | undefined { + if (!testUuid) { + return undefined + } + return this.store.get(testUuid) + } + + /** Drop the accumulated entry once flushed (avoid unbounded growth). */ + clear(testUuid: string | undefined): void { + if (testUuid) { + this.store.delete(testUuid) + } + } +} diff --git a/packages/browserstack-service/src/index.ts b/packages/browserstack-service/src/index.ts index 32b65e4..1c9ca3e 100644 --- a/packages/browserstack-service/src/index.ts +++ b/packages/browserstack-service/src/index.ts @@ -20,6 +20,14 @@ export * from './types.js' declare global { namespace WebdriverIO { interface ServiceOption extends BrowserstackConfig {} + + interface Browser { + getAccessibilityResultsSummary: () => Promise>, + getAccessibilityResults: () => Promise>>, + performScan: () => Promise | undefined>, + startA11yScanning: () => Promise, + stopA11yScanning: () => Promise + } } interface State { value: number, diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index af202d3..8139506 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -19,6 +19,8 @@ import { getUniqueIdentifier, getUniqueIdentifierForCucumber, isBrowserstackSession, + isLoadTestingSession, + getLtsSessionId, isScreenshotCommand, isUndefined, o11yClassErrorHandler, @@ -44,6 +46,7 @@ import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { HookState } from './cli/states/hookState.js' import PerformanceTester from './instrumentation/performance/performance-tester.js' import * as PERFORMANCE_SDK_EVENTS from './instrumentation/performance/constants.js' +import CustomTagsHandler from './custom-tags-handler.js' class _InsightsHandler { private _tests: Record = {} @@ -65,6 +68,13 @@ class _InsightsHandler { public currentTestId: string | undefined public cbtQueue: Array = [] + // LTS gates cached at construction. Env vars are stable for the JVM/process + // lifetime — recomputing on every event was 6 process.env reads per test + // event (5 inline ternaries across getRunData/cucumber paths + 1 in + // getIntegrationsObject). Cache once + reuse. + private readonly _ltsActive: boolean + private readonly _ltsSessionId: string + constructor (private _browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser, private _framework?: string, _userCaps?: Capabilities.ResolvedTestrunnerCapabilities, _options?: BrowserstackConfig & BrowserstackOptions) { const caps = (this._browser as WebdriverIO.Browser).capabilities as WebdriverIO.Capabilities const sessionId = (this._browser as WebdriverIO.Browser).sessionId @@ -72,6 +82,19 @@ class _InsightsHandler { this._userCaps = _userCaps this._options = _options + this._ltsActive = isLoadTestingSession() + this._ltsSessionId = this._ltsActive ? getLtsSessionId() : '' + // One-time misconfig warning: BROWSERSTACK_LTS=true is the documented + // local-repro opt-in flag, but without BROWSERSTACK_LTS_SESSION_ID + // (which the LTS pod infrastructure normally injects), event emission + // will pair product='loadTesting' with the WebDriver-assigned + // session_id rather than the pod-iteration id. Backend won't be able + // to correlate the row to an LTS pod iteration. Surface the + // misconfiguration once instead of silently shipping mismatched rows. + if (this._ltsActive && !this._ltsSessionId) { + BStackLogger.warn('LTS active (BROWSERSTACK_LTS=true) but BROWSERSTACK_LTS_SESSION_ID is not set — event payload will pair product=loadTesting with WebDriver-assigned session_id rather than the pod-iteration id. Set BROWSERSTACK_LTS_SESSION_ID to silence this warning.') + } + this._platformMeta = { browserName: caps?.browserName, browserVersion: caps?.browserVersion, @@ -269,7 +292,13 @@ class _InsightsHandler { this._tests[fullTitle] = { uuid: hookUUID, - startedAt: (new Date()).toISOString() + startedAt: (new Date()).toISOString(), + // Tag as a hook and stash identity so a teardown sweep can synthesise a terminal + // HookRunFinished if this hook never returns (e.g. a hang with timeouts disabled). + kind: 'hook', + name: test.title || test.description, + scopes: this.getHierarchy(test), + fileName: test.file || this._suiteFile } this.setCurrentHook({ uuid: hookUUID }) this.attachHookData(context, hookUUID) @@ -296,7 +325,17 @@ class _InsightsHandler { } this.setCurrentHook({ uuid: this._tests[fullTitle].uuid, finished: true }) - this.listener.hookFinished(this.getRunData(test, 'HookRunFinished', result)) + const hookData = this.getRunData(test, 'HookRunFinished', result) + // never emit a finish with a null/undefined uuid — the backend cannot match it to a + // HookRunStarted, leaving the hook orphaned. Same guard as afterTest: a missing entry + // (mislabelled identity upstream) degrades getRunData to an empty meta with no uuid. + // The skip-propagation below seeds its own fresh uuids per skipped test, so it stays + // outside the guard. + if (hookData.uuid) { + this.listener.hookFinished(hookData) + } else { + BStackLogger.warn(`Skipping HookRunFinished for '${fullTitle}' — resolved uuid is missing; not emitting an unmatched finish.`) + } const hookType = getHookType(test.title) /* @@ -374,7 +413,8 @@ class _InsightsHandler { if (eventType === 'HookRunStarted') { testData.integrations = {} if (this._browser && this._platformMeta) { - const provider = getCloudProvider(this._browser) + // LTS reporter override (see getCloudProvider note in util.ts) + const provider = this._ltsActive ? 'browserstack' : getCloudProvider(this._browser) testData.integrations[provider] = this.getIntegrationsObject() } } @@ -393,7 +433,13 @@ class _InsightsHandler { const fullTitle = getUniqueIdentifier(test, this._framework) this._tests[fullTitle] = { uuid, - startedAt: (new Date()).toISOString() + startedAt: (new Date()).toISOString(), + // Tag as a test and stash identity so a teardown sweep can synthesise a terminal + // TestRunFinished if this test never reaches afterTest (e.g. mocha advanced past a hang). + kind: 'test', + name: test.title || test.description, + scopes: this.getHierarchy(test), + fileName: test.file || this._suiteFile } this.listener.testStarted(this.getRunData(test, 'TestRunStarted')) } @@ -409,6 +455,14 @@ class _InsightsHandler { } this.flushCBTDataQueue() const testData = this.getRunData(test, 'TestRunFinished', result) + // never emit a finish with a null/undefined uuid — the backend cannot match it to + // a TestRunStarted, leaving the started test orphaned (status_stats in_progress). The + // snapshotted-identity correlation in service.ts (afterTest trusts originalTest) should + // prevent this; this guard is defence-in-depth. + if (!testData.uuid) { + BStackLogger.warn(`Skipping TestRunFinished for '${getUniqueIdentifier(test, this._framework)}' — resolved uuid is missing; not emitting an unmatched finish.`) + return + } this.listener.testFinished(testData) const testFinishHashCode = generateHashCodeFromFields( [ @@ -425,6 +479,117 @@ class _InsightsHandler { TestReporter.hashCodeToHandleTestSkip[testFinishHashCode] = testData.uuid ?? '' } + /** + * Teardown safety net: synthesise a terminal finish for any started-but-unfinished + * test/hook so the backend closes the entity instead of leaving it in_progress and + * letting the build watchdog inflate the duration. + * + * Scoped to mocha. Cucumber hooks are keyed differently (getCucumberHookUniqueId) and + * their lifecycle differs, so they are intentionally left out here — closing them safely + * would need cucumber-specific keying and is a known gap for a follow-up. + * + * Each entry was tagged with kind at start time so we emit HookRunFinished for hooks and + * TestRunFinished for tests without re-deriving the kind from the title. Entries without a + * uuid are skipped (same null-uuid guard as afterTest): a finish the backend can't match to a + * start is worse than none. The result is marked failed/incomplete via getFailureObject so the + * entity lands in a terminal state rather than pending. + */ + public async sweepUnfinished() { + if (this._framework !== 'mocha') { + return + } + + for (const fullTitle of Object.keys(this._tests)) { + const meta = this._tests[fullTitle] + // Already finished, or no kind tag — nothing to sweep. + // + // Kind-less entries are CLI/gRPC-path tests seeded by setTestData (cucumber/legacy + // entries are also kind-less and skipped for the same reason). A CLI-path test's + // test_run lifecycle is owned by the binary (trackEvent -> gRPC -> stopBinSession), NOT + // the JS listener pipeline this sweep emits on. Sweeping them would double-emit a finish + // on a transport that never opened them, so they are correctly skipped here. Any genuine + // CLI-path orphan is a binary-side concern. + if (!meta || meta.finishedAt || (meta.kind !== 'hook' && meta.kind !== 'test')) { + continue + } + // Only sweep entries that actually started. + if (!meta.startedAt) { + continue + } + // Never emit a finish with a missing uuid — the backend cannot match it to a start, + // which would orphan the entity rather than close it. + if (!meta.uuid) { + BStackLogger.warn('Skipping synthetic finish for ' + fullTitle + ' — stashed uuid is missing.') + continue + } + + // Stamp the finish time into a local and build the payload from it. We only mark the + // stashed meta as finished AFTER the listener call succeeds — if the emit throws, the + // entry must stay unfinished so a later pass does not skip (and silently re-orphan) the + // exact entity this sweep exists to close. + const finishedAt = (new Date()).toISOString() + try { + const testData = this.buildSweepFinishData(fullTitle, meta, finishedAt) + + if (meta.kind === 'hook') { + this.listener.hookFinished(testData) + } else { + this.listener.testFinished(testData) + } + meta.finishedAt = finishedAt + BStackLogger.debug('Emitted synthetic ' + (meta.kind === 'hook' ? 'HookRunFinished' : 'TestRunFinished') + ' for unfinished ' + fullTitle + '.') + } catch (err) { + // A failed emit means the orphan is still open — that is a persisted orphan, so warn. + // The entry is left unstamped on purpose; per-item isolation keeps the loop sweeping + // the remaining entries. + BStackLogger.warn('Failed to emit synthetic finish while sweeping unfinished ' + fullTitle + ': ' + err) + } + } + } + + /** + * Build a terminal (failed/incomplete) finish payload for a swept entry directly from the + * stashed TestMeta, since the live framework test object is no longer available at teardown. + */ + private buildSweepFinishData(fullTitle: string, meta: TestMeta, finishedAt: string): TestData { + const filename = meta.fileName + const testData: TestData = { + uuid: meta.uuid, + type: meta.kind === 'hook' ? 'hook' : 'test', + name: meta.name, + body: { + lang: 'webdriverio', + code: null + }, + scope: fullTitle, + scopes: meta.scopes, + identifier: fullTitle, + file_name: filename ? path.relative(process.cwd(), filename) : undefined, + location: filename ? path.relative(process.cwd(), filename) : undefined, + vc_filepath: (this._gitConfigPath && filename) ? path.relative(this._gitConfigPath, filename) : undefined, + started_at: meta.startedAt, + finished_at: finishedAt, + framework: this._framework, + // getFailureObject only returns failure/failure_reason/failure_type — never a result + // key — but set result AFTER the spread so the synthetic 'failed' status stays + // authoritative even if that ever changes. + ...getFailureObject(new Error('Test/hook did not finish before the session ended (incomplete).')), + result: 'failed' + } + + testData.integrations = {} + if (this._browser && this._platformMeta) { + const provider = getCloudProvider(this._browser) + testData.integrations[provider] = this.getIntegrationsObject() + } + + if (meta.kind === 'hook') { + testData.hook_type = meta.name ? getHookType(meta.name.toLowerCase()) : 'undefined' + } + + return testData + } + /** * Cucumber Only */ @@ -684,7 +849,10 @@ class _InsightsHandler { private getRunData (test: Frameworks.Test, eventType: string, results?: Frameworks.TestResult) { const fullTitle = getUniqueIdentifier(test, this._framework) - const testMetaData = this._tests[fullTitle] + // never let a missing meta entry crash the finish (which would silently drop the + // event and orphan the test). A start always seeds this._tests[fullTitle]; if it is absent + // here the identity was mislabelled upstream — degrade gracefully with an empty meta. + const testMetaData = this._tests[fullTitle] || ({} as TestMeta) const filename = test.file || this._suiteFile this.currentTestId = testMetaData.uuid @@ -716,7 +884,8 @@ class _InsightsHandler { if ((eventType === 'TestRunFinished' || eventType === 'HookRunFinished') && results) { testData.integrations = {} if (this._browser && this._platformMeta) { - const provider = getCloudProvider(this._browser) + // LTS reporter override (see getCloudProvider note in util.ts) + const provider = this._ltsActive ? 'browserstack' : getCloudProvider(this._browser) testData.integrations[provider] = this.getIntegrationsObject() } const { error, passed } = results @@ -736,12 +905,22 @@ class _InsightsHandler { if (this._hooks[fullTitle]) { testData.hooks = this._hooks[fullTitle] } + + // Drain any custom tags (multi Test-Case-ID) recorded for this test via + // browser.setCustomTags on the legacy/listener path. Keyed on the test + // uuid (the same uuid set in beforeTest / setTestData). The accumulator + // already completed union+dedupe SDK-side; the binary is pass-through. + const customMetadata = CustomTagsHandler.drain(testMetaData.uuid) + if (customMetadata && Object.keys(customMetadata).length > 0) { + testData.custom_metadata = customMetadata + } } if (eventType === 'TestRunStarted' || eventType === 'TestRunSkipped' || eventType === 'HookRunStarted') { testData.integrations = {} if (this._browser && this._platformMeta) { - const provider = getCloudProvider(this._browser) + // LTS reporter override (see getCloudProvider note in util.ts) + const provider = this._ltsActive ? 'browserstack' : getCloudProvider(this._browser) testData.integrations[provider] = this.getIntegrationsObject() } } @@ -851,7 +1030,8 @@ class _InsightsHandler { if (eventType === 'TestRunStarted' || eventType === 'TestRunSkipped') { testData.integrations = {} if (this._browser && this._platformMeta) { - const provider = getCloudProvider(this._browser) + // LTS reporter override (see getCloudProvider note in util.ts) + const provider = this._ltsActive ? 'browserstack' : getCloudProvider(this._browser) testData.integrations[provider] = this.getIntegrationsObject() } } @@ -918,7 +1098,8 @@ class _InsightsHandler { const integrationsData: Record = {} if (this._browser && this._platformMeta) { - const provider = getCloudProvider(this._browser) as keyof IntegrationObject + // LTS reporter override (see getCloudProvider note in util.ts) + const provider = (this._ltsActive ? 'browserstack' : getCloudProvider(this._browser)) as keyof IntegrationObject integrationsData[provider] = this.getIntegrationsObject() } @@ -943,13 +1124,19 @@ class _InsightsHandler { BStackLogger.debug(`Driver capabilities used for integration object: ${JSON.stringify(caps)}`) BStackLogger.debug(`User capabilities used for integration object: ${JSON.stringify(this._userCaps)}`) + // LTS: override session_id with the pod-iteration LTS env id and + // force product='loadTesting' so TestHub's o11y classifier resolves + // test_run.origin=LoadTesting. Mirrors py-sdk 0efca1ae (session_id + // override at event-emission layer) + a245a814 (product=loadTesting + // tag on AutomationSession). Non-LTS runs see zero behavior change. + // ltsActive + ltsSessionId are cached at construction (see constructor). return { capabilities: caps, - session_id: sessionId, + session_id: (this._ltsActive && this._ltsSessionId) ? this._ltsSessionId : sessionId, browser: caps?.browserName, browser_version: caps?.browserVersion, platform: caps?.platformName, - product: this._platformMeta?.product, + product: this._ltsActive ? 'loadTesting' : this._platformMeta?.product, platform_version: getPlatformVersion(caps, this._userCaps as WebdriverIO.Capabilities), device: getResolvedDeviceName(caps, this._userCaps as WebdriverIO.Capabilities) } diff --git a/packages/browserstack-service/src/instrumentation/funnelInstrumentation.ts b/packages/browserstack-service/src/instrumentation/funnelInstrumentation.ts index fc75c22..fb6bb71 100644 --- a/packages/browserstack-service/src/instrumentation/funnelInstrumentation.ts +++ b/packages/browserstack-service/src/instrumentation/funnelInstrumentation.ts @@ -157,6 +157,13 @@ function buildEventData(eventType: string, config: BrowserStackConfig, isCLIEnab if (process.env.BSTACK_A11Y_POLLING_TIMEOUT) { eventProperties.pollingTimeout = process.env.BSTACK_A11Y_POLLING_TIMEOUT as string } + // If any worker called browser.reloadSession(), mark the build so the + // session-linking dashboard can exclude its (expected) reload-orphaned sessions. + type WorkerRecord = { reloadHappened?: boolean; usageStats?: unknown } + const reloadHappened = workerData.some((worker) => (worker as WorkerRecord).reloadHappened === true) + if (reloadHappened) { + eventProperties.finishedMetadata = { reason: 'session_reloaded' } + } } return { diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 3d0ed2d..4fd6b73 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -11,6 +11,8 @@ import type { CurrentRunInfo, StdLog } from './types.js' import type { BrowserstackConfig, TestData, TestMeta } from './types.js' import { getCloudProvider, + isLoadTestingSession, + getLtsSessionId, o11yClassErrorHandler, getGitMetaData, removeAnsiColors, @@ -280,8 +282,19 @@ class _TestReporter extends WDIOReporter { } if (eventType.startsWith('TestRun') || eventType === 'HookRunStarted') { + // LTS pod-iterations run against a local Selenium hub, so the + // hostname-based check in getCloudProvider would resolve to + // 'unknown_grid' and TestHub's o11y classifier would land the row + // with origin=UnknownGrid. Force 'browserstack' here ONLY in the + // reporter path — getCloudProvider stays untouched so Automate + // guards (markSessionName/Status, KEY_IS_BROWSERSTACK_HUB) keep + // their original semantics under LTS. + const ltsActive = isLoadTestingSession() + const ltsSessionId = ltsActive ? getLtsSessionId() : '' /* istanbul ignore next */ - const cloudProvider = getCloudProvider({ options: { hostname: this._config?.hostname } } as WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser) + const cloudProvider = ltsActive + ? 'browserstack' + : getCloudProvider({ options: { hostname: this._config?.hostname } } as WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser) testData.integrations = {} // For Appium / App-Automate, server-resolved fields like `deviceModel` // live on the live driver session. The user-requested input caps — @@ -296,12 +309,13 @@ class _TestReporter extends WDIOReporter { /* istanbul ignore next */ testData.integrations[cloudProvider] = { capabilities: this._capabilities, - session_id: this._sessionId, + session_id: (ltsActive && ltsSessionId) ? ltsSessionId : this._sessionId, browser: this._capabilities?.browserName, browser_version: this._capabilities?.browserVersion, platform: this._capabilities?.platformName, platform_version: getPlatformVersion(this._capabilities, this._userCaps as WebdriverIO.Capabilities), - device: getResolvedDeviceName(liveBrowserCaps, this._userCaps as WebdriverIO.Capabilities) + device: getResolvedDeviceName(liveBrowserCaps, this._userCaps as WebdriverIO.Capabilities), + ...(ltsActive ? { product: 'loadTesting' } : {}) } } diff --git a/packages/browserstack-service/src/request-handler.ts b/packages/browserstack-service/src/request-handler.ts index 294ef39..47eeb36 100644 --- a/packages/browserstack-service/src/request-handler.ts +++ b/packages/browserstack-service/src/request-handler.ts @@ -70,7 +70,14 @@ export default class RequestQueueHandler { callCallback = async (data: UploadType[], kind: string) => { BStackLogger.debug('calling callback with kind ' + kind) - await this.callback?.(data) + // SDK-6275: isolate per-batch failures so one failing upload never aborts + // the rest of the queue flush (notably the shutdown() drain loop, which + // awaits this for each batch and would otherwise stop on the first error). + try { + await this.callback?.(data) + } catch (e) { + BStackLogger.debug(`Exception while sending data batch (${kind}); continuing with remaining batches: ${e}`) + } } resetEventBatchPolling () { diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 7263117..9a29000 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -7,7 +7,8 @@ import { getParentSuiteName, isBrowserstackSession, patchConsoleLogs, - isTrue + isTrue, + getUniqueIdentifier } from './util.js' import type { BrowserstackConfig, BrowserstackOptions, MultiRemoteAction } from './types.js' import type { Pickle, Feature, ITestCaseHookParameter, CucumberHook } from './cucumber-types.js' @@ -16,6 +17,7 @@ import TestReporter from './reporter.js' import { DEFAULT_OPTIONS, NOT_ALLOWED_KEYS_IN_CAPS, PERF_MEASUREMENT_ENV } from './constants.js' import CrashReporter from './crash-reporter.js' import AccessibilityHandler from './accessibility-handler.js' +import CustomTagsHandler from './custom-tags-handler.js' import { BStackLogger } from './bstackLogger.js' import PercyHandler from './Percy/Percy-Handler.js' import Listener from './testOps/listener.js' @@ -49,6 +51,7 @@ export default class BrowserstackService implements Services.ServiceInstance { private _scenariosThatRan: string[] = [] private _lastScenarioName?: string // Track last scenario for preferScenarioName feature private _scenariosRanCount: number = 0 // Count of non-skipped scenarios + private _reloadHappened: boolean = false // Set when browser.reloadSession() is called; surfaced as finishedMetadata on SDKTestSuccessful so reload-orphaned builds can be excluded from session-linking private _failureStatuses: string[] = ['failed', 'ambiguous', 'undefined', 'unknown'] private _browser?: WebdriverIO.Browser private _suiteTitle?: string @@ -58,9 +61,24 @@ export default class BrowserstackService implements Services.ServiceInstance { private _specsRan: boolean = false private _observability private _currentTest?: Frameworks.Test | ITestCaseHookParameter + /** + * CLI/gRPC path: map of test identity -> the test_uuid minted for it at + * INIT_TEST/PRE. The CLI mints a fresh uuid into a SINGLE mutable per-worker tracked instance + * (`trackWdioMochaInstance` overwrites `TestFramework.instances[ctxId]` wholesale on every + * INIT_TEST), and the gRPC test-finish reads the uuid from that single slot. When a test hangs + * past the mocha timeout, the NEXT test's INIT_TEST overwrites the slot before the hung test's + * afterTest POST fires, so the POST would carry the wrong (next) test's uuid and the binary + * would close the wrong test_run — orphaning the hung one. We snapshot each test's uuid here, + * keyed by the same identity the legacy `_tests` map uses, and at afterTest we restore the + * finishing test's minted uuid onto the tracked instance before the POST so it closes the + * correct test_run. The finishing identity is `originalTest` (already snapshotted pre-await by + * the testFnWrapper, so it is the timed-out runnable's own identity, not the next test's). + */ + private _cliTestUuids: Map = new Map() private _insightsHandler?: InsightsHandler private _accessibility private _accessibilityHandler?: AccessibilityHandler + private _customTagsHandler?: CustomTagsHandler private _percy private _percyCaptureMode: string | undefined = undefined private _percyHandler?: PercyHandler @@ -267,6 +285,25 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._insightsHandler.before() } + /** + * register custom-tag (multi Test-Case-ID) browser method on the + * legacy/listener path. The CLI/gRPC path registers it via + * CustomTagsModule.onBeforeExecute instead (both handlers' before() + * are skipped when the binary is up). + */ + if (!BrowserstackCLI.getInstance().isRunning()) { + try { + this._customTagsHandler = new CustomTagsHandler( + this._browser, + this._caps, + this._config.framework + ) + this._customTagsHandler.before() + } catch (err) { + BStackLogger.error(`[Custom Tags] Error registering setCustomTags: ${err}`) + } + } + /** * register command event */ @@ -388,6 +425,12 @@ export default class BrowserstackService implements Services.ServiceInstance { if (BrowserstackCLI.getInstance().isRunning()) { await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test }) const uuid = TestFramework.getState(TestFramework.getTrackedInstance(), TestFrameworkConstants.KEY_TEST_UUID) + // snapshot this test's freshly-minted uuid keyed by its identity, so a later + // afterTest can restore it even after a subsequent INIT_TEST has overwritten the single + // mutable tracked-instance slot. Keyed exactly like the legacy `_tests` map. + if (this._config.framework === 'mocha' && uuid) { + this._cliTestUuids.set(getUniqueIdentifier(test, this._config.framework), uuid as string) + } this._insightsHandler?.setTestData(test, uuid) await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }) return @@ -400,7 +443,11 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterTest' }) - async afterTest(test: Frameworks.Test, context: never, results: Frameworks.TestResult) { + async afterTest(originalTest: Frameworks.Test, context: never, results: Frameworks.TestResult) { + // `originalTest` is the identity the testFnWrapper snapshotted BEFORE awaiting the spec body, + // so on a timeout it is the runnable that actually started (not the next test mocha advanced + // its pointer to). Trust it directly as the finishing test for both reporting paths. + const test = originalTest this._specsRan = true const { error, passed, skipped } = results if (!passed && !skipped) { @@ -412,8 +459,26 @@ export default class BrowserstackService implements Services.ServiceInstance { } if (BrowserstackCLI.getInstance().isRunning()) { + // the CLI test-finish reads test_uuid from the single mutable per-worker + // tracked instance, which a later INIT_TEST may have overwritten with the NEXT test's + // uuid. `test` is `originalTest` — already the correct (timed-out) identity, snapshotted + // pre-await by the testFnWrapper — so restore THAT test's minted uuid onto the tracked + // instance so the POST carries it and the binary closes the correct test_run. Without + // this, the finish would carry the next test's uuid and orphan the finishing one. + if (this._config.framework === 'mocha') { + const identifier = getUniqueIdentifier(test, this._config.framework) + const resolvedUuid = this._cliTestUuids.get(identifier) + if (resolvedUuid) { + const trackedInstance = TestFramework.getTrackedInstance() + if (trackedInstance) { + TestFramework.setState(trackedInstance, TestFrameworkConstants.KEY_TEST_UUID, resolvedUuid) + } + // Clean up so the per-worker map does not grow across the run. + this._cliTestUuids.delete(identifier) + } + } await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test, result: results }) - await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.POST, { test, result: results, suiteTite: this._suiteTitle }) + await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.POST, { test, result: results, suiteTitle: this._suiteTitle }) return } @@ -503,6 +568,23 @@ export default class BrowserstackService implements Services.ServiceInstance { } })() + // Teardown safety net: close any started-but-unfinished test/hook with a synthetic + // finish so the backend does not orphan them to the build watchdog (which would inflate + // the reported duration). This MUST run before onWorkerEnd, which shuts the event + // batcher down inside teardown; events enqueued after that are dropped. Wrapped so a + // sweep failure never breaks the session teardown. + try { + await this._insightsHandler?.sweepUnfinished() + } catch (sweepErr) { + BStackLogger.debug('Exception in sweepUnfinished during after(): ' + util.format(sweepErr)) + } + // The sweep closes the _tests entries, but the CLI uuid snapshots (_cliTestUuids) are + // only drained in afterTest — the callback that never fires for a test the sweep just + // handled (e.g. one that timed out). Clear them here at per-worker teardown so stale + // snapshots cannot leak across the worker. Safe to clear: this runs after the sweep and + // no further afterTest will consume them in this worker. + this._cliTestUuids.clear() + // Track Listener cleanup PerformanceTester.start(EVENTS.SDK_LISTENER_WORKER_END) await Listener.getInstance().onWorkerEnd() @@ -643,6 +725,8 @@ export default class BrowserstackService implements Services.ServiceInstance { return Promise.resolve() } + this._reloadHappened = true + const { setSessionName, setSessionStatus } = this._options const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true @@ -862,7 +946,8 @@ export default class BrowserstackService implements Services.ServiceInstance { private saveWorkerData() { saveWorkerData({ - usageStats: UsageStats.getInstance().getDataToSave() + usageStats: UsageStats.getInstance().getDataToSave(), + reloadHappened: this._reloadHappened }) } } diff --git a/packages/browserstack-service/src/testHub/utils.ts b/packages/browserstack-service/src/testHub/utils.ts index aa6549f..f66e3cb 100644 --- a/packages/browserstack-service/src/testHub/utils.ts +++ b/packages/browserstack-service/src/testHub/utils.ts @@ -2,7 +2,7 @@ import { BROWSERSTACK_PERCY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_ACCESSIBILITY } from '../constants.js' import type BrowserStackConfig from '../config.js' import { BStackLogger } from '../bstackLogger.js' -import { isTrue } from '../util.js' +import { isTrue, isLoadTestingSession } from '../util.js' interface ErrorType { key: string @@ -14,17 +14,39 @@ export interface Errors { } export const getProductMap = (config: BrowserStackConfig): { [key: string]: boolean } => { + // LTS gating: under LTS, automate is forced false (LTS builds route to + // the LTS internal hub; keeping automate=true tags TestHub source as + // TO,AUT,LTS instead of TO,LTS — see py-sdk ea53d914). The `lts` key + // itself is only emitted under LTS so non-LTS payloads stay byte- + // identical to the pre-LTS-PR shape (avoids surprising any backend with + // strict unknown-key validation). + const lts = isLoadTestingSession() const entries: [string, boolean | undefined][] = [ ['observability', config.testObservability.enabled], ['accessibility', !!config.accessibility], ['percy', config.percy], - ['automate', config.automate], + ['automate', lts ? false : config.automate], ['app_automate', config.appAutomate], ] + if (lts) { + entries.push(['lts', true]) + } return Object.fromEntries(entries.filter(([, v]) => v !== null)) as { [key: string]: boolean } } export const shouldProcessEventForTesthub = (eventType: string): boolean => { + // LTS short-circuit: when running as an LTS load test (BROWSERSTACK_LTS=true + // for local repro, or BROWSERSTACK_LTS_SESSION_ID set by the pod), TestHub + // event emission must always flow regardless of the observability / + // accessibility / percy env flags. Pod runs do export + // BROWSERSTACK_OBSERVABILITY, but the local-repro flag path doesn't + // guarantee it — without this short-circuit, getProductMap correctly + // emits lts: true while every onTestStart / onTestEnd / onHookStart + // silently no-ops. Mirror of the unconditional LTS event flow in py-sdk's + // should_send_event_for_testhub() (PR #993). + if (isLoadTestingSession()) { + return true + } if (isTrue(process.env[BROWSERSTACK_OBSERVABILITY])) { return true } @@ -75,12 +97,21 @@ export const logBuildError = (error: Errors | null, product: string = ''): void } export const getProductMapForBuildStartCall = (config: BrowserStackConfig, accessibilityAutomation?: boolean | null): { [key: string]: boolean } => { + // LTS gating: under LTS, automate is forced false (LTS builds route to + // the LTS internal hub; keeping automate=true tags TestHub source as + // TO,AUT,LTS instead of TO,LTS — see py-sdk ea53d914). The `lts` key + // itself is only emitted under LTS to keep non-LTS payloads byte- + // identical to the pre-LTS-PR shape. + const lts = isLoadTestingSession() const entries: [string, boolean | undefined | null][] = [ ['observability', config.testObservability.enabled], ['accessibility', accessibilityAutomation], ['percy', config.percy], - ['automate', config.automate], + ['automate', lts ? false : config.automate], ['app_automate', config.appAutomate], ] + if (lts) { + entries.push(['lts', true]) + } return Object.fromEntries(entries.filter(([, v]) => v !== null)) as { [key: string]: boolean } } diff --git a/packages/browserstack-service/src/types.ts b/packages/browserstack-service/src/types.ts index 256eb83..a41d085 100644 --- a/packages/browserstack-service/src/types.ts +++ b/packages/browserstack-service/src/types.ts @@ -1,5 +1,6 @@ import type { Capabilities, Options, Frameworks } from '@wdio/types' import type { Options as BSOptions } from 'browserstack-local' +import type { CustomMetadata } from './customTags.js' export type MultiRemoteAction = (sessionId: string, browserName?: string) => Promise @@ -241,7 +242,16 @@ export interface TestMeta { scenario?: { name: string }, examples?: string[], hookType?: string, - testRunId?: string + testRunId?: string, + // Explicitly records whether this entry is a hook or a test so a teardown sweep can emit the + // correct synthetic finish event without re-deriving the kind from the title. Tagged at + // beforeHook/beforeTest time. Only used for the mocha never-finished sweep. + kind?: 'hook' | 'test', + // Identity captured at start time so the sweep can build a terminal finish payload without the + // live framework test object (which is gone by teardown). + name?: string, + scopes?: string[], + fileName?: string } export interface CurrentRunInfo { @@ -277,7 +287,8 @@ export interface TestData { meta?: TestMeta, tags?: string[], test_run_id?: string, - product_map?: {} + product_map?: {}, + custom_metadata?: CustomMetadata } export interface UserConfig { @@ -445,6 +456,9 @@ export interface EventProperties { } } isCLIEnabled?: boolean + finishedMetadata?: { + reason: string + } } export interface FunnelData { diff --git a/packages/browserstack-service/src/util.ts b/packages/browserstack-service/src/util.ts index 27a6ff5..284167a 100644 --- a/packages/browserstack-service/src/util.ts +++ b/packages/browserstack-service/src/util.ts @@ -31,6 +31,8 @@ import { BROWSERSTACK_TESTHUB_JWT, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_ACCESSIBILITY, + BROWSERSTACK_LTS, + BROWSERSTACK_LTS_SESSION_ID, TESTOPS_SCREENSHOT_ENV, BROWSERSTACK_TESTHUB_UUID, PERF_MEASUREMENT_ENV, @@ -1028,7 +1030,36 @@ export function getUniqueIdentifierForCucumber(world: ITestCaseHookParameter): s return world.pickle.uri + '_' + world.pickle.astNodeIds.join(',') } +// Load Testing Service (LTS) gating helpers — mirror of bstack_utils/helper.py +// is_load_testing_session() / get_lts_session_id() in browserstack-python-sdk +// (branch LTS-tra-python-support). LTS pod-iterations export +// BROWSERSTACK_LTS_SESSION_ID before invoking the test runner; presence of +// that env var is the single source of truth for "this run is an LTS pod +// iteration". Optional BROWSERSTACK_LTS=true|1 lets the runner opt-in without +// supplying a session id (useful for local repro). +export function isLoadTestingSession(): boolean { + if (process.env[BROWSERSTACK_LTS_SESSION_ID]) { + return true + } + const flag = process.env[BROWSERSTACK_LTS] + return flag === 'true' || flag === '1' +} + +export function getLtsSessionId(): string { + return process.env[BROWSERSTACK_LTS_SESSION_ID] || '' +} + export function getCloudProvider(browser: WebdriverIO.Browser | WebdriverIO.MultiRemoteBrowser): string { + // NOTE: do NOT branch on isLoadTestingSession() here. getCloudProvider is + // shared with Automate-side guards (automateModule onBefore/onAfterTest, + // webdriverIOModule KEY_IS_BROWSERSTACK_HUB, accessibility-handler, + // service.ts) — flipping it to 'browserstack' under LTS would make + // isBrowserstackSession() return true for local-Selenium pod sessions, + // causing markSessionName / markSessionStatus to PUT against Automate + // REST APIs with session ids that don't exist there. TestHub-reporting + // callers that need 'browserstack' under LTS derive it locally with + // `isLoadTestingSession() ? 'browserstack' : getCloudProvider(browser)` + // (see reporter.ts, insights-handler.ts). if (browser && 'instances' in browser) { // Loop through all instances for (const instanceName of browser.instances) {