From 447c4d341b4fb486aebf5ae4b4ec6d40087b6426 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 2 Sep 2026 22:22:53 +0530 Subject: [PATCH 01/25] SDK-7414: open the CLI flow for WebdriverIO-cucumber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 'cucumber' to CLISupportedFrameworks, which is the single gate both the launcher and the worker read. The binary side already registers 'WebdriverIO-cucumber' and the name reaches it unchanged: setFrameworkDetail takes WDIO_NAMING_PREFIX + config.framework verbatim, so no session-start branch is needed here. setupTestFramework's if had no else, so an unmatched name left testFramework null and every CLI event no-opped without an error anywhere. The framework class lands next, so cucumber takes that arm for now — log it rather than leave the silence. Co-Authored-By: Claude Opus 5 (1M context) --- packages/browserstack-service/src/cli/cliUtils.ts | 2 +- packages/browserstack-service/src/cli/index.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 03b335f..3c6e5c0 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -47,7 +47,7 @@ const CLI_DOWNLOAD_TMP_SUFFIX = '.zip' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha'] + static CLISupportedFrameworks = ['mocha', 'cucumber'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index b1b3cc3..2587956 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -548,7 +548,11 @@ export class BrowserstackCLI { const testFrameworkDetail = CLIUtils.getTestFrameworkDetail() if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-mocha') { this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return } + // An unmatched name leaves testFramework null, and every CLI event then no-ops with no + // error of any kind. Name it so the silence is diagnosable. + this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) } /** From 78a90f2b8a3188a5d41df1dc05f2cca02fab1cfc Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 2 Sep 2026 23:23:27 +0530 Subject: [PATCH 02/25] feat(cli): add WdioCucumberTestFramework and drive it from cucumber's own states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cucumber's unit of work is the scenario, and every cli/modules/* observer subscribes to TestFrameworkState.TEST — so a scenario raises TEST/PRE at beforeScenario and TEST/POST at afterScenario, and the whole module set works unchanged. Extends the base TestFramework, not WdioMochaTestFramework: WDIO never calls beforeTest/afterTest or titled hooks for cucumber, so mocha's INIT_TEST/TEST/hook boundary semantics have no source here. - Hooks classify via a _cucumberData state machine, not util.ts getHookType() — a cucumber hook carries no title and BeforeAll/AfterAll pass no hook object at all, so getHookType would throw the moment the flow gate opened. Step-scoped hooks stay unreported. - Scenario results set test_result_at. Without it testHubModule marks the test deferred and waits on LOG_REPORT, a state cucumber never emits. - Duration comes from cucumber's protobuf Duration, not an ended_at - started_at delta; the failure backtrace is one element, not mocha's two; tags keep their leading '@'; identifier stays the raw pickle name while name/scope carry the examples qualifier. - The feature path is sent absolute — the binary re-bases it (SDK-7233). - Logs route to the open hook's uuid when one is in flight, else the scenario's. Mocha's path is unchanged: both service.ts hook edits add an instanceof arm ahead of the existing block, and the factory's mocha branch still returns first. SDK-7414 --- .../frameworks/wdioCucumberTestFramework.ts | 522 ++++++++++++++++++ .../browserstack-service/src/cli/index.ts | 7 +- packages/browserstack-service/src/service.ts | 119 +++- 3 files changed, 644 insertions(+), 4 deletions(-) create mode 100644 packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts new file mode 100644 index 0000000..db6613f --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -0,0 +1,522 @@ +import { v4 as uuidv4 } from 'uuid' +import path from 'node:path' + +import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' +import { BStackLogger as logger } from '../cliLogger.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' +import { getScenarioExamples, removeAnsiColors } from '../../util.js' + +import type { Frameworks } from '@wdio/types' +import type { CucumberHook, Feature, ITestCaseHookParameter, Pickle } from '../../cucumber-types.js' + +/** + * `test_duration` and `bdd_meta_info` are read by the binary's WebdriverIO-cucumber module but + * have no entry in TestFrameworkConstants, which is shared with the mocha path. Kept local rather + * than appended there so this framework owns its own wire keys. + */ +const KEY_TEST_DURATION = 'test_duration' +const KEY_BDD_META_INFO = 'bdd_meta_info' + +type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' + +const HOOK_STATES: Record = { + BEFORE_ALL: TestFrameworkState.BEFORE_ALL, + AFTER_ALL: TestFrameworkState.AFTER_ALL, + BEFORE_EACH: TestFrameworkState.BEFORE_EACH, + AFTER_EACH: TestFrameworkState.AFTER_EACH, +} + +interface StepMeta { + id: string + text: string + keyword: string + started_at?: string + finished_at?: string + result?: string + duration?: unknown + failure?: string +} + +/** + * File-path pair sent with every scenario/hook event. + * + * The ABSOLUTE feature path is deliberate: the binary re-bases it itself + * (`path.relative(session.pathProject, …)` for `file_name`/`location` and against the git root for + * `vc_filepath`). Legacy reported both pre-relativised — sending that shape here made both fields + * come out wrong, and sending `undefined` threw inside the binary and dropped the event (SDK-7233). + */ +const resolveFeatureFilePaths = (featurePath: string | undefined) => ({ + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: featurePath, + [TestFrameworkConstants.KEY_TEST_LOCATION]: featurePath + ? path.relative(process.cwd(), featurePath) + : undefined, +}) + +/** + * CLI test framework for `framework: 'cucumber'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: WDIO does not call + * `beforeTest`/`afterTest`/titled hooks for cucumber at all, so mocha's INIT_TEST/TEST/hook + * boundary semantics have no source here and borrowing them would report cucumber through the + * wrong runner's event model. + * + * Cucumber's unit of work is the scenario, and every `cli/modules/*` observer subscribes to + * `TestFrameworkState.TEST` — so a scenario raises TEST/PRE at `beforeScenario` and TEST/POST at + * `afterScenario`, and the module set works unchanged. + */ +export default class WdioCucumberTestFramework extends TestFramework { + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + + /** + * The bookkeeping `classifyHookType()` derives hook types from. A cucumber hook invocation + * carries no title, and `BeforeAll`/`AfterAll` pass no hook object at all, so classification + * is state-machine derived — `util.ts → getHookType()` is Mocha-title-shaped and can only + * return 'unknown' or throw here. + */ + private cucumberData: { + feature?: Feature + uri?: string + scenario?: Pickle + scenariosStarted: boolean + stepsStarted: boolean + stepDepth: number + } = { scenariosStarted: false, stepsStarted: false, stepDepth: 0 } + + /** + * Steps accumulated for the scenario in flight. Re-allocated (never cleared in place) at each + * scenario start so a payload already built from the previous scenario can never observe the + * next one's steps. + */ + private scenarioSteps: StepMeta[] = [] + + /** The hook currently open on this worker — started and not yet finished. */ + private openHook: { key: string, hookId: string } | null = null + + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioCucumberTestFramework: constructed') + } + + /** + * Feature bookkeeping. Raises no framework state — cucumber has no feature-level wire event, + * and `beforeSuite`/`afterSuite` are not part of its WDIO surface. + */ + onFeatureStart(uri: string, feature: Feature) { + logger.debug(`onFeatureStart: uri=${uri} feature=${feature?.name}`) + this.cucumberData.scenariosStarted = false + this.cucumberData.feature = feature + this.cucumberData.uri = uri + } + + /** + * Step bookkeeping. Steps travel inside the scenario payload's BDD meta, never as their own + * event — `TestFrameworkState.STEP` has zero producers anywhere in this SDK. + */ + onStepStart(step: Frameworks.PickleStep) { + this.cucumberData.stepsStarted = true + this.cucumberData.stepDepth++ + this.scenarioSteps.push({ + id: step.id, + text: step.text, + keyword: step.keyword, + started_at: (new Date()).toISOString(), + }) + } + + onStepEnd(step: Frameworks.PickleStep, result: Frameworks.PickleResult) { + this.cucumberData.stepDepth = Math.max(0, this.cucumberData.stepDepth - 1) + const stepMeta = this.scenarioSteps.find(item => item.id === step.id) + if (!stepMeta) { + return + } + stepMeta.finished_at = (new Date()).toISOString() + stepMeta.result = result.passed ? 'PASSED' : 'FAILED' + stepMeta.duration = result.duration + if (result.error) { + stepMeta.failure = removeAnsiColors(result.error) + } + } + + /** + * Classify a cucumber hook invocation from the bookkeeping state. + * + * Returns null for a step-scoped hook (`BeforeStep`/`AfterStep`), which is never reported — + * reporting it would change the dashboard hook count, which is a feature and not parity. + */ + classifyHookType(test: CucumberHook | undefined): CucumberHookType | null { + if (!test) { + return this.cucumberData.scenariosStarted ? 'AFTER_ALL' : 'BEFORE_ALL' + } + if (!this.cucumberData.stepsStarted) { + return 'BEFORE_EACH' + } + if (this.cucumberData.stepDepth > 0) { + return null + } + return 'AFTER_EACH' + } + + /** The TestFrameworkState a cucumber hook invocation maps to, or null when unreported. */ + classifyHookState(test: CucumberHook | undefined): State | null { + const hookType = this.classifyHookType(test) + return hookType ? HOOK_STATES[hookType] : null + } + + /** + * ` for ` — the separator is a literal ' ' + 'for' + ' '. + */ + private hookName(hookType: CucumberHookType): string { + switch (hookType) { + case 'BEFORE_EACH': + case 'AFTER_EACH': + return `${hookType} for ${this.cucumberData.scenario?.name}` + case 'BEFORE_ALL': + case 'AFTER_ALL': + return `${hookType} for ${this.cucumberData.feature?.name}` + } + } + + private featurePath(): string | undefined { + const uri = this.cucumberData.uri + return uri ? path.resolve(process.cwd(), uri) : undefined + } + + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) + await super.trackEvent(testFrameworkState, hookState, args) + + const instance = this.resolveInstance(testFrameworkState, hookState) + if (!instance) { + // Console output emitted before the first scenario (or before BeforeAll) has nothing + // to attach to. The legacy path drops it just as silently, so this is expected rather + // than a failure. + const detail = `trackEvent: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}` + if (testFrameworkState === TestFrameworkState.LOG) { + logger.debug(detail) + } else { + logger.error(detail) + } + return + } + + const shortState = testFrameworkState.toString().split('.')[1] + const isHook = CLIUtils.matchHookRegex(shortState) + + try { + if (isHook && hookState === HookState.PRE) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_HOOK_ID]: uuidv4(), + }) + } + + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + this.loadScenarioData(instance, args.world as ITestCaseHookParameter) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: new Date().toISOString(), + }) + } else if (hookState === HookState.POST) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: new Date().toISOString(), + }) + this.loadScenarioResult(instance, args) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.loadLogEntry(instance, args.logEntry as Record) + } + + if (isHook) { + this.trackHookEvents(instance, shortState, hookState, args) + } + } catch (error) { + logger.error(`trackEvent: Error in tracking events: ${error} hookState=${hookState} testFrameworkState=${testFrameworkState}`) + } + + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + /** + * One instance per scenario, keyed by the worker (`pid:threadId`) — the single lookup every + * `cli/modules/*` reads, via `TestFramework.getTrackedInstance()`. WDIO forks one worker + * process per spec file, so the worker key is the scenario's execution context; a thread id + * alone would collide once threads are reused. + */ + private resolveInstance(testFrameworkState: State, hookState: State): TestFrameworkInstance | null { + let instance = TestFramework.getTrackedInstance() + const isHook = CLIUtils.matchHookRegex(testFrameworkState.toString().split('.')[1]) + + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + // Every scenario is its own test. BEFORE_EACH hooks fire after beforeScenario, so + // they land on the instance minted here and never on the previous scenario's. + this.trackWdioCucumberInstance(testFrameworkState) + } else if (isHook && hookState === HookState.PRE && !instance) { + // BEFORE_ALL runs before any scenario exists, so it has no instance to attach to. + this.trackWdioCucumberInstance(testFrameworkState) + } + + instance = TestFramework.getTrackedInstance() + if (!instance) { + logger.debug(`resolveInstance: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}`) + return null + } + this.updateInstanceState(instance, testFrameworkState, hookState) + return instance + } + + private trackWdioCucumberInstance(testFrameworkState: State) { + const target = CLIUtils.getCurrentInstanceName() + const trackedContext = TrackedInstance.createContext(target) + + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + testFrameworkState, + HookState.NONE + ) + + const frameworkName = this.getTestFrameworks()[0] + const testUuid = uuidv4() + + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: testUuid, + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + }) + + // Read by the A11y and App-A11y scan paths. + process.env[TEST_ANALYTICS_ID] = testUuid + + TestFramework.setTrackedInstance(trackedContext, instance) + logger.debug(`trackWdioCucumberInstance: contextId=${trackedContext.getId()} target=${target} testUuid=${testUuid}`) + } + + /** + * Scenario identity. Every field below is fixed by a parity row and the asymmetries are + * deliberate — see `wdioCucumberTestFramework` notes in the SDK-7414 parity table. + */ + private loadScenarioData(instance: TestFrameworkInstance, world: ITestCaseHookParameter) { + if (!world?.pickle) { + logger.error('loadScenarioData: no pickle on the world object; scenario identity will be empty') + return + } + const pickle = world.pickle + const feature = world.gherkinDocument?.feature + this.cucumberData.scenario = pickle + this.cucumberData.scenariosStarted = true + this.cucumberData.stepsStarted = false + this.cucumberData.stepDepth = 0 + // Fresh array, never a clear-in-place: the previous scenario's payload must not be able to + // observe this scenario's steps through a retained reference. + this.scenarioSteps = [] + + const examples = getScenarioExamples(world) + // Exactly one space before '(' and ', ' between cells — asserted character-for-character. + const qualifiedName = examples + ? pickle.name + ' (' + examples.join(', ') + ')' + : pickle.name + const featurePath = this.featurePath() + + instance.updateMultipleEntries({ + // The RAW pickle name, deliberately WITHOUT the examples qualifier that `name`/`scope` + // carry. The binary maps this to `identifier`; collapsing the two would change how the + // dashboard groups Scenario Outline rows. + [TestFrameworkConstants.KEY_TEST_ID]: pickle.name, + [TestFrameworkConstants.KEY_TEST_NAME]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPE]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature?.name || ''], + // Step source is never reported for cucumber. + [TestFrameworkConstants.KEY_TEST_CODE]: null, + // Gherkin tag text INCLUDING the leading '@', source order, no dedupe, no lowercasing. + // `.map` allocates a new array — the pickle's own tag collection is never mutated. + [TestFrameworkConstants.KEY_TEST_TAGS]: pickle.tags.map(({ name }: { name: string }) => name), + ...resolveFeatureFilePaths(featurePath), + [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, featurePath, examples), + }) + } + + private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, featurePath: string | undefined, examples: string[] | undefined) { + return { + feature: { + name: feature?.name, + path: featurePath, + description: feature?.description, + }, + scenario: { name: pickle.name }, + steps: this.scenarioSteps.map(step => ({ ...step })), + examples: examples ?? [], + } + } + + /** + * Scenario result, loaded at the real "scenario ends" state. + * + * KEY_TEST_RESULT_AT is load-bearing, not decoration: `testHubModule → onAllTestEvents()` + * treats a TEST/POST without it as result-less, marks the test deferred, and then waits for a + * `LOG_REPORT` POST to recover it — a state cucumber never emits. + */ + private loadScenarioResult(instance: TestFrameworkInstance, args: Record) { + const world = args.world as ITestCaseHookParameter | undefined + const pickle = world?.pickle ?? this.cucumberData.scenario + const feature = world?.gherkinDocument?.feature ?? this.cucumberData.feature + + const updates: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + + if (pickle) { + updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, this.featurePath(), getScenarioExamples(world as ITestCaseHookParameter)) + } + + const result = world?.result + if (result) { + let testResult = result.status.toLowerCase() + if (testResult !== 'passed' && testResult !== 'failed') { + // UNKNOWN / UNDEFINED / AMBIGUOUS / PENDING / SKIPPED all collapse to skipped. + testResult = 'skipped' + } + + // A scenario that failed only because of a hook is reported as passed when the user + // has declared ignoreHooksStatus. The same flag independently gates the session-status + // accumulation in service.ts — two sites, one flag. + if (args.ignoreHooksStatus === true && testResult === 'failed' && !this.hasStepFailures()) { + testResult = 'passed' + } + + updates[TestFrameworkConstants.KEY_TEST_RESULT] = testResult + // Cucumber's own protobuf Duration, NOT an ended_at - started_at delta. + updates[KEY_TEST_DURATION] = result.duration + ? result.duration.seconds * 1000 + result.duration.nanos / 1000000 + : undefined + + if (testResult === 'failed') { + const message = result.message + // A ONE-element backtrace. The mocha path sends two entries (message + stack); + // cucumber's result carries a single combined message. + updates[TestFrameworkConstants.KEY_TEST_FAILURE] = [ + { backtrace: [message ? removeAnsiColors(message) : 'unknown'] } + ] + updates[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = message ? removeAnsiColors(message) : message + if (message) { + updates[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = message.match(/AssertionError/) + ? 'AssertionError' + : 'UnhandledError' + } + } + } + + instance.updateMultipleEntries(updates) + this.cucumberData.scenario = undefined + } + + private hasStepFailures(): boolean { + return this.scenarioSteps.some(step => step.result === 'FAILED') + } + + /** + * Route a console log to the row it belongs on: the open hook's uuid while a hook is in + * flight and unfinished, otherwise the scenario's uuid. + * + * The record itself always goes into KEY_TEST_LOGS, because the send path + * (`testHubModule → onAllTestEvents`, unchanged) collects test logs plus the last FINISHED + * hook's logs — a record parked on a still-open hook's own array would never be picked up. + * The routing is carried by KEY_HOOK_ID on the record, which `sendLogCreatedEvent` reads in + * preference to the test uuid. + */ + private loadLogEntry(instance: TestFrameworkInstance, logEntry: Record) { + if (!logEntry) { + return + } + const { level, message, timestamp } = logEntry + const logRecord: Record = { + kind: TestFrameworkConstants.KIND_LOG, + message: Buffer.from(message as string), + level, + timestamp, + } + + if (this.openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = this.openHook.hookId + } + + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_LOGS]: entries, + }) + } + + /** + * Hook lifecycle. Entries are keyed by the short state name, matching how the binary looks + * them up via `event.test_hooks_started[request.testFrameworkState]`. + * + * A finish with no recorded start is dropped rather than emitted — an unmatched + * HookRunFinished orphans a hook row the backend cannot pair. + */ + private trackHookEvents(instance: TestFrameworkInstance, key: string, hookState: State, args: Record) { + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + if (!hooksStarted.has(key)) { + hooksStarted.set(key, []) + } + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map + if (!hooksFinished.has(key)) { + hooksFinished.set(key, []) + } + + const updates: Record = { + [TestFrameworkConstants.KEY_HOOKS_STARTED]: hooksStarted, + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: hooksFinished, + } + const featurePath = this.featurePath() + + if (hookState === HookState.PRE) { + const hookId = (TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) || '') as string + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: new Date().toISOString(), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: this.hookName(key as CucumberHookType), + ...resolveFeatureFilePaths(featurePath), + } + hooksStarted.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_STARTED] = key + this.openHook = { key, hookId } + logger.debug(`trackHookEvents: hook started key=${key} name=${hook[TestFrameworkConstants.KEY_HOOK_NAME]}`) + } else if (hookState === HookState.POST) { + const hooksList = hooksStarted.get(key) || [] + if (hooksList.length === 0) { + logger.warn(`trackHookEvents: dropping hook finish for '${key}' — no matching start was recorded`) + this.openHook = null + return + } + + const hook = hooksList.pop() as Record + const hookResult = args.result as Frameworks.TestResult | undefined + // passed / failed only — no 'skipped' arm, unlike the scenario result path. + if (hookResult) { + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = hookResult.passed ? 'passed' : 'failed' + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = new Date().toISOString() + hooksFinished.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_FINISHED] = key + this.openHook = null + logger.debug(`trackHookEvents: hook finished key=${key} result=${hook[TestFrameworkConstants.KEY_HOOK_RESULT]}`) + } + + instance.updateMultipleEntries(updates) + } +} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 2587956..444ec6d 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -16,6 +16,7 @@ import { BROWSERSTACK_ACCESSIBILITY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_TE import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -47,7 +48,7 @@ export class BrowserstackCLI { modulesLoaded = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|null = null + testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -550,6 +551,10 @@ export class BrowserstackCLI { this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) return } + if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-cucumber') { + this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return + } // An unmatched name leaves testFramework null, and every CLI event then no-ops with no // error of any kind. Name it so the silence is diagnosable. this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index b996be4..84f9a9e 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -44,6 +44,7 @@ import { AutomationFrameworkState } from './cli/states/automationFrameworkState. import { HookState } from './cli/states/hookState.js' import { AutomationFrameworkConstants } from './cli/frameworks/constants/automationFrameworkConstants.js' import TestFramework from './cli/frameworks/testFramework.js' +import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import AccessibilityModule from './cli/modules/accessibilityModule.js' @@ -439,7 +440,17 @@ export default class BrowserstackService implements Services.ServiceInstance { // during any startup race, so a `!` here could throw a TypeError inside this awaited // WDIO hook and break the user's suite. Instrumentation must degrade quietly. const framework = BrowserstackCLI.getInstance().getTestFramework() - if (framework) { + if (framework instanceof WdioCucumberTestFramework) { + // A cucumber hook invocation carries no title, and BeforeAll/AfterAll pass no hook + // object at all, so getHookType — which matches Mocha's quoted titles — can only + // return 'unknown' here or throw on the property access. The framework classifies + // them from its own bookkeeping, and returns null for the step-scoped hooks that + // are deliberately never reported. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) + } + } else if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) @@ -476,6 +487,16 @@ export default class BrowserstackService implements Services.ServiceInstance { // Null-check the tracker rather than asserting (see beforeHook) so a missing tracker // degrades quietly instead of throwing inside this awaited hook. const framework = BrowserstackCLI.getInstance().getTestFramework() + if (framework instanceof WdioCucumberTestFramework) { + // See beforeHook: cucumber's taxonomy, not Mocha's titles. The suite-skip cascade + // below is Mocha-shaped (it walks `test.ctx.test.parent`) and has no cucumber + // counterpart here. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) + } + return + } if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { @@ -815,9 +836,62 @@ export default class BrowserstackService implements Services.ServiceInstance { this._suiteTitle = feature.name await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) + + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onFeatureStart(uri, feature) + return + } await this._insightsHandler?.beforeFeature(uri, feature) } + /** + * The CLI test framework, only when it is the cucumber one and the CLI is actually running. + * Returns null on the legacy flow and on any startup race, so every call site degrades to the + * classic handlers instead of throwing inside an awaited WDIO hook. + */ + private _cliCucumberFramework(): WdioCucumberTestFramework|null { + if (!BrowserstackCLI.getInstance().isRunning()) { + return null + } + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioCucumberTestFramework ? framework : null + } + + /** + * A `Frameworks.Test`-shaped view of a scenario, for the cli/modules that read `args.test`. + * + * `fullName` is set deliberately. automateModule's session naming is SHAPE-keyed + * (`else if (test && !test.fullName)`), so leaving it unset drops the scenario into the mocha + * arm and names the session ` - `; legacy names the session after the + * feature alone, which is what the populated `fullName` preserves. + */ + private _cucumberTestView(world: ITestCaseHookParameter): Frameworks.Test { + const scenarioName = world.pickle?.name ?? '' + return { + title: scenarioName, + fullName: scenarioName, + parent: this._suiteTitle ?? '', + file: world.gherkinDocument?.uri, + } as unknown as Frameworks.Test + } + + /** + * A `Frameworks.TestResult`-shaped view of a scenario result, for automateModule's + * session-status marking. Anything cucumber reports that is neither passed nor failed is a + * skip — the same collapse the scenario's own reported result applies. + */ + private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { + const status = world.result?.status?.toLowerCase() + return { + passed: status === 'passed', + skipped: status !== undefined && status !== 'passed' && status !== 'failed', + error: world.result?.message ? new Error(world.result.message) : undefined, + duration: 0, + retries: { attempts: 0, limit: 0 }, + } as unknown as Frameworks.TestResult + } + /** * Runs before a Cucumber Scenario. * @param world world object containing information on pickle and test step @@ -825,9 +899,24 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeScenario' }) async beforeScenario (world: ITestCaseHookParameter) { this._currentTest = world + const scenarioName = world.pickle.name || 'unknown scenario' + + // The scenario IS the unit of work, so it raises TEST/PRE — the state every cli/modules + // observer subscribes to. WDIO never calls beforeTest for cucumber, so there is no other + // moment at which the modules could be driven. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + }) + await this._setAnnotation(`Scenario: ${scenarioName}`) + return + } + await this._accessibilityHandler?.beforeScenario(world) await this._insightsHandler?.beforeScenario(world) - const scenarioName = world.pickle.name || 'unknown scenario' await this._setAnnotation(`Scenario: ${scenarioName}`) } @@ -869,6 +958,18 @@ export default class BrowserstackService implements Services.ServiceInstance { } } + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.POST, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + result: this._cucumberTestResult(world), + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + }) + return + } + await this._accessibilityHandler?.afterScenario(world) await this._insightsHandler?.afterScenario(world) await this._percyHandler?.afterScenario() @@ -876,12 +977,24 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeStep' }) async beforeStep (step: Frameworks.PickleStep, scenario: Pickle) { - await this._insightsHandler?.beforeStep(step, scenario) + // Steps travel inside the scenario payload, never as their own wire event — this only + // feeds the bookkeeping the hook classifier reads. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onStepStart(step) + } else { + await this._insightsHandler?.beforeStep(step, scenario) + } await this._setAnnotation(`Step: ${step.keyword}${step.text}`) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterStep' }) async afterStep (step: Frameworks.PickleStep, scenario: Pickle, result: Frameworks.PickleResult) { + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onStepEnd(step, result) + return + } await this._insightsHandler?.afterStep(step, scenario, result) } From 72993857e4236ae11ca0208f0ba6db2154fb92e5 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 3 Sep 2026 01:03:03 +0530 Subject: [PATCH 03/25] feat(cli): widen cucumber's shared dispatch gates and hook payload Phase 7 of SDK-7414. 26 shared dispatch sites were enumerated before any edit; 19 needed no change and are recorded as such. accessibilityModule.onBeforeTest now calls shouldScanTestForAccessibility in its 6-arg form, passing the cucumber world and the tag-filter flag. The 3-arg form matches include/exclude tags against the test title, so a cucumber user's tag filters were silently ignored and every scenario was scanned. Only the call arity changed; the helper itself is untouched, and args.world is populated solely on the cucumber path, so mocha and jasmine keep the exact title-matching behaviour. wdioCucumberTestFramework stamps hook_scope, hook_retries and hook_duration onto the hook record. The binary cannot derive any of them from the event: a hook's scope is the feature name while the event carries the examples-qualified scenario name, and BEFORE_ALL/AFTER_ALL fire on an instance with no scenario data at all. --- .../cli/frameworks/wdioCucumberTestFramework.ts | 15 +++++++++++++++ .../src/cli/modules/accessibilityModule.ts | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index db6613f..94a3de6 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -23,6 +23,16 @@ import type { CucumberHook, Feature, ITestCaseHookParameter, Pickle } from '../. const KEY_TEST_DURATION = 'test_duration' const KEY_BDD_META_INFO = 'bdd_meta_info' +/** + * Per-hook wire keys. The binary cannot derive any of the three from the event: a hook's scope is + * the FEATURE name (parity row 16) while the event carries the examples-qualified SCENARIO name, + * and BEFORE_ALL/AFTER_ALL fire on an instance that has no scenario data at all. Retries and + * duration (row 26) come from WDIO's hook result, which only this side sees. + */ +const KEY_HOOK_SCOPE = 'hook_scope' +const KEY_HOOK_RETRIES = 'hook_retries' +const KEY_HOOK_DURATION = 'hook_duration' + type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' const HOOK_STATES: Record = { @@ -490,6 +500,7 @@ export default class WdioCucumberTestFramework extends TestFramework { [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: new Date().toISOString(), [TestFrameworkConstants.KEY_HOOK_LOGS]: [], [TestFrameworkConstants.KEY_HOOK_NAME]: this.hookName(key as CucumberHookType), + [KEY_HOOK_SCOPE]: this.cucumberData.feature?.name, ...resolveFeatureFilePaths(featurePath), } hooksStarted.get(key)?.push(hook) @@ -509,6 +520,10 @@ export default class WdioCucumberTestFramework extends TestFramework { // passed / failed only — no 'skipped' arm, unlike the scenario result path. if (hookResult) { hook[TestFrameworkConstants.KEY_HOOK_RESULT] = hookResult.passed ? 'passed' : 'failed' + // WDIO reports hook duration in plain ms on the result, not as cucumber's protobuf + // Duration — legacy sends it through unchanged and so does this. + hook[KEY_HOOK_RETRIES] = hookResult.retries + hook[KEY_HOOK_DURATION] = hookResult.duration } hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = new Date().toISOString() hooksFinished.get(key)?.push(hook) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 8e44b11..6f6ea0e 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -314,7 +314,13 @@ export default class AccessibilityModule extends BaseModule { const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) const accessibilityOptions = this.config.accessibilityOptions - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined) && this.accessibility + // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg + // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags + // against the test title instead and so silently scans every scenario. `args.world` is + // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg + // behaviour — both extra args arrive undefined/false and the tag branch is not taken. + const world = args.world as { [key: string]: unknown } | undefined + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) From 71fa9bba5983e8f5fc1c77f1db1c3cbfcadfdb52 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 3 Sep 2026 01:43:41 +0530 Subject: [PATCH 04/25] fix(cli): honour ignoreHooksStatus when marking cucumber sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the CLI flow the Automate session status comes solely from the result view service.afterScenario builds — service.after()'s _failReasons path is gated on the binary not running. That view ignored testObservabilityOptions.ignoreHooksStatus, so a scenario that failed only in a hook marked the session failed where the legacy flow marked it passed. Reuses the framework class's own hasStepFailures(), the same predicate the observability result already applies, so both surfaces of the flag agree. InsightsHandler.hasTestStepFailures is unusable here: it reads _tests, which the CLI branch never populates. Co-Authored-By: Claude Opus 5 (1M context) --- .../frameworks/wdioCucumberTestFramework.ts | 7 ++- packages/browserstack-service/src/service.ts | 11 ++++- .../tests/service.test.ts | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index 94a3de6..54c3ab8 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -431,7 +431,12 @@ export default class WdioCucumberTestFramework extends TestFramework { this.cucumberData.scenario = undefined } - private hasStepFailures(): boolean { + /** + * Whether the scenario in flight failed in a STEP, as opposed to failing only in a hook. + * Public because `ignoreHooksStatus` has two surfaces (parity row 41): the o11y result below, + * and the Automate session status, which `service.afterScenario()` derives from the same answer. + */ + hasStepFailures(): boolean { return this.scenarioSteps.some(step => step.result === 'FAILED') } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 84f9a9e..3d25b0c 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -883,8 +883,17 @@ export default class BrowserstackService implements Services.ServiceInstance { */ private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { const status = world.result?.status?.toLowerCase() + + // `ignoreHooksStatus` has to reach session marking as well as the o11y result (parity row + // 41). On the CLI flow automateModule derives the session status from this view alone — + // service.after()'s _failReasons accumulation, which applied the flag on the legacy path, + // is gated off while the binary is up. A missing framework keeps the raw status. + const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true + const hasStepFailures = this._cliCucumberFramework()?.hasStepFailures() ?? true + const hookOnlyFailure = ignoreHooksStatus && status === 'failed' && !hasStepFailures + return { - passed: status === 'passed', + passed: status === 'passed' || hookOnlyFailure, skipped: status !== undefined && status !== 'passed' && status !== 'failed', error: world.result?.message ? new Error(world.result.message) : undefined, duration: 0, diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index bf544f9..29afdd2 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -10,6 +10,7 @@ import { BrowserstackCLI } from '../src/cli/index.js' import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' +import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFramework.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' @@ -2773,3 +2774,49 @@ describe('afterTest bail skip cascade (SDK-7063)', () => { expect(skippedTitles(trackEvent)).toEqual([]) }) }) + +describe('afterScenario session-status view honours ignoreHooksStatus (parity row 41)', () => { + let getInstanceSpy: ReturnType + + const makeService = (ignoreHooksStatus: boolean) => new BrowserstackService( + { testObservability: false, testObservabilityOptions: { ignoreHooksStatus } } as any, + [] as any, + { user: 'foo', key: 'bar', framework: 'cucumber' } as any + ) + + const runAfterScenario = async (svc: BrowserstackService, hadStepFailures: boolean) => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + vi.spyOn(framework, 'hasStepFailures').mockReturnValue(hadStepFailures) + const trackEvent = vi.spyOn(framework, 'trackEvent').mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework + } as any) + + await svc.afterScenario({ + pickle: { name: 'a scenario' }, + result: { status: 'FAILED', message: 'hook blew up' } + } as any) + + return (trackEvent.mock.calls.at(-1)?.[2] as any)?.result + } + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('reports a hook-only failure as passed so the session is not marked failed', async () => { + const result = await runAfterScenario(makeService(true), false) + expect(result.passed).toBe(true) + }) + + it('still reports a step failure as failed under the same flag', async () => { + const result = await runAfterScenario(makeService(true), true) + expect(result.passed).toBe(false) + }) + + it('leaves a hook-only failure failed when the flag is not set', async () => { + const result = await runAfterScenario(makeService(false), false) + expect(result.passed).toBe(false) + }) +}) From e6596eb48615e14c6096a6acb7c57659bbee05f5 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 3 Sep 2026 18:01:27 +0530 Subject: [PATCH 05/25] fix(cli): route turboscale marking, hook-failure verdicts, a11y gate and custom tags Four product fixes on the CLI/binary flow, all in cli/modules/*. Turboscale sessions were marked against the Automate REST API: the URL was a two-way ternary (app-automate / automate) and the verb was always PUT, where turboscale needs PATCH against /automate-turboscale/v1/sessions. Both markers now share one three-way resolver, so the path and the verb cannot drift apart. wdio_mocha carried the same defect and is repaired by the same change. A failing BeforeAll/AfterAll produced no scenario result, so it could never enter the per-test map onAfterExecute aggregates and the session came back passed. Cucumber-gated, and it honours ignoreHooksStatus the same way the scenario surface does. The scenarios a failed BeforeAll abandons now reach Test Observability as skipped rather than vanishing. The cascade publishes straight to TestHub, as the legacy listener did, so it does not rename the session or fire a scan or a Percy teardown per skipped row. TestHub's v2 pipeline builds the test row from the start event, so each row sends a start followed by the skip; a lone TestRunSkipped is accepted and counted in no bucket. accessibilityModule.onHookStart re-opened the scan gate for every framework, resting on beforeEach preceding beforeTest. Cucumber inverts that ordering, so the write landed last and scanned every scenario regardless of the tag filters. Narrowed to mocha, matching the legacy handler. setCustomTags had no framework gate and had quietly started working for cucumber, where the legacy handler warns and no-ops. Gated back. Co-Authored-By: Claude Opus 5 (1M context) --- .../frameworks/wdioCucumberTestFramework.ts | 96 +++++++- .../src/cli/modules/accessibilityModule.ts | 13 +- .../src/cli/modules/automateModule.ts | 155 +++++++++++-- .../src/cli/modules/customTagsModule.ts | 15 ++ packages/browserstack-service/src/service.ts | 63 +++++- .../cli/modules/accessibilityModule.test.ts | 44 +++- .../cli/modules/automateModule.phase8.test.ts | 207 ++++++++++++++++++ .../tests/cli/modules/automateModule.test.ts | 16 +- .../cli/modules/customTagsModule.test.ts | 105 +++++++++ .../tests/service.test.ts | 121 ++++++++++ 10 files changed, 803 insertions(+), 32 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts create mode 100644 packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index 54c3ab8..e1d46b4 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -13,7 +13,7 @@ import { TEST_ANALYTICS_ID } from '../../constants.js' import { getScenarioExamples, removeAnsiColors } from '../../util.js' import type { Frameworks } from '@wdio/types' -import type { CucumberHook, Feature, ITestCaseHookParameter, Pickle } from '../../cucumber-types.js' +import type { CucumberHook, Feature, FeatureChild, ITestCaseHookParameter, Pickle, Scenario, Step } from '../../cucumber-types.js' /** * `test_duration` and `bdd_meta_info` are read by the binary's WebdriverIO-cucumber module but @@ -33,6 +33,14 @@ const KEY_HOOK_SCOPE = 'hook_scope' const KEY_HOOK_RETRIES = 'hook_retries' const KEY_HOOK_DURATION = 'hook_duration' +/** + * Marks a TEST/POST event as a BEFORE_ALL cascade row rather than a real scenario finish, so the + * binary's cucumber module emits `TestRunSkipped` — legacy's wire event for this case — instead of + * `TestRunFinished`. The state->event mapping in the WDIO language index is shared with mocha and + * is deliberately not touched. + */ +const KEY_TEST_SKIPPED_CASCADE = 'test_skipped_cascade' + type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' const HOOK_STATES: Record = { @@ -431,6 +439,92 @@ export default class WdioCucumberTestFramework extends TestFramework { this.cucumberData.scenario = undefined } + /** + * Synthesise one detached instance per scenario the feature never got to run, for the + * BEFORE_ALL failure cascade (parity row 15). Rule-nested scenarios included. + * + * Detached is load-bearing: these are NOT registered via `setTrackedInstance`, so the real + * per-scenario instance and `process.env[TEST_ANALYTICS_ID]` are untouched. The caller sends + * each one straight to TestHub rather than through `runHooks`, mirroring legacy — whose + * cascade called `listener.testFinished()` directly and so never reached the Automate, + * Accessibility or Percy handlers. Dispatching these through the observer set instead would + * rename the session, fire an a11y stop event and run a Percy teardown per skipped row, none + * of which legacy does. + * + * Parity row 18: no tags — legacy's cascade payload has no `world`, so `test_tags` is absent. + */ + buildSkippedScenarioInstances(): TestFrameworkInstance[] { + const feature = this.cucumberData.feature + if (!feature) { + logger.debug('buildSkippedScenarioInstances: no feature recorded; nothing to cascade') + return [] + } + + const scenarios: Scenario[] = [] + for (const child of (feature.children || []) as FeatureChild[]) { + if (child.rule) { + for (const ruleChild of (child.rule.children || [])) { + if (ruleChild.scenario) { + scenarios.push(ruleChild.scenario) + } + } + } else if (child.scenario) { + scenarios.push(child.scenario) + } + } + + const featurePath = this.featurePath() + return scenarios.map(scenario => this.buildSkippedScenarioInstance(scenario, feature, featurePath)) + } + + private buildSkippedScenarioInstance(scenario: Scenario, feature: Feature, featurePath: string | undefined): TestFrameworkInstance { + const now = new Date().toISOString() + const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.TEST, + HookState.POST + ) + + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + // A cascade row is identified by the RAW scenario name on both fields: the feature + // never ran, so no Examples row was ever selected and there is nothing to qualify. + [TestFrameworkConstants.KEY_TEST_ID]: scenario.name, + [TestFrameworkConstants.KEY_TEST_NAME]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPE]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature.name || ''], + [TestFrameworkConstants.KEY_TEST_CODE]: null, + [TestFrameworkConstants.KEY_TEST_RESULT]: 'skipped', + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: now, + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: now, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, + ...resolveFeatureFilePaths(featurePath), + [KEY_TEST_SKIPPED_CASCADE]: true, + [KEY_BDD_META_INFO]: { + feature: { name: feature.name, path: featurePath, description: feature.description }, + scenario: { name: scenario.name }, + steps: (scenario.steps || []).map((step: Step) => ({ + id: step.id, + text: step.text, + keyword: step.keyword, + result: 'skipped', + })), + examples: [], + }, + }) + + return instance + } + /** * Whether the scenario in flight failed in a STEP, as opposed to failing only in a hook. * Public because `ignoreHooksStatus` has two surfaces (parity row 41): the o11y result below, diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 6f6ea0e..66edfd3 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -89,7 +89,18 @@ export default class AccessibilityModule extends BaseModule { // Open the scan gate for the hook window so DOM-changing commands issued inside // before/beforeEach/afterEach/after hooks trigger scans (web per-command path). The // following onBeforeTest re-computes the per-test gate, so this only affects the hook. - if (this.autoScanning && sessionId !== undefined && sessionId !== null) { + // + // Mocha-only, exactly as legacy gates the identical write (accessibility-handler + // beforeHook, `this._framework === 'mocha'`). The "onBeforeTest re-computes it after" + // invariant above holds only where beforeEach precedes beforeTest. Cucumber inverts + // that — WDIO raises the scenario boundary BEFORE cucumber's own Before hooks — so + // there the write lands last and permanently forces the gate open, scanning every + // scenario regardless of includeTagsInTestingScope / excludeTagsInTestingScope and + // undoing a user's stopA11yScanning(). The currentHookRunUuid capture above is + // correct for every framework and stays outside this gate. + const frameworkName = String(TestFramework.getState(testInstance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + const reopensGateForHook = frameworkName.toLowerCase().includes('mocha') + if (this.autoScanning && reopensGateForHook && sessionId !== undefined && sessionId !== null) { this.accessibilityMap.set(sessionId, true) } } catch (error) { diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index fcdc336..a9d6bce 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -46,6 +46,10 @@ export default class AutomateModule extends BaseModule { TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) TestFramework.registerObserver(AutomationFrameworkState.EXECUTE, HookState.POST, this.onAfterExecute.bind(this)) + // Build-level hooks carry no scenario result, so they reach the session verdict only + // through their own state. See onBuildLevelHookEnd — cucumber-gated inside the handler. + TestFramework.registerObserver(TestFrameworkState.BEFORE_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'BEFORE_ALL')) + TestFramework.registerObserver(TestFrameworkState.AFTER_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'AFTER_ALL')) } getModuleName(): string { @@ -217,6 +221,86 @@ export default class AutomateModule extends BaseModule { TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_REASON, reason) } + /** + * A `BeforeAll` / `AfterAll` failure produces no scenario result, so it can never enter the + * per-test `testResults` map that onAfterExecute aggregates — a run whose BeforeAll blew up + * reports its session as PASSED. Legacy pushed the hook error into `_failReasons` and + * `after()` marked the session failed; that whole accumulation is gated + * `setSessionStatus && !BrowserstackCLI.isRunning()`, so it is dead while the binary is up. + * + * Cucumber-gated deliberately. `wdio_mocha` has the identical latent shape on this flow, but + * legacy mocha behaved the same way, so repairing it here would be an unrequested behaviour + * change to the one framework already working on the CLI flow. + */ + async onBuildLevelHookEnd(hookKey: string, args: Record) { + try { + const instance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!instance || !this.isCucumberInstance(instance)) { + return + } + + const result = args?.result as { passed?: boolean, error?: Error } | undefined + if (!result || result.passed) { + return + } + + // Parity row 41, third surface: with ignoreHooksStatus declared, a failure that exists + // only in a hook must leave the session passed. Legacy expresses this by skipping the + // `_failReasons` push in afterHook; skipping the record here is the same decision. + if (isTrue(args?.ignoreHooksStatus)) { + this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) + return + } + + const testContextOptions = this.config.testContextOptions as TestContextOptions + if (testContextOptions?.skipSessionStatus) { + return + } + + const autoInstance = AutomationFramework.getTrackedInstance() + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + if (!sessionId) { + this.logger.debug(`onBuildLevelHookEnd: no session id resolved for ${hookKey}; nothing to mark`) + return + } + + const sessionData = this.sessionMap.get(sessionId) + if (!sessionData) { + // A BeforeAll can fail before any scenario ran, so the session may not be + // registered yet. `lastTestName` stays empty on purpose — flushSessionName + // early-returns on it, so registering here cannot rename the session. + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + } + + const name = this.resolveHookName(instance, hookKey) + this.sessionMap.get(sessionId)!.testResults.set(name, { + testName: name, + status: 'failed', + reason: (result.error && result.error.message) || 'Hook failed' + }) + this.logger.info(`onBuildLevelHookEnd: recorded ${hookKey} failure against session ${sessionId}`) + } catch (error) { + this.logger.error(`Exception in automate onBuildLevelHookEnd: ${error}`) + } + } + + private isCucumberInstance(instance: TestFrameworkInstance): boolean { + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + return frameworkName.toLowerCase().includes('cucumber') + } + + /** The hook's reported name (`BEFORE_ALL for `), so the session reason names the hook. */ + private resolveHookName(instance: TestFrameworkInstance, hookKey: string): string { + try { + const finished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> | undefined + const hooks = finished?.get(hookKey) + const hookName = hooks?.length ? hooks[hooks.length - 1][TestFrameworkConstants.KEY_HOOK_NAME] : undefined + return (hookName as string) || hookKey + } catch { + return hookKey + } + } + async onAfterExecute() { this.logger.debug('onAfterExecute: inside automate module after execute hook!') @@ -266,29 +350,64 @@ export default class AutomateModule extends BaseModule { return this.hasAppCapInFrameworkState() } + // The binary echoes the parsed `turboScale` flag back on the session config; the env var is + // written unconditionally by the service constructor in this same worker process, so it stands + // in when a config shape predates the flag. + private isTurboScale(): boolean { + return isTrue(this.config.turboScale) || isTrue(process.env.BROWSERSTACK_TURBOSCALE_INTERNAL) + } + + /** + * Resolve the REST endpoint a session marker must hit. + * + * Turboscale is not a variant of Automate here — it is a different API on a different path + * with a different VERB (PATCH, not PUT). The legacy path expressed this through + * `_sessionBaseUrl` + `_update()`, both gated `!BrowserstackCLI.isRunning()`, so neither + * survives onto the CLI flow and nothing in the binary compensates. + * + * Precedence mirrors legacy's assignment order in `beforeSession()`: the turboscale base URL + * is assigned AFTER the app-automate one, so a turboscale grid wins even with an app cap set. + * + * Single resolver for both markers deliberately: naming and status previously duplicated the + * ternary, which is how the two can drift apart. + */ + private resolveSessionApi(sessionId: string): { url: string, method: 'PUT' | 'PATCH', product: string } { + if (this.isTurboScale()) { + return { + url: `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate-turboscale/v1/sessions/${sessionId}.json`, + method: 'PATCH', + product: 'Automate TurboScale' + } + } + if (this.isAppAutomate()) { + return { + url: `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json`, + method: 'PUT', + product: 'App Automate' + } + } + return { + url: `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json`, + method: 'PUT', + product: 'Automate' + } + } + async markSessionName(sessionId: string, sessionName: string, config: { user: string; key: string; }): Promise { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_NAME, async (sessionId: string, sessionName: string, config: { user: string; key: string; }) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') - const isAppAutomate = this.isAppAutomate() - if (isAppAutomate) { - this.logger.info('Marking session name for App Automate') - } else { - this.logger.info('Marking session name for Automate') - } - - const sessionNameApiUrl = isAppAutomate - ? `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json` - : `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json` + const { url: sessionNameApiUrl, method, product } = this.resolveSessionApi(sessionId) + this.logger.info(`Marking session name for ${product}`) const requestBody = { name: sessionName } const options = { - method: 'PUT', + method, headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' @@ -312,16 +431,8 @@ export default class AutomateModule extends BaseModule { async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') - const isAppAutomate = this.isAppAutomate() - if (isAppAutomate) { - this.logger.info('Marking session status for App Automate') - } else { - this.logger.info('Marking session status for Automate') - } - - const sessionStatusApiUrl = isAppAutomate - ? `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json` - : `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json` + const { url: sessionStatusApiUrl, method, product } = this.resolveSessionApi(sessionId) + this.logger.info(`Marking session status for ${product}`) const body = { status: sessionStatus, @@ -329,7 +440,7 @@ export default class AutomateModule extends BaseModule { } const options = { - method: 'PUT', + method, headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' diff --git a/packages/browserstack-service/src/cli/modules/customTagsModule.ts b/packages/browserstack-service/src/cli/modules/customTagsModule.ts index ce24ae6..b70b679 100644 --- a/packages/browserstack-service/src/cli/modules/customTagsModule.ts +++ b/packages/browserstack-service/src/cli/modules/customTagsModule.ts @@ -11,6 +11,7 @@ import { TestFrameworkState } from '../states/testFrameworkState.js' import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' import { CLIUtils } from '../cliUtils.js' import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' +import { BrowserstackCLI } from '../index.js' import { mergeIntoTags, parseCommaSeparatedValues, getCurrentMochaHookWindow } from '../../customTags.js' import type { CustomMetadata } from '../../customTags.js' @@ -54,6 +55,10 @@ export default class CustomTagsModule extends BaseModule { return CustomTagsModule.MODULE_NAME } + private isMochaFramework(): boolean { + return BrowserstackCLI.getInstance().getTestFramework() instanceof WdioMochaTestFramework + } + async onBeforeExecute() { try { const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() @@ -70,6 +75,16 @@ export default class CustomTagsModule extends BaseModule { (browser as WebdriverIO.Browser).setCustomTags = async (key: string, value: string): Promise => { try { + // Mocha-only, matching the legacy handler (custom-tags-handler, which warns + // and no-ops for any framework other than mocha). The method is still + // registered so the call resolves and the warning reaches the user, exactly + // as on the legacy flow. Without this gate the CLI path silently STARTS + // supporting custom tags for other frameworks — a widening, but an + // unrequested behaviour change all the same. + if (!this.isMochaFramework()) { + this.logger.warn('setCustomTags is only supported for the mocha framework; ignoring call') + return + } if (!key || !value) { this.logger.warn('setCustomTags: key and value are required; ignoring call') return diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 3d25b0c..a727f2a 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -489,11 +489,22 @@ export default class BrowserstackService implements Services.ServiceInstance { const framework = BrowserstackCLI.getInstance().getTestFramework() if (framework instanceof WdioCucumberTestFramework) { // See beforeHook: cucumber's taxonomy, not Mocha's titles. The suite-skip cascade - // below is Mocha-shaped (it walks `test.ctx.test.parent`) and has no cucumber - // counterpart here. + // in the mocha arm below is Mocha-shaped (it walks `test.ctx.test.parent`); + // cucumber's own cascade is _reportCucumberScenariosSkipped, further down. const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) if (hookFrameworkState) { - await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) + // ignoreHooksStatus rides the event so automateModule can apply the same flag + // to the session verdict that loadScenarioResult applies to the o11y result + // (parity row 41). The module cannot read it — the binary-supplied config it + // holds carries no testObservabilityOptions. + await framework.trackEvent(hookFrameworkState, HookState.POST, { + test, + result, + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true + }) + } + if (hookFrameworkState === TestFrameworkState.BEFORE_ALL && result && !result.passed) { + await this._reportCucumberScenariosSkipped(framework) } return } @@ -518,6 +529,52 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._accessibilityHandler?.afterHook() } + /** + * BEFORE_ALL failure cascade — parity row 15. Cucumber abandons the whole feature when a + * `BeforeAll` throws, so every scenario in it (Rule-nested ones included) must be reported + * SKIPPED rather than simply vanishing. + * + * Legacy did this from `insights-handler.afterHook` via `sendScenarioObjectSkipped()`, which + * publishes through the legacy HTTP listener (`api/v1/batch`) — inert once the binary is up. + * That is escape class 3 / SDK-7047 in its documented form, and the repair is to give the + * cascade a CLI/gRPC publisher. + * + * Sent straight to TestHub rather than through `framework.trackEvent()`: legacy's cascade + * called `listener.testFinished()` directly and so bypassed every product handler. Routing + * these through the observer set would rename the Automate session, fire an accessibility + * stop event and run a Percy teardown once per skipped row — none of which legacy does. + * + * Cucumber-only: private, one call site, in the `instanceof WdioCucumberTestFramework` arm. + */ + private async _reportCucumberScenariosSkipped(framework: WdioCucumberTestFramework) { + try { + const testHubModule = BrowserstackCLI.getInstance().modules.TestHubModule as TestHubModule | undefined + if (!testHubModule) { + BStackLogger.debug('BEFORE_ALL cascade: TestHub module not loaded; skipped scenarios will not be reported') + return + } + + const instances = framework.buildSkippedScenarioInstances() + for (const instance of instances) { + // Both halves, because TestHub's v2 batch pipeline creates the test row from the + // START event and treats TestRunSkipped as its terminal — a lone TestRunSkipped is + // accepted and then counted in no bucket at all. The legacy v1 listener created the + // row from the skip event itself, which is why it sent only one. + await testHubModule.sendTestFrameworkEvent( + { instance }, + { testFrameworkState: 'TEST', testHookState: 'PRE' } + ) + await testHubModule.sendTestFrameworkEvent( + { instance }, + { testFrameworkState: 'TEST', testHookState: 'POST' } + ) + } + BStackLogger.debug(`BEFORE_ALL cascade: reported ${instances.length} scenario(s) as skipped`) + } catch (err) { + BStackLogger.debug(`Exception reporting the BEFORE_ALL skip cascade: ${util.format(err)}`) + } + } + @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeTest' }) async beforeTest (test: Frameworks.Test) { this._currentTest = test diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 0c77511..4f0eda4 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -534,20 +534,58 @@ describe('AccessibilityModule', () => { } }) - it('captures the hook run uuid and opens the scan gate at hook start', async () => { - vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + // The gate write is Mocha-only, exactly as legacy gates it. Keyed rather than blanket + // mock: the handler now reads the framework name off the instance too. + const mockInstanceState = (frameworkName: string) => { + vi.mocked(TestFramework.getState).mockImplementation((_i: any, key: string) => + (key === 'test_framework_name' ? frameworkName : 'hook-uuid-123') as any) vi.mocked(AutomationFramework.getState).mockImplementation((instance: any, key: string) => (key.includes('session_id') ? 12345 : {}) as any) + } + + it('captures the hook run uuid and opens the scan gate at hook start', async () => { + mockInstanceState('WebdriverIO-mocha') + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(true) + }) + + // 8-C. Discriminating: identical call, opposite answers. Mocha's beforeEach precedes + // beforeTest so the re-open is harmless; cucumber's scenario boundary precedes its Before + // hooks, so the same write would permanently force the gate open and defeat the + // includeTagsInTestingScope / excludeTagsInTestingScope filtering (parity row 35). + it('does NOT re-open the scan gate for cucumber — the per-test gate stands', async () => { + mockInstanceState('WebdriverIO-cucumber') + accessibilityModule.accessibilityMap.set(12345, false) await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(false) + }) + + it('re-opens a closed gate for mocha — the opposite answer on the same input', async () => { + mockInstanceState('WebdriverIO-mocha') + accessibilityModule.accessibilityMap.set(12345, false) + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(true) }) + it('still captures the hook run uuid for cucumber (app-a11y hook-scan stamping)', async () => { + mockInstanceState('WebdriverIO-cucumber') + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + }) + it('does not open the scan gate when accessibility is disabled', async () => { accessibilityModule.accessibility = false - vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + mockInstanceState('WebdriverIO-mocha') await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts new file mode 100644 index 0000000..8277245 --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' +import AutomateModule from '../../../src/cli/modules/automateModule.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import { _fetch as fetch } from '../../../src/fetchWrapper.js' +import type { Options } from '@wdio/types' + +vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ + default: { + registerObserver: vi.fn(), + setState: vi.fn(), + getState: vi.fn(), + getTrackedInstance: vi.fn() + } +})) + +vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ + default: { + getTrackedInstance: vi.fn(), + getState: vi.fn(), + getDriver: vi.fn() + } +})) + +vi.mock('../../../src/cli/cliLogger.js', () => ({ + BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } +})) + +vi.mock('../../../src/util.js', () => ({ + isBrowserstackSession: vi.fn(() => true), + isTrue: vi.fn((value) => (value + '').toLowerCase() === 'true'), + hasAppCap: vi.fn(() => false) +})) + +vi.mock('../../../src/instrumentation/performance/performance-tester.js', () => ({ + default: { measureWrapper: vi.fn((event, fn) => fn) } +})) + +vi.mock('../../../src/fetchWrapper.js', () => ({ _fetch: vi.fn() })) + +const cucumberInstance = { framework: 'WebdriverIO-cucumber' } +const mochaInstance = { framework: 'WebdriverIO-mocha' } + +function stateFor(instance: unknown, key: string) { + if (key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) { + return (instance as { framework: string }).framework + } + if (key === TestFrameworkConstants.KEY_HOOKS_FINISHED) { + return new Map([['BEFORE_ALL', [{ [TestFrameworkConstants.KEY_HOOK_NAME]: 'BEFORE_ALL for Login' }]]]) + } + return undefined +} + +function newModule(config: Record = {}) { + const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) + mod.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey', + ...config + } as never + return mod +} + +describe('AutomateModule — Phase 8 remediations', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(TestFramework.getState).mockImplementation((instance, key) => stateFor(instance, key)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + afterEach(() => { + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + /** + * 8-A. Discriminating: the SAME call produces opposite verbs and different hosts/paths + * depending only on the turboscale flag. + */ + describe('8-A — turboscale session marking routes to its own API with PATCH', () => { + it('PATCHes the turboscale endpoint when turboScale is configured', async () => { + const mod = newModule({ turboScale: true }) + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate-turboscale/v1/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PATCH') + }) + + it('PUTs the automate endpoint when turboScale is not configured', async () => { + const mod = newModule() + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PUT') + }) + + it('honours the BROWSERSTACK_TURBOSCALE_INTERNAL fallback', async () => { + process.env.BROWSERSTACK_TURBOSCALE_INTERNAL = 'true' + const mod = newModule() + await mod.markSessionName('sess-1', 'a name', { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toContain('/automate-turboscale/v1/sessions/') + expect((options as { method: string }).method).toBe('PATCH') + }) + + it('takes precedence over app-automate, mirroring legacy assignment order', async () => { + const mod = newModule({ turboScale: true, app: 'bs://app' }) + await mod.markSessionStatus('sess-1', 'failed', 'boom', { user: 'u', key: 'k' }) + + expect(vi.mocked(fetch).mock.calls[0][0]).toContain('/automate-turboscale/v1/sessions/') + }) + + it('names and statuses agree on verb and path', async () => { + const mod = newModule({ turboScale: true }) + await mod.markSessionName('sess-1', 'a name', { user: 'u', key: 'k' }) + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [nameUrl, nameOpts] = vi.mocked(fetch).mock.calls[0] + const [statusUrl, statusOpts] = vi.mocked(fetch).mock.calls[1] + expect(nameUrl).toBe(statusUrl) + expect((nameOpts as { method: string }).method).toBe((statusOpts as { method: string }).method) + }) + }) + + /** + * 8-B (session verdict). Discriminating: the SAME failing build-level hook fails the session + * for cucumber and leaves mocha's verdict untouched. + */ + describe('8-B — build-level hook failures reach the session verdict', () => { + const failing = { passed: false, error: new Error('BeforeAll blew up') } + + it('marks the session failed for cucumber when a BeforeAll fails', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + const statusCall = vi.mocked(fetch).mock.calls.find(([, o]) => + JSON.parse((o as { body: string }).body).status !== undefined)! + const body = JSON.parse((statusCall[1] as { body: string }).body) + expect(body.status).toBe('failed') + expect(body.reason).toBe('BeforeAll blew up') + }) + + it('names the hook in the failure reason', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: false, error: new Error('teardown') } }) + await mod.onAfterExecute() + + const body = JSON.parse((vi.mocked(fetch).mock.calls[0][1] as { body: string }).body) + expect(body.reason).toContain('BEFORE_ALL for Login') + }) + + it('leaves wdio_mocha untouched — the identical failure records nothing', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: mochaInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('records nothing when the hook passed', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: { passed: true } }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the session PASSED under ignoreHooksStatus (parity row 41)', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { + instance: cucumberInstance, + result: failing, + ignoreHooksStatus: true + }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('respects skipSessionStatus', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: false, skipSessionStatus: true } }) + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('does not rename the session when the hook failure is the only record', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + const bodies = vi.mocked(fetch).mock.calls.map(([, o]) => JSON.parse((o as { body: string }).body)) + expect(bodies.some(b => b.name !== undefined)).toBe(false) + }) + }) +}) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 3fca63a..6cc18e4 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -14,7 +14,9 @@ import type { Options } from '@wdio/types' vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ default: { registerObserver: vi.fn(), - setState: vi.fn() + setState: vi.fn(), + getState: vi.fn(), + getTrackedInstance: vi.fn() } })) @@ -113,7 +115,7 @@ describe('AutomateModule', () => { // Create new instance to test observer registration (constructor registers the observers) new AutomateModule(mockConfig) - expect(TestFramework.registerObserver).toHaveBeenCalledTimes(3) + expect(TestFramework.registerObserver).toHaveBeenCalledTimes(5) expect(TestFramework.registerObserver).toHaveBeenCalledWith( TestFrameworkState.TEST, HookState.PRE, @@ -129,6 +131,16 @@ describe('AutomateModule', () => { HookState.POST, expect.any(Function) ) + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.BEFORE_ALL, + HookState.POST, + expect.any(Function) + ) + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.AFTER_ALL, + HookState.POST, + expect.any(Function) + ) }) it('should have correct static MODULE_NAME', () => { diff --git a/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts new file mode 100644 index 0000000..83c0c3c --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' + +vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ + default: class MockTestFramework { + static registerObserver = vi.fn() + static getTrackedInstance = vi.fn() + static getState = vi.fn() + } +})) + +vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ + default: class MockAutomationFramework { + static registerObserver = vi.fn() + static getTrackedInstance = vi.fn() + static getDriver = vi.fn() + } +})) + +vi.mock('../../../src/cli/cliLogger.js', () => ({ + BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } +})) + +vi.mock('../../../src/cli/index.js', () => ({ + BrowserstackCLI: { getInstance: vi.fn() } +})) + +import CustomTagsModule from '../../../src/cli/modules/customTagsModule.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import WdioMochaTestFramework from '../../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../../src/cli/frameworks/wdioCucumberTestFramework.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' +import { BStackLogger } from '../../../src/cli/cliLogger.js' + +/** + * 8-D — parity row 19. `setCustomTags` warns and no-ops for every framework except mocha, which + * is what the legacy custom-tags-handler does. Discriminating: the SAME call merges tags under + * mocha and merges nothing under cucumber. + */ +describe('CustomTagsModule — framework gate (parity row 19)', () => { + let module: CustomTagsModule + let browser: Record + let instance: { updateMultipleEntries: ReturnType, getCurrentTestState: ReturnType } + + const useFramework = (framework: unknown) => { + vi.mocked(BrowserstackCLI.getInstance).mockReturnValue({ + getTestFramework: () => framework + } as never) + } + + beforeEach(async () => { + vi.clearAllMocks() + browser = {} + instance = { + updateMultipleEntries: vi.fn(), + getCurrentTestState: vi.fn().mockReturnValue('TestFrameworkState.TEST') + } + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getDriver).mockReturnValue(browser as never) + vi.mocked(TestFramework.getTrackedInstance).mockReturnValue(instance as never) + vi.mocked(TestFramework.getState).mockReturnValue({} as never) + + module = new CustomTagsModule() + }) + + const register = async () => { + await module.onBeforeExecute() + return browser.setCustomTags as (k: string, v: string) => Promise + } + + it('registers setCustomTags regardless of framework, so the call always resolves', async () => { + useFramework(Object.create(WdioCucumberTestFramework.prototype)) + expect(await register()).toBeTypeOf('function') + }) + + it('merges tags for mocha', async () => { + useFramework(Object.create(WdioMochaTestFramework.prototype)) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1,TC-2') + + expect(instance.updateMultipleEntries).toHaveBeenCalled() + }) + + it('warns and no-ops for cucumber — the opposite answer on the same call', async () => { + useFramework(Object.create(WdioCucumberTestFramework.prototype)) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1,TC-2') + + expect(instance.updateMultipleEntries).not.toHaveBeenCalled() + expect(BStackLogger.warn).toHaveBeenCalledWith( + 'setCustomTags is only supported for the mocha framework; ignoring call' + ) + }) + + it('warns and no-ops when no CLI test framework is registered', async () => { + useFramework(null) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1') + + expect(instance.updateMultipleEntries).not.toHaveBeenCalled() + }) +}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 29afdd2..44eed94 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -11,6 +11,8 @@ import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFramework.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' @@ -2820,3 +2822,122 @@ describe('afterScenario session-status view honours ignoreHooksStatus (parity ro expect(result.passed).toBe(false) }) }) + +describe('BEFORE_ALL skip cascade + hook flag pass-through (parity row 15, escape class 3 / SDK-7047)', () => { + let getInstanceSpy: ReturnType + + const feature = { + name: 'Login', + description: 'a description', + children: [ + { scenario: { name: 'Scenario A', steps: [{ id: 's1', text: 'I log in', keyword: 'Given ' }] } }, + { background: {} }, + { rule: { children: [{ background: {} }, { scenario: { name: 'Rule-nested B', steps: [] } }] } } + ] + } + + const makeService = (ignoreHooksStatus = false) => new BrowserstackService( + { testObservability: false, testObservabilityOptions: { ignoreHooksStatus } } as any, + [] as any, + { user: 'foo', key: 'bar', framework: 'cucumber' } as any + ) + + const runAfterHook = async (svc: BrowserstackService, result: unknown, scenariosStarted = false) => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + framework.onFeatureStart('features/login.feature', feature as any) + if (scenariosStarted) { + // flips classifyHookType from BEFORE_ALL to AFTER_ALL + ;(framework as any).cucumberData.scenariosStarted = true + } + const trackEvent = vi.spyOn(framework, 'trackEvent').mockResolvedValue(undefined) + const sendTestFrameworkEvent = vi.fn().mockResolvedValue(true) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + modules: { TestHubModule: { sendTestFrameworkEvent } } + } as any) + + await svc.afterHook(undefined as any, {}, result as any) + return { sendTestFrameworkEvent, trackEvent } + } + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('reports every not-run scenario as skipped, Rule-nested ones included', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + // two scenarios x (start, skip) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(4) + const names = sendTestFrameworkEvent.mock.calls.map(([args]: any[]) => + args.instance.getAllData().get('test_name')) + expect(names).toEqual(['Scenario A', 'Scenario A', 'Rule-nested B', 'Rule-nested B']) + }) + + // TestHub's v2 batch pipeline creates the test row from the START event; a lone TestRunSkipped + // is accepted and lands in no bucket at all (observed: O11Y build ad39yy… reported skipped=0 + // for three sent TestRunSkipped events, lwbpu2… reported skipped=3 for the pair). + it('sends a start then a skip per scenario, so the binary emits TestRunStarted + TestRunSkipped', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + const overrides = sendTestFrameworkEvent.mock.calls.map(([, o]: any[]) => o.testHookState) + expect(overrides).toEqual(['PRE', 'POST', 'PRE', 'POST']) + + for (const [args, override] of sendTestFrameworkEvent.mock.calls as any[]) { + expect(override.testFrameworkState).toBe('TEST') + const data = args.instance.getAllData() + expect(data.get('test_result')).toBe('skipped') + expect(data.get('test_skipped_cascade')).toBe(true) + // parity row 18 — the cascade payload has no world, so no tags + expect(data.get('test_tags')).toBeUndefined() + expect(data.get('test_scopes')).toEqual(['Login']) + } + }) + + it('marks every step of a skipped scenario skipped', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + const meta = sendTestFrameworkEvent.mock.calls[0][0].instance.getAllData().get('bdd_meta_info') as any + expect(meta.steps).toEqual([{ id: 's1', text: 'I log in', keyword: 'Given ', result: 'skipped' }]) + expect(meta.feature.name).toBe('Login') + }) + + it('does not cascade when the BEFORE_ALL passed', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: true }) + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('does not cascade for an AFTER_ALL failure — the scenarios already ran', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }, true) + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('passes ignoreHooksStatus on the hook event so the module can honour it', async () => { + const { trackEvent } = await runAfterHook(makeService(true), { passed: false, error: new Error('boom') }) + expect((trackEvent.mock.calls.at(-1)?.[2] as any).ignoreHooksStatus).toBe(true) + + const off = await runAfterHook(makeService(false), { passed: false, error: new Error('boom') }) + expect((off.trackEvent.mock.calls.at(-1)?.[2] as any).ignoreHooksStatus).toBe(false) + }) + + it('reports the hook failure on the hook event itself, not as passed', async () => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + framework.onFeatureStart('features/login.feature', feature as any) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + modules: { TestHubModule: { sendTestFrameworkEvent: vi.fn().mockResolvedValue(true) } } + } as any) + + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.PRE, { test: undefined }) + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { + test: undefined, + result: { passed: false, error: new Error('boom'), duration: 12, retries: { attempts: 0, limit: 0 } } + }) + + const instance = (framework as any).constructor.getTrackedInstance() + const finished = instance.getAllData().get('test_hooks_finished') as Map + expect(finished.get('BEFORE_ALL')![0].hook_result).toBe('failed') + }) +}) From bdf9cb120c06f8ce69e3e184f108a8b3800642b7 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 3 Sep 2026 20:39:59 +0530 Subject: [PATCH 06/25] fix(cli): mark the cucumber session failed when a build hook fails with no scenarios A failing BeforeAll under ignoreHooksStatus left the Automate session unmarked and so invisible on the dashboard. Legacy marks it failed through the !_specsRan arm of after(), which the flag never reaches. Keys the skip on a scenario result having been recorded, not on the flag. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/cli/modules/automateModule.ts | 23 +++++--- .../cli/modules/automateModule.phase8.test.ts | 55 ++++++++++++++++++- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index a9d6bce..0b57e85 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -244,14 +244,6 @@ export default class AutomateModule extends BaseModule { return } - // Parity row 41, third surface: with ignoreHooksStatus declared, a failure that exists - // only in a hook must leave the session passed. Legacy expresses this by skipping the - // `_failReasons` push in afterHook; skipping the record here is the same decision. - if (isTrue(args?.ignoreHooksStatus)) { - this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) - return - } - const testContextOptions = this.config.testContextOptions as TestContextOptions if (testContextOptions?.skipSessionStatus) { return @@ -265,6 +257,21 @@ export default class AutomateModule extends BaseModule { } const sessionData = this.sessionMap.get(sessionId) + // Parity row 41, third surface: with ignoreHooksStatus declared, a failure that exists + // only in a hook must leave the session passed. Legacy expresses that in the + // `ignoreHooksStatus && this._specsRan` arm of `after()`, and that arm needs BOTH. With + // no scenario recorded, legacy instead falls through to the arm that marks `failed` + // unconditionally — flag or no flag — so honouring the flag here would leave the session + // unmarked where legacy marks it, and an unmarked session is invisible on the dashboard. + // Keyed on the absence of scenario results, never on the flag. A cucumber `BeforeAll` + // failure aborts the run outright, so nothing can arrive after this point; by `AfterAll` + // every scenario that ran has already been recorded. + const specsRan = (sessionData?.testResults.size ?? 0) > 0 + if (specsRan && isTrue(args?.ignoreHooksStatus)) { + this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) + return + } + if (!sessionData) { // A BeforeAll can fail before any scenario ran, so the session may not be // registered yet. `lastTestName` stays empty on purpose — flushSessionName diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts index 8277245..87cdc85 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts @@ -137,6 +137,20 @@ describe('AutomateModule — Phase 8 remediations', () => { describe('8-B — build-level hook failures reach the session verdict', () => { const failing = { passed: false, error: new Error('BeforeAll blew up') } + /** Drives one scenario through TEST/POST so `testResults` carries a real scenario result. */ + const runScenario = (mod: AutomateModule, passed: boolean) => mod.onAfterTest({ + instance: cucumberInstance, + result: { error: passed ? null : new Error('step failed'), passed }, + test: { title: 'a scenario', fullName: 'Feature: a scenario' }, + suiteTitle: 'Feature' + }) + + const statusBody = () => { + const call = vi.mocked(fetch).mock.calls.find(([, o]) => + JSON.parse((o as { body: string }).body).status !== undefined) + return call ? JSON.parse((call[1] as { body: string }).body) : undefined + } + it('marks the session failed for cucumber when a BeforeAll fails', async () => { const mod = newModule() await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) @@ -175,7 +189,25 @@ describe('AutomateModule — Phase 8 remediations', () => { expect(fetch).not.toHaveBeenCalled() }) - it('keeps the session PASSED under ignoreHooksStatus (parity row 41)', async () => { + it('keeps the session PASSED under ignoreHooksStatus once a scenario has run (parity row 41)', async () => { + const mod = newModule() + await runScenario(mod, true) + await mod.onBuildLevelHookEnd('AFTER_ALL', { + instance: cucumberInstance, + result: failing, + ignoreHooksStatus: true + }) + await mod.onAfterExecute() + + expect(statusBody()).toEqual({ status: 'passed' }) + }) + + /** + * Zero scenarios is legacy's `!_specsRan` arm, which marks failed with no regard for the + * flag. Discriminating against the case directly above: identical hook, identical flag, + * opposite verdicts — the scenario having run is the only difference. + */ + it('marks the session FAILED under ignoreHooksStatus when no scenario ran', async () => { const mod = newModule() await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, @@ -184,9 +216,30 @@ describe('AutomateModule — Phase 8 remediations', () => { }) await mod.onAfterExecute() + expect(statusBody().status).toBe('failed') + }) + + it('leaves wdio_mocha unmarked on that same zero-scenario case', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { + instance: mochaInstance, + result: failing, + ignoreHooksStatus: true + }) + await mod.onAfterExecute() + expect(fetch).not.toHaveBeenCalled() }) + it('does not fail a session whose scenarios all passed and whose hooks all passed', async () => { + const mod = newModule() + await runScenario(mod, true) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: true } }) + await mod.onAfterExecute() + + expect(statusBody()).toEqual({ status: 'passed' }) + }) + it('respects skipSessionStatus', async () => { const mod = newModule({ testContextOptions: { skipSessionName: false, skipSessionStatus: true } }) await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) From 650ca237c9940cea04474f31537b68b386fabd80 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Fri, 4 Sep 2026 14:09:25 +0530 Subject: [PATCH 07/25] fix(cli): fail the cucumber session when any scenario fails, whatever the order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects collapsed a cucumber feature's scenario results into a single verdict, so a feature whose last scenario passed reported a passed Automate session however many earlier scenarios had failed. Legacy marks it failed. automateModule.onAfterTest derived one `name` and used it for two different jobs: the session name and the testResults accumulator key. Cucumber's test view carries a fullName, so `name` stayed the Feature title — shared by every scenario — and the Map collapsed N scenarios into one last-write-wins entry. Key the accumulator on fullName where the framework supplies one. Mocha leaves fullName undefined, so its key is unchanged and its path is byte-identical. _cucumberTestResult also read the observability passed/failed collapse for session status. The two views are not the same: UNDEFINED / AMBIGUOUS / UNKNOWN fail the session on legacy while still reporting to Observability as skipped, and PENDING joins them under cucumberOpts.strict. Read _failureStatuses. Verified with an ordering probe — the same two scenarios, order the only variable. Both arms now report failed with legacy's own reason for the shared failure mode, where the failure-first arm previously reported passed. SDK-7414 Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 8 +- packages/browserstack-service/src/service.ts | 16 +++- .../tests/cli/modules/automateModule.test.ts | 87 ++++++++++++++++++- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 0b57e85..f3646e7 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -213,7 +213,13 @@ export default class AutomateModule extends BaseModule { const sessionData = this.sessionMap.get(sessionId) if (sessionData) { - sessionData.testResults.set(name, testResult) + // `name` is the session NAME, which for cucumber is the Feature title and therefore + // shared by every scenario in the file — keying the results map on it collapses N + // scenarios into one last-write-wins entry, so a feature whose last scenario passes + // reports a passed session however many earlier ones failed. Mocha leaves `fullName` + // undefined, so the key is unchanged there. + const resultKey = (test && test.fullName) ? String(test.fullName) : name + sessionData.testResults.set(resultKey, testResult) this.sessionMap.set(sessionId, sessionData) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index a727f2a..190dacb 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -935,8 +935,13 @@ export default class BrowserstackService implements Services.ServiceInstance { /** * A `Frameworks.TestResult`-shaped view of a scenario result, for automateModule's - * session-status marking. Anything cucumber reports that is neither passed nor failed is a - * skip — the same collapse the scenario's own reported result applies. + * session-status marking. + * + * Which statuses count as a session failure is `_failureStatuses`, NOT the passed/failed + * collapse the o11y result applies: cucumber's UNDEFINED / AMBIGUOUS / UNKNOWN fail the + * session on legacy while still being reported to Observability as `skipped`, and PENDING + * joins them only under `cucumberOpts.strict`. Reading the o11y collapse here instead marked + * a feature with an undefined step `passed` where legacy marks it `failed`. */ private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { const status = world.result?.status?.toLowerCase() @@ -949,9 +954,12 @@ export default class BrowserstackService implements Services.ServiceInstance { const hasStepFailures = this._cliCucumberFramework()?.hasStepFailures() ?? true const hookOnlyFailure = ignoreHooksStatus && status === 'failed' && !hasStepFailures + const passed = status === 'passed' || hookOnlyFailure + const failed = !passed && status !== undefined && this._failureStatuses.includes(status) + return { - passed: status === 'passed' || hookOnlyFailure, - skipped: status !== undefined && status !== 'passed' && status !== 'failed', + passed, + skipped: !passed && !failed, error: world.result?.message ? new Error(world.result.message) : undefined, duration: 0, retries: { attempts: 0, limit: 0 }, diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 6cc18e4..ec07fa3 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -642,4 +642,89 @@ describe('AutomateModule', () => { expect(moduleWithoutConfig).toBeInstanceOf(AutomateModule) expect(moduleWithoutConfig.getModuleName()).toBe('AutomateModule') }) -}) \ No newline at end of file +}) + +describe('AutomateModule testResults keying (SDK-7414)', () => { + let automateModule: AutomateModule + let mockTestInstance: any + + const FEATURE = 'A cucumber feature' + + // A cucumber test view carries fullName (the scenario); a mocha one never does. + const cucumberTest = (scenario: string) => ({ title: scenario, fullName: scenario, parent: FEATURE }) + const mochaTest = (title: string) => ({ title, parent: FEATURE }) + + const afterTest = (test: any, passed: boolean) => ({ + instance: mockTestInstance, + result: { error: passed ? null : new Error(`${test.title} failed`), passed }, + test, + suiteTitle: FEATURE + }) + + const sessionData = () => (automateModule as any).sessionMap.get('test-session-id') + + beforeEach(() => { + vi.clearAllMocks() + + const mockAutoInstance = { getId: vi.fn().mockReturnValue(1) } + mockTestInstance = { getId: vi.fn().mockReturnValue(1) } + + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue(mockAutoInstance) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'test-session-id' }) + vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { + if (key === 'framework_session_id') {return 'test-session-id'} + if (key.includes('CAPABILITIES')) {return { browserName: 'chrome' }} + return {} + }) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) } as any) + + automateModule = new AutomateModule({ user: 'testuser', key: 'testkey' } as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey' + } as any + }) + + // The defect: every scenario in a feature shares the session name, so keying testResults on it + // collapsed N scenarios into one last-write-wins entry and a trailing pass hid earlier failures. + it('keeps one entry per scenario for cucumber, so a trailing pass cannot mask an earlier failure', async () => { + await automateModule.onAfterTest(afterTest(cucumberTest('scenario one fails'), false)) + await automateModule.onAfterTest(afterTest(cucumberTest('scenario two passes'), true)) + + const results = sessionData().testResults + expect([...results.keys()]).toEqual(['scenario one fails', 'scenario two passes']) + expect(results.size).toBe(2) + expect([...results.values()].map((r: any) => r.status)).toEqual(['failed', 'passed']) + // Parity row 31: the session NAME stays the feature title even though the keys do not. + expect(sessionData().lastTestName).toBe(FEATURE) + }) + + // The discriminating pair: identical input shape, opposite answers through the resultKey branch. + it('keys on the scenario for cucumber and leaves the key unchanged for mocha', async () => { + await automateModule.onAfterTest(afterTest(cucumberTest('a scenario'), true)) + const cucumberKey = [...sessionData().testResults.keys()][0] + const cucumberName = sessionData().lastTestName + + vi.clearAllMocks() + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'test-session-id' }) + vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { + if (key === 'framework_session_id') {return 'test-session-id'} + if (key.includes('CAPABILITIES')) {return { browserName: 'chrome' }} + return {} + }) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) } as any) + ;(automateModule as any).sessionMap = new Map() + + await automateModule.onAfterTest(afterTest(mochaTest('a test'), true)) + const mochaKey = [...sessionData().testResults.keys()][0] + const mochaName = sessionData().lastTestName + + expect(cucumberKey).toBe('a scenario') + expect(cucumberKey).not.toBe(cucumberName) + expect(mochaKey).toBe(mochaName) + expect(mochaKey).toBe(`${FEATURE} - a test`) + }) +}) From 28aa160a0bff30c27fe15ee476c47a7225860d3c Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sun, 6 Sep 2026 17:00:00 +0530 Subject: [PATCH 08/25] fix(a11y): decode goog:chromeOptions when it arrives over gRPC The binary types AccessibilityCapability.value as a proto string, so the object-valued goog:chromeOptions capability reaches the SDK as "[object Object]" on the CLI flow where the HTTP launch response delivers a real object. AccessibilityScripts.update() stored that string verbatim, and the non-BrowserStack-infra accessibility path in the launcher then wrote it into a W3C capability, which the hub rejects outright: The property '#/alwaysMatch/goog:chromeOptions' of type String did not match the following type: object No session is created, so every assertion downstream fails for want of one. Accept both shapes and drop anything that is not an object, so a value that cannot become a capability is never written as one. Applying this in update() rather than at the response site also covers the value read back from a commands.json poisoned by an earlier run. Co-Authored-By: Claude Opus 5 --- .../src/scripts/accessibility-scripts.ts | 26 +++++++++++++-- .../tests/accessibility-scripts.test.ts | 33 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/scripts/accessibility-scripts.ts b/packages/browserstack-service/src/scripts/accessibility-scripts.ts index 1e1dff0..def7086 100644 --- a/packages/browserstack-service/src/scripts/accessibility-scripts.ts +++ b/packages/browserstack-service/src/scripts/accessibility-scripts.ts @@ -14,6 +14,27 @@ interface Command { class: string } +/** + * The binary types AccessibilityCapability.value as a proto string, so goog:chromeOptions + * arrives JSON-encoded over gRPC where the HTTP launch response delivers a plain object. + * Accept both. Anything that does not resolve to an object is dropped rather than written + * into a W3C capability, which the hub rejects outright. + */ +function toChromeOptions(value: unknown): { [key: string]: unknown } | null { + let parsed = value + if (typeof parsed === 'string') { + try { + parsed = JSON.parse(parsed) + } catch { + return null + } + } + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as { [key: string]: unknown } + } + return null +} + class AccessibilityScripts { private static instance: AccessibilityScripts | null = null @@ -88,8 +109,9 @@ class AccessibilityScripts { if (data.commands && data.commands.length) { this.commandsToWrap = data.commands } - if (data.nonBStackInfraA11yChromeOptions){ - this.ChromeExtension = data.nonBStackInfraA11yChromeOptions + const chromeOptions = toChromeOptions(data.nonBStackInfraA11yChromeOptions) + if (chromeOptions){ + this.ChromeExtension = chromeOptions } } diff --git a/packages/browserstack-service/tests/accessibility-scripts.test.ts b/packages/browserstack-service/tests/accessibility-scripts.test.ts index 609d4b0..9aed2c4 100644 --- a/packages/browserstack-service/tests/accessibility-scripts.test.ts +++ b/packages/browserstack-service/tests/accessibility-scripts.test.ts @@ -145,3 +145,36 @@ describe('getWritableDir', () => { expect(writableDir).toBe(process.cwd()) // Should return the second path }) }) + +describe('nonBStackInfraA11yChromeOptions across flows', () => { + const scripts: typeof AccessibilityScripts = AccessibilityScripts + const payload = (chromeOptions: unknown) => ({ + commands: [], + scripts: { scan: 'scan', getResults: 'getResults', getResultsSummary: 'getResultsSummary', saveResults: 'saveResults' }, + nonBStackInfraA11yChromeOptions: chromeOptions + } as unknown as Parameters[0]) + + beforeEach(() => { + scripts.ChromeExtension = {} + }) + + it('keeps the object the HTTP launch response delivers', () => { + scripts.update(payload({ args: ['--headless=new'], extensions: ['b64'] })) + expect(scripts.ChromeExtension).to.deep.equal({ args: ['--headless=new'], extensions: ['b64'] }) + }) + + it('decodes the JSON string the gRPC response delivers', () => { + scripts.update(payload('{"args":["--headless=new"],"extensions":["b64"]}')) + expect(scripts.ChromeExtension).to.deep.equal({ args: ['--headless=new'], extensions: ['b64'] }) + }) + + it('drops a value that is not an object instead of writing it into a capability', () => { + scripts.update(payload('[object Object]')) + expect(scripts.ChromeExtension).to.deep.equal({}) + }) + + it('leaves the extension untouched when the capability set has no chrome options', () => { + scripts.update(payload(undefined)) + expect(scripts.ChromeExtension).to.deep.equal({}) + }) +}) From e7f74f532071f254be3afee5648613c97a76633f Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Mon, 7 Sep 2026 02:15:08 +0530 Subject: [PATCH 09/25] fix(cli): honour preferScenarioName on the CLI flow, and keep it out of caps Parity row 40 was failing twice over, independently. The option reached bstack:options. cliUtils.getBinConfig passes the service options through verbatim, and the binary's getBstackOptions() copies every config key it does not recognise into the outgoing W3C payload, so the hub rejected the session before it existed: The property '#/alwaysMatch/bstack:options' contains additional properties ["preferScenarioName"] outside of the schema when none are allowed Excluded on the SDK side rather than in the binary's EXCLUDED_CAPS. preferScenarioName is a wdio-service option, not a BrowserStack capability, so NOT_ALLOWED_KEYS_IN_CAPS is both the narrower blast radius (WDIO frameworks, not every language SDK) and the more correct home - includeTagsInTestingScope is already there for exactly this reason. turboScaleOptions belonged in EXCLUDED_CAPS because that key genuinely is a capability. This half is not cucumber-specific: the same leak broke preferScenarioName on wdio_mocha, and fixing it here repairs that too. And the gate had no implementation on this flow. service.after() sets _fullTitle, but every _updateJob call site that consumes it is gated !BrowserstackCLI.isRunning(), so the name never moved; automateModule owns the name here and was still applying the feature title. after() now pushes the rename to a new automateModule.overrideSessionName(), which writes sessionMap and re-flushes - flushSessionName's appliedName de-dupe keeps a no-op override free, and skipSessionName still wins, matching legacy omitting `name` from its _updateJob payload under setSessionName: false. Legacy's `=== 1` exactness is reproduced, not widened: the new branch sits inside the existing guard. wdio_mocha cannot reach it - _scenariosRanCount and _lastScenarioName are written only by cucumber's afterScenario - which the discriminating test pins on identical input. Separately, _cucumberTestResult() now mirrors legacy afterScenario()'s failure message. The statuses that only fail a session via _failureStatuses carry no world.result.message - PENDING under cucumberOpts.strict, equally UNDEFINED and AMBIGUOUS - so automateModule fell back to 'Unknown Error' where legacy reports `Some steps/hooks are pending for scenario "..."`. A failure that carries a real message is unaffected. Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 44 ++++++ .../browserstack-service/src/constants.ts | 2 +- packages/browserstack-service/src/service.ts | 31 +++- .../automateModule.preferScenarioName.test.ts | 134 ++++++++++++++++++ .../service.preferScenarioName.cli.test.ts | 134 ++++++++++++++++++ 5 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts create mode 100644 packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index f3646e7..628eee8 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -138,6 +138,50 @@ export default class AutomateModule extends BaseModule { this.sessionMap.set(sessionId, sessionData) } + /** + * Apply a session-name override decided at worker teardown rather than per test — the shape + * `preferScenarioName` needs, since "exactly one scenario ran" is only known once the worker + * is done. Legacy expresses it as `_updateJob({ name: this._fullTitle })` in service.after(), + * which is gated `!BrowserstackCLI.isRunning()`; here the name lives in `sessionMap`, so the + * override is written there and re-flushed. `flushSessionName`'s `appliedName` de-dupe means + * a no-op override costs no API call. + * + * `setSessionName: false` still wins: legacy omits `name` from its `_updateJob` payload in + * that case, and the same flag short-circuits here. + */ + async overrideSessionName(name: string): Promise { + try { + if (!name) { + return + } + + const testContextOptions = this.config.testContextOptions as TestContextOptions + if (testContextOptions?.skipSessionName) { + return + } + + const autoInstance = AutomationFramework.getTrackedInstance() + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + if (!sessionId) { + this.logger.debug('overrideSessionName: no session id resolved; nothing to rename') + return + } + + const sessionData = this.sessionMap.get(sessionId) + if (!sessionData) { + this.logger.debug(`overrideSessionName: session ${sessionId} is not registered; nothing to rename`) + return + } + + sessionData.lastTestName = name + this.sessionMap.set(sessionId, sessionData) + await this.flushSessionName(sessionId) + this.logger.info(`overrideSessionName: renamed session ${sessionId} to "${name}"`) + } catch (error) { + this.logger.error(`Exception in automate overrideSessionName: ${error}`) + } + } + async onAfterTest(args: Record) { this.logger.debug('onAfterTest: inside automate module after test hook!') const instace = args.instance as TestFrameworkInstance diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index c64bfa8..ae00025 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -42,7 +42,7 @@ export const DEFAULT_WAIT_TIMEOUT_FOR_PENDING_UPLOADS = 5000 // 5s export const DEFAULT_WAIT_INTERVAL_FOR_PENDING_UPLOADS = 100 // 100ms export const BSTACK_SERVICE_VERSION = bstackServiceVersion -export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride'] +export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName'] export const BROWSERSTACK_TEST_PLAN_ID = 'BROWSERSTACK_TEST_PLAN_ID' export const LOGS_FILE = 'logs/bstack-wdio-service.log' diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 190dacb..8a22128 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -23,6 +23,7 @@ import AccessibilityHandler from './accessibility-handler.js' import CustomTagsHandler from './custom-tags-handler.js' import { classifyMochaHookTitle, setCurrentMochaHookWindow } from './customTags.js' import type TestHubModule from './cli/modules/testHubModule.js' +import type AutomateModule from './cli/modules/automateModule.js' import { BStackLogger } from './bstackLogger.js' import PercyHandler from './Percy/Percy-Handler.js' import Listener from './testOps/listener.js' @@ -725,6 +726,21 @@ export default class BrowserstackService implements Services.ServiceInstance { // use the scenario name instead of the feature name if (preferScenarioName && this._scenariosRanCount === 1 && this._lastScenarioName) { this._fullTitle = this._lastScenarioName + // `_fullTitle` only reaches the session through `_updateJob`, and every one of its + // call sites is gated `!BrowserstackCLI.isRunning()`, so on the CLI flow the rename + // has to be pushed to automateModule, which owns the name there. + // + // Unreachable for mocha and jasmine: `_scenariosRanCount` and `_lastScenarioName` + // are written only by cucumber's `afterScenario`, so the guard above is false for + // any other framework however `preferScenarioName` is set. + if (BrowserstackCLI.getInstance().isRunning()) { + try { + const automateModule = BrowserstackCLI.getInstance().modules.AutomateModule as AutomateModule | undefined + await automateModule?.overrideSessionName(this._lastScenarioName) + } catch (renameErr) { + BStackLogger.debug(`Exception applying preferScenarioName in after(): ${util.format(renameErr)}`) + } + } } if (BrowserstackCLI.getInstance().isRunning()) { @@ -957,10 +973,23 @@ export default class BrowserstackService implements Services.ServiceInstance { const passed = status === 'passed' || hookOnlyFailure const failed = !passed && status !== undefined && this._failureStatuses.includes(status) + // The statuses that only fail the session via `_failureStatuses` carry no `world.result + // .message` — PENDING under `cucumberOpts.strict`, and equally UNDEFINED / AMBIGUOUS — so + // automateModule falls back to its generic 'Unknown Error'. Legacy synthesises the reason + // in the same afterScenario() block that consults `_failureStatuses`; mirror it verbatim. + let error: Error | undefined + if (failed) { + error = new Error(world.result?.message || (status === 'pending' + ? `Some steps/hooks are pending for scenario "${world.pickle.name}"` + : 'Unknown Error')) + } else if (world.result?.message) { + error = new Error(world.result.message) + } + return { passed, skipped: !passed && !failed, - error: world.result?.message ? new Error(world.result.message) : undefined, + error, duration: 0, retries: { attempts: 0, limit: 0 }, } as unknown as Frameworks.TestResult diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts new file mode 100644 index 0000000..a223d03 --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' +import AutomateModule from '../../../src/cli/modules/automateModule.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import { _fetch as fetch } from '../../../src/fetchWrapper.js' +import type { Options } from '@wdio/types' + +vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ + default: { + registerObserver: vi.fn(), + setState: vi.fn(), + getState: vi.fn(), + getTrackedInstance: vi.fn() + } +})) + +vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ + default: { + getTrackedInstance: vi.fn(), + getState: vi.fn(), + getDriver: vi.fn() + } +})) + +vi.mock('../../../src/cli/cliLogger.js', () => ({ + BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } +})) + +vi.mock('../../../src/util.js', () => ({ + isBrowserstackSession: vi.fn(() => true), + isTrue: vi.fn((value) => (value + '').toLowerCase() === 'true'), + hasAppCap: vi.fn(() => false) +})) + +vi.mock('../../../src/instrumentation/performance/performance-tester.js', () => ({ + default: { measureWrapper: vi.fn((event, fn) => fn) } +})) + +vi.mock('../../../src/fetchWrapper.js', () => ({ _fetch: vi.fn() })) + +function newModule(config: Record = {}) { + const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) + mod.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey', + ...config + } as never + return mod +} + +function register(mod: AutomateModule, sessionId: string, lastTestName: string, appliedName?: string) { + const sessionMap = mod['sessionMap'] as Map }> + sessionMap.set(sessionId, { lastTestName, appliedName, testResults: new Map() }) + return sessionMap +} + +function namesPUT() { + return vi.mocked(fetch).mock.calls.map(([, init]) => { + try { + return JSON.parse((init as { body: string }).body).name + } catch { + return undefined + } + }) +} + +describe('AutomateModule.overrideSessionName — parity row 40 (preferScenarioName)', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + }) + + it('renames a registered session to the scenario name', async () => { + const mod = newModule() + register(mod, 'sess-1', 'Login Feature', 'Login Feature') + + await mod.overrideSessionName('Can do something single') + + expect(namesPUT()).toContain('Can do something single') + expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Can do something single') + }) + + // The rename is the same opt-out legacy applies: service.after() omits `name` from its + // _updateJob payload when setSessionName is false, so the override must not sneak one in. + it('honours setSessionName: false and issues no rename', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) + register(mod, 'sess-1', 'Login Feature', 'Login Feature') + + await mod.overrideSessionName('Can do something single') + + expect(fetch).not.toHaveBeenCalled() + expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Login Feature') + }) + + it('no-ops when the session was never registered', async () => { + const mod = newModule() + + await mod.overrideSessionName('Can do something single') + + expect(fetch).not.toHaveBeenCalled() + }) + + it('no-ops when no session id resolves', async () => { + const mod = newModule() + register(mod, 'sess-1', 'Login Feature', 'Login Feature') + vi.mocked(AutomationFramework.getState).mockReturnValue(undefined as never) + + await mod.overrideSessionName('Can do something single') + + expect(fetch).not.toHaveBeenCalled() + }) + + it('no-ops on an empty name', async () => { + const mod = newModule() + register(mod, 'sess-1', 'Login Feature', 'Login Feature') + + await mod.overrideSessionName('') + + expect(fetch).not.toHaveBeenCalled() + expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Login Feature') + }) + + it('costs no API call when the override matches the name already applied', async () => { + const mod = newModule() + register(mod, 'sess-1', 'Can do something single', 'Can do something single') + + await mod.overrideSessionName('Can do something single') + + expect(fetch).not.toHaveBeenCalled() + }) +}) diff --git a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts new file mode 100644 index 0000000..f564390 --- /dev/null +++ b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' +import BrowserstackService from '../src/service.js' +import { BrowserstackCLI } from '../src/cli/index.js' + +vi.mock('../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: () => ({ + isRunning: () => false, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + }) + } +})) + +describe('preferScenarioName on the CLI flow — parity row 40', () => { + let getInstanceSpy: ReturnType | undefined + let overrideSessionName: ReturnType + + const makeService = (framework: string, options: Record = {}) => new BrowserstackService( + { testObservability: false, preferScenarioName: true, setSessionName: true, setSessionStatus: true, ...options } as never, + [] as never, + { user: 'foo', key: 'bar', framework, cucumberOpts: { strict: false } } as never + ) + + beforeEach(() => { + overrideSessionName = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }), + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }), + modules: { AutomateModule: { overrideSessionName } } + } as never) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('renames the session to the scenario name when exactly one scenario ran', async () => { + const service = makeService('cucumber') + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + await service.after(0) + + expect(overrideSessionName).toHaveBeenCalledTimes(1) + expect(overrideSessionName).toHaveBeenCalledWith('Can do something single') + }) + + // Legacy's `=== 1` is the specification, not a lower bound. + it('does NOT rename when two scenarios ran', async () => { + const service = makeService('cucumber') + await service.afterScenario({ pickle: { name: 'Scenario one' }, result: { status: 'passed' } } as never) + await service.afterScenario({ pickle: { name: 'Scenario two' }, result: { status: 'passed' } } as never) + + await service.after(0) + + expect(overrideSessionName).not.toHaveBeenCalled() + }) + + it('does NOT rename when preferScenarioName is absent', async () => { + const service = makeService('cucumber', { preferScenarioName: undefined }) + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + await service.after(0) + + expect(overrideSessionName).not.toHaveBeenCalled() + }) + + it('does NOT rename when the only scenario was skipped', async () => { + const service = makeService('cucumber') + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'skipped' } } as never) + + await service.after(0) + + expect(overrideSessionName).not.toHaveBeenCalled() + }) + + // The discriminating pair: identical options and an identical "exactly one unit ran", opposite + // answers. `_scenariosRanCount` / `_lastScenarioName` are written only by cucumber's + // afterScenario, so wdio_mocha cannot reach the rename however preferScenarioName is set. + it('leaves wdio_mocha untouched on the same input', async () => { + const service = makeService('mocha') + await service.afterTest( + { title: 'a test', parent: 'a suite' } as never, + undefined as never, + { passed: true, duration: 1, retries: { attempts: 0, limit: 0 }, exception: '', status: 'passed' } as never + ) + + await service.after(0) + + expect(overrideSessionName).not.toHaveBeenCalled() + expect(service['_scenariosRanCount']).toBe(0) + }) +}) + +describe('_cucumberTestResult failure reason — parity row 39 adjacent', () => { + const makeService = (strict: boolean) => new BrowserstackService( + { testObservability: false } as never, + [] as never, + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict } } as never + ) + + const world = (status: string, message?: string) => ({ + pickle: { name: 'CfgGate pending scenario' }, + result: message ? { status, message } : { status } + }) + + it('synthesises legacy\'s pending reason when strict makes a pending scenario fail', () => { + const result = makeService(true)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.error?.message).toBe('Some steps/hooks are pending for scenario "CfgGate pending scenario"') + }) + + it('leaves a pending scenario unfailed — and unreasoned — when strict is off', () => { + const result = makeService(false)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.skipped).toBe(true) + expect(result.error).toBeUndefined() + }) + + it('keeps the real message when the result carries one', () => { + const result = makeService(false)['_cucumberTestResult'](world('FAILED', 'AssertionError: nope') as never) + + expect(result.error?.message).toBe('AssertionError: nope') + }) + + it('falls back to Unknown Error for a message-less non-pending failure', () => { + const result = makeService(false)['_cucumberTestResult'](world('UNDEFINED') as never) + + expect(result.error?.message).toBe('Unknown Error') + }) +}) From f1444f6e970dfc88907326feadcc95243e16b80b Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 8 Sep 2026 18:36:44 +0530 Subject: [PATCH 10/25] refactor(a11y): simplify the mocha-only guard on the hook scan gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Readability only; no behaviour change beyond the session-id guard noted below. `reopensGateForHook` named the effect rather than the condition — renamed to `isMocha`, matching the file's neighbours (`isBrowserstackSession`, `isPreTestWindow`). `sessionId !== undefined && sessionId !== null` collapsed to a truthiness check. Line 287 of this file already guards the same value that way, so the long form was the outlier. The substring match stays. `KEY_TEST_FRAMEWORK_NAME` holds the vendor-qualified name — `WebdriverIO-mocha`, observed in the wire payload — so `=== 'mocha'` would never match and the mocha hook window would silently stop scanning. `testHubModule` gates on the same value the same way. Comment cut from eleven lines to five, keeping only what the code cannot say: that this mirrors legacy's `_framework === 'mocha'` gate, and why it has to be mocha-only. Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/accessibilityModule.ts | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 66edfd3..1a482f3 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -86,21 +86,14 @@ export default class AccessibilityModule extends BaseModule { if (!this.accessibility) { return } - // Open the scan gate for the hook window so DOM-changing commands issued inside - // before/beforeEach/afterEach/after hooks trigger scans (web per-command path). The - // following onBeforeTest re-computes the per-test gate, so this only affects the hook. - // - // Mocha-only, exactly as legacy gates the identical write (accessibility-handler - // beforeHook, `this._framework === 'mocha'`). The "onBeforeTest re-computes it after" - // invariant above holds only where beforeEach precedes beforeTest. Cucumber inverts - // that — WDIO raises the scenario boundary BEFORE cucumber's own Before hooks — so - // there the write lands last and permanently forces the gate open, scanning every - // scenario regardless of includeTagsInTestingScope / excludeTagsInTestingScope and - // undoing a user's stopA11yScanning(). The currentHookRunUuid capture above is - // correct for every framework and stays outside this gate. + // Open the scan gate for the hook window, so commands issued inside hooks are scanned. + // Mocha-only, as legacy gates the identical write (`this._framework === 'mocha'`): it + // relies on the following onBeforeTest re-computing the gate, which holds only where + // beforeEach precedes beforeTest. Cucumber inverts that ordering, so there the write + // would land last and force the gate permanently open. const frameworkName = String(TestFramework.getState(testInstance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') - const reopensGateForHook = frameworkName.toLowerCase().includes('mocha') - if (this.autoScanning && reopensGateForHook && sessionId !== undefined && sessionId !== null) { + const isMocha = frameworkName.toLowerCase().includes('mocha') + if (this.autoScanning && isMocha && sessionId) { this.accessibilityMap.set(sessionId, true) } } catch (error) { From 757339339ae6526f950d60850d1fa2c02760ed8c Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 8 Sep 2026 21:36:11 +0530 Subject: [PATCH 11/25] docs(cli): trim the cucumber hook comments in service.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments only; no code change. The BEFORE_ALL cascade doc dropped its parity-row and ticket citations and the "one call site" note, keeping what the code cannot say: what cucumber does to a feature when BeforeAll throws, and why the cascade goes straight to TestHub instead of through trackEvent. Two comments overstated a constraint. Both said automateModule "cannot read" an option, which is only true of `this.config` — the binary-supplied one. A module can reach service options via `BrowserstackCLI.getInstance().options`, as accessibilityModule already does. They now give the actual reason for the placement instead of implying an access restriction that does not exist. Co-Authored-By: Claude Opus 5 --- packages/browserstack-service/src/service.ts | 51 ++++++++------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 8a22128..409d4e9 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -489,15 +489,11 @@ export default class BrowserstackService implements Services.ServiceInstance { // degrades quietly instead of throwing inside this awaited hook. const framework = BrowserstackCLI.getInstance().getTestFramework() if (framework instanceof WdioCucumberTestFramework) { - // See beforeHook: cucumber's taxonomy, not Mocha's titles. The suite-skip cascade - // in the mocha arm below is Mocha-shaped (it walks `test.ctx.test.parent`); - // cucumber's own cascade is _reportCucumberScenariosSkipped, further down. + // Cucumber's taxonomy, not Mocha's titles — see beforeHook. const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) if (hookFrameworkState) { - // ignoreHooksStatus rides the event so automateModule can apply the same flag - // to the session verdict that loadScenarioResult applies to the o11y result - // (parity row 41). The module cannot read it — the binary-supplied config it - // holds carries no testObservabilityOptions. + // ignoreHooksStatus rides the event so the cucumber-only policy stays out of + // automateModule, which mocha and jasmine share. await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result, @@ -531,21 +527,15 @@ export default class BrowserstackService implements Services.ServiceInstance { } /** - * BEFORE_ALL failure cascade — parity row 15. Cucumber abandons the whole feature when a - * `BeforeAll` throws, so every scenario in it (Rule-nested ones included) must be reported - * SKIPPED rather than simply vanishing. + * Cucumber abandons the whole feature when a `BeforeAll` throws, so every scenario in it + * (Rule-nested included) is reported SKIPPED rather than vanishing. Ports + * `insights-handler.processCucumberHook`, whose cascade publishes over the legacy HTTP + * listener and is inert once the binary is up. * - * Legacy did this from `insights-handler.afterHook` via `sendScenarioObjectSkipped()`, which - * publishes through the legacy HTTP listener (`api/v1/batch`) — inert once the binary is up. - * That is escape class 3 / SDK-7047 in its documented form, and the repair is to give the - * cascade a CLI/gRPC publisher. - * - * Sent straight to TestHub rather than through `framework.trackEvent()`: legacy's cascade - * called `listener.testFinished()` directly and so bypassed every product handler. Routing - * these through the observer set would rename the Automate session, fire an accessibility - * stop event and run a Percy teardown once per skipped row — none of which legacy does. - * - * Cucumber-only: private, one call site, in the `instanceof WdioCucumberTestFramework` arm. + * Sent straight to TestHub rather than via `framework.trackEvent()`: legacy called + * `listener.testFinished()` directly, so routing these through the observers would rename the + * session, stop accessibility and run a Percy teardown per skipped row — none of which legacy + * does. */ private async _reportCucumberScenariosSkipped(framework: WdioCucumberTestFramework) { try { @@ -557,10 +547,9 @@ export default class BrowserstackService implements Services.ServiceInstance { const instances = framework.buildSkippedScenarioInstances() for (const instance of instances) { - // Both halves, because TestHub's v2 batch pipeline creates the test row from the - // START event and treats TestRunSkipped as its terminal — a lone TestRunSkipped is - // accepted and then counted in no bucket at all. The legacy v1 listener created the - // row from the skip event itself, which is why it sent only one. + // Both halves: TestHub's v2 pipeline creates the row from the START event, so a + // lone TestRunSkipped is accepted and counted in no bucket. Legacy's v1 listener + // created the row from the skip itself, which is why it sent only one. await testHubModule.sendTestFrameworkEvent( { instance }, { testFrameworkState: 'TEST', testHookState: 'PRE' } @@ -726,13 +715,11 @@ export default class BrowserstackService implements Services.ServiceInstance { // use the scenario name instead of the feature name if (preferScenarioName && this._scenariosRanCount === 1 && this._lastScenarioName) { this._fullTitle = this._lastScenarioName - // `_fullTitle` only reaches the session through `_updateJob`, and every one of its - // call sites is gated `!BrowserstackCLI.isRunning()`, so on the CLI flow the rename - // has to be pushed to automateModule, which owns the name there. - // - // Unreachable for mocha and jasmine: `_scenariosRanCount` and `_lastScenarioName` - // are written only by cucumber's `afterScenario`, so the guard above is false for - // any other framework however `preferScenarioName` is set. + // `_fullTitle` reaches the session only through `_updateJob`, whose call sites are + // all gated `!isRunning()`, so on the CLI flow the rename is pushed to + // automateModule instead. The decision stays here because `_scenariosRanCount` + // does — it counts non-skipped scenarios and is written only by cucumber's + // `afterScenario`. if (BrowserstackCLI.getInstance().isRunning()) { try { const automateModule = BrowserstackCLI.getInstance().modules.AutomateModule as AutomateModule | undefined From ed012cf35a918e5da09ecf17da64d156fef5c218 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 8 Sep 2026 21:53:25 +0530 Subject: [PATCH 12/25] refactor(cli): let automateModule own the preferScenarioName rename MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rename lived in service.after(), which reached into the module through a public overrideSessionName(). It does not need to: the module already sees every scenario event with its status, so it can keep the tally itself and apply the name on the path that already flushes it. onAfterTest counts non-skipped scenarios behind the existing isCucumberInstance gate; onAfterExecute applies the scenario name just before the final flushSessionName sweep, so skipSessionName keeps working through that guard. service.ts loses the module import, the try/catch and the cross-component ordering dependency on after() running before the session closes. The flag rides the scenario event rather than being read from the module. A module CAN reach service options via BrowserstackCLI.getInstance().options — accessibilityModule does — but importing the CLI singleton here is a cycle (automateModule -> index -> testHubModule -> wdioMochaTestFramework), and it breaks class construction at load. Riding the event is the route ignoreHooksStatus already takes. Legacy's `=== 1` exactness is reproduced, not widened; both tests were falsified against a deliberately broken guard. The five service-level cases move to the module suite, three new ones cover the event seam, and the four _cucumberTestResult reason cases that shared the old file are kept. Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 68 +++++--------- packages/browserstack-service/src/service.ts | 21 +---- .../automateModule.preferScenarioName.test.ts | 93 ++++++++++++------- .../service.preferScenarioName.cli.test.ts | 86 +++++++---------- 4 files changed, 117 insertions(+), 151 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 628eee8..b5c906d 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -27,6 +27,9 @@ interface SessionData { lastTestName: string appliedName?: string // last name successfully PUT for this session, for de-duping testResults: Map // testName -> TestResult + scenariosRan: number // non-skipped cucumber scenarios, for preferScenarioName + lastScenarioName?: string + preferScenarioName?: boolean } export default class AutomateModule extends BaseModule { @@ -92,7 +95,8 @@ export default class AutomateModule extends BaseModule { if (!existingSession) { this.sessionMap.set(sessionId, { lastTestName: name, - testResults: new Map() + testResults: new Map(), + scenariosRan: 0 }) } else { existingSession.lastTestName = name @@ -138,50 +142,6 @@ export default class AutomateModule extends BaseModule { this.sessionMap.set(sessionId, sessionData) } - /** - * Apply a session-name override decided at worker teardown rather than per test — the shape - * `preferScenarioName` needs, since "exactly one scenario ran" is only known once the worker - * is done. Legacy expresses it as `_updateJob({ name: this._fullTitle })` in service.after(), - * which is gated `!BrowserstackCLI.isRunning()`; here the name lives in `sessionMap`, so the - * override is written there and re-flushed. `flushSessionName`'s `appliedName` de-dupe means - * a no-op override costs no API call. - * - * `setSessionName: false` still wins: legacy omits `name` from its `_updateJob` payload in - * that case, and the same flag short-circuits here. - */ - async overrideSessionName(name: string): Promise { - try { - if (!name) { - return - } - - const testContextOptions = this.config.testContextOptions as TestContextOptions - if (testContextOptions?.skipSessionName) { - return - } - - const autoInstance = AutomationFramework.getTrackedInstance() - const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) - if (!sessionId) { - this.logger.debug('overrideSessionName: no session id resolved; nothing to rename') - return - } - - const sessionData = this.sessionMap.get(sessionId) - if (!sessionData) { - this.logger.debug(`overrideSessionName: session ${sessionId} is not registered; nothing to rename`) - return - } - - sessionData.lastTestName = name - this.sessionMap.set(sessionId, sessionData) - await this.flushSessionName(sessionId) - this.logger.info(`overrideSessionName: renamed session ${sessionId} to "${name}"`) - } catch (error) { - this.logger.error(`Exception in automate overrideSessionName: ${error}`) - } - } - async onAfterTest(args: Record) { this.logger.debug('onAfterTest: inside automate module after test hook!') const instace = args.instance as TestFrameworkInstance @@ -238,7 +198,7 @@ export default class AutomateModule extends BaseModule { // and a `setSessionName: false` user must not be pulled into sessionMap — that would hand // onAfterExecute a session to status-mark where it previously had none. if (sessionId && !testContextOptions.skipSessionName && !this.sessionMap.has(sessionId)) { - this.sessionMap.set(sessionId, { lastTestName: name, testResults: new Map() }) + this.sessionMap.set(sessionId, { lastTestName: name, testResults: new Map(), scenariosRan: 0 }) } // No-op for the steady state: when no mid-test reload happened, onBeforeTest already // applied this exact name and `appliedName` de-dupes it away — no extra API call. @@ -264,6 +224,11 @@ export default class AutomateModule extends BaseModule { // undefined, so the key is unchanged there. const resultKey = (test && test.fullName) ? String(test.fullName) : name sessionData.testResults.set(resultKey, testResult) + if (!skipped && this.isCucumberInstance(instace)) { + sessionData.scenariosRan++ + sessionData.lastScenarioName = testTitle + sessionData.preferScenarioName = isTrue(args.preferScenarioName) + } this.sessionMap.set(sessionId, sessionData) } @@ -326,7 +291,7 @@ export default class AutomateModule extends BaseModule { // A BeforeAll can fail before any scenario ran, so the session may not be // registered yet. `lastTestName` stays empty on purpose — flushSessionName // early-returns on it, so registering here cannot rename the session. - this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map() }) + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) } const name = this.resolveHookName(instance, hookKey) @@ -383,6 +348,15 @@ export default class AutomateModule extends BaseModule { } } + // preferScenarioName: cucumber names the session after the FEATURE, but when + // exactly one non-skipped scenario ran the user can ask for that scenario's name + // instead. Only decidable here — "exactly one" is not knowable while tests are + // still arriving. `skipSessionName` still wins, inside flushSessionName. + if (sessionData.preferScenarioName && sessionData.scenariosRan === 1 && sessionData.lastScenarioName) { + sessionData.lastTestName = sessionData.lastScenarioName + this.sessionMap.set(sessionId, sessionData) + } + // Final sweep — a no-op for sessions already named per-test in onBeforeTest. await this.flushSessionName(sessionId) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 409d4e9..261f910 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -23,7 +23,6 @@ import AccessibilityHandler from './accessibility-handler.js' import CustomTagsHandler from './custom-tags-handler.js' import { classifyMochaHookTitle, setCurrentMochaHookWindow } from './customTags.js' import type TestHubModule from './cli/modules/testHubModule.js' -import type AutomateModule from './cli/modules/automateModule.js' import { BStackLogger } from './bstackLogger.js' import PercyHandler from './Percy/Percy-Handler.js' import Listener from './testOps/listener.js' @@ -711,23 +710,12 @@ export default class BrowserstackService implements Services.ServiceInstance { PerformanceTester.start(PERFORMANCE_SDK_EVENTS.DRIVER_EVENT.QUIT) const { preferScenarioName, setSessionName, setSessionStatus } = this._options - // For Cucumber: If only 1 Scenario ran and preferScenarioName is enabled, - // use the scenario name instead of the feature name + // One scenario and preferScenarioName set: name the session after the scenario + // rather than the feature. Legacy-flow only — `_fullTitle` reaches the session through + // `_updateJob`, whose call sites are all gated `!isRunning()`; on the CLI flow + // automateModule keeps its own tally and applies this in onAfterExecute. if (preferScenarioName && this._scenariosRanCount === 1 && this._lastScenarioName) { this._fullTitle = this._lastScenarioName - // `_fullTitle` reaches the session only through `_updateJob`, whose call sites are - // all gated `!isRunning()`, so on the CLI flow the rename is pushed to - // automateModule instead. The decision stays here because `_scenariosRanCount` - // does — it counts non-skipped scenarios and is written only by cucumber's - // `afterScenario`. - if (BrowserstackCLI.getInstance().isRunning()) { - try { - const automateModule = BrowserstackCLI.getInstance().modules.AutomateModule as AutomateModule | undefined - await automateModule?.overrideSessionName(this._lastScenarioName) - } catch (renameErr) { - BStackLogger.debug(`Exception applying preferScenarioName in after(): ${util.format(renameErr)}`) - } - } } if (BrowserstackCLI.getInstance().isRunning()) { @@ -1056,6 +1044,7 @@ export default class BrowserstackService implements Services.ServiceInstance { suiteTitle: this._suiteTitle, result: this._cucumberTestResult(world), ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + preferScenarioName: this._options.preferScenarioName === true, }) return } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts index a223d03..ab0c83b 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts @@ -48,9 +48,14 @@ function newModule(config: Record = {}) { return mod } -function register(mod: AutomateModule, sessionId: string, lastTestName: string, appliedName?: string) { - const sessionMap = mod['sessionMap'] as Map }> - sessionMap.set(sessionId, { lastTestName, appliedName, testResults: new Map() }) +function register(mod: AutomateModule, sessionId: string, lastTestName: string, seed: Record = {}) { + const sessionMap = mod['sessionMap'] as Map> + sessionMap.set(sessionId, { + lastTestName, + testResults: new Map(), + scenariosRan: 0, + ...seed + }) return sessionMap } @@ -64,7 +69,7 @@ function namesPUT() { }) } -describe('AutomateModule.overrideSessionName — parity row 40 (preferScenarioName)', () => { +describe('AutomateModule preferScenarioName — parity row 40', () => { beforeEach(() => { vi.clearAllMocks() vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) @@ -73,62 +78,80 @@ describe('AutomateModule.overrideSessionName — parity row 40 (preferScenarioNa vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) }) - it('renames a registered session to the scenario name', async () => { + // Exactly one non-skipped scenario ran and the flag is set: the session takes the scenario + // name. This is the only branch where legacy departs from the feature name. + it('renames to the scenario name when exactly one scenario ran', async () => { const mod = newModule() - register(mod, 'sess-1', 'Login Feature', 'Login Feature') + register(mod, 'sess-1', 'Login Feature', { + scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true + }) - await mod.overrideSessionName('Can do something single') + await mod.onAfterExecute() - expect(namesPUT()).toContain('Can do something single') - expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Can do something single') + expect(namesPUT()).toContain('Can log in') }) - // The rename is the same opt-out legacy applies: service.after() omits `name` from its - // _updateJob payload when setSessionName is false, so the override must not sneak one in. - it('honours setSessionName: false and issues no rename', async () => { - const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) - register(mod, 'sess-1', 'Login Feature', 'Login Feature') + // The `=== 1` exactness legacy applies: two scenarios keep the feature name. Reproduced, not + // widened — a `>= 1` here would rename every multi-scenario feature. + it('keeps the feature name when two scenarios ran', async () => { + const mod = newModule() + register(mod, 'sess-1', 'Login Feature', { + scenariosRan: 2, lastScenarioName: 'Second scenario', preferScenarioName: true + }) - await mod.overrideSessionName('Can do something single') + await mod.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() - expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Login Feature') + expect(namesPUT()).not.toContain('Second scenario') + expect(namesPUT()).toContain('Login Feature') }) - it('no-ops when the session was never registered', async () => { + it('keeps the feature name when the flag is absent', async () => { const mod = newModule() + register(mod, 'sess-1', 'Login Feature', { + scenariosRan: 1, lastScenarioName: 'Can log in' + }) - await mod.overrideSessionName('Can do something single') + await mod.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() + expect(namesPUT()).not.toContain('Can log in') }) - it('no-ops when no session id resolves', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Login Feature', 'Login Feature') - vi.mocked(AutomationFramework.getState).mockReturnValue(undefined as never) + // Legacy omits `name` from its _updateJob payload when setSessionName is false, so the + // rename must not sneak one in. + it('honours setSessionName: false and issues no rename', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) + register(mod, 'sess-1', 'Login Feature', { + scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true + }) - await mod.overrideSessionName('Can do something single') + await mod.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() + expect(namesPUT()).not.toContain('Can log in') }) - it('no-ops on an empty name', async () => { + // A skipped scenario is not a scenario that ran — legacy's counter is gated the same way, + // so a feature whose only non-skipped scenario is absent must not be renamed. + it('does not count a skipped scenario', async () => { const mod = newModule() - register(mod, 'sess-1', 'Login Feature', 'Login Feature') + register(mod, 'sess-1', 'Login Feature', { preferScenarioName: true }) + const sessionData = mod['sessionMap'].get('sess-1')! + + expect(sessionData.scenariosRan).toBe(0) - await mod.overrideSessionName('') + await mod.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() - expect(mod['sessionMap'].get('sess-1')!.lastTestName).toBe('Login Feature') + expect(namesPUT()).not.toContain('Can log in') }) - it('costs no API call when the override matches the name already applied', async () => { + // mocha never reaches the counter (it is gated on isCucumberInstance), so its session name + // is whatever onBeforeTest applied — the discriminating case against cucumber above. + it('leaves a session with no cucumber scenarios untouched', async () => { const mod = newModule() - register(mod, 'sess-1', 'Can do something single', 'Can do something single') + register(mod, 'sess-1', 'Testing with BStackDemo - add product to cart', { + scenariosRan: 0, preferScenarioName: true + }) - await mod.overrideSessionName('Can do something single') + await mod.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() }) }) diff --git a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts index f564390..d8a0972 100644 --- a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts +++ b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' import BrowserstackService from '../src/service.js' import { BrowserstackCLI } from '../src/cli/index.js' +import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFramework.js' vi.mock('../src/cli/index.js', () => ({ BrowserstackCLI: { @@ -12,23 +13,33 @@ vi.mock('../src/cli/index.js', () => ({ } })) -describe('preferScenarioName on the CLI flow — parity row 40', () => { +/** + * The seam between the two halves of parity row 40: automateModule decides the rename (it is the + * only place that knows the final scenario count), but it cannot read service options, so the flag + * rides the scenario event — the same route `ignoreHooksStatus` takes. The decision itself is + * covered in tests/cli/modules/automateModule.preferScenarioName.test.ts. + */ +describe('preferScenarioName reaches the module — parity row 40', () => { let getInstanceSpy: ReturnType | undefined - let overrideSessionName: ReturnType + let trackEvent: ReturnType - const makeService = (framework: string, options: Record = {}) => new BrowserstackService( - { testObservability: false, preferScenarioName: true, setSessionName: true, setSessionStatus: true, ...options } as never, + const makeService = (options: Record = {}) => new BrowserstackService( + { testObservability: false, setSessionName: true, setSessionStatus: true, ...options } as never, [] as never, - { user: 'foo', key: 'bar', framework, cucumberOpts: { strict: false } } as never + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict: false } } as never ) + const scenarioEventArgs = () => trackEvent.mock.calls.at(-1)?.[2] as Record + beforeEach(() => { - overrideSessionName = vi.fn().mockResolvedValue(undefined) + trackEvent = vi.fn().mockResolvedValue(undefined) + const cucumberFramework = Object.create(WdioCucumberTestFramework.prototype) + cucumberFramework.trackEvent = trackEvent + cucumberFramework.hasStepFailures = () => false getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ isRunning: () => true, - getTestFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }), - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }), - modules: { AutomateModule: { overrideSessionName } } + getTestFramework: () => cucumberFramework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) } as never) }) @@ -36,59 +47,28 @@ describe('preferScenarioName on the CLI flow — parity row 40', () => { getInstanceSpy?.mockRestore() }) - it('renames the session to the scenario name when exactly one scenario ran', async () => { - const service = makeService('cucumber') + it('carries preferScenarioName: true on the scenario event when set', async () => { + const service = makeService({ preferScenarioName: true }) await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) - await service.after(0) - - expect(overrideSessionName).toHaveBeenCalledTimes(1) - expect(overrideSessionName).toHaveBeenCalledWith('Can do something single') - }) - - // Legacy's `=== 1` is the specification, not a lower bound. - it('does NOT rename when two scenarios ran', async () => { - const service = makeService('cucumber') - await service.afterScenario({ pickle: { name: 'Scenario one' }, result: { status: 'passed' } } as never) - await service.afterScenario({ pickle: { name: 'Scenario two' }, result: { status: 'passed' } } as never) - - await service.after(0) - - expect(overrideSessionName).not.toHaveBeenCalled() + expect(scenarioEventArgs().preferScenarioName).toBe(true) }) - it('does NOT rename when preferScenarioName is absent', async () => { - const service = makeService('cucumber', { preferScenarioName: undefined }) + // Absent must travel as an explicit false, not undefined: the module treats the field as the + // whole opt-in, so a missing value and an opted-out value must be indistinguishable there. + it('carries preferScenarioName: false when the option is absent', async () => { + const service = makeService() await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) - await service.after(0) - - expect(overrideSessionName).not.toHaveBeenCalled() - }) - - it('does NOT rename when the only scenario was skipped', async () => { - const service = makeService('cucumber') - await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'skipped' } } as never) - - await service.after(0) - - expect(overrideSessionName).not.toHaveBeenCalled() + expect(scenarioEventArgs().preferScenarioName).toBe(false) }) - // The discriminating pair: identical options and an identical "exactly one unit ran", opposite - // answers. `_scenariosRanCount` / `_lastScenarioName` are written only by cucumber's - // afterScenario, so wdio_mocha cannot reach the rename however preferScenarioName is set. - it('leaves wdio_mocha untouched on the same input', async () => { - const service = makeService('mocha') - await service.afterTest( - { title: 'a test', parent: 'a suite' } as never, - undefined as never, - { passed: true, duration: 1, retries: { attempts: 0, limit: 0 }, exception: '', status: 'passed' } as never - ) - - await service.after(0) + // The count itself stays on the service side too, because `_scenariosRanCount` is what legacy + // reads; the module keeps its own tally for the CLI flow. Both must ignore skipped scenarios. + it('does not count a skipped scenario toward the service-side tally', async () => { + const service = makeService({ preferScenarioName: true }) + await service.afterScenario({ pickle: { name: 'Skipped one' }, result: { status: 'skipped' } } as never) - expect(overrideSessionName).not.toHaveBeenCalled() expect(service['_scenariosRanCount']).toBe(0) }) }) From a16ef314de53346ef60edc8bd95b783e300bcb89 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 05:43:52 +0530 Subject: [PATCH 13/25] docs(cli): drop parity-table references from cucumber comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments only; no code change. Six comments pointed at the SDK-7414 parity table by row number. That table is a migration working artifact, not something a future reader of this file will have, so the pointers were about to become dangling references. Each now states the constraint directly instead of citing where it was recorded. Jira ticket references are kept — SDK-7233 is durable and lookupable, which a row number is not. Co-Authored-By: Claude Opus 5 --- .../frameworks/wdioCucumberTestFramework.ts | 19 ++++++++----------- packages/browserstack-service/src/service.ts | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index e1d46b4..d6541ad 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -25,9 +25,9 @@ const KEY_BDD_META_INFO = 'bdd_meta_info' /** * Per-hook wire keys. The binary cannot derive any of the three from the event: a hook's scope is - * the FEATURE name (parity row 16) while the event carries the examples-qualified SCENARIO name, - * and BEFORE_ALL/AFTER_ALL fire on an instance that has no scenario data at all. Retries and - * duration (row 26) come from WDIO's hook result, which only this side sees. + * the FEATURE name while the event carries the examples-qualified SCENARIO name, and + * BEFORE_ALL/AFTER_ALL fire on an instance that has no scenario data at all. Retries and duration + * come from WDIO's hook result, which only this side sees. */ const KEY_HOOK_SCOPE = 'hook_scope' const KEY_HOOK_RETRIES = 'hook_retries' @@ -321,10 +321,7 @@ export default class WdioCucumberTestFramework extends TestFramework { logger.debug(`trackWdioCucumberInstance: contextId=${trackedContext.getId()} target=${target} testUuid=${testUuid}`) } - /** - * Scenario identity. Every field below is fixed by a parity row and the asymmetries are - * deliberate — see `wdioCucumberTestFramework` notes in the SDK-7414 parity table. - */ + /** Scenario identity. Each field matches what the legacy flow sent; the asymmetries are deliberate. */ private loadScenarioData(instance: TestFrameworkInstance, world: ITestCaseHookParameter) { if (!world?.pickle) { logger.error('loadScenarioData: no pickle on the world object; scenario identity will be empty') @@ -441,7 +438,7 @@ export default class WdioCucumberTestFramework extends TestFramework { /** * Synthesise one detached instance per scenario the feature never got to run, for the - * BEFORE_ALL failure cascade (parity row 15). Rule-nested scenarios included. + * BEFORE_ALL failure cascade. Rule-nested scenarios included. * * Detached is load-bearing: these are NOT registered via `setTrackedInstance`, so the real * per-scenario instance and `process.env[TEST_ANALYTICS_ID]` are untouched. The caller sends @@ -451,7 +448,7 @@ export default class WdioCucumberTestFramework extends TestFramework { * rename the session, fire an a11y stop event and run a Percy teardown per skipped row, none * of which legacy does. * - * Parity row 18: no tags — legacy's cascade payload has no `world`, so `test_tags` is absent. + * No tags: legacy's cascade payload has no `world`, so `test_tags` is absent. */ buildSkippedScenarioInstances(): TestFrameworkInstance[] { const feature = this.cucumberData.feature @@ -527,8 +524,8 @@ export default class WdioCucumberTestFramework extends TestFramework { /** * Whether the scenario in flight failed in a STEP, as opposed to failing only in a hook. - * Public because `ignoreHooksStatus` has two surfaces (parity row 41): the o11y result below, - * and the Automate session status, which `service.afterScenario()` derives from the same answer. + * Public because `ignoreHooksStatus` has two surfaces: the o11y result below, and the Automate + * session status, which `service.afterScenario()` derives from the same answer. */ hasStepFailures(): boolean { return this.scenarioSteps.some(step => step.result === 'FAILED') diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 261f910..8cb6907 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -937,8 +937,8 @@ export default class BrowserstackService implements Services.ServiceInstance { private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { const status = world.result?.status?.toLowerCase() - // `ignoreHooksStatus` has to reach session marking as well as the o11y result (parity row - // 41). On the CLI flow automateModule derives the session status from this view alone — + // `ignoreHooksStatus` has to reach session marking as well as the o11y result. On the CLI + // flow automateModule derives the session status from this view alone — // service.after()'s _failReasons accumulation, which applied the flag on the legacy path, // is gated off while the binary is up. A missing framework keeps the raw status. const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true From f8f5f6a6324d64d13907acdaa16e585b14f70182 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 05:51:17 +0530 Subject: [PATCH 14/25] docs(cli): condense the BEFORE_ALL cascade comment Comment only; no code change. Fourteen lines to seven, keeping why the instances are detached and why they bypass the observer set. Co-Authored-By: Claude Opus 5 --- .../frameworks/wdioCucumberTestFramework.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index d6541ad..1a50d88 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -437,18 +437,12 @@ export default class WdioCucumberTestFramework extends TestFramework { } /** - * Synthesise one detached instance per scenario the feature never got to run, for the - * BEFORE_ALL failure cascade. Rule-nested scenarios included. - * - * Detached is load-bearing: these are NOT registered via `setTrackedInstance`, so the real - * per-scenario instance and `process.env[TEST_ANALYTICS_ID]` are untouched. The caller sends - * each one straight to TestHub rather than through `runHooks`, mirroring legacy — whose - * cascade called `listener.testFinished()` directly and so never reached the Automate, - * Accessibility or Percy handlers. Dispatching these through the observer set instead would - * rename the session, fire an a11y stop event and run a Percy teardown per skipped row, none - * of which legacy does. - * - * No tags: legacy's cascade payload has no `world`, so `test_tags` is absent. + * One detached instance per scenario the feature never ran, for the BEFORE_ALL cascade + * (Rule-nested included). Detached is load-bearing: not registered via `setTrackedInstance`, + * so the live instance and `process.env[TEST_ANALYTICS_ID]` are untouched, and the caller + * sends each straight to TestHub — routing them through the observers would rename the + * session, stop accessibility and run a Percy teardown per row. No tags: legacy's cascade + * payload has no `world`. */ buildSkippedScenarioInstances(): TestFrameworkInstance[] { const feature = this.cucumberData.feature From 1382903b23a9358ee362638712437339fdbbad35 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 05:58:36 +0530 Subject: [PATCH 15/25] fix(cli): send the raw uri in bdd_meta_info.feature.path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `featurePath()` fed two consumers with one absolute value, and only one of them wants it. `test_file_path` must stay absolute: the binary's cucumber module re-bases it (`path.relative(session.pathProject, absoluteTestFilePath)`), so a pre-relativised value there is resolved against cwd first and both `file_name` and `vc_filepath` come out wrong (SDK-7233). Unchanged. `bdd_meta_info` is never read anywhere in the binary's node path, so it reaches the dashboard verbatim. Legacy builds it as `feature = { path: gherkinDocument.uri, … }` — the raw uri — while relativising separately for `file_name`/`location`. Ours sent the absolute path, so the dashboard showed `/Users//…/features/x.feature` where legacy showed `features/x.feature`, publishing the developer's home directory. Both bdd-meta sites now read `cucumberData.uri` directly rather than taking a path parameter, so no caller can pass the absolute value back in. Tests assert both shapes and were falsified: restoring the absolute value fails the two meta assertions and leaves the test_file_path one green. Co-Authored-By: Claude Opus 5 --- .../frameworks/wdioCucumberTestFramework.ts | 16 ++++-- ...oCucumberTestFramework.featurePath.test.ts | 53 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index 1a50d88..fa71fc5 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -358,15 +358,21 @@ export default class WdioCucumberTestFramework extends TestFramework { // `.map` allocates a new array — the pickle's own tag collection is never mutated. [TestFrameworkConstants.KEY_TEST_TAGS]: pickle.tags.map(({ name }: { name: string }) => name), ...resolveFeatureFilePaths(featurePath), - [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, featurePath, examples), + [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, examples), }) } - private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, featurePath: string | undefined, examples: string[] | undefined) { + /** + * `feature.path` is the RAW gherkin uri, matching legacy (`insights-handler` builds + * `feature = { path: gherkinDocument.uri, … }`). It is NOT the absolute path used for + * `test_file_path`: the binary re-bases that one, but never touches this blob, so an absolute + * value here reaches the dashboard as-is and carries the developer's home directory with it. + */ + private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, examples: string[] | undefined) { return { feature: { name: feature?.name, - path: featurePath, + path: this.cucumberData.uri, description: feature?.description, }, scenario: { name: pickle.name }, @@ -392,7 +398,7 @@ export default class WdioCucumberTestFramework extends TestFramework { } if (pickle) { - updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, this.featurePath(), getScenarioExamples(world as ITestCaseHookParameter)) + updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, getScenarioExamples(world as ITestCaseHookParameter)) } const result = world?.result @@ -501,7 +507,7 @@ export default class WdioCucumberTestFramework extends TestFramework { ...resolveFeatureFilePaths(featurePath), [KEY_TEST_SKIPPED_CASCADE]: true, [KEY_BDD_META_INFO]: { - feature: { name: feature.name, path: featurePath, description: feature.description }, + feature: { name: feature.name, path: this.cucumberData.uri, description: feature.description }, scenario: { name: scenario.name }, steps: (scenario.steps || []).map((step: Step) => ({ id: step.id, diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts new file mode 100644 index 0000000..4ab87e4 --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' +import path from 'node:path' +import * as bstackLogger from '../../src/bstackLogger.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +// A cwd-relative uri, as WDIO supplies it on gherkinDocument.uri. +const URI = 'features/checkout.feature' +const FEATURE = { name: 'Checkout', description: 'a feature', children: [] } + +/** + * The two path fields are deliberately different shapes and must not be conflated: + * + * - `test_file_path` is ABSOLUTE. The binary re-bases it itself + * (`path.relative(session.pathProject, absoluteTestFilePath)` in the cucumber module), so a + * pre-relativised value there gets resolved against cwd first and comes out wrong (SDK-7233). + * - `bdd_meta_info.feature.path` is the RAW uri, matching legacy's + * `feature = { path: gherkinDocument.uri, … }`. Nothing in the binary's node path reads this + * blob, so whatever is sent reaches the dashboard verbatim — an absolute value there leaks the + * developer's home directory. + */ +describe('cucumber feature paths — absolute on the wire field, raw uri in the bdd meta', () => { + let framework: WdioCucumberTestFramework + + beforeEach(() => { + framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') + framework.onFeatureStart(URI, FEATURE as never) + }) + + it('sends test_file_path as an absolute path for the binary to re-base', () => { + const featurePath = framework['featurePath']() + + expect(path.isAbsolute(featurePath as string)).toBe(true) + expect(featurePath).toBe(path.resolve(process.cwd(), URI)) + }) + + it('puts the RAW uri in bdd_meta_info.feature.path, not the absolute path', () => { + const meta = framework['buildBddMetaInfo']({ name: 'a scenario', tags: [] } as never, FEATURE as never, []) + + expect(meta.feature.path).toBe(URI) + expect(path.isAbsolute(meta.feature.path as string)).toBe(false) + }) + + // The regression this guards: the two fields were fed from one absolute value, so the meta + // blob reached the dashboard carrying the developer's home directory. + it('does not put the absolute path in the bdd meta', () => { + const meta = framework['buildBddMetaInfo']({ name: 'a scenario', tags: [] } as never, FEATURE as never, []) + + expect(meta.feature.path).not.toBe(framework['featurePath']()) + expect(meta.feature.path).not.toContain(process.cwd()) + }) +}) From e35a3d1c22c3e57fdcefda2991ff08c2704d3516 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 06:04:00 +0530 Subject: [PATCH 16/25] test(cli): name the cucumber framework suite as the main test file Renamed wdioCucumberTestFramework.featurePath.test.ts to wdioCucumberTestFramework.test.ts. No content change. The class had no test file at all, so a concern-scoped name was the wrong shape: the repo uses .test.ts as the main suite and adds ..test.ts alongside it, and the next test for this class now has an obvious home instead of spawning a second file. Co-Authored-By: Claude Opus 5 --- ...work.featurePath.test.ts => wdioCucumberTestFramework.test.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/browserstack-service/tests/cli/{wdioCucumberTestFramework.featurePath.test.ts => wdioCucumberTestFramework.test.ts} (100%) diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts similarity index 100% rename from packages/browserstack-service/tests/cli/wdioCucumberTestFramework.featurePath.test.ts rename to packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts From 8c550bd6f9fe8f7b7961d8850bbe82ee9b6e2f4f Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 06:07:37 +0530 Subject: [PATCH 17/25] test(cli): drop migration-plan references from test names and labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renamed automateModule.phase8.test.ts to automateModule.sessionMarking.test.ts and removed the "Phase 8", "8-A", "8-B" describe labels. The file covers two halves of one subject — which API a session mark is sent to, and what verdict a build-level hook failure produces — so the subject names it better than the phase that happened to introduce it. Also stripped parity-row citations from six test files. Same reasoning as the source comments: the row numbers point at a migration working artifact a future reader will not have, and were about to become dangling references. The assertions already state what they check. No test content changed; 1271 still pass. Co-Authored-By: Claude Opus 5 --- .../tests/cli/modules/accessibilityModule.test.ts | 2 +- .../automateModule.preferScenarioName.test.ts | 2 +- ...test.ts => automateModule.sessionMarking.test.ts} | 12 ++++++------ .../tests/cli/modules/customTagsModule.test.ts | 4 ++-- .../tests/service.preferScenarioName.cli.test.ts | 6 +++--- packages/browserstack-service/tests/service.test.ts | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) rename packages/browserstack-service/tests/cli/modules/{automateModule.phase8.test.ts => automateModule.sessionMarking.test.ts} (95%) diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 4f0eda4..78027b5 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -555,7 +555,7 @@ describe('AccessibilityModule', () => { // 8-C. Discriminating: identical call, opposite answers. Mocha's beforeEach precedes // beforeTest so the re-open is harmless; cucumber's scenario boundary precedes its Before // hooks, so the same write would permanently force the gate open and defeat the - // includeTagsInTestingScope / excludeTagsInTestingScope filtering (parity row 35). + // includeTagsInTestingScope / excludeTagsInTestingScope filtering. it('does NOT re-open the scan gate for cucumber — the per-test gate stands', async () => { mockInstanceState('WebdriverIO-cucumber') accessibilityModule.accessibilityMap.set(12345, false) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts index ab0c83b..84559c3 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts @@ -69,7 +69,7 @@ function namesPUT() { }) } -describe('AutomateModule preferScenarioName — parity row 40', () => { +describe('AutomateModule preferScenarioName', () => { beforeEach(() => { vi.clearAllMocks() vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts similarity index 95% rename from packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts rename to packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts index 87cdc85..d480f52 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.phase8.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts @@ -63,7 +63,7 @@ function newModule(config: Record = {}) { return mod } -describe('AutomateModule — Phase 8 remediations', () => { +describe('AutomateModule — session marking', () => { beforeEach(() => { vi.clearAllMocks() vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) @@ -79,10 +79,10 @@ describe('AutomateModule — Phase 8 remediations', () => { }) /** - * 8-A. Discriminating: the SAME call produces opposite verbs and different hosts/paths + * Discriminating: the SAME call produces opposite verbs and different hosts/paths * depending only on the turboscale flag. */ - describe('8-A — turboscale session marking routes to its own API with PATCH', () => { + describe('turboscale session marking routes to its own API with PATCH', () => { it('PATCHes the turboscale endpoint when turboScale is configured', async () => { const mod = newModule({ turboScale: true }) await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) @@ -131,10 +131,10 @@ describe('AutomateModule — Phase 8 remediations', () => { }) /** - * 8-B (session verdict). Discriminating: the SAME failing build-level hook fails the session + * Session verdict. Discriminating: the SAME failing build-level hook fails the session * for cucumber and leaves mocha's verdict untouched. */ - describe('8-B — build-level hook failures reach the session verdict', () => { + describe('build-level hook failures reach the session verdict', () => { const failing = { passed: false, error: new Error('BeforeAll blew up') } /** Drives one scenario through TEST/POST so `testResults` carries a real scenario result. */ @@ -189,7 +189,7 @@ describe('AutomateModule — Phase 8 remediations', () => { expect(fetch).not.toHaveBeenCalled() }) - it('keeps the session PASSED under ignoreHooksStatus once a scenario has run (parity row 41)', async () => { + it('keeps the session PASSED under ignoreHooksStatus once a scenario has run', async () => { const mod = newModule() await runScenario(mod, true) await mod.onBuildLevelHookEnd('AFTER_ALL', { diff --git a/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts index 83c0c3c..cf4a435 100644 --- a/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts @@ -33,11 +33,11 @@ import { BrowserstackCLI } from '../../../src/cli/index.js' import { BStackLogger } from '../../../src/cli/cliLogger.js' /** - * 8-D — parity row 19. `setCustomTags` warns and no-ops for every framework except mocha, which + * 8-D. `setCustomTags` warns and no-ops for every framework except mocha, which * is what the legacy custom-tags-handler does. Discriminating: the SAME call merges tags under * mocha and merges nothing under cucumber. */ -describe('CustomTagsModule — framework gate (parity row 19)', () => { +describe('CustomTagsModule — framework gate', () => { let module: CustomTagsModule let browser: Record let instance: { updateMultipleEntries: ReturnType, getCurrentTestState: ReturnType } diff --git a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts index d8a0972..b365792 100644 --- a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts +++ b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts @@ -14,12 +14,12 @@ vi.mock('../src/cli/index.js', () => ({ })) /** - * The seam between the two halves of parity row 40: automateModule decides the rename (it is the + * The seam between the two halves of automateModule decides the rename (it is the * only place that knows the final scenario count), but it cannot read service options, so the flag * rides the scenario event — the same route `ignoreHooksStatus` takes. The decision itself is * covered in tests/cli/modules/automateModule.preferScenarioName.test.ts. */ -describe('preferScenarioName reaches the module — parity row 40', () => { +describe('preferScenarioName reaches the module', () => { let getInstanceSpy: ReturnType | undefined let trackEvent: ReturnType @@ -73,7 +73,7 @@ describe('preferScenarioName reaches the module — parity row 40', () => { }) }) -describe('_cucumberTestResult failure reason — parity row 39 adjacent', () => { +describe('_cucumberTestResult failure reason adjacent', () => { const makeService = (strict: boolean) => new BrowserstackService( { testObservability: false } as never, [] as never, diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 44eed94..32e0012 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2777,7 +2777,7 @@ describe('afterTest bail skip cascade (SDK-7063)', () => { }) }) -describe('afterScenario session-status view honours ignoreHooksStatus (parity row 41)', () => { +describe('afterScenario session-status view honours ignoreHooksStatus', () => { let getInstanceSpy: ReturnType const makeService = (ignoreHooksStatus: boolean) => new BrowserstackService( @@ -2823,7 +2823,7 @@ describe('afterScenario session-status view honours ignoreHooksStatus (parity ro }) }) -describe('BEFORE_ALL skip cascade + hook flag pass-through (parity row 15, escape class 3 / SDK-7047)', () => { +describe('BEFORE_ALL skip cascade + hook flag pass-through (legacy parity, escape class 3 / SDK-7047)', () => { let getInstanceSpy: ReturnType const feature = { @@ -2889,7 +2889,7 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (parity row 15, escap const data = args.instance.getAllData() expect(data.get('test_result')).toBe('skipped') expect(data.get('test_skipped_cascade')).toBe(true) - // parity row 18 — the cascade payload has no world, so no tags + // the cascade payload has no world, so no tags expect(data.get('test_tags')).toBeUndefined() expect(data.get('test_scopes')).toEqual(['Login']) } From 17ed99f123bb4332070c5ce9df496ef23d08bb13 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 13:04:34 +0530 Subject: [PATCH 18/25] fix(cli): relativise bdd_meta_info.feature.path, which 1382903 did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `1382903` swapped `bdd_meta_info.feature.path` from the absolute `featurePath()` to the raw `cucumberData.uri` on the premise that WDIO supplies a cwd-relative uri. It does not. WDIO hands `beforeFeature` an ABSOLUTE path — verified on the wire: onFeatureStart: uri=/Users/…/automate-wdio_cucumber/features/cfg-one.feature so `path.resolve(cwd, uri) === uri` and the swap was a no-op at runtime. The dashboard still showed the developer's home directory. The unit test stayed green only because its fixture fed `onFeatureStart` a relative uri, a shape WDIO never produces. Legacy never reads WDIO's uri: `insights-handler` builds the blob off the cucumber world's `gherkinDocument.uri`, which is cwd-relative. Both bdd-meta sites now go through `featureUriForMeta()`, which relativises against cwd and reproduces that value exactly. `test_file_path` is untouched and stays ABSOLUTE — the binary re-bases it itself, and pre-relativising it corrupts `file_name` and `vc_filepath` (SDK-7233). Verified on the O11Y dashboard, CLI against a published-9.35.1 legacy control on the same feature file: meta.feature.path features/cfg-one.feature == legacy (was an absolute path) file_name features/cfg-one.feature == legacy (unchanged) location features/cfg-one.feature (unchanged) vc_filepath '' == legacy (unchanged) No `/Users/` string survives anywhere in the CLI run's payload — 2 occurrences before, 0 after. The fixture now runs both uri shapes. Falsified: restoring `cucumberData.uri` fails the two meta assertions on the absolute-uri arm only and leaves every `test_file_path` assertion green — the same falsification 1382903 claimed, which its relative-only fixture could not actually perform. Co-Authored-By: Claude Opus 5 --- .../frameworks/wdioCucumberTestFramework.ts | 24 ++++++++++++++----- .../cli/wdioCucumberTestFramework.test.ts | 23 +++++++++++------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index fa71fc5..8cda05f 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -206,6 +206,18 @@ export default class WdioCucumberTestFramework extends TestFramework { return uri ? path.resolve(process.cwd(), uri) : undefined } + /** + * The feature path as the bdd meta blob wants it — cwd-relative, matching legacy. + * + * `cucumberData.uri` cannot be forwarded raw: WDIO hands `beforeFeature` an ABSOLUTE uri, so + * the raw value carries the developer's home directory. Legacy never sees that value — it + * reads the cucumber world's `gherkinDocument.uri`, which is cwd-relative. + */ + private featureUriForMeta(): string | undefined { + const absolute = this.featurePath() + return absolute ? path.relative(process.cwd(), absolute) : undefined + } + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) await super.trackEvent(testFrameworkState, hookState, args) @@ -363,16 +375,16 @@ export default class WdioCucumberTestFramework extends TestFramework { } /** - * `feature.path` is the RAW gherkin uri, matching legacy (`insights-handler` builds - * `feature = { path: gherkinDocument.uri, … }`). It is NOT the absolute path used for - * `test_file_path`: the binary re-bases that one, but never touches this blob, so an absolute - * value here reaches the dashboard as-is and carries the developer's home directory with it. + * `feature.path` is cwd-relative, matching legacy (`insights-handler` builds + * `feature = { path: gherkinDocument.uri, … }` off the cucumber world). It is NOT the absolute + * path used for `test_file_path`: the binary re-bases that one, but never touches this blob, so + * an absolute value here reaches the dashboard as-is, home directory and all. */ private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, examples: string[] | undefined) { return { feature: { name: feature?.name, - path: this.cucumberData.uri, + path: this.featureUriForMeta(), description: feature?.description, }, scenario: { name: pickle.name }, @@ -507,7 +519,7 @@ export default class WdioCucumberTestFramework extends TestFramework { ...resolveFeatureFilePaths(featurePath), [KEY_TEST_SKIPPED_CASCADE]: true, [KEY_BDD_META_INFO]: { - feature: { name: feature.name, path: this.cucumberData.uri, description: feature.description }, + feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, scenario: { name: scenario.name }, steps: (scenario.steps || []).map((step: Step) => ({ id: step.id, diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts index 4ab87e4..53d4438 100644 --- a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts @@ -5,8 +5,8 @@ import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTest vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) -// A cwd-relative uri, as WDIO supplies it on gherkinDocument.uri. -const URI = 'features/checkout.feature' +const RELATIVE_URI = 'features/checkout.feature' +const ABSOLUTE_URI = path.resolve(process.cwd(), RELATIVE_URI) const FEATURE = { name: 'Checkout', description: 'a feature', children: [] } /** @@ -15,30 +15,37 @@ const FEATURE = { name: 'Checkout', description: 'a feature', children: [] } * - `test_file_path` is ABSOLUTE. The binary re-bases it itself * (`path.relative(session.pathProject, absoluteTestFilePath)` in the cucumber module), so a * pre-relativised value there gets resolved against cwd first and comes out wrong (SDK-7233). - * - `bdd_meta_info.feature.path` is the RAW uri, matching legacy's + * - `bdd_meta_info.feature.path` is cwd-relative, matching legacy's * `feature = { path: gherkinDocument.uri, … }`. Nothing in the binary's node path reads this * blob, so whatever is sent reaches the dashboard verbatim — an absolute value there leaks the * developer's home directory. + * + * Both uri shapes are exercised because WDIO hands `beforeFeature` the ABSOLUTE one. A fixture that + * only feeds the relative shape stays green against a `path` that forwards the uri raw, which is + * exactly how an absolute path reached the dashboard under a passing suite. */ -describe('cucumber feature paths — absolute on the wire field, raw uri in the bdd meta', () => { +describe.each([ + ['a relative uri', RELATIVE_URI], + ['the absolute uri WDIO supplies', ABSOLUTE_URI], +])('cucumber feature paths — %s', (_shape, uri) => { let framework: WdioCucumberTestFramework beforeEach(() => { framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') - framework.onFeatureStart(URI, FEATURE as never) + framework.onFeatureStart(uri, FEATURE as never) }) it('sends test_file_path as an absolute path for the binary to re-base', () => { const featurePath = framework['featurePath']() expect(path.isAbsolute(featurePath as string)).toBe(true) - expect(featurePath).toBe(path.resolve(process.cwd(), URI)) + expect(featurePath).toBe(ABSOLUTE_URI) }) - it('puts the RAW uri in bdd_meta_info.feature.path, not the absolute path', () => { + it('puts a cwd-relative path in bdd_meta_info.feature.path', () => { const meta = framework['buildBddMetaInfo']({ name: 'a scenario', tags: [] } as never, FEATURE as never, []) - expect(meta.feature.path).toBe(URI) + expect(meta.feature.path).toBe(RELATIVE_URI) expect(path.isAbsolute(meta.feature.path as string)).toBe(false) }) From f45497d80f31abdef3cdc27afbe8e53bc132fa89 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 13:37:09 +0530 Subject: [PATCH 19/25] docs(cli): condense the build-level hook comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment only; no code change. Nine lines to four on the specsRan guard in onBuildLevelHookEnd, keeping why it keys on the absence of scenario results rather than on ignoreHooksStatus. Also removes two parity-table references my earlier sweep missed — it was case-sensitive and these read "Parity row". Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 13 ++++--------- .../tests/cli/modules/automateModule.test.ts | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index b5c906d..2bb9ecc 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -272,15 +272,10 @@ export default class AutomateModule extends BaseModule { } const sessionData = this.sessionMap.get(sessionId) - // Parity row 41, third surface: with ignoreHooksStatus declared, a failure that exists - // only in a hook must leave the session passed. Legacy expresses that in the - // `ignoreHooksStatus && this._specsRan` arm of `after()`, and that arm needs BOTH. With - // no scenario recorded, legacy instead falls through to the arm that marks `failed` - // unconditionally — flag or no flag — so honouring the flag here would leave the session - // unmarked where legacy marks it, and an unmarked session is invisible on the dashboard. - // Keyed on the absence of scenario results, never on the flag. A cucumber `BeforeAll` - // failure aborts the run outright, so nothing can arrive after this point; by `AfterAll` - // every scenario that ran has already been recorded. + // Keyed on the absence of scenario results, never on the flag: legacy's + // `ignoreHooksStatus && this._specsRan` arm needs BOTH, and with no scenario recorded it + // falls through to marking `failed` regardless of the flag. The count is final here — a + // `BeforeAll` failure aborts the run, and by `AfterAll` every scenario has been recorded. const specsRan = (sessionData?.testResults.size ?? 0) > 0 if (specsRan && isTrue(args?.ignoreHooksStatus)) { this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index ec07fa3..d3a0b40 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -697,7 +697,7 @@ describe('AutomateModule testResults keying (SDK-7414)', () => { expect([...results.keys()]).toEqual(['scenario one fails', 'scenario two passes']) expect(results.size).toBe(2) expect([...results.values()].map((r: any) => r.status)).toEqual(['failed', 'passed']) - // Parity row 31: the session NAME stays the feature title even though the keys do not. + // the session NAME stays the feature title even though the keys do not. expect(sessionData().lastTestName).toBe(FEATURE) }) From f84fc3935bcf3cf11adf4854452e43c13f5bf045 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 14:46:03 +0530 Subject: [PATCH 20/25] fix(cli): status-mark a session even when setSessionName is false MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opting out of session NAMING silently opted you out of status MARKING: the session showed `done`/unmarked on the dashboard instead of passed or failed. sessionMap registration was gated on skipSessionName in both places that do it — onBeforeTest returned before registering, and onAfterTest's repair carried the same conjunct — so testResults was never populated and the onAfterExecute sweep had nothing to mark. Registration is now independent of the flag; the name is what it suppresses. flushSessionName already hard-returns on skipSessionName and on an empty lastTestName, so a registered session cannot leak a name. Legacy gates its after() status block on setSessionStatus alone — setSessionName never enters the condition. Measured on both frameworks' legacy arms: with setSessionName:false, cucumber (9.35.1) and mocha (9.20.1, the last release before mocha was platformised) both report name '' and status passed, while both CLI arms report unmarked. So this regressed every framework on the CLI path, not just cucumber. G7: this changes wdio_mocha's behaviour too, and deliberately — mocha is equally broken today and the fix repairs it. The inverse-leak guard test asserted the opposite, having encoded the CLI's own prior behaviour rather than parity with legacy. Rewritten to assert exactly one status call carrying no name field; falsified against the reverted code. Verified at unit level only. Dashboard verification pending. Co-Authored-By: Claude Opus 5 --- .../src/cli/modules/automateModule.ts | 26 ++++++++++++++----- .../tests/cli/modules/automateModule.test.ts | 15 ++++++++--- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 2bb9ecc..a62efb5 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -70,8 +70,20 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions - if (testContextOptions.skipSessionName || !isBrowserstackSession(browser)) { + if (!isBrowserstackSession(browser)) { + return + } + + // `setSessionName: false` suppresses the NAME, not the registration. The session still has + // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it + // either way — its `after()` status block gates on setSessionStatus alone. Registering with + // an empty lastTestName is safe: flushSessionName early-returns both on the flag and on an + // empty name, so no name can be sent from here. + if (testContextOptions.skipSessionName) { this.logger.info('Skipping session name update as per configuration') + if (sessionId && !this.sessionMap.has(sessionId)) { + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) + } return } @@ -193,12 +205,12 @@ export default class AutomateModule extends BaseModule { // (service.onReload has already pointed KEY_FRAMEWORK_SESSION_ID at it) and adopt it while // it is still open. // - // Deliberately gated on skipSessionName, NOT skipSessionStatus: naming and status are - // independent options, so a `setSessionStatus: false` user must still get the name repair, - // and a `setSessionName: false` user must not be pulled into sessionMap — that would hand - // onAfterExecute a session to status-mark where it previously had none. - if (sessionId && !testContextOptions.skipSessionName && !this.sessionMap.has(sessionId)) { - this.sessionMap.set(sessionId, { lastTestName: name, testResults: new Map(), scenariosRan: 0 }) + // Registration is independent of both opt-outs: `setSessionStatus: false` must still get the + // name repair, and `setSessionName: false` must still be status-marked. The name is what the + // flag suppresses, so a skipped-name session registers with an empty lastTestName. + if (sessionId && !this.sessionMap.has(sessionId)) { + const repairName = testContextOptions.skipSessionName ? '' : name + this.sessionMap.set(sessionId, { lastTestName: repairName, testResults: new Map(), scenariosRan: 0 }) } // No-op for the steady state: when no mid-test reload happened, onBeforeTest already // applied this exact name and `appliedName` de-dupes it away — no extra API call. diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index d3a0b40..e4b127e 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -306,9 +306,12 @@ describe('AutomateModule', () => { ) }) - it('adds no status traffic for skipSessionName users (SDK-7270 inverse-leak guard)', async () => { - // With naming off, onBeforeTest never registers the session. onAfterTest must not adopt it - // either, or onAfterExecute would start status-marking sessions it previously ignored. + it('still status-marks a skipSessionName session, and sends no name', async () => { + // `setSessionName: false` suppresses the NAME only. Legacy gates its `after()` status block + // on setSessionStatus alone, and marks `passed` with an empty name — measured on BOTH + // frameworks' legacy arms. This test previously asserted the opposite (no traffic at all), + // which encoded the CLI's own behaviour rather than parity with legacy, and left every + // CLI-flow session unmarked whenever a user opted out of naming. (automateModule.config as any).testContextOptions.skipSessionName = true vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) @@ -327,7 +330,11 @@ describe('AutomateModule', () => { await automateModule.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() + const bodies = vi.mocked(fetch).mock.calls.map(([, init]) => (init as { body: string }).body) + // exactly one call, and it is the status mark — no name field anywhere + expect(bodies).toHaveLength(1) + expect(JSON.parse(bodies[0])).not.toHaveProperty('name') + expect(JSON.parse(bodies[0]).status).toBe('passed') }) it('should skip session status update when skipSessionStatus is true', async () => { From 59a03dee30a668efcb8b1ac2529726b693c034fe Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 17:37:54 +0530 Subject: [PATCH 21/25] fix(cli): register the driver when every product is turned off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With observability, accessibility and Percy all off, the Automate session went unnamed and unmarked and browser.setCustomTags was never defined — a call to it threw, where legacy defines the method and warns. service.ts raised the CREATE/POST driver-registration event only inside `if (shouldProcessEventForTesthub(''))`. That predicate is a disjunction over the three product flags, called with an empty eventType, so accessibility or Percy being on holds it open and only all-three-off closes it. With it closed the event never fires: webdriverIOModule.onDriverCreated never runs, the driver is unregistered, isBrowserstackSession() is falsy, and automateModule skips both naming and status marking. customTagsModule.onBeforeExecute never assigns setCustomTags either — the module IS constructed, so its absence is not a construction problem but a missing event. Measured over 8 runs, both flows: with a11y or Percy on, CLI is indistinguishable from legacy. With everything off, CLI gives name '' / status done / setCustomTags undefined against legacy's named, marked, defined. Deliberately raised only where the gate would have swallowed it, rather than hoisted out of the block. Hoisting reads better and is the right refactor later, but it would reorder the event ahead of `new InsightsHandler(...)` on every configuration that already works. This shape cannot execute when the gate is open, so no working configuration changes and verification narrows to the all-products-off arm. Tests falsified in both directions: reverting the fix fails the all-off case; making the new branch unconditional fails both double-fire cases. Scope: service side only. The binary also creates an empty TestHub build in this configuration where legacy creates none — out of scope here, still open. Co-Authored-By: Claude Opus 5 --- packages/browserstack-service/src/service.ts | 11 +++ .../service.driverRegistration.cli.test.ts | 83 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 packages/browserstack-service/tests/service.driverRegistration.cli.test.ts diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 8cb6907..0cf63c7 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -302,6 +302,17 @@ export default class BrowserstackService implements Services.ServiceInstance { BStackLogger.error(`[Accessibility Test Run] Error in service class before function: ${err}`) } + // The driver registration is an automation-lifecycle concern, not an observability + // one. With EVERY product off, shouldProcessEventForTesthub() closes (it is a + // disjunction over the three product flags, and '' bypasses the eventType guards), + // so the CREATE/POST event inside the block below never fires: onDriverCreated never + // runs, and the session goes unnamed and unmarked while browser.setCustomTags is + // never defined. This covers exactly that case — when the gate is open the block + // below still raises the event, so nothing fires twice. + if (BrowserstackCLI.getInstance().isRunning() && !shouldProcessEventForTesthub('')) { + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) + } + if (shouldProcessEventForTesthub('')) { patchConsoleLogs() diff --git a/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts b/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts new file mode 100644 index 0000000..a252acf --- /dev/null +++ b/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' +import BrowserstackService from '../src/service.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' + +vi.mock('../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: () => ({ + isRunning: () => false, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + }) + } +})) + +/** + * The CREATE/POST event is the driver registration — `webdriverIOModule.onDriverCreated` and the + * other product modules' init handlers all hang off it. It used to be raised ONLY inside + * `if (shouldProcessEventForTesthub(''))`, which is a disjunction over the three product flags: with + * every product off the gate closes, the event never fires, and the session goes unnamed and + * unmarked while `browser.setCustomTags` is never defined. + * + * Both assertions matter. The first is the fix. The second is the safety property of its shape — + * it must not double-raise on the configurations that already worked. + */ +describe('driver registration is not gated on observability', () => { + const PRODUCT_ENV = ['BROWSERSTACK_OBSERVABILITY', 'BROWSERSTACK_ACCESSIBILITY', 'BROWSERSTACK_PERCY'] + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + const saved: Record = {} + + const createPostCalls = () => trackEvent.mock.calls.filter( + ([state, hook]) => state === AutomationFrameworkState.CREATE && hook === HookState.POST) + + const runBefore = async () => { + const service = new BrowserstackService({} as never, [{}] as never, { capabilities: {} } as never) + await service.beforeSession({} as never) + await service.before(service['_config'] as never, [], { sessionId: 'sess-1' } as never) + } + + beforeEach(() => { + PRODUCT_ENV.forEach(k => { saved[k] = process.env[k]; delete process.env[k] }) + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + }) + + afterEach(() => { + PRODUCT_ENV.forEach(k => { if (saved[k] === undefined) { delete process.env[k] } else { process.env[k] = saved[k] } }) + getInstanceSpy?.mockRestore() + }) + + // The fix: every product off is the one configuration that closes the gate. + it('registers the driver with every product turned off', async () => { + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + // The safety property: on a configuration that already worked, the event must fire once, not + // twice — the gated block still raises it and this must not add a second. + it('does not double-register when observability is on', async () => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + // Accessibility alone also holds the gate open, so the same single-fire rule applies. + it('does not double-register when only accessibility is on', async () => { + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) +}) From 31c5cc78c9d60acb8c9702ffb9c82135887efc65 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 18:10:37 +0530 Subject: [PATCH 22/25] test(cli): merge the driver-registration tests into service.test.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved the three cases out of service.driverRegistration.cli.test.ts and deleted that file. service.test.ts is the main suite for service.ts and already mocks cli/index.js in the same shape, so the standalone file added a second copy of that setup for no benefit. Comment condensed from ten lines to five. No test content changed; still falsified in both directions — reverting the fix fails one case, making the new branch unconditional fails the other two. Co-Authored-By: Claude Opus 5 --- .../service.driverRegistration.cli.test.ts | 83 ------------------- .../tests/service.test.ts | 60 ++++++++++++++ 2 files changed, 60 insertions(+), 83 deletions(-) delete mode 100644 packages/browserstack-service/tests/service.driverRegistration.cli.test.ts diff --git a/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts b/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts deleted file mode 100644 index a252acf..0000000 --- a/packages/browserstack-service/tests/service.driverRegistration.cli.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import BrowserstackService from '../src/service.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' -import { HookState } from '../src/cli/states/hookState.js' - -vi.mock('../src/cli/index.js', () => ({ - BrowserstackCLI: { - getInstance: () => ({ - isRunning: () => false, - getTestFramework: () => null, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - }) - } -})) - -/** - * The CREATE/POST event is the driver registration — `webdriverIOModule.onDriverCreated` and the - * other product modules' init handlers all hang off it. It used to be raised ONLY inside - * `if (shouldProcessEventForTesthub(''))`, which is a disjunction over the three product flags: with - * every product off the gate closes, the event never fires, and the session goes unnamed and - * unmarked while `browser.setCustomTags` is never defined. - * - * Both assertions matter. The first is the fix. The second is the safety property of its shape — - * it must not double-raise on the configurations that already worked. - */ -describe('driver registration is not gated on observability', () => { - const PRODUCT_ENV = ['BROWSERSTACK_OBSERVABILITY', 'BROWSERSTACK_ACCESSIBILITY', 'BROWSERSTACK_PERCY'] - let trackEvent: ReturnType - let getInstanceSpy: ReturnType | undefined - const saved: Record = {} - - const createPostCalls = () => trackEvent.mock.calls.filter( - ([state, hook]) => state === AutomationFrameworkState.CREATE && hook === HookState.POST) - - const runBefore = async () => { - const service = new BrowserstackService({} as never, [{}] as never, { capabilities: {} } as never) - await service.beforeSession({} as never) - await service.before(service['_config'] as never, [], { sessionId: 'sess-1' } as never) - } - - beforeEach(() => { - PRODUCT_ENV.forEach(k => { saved[k] = process.env[k]; delete process.env[k] }) - trackEvent = vi.fn().mockResolvedValue(undefined) - getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => true, - getTestFramework: () => null, - getAutomationFramework: () => ({ trackEvent }), - modules: {} - } as never) - }) - - afterEach(() => { - PRODUCT_ENV.forEach(k => { if (saved[k] === undefined) { delete process.env[k] } else { process.env[k] = saved[k] } }) - getInstanceSpy?.mockRestore() - }) - - // The fix: every product off is the one configuration that closes the gate. - it('registers the driver with every product turned off', async () => { - await runBefore() - - expect(createPostCalls()).toHaveLength(1) - }) - - // The safety property: on a configuration that already worked, the event must fire once, not - // twice — the gated block still raises it and this must not add a second. - it('does not double-register when observability is on', async () => { - process.env.BROWSERSTACK_OBSERVABILITY = 'true' - - await runBefore() - - expect(createPostCalls()).toHaveLength(1) - }) - - // Accessibility alone also holds the gate open, so the same single-fire rule applies. - it('does not double-register when only accessibility is on', async () => { - process.env.BROWSERSTACK_ACCESSIBILITY = 'true' - - await runBefore() - - expect(createPostCalls()).toHaveLength(1) - }) -}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 32e0012..d54bb12 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -14,6 +14,7 @@ import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFra import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' +import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' @@ -2941,3 +2942,62 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (legacy parity, escap expect(finished.get('BEFORE_ALL')![0].hook_result).toBe('failed') }) }) + +describe('driver registration is not gated on observability', () => { + // CREATE/POST is the driver registration — onDriverCreated and the product modules' init + // handlers hang off it. It used to be raised only inside `shouldProcessEventForTesthub('')`, + // a disjunction over the three product flags, so with every product off the gate closed and + // the session went unnamed and unmarked with setCustomTags undefined. The second and third + // cases guard the fix's shape: it must not double-raise where the gate is already open. + const PRODUCT_ENV = ['BROWSERSTACK_OBSERVABILITY', 'BROWSERSTACK_ACCESSIBILITY', 'BROWSERSTACK_PERCY'] + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + const saved: Record = {} + + const createPostCalls = () => trackEvent.mock.calls.filter( + ([state, hook]) => state === AutomationFrameworkState.CREATE && hook === HookState.POST) + + const runBefore = async () => { + const svc = new BrowserstackService({} as never, [{}] as never, { capabilities: {} } as never) + await svc.beforeSession({} as never) + await svc.before(svc['_config'] as never, [], { sessionId: 'sess-1' } as never) + } + + beforeEach(() => { + PRODUCT_ENV.forEach(k => { saved[k] = process.env[k]; delete process.env[k] }) + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + }) + + afterEach(() => { + PRODUCT_ENV.forEach(k => { if (saved[k] === undefined) { delete process.env[k] } else { process.env[k] = saved[k] } }) + getInstanceSpy?.mockRestore() + }) + + it('registers the driver with every product turned off', async () => { + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + it('does not double-register when observability is on', async () => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + it('does not double-register when only accessibility is on', async () => { + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) +}) From 8d63e5faae76d5cf92ceeb57f663e164c898e5da Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 18:15:20 +0530 Subject: [PATCH 23/25] test(cli): fold service.preferScenarioName.cli.test.ts into service.test.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second and last of the per-concern service suites. All its imports and its cli/index.js mock already existed in service.test.ts, so the separate file was duplicated setup around seven tests — three on the preferScenarioName event seam and four on _cucumberTestResult's failure reason. Repaired a comment my earlier parity-row sweep had mangled: removing "parity row 40:" left "the two halves of automateModule decides the rename", which no longer parsed. Rewritten and condensed. Also dropped an "escape class 3" reference the sweep missed — same migration-plan category as the row numbers. SDK-7047 kept; a ticket outlives the plan. No test content changed. 1277 pass. Co-Authored-By: Claude Opus 5 --- .../service.preferScenarioName.cli.test.ts | 114 ------------------ .../tests/service.test.ts | 99 ++++++++++++++- 2 files changed, 98 insertions(+), 115 deletions(-) delete mode 100644 packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts diff --git a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts b/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts deleted file mode 100644 index b365792..0000000 --- a/packages/browserstack-service/tests/service.preferScenarioName.cli.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import BrowserstackService from '../src/service.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFramework.js' - -vi.mock('../src/cli/index.js', () => ({ - BrowserstackCLI: { - getInstance: () => ({ - isRunning: () => false, - getTestFramework: () => null, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - }) - } -})) - -/** - * The seam between the two halves of automateModule decides the rename (it is the - * only place that knows the final scenario count), but it cannot read service options, so the flag - * rides the scenario event — the same route `ignoreHooksStatus` takes. The decision itself is - * covered in tests/cli/modules/automateModule.preferScenarioName.test.ts. - */ -describe('preferScenarioName reaches the module', () => { - let getInstanceSpy: ReturnType | undefined - let trackEvent: ReturnType - - const makeService = (options: Record = {}) => new BrowserstackService( - { testObservability: false, setSessionName: true, setSessionStatus: true, ...options } as never, - [] as never, - { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict: false } } as never - ) - - const scenarioEventArgs = () => trackEvent.mock.calls.at(-1)?.[2] as Record - - beforeEach(() => { - trackEvent = vi.fn().mockResolvedValue(undefined) - const cucumberFramework = Object.create(WdioCucumberTestFramework.prototype) - cucumberFramework.trackEvent = trackEvent - cucumberFramework.hasStepFailures = () => false - getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => true, - getTestFramework: () => cucumberFramework, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - } as never) - }) - - afterEach(() => { - getInstanceSpy?.mockRestore() - }) - - it('carries preferScenarioName: true on the scenario event when set', async () => { - const service = makeService({ preferScenarioName: true }) - await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) - - expect(scenarioEventArgs().preferScenarioName).toBe(true) - }) - - // Absent must travel as an explicit false, not undefined: the module treats the field as the - // whole opt-in, so a missing value and an opted-out value must be indistinguishable there. - it('carries preferScenarioName: false when the option is absent', async () => { - const service = makeService() - await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) - - expect(scenarioEventArgs().preferScenarioName).toBe(false) - }) - - // The count itself stays on the service side too, because `_scenariosRanCount` is what legacy - // reads; the module keeps its own tally for the CLI flow. Both must ignore skipped scenarios. - it('does not count a skipped scenario toward the service-side tally', async () => { - const service = makeService({ preferScenarioName: true }) - await service.afterScenario({ pickle: { name: 'Skipped one' }, result: { status: 'skipped' } } as never) - - expect(service['_scenariosRanCount']).toBe(0) - }) -}) - -describe('_cucumberTestResult failure reason adjacent', () => { - const makeService = (strict: boolean) => new BrowserstackService( - { testObservability: false } as never, - [] as never, - { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict } } as never - ) - - const world = (status: string, message?: string) => ({ - pickle: { name: 'CfgGate pending scenario' }, - result: message ? { status, message } : { status } - }) - - it('synthesises legacy\'s pending reason when strict makes a pending scenario fail', () => { - const result = makeService(true)['_cucumberTestResult'](world('PENDING') as never) - - expect(result.passed).toBe(false) - expect(result.error?.message).toBe('Some steps/hooks are pending for scenario "CfgGate pending scenario"') - }) - - it('leaves a pending scenario unfailed — and unreasoned — when strict is off', () => { - const result = makeService(false)['_cucumberTestResult'](world('PENDING') as never) - - expect(result.passed).toBe(false) - expect(result.skipped).toBe(true) - expect(result.error).toBeUndefined() - }) - - it('keeps the real message when the result carries one', () => { - const result = makeService(false)['_cucumberTestResult'](world('FAILED', 'AssertionError: nope') as never) - - expect(result.error?.message).toBe('AssertionError: nope') - }) - - it('falls back to Unknown Error for a message-less non-pending failure', () => { - const result = makeService(false)['_cucumberTestResult'](world('UNDEFINED') as never) - - expect(result.error?.message).toBe('Unknown Error') - }) -}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index d54bb12..6ac857c 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2824,7 +2824,7 @@ describe('afterScenario session-status view honours ignoreHooksStatus', () => { }) }) -describe('BEFORE_ALL skip cascade + hook flag pass-through (legacy parity, escape class 3 / SDK-7047)', () => { +describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => { let getInstanceSpy: ReturnType const feature = { @@ -3001,3 +3001,100 @@ describe('driver registration is not gated on observability', () => { expect(createPostCalls()).toHaveLength(1) }) }) + +// automateModule decides the rename — it is the only place that knows the final scenario count — +// but it cannot read service options, so the flag rides the scenario event, as `ignoreHooksStatus` +// does. The decision itself is covered in tests/cli/modules/automateModule.preferScenarioName.test.ts. +describe('preferScenarioName reaches the module', () => { + let getInstanceSpy: ReturnType | undefined + let trackEvent: ReturnType + + const makeService = (options: Record = {}) => new BrowserstackService( + { testObservability: false, setSessionName: true, setSessionStatus: true, ...options } as never, + [] as never, + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict: false } } as never + ) + + const scenarioEventArgs = () => trackEvent.mock.calls.at(-1)?.[2] as Record + + beforeEach(() => { + trackEvent = vi.fn().mockResolvedValue(undefined) + const cucumberFramework = Object.create(WdioCucumberTestFramework.prototype) + cucumberFramework.trackEvent = trackEvent + cucumberFramework.hasStepFailures = () => false + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => cucumberFramework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as never) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('carries preferScenarioName: true on the scenario event when set', async () => { + const service = makeService({ preferScenarioName: true }) + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + expect(scenarioEventArgs().preferScenarioName).toBe(true) + }) + + // Absent must travel as an explicit false, not undefined: the module treats the field as the + // whole opt-in, so a missing value and an opted-out value must be indistinguishable there. + it('carries preferScenarioName: false when the option is absent', async () => { + const service = makeService() + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + expect(scenarioEventArgs().preferScenarioName).toBe(false) + }) + + // The count itself stays on the service side too, because `_scenariosRanCount` is what legacy + // reads; the module keeps its own tally for the CLI flow. Both must ignore skipped scenarios. + it('does not count a skipped scenario toward the service-side tally', async () => { + const service = makeService({ preferScenarioName: true }) + await service.afterScenario({ pickle: { name: 'Skipped one' }, result: { status: 'skipped' } } as never) + + expect(service['_scenariosRanCount']).toBe(0) + }) +}) + +describe('_cucumberTestResult failure reason adjacent', () => { + const makeService = (strict: boolean) => new BrowserstackService( + { testObservability: false } as never, + [] as never, + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict } } as never + ) + + const world = (status: string, message?: string) => ({ + pickle: { name: 'CfgGate pending scenario' }, + result: message ? { status, message } : { status } + }) + + it('synthesises legacy\'s pending reason when strict makes a pending scenario fail', () => { + const result = makeService(true)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.error?.message).toBe('Some steps/hooks are pending for scenario "CfgGate pending scenario"') + }) + + it('leaves a pending scenario unfailed — and unreasoned — when strict is off', () => { + const result = makeService(false)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.skipped).toBe(true) + expect(result.error).toBeUndefined() + }) + + it('keeps the real message when the result carries one', () => { + const result = makeService(false)['_cucumberTestResult'](world('FAILED', 'AssertionError: nope') as never) + + expect(result.error?.message).toBe('AssertionError: nope') + }) + + it('falls back to Unknown Error for a message-less non-pending failure', () => { + const result = makeService(false)['_cucumberTestResult'](world('UNDEFINED') as never) + + expect(result.error?.message).toBe('Unknown Error') + }) +}) From 5fd444d1a0d57abf03bd9f6e875bdea1d004e50f Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 21:54:34 +0530 Subject: [PATCH 24/25] test(cli): fold the automateModule satellites into its main suite Merges automateModule.sessionMarking.test.ts and automateModule.preferScenarioName.test.ts into automateModule.test.ts and deletes both. All three mocked the identical six modules, so the satellites were duplicated setup. Trimmed 21 cases to 15 rather than concatenating. Dropped: the BROWSERSTACK_TURBOSCALE_INTERNAL variant and the name/status-agree check (one resolver, already covered by the PATCH/PUT pair); a second mocha guard on the zero-scenario case; an all-passed baseline; a no-rename side-effect assertion; and a no-cucumber-scenarios case that repeats the scenariosRan != 1 branch. Every discriminating pair is kept, confirmed by falsifying the trimmed suite against all three shipped fixes: === 1 -> >= 1 fails 1, reverting the resultKey collapse fails 2, re-gating registration on skipSessionName fails 1. Co-Authored-By: Claude Opus 5 --- .../automateModule.preferScenarioName.test.ts | 157 ----------- .../automateModule.sessionMarking.test.ts | 260 ------------------ .../tests/cli/modules/automateModule.test.ts | 224 ++++++++++++++- 3 files changed, 223 insertions(+), 418 deletions(-) delete mode 100644 packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts delete mode 100644 packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts deleted file mode 100644 index 84559c3..0000000 --- a/packages/browserstack-service/tests/cli/modules/automateModule.preferScenarioName.test.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { describe, expect, it, vi, beforeEach } from 'vitest' -import AutomateModule from '../../../src/cli/modules/automateModule.js' -import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' -import { _fetch as fetch } from '../../../src/fetchWrapper.js' -import type { Options } from '@wdio/types' - -vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ - default: { - registerObserver: vi.fn(), - setState: vi.fn(), - getState: vi.fn(), - getTrackedInstance: vi.fn() - } -})) - -vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ - default: { - getTrackedInstance: vi.fn(), - getState: vi.fn(), - getDriver: vi.fn() - } -})) - -vi.mock('../../../src/cli/cliLogger.js', () => ({ - BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } -})) - -vi.mock('../../../src/util.js', () => ({ - isBrowserstackSession: vi.fn(() => true), - isTrue: vi.fn((value) => (value + '').toLowerCase() === 'true'), - hasAppCap: vi.fn(() => false) -})) - -vi.mock('../../../src/instrumentation/performance/performance-tester.js', () => ({ - default: { measureWrapper: vi.fn((event, fn) => fn) } -})) - -vi.mock('../../../src/fetchWrapper.js', () => ({ _fetch: vi.fn() })) - -function newModule(config: Record = {}) { - const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) - mod.config = { - testContextOptions: { skipSessionName: false, skipSessionStatus: false }, - userName: 'testuser', - accessKey: 'testkey', - ...config - } as never - return mod -} - -function register(mod: AutomateModule, sessionId: string, lastTestName: string, seed: Record = {}) { - const sessionMap = mod['sessionMap'] as Map> - sessionMap.set(sessionId, { - lastTestName, - testResults: new Map(), - scenariosRan: 0, - ...seed - }) - return sessionMap -} - -function namesPUT() { - return vi.mocked(fetch).mock.calls.map(([, init]) => { - try { - return JSON.parse((init as { body: string }).body).name - } catch { - return undefined - } - }) -} - -describe('AutomateModule preferScenarioName', () => { - beforeEach(() => { - vi.clearAllMocks() - vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) - vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => - key === 'framework_session_id' ? 'sess-1' : ({} as never)) - vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) - }) - - // Exactly one non-skipped scenario ran and the flag is set: the session takes the scenario - // name. This is the only branch where legacy departs from the feature name. - it('renames to the scenario name when exactly one scenario ran', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Login Feature', { - scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true - }) - - await mod.onAfterExecute() - - expect(namesPUT()).toContain('Can log in') - }) - - // The `=== 1` exactness legacy applies: two scenarios keep the feature name. Reproduced, not - // widened — a `>= 1` here would rename every multi-scenario feature. - it('keeps the feature name when two scenarios ran', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Login Feature', { - scenariosRan: 2, lastScenarioName: 'Second scenario', preferScenarioName: true - }) - - await mod.onAfterExecute() - - expect(namesPUT()).not.toContain('Second scenario') - expect(namesPUT()).toContain('Login Feature') - }) - - it('keeps the feature name when the flag is absent', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Login Feature', { - scenariosRan: 1, lastScenarioName: 'Can log in' - }) - - await mod.onAfterExecute() - - expect(namesPUT()).not.toContain('Can log in') - }) - - // Legacy omits `name` from its _updateJob payload when setSessionName is false, so the - // rename must not sneak one in. - it('honours setSessionName: false and issues no rename', async () => { - const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) - register(mod, 'sess-1', 'Login Feature', { - scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true - }) - - await mod.onAfterExecute() - - expect(namesPUT()).not.toContain('Can log in') - }) - - // A skipped scenario is not a scenario that ran — legacy's counter is gated the same way, - // so a feature whose only non-skipped scenario is absent must not be renamed. - it('does not count a skipped scenario', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Login Feature', { preferScenarioName: true }) - const sessionData = mod['sessionMap'].get('sess-1')! - - expect(sessionData.scenariosRan).toBe(0) - - await mod.onAfterExecute() - - expect(namesPUT()).not.toContain('Can log in') - }) - - // mocha never reaches the counter (it is gated on isCucumberInstance), so its session name - // is whatever onBeforeTest applied — the discriminating case against cucumber above. - it('leaves a session with no cucumber scenarios untouched', async () => { - const mod = newModule() - register(mod, 'sess-1', 'Testing with BStackDemo - add product to cart', { - scenariosRan: 0, preferScenarioName: true - }) - - await mod.onAfterExecute() - - }) -}) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts deleted file mode 100644 index d480f52..0000000 --- a/packages/browserstack-service/tests/cli/modules/automateModule.sessionMarking.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import AutomateModule from '../../../src/cli/modules/automateModule.js' -import TestFramework from '../../../src/cli/frameworks/testFramework.js' -import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' -import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' -import { _fetch as fetch } from '../../../src/fetchWrapper.js' -import type { Options } from '@wdio/types' - -vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ - default: { - registerObserver: vi.fn(), - setState: vi.fn(), - getState: vi.fn(), - getTrackedInstance: vi.fn() - } -})) - -vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ - default: { - getTrackedInstance: vi.fn(), - getState: vi.fn(), - getDriver: vi.fn() - } -})) - -vi.mock('../../../src/cli/cliLogger.js', () => ({ - BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } -})) - -vi.mock('../../../src/util.js', () => ({ - isBrowserstackSession: vi.fn(() => true), - isTrue: vi.fn((value) => (value + '').toLowerCase() === 'true'), - hasAppCap: vi.fn(() => false) -})) - -vi.mock('../../../src/instrumentation/performance/performance-tester.js', () => ({ - default: { measureWrapper: vi.fn((event, fn) => fn) } -})) - -vi.mock('../../../src/fetchWrapper.js', () => ({ _fetch: vi.fn() })) - -const cucumberInstance = { framework: 'WebdriverIO-cucumber' } -const mochaInstance = { framework: 'WebdriverIO-mocha' } - -function stateFor(instance: unknown, key: string) { - if (key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) { - return (instance as { framework: string }).framework - } - if (key === TestFrameworkConstants.KEY_HOOKS_FINISHED) { - return new Map([['BEFORE_ALL', [{ [TestFrameworkConstants.KEY_HOOK_NAME]: 'BEFORE_ALL for Login' }]]]) - } - return undefined -} - -function newModule(config: Record = {}) { - const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) - mod.config = { - testContextOptions: { skipSessionName: false, skipSessionStatus: false }, - userName: 'testuser', - accessKey: 'testkey', - ...config - } as never - return mod -} - -describe('AutomateModule — session marking', () => { - beforeEach(() => { - vi.clearAllMocks() - vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) - vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => - key === 'framework_session_id' ? 'sess-1' : ({} as never)) - vi.mocked(TestFramework.getState).mockImplementation((instance, key) => stateFor(instance, key)) - vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) - delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL - }) - - afterEach(() => { - delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL - }) - - /** - * Discriminating: the SAME call produces opposite verbs and different hosts/paths - * depending only on the turboscale flag. - */ - describe('turboscale session marking routes to its own API with PATCH', () => { - it('PATCHes the turboscale endpoint when turboScale is configured', async () => { - const mod = newModule({ turboScale: true }) - await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) - - const [url, options] = vi.mocked(fetch).mock.calls[0] - expect(url).toBe('https://api.browserstack.com/automate-turboscale/v1/sessions/sess-1.json') - expect((options as { method: string }).method).toBe('PATCH') - }) - - it('PUTs the automate endpoint when turboScale is not configured', async () => { - const mod = newModule() - await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) - - const [url, options] = vi.mocked(fetch).mock.calls[0] - expect(url).toBe('https://api.browserstack.com/automate/sessions/sess-1.json') - expect((options as { method: string }).method).toBe('PUT') - }) - - it('honours the BROWSERSTACK_TURBOSCALE_INTERNAL fallback', async () => { - process.env.BROWSERSTACK_TURBOSCALE_INTERNAL = 'true' - const mod = newModule() - await mod.markSessionName('sess-1', 'a name', { user: 'u', key: 'k' }) - - const [url, options] = vi.mocked(fetch).mock.calls[0] - expect(url).toContain('/automate-turboscale/v1/sessions/') - expect((options as { method: string }).method).toBe('PATCH') - }) - - it('takes precedence over app-automate, mirroring legacy assignment order', async () => { - const mod = newModule({ turboScale: true, app: 'bs://app' }) - await mod.markSessionStatus('sess-1', 'failed', 'boom', { user: 'u', key: 'k' }) - - expect(vi.mocked(fetch).mock.calls[0][0]).toContain('/automate-turboscale/v1/sessions/') - }) - - it('names and statuses agree on verb and path', async () => { - const mod = newModule({ turboScale: true }) - await mod.markSessionName('sess-1', 'a name', { user: 'u', key: 'k' }) - await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) - - const [nameUrl, nameOpts] = vi.mocked(fetch).mock.calls[0] - const [statusUrl, statusOpts] = vi.mocked(fetch).mock.calls[1] - expect(nameUrl).toBe(statusUrl) - expect((nameOpts as { method: string }).method).toBe((statusOpts as { method: string }).method) - }) - }) - - /** - * Session verdict. Discriminating: the SAME failing build-level hook fails the session - * for cucumber and leaves mocha's verdict untouched. - */ - describe('build-level hook failures reach the session verdict', () => { - const failing = { passed: false, error: new Error('BeforeAll blew up') } - - /** Drives one scenario through TEST/POST so `testResults` carries a real scenario result. */ - const runScenario = (mod: AutomateModule, passed: boolean) => mod.onAfterTest({ - instance: cucumberInstance, - result: { error: passed ? null : new Error('step failed'), passed }, - test: { title: 'a scenario', fullName: 'Feature: a scenario' }, - suiteTitle: 'Feature' - }) - - const statusBody = () => { - const call = vi.mocked(fetch).mock.calls.find(([, o]) => - JSON.parse((o as { body: string }).body).status !== undefined) - return call ? JSON.parse((call[1] as { body: string }).body) : undefined - } - - it('marks the session failed for cucumber when a BeforeAll fails', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) - await mod.onAfterExecute() - - const statusCall = vi.mocked(fetch).mock.calls.find(([, o]) => - JSON.parse((o as { body: string }).body).status !== undefined)! - const body = JSON.parse((statusCall[1] as { body: string }).body) - expect(body.status).toBe('failed') - expect(body.reason).toBe('BeforeAll blew up') - }) - - it('names the hook in the failure reason', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) - await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: false, error: new Error('teardown') } }) - await mod.onAfterExecute() - - const body = JSON.parse((vi.mocked(fetch).mock.calls[0][1] as { body: string }).body) - expect(body.reason).toContain('BEFORE_ALL for Login') - }) - - it('leaves wdio_mocha untouched — the identical failure records nothing', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: mochaInstance, result: failing }) - await mod.onAfterExecute() - - expect(fetch).not.toHaveBeenCalled() - }) - - it('records nothing when the hook passed', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: { passed: true } }) - await mod.onAfterExecute() - - expect(fetch).not.toHaveBeenCalled() - }) - - it('keeps the session PASSED under ignoreHooksStatus once a scenario has run', async () => { - const mod = newModule() - await runScenario(mod, true) - await mod.onBuildLevelHookEnd('AFTER_ALL', { - instance: cucumberInstance, - result: failing, - ignoreHooksStatus: true - }) - await mod.onAfterExecute() - - expect(statusBody()).toEqual({ status: 'passed' }) - }) - - /** - * Zero scenarios is legacy's `!_specsRan` arm, which marks failed with no regard for the - * flag. Discriminating against the case directly above: identical hook, identical flag, - * opposite verdicts — the scenario having run is the only difference. - */ - it('marks the session FAILED under ignoreHooksStatus when no scenario ran', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { - instance: cucumberInstance, - result: failing, - ignoreHooksStatus: true - }) - await mod.onAfterExecute() - - expect(statusBody().status).toBe('failed') - }) - - it('leaves wdio_mocha unmarked on that same zero-scenario case', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { - instance: mochaInstance, - result: failing, - ignoreHooksStatus: true - }) - await mod.onAfterExecute() - - expect(fetch).not.toHaveBeenCalled() - }) - - it('does not fail a session whose scenarios all passed and whose hooks all passed', async () => { - const mod = newModule() - await runScenario(mod, true) - await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: true } }) - await mod.onAfterExecute() - - expect(statusBody()).toEqual({ status: 'passed' }) - }) - - it('respects skipSessionStatus', async () => { - const mod = newModule({ testContextOptions: { skipSessionName: false, skipSessionStatus: true } }) - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) - await mod.onAfterExecute() - - expect(fetch).not.toHaveBeenCalled() - }) - - it('does not rename the session when the hook failure is the only record', async () => { - const mod = newModule() - await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) - await mod.onAfterExecute() - - const bodies = vi.mocked(fetch).mock.calls.map(([, o]) => JSON.parse((o as { body: string }).body)) - expect(bodies.some(b => b.name !== undefined)).toBe(false) - }) - }) -}) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index e4b127e..1378920 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1,10 +1,11 @@ -import { describe, expect, it, vi, beforeEach } from 'vitest' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' import AutomateModule from '../../../src/cli/modules/automateModule.js' import TestFramework from '../../../src/cli/frameworks/testFramework.js' import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' @@ -735,3 +736,224 @@ describe('AutomateModule testResults keying (SDK-7414)', () => { expect(mochaKey).toBe(`${FEATURE} - a test`) }) }) + +const cucumberInstance = { framework: 'WebdriverIO-cucumber' } +const mochaInstance = { framework: 'WebdriverIO-mocha' } + +function stateFor(instance: unknown, key: string) { + if (key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) { + return (instance as { framework: string }).framework + } + if (key === TestFrameworkConstants.KEY_HOOKS_FINISHED) { + return new Map([['BEFORE_ALL', [{ [TestFrameworkConstants.KEY_HOOK_NAME]: 'BEFORE_ALL for Login' }]]]) + } + return undefined +} + +function newModule(config: Record = {}) { + const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) + mod.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey', + ...config + } as never + + return mod +} + +describe('AutomateModule — session marking', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(TestFramework.getState).mockImplementation((instance, key) => stateFor(instance, key)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + afterEach(() => { + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + // Discriminating: the same call yields opposite verbs and different hosts on the flag alone. + describe('turboscale routes to its own API with PATCH', () => { + it('PATCHes the turboscale endpoint when turboScale is configured', async () => { + const mod = newModule({ turboScale: true }) + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate-turboscale/v1/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PATCH') + }) + + it('PUTs the automate endpoint when turboScale is not configured', async () => { + const mod = newModule() + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PUT') + }) + + it('takes precedence over app-automate, mirroring legacy assignment order', async () => { + const mod = newModule({ turboScale: true, app: 'bs://app' }) + await mod.markSessionStatus('sess-1', 'failed', 'boom', { user: 'u', key: 'k' }) + + expect(vi.mocked(fetch).mock.calls[0][0]).toContain('/automate-turboscale/v1/sessions/') + }) + }) + + // A build-level hook has no test tied to it, so its failure reaches the verdict only here. + // Discriminating: the same failing hook fails cucumber and leaves mocha untouched. + describe('build-level hook failures reach the session verdict', () => { + const failing = { passed: false, error: new Error('BeforeAll blew up') } + + const runScenario = (mod: AutomateModule, passed: boolean) => mod.onAfterTest({ + instance: cucumberInstance, + result: { error: passed ? null : new Error('step failed'), passed }, + test: { title: 'a scenario', fullName: 'Feature: a scenario' }, + suiteTitle: 'Feature' + }) + + const statusBody = () => { + const call = vi.mocked(fetch).mock.calls.find(([, o]) => + JSON.parse((o as { body: string }).body).status !== undefined) + + return call ? JSON.parse((call[1] as { body: string }).body) : undefined + } + + it('marks the session failed for cucumber when a BeforeAll fails', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + expect(statusBody().status).toBe('failed') + expect(statusBody().reason).toBe('BeforeAll blew up') + }) + + it('names the hook in the failure reason when several fail', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: false, error: new Error('teardown') } }) + await mod.onAfterExecute() + + expect(statusBody().reason).toContain('BEFORE_ALL for Login') + }) + + it('leaves wdio_mocha untouched — the identical failure records nothing', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: mochaInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('records nothing when the hook passed', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: { passed: true } }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the session PASSED under ignoreHooksStatus once a scenario has run', async () => { + const mod = newModule() + await runScenario(mod, true) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: failing, ignoreHooksStatus: true }) + await mod.onAfterExecute() + + expect(statusBody()).toEqual({ status: 'passed' }) + }) + + // Zero scenarios is legacy's `!_specsRan` arm, which marks failed regardless of the flag. + // Discriminating against the case above: same hook, same flag, opposite verdicts. + it('marks the session FAILED under ignoreHooksStatus when no scenario ran', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing, ignoreHooksStatus: true }) + await mod.onAfterExecute() + + expect(statusBody().status).toBe('failed') + }) + + it('respects skipSessionStatus', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: false, skipSessionStatus: true } }) + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + }) +}) + +describe('AutomateModule preferScenarioName', () => { + const register = (mod: AutomateModule, lastTestName: string, seed: Record = {}) => { + (mod['sessionMap'] as Map>) + .set('sess-1', { lastTestName, testResults: new Map(), scenariosRan: 0, ...seed }) + } + + const namesPUT = () => vi.mocked(fetch).mock.calls.map(([, init]) => { + try { + return JSON.parse((init as { body: string }).body).name + } catch { + return undefined + } + }) + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + }) + + it('renames to the scenario name when exactly one scenario ran', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).toContain('Can log in') + }) + + // The `=== 1` exactness legacy applies — a `>= 1` here renames every multi-scenario feature. + it('keeps the feature name when two scenarios ran', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 2, lastScenarioName: 'Second scenario', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Second scenario') + expect(namesPUT()).toContain('Login Feature') + }) + + it('keeps the feature name when the flag is absent', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in' }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) + + it('honours setSessionName: false and issues no rename', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) + + // A skipped scenario is not a scenario that ran; legacy's counter is gated the same way. + it('does not count a skipped scenario', async () => { + const mod = newModule() + register(mod, 'Login Feature', { preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) +}) From 6a73c320d5f5de605c643b23ef5f5fecb9e9b178 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 9 Sep 2026 22:07:07 +0530 Subject: [PATCH 25/25] style(test): drop an unnecessary semicolon failing lint `npm run lint` exited 1 on no-extra-semi in service.test.ts. The leading `;` guarded against ASI before a `(`-initial line, but the preceding token is the `{` of an if-block, so there is nothing to guard. Pre-existing, but in scope for this branch: merging the per-concern suites into this file is what shifted the reported line to 2851. eslint clean over src and tests; 1276 tests pass on the post-merge tree. Co-Authored-By: Claude Opus 5 --- packages/browserstack-service/tests/service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 6ac857c..67f1d59 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2848,7 +2848,7 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => { framework.onFeatureStart('features/login.feature', feature as any) if (scenariosStarted) { // flips classifyHookType from BEFORE_ALL to AFTER_ALL - ;(framework as any).cucumberData.scenariosStarted = true + (framework as any).cucumberData.scenariosStarted = true } const trackEvent = vi.spyOn(framework, 'trackEvent').mockResolvedValue(undefined) const sendTestFrameworkEvent = vi.fn().mockResolvedValue(true)