diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 03b335f..3c6e5c0 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -47,7 +47,7 @@ const CLI_DOWNLOAD_TMP_SUFFIX = '.zip' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha'] + static CLISupportedFrameworks = ['mocha', 'cucumber'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts new file mode 100644 index 0000000..8cda05f --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -0,0 +1,645 @@ +import { v4 as uuidv4 } from 'uuid' +import path from 'node:path' + +import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' +import { BStackLogger as logger } from '../cliLogger.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' +import { getScenarioExamples, removeAnsiColors } from '../../util.js' + +import type { Frameworks } from '@wdio/types' +import type { CucumberHook, Feature, FeatureChild, ITestCaseHookParameter, Pickle, Scenario, Step } from '../../cucumber-types.js' + +/** + * `test_duration` and `bdd_meta_info` are read by the binary's WebdriverIO-cucumber module but + * have no entry in TestFrameworkConstants, which is shared with the mocha path. Kept local rather + * than appended there so this framework owns its own wire keys. + */ +const KEY_TEST_DURATION = 'test_duration' +const KEY_BDD_META_INFO = 'bdd_meta_info' + +/** + * Per-hook wire keys. The binary cannot derive any of the three from the event: a hook's scope is + * the FEATURE name while the event carries the examples-qualified SCENARIO name, and + * BEFORE_ALL/AFTER_ALL fire on an instance that has no scenario data at all. Retries and duration + * come from WDIO's hook result, which only this side sees. + */ +const KEY_HOOK_SCOPE = 'hook_scope' +const KEY_HOOK_RETRIES = 'hook_retries' +const KEY_HOOK_DURATION = 'hook_duration' + +/** + * Marks a TEST/POST event as a BEFORE_ALL cascade row rather than a real scenario finish, so the + * binary's cucumber module emits `TestRunSkipped` — legacy's wire event for this case — instead of + * `TestRunFinished`. The state->event mapping in the WDIO language index is shared with mocha and + * is deliberately not touched. + */ +const KEY_TEST_SKIPPED_CASCADE = 'test_skipped_cascade' + +type CucumberHookType = 'BEFORE_ALL' | 'AFTER_ALL' | 'BEFORE_EACH' | 'AFTER_EACH' + +const HOOK_STATES: Record = { + BEFORE_ALL: TestFrameworkState.BEFORE_ALL, + AFTER_ALL: TestFrameworkState.AFTER_ALL, + BEFORE_EACH: TestFrameworkState.BEFORE_EACH, + AFTER_EACH: TestFrameworkState.AFTER_EACH, +} + +interface StepMeta { + id: string + text: string + keyword: string + started_at?: string + finished_at?: string + result?: string + duration?: unknown + failure?: string +} + +/** + * File-path pair sent with every scenario/hook event. + * + * The ABSOLUTE feature path is deliberate: the binary re-bases it itself + * (`path.relative(session.pathProject, …)` for `file_name`/`location` and against the git root for + * `vc_filepath`). Legacy reported both pre-relativised — sending that shape here made both fields + * come out wrong, and sending `undefined` threw inside the binary and dropped the event (SDK-7233). + */ +const resolveFeatureFilePaths = (featurePath: string | undefined) => ({ + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: featurePath, + [TestFrameworkConstants.KEY_TEST_LOCATION]: featurePath + ? path.relative(process.cwd(), featurePath) + : undefined, +}) + +/** + * CLI test framework for `framework: 'cucumber'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: WDIO does not call + * `beforeTest`/`afterTest`/titled hooks for cucumber at all, so mocha's INIT_TEST/TEST/hook + * boundary semantics have no source here and borrowing them would report cucumber through the + * wrong runner's event model. + * + * Cucumber's unit of work is the scenario, and every `cli/modules/*` observer subscribes to + * `TestFrameworkState.TEST` — so a scenario raises TEST/PRE at `beforeScenario` and TEST/POST at + * `afterScenario`, and the module set works unchanged. + */ +export default class WdioCucumberTestFramework extends TestFramework { + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + + /** + * The bookkeeping `classifyHookType()` derives hook types from. A cucumber hook invocation + * carries no title, and `BeforeAll`/`AfterAll` pass no hook object at all, so classification + * is state-machine derived — `util.ts → getHookType()` is Mocha-title-shaped and can only + * return 'unknown' or throw here. + */ + private cucumberData: { + feature?: Feature + uri?: string + scenario?: Pickle + scenariosStarted: boolean + stepsStarted: boolean + stepDepth: number + } = { scenariosStarted: false, stepsStarted: false, stepDepth: 0 } + + /** + * Steps accumulated for the scenario in flight. Re-allocated (never cleared in place) at each + * scenario start so a payload already built from the previous scenario can never observe the + * next one's steps. + */ + private scenarioSteps: StepMeta[] = [] + + /** The hook currently open on this worker — started and not yet finished. */ + private openHook: { key: string, hookId: string } | null = null + + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioCucumberTestFramework: constructed') + } + + /** + * Feature bookkeeping. Raises no framework state — cucumber has no feature-level wire event, + * and `beforeSuite`/`afterSuite` are not part of its WDIO surface. + */ + onFeatureStart(uri: string, feature: Feature) { + logger.debug(`onFeatureStart: uri=${uri} feature=${feature?.name}`) + this.cucumberData.scenariosStarted = false + this.cucumberData.feature = feature + this.cucumberData.uri = uri + } + + /** + * Step bookkeeping. Steps travel inside the scenario payload's BDD meta, never as their own + * event — `TestFrameworkState.STEP` has zero producers anywhere in this SDK. + */ + onStepStart(step: Frameworks.PickleStep) { + this.cucumberData.stepsStarted = true + this.cucumberData.stepDepth++ + this.scenarioSteps.push({ + id: step.id, + text: step.text, + keyword: step.keyword, + started_at: (new Date()).toISOString(), + }) + } + + onStepEnd(step: Frameworks.PickleStep, result: Frameworks.PickleResult) { + this.cucumberData.stepDepth = Math.max(0, this.cucumberData.stepDepth - 1) + const stepMeta = this.scenarioSteps.find(item => item.id === step.id) + if (!stepMeta) { + return + } + stepMeta.finished_at = (new Date()).toISOString() + stepMeta.result = result.passed ? 'PASSED' : 'FAILED' + stepMeta.duration = result.duration + if (result.error) { + stepMeta.failure = removeAnsiColors(result.error) + } + } + + /** + * Classify a cucumber hook invocation from the bookkeeping state. + * + * Returns null for a step-scoped hook (`BeforeStep`/`AfterStep`), which is never reported — + * reporting it would change the dashboard hook count, which is a feature and not parity. + */ + classifyHookType(test: CucumberHook | undefined): CucumberHookType | null { + if (!test) { + return this.cucumberData.scenariosStarted ? 'AFTER_ALL' : 'BEFORE_ALL' + } + if (!this.cucumberData.stepsStarted) { + return 'BEFORE_EACH' + } + if (this.cucumberData.stepDepth > 0) { + return null + } + return 'AFTER_EACH' + } + + /** The TestFrameworkState a cucumber hook invocation maps to, or null when unreported. */ + classifyHookState(test: CucumberHook | undefined): State | null { + const hookType = this.classifyHookType(test) + return hookType ? HOOK_STATES[hookType] : null + } + + /** + * ` for ` — the separator is a literal ' ' + 'for' + ' '. + */ + private hookName(hookType: CucumberHookType): string { + switch (hookType) { + case 'BEFORE_EACH': + case 'AFTER_EACH': + return `${hookType} for ${this.cucumberData.scenario?.name}` + case 'BEFORE_ALL': + case 'AFTER_ALL': + return `${hookType} for ${this.cucumberData.feature?.name}` + } + } + + private featurePath(): string | undefined { + const uri = this.cucumberData.uri + return uri ? path.resolve(process.cwd(), uri) : undefined + } + + /** + * The feature path as the bdd meta blob wants it — cwd-relative, matching legacy. + * + * `cucumberData.uri` cannot be forwarded raw: WDIO hands `beforeFeature` an ABSOLUTE uri, so + * the raw value carries the developer's home directory. Legacy never sees that value — it + * reads the cucumber world's `gherkinDocument.uri`, which is cwd-relative. + */ + private featureUriForMeta(): string | undefined { + const absolute = this.featurePath() + return absolute ? path.relative(process.cwd(), absolute) : undefined + } + + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) + await super.trackEvent(testFrameworkState, hookState, args) + + const instance = this.resolveInstance(testFrameworkState, hookState) + if (!instance) { + // Console output emitted before the first scenario (or before BeforeAll) has nothing + // to attach to. The legacy path drops it just as silently, so this is expected rather + // than a failure. + const detail = `trackEvent: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}` + if (testFrameworkState === TestFrameworkState.LOG) { + logger.debug(detail) + } else { + logger.error(detail) + } + return + } + + const shortState = testFrameworkState.toString().split('.')[1] + const isHook = CLIUtils.matchHookRegex(shortState) + + try { + if (isHook && hookState === HookState.PRE) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_HOOK_ID]: uuidv4(), + }) + } + + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + this.loadScenarioData(instance, args.world as ITestCaseHookParameter) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: new Date().toISOString(), + }) + } else if (hookState === HookState.POST) { + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: new Date().toISOString(), + }) + this.loadScenarioResult(instance, args) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.loadLogEntry(instance, args.logEntry as Record) + } + + if (isHook) { + this.trackHookEvents(instance, shortState, hookState, args) + } + } catch (error) { + logger.error(`trackEvent: Error in tracking events: ${error} hookState=${hookState} testFrameworkState=${testFrameworkState}`) + } + + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + /** + * One instance per scenario, keyed by the worker (`pid:threadId`) — the single lookup every + * `cli/modules/*` reads, via `TestFramework.getTrackedInstance()`. WDIO forks one worker + * process per spec file, so the worker key is the scenario's execution context; a thread id + * alone would collide once threads are reused. + */ + private resolveInstance(testFrameworkState: State, hookState: State): TestFrameworkInstance | null { + let instance = TestFramework.getTrackedInstance() + const isHook = CLIUtils.matchHookRegex(testFrameworkState.toString().split('.')[1]) + + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + // Every scenario is its own test. BEFORE_EACH hooks fire after beforeScenario, so + // they land on the instance minted here and never on the previous scenario's. + this.trackWdioCucumberInstance(testFrameworkState) + } else if (isHook && hookState === HookState.PRE && !instance) { + // BEFORE_ALL runs before any scenario exists, so it has no instance to attach to. + this.trackWdioCucumberInstance(testFrameworkState) + } + + instance = TestFramework.getTrackedInstance() + if (!instance) { + logger.debug(`resolveInstance: no instance for testFrameworkState=${testFrameworkState} hookState=${hookState}`) + return null + } + this.updateInstanceState(instance, testFrameworkState, hookState) + return instance + } + + private trackWdioCucumberInstance(testFrameworkState: State) { + const target = CLIUtils.getCurrentInstanceName() + const trackedContext = TrackedInstance.createContext(target) + + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + testFrameworkState, + HookState.NONE + ) + + const frameworkName = this.getTestFrameworks()[0] + const testUuid = uuidv4() + + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: testUuid, + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + }) + + // Read by the A11y and App-A11y scan paths. + process.env[TEST_ANALYTICS_ID] = testUuid + + TestFramework.setTrackedInstance(trackedContext, instance) + logger.debug(`trackWdioCucumberInstance: contextId=${trackedContext.getId()} target=${target} testUuid=${testUuid}`) + } + + /** Scenario identity. Each field matches what the legacy flow sent; the asymmetries are deliberate. */ + private loadScenarioData(instance: TestFrameworkInstance, world: ITestCaseHookParameter) { + if (!world?.pickle) { + logger.error('loadScenarioData: no pickle on the world object; scenario identity will be empty') + return + } + const pickle = world.pickle + const feature = world.gherkinDocument?.feature + this.cucumberData.scenario = pickle + this.cucumberData.scenariosStarted = true + this.cucumberData.stepsStarted = false + this.cucumberData.stepDepth = 0 + // Fresh array, never a clear-in-place: the previous scenario's payload must not be able to + // observe this scenario's steps through a retained reference. + this.scenarioSteps = [] + + const examples = getScenarioExamples(world) + // Exactly one space before '(' and ', ' between cells — asserted character-for-character. + const qualifiedName = examples + ? pickle.name + ' (' + examples.join(', ') + ')' + : pickle.name + const featurePath = this.featurePath() + + instance.updateMultipleEntries({ + // The RAW pickle name, deliberately WITHOUT the examples qualifier that `name`/`scope` + // carry. The binary maps this to `identifier`; collapsing the two would change how the + // dashboard groups Scenario Outline rows. + [TestFrameworkConstants.KEY_TEST_ID]: pickle.name, + [TestFrameworkConstants.KEY_TEST_NAME]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPE]: qualifiedName, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature?.name || ''], + // Step source is never reported for cucumber. + [TestFrameworkConstants.KEY_TEST_CODE]: null, + // Gherkin tag text INCLUDING the leading '@', source order, no dedupe, no lowercasing. + // `.map` allocates a new array — the pickle's own tag collection is never mutated. + [TestFrameworkConstants.KEY_TEST_TAGS]: pickle.tags.map(({ name }: { name: string }) => name), + ...resolveFeatureFilePaths(featurePath), + [KEY_BDD_META_INFO]: this.buildBddMetaInfo(pickle, feature, examples), + }) + } + + /** + * `feature.path` is cwd-relative, matching legacy (`insights-handler` builds + * `feature = { path: gherkinDocument.uri, … }` off the cucumber world). It is NOT the absolute + * path used for `test_file_path`: the binary re-bases that one, but never touches this blob, so + * an absolute value here reaches the dashboard as-is, home directory and all. + */ + private buildBddMetaInfo(pickle: Pickle, feature: Feature | undefined, examples: string[] | undefined) { + return { + feature: { + name: feature?.name, + path: this.featureUriForMeta(), + description: feature?.description, + }, + scenario: { name: pickle.name }, + steps: this.scenarioSteps.map(step => ({ ...step })), + examples: examples ?? [], + } + } + + /** + * Scenario result, loaded at the real "scenario ends" state. + * + * KEY_TEST_RESULT_AT is load-bearing, not decoration: `testHubModule → onAllTestEvents()` + * treats a TEST/POST without it as result-less, marks the test deferred, and then waits for a + * `LOG_REPORT` POST to recover it — a state cucumber never emits. + */ + private loadScenarioResult(instance: TestFrameworkInstance, args: Record) { + const world = args.world as ITestCaseHookParameter | undefined + const pickle = world?.pickle ?? this.cucumberData.scenario + const feature = world?.gherkinDocument?.feature ?? this.cucumberData.feature + + const updates: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + + if (pickle) { + updates[KEY_BDD_META_INFO] = this.buildBddMetaInfo(pickle, feature, getScenarioExamples(world as ITestCaseHookParameter)) + } + + const result = world?.result + if (result) { + let testResult = result.status.toLowerCase() + if (testResult !== 'passed' && testResult !== 'failed') { + // UNKNOWN / UNDEFINED / AMBIGUOUS / PENDING / SKIPPED all collapse to skipped. + testResult = 'skipped' + } + + // A scenario that failed only because of a hook is reported as passed when the user + // has declared ignoreHooksStatus. The same flag independently gates the session-status + // accumulation in service.ts — two sites, one flag. + if (args.ignoreHooksStatus === true && testResult === 'failed' && !this.hasStepFailures()) { + testResult = 'passed' + } + + updates[TestFrameworkConstants.KEY_TEST_RESULT] = testResult + // Cucumber's own protobuf Duration, NOT an ended_at - started_at delta. + updates[KEY_TEST_DURATION] = result.duration + ? result.duration.seconds * 1000 + result.duration.nanos / 1000000 + : undefined + + if (testResult === 'failed') { + const message = result.message + // A ONE-element backtrace. The mocha path sends two entries (message + stack); + // cucumber's result carries a single combined message. + updates[TestFrameworkConstants.KEY_TEST_FAILURE] = [ + { backtrace: [message ? removeAnsiColors(message) : 'unknown'] } + ] + updates[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = message ? removeAnsiColors(message) : message + if (message) { + updates[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = message.match(/AssertionError/) + ? 'AssertionError' + : 'UnhandledError' + } + } + } + + instance.updateMultipleEntries(updates) + this.cucumberData.scenario = undefined + } + + /** + * One detached instance per scenario the feature never ran, for the BEFORE_ALL cascade + * (Rule-nested included). Detached is load-bearing: not registered via `setTrackedInstance`, + * so the live instance and `process.env[TEST_ANALYTICS_ID]` are untouched, and the caller + * sends each straight to TestHub — routing them through the observers would rename the + * session, stop accessibility and run a Percy teardown per row. No tags: legacy's cascade + * payload has no `world`. + */ + buildSkippedScenarioInstances(): TestFrameworkInstance[] { + const feature = this.cucumberData.feature + if (!feature) { + logger.debug('buildSkippedScenarioInstances: no feature recorded; nothing to cascade') + return [] + } + + const scenarios: Scenario[] = [] + for (const child of (feature.children || []) as FeatureChild[]) { + if (child.rule) { + for (const ruleChild of (child.rule.children || [])) { + if (ruleChild.scenario) { + scenarios.push(ruleChild.scenario) + } + } + } else if (child.scenario) { + scenarios.push(child.scenario) + } + } + + const featurePath = this.featurePath() + return scenarios.map(scenario => this.buildSkippedScenarioInstance(scenario, feature, featurePath)) + } + + private buildSkippedScenarioInstance(scenario: Scenario, feature: Feature, featurePath: string | undefined): TestFrameworkInstance { + const now = new Date().toISOString() + const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.TEST, + HookState.POST + ) + + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + // A cascade row is identified by the RAW scenario name on both fields: the feature + // never ran, so no Examples row was ever selected and there is nothing to qualify. + [TestFrameworkConstants.KEY_TEST_ID]: scenario.name, + [TestFrameworkConstants.KEY_TEST_NAME]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPE]: scenario.name, + [TestFrameworkConstants.KEY_TEST_SCOPES]: [feature.name || ''], + [TestFrameworkConstants.KEY_TEST_CODE]: null, + [TestFrameworkConstants.KEY_TEST_RESULT]: 'skipped', + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: now, + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: now, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, + ...resolveFeatureFilePaths(featurePath), + [KEY_TEST_SKIPPED_CASCADE]: true, + [KEY_BDD_META_INFO]: { + feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, + scenario: { name: scenario.name }, + steps: (scenario.steps || []).map((step: Step) => ({ + id: step.id, + text: step.text, + keyword: step.keyword, + result: 'skipped', + })), + examples: [], + }, + }) + + return instance + } + + /** + * Whether the scenario in flight failed in a STEP, as opposed to failing only in a hook. + * Public because `ignoreHooksStatus` has two surfaces: the o11y result below, and the Automate + * session status, which `service.afterScenario()` derives from the same answer. + */ + hasStepFailures(): boolean { + return this.scenarioSteps.some(step => step.result === 'FAILED') + } + + /** + * Route a console log to the row it belongs on: the open hook's uuid while a hook is in + * flight and unfinished, otherwise the scenario's uuid. + * + * The record itself always goes into KEY_TEST_LOGS, because the send path + * (`testHubModule → onAllTestEvents`, unchanged) collects test logs plus the last FINISHED + * hook's logs — a record parked on a still-open hook's own array would never be picked up. + * The routing is carried by KEY_HOOK_ID on the record, which `sendLogCreatedEvent` reads in + * preference to the test uuid. + */ + private loadLogEntry(instance: TestFrameworkInstance, logEntry: Record) { + if (!logEntry) { + return + } + const { level, message, timestamp } = logEntry + const logRecord: Record = { + kind: TestFrameworkConstants.KIND_LOG, + message: Buffer.from(message as string), + level, + timestamp, + } + + if (this.openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = this.openHook.hookId + } + + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_LOGS]: entries, + }) + } + + /** + * Hook lifecycle. Entries are keyed by the short state name, matching how the binary looks + * them up via `event.test_hooks_started[request.testFrameworkState]`. + * + * A finish with no recorded start is dropped rather than emitted — an unmatched + * HookRunFinished orphans a hook row the backend cannot pair. + */ + private trackHookEvents(instance: TestFrameworkInstance, key: string, hookState: State, args: Record) { + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + if (!hooksStarted.has(key)) { + hooksStarted.set(key, []) + } + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map + if (!hooksFinished.has(key)) { + hooksFinished.set(key, []) + } + + const updates: Record = { + [TestFrameworkConstants.KEY_HOOKS_STARTED]: hooksStarted, + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: hooksFinished, + } + const featurePath = this.featurePath() + + if (hookState === HookState.PRE) { + const hookId = (TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) || '') as string + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: new Date().toISOString(), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: this.hookName(key as CucumberHookType), + [KEY_HOOK_SCOPE]: this.cucumberData.feature?.name, + ...resolveFeatureFilePaths(featurePath), + } + hooksStarted.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_STARTED] = key + this.openHook = { key, hookId } + logger.debug(`trackHookEvents: hook started key=${key} name=${hook[TestFrameworkConstants.KEY_HOOK_NAME]}`) + } else if (hookState === HookState.POST) { + const hooksList = hooksStarted.get(key) || [] + if (hooksList.length === 0) { + logger.warn(`trackHookEvents: dropping hook finish for '${key}' — no matching start was recorded`) + this.openHook = null + return + } + + const hook = hooksList.pop() as Record + const hookResult = args.result as Frameworks.TestResult | undefined + // passed / failed only — no 'skipped' arm, unlike the scenario result path. + if (hookResult) { + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = hookResult.passed ? 'passed' : 'failed' + // WDIO reports hook duration in plain ms on the result, not as cucumber's protobuf + // Duration — legacy sends it through unchanged and so does this. + hook[KEY_HOOK_RETRIES] = hookResult.retries + hook[KEY_HOOK_DURATION] = hookResult.duration + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = new Date().toISOString() + hooksFinished.get(key)?.push(hook) + updates[WdioCucumberTestFramework.KEY_HOOK_LAST_FINISHED] = key + this.openHook = null + logger.debug(`trackHookEvents: hook finished key=${key} result=${hook[TestFrameworkConstants.KEY_HOOK_RESULT]}`) + } + + instance.updateMultipleEntries(updates) + } +} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index b1b3cc3..444ec6d 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -16,6 +16,7 @@ import { BROWSERSTACK_ACCESSIBILITY, BROWSERSTACK_OBSERVABILITY, BROWSERSTACK_TE import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -47,7 +48,7 @@ export class BrowserstackCLI { modulesLoaded = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|null = null + testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -548,7 +549,15 @@ export class BrowserstackCLI { const testFrameworkDetail = CLIUtils.getTestFrameworkDetail() if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-mocha') { this.testFramework = new WdioMochaTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return + } + if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-cucumber') { + this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return } + // An unmatched name leaves testFramework null, and every CLI event then no-ops with no + // error of any kind. Name it so the silence is diagnosable. + this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) } /** diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 6c38c7e..edcb1b3 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -89,10 +89,14 @@ export default class AccessibilityModule extends BaseModule { if (!this.accessibility) { return } - // Open the scan gate for the hook window so DOM-changing commands issued inside - // before/beforeEach/afterEach/after hooks trigger scans (web per-command path). The - // following onBeforeTest re-computes the per-test gate, so this only affects the hook. - if (this.autoScanning && sessionId !== undefined && sessionId !== null) { + // Open the scan gate for the hook window, so commands issued inside hooks are scanned. + // Mocha-only, as legacy gates the identical write (`this._framework === 'mocha'`): it + // relies on the following onBeforeTest re-computing the gate, which holds only where + // beforeEach precedes beforeTest. Cucumber inverts that ordering, so there the write + // would land last and force the gate permanently open. + const frameworkName = String(TestFramework.getState(testInstance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + const isMocha = frameworkName.toLowerCase().includes('mocha') + if (this.autoScanning && isMocha && sessionId) { this.accessibilityMap.set(sessionId, true) } } catch (error) { @@ -343,7 +347,13 @@ export default class AccessibilityModule extends BaseModule { const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) const accessibilityOptions = this.config.accessibilityOptions - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined) && this.accessibility + // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg + // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags + // against the test title instead and so silently scans every scenario. `args.world` is + // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg + // behaviour — both extra args arrive undefined/false and the tag branch is not taken. + const world = args.world as { [key: string]: unknown } | undefined + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index fcdc336..a62efb5 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -27,6 +27,9 @@ interface SessionData { lastTestName: string appliedName?: string // last name successfully PUT for this session, for de-duping testResults: Map // testName -> TestResult + scenariosRan: number // non-skipped cucumber scenarios, for preferScenarioName + lastScenarioName?: string + preferScenarioName?: boolean } export default class AutomateModule extends BaseModule { @@ -46,6 +49,10 @@ export default class AutomateModule extends BaseModule { TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) TestFramework.registerObserver(AutomationFrameworkState.EXECUTE, HookState.POST, this.onAfterExecute.bind(this)) + // Build-level hooks carry no scenario result, so they reach the session verdict only + // through their own state. See onBuildLevelHookEnd — cucumber-gated inside the handler. + TestFramework.registerObserver(TestFrameworkState.BEFORE_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'BEFORE_ALL')) + TestFramework.registerObserver(TestFrameworkState.AFTER_ALL, HookState.POST, this.onBuildLevelHookEnd.bind(this, 'AFTER_ALL')) } getModuleName(): string { @@ -63,8 +70,20 @@ export default class AutomateModule extends BaseModule { const suiteTitle = args.suiteTitle as string const testContextOptions = this.config.testContextOptions as TestContextOptions - if (testContextOptions.skipSessionName || !isBrowserstackSession(browser)) { + if (!isBrowserstackSession(browser)) { + return + } + + // `setSessionName: false` suppresses the NAME, not the registration. The session still has + // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it + // either way — its `after()` status block gates on setSessionStatus alone. Registering with + // an empty lastTestName is safe: flushSessionName early-returns both on the flag and on an + // empty name, so no name can be sent from here. + if (testContextOptions.skipSessionName) { this.logger.info('Skipping session name update as per configuration') + if (sessionId && !this.sessionMap.has(sessionId)) { + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) + } return } @@ -88,7 +107,8 @@ export default class AutomateModule extends BaseModule { if (!existingSession) { this.sessionMap.set(sessionId, { lastTestName: name, - testResults: new Map() + testResults: new Map(), + scenariosRan: 0 }) } else { existingSession.lastTestName = name @@ -185,12 +205,12 @@ export default class AutomateModule extends BaseModule { // (service.onReload has already pointed KEY_FRAMEWORK_SESSION_ID at it) and adopt it while // it is still open. // - // Deliberately gated on skipSessionName, NOT skipSessionStatus: naming and status are - // independent options, so a `setSessionStatus: false` user must still get the name repair, - // and a `setSessionName: false` user must not be pulled into sessionMap — that would hand - // onAfterExecute a session to status-mark where it previously had none. - if (sessionId && !testContextOptions.skipSessionName && !this.sessionMap.has(sessionId)) { - this.sessionMap.set(sessionId, { lastTestName: name, testResults: new Map() }) + // Registration is independent of both opt-outs: `setSessionStatus: false` must still get the + // name repair, and `setSessionName: false` must still be status-marked. The name is what the + // flag suppresses, so a skipped-name session registers with an empty lastTestName. + if (sessionId && !this.sessionMap.has(sessionId)) { + const repairName = testContextOptions.skipSessionName ? '' : name + this.sessionMap.set(sessionId, { lastTestName: repairName, testResults: new Map(), scenariosRan: 0 }) } // No-op for the steady state: when no mid-test reload happened, onBeforeTest already // applied this exact name and `appliedName` de-dupes it away — no extra API call. @@ -209,7 +229,18 @@ export default class AutomateModule extends BaseModule { const sessionData = this.sessionMap.get(sessionId) if (sessionData) { - sessionData.testResults.set(name, testResult) + // `name` is the session NAME, which for cucumber is the Feature title and therefore + // shared by every scenario in the file — keying the results map on it collapses N + // scenarios into one last-write-wins entry, so a feature whose last scenario passes + // reports a passed session however many earlier ones failed. Mocha leaves `fullName` + // undefined, so the key is unchanged there. + const resultKey = (test && test.fullName) ? String(test.fullName) : name + sessionData.testResults.set(resultKey, testResult) + if (!skipped && this.isCucumberInstance(instace)) { + sessionData.scenariosRan++ + sessionData.lastScenarioName = testTitle + sessionData.preferScenarioName = isTrue(args.preferScenarioName) + } this.sessionMap.set(sessionId, sessionData) } @@ -217,6 +248,88 @@ export default class AutomateModule extends BaseModule { TestFramework.setState(instace, TestFrameworkConstants.KEY_AUTOMATE_SESSION_REASON, reason) } + /** + * A `BeforeAll` / `AfterAll` failure produces no scenario result, so it can never enter the + * per-test `testResults` map that onAfterExecute aggregates — a run whose BeforeAll blew up + * reports its session as PASSED. Legacy pushed the hook error into `_failReasons` and + * `after()` marked the session failed; that whole accumulation is gated + * `setSessionStatus && !BrowserstackCLI.isRunning()`, so it is dead while the binary is up. + * + * Cucumber-gated deliberately. `wdio_mocha` has the identical latent shape on this flow, but + * legacy mocha behaved the same way, so repairing it here would be an unrequested behaviour + * change to the one framework already working on the CLI flow. + */ + async onBuildLevelHookEnd(hookKey: string, args: Record) { + try { + const instance = (args?.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!instance || !this.isCucumberInstance(instance)) { + return + } + + const result = args?.result as { passed?: boolean, error?: Error } | undefined + if (!result || result.passed) { + return + } + + const testContextOptions = this.config.testContextOptions as TestContextOptions + if (testContextOptions?.skipSessionStatus) { + return + } + + const autoInstance = AutomationFramework.getTrackedInstance() + const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) + if (!sessionId) { + this.logger.debug(`onBuildLevelHookEnd: no session id resolved for ${hookKey}; nothing to mark`) + return + } + + const sessionData = this.sessionMap.get(sessionId) + // Keyed on the absence of scenario results, never on the flag: legacy's + // `ignoreHooksStatus && this._specsRan` arm needs BOTH, and with no scenario recorded it + // falls through to marking `failed` regardless of the flag. The count is final here — a + // `BeforeAll` failure aborts the run, and by `AfterAll` every scenario has been recorded. + const specsRan = (sessionData?.testResults.size ?? 0) > 0 + if (specsRan && isTrue(args?.ignoreHooksStatus)) { + this.logger.debug(`onBuildLevelHookEnd: ${hookKey} failed but ignoreHooksStatus is set; not failing the session`) + return + } + + if (!sessionData) { + // A BeforeAll can fail before any scenario ran, so the session may not be + // registered yet. `lastTestName` stays empty on purpose — flushSessionName + // early-returns on it, so registering here cannot rename the session. + this.sessionMap.set(sessionId, { lastTestName: '', testResults: new Map(), scenariosRan: 0 }) + } + + const name = this.resolveHookName(instance, hookKey) + this.sessionMap.get(sessionId)!.testResults.set(name, { + testName: name, + status: 'failed', + reason: (result.error && result.error.message) || 'Hook failed' + }) + this.logger.info(`onBuildLevelHookEnd: recorded ${hookKey} failure against session ${sessionId}`) + } catch (error) { + this.logger.error(`Exception in automate onBuildLevelHookEnd: ${error}`) + } + } + + private isCucumberInstance(instance: TestFrameworkInstance): boolean { + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + return frameworkName.toLowerCase().includes('cucumber') + } + + /** The hook's reported name (`BEFORE_ALL for `), so the session reason names the hook. */ + private resolveHookName(instance: TestFrameworkInstance, hookKey: string): string { + try { + const finished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> | undefined + const hooks = finished?.get(hookKey) + const hookName = hooks?.length ? hooks[hooks.length - 1][TestFrameworkConstants.KEY_HOOK_NAME] : undefined + return (hookName as string) || hookKey + } catch { + return hookKey + } + } + async onAfterExecute() { this.logger.debug('onAfterExecute: inside automate module after execute hook!') @@ -242,6 +355,15 @@ export default class AutomateModule extends BaseModule { } } + // preferScenarioName: cucumber names the session after the FEATURE, but when + // exactly one non-skipped scenario ran the user can ask for that scenario's name + // instead. Only decidable here — "exactly one" is not knowable while tests are + // still arriving. `skipSessionName` still wins, inside flushSessionName. + if (sessionData.preferScenarioName && sessionData.scenariosRan === 1 && sessionData.lastScenarioName) { + sessionData.lastTestName = sessionData.lastScenarioName + this.sessionMap.set(sessionId, sessionData) + } + // Final sweep — a no-op for sessions already named per-test in onBeforeTest. await this.flushSessionName(sessionId) @@ -266,29 +388,64 @@ export default class AutomateModule extends BaseModule { return this.hasAppCapInFrameworkState() } + // The binary echoes the parsed `turboScale` flag back on the session config; the env var is + // written unconditionally by the service constructor in this same worker process, so it stands + // in when a config shape predates the flag. + private isTurboScale(): boolean { + return isTrue(this.config.turboScale) || isTrue(process.env.BROWSERSTACK_TURBOSCALE_INTERNAL) + } + + /** + * Resolve the REST endpoint a session marker must hit. + * + * Turboscale is not a variant of Automate here — it is a different API on a different path + * with a different VERB (PATCH, not PUT). The legacy path expressed this through + * `_sessionBaseUrl` + `_update()`, both gated `!BrowserstackCLI.isRunning()`, so neither + * survives onto the CLI flow and nothing in the binary compensates. + * + * Precedence mirrors legacy's assignment order in `beforeSession()`: the turboscale base URL + * is assigned AFTER the app-automate one, so a turboscale grid wins even with an app cap set. + * + * Single resolver for both markers deliberately: naming and status previously duplicated the + * ternary, which is how the two can drift apart. + */ + private resolveSessionApi(sessionId: string): { url: string, method: 'PUT' | 'PATCH', product: string } { + if (this.isTurboScale()) { + return { + url: `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate-turboscale/v1/sessions/${sessionId}.json`, + method: 'PATCH', + product: 'Automate TurboScale' + } + } + if (this.isAppAutomate()) { + return { + url: `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json`, + method: 'PUT', + product: 'App Automate' + } + } + return { + url: `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json`, + method: 'PUT', + product: 'Automate' + } + } + async markSessionName(sessionId: string, sessionName: string, config: { user: string; key: string; }): Promise { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_NAME, async (sessionId: string, sessionName: string, config: { user: string; key: string; }) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') - const isAppAutomate = this.isAppAutomate() - if (isAppAutomate) { - this.logger.info('Marking session name for App Automate') - } else { - this.logger.info('Marking session name for Automate') - } - - const sessionNameApiUrl = isAppAutomate - ? `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json` - : `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json` + const { url: sessionNameApiUrl, method, product } = this.resolveSessionApi(sessionId) + this.logger.info(`Marking session name for ${product}`) const requestBody = { name: sessionName } const options = { - method: 'PUT', + method, headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' @@ -312,16 +469,8 @@ export default class AutomateModule extends BaseModule { async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') - const isAppAutomate = this.isAppAutomate() - if (isAppAutomate) { - this.logger.info('Marking session status for App Automate') - } else { - this.logger.info('Marking session status for Automate') - } - - const sessionStatusApiUrl = isAppAutomate - ? `${APIUtils.BROWSERSTACK_AA_API_URL}/app-automate/sessions/${sessionId}.json` - : `${APIUtils.BROWSERSTACK_AUTOMATE_API_URL}/automate/sessions/${sessionId}.json` + const { url: sessionStatusApiUrl, method, product } = this.resolveSessionApi(sessionId) + this.logger.info(`Marking session status for ${product}`) const body = { status: sessionStatus, @@ -329,7 +478,7 @@ export default class AutomateModule extends BaseModule { } const options = { - method: 'PUT', + method, headers: { Authorization: `Basic ${auth}`, 'Content-Type': 'application/json' diff --git a/packages/browserstack-service/src/cli/modules/customTagsModule.ts b/packages/browserstack-service/src/cli/modules/customTagsModule.ts index ce24ae6..b70b679 100644 --- a/packages/browserstack-service/src/cli/modules/customTagsModule.ts +++ b/packages/browserstack-service/src/cli/modules/customTagsModule.ts @@ -11,6 +11,7 @@ import { TestFrameworkState } from '../states/testFrameworkState.js' import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' import { CLIUtils } from '../cliUtils.js' import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' +import { BrowserstackCLI } from '../index.js' import { mergeIntoTags, parseCommaSeparatedValues, getCurrentMochaHookWindow } from '../../customTags.js' import type { CustomMetadata } from '../../customTags.js' @@ -54,6 +55,10 @@ export default class CustomTagsModule extends BaseModule { return CustomTagsModule.MODULE_NAME } + private isMochaFramework(): boolean { + return BrowserstackCLI.getInstance().getTestFramework() instanceof WdioMochaTestFramework + } + async onBeforeExecute() { try { const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() @@ -70,6 +75,16 @@ export default class CustomTagsModule extends BaseModule { (browser as WebdriverIO.Browser).setCustomTags = async (key: string, value: string): Promise => { try { + // Mocha-only, matching the legacy handler (custom-tags-handler, which warns + // and no-ops for any framework other than mocha). The method is still + // registered so the call resolves and the warning reaches the user, exactly + // as on the legacy flow. Without this gate the CLI path silently STARTS + // supporting custom tags for other frameworks — a widening, but an + // unrequested behaviour change all the same. + if (!this.isMochaFramework()) { + this.logger.warn('setCustomTags is only supported for the mocha framework; ignoring call') + return + } if (!key || !value) { this.logger.warn('setCustomTags: key and value are required; ignoring call') return diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index c64bfa8..ae00025 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -42,7 +42,7 @@ export const DEFAULT_WAIT_TIMEOUT_FOR_PENDING_UPLOADS = 5000 // 5s export const DEFAULT_WAIT_INTERVAL_FOR_PENDING_UPLOADS = 100 // 100ms export const BSTACK_SERVICE_VERSION = bstackServiceVersion -export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride'] +export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName'] export const BROWSERSTACK_TEST_PLAN_ID = 'BROWSERSTACK_TEST_PLAN_ID' export const LOGS_FILE = 'logs/bstack-wdio-service.log' diff --git a/packages/browserstack-service/src/scripts/accessibility-scripts.ts b/packages/browserstack-service/src/scripts/accessibility-scripts.ts index 1e1dff0..def7086 100644 --- a/packages/browserstack-service/src/scripts/accessibility-scripts.ts +++ b/packages/browserstack-service/src/scripts/accessibility-scripts.ts @@ -14,6 +14,27 @@ interface Command { class: string } +/** + * The binary types AccessibilityCapability.value as a proto string, so goog:chromeOptions + * arrives JSON-encoded over gRPC where the HTTP launch response delivers a plain object. + * Accept both. Anything that does not resolve to an object is dropped rather than written + * into a W3C capability, which the hub rejects outright. + */ +function toChromeOptions(value: unknown): { [key: string]: unknown } | null { + let parsed = value + if (typeof parsed === 'string') { + try { + parsed = JSON.parse(parsed) + } catch { + return null + } + } + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as { [key: string]: unknown } + } + return null +} + class AccessibilityScripts { private static instance: AccessibilityScripts | null = null @@ -88,8 +109,9 @@ class AccessibilityScripts { if (data.commands && data.commands.length) { this.commandsToWrap = data.commands } - if (data.nonBStackInfraA11yChromeOptions){ - this.ChromeExtension = data.nonBStackInfraA11yChromeOptions + const chromeOptions = toChromeOptions(data.nonBStackInfraA11yChromeOptions) + if (chromeOptions){ + this.ChromeExtension = chromeOptions } } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index eef859c..4d7ce44 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -44,6 +44,7 @@ import { AutomationFrameworkState } from './cli/states/automationFrameworkState. import { HookState } from './cli/states/hookState.js' import { AutomationFrameworkConstants } from './cli/frameworks/constants/automationFrameworkConstants.js' import TestFramework from './cli/frameworks/testFramework.js' +import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import AccessibilityModule from './cli/modules/accessibilityModule.js' @@ -301,6 +302,17 @@ export default class BrowserstackService implements Services.ServiceInstance { BStackLogger.error(`[Accessibility Test Run] Error in service class before function: ${err}`) } + // The driver registration is an automation-lifecycle concern, not an observability + // one. With EVERY product off, shouldProcessEventForTesthub() closes (it is a + // disjunction over the three product flags, and '' bypasses the eventType guards), + // so the CREATE/POST event inside the block below never fires: onDriverCreated never + // runs, and the session goes unnamed and unmarked while browser.setCustomTags is + // never defined. This covers exactly that case — when the gate is open the block + // below still raises the event, so nothing fires twice. + if (BrowserstackCLI.getInstance().isRunning() && !shouldProcessEventForTesthub('')) { + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) + } + if (shouldProcessEventForTesthub('')) { patchConsoleLogs() @@ -439,7 +451,17 @@ export default class BrowserstackService implements Services.ServiceInstance { // during any startup race, so a `!` here could throw a TypeError inside this awaited // WDIO hook and break the user's suite. Instrumentation must degrade quietly. const framework = BrowserstackCLI.getInstance().getTestFramework() - if (framework) { + if (framework instanceof WdioCucumberTestFramework) { + // A cucumber hook invocation carries no title, and BeforeAll/AfterAll pass no hook + // object at all, so getHookType — which matches Mocha's quoted titles — can only + // return 'unknown' here or throw on the property access. The framework classifies + // them from its own bookkeeping, and returns null for the step-scoped hooks that + // are deliberately never reported. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) + } + } else if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) @@ -476,6 +498,23 @@ export default class BrowserstackService implements Services.ServiceInstance { // Null-check the tracker rather than asserting (see beforeHook) so a missing tracker // degrades quietly instead of throwing inside this awaited hook. const framework = BrowserstackCLI.getInstance().getTestFramework() + if (framework instanceof WdioCucumberTestFramework) { + // Cucumber's taxonomy, not Mocha's titles — see beforeHook. + const hookFrameworkState = framework.classifyHookState(test as CucumberHook|undefined) + if (hookFrameworkState) { + // ignoreHooksStatus rides the event so the cucumber-only policy stays out of + // automateModule, which mocha and jasmine share. + await framework.trackEvent(hookFrameworkState, HookState.POST, { + test, + result, + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true + }) + } + if (hookFrameworkState === TestFrameworkState.BEFORE_ALL && result && !result.passed) { + await this._reportCucumberScenariosSkipped(framework) + } + return + } if (framework) { const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] if (hookFrameworkState) { @@ -497,6 +536,45 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._accessibilityHandler?.afterHook() } + /** + * Cucumber abandons the whole feature when a `BeforeAll` throws, so every scenario in it + * (Rule-nested included) is reported SKIPPED rather than vanishing. Ports + * `insights-handler.processCucumberHook`, whose cascade publishes over the legacy HTTP + * listener and is inert once the binary is up. + * + * Sent straight to TestHub rather than via `framework.trackEvent()`: legacy called + * `listener.testFinished()` directly, so routing these through the observers would rename the + * session, stop accessibility and run a Percy teardown per skipped row — none of which legacy + * does. + */ + private async _reportCucumberScenariosSkipped(framework: WdioCucumberTestFramework) { + try { + const testHubModule = BrowserstackCLI.getInstance().modules.TestHubModule as TestHubModule | undefined + if (!testHubModule) { + BStackLogger.debug('BEFORE_ALL cascade: TestHub module not loaded; skipped scenarios will not be reported') + return + } + + const instances = framework.buildSkippedScenarioInstances() + for (const instance of instances) { + // Both halves: TestHub's v2 pipeline creates the row from the START event, so a + // lone TestRunSkipped is accepted and counted in no bucket. Legacy's v1 listener + // created the row from the skip itself, which is why it sent only one. + await testHubModule.sendTestFrameworkEvent( + { instance }, + { testFrameworkState: 'TEST', testHookState: 'PRE' } + ) + await testHubModule.sendTestFrameworkEvent( + { instance }, + { testFrameworkState: 'TEST', testHookState: 'POST' } + ) + } + BStackLogger.debug(`BEFORE_ALL cascade: reported ${instances.length} scenario(s) as skipped`) + } catch (err) { + BStackLogger.debug(`Exception reporting the BEFORE_ALL skip cascade: ${util.format(err)}`) + } + } + @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeTest' }) async beforeTest (test: Frameworks.Test) { this._currentTest = test @@ -643,8 +721,10 @@ export default class BrowserstackService implements Services.ServiceInstance { PerformanceTester.start(PERFORMANCE_SDK_EVENTS.DRIVER_EVENT.QUIT) const { preferScenarioName, setSessionName, setSessionStatus } = this._options - // For Cucumber: If only 1 Scenario ran and preferScenarioName is enabled, - // use the scenario name instead of the feature name + // One scenario and preferScenarioName set: name the session after the scenario + // rather than the feature. Legacy-flow only — `_fullTitle` reaches the session through + // `_updateJob`, whose call sites are all gated `!isRunning()`; on the CLI flow + // automateModule keeps its own tally and applies this in onAfterExecute. if (preferScenarioName && this._scenariosRanCount === 1 && this._lastScenarioName) { this._fullTitle = this._lastScenarioName } @@ -819,9 +899,92 @@ export default class BrowserstackService implements Services.ServiceInstance { this._suiteTitle = feature.name await this._setSessionName(feature.name) await this._setAnnotation(`Feature: ${feature.name}`) + + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onFeatureStart(uri, feature) + return + } await this._insightsHandler?.beforeFeature(uri, feature) } + /** + * The CLI test framework, only when it is the cucumber one and the CLI is actually running. + * Returns null on the legacy flow and on any startup race, so every call site degrades to the + * classic handlers instead of throwing inside an awaited WDIO hook. + */ + private _cliCucumberFramework(): WdioCucumberTestFramework|null { + if (!BrowserstackCLI.getInstance().isRunning()) { + return null + } + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioCucumberTestFramework ? framework : null + } + + /** + * A `Frameworks.Test`-shaped view of a scenario, for the cli/modules that read `args.test`. + * + * `fullName` is set deliberately. automateModule's session naming is SHAPE-keyed + * (`else if (test && !test.fullName)`), so leaving it unset drops the scenario into the mocha + * arm and names the session ` - `; legacy names the session after the + * feature alone, which is what the populated `fullName` preserves. + */ + private _cucumberTestView(world: ITestCaseHookParameter): Frameworks.Test { + const scenarioName = world.pickle?.name ?? '' + return { + title: scenarioName, + fullName: scenarioName, + parent: this._suiteTitle ?? '', + file: world.gherkinDocument?.uri, + } as unknown as Frameworks.Test + } + + /** + * A `Frameworks.TestResult`-shaped view of a scenario result, for automateModule's + * session-status marking. + * + * Which statuses count as a session failure is `_failureStatuses`, NOT the passed/failed + * collapse the o11y result applies: cucumber's UNDEFINED / AMBIGUOUS / UNKNOWN fail the + * session on legacy while still being reported to Observability as `skipped`, and PENDING + * joins them only under `cucumberOpts.strict`. Reading the o11y collapse here instead marked + * a feature with an undefined step `passed` where legacy marks it `failed`. + */ + private _cucumberTestResult(world: ITestCaseHookParameter): Frameworks.TestResult { + const status = world.result?.status?.toLowerCase() + + // `ignoreHooksStatus` has to reach session marking as well as the o11y result. On the CLI + // flow automateModule derives the session status from this view alone — + // service.after()'s _failReasons accumulation, which applied the flag on the legacy path, + // is gated off while the binary is up. A missing framework keeps the raw status. + const ignoreHooksStatus = this._options.testObservabilityOptions?.ignoreHooksStatus === true + const hasStepFailures = this._cliCucumberFramework()?.hasStepFailures() ?? true + const hookOnlyFailure = ignoreHooksStatus && status === 'failed' && !hasStepFailures + + const passed = status === 'passed' || hookOnlyFailure + const failed = !passed && status !== undefined && this._failureStatuses.includes(status) + + // The statuses that only fail the session via `_failureStatuses` carry no `world.result + // .message` — PENDING under `cucumberOpts.strict`, and equally UNDEFINED / AMBIGUOUS — so + // automateModule falls back to its generic 'Unknown Error'. Legacy synthesises the reason + // in the same afterScenario() block that consults `_failureStatuses`; mirror it verbatim. + let error: Error | undefined + if (failed) { + error = new Error(world.result?.message || (status === 'pending' + ? `Some steps/hooks are pending for scenario "${world.pickle.name}"` + : 'Unknown Error')) + } else if (world.result?.message) { + error = new Error(world.result.message) + } + + return { + passed, + skipped: !passed && !failed, + error, + duration: 0, + retries: { attempts: 0, limit: 0 }, + } as unknown as Frameworks.TestResult + } + /** * Runs before a Cucumber Scenario. * @param world world object containing information on pickle and test step @@ -829,9 +992,24 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeScenario' }) async beforeScenario (world: ITestCaseHookParameter) { this._currentTest = world + const scenarioName = world.pickle.name || 'unknown scenario' + + // The scenario IS the unit of work, so it raises TEST/PRE — the state every cli/modules + // observer subscribes to. WDIO never calls beforeTest for cucumber, so there is no other + // moment at which the modules could be driven. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + }) + await this._setAnnotation(`Scenario: ${scenarioName}`) + return + } + await this._accessibilityHandler?.beforeScenario(world) await this._insightsHandler?.beforeScenario(world) - const scenarioName = world.pickle.name || 'unknown scenario' await this._setAnnotation(`Scenario: ${scenarioName}`) } @@ -873,6 +1051,19 @@ export default class BrowserstackService implements Services.ServiceInstance { } } + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + await cliFramework.trackEvent(TestFrameworkState.TEST, HookState.POST, { + world, + test: this._cucumberTestView(world), + suiteTitle: this._suiteTitle, + result: this._cucumberTestResult(world), + ignoreHooksStatus: this._options.testObservabilityOptions?.ignoreHooksStatus === true, + preferScenarioName: this._options.preferScenarioName === true, + }) + return + } + await this._accessibilityHandler?.afterScenario(world) await this._insightsHandler?.afterScenario(world) await this._percyHandler?.afterScenario() @@ -880,12 +1071,24 @@ export default class BrowserstackService implements Services.ServiceInstance { @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeStep' }) async beforeStep (step: Frameworks.PickleStep, scenario: Pickle) { - await this._insightsHandler?.beforeStep(step, scenario) + // Steps travel inside the scenario payload, never as their own wire event — this only + // feeds the bookkeeping the hook classifier reads. + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onStepStart(step) + } else { + await this._insightsHandler?.beforeStep(step, scenario) + } await this._setAnnotation(`Step: ${step.keyword}${step.text}`) } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterStep' }) async afterStep (step: Frameworks.PickleStep, scenario: Pickle, result: Frameworks.PickleResult) { + const cliFramework = this._cliCucumberFramework() + if (cliFramework) { + cliFramework.onStepEnd(step, result) + return + } await this._insightsHandler?.afterStep(step, scenario, result) } diff --git a/packages/browserstack-service/tests/accessibility-scripts.test.ts b/packages/browserstack-service/tests/accessibility-scripts.test.ts index 609d4b0..9aed2c4 100644 --- a/packages/browserstack-service/tests/accessibility-scripts.test.ts +++ b/packages/browserstack-service/tests/accessibility-scripts.test.ts @@ -145,3 +145,36 @@ describe('getWritableDir', () => { expect(writableDir).toBe(process.cwd()) // Should return the second path }) }) + +describe('nonBStackInfraA11yChromeOptions across flows', () => { + const scripts: typeof AccessibilityScripts = AccessibilityScripts + const payload = (chromeOptions: unknown) => ({ + commands: [], + scripts: { scan: 'scan', getResults: 'getResults', getResultsSummary: 'getResultsSummary', saveResults: 'saveResults' }, + nonBStackInfraA11yChromeOptions: chromeOptions + } as unknown as Parameters[0]) + + beforeEach(() => { + scripts.ChromeExtension = {} + }) + + it('keeps the object the HTTP launch response delivers', () => { + scripts.update(payload({ args: ['--headless=new'], extensions: ['b64'] })) + expect(scripts.ChromeExtension).to.deep.equal({ args: ['--headless=new'], extensions: ['b64'] }) + }) + + it('decodes the JSON string the gRPC response delivers', () => { + scripts.update(payload('{"args":["--headless=new"],"extensions":["b64"]}')) + expect(scripts.ChromeExtension).to.deep.equal({ args: ['--headless=new'], extensions: ['b64'] }) + }) + + it('drops a value that is not an object instead of writing it into a capability', () => { + scripts.update(payload('[object Object]')) + expect(scripts.ChromeExtension).to.deep.equal({}) + }) + + it('leaves the extension untouched when the capability set has no chrome options', () => { + scripts.update(payload(undefined)) + expect(scripts.ChromeExtension).to.deep.equal({}) + }) +}) diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 11cf8a9..edd23a2 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -648,20 +648,58 @@ describe('AccessibilityModule', () => { } }) - it('captures the hook run uuid and opens the scan gate at hook start', async () => { - vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + // The gate write is Mocha-only, exactly as legacy gates it. Keyed rather than blanket + // mock: the handler now reads the framework name off the instance too. + const mockInstanceState = (frameworkName: string) => { + vi.mocked(TestFramework.getState).mockImplementation((_i: any, key: string) => + (key === 'test_framework_name' ? frameworkName : 'hook-uuid-123') as any) vi.mocked(AutomationFramework.getState).mockImplementation((instance: any, key: string) => (key.includes('session_id') ? 12345 : {}) as any) + } + + it('captures the hook run uuid and opens the scan gate at hook start', async () => { + mockInstanceState('WebdriverIO-mocha') + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(true) + }) + + // 8-C. Discriminating: identical call, opposite answers. Mocha's beforeEach precedes + // beforeTest so the re-open is harmless; cucumber's scenario boundary precedes its Before + // hooks, so the same write would permanently force the gate open and defeat the + // includeTagsInTestingScope / excludeTagsInTestingScope filtering. + it('does NOT re-open the scan gate for cucumber — the per-test gate stands', async () => { + mockInstanceState('WebdriverIO-cucumber') + accessibilityModule.accessibilityMap.set(12345, false) await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(false) + }) + + it('re-opens a closed gate for mocha — the opposite answer on the same input', async () => { + mockInstanceState('WebdriverIO-mocha') + accessibilityModule.accessibilityMap.set(12345, false) + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + expect(accessibilityModule.accessibilityMap.get(12345)).toBe(true) }) + it('still captures the hook run uuid for cucumber (app-a11y hook-scan stamping)', async () => { + mockInstanceState('WebdriverIO-cucumber') + + await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) + + expect(accessibilityModule.currentHookRunUuid).toBe('hook-uuid-123') + }) + it('does not open the scan gate when accessibility is disabled', async () => { accessibilityModule.accessibility = false - vi.mocked(TestFramework.getState).mockReturnValue('hook-uuid-123') + mockInstanceState('WebdriverIO-mocha') await accessibilityModule.onHookStart({ instance: mockTestInstance } as any) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 3fca63a..1378920 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1,10 +1,11 @@ -import { describe, expect, it, vi, beforeEach } from 'vitest' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' import AutomateModule from '../../../src/cli/modules/automateModule.js' import TestFramework from '../../../src/cli/frameworks/testFramework.js' import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' @@ -14,7 +15,9 @@ import type { Options } from '@wdio/types' vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ default: { registerObserver: vi.fn(), - setState: vi.fn() + setState: vi.fn(), + getState: vi.fn(), + getTrackedInstance: vi.fn() } })) @@ -113,7 +116,7 @@ describe('AutomateModule', () => { // Create new instance to test observer registration (constructor registers the observers) new AutomateModule(mockConfig) - expect(TestFramework.registerObserver).toHaveBeenCalledTimes(3) + expect(TestFramework.registerObserver).toHaveBeenCalledTimes(5) expect(TestFramework.registerObserver).toHaveBeenCalledWith( TestFrameworkState.TEST, HookState.PRE, @@ -129,6 +132,16 @@ describe('AutomateModule', () => { HookState.POST, expect.any(Function) ) + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.BEFORE_ALL, + HookState.POST, + expect.any(Function) + ) + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.AFTER_ALL, + HookState.POST, + expect.any(Function) + ) }) it('should have correct static MODULE_NAME', () => { @@ -294,9 +307,12 @@ describe('AutomateModule', () => { ) }) - it('adds no status traffic for skipSessionName users (SDK-7270 inverse-leak guard)', async () => { - // With naming off, onBeforeTest never registers the session. onAfterTest must not adopt it - // either, or onAfterExecute would start status-marking sessions it previously ignored. + it('still status-marks a skipSessionName session, and sends no name', async () => { + // `setSessionName: false` suppresses the NAME only. Legacy gates its `after()` status block + // on setSessionStatus alone, and marks `passed` with an empty name — measured on BOTH + // frameworks' legacy arms. This test previously asserted the opposite (no traffic at all), + // which encoded the CLI's own behaviour rather than parity with legacy, and left every + // CLI-flow session unmarked whenever a user opted out of naming. (automateModule.config as any).testContextOptions.skipSessionName = true vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) @@ -315,7 +331,11 @@ describe('AutomateModule', () => { await automateModule.onAfterExecute() - expect(fetch).not.toHaveBeenCalled() + const bodies = vi.mocked(fetch).mock.calls.map(([, init]) => (init as { body: string }).body) + // exactly one call, and it is the status mark — no name field anywhere + expect(bodies).toHaveLength(1) + expect(JSON.parse(bodies[0])).not.toHaveProperty('name') + expect(JSON.parse(bodies[0]).status).toBe('passed') }) it('should skip session status update when skipSessionStatus is true', async () => { @@ -630,4 +650,310 @@ describe('AutomateModule', () => { expect(moduleWithoutConfig).toBeInstanceOf(AutomateModule) expect(moduleWithoutConfig.getModuleName()).toBe('AutomateModule') }) -}) \ No newline at end of file +}) + +describe('AutomateModule testResults keying (SDK-7414)', () => { + let automateModule: AutomateModule + let mockTestInstance: any + + const FEATURE = 'A cucumber feature' + + // A cucumber test view carries fullName (the scenario); a mocha one never does. + const cucumberTest = (scenario: string) => ({ title: scenario, fullName: scenario, parent: FEATURE }) + const mochaTest = (title: string) => ({ title, parent: FEATURE }) + + const afterTest = (test: any, passed: boolean) => ({ + instance: mockTestInstance, + result: { error: passed ? null : new Error(`${test.title} failed`), passed }, + test, + suiteTitle: FEATURE + }) + + const sessionData = () => (automateModule as any).sessionMap.get('test-session-id') + + beforeEach(() => { + vi.clearAllMocks() + + const mockAutoInstance = { getId: vi.fn().mockReturnValue(1) } + mockTestInstance = { getId: vi.fn().mockReturnValue(1) } + + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue(mockAutoInstance) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'test-session-id' }) + vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { + if (key === 'framework_session_id') {return 'test-session-id'} + if (key.includes('CAPABILITIES')) {return { browserName: 'chrome' }} + return {} + }) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) } as any) + + automateModule = new AutomateModule({ user: 'testuser', key: 'testkey' } as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey' + } as any + }) + + // The defect: every scenario in a feature shares the session name, so keying testResults on it + // collapsed N scenarios into one last-write-wins entry and a trailing pass hid earlier failures. + it('keeps one entry per scenario for cucumber, so a trailing pass cannot mask an earlier failure', async () => { + await automateModule.onAfterTest(afterTest(cucumberTest('scenario one fails'), false)) + await automateModule.onAfterTest(afterTest(cucumberTest('scenario two passes'), true)) + + const results = sessionData().testResults + expect([...results.keys()]).toEqual(['scenario one fails', 'scenario two passes']) + expect(results.size).toBe(2) + expect([...results.values()].map((r: any) => r.status)).toEqual(['failed', 'passed']) + // the session NAME stays the feature title even though the keys do not. + expect(sessionData().lastTestName).toBe(FEATURE) + }) + + // The discriminating pair: identical input shape, opposite answers through the resultKey branch. + it('keys on the scenario for cucumber and leaves the key unchanged for mocha', async () => { + await automateModule.onAfterTest(afterTest(cucumberTest('a scenario'), true)) + const cucumberKey = [...sessionData().testResults.keys()][0] + const cucumberName = sessionData().lastTestName + + vi.clearAllMocks() + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'test-session-id' }) + vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { + if (key === 'framework_session_id') {return 'test-session-id'} + if (key.includes('CAPABILITIES')) {return { browserName: 'chrome' }} + return {} + }) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: vi.fn().mockResolvedValue({ success: true }) } as any) + ;(automateModule as any).sessionMap = new Map() + + await automateModule.onAfterTest(afterTest(mochaTest('a test'), true)) + const mochaKey = [...sessionData().testResults.keys()][0] + const mochaName = sessionData().lastTestName + + expect(cucumberKey).toBe('a scenario') + expect(cucumberKey).not.toBe(cucumberName) + expect(mochaKey).toBe(mochaName) + expect(mochaKey).toBe(`${FEATURE} - a test`) + }) +}) + +const cucumberInstance = { framework: 'WebdriverIO-cucumber' } +const mochaInstance = { framework: 'WebdriverIO-mocha' } + +function stateFor(instance: unknown, key: string) { + if (key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) { + return (instance as { framework: string }).framework + } + if (key === TestFrameworkConstants.KEY_HOOKS_FINISHED) { + return new Map([['BEFORE_ALL', [{ [TestFrameworkConstants.KEY_HOOK_NAME]: 'BEFORE_ALL for Login' }]]]) + } + return undefined +} + +function newModule(config: Record = {}) { + const mod = new AutomateModule({ user: 'u', key: 'k' } as Options.Testrunner) + mod.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'testuser', + accessKey: 'testkey', + ...config + } as never + + return mod +} + +describe('AutomateModule — session marking', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(TestFramework.getState).mockImplementation((instance, key) => stateFor(instance, key)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + afterEach(() => { + delete process.env.BROWSERSTACK_TURBOSCALE_INTERNAL + }) + + // Discriminating: the same call yields opposite verbs and different hosts on the flag alone. + describe('turboscale routes to its own API with PATCH', () => { + it('PATCHes the turboscale endpoint when turboScale is configured', async () => { + const mod = newModule({ turboScale: true }) + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate-turboscale/v1/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PATCH') + }) + + it('PUTs the automate endpoint when turboScale is not configured', async () => { + const mod = newModule() + await mod.markSessionStatus('sess-1', 'passed', undefined, { user: 'u', key: 'k' }) + + const [url, options] = vi.mocked(fetch).mock.calls[0] + expect(url).toBe('https://api.browserstack.com/automate/sessions/sess-1.json') + expect((options as { method: string }).method).toBe('PUT') + }) + + it('takes precedence over app-automate, mirroring legacy assignment order', async () => { + const mod = newModule({ turboScale: true, app: 'bs://app' }) + await mod.markSessionStatus('sess-1', 'failed', 'boom', { user: 'u', key: 'k' }) + + expect(vi.mocked(fetch).mock.calls[0][0]).toContain('/automate-turboscale/v1/sessions/') + }) + }) + + // A build-level hook has no test tied to it, so its failure reaches the verdict only here. + // Discriminating: the same failing hook fails cucumber and leaves mocha untouched. + describe('build-level hook failures reach the session verdict', () => { + const failing = { passed: false, error: new Error('BeforeAll blew up') } + + const runScenario = (mod: AutomateModule, passed: boolean) => mod.onAfterTest({ + instance: cucumberInstance, + result: { error: passed ? null : new Error('step failed'), passed }, + test: { title: 'a scenario', fullName: 'Feature: a scenario' }, + suiteTitle: 'Feature' + }) + + const statusBody = () => { + const call = vi.mocked(fetch).mock.calls.find(([, o]) => + JSON.parse((o as { body: string }).body).status !== undefined) + + return call ? JSON.parse((call[1] as { body: string }).body) : undefined + } + + it('marks the session failed for cucumber when a BeforeAll fails', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + expect(statusBody().status).toBe('failed') + expect(statusBody().reason).toBe('BeforeAll blew up') + }) + + it('names the hook in the failure reason when several fail', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: { passed: false, error: new Error('teardown') } }) + await mod.onAfterExecute() + + expect(statusBody().reason).toContain('BEFORE_ALL for Login') + }) + + it('leaves wdio_mocha untouched — the identical failure records nothing', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: mochaInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('records nothing when the hook passed', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: { passed: true } }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the session PASSED under ignoreHooksStatus once a scenario has run', async () => { + const mod = newModule() + await runScenario(mod, true) + await mod.onBuildLevelHookEnd('AFTER_ALL', { instance: cucumberInstance, result: failing, ignoreHooksStatus: true }) + await mod.onAfterExecute() + + expect(statusBody()).toEqual({ status: 'passed' }) + }) + + // Zero scenarios is legacy's `!_specsRan` arm, which marks failed regardless of the flag. + // Discriminating against the case above: same hook, same flag, opposite verdicts. + it('marks the session FAILED under ignoreHooksStatus when no scenario ran', async () => { + const mod = newModule() + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing, ignoreHooksStatus: true }) + await mod.onAfterExecute() + + expect(statusBody().status).toBe('failed') + }) + + it('respects skipSessionStatus', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: false, skipSessionStatus: true } }) + await mod.onBuildLevelHookEnd('BEFORE_ALL', { instance: cucumberInstance, result: failing }) + await mod.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + }) +}) + +describe('AutomateModule preferScenarioName', () => { + const register = (mod: AutomateModule, lastTestName: string, seed: Record = {}) => { + (mod['sessionMap'] as Map>) + .set('sess-1', { lastTestName, testResults: new Map(), scenariosRan: 0, ...seed }) + } + + const namesPUT = () => vi.mocked(fetch).mock.calls.map(([, init]) => { + try { + return JSON.parse((init as { body: string }).body).name + } catch { + return undefined + } + }) + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => + key === 'framework_session_id' ? 'sess-1' : ({} as never)) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({ ok: true }) } as never) + }) + + it('renames to the scenario name when exactly one scenario ran', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).toContain('Can log in') + }) + + // The `=== 1` exactness legacy applies — a `>= 1` here renames every multi-scenario feature. + it('keeps the feature name when two scenarios ran', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 2, lastScenarioName: 'Second scenario', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Second scenario') + expect(namesPUT()).toContain('Login Feature') + }) + + it('keeps the feature name when the flag is absent', async () => { + const mod = newModule() + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in' }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) + + it('honours setSessionName: false and issues no rename', async () => { + const mod = newModule({ testContextOptions: { skipSessionName: true, skipSessionStatus: false } }) + register(mod, 'Login Feature', { scenariosRan: 1, lastScenarioName: 'Can log in', preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) + + // A skipped scenario is not a scenario that ran; legacy's counter is gated the same way. + it('does not count a skipped scenario', async () => { + const mod = newModule() + register(mod, 'Login Feature', { preferScenarioName: true }) + + await mod.onAfterExecute() + + expect(namesPUT()).not.toContain('Can log in') + }) +}) diff --git a/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts new file mode 100644 index 0000000..cf4a435 --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/customTagsModule.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' + +vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ + default: class MockTestFramework { + static registerObserver = vi.fn() + static getTrackedInstance = vi.fn() + static getState = vi.fn() + } +})) + +vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ + default: class MockAutomationFramework { + static registerObserver = vi.fn() + static getTrackedInstance = vi.fn() + static getDriver = vi.fn() + } +})) + +vi.mock('../../../src/cli/cliLogger.js', () => ({ + BStackLogger: { info: vi.fn(), debug: vi.fn(), error: vi.fn(), warn: vi.fn() } +})) + +vi.mock('../../../src/cli/index.js', () => ({ + BrowserstackCLI: { getInstance: vi.fn() } +})) + +import CustomTagsModule from '../../../src/cli/modules/customTagsModule.js' +import TestFramework from '../../../src/cli/frameworks/testFramework.js' +import AutomationFramework from '../../../src/cli/frameworks/automationFramework.js' +import WdioMochaTestFramework from '../../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../../src/cli/frameworks/wdioCucumberTestFramework.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' +import { BStackLogger } from '../../../src/cli/cliLogger.js' + +/** + * 8-D. `setCustomTags` warns and no-ops for every framework except mocha, which + * is what the legacy custom-tags-handler does. Discriminating: the SAME call merges tags under + * mocha and merges nothing under cucumber. + */ +describe('CustomTagsModule — framework gate', () => { + let module: CustomTagsModule + let browser: Record + let instance: { updateMultipleEntries: ReturnType, getCurrentTestState: ReturnType } + + const useFramework = (framework: unknown) => { + vi.mocked(BrowserstackCLI.getInstance).mockReturnValue({ + getTestFramework: () => framework + } as never) + } + + beforeEach(async () => { + vi.clearAllMocks() + browser = {} + instance = { + updateMultipleEntries: vi.fn(), + getCurrentTestState: vi.fn().mockReturnValue('TestFrameworkState.TEST') + } + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as never) + vi.mocked(AutomationFramework.getDriver).mockReturnValue(browser as never) + vi.mocked(TestFramework.getTrackedInstance).mockReturnValue(instance as never) + vi.mocked(TestFramework.getState).mockReturnValue({} as never) + + module = new CustomTagsModule() + }) + + const register = async () => { + await module.onBeforeExecute() + return browser.setCustomTags as (k: string, v: string) => Promise + } + + it('registers setCustomTags regardless of framework, so the call always resolves', async () => { + useFramework(Object.create(WdioCucumberTestFramework.prototype)) + expect(await register()).toBeTypeOf('function') + }) + + it('merges tags for mocha', async () => { + useFramework(Object.create(WdioMochaTestFramework.prototype)) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1,TC-2') + + expect(instance.updateMultipleEntries).toHaveBeenCalled() + }) + + it('warns and no-ops for cucumber — the opposite answer on the same call', async () => { + useFramework(Object.create(WdioCucumberTestFramework.prototype)) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1,TC-2') + + expect(instance.updateMultipleEntries).not.toHaveBeenCalled() + expect(BStackLogger.warn).toHaveBeenCalledWith( + 'setCustomTags is only supported for the mocha framework; ignoring call' + ) + }) + + it('warns and no-ops when no CLI test framework is registered', async () => { + useFramework(null) + const setCustomTags = await register() + + await setCustomTags('TC', 'TC-1') + + expect(instance.updateMultipleEntries).not.toHaveBeenCalled() + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts new file mode 100644 index 0000000..53d4438 --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' +import path from 'node:path' +import * as bstackLogger from '../../src/bstackLogger.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const RELATIVE_URI = 'features/checkout.feature' +const ABSOLUTE_URI = path.resolve(process.cwd(), RELATIVE_URI) +const FEATURE = { name: 'Checkout', description: 'a feature', children: [] } + +/** + * The two path fields are deliberately different shapes and must not be conflated: + * + * - `test_file_path` is ABSOLUTE. The binary re-bases it itself + * (`path.relative(session.pathProject, absoluteTestFilePath)` in the cucumber module), so a + * pre-relativised value there gets resolved against cwd first and comes out wrong (SDK-7233). + * - `bdd_meta_info.feature.path` is cwd-relative, matching legacy's + * `feature = { path: gherkinDocument.uri, … }`. Nothing in the binary's node path reads this + * blob, so whatever is sent reaches the dashboard verbatim — an absolute value there leaks the + * developer's home directory. + * + * Both uri shapes are exercised because WDIO hands `beforeFeature` the ABSOLUTE one. A fixture that + * only feeds the relative shape stays green against a `path` that forwards the uri raw, which is + * exactly how an absolute path reached the dashboard under a passing suite. + */ +describe.each([ + ['a relative uri', RELATIVE_URI], + ['the absolute uri WDIO supplies', ABSOLUTE_URI], +])('cucumber feature paths — %s', (_shape, uri) => { + let framework: WdioCucumberTestFramework + + beforeEach(() => { + framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') + framework.onFeatureStart(uri, FEATURE as never) + }) + + it('sends test_file_path as an absolute path for the binary to re-base', () => { + const featurePath = framework['featurePath']() + + expect(path.isAbsolute(featurePath as string)).toBe(true) + expect(featurePath).toBe(ABSOLUTE_URI) + }) + + it('puts a cwd-relative path in bdd_meta_info.feature.path', () => { + const meta = framework['buildBddMetaInfo']({ name: 'a scenario', tags: [] } as never, FEATURE as never, []) + + expect(meta.feature.path).toBe(RELATIVE_URI) + expect(path.isAbsolute(meta.feature.path as string)).toBe(false) + }) + + // The regression this guards: the two fields were fed from one absolute value, so the meta + // blob reached the dashboard carrying the developer's home directory. + it('does not put the absolute path in the bdd meta', () => { + const meta = framework['buildBddMetaInfo']({ name: 'a scenario', tags: [] } as never, FEATURE as never, []) + + expect(meta.feature.path).not.toBe(framework['featurePath']()) + expect(meta.feature.path).not.toContain(process.cwd()) + }) +}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index bf544f9..67f1d59 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -10,7 +10,11 @@ import { BrowserstackCLI } from '../src/cli/index.js' import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' +import WdioCucumberTestFramework from '../src/cli/frameworks/wdioCucumberTestFramework.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' +import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' @@ -2773,3 +2777,324 @@ describe('afterTest bail skip cascade (SDK-7063)', () => { expect(skippedTitles(trackEvent)).toEqual([]) }) }) + +describe('afterScenario session-status view honours ignoreHooksStatus', () => { + let getInstanceSpy: ReturnType + + const makeService = (ignoreHooksStatus: boolean) => new BrowserstackService( + { testObservability: false, testObservabilityOptions: { ignoreHooksStatus } } as any, + [] as any, + { user: 'foo', key: 'bar', framework: 'cucumber' } as any + ) + + const runAfterScenario = async (svc: BrowserstackService, hadStepFailures: boolean) => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + vi.spyOn(framework, 'hasStepFailures').mockReturnValue(hadStepFailures) + const trackEvent = vi.spyOn(framework, 'trackEvent').mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework + } as any) + + await svc.afterScenario({ + pickle: { name: 'a scenario' }, + result: { status: 'FAILED', message: 'hook blew up' } + } as any) + + return (trackEvent.mock.calls.at(-1)?.[2] as any)?.result + } + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('reports a hook-only failure as passed so the session is not marked failed', async () => { + const result = await runAfterScenario(makeService(true), false) + expect(result.passed).toBe(true) + }) + + it('still reports a step failure as failed under the same flag', async () => { + const result = await runAfterScenario(makeService(true), true) + expect(result.passed).toBe(false) + }) + + it('leaves a hook-only failure failed when the flag is not set', async () => { + const result = await runAfterScenario(makeService(false), false) + expect(result.passed).toBe(false) + }) +}) + +describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => { + let getInstanceSpy: ReturnType + + const feature = { + name: 'Login', + description: 'a description', + children: [ + { scenario: { name: 'Scenario A', steps: [{ id: 's1', text: 'I log in', keyword: 'Given ' }] } }, + { background: {} }, + { rule: { children: [{ background: {} }, { scenario: { name: 'Rule-nested B', steps: [] } }] } } + ] + } + + const makeService = (ignoreHooksStatus = false) => new BrowserstackService( + { testObservability: false, testObservabilityOptions: { ignoreHooksStatus } } as any, + [] as any, + { user: 'foo', key: 'bar', framework: 'cucumber' } as any + ) + + const runAfterHook = async (svc: BrowserstackService, result: unknown, scenariosStarted = false) => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + framework.onFeatureStart('features/login.feature', feature as any) + if (scenariosStarted) { + // flips classifyHookType from BEFORE_ALL to AFTER_ALL + (framework as any).cucumberData.scenariosStarted = true + } + const trackEvent = vi.spyOn(framework, 'trackEvent').mockResolvedValue(undefined) + const sendTestFrameworkEvent = vi.fn().mockResolvedValue(true) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + modules: { TestHubModule: { sendTestFrameworkEvent } } + } as any) + + await svc.afterHook(undefined as any, {}, result as any) + return { sendTestFrameworkEvent, trackEvent } + } + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('reports every not-run scenario as skipped, Rule-nested ones included', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + // two scenarios x (start, skip) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(4) + const names = sendTestFrameworkEvent.mock.calls.map(([args]: any[]) => + args.instance.getAllData().get('test_name')) + expect(names).toEqual(['Scenario A', 'Scenario A', 'Rule-nested B', 'Rule-nested B']) + }) + + // TestHub's v2 batch pipeline creates the test row from the START event; a lone TestRunSkipped + // is accepted and lands in no bucket at all (observed: O11Y build ad39yy… reported skipped=0 + // for three sent TestRunSkipped events, lwbpu2… reported skipped=3 for the pair). + it('sends a start then a skip per scenario, so the binary emits TestRunStarted + TestRunSkipped', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + const overrides = sendTestFrameworkEvent.mock.calls.map(([, o]: any[]) => o.testHookState) + expect(overrides).toEqual(['PRE', 'POST', 'PRE', 'POST']) + + for (const [args, override] of sendTestFrameworkEvent.mock.calls as any[]) { + expect(override.testFrameworkState).toBe('TEST') + const data = args.instance.getAllData() + expect(data.get('test_result')).toBe('skipped') + expect(data.get('test_skipped_cascade')).toBe(true) + // the cascade payload has no world, so no tags + expect(data.get('test_tags')).toBeUndefined() + expect(data.get('test_scopes')).toEqual(['Login']) + } + }) + + it('marks every step of a skipped scenario skipped', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }) + + const meta = sendTestFrameworkEvent.mock.calls[0][0].instance.getAllData().get('bdd_meta_info') as any + expect(meta.steps).toEqual([{ id: 's1', text: 'I log in', keyword: 'Given ', result: 'skipped' }]) + expect(meta.feature.name).toBe('Login') + }) + + it('does not cascade when the BEFORE_ALL passed', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: true }) + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('does not cascade for an AFTER_ALL failure — the scenarios already ran', async () => { + const { sendTestFrameworkEvent } = await runAfterHook(makeService(), { passed: false, error: new Error('boom') }, true) + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('passes ignoreHooksStatus on the hook event so the module can honour it', async () => { + const { trackEvent } = await runAfterHook(makeService(true), { passed: false, error: new Error('boom') }) + expect((trackEvent.mock.calls.at(-1)?.[2] as any).ignoreHooksStatus).toBe(true) + + const off = await runAfterHook(makeService(false), { passed: false, error: new Error('boom') }) + expect((off.trackEvent.mock.calls.at(-1)?.[2] as any).ignoreHooksStatus).toBe(false) + }) + + it('reports the hook failure on the hook event itself, not as passed', async () => { + const framework = new WdioCucumberTestFramework(['cucumber'], { cucumber: '10.0.0' }, 'bin-session') + framework.onFeatureStart('features/login.feature', feature as any) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + modules: { TestHubModule: { sendTestFrameworkEvent: vi.fn().mockResolvedValue(true) } } + } as any) + + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.PRE, { test: undefined }) + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { + test: undefined, + result: { passed: false, error: new Error('boom'), duration: 12, retries: { attempts: 0, limit: 0 } } + }) + + const instance = (framework as any).constructor.getTrackedInstance() + const finished = instance.getAllData().get('test_hooks_finished') as Map + expect(finished.get('BEFORE_ALL')![0].hook_result).toBe('failed') + }) +}) + +describe('driver registration is not gated on observability', () => { + // CREATE/POST is the driver registration — onDriverCreated and the product modules' init + // handlers hang off it. It used to be raised only inside `shouldProcessEventForTesthub('')`, + // a disjunction over the three product flags, so with every product off the gate closed and + // the session went unnamed and unmarked with setCustomTags undefined. The second and third + // cases guard the fix's shape: it must not double-raise where the gate is already open. + const PRODUCT_ENV = ['BROWSERSTACK_OBSERVABILITY', 'BROWSERSTACK_ACCESSIBILITY', 'BROWSERSTACK_PERCY'] + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + const saved: Record = {} + + const createPostCalls = () => trackEvent.mock.calls.filter( + ([state, hook]) => state === AutomationFrameworkState.CREATE && hook === HookState.POST) + + const runBefore = async () => { + const svc = new BrowserstackService({} as never, [{}] as never, { capabilities: {} } as never) + await svc.beforeSession({} as never) + await svc.before(svc['_config'] as never, [], { sessionId: 'sess-1' } as never) + } + + beforeEach(() => { + PRODUCT_ENV.forEach(k => { saved[k] = process.env[k]; delete process.env[k] }) + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + }) + + afterEach(() => { + PRODUCT_ENV.forEach(k => { if (saved[k] === undefined) { delete process.env[k] } else { process.env[k] = saved[k] } }) + getInstanceSpy?.mockRestore() + }) + + it('registers the driver with every product turned off', async () => { + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + it('does not double-register when observability is on', async () => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) + + it('does not double-register when only accessibility is on', async () => { + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + + await runBefore() + + expect(createPostCalls()).toHaveLength(1) + }) +}) + +// automateModule decides the rename — it is the only place that knows the final scenario count — +// but it cannot read service options, so the flag rides the scenario event, as `ignoreHooksStatus` +// does. The decision itself is covered in tests/cli/modules/automateModule.preferScenarioName.test.ts. +describe('preferScenarioName reaches the module', () => { + let getInstanceSpy: ReturnType | undefined + let trackEvent: ReturnType + + const makeService = (options: Record = {}) => new BrowserstackService( + { testObservability: false, setSessionName: true, setSessionStatus: true, ...options } as never, + [] as never, + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict: false } } as never + ) + + const scenarioEventArgs = () => trackEvent.mock.calls.at(-1)?.[2] as Record + + beforeEach(() => { + trackEvent = vi.fn().mockResolvedValue(undefined) + const cucumberFramework = Object.create(WdioCucumberTestFramework.prototype) + cucumberFramework.trackEvent = trackEvent + cucumberFramework.hasStepFailures = () => false + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => cucumberFramework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as never) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + }) + + it('carries preferScenarioName: true on the scenario event when set', async () => { + const service = makeService({ preferScenarioName: true }) + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + expect(scenarioEventArgs().preferScenarioName).toBe(true) + }) + + // Absent must travel as an explicit false, not undefined: the module treats the field as the + // whole opt-in, so a missing value and an opted-out value must be indistinguishable there. + it('carries preferScenarioName: false when the option is absent', async () => { + const service = makeService() + await service.afterScenario({ pickle: { name: 'Can do something single' }, result: { status: 'passed' } } as never) + + expect(scenarioEventArgs().preferScenarioName).toBe(false) + }) + + // The count itself stays on the service side too, because `_scenariosRanCount` is what legacy + // reads; the module keeps its own tally for the CLI flow. Both must ignore skipped scenarios. + it('does not count a skipped scenario toward the service-side tally', async () => { + const service = makeService({ preferScenarioName: true }) + await service.afterScenario({ pickle: { name: 'Skipped one' }, result: { status: 'skipped' } } as never) + + expect(service['_scenariosRanCount']).toBe(0) + }) +}) + +describe('_cucumberTestResult failure reason adjacent', () => { + const makeService = (strict: boolean) => new BrowserstackService( + { testObservability: false } as never, + [] as never, + { user: 'foo', key: 'bar', framework: 'cucumber', cucumberOpts: { strict } } as never + ) + + const world = (status: string, message?: string) => ({ + pickle: { name: 'CfgGate pending scenario' }, + result: message ? { status, message } : { status } + }) + + it('synthesises legacy\'s pending reason when strict makes a pending scenario fail', () => { + const result = makeService(true)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.error?.message).toBe('Some steps/hooks are pending for scenario "CfgGate pending scenario"') + }) + + it('leaves a pending scenario unfailed — and unreasoned — when strict is off', () => { + const result = makeService(false)['_cucumberTestResult'](world('PENDING') as never) + + expect(result.passed).toBe(false) + expect(result.skipped).toBe(true) + expect(result.error).toBeUndefined() + }) + + it('keeps the real message when the result carries one', () => { + const result = makeService(false)['_cucumberTestResult'](world('FAILED', 'AssertionError: nope') as never) + + expect(result.error?.message).toBe('AssertionError: nope') + }) + + it('falls back to Unknown Error for a message-less non-pending failure', () => { + const result = makeService(false)['_cucumberTestResult'](world('UNDEFINED') as never) + + expect(result.error?.message).toBe('Unknown Error') + }) +})