test(evals): exercise the workspace V4 CLI

.changeset/brave-browsers-migrate.md

@@ -0,0 +1,5 @@
+---
+"browse": minor
+---
+
+migrate the Browse CLI runtime to Stagehand V4 and remove the `--return-xpath` option from coordinate actions

packages/cli/README.md

@@ -111,7 +111,7 @@ browse highlight @0-12                         # Highlight an element (--duratio
 
 ### Mouse (raw coordinates)
 
-Use these when you need pixel coordinates instead of a ref. Add `--return-xpath` to get the XPath under the cursor.
+Use these when you need pixel coordinates instead of a ref.
 
 ```bash
 browse mouse click 240 320      # Click coordinates (--button, --click-count)

packages/cli/src/commands/mouse/click.ts

@@ -15,7 +15,7 @@ export default class MouseClick extends BrowseCommand {
   static override examples = [
     "browse mouse click 240 320",
     "browse mouse click 240 320 --button right",
-    "browse mouse click 240 320 --click-count 2 --return-xpath",
+    "browse mouse click 240 320 --click-count 2",
   ];
 
   static override args = {
@@ -31,10 +31,6 @@ export default class MouseClick extends BrowseCommand {
       description: "Number of clicks to send.",
       helpValue: "<count>",
     }),
-    "return-xpath": Flags.boolean({
-      description:
-        "Include the XPath under the coordinate when the driver can return it.",
-    }),
   };
 
   async run(): Promise<void> {
@@ -44,7 +40,6 @@ export default class MouseClick extends BrowseCommand {
       {
         button: flags.button,
         clickCount: flags["click-count"],
-        returnXPath: flags["return-xpath"],
         x: parseNumber(args.x, "x"),
         y: parseNumber(args.y, "y"),
       },

packages/cli/src/commands/mouse/drag.ts

@@ -15,7 +15,6 @@ export default class MouseDrag extends BrowseCommand {
   static override examples = [
     "browse mouse drag 100 100 400 400",
     "browse mouse drag 100 100 400 400 --steps 20 --delay 10",
-    "browse mouse drag 100 100 400 400 --return-xpath",
   ];
 
   static override args = {
@@ -45,10 +44,6 @@ export default class MouseDrag extends BrowseCommand {
       description: "Delay between drag steps in milliseconds.",
       helpValue: "<ms>",
     }),
-    "return-xpath": Flags.boolean({
-      description:
-        "Include the XPath under the start/end coordinates when the driver can return it.",
-    }),
     steps: Flags.integer({
       default: 10,
       description: "Number of intermediate drag steps.",
@@ -65,7 +60,6 @@ export default class MouseDrag extends BrowseCommand {
         delay: flags.delay,
         fromX: parseNumber(args.fromX, "fromX"),
         fromY: parseNumber(args.fromY, "fromY"),
-        returnXPath: flags["return-xpath"],
         steps: flags.steps,
         toX: parseNumber(args.toX, "toX"),
         toY: parseNumber(args.toY, "toY"),

packages/cli/src/commands/mouse/hover.ts

@@ -1,4 +1,4 @@
-import { Args, Flags } from "@oclif/core";
+import { Args } from "@oclif/core";
 
 import { BrowseCommand } from "../../base.js";
 import {
@@ -13,7 +13,6 @@ export default class MouseHover extends BrowseCommand {
 
   static override examples = [
     "browse mouse hover 240 320",
-    "browse mouse hover 240 320 --return-xpath",
     "browse mouse hover 240 320 --session research",
   ];
 
@@ -24,18 +23,13 @@ export default class MouseHover extends BrowseCommand {
 
   static override flags = {
     ...driverCommandFlags,
-    "return-xpath": Flags.boolean({
-      description:
-        "Include the XPath under the coordinate when the driver can return it.",
-    }),
   };
 
   async run(): Promise<void> {
     const { args, flags } = await this.parse(MouseHover);
     await runDriverCommandFromFlags(
       "mouse.hover",
       {
-        returnXPath: flags["return-xpath"],
         x: parseNumber(args.x, "x"),
         y: parseNumber(args.y, "y"),
       },

packages/cli/src/commands/mouse/scroll.ts

@@ -1,4 +1,4 @@
-import { Args, Flags } from "@oclif/core";
+import { Args } from "@oclif/core";
 
 import { BrowseCommand } from "../../base.js";
 import {
@@ -14,7 +14,6 @@ export default class MouseScroll extends BrowseCommand {
   static override examples = [
     "browse mouse scroll 400 500 0 600",
     "browse mouse scroll 400 500 0 -600",
-    "browse mouse scroll 400 500 0 600 --return-xpath",
   ];
 
   static override args = {
@@ -32,10 +31,6 @@ export default class MouseScroll extends BrowseCommand {
 
   static override flags = {
     ...driverCommandFlags,
-    "return-xpath": Flags.boolean({
-      description:
-        "Include the XPath under the coordinate when the driver can return it.",
-    }),
   };
 
   async run(): Promise<void> {
@@ -45,7 +40,6 @@ export default class MouseScroll extends BrowseCommand {
       {
         deltaX: parseNumber(args.deltaX, "deltaX"),
         deltaY: parseNumber(args.deltaY, "deltaY"),
-        returnXPath: flags["return-xpath"],
         x: parseNumber(args.x, "x"),
         y: parseNumber(args.y, "y"),
       },

packages/cli/src/lib/driver/commands/mouse.ts

@@ -1,22 +1,24 @@
 import { z } from "zod";
 
+import { updateCursorOverlayPosition } from "../cursor-overlay.js";
+import type { DriverPage, DriverSessionManager } from "../session-manager.js";
 import type { DriverCommandHandlers } from "./types.js";
 
 const ButtonSchema = z.enum(["left", "right", "middle"]).optional();
 
 export const mouseHandlers: DriverCommandHandlers = {
   async "mouse.click"(manager, params) {
-    const { button, clickCount, returnXPath, x, y } = z
+    const { button, clickCount, x, y } = z
       .object({
         button: ButtonSchema,
         clickCount: z.number().int().positive().optional(),
-        returnXPath: z.boolean().optional(),
         x: z.number(),
         y: z.number(),
       })
+      .strict()
       .parse(params);
-    assertXPathUnavailable(returnXPath);
     const page = await manager.activePage();
+    await positionCursorOverlay(manager, page, x, y);
     await page.click(x, y, {
       ...(button === undefined ? {} : { button }),
       ...(clickCount === undefined ? {} : { clickCount }),
@@ -25,61 +27,69 @@ export const mouseHandlers: DriverCommandHandlers = {
   },
 
   async "mouse.hover"(manager, params) {
-    const { returnXPath, x, y } = z
+    const { x, y } = z
       .object({
-        returnXPath: z.boolean().optional(),
         x: z.number(),
         y: z.number(),
       })
+      .strict()
       .parse(params);
-    assertXPathUnavailable(returnXPath);
     const page = await manager.activePage();
+    await positionCursorOverlay(manager, page, x, y);
     await page.hover(x, y);
     return { hovered: true };
   },
 
   async "mouse.scroll"(manager, params) {
-    const { deltaX, deltaY, returnXPath, x, y } = z
+    const { deltaX, deltaY, x, y } = z
       .object({
         deltaX: z.number(),
         deltaY: z.number(),
-        returnXPath: z.boolean().optional(),
         x: z.number(),
         y: z.number(),
       })
+      .strict()
       .parse(params);
-    assertXPathUnavailable(returnXPath);
     const page = await manager.activePage();
+    await positionCursorOverlay(manager, page, x, y);
     await page.scroll(x, y, deltaX, deltaY);
     return { scrolled: true };
   },
 
   async "mouse.drag"(manager, params) {
-    const { button, delay, fromX, fromY, returnXPath, steps, toX, toY } = z
+    const { button, delay, fromX, fromY, steps, toX, toY } = z
       .object({
         button: ButtonSchema,
         delay: z.number().int().nonnegative().optional(),
         fromX: z.number(),
         fromY: z.number(),
-        returnXPath: z.boolean().optional(),
         steps: z.number().int().positive().optional(),
         toX: z.number(),
         toY: z.number(),
       })
+      .strict()
       .parse(params);
-    assertXPathUnavailable(returnXPath);
     const page = await manager.activePage();
+    await positionCursorOverlay(manager, page, fromX, fromY);
     await page.dragAndDrop(fromX, fromY, toX, toY, {
       ...(button === undefined ? {} : { button }),
       ...(delay === undefined ? {} : { delay }),
       ...(steps === undefined ? {} : { steps }),
     });
+    // A successful drag may navigate and destroy the old execution context.
+    // The final marker position is visual-only, so do not turn that race into a
+    // reported drag failure.
+    await positionCursorOverlay(manager, page, toX, toY).catch(() => undefined);
     return { dragged: true };
   },
 };
 
-function assertXPathUnavailable(returnXPath: boolean | undefined): void {
-  if (returnXPath) {
-    throw new Error("Coordinate XPath lookup is not exposed by Stagehand V4");
-  }
+async function positionCursorOverlay(
+  manager: DriverSessionManager,
+  page: DriverPage,
+  x: number,
+  y: number,
+): Promise<void> {
+  if (!manager.isCursorOverlayEnabled(page)) return;
+  await page.evaluate(updateCursorOverlayPosition, { x, y });
 }

packages/cli/src/lib/driver/commands/network.ts

@@ -3,7 +3,8 @@ import type { DriverCommandHandlers } from "./types.js";
 export const networkHandlers: DriverCommandHandlers = {
   async "network.on"(manager) {
     const page = await manager.activePage();
-    return manager.network.enable(page);
+    const websocketUrl = await manager.networkWebSocketDebuggerUrl();
+    return manager.network.enable(page, websocketUrl);
   },
 
   async "network.off"(manager) {

packages/cli/src/lib/driver/commands/runtime.ts

@@ -2,8 +2,8 @@ import { promises as fs } from "node:fs";
 
 import { z } from "zod";
 
+import { CURSOR_OVERLAY_SCRIPT } from "../cursor-overlay.js";
 import type { DriverCommandHandlers } from "./types.js";
-import { unavailableCursorOverlay } from "./unavailable.js";
 
 export const runtimeHandlers: DriverCommandHandlers = {
   async screenshot(manager, params) {
@@ -89,7 +89,13 @@ export const runtimeHandlers: DriverCommandHandlers = {
     return { waited: true };
   },
 
-  cursor: unavailableCursorOverlay,
+  async cursor(manager) {
+    const page = await manager.activePage();
+    await page.addInitScript(CURSOR_OVERLAY_SCRIPT);
+    await page.evaluate(CURSOR_OVERLAY_SCRIPT);
+    manager.markCursorOverlayEnabled(page);
+    return { enabled: true };
+  },
 };
 
 function parseTimeoutMs(value: string | undefined): number {

packages/cli/src/lib/driver/cursor-overlay.ts

@@ -0,0 +1,65 @@
+export const CURSOR_OVERLAY_SCRIPT = `(() => {
+  if (globalThis !== globalThis.top) return;
+
+  const cursorId = "__browse_cursor_overlay__";
+  const ensureCursor = () => {
+    const existing = document.getElementById(cursorId);
+    if (existing instanceof HTMLDivElement) return existing;
+
+    const root = document.documentElement || document.body;
+    if (!root) return null;
+
+    const cursor = document.createElement("div");
+    cursor.id = cursorId;
+    cursor.setAttribute("aria-hidden", "true");
+    Object.assign(cursor.style, {
+      contain: "layout style paint",
+      height: "24px",
+      left: "0px",
+      mixBlendMode: "normal",
+      pointerEvents: "none",
+      position: "fixed",
+      top: "0px",
+      userSelect: "none",
+      width: "16px",
+      willChange: "left,top",
+      zIndex: "2147483647",
+    });
+    cursor.innerHTML =
+      '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="24" viewBox="0 0 16 24"><path d="M1 0 L1 22 L6 14 L15 14 Z" fill="black" stroke="white" stroke-width="0.7"/></svg>';
+    root.appendChild(cursor);
+    return cursor;
+  };
+
+  const moveCursor = (x, y) => {
+    const cursor = ensureCursor();
+    if (!cursor) return;
+    cursor.style.left = Math.max(0, x) + "px";
+    cursor.style.top = Math.max(0, y) + "px";
+  };
+
+  globalThis.__browseMoveCursorOverlay__ = moveCursor;
+  ensureCursor();
+  if (!globalThis.__browseCursorOverlayListenerInstalled__) {
+    document.addEventListener(
+      "mousemove",
+      (event) => {
+        moveCursor(event.clientX, event.clientY);
+      },
+      { capture: true },
+    );
+    globalThis.__browseCursorOverlayListenerInstalled__ = true;
+  }
+})()`;
+
+export function updateCursorOverlayPosition(position: {
+  x: number;
+  y: number;
+}): void {
+  const moveCursor = (
+    globalThis as typeof globalThis & {
+      __browseMoveCursorOverlay__?: (x: number, y: number) => void;
+    }
+  ).__browseMoveCursorOverlay__;
+  moveCursor?.(position.x, position.y);
+}