From b2d5b722ec8c45c2ad9e51dbccddf4863977a848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Andr=C3=A9?= Date: Mon, 19 Oct 2026 08:48:14 +0200 Subject: [PATCH] feat(automation): complete safe native window interactions (#1018) Complete application-window interaction on the shared page engine introduced by #1015, while preserving native window/run and document-scoped ref authority. Add real pointer clicks, hover, stepped pointer dragging, wheel scrolling, keyboard chords and bounded text waits. Flat tool commands are validated at both bridge boundaries. Clicks refuse covered targets; press(ref) refuses a target that cannot acquire focus, so Enter never reaches an unrelated draft. Drag requires both ends visible instead of scrolling under a measured source. Preferences remains outside automation; focusing Preferences or a remote window falls back to the most recently used local application window. Trusted native input carries monotonic provenance intervals. Human Form decisions check the originating event timestamp, not the time a stalled renderer handles it; delayed keys and their implicit submit clicks remain automated after lease expiry. Fixed page functions keep their depth marker. Native events use the page's epoch clock, and cleanup preserves the original press timestamp, independently of configured command timeouts. Failed sequences best-effort release held buttons and keys, including when the cleanup mark write itself times out. An admitted input timeout is reported as an unknown outcome with inspection-before-retry guidance: the renderer may deliver the input or finish the click later. Never replay these inputs. Validation covers the actual Mission editor and InterruptionDock in isolated Chromium, including a 16-second renderer freeze and a control human click. Targeted server tests cover key mapping, both bridge parsers, focus/target fences, failure cleanup and late input outcomes. Full server validation, isolated OpenCode 2.0.31 automation, bundle regression and Tauri compilation also pass; typecheck diagnostics remain at baseline. No installed application or shared daemon was changed or restarted. Four independent gatekeeper passes; the final review reports zero actionable P0-P3 findings at 0ffb5a53b425500336854902c7068cd2233695f4. Refs #1008. --- .../skills/codenomad-automation/SKILL.md | 2 +- AGENTS.md | 2 +- dev-docs/BROWSER_AUTOMATION.md | 10 +- packages/electron-app/electron/main/main.ts | 4 +- .../server/src/automation/page/actions.ts | 211 ++++++++++-------- .../server/src/automation/page/input.test.ts | 65 ++++++ packages/server/src/automation/page/input.ts | 140 ++++++++++++ .../server/src/automation/page/keys.test.ts | 35 +++ packages/server/src/automation/page/keys.ts | 115 ++++++++++ packages/server/src/automation/page/parse.ts | 86 +++++++ .../server/src/automation/page/protocol.ts | 151 +++++++++++++ packages/server/src/developer-cdp.test.ts | 65 ++++-- packages/server/src/developer-cdp.ts | 23 +- .../src/opencode/automation-plugin.test.ts | 27 +++ .../server/src/opencode/automation-plugin.ts | 35 +-- .../server/routes/automation-plugin.test.ts | 10 +- .../src/server/routes/automation-plugin.ts | 5 +- .../tauri-app/src-tauri/src/developer_mode.rs | 4 +- packages/ui/src/lib/automation-input.ts | 7 + .../ui/src/lib/form-answer-interaction.ts | 13 +- .../ui/tests/browser/automation-forms.test.ts | 117 ++++------ .../ui/tests/browser/automation-input.test.ts | 186 +++++++++++++++ .../ui/tests/browser/automation-session.ts | 45 ++++ 23 files changed, 1118 insertions(+), 240 deletions(-) create mode 100644 packages/server/src/automation/page/input.test.ts create mode 100644 packages/server/src/automation/page/input.ts create mode 100644 packages/server/src/automation/page/keys.test.ts create mode 100644 packages/server/src/automation/page/keys.ts create mode 100644 packages/server/src/automation/page/parse.ts create mode 100644 packages/server/src/automation/page/protocol.ts create mode 100644 packages/ui/src/lib/automation-input.ts create mode 100644 packages/ui/tests/browser/automation-input.test.ts create mode 100644 packages/ui/tests/browser/automation-session.ts diff --git a/.opencode/skills/codenomad-automation/SKILL.md b/.opencode/skills/codenomad-automation/SKILL.md index 8322a916d..b6ef1b793 100644 --- a/.opencode/skills/codenomad-automation/SKILL.md +++ b/.opencode/skills/codenomad-automation/SKILL.md @@ -8,7 +8,7 @@ description: Inspect, modify, rebuild, relaunch, and verify CodeNomad desktop bu Use this workflow when changing CodeNomad itself: 1. Call `codenomad.inspect` before editing. It identifies the local CodeNomad window and returns an indented accessibility outline with refs, plus runtime diagnostics. Large parts collapse with `(+N hidden)`/`(+N more)`: pass `ref` to expand one, or `query` to find elements by text or role. -2. Use refs from the latest inspections with `codenomad.act`: `click`, `type` (replaces the field's text; `clear: false` appends) and `select` (native `` options by label or value), `hover`, `drag` onto another ref, `scroll`, `press` a key or chord such as `Enter`, `Escape`, `Shift+Tab` or `Mod+K`, and `wait` for text to appear or disappear. Results return the field value, chosen options or focused element to check. Inspect again after navigation, reload or failed actions. 3. Make the smallest source change and run the relevant typecheck, native test, or build. Automation does not own OpenCode: the shared `opencode2` daemon and this session remain alive if CodeNomad closes. 4. After updated artifacts are ready, call `codenomad.act` with `action: "restart"`. The call gracefully relaunches the host pinned by the latest inspection, waits for the same artifact identity to reconnect to the OpenCode session, and returns a fresh inspection. It never switches to another worktree's build. 5. Call `codenomad.screenshot` for final visual evidence and report the checks that ran. diff --git a/AGENTS.md b/AGENTS.md index 8a5771713..009ede393 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,7 @@ - One bundled `codenomad.automation` V2 plugin uses native discovery and backend presence. All browser/developer tools are available without a Developer Mode toggle; native instrumentation starts with the desktop host. Loading, tool availability and execution targeting are separate: retain the authenticated bridge and session/window fences. Tauri preview children receive no application capabilities; primary-renderer reload must dispose them and final-window checks must count native windows. See `dev-docs/BROWSER_AUTOMATION.md`. - Developer UI tools target the inspected native window/run, not the currently selected project or conversation. Keep backend ownership discovery separate from UI focus; agents must be able to inspect and navigate back from another session themselves. Accessibility refs expire on document/target replacement, not merely on session selection. Browser previews retain their own session attachments. -- Page automation (outline, refs, click/type/select) lives once in the transport-agnostic `packages/server/src/automation/page/` engine; inspections stay budgeted plain-text outlines that announce what they collapse, never a silent cut. In the CodeNomad window page functions carry `__codenomadAutomationDepth`, clicks stay programmatic and native text insertion yields only input events, so automation can never produce the trusted dock click/key press a human Form answer requires: never add unmarked trusted input or arbitrary script evaluation there. See `dev-docs/BROWSER_AUTOMATION.md`. +- Page automation (outline, refs, click/type/select/hover/drag/scroll/press/wait) lives once in the transport-agnostic `packages/server/src/automation/page/` engine, with tool input validated by its `parse.ts` on both sides of the bridge; inspections stay budgeted plain-text outlines that announce what they collapse, never a silent cut. Pointer actions aim at the visible centre and refuse covered elements. In the CodeNomad window page functions carry `__codenomadAutomationDepth` and every trusted DevTools input the expiring `__codenomadAutomationInputUntil` lease, both read by `lib/automation-input.ts`, so automation can never produce the trusted dock click/key press a human Form answer requires: never add unmarked trusted input or arbitrary script evaluation there. See `dev-docs/BROWSER_AUTOMATION.md`. - Desktop automation and pruning provisioning follows the authenticated OpenCode connection and its `config.get` global discovery directory, including reconnects. Never derive a running daemon's roots from backend/startup environment or CLI `debug paths`. OpenCode watches the entire discovery root recursively: keep changing presence leases outside it, in the sibling `.codenomad//` namespace. Retain existing outside-root storage; migrate inside-root storage while reading older backends' leases without writing there. WSL translates the daemon-reported paths through the selected distro only for filesystem access. Verify heartbeat stability with the isolated native automation fixture. - Plugins register only at their next presence check, so a backend's first plugin RPC could arrive before registration and be refused. Followers acknowledge a lease with a `.ready` marker (current namespace only) once every follower of that directory in the native process has observed it while registered; routes calling bundled-plugin RPCs await `workspaceManager.desktopPluginsReady()` before reading or authorizing the session, so ownership checks and deletion-fence entry stay adjacent (bounded to 5 s, never rejects). The bound is paid in full only when no marker can arrive: no Location loaded yet, or pre-marker plugin copies still loaded after an upgrade. Do not replace this ordering with retries of refused reads. diff --git a/dev-docs/BROWSER_AUTOMATION.md b/dev-docs/BROWSER_AUTOMATION.md index ef9e85df1..62530b08b 100644 --- a/dev-docs/BROWSER_AUTOMATION.md +++ b/dev-docs/BROWSER_AUTOMATION.md @@ -20,13 +20,17 @@ The bridge remains loopback-only, token-authenticated, bounded, and session-loca ## Page engine -`codenomad.inspect` and `codenomad.act` (the CodeNomad window) run on the shared page engine in `packages/server/src/automation/page/`, which only needs a DevTools protocol channel: `snapshot.ts` renders the accessibility tree, `actions.ts` acts through fixed page functions, and `refs.ts` names DOM nodes. Browser-preview tools still use their native controllers. +`codenomad.inspect` and `codenomad.act` (the CodeNomad window) run on the shared page engine in `packages/server/src/automation/page/`, which only needs a DevTools protocol channel: `snapshot.ts` renders the accessibility tree, `actions.ts` performs actions, `input.ts` dispatches trusted pointer, wheel and keyboard input, `keys.ts` maps key chords, `parse.ts` validates tool input on both sides of the bridge, `protocol.ts` holds the shared protocol primitives and `refs.ts` names DOM nodes. Browser-preview tools still use their native controllers. - Inspection is an indented outline with refs, read as plain text. Wrappers, unnamed images and text repeating its parent's name are dropped; fields show their value instead of their inner nodes; native selects list their options with `selected`/`disabled`. A character budget (32 000 by default) is shared among siblings: small parts stay whole while large ones collapse in place with `(+N hidden)` or `(+N more)`, keeping their first and last items, so nothing is cut silently. `ref` expands one subtree and `query` keeps elements whose name, value or description contains it, or whose role is exactly it, with their contents and ancestors. - A ref names one node within one target, run and navigation epoch. It keeps its name and stays usable for the life of that page generation, whatever is inspected later; a node that left the page fails its action explicitly. - `type` focuses the field and refuses one that cannot take focus (hidden, disabled, behind a modal dialog), because native text insertion goes to the focused element. It then replaces the field's text by default (select, then native insertion, so framework input handlers fire); `clear: false` appends. Date, time, range and color inputs are validated before anything changes, then set with `input`/`change`, keeping the browser's normalization; email and number inputs append as a value. `select` picks native ` options by label or value. restart gracefully relaunches CodeNomad and waits for reconnection.", + description: "Act on refs from the latest codenomad.inspect like a user: click (real pointer at the element's visible centre, refused when covered), type into a field (replacing its text unless clear is false), select native @@ -52,6 +49,7 @@ const EDGES = `
for (const element of document.querySelectorAll("input")) for (const type of ["input", "change"]) element.addEventListener(type, () => window.events.push(type + ":" + element.id)) ` + before(async () => { const cache = await createFixtureCache(), shutdown = createFixtureShutdown(cache) server = await createServer({ configFile: false, root: fileURLToPath(new URL("../..", import.meta.url)), logLevel: "error", cacheDir: cache.cacheDir, @@ -66,44 +64,15 @@ before(async () => { shutdown.own(server) await server.listen() base = `http://127.0.0.1:${(server.httpServer!.address() as { port: number }).port}` - profile = await mkdtemp(path.join(os.tmpdir(), "codenomad-automation-forms-")) - context = await chromium.launchPersistentContext(profile, { executablePath: process.env.CODENOMAD_BROWSER_PATH || undefined, locale: "en-US", - args: ["--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0"] }) - let port = "" - for (let attempt = 0; attempt < 100 && !port; attempt++) { - port = (await readFile(path.join(profile, "DevToolsActivePort"), "utf8").catch(() => "")).split(/\r?\n/, 1)[0] - if (!port) await new Promise(resolve => setTimeout(resolve, 50)) - } - assert.ok(port, "Chromium published its DevTools port") - endpoint = `http://127.0.0.1:${port}` + automation = await launchAutomationBrowser() }) after(async () => { - await context?.close() + await automation?.close() await server?.close() - if (profile) await rm(profile, { recursive: true, force: true }) }) -/** A page that identifies as one CodeNomad window, and the controller pinned to it. */ -async function open(pathname: string, setup?: (page: Page) => Promise) { - const windowId = `automation-window-${++windows}` - const page = await context.newPage() - page.setDefaultTimeout(15_000) - await page.addInitScript(`window.__CODENOMAD_WINDOW_ID__ = ${JSON.stringify(windowId)}`) - await setup?.(page) - await page.goto(`${base}${pathname}`) - const cdp = new DeveloperCdp({ timeoutMs: 10_000 }) - const identity: DeveloperCdpIdentity = { endpoint, runId: `run-${windowId}`, windowId } - const refFor = async (pattern: RegExp, query?: string) => { - const { snapshot } = await cdp.inspect(identity, query ? { query } : {}) - const match = new RegExp(`${pattern.source}[^\\n]*?\\[(e\\d+)\\]`).exec(snapshot) - assert.ok(match, `${pattern} in:\n${snapshot}`) - return match[1] - } - const act = (action: { kind: "click"; ref: string } | { kind: "type"; ref: string; text: string; clear?: boolean } | { kind: "select"; ref: string; values: string[] }) => - cdp.act(action.kind === "type" ? { ...identity, ...action, clear: action.clear !== false } : { ...identity, ...action }) - return { page, cdp, identity, refFor, act, close: async () => { cdp.close(); await page.close() } } -} +const open = (pathname: string, setup?: (page: Page) => Promise) => openAutomationPage(automation, `${base}${pathname}`, setup) test("typing replaces a prefilled field by default, clear:false appends, and selects fire input/change like a user", async () => { const run = await open("/automation-form") @@ -114,32 +83,32 @@ test("typing replaces a prefilled field by default, clear:false appends, and sel assert.match(outline, /combobox "Scenario" \[e\d+\] value="Flexible mission" collapsed\n {4}- option "Flexible mission" \[e\d+\] selected\n {4}- option "Exploration" \[e\d+\]\n {4}- option "Debugging" \[e\d+\] disabled/) const title = await refFor(/textbox "Title"/) - assert.deepEqual(await act({ kind: "type", ref: title, text: "Mission title" }), { value: "Mission title" }) - assert.deepEqual(await act({ kind: "type", ref: title, text: " suffix", clear: false }), { value: "Mission title suffix" }) + assert.deepEqual(await act({ action: "type", ref: title, text: "Mission title" }), { value: "Mission title" }) + assert.deepEqual(await act({ action: "type", ref: title, text: " suffix", clear: false }), { value: "Mission title suffix" }) assert.equal(await page.inputValue("#title"), "Mission title suffix") - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Notes"/), text: "" }), { value: "" }) - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Rich"/), text: "New rich" }), { value: "New rich" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Notes"/), text: "" }), { value: "" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Rich"/), text: "New rich" }), { value: "New rich" }) const due = await refFor(/Date "Due"/) - assert.deepEqual(await act({ kind: "type", ref: due, text: "2026-10-19" }), { value: "2026-10-19" }) + assert.deepEqual(await act({ action: "type", ref: due, text: "2026-10-19" }), { value: "2026-10-19" }) assert.match((await run.cdp.inspect(run.identity, { ref: due })).snapshot, /^- Date "Due" \[e\d+\][^\n]*$/, "a date field is one line, without its inner spinbuttons") const scenario = await refFor(/combobox "Scenario"/) await page.evaluate(() => { (window as any).events.length = 0 }) - assert.deepEqual(await act({ kind: "select", ref: scenario, values: ["Exploration"] }), { selected: ["Exploration"] }) + assert.deepEqual(await act({ action: "select", ref: scenario, values: ["Exploration"] }), { selected: ["Exploration"] }) assert.equal(await page.inputValue("#scenario"), "wayfinder") assert.deepEqual(await page.evaluate(() => (window as any).events), ["input:scenario:1", "change:scenario:1"], "framework handlers see input then change, marked as automation") assert.equal(await page.evaluate(() => (globalThis as any).__codenomadAutomationDepth), 0) - assert.deepEqual(await act({ kind: "select", ref: scenario, values: ["custom"] }), { selected: ["Flexible mission"] }, "values match too") - await assert.rejects(act({ kind: "select", ref: scenario, values: ["Debugging"] }), /Option "Debugging" of e\d+ is disabled/) - await assert.rejects(act({ kind: "select", ref: scenario, values: ["Nope"] }), /No option "Nope" in e\d+/) - await assert.rejects(act({ kind: "select", ref: scenario, values: [] }), /takes exactly one value/) - await assert.rejects(act({ kind: "type", ref: scenario, text: "x" }), /is not a text field/) - await assert.rejects(act({ kind: "type", ref: await refFor(/checkbox "Agree"/), text: "x" }), /is not a text field; use click/) - await assert.rejects(act({ kind: "select", ref: title, values: ["x"] }), /is not a native select/) + assert.deepEqual(await act({ action: "select", ref: scenario, values: ["custom"] }), { selected: ["Flexible mission"] }, "values match too") + await assert.rejects(act({ action: "select", ref: scenario, values: ["Debugging"] }), /Option "Debugging" of e\d+ is disabled/) + await assert.rejects(act({ action: "select", ref: scenario, values: ["Nope"] }), /No option "Nope" in e\d+/) + await assert.rejects(act({ action: "select", ref: scenario, values: [] }), /takes exactly one value/) + await assert.rejects(act({ action: "type", ref: scenario, text: "x" }), /is not a text field/) + await assert.rejects(act({ action: "type", ref: await refFor(/checkbox "Agree"/), text: "x" }), /is not a text field; use click/) + await assert.rejects(act({ action: "select", ref: title, values: ["x"] }), /is not a native select/) // An option ref is clickable: it picks that option, since the native popup is not in the page. - assert.deepEqual(await act({ kind: "click", ref: await refFor(/option "Exploration"/, "Scenario") }), { selected: ["Exploration"] }) + assert.deepEqual(await act({ action: "click", ref: await refFor(/option "Exploration"/, "Scenario") }), { selected: ["Exploration"] }) assert.match((await run.cdp.inspect(run.identity, { query: "scenario" })).snapshot, /combobox "Scenario" \[e\d+\] value="Exploration"/) } finally { await run.close() } }) @@ -159,40 +128,40 @@ test("typing never lands in another field, and normalized inputs keep the browse await page.evaluate(() => { document.querySelector("details")!.open = true }) const hidden = await refFor(/textbox "Hidden"/) await page.evaluate(() => { document.querySelector("details")!.open = false }) - await assert.rejects(act({ kind: "type", ref: hidden, text: "Lost" }), /cannot take focus/) - await assert.rejects(act({ kind: "type", ref: await refFor(/textbox "Locked"/), text: "Lost" }), /is disabled/) + await assert.rejects(act({ action: "type", ref: hidden, text: "Lost" }), /cannot take focus/) + await assert.rejects(act({ action: "type", ref: await refFor(/textbox "Locked"/), text: "Lost" }), /is disabled/) assert.equal(await page.evaluate(() => document.activeElement?.id), "outside") assert.equal(await page.inputValue("#outside"), "Kept", "nothing was typed into the focused field instead") const modalOutside = await refFor(/textbox "Outside"/) await page.evaluate(() => (document.getElementById("modal") as HTMLDialogElement).showModal()) - await assert.rejects(act({ kind: "type", ref: modalOutside, text: "Lost" }), /cannot take focus/) + await assert.rejects(act({ action: "type", ref: modalOutside, text: "Lost" }), /cannot take focus/) assert.equal(await page.inputValue("#modal-field"), "", "the dialog's field received nothing") await page.evaluate(() => (document.getElementById("modal") as HTMLDialogElement).close()) - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Mail"/), text: "c", clear: false }), { value: "abc" }) - assert.deepEqual(await act({ kind: "type", ref: await refFor(/spinbutton "Count"/), text: "3", clear: false }), { value: "123" }) - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Mail"/), text: "x@y.z" }), { value: "x@y.z" }, "clearing works there too") + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Mail"/), text: "c", clear: false }), { value: "abc" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/spinbutton "Count"/), text: "3", clear: false }), { value: "123" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Mail"/), text: "x@y.z" }), { value: "x@y.z" }, "clearing works there too") // Through its label's text, which designates the labelled control. - assert.deepEqual(await act({ kind: "type", ref: await refFor(/text "Tint"/), text: "#FF0000" }), { value: "#ff0000" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/text "Tint"/), text: "#FF0000" }), { value: "#ff0000" }) const volume = await refFor(/slider "Volume"/) - assert.deepEqual(await act({ kind: "type", ref: volume, text: "55" }), { value: "60" }, "ranges snap to their step") + assert.deepEqual(await act({ action: "type", ref: volume, text: "55" }), { value: "60" }, "ranges snap to their step") await page.evaluate(() => { (window as any).events.length = 0 }) for (const text of ["0x50", " 70", ""]) { - await assert.rejects(act({ kind: "type", ref: volume, text }), /is not a valid value/, `range ${JSON.stringify(text)}`) + await assert.rejects(act({ action: "type", ref: volume, text }), /is not a valid value/, `range ${JSON.stringify(text)}`) } - await assert.rejects(act({ kind: "type", ref: await refFor(/text "Tint"/), text: "" }), /is not a valid value/, "a color has no empty value") - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Mail"/), text: "", clear: false }), { value: "x@y.z" }) + await assert.rejects(act({ action: "type", ref: await refFor(/text "Tint"/), text: "" }), /is not a valid value/, "a color has no empty value") + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Mail"/), text: "", clear: false }), { value: "x@y.z" }) assert.deepEqual([await page.inputValue("#volume"), await page.inputValue("#tint")], ["60", "#ff0000"]) assert.deepEqual(await page.evaluate(() => (window as any).events), [], "refused or unchanged values fire nothing") - await assert.rejects(act({ kind: "type", ref: await refFor(/Date "Day"/), text: "2026-13-45" }), /"2026-13-45" is not a valid value/) + await assert.rejects(act({ action: "type", ref: await refFor(/Date "Day"/), text: "2026-13-45" }), /"2026-13-45" is not a valid value/) assert.equal(await page.inputValue("#day"), "2026-01-02", "an invalid date changes nothing") assert.deepEqual(await page.evaluate(() => (window as any).events), []) const second = /option "Same" \[(e\d+)\][^\n]*\n\s*- option "Same" \[(e\d+)\]/.exec((await run.cdp.inspect(run.identity, { query: "Twins" })).snapshot)![2] - await act({ kind: "click", ref: second }) + await act({ action: "click", ref: second }) assert.equal(await page.inputValue("#twins"), "s2", "an option click picks that exact option, not the first with its label") - assert.deepEqual(await act({ kind: "type", ref: await refFor(/textbox "Story"/), text: " end", clear: false }), { value: "one\ntwo end" }) + assert.deepEqual(await act({ action: "type", ref: await refFor(/textbox "Story"/), text: " end", clear: false }), { value: "one\ntwo end" }) } finally { await run.close() } }) @@ -222,17 +191,17 @@ test("an agent fills the real Mission editor: the title is replaced, not concate }) try { const { page, act, refFor } = run - await act({ kind: "click", ref: await refFor(/button "Create mission"/, "Create mission") }) - await act({ kind: "type", ref: await refFor(/textbox "What should the mission do\?"/, "What should the mission do?"), text: "Chart the module seams" }) + await act({ action: "click", ref: await refFor(/button "Create mission"/, "Create mission") }) + await act({ action: "type", ref: await refFor(/textbox "What should the mission do\?"/, "What should the mission do?"), text: "Chart the module seams" }) const title = await refFor(/textbox "Title"/, "Title") - await act({ kind: "type", ref: title, text: "First draft" }) - assert.deepEqual(await act({ kind: "type", ref: title, text: "Seam survey" }), { value: "Seam survey" }, "a second type replaces the first") - assert.deepEqual(await act({ kind: "type", ref: title, text: " (pilot)", clear: false }), { value: "Seam survey (pilot)" }) - await act({ kind: "click", ref: await refFor(/DisclosureTriangle "Options"/, "Options") }) - assert.deepEqual(await act({ kind: "select", ref: await refFor(/combobox "Playbook"/, "Playbook"), values: ["Exploration"] }), { selected: ["Exploration"] }) + await act({ action: "type", ref: title, text: "First draft" }) + assert.deepEqual(await act({ action: "type", ref: title, text: "Seam survey" }), { value: "Seam survey" }, "a second type replaces the first") + assert.deepEqual(await act({ action: "type", ref: title, text: " (pilot)", clear: false }), { value: "Seam survey (pilot)" }) + await act({ action: "click", ref: await refFor(/DisclosureTriangle "Options"/, "Options") }) + assert.deepEqual(await act({ action: "select", ref: await refFor(/combobox "Playbook"/, "Playbook"), values: ["Exploration"] }), { selected: ["Exploration"] }) assert.match((await run.cdp.inspect(run.identity, { query: "Playbook" })).snapshot, /combobox "Playbook" \[e\d+\] value="Exploration"/) await page.waitForFunction(() => !(document.querySelector('form.mission-editor button[type="submit"]') as HTMLButtonElement)?.disabled) - await act({ kind: "click", ref: await refFor(/button "Create"(?= \[)/, "Create") }) + await act({ action: "click", ref: await refFor(/button "Create"(?= \[)/, "Create") }) await page.waitForFunction(() => document.querySelector("form.mission-editor") === null) assert.equal(creates.length, 1) assert.deepEqual([creates[0].title, creates[0].objective, creates[0].template], ["Seam survey (pilot)", "Chart the module seams", "wayfinder"]) diff --git a/packages/ui/tests/browser/automation-input.test.ts b/packages/ui/tests/browser/automation-input.test.ts new file mode 100644 index 000000000..8a39bae05 --- /dev/null +++ b/packages/ui/tests/browser/automation-input.test.ts @@ -0,0 +1,186 @@ +import assert from "node:assert/strict" +import { after, before, test } from "node:test" +import { fileURLToPath } from "node:url" +import { createServer, type ViteDevServer } from "vite" +import solid from "vite-plugin-solid" +import { launchAutomationBrowser, openAutomationPage } from "./automation-session" +import { createFixtureCache } from "./fixture-cache" +import { createFixtureShutdown } from "./fixture-shutdown" +import { prepareInterruptionDock } from "./fixtures/interruption-dock-preparation" + +// Trusted pointer, wheel and keyboard input from the real codenomad.act controller. +let server: ViteDevServer, automation: Awaited>, base: string + +const INPUTS = `
+ + idle + +
tall
+
+
+
+
+
+
+

+
` + +before(async () => { + const cache = await createFixtureCache(), shutdown = createFixtureShutdown(cache) + server = await createServer({ configFile: false, root: fileURLToPath(new URL("../..", import.meta.url)), logLevel: "error", cacheDir: cache.cacheDir, + plugins: [solid(), shutdown.plugin, { name: "automation-input", configureServer(s) { + s.middlewares.use("/automation-inputs", (_req, res) => { res.setHeader("Content-Type", "text/html"); res.end(INPUTS) }) + s.middlewares.use("/automation-dock", async (_req, res) => { + res.setHeader("Content-Type", "text/html") + res.end(await s.transformIndexHtml("/automation-dock", '
')) + }) + } }], resolve: { dedupe: ["solid-js"] }, optimizeDeps: { exclude: ["lucide-solid"] }, server: { host: "127.0.0.1", port: 0, hmr: false, watch: null } }) + shutdown.own(server) + await server.listen() + await prepareInterruptionDock(server) + base = `http://127.0.0.1:${(server.httpServer!.address() as { port: number }).port}` + automation = await launchAutomationBrowser() +}) + +after(async () => { + await automation?.close() + await server?.close() +}) + +const log = (page: import("playwright").Page) => page.evaluate(() => (window as any).log.splice(0) as string[]) + +test("keys, hover, wheel, drag and pointer clicks behave like a user and are always marked as automation", async () => { + const run = await openAutomationPage(automation, `${base}/automation-inputs`) + try { + const { page, act, refFor } = run + const query = await refFor(/textbox "Query"/) + await act({ action: "type", ref: query, text: "hello" }) + await log(page) + assert.deepEqual(await act({ action: "press", key: "Enter", ref: query }), { focused: "input \"Query\"" }) + assert.deepEqual(await log(page), ["keydown:query:Enter:true:true", "click:BUTTON::true:true", "submit:hello"], + "a trusted Enter submits the form through its default button; the key and the implicit click are both marked") + assert.deepEqual(await act({ action: "press", key: "Tab" }), { focused: "button \"Go\"" }, "Tab moves focus like a keyboard") + assert.equal(await page.evaluate(() => (globalThis as any).__codenomadAutomationInputUntil <= performance.now()), true, + "the mark ends after each action but retains its timestamp for queued events") + + await act({ action: "hover", ref: await refFor(/button "Hover me"/) }) + assert.equal(await page.textContent("#hover-state"), "hovered") + + await log(page) + await act({ action: "click", ref: await refFor(/button "Open menu"/) }) + assert.deepEqual((await log(page)).filter(entry => entry.startsWith("pointerdown")), ["pointerdown:menu-trigger::true:true"]) + await act({ action: "click", ref: await refFor(/menuitem "Menu item"/, "Menu item") }) + assert.ok((await log(page)).includes("menu-item"), "a pointerdown-only menu opens and its item is clickable") + + await act({ action: "scroll", ref: await refFor(/region "Scroller"/), direction: "down", amount: 500 }) + await page.waitForFunction(() => document.getElementById("scroller")!.scrollTop > 0) + + await act({ action: "drag", ref: await refFor(/button "Handle"/), to: await refFor(/region "Drop zone"/) }) + assert.equal(await page.evaluate(() => (window as any).dropped), true) + + await assert.rejects(act({ action: "click", ref: await refFor(/button "Under"/) }), /is covered by div "Cover"/) + assert.deepEqual((await log(page)).filter(entry => entry.includes("under")), [], "a covered element receives nothing") + + assert.equal((await act({ action: "wait", text: "Ready now", timeout: 5_000 })).found, true) + assert.equal((await act({ action: "wait", text: "Hover me", gone: true, timeout: 300 }).catch(error => error.message)), "\"Hover me\" is still shown after 300 ms") + await assert.rejects(act({ action: "press", key: "Hyper+Q" }), /Unknown modifier "Hyper"/) + } finally { await run.close() } +}) + +test("a hidden press target never submits the focused draft, and dragging cannot scroll under its start point", async () => { + const run = await openAutomationPage(automation, `${base}/automation-inputs`) + try { + const { page, act, refFor } = run + await page.evaluate(() => { + const details = document.createElement("details") + details.open = true + details.innerHTML = 'Target' + document.body.append(details) + }) + const hidden = await refFor(/textbox "Hidden target"/) + await page.evaluate(() => { document.querySelector("details")!.open = false }) + await page.focus("#query") + await log(page) + await assert.rejects(act({ action: "press", ref: hidden, key: "Enter" }), /cannot take focus/) + assert.deepEqual(await log(page), [], "no key or implicit submit reached the focused field") + + await page.evaluate(() => { + const list = document.createElement("section") + list.innerHTML = Array.from({ length: 35 }, (_, index) => ``).join("") + document.body.append(list) + }) + const first = await refFor(/button "Row 1"/), last = await refFor(/button "Row 35"/) + await page.getByRole("button", { name: "Row 1", exact: true }).scrollIntoViewIfNeeded() + const before = await page.evaluate(() => scrollY) + await log(page) + await assert.rejects(act({ action: "drag", ref: first, to: last }), /no visible box to drag to/) + assert.equal(await page.evaluate(() => scrollY), before, "dragging never scrolls the page under a saved start point") + assert.deepEqual(await log(page), [], "no row was pressed") + } finally { await run.close() } +}) + +/** Answers the real InterruptionDock question and returns the human-answer header it sent. */ +async function answerDock(via: "click" | "press" | "press-stalled" | "click-stalled" | "human"): Promise { + const run = await openAutomationPage(automation, `${base}/automation-dock`, async page => { + await page.route("**/api/**", route => route.fulfill({ contentType: "application/json", body: "{}" })) + }) + try { + const { page, act, refFor } = run + await page.waitForFunction(() => Boolean((window as any).fixture?.snapshot().ids.length)) + await page.evaluate(() => (window as any).fixture.ask()) + const field = page.locator('.interruption-dock input[type="text"]:visible') + await field.waitFor() + if (via === "human") { + await field.fill("Human answer") + await page.locator('.interruption-dock button[type="submit"]:visible').click() + } else { + const ref = await refFor(/textbox/, "textbox") + await act({ action: "type", ref, text: `Automation ${via}` }) + if (via === "press-stalled" || via === "click-stalled") { + await page.evaluate(({ event, duration }) => window.addEventListener(event, () => { + const end = performance.now() + duration + while (performance.now() < end) { /* Renderer stalled beyond the 15-second input lease. */ } + }, { capture: true, once: true }), { event: via === "press-stalled" ? "keydown" : "pointerdown", duration: via === "press-stalled" ? 16_000 : 25_000 }) + const input = via === "press-stalled" ? { action: "press", key: "Enter", ref } + : { action: "click", ref: await refFor(/button "Submit"/, "Submit") } + await act(input).catch(error => { + if (!/timed out/.test(error.message)) throw error + }) + } else if (via === "press") await act({ action: "press", key: "Enter", ref }) + else await act({ action: "click", ref: await refFor(/button "Submit"/, "Submit") }) + } + await page.waitForFunction(() => (window as any).fixture.replies.length === 1) + const sent = await page.evaluate(() => ({ answer: (window as any).fixture.replies[0].answer, options: (window as any).fixture.replyOptions[0] })) + assert.deepEqual(sent.answer, { q0: via === "human" ? "Human answer" : `Automation ${via}` }) + return sent.options?.headers?.["x-codenomad-human-answer"] + } finally { await run.close() } +} + +test("a dock answer given through trusted automation clicks or keys never carries the human-answer mark", async () => { + assert.equal(await answerDock("click"), undefined, "a trusted automation click is not a human answer") + assert.equal(await answerDock("press"), undefined, "a trusted automation Enter is not a human answer") + assert.equal(await answerDock("press-stalled"), undefined, "a key queued past lease expiry and its default click still are not human") + assert.equal(await answerDock("click-stalled"), undefined, "a cleanup release after two 10-second command timeouts retains its original provenance") + assert.equal(await answerDock("human"), "1", "control: a real user click on the same dock is") +}) diff --git a/packages/ui/tests/browser/automation-session.ts b/packages/ui/tests/browser/automation-session.ts new file mode 100644 index 000000000..2ab8fa525 --- /dev/null +++ b/packages/ui/tests/browser/automation-session.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { chromium, type BrowserContext, type Page } from "playwright" +import { DeveloperCdp, type DeveloperCdpIdentity } from "../../../server/src/developer-cdp" +import { parsePageAction } from "../../../server/src/automation/page/parse" + +/** A real Chromium with its DevTools endpoint, driven by the real codenomad.inspect/act + * controller exactly as it drives the CodeNomad window. */ +export async function launchAutomationBrowser() { + const profile = await mkdtemp(path.join(os.tmpdir(), "codenomad-automation-")) + const context = await chromium.launchPersistentContext(profile, { executablePath: process.env.CODENOMAD_BROWSER_PATH || undefined, locale: "en-US", + args: ["--remote-debugging-address=127.0.0.1", "--remote-debugging-port=0"] }) + let port = "" + for (let attempt = 0; attempt < 100 && !port; attempt++) { + port = (await readFile(path.join(profile, "DevToolsActivePort"), "utf8").catch(() => "")).split(/\r?\n/, 1)[0] + if (!port) await new Promise(resolve => setTimeout(resolve, 50)) + } + assert.ok(port, "Chromium published its DevTools port") + return { context, endpoint: `http://127.0.0.1:${port}`, close: async () => { await context.close(); await rm(profile, { recursive: true, force: true }) } } +} + +let windows = 0 + +/** A page that identifies as one CodeNomad window, and the controller pinned to it. */ +export async function openAutomationPage(browser: { context: BrowserContext; endpoint: string }, url: string, setup?: (page: Page) => Promise) { + const windowId = `automation-window-${++windows}` + const page = await browser.context.newPage() + page.setDefaultTimeout(15_000) + await page.addInitScript(`window.__CODENOMAD_WINDOW_ID__ = ${JSON.stringify(windowId)}`) + await setup?.(page) + await page.goto(url) + const cdp = new DeveloperCdp({ timeoutMs: 10_000 }) + const identity: DeveloperCdpIdentity = { endpoint: browser.endpoint, runId: `run-${windowId}`, windowId } + const refFor = async (pattern: RegExp, query?: string) => { + const { snapshot } = await cdp.inspect(identity, query ? { query } : {}) + const match = new RegExp(`${pattern.source}[^\\n]*?\\[(e\\d+)\\]`).exec(snapshot) + assert.ok(match, `${pattern} in:\n${snapshot}`) + return match[1] + } + /** Tool-shaped input, validated by the same parser as codenomad.act. */ + const act = async (input: Record) => cdp.act(identity, parsePageAction(input, "inspection")!) + return { page, cdp, identity, refFor, act, close: async () => { cdp.close(); await page.close() } } +}