vrr/qwen-code
2
0
Fork 0
mirror of https://github.com/QwenLM/qwen-code.git synced 2026-10-21 14:21:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 29

feat(cua-driver): add versioned Computer Use SDK and release pipeline (#9587)

Browse source 
* chore(cua-driver): sync upstream v0.20.0

* feat(cua-driver): add Computer Use SDK with versioned observation revisions

Wrap the typed driver SDK in a standalone Node wrapper and add accessibility.observation_revision.v1: base-anchored validated diffs with opaque element tokens, explicit full-resync reasons, full-only answers on Windows/Linux. Fix portable include_screenshot schema type and classify stable kAXErrorFailure refusals as complete in the macOS capture tracker.

* feat(cua-driver): complete typed Computer Use capabilities

* fix(cua-driver): use Qwen-owned npm identity

* feat(cua-driver): publish one Qwen CUA SDK package

* fix(cua-driver): verify Windows Rust targets

* fix(cua-driver): verify macOS Rust targets

* fix(cua-driver): pin release Rust toolchain

* fix(cua-sdk): fail closed on incomplete releases

* test(cua-sdk): import workflow test globals

* ci(cua-sdk): retry Debian package downloads

* fix(cua-driver): harden lifecycle and release gates

---------

Co-authored-by: tutu <tutu@U-RD4R9MQQ-2235.local>
This commit is contained in:
顾盼 2026-08-23 14:20:14 +00:00 • committed by GitHub
parent 0b953b7929
commit b5fbdb22d3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
278 changed files with 53164 additions and 5784 deletions
Download patch file Download diff file Expand all files Collapse all files

348
.github/workflows/cd-cua-driver.yml vendored
View file

@ -17,6 +17,8 @@ name: 'CD: Qwen CUA Driver'
# QwenCuaDriver.app) + *-binary.tar.gz (runtime + SDK payload)
# Linux : cua-driver-rs-<v>-linux-{x86_64,arm64}.tar.gz + *-binary.tar.gz
# Windows: cua-driver-rs-<v>-windows-{x86_64,arm64}.zip + *-binary.zip
# npm : one platform-neutral @qwen-code/cua-sdk package. Its postinstall
# downloads and verifies the matching *-binary archive above.
on:
push:
tags: ['cua-driver-rs-v*']
@ -25,16 +27,18 @@ on:
version:
description: 'Version to release (without v prefix)'
required: true
default: '0.17.0'
default: '0.20.0'
notarize:
description: 'Codesign + notarize the macOS artifacts (false to skip during
description:
'Codesign + notarize the macOS artifacts (false to skip during
iteration)'
required: false
type: 'boolean'
default: true
dry_run:
description: 'Build + package only; do NOT create/update a GitHub Release.
Leave on (with notarize=false) to rehearse the build pipeline.'
description:
'Build all release assets and clean-install the npm package without
creating a GitHub Release, npm version, tag, or installer sync PR.'
required: false
type: 'boolean'
default: true
@ -44,11 +48,17 @@ jobs:
validate-version:
name: 'validate-release-version'
runs-on: 'ubuntu-latest'
outputs:
version: '${{ steps.release.outputs.version }}'
steps:
- uses: 'actions/checkout@v4'
with:
fetch-depth: 0
- name: 'Validate release version'
id: 'release'
shell: 'bash'
run: |
set -euo pipefail
if [[ "$GITHUB_REF" == refs/tags/cua-driver-rs-v* ]]; then
VERSION="${GITHUB_REF#refs/tags/cua-driver-rs-v}"
else
@ -60,7 +70,35 @@ jobs:
echo "::error::Release version $VERSION does not match Cargo version $CARGO_VERSION"
exit 1
fi
- uses: 'dtolnay/rust-toolchain@stable'
SDK_NAME=$(node -p "require('./packages/cua-driver/typescript/package.json').name")
SDK_VERSION=$(node -p "require('./packages/cua-driver/typescript/package.json').version")
if [[ "$SDK_NAME" != "@qwen-code/cua-sdk" ]]; then
echo "::error::Expected @qwen-code/cua-sdk, found $SDK_NAME"
exit 1
fi
if [[ "$SDK_VERSION" != "$VERSION" ]]; then
echo "::error::SDK version $SDK_VERSION does not match release version $VERSION"
exit 1
fi
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "$GITHUB_REF" != "refs/heads/main" ]]; then
echo "::error::A production dispatch must run from protected main"
exit 1
fi
if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.dry_run }}" != "true" && "${{ inputs.notarize }}" != "true" ]]; then
echo "::error::A production dispatch must enable macOS codesigning and notarization"
exit 1
fi
if [[ "$GITHUB_REF" == refs/tags/cua-driver-rs-v* ]]; then
git fetch origin main
if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then
echo "::error::Release tag must point to a commit reachable from main"
exit 1
fi
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- uses: 'dtolnay/rust-toolchain@1.97.1'
- name: 'Test model payload filtering'
working-directory: 'packages/cua-driver/rust'
run: 'cargo test -p cua-driver-core --locked model_payload'
@ -88,8 +126,8 @@ jobs:
steps:
- name: 'Install base tooling (container is bare)'
run: |
apt-get update
apt-get install -y --no-install-recommends \
apt-get -o Acquire::Retries=3 update
apt-get -o Acquire::Retries=3 install -y --no-install-recommends \
git ca-certificates curl build-essential pkg-config \
libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \
libxkbcommon-dev
@ -109,7 +147,7 @@ jobs:
VERSION="${{ inputs.version }}"
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- uses: 'dtolnay/rust-toolchain@stable'
- uses: 'dtolnay/rust-toolchain@1.97.1'
with:
targets: '${{ matrix.target }}'
- uses: 'actions/cache@v4'
@ -118,8 +156,9 @@ jobs:
~/.cargo/registry
~/.cargo/git
packages/cua-driver/rust/target
key: 'cua-driver-linux-${{ matrix.arch }}-${{ hashFiles(''packages/cua-driver/rust/Cargo.lock'')
}}'
key:
"cua-driver-linux-${{ matrix.arch }}-${{ hashFiles('packages/cua-driver/rust/Cargo.lock')
}}"
restore-keys: |
cua-driver-linux-${{ matrix.arch }}-
- name: 'Build (release)'
@ -187,7 +226,7 @@ jobs:
VERSION="${{ inputs.version }}"
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- uses: 'dtolnay/rust-toolchain@stable'
- uses: 'dtolnay/rust-toolchain@1.97.1'
with:
targets: '${{ matrix.target }}'
- uses: 'actions/cache@v4'
@ -196,8 +235,9 @@ jobs:
~/.cargo/registry
~/.cargo/git
packages/cua-driver/rust/target
key: 'cua-driver-windows-${{ matrix.arch }}-${{ hashFiles(''packages/cua-driver/rust/Cargo.lock'')
}}'
key:
"cua-driver-windows-${{ matrix.arch }}-${{ hashFiles('packages/cua-driver/rust/Cargo.lock')
}}"
restore-keys: |
cua-driver-windows-${{ matrix.arch }}-
- name: 'Build (release, unsigned)'
@ -243,7 +283,7 @@ jobs:
env:
# Sign+notarize on tag push (real release); on manual dispatch honor the
# `notarize` input so the pipeline can be iterated without Apple's notary.
DO_NOTARIZE: '${{ startsWith(github.ref, ''refs/tags/cua-driver-rs-v'') || inputs.notarize == true }}'
DO_NOTARIZE: "${{ startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || inputs.notarize == true }}"
steps:
- uses: 'actions/checkout@v4'
- uses: 'actions/setup-node@v4'
@ -261,7 +301,7 @@ jobs:
VERSION="${{ inputs.version }}"
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
- uses: 'dtolnay/rust-toolchain@stable'
- uses: 'dtolnay/rust-toolchain@1.97.1'
with:
targets: 'aarch64-apple-darwin,x86_64-apple-darwin'
- uses: 'actions/cache@v4'
@ -270,7 +310,7 @@ jobs:
~/.cargo/registry
~/.cargo/git
packages/cua-driver/rust/target
key: 'cua-driver-darwin-universal-${{ hashFiles(''packages/cua-driver/rust/Cargo.lock'') }}'
key: "cua-driver-darwin-universal-${{ hashFiles('packages/cua-driver/rust/Cargo.lock') }}"
restore-keys: |
cua-driver-darwin-universal-
- name: 'Build arm64'
@ -324,7 +364,7 @@ jobs:
otool -l release/universal/libcua_driver_sdk.dylib | \
grep -A2 LC_RPATH | grep -Fq 'path @loader_path'
- name: 'Import signing certificate + discover identity'
if: 'env.DO_NOTARIZE == ''true'''
if: "env.DO_NOTARIZE == 'true'"
env:
MAC_CSC_LINK: '${{ secrets.MAC_CSC_LINK }}'
MAC_CSC_KEY_PASSWORD: '${{ secrets.MAC_CSC_KEY_PASSWORD }}'
@ -351,7 +391,7 @@ jobs:
echo "SIGN_IDENTITY=$IDENTITY" >> "$GITHUB_ENV"
echo "Signing identity: $IDENTITY"
- name: 'Codesign universal binary (hardened runtime)'
if: 'env.DO_NOTARIZE == ''true'''
if: "env.DO_NOTARIZE == 'true'"
working-directory: 'packages/cua-driver/rust'
run: |
codesign --force --timestamp --options runtime \
@ -374,13 +414,15 @@ jobs:
cp release/universal/cua-cursor-theme release/QwenCuaDriver.app/Contents/MacOS/cua-cursor-theme
chmod +x release/QwenCuaDriver.app/Contents/MacOS/qwen-cua-driver
chmod +x release/QwenCuaDriver.app/Contents/MacOS/cua-cursor-theme
plutil -replace CFBundleShortVersionString -string "$VERSION" release/QwenCuaDriver.app/Contents/Info.plist
plutil -replace CFBundleVersion -string "$VERSION" release/QwenCuaDriver.app/Contents/Info.plist
BUNDLE_VERSION="${VERSION%%-*}"
plutil -replace CFBundleShortVersionString -string "$BUNDLE_VERSION" release/QwenCuaDriver.app/Contents/Info.plist
plutil -replace CFBundleVersion -string "$BUNDLE_VERSION" release/QwenCuaDriver.app/Contents/Info.plist
rm -f release/QwenCuaDriver.app/Contents/MacOS/.gitkeep
plutil -p release/QwenCuaDriver.app/Contents/Info.plist | grep -E 'CFBundle(Short)?Version'
- name: 'Codesign + notarize + staple QwenCuaDriver.app (App Store Connect API
- name:
'Codesign + notarize + staple QwenCuaDriver.app (App Store Connect API
key)'
if: 'env.DO_NOTARIZE == ''true'''
if: "env.DO_NOTARIZE == 'true'"
working-directory: 'packages/cua-driver/rust'
env:
APPLE_NOTARY_API_KEY_P8_BASE64: '${{ secrets.APPLE_NOTARY_API_KEY_P8_BASE64 }}'
@ -402,7 +444,7 @@ jobs:
--issuer "$APPLE_NOTARY_ISSUER_ID" \
--wait --timeout 20m
xcrun stapler staple release/QwenCuaDriver.app
spctl -a -vv -t exec release/QwenCuaDriver.app || true
spctl -a -vv -t exec release/QwenCuaDriver.app
rm -f release/QwenCuaDriver.app.zip notary_key.p8
- name: 'Package'
working-directory: 'packages/cua-driver/rust'
@ -470,20 +512,95 @@ jobs:
esac
done
verify-sdk-package:
name: 'single npm package dry-run'
needs: ['validate-version', 'build-linux', 'verify-release-artifacts']
runs-on: 'ubuntu-latest'
permissions:
contents: 'read'
steps:
- uses: 'actions/checkout@v4'
- uses: 'actions/setup-node@v4'
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: 'packages/cua-driver/typescript/package-lock.json'
registry-url: 'https://registry.npmjs.org'
scope: '@qwen-code'
- name: 'Install npm 11'
run: 'npm install --global npm@11.19.0'
- uses: 'actions/download-artifact@v4'
with:
name: 'cua-driver-rs-linux-x86_64'
path: '${{ runner.temp }}/cua-driver-linux'
- name: 'Extract verified native payload'
id: 'native'
shell: 'bash'
run: |
set -euo pipefail
ARCHIVE=$(find "${RUNNER_TEMP}/cua-driver-linux" -type f -name '*-binary.tar.gz' -print -quit)
if [[ -z "$ARCHIVE" ]]; then
echo '::error::Linux x86_64 binary archive is missing'
exit 1
fi
NATIVE_DIR="${RUNNER_TEMP}/cua-sdk-native"
RELEASE_DIR="${RUNNER_TEMP}/cua-sdk-release-fixture"
mkdir -p "$NATIVE_DIR"
mkdir -p "$RELEASE_DIR"
tar -xzf "$ARCHIVE" -C "$NATIVE_DIR"
test -f "$NATIVE_DIR/libcua_driver_sdk.so"
test -f "$NATIVE_DIR/cua_driver_node_runtime.node"
cp "$ARCHIVE" "$RELEASE_DIR/"
(cd "$RELEASE_DIR" && sha256sum "$(basename "$ARCHIVE")" > checksums.txt)
echo "directory=$NATIVE_DIR" >> "$GITHUB_OUTPUT"
echo "release_directory=$RELEASE_DIR" >> "$GITHUB_OUTPUT"
- name: 'Install SDK dependencies'
working-directory: 'packages/cua-driver/typescript'
run: 'npm ci --ignore-scripts --no-audit --no-fund'
- name: 'Test SDK and native loader'
working-directory: 'packages/cua-driver/typescript'
env:
CUA_DRIVER_REQUIRE_UNIFFI: '1'
QWEN_CUA_SDK_NATIVE_DIR: '${{ steps.native.outputs.directory }}'
run: 'npm test'
- name: 'Pack, publish-dry-run, and clean-install one SDK tarball'
env:
QWEN_CUA_SDK_NATIVE_DIR: '${{ steps.native.outputs.directory }}'
run: |
node packages/cua-driver/scripts/verify-cua-sdk-package.mjs \
--native-dir "${QWEN_CUA_SDK_NATIVE_DIR}" \
--release-dir "${{ steps.native.outputs.release_directory }}" \
--output-dir "${RUNNER_TEMP}/cua-sdk-package"
- uses: 'actions/upload-artifact@v4'
with:
name: 'cua-sdk-npm-${{ needs.validate-version.outputs.version }}'
path: '${{ runner.temp }}/cua-sdk-package/*.tgz'
if-no-files-found: 'error'
retention-days: 7
# ── Release ──────────────────────────────────────────────────────────────
release:
name: 'Create GitHub Release'
needs: ['build-linux', 'build-windows', 'build-macos', 'verify-release-artifacts']
needs:
[
'build-linux',
'build-windows',
'build-macos',
'verify-release-artifacts',
'verify-sdk-package',
]
runs-on: 'ubuntu-latest'
outputs:
version: '${{ steps.version.outputs.version }}'
if: 'startsWith(github.ref, ''refs/tags/cua-driver-rs-v'') || (github.event_name
== ''workflow_dispatch'' && inputs.dry_run == false)'
if:
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
== 'workflow_dispatch' && inputs.dry_run == false)"
steps:
- uses: 'actions/checkout@v4'
- uses: 'actions/download-artifact@v4'
with:
path: 'release-upload'
pattern: 'cua-driver-rs-*'
merge-multiple: true
- name: 'Determine version + tag'
id: 'version'
@ -511,8 +628,54 @@ jobs:
mkdir -p "$SKILLS_STAGE"
cp -R packages/cua-driver/rust/Skills/cua-driver/. "$SKILLS_STAGE/"
tar -czf "release-upload/${SKILLS_STAGE}.tar.gz" "$SKILLS_STAGE"
(cd release-upload && sha256sum cua-driver-rs-*.tar.gz cua-driver-rs-*.zip > checksums.txt)
(
cd release-upload
: > checksums.txt
for file in *; do
[[ -f "$file" && "$file" != checksums.txt ]] || continue
sha256sum "$file" >> checksums.txt
done
)
- name: 'Refuse to replace an existing release'
id: 'release-state'
env:
GH_TOKEN: '${{ github.token }}'
TAG: '${{ steps.version.outputs.tag }}'
shell: 'bash'
run: |
set -euo pipefail
if ! gh release view "$TAG" >/dev/null 2>&1; then
echo 'exists=false' >> "$GITHUB_OUTPUT"
exit 0
fi
git fetch origin "refs/tags/${TAG}:refs/tags/${TAG}"
TAG_COMMIT=$(git rev-list -n 1 "$TAG")
if [[ "$TAG_COMMIT" != "$GITHUB_SHA" ]]; then
echo "::error::Existing $TAG points to $TAG_COMMIT, not $GITHUB_SHA"
exit 1
fi
EXISTING=$(mktemp -d)
trap 'rm -rf "$EXISTING"' EXIT
gh release download "$TAG" --dir "$EXISTING"
find release-upload -maxdepth 1 -type f -exec basename {} \; | sort > "$EXISTING/expected-assets.txt"
gh release view "$TAG" --json assets --jq '.assets[].name' | sort > "$EXISTING/actual-assets.txt"
if ! diff -u "$EXISTING/expected-assets.txt" "$EXISTING/actual-assets.txt"; then
echo "::error::Existing $TAG asset set is incomplete or unexpected"
exit 1
fi
grep -v '^checksums.txt$' "$EXISTING/expected-assets.txt" > "$EXISTING/expected-checksums.txt"
sed -nE 's/^[0-9a-f]{64} [ *](.+)$/\1/p' "$EXISTING/checksums.txt" | sort > "$EXISTING/actual-checksums.txt"
if ! diff -u "$EXISTING/expected-checksums.txt" "$EXISTING/actual-checksums.txt"; then
echo "::error::Existing $TAG checksum manifest is incomplete or unexpected"
exit 1
fi
(cd "$EXISTING" && sha256sum --check checksums.txt)
echo "Existing $TAG is complete and checksum-valid; reusing it without replacement"
echo 'exists=true' >> "$GITHUB_OUTPUT"
- name: 'Create GitHub Release'
if: "steps.release-state.outputs.exists != 'true'"
uses: 'softprops/action-gh-release@v2'
with:
tag_name: '${{ steps.version.outputs.tag }}'
@ -528,6 +691,9 @@ jobs:
- **macOS**: codesigned + notarized universal binary + `QwenCuaDriver.app`
- **Linux**: unsigned (x86_64 + arm64, glibc 2.31 floor)
- **Windows**: unsigned (x86_64 + arm64)
- **Node.js**: the same workflow publishes the single
`@qwen-code/cua-sdk` npm package after a clean install against
these release assets
Enable relative coordinates: `CUA_DRIVER_RS_COORDINATE_SPACE=1`
(default `0` = off; optional `CUA_DRIVER_RS_COORDINATE_SCALE=1000`).
@ -535,9 +701,135 @@ jobs:
Enable textual MCP payload filtering for affected model API routes:
`MCP_MODEL_PAYLOAD_FILTER=1` (default off).
publish-sdk:
name: 'Publish @qwen-code/cua-sdk'
needs: ['validate-version', 'verify-sdk-package', 'release']
if:
"startsWith(github.ref, 'refs/tags/cua-driver-rs-v') || (github.event_name
== 'workflow_dispatch' && inputs.dry_run == false)"
runs-on: 'ubuntu-latest'
environment:
name: 'production-release'
url: 'https://www.npmjs.com/package/@qwen-code/cua-sdk/v/${{ needs.validate-version.outputs.version }}'
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: 'actions/setup-node@v4'
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
scope: '@qwen-code'
- name: 'Install npm 11'
run: 'npm install --global npm@11.19.0'
- uses: 'actions/download-artifact@v4'
with:
name: 'cua-sdk-npm-${{ needs.validate-version.outputs.version }}'
path: '${{ runner.temp }}/cua-sdk-package'
- name: 'Require npm publishing token'
env:
NODE_AUTH_TOKEN: '${{ secrets.NPM_TOKEN }}'
run: |
if [[ -z "$NODE_AUTH_TOKEN" ]]; then
echo '::error::production-release NPM_TOKEN is required for the first @qwen-code/cua-sdk publication'
exit 1
fi
- name: 'Clean-install against the published GitHub Release'
env:
QWEN_CUA_SDK_CACHE_DIR: '${{ runner.temp }}/public-release-cache'
VERSION: '${{ needs.validate-version.outputs.version }}'
run: |
set -euo pipefail
TARBALL=$(find "${RUNNER_TEMP}/cua-sdk-package" -type f -name '*.tgz' -print -quit)
if [[ -z "$TARBALL" ]]; then
echo '::error::Packed @qwen-code/cua-sdk tarball is missing'
exit 1
fi
CHECKSUMS_URL="https://github.com/QwenLM/qwen-code/releases/download/cua-driver-rs-v${VERSION}/checksums.txt"
for attempt in {1..12}; do
if curl -fsSL --connect-timeout 15 --max-time 60 "$CHECKSUMS_URL" -o "${RUNNER_TEMP}/checksums.txt"; then
break
fi
if [[ "$attempt" == 12 ]]; then
echo '::error::Published release checksums did not become available'
exit 1
fi
sleep 10
done
CONSUMER=$(mktemp -d)
trap 'rm -rf "$CONSUMER"' EXIT
cd "$CONSUMER"
npm init --yes >/dev/null
npm install "$TARBALL" --no-audit --no-fund
node --input-type=module --eval '
import { CuaDriver } from "@qwen-code/cua-sdk";
import { ComputerUse } from "@qwen-code/cua-sdk/computer-use";
const driver = CuaDriver.create(undefined);
const computer = await ComputerUse.create({ session: "release-registry-smoke" });
try {
const listing = JSON.parse(await driver.listToolsJson());
if (!Array.isArray(listing.tools) || listing.tools.length === 0) {
throw new Error("native SDK returned no tools");
}
const revisionSupported = await computer.supportsObservationRevision();
if (typeof revisionSupported !== "boolean") {
throw new Error("ComputerUse capability probe returned an invalid result");
}
console.log(JSON.stringify({
toolCount: listing.tools.length,
revisionSupported,
}));
} finally {
await computer.close();
await driver.shutdown();
driver.uniffiDestroy();
}
'
- name: 'Publish immutable SDK tarball'
env:
NODE_AUTH_TOKEN: '${{ secrets.NPM_TOKEN }}'
VERSION: '${{ needs.validate-version.outputs.version }}'
run: |
set -euo pipefail
TARBALL=$(find "${RUNNER_TEMP}/cua-sdk-package" -type f -name '*.tgz' -print -quit)
LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)"
REMOTE_INTEGRITY=$(npm view "@qwen-code/cua-sdk@${VERSION}" dist.integrity 2>/dev/null || true)
if [[ -n "$REMOTE_INTEGRITY" ]]; then
if [[ "$REMOTE_INTEGRITY" != "$LOCAL_INTEGRITY" ]]; then
echo "::error::@qwen-code/cua-sdk@${VERSION} already exists with different integrity"
exit 1
fi
echo "@qwen-code/cua-sdk@${VERSION} already contains this exact tarball"
exit 0
fi
NPM_TAG=latest
[[ "$VERSION" == *-* ]] && NPM_TAG=next
npm publish "$TARBALL" --provenance --access public --tag "$NPM_TAG"
- name: 'Verify npm registry integrity'
env:
VERSION: '${{ needs.validate-version.outputs.version }}'
run: |
set -euo pipefail
TARBALL=$(find "${RUNNER_TEMP}/cua-sdk-package" -type f -name '*.tgz' -print -quit)
LOCAL_INTEGRITY="sha512-$(openssl dgst -sha512 -binary "$TARBALL" | openssl base64 -A)"
for attempt in {1..20}; do
REMOTE_INTEGRITY=$(npm view "@qwen-code/cua-sdk@${VERSION}" dist.integrity 2>/dev/null || true)
if [[ "$REMOTE_INTEGRITY" == "$LOCAL_INTEGRITY" ]]; then
echo "Verified npm registry integrity on attempt $attempt"
exit 0
fi
if [[ -n "$REMOTE_INTEGRITY" ]]; then
echo '::error::Published npm artifact integrity does not match the verified tarball'
exit 1
fi
sleep 10
done
echo '::error::Published npm version did not become visible'
exit 1
sync-installer-version:
name: 'Sync installer version to main'
needs: ['release']
needs: ['release', 'publish-sdk']
runs-on: 'ubuntu-latest'
permissions:
contents: 'write'

70
docs/design/cua-driver-0.20.0-upstream-sync.md Normal file
View file

@ -0,0 +1,70 @@
# CUA Driver 0.20.0 Upstream Sync
## Source of truth
The synchronization target is the stable Cua Driver release
`cua-driver-rs-v0.20.0` from `trycua/cua`, resolved to commit
`bb8c86049cad1bf0853c6d25c03c14875d0d047f`. The npm `latest` dist-tag for
`@trycua/cua-driver` also resolves to `0.20.0`. The newer `0.20.1` artifacts
are nightly prereleases and are not used as a release baseline.
The existing Qwen snapshot is `cua-driver-rs-v0.17.0` at commit
`10279552e2bbe479e367a082f78b1b98ee85a697`. The supported
`packages/cua-driver/scripts/sync-from-upstream.sh` delta is checked against a
three-way merge with 0.17.0 as the common ancestor.
## Adopted upstream behavior
The 0.20.0 runtime becomes the vendored base, including:
- transport-owned implicit lifecycle sessions and explicit per-action targets;
- capability manifests applied across permission profiles;
- typed SDK, contract, and generated binding updates;
- foreground-focus verification and background-input hardening;
- policy-filtered tool discovery and named CLI session behavior;
- stable/nightly release-channel support; and
- removal of legacy browser approval tokens.
The removed browser-token path is not retained as a Qwen compatibility fork.
Existing-profile access continues only through trusted launch grants, bounded
manifests, or an embedding host's authorization callback.
## Qwen invariants
The sync must preserve these downstream-owned boundaries:
- executable, app, bundle, service, install, update, and release identities
remain Qwen-owned;
- telemetry remains disabled by default and, when explicitly enabled, goes
directly to the documented upstream endpoint;
- `MCP_MODEL_PAYLOAD_FILTER=1` remains an explicit Qwen-facing payload filter;
- `CUA_DRIVER_RS_COORDINATE_SPACE=1` remains the opt-in normalized-coordinate
adapter, with absolute pixels as the default;
- the release installation continues to use `~/.cua-driver` for compatibility,
while source builds retain their separate Qwen identity; and
- the still-open `trycua/cua#2021` empty-title Windows-window patch remains
carried and documented.
## Release and packaging boundary
The Qwen release workflow keeps its existing signing, notarization, artifact
names, and publication ownership. It advances its manual default to 0.20.0,
uses the restored upstream TypeScript lockfile required by `npm ci`, and keeps
macOS bundle versions valid when a prerelease suffix is present.
The upstream tag left its standalone Agent SDK examples pinned to 0.19.2 even
though both npm and PyPI publish 0.20.0. The vendored examples and their npm
lockfile are aligned to 0.20.0 so they exercise the synchronized contract.
Upstream's monorepo-wide nightly orchestration, release-attribution services,
Python publishing, and documentation publishing are not copied into Qwen Code.
They depend on upstream-only scripts, secrets, repositories, and release
governance. Cross-platform signed release publication remains a CI gate.
## Out of scope
This change does not update Qwen Code's current built-in downloader pin,
Computer Use MCP adapter, bootstrap flow, permission UX, or model-visible tool
schemas. It also does not implement Issue #9334. Those integration changes
remain independently reviewable follow-ups on top of the verified 0.20.0
runtime and TypeScript SDK.

143
docs/design/cua-driver-computer-use-sdk.md Normal file
View file

@ -0,0 +1,143 @@
# CUA Driver Computer Use SDK
## Goal
Upgrade the vendored CUA Driver 0.20.0 implementation with a native, versioned accessibility observation revision protocol, then expose the resulting capability through its typed SDKs and a small JavaScript wrapper.
The finished SDK must be directly importable from an ordinary Node.js program and independently testable without Qwen Code, Node REPL, a Skill, or any Qwen host integration. Stage 3 will teach Qwen Code to call this SDK from the Node REPL delivered by #9333.
## Stage boundary
This stage includes only:
- cua-driver core and native platform changes;
- Rust, Python, and TypeScript SDK contract and generated bindings;
- a thin JavaScript Computer Use wrapper over the TypeScript cua-driver SDK;
- independent unit, compatibility, platform E2E, packaging, and release validation.
This stage does not:
- modify Qwen Code core, CLI, ACP, TUI, tool registry, scheduler, or permission manager;
- register any SDK or capability in the Node REPL;
- add a hidden Qwen bridge or Qwen-specific runtime session;
- track which observation reached a model;
- add a Computer Use Skill, prompt, default migration, or direct-tool replacement.
## Runtime topology
```text
ordinary Node.js program
-> @qwen-code/cua-sdk/computer-use
-> @qwen-code/cua-sdk typed driver API
-> cua-driver runtime
-> AX / UIA / AT-SPI platform implementation
```
The wrapper calls the typed SDK directly. It does not route calls through Qwen Code or a second tool protocol. It preserves cua-driver's existing runtime, permission, transport, and lifecycle behavior.
The TypeScript SDK and Computer Use wrapper are distributed together as the
single `@qwen-code/cua-sdk` npm package. The package root exposes the generated
typed driver API and the `/computer-use` subpath exposes the high-level wrapper.
There is no driver npm package and there are no platform npm packages.
The matching `qwen-cua-driver` GitHub Release remains the only native artifact
channel. During npm installation, `@qwen-code/cua-sdk` downloads the exact
same-version binary archive, verifies it against that release's
`checksums.txt`, and caches only the SDK library plus Node runtime. An explicit
native-directory override supports source builds and release dry-runs without
changing the production resolution path. The synchronized TryCua release is
source provenance only; no published Qwen artifact imports or resolves the
upstream npm package.
## Release contract
One version identifies both release surfaces:
1. `cua-driver-rs-v<version>` publishes the driver, SDK library, Node runtime,
installers, and checksums to the Qwen GitHub Release.
2. `@qwen-code/cua-sdk@<version>` publishes the platform-neutral JavaScript,
generated bindings, declarations, downloader, and Computer Use wrapper.
Production publication is ordered. The GitHub Release must be complete before
the packed npm artifact is installed without overrides against the public
release. Only after that real installation and native-load smoke test succeeds
may npm publication run. A retry accepts an already-published npm version only
when its registry integrity matches the packed artifact.
The workflow dry-run builds every native target, verifies the release archive
contract, packs exactly one npm tarball, installs it into a clean consumer
project using the just-built native payload, and runs the native-load smoke
test. It creates no tag, GitHub Release, npm version, installer-version PR, or
machine-wide driver installation.
## Observation revision contract
CUA Driver adds the opt-in `accessibility.observation_revision.v1` capability. Existing callers that do not opt in continue receiving the current full snapshot and snapshot-token behavior.
A revision request explicitly supplies:
- protocol version;
- exact target;
- optional base revision ID;
- optional force-full flag;
- serializer/projection version.
A response reports:
- `full | diff | no_change` mode;
- current and actual base revision IDs;
- lineage, serializer, and projection versions;
- stable-element support;
- a closed resynchronization reason when full output is required.
The caller, not cua-driver, selects the base revision. Missing, expired, foreign, or incompatible bases produce a full response. The driver never guesses whether a revision reached a model.
## Stable identity and diff correctness
Within one trusted driver session, runtime generation, exact target, and serializer lineage:
- the same native element keeps its stable ID;
- rename, value change, reorder, and reparent retain the ID;
- inserted elements receive new IDs;
- removed IDs retire;
- destroyed and recreated look-alikes receive new IDs.
Every candidate diff must replay from the requested base to the canonical current full rendering. Incomplete capture, unavailable identity, incompatible lineage, failed replay, or a diff not smaller than the full rendering returns full output with an explicit reason.
Stable action tokens resolve through the current revision. An unchanged element remains actionable after compatible diffs; removed, recreated, foreign-session, or stale-generation tokens fail before native dispatch.
## Platform identity
| Platform path | Identity rule | Revision v1 behavior |
| ------------------ | ----------------------------------------------------------------- | ----------------------------------------------- |
| macOS AX | retained `AXUIElementRef`, compared with Core Foundation equality | diff when capture and identity are complete |
| Windows UIA | RuntimeId candidate confirmed by `IUIAutomation::CompareElements` | diff when identity is confirmed |
| Windows MSAA | no approved stable identity | explicit full-only fallback |
| Linux AT-SPI | unique D-Bus owner plus object path | diff while the owner remains live and unchanged |
| Linux X11 fallback | no approved stable identity | explicit full-only fallback |
Provider invalidation, truncation, subtree read failure, target ambiguity, or fallback capture forces full output.
## SDK wrapper
The JavaScript wrapper exposes a small Computer Use API while directly using the generated TypeScript cua-driver SDK. It hides raw low-level constructors and arbitrary tool dispatch from its public surface, but it does not depend on Qwen Code.
The observation API returns the revision ID and accepts an explicit base revision ID on the next call. The caller owns that state. Actions consume opaque element tokens returned by the driver. The wrapper does not compute a second semantic diff or invent element identity.
The wrapper must run in a standalone Node.js integration test before Stage 3 begins.
## Validation
Completion requires:
- deterministic core tests for full, diff, no-change, replay, eviction, isolation, and forced-full reasons;
- unchanged compatibility fixtures for existing Rust, Python, and TypeScript applications;
- generated SDK drift checks;
- real signed macOS, Windows UIA, and Linux AT-SPI E2E evidence;
- explicit MSAA and X11 full-only evidence;
- stable-token action tests after rename, insertion, reparent, removal, and recreation;
- a standalone Node.js test importing and using the JavaScript wrapper directly;
- at least 30 deterministic real transitions and a median accessibility-text reduction of at least 40% for small UI changes;
- clean build, typecheck, packaging, and release artifacts.
No Qwen Code or model-in-the-loop result is part of this stage's completion claim.

39
docs/plans/2026-08-20-cua-driver-0.20.0-upstream-sync.md Normal file
View file

@ -0,0 +1,39 @@
# CUA Driver 0.20.0 Upstream Sync Plan
## Delivery boundary
Deliver one pure prerequisite change before Issue #9334: move the vendored CUA
Driver from the released 0.17.0 snapshot to released 0.20.0 while preserving
the documented Qwen distribution and compatibility patches. Do not change the
built-in Computer Use adapter or downloader pin.
## Sequence
1. Verify the official Git tag, commit, npm dist-tag, and release metadata.
2. Apply the 0.17.0 to 0.20.0 `libs/cua-driver` delta with the repository sync
script.
3. Cross-check every reject through a three-way merge using upstream 0.17.0 as
the common ancestor.
4. Adopt 0.20 lifecycle, target, authorization, SDK, and browser-token behavior;
retain Qwen identity, telemetry, coordinate, payload-filter, and Windows
empty-title-window changes.
5. Restore the package lockfiles required by clean `npm ci` builds, align the
standalone SDK examples with the published 0.20.0 packages, and advance
Qwen release metadata to 0.20.0.
6. Validate source provenance, version parity, generated contracts, language
SDKs, Rust packages, installer behavior, release wiring, and the enclosing
Qwen workspace.
7. Run a debug runtime smoke test and record platform-specific or signed-release
gaps separately from source failures.
8. Read the complete tracked and untracked diff in open-ended passes and require
two consecutive clean passes after the final fix.
## Stop conditions requiring a new decision
- changing Qwen's built-in Computer Use downloader or bootstrap behavior;
- replacing Qwen identities, install paths, signing ownership, or telemetry
default with upstream values;
- copying upstream-only publication infrastructure or introducing new release
secrets;
- dropping the documented local Windows patch before it is merged upstream; or
- adding Issue #9334 model-facing APIs to this synchronization change.

34
docs/plans/2026-08-20-cua-driver-computer-use-sdk.md Normal file
View file

@ -0,0 +1,34 @@
# CUA Driver Computer Use SDK implementation plan
1. Add the opt-in `accessibility.observation_revision.v1` request and response contract while preserving all legacy full-snapshot callers.
2. Finish the shared revision lineage, deterministic renderer, replay validation, bounded retention, session cleanup, stable IDs, and current-element resolution in cua-driver core.
3. Complete macOS AX identity, capture-completeness, invalidation/refetch, and signed real-platform E2E.
4. Implement Windows UIA identity with RuntimeId candidates plus `CompareElements`; keep MSAA explicitly full-only.
5. Implement Linux AT-SPI identity with unique D-Bus owner plus object path; keep X11 fallback explicitly full-only.
6. Expose revision-v1 through the Rust contract and generated Python and TypeScript SDKs, including compatibility/version negotiation.
7. Publish the generated TypeScript SDK and thin standalone Computer Use wrapper
as one `@qwen-code/cua-sdk` package. Load native code only from the matching
Qwen GitHub Release; do not create driver or platform npm packages.
8. Make the CUA Driver release workflow build every native target, pack and
clean-install the single SDK package in dry-run, and publish the GitHub
Release before the npm package in production.
9. Validate deterministic transitions, stable-token actions, lifecycle cleanup,
compatibility fixtures, generated binding drift, platform E2E, context
reduction, packaging, and release artifacts.
## Explicit exclusions
Do not modify Qwen Code core, CLI, ACP, TUI, Node REPL, tool registration, scheduler, permission manager, prompts, or Skills in this stage. Do not add a Qwen host bridge or model-delivery tracking. Those belong to Stage 3 (#9335).
## Current checkpoint
The shared revision core, macOS AX, Windows UIA, Linux AT-SPI, generated SDK
exposure, direct JavaScript wrapper, and lifecycle cleanup are present in the
working tree. macOS has real native E2E evidence, including 30 deterministic
transitions and token-bound actions. Windows UIA/MSAA and Linux AT-SPI/X11 have
compile-time and deterministic-test coverage but still require real target E2E
before this stage can claim cross-platform completion. The earlier root-plus-
platform npm staging result is invalid for the approved distribution model. A
single-package local clean install, Release-fixture postinstall, native smoke,
and packaged AppKit E2E now pass; the GitHub-hosted all-platform workflow
dry-run remains the release gate.

2
packages/cua-driver/.vendored-from
View file

@ -1 +1 @@
cua-driver-rs-v0.17.0
cua-driver-rs-v0.20.0

4
packages/cua-driver/.vendored-patches.md
View file

@ -1,12 +1,12 @@
# Cherry-picked upstream PRs
This vendored copy is based on the tag in [`.vendored-from`](./.vendored-from)
(`cua-driver-rs-v0.17.0`), **plus** a small set of upstream trycua/cua pull
(`cua-driver-rs-v0.20.0`), **plus** a small set of upstream trycua/cua pull
requests applied as ordinary commits on top of the base snapshot.
| Upstream PR | Fix | Status | Touches |
|---|---|---|---|
| [trycua/cua#2021](https://github.com/trycua/cua/pull/2021) | `list_windows` includes empty-/null-title top-level windows | open (draft); `resolve_uwp_host_window` is already in the 0.17.0 base | `platform-windows` |
| [trycua/cua#2021](https://github.com/trycua/cua/pull/2021) | `list_windows` includes empty-/null-title top-level windows | open; `resolve_uwp_host_window` is already in the 0.20.0 base | `platform-windows` |
## Retired patches (subsumed by 0.7.0)

33
packages/cua-driver/README.md
View file

@ -4,7 +4,7 @@ Qwen Code's vendored distribution of the cross-platform Cua Driver runtime.
It provides native desktop and browser automation over MCP, a one-shot CLI,
and in-process Python and TypeScript SDKs.
This tree is based on upstream `cua-driver-rs-v0.17.0`. The upstream snapshot
This tree is based on upstream `cua-driver-rs-v0.20.0`. The upstream snapshot
is recorded in [`.vendored-from`](.vendored-from); Qwen-owned differences are
documented in [`.vendored-patches.md`](.vendored-patches.md) and
[`docs/relative-coordinates-design.md`](docs/relative-coordinates-design.md).
@ -14,18 +14,18 @@ documented in [`.vendored-patches.md`](.vendored-patches.md) and
macOS and Linux:
```bash
CUA_DRIVER_RS_VERSION=0.17.0 \
CUA_DRIVER_RS_VERSION=0.20.0 \
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.sh)"
```
Windows PowerShell:
```powershell
$env:CUA_DRIVER_RS_VERSION = "0.17.0"
$env:CUA_DRIVER_RS_VERSION = "0.20.0"
irm https://raw.githubusercontent.com/QwenLM/qwen-code/main/packages/cua-driver/scripts/install.ps1 | iex
```
Expected: qwen-cua-driver 0.17.0.
Expected: qwen-cua-driver 0.20.0.
The released product uses Qwen-owned identities throughout:
@ -59,16 +59,20 @@ qwen mcp add cua-driver qwen-cua-driver mcp
Other MCP clients can use the same executable and arguments. Shell-oriented
automation can call tools through `qwen-cua-driver call`.
## 0.17 runtime and SDK surface
## 0.20 runtime and SDK surface
The 0.17 base includes the SDK-owned runtime and versioned C ABI, generated
The 0.20 base includes the SDK-owned runtime and versioned C ABI, generated
Python and TypeScript UniFFI bindings, typed browser automation, permission
modes, runtime-owned consent adapters, per-session capture scope,
snapshot-bound element tokens, closed action results, `verify_state`, native
menu invocation, clipboard tools, window framing, and semantic cursor themes.
modes, runtime-owned consent adapters, transport-owned implicit lifecycle
sessions, per-action target selection, capability manifests across permission
profiles, snapshot-bound element tokens, closed action results, `verify_state`,
foreground-focus verification, native menu and clipboard operations, window
framing, and semantic cursor themes. Browser approval tokens are retired;
existing-profile access now requires a trusted launch grant, bounded manifest,
or embedding-host authorization.
Python applications import `cua_driver`. TypeScript applications retain the
upstream-compatible `@trycua/cua-driver` package name. Both use the same
Python applications import `cua_driver`. TypeScript applications import the
Qwen-owned `@qwen-code/cua-sdk` package. Both use the same
in-process native runtime; MCP remains the agent-facing boundary implemented by
`qwen-cua-driver`.
@ -82,6 +86,13 @@ and generated bindings are documented in [`contract/README.md`](contract/README.
admits only reviewed tools and resources. `unrestricted` requires
`--dangerously-bypass-approvals`.
The mode belongs to the process that owns the runtime and is fixed at launch:
`qwen-cua-driver serve` takes the flags, while `qwen-cua-driver mcp` and embedding hosts
use the matching `CUA_DRIVER_PERMISSION_MODE`,
`CUA_DRIVER_CAPABILITY_MANIFEST_FILE`, and
`CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED` variables. Choose it before starting
the daemon; a running daemon must be restarted to change it.
Attaching to an existing logged-in Chromium profile remains explicit:
```bash

12
packages/cua-driver/compat-fixtures/cli.json
View file

@ -2,14 +2,12 @@
"help_lines": [
"cross-platform computer-use automation driver",
"Usage: cua-driver [SUBCOMMAND] [OPTIONS]",
"Subcommands: mcp, list-tools, describe, call, serve, stop, revoke, status, config, telemetry, recording, update, check-update, doctor, diagnose, permissions, autostart, skills, browser-approve, manifest"
"Subcommands: mcp, list-tools, describe, call, serve, stop, revoke, status, config, telemetry, recording, update, check-update, doctor, diagnose, permissions, autostart, skills, manifest"
],
"manifest": {
"schema_version": "1",
"binary_version_format": "semver",
"mcp_args": [
"mcp"
],
"mcp_args": ["mcp"],
"subcommands": {
"mcp": [
["--socket", "string"],
@ -23,6 +21,8 @@
["--permission-mode", "string"],
["--grant", "repeatable-string"],
["--dangerously-bypass-approvals", "flag"],
["--capability-manifest", "string"],
["--approve-capability-manifest", "flag"],
["--session-policy", "string"],
["--approve-session-policy", "flag"],
["--no-permissions-gate", "flag"],
@ -35,9 +35,7 @@
["--screenshot-out-file", "string"],
["--socket", "string"]
],
"manifest": [
["--pretty", "flag"]
]
"manifest": [["--pretty", "flag"]]
}
}
}

22
packages/cua-driver/contract/README.md
View file

@ -18,19 +18,25 @@ surface through their runtime's existing MCP client.
The typed slice covers the cross-platform session lifecycle tools:
- `start_session`
- `escalate_session`
- `get_session_state`
- `get_session`
- `list_sessions`
- `end_session`
Ordinary calls do not need `start_session`. The runtime creates one implicit
session for the authenticated transport lease and reuses it until transport
close, explicit end, or five minutes of inactivity. `escalate_session` and
`get_session_state` remain deprecated compatibility tools for legacy
capture-scope sessions. There is no `deescalate_session` tool.
It also covers the portable whole-desktop loop:
- `get_desktop_state`
- `get_screen_size`
- `get_cursor_position`
- `move_cursor` with the required `scope="desktop"`
- `move_cursor` with an exact per-call window or desktop `target`
- `set_window_frame` for exact, read-back-verified top-level window geometry
- `invoke_menu` for an exact native application-menu path resolved live at each hop
- `click` with the required `scope="desktop"`
- `click` with an exact per-call window or desktop `target`
- `drag` and `scroll` in native desktop coordinates
- `type_text`, `press_key`, and `hotkey` against the foreground application
- `clipboard_read` for available types and opt-in plain-text readback
@ -61,7 +67,11 @@ The checked-observation slice is also shared by MCP and both generated SDKs:
harness to interpret. Cua Driver does not OCR or assign task meaning to it.
Session contracts are marked `canonical_runtime`: the same typed Rust input,
output, and metadata declaration builds the live MCP tool. Desktop contracts
output, and metadata declaration builds the live MCP tool. The preferred action
target is a tagged union: `{kind:"window", pid, window_id}` or
`{kind:"desktop", display_id:"primary"}`. Legacy flat `scope`, `pid`, and
`window_id` fields remain accepted during the compatibility window but cannot
be mixed with `target`. Desktop contracts
are marked `portable_subset`: their typed Rust inputs are a deliberately
narrower projection of the richer macOS, Linux, and Windows runtime schemas.
Each platform's desktop branch deserializes that projection before acting,
@ -89,7 +99,7 @@ Compatibility is tracked separately at each boundary:
| Field | Current | Meaning |
| --- | --- | --- |
| `contract_version` | `0.6.0` | Generated manifest and typed SDK shape |
| `contract_version` | `0.7.0` | Generated manifest and typed SDK shape |
| `tools_list_schema_version` | `1` | cua-driver `tools/list` extension shape |
| `capability_version` | `1` | Additive capability-token vocabulary |
| `mcp_protocol_version` | `2025-06-18` | MCP initialization protocol served to agent runtimes |

1229
packages/cua-driver/contract/manifest.json
View file

File diff suppressed because it is too large Load diff

30
packages/cua-driver/docs/action-icon-catalog.md
View file

@ -170,10 +170,12 @@ distinct resolved action IDs.
| Public tool | Resolved action keys |
| -------------------------- | ----------------------------------------------------- |
| `start_session` | `session.start.{auto\|window\|desktop}` |
| `escalate_session` | `session.escalate.desktop` |
| `get_session_state` | `session.inspect` |
| `start_session` | `session.start.{implicit\|named}` |
| `get_session` | `session.inspect.one` |
| `list_sessions` | `session.inspect.list` |
| `end_session` | `session.end` |
| `escalate_session` | `session.legacy.escalate.desktop` |
| `get_session_state` | `session.legacy.inspect_capture` |
| `start_recording` | `recording.start.trajectory`, `recording.start.video` |
| `stop_recording` | `recording.stop` |
| `get_recording_state` | `recording.inspect` |
@ -215,7 +217,7 @@ public tool names.
| 16 | `get_screen_size` | Inspection | All | Screen dimensions |
| 17 | `get_desktop_state` | Inspection | All | Desktop capture |
| 18 | `get_cursor_position` | Inspection | All | Cursor location |
| 19 | `move_cursor` | Pointer/overlay input | All | Agent-cursor move in window scope; real-pointer move in desktop scope |
| 19 | `move_cursor` | Pointer/overlay input | All | Agent-cursor or real-pointer move selected by the per-call target |
| 20 | `set_agent_cursor_enabled` | Agent cursor | All | Show/hide cursor |
| 21 | `set_agent_cursor_motion` | Agent cursor | All | Motion configuration |
| 22 | `set_agent_cursor_theme` | Agent cursor | All | Select an installed visual theme |
@ -241,16 +243,18 @@ public tool names.
| 42 | `get_recording_state` | Recording | All | Recording inspect |
| 43 | `replay_trajectory` | Recording | All | Replay |
| 44 | `install_ffmpeg` | Maintenance | Windows/Linux; no-op when unnecessary | Plan/install dependency |
| 45 | `start_session` | Session lifecycle | All | Start session plus capture scope |
| 46 | `escalate_session` | Session lifecycle | All | Escalate to desktop |
| 47 | `get_session_state` | Session lifecycle | All | Session inspect |
| 45 | `start_session` | Session lifecycle | All | Optional implicit or named lifecycle start |
| 46 | `get_session` | Session lifecycle | All | Inspect one visible lifecycle session |
| 47 | `list_sessions` | Session lifecycle | All | List transport-visible lifecycle sessions |
| 48 | `end_session` | Session lifecycle | All | End session |
| 49 | `check_for_update` | Maintenance | All | Update check |
| 50 | `debug_window_info` | Diagnostic | Windows only | Window diagnostic |
| 51 | `mouse_button_down` | Low-level pointer | Linux only | Hold button |
| 52 | `mouse_drag` | Low-level pointer | Linux only | Move held pointer |
| 53 | `mouse_button_up` | Low-level pointer | Linux only | Release button |
| 54 | `parallel_mouse_drag` | Multi-pointer | Linux only | Concurrent drags |
| 49 | `escalate_session` | Legacy session compatibility | All | Deprecated capture-scope escalation |
| 50 | `get_session_state` | Legacy session compatibility | All | Deprecated capture-scope inspect |
| 51 | `check_for_update` | Maintenance | All | Update check |
| 52 | `debug_window_info` | Diagnostic | Windows only | Window diagnostic |
| 53 | `mouse_button_down` | Low-level pointer | Linux only | Hold button |
| 54 | `mouse_drag` | Low-level pointer | Linux only | Move held pointer |
| 55 | `mouse_button_up` | Low-level pointer | Linux only | Release button |
| 56 | `parallel_mouse_drag` | Multi-pointer | Linux only | Concurrent drags |
## Linux low-level pointer action keys

3
packages/cua-driver/docs/action-result-contract.md
View file

@ -39,7 +39,8 @@ The action-result tools are:
`click`, `double_click`, `right_click`, `scroll`, `drag`, `mouse_drag`,
`parallel_mouse_drag`, `move_cursor`, `mouse_button_down`, `mouse_button_up`,
`type_text`, `type_text_chars`, `press_key`, `hotkey`, `set_value`,
`browser_click`, `browser_pointer`, and `browser_type`.
`set_window_frame`, `invoke_menu`, `browser_click`, `browser_pointer`, and
`browser_type`.
Other mutating tools such as application launch, window activation, browser
navigation, dialogs, uploads, and downloads retain their own typed results.

11
packages/cua-driver/docs/cursor-modifier-badge-restructuring-plan.md
View file

@ -13,7 +13,7 @@ Cua Driver should present two kinds of cursor information in two clear places:
- delivery: background or foreground
- target: AX, pixel, browser, or desktop
The badge remains lightweight. The session name fades using the existing two-second hold and 400-millisecond fade. Modifier chips appear only while their action is active, followed by a 400-millisecond trailing fade. When the name is hidden, the badge contracts to a compact session orb and active modifier chips. Hovering the pointer restores the full session name where hover observation is supported.
The badge remains lightweight. The session name fades using the existing two-second hold and 400-millisecond fade. Modifier chips appear only while their action is active, followed by a 400-millisecond trailing fade. When the name is hidden, the badge contracts to a compact session-colored capsule and active modifier chips. Hovering the pointer restores the full session name where hover observation is supported.
This is visual feedback only. It is not an authorization indicator, a security boundary, or evidence that an action succeeded.
@ -40,13 +40,13 @@ Use one pill rather than adding a second persistent status capsule.
| Badge content | Visibility |
| ------------------------- | --------------------------------------------------------------------------------------- |
| Session orb and name | Existing reveal, two-second hold, 400-millisecond fade; hover restores full opacity |
| Session name | Existing reveal, two-second hold, 400-millisecond fade; hover restores full opacity |
| Delivery and target chips | Full opacity while modifiers are active; 400-millisecond trailing fade after they clear |
| Badge chrome | Visible while either the name or modifier chips are visible |
Starting an action must not restart the session-name timer. Otherwise a long session name would flash on every tool call. Modifier changes reveal or update the chips only.
When the name has faded, the pill contracts to the session orb and active chips. If no public session label exists, modifier-bearing actions still get this compact session-colored capsule. This preserves context without requiring a display name.
When the name has faded, the pill contracts to the session-colored capsule and active chips. If no public session label exists, modifier-bearing actions still get this compact session-colored capsule. This preserves context without requiring a display name.
### Host-owned modifier glyphs
@ -127,7 +127,6 @@ pub struct SessionBadgeInput<'a> {
pub struct SessionBadgeLayout {
pub rect: Rect,
pub orb: Rect,
pub label: Option<BadgeLabelLayout>,
pub delivery_chip: Option<BadgeChip>,
pub target_chip: Option<BadgeChip>,
@ -284,8 +283,8 @@ Keep the D-Bus action payload unchanged. Bump the bundled helper version and its
3. Add:
- full label with one chip
- full label with two chips
- contracted orb with one chip
- contracted orb with two chips
- contracted capsule with one chip
- contracted capsule with two chips
- no-label modifier capsule
4. Regenerate the public delivery and target GIF.
5. Update cursor personalization documentation and theme author instructions.

7
packages/cua-driver/docs/embedded-host-uniffi-implementation-journal.md
View file

@ -46,9 +46,10 @@ surfaces:
not inherited by an embedded child.
- `/embedded` may provide ergonomic adapters, but canonical lifecycle behavior
belongs to Rust and is generated into Python and TypeScript.
- Python wheels, the public npm package, and native npm platform packages ship
from the same Cua Driver tag and exact version. Release validation rejects
source version drift before registry publication.
- Python wheels and the single public `@qwen-code/cua-sdk` package ship from
the same Cua Driver tag and exact version. Native Node payloads remain GitHub
Release assets. Release validation rejects source version drift before
registry publication.
## Validation matrix

920
packages/cua-driver/docs/macos-background-input-v1-plan.md Normal file
View file

@ -0,0 +1,920 @@
# Exact macOS Background Input v1
**Status:** Proposed implementation plan
**Base:** `origin/main` at `d21e3447f9b08c761c090946648d5aca5e6c9cf1`
**Date:** 2026-08-04
## Goal
Let an agent operate ordinary macOS applications in the background without
moving the real pointer, changing the user's active application, or silently
acting on another window owned by the same process.
V1 is deliberately conservative. It expands the routes for which Cua Driver can
prove an exact `(pid, CGWindowID)` destination and returns a structured refusal
for every other route. A refused action is preferable to a process-scoped key
event that appears successful after mutating a sibling window.
## Success criteria
V1 is complete when all of the following are true:
1. Every window-scoped macOS mutation carries the requested `(pid,
CGWindowID)` from target resolution through route selection, dispatch, and
post-action verification.
2. A result can be `confirmed` only from evidence belonging to that exact
window or to an exact browser target bound to it. State from another
same-process window is never accepted as evidence.
3. Background routing is ordered as semantic Accessibility, exact browser/CDP,
exact window-local pointer, and finally PID keyboard with an independent
exact-delivery proof. If none is safe, the driver refuses before sending
input.
4. Background mode never silently crosses to global HID, activates another
application, moves the real pointer, changes Space, or restores a minimized
window.
5. Minimized and hidden windows retain safe semantic AX actions and current
one-shot capture. Raw key commits are refused; callers use `AXConfirm`, an
exact `AXPress`, or an exact browser action when one exists.
6. An off-Space window that is absent from the process's `AXWindows` is
observation-only. Cua Driver does not use another window's AX tree or send
process-scoped input while the target is unresolved.
7. Ordinary Electron applications get per-process AX enablement, exact
`BrowserWindow` mapping, bounded fresh-tree acquisition, delayed effect
verification, and explicit two-window safety coverage.
8. Existing action request schemas and the closed `ActionResult` wire shape
remain compatible.
9. Focused unit/contract tests run in ordinary CI. Exact-SHA native evidence is
collected separately on a signed macOS Namespace runner and in the
canonical Lume acceptance lane.
## Non-goals
V1 does not attempt to provide:
- live minimized-window PiP or a window-scoped ScreenCaptureKit stream;
- continuously changing Chromium pixels while the application is occluded,
hidden, or minimized;
- full Accessibility parity across Spaces;
- automatic Space switching, temporary Space membership, sticky windows, or
switch-and-restore input;
- deminiaturize/order-behind/re-minimize transactions;
- minimize-button interception or application-specific window swizzling;
- a background raw-key guarantee for multi-window applications;
- a global-HID fallback in background mode;
- background control of games, Metal surfaces, protected video, secure input,
or other surfaces that do not expose an exact semantic or routed target;
- generic WKWebView mutation, Safari JavaScript fallback expansion, or a new
browser attachment mechanism;
- a redesign of the public action-result contract; or
- changes to application-owned Chromium settings such as
`backgroundThrottling`.
The existing caller-selected `delivery_mode: "foreground"` remains a separate
last-resort behavior. This plan does not remove it, but the background router
must never enter it implicitly.
## Current source-grounded state
### Exact AX scoping exists
[`ax/window_scope.rs`](../rust/crates/platform-macos/src/ax/window_scope.rs)
already defines `Matched`, `NotFound`, `OwnerPidMismatch`, and `AxUnresolved`.
Its structural invariant is the right foundation: every non-`Matched` result
walks no elements.
[`ax/tree.rs`](../rust/crates/platform-macos/src/ax/tree.rs) unions
`AXChildren` and `AXWindows`, maps top-level AX windows to CG window IDs through
`_AXUIElementGetWindow`, and bounds the default walk at 2,000 nodes and depth
25. [`get_window_state.rs`](../rust/crates/platform-macos/src/tools/get_window_state.rs)
preflights WindowServer ownership, empties the element cache when the requested
window does not resolve, and still returns an exact-window screenshot for the
`AxUnresolved` degradation.
These checks prevent an unresolved request from returning a sibling window's
tree. Mutation needs to carry the same invariant farther than it does today.
### Explicit `window_id` does not make PID keyboard delivery window-scoped
[`window_target.rs`](../rust/crates/cua-driver-core/src/window_target.rs)
correctly promotes a unique PID-only target and refuses a PID with multiple
eligible top-level windows. Explicit `window_id` and element-token requests,
however, pass through that cardinality guard unchanged.
On macOS, [`keyboard.rs`](../rust/crates/platform-macos/src/input/keyboard.rs)
posts through `SLEventPostToPid` with a public `CGEvent::post_to_pid` fallback.
Both transports are process-scoped. [`press_key.rs`](../rust/crates/platform-macos/src/tools/press_key.rs)
best-effort focuses an element and then posts the key to the PID. Its pixel form
first invokes the click path to focus `(x, y)` and then posts the same
process-scoped key, so that combined pointer-plus-key branch needs one exact
route decision rather than bypassing keyboard safety after the focus click.
[`type_text.rs`](../rust/crates/platform-macos/src/tools/type_text.rs) has a
second exactness gap: when no explicit element pointer is supplied,
`read_axvalue(pid, None)` reads `focused_element_of_pid(pid)`. Its before/after
verification can therefore observe whichever same-process field is focused,
not necessarily a descendant of the requested `window_id`. Passing an explicit
window ID does not change that readback.
The required same-PID regression test follows directly from these source
semantics: unresolved target window A, focused sibling window B, process-routed
key, and PID-global focused-element readback. V1 must refuse before the key is
posted and prove from fixture journals that neither window changed.
### Exact semantic and window-local primitives already exist
[`ax_actions.rs`](../rust/crates/platform-macos/src/input/ax_actions.rs)
supports `AXPress`, `AXConfirm`, selection, focus, and enabled-state checks.
[`set_value.rs`](../rust/crates/platform-macos/src/tools/set_value.rs) already
does native AX value readback and correctly distrusts AXValue echo from an
`AXWebArea`.
[`mouse.rs`](../rust/crates/platform-macos/src/input/mouse.rs) has a routed
pointer primitive that stamps window-local location plus the target PID and
CGWindowID fields before SkyLight/PID posting. It does not move the real
pointer. The same file also contains private no-raise activation and
Chromium-specific event recipes. These are implementation candidates, not by
themselves proof that an action reached the requested window. V1 must gate them
on exact target facts and exact post-action evidence.
[`click.rs`](../rust/crates/platform-macos/src/tools/click.rs) already prefers
AX, can perform window-local pointer fallback for specific selection controls,
and distinguishes verified AX selection from an unverified dispatch. The new
router should consolidate those decisions rather than add a second hidden
fallback ladder.
### Electron AX enablement exists but needs lifecycle and mutation discipline
[`ax/bindings.rs`](../rust/crates/platform-macos/src/ax/bindings.rs) writes
`AXManualAccessibility` and falls back to `AXEnhancedUserInterface` only when
the modern attribute is unsupported. The tree walker enables it once per PID,
waits 500 ms when accepted, and then walks the bounded tree.
The current cache is a set of PIDs. V1 should associate enablement with the
observed process lifetime so PID reuse cannot inherit a stale "enabled"
decision. Electron mutation must reacquire an exact bounded tree rather than
using title order, the first AX window, or stale process-global focus.
### Exact browser mutation is already unique-or-refuse
[`browser/binding.rs`](../rust/crates/cua-driver-core/src/browser/binding.rs)
contains the pure native-window-to-CDP correlation. [`browser/engine.rs`](../rust/crates/cua-driver-core/src/browser/engine.rs)
checks native ownership, process fingerprint, endpoint ownership, CDP window
geometry/cardinality, tab identity, and generation again before every
mutation. [`browser/tools.rs`](../rust/crates/cua-driver-core/src/browser/tools.rs)
uses those exact capabilities for typed browser actions.
V1 should reuse this rung rather than creating a macOS-specific browser
shortcut. The bounded Electron fallback remains exact only when the endpoint
exposes one page and the owner has exactly one native window. Two Electron
`BrowserWindow`s must not use that fallback; they need a proven CDP-window
mapping or a refusal. WKWebView mutation remains unsupported.
### Capture and Spaces metadata are limited
[`capture.rs`](../rust/crates/platform-macos/src/capture.rs) currently obtains
a one-shot window image with `screencapture -l <CGWindowID> -x -o`. The
ScreenCaptureKit implementation is display-scoped recording, not a
window-scoped live stream. V1 keeps the current one-shot path and does not
claim frame freshness from API success.
[`list_windows.rs`](../rust/crates/platform-macos/src/tools/list_windows.rs)
describes minimized, hidden, and off-Space enumeration, but the current macOS
records do not reliably populate `current_space_id`, `on_current_space`, or
`space_ids`. V1 must therefore use exact AX-window presence as its input safety
gate instead of inventing a complete Spaces model.
### The result contract is intentionally closed
[`outputs.rs`](../rust/crates/cua-driver-contract/src/outputs.rs) defines the
closed `ActionResult`: effect, route, delivery, evidence, and escalation.
[`action_record.rs`](../rust/crates/cua-driver-core/src/action_record.rs) is the
functional core that conservatively projects native attempts to that public
shape. [`action-result-contract.md`](action-result-contract.md) explicitly
excludes echoed selectors, coordinates, and request targets.
V1 keeps that public result shape. Exact target facts belong to the request,
the internal execution record, a read-only capability report, and structured
refusals. Any future proposal to add optional result fields must update the
contract and every generated strict SDK together; it is not part of this work.
## Locked design decisions
### 1. Exact-window invariant
The canonical target key is the requested `(pid, CGWindowID)`. It is carried
unchanged through planning, execution, and verification.
The following rules are non-negotiable:
- WindowServer must attribute the CGWindowID to the requested PID immediately
before mutation.
- An AX element route must prove that the live element ascends to an AX window
whose `_AXUIElementGetWindow` is the requested CGWindowID.
- A browser route must revalidate its existing exact native-window capability
immediately before mutation.
- A pointer route must use the requested window's validated pixel frame and
stamped CGWindowID, then verify an effect on that same target.
- A keyboard readback must read the addressed element or a fresh element
proven to descend from the requested AX window. It must never call a
PID-global focused-element reader as target evidence.
- A sibling window's state may be retained as a negative canary, but it can
never confirm the requested action.
- If the target disappears, changes owner, becomes AX-unresolved where the
chosen route needs AX, or becomes ambiguous between planning and dispatch,
no fallback actuator runs.
An explicit `window_id` is an address supplied by the caller, not proof that a
process-scoped actuator will honor it.
### 2. Functional core and imperative shell
Add a small pure background-input decision module in
`cua-driver-core`. It owns no macOS objects and performs no I/O. Its inputs are
facts collected by a platform shell:
- exact target and WindowServer ownership;
- AX scope (`matched`, `not_found`, `owner_mismatch`, `unresolved`);
- requested action semantics and trust class;
- target element ancestry and available semantic actions;
- target state (`visible`, `occluded`, `offscreen`, `minimized`, `hidden`, or
unknown where macOS cannot distinguish safely);
- same-PID top-level-window cardinality;
- exact browser-binding eligibility;
- exact window-local pointer eligibility;
- keyboard destination and available verifier;
- baseline foreground PID, real pointer position, and Space posture; and
- permission/private-symbol readiness.
The pure output is one route with prerequisites and a verification plan, or a
typed refusal reason. Keep names internal, but model at least:
```text
ExactWindowTarget(pid, window_id)
TargetResolution
BackgroundRoute
VerificationPlan
BackgroundInputDecision = Execute(...) | Refuse(...)
```
Use a pure truth table for route choice and result classification. Extend the
existing `ActionExecutionRecord` projection rather than adding macOS wire
semantics. Platform adapters translate a chosen internal route to the existing
public `ActionRoute` values.
The macOS imperative shell:
1. serializes mutation planning per PID;
2. gathers fresh WindowServer, AX, browser, and foreground facts;
3. asks the pure core for exactly one decision;
4. revalidates route-specific facts immediately before dispatch;
5. invokes only that actuator;
6. polls only the planned exact-target verifier within a bounded deadline;
7. checks foreground PID, real pointer, and Space posture; and
8. emits the existing action result or a structured refusal.
The core never says "try all routes." The shell cannot improvise a second
actuator after a failed proof. A caller may take a newly reported route after
fresh state, preserving the existing agent-owned action ladder.
### 3. Background routing ladder
The decision order is:
| Order | Route | Required exactness | V1 behavior |
| --- | --- | --- | --- |
| 1 | AX semantic action/value | Exact AX window and exact element ancestry | Prefer `AXPress`, `AXConfirm`, selection, or value mutation with target-bound readback. AX API acceptance alone is unverified. |
| 2 | Browser/CDP | Existing exact native-window, endpoint, CDP-window, tab, generation, and ref proof | Reuse typed browser tools. Do not guess or auto-convert an AX token into a DOM ref. |
| 3 | Window-local routed pointer | Exact owner, AX-window presence, validated pixel frame, stamped CGWindowID, supported target state, and exact postcondition | Never move the real pointer. Private SPI absence or failed revalidation refuses. |
| 4 | PID keyboard | Exact target element/window, no competing same-PID keyboard destination, and an exact verifier available before posting | Native single-window text insertion may qualify. Generic keys, multi-window apps, minimized/hidden windows, and unresolved off-Space windows refuse. |
| 5 | Refusal | No safe route | Return the exact reason and available alternatives without sending input. |
The ladder preserves action semantics. It must not silently replace a trusted
browser click with a synthetic DOM event, infer a submit button from a label,
or turn a desktop request into a browser request without an exact capability.
Where request shapes cannot be translated exactly, the result advises the
explicit tool the caller may use next.
Global HID is not in this table. Only an explicit foreground request may use
the existing guarded foreground/HID path.
### 4. Semantic AX behavior and commit policy
For a token/index action, revalidate the retained element's ancestry against
the requested top-level AX window. A cached element under the right cache key
is not sufficient after a lifecycle or Space transition.
Safe commit behavior is explicit:
- use an advertised `AXConfirm` on the exact field when available;
- use `AXPress` on an explicitly addressed commit button;
- use an exact browser action when the caller has a live page/ref capability;
- otherwise refuse a background `Return` for minimized, hidden, unresolved,
or multi-window targets.
Do not heuristically find the first default button or press another window's
button. Do not turn `press_key(Return)` into `AXPress` unless the request itself
identifies the semantic target.
Native `AXValue` equality can confirm a value operation on the exact retained
element. AX values under `AXWebArea` remain untrusted because the accessibility
shim can echo a write that the renderer did not consume. Electron/web content
requires browser/DOM readback or another exact application effect.
### 5. Electron policy
Electron support is part of v1, not a generic Chromium exception.
For each observed process lifetime:
1. Create the AX application element and try `AXManualAccessibility=true`.
2. Fall back to `AXEnhancedUserInterface=true` only when the modern attribute
returns `kAXErrorAttributeUnsupported`; transient failures do not trigger a
second private write or a success claim.
3. If enablement is accepted, wait the existing bounded settle and perform a
fresh exact-window walk. Key the enablement cache by process fingerprint or
generation, not an immortal numeric PID.
4. Build the top-level `BrowserWindow` map only from `_AXUIElementGetWindow`.
Window title, array order, focused-window status, and first-match selection
are not identity proofs.
5. Require the requested BrowserWindow to be in `AXWindows` before native
mutation. An absent target returns no elements and no input route.
6. Before an element action, prove its ancestry to that BrowserWindow. After
the action, poll a fresh bounded exact-window tree or exact CDP/DOM state.
Never verify against process-global focus or a stale cached tree.
7. Use a delayed bounded verifier because renderer and AX state publication are
asynchronous. Timeouts produce `unverifiable` or a proven delivery failure,
never a fabricated confirmation.
8. Retain the existing distrust of web-area AXValue echo. Normal DOM text and
key semantics should use the exact browser rung when a binding is available.
9. With two BrowserWindows, disable the single-page/single-native-window CDP
shortcut and every raw PID-key route. A route is available only when the
exact native/CDP mapping or exact semantic AX action survives revalidation.
The signed fixture launches once without `--force-renderer-accessibility` to
prove the driver's per-PID enablement works for a normal `BrowserWindow`, and
once with the existing fixture flag as a diagnostic control. It also covers
default `backgroundThrottling` and `backgroundThrottling:false`. The setting
may affect changing pixels, but it must not weaken exact-target input policy or
become a driver prerequisite.
### 6. Minimized and hidden windows
For minimized or application-hidden targets:
- keep exact one-shot capture as observation when it succeeds;
- label capture freshness as unknown rather than promising live frames;
- allow exact semantic AX actions with target-bound verification;
- allow an already-exact browser/CDP action only if all native and browser
binding checks still pass;
- use semantic commit rather than a raw Return key;
- refuse PID keyboard and native routed pointer in v1; and
- do not deminiaturize, unhide, activate, reorder, move, or re-minimize the
window as an implementation detail.
This preserves foreground state by construction. A no-activate restore
transaction can be investigated as a later opt-in mode, but it is not a v1
fallback.
### 7. Other-Space policy
Observation is allowed: `list_windows`, exact one-shot capture, and any honest
degradation metadata may still be returned.
For mutation, query a fresh application `AXWindows` and map every candidate via
`_AXUIElementGetWindow`:
- If the requested CGWindowID is absent, refuse all input with an
`off_space_or_ax_unresolved` reason. Do not inspect or act on a sibling
window, even if it is on the current Space.
- If the exact target is present, v1 may use a semantic AX or exact browser
route whose own preconditions and postcondition pass. A native pixel or
PID-key route is not promoted merely because capture succeeded.
- Unknown Space metadata remains unknown. Do not claim `on_current_space`
without a real source.
Explicit switch/restore, temporarily adding a window to the current Space, and
sticky-window behavior require a later opt-in design with its own visible-state
and cleanup contract. V1 adds no SkyLight Spaces manipulation.
### 8. Exact pointer route
The existing window-local pointer implementation may be selected only when:
- WindowServer ownership and AX-window identity both match the requested
target immediately before dispatch;
- the current screenshot dimensions have a validated mapping to the requested
WindowServer bounds;
- the local point falls inside that exact window frame;
- the target is neither minimized, hidden, nor AX-unresolved/off-Space;
- the required private symbols resolve on the running macOS version;
- baseline frontmost PID and real pointer position have been captured; and
- an exact postcondition exists for the selected control.
Use the existing stamped local coordinate and CGWindowID fields. Treat private
no-raise activation as part of the actuator, not as delivery proof. If it
changes the user's frontmost application, pointer, or Space, mark the route
failed, restore only state owned by the driver when that is safe, and disable
the route for the remainder of the process session.
The v1 router does not use unverified pixel input for a mutation whose only
result is "event posted." It may remain available through current APIs as
`unverifiable` outside the new exact capability, but it must not be advertised
as exact background input.
### 9. PID keyboard gate
PID-routed keyboard is a last background rung because macOS accepts a process,
not a CGWindowID.
Before posting text, require all of:
- a live explicit element proven to descend from the requested AX window;
- that exact element is the PID's focused UI element and the PID's focused AX
window maps to the requested CGWindowID;
- the requested target is the only eligible same-PID keyboard window at the
pre-dispatch revalidation point;
- the target is visible or occluded on the current Space, not minimized or
hidden;
- a target-bound value verifier is readable before dispatch; and
- the value surface is native, not an `AXWebArea` echo surface.
Post the event only after those facts pass, then poll the same retained target
or a freshly reacquired equivalent inside the exact requested tree. If any
precondition cannot be proven, refuse before input. Generic `press_key` and
hotkeys normally lack a safe target-specific postcondition and therefore
refuse in background exact mode unless a future action-specific verifier is
added.
The singleton rule is a conservative v1 proof, not a claim that
`SLEventPostToPid` is window-addressed. It prevents the known class of
same-process sibling delivery while semantic and browser routes cover most
ordinary controls.
### 10. Capability and result reporting
Add an additive `background_input` section to the read-only exact-window state
output. It should be produced from the same pure decision facts and contain no
window titles, control values, or private symbol addresses. A representative
shape is:
```json
{
"background_input": {
"exact_window": {
"status": "matched",
"pid": 123,
"window_id": 456,
"ax_window": "matched"
},
"routes": [
{"route": "accessibility", "status": "available"},
{
"route": "pid_keyboard",
"status": "refused",
"reason": "same_pid_sibling_windows"
}
],
"observation": {
"one_shot_capture": "available",
"frame_freshness": "unknown"
}
}
}
```
The capability route and status strings above are illustrative and are not new
values for the closed public `ActionRoute` enum.
The exact enums and generated types should be reviewed with the implementation,
but the semantics are fixed:
- `available` means route prerequisites are currently proven, not that a
future call is guaranteed to succeed;
- every action revalidates instead of trusting this report;
- unknown facts stay unknown;
- route refusal includes one stable machine-readable reason; and
- the report does not expose user content.
Keep successful action outputs on the current `ActionResult` contract:
- `route` uses the existing enum, including accessibility, synthetic events,
system API, DOM, and trusted input for relevant background actions;
- `delivery.mode` reports what actually occurred;
- `confirmed` requires publishable exact-target readback;
- `unverifiable` means the route ran but exact effect is unknown;
- `refused` means no actuator ran; and
- escalation is advice for an explicit caller decision.
Structured refusals should include `code`, `effect: "refused"`, the requested
`pid` and `window_id`, the failed exactness fact, and the safe next route when
one exists. Initial codes should distinguish at least ownership mismatch,
AX-window unresolved/off-Space, same-PID keyboard ambiguity, unavailable
verification, and unavailable private route.
Do not report `delivery_failed` merely because the driver lacks evidence.
Use it only when exact negative evidence proves delivery did not occur. A
missing/unreadable postcondition is `unverifiable` with
`effect_unconfirmed`; an unchanged readable exact target after the bounded
delivery window may be `suspected_noop` or delivery failure according to the
existing action's semantics. API acceptance is never confirmation.
## State policy matrix
This is the required decision policy, not a claim that every route already has
native evidence:
| Target state | Observe | AX semantic | Exact browser | Window-local pointer | PID keyboard |
| --- | --- | --- | --- | --- | --- |
| Frontmost | Current state/capture | Yes when exact | Yes when exact | Yes when exact and verified | Only with singleton exact element and verifier |
| Background visible | Current state/capture | Yes when exact | Yes when exact | Yes when exact and verified | Same conservative gate |
| Fully occluded | One-shot capture; freshness honest | Yes when exact | Yes when exact | Yes only after signed proof and exact verifier | Same conservative gate |
| Fully offscreen on current Space | One-shot capture when valid | Yes when exact | Yes when exact | Signed-evidence gated; no geometry guess | Refuse in v1 |
| Minimized | One-shot capture, freshness unknown | Yes when exact | Yes only if exact binding revalidates | Refuse | Refuse; semantic commit instead |
| Application hidden | One-shot capture, freshness unknown | Yes when exact | Yes only if exact binding revalidates | Refuse | Refuse; semantic commit instead |
| Other Space, exact AX window present | Observation allowed | Yes when exact | Yes when exact | Refuse in v1 | Refuse in v1 |
| Other Space/AX window absent | Observation only | Refuse | Refuse | Refuse | Refuse |
| Owner mismatch/stale window | Refuse stale target; report owner when safe | Refuse | Refuse | Refuse | Refuse |
## Compatibility expectations
These are route expectations to certify, not unsupported parity claims:
| Surface | V1 expectation | Evidence required |
| --- | --- | --- |
| AppKit | Primary supported lane: exact AX actions/value, verified window-local pointer where semantic AX is unavailable, narrowly gated native text | Runtime AppKit fixture, including two windows under one PID |
| SwiftUI | Exact AX on the current Space; truthful refusal when an off-Space window disappears from `AXWindows` | Runtime SwiftUI fixture on current and another Space |
| Electron DOM | AX enablement and exact BrowserWindow tree; exact CDP route when bound; no multi-window PID keys | Runtime two-BrowserWindow fixture with per-window DOM journals |
| Mac Catalyst | Semantic AX or verified pointer only; raw keyboard remains gated | Source-supported candidate; runtime row required before advertising |
| Qt/Java | Use exposed exact AX semantics; refuse absent/custom AX and raw-key ambiguity | Source-based until dedicated signed fixtures exist |
| Browser canvas/WebGL | Exact browser action when a supported ref/route exists; otherwise no semantic claim | Source-based browser binding plus route-specific runtime tests |
| Games/Metal/custom event loops | No v1 background guarantee | Explicitly unsupported until an exact app-owned route exists |
| Secure/protected surfaces and secure input | No capture or input bypass | Refusal/permission tests; never weaken platform protection |
| System UI and out-of-process panels | Require actual WindowServer owner and exact AX window; never follow a PID silently | Existing owner-mismatch tests plus signed panel coverage |
`backgroundThrottling:false` may preserve Electron painting in cases where the
default renderer throttles. That is an application compatibility option, not a
Cua Driver route guarantee. Canvas/WebGL, trusted-event filters, and custom
event loops can still behave differently from ordinary DOM controls.
## Staged implementation
### Stage 0: Lock the safety regression
- Extend the AppKit and Electron fixtures with two fixed-title windows, one
field and one commit control per window, and a run-owned per-window journal.
- Reproduce process-scoped keyboard delivery with target A unresolved or
minimized and sibling B focused.
- Add a test that requests A by exact `(pid, window_id)` and requires a refusal
before dispatch; both journals and both field values must remain unchanged.
- Add a unit regression showing that a sibling's focused-element readback can
never satisfy A's verification plan.
Do not refactor routes until this failing behavioral shape and pure regression
are both captured.
### Stage 1: Pure exact-target router
- Add the target facts, route decision, verifier requirement, and refusal
reasons to common core.
- Encode the complete state matrix as table-driven unit tests.
- Make "explicit `window_id`" distinct from "exact delivery proven."
- Extend internal action records with diagnostic exact-target proof state only
as needed; do not change the public `ActionResult` shape.
- Add a per-PID mutation coordinator so two concurrent background decisions
cannot change process focus underneath one another.
### Stage 2: Fresh macOS target acquisition
- Centralize WindowServer owner preflight and fresh `AXWindows` mapping.
- Add a bounded ancestry check from a retained element to its exact AX window.
- Make Chromium/Electron AX enablement process-lifetime aware and preserve the
modern-attribute/fallback error distinction.
- Reacquire bounded exact-window state for delayed postconditions and clear
stale caches on every unresolved result.
- Change the `AxUnresolved` input advice from unconditional pixel mutation to
observation-only/refusal when exact input cannot be proven.
### Stage 3: Semantic actions and exact verification
- Route exact `AXPress`, `AXConfirm`, selection, and native value mutations
first.
- Bind every readback to the requested target element/window.
- Preserve web-area AX echo distrust and add browser/DOM verification where an
exact capability already exists.
- Make process-level window changes diagnostic only unless the changed window
is the exact target postcondition.
- Normalize `confirmed`, `unverifiable`, `suspected_noop`, and proven delivery
failure through the functional core.
### Stage 4: Browser, pointer, and keyboard rungs
- Reuse browser mutation revalidation without a second Electron-only binding
implementation.
- Gate the existing window-local pointer route on exact target, state, pixel
frame, private-symbol readiness, and verifier facts.
- Route `press_key`'s existing pixel-focus-then-key form through the same pure
decision. A successful focus click cannot waive the exact AX-window,
singleton destination, target-bound verifier, or state gates for the key.
- Remove PID-global focused-element readback from window-scoped keyboard
verification.
- Permit native PID text only under the singleton exact-element/verifier gate.
- Refuse generic keys and every unsafe minimized/hidden/off-Space raw-key case.
- Assert that the background router has no call edge to global HID or
foreground assist.
### Stage 5: Capabilities, documentation, and rollout
- Add the read-only per-window capability report and structured refusal codes.
- Keep request and `ActionResult` schemas compatible; regenerate/check SDK
artifacts only if the read-only output has a generated typed surface.
- Update the background-delivery and known-limit documentation to match the
certified matrix, not the intended matrix.
- Ship the high-risk same-PID refusal before expanding routes. If a new route
regresses, disable that route while retaining the exact-target guard.
- Run the full desktop acceptance matrix once against the final exact SHA, as
required by repository guidance.
## Test and acceptance plan
### Ordinary CI
Run these on every implementation pull request without a GUI or TCC grants:
- pure route-decision table for every state and route;
- owner mismatch, stale ID, AX-unresolved, and exact matched scope;
- explicit-window versus exact-delivery distinction;
- same-PID sibling rejection before actuator invocation;
- wrong-sibling readback ignored by verification;
- pixel-focus-then-key cannot bypass keyboard refusal or dispatch a second
actuator after target facts change;
- no background decision can produce a global-HID transport;
- native value, missing readback, unchanged value, partial text, delayed
success, and negative-delivery result classification;
- Electron PID-lifetime enablement cache and fallback behavior;
- exact BrowserWindow map cardinality and no first/title-only match;
- existing CDP binding/revalidation, including one-page/one-window fallback
refusal after a second native window appears;
- minimized/hidden/off-Space policy truth tables;
- read-only capability and structured-refusal schema snapshots;
- existing closed `ActionResult` invariants and generated SDK parity; and
- fixture build/lint tests.
### Signed macOS behavior matrix
The native harness must use only repository fixtures and a fixture-owned
occlusion sentinel. Each action records pre/post foreground PID, real pointer
position, requested CGWindowID, AX window IDs, route, result, exact fixture
journal entry, and cleanup state.
Required rows:
1. **AppKit:** frontmost, background visible, fully occluded, offscreen,
minimized, and hidden. Cover AXPress, AXConfirm, AXValue, exact pointer, and
singleton native text.
2. **AppKit same PID/two windows:** request each window independently; minimize
or make one AX-unresolved while the sibling is focused; raw key/text must
refuse and neither sibling journal may change.
3. **SwiftUI:** background AX actions and value readback on the current Space;
move only the fixture window to another preconfigured Space, query from the
other Space, and require refusal whenever its CGWindowID is absent from
`AXWindows`.
4. **Electron DOM/two BrowserWindows:** run without force-renderer AX first,
then the diagnostic flag; run default throttling and
`backgroundThrottling:false`; prove exact AX window mapping, AXPress,
semantic commit, exact browser type/click, delayed readback, and raw-key
refusal for minimized/unresolved targets. Every effect must appear in the
addressed window's own fixture journal and not its sibling's.
5. **Browser routes:** exact bind and mutation, stale generation, geometry
mismatch, second native window, ambiguous tab/window, and unavailable
endpoint. Every ambiguous row refuses without mutation.
6. **Foreground preservation:** for every background success and refusal, the
fixture sentinel remains frontmost, the real pointer is unchanged, and the
current Space is unchanged.
7. **Capture:** record whether one-shot capture succeeds in each state and
never infer live animation from a successful PNG alone.
8. **Permissions/private APIs:** missing Accessibility, missing Screen
Recording, and missing private symbols return an honest unavailable/refused
result without prompting or global fallback.
9. **Cleanup:** close fixture windows, stop all fixture/driver processes,
restore the initial fixture foreground, pointer, and Space, and leave no
run-owned artifact outside the selected artifact directory.
The other-Space setup belongs to the disposable runner image or a test-only
harness. Product code must not create, join, or switch Spaces. The test records
its starting Space and always restores it in a trap/finally path.
### Canonical Lume gate
The repository's canonical signed/TCC acceptance remains
[`tests/runners/macos-lume/README.md`](../tests/runners/macos-lume/README.md)
and [`run-all.sh`](../tests/runners/macos-lume/run-all.sh). It provides the
logged-in Aqua session, certificate-backed `CuaDriverLocal.app`, normal TCC
grant flow, and exact-source checks.
After focused Namespace evidence passes, add the new fixture rows to the Lume
entrypoint and run once on the final candidate SHA. Preserve its existing
requirements:
- use a disposable worker cloned from the stopped private seed;
- install with `install-local.sh --release --autostart
--require-stable-signing`;
- never edit `TCC.db`;
- execute the GUI suite from the guest's logged-in Terminal, not SSH;
- retain exact SHA, environment, code-signing requirement, permission status,
structured results, fixture journals, screenshots/recording where allowed,
and cleanup evidence; and
- retrieve evidence, then delete the worker.
Namespace validation complements this gate; it does not replace the Lume
golden-image contract.
## Namespace macOS runner workflow
### Discovery recorded for this plan
The planning checkout was inspected on 2026-08-04:
- project `.amp/plugins` was absent;
- the surrounding workspace `.amp/plugins` was absent;
- user `~/.config/amp/plugins` contained
`namespace-mac-runners.ts` and `namespace-mac-runners.js`;
- no repository-local Namespace runner instructions were found; and
- the TypeScript source registered the exact tool name
`namespace_mac_runner`.
The user plugin is not versioned with this repository, so a later agent must
repeat discovery before use. Check project `.amp/plugins`, user
`~/.config/amp/plugins`, the active workspace plugin location, Amp plugin
documentation, and the loaded plugin source. If the tool is absent, record
those checks and make plugin discovery/setup step one; do not guess an API.
The discovered tool accepts:
```text
action: list | spawn | destroy
runner_id: optional hostname-compatible ID for spawn
repository_url: optional HTTPS github.com URL for spawn
duration_minutes: optional integer from 5 through 720
instance_id: required for destroy
```
The discovered implementation defaults to macOS `26.x`, arm64, 6 vCPU,
14,336 MB, and a 60-minute TTL. It requires `nsc login`. Spawn creates billable
infrastructure immediately without confirmation, clones into
`$HOME/workspace/repository` through Amp's GitHub credential helper, starts
`amp --no-tui --runner-id "$AMP_RUNNER_ID" --remote-control-terminal` in tmux,
and waits for `Registered. Create threads`. Failed provisioning attempts to
destroy the instance it created. Destroy accepts only plugin-managed instance
IDs and asks for confirmation. These details must be re-read from the installed
plugin because it can change independently of this plan.
### Exact later validation workflow
1. Re-run the plugin discovery above and record the resolved source/config.
2. Call `namespace_mac_runner` with `{"action":"list"}`. Do not modify or
destroy unrelated runners.
3. Obtain explicit approval for the billable spawn. The approval for this plan
PR does not authorize a later infrastructure charge.
4. Push the implementation candidate and record its full SHA. Spawn with a
unique hostname-compatible ID and bounded TTL, for example:
```json
{
"action": "spawn",
"runner_id": "cua-bg-input-<unique-suffix>",
"repository_url": "https://github.com/trycua/cua.git",
"duration_minutes": 180
}
```
5. Record the returned runner ID, Namespace instance ID, URL, resolved macOS
selector, repository URL, and deadline. TTL is a backstop, not cleanup.
6. Call Amp's `list_runners` immediately before `create_thread`. Start a child
with `executor: "runner"` and that exact live runner ID; do not silently use
an orb. Instruct it to fetch and detach-checkout the exact candidate SHA,
because the plugin's shallow clone starts from repository default HEAD.
7. On the Namespace Mac, record `sw_vers`, OS build, architecture, SIP status,
Xcode/Rust/Node versions, exact git SHA, and a clean worktree before testing.
8. Use repository-supported `CuaDriverLocal.app` setup only. Follow
[`scripts/README.md`](../scripts/README.md), use an ephemeral
certificate-backed local signing identity, and install with
`--require-stable-signing`. Never copy a maintainer private key or signing
keychain to Namespace. Never edit `TCC.db` or bypass TCC.
9. Grant Accessibility and Screen Recording only through normal macOS UI if the
disposable Namespace environment and explicit test approval support it.
Record `permissions status --json` and the `codesign -d -r-` designated
requirement. The Namespace plugin provisions neither a signing identity nor
TCC. If stable signing, interactive UI, or either grant is unavailable,
mark the signed native lane blocked and run only pure/build tests; do not
claim native behavior passed.
10. Use only fixed-title repository fixtures and run-owned artifact paths.
Capture the matrix evidence listed above, including exact fixture journal
effects, requested and sibling window IDs, AX counts/mapping, result
payloads, foreground PID, real pointer, Space posture, and action timing.
Do not collect unrelated application state, window titles, or user data.
11. In a finally/trap path, stop fixture and driver processes, close fixture
windows, restore fixture focus/pointer/Space state, remove run-owned
temporary files, and write a cleanup assertion. Retrieve only sanitized
evidence needed for review.
12. From the controlling thread, call
`namespace_mac_runner` with
`{"action":"destroy","instance_id":"<recorded-instance-id>"}` and confirm
destruction. Call `{"action":"list"}` again and require the instance to be
absent.
## Risk analysis
### macOS and private APIs
`_AXUIElementGetWindow`, `AXManualAccessibility`,
`AXEnhancedUserInterface`, SkyLight event posting, private event fields, and
no-raise activation may change between macOS releases. Every lookup must be
bounded and fail closed. Missing symbols remove a route; they never cause a
global or PID-only fallback.
V1 adds no private Spaces API. Space membership remains incomplete, and the
policy deliberately refuses when AX cannot prove the exact target.
### TCC and signing identity
Accessibility and capture evidence is valid only for the process identity that
performed it. Ad-hoc rebuilds can invalidate grants. Native acceptance requires
the certificate-backed `CuaDriverLocal.app` flow and normal permission UI.
Neither CI mocks nor a successful build substitutes for this evidence.
### Application semantic differences
AX API success may be a no-op, Chromium AXValue may be an echo, and custom
renderers may reject synthetic input. Confirmation always comes from an
independent exact-target effect. Unsupported applications receive narrower
capabilities, not a misleading parity claim.
### Races and lifecycle
Windows can close, change owner, add a modal, or restart between observation
and input. Per-PID serialization and immediate revalidation reduce the window;
they do not make stale capabilities permanent. Any changed fact invalidates
the decision and requires fresh state.
### Compatibility and rollout
Request schemas and successful action results remain stable. Some calls that
currently post best-effort PID keys will begin refusing when exact delivery is
not provable. That is an intentional safety correction. Refusals include a
recoverable semantic/browser/foreground alternative when one actually exists.
Roll out one route at a time after its fixture lane passes. The rollback is to
disable the new route decision while retaining the same-PID/unresolved-target
guard. Do not restore the unsafe process-scoped fallback to improve success
rates.
Telemetry and retained artifacts may contain route, refusal reason, OS build,
target PID/window ID, timing, and boolean invariants. They must not contain
window titles, field values, typed text, screenshots of unrelated apps,
browser URLs, or fixture logs outside the fixed test vocabulary.
## Alternatives rejected for v1
- **Post to the PID and verify any focused field.** This is the wrong-window
bug, not a verification strategy.
- **Assume explicit `window_id` makes `post_to_pid` exact.** The keyboard
transport has no window parameter.
- **Restore without activation, order behind, act, and re-minimize.** This
changes target state, can still change internal key-window routing, and adds
cleanup races. It belongs in a later opt-in design.
- **Move the window offscreen or to another Space.** This mutates user desktop
state and does not solve AX or renderer suspension reliably.
- **Temporarily make a window sticky or add it to the current Space.** This is
full Spaces manipulation, outside v1's contract.
- **Treat capture success as input exactness.** A CGWindowID image proves the
capture target, not the destination of a process-scoped key.
- **Treat AX API acceptance or event-post return as confirmation.** Both can
report success without the intended application effect.
- **Silently use foreground HID.** It violates the background contract and can
act on the user's current application.
- **Add request echoes to the closed `ActionResult`.** Callers already own the
request; a wire change would require coordinated strict-SDK migration without
improving route safety.
## Open implementation questions
Resolve these during Stage 0/1 review, before enabling a route:
1. Which existing read-only output surface should own the additive
`background_input` capability in every SDK without duplicating native
window state?
2. Which stable process fingerprint should replace the PID-only Chromium AX
enablement cache on macOS?
3. Which native text-field properties are sufficient to predeclare a safe
postcondition for the singleton PID-key rung?
4. Can the current private no-raise pointer preparation satisfy foreground,
pointer, and Space invariants on every supported macOS build? Builds without
signed evidence must keep the route disabled.
5. What bounded delayed-verification deadline gives Electron enough time while
staying below the public tool timeout? Use evidence from the fixture rather
than reusing an unrelated settle constant blindly.
6. How will the disposable Lume/Namespace image provide a deterministic second
Space for tests while guaranteeing restoration? This is a test-environment
decision, not a product Space API.

20
packages/cua-driver/docs/test-harnesses-guide.md
View file

@ -30,6 +30,26 @@ The OS workflow may fan the complete matrix out into independent jobs for
reporting and failure isolation. That is an execution detail; contributors
should think of it as one canonical suite.
### Evidence authority
A canonical result is the complete repository harness run at the exact source
SHA. A one-off app smoke, manually assembled script, video, or environment
replay can diagnose a failure or provide release presentation evidence, but it
does not replace the complete matrix.
Windows and Linux use the repository's GitHub-hosted workflows when their
strict environment preflights pass. Windows Azure RDP runs are optional
environment-parity replays or a fallback when the hosted preflight cannot prove
a required capability. macOS uses the logged-in Lume maintainer wrapper.
For browser-facing changes or browser-use release certification, also run the
standalone Chrome/Edge matrix: the macOS wrapper accepts
`--standalone-browser`, while the Windows and Linux workflow is
`.github/workflows/e2e-rust-standalone-browsers.yml`.
Historical `*-plan.md`, `*-journal.md`, and release evidence documents record
what was run at that time. They are not current execution instructions and do
not override this guide or `scripts/ci/README.md`.
## Repository Map
```text

8
packages/cua-driver/docs/test-matrix.md
View file

@ -45,7 +45,7 @@ These run without the repo-local GUI applications:
| --- | --- | --- |
| Core driver logic | `rust/crates/*/src/**` | Protocol values, sessions, schemas, image helpers, input helpers, configuration, telemetry, CLI behavior |
| MCP and CLI boundary | `rust/crates/cua-driver/tests/protocol_*` | Handshake, tool registration, tool calls, media, sessions, and errors |
| Session capture scope | `session_capture_scope_test.rs` plus core `capture_scope` tests | Per-session isolation, immutable live policy, explicit auto escalation, end/revive race safety, transport-mirror refusal, and retired persistent key |
| Lifecycle sessions and target migration | Core `session`, `session_tools`, `action_target`, and compatibility `capture_scope` tests plus `session_capture_scope_test.rs` | One implicit session per transport, five-minute idle policy, in-flight and exact-once cleanup, owner-scoped inspection and revival, per-call target validation, and legacy capture-scope compatibility |
| Tool-contract gate | `schema_consistency_test.rs` | Shared tool schema parity and reviewed risk metadata across OS backends |
| Configuration transport | `transport_config_persistence_test.rs` | CLI and MCP configuration persistence |
| Token and protocol surfaces | `protocol_element_token_test.rs`, related tests | JSON-RPC-visible contract behavior |
@ -109,7 +109,7 @@ Windows native harnesses are repo-local applications built from source:
| WPF | `harness_wpf_test.rs` | UIA controls, text, keys, pointer actions, scroll, drag, popups, menus, modal windows |
| WinUI3 | `harness_winui3_test.rs` | XAML controls, text, checkbox/radio, slider, combo, popup |
| WebView2 | `harness_web_test.rs` | Window discovery, CDP page access, JavaScript, DOM click path |
| Desktop scope | `desktop_scope_windows_test.rs` | Concurrent strict window/desktop sessions, full-display capture, screen-absolute click/scroll, and strict rejection |
| Desktop target | `desktop_scope_windows_test.rs` | Per-call window/desktop modality, full-display capture, screen-absolute click/scroll, and strict rejection |
| Desktop invariants | Testkit `DesktopObserver` plus typed launch/capture/cursor owners | Cross-cutting focus, z-order, minimized-launch, screenshot, cursor, and desktop checks |
Native controls use AX/UIA state as their oracle. Pointer actions also use PX
@ -123,7 +123,7 @@ background delivery and attach desktop-side-effect oracles.
| AppKit | `harness_appkit_test.rs` | AX tree/capture, AX value/text, AX scroll, PX clicks, and foreground slider drag across the proven delivery modes; background drag is an exact refusal |
| SwiftUI | `harness_swiftui_test.rs` | AX tree/capture, background click/value, and foreground popover-trigger state |
| WKWebView | `cross_platform_behavior_test.rs` | Dedicated native host running the full 40-cell shared web catalog |
| Desktop scope | `desktop_scope_macos_test.rs` | Concurrent strict window/desktop sessions, screen-absolute action delivery, and strict rejection |
| Desktop target | `desktop_scope_macos_test.rs` | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection |
| Installed-app launch/focus | `installed_app_launch_macos_test.rs` | Real Calculator/TextEdit launch and focus behavior in the canonical logged-in lane |
| Installed-app text | `installed_app_textedit_macos_test.rs` | Real TextEdit AX background write and verification in the canonical logged-in lane |
@ -143,7 +143,7 @@ transient-panel AX discovery gap.
| Electron | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix |
| Tauri | `cross_platform_behavior_test.rs` | X11 and hosted Sway | Shared web action matrix |
| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI and Wayland/AT-SPI where configured | Native GTK controls and input |
| Desktop scope | `desktop_scope_linux_test.rs` | X11/Wayland | Concurrent strict window/desktop sessions, screen-absolute action delivery, and strict rejection |
| Desktop target | `desktop_scope_linux_test.rs` | X11/Wayland | Per-call window/desktop modality, screen-absolute action delivery, and strict rejection |
Nix provides the Linux build and desktop environment. X11 and Wayland are
separate matrix dimensions because their capture and input contracts differ.

33
packages/cua-driver/docs/uniffi-sdk-implementation-journal.md
View file

@ -41,7 +41,7 @@ The public integration split remains:
The product boundary is reflected directly in packaging:
- Python client applications import the Rust-backed SDK from `cua_driver`.
- TypeScript client applications import it from `@trycua/cua-driver`.
- TypeScript client applications import it from `@qwen-code/cua-sdk`.
- Agents configure `cua-driver mcp` through their runtime's existing MCP client
and do not import either language package.
- The language-native MCP facades and their contract generator were removed.
@ -121,9 +121,10 @@ The product boundary is reflected directly in packaging:
- A built macOS arm64 wheel was installed into a clean Python 3.12 environment;
`cua_driver.CuaDriver.connect(None)` loaded the packaged library and
returned the canonical daemon socket path.
- An actual 382.3 kB npm tarball (1.2 MB unpacked) was installed into a clean
project; the root imported without native code and
`@trycua/cua-driver` loaded the packaged dylib successfully.
- A pre-rename 382.3 kB npm tarball (1.2 MB unpacked) was installed into a
clean project and loaded the packaged dylib successfully. That result used
the upstream npm identity and is historical only; it is not current Qwen
release evidence.
- The macOS dylib install ID is the relocatable
`@rpath/libcua_driver_sdk.dylib`, and release CI asserts it before signing.
- After promoting the SDK to each package root, a fresh macOS arm64 wheel and
@ -143,9 +144,25 @@ The product boundary is reflected directly in packaging:
- Rust release artifacts now carry the SDK library on Linux, macOS, and
Windows. The Python release matrix installs every built wheel and imports the
generated binding before PyPI publication.
- Release CI assembles optional native npm packages for every supported Node
OS/architecture from the verified Cua Driver release assets, smoke-tests the
installed root package, and publishes all artifacts at the Rust tag version.
A developer's host-local tarball is never published.
- Release CI packs one platform-neutral `@qwen-code/cua-sdk` artifact,
clean-installs it against the verified Cua Driver release payload, and
publishes it at the Rust tag version only after the GitHub Release exists. A
developer's host-local tarball is never published.
- MCP/CLI remains unchanged as the agent boundary; the language package roots
are the Rust-backed application SDK.
### Qwen-owned npm identity
- The TypeScript SDK and `/computer-use` wrapper are one
`@qwen-code/cua-sdk` package. There is no driver npm package and there are no
platform npm packages.
- The package version identifies the matching Qwen CUA Driver GitHub Release.
Its postinstall verifies `checksums.txt` before caching the SDK library and
Node runtime. It never imports or resolves the published upstream driver
package.
- Release dry-run clean-installs that one tarball against the just-built native
payload. Production repeats the install against the public Qwen Release
before npm publication.
- The frozen 0.12.6 TypeScript compatibility fixture keeps its historical
import unchanged and passes when the Qwen SDK tarball is installed under a
test-only npm alias. It is not a production dependency or publication target.

22
packages/cua-driver/docs/why-cua-driver-uses-mcp-instead-of-uniffi.md
View file

@ -20,7 +20,7 @@ are not competing protocol choices.
| Surface | Consumer | Public shape | What provides runtime portability |
| -------------------------- | --------------------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
| Cua as an agent MCP or CLI | Codex, Claude Code, another agent, or a shell | `qwen-cua-driver mcp` and `qwen-cua-driver call` | MCP and the executable protocol already work from any capable runtime. |
| Cua as an imported SDK | Python, TypeScript, Swift, Kotlin, or another application | `cua_driver` or `@trycua/cua-driver` | Python and Node call a shared Rust daemon-client implementation through experimental UniFFI bindings. |
| Cua as an imported SDK | Python, TypeScript, Swift, Kotlin, or another application | `cua_driver` or `@qwen-code/cua-sdk` | Python and Node call a shared Rust daemon-client implementation through experimental UniFFI bindings. |
The [Codex and Claude Agent SDK examples](../examples/agent-sdks/README.md)
make the first surface concrete. Each agent SDK receives the same stdio MCP
@ -31,7 +31,7 @@ imports a generated Cua client.
1. Keep MCP and the CLI as the canonical agent boundary. Do not require a Cua
language package merely to connect an MCP-capable agent.
2. Make `cua-driver` and `@trycua/cua-driver` application SDK packages backed
2. Make `cua-driver` and `@qwen-code/cua-sdk` application SDK packages backed
by UniFFI. Remove their language-native MCP facades because those duplicate
a runtime-neutral protocol client that agent runtimes already provide. This
deliberately makes the package API breaking before publication.
@ -122,7 +122,7 @@ The package name is the SDK entrypoint; no transport suffix is required:
| Runtime | Rust-backed application SDK | Agent integration |
| --- | --- | --- |
| Python | `cua_driver` | Configure `qwen-cua-driver mcp` in the agent runtime |
| TypeScript | `@trycua/cua-driver` | Configure `qwen-cua-driver mcp` in the agent runtime |
| TypeScript | `@qwen-code/cua-sdk` | Configure `qwen-cua-driver mcp` in the agent runtime |
There is no public `/sdk`, `/mcp`, or `/native` entrypoint. “Native” describes
the private generated loader and platform library, not the product API. The
@ -330,7 +330,7 @@ architecture is described in the [contract README](../contract/README.md).
Mozilla UniFFI `0.31.0` generates Python from the compiled `cdylib` metadata.
Node generation uses the separately maintained
`uniffi-bindgen-react-native`/UBRN `0.31.0-3` N-API target plus pinned
`@ubjs/core` and `@ubjs/node` runtimes. The checked-in outputs are regenerated
`@ubjs/core`. The checked-in outputs are regenerated
into temporary roots, tracked by ownership manifests, and compared byte for
byte in CI.
@ -345,12 +345,12 @@ Python wheels are platform-specific and the Rust release workflow
places the matching SDK library beside the CLI in each release runtime archive.
The wheel builder moves that library next to the generated Python modules, and
CI inspects the wheel and runs it across the FFI boundary. The public Node
package uses generated platform resolution and optional native packages for
macOS arm64/x64, Linux glibc arm64/x64, and Windows arm64/x64. Release CI builds
those packages from the already verified Cua Driver assets, smoke-tests the
installed root plus matching native package, publishes native packages first,
then publishes the root. Python, npm, and Rust versions come from one release
tag and are attached to the same GitHub release.
package is the single platform-neutral `@qwen-code/cua-sdk`. Its postinstall
selects the matching macOS, glibc Linux, or Windows release archive, verifies
the archive checksum, and caches the SDK library plus Node runtime. Release CI
smoke-tests the installed npm tarball against the native release payload and
publishes npm only after the GitHub Release exists. Python, npm, and Rust
versions come from one release tag.
## UniFFI follow-up gates
@ -359,7 +359,7 @@ Evaluate the two SDK targets separately.
Before expanding the UniFFI path to another runtime or platform:
1. Load-test the native artifact on the added release OS and architecture.
2. Add the platform package to generated resolution and the release matrix.
2. Add the target archive to native resolution and the release matrix.
3. Preserve release signing/notarization evidence for the native library everywhere
the platform requires it.
4. Run the executable MCP boundary and imported SDK through the same daemon

9
packages/cua-driver/examples/agent-sdks/README.md
View file

@ -86,13 +86,16 @@ npm run codex -- \
"Inspect the active app and summarize what is visible without changing it"
```
Set `CODEX_MODEL` optionally. Both scripts create a unique Cua session outside
the agent, configure `qwen-cua-driver mcp`, and end the session in `finally`.
Set `CODEX_MODEL` optionally. Both scripts configure one long-lived
`qwen-cua-driver mcp` transport. Its first admitted stateful call creates an
implicit session, later unnamed calls reuse it, and transport shutdown runs
the same cleanup as an explicit `end_session`.
## Environment and safety
- Put `qwen-cua-driver` on `PATH`, or set `CUA_DRIVER_BIN` for MCP examples.
- Set `CUA_CAPTURE_SCOPE` to `auto`, `window`, or `desktop` for MCP examples.
- Select an exact window or desktop target on each action. Sessions do not
store capture modality.
- Authenticate the chosen agent SDK using its normal local login or API key.
- Run only trusted tasks. These examples remove interactive approval prompts.
- Keep purchases, messages, deletion, credential entry, and other irreversible

98
packages/cua-driver/examples/agent-sdks/claude_agent.py
View file

@ -4,12 +4,10 @@ from __future__ import annotations
import argparse
import asyncio
import json
import os
import shutil
from pathlib import Path
from typing import Literal, cast
from uuid import uuid4
from typing import Literal
from claude_agent_sdk import (
ClaudeAgentOptions,
@ -21,7 +19,6 @@ from claude_agent_sdk import (
from native_tools import NativeDesktopTools
CaptureScope = Literal["auto", "window", "desktop"]
Route = Literal["native", "mcp"]
@ -47,45 +44,20 @@ def driver_binary() -> str:
raise RuntimeError("qwen-cua-driver is not on PATH; set CUA_DRIVER_BIN")
def capture_scope() -> CaptureScope:
value = os.environ.get("CUA_CAPTURE_SCOPE", "auto")
if value not in {"auto", "window", "desktop"}:
raise ValueError("CUA_CAPTURE_SCOPE must be auto, window, or desktop")
return cast(CaptureScope, value)
async def driver_call(binary: str, name: str, arguments: dict[str, str]) -> None:
process = await asyncio.create_subprocess_exec(
binary,
"call",
name,
json.dumps(arguments),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=30)
except TimeoutError:
process.kill()
await process.wait()
raise RuntimeError(f"qwen-cua-driver call {name} timed out") from None
if process.returncode != 0:
detail = stderr.decode().strip() or stdout.decode().strip()
raise RuntimeError(f"qwen-cua-driver call {name} failed: {detail}")
def prompt(task: str, route: Route, session: str | None = None) -> str:
def prompt(task: str, route: Route) -> str:
lifecycle = (
f"The host already started session {session!r}. Pass it to every Cua "
"tool that accepts a session; do not call start_session or end_session."
if session
else "The host owns the native driver session; the custom tools do not expose lifecycle."
"The MCP transport owns one implicit lifecycle session. Omit the session "
"field for ordinary calls so the runtime creates and reuses it."
if route == "mcp"
else "The native SDK client owns one implicit lifecycle session; custom "
"tools do not expose lifecycle."
)
return f"""Complete this trusted desktop task through Cua Driver:
{task}
Route: {route}. {lifecycle}
Route: {route}. {lifecycle} Select an exact window or desktop target for every
action; session identity never selects capture modality or permission authority.
Use only the supplied Cua tools for desktop observation and interaction. Inspect
state before each action and verify state after it. A timeout after a mutation
means the outcome is unknown: observe before retrying and never blindly replay.
@ -152,7 +124,6 @@ async def run_agent(options: ClaudeAgentOptions, task_prompt: str) -> None:
async def run_native(task: str) -> None:
runtime = NativeDesktopTools()
try:
await runtime.start()
options = ClaudeAgentOptions(
model=os.environ.get("CLAUDE_MODEL"),
cwd=Path.cwd(),
@ -175,38 +146,25 @@ async def run_native(task: str) -> None:
async def run_mcp(task: str) -> None:
binary = driver_binary()
scope = capture_scope()
session = f"claude-mcp-{uuid4().hex[:12]}"
started = False
try:
await driver_call(
binary,
"start_session",
{"session": session, "capture_scope": scope},
)
started = True
options = ClaudeAgentOptions(
model=os.environ.get("CLAUDE_MODEL"),
cwd=Path.cwd(),
tools=[],
mcp_servers={
"cua_driver": {
"type": "stdio",
"command": binary,
"args": ["mcp"],
}
},
strict_mcp_config=True,
# Claude's MCP allowlist accepts the server name to enable all
# tools from that server; wildcard tool-name globs are unsupported.
allowed_tools=["mcp__cua_driver"],
permission_mode="dontAsk",
max_turns=40,
)
await run_agent(options, prompt(task, "mcp", session))
finally:
if started:
await driver_call(binary, "end_session", {"session": session})
options = ClaudeAgentOptions(
model=os.environ.get("CLAUDE_MODEL"),
cwd=Path.cwd(),
tools=[],
mcp_servers={
"cua_driver": {
"type": "stdio",
"command": binary,
"args": ["mcp"],
}
},
strict_mcp_config=True,
# Claude's MCP allowlist accepts the server name to enable all
# tools from that server; wildcard tool-name globs are unsupported.
allowed_tools=["mcp__cua_driver"],
permission_mode="dontAsk",
max_turns=40,
)
await run_agent(options, prompt(task, "mcp"))
async def main() -> None:

75
packages/cua-driver/examples/agent-sdks/claude_agent.ts
View file

@ -1,9 +1,5 @@
/** Run a Claude Agent SDK desktop task through native Cua tools or MCP. */
import { execFile } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { promisify } from 'node:util';
import {
createSdkMcpServer,
query,
@ -16,9 +12,6 @@ import { z } from 'zod';
import { NativeDesktopTools } from './native-tools.js';
type Route = 'native' | 'mcp';
type CaptureScope = 'auto' | 'window' | 'desktop';
const execFileAsync = promisify(execFile);
function argumentsFromCli(): { route: Route; task: string } {
const args = process.argv.slice(2);
@ -37,22 +30,6 @@ function argumentsFromCli(): { route: Route; task: string } {
return { route, task };
}
function captureScope(): CaptureScope {
const value = process.env.CUA_CAPTURE_SCOPE ?? 'auto';
if (value !== 'auto' && value !== 'window' && value !== 'desktop') {
throw new Error('CUA_CAPTURE_SCOPE must be auto, window, or desktop');
}
return value;
}
async function driverCall(
binary: string,
name: string,
args: Record<string, string>
): Promise<void> {
await execFileAsync(binary, ['call', name, JSON.stringify(args)], { timeout: 30_000 });
}
async function runAgent(options: Options, prompt: string): Promise<void> {
let result: SDKResultMessage | undefined;
for await (const message of query({ prompt, options })) {
@ -98,15 +75,17 @@ function nativeServer(runtime: NativeDesktopTools) {
});
}
function taskPrompt(task: string, route: Route, session?: string): string {
const lifecycle = session
? `The host already started session ${JSON.stringify(session)}. Pass it to every Cua tool that accepts a session; do not call start_session or end_session.`
: 'The host owns the native driver session; custom tools do not expose lifecycle.';
function taskPrompt(task: string, route: Route): string {
const lifecycle =
route === 'mcp'
? 'The MCP transport owns one implicit lifecycle session. Omit the session field for ordinary calls so the runtime creates and reuses it.'
: 'The native SDK client owns one implicit lifecycle session; custom tools do not expose lifecycle.';
return `Complete this trusted desktop task through Cua Driver:
${task}
Route: ${route}. ${lifecycle}
Route: ${route}. ${lifecycle} Select an exact window or desktop target for every
action; session identity never selects capture modality or permission authority.
Use only supplied Cua tools for desktop observation and interaction. Inspect
before each action and verify afterward. If a mutation times out, observe before
any retry; never blindly replay an action with an unknown outcome. Do not
@ -117,7 +96,6 @@ action unless the task explicitly requests it. Name anything unverified.`;
async function runNative(task: string): Promise<void> {
const runtime = new NativeDesktopTools();
try {
await runtime.start();
await runAgent(
{
model: process.env.CLAUDE_MODEL,
@ -142,31 +120,22 @@ async function runNative(task: string): Promise<void> {
async function runMcp(task: string): Promise<void> {
const binary = process.env.CUA_DRIVER_BIN ?? 'qwen-cua-driver';
const scope = captureScope();
const session = `claude-mcp-${randomUUID().slice(0, 12)}`;
let started = false;
try {
await driverCall(binary, 'start_session', { session, capture_scope: scope });
started = true;
await runAgent(
{
model: process.env.CLAUDE_MODEL,
tools: [],
mcpServers: {
cua_driver: { type: 'stdio', command: binary, args: ['mcp'] },
},
strictMcpConfig: true,
// A bare MCP server name enables all of its tools. Claude does not
// support wildcard tool-name globs in this allowlist.
allowedTools: ['mcp__cua_driver'],
permissionMode: 'dontAsk',
maxTurns: 40,
await runAgent(
{
model: process.env.CLAUDE_MODEL,
tools: [],
mcpServers: {
cua_driver: { type: 'stdio', command: binary, args: ['mcp'] },
},
taskPrompt(task, 'mcp', session)
);
} finally {
if (started) await driverCall(binary, 'end_session', { session });
}
strictMcpConfig: true,
// A bare MCP server name enables all of its tools. Claude does not
// support wildcard tool-name globs in this allowlist.
allowedTools: ['mcp__cua_driver'],
permissionMode: 'dontAsk',
maxTurns: 40,
},
taskPrompt(task, 'mcp')
);
}
const args = argumentsFromCli();

87
packages/cua-driver/examples/agent-sdks/codex_agent.py
View file

@ -7,13 +7,9 @@ import asyncio
import json
import os
import shutil
from typing import Literal, cast
from uuid import uuid4
from openai_codex import AsyncCodex, CodexConfig, Sandbox
CaptureScope = Literal["auto", "window", "desktop"]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
@ -31,76 +27,37 @@ def driver_binary() -> str:
raise RuntimeError("qwen-cua-driver is not on PATH; set CUA_DRIVER_BIN")
def capture_scope() -> CaptureScope:
value = os.environ.get("CUA_CAPTURE_SCOPE", "auto")
if value not in {"auto", "window", "desktop"}:
raise ValueError("CUA_CAPTURE_SCOPE must be auto, window, or desktop")
return cast(CaptureScope, value)
async def driver_call(binary: str, name: str, arguments: dict[str, str]) -> None:
process = await asyncio.create_subprocess_exec(
binary,
"call",
name,
json.dumps(arguments),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=30)
except TimeoutError:
process.kill()
await process.wait()
raise RuntimeError(f"qwen-cua-driver call {name} timed out") from None
if process.returncode != 0:
detail = stderr.decode().strip() or stdout.decode().strip()
raise RuntimeError(f"qwen-cua-driver call {name} failed: {detail}")
async def main() -> None:
args = parse_args()
binary = driver_binary()
scope = capture_scope()
session = f"codex-python-{uuid4().hex[:12]}"
started = False
try:
await driver_call(
binary,
"start_session",
{"session": session, "capture_scope": scope},
config = CodexConfig(
config_overrides=(
f"mcp_servers.cua_driver.command={json.dumps(binary)}",
'mcp_servers.cua_driver.args=["mcp"]',
"mcp_servers.cua_driver.required=true",
)
started = True
config = CodexConfig(
config_overrides=(
f'mcp_servers.cua_driver.command={json.dumps(binary)}',
'mcp_servers.cua_driver.args=["mcp"]',
"mcp_servers.cua_driver.required=true",
)
)
async with AsyncCodex(config) as codex:
thread = await codex.thread_start(
model=os.environ.get("CODEX_MODEL"),
sandbox=Sandbox.read_only,
)
async with AsyncCodex(config) as codex:
thread = await codex.thread_start(
model=os.environ.get("CODEX_MODEL"),
sandbox=Sandbox.read_only,
)
result = await thread.run(
f"""Complete this trusted desktop task through the cua_driver MCP server:
result = await thread.run(
f"""Complete this trusted desktop task through the cua_driver MCP server:
{args.task}
The host already started session {session!r} with capture scope {scope!r}.
Use only cua_driver MCP tools for desktop observation and interaction. Pass
session {session!r} to every tool that accepts a session. Inspect before each
action and verify afterward. Do not call start_session or end_session. If a
mutation times out, observe before any retry; never blindly replay an action
with an unknown outcome. Return a concise result and name anything unverified.
This MCP transport owns one implicit lifecycle session. Omit the session field
for ordinary calls so the runtime creates and reuses it. Select an exact window
or desktop target on every action; session identity never selects capture
modality or permission authority. Use only cua_driver MCP tools for desktop
observation and interaction. Inspect before each action and verify afterward.
If a mutation times out, observe before any retry; never blindly replay an
action with an unknown outcome. Return a concise result and name anything
unverified.
"""
)
print(result.final_response)
finally:
if started:
await driver_call(binary, "end_session", {"session": session})
)
print(result.final_response)
if __name__ == "__main__":

95
packages/cua-driver/examples/agent-sdks/codex_agent.ts
View file

@ -1,16 +1,9 @@
/** Run a Codex SDK desktop task through Cua Driver's MCP server. */
import { execFile } from 'node:child_process';
import process from 'node:process';
import { promisify } from 'node:util';
import { randomUUID } from 'node:crypto';
import { Codex } from '@openai/codex-sdk';
type CaptureScope = 'auto' | 'window' | 'desktop';
const execFileAsync = promisify(execFile);
function taskFromArgs(): string {
const task = process.argv.slice(2).join(' ').trim();
if (!task) {
@ -19,37 +12,19 @@ function taskFromArgs(): string {
return task;
}
function captureScope(): CaptureScope {
const value = process.env.CUA_CAPTURE_SCOPE ?? 'auto';
if (value !== 'auto' && value !== 'window' && value !== 'desktop') {
throw new Error('CUA_CAPTURE_SCOPE must be auto, window, or desktop');
}
return value;
}
async function driverCall(
binary: string,
tool: string,
args: Record<string, string>
): Promise<void> {
await execFileAsync(binary, ['call', tool, JSON.stringify(args)], {
timeout: 30_000,
});
}
function agentPrompt(task: string, session: string, scope: CaptureScope): string {
function agentPrompt(task: string): string {
return `Complete this desktop task through the cua_driver MCP server:
${task}
The host has already started session ${JSON.stringify(session)} with capture
scope ${JSON.stringify(scope)}. Use only cua_driver MCP tools for desktop
This MCP transport owns one implicit lifecycle session. Omit the session field
for ordinary calls so the runtime creates and reuses it. Select an exact window
or desktop target on every action; session identity never selects capture
modality or permission authority. Use only cua_driver MCP tools for desktop
observation and interaction. Do not substitute shell, filesystem, web, or code
execution. Pass session ${JSON.stringify(session)} to every Cua tool that
accepts a session. Inspect state before each action and verify state after it.
Do not call start_session or end_session; the host owns lifecycle cleanup. Do
not blindly retry a mutation after a timeout or disconnect; observe first. Do
not perform purchases, send messages, delete data, expose credentials, or take
execution. Inspect state before each action and verify state after it. Do not
blindly retry a mutation after a timeout or disconnect; observe first. Do not
perform purchases, send messages, delete data, expose credentials, or take
another irreversible action unless the task explicitly requests that exact
action. Return a concise result and mention any step you could not verify.`;
}
@ -57,44 +32,28 @@ action. Return a concise result and mention any step you could not verify.`;
async function main(): Promise<void> {
const task = taskFromArgs();
const binary = process.env.CUA_DRIVER_BIN ?? 'qwen-cua-driver';
const scope = captureScope();
const session = `codex-agent-${randomUUID().slice(0, 12)}`;
let started = false;
try {
await driverCall(binary, 'start_session', {
session,
capture_scope: scope,
});
started = true;
const codex = new Codex({
config: {
mcp_servers: {
cua_driver: {
command: binary,
args: ['mcp'],
startup_timeout_sec: 30,
required: true,
},
const codex = new Codex({
config: {
mcp_servers: {
cua_driver: {
command: binary,
args: ['mcp'],
startup_timeout_sec: 30,
required: true,
},
},
});
const model = process.env.CODEX_MODEL;
const thread = codex.startThread({
...(model ? { model } : {}),
sandboxMode: 'read-only',
approvalPolicy: 'never',
skipGitRepoCheck: true,
});
},
});
const model = process.env.CODEX_MODEL;
const thread = codex.startThread({
...(model ? { model } : {}),
sandboxMode: 'read-only',
approvalPolicy: 'never',
skipGitRepoCheck: true,
});
const result = await thread.run(agentPrompt(task, session, scope));
console.log(result.finalResponse);
} finally {
if (started) {
await driverCall(binary, 'end_session', { session });
}
}
const result = await thread.run(agentPrompt(task));
console.log(result.finalResponse);
}
await main();

54
packages/cua-driver/examples/agent-sdks/native-tools.ts
View file

@ -1,20 +1,15 @@
/** Narrow agent-tool adapter over the same-process Cua Driver TypeScript SDK. */
import { randomUUID } from 'node:crypto';
import {
CaptureScope,
ActionTarget,
ClickButton,
ClickInput,
CuaDriver,
DesktopScope,
EndSessionInput,
GetDesktopStateInput,
PressKeyInput,
StartSessionInput,
ToolResult,
TypeTextInput,
} from '@trycua/cua-driver';
} from '@qwen-code/cua-sdk';
type McpContent =
| { type: 'text'; text: string }
@ -27,47 +22,23 @@ export type NativeToolResult = {
export class NativeDesktopTools {
readonly driver = CuaDriver.create(undefined);
readonly session = `claude-native-${randomUUID().slice(0, 12)}`;
private started = false;
readonly desktopTarget = new ActionTarget.Desktop({ displayId: 'primary' });
constructor(private readonly timeoutMs = 30_000) {}
async start(): Promise<void> {
await this.bounded(
this.driver.startSession(
StartSessionInput.new({
session: this.session,
captureScope: CaptureScope.Desktop,
})
),
'start_session'
);
this.started = true;
}
async close(): Promise<void> {
try {
if (this.started) {
await this.bounded(
this.driver.endSession(EndSessionInput.new({ session: this.session })),
'end_session'
);
this.started = false;
}
await this.driver.shutdown();
} finally {
try {
await this.driver.shutdown();
} finally {
// Generated UniFFI bindings expose deterministic handle release
// separately from asynchronous runtime shutdown.
(this.driver as unknown as { uniffiDestroy(): void }).uniffiDestroy();
}
// Generated UniFFI bindings expose deterministic handle release
// separately from asynchronous runtime shutdown.
(this.driver as unknown as { uniffiDestroy(): void }).uniffiDestroy();
}
}
async observe(): Promise<NativeToolResult> {
const result = await this.bounded(
this.driver.getDesktopState(GetDesktopStateInput.new({ session: this.session })),
this.driver.getDesktopState(GetDesktopStateInput.new({})),
'get_desktop_state'
);
return this.content(result);
@ -79,8 +50,7 @@ export class NativeDesktopTools {
ClickInput.new({
x,
y,
scope: DesktopScope.Desktop,
session: this.session,
target: this.desktopTarget,
button: ClickButton.Left,
count: 1,
})
@ -93,8 +63,7 @@ export class NativeDesktopTools {
this.driver.typeText(
TypeTextInput.new({
text,
scope: DesktopScope.Desktop,
session: this.session,
target: this.desktopTarget,
})
)
);
@ -105,8 +74,7 @@ export class NativeDesktopTools {
this.driver.pressKey(
PressKeyInput.new({
key,
scope: DesktopScope.Desktop,
session: this.session,
target: this.desktopTarget,
})
)
);

39
packages/cua-driver/examples/agent-sdks/native_driver.py
View file

@ -9,13 +9,10 @@ from urllib.request import Request, urlopen
from uuid import uuid4
from cua_driver import (
CaptureScope,
ActionTarget,
CuaDriver,
DesktopScope,
EndSessionInput,
GetDesktopStateInput,
PressKeyInput,
StartSessionInput,
TypeTextInput,
)
@ -47,25 +44,13 @@ def parse_args() -> argparse.Namespace:
async def main() -> None:
args = parse_args()
token = f"cua-{uuid4().hex[:10]}"
session = f"native-python-{uuid4().hex[:10]}"
await asyncio.to_thread(fixture_request, args.fixture, "/reset", method="POST")
driver = CuaDriver.create()
started = False
mutation_outcome_unknown = False
try:
await asyncio.wait_for(
driver.start_session(
StartSessionInput(session=session, capture_scope=CaptureScope.DESKTOP)
),
args.timeout,
)
started = True
before = await asyncio.wait_for(
driver.get_desktop_state(
GetDesktopStateInput(session=session, screenshot_out_file=None)
),
driver.get_desktop_state(GetDesktopStateInput(session=None, screenshot_out_file=None)),
args.timeout,
)
if before.is_error or not before.images:
@ -77,8 +62,9 @@ async def main() -> None:
driver.type_text(
TypeTextInput(
text=token,
scope=DesktopScope.DESKTOP,
session=session,
target=ActionTarget.DESKTOP(display_id="primary"),
scope=None,
session=None,
)
),
args.timeout,
@ -90,8 +76,9 @@ async def main() -> None:
PressKeyInput(
key="ENTER",
modifiers=None,
scope=DesktopScope.DESKTOP,
session=session,
target=ActionTarget.DESKTOP(display_id="primary"),
scope=None,
session=None,
)
),
args.timeout,
@ -105,9 +92,7 @@ async def main() -> None:
# the independent fixture first; it may have consumed the action.
await wait_for_submission(args.fixture, token, args.timeout)
after = await asyncio.wait_for(
driver.get_desktop_state(
GetDesktopStateInput(session=session, screenshot_out_file=None)
),
driver.get_desktop_state(GetDesktopStateInput(session=None, screenshot_out_file=None)),
args.timeout,
)
if after.is_error or not after.images:
@ -116,11 +101,7 @@ async def main() -> None:
if mutation_outcome_unknown:
print("the driver response was uncertain; the external postcondition resolved it")
finally:
try:
if started:
await driver.end_session(EndSessionInput(session=session))
finally:
await driver.shutdown()
await driver.shutdown()
if __name__ == "__main__":

45
packages/cua-driver/examples/agent-sdks/native_driver.ts
View file

@ -3,18 +3,16 @@
import { randomUUID } from 'node:crypto';
import {
CaptureScope,
ActionTarget,
CuaDriver,
DesktopScope,
EndSessionInput,
GetDesktopStateInput,
PressKeyInput,
StartSessionInput,
TypeTextInput,
} from '@trycua/cua-driver';
} from '@qwen-code/cua-sdk';
const fixture = process.env.CUA_FIXTURE_URL ?? 'http://127.0.0.1:8765';
const timeoutMs = 10_000;
const desktopTarget = new ActionTarget.Desktop({ displayId: 'primary' });
async function bounded<T>(promise: Promise<T>, label: string): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
@ -47,27 +45,14 @@ async function waitForSubmission(token: string): Promise<void> {
}
const token = `cua-${randomUUID().slice(0, 10)}`;
const session = `native-typescript-${randomUUID().slice(0, 10)}`;
await resetFixture();
const driver = CuaDriver.create(undefined);
let started = false;
let mutationOutcomeUnknown = false;
try {
await bounded(
driver.startSession(
StartSessionInput.new({
session,
captureScope: CaptureScope.Desktop,
})
),
'start_session'
);
started = true;
const before = await bounded(
driver.getDesktopState(GetDesktopStateInput.new({ session })),
driver.getDesktopState(GetDesktopStateInput.new({})),
'get_desktop_state'
);
if (before.isError || before.images.length === 0) {
@ -80,8 +65,7 @@ try {
driver.typeText(
TypeTextInput.new({
text: token,
scope: DesktopScope.Desktop,
session,
target: desktopTarget,
})
),
'type_text'
@ -92,8 +76,7 @@ try {
driver.pressKey(
PressKeyInput.new({
key: 'ENTER',
scope: DesktopScope.Desktop,
session,
target: desktopTarget,
})
),
'press_key'
@ -107,7 +90,7 @@ try {
// fixture before considering any retry.
await waitForSubmission(token);
const after = await bounded(
driver.getDesktopState(GetDesktopStateInput.new({ session })),
driver.getDesktopState(GetDesktopStateInput.new({})),
'post-action get_desktop_state'
);
if (after.isError || after.images.length === 0) {
@ -119,16 +102,10 @@ try {
}
} finally {
try {
if (started) {
await driver.endSession(EndSessionInput.new({ session }));
}
await driver.shutdown();
} finally {
try {
await driver.shutdown();
} finally {
// shutdown() drains the runtime; this generated destructor releases the
// native UniFFI handle immediately instead of waiting for JavaScript GC.
(driver as unknown as { uniffiDestroy(): void }).uniffiDestroy();
}
// shutdown() drains the runtime; this generated destructor releases the
// native UniFFI handle immediately instead of waiting for JavaScript GC.
(driver as unknown as { uniffiDestroy(): void }).uniffiDestroy();
}
}

47
packages/cua-driver/examples/agent-sdks/native_tools.py
View file

@ -5,18 +5,14 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from typing import TypeVar
from uuid import uuid4
from cua_driver import (
CaptureScope,
ActionTarget,
ClickButton,
ClickInput,
CuaDriver,
DesktopScope,
EndSessionInput,
GetDesktopStateInput,
PressKeyInput,
StartSessionInput,
ToolResult,
TypeTextInput,
)
@ -26,40 +22,20 @@ T = TypeVar("T")
class NativeDesktopTools:
"""Own one in-process driver runtime and one desktop-scoped session."""
"""Own one in-process runtime with one implicit lifecycle session."""
def __init__(self, timeout: float = 30.0) -> None:
self.driver = CuaDriver.create()
self.session = f"claude-native-{uuid4().hex[:12]}"
self.timeout = timeout
self.started = False
async def start(self) -> None:
await self._bounded(
self.driver.start_session(
StartSessionInput(
session=self.session,
capture_scope=CaptureScope.DESKTOP,
)
)
)
self.started = True
async def close(self) -> None:
try:
if self.started:
await self._bounded(
self.driver.end_session(EndSessionInput(session=self.session))
)
self.started = False
finally:
await self.driver.shutdown()
await self.driver.shutdown()
async def observe(self) -> dict[str, object]:
result = await self._bounded(
self.driver.get_desktop_state(
GetDesktopStateInput(
session=self.session,
session=None,
screenshot_out_file=None,
)
)
@ -72,8 +48,9 @@ class NativeDesktopTools:
ClickInput(
x=x,
y=y,
scope=DesktopScope.DESKTOP,
session=self.session,
target=ActionTarget.DESKTOP(display_id="primary"),
scope=None,
session=None,
button=ClickButton.LEFT,
count=1,
)
@ -85,8 +62,9 @@ class NativeDesktopTools:
lambda: self.driver.type_text(
TypeTextInput(
text=text,
scope=DesktopScope.DESKTOP,
session=self.session,
target=ActionTarget.DESKTOP(display_id="primary"),
scope=None,
session=None,
)
)
)
@ -97,8 +75,9 @@ class NativeDesktopTools:
PressKeyInput(
key=key,
modifiers=None,
scope=DesktopScope.DESKTOP,
session=self.session,
target=ActionTarget.DESKTOP(display_id="primary"),
scope=None,
session=None,
)
)
)

2208
packages/cua-driver/examples/agent-sdks/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

2
packages/cua-driver/examples/agent-sdks/package.json
View file

@ -11,7 +11,7 @@
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "0.3.218",
"@openai/codex-sdk": "0.145.0",
"@trycua/cua-driver": "0.12.2",
"@qwen-code/cua-sdk": "file:../../typescript",
"zod": "4.1.12"
},
"devDependencies": {

2
packages/cua-driver/examples/agent-sdks/requirements.txt
View file

@ -1,3 +1,3 @@
claude-agent-sdk==0.2.124
openai-codex==0.144.4
cua-driver==0.12.2
cua-driver==0.20.0

11
packages/cua-driver/python/README.md
View file

@ -38,7 +38,6 @@ in the importing process and does not require the executable or daemon.
import asyncio
from cua_driver import (
CaptureScope,
CuaDriver,
CursorReducedMotion,
EndSessionInput,
@ -50,7 +49,7 @@ from cua_driver import (
async def main() -> None:
driver = CuaDriver.create()
await driver.start_session(
StartSessionInput(session="demo", capture_scope=CaptureScope.DESKTOP)
StartSessionInput(session="demo", capture_scope=None, cursor_theme=None)
)
try:
await driver.set_agent_cursor_theme(
@ -77,7 +76,13 @@ text, images, verification/error metadata, and `structured_json` / `raw_json`
for platform-extensible results. Session lifecycle calls return dedicated
generated records.
The agent cursor is session-owned. Its default theme and custom dotLottie
`start_session` is optional for ordinary calls. The runtime creates one
implicit session for this SDK transport and reuses it until shutdown, explicit
end, or five minutes of inactivity. Use a named session when application code
needs to configure or inspect that run explicitly.
The agent cursor is session-owned and initializes on the first cursor-bearing
action, including `move_cursor`. Its default theme and custom dotLottie
authoring workflow are documented in
[`docs/cursor-themes.md`](../docs/cursor-themes.md). Custom source is compiled
and installed with the local CLI; SDK and MCP tools select only an installed

2
packages/cua-driver/python/pyproject.toml
View file

@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "cua-driver"
version = "0.17.0"
version = "0.20.0"
description = "Rust-backed Cua Driver SDK and bundled executable for client applications"
readme = "README.md"
license = { text = "MIT" }

6
packages/cua-driver/python/src/cua_driver/__init__.py
View file

@ -4,7 +4,7 @@ Agents should configure the bundled ``qwen-cua-driver mcp`` executable directly
through their runtime's MCP client instead of importing a language MCP facade.
"""
__version__ = "0.17.0" # x-release-please-version
__version__ = "0.20.0" # x-release-please-version
from ._native import (
ActionCompletion,
@ -79,9 +79,11 @@ from ._native_contract import (
GetDesktopStateInput,
GetScreenSizeInput,
GetSessionStateInput,
GetWindowStateInput,
HotkeyInput,
InvokeMenuInput,
MoveCursorInput,
ObservationRevisionInput,
Platform,
PredicateOutcome,
PressKeyInput,
@ -227,11 +229,13 @@ __all__ = [
"GetDesktopStateInput",
"GetScreenSizeInput",
"GetSessionStateInput",
"GetWindowStateInput",
"HotkeyInput",
"InvokeMenuInput",
"ImageContent",
"MacOsPermissionStatus",
"MoveCursorInput",
"ObservationRevisionInput",
"Platform",
"PredicateOutcome",
"PrivateWorkerOptions",

984
packages/cua-driver/python/src/cua_driver/_native.py
View file

File diff suppressed because it is too large Load diff

1673
packages/cua-driver/python/src/cua_driver/_native_contract.py
View file

File diff suppressed because it is too large Load diff

7
packages/cua-driver/python/tests/test_uniffi_loader.py
View file

@ -58,7 +58,7 @@ while True:
if request["method"] == "metadata":
result = {
"driver_version": "0.10.0",
"contract_version": "0.6.0",
"contract_version": "0.7.0",
"tools_list_schema_version": "1",
"capability_version": "1",
"mcp_protocol_version": "2025-06-18",
@ -142,7 +142,7 @@ except FileNotFoundError:
if request["method"] == "metadata":
result = {
"driver_version": "0.12.6",
"contract_version": "0.6.0",
"contract_version": "0.7.0",
"tools_list_schema_version": "1",
"capability_version": "1",
"mcp_protocol_version": "2025-06-18",
@ -186,6 +186,8 @@ except FileNotFoundError:
expected_methods = {
"start_session",
"escalate_session",
"get_session",
"list_sessions",
"get_session_state",
"end_session",
"get_desktop_state",
@ -224,6 +226,7 @@ except FileNotFoundError:
ClickInput(
x=12.0,
y=34.0,
target=None,
scope=DesktopScope.DESKTOP,
session="python-run",
button=ClickButton.LEFT,

2
packages/cua-driver/python/uv.lock generated
View file

@ -13,7 +13,7 @@ wheels = [
[[package]]
name = "cua-driver"
version = "0.17.0"
version = "0.20.0"
source = { editable = "." }
[package.optional-dependencies]

114
packages/cua-driver/rust/CHANGELOG.md
View file

@ -1,5 +1,119 @@
# Changelog
## [0.20.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.3...cua-driver-rs-v0.20.0) (2026-08-15)
### ⚠ BREAKING CHANGES
* **cua-driver:** remove browser approval tokens ([#3185](https://github.com/trycua/cua/issues/3185))
### Features
* add immutable Driver and Lume nightly releases ([8cef90b](https://github.com/trycua/cua/commit/8cef90b7ce6f85793a0ca3a3ddc050eac01b6b83))
* add persistent Driver and Lume release channels ([0813659](https://github.com/trycua/cua/commit/0813659ac237cd46da21c2a51f9a95f27a3e7845))
* **cua-driver:** add implicit lifecycle sessions ([#3013](https://github.com/trycua/cua/issues/3013)) ([858ef13](https://github.com/trycua/cua/commit/858ef13a5b1f2630b074d8f588a25cb0b19cab25))
* **cua-driver:** apply capability manifests across permission profiles ([#3015](https://github.com/trycua/cua/issues/3015)) ([3a53e53](https://github.com/trycua/cua/commit/3a53e53f99dcf8d5ed761c5cd0b5dd039c916121))
* **cua-driver:** remove browser approval tokens ([#3185](https://github.com/trycua/cua/issues/3185)) ([0d7c011](https://github.com/trycua/cua/commit/0d7c011eb31526d9c2b500806f35523dc2bdfb0e))
### Bug Fixes
* **cua-driver:** align agent guidance with lifecycle sessions ([#3041](https://github.com/trycua/cua/issues/3041)) ([dbde09e](https://github.com/trycua/cua/commit/dbde09e0bfd39abbc374e2e9aade86086e773bb8))
* **cua-driver:** hide policy-disabled MCP tools ([#3132](https://github.com/trycua/cua/issues/3132)) ([1532830](https://github.com/trycua/cua/commit/1532830d3d5c6e5db22198130d82a4741a7b4a2f))
* **cua-driver:** preserve direct MCP session ownership ([#3079](https://github.com/trycua/cua/issues/3079)) ([d509687](https://github.com/trycua/cua/commit/d5096871d088c2e5b5942d678f2c127840061d7d))
* **cua-driver:** preserve named CLI sessions ([#3144](https://github.com/trycua/cua/issues/3144)) ([1803e52](https://github.com/trycua/cua/commit/1803e52d0c4a94d88787706cfe89eccb6dc723db))
* **cua-driver:** reject misplaced MCP permission flags ([#3085](https://github.com/trycua/cua/issues/3085)) ([a5bc265](https://github.com/trycua/cua/commit/a5bc2651c10dc64f4779f2c9fd99bfddb9905ffb))
* **cua-driver:** stage current skill pack on Windows local installs ([#3083](https://github.com/trycua/cua/issues/3083)) ([b4bc58a](https://github.com/trycua/cua/commit/b4bc58a3a262f30fcd9d6772851cde0f3429deb6))
* **cua-driver:** verify foreground focus before input ([#3068](https://github.com/trycua/cua/issues/3068)) ([f9192dd](https://github.com/trycua/cua/commit/f9192dde69dd85ff6c5ad0bf4731d33ebc7d20d3))
* **cua-driver:** write the update-check cache to the canonical home ([#3032](https://github.com/trycua/cua/issues/3032)) ([69e5774](https://github.com/trycua/cua/commit/69e57742751407258f2189ec2c4bb21fba78da2e))
## [0.19.3](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.2...cua-driver-rs-v0.19.3) (2026-08-10)
### Bug Fixes
* **cua-driver:** remove Windows npm VC runtime prerequisite ([#3038](https://github.com/trycua/cua/issues/3038)) ([c167125](https://github.com/trycua/cua/commit/c167125e1bb0c4fe515beddc0d480f573f3c6077))
## [0.19.2](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.1...cua-driver-rs-v0.19.2) (2026-08-07)
### Bug Fixes
* **cua-driver:** constrain incompatible zune-core resolution ([#2984](https://github.com/trycua/cua/issues/2984)) ([1979ab7](https://github.com/trycua/cua/commit/1979ab73fdcdba130a54467ca7072c447011f7ab))
* **cua-driver:** make MCP output schemas object-rooted ([96d87ad](https://github.com/trycua/cua/commit/96d87adb248d746aed2a5427b1a256dbd3f1ae1a))
* **cua-driver:** reap Linux launch_app children instead of leaking zombies ([#2974](https://github.com/trycua/cua/issues/2974)) ([f91d78e](https://github.com/trycua/cua/commit/f91d78e49a74fd7a690100957886e93e185655e2))
## [0.19.1](https://github.com/trycua/cua/compare/cua-driver-rs-v0.19.0...cua-driver-rs-v0.19.1) (2026-08-07)
### Bug Fixes
* **cua-driver:** advertise refusals in the MCP outputSchema ([#2968](https://github.com/trycua/cua/issues/2968)) ([dc6f32c](https://github.com/trycua/cua/commit/dc6f32cd4d32bb0a60e18674086f443f9d9d8288))
* **cua-driver:** record launch provenance for sessionless launch_app so kill_app can reprove it ([#2966](https://github.com/trycua/cua/issues/2966)) ([b065550](https://github.com/trycua/cua/commit/b06555075fc0ccc14df7e17d33b7eb27d97c9c53)), closes [#2965](https://github.com/trycua/cua/issues/2965)
* **cua-driver:** resolve Linux launch_app names via .desktop entries and surface xdg-open failures ([#2954](https://github.com/trycua/cua/issues/2954)) ([fb4b492](https://github.com/trycua/cua/commit/fb4b492cd997dd3e8659298adfd98825974ebb7e))
* **cua-driver:** survive z-order BadMatch under reparenting WMs so the Linux agent cursor can paint ([#2957](https://github.com/trycua/cua/issues/2957)) ([9522358](https://github.com/trycua/cua/commit/9522358294f34aba138221e87d3dbb79486b199a)), closes [#2955](https://github.com/trycua/cua/issues/2955)
## [0.19.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.18.0...cua-driver-rs-v0.19.0) (2026-08-06)
### Features
* **cua-driver:** support Prime Agent skill onboarding ([#2944](https://github.com/trycua/cua/issues/2944)) ([0f796d0](https://github.com/trycua/cua/commit/0f796d0659877bee3404d11eef566cfbcbf8edf1))
### Bug Fixes
* **cua-driver:** Linux cursor overlay compositing, xrdp-family servers, and resting float ([#2940](https://github.com/trycua/cua/issues/2940)) ([997d6d8](https://github.com/trycua/cua/commit/997d6d86433254ccf12e5dbf1edcb46bf05eb5ac))
* **cua-driver:** resolve SDK release from run artifacts ([#2926](https://github.com/trycua/cua/issues/2926)) ([3cd9de9](https://github.com/trycua/cua/commit/3cd9de9a32bf6b4a98a2186c1535054bd740bb21))
## [0.18.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.17.0...cua-driver-rs-v0.18.0) (2026-08-05)
### Features
* **cua-driver:** add macOS Spaces awareness to list_windows ([#2850](https://github.com/trycua/cua/issues/2850)) ([f7404b4](https://github.com/trycua/cua/commit/f7404b40287c128b15afa30b6c6f9024cbf081b8))
* **cua-driver:** drop the session orb from the cursor badge ([#2842](https://github.com/trycua/cua/issues/2842)) ([59f280e](https://github.com/trycua/cua/commit/59f280e485be4f58811bda6a9e2dbe7632b9b63c))
* **cua-driver:** exact-target macOS background input v1 ([#2837](https://github.com/trycua/cua/issues/2837)) ([1b2cb5a](https://github.com/trycua/cua/commit/1b2cb5a706c3e5d636b683ab15336dbf35e579e0))
### Bug Fixes
* **cua-driver:** avoid accessibility advertise on Cinnamon ([#2784](https://github.com/trycua/cua/issues/2784)) ([b3ba121](https://github.com/trycua/cua/commit/b3ba12126188e1540b93d7c51d55926299fd36ab))
* **cua-driver:** avoid retrying deferred text writes ([7e0d6bc](https://github.com/trycua/cua/commit/7e0d6bcb5d7ca55d9da1ff3e8acc1bdddf6fa422))
* **cua-driver:** bound large macOS text synthesis ([#2863](https://github.com/trycua/cua/issues/2863)) ([a6b87e0](https://github.com/trycua/cua/commit/a6b87e01af9c3f291a40a9d7f4e6301418b2a8ab))
* **cua-driver:** bound Linux AT-SPI listener startup ([#2827](https://github.com/trycua/cua/issues/2827)) ([ed3c365](https://github.com/trycua/cua/commit/ed3c365c954446c774c422e8018541e56155dd6a))
* **cua-driver:** bound Linux snapshots after app exit ([#2822](https://github.com/trycua/cua/issues/2822)) ([d1c5a7a](https://github.com/trycua/cua/commit/d1c5a7a90b23a37a18838d9c2122d003d00e7ecf))
* **cua-driver:** bound Windows app-name lookup ([#2858](https://github.com/trycua/cua/issues/2858)) ([2be7aab](https://github.com/trycua/cua/commit/2be7aab3de35bdffa9d8ad767471c0699d36d6e5))
* **cua-driver:** bound Windows installed-app discovery ([#2855](https://github.com/trycua/cua/issues/2855)) ([ea971ef](https://github.com/trycua/cua/commit/ea971ef06d4ebeda16c6947cb2bca2dd875d2ddc))
* **cua-driver:** clarify synthetic browser click outcomes ([#2866](https://github.com/trycua/cua/issues/2866)) ([197c518](https://github.com/trycua/cua/commit/197c518c9c364bd30da8bc076748ea82ed093e27))
* **cua-driver:** explain permanent scope recovery ([#2857](https://github.com/trycua/cua/issues/2857)) ([9110d9f](https://github.com/trycua/cua/commit/9110d9fd69c7fda41caab3aaac9df80c87bdab2f))
* **cua-driver:** fail closed on X11 overlay shape errors ([#1818](https://github.com/trycua/cua/issues/1818)) ([0462954](https://github.com/trycua/cua/commit/0462954cfb9843ca0cc43f49262acf0e69e3517d))
* **cua-driver:** hard-bound Windows UIA provider calls ([#2117](https://github.com/trycua/cua/issues/2117)) ([c9c6607](https://github.com/trycua/cua/commit/c9c660770a26c2f0024b9be1079fac0d4c9effff))
* **cua-driver:** harden Windows discovery and foreground input ([#2812](https://github.com/trycua/cua/issues/2812)) ([441cae0](https://github.com/trycua/cua/commit/441cae0d4d4048524ce2f7eeabfb63a6c9a422d3))
* **cua-driver:** ignore macOS compositor sibling surfaces ([#2908](https://github.com/trycua/cua/issues/2908)) ([df57e61](https://github.com/trycua/cua/commit/df57e610d3f2e9c07aac59d4896d48656490e1cc))
* **cua-driver:** keep Windows autostart on the junction path ([#2809](https://github.com/trycua/cua/issues/2809)) ([06f0c04](https://github.com/trycua/cua/commit/06f0c048fb11b1780e61252eb41a2b968865a8a7))
* **cua-driver:** keep Windows cursor visible without focus theft ([#2864](https://github.com/trycua/cua/issues/2864)) ([2e7a412](https://github.com/trycua/cua/commit/2e7a41215ec9e3594567029aad07450c3d731721))
* **cua-driver:** keep remote debugging enabled when a restart is needed ([#2911](https://github.com/trycua/cua/issues/2911)) ([f5b15cc](https://github.com/trycua/cua/commit/f5b15cccf6a9f0424dd2d83f730b523c4be21df0))
* **cua-driver:** launch desktop AppsFolder apps on Windows ([#2862](https://github.com/trycua/cua/issues/2862)) ([fb2a6f6](https://github.com/trycua/cua/commit/fb2a6f60522ebf132270a3377c9a0496d2c0ba98))
* **cua-driver:** prevent Linux uinput pointer panics ([#2736](https://github.com/trycua/cua/issues/2736)) ([9fde53e](https://github.com/trycua/cua/commit/9fde53e1fb190834054eb36e7574bc0b523831e2))
* **cua-driver:** quiesce idle Wayland overlay ([#2828](https://github.com/trycua/cua/issues/2828)) ([35b71a0](https://github.com/trycua/cua/commit/35b71a0ebd081c333bf53af6b386ef4798dd05b1))
* **cua-driver:** refuse minimized Windows element actions ([8a9b7ba](https://github.com/trycua/cua/commit/8a9b7baa32bb5b7ea48a1ee6ffc1d87a433405e4))
* **cua-driver:** repair X11 overlay after display changes ([#2354](https://github.com/trycua/cua/issues/2354)) ([81019b5](https://github.com/trycua/cua/commit/81019b54b89d577ccb5d7758eba0fd33861fc9d9))
* **cua-driver:** report honest macOS browser chrome coverage ([#2919](https://github.com/trycua/cua/issues/2919)) ([7c52b84](https://github.com/trycua/cua/commit/7c52b8449fab36cdda9fe4310e6afd3be202c955))
* **cua-driver:** report macOS key delivery truthfully ([#2830](https://github.com/trycua/cua/issues/2830)) ([3dc0fc0](https://github.com/trycua/cua/commit/3dc0fc0e345f6d8107899a18c74245615e294c03))
* **cua-driver:** report PostMessage pixel fallback truthfully ([4dc537e](https://github.com/trycua/cua/commit/4dc537ef34e715c099ab30c7d45618e791da2ad1))
* **cua-driver:** scope the Linux AT-SPI walk to the requested window ([#2895](https://github.com/trycua/cua/issues/2895)) ([8c515ca](https://github.com/trycua/cua/commit/8c515ca74d7f9dad2b3ae9d74b59294511824fa1))
* **cua-driver:** select the correct Linux AT-SPI application tree ([#2740](https://github.com/trycua/cua/issues/2740)) ([d86e49b](https://github.com/trycua/cua/commit/d86e49ba4bf1c456fd16eca220184a38600ed927))
* **cua-driver:** stage local installs over a running driver ([#2889](https://github.com/trycua/cua/issues/2889)) ([157816e](https://github.com/trycua/cua/commit/157816e6144a79162253762b79d0abdfffa8e486))
* **cua-driver:** stop Windows update --apply from killing itself ([#2805](https://github.com/trycua/cua/issues/2805)) ([aebd996](https://github.com/trycua/cua/commit/aebd9962d2686e75ff9e17e0a3735e303ff96981))
* **cua-driver:** verify exact macOS window activation ([#2829](https://github.com/trycua/cua/issues/2829)) ([cd5c6f3](https://github.com/trycua/cua/commit/cd5c6f3ec5c2f71a8fbc64563d42933d381008ba))
### Performance Improvements
* **cua-driver:** accelerate Linux X11 capture with MIT-SHM ([#2796](https://github.com/trycua/cua/issues/2796)) ([5448c44](https://github.com/trycua/cua/commit/5448c449c81c81a7e48a418f8276d257f0e4c563))
* **cua-driver:** accelerate macOS window capture with ScreenCaptureKit ([#2795](https://github.com/trycua/cua/issues/2795)) ([bc90373](https://github.com/trycua/cua/commit/bc90373362cb7c521b1ff03f94457d5de618095c))
## [0.17.0](https://github.com/trycua/cua/compare/cua-driver-rs-v0.16.0...cua-driver-rs-v0.17.0) (2026-08-02)

32
packages/cua-driver/rust/Cargo.lock generated
View file

@ -981,11 +981,12 @@ dependencies = [
[[package]]
name = "cua-driver"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"async-trait",
"base64",
"core-foundation",
"cua-driver-contract",
"cua-driver-core",
"cua-driver-sdk",
@ -1016,7 +1017,7 @@ dependencies = [
[[package]]
name = "cua-driver-bindgen"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"cbindgen",
"uniffi",
@ -1024,7 +1025,7 @@ dependencies = [
[[package]]
name = "cua-driver-contract"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"schemars",
"serde",
@ -1034,7 +1035,7 @@ dependencies = [
[[package]]
name = "cua-driver-core"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"async-trait",
@ -1042,6 +1043,7 @@ dependencies = [
"cua-driver-contract",
"futures-util",
"image",
"jsonschema",
"libc",
"regex",
"regorus",
@ -1057,11 +1059,12 @@ dependencies = [
"url",
"uuid",
"windows 0.58.0",
"zune-core",
]
[[package]]
name = "cua-driver-sdk"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"async-trait",
"core-foundation",
@ -1083,7 +1086,7 @@ dependencies = [
[[package]]
name = "cua-driver-testkit"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"core-foundation",
"libc",
@ -1097,7 +1100,7 @@ dependencies = [
[[package]]
name = "cua-driver-uia"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"cua-driver-core",
@ -1112,7 +1115,7 @@ dependencies = [
[[package]]
name = "cursor-overlay"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"cua-driver-contract",
@ -1129,7 +1132,7 @@ dependencies = [
[[package]]
name = "cursor-theme-cli"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"cursor-overlay",
@ -2897,7 +2900,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "pip-preview"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"serde_json",
@ -2957,7 +2960,7 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "platform-linux"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"ashpd",
@ -2976,6 +2979,7 @@ dependencies = [
"image",
"libc",
"libspa",
"memmap2",
"meval",
"pip-preview",
"pipewire",
@ -2988,6 +2992,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"url",
"uuid",
"wayland-backend",
"wayland-client",
"wayland-protocols",
@ -3001,7 +3006,7 @@ dependencies = [
[[package]]
name = "platform-macos"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"async-trait",
@ -3036,7 +3041,7 @@ dependencies = [
[[package]]
name = "platform-windows"
version = "0.17.0"
version = "0.20.0"
dependencies = [
"anyhow",
"async-trait",
@ -3054,6 +3059,7 @@ dependencies = [
"tiny-skia",
"tokio",
"tracing",
"uuid",
"windows 0.58.0",
]

6
packages/cua-driver/rust/Cargo.toml
View file

@ -17,7 +17,7 @@ members = [
]
[workspace.package]
version = "0.17.0"
version = "0.20.0"
edition = "2021"
authors = ["trycua"]
license = "MIT"
@ -34,6 +34,10 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
base64 = "0.22"
uuid = { version = "1", features = ["v4"] }
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
# zune-jpeg 0.5.15 accepts zune-core ^0.5, but zune-core 0.5.2 changed its
# disabled logging macros in a source-incompatible way. Keep fresh downstream
# resolution on the last compatible pair until zune-jpeg publishes its fix.
zune-core = "=0.5.1"
uniffi = "=0.31.0"
postcard = { version = "1", features = ["alloc"] }
sha2 = "0.10"

64
packages/cua-driver/rust/Skills/cua-driver/BROWSER.md
View file

@ -15,21 +15,40 @@ mints session-scoped tab and element capabilities.
The canonical loop is:
```text
start_session
start_session(session?) # optional; can name before acting
list_windows or launch_app
get_browser_state(pid, window_id, session) # bind
get_browser_state(target_id, tab_id, session,
snapshot_format=semantic_v2) # snapshot
get_browser_state(pid, window_id, session?) # bind
get_browser_state(target_id, tab_id, session?,
snapshot_format=semantic_v2) # snapshot
browser_navigate / browser_click / browser_type / browser_pointer
browser_dialog / browser_set_input_files / browser_download
get_browser_state(target_id, tab_id, session,
snapshot_format=semantic_v2) # verify and refresh refs
end_session
get_browser_state(target_id, tab_id, session?,
snapshot_format=semantic_v2) # verify and refresh refs
end_session(session?) # optional cleanup
```
Use one explicit `session` value throughout. Never substitute a raw CDP
target id, tab ordinal, URL match, or remembered ref for a capability returned
by `get_browser_state`.
For a multi-call browser workflow, prefer a short `session` label and pass the
same value on every call that accepts it. Passing it once is not sticky; a later
omitted value uses the transport's implicit session. One long-lived MCP or SDK
transport may omit `session` for one-off or deliberately unlabeled work; its
first admitted call creates one implicit session and later unnamed calls reuse
it. Direct one-shot CLI calls use disposable transports. Never substitute a raw
CDP target id, tab ordinal, URL match, or remembered ref for a capability
returned by `get_browser_state`.
### Copy page content to the system clipboard
If the requested outcome is exact page content on the system clipboard—not a
literal text-selection gesture—read the content from a fresh semantic browser
snapshot, call `clipboard_write` with the exact observed value, and verify it
with `clipboard_read`. This path is background-safe and does not require a
clickable ref: passive headings and text nodes are evidence sources, not
controls that must be clicked before their value can be copied.
Fall back to visual text selection and the platform copy hotkey only when the
user explicitly requires that gesture or clipboard tools are unavailable.
That fallback is native input, not a typed page mutation, and may require the
foreground escalation rules in `SKILL.md`.
### Browser recording feedback
@ -88,15 +107,17 @@ Prefer an isolated profile when the task does not need the user's existing
cookies or login state:
```bash
# Direct CLI/raw clients mint this token interactively. MCP hosts can use their
# destructive-tool approval flow instead.
qwen-cua-driver browser-approve --pid 4242 --profile-mode isolated_new
qwen-cua-driver browser_prepare \
'{"pid":4242,"session":"browser-run-1","allow_launch":true,
"profile":{"mode":"isolated_new"},"approval_token":"<token>"}'
"profile":{"mode":"isolated_new"}}'
```
Isolated preparation follows the runtime permission mode and optional
capability manifest. Standard mode treats it as routine, bounded mode requires
a matching manifest, and unrestricted mode requires the launcher's dangerous
acknowledgement. `allow_launch: true` states that this call may create the
separate process; it does not widen runtime authorization.
Use `isolated_named` with a path-safe `name` for a reusable driver-managed
profile. Preparation launches a separate browser and never copies, modifies,
or terminates the requested personal profile. The result returns a
@ -282,9 +303,11 @@ qwen-cua-driver browser_click \
`dom_event` calls the page element's click behavior without pretending that a
trusted pointer event occurred. It requires a ref and is the full-background
alternative where supported. Never silently change trust class after a
refusal. Coordinate clicks accept viewport CSS `x` and `y`, but only on the
trusted route; prefer refs.
alternative where supported. Dispatch is not proof that the control activated:
trust-gated controls can ignore synthetic events, so refresh page state and
verify the expected postcondition. Never silently change trust class or
foreground the browser after a refusal. Coordinate clicks accept viewport CSS
`x` and `y`, but only on the trusted route; prefer refs.
### Type
@ -409,8 +432,9 @@ result from the current host, process, window, session, and tab.
- `browser_requires_setup`: obtain explicit approval and call
`browser_prepare`; never make setup a hidden read side effect.
- `browser_consent_required`: restart standard mode with the trusted launch
grant, use a matching bounded manifest, or let the embedding host decide the
attested request. Do not automate a generic approval dialog.
grant, use a capability manifest that admits the exact resource while the
selected profile remains independently binding, or let the embedding host
decide the attested request. Do not automate a generic approval dialog.
- `browser_binding_ambiguous` or heuristic binding: resolve the native-window
ambiguity and bind again; do not mutate.
- `browser_ref_stale`: snapshot again and use a new ref.

13
packages/cua-driver/rust/Skills/cua-driver/EMBEDDING.md
View file

@ -47,7 +47,7 @@ create `CuaDriver` directly. This path does not start an executable or open a
socket, and TCC checks execute as the importing application:
```ts
import { CuaDriver } from '@trycua/cua-driver';
import { CuaDriver } from '@qwen-code/cua-sdk';
const driver = CuaDriver.create(undefined);
try {
@ -140,13 +140,13 @@ migration.
### Node and Electron daemon hosts
Use the embedded host in `@trycua/cua-driver` instead of implementing process
Use the embedded host in `@qwen-code/cua-sdk` instead of implementing process
and socket management in every host. It starts a private daemon directly, waits
until its socket accepts connections, returns SDK and MCP connection details,
and owns restart and cleanup:
```ts
import { CuaDriver, EmbeddedCuaDriverHost } from '@trycua/cua-driver';
import { CuaDriver, EmbeddedCuaDriverHost } from '@qwen-code/cua-sdk';
const embedded = new EmbeddedCuaDriverHost(
'/path/inside/YourApp.app/Contents/Resources/qwen-cua-driver',
@ -194,8 +194,9 @@ daemon child.
completion is unknown.
- The Rust owner holds a parent-liveness pipe, so host death closes the daemon;
orderly shutdown should still await `stop()`.
- Capture scope belongs to each session. One embedded daemon can concurrently
serve `auto`, strict `window`, and strict `desktop` sessions.
- Capture modality belongs to each observation or action target, not to the
lifecycle session. One embedded daemon can concurrently serve exact window
and desktop calls without changing session state.
- Permission changes require destroying clients, restarting the daemon, and
reconnecting. A connection from the old generation is never reusable.
@ -272,7 +273,7 @@ a dialog (the `prompt` argument is ignored) and returns:
"embedded": true,
"pid": 12345,
"responsible_ppid": 12300,
"executable": "/path/to/cua-driver",
"executable": "/path/to/qwen-cua-driver",
"disclaim_env": false,
"note": "Embedded mode: these booleans reflect the HOST app's TCC grant…"
}

12
packages/cua-driver/rust/Skills/cua-driver/LINUX.md
View file

@ -127,11 +127,13 @@ CI so the three surfaces can't drift. Linux-relevant notes:
Linux builds rejected it via `additionalProperties:false` (it was
effectively macOS-only); it is now uniformly schema-accepted — Linux
glides a per-session cursor on X11 where the overlay is available.
- **Windowless screen-absolute actions** pass `scope:"desktop"` with no
pid/window_id, uniformly with macOS and Windows. The session must have
effective desktop scope (`start_session(..., capture_scope:"desktop")`, or
an explicitly escalated `auto` session). Strict window sessions receive
`desktop_scope_disabled`; no persistent config is read or written.
- **Windowless screen-absolute actions** use
`target:{"kind":"desktop","display_id":"primary"}`, uniformly with macOS
and Windows. Exact window actions use
`target:{"kind":"window","pid":PID,"window_id":WINDOW_ID}`. Legacy flat
`scope`, `pid`, and `window_id` fields remain compatibility inputs, but do
not combine them with `target`; capture modality no longer changes lifecycle
session state.
## Native application menus

11
packages/cua-driver/rust/Skills/cua-driver/MACOS.md
View file

@ -324,11 +324,12 @@ _Cross-platform parameter contract_):
- **`session` always worked on macOS;** the cross-platform change is that
Windows/Linux stopped _rejecting_ it. No macOS-side change to how you
pass it.
- **`scope`** (`window` / `desktop`) selects the action form uniformly on all
platforms. Pass `scope:"desktop"` with no pid/window_id for screen-absolute
pointer actions or foreground keyboard actions. The session's immutable
`capture_scope` policy must permit that form; set it with `start_session`,
never persistent config.
- **`target`** selects the action coordinate space uniformly on all platforms.
Use `target:{"kind":"desktop","display_id":"primary"}` for
screen-absolute pointer actions or foreground keyboard actions, and
`target:{"kind":"window","pid":PID,"window_id":WINDOW_ID}` for exact
window coordinates. Legacy flat `scope`, `pid`, and `window_id` fields remain
compatibility inputs, but do not combine them with `target`.
### Canvases, viewports, games (Blender, Unity, GHOST, Qt, wxWidgets)

198
packages/cua-driver/rust/Skills/cua-driver/SKILL.md
View file

@ -1,7 +1,7 @@
---
name: cua-driver
description: Drive a native GUI app (macOS, Windows, Linux) via the Qwen Cua Driver CLI (default) or MCP server; snapshot its accessibility tree, act through snapshot-bound element tokens, native menu paths, exact window geometry, or pixel coordinates, and verify from fresh state. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real application on the host.
version: 0.17.0 # x-release-please-version
version: 0.20.0 # x-release-please-version
metadata:
openclaw:
requires:
@ -86,7 +86,8 @@ before stopping or advancing:
3. **Background pixel action.** Use the pixels from the same state snapshot.
4. **Foreground delivery.** Retry only the action that evidence says could not
land in the background.
5. **Desktop fallback.** Enter this explicit, one-way session phase last.
5. **Desktop fallback.** Select an exact desktop target for that call only.
Later calls may return to an exact window target in the same session.
Use Cua Driver when the outcome lives in an application's UI or window state,
or when the user explicitly asks to operate that GUI. Once the task crosses
@ -123,10 +124,27 @@ and keep each claim narrow:
partial result, stop that GUI path and surface the unresolved state instead
of retrying blindly.
### Clipboard outcomes and GUI fallbacks
When the requested postcondition is an exact value on the system clipboard,
rather than the literal gesture of selecting and copying it, keep the operation
semantic. Read the value from the narrowest typed source, call
`clipboard_write`, then prove the real clipboard state with `clipboard_read`.
For browser content, this means reading the page with `get_browser_state` and
writing the exact observed text; a passive page-text ref does not need to be
clicked first.
Use visual selection followed by the platform copy hotkey only when the user
explicitly asks for that gesture, the source cannot expose the value
semantically, or direct clipboard tools are unavailable. Treat that as a GUI
fallback: re-snapshot before acting, verify the selected range when the
application exposes it, and escalate only the delivery step that cannot land
in the background.
## The no-foreground principle (window phase)
In a strict `window` session, and during the initial window phase of an
`auto` session, **the user's frontmost app MUST NOT change.** Every platform
During window-targeted background actions, **the user's frontmost app MUST NOT
change.** Every platform
has its own list of forbidden commands:
- macOS: any `open` invocation, any `osascript` that mutates GUI
@ -140,11 +158,11 @@ If you reach for a command that says "activate", "foreground",
"raise", or "make key", stop and translate to the cua-driver tool
that does the same intent without focus-stealing.
A strict `desktop` session is an explicit user choice to operate the visible
desktop and therefore uses foreground/system input. An `auto` session may enter
that phase only after the complete window ladder below has been attempted and
verified, followed by `escalate_session`. Never infer desktop permission from a
failed action or a proxy/transport session id.
A desktop target is an explicit per-call choice to operate the visible desktop
and therefore uses foreground/system input. Use it only after the narrower
window ladder has been attempted and verified. Permission policy must still
admit the display resource. Never infer desktop permission from a failed action
or a public session label.
## GUI transport defaults — prefer cua-driver over GUI shell shims
@ -215,7 +233,7 @@ qwen-cua-driver stop
```
For Chromium page content, keep the same native window selection but switch to
the browser capability loop: `start_session`, bind `(pid, window_id)` with
the browser capability loop: use one lifecycle session, bind `(pid, window_id)` with
`get_browser_state`, snapshot the returned tab, then use `browser_click`,
`browser_type`, or `browser_navigate`. Read `BROWSER.md` before using this
route. Browser target ids, tab ids, and refs are session-scoped and stale refs
@ -223,8 +241,9 @@ must be replaced by a fresh snapshot.
## Agent cursor overlay
Visual cursor overlay for demos and screen recordings. It is enabled by
default for declared sessions; anonymous actions remain cursor-less. Toggle with
Visual cursor overlay for demos and screen recordings. It initializes on the
first cursor-bearing action, including `move_cursor`, and follows the
transport's implicit or named lifecycle session. Toggle a named cursor with
`set_agent_cursor_enabled` to hide or re-show it. The embedded
`cua.default` theme uses a session-colored pointer over a larger,
cursor-shaped glow in the same session color. The glow fades to transparent
@ -266,6 +285,11 @@ recording, do a pixel click (`click({pid,x,y})`) or a `move_cursor`
first to put the cursor on-screen; subsequent AX actions then glide the
full path normally.
Pixel `click` already glides the overlay. Do not call `move_cursor`
immediately before `click` on the same target; that plays two glides.
Use `move_cursor` to place the overlay without clicking, or as the
one-time seed above before AX actions.
Requires a suitable UI event loop. Service and private-worker runtimes provide
one. On macOS, a same-process SDK runtime or `qwen-cua-driver mcp --direct` without
a certified host main-thread adapter returns a structured
@ -328,9 +352,11 @@ The route vocabulary is intentionally cross-platform:
An optional escalation is a harness instruction, never an automatic retry:
- `pixel`: refresh visual state and choose an exact pixel target;
- `foreground`: explicitly select foreground delivery if session policy allows;
- `foreground`: explicitly select foreground delivery if the authorization
stack admits the tool and exact target;
- `page`: bind the native window to a supported browser page route;
- `session`: prepare or explicitly widen the session only when policy permits.
- `session`: a legacy compatibility signal from an older capture-scope daemon;
current callers choose a desktop target on the specific action instead.
Branch on the closed reason vocabulary:
`route_unavailable`, `delivery_failed`, `effect_unconfirmed`,
@ -340,36 +366,67 @@ After any action, keep using `verify_state` or a fresh state snapshot for the
actual task postcondition. The multimodal harness owns visual reading and the
decision to stop, retry, or advance the ladder.
## Choose capture scope when the session starts
## Choose the target on each action
`capture_scope` is a per-session policy, not persistent configuration. Declare
it with `start_session`; it is immutable until that session ends. Concurrent
sessions may choose different policies safely.
A session owns lifecycle, cursor, recording, cleanup, and telemetry state. It
does not store the current capture modality. Select an exact target on each
action:
- `auto` (default): begins with effective scope `window`. Desktop perception
and actions are locked until the window ladder is exhausted, each attempted
action is verified, and the caller explicitly invokes `escalate_session`.
Escalation is one-way for the live session.
- `window`: strict window-only perception and actions. Desktop tools are always
rejected with `desktop_scope_disabled`.
- `desktop`: strict full-desktop perception and foreground/system actions.
Window-scoped perception and actions are rejected with
`window_scope_disabled`.
```bash
qwen-cua-driver start_session '{"session":"research-1","capture_scope":"auto"}'
qwen-cua-driver get_session_state '{"session":"research-1"}'
```jsonc
{"target":{"kind":"window","pid":844,"window_id":10725}}
{"target":{"kind":"desktop","display_id":"primary"}}
```
Do not use `config set capture_scope` or `set_config`; that key is retired and
stale values on disk are ignored. Always pass the public `session` field on
state and action calls. Reserved fields such as `_session_id` are transport
metadata and cannot create or change policy.
The window target uses window-local coordinates and the background/foreground
delivery ladder. The desktop target uses screen coordinates and foreground
delivery. A desktop action does not disable window tools for later calls.
During a mixed-version rollout, require `tools/list` to advertise
`session.capture_scope` (and `session.capture_scope.escalate` for `auto`). If an
older daemon does not advertise them, fail closed and ask for an upgrade; never
fall back to the retired global config key.
`start_session` is optional. For a multi-call run, prefer a short public
`session` label and pass the same label on every call that accepts it. The label
is call-scoped: if a later call omits it, that call uses the authenticated
transport's implicit session instead. Unnamed calls on one transport reuse that
implicit identity. The default idle TTL is five minutes. Call
`start_session(session)` to name or configure a run before acting, or to revive
an ended name.
Do not use `config set capture_scope` or `set_config`; that key is retired and
stale values on disk are ignored. `start_session.capture_scope`,
`get_session_state`, and `escalate_session` are deprecated compatibility
surfaces. There is no `deescalate_session`. Reserved fields such as
`_session_id` are transport metadata and cannot create authority.
## Keep authorization separate from sessions
The trusted host selects one permission profile at startup. `standard` keeps
the normal profile behavior and residual approval requirements, `bounded`
requires a reviewed capability manifest and has no runtime approval path, and
`unrestricted` bypasses Cua approval prompts after explicit risk acceptance.
Hard invariants plus managed and user policy remain binding in every profile.
An optional capability manifest is a deny-by-default ceiling in `standard` and
`unrestricted`; `bounded` requires one. It can remove tools or typed resources
from the selected profile, but it cannot grant a tool, resource, or approval
bypass that another authorization layer denies. Approval is considered only
after the tool and every adapter-attested resource are inside manifest scope.
Use the canonical startup pair together:
```bash
qwen-cua-driver mcp \
--permission-mode standard \
--capability-manifest ./capabilities.yaml \
--approve-capability-manifest
```
Capability manifest v3 omits file-level `mode` and `ask.tools`. Its
`allow.tools` list is nonempty. Lifetime fields are optional in `standard` and
`unrestricted`; `bounded` requires both `expires_after` and `idle_timeout`.
The older `--session-policy` names remain compatibility aliases and must not be
used in new configurations.
Starting, ending, naming, reconnecting, or omitting a session never changes
permission authority. A public session label is lifecycle metadata, never a
grant, caller identity, or bearer credential.
### Why window selection is the caller's job now
@ -483,6 +540,13 @@ success” above. The old `verified`, `path`, coordinates, scope, and
The full wire contract and 0.14 migration notes are in
`../../../docs/action-result-contract.md`.
A successful accessibility value write can still return
`effect:"unverifiable"` when the provider publishes its new value only after
the action call unwinds. Take a fresh snapshot before retrying; an immediate
retry can duplicate text. An explicit pixel escalation is reserved for a web
surface whose accessibility layer echoed the write without proving that the
renderer observed it.
`get_window_state` itself, when the AX tree comes back empty (a non-AX
surface like Electron/Chromium/canvas), returns `degraded: true`
plus an observation-specific escalation hint — normally pointing at pixels (you
@ -547,15 +611,12 @@ if resp.escalation.target == "foreground"
# unfocused window there; see LINUX.md
verify again
# Route 5 — desktop fallback (auto sessions only, explicit and one-way)
# Route 5 — per-call desktop fallback
# Reach this only after semantic, AX, window-pixel, and foreground-window
# delivery have all been exhausted and verified ineffective.
escalate_session(session,
reason="foreground_ineffective", # or another advertised reason
detail="bounded non-sensitive summary")
get_desktop_state(session) # full primary display
desktop_action(session, scope="desktop", ...) # no pid/window_id
get_desktop_state(session) # verify in the same coordinate frame
get_desktop_state() # full primary display
desktop_action(target={kind:"desktop", display_id:"primary"}, ...)
get_desktop_state() # verify in the same coordinate frame
```
The two ideas to hold onto: (1) the AX tree **lies** on canvas / web /
@ -586,44 +647,45 @@ last resort.
## The canonical loop
```
start_session(session, capture_scope="auto") # once per run; policy is immutable
launch_app(target)
# for multi-call work, repeat the same session label on every call that accepts it
launch_app(target, session)
→ pick window_id from the returned `windows` array
(or call list_windows(pid) separately)
→ get_window_state(pid, window_id)
→ [act] # every action also takes (pid, window_id) + your `session`
→ [act] # pass target={kind:"window", pid, window_id}
→ verify_state(pid, window_id, expect) # structured check; optional image
end_session(session) # when the run finishes
end_session(session?) # optional explicit cleanup
```
For strict desktop sessions, replace the window portion with
`get_desktop_state(session) → action(session, scope="desktop", ...) →
get_desktop_state(session)`. Desktop actions use screen-absolute coordinates
from that exact full-display image and omit `pid`/`window_id`. The global
`get_screen_size` and `get_cursor_position` helpers are desktop-scoped too.
For screen-absolute work, replace the window portion with
`get_desktop_state() → action(target={kind:"desktop",display_id:"primary"}, ...)
→ get_desktop_state()`. Desktop actions use coordinates from that exact
full-display image.
`launch_app` now returns a `windows` array alongside the pid, so the
common case collapses to two calls (`launch_app` → `get_window_state`)
without a separate `list_windows` hop.
**Declare a session.** A session is _your run's_ identity — a stable id
you choose (`"research-1"`), declared with `start_session` and passed as
`session` on every action. It owns your agent cursor and capture policy (a
distinct colour and one immutable policy per id), follows the run across any
apps/windows, and is the same whether
you drive over MCP, the CLI, or the socket. Declaring the session creates the
cursor; anonymous actions remain cursor-less.
End with `end_session` (or the idle-TTL reclaims it).
**Prefer a named session for multi-call work.** Choose a short label (for
example, `session: "research-1"`) and pass the same value on every call that
accepts it. Passing it once is not sticky: a later call that omits `session`
uses the transport's implicit session. Call `start_session(session)` when you
need to name or configure the run before acting, or to revive a name after
`end_session`. For one-off or deliberately unlabeled work, omission is valid
and the transport still gets one private lifecycle identity and visible agent
cursor. A public label makes inspection and cleanup easier, but it is not a
credential. End with `end_session` when useful; transport close or the
five-minute idle TTL also reclaims it.
**Concurrent runs/subagents:** each run may independently choose `auto`,
`window`, or `desktop`; one session's escalation never changes another. Also,
**Concurrent runs/subagents:** each transport gets its own implicit session.
Also,
`launch_app` is idempotent — two runs that
launch the same app get the **same** instance (and on single-instance apps
like Calculator, the same window), so they clobber each other. Give each run
its **own `session`** (→ its own cursor) AND pass
its **own connection** (for independent lifecycle/cursor ownership) AND pass
`creates_new_application_instance: true` to `launch_app` (→ its own window).
The element cache is keyed on `(pid, window_id)` and the cursor on `session`,
so distinct instances + distinct sessions keep the runs fully separated.
The element cache is keyed on `(pid, window_id)` and the cursor on the private
lifecycle session, so distinct instances and transports keep the runs isolated.
**Parallelism vs. ordering.** Distinct sessions give distinct _cursors_, not
distinct _connections_. Subagents that share one `qwen-cua-driver mcp` (stdio)

63
packages/cua-driver/rust/Skills/cua-driver/WINDOWS.md
View file

@ -412,15 +412,18 @@ When a qwen-cua-driver call surprises you, diagnose qwen-cua-driver first:
`capture_mode` param is **deprecated and ignored** — it's still accepted
so old callers don't error, but both the tree and the image come back
regardless of what you pass.
- **`get_desktop_state` returns `desktop_scope_disabled`?** That's
intended: full-display capture is a **desktop-scope** operation, gated
by the caller-declared session policy. To verify a specific window use
`get_window_state(pid, window_id)` (works backgrounded). Use
`get_desktop_state` only in a strict desktop session or after explicitly
escalating an `auto` session once the window ladder is exhausted. Desktop
actions pass `scope:"desktop"` with no pid/window_id. Don't reach for
`get_desktop_state` as a casual screenshot — it's the capture surface for
desktop-scope coordinate loops, not window inspection.
- **`list_windows` returns Win32 windows but misses UWP / WebView2
windows?** UIA desktop enumeration may be degraded because a provider
is unresponsive. `list_windows` falls back to Win32-only output instead
of hanging; run `cua-driver doctor` and retry after the provider
recovers.
- **Need full-display capture?** Use `get_desktop_state` only for a desktop
coordinate loop. To verify a specific window, use
`get_window_state(pid, window_id)` instead; it works while backgrounded.
Desktop actions use
`target:{"kind":"desktop","display_id":"primary"}`. Don't reach for
`get_desktop_state` as a casual screenshot—it is the desktop capture
surface, not window inspection.
- **`Calc display stuck at 0 after my clicks`?** Almost always
means UWP and you're on the PostMessage path. UWP processes
pointer input via `Windows.UI.Input`, NOT through HWND message
@ -482,8 +485,9 @@ your prior tool calls earned.
schtasks /Run /TN qwen-cua-driver-serve
```
3. **Run `qwen-cua-driver doctor`** — reports session ID, COM apartment
status, UIA reachability, install paths, version. If anything reads
`false` / `error`, fix that before tool-calling.
status, UIA desktop-enumeration reachability, install paths,
version. If anything reads `false` / `error`, fix that before
tool-calling.
4. **Permissions** — Windows has no TCC equivalent. cua-driver-rs
needs:
- No admin elevation for normal use (UIA, PostMessage, UWP
@ -781,21 +785,26 @@ typed browser tools yet.
- **`hotkey` / `press_key`** (keystroke + key-combo): `delivery_mode:"background"`
surfaces a `background_unavailable` error for VCL.
- **`type_text`** does a **UIA read-back** and returns the shared
`ActionResult`: `effect:"confirmed"` with `evidence:[{"kind":
"value_readback"}]` when the value reflects the text, and
`effect:"unverifiable"` when the value is unchanged or unreadable. Use the
optional `escalation` to choose the next rung. **Pass an
`element_index`** for reliable verification: the read-back then reads
_that specific element_ by handle (ValuePattern → TextPattern), which is
**focus-independent** — it can confirm or disprove a change whether or not
the target is foreground. (Verified live against the WPF harness: typed
via element_index, read back confirmed, value independently present in
the next snapshot — app never fronted.) **Without** an element_index it
falls back to system-wide `GetFocusedElement`, which on Windows only
resolves when the target is the **foreground** app (no per-app
`AXFocusedUIElement` like macOS); a backgrounded target then reads
an unverifiable result even when the text actually landed — so it is NOT a
failure signal; call `verify_state` or inspect a fresh screenshot.
`ActionResult`. With an `element_index`, the ValuePattern path returns
`effect:"confirmed"` with `evidence:[{"kind":"value_readback"}]` only
when the complete expected value is synchronously visible and differs from
the prior value. If SetValue succeeded but the provider still exposes the
old or no value, the result is `effect:"unverifiable"` with no escalation.
This is common for deferred providers such as AccessKit: take a fresh
snapshot before retrying because the value may publish only after
`type_text` returns and an immediate retry can duplicate text. A pixel
escalation is reserved for an Electron/web accessibility echo that does
not prove the renderer observed the write. The PostMessage fallback keeps
its separate delivery/read-back behavior. Even when the value changes and
contains the requested text, the result stays `effect:"unverifiable"` because
WM_CHAR does not expose the insertion point. Take a fresh snapshot before
retrying. It may recommend foreground when a background insert appears
dropped. Passing an `element_index` makes the
read-back target that exact element by handle (ValuePattern → TextPattern),
independent of foreground focus. Without one, PostMessage verification
falls back to system-wide `GetFocusedElement`, which normally resolves only
for the foreground app; an unreadable result is therefore not proof of
failure.
Escalate to `delivery_mode:"foreground"` for both (SendInput Unicode /
accelerator). **But** foreground needs the swap to actually land — if the
daemon lacks UIAccess and `bring_to_front` returns `landed_on_target:false`
@ -828,7 +837,7 @@ typed browser tools yet.
- Daemon version and install paths
- Current session ID (must be ≥1)
- COM apartment status (STA / MTA / uninitialized)
- UIA reachability (can we connect to `CUIAutomation`?)
- UIA reachability (can we create `CUIAutomation` and enumerate desktop children?)
- AppX broker reachability (for packaged-app activation)
- PATH state (is `qwen-cua-driver` actually on PATH?)
- Autostart Scheduled Task status

2
packages/cua-driver/rust/VERSION
View file

@ -1 +1 @@
0.17.0
0.20.0

19
packages/cua-driver/rust/crates/cua-driver-contract/src/compatibility.rs
View file

@ -43,6 +43,14 @@ pub fn schema_subset_violations(portable: &Value, live: &Value) -> Vec<String> {
}
fn compare_schema(path: &str, portable: &Value, live: &Value, violations: &mut Vec<String>) {
// Exact schema equality is itself a proof of subset compatibility, even
// when the shared schema contains a construct (for example a generated
// tagged-union `anyOf`) that this intentionally small implication engine
// does not otherwise interpret.
if portable == live {
return;
}
let Some(portable_object) = portable.as_object() else {
violations.push(format!("{path}: portable schema must be an object"));
return;
@ -370,6 +378,17 @@ mod tests {
assert!(violations[0].contains("unsupported portable schema keyword `pattern`"));
}
#[test]
fn identical_unknown_schema_is_a_proven_subset() {
let schema = json!({
"anyOf": [
{ "type": "string" },
{ "type": "null" }
]
});
assert!(schema_subset_violations(&schema, &schema).is_empty());
}
#[test]
fn additional_property_domain_must_be_accepted_live() {
let portable = json!({ "type": "object" });

76
packages/cua-driver/rust/crates/cua-driver-contract/src/desktop.rs
View file

@ -10,10 +10,11 @@
use crate::{
ActionResult, ClickInput, ClipboardReadInput, ClipboardReadOutput, ClipboardWriteInput,
ClipboardWriteOutput, CursorAction, CursorPositionOutput, CursorSemantics, DesktopStateOutput,
DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, HotkeyInput,
InvokeMenuInput, MoveCursorInput, Platform, PressKeyInput, SchemaMode, ScreenSizeOutput,
ScrollInput, SetWindowFrameInput, ToolAnnotations, ToolContract, ToolInput, ToolOutput,
TypeTextInput,
DragInput, GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput,
GetWindowStateInput, HotkeyInput, InvokeMenuInput, MoveCursorInput,
PerformSecondaryActionInput, Platform, PressKeyInput, SchemaMode, ScreenSizeOutput,
ScrollInput, SetValueInput, SetWindowFrameInput, ToolAnnotations, ToolContract, ToolInput,
ToolOutput, TypeTextInput, WindowStateOutput,
};
const ALL_PLATFORMS: [Platform; 3] = [Platform::Macos, Platform::Windows, Platform::Linux];
@ -21,6 +22,7 @@ const ALL_PLATFORMS: [Platform; 3] = [Platform::Macos, Platform::Windows, Platfo
pub fn contracts() -> Vec<ToolContract> {
vec![
get_desktop_state(),
get_window_state(),
get_screen_size(),
get_cursor_position(),
move_cursor(),
@ -29,6 +31,8 @@ pub fn contracts() -> Vec<ToolContract> {
click(),
drag(),
scroll(),
set_value(),
perform_secondary_action(),
clipboard_read(),
clipboard_write(),
type_text(),
@ -157,6 +161,35 @@ fn get_desktop_state() -> ToolContract {
)
}
// The window snapshot payload stays platform-owned (markdown + structured
// elements shapes are still converging), so this contract commits only the
// portable input projection and the versioned observation-revision envelope;
// the remaining output fields stay additive extensions — the same stance
// `list_windows` takes above.
fn get_window_state() -> ToolContract {
contract::<GetWindowStateInput, WindowStateOutput>(
"get_window_state",
"Walk one exact window's accessibility tree and return the actionable element snapshot, optionally as a versioned observation revision (`accessibility.observation_revision.v1`) diffed against a caller-supplied base.",
&[
"accessibility.window_state",
"accessibility.tree",
"accessibility.tree.structured",
"accessibility.tree.bounded",
"accessibility.element_tokens",
"accessibility.observation_revision.v1",
"screen.capture",
"screen.capture.window",
],
ToolAnnotations {
read_only: true,
destructive: false,
idempotent: false,
open_world: false,
},
CursorAction::Observe,
)
}
fn get_screen_size() -> ToolContract {
contract::<GetScreenSizeInput, ScreenSizeOutput>(
"get_screen_size",
@ -281,6 +314,39 @@ fn scroll() -> ToolContract {
)
}
fn set_value() -> ToolContract {
contract::<SetValueInput, ActionResult>(
"set_value",
"Set the value of the exact current accessibility element named by an opaque element token.",
&["accessibility.value.set", "accessibility.element_tokens"],
ToolAnnotations {
read_only: false,
destructive: true,
idempotent: true,
open_world: true,
},
CursorAction::Text,
)
}
fn perform_secondary_action() -> ToolContract {
contract::<PerformSecondaryActionInput, ActionResult>(
"perform_secondary_action",
"Perform one explicitly named action advertised by the exact current accessibility element. Missing, ambiguous, disabled, or stale actions fail closed without a pixel fallback.",
&[
"accessibility.action.secondary",
"accessibility.element_tokens",
],
ToolAnnotations {
read_only: false,
destructive: true,
idempotent: false,
open_world: true,
},
CursorAction::App,
)
}
fn type_text() -> ToolContract {
contract::<TypeTextInput, ActionResult>(
"type_text",
@ -319,7 +385,7 @@ fn hotkey() -> ToolContract {
contract::<HotkeyInput, ActionResult>(
"hotkey",
"Press a key chord in the foreground desktop application.",
&["input.keyboard.hotkey"],
&["input.keyboard.hotkey", "accessibility.element_tokens"],
ToolAnnotations {
read_only: false,
destructive: true,

574
packages/cua-driver/rust/crates/cua-driver-contract/src/inputs.rs
View file

@ -58,6 +58,14 @@ fn string_schema(generator: &mut SchemaGenerator) -> Schema {
String::json_schema(generator)
}
fn boolean_schema(generator: &mut SchemaGenerator) -> Schema {
bool::json_schema(generator)
}
pub const MULTI_CALL_SESSION_DESCRIPTION: &str =
"For multi-call work, prefer a short public session label and repeat it on every call that \
accepts it. Omit it to use the authenticated transport's implicit lifecycle session.";
fn string_list_schema(generator: &mut SchemaGenerator) -> Schema {
Vec::<String>::json_schema(generator)
}
@ -83,6 +91,23 @@ fn positive_integer_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "integer", "minimum": 1 })
}
fn observation_revision_schema(_: &mut SchemaGenerator) -> Schema {
// Mirrors the live platform subschema so the portable projection stays a
// provable subset of every runtime `get_window_state` schema.
json_schema!({
"type": "object",
"required": ["version", "serializer_version", "projection_version"],
"properties": {
"version": { "type": "integer", "const": 1 },
"serializer_version": { "type": "string", "minLength": 1, "maxLength": 128 },
"projection_version": { "type": "string", "minLength": 1, "maxLength": 128 },
"base_revision_id": { "type": "string", "minLength": 1, "maxLength": 256 },
"force_full": { "type": "boolean", "default": false }
},
"additionalProperties": false
})
}
fn click_button_schema(generator: &mut SchemaGenerator) -> Schema {
ClickButton::json_schema(generator)
}
@ -103,6 +128,10 @@ fn drag_steps_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "integer", "minimum": 1, "maximum": 200 })
}
fn delivery_mode_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "string", "enum": ["background", "foreground"] })
}
fn scroll_amount_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "integer", "minimum": 1, "maximum": 50 })
}
@ -111,6 +140,10 @@ fn cursor_theme_id_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "string", "minLength": 1, "maxLength": 200 })
}
fn element_token_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "string", "minLength": 1, "maxLength": 512 })
}
fn escalation_detail_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({ "type": "string", "maxLength": 200 })
}
@ -118,8 +151,7 @@ fn escalation_detail_schema(_: &mut SchemaGenerator) -> Schema {
fn capture_scope_schema(_: &mut SchemaGenerator) -> Schema {
json_schema!({
"type": "string",
"enum": ["auto", "window", "desktop"],
"default": "auto"
"enum": ["auto", "window", "desktop"]
})
}
@ -198,6 +230,42 @@ pub enum DesktopScope {
Desktop,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
#[serde(rename_all = "snake_case")]
pub enum DeliveryMode {
Background,
Foreground,
}
fn desktop_scope_schema(generator: &mut SchemaGenerator) -> Schema {
DesktopScope::json_schema(generator)
}
/// Exact capture/input target selected independently for each action.
///
/// `display_id="primary"` is the portable desktop target in this release.
/// Platforms that cannot address another display reject it explicitly rather
/// than silently changing coordinate spaces.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum ActionTarget {
Window { pid: u32, window_id: u64 },
Desktop { display_id: String },
}
pub fn action_target_schema() -> Value {
let settings = schemars::generate::SchemaSettings::draft2020_12().with(|settings| {
settings.meta_schema = None;
settings.inline_subschemas = true;
});
let schema = settings
.into_generator()
.into_root_schema_for::<ActionTarget>();
let mut value = serde_json::to_value(schema).expect("action target schema serializes");
normalize_schema(&mut value);
value
}
impl JsonSchema for DesktopScope {
fn schema_name() -> std::borrow::Cow<'static, str> {
"DesktopScope".into()
@ -264,9 +332,14 @@ impl ScrollBy {
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
pub struct StartSessionInput {
/// Stable session id for this run (e.g. "research-run-1").
pub session: String,
/// Per-session perception/action modality. auto starts window-only and requires explicit escalation before desktop tools; window and desktop are strict. Immutable for the live session.
/// Optional stable public label for this run (e.g. "research-run-1").
/// When omitted, the authenticated transport lease's implicit session is
/// created or returned.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
/// Deprecated compatibility policy. New callers select window or desktop
/// modality on each action instead of storing it on the session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "capture_scope_schema")]
pub capture_scope: Option<CaptureScope>,
@ -296,17 +369,53 @@ impl ToolInput for EscalateSessionInput {
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
pub struct GetSessionStateInput {
pub session: String,
/// Optional public label. When omitted, inspect the caller's attached
/// implicit session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
}
impl ToolInput for GetSessionStateInput {
const TOOL_NAME: &'static str = "get_session_state";
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
pub struct GetSessionInput {
/// Optional public label. When omitted, inspect the caller's attached
/// implicit session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
}
impl ToolInput for GetSessionInput {
const TOOL_NAME: &'static str = "get_session";
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
pub struct ListSessionsInput {
/// Maximum number of content-free summaries to return (default 50, max
/// 100). Ordinary agent transports are scoped to their own lease.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub limit: Option<u32>,
/// Opaque continuation cursor returned by a previous call.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub cursor: Option<String>,
}
impl ToolInput for ListSessionsInput {
const TOOL_NAME: &'static str = "list_sessions";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
pub struct EndSessionInput {
/// The session id to end.
pub session: String,
/// Optional public label to end. When omitted, end the caller's attached
/// implicit session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
@ -370,7 +479,8 @@ impl ToolInput for EndSessionInput {
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct GetDesktopStateInput {
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -384,10 +494,103 @@ impl ToolInput for GetDesktopStateInput {
const TOOL_NAME: &'static str = "get_desktop_state";
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct ListAppsInput {}
impl ToolInput for ListAppsInput {
const TOOL_NAME: &'static str = "list_apps";
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct ListWindowsInput {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub pid: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub on_screen_only: Option<bool>,
}
impl ToolInput for ListWindowsInput {
const TOOL_NAME: &'static str = "list_windows";
}
/// Opt in to the versioned `accessibility.observation_revision.v1` protocol on
/// `get_window_state`. Omitting the whole record preserves the legacy
/// full-snapshot contract byte for byte.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct ObservationRevisionInput {
/// Protocol version. Only `1` is defined; any other value is rejected
/// with a closed `invalid_observation_revision` error.
pub version: u32,
/// Canonical accessibility serializer expected by the caller. A mismatch
/// returns a full response with `serializer_changed`.
pub serializer_version: String,
/// Canonical tree projection expected by the caller. A mismatch returns a
/// full response with `projection_changed`.
pub projection_version: String,
/// Revision the caller wants to diff against. The caller — never the
/// driver — decides which revision was actually delivered downstream.
/// Missing, expired, foreign, or incompatible bases yield a full response.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub base_revision_id: Option<String>,
/// Force a full resynchronization even when a compatible base exists.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub force_full: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct GetWindowStateInput {
/// Target process ID.
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
/// Exact window to observe.
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: u64,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
/// Case-insensitive projection filter. Incompatible with `observation_revision`.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub query: Option<String>,
/// Default true. Set false to skip the screenshot and return the tree only.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "boolean_schema")]
pub include_screenshot: Option<bool>,
/// Write the PNG here instead of returning base64.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub screenshot_out_file: Option<String>,
/// Cap on the total number of accessibility nodes walked.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub max_elements: Option<u32>,
/// Cap on the accessibility-tree walk depth.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub max_depth: Option<u32>,
/// Opt in to `accessibility.observation_revision.v1`. Requires a bound
/// driver session. Omit to preserve the legacy full-snapshot contract.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "observation_revision_schema")]
pub observation_revision: Option<ObservationRevisionInput>,
}
impl ToolInput for GetWindowStateInput {
const TOOL_NAME: &'static str = "get_window_state";
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct GetScreenSizeInput {
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -400,7 +603,8 @@ impl ToolInput for GetScreenSizeInput {
#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct GetCursorPositionInput {
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -417,8 +621,15 @@ pub struct MoveCursorInput {
pub x: f64,
#[schemars(schema_with = "number_schema")]
pub y: f64,
pub scope: DesktopScope,
/// Optional session id.
/// Preferred per-call target. New callers should set this field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -439,7 +650,8 @@ pub struct SetWindowFrameInput {
pub width: f64,
#[schemars(schema_with = "positive_number_schema")]
pub height: f64,
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -461,7 +673,8 @@ pub struct InvokeMenuInput {
pub window_id: u64,
#[schemars(schema_with = "menu_path_schema")]
pub path: Vec<String>,
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -482,8 +695,14 @@ pub struct ClickInput {
pub x: f64,
#[schemars(schema_with = "number_schema")]
pub y: f64,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -499,6 +718,86 @@ impl ToolInput for ClickInput {
const TOOL_NAME: &'static str = "click";
}
/// Exact-window click input for the generated SDK. The existing [`ClickInput`]
/// remains the portable desktop-coordinate form.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowClickInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub y: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "click_button_schema")]
pub button: Option<ClickButton>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "click_count_schema")]
pub count: Option<u32>,
}
impl ToolInput for WindowClickInput {
const TOOL_NAME: &'static str = "click";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct DoubleClickInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub y: Option<f64>,
}
impl ToolInput for DoubleClickInput {
const TOOL_NAME: &'static str = "double_click";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct RightClickInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub y: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_list_schema")]
pub modifier: Option<Vec<String>>,
}
impl ToolInput for RightClickInput {
const TOOL_NAME: &'static str = "right_click";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct DragInput {
@ -510,8 +809,14 @@ pub struct DragInput {
pub to_x: f64,
#[schemars(schema_with = "number_schema")]
pub to_y: f64,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -533,6 +838,44 @@ impl ToolInput for DragInput {
const TOOL_NAME: &'static str = "drag";
}
/// Exact-window drag input for the generated SDK. The existing [`DragInput`]
/// keeps its established UniFFI record layout.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowDragInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: u64,
#[schemars(schema_with = "number_schema")]
pub from_x: f64,
#[schemars(schema_with = "number_schema")]
pub from_y: f64,
#[schemars(schema_with = "number_schema")]
pub to_x: f64,
#[schemars(schema_with = "number_schema")]
pub to_y: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "drag_duration_schema")]
pub duration_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "drag_steps_schema")]
pub steps: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "delivery_mode_schema")]
pub delivery_mode: Option<DeliveryMode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "click_button_schema")]
pub button: Option<ClickButton>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_list_schema")]
pub modifier: Option<Vec<String>>,
}
impl ToolInput for WindowDragInput {
const TOOL_NAME: &'static str = "drag";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct ScrollInput {
@ -541,8 +884,14 @@ pub struct ScrollInput {
#[schemars(schema_with = "number_schema")]
pub y: f64,
pub direction: ScrollDirection,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -558,12 +907,48 @@ impl ToolInput for ScrollInput {
const TOOL_NAME: &'static str = "scroll";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowScrollInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "number_schema")]
pub y: Option<f64>,
pub direction: ScrollDirection,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "scroll_by_schema")]
pub by: Option<ScrollBy>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "scroll_amount_schema")]
pub amount: Option<u64>,
}
impl ToolInput for WindowScrollInput {
const TOOL_NAME: &'static str = "scroll";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct TypeTextInput {
pub text: String,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -573,6 +958,43 @@ impl ToolInput for TypeTextInput {
const TOOL_NAME: &'static str = "type_text";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowTypeTextInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
pub text: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub delay_ms: Option<u64>,
}
impl ToolInput for WindowTypeTextInput {
const TOOL_NAME: &'static str = "type_text";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct SetValueInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[schemars(schema_with = "element_token_schema")]
pub element_token: String,
pub value: String,
}
impl ToolInput for SetValueInput {
const TOOL_NAME: &'static str = "set_value";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct ClipboardReadInput {
@ -580,7 +1002,8 @@ pub struct ClipboardReadInput {
/// Clipboard content is privacy-sensitive and is never retained in telemetry.
#[serde(default)]
pub include_text: bool,
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -605,7 +1028,8 @@ pub struct ClipboardWriteInput {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub file_path: Option<String>,
/// Optional session id.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -619,8 +1043,14 @@ impl ToolInput for ClipboardWriteInput {
#[serde(deny_unknown_fields)]
pub struct PressKeyInput {
pub key: String,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -633,13 +1063,40 @@ impl ToolInput for PressKeyInput {
const TOOL_NAME: &'static str = "press_key";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowPressKeyInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
pub key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_list_schema")]
pub modifiers: Option<Vec<String>>,
}
impl ToolInput for WindowPressKeyInput {
const TOOL_NAME: &'static str = "press_key";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct HotkeyInput {
#[schemars(length(min = 2))]
pub keys: Vec<String>,
pub scope: DesktopScope,
/// Optional session id.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target: Option<ActionTarget>,
/// Deprecated flat desktop target retained for wire compatibility.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "desktop_scope_schema")]
pub scope: Option<DesktopScope>,
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,
@ -649,6 +1106,42 @@ impl ToolInput for HotkeyInput {
const TOOL_NAME: &'static str = "hotkey";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct WindowHotkeyInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "element_token_schema")]
pub element_token: Option<String>,
#[schemars(length(min = 2))]
pub keys: Vec<String>,
}
impl ToolInput for WindowHotkeyInput {
const TOOL_NAME: &'static str = "hotkey";
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, uniffi::Record)]
#[serde(deny_unknown_fields)]
pub struct PerformSecondaryActionInput {
#[schemars(schema_with = "positive_integer_schema")]
pub pid: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "positive_integer_schema")]
pub window_id: Option<u64>,
#[schemars(schema_with = "element_token_schema")]
pub element_token: String,
pub action: String,
}
impl ToolInput for PerformSecondaryActionInput {
const TOOL_NAME: &'static str = "perform_secondary_action";
}
#[cfg(test)]
mod tests {
use serde_json::json;
@ -660,8 +1153,9 @@ mod tests {
let schema = ClickInput::input_schema();
assert_eq!(schema["type"], "object");
assert_eq!(schema["additionalProperties"], false);
assert_eq!(schema["required"], json!(["x", "y", "scope"]));
assert_eq!(schema["properties"]["scope"], json!({ "const": "desktop" }));
assert_eq!(schema["required"], json!(["x", "y"]));
assert_eq!(schema["properties"]["scope"]["const"], "desktop");
assert!(schema["properties"]["target"]["anyOf"].is_array());
assert_eq!(
schema["properties"]["button"],
json!({ "type": "string", "enum": ["left", "right", "middle"] })
@ -673,11 +1167,22 @@ mod tests {
}
#[test]
fn generated_session_schema_preserves_default_and_open_input() {
fn generated_drag_schema_exposes_the_runtime_delivery_ladder() {
let schema = WindowDragInput::input_schema();
assert_eq!(
schema["properties"]["delivery_mode"],
json!({ "type": "string", "enum": ["background", "foreground"] })
);
}
#[test]
fn generated_session_schema_makes_name_and_legacy_capture_optional() {
let schema = StartSessionInput::input_schema();
assert_eq!(schema["additionalProperties"], true);
assert_eq!(schema["required"], json!(["session"]));
assert_eq!(schema["properties"]["capture_scope"]["default"], "auto");
assert_eq!(schema["required"], json!([]));
assert!(schema["properties"]["capture_scope"]
.get("default")
.is_none());
assert_eq!(
schema["properties"]["capture_scope"]["enum"],
json!(["auto", "window", "desktop"])
@ -689,7 +1194,6 @@ mod tests {
let error = serde_json::from_value::<ClickInput>(json!({
"x": 1,
"y": 2,
"scope": "desktop",
"pid": 3
}))
.expect_err("portable input must reject runtime-only fields");

127
packages/cua-driver/rust/crates/cua-driver-contract/src/lib.rs
View file

@ -27,21 +27,29 @@ pub use cursor::{
CursorSemantics, CursorTarget, CursorThemeSelection,
};
pub use inputs::{
CaptureScope, ClickButton, ClickInput, ClipboardReadInput, ClipboardWriteInput, DesktopScope,
DragInput, EndSessionInput, EscalateSessionInput, EscalationReason, GetAgentCursorStateInput,
GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, GetSessionStateInput,
HotkeyInput, InvokeMenuInput, MoveCursorInput, PressKeyInput, ScrollBy, ScrollDirection,
ScrollInput, SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput,
SetWindowFrameInput, StartSessionInput, ToolInput, TypeTextInput,
action_target_schema, ActionTarget, CaptureScope, ClickButton, ClickInput, ClipboardReadInput,
ClipboardWriteInput, DeliveryMode, DesktopScope, DoubleClickInput, DragInput, EndSessionInput,
EscalateSessionInput, EscalationReason, GetAgentCursorStateInput, GetCursorPositionInput,
GetDesktopStateInput, GetScreenSizeInput, GetSessionInput, GetSessionStateInput,
GetWindowStateInput, HotkeyInput, InvokeMenuInput, ListAppsInput, ListSessionsInput,
ListWindowsInput, MoveCursorInput, ObservationRevisionInput, PerformSecondaryActionInput,
PressKeyInput, RightClickInput, ScrollBy, ScrollDirection, ScrollInput,
SetAgentCursorEnabledInput, SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetValueInput,
SetWindowFrameInput, StartSessionInput, ToolInput, TypeTextInput, WindowClickInput,
WindowDragInput, WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput,
WindowTypeTextInput, MULTI_CALL_SESSION_DESCRIPTION,
};
pub use outputs::{
ActionDelivery, ActionDeliveryMode, ActionEffect, ActionEscalation, ActionEscalationReason,
ActionEscalationTarget, ActionEvidence, ActionEvidenceKind, ActionResult,
ActionResultValidationError, ActionRoute, ClipboardReadOutput, ClipboardWriteOutput,
CursorMotionOutput, CursorPointOutput, CursorPositionOutput, CursorThemeOutput,
CursorVisualOutput, DesktopStateOutput, EffectiveScope, EndSessionOutput,
GetAgentCursorStateOutput, ScreenSizeOutput, SessionStateOutput, SetAgentCursorEnabledOutput,
advertised_output_schema, refusal_envelope_schema, ActionDelivery, ActionDeliveryMode,
ActionEffect, ActionEscalation, ActionEscalationReason, ActionEscalationTarget, ActionEvidence,
ActionEvidenceKind, ActionResult, ActionResultValidationError, ActionRoute,
ClipboardReadOutput, ClipboardWriteOutput, CursorMotionOutput, CursorPointOutput,
CursorPositionOutput, CursorThemeOutput, CursorVisualOutput, DesktopStateOutput,
EffectiveScope, EndSessionOutput, GetAgentCursorStateOutput, ListSessionsOutput,
ObservationRevisionOutput, ScreenSizeOutput, SessionClientKindOutput, SessionLifecycleState,
SessionOutput, SessionStateOutput, SessionTransportOutput, SetAgentCursorEnabledOutput,
SetAgentCursorMotionOutput, SetAgentCursorThemeOutput, StartSessionOutput, ToolOutput,
WindowStateOutput,
};
pub use verification::{
BoundsExpectation, ElementPredicate, ElementSelector, PredicateOutcome, StatePredicate,
@ -56,7 +64,7 @@ pub const TOOLS_LIST_SCHEMA_VERSION: &str = "1";
pub const CAPABILITY_VERSION: &str = "1";
/// Shape version for the checked-in generated client contract.
pub const CONTRACT_VERSION: &str = "0.6.0";
pub const CONTRACT_VERSION: &str = "0.7.0";
/// MCP protocol version used by current cua-driver clients.
pub const MCP_PROTOCOL_VERSION: &str = "2025-06-18";
@ -81,6 +89,7 @@ pub const ACTION_RESULT_TOOLS: &[&str] = &[
"press_key",
"hotkey",
"set_value",
"perform_secondary_action",
"set_window_frame",
"invoke_menu",
"browser_click",
@ -282,6 +291,57 @@ pub fn validate_success_output(name: &str, value: Value) -> Result<bool, String>
mod tests {
use super::*;
#[test]
fn get_window_state_input_serializes_to_the_live_wire_shape() {
let minimal = serde_json::to_value(GetWindowStateInput {
pid: 42,
window_id: 7,
session: None,
query: None,
include_screenshot: None,
screenshot_out_file: None,
max_elements: None,
max_depth: None,
observation_revision: None,
})
.unwrap();
assert_eq!(minimal, serde_json::json!({ "pid": 42, "window_id": 7 }));
let revision = serde_json::to_value(GetWindowStateInput {
pid: 42,
window_id: 7,
session: Some("s1".into()),
query: None,
include_screenshot: Some(false),
screenshot_out_file: None,
max_elements: None,
max_depth: None,
observation_revision: Some(ObservationRevisionInput {
version: 1,
serializer_version: "accessibility-render-v1".into(),
projection_version: "full-tree-v1".into(),
base_revision_id: Some("r_1".into()),
force_full: None,
}),
})
.unwrap();
assert_eq!(
revision,
serde_json::json!({
"pid": 42,
"window_id": 7,
"session": "s1",
"include_screenshot": false,
"observation_revision": {
"version": 1,
"serializer_version": "accessibility-render-v1",
"projection_version": "full-tree-v1",
"base_revision_id": "r_1"
}
})
);
}
#[test]
fn manifest_is_sorted_and_versioned() {
let manifest = manifest();
@ -293,7 +353,7 @@ mod tests {
let mut sorted = names.clone();
sorted.sort_unstable();
assert_eq!(names, sorted);
assert_eq!(manifest.contract_version, "0.6.0");
assert_eq!(manifest.contract_version, "0.7.0");
assert!(manifest.experimental);
}
@ -342,6 +402,45 @@ mod tests {
);
}
#[test]
fn portable_action_sessions_explain_named_multi_call_runs() {
for name in [
"click",
"clipboard_read",
"clipboard_write",
"drag",
"get_cursor_position",
"get_desktop_state",
"get_screen_size",
"hotkey",
"invoke_menu",
"move_cursor",
"press_key",
"scroll",
"set_window_frame",
"type_text",
] {
let contract = tool_contract(name).expect("portable action contract");
let description = contract.input_schema["properties"]["session"]["description"]
.as_str()
.expect("portable session description")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
assert_eq!(description, MULTI_CALL_SESSION_DESCRIPTION, "{name}");
}
let verify = tool_contract("verify_state").expect("verify_state contract");
let description = verify.input_schema["properties"]["session"]["description"]
.as_str()
.expect("verify_state session description")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
assert!(description.starts_with(MULTI_CALL_SESSION_DESCRIPTION));
assert!(description.contains("never selects capture modality or authorization"));
}
#[test]
fn every_contract_has_success_schema_and_platforms() {
for tool in manifest().tools {

193
packages/cua-driver/rust/crates/cua-driver-contract/src/outputs.rs
View file

@ -18,6 +18,43 @@ pub trait ToolOutput: Serialize + DeserializeOwned + JsonSchema {
}
}
/// Schema for the refusal payload a tool emits alongside `isError: true`.
///
/// Refusals answer with a diagnostic shape rather than the success shape, and
/// two are in service: `{"status":"refused","refusal":{code,message,detail}}`
/// on the element-token and daemon paths, and `{"code":…,"effect":"refused",…}`
/// on the window-target paths. Both carry tool-specific diagnostic keys, so the
/// envelope stays open — the marker keys are what make it recognisably a
/// refusal rather than a malformed success.
pub fn refusal_envelope_schema() -> Value {
serde_json::json!({
"type": "object",
"additionalProperties": true,
"anyOf": [
{"required": ["refusal"]},
{"required": ["status"]},
{"required": ["code"]},
],
})
}
/// Wrap a success schema into the shape advertised as the MCP `outputSchema`.
///
/// MCP requires every `structuredContent` a tool emits to validate against its
/// advertised `outputSchema` — including payloads that accompany
/// `isError: true`. Advertising the success shape alone made strict clients
/// reject refusals outright, replacing the actionable message the driver had
/// already placed in `content` ("element_token is stale; call get_window_state
/// again to refresh") with an opaque schema-validation error. Agents then had
/// no signal to re-snapshot and fell back to blind pixel clicking.
///
/// The success variant is kept exactly as generated — still closed, so success
/// payloads stay strictly checked — and the refusal envelope joins it as a
/// sibling variant.
pub fn advertised_output_schema(success: Value) -> Value {
serde_json::json!({ "type": "object", "anyOf": [success, refusal_envelope_schema()] })
}
fn output_schema_with_additional_properties<T: JsonSchema>(additional_properties: bool) -> Value {
let mut settings = SchemaSettings::draft2020_12();
settings.inline_subschemas = true;
@ -82,6 +119,59 @@ pub struct SessionStateOutput {
impl ToolOutput for SessionStateOutput {}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
#[serde(rename_all = "snake_case")]
pub enum SessionLifecycleState {
Active,
Ending,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
#[serde(rename_all = "snake_case")]
pub enum SessionClientKindOutput {
Cli,
Direct,
Mcp,
PythonSdk,
TypescriptSdk,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Enum)]
#[serde(rename_all = "snake_case")]
pub enum SessionTransportOutput {
Cli,
Daemon,
McpStdio,
McpHttp,
}
/// Content-free lifecycle state safe for an ordinary agent transport.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)]
pub struct SessionOutput {
/// Sanitized public label, or null for an unnamed implicit session.
#[schemars(required, schema_with = "nullable_string_schema")]
pub session: Option<String>,
pub implicit: bool,
pub state: SessionLifecycleState,
pub client_kind: SessionClientKindOutput,
pub transport: SessionTransportOutput,
pub cursor_visible: bool,
pub recording_active: bool,
pub idle_seconds: u64,
pub expires_in_seconds: u64,
}
impl ToolOutput for SessionOutput {}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)]
pub struct ListSessionsOutput {
pub sessions: Vec<SessionOutput>,
#[schemars(required, schema_with = "nullable_string_schema")]
pub next_cursor: Option<String>,
}
impl ToolOutput for ListSessionsOutput {}
/// Successful structured result returned by `start_session`.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq, uniffi::Record)]
pub struct StartSessionOutput {
@ -229,6 +319,109 @@ fn png_mime_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({ "const": "image/png" })
}
/// Versioned `accessibility.observation_revision.v1` envelope attached to a
/// `get_window_state` success when the caller opted in. Platform runtimes add
/// diagnostic fields beyond this committed core; they flow into `extensions`.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
pub struct ObservationRevisionOutput {
#[schemars(schema_with = "observation_revision_capability_schema")]
pub capability: String,
#[schemars(schema_with = "integer_schema")]
pub version: u64,
#[schemars(schema_with = "string_schema_required")]
pub serializer_version: String,
#[schemars(schema_with = "string_schema_required")]
pub projection_version: String,
#[schemars(schema_with = "observation_revision_mode_schema")]
pub mode: String,
#[schemars(schema_with = "string_schema_required")]
pub lineage_id: String,
#[schemars(schema_with = "string_schema_required")]
pub revision_id: String,
/// Base the driver actually diffed against. Absent on full responses
/// without a usable base.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub base_revision_id: Option<String>,
/// Whether emitted element tokens are stable revision tokens that survive
/// compatible diffs. False means the response is a non-retained full.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stable_element_ids: Option<bool>,
/// Closed reason naming why a full resynchronization was required.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub resync_reason: Option<String>,
#[serde(flatten)]
pub extensions: BTreeMap<String, Value>,
}
impl ObservationRevisionOutput {
fn validate(&self) -> Result<(), String> {
if self.capability != "accessibility.observation_revision.v1" {
return Err(
"observation_revision.capability must be accessibility.observation_revision.v1"
.into(),
);
}
if self.version != 1 {
return Err("observation_revision.version must be 1".into());
}
if self.serializer_version.is_empty() || self.projection_version.is_empty() {
return Err(
"observation_revision serializer_version and projection_version must be non-empty"
.into(),
);
}
if !matches!(self.mode.as_str(), "full" | "diff" | "no_change") {
return Err("observation_revision.mode must be full, diff, or no_change".into());
}
if self.lineage_id.is_empty() || self.revision_id.is_empty() {
return Err("observation_revision lineage_id and revision_id must be non-empty".into());
}
if matches!(self.mode.as_str(), "diff" | "no_change") && self.base_revision_id.is_none() {
return Err(
"observation_revision diff/no_change responses must name their base_revision_id"
.into(),
);
}
Ok(())
}
}
fn observation_revision_capability_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({ "const": "accessibility.observation_revision.v1" })
}
fn observation_revision_mode_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({ "type": "string", "enum": ["full", "diff", "no_change"] })
}
fn string_schema_required(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({ "type": "string", "minLength": 1 })
}
/// Portable projection of a `get_window_state` success. The snapshot payload
/// (markdown rendering, structured elements, screenshot fields) remains
/// platform-owned and still converging, so only the versioned observation
/// revision envelope is committed here; everything else flows through
/// `extensions` unvalidated.
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
pub struct WindowStateOutput {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub observation_revision: Option<ObservationRevisionOutput>,
#[serde(flatten)]
pub extensions: BTreeMap<String, Value>,
}
impl ToolOutput for WindowStateOutput {
fn validate(&self) -> Result<(), String> {
match &self.observation_revision {
Some(revision) => revision.validate(),
None => Ok(()),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)]
pub struct ScreenSizeOutput {
#[schemars(schema_with = "number_schema")]

58
packages/cua-driver/rust/crates/cua-driver-contract/src/session.rs
View file

@ -3,14 +3,15 @@
use crate::{
CursorAction, CursorSemantics, EndSessionInput, EndSessionOutput, EscalateSessionInput,
GetSessionStateInput, Platform, SchemaMode, SessionStateOutput, StartSessionInput,
StartSessionOutput, ToolAnnotations, ToolContract, ToolInput, ToolOutput,
GetSessionInput, GetSessionStateInput, ListSessionsInput, ListSessionsOutput, Platform,
SchemaMode, SessionOutput, SessionStateOutput, StartSessionInput, StartSessionOutput,
ToolAnnotations, ToolContract, ToolInput, ToolOutput,
};
const ALL_PLATFORMS: [Platform; 3] = [Platform::Macos, Platform::Windows, Platform::Linux];
pub fn contracts() -> Vec<ToolContract> {
vec![start(), escalate(), get_state(), end()]
vec![start(), escalate(), get(), list(), get_state(), end()]
}
fn contract<I: ToolInput, O: ToolOutput>(
@ -38,7 +39,7 @@ fn contract<I: ToolInput, O: ToolOutput>(
fn start() -> ToolContract {
contract::<StartSessionInput, StartSessionOutput>(
"start_session",
"Declare a session — a named, color-coded identity for THIS agent run. Pass a stable `session` id and choose capture_scope=auto|window|desktop; the agent cursor, capture policy, per-session config, and recording all key on it, and it follows the run across any apps/windows. The cursor's color is derived from the id, so distinct runs are visually distinct. A cursor is shown only for a declared session — call this (or pass `session` on your first action) to opt in. Idempotent: re-calling with the same id just refreshes its idle-TTL. End it with `end_session` (or let the idle-TTL reclaim it). Concurrent runs/subagents each pass their own `session` to get their own cursor.",
"Optionally create or return a lifecycle session before acting. For multi-call work, prefer a short public `session` label and repeat it on every call that accepts it; an omitted value uses the authenticated transport lease's implicit session instead. This tool is optional because an ordinary action can create or reuse a named run directly. Use it to set the initial cursor theme before acting or to revive a public name after it has ended; ordinary actions never revive ended names. `capture_scope` is deprecated compatibility input; new callers select window or desktop modality per action. Idempotent.",
&["session.lifecycle.start", "session.capture_scope"],
ToolAnnotations {
read_only: false,
@ -52,7 +53,7 @@ fn start() -> ToolContract {
fn escalate() -> ToolContract {
contract::<EscalateSessionInput, SessionStateOutput>(
"escalate_session",
"Unlock the desktop phase of an auto capture-scope session after the window action ladder has been exhausted and verified. This is a one-way transition for the live session and records a bounded reason.",
"Deprecated compatibility tool for legacy capture-scope sessions. New callers select window or desktop modality on each action. No deescalate_session tool exists.",
&["session.capture_scope.escalate"],
ToolAnnotations {
read_only: false,
@ -66,7 +67,7 @@ fn escalate() -> ToolContract {
fn get_state() -> ToolContract {
contract::<GetSessionStateInput, SessionStateOutput>(
"get_session_state",
"Read the live session's capture policy and effective scope.",
"Deprecated compatibility alias that reads a live legacy session's capture policy. Use get_session for lifecycle state.",
&["session.capture_scope.read"],
ToolAnnotations {
read_only: true,
@ -77,10 +78,38 @@ fn get_state() -> ToolContract {
)
}
fn get() -> ToolContract {
contract::<GetSessionInput, SessionOutput>(
"get_session",
"Read content-free lifecycle, cursor, recording, and idle status for one session visible to this authenticated transport. Omit `session` to inspect its implicit session.",
&["session.lifecycle.read"],
ToolAnnotations {
read_only: true,
destructive: false,
idempotent: true,
open_world: false,
},
)
}
fn list() -> ToolContract {
contract::<ListSessionsInput, ListSessionsOutput>(
"list_sessions",
"List content-free lifecycle summaries attached to this authenticated transport lease. It does not enumerate other callers' sessions.",
&["session.lifecycle.list"],
ToolAnnotations {
read_only: true,
destructive: false,
idempotent: true,
open_world: false,
},
)
}
fn end() -> ToolContract {
contract::<EndSessionInput, EndSessionOutput>(
"end_session",
"End a session declared with `start_session`: removes its agent cursor, stops any recording it owns, and clears its per-session config. Call this when a run finishes so its cursor doesn't linger (otherwise the idle-TTL reclaims it after a period of inactivity). Idempotent.",
"End one visible lifecycle session and run its cursor, recording, configuration, and other cleanup hooks exactly once. Omit `session` to end the authenticated transport's implicit session. Idempotent.",
&["session.lifecycle.end"],
ToolAnnotations {
read_only: false,
@ -90,3 +119,18 @@ fn end() -> ToolContract {
},
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn start_description_explains_direct_naming_and_explicit_revival() {
let description = start().description;
assert!(description.contains("prefer a short public `session` label"));
assert!(description.contains("repeat it on every call that accepts it"));
assert!(description.contains("omitted value uses the authenticated transport"));
assert!(description.contains("revive a public name after it has ended"));
assert!(description.contains("ordinary actions never revive ended names"));
}
}

4
packages/cua-driver/rust/crates/cua-driver-contract/src/verification.rs
View file

@ -156,7 +156,9 @@ pub struct VerifyStateInput {
/// One to eight predicates, combined with logical AND.
#[schemars(length(min = 1, max = 8))]
pub expect: Vec<StatePredicate>,
/// Optional session id for capture-scope and authorization continuity.
/// For multi-call work, prefer a short public session label and repeat it on every call that
/// accepts it. Omit it to use the authenticated transport's implicit lifecycle session. This
/// field never selects capture modality or authorization.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "string_schema")]
pub session: Option<String>,

11
packages/cua-driver/rust/crates/cua-driver-core/Cargo.toml
View file

@ -25,6 +25,10 @@ regorus = { version = "0.10.1", optional = true }
# helpers extracted from `platform-{macos,windows,linux}/src/capture.rs` so
# all three platforms call the same code path.
image = { workspace = true }
# Compatibility constraint for image's zune-jpeg decoder. This dependency is
# intentionally direct so SDK consumers without this workspace lockfile cannot
# resolve the incompatible zune-core 0.5.2 release.
zune-core = { workspace = true }
# Used by the `browser` module's pooled CDP browser-endpoint WebSocket.
# Same versions platform-macos already depends on, to avoid duplicate
# crate versions in the workspace graph. No TLS features: the endpoint
@ -44,6 +48,13 @@ libc = "0.2"
# a duplicate crate version.
[dev-dependencies]
tempfile = "3"
# Test-only. Asserts the advertised `outputSchema` the way a strict MCP client
# does — by validating real payloads against it. The runtime itself validates
# success output by deserialising into the Rust type, so this stays out of the
# shipped dependency graph.
# Version matches the one regorus already pulls into this crate's graph, so
# the test dependency reuses that build instead of adding a second copy.
jsonschema = { version = "0.46", default-features = false }
[target.'cfg(windows)'.dependencies]
windows = { version = "0.58", features = ["Win32_Foundation", "Win32_UI_WindowsAndMessaging"] }

41
packages/cua-driver/rust/crates/cua-driver-core/src/action_record.rs
View file

@ -1820,4 +1820,45 @@ mod tests {
);
}
}
#[test]
fn every_legacy_text_only_action_is_conservatively_unverifiable() {
let tools = [
"click",
"double_click",
"right_click",
"scroll",
"drag",
"mouse_drag",
"parallel_mouse_drag",
"move_cursor",
"mouse_button_down",
"mouse_button_up",
"type_text",
"type_text_chars",
"press_key",
"hotkey",
"set_value",
"browser_click",
"browser_pointer",
"browser_type",
];
for tool in tools {
let record = ActionExecutionRecord::from_legacy(
tool,
&serde_json::json!({}),
&serde_json::Value::Null,
)
.unwrap_or_else(|| panic!("legacy text-only action {tool} must normalize"));
assert_eq!(
record.effect,
ActionEffect::Unverifiable,
"legacy text-only action {tool} must never imply confirmation"
);
record
.public_result()
.unwrap_or_else(|error| panic!("{tool} must publish: {error:?}"));
}
}
}

146
packages/cua-driver/rust/crates/cua-driver-core/src/action_target.rs Normal file
View file

@ -0,0 +1,146 @@
//! Per-call target normalization for lifecycle-independent capture modality.
use crate::protocol::ToolResult;
use serde_json::{json, Value};
const TARGETED_TOOLS: &[&str] = &[
"move_cursor",
"click",
"drag",
"scroll",
"type_text",
"press_key",
"hotkey",
];
pub fn supports_typed_target(tool_name: &str) -> bool {
TARGETED_TOOLS.contains(&tool_name)
}
fn invalid_target(message: impl Into<String>) -> ToolResult {
ToolResult::error(message.into()).with_structured(json!({
"code": "invalid_action_target",
}))
}
/// Normalize the generated tagged union into the legacy flat fields consumed
/// by the current thin platform adapters. This runs once at the canonical
/// dispatch boundary before authorization, so policy/resource checks and the
/// platform worker see the same exact target.
pub fn normalize_action_target(tool_name: &str, args: &mut Value) -> Result<(), ToolResult> {
let Some(object) = args.as_object_mut() else {
return Ok(());
};
let Some(target) = object.remove("target") else {
if object.get("scope").and_then(Value::as_str) == Some("desktop")
&& (object.contains_key("pid") || object.contains_key("window_id"))
{
return Err(invalid_target(
"desktop scope cannot be combined with pid or window_id",
));
}
return Ok(());
};
if !supports_typed_target(tool_name) {
return Err(invalid_target(format!(
"{tool_name} does not accept a per-call target"
)));
}
if object.contains_key("scope")
|| object.contains_key("pid")
|| object.contains_key("window_id")
{
return Err(invalid_target(
"target cannot be combined with legacy scope, pid, or window_id fields",
));
}
let Some(target) = target.as_object() else {
return Err(invalid_target("target must be an object"));
};
match target.get("kind").and_then(Value::as_str) {
Some("window") => {
let pid = target
.get("pid")
.and_then(Value::as_u64)
.filter(|pid| *pid > 0 && *pid <= u32::MAX as u64)
.ok_or_else(|| invalid_target("window target requires a positive 32-bit pid"))?;
let window_id = target
.get("window_id")
.and_then(Value::as_u64)
.filter(|window_id| *window_id > 0)
.ok_or_else(|| invalid_target("window target requires a positive window_id"))?;
if target.len() != 3 {
return Err(invalid_target(
"window target accepts only kind, pid, and window_id",
));
}
object.insert("scope".into(), Value::String("window".into()));
object.insert("pid".into(), Value::Number(pid.into()));
object.insert("window_id".into(), Value::Number(window_id.into()));
}
Some("desktop") => {
let display_id = target
.get("display_id")
.and_then(Value::as_str)
.filter(|display_id| !display_id.is_empty())
.ok_or_else(|| invalid_target("desktop target requires display_id"))?;
if target.len() != 2 {
return Err(invalid_target(
"desktop target accepts only kind and display_id",
));
}
if display_id != "primary" {
return Err(invalid_target(
"this release supports only display_id='primary'",
));
}
object.insert("scope".into(), Value::String("desktop".into()));
}
Some(_) => return Err(invalid_target("target.kind must be window or desktop")),
None => return Err(invalid_target("target.kind is required")),
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn exact_targets_normalize_to_one_unambiguous_legacy_shape() {
let mut window = json!({
"x": 1,
"y": 2,
"target": {"kind": "window", "pid": 7, "window_id": 9}
});
normalize_action_target("click", &mut window).unwrap();
assert_eq!(window["scope"], "window");
assert_eq!(window["pid"], 7);
assert_eq!(window["window_id"], 9);
assert!(window.get("target").is_none());
let mut desktop = json!({
"x": 1,
"y": 2,
"target": {"kind": "desktop", "display_id": "primary"}
});
normalize_action_target("click", &mut desktop).unwrap();
assert_eq!(desktop["scope"], "desktop");
assert!(desktop.get("pid").is_none());
}
#[test]
fn ambiguous_or_unsupported_targets_fail_closed() {
for mut args in [
json!({
"scope": "desktop",
"target": {"kind": "desktop", "display_id": "primary"}
}),
json!({"scope": "desktop", "pid": 7}),
json!({"target": {"kind": "desktop", "display_id": "secondary"}}),
json!({"target": {"kind": "window", "pid": 7, "window_id": 0}}),
] {
assert!(normalize_action_target("click", &mut args).is_err());
}
}
}

502
packages/cua-driver/rust/crates/cua-driver-core/src/authorization.rs
View file

@ -14,8 +14,6 @@ use serde_json::Value;
pub const PERMISSION_MODE_ENV: &str = "CUA_DRIVER_PERMISSION_MODE";
pub const DANGEROUS_BYPASS_ENV: &str = "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS";
pub const DISABLE_UNRESTRICTED_ENV: &str = "CUA_DRIVER_DISABLE_UNRESTRICTED";
pub const LEGACY_EXISTING_PROFILE_APPROVAL_ENV: &str =
"CUA_DRIVER_ALLOW_LEGACY_EXISTING_PROFILE_APPROVAL";
pub const RISK_METADATA_VERSION: &str = "1";
static LAUNCH_GRANTS: OnceLock<BTreeSet<String>> = OnceLock::new();
@ -116,8 +114,10 @@ pub struct EnforcementAdapterDescriptor {
/// Mode-specific truth. `state` remains the standard-mode value for
/// backward-compatible inventory consumers.
pub enforcement_by_mode: AdapterEnforcement,
/// Runtime behavior selected after hard invariants and policy admission.
pub behavior_by_mode: AdapterModeBehavior,
/// Factored profile behavior selected after hard invariants, configured
/// policy, and optional capability-manifest scope admission.
#[serde(rename = "profile_behavior_class")]
pub profile_behavior: AdapterProfileBehavior,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
@ -151,39 +151,58 @@ pub enum ModeBehavior {
Allow,
Deny,
RequireGrant,
Manifest,
/// Bounded-profile compatibility spelling on the public inventory. The
/// factored profile class explains that the manifest is a separate scope
/// ceiling rather than the approval behavior itself.
#[serde(rename = "manifest")]
AllowWithoutGrant,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct AdapterModeBehavior {
pub standard: ModeBehavior,
pub bounded: ModeBehavior,
pub unrestricted: ModeBehavior,
#[serde(rename_all = "snake_case")]
pub enum AdapterProfileBehavior {
Routine,
GrantInStandard,
BoundedOrUnrestricted,
UnrestrictedOnly,
Denied,
}
impl AdapterModeBehavior {
pub const fn new(
standard: ModeBehavior,
bounded: ModeBehavior,
unrestricted: ModeBehavior,
) -> Self {
Self {
standard,
bounded,
unrestricted,
}
}
impl AdapterProfileBehavior {
pub const fn for_mode(self, mode: PermissionMode) -> ModeBehavior {
match mode {
PermissionMode::Standard => self.standard,
PermissionMode::Bounded => self.bounded,
PermissionMode::Unrestricted => self.unrestricted,
match (self, mode) {
(Self::Routine, PermissionMode::Standard | PermissionMode::Unrestricted) => {
ModeBehavior::Allow
}
(Self::Routine, PermissionMode::Bounded) => ModeBehavior::AllowWithoutGrant,
(Self::GrantInStandard, PermissionMode::Standard) => ModeBehavior::RequireGrant,
(Self::GrantInStandard, PermissionMode::Bounded) => ModeBehavior::AllowWithoutGrant,
(Self::GrantInStandard, PermissionMode::Unrestricted) => ModeBehavior::Allow,
(Self::BoundedOrUnrestricted, PermissionMode::Standard) => ModeBehavior::Deny,
(Self::BoundedOrUnrestricted, PermissionMode::Bounded) => {
ModeBehavior::AllowWithoutGrant
}
(Self::BoundedOrUnrestricted, PermissionMode::Unrestricted) => ModeBehavior::Allow,
(Self::UnrestrictedOnly, PermissionMode::Unrestricted) => ModeBehavior::Allow,
(Self::UnrestrictedOnly, PermissionMode::Standard | PermissionMode::Bounded)
| (Self::Denied, _) => ModeBehavior::Deny,
}
}
}
const EXISTING_PROFILE_OPERATIONS: &[&str] = &["browser_prepare[strategy.kind=existing_profile]"];
const ISOLATED_BROWSER_OPERATIONS: &[&str] = &["browser_prepare[profile.mode=isolated]"];
const ISOLATED_BROWSER_SCOPE_KEYS: &[&str] = &[
"daemon_generation",
"public_session",
"transport_session",
"pid",
"profile_mode",
"profile_name",
"permission_mode",
"managed_policy_sha256",
"user_policy_sha256",
];
const EXISTING_PROFILE_SCOPE_KEYS: &[&str] = &[
"daemon_generation",
"public_session",
@ -253,6 +272,7 @@ const DESKTOP_INPUT_OPERATIONS: &[&str] = &[
"press_key",
"hotkey",
"set_value",
"perform_secondary_action",
"bring_to_front",
"set_window_frame",
];
@ -357,6 +377,23 @@ const SESSION_REVOCATION: &[&str] = &[
];
pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
EnforcementAdapterDescriptor {
id: "browser_prepare.isolated",
operations: ISOLATED_BROWSER_OPERATIONS,
state: RiskEnforcement::Active,
risk_class: RiskClass::R1,
resource_kind: "driver_owned_browser_profile",
scope_keys: ISOLATED_BROWSER_SCOPE_KEYS,
grant_type: None,
idle_ttl_seconds: None,
absolute_ttl_seconds: None,
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "browser_prepare.existing_profile",
operations: EXISTING_PROFILE_OPERATIONS,
@ -367,17 +404,13 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("existing_profile_session_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "launch_grant_or_host_in_standard; manifest_in_bounded",
authorization_requirement: "profile_approval_behavior_and_optional_capability_manifest",
revocation_triggers: EXISTING_PROFILE_REVOCATION,
refusal_code: Some("browser_consent_required"),
authorization_source:
"launch_grant_or_authorization_host_in_standard; manifest_in_bounded; trusted_launch_acknowledgement_in_unrestricted",
"launch_grant_or_authorization_host_in_standard; unattended_in_bounded; trusted_launch_acknowledgement_in_unrestricted; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::RequireGrant,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::GrantInStandard,
},
EnforcementAdapterDescriptor {
id: "private_observation",
@ -389,16 +422,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "desktop_input",
@ -410,16 +439,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "file_transfer_and_output",
@ -431,16 +456,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "browser_consequential_action",
@ -452,16 +473,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(5 * 60),
absolute_ttl_seconds: Some(30 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "browser_unbounded_script",
@ -478,11 +495,7 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
refusal_code: Some("unbounded_operation_requires_unrestricted"),
authorization_source: "unrestricted_mode_with_trusted_launch_time_risk_acknowledgement",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Deny,
ModeBehavior::Deny,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::UnrestrictedOnly,
},
EnforcementAdapterDescriptor {
id: "browser_bound_input",
@ -494,16 +507,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "process_control",
@ -515,17 +524,13 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(2 * 60),
absolute_ttl_seconds: Some(10 * 60),
authorization_requirement: "driver_ownership_in_standard; manifest_in_bounded",
authorization_requirement: "driver_ownership_in_standard; unattended_in_bounded; optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source:
"driver_ownership_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
"driver_ownership_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Deny,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::BoundedOrUnrestricted,
},
EnforcementAdapterDescriptor {
id: "os_permission_prompt",
@ -542,11 +547,7 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
refusal_code: Some("os_permission_prompt_requires_trusted_host"),
authorization_source: "trusted_host_setup_outside_the_agent_tool_path",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Deny,
ModeBehavior::Deny,
ModeBehavior::Deny,
),
profile_behavior: AdapterProfileBehavior::Denied,
},
EnforcementAdapterDescriptor {
id: "driver_configuration",
@ -563,16 +564,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(5 * 60),
absolute_ttl_seconds: Some(30 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "clipboard",
@ -592,16 +589,12 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
grant_type: Some("protected_resource_grant"),
idle_ttl_seconds: Some(30 * 60),
absolute_ttl_seconds: Some(8 * 60 * 60),
authorization_requirement: "manifest_only_in_bounded",
authorization_requirement: "profile_behavior_and_optional_capability_manifest",
revocation_triggers: SESSION_REVOCATION,
refusal_code: Some("authorization_required"),
authorization_source: "built_in_standard; manifest_in_bounded; trusted_unrestricted_mode",
authorization_source: "built_in_standard; unattended_bounded_profile; trusted_unrestricted_mode; optional_capability_manifest_ceiling",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::Active),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Allow,
ModeBehavior::Manifest,
ModeBehavior::Allow,
),
profile_behavior: AdapterProfileBehavior::Routine,
},
EnforcementAdapterDescriptor {
id: "devices",
@ -618,11 +611,7 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
refusal_code: None,
authorization_source: "capability_not_exposed",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::NotExposed),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Deny,
ModeBehavior::Deny,
ModeBehavior::Deny,
),
profile_behavior: AdapterProfileBehavior::Denied,
},
EnforcementAdapterDescriptor {
id: "shell_and_network",
@ -639,16 +628,32 @@ pub const ENFORCEMENT_ADAPTERS: &[EnforcementAdapterDescriptor] = &[
refusal_code: None,
authorization_source: "capability_not_exposed",
enforcement_by_mode: AdapterEnforcement::uniform(RiskEnforcement::NotExposed),
behavior_by_mode: AdapterModeBehavior::new(
ModeBehavior::Deny,
ModeBehavior::Deny,
ModeBehavior::Deny,
),
profile_behavior: AdapterProfileBehavior::Denied,
},
];
pub fn enforcement_adapter_inventory_json() -> Value {
serde_json::to_value(ENFORCEMENT_ADAPTERS).expect("static adapter inventory serializes")
Value::Array(
ENFORCEMENT_ADAPTERS
.iter()
.map(|adapter| {
let mut value =
serde_json::to_value(adapter).expect("static adapter inventory serializes");
value
.as_object_mut()
.expect("adapter descriptor serializes as an object")
.insert(
"behavior_by_mode".to_owned(),
serde_json::json!({
"standard": adapter.profile_behavior.for_mode(PermissionMode::Standard),
"bounded": adapter.profile_behavior.for_mode(PermissionMode::Bounded),
"unrestricted": adapter.profile_behavior.for_mode(PermissionMode::Unrestricted),
}),
);
value
})
.collect(),
)
}
pub fn adapter_ids_with_state(state: RiskEnforcement) -> Vec<&'static str> {
@ -692,6 +697,13 @@ pub fn enforcement_adapters_for_call(
&& args.pointer("/strategy/kind").and_then(Value::as_str) == Some("existing_profile")
{
add("browser_prepare.existing_profile");
} else if tool == "browser_prepare"
&& matches!(
args.pointer("/profile/mode").and_then(Value::as_str),
Some("isolated_new" | "isolated_named")
)
{
add("browser_prepare.isolated");
}
if matches!(
@ -822,6 +834,8 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
"get_screen_size"
| "get_cursor_position"
| "get_config"
| "get_session"
| "list_sessions"
| "get_session_state"
| "get_agent_cursor_state"
| "get_recording_state"
@ -845,6 +859,7 @@ pub fn advertised_risk_for(tool: &str) -> RiskAssessment {
| "press_key"
| "hotkey"
| "set_value"
| "perform_secondary_action"
| "invoke_menu"
| "launch_app"
| "bring_to_front"
@ -919,7 +934,7 @@ pub fn classify_tool_call(tool: &str, args: &Value) -> RiskAssessment {
} else {
RiskAssessment {
class: RiskClass::R1,
enforcement: RiskEnforcement::MetadataOnly,
enforcement: RiskEnforcement::Active,
operation_sensitive: true,
}
}
@ -1086,46 +1101,32 @@ pub fn authorize_tool_call_with_context(
)));
}
let mode = context.mode();
if mode == PermissionMode::Bounded {
let manifest = context.bounded_manifest().ok_or_else(|| {
crate::policy::AuthorizationError::Loading(
"bounded session policy is unavailable".to_owned(),
)
})?;
if manifest.is_expired() {
return Err(crate::policy::AuthorizationError::Denied(
"bounded session policy expired".to_owned(),
));
}
if mode == PermissionMode::Bounded && context.capability_manifest().is_none() {
return Err(crate::policy::AuthorizationError::Loading(
"bounded mode capability manifest is unavailable".to_owned(),
));
}
if let Some(manifest) = context.capability_manifest() {
manifest
.validate_for_mode(mode)
.map_err(crate::policy::AuthorizationError::Denied)?;
match manifest.decision(tool) {
crate::session_manifest::ManifestDecision::Allow => {
manifest
.authorize_call(tool, args)
.map_err(crate::policy::AuthorizationError::Denied)?;
manifest
.authorize_dispatch()
.map_err(crate::policy::AuthorizationError::Denied)?;
}
crate::session_manifest::ManifestDecision::Deny => {
crate::session_manifest::ManifestDecision::Allow => manifest
.authorize_call(tool, args)
.map_err(crate::policy::AuthorizationError::Denied)?,
crate::session_manifest::ManifestDecision::Deny
| crate::session_manifest::ManifestDecision::Ask => {
return Err(crate::policy::AuthorizationError::Denied(format!(
"bounded session policy denies tool '{tool}'"
)))
}
crate::session_manifest::ManifestDecision::Ask => {
return Err(crate::policy::AuthorizationError::Denied(format!(
"bounded session policy requires protected approval for tool '{tool}'; unattended dispatch cannot auto-accept"
"capability manifest denies tool '{tool}'"
)))
}
crate::session_manifest::ManifestDecision::Undeclared => {
return Err(crate::policy::AuthorizationError::Denied(format!(
"tool '{tool}' is outside the bounded session policy"
"tool '{tool}' is outside the capability manifest"
)))
}
}
}
context
.authorize_dispatch()
.map_err(crate::policy::AuthorizationError::Denied)?;
Ok(risk)
}
@ -1150,6 +1151,7 @@ fn enforce_hard_invariants(
| "press_key"
| "hotkey"
| "set_value"
| "perform_secondary_action"
| "kill_app"
| "bring_to_front"
| "get_accessibility_tree"
@ -1198,20 +1200,16 @@ pub enum PermissionModeError {
MissingDangerAcknowledgement,
#[error("--dangerously-bypass-approvals cannot be combined with a non-unrestricted --permission-mode")]
UnexpectedDangerAcknowledgement,
#[error("permission mode bounded requires --session-policy <path> at trusted daemon startup")]
MissingSessionPolicy,
#[error("permission mode bounded requires --approve-session-policy at trusted daemon startup")]
MissingSessionPolicyApproval,
#[error(
"--session-policy/--approve-session-policy are valid only with --permission-mode bounded"
"permission mode bounded requires --capability-manifest <path> at trusted daemon startup"
)]
MissingSessionPolicy,
#[error("a configured capability manifest requires --approve-capability-manifest at trusted daemon startup")]
MissingSessionPolicyApproval,
#[error("--approve-capability-manifest requires --capability-manifest <path>")]
UnexpectedSessionPolicy,
#[error("permission mode unrestricted is disabled by managed startup configuration")]
UnrestrictedDisabled,
#[error(
"--allow-legacy-existing-profile-approval is valid only with --permission-mode standard"
)]
UnexpectedLegacyApproval,
}
fn parse_permission_mode(
@ -1244,21 +1242,6 @@ fn env_flag(name: &str) -> bool {
})
}
/// Temporary migration escape hatch for the same-user-writable browser
/// approval artifact. It is deliberately outside the public tool protocol and
/// must never be described as protected human consent.
pub fn legacy_existing_profile_approval_enabled() -> bool {
// Core's browser unit fixtures exercise the legacy artifact's binding,
// setup, reconnect, and cleanup mechanics directly. Integration tests
// compile this crate without `cfg(test)` and own the production-default
// refusal contract.
#[cfg(test)]
return true;
#[cfg(not(test))]
env_flag(LEGACY_EXISTING_PROFILE_APPROVAL_ENV)
}
static CONFIGURED_PERMISSION_MODE: OnceLock<Result<PermissionMode, String>> = OnceLock::new();
/// Return the immutable process permission mode. An unset mode is `standard`.
@ -1281,32 +1264,23 @@ pub fn validate_startup_authorization() -> anyhow::Result<()> {
if mode == PermissionMode::Unrestricted && env_flag(DISABLE_UNRESTRICTED_ENV) {
return Err(PermissionModeError::UnrestrictedDisabled.into());
}
if mode != PermissionMode::Standard && env_flag(LEGACY_EXISTING_PROFILE_APPROVAL_ENV) {
return Err(PermissionModeError::UnexpectedLegacyApproval.into());
let manifest_configured = crate::session_manifest::capability_manifest_configured();
let manifest_approved = crate::session_manifest::capability_manifest_approved();
if mode == PermissionMode::Bounded && !manifest_configured {
return Err(PermissionModeError::MissingSessionPolicy.into());
}
let session_policy_configured =
std::env::var_os(crate::session_manifest::SESSION_POLICY_FILE_ENV).is_some();
let session_policy_approved = env_flag(crate::session_manifest::SESSION_POLICY_APPROVED_ENV);
match mode {
PermissionMode::Bounded => {
if !session_policy_configured {
return Err(PermissionModeError::MissingSessionPolicy.into());
}
if !session_policy_approved {
return Err(PermissionModeError::MissingSessionPolicyApproval.into());
}
let manifest = crate::session_manifest::configured_session_manifest()
.map_err(anyhow::Error::msg)?
.ok_or(PermissionModeError::MissingSessionPolicy)?;
if manifest.is_expired() {
anyhow::bail!("bounded session policy is already expired");
}
}
PermissionMode::Standard | PermissionMode::Unrestricted => {
if session_policy_configured || session_policy_approved {
return Err(PermissionModeError::UnexpectedSessionPolicy.into());
}
}
if manifest_configured && !manifest_approved {
return Err(PermissionModeError::MissingSessionPolicyApproval.into());
}
if manifest_approved && !manifest_configured {
return Err(PermissionModeError::UnexpectedSessionPolicy.into());
}
if let Some(manifest) =
crate::session_manifest::configured_capability_manifest().map_err(anyhow::Error::msg)?
{
manifest
.validate_for_mode(mode)
.map_err(anyhow::Error::msg)?;
}
Ok(())
}
@ -1342,8 +1316,8 @@ pub fn status_json_with_provider(authorization_host: Option<&'static str>) -> se
.as_ref()
.map(|mode| adapter_ids_with_state_for_mode(*mode, RiskEnforcement::NotExposed))
.unwrap_or_default();
let session_policy = crate::session_manifest::configured_session_manifest();
let session_policy_status = session_policy
let capability_manifest = crate::session_manifest::configured_capability_manifest();
let capability_manifest_status = capability_manifest
.as_ref()
.ok()
.and_then(|manifest| *manifest)
@ -1353,7 +1327,9 @@ pub fn status_json_with_provider(authorization_host: Option<&'static str>) -> se
"version": manifest.version(),
"sha256": manifest.sha256(),
"expires_unix_ms": manifest.expires_unix_ms(),
"idle_timeout_seconds": manifest.idle_timeout().as_secs(),
"idle_timeout_seconds": manifest.idle_timeout().map(|duration| duration.as_secs()),
"expired": manifest.is_expired(),
"idle_expired": manifest.is_idle_expired(),
"allow_count": allow,
"deny_count": deny,
"ask_count": ask,
@ -1377,7 +1353,6 @@ pub fn status_json_with_provider(authorization_host: Option<&'static str>) -> se
"managed_policy_valid": managed_policy.is_ok(),
"managed_policy_sha256": managed_policy_sha256,
"built_in_ceiling": "reviewed_tool_and_risk_map_v1",
"legacy_existing_profile_approval": legacy_existing_profile_approval_enabled(),
"risk_metadata_version": RISK_METADATA_VERSION,
"active_risk_enforcement": adapter_ids_with_state(RiskEnforcement::Active),
"metadata_only_risk_enforcement": adapter_ids_with_state(RiskEnforcement::MetadataOnly),
@ -1388,10 +1363,15 @@ pub fn status_json_with_provider(authorization_host: Option<&'static str>) -> se
"enforcement_adapters": enforcement_adapter_inventory_json(),
"authorization_host": authorization_host,
"authorization_host_assurance": authorization_host_assurance,
"session_policy_configured": std::env::var_os(crate::session_manifest::SESSION_POLICY_FILE_ENV).is_some(),
"session_policy_approved_at_startup": env_flag(crate::session_manifest::SESSION_POLICY_APPROVED_ENV),
"session_policy_valid": session_policy.is_ok(),
"session_policy": session_policy_status,
"capability_manifest_configured": crate::session_manifest::capability_manifest_configured(),
"capability_manifest_approved_at_startup": crate::session_manifest::capability_manifest_approved(),
"capability_manifest_valid": capability_manifest.is_ok(),
"capability_manifest": capability_manifest_status,
// Deprecated status aliases retained for one compatibility window.
"session_policy_configured": crate::session_manifest::capability_manifest_configured(),
"session_policy_approved_at_startup": crate::session_manifest::capability_manifest_approved(),
"session_policy_valid": capability_manifest.is_ok(),
"session_policy": capability_manifest_status,
});
let session_status = crate::session_authorization::status_json();
if let (Some(target), Some(session_status)) =
@ -1467,7 +1447,7 @@ mod tests {
}
#[test]
fn existing_profile_is_the_first_actively_enforced_risk_operation() {
fn browser_prepare_operations_are_actively_enforced() {
assert_eq!(
advertised_risk_for("browser_prepare").enforcement,
RiskEnforcement::Active,
@ -1485,7 +1465,7 @@ mod tests {
&serde_json::json!({"profile": {"mode": "isolated_new"}}),
);
assert_eq!(isolated.class, RiskClass::R1);
assert_eq!(isolated.enforcement, RiskEnforcement::MetadataOnly);
assert_eq!(isolated.enforcement, RiskEnforcement::Active);
}
#[test]
@ -1618,6 +1598,7 @@ mod tests {
assert_eq!(
adapter_ids_with_state(RiskEnforcement::Active),
vec![
"browser_prepare.isolated",
"browser_prepare.existing_profile",
"private_observation",
"desktop_input",
@ -1642,6 +1623,7 @@ mod tests {
assert_eq!(
adapter_ids_with_state_for_mode(PermissionMode::Standard, RiskEnforcement::Active),
vec![
"browser_prepare.isolated",
"browser_prepare.existing_profile",
"private_observation",
"desktop_input",
@ -1672,11 +1654,12 @@ mod tests {
.iter()
.find(|adapter| adapter.id == id)
.unwrap()
.behavior_by_mode
.profile_behavior
.for_mode(mode)
};
for id in [
"browser_prepare.isolated",
"private_observation",
"desktop_input",
"file_transfer_and_output",
@ -1688,7 +1671,7 @@ mod tests {
assert_eq!(behavior(id, PermissionMode::Standard), ModeBehavior::Allow);
assert_eq!(
behavior(id, PermissionMode::Bounded),
ModeBehavior::Manifest
ModeBehavior::AllowWithoutGrant
);
assert_eq!(
behavior(id, PermissionMode::Unrestricted),
@ -1713,6 +1696,87 @@ mod tests {
);
}
#[test]
fn adding_a_manifest_is_monotonic_for_standard_and_unrestricted_profile_behavior() {
let authority = |behavior| match behavior {
ModeBehavior::Deny => 0,
ModeBehavior::RequireGrant => 1,
ModeBehavior::Allow | ModeBehavior::AllowWithoutGrant => 2,
};
// Manifest admission is evaluated before the unchanged profile
// behavior. Exhaust every adapter and every manifest decision: an
// in-scope allow preserves the profile result, while deny, legacy ask,
// and undeclared scope all reduce it to deny.
for mode in [PermissionMode::Standard, PermissionMode::Unrestricted] {
for adapter in ENFORCEMENT_ADAPTERS {
let baseline = adapter.profile_behavior.for_mode(mode);
for manifest_decision in [
crate::session_manifest::ManifestDecision::Allow,
crate::session_manifest::ManifestDecision::Deny,
crate::session_manifest::ManifestDecision::Ask,
crate::session_manifest::ManifestDecision::Undeclared,
] {
let with_manifest = match manifest_decision {
crate::session_manifest::ManifestDecision::Allow => baseline,
crate::session_manifest::ManifestDecision::Deny
| crate::session_manifest::ManifestDecision::Ask
| crate::session_manifest::ManifestDecision::Undeclared => {
ModeBehavior::Deny
}
};
assert!(
authority(with_manifest) <= authority(baseline),
"adding {manifest_decision:?} scope widened {} in {mode:?} from {baseline:?} to {with_manifest:?}",
adapter.id,
);
}
}
}
}
#[test]
fn standard_manifest_scope_is_checked_before_existing_profile_approval() {
use std::io::Write as _;
use std::sync::Arc;
let mut file = tempfile::NamedTempFile::new().unwrap();
writeln!(
file,
"version: 3\nallow:\n tools: [browser_prepare]\nresources:\n browser:\n existing_profiles:\n - pid: 42\n window_id: 7"
)
.unwrap();
let manifest = Arc::new(crate::session_manifest::load_manifest(file.path()).unwrap());
let ceiling = crate::session_authorization::SessionModeCeiling::for_trusted_sessions(
[PermissionMode::Standard],
false,
std::time::Duration::from_secs(60),
std::time::Duration::from_secs(30),
)
.unwrap();
let context =
crate::session_authorization::SessionAuthorizationRegistry::with_ceiling(ceiling)
.compatibility_context(PermissionMode::Standard, Some(manifest))
.unwrap();
let call = |window_id| {
authorize_tool_call_with_context(
"browser_prepare",
&serde_json::json!({
"pid": 42,
"window_id": window_id,
"strategy": {"kind": "existing_profile"},
}),
&context,
)
};
assert!(call(7).is_ok());
assert!(call(8)
.unwrap_err()
.to_string()
.contains("outside the capability manifest"));
}
#[test]
fn exact_call_inventory_composes_overlapping_resource_boundaries() {
let ids = |tool, args: Value| {
@ -1735,6 +1799,20 @@ mod tests {
);
assert!(ids("page", serde_json::json!({})).is_empty());
assert!(ids("page", serde_json::json!({"action": "unknown"})).is_empty());
assert_eq!(
ids(
"browser_prepare",
serde_json::json!({"profile": {"mode": "isolated_new"}})
),
vec!["browser_prepare.isolated"]
);
assert_eq!(
ids(
"browser_prepare",
serde_json::json!({"strategy": {"kind": "existing_profile"}})
),
vec!["browser_prepare.existing_profile"]
);
assert_eq!(
ids("browser_dialog", serde_json::json!({"action": "accept"})),
vec!["browser_consequential_action"]
@ -1820,6 +1898,7 @@ mod tests {
assert_eq!(
status["active_risk_enforcement"],
serde_json::json!([
"browser_prepare.isolated",
"browser_prepare.existing_profile",
"private_observation",
"desktop_input",
@ -1844,6 +1923,7 @@ mod tests {
assert_eq!(
status["effective_active_risk_enforcement"],
serde_json::json!([
"browser_prepare.isolated",
"browser_prepare.existing_profile",
"private_observation",
"desktop_input",
@ -1865,6 +1945,24 @@ mod tests {
status["enforcement_adapters"],
enforcement_adapter_inventory_json()
);
let existing_profile = status["enforcement_adapters"]
.as_array()
.unwrap()
.iter()
.find(|adapter| adapter["id"] == "browser_prepare.existing_profile")
.unwrap();
assert_eq!(
existing_profile["profile_behavior_class"],
"grant_in_standard"
);
assert_eq!(
existing_profile["behavior_by_mode"],
serde_json::json!({
"standard": "require_grant",
"bounded": "manifest",
"unrestricted": "allow",
})
);
assert_eq!(status["authorization_host_assurance"], "unavailable");
assert_eq!(
status_json_with_provider(Some("cua_local_desktop_confirmation_v1"))

698
packages/cua-driver/rust/crates/cua-driver-core/src/background_input.rs Normal file
View file

@ -0,0 +1,698 @@
//! Pure exact-target decisions for macOS-style background input (v1).
//!
//! Background mutation must carry one exact `(pid, CGWindowID)` target from
//! request to postcondition. The platform shell gathers fresh facts about that
//! target immediately before dispatch and asks this module for exactly one
//! decision: execute the requested route, or refuse with a stable
//! machine-readable reason. The shell never improvises a second actuator after
//! a refusal, and a sibling window's state can never confirm the requested
//! action.
//!
//! This module owns no platform objects and performs no I/O, so the complete
//! state-policy matrix is testable in ordinary CI. See
//! `docs/macos-background-input-v1-plan.md` for the design contract.
use serde_json::{json, Value};
/// The canonical target key: the requested `(pid, CGWindowID)`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ExactWindowTarget {
pub pid: i32,
pub window_id: u32,
}
/// WindowServer's attribution of the requested CGWindowID, resolved
/// immediately before the decision.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum WindowServerOwnership {
/// The window exists and the requested pid owns it.
SamePid,
/// WindowServer has no record of the id — closed, stale, or fabricated.
NotFound,
/// The window exists but another process owns it (out-of-process panels).
ForeignPid { owner_pid: i32 },
}
/// Whether an explicitly addressed element was proven to belong to the
/// requested window.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ElementAncestry {
/// The action does not address a retained element.
NotAddressed,
/// The live element ascends to an AX window whose CGWindowID is the
/// requested one.
ProvenDescendant,
/// The live element ascends to a different window of the same process.
OutsideTargetWindow,
/// Ancestry could not be resolved (dead element, SPI failure). An address
/// the shell cannot re-prove is not an exact target.
Unproven,
}
/// Fresh facts about one exact target, gathered by the platform shell.
///
/// Unknown facts must stay unknown: gatherers must not guess a value merely to
/// unlock a route.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BackgroundTargetFacts {
pub window_server: WindowServerOwnership,
/// The requested CGWindowID is claimed by one of the application's fresh
/// `AXWindows` (mapped through `_AXUIElementGetWindow`). Absent means
/// off-Space or AX-unresolved: observation-only.
pub ax_window_present: bool,
/// `AXMinimized` on the exact target window. `None` means the attribute
/// could not be read — an unproven fact, which fails closed for pointer
/// and keyboard routes (it never unlocks them).
pub target_minimized: Option<bool>,
/// `AXHidden` on the owning application. `None` fails closed like
/// `target_minimized`.
pub app_hidden: Option<bool>,
/// Same-pid top-level windows, other than the target, that could receive
/// process-scoped keyboard input (enumerated from WindowServer so an
/// off-Space sibling that AX cannot see still counts; proven-minimized
/// siblings are excluded because they cannot be the key window).
pub competing_keyboard_destinations: usize,
/// Ancestry proof for an explicitly addressed element, when one exists.
pub element: ElementAncestry,
}
/// The background action classes v1 distinguishes. Route order and trust are
/// documented in the plan's routing ladder.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BackgroundAction {
/// Semantic AX action or value mutation on an exact element
/// (`AXPress`, `AXConfirm`, selection, `AXValue`).
AxSemantic,
/// Window-local routed pointer (stamped CGWindowID; real pointer unmoved).
WindowPointer,
/// Process-scoped text insertion (AX selected-text write or CGEvent
/// keystrokes) with a target-bound value readback.
InsertText,
/// Process-scoped key/hotkey without an action-specific verifier.
GenericKey,
}
impl BackgroundAction {
fn is_pid_keyboard(self) -> bool {
matches!(
self,
BackgroundAction::InsertText | BackgroundAction::GenericKey
)
}
}
/// Stable machine-readable refusal codes. These appear in structured tool
/// results; do not rename them without a compatibility review.
pub mod refusal_codes {
pub const WINDOW_NOT_FOUND: &str = "window_not_found";
pub const OWNER_PID_MISMATCH: &str = "owner_pid_mismatch";
pub const OFF_SPACE_OR_AX_UNRESOLVED: &str = "off_space_or_ax_unresolved";
pub const MINIMIZED_OR_HIDDEN: &str = "minimized_or_hidden_window";
pub const SAME_PID_KEYBOARD_AMBIGUITY: &str = "same_pid_keyboard_ambiguity";
pub const ELEMENT_OUTSIDE_TARGET_WINDOW: &str = "element_outside_target_window";
}
/// A typed refusal: no actuator ran and none may run for this decision.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BackgroundRefusal {
pub code: &'static str,
pub reason: String,
/// The safe next route when one actually exists (advice for an explicit
/// caller decision, never an implicit fallback).
pub advice: Option<&'static str>,
}
/// Verification binding for an executed route: evidence is acceptable only
/// when it belongs to the exact requested window.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TargetBoundVerification {
pub window_id: u32,
}
impl TargetBoundVerification {
/// Whether a readback observed on an element whose resolved window is
/// `evidence_window_id` may confirm the requested action. Unresolvable
/// evidence (`None`) is never accepted — it is unverifiable, not proof —
/// and a sibling window's state can never confirm the requested target.
pub fn accepts_evidence_from(&self, evidence_window_id: Option<u32>) -> bool {
evidence_window_id == Some(self.window_id)
}
}
/// One decision for one gathered-facts snapshot.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum BackgroundInputDecision {
Execute {
/// The only evidence source that may report `confirmed`.
verification: TargetBoundVerification,
},
Refuse(BackgroundRefusal),
}
impl BackgroundInputDecision {
pub fn is_execute(&self) -> bool {
matches!(self, BackgroundInputDecision::Execute { .. })
}
}
fn refuse(
code: &'static str,
reason: String,
advice: Option<&'static str>,
) -> BackgroundInputDecision {
BackgroundInputDecision::Refuse(BackgroundRefusal {
code,
reason,
advice,
})
}
/// Decide whether one background action may run against one exact target.
///
/// The rules encode the v1 state-policy matrix:
/// - a stale or foreign CGWindowID refuses every mutation;
/// - a target absent from fresh `AXWindows` (off-Space or AX-unresolved) is
/// observation-only;
/// - an addressed element must prove ancestry to the requested window for
/// every route, including semantic AX;
/// - semantic AX actions remain available for minimized/hidden targets;
/// - the routed pointer requires a visible (possibly occluded) target; and
/// - process-scoped keyboard additionally requires that the target is the
/// only eligible same-pid keyboard destination, because the transport
/// addresses a process, not a window.
pub fn decide_background_input(
target: ExactWindowTarget,
facts: &BackgroundTargetFacts,
action: BackgroundAction,
) -> BackgroundInputDecision {
match &facts.window_server {
WindowServerOwnership::NotFound => {
return refuse(
refusal_codes::WINDOW_NOT_FOUND,
format!(
"WindowServer has no record of window {} (closed or stale); \
re-enumerate windows and re-target",
target.window_id
),
None,
);
}
WindowServerOwnership::ForeignPid { owner_pid } => {
return refuse(
refusal_codes::OWNER_PID_MISMATCH,
format!(
"window {} is owned by pid {owner_pid}, not requested pid {}; \
re-target the actual owner",
target.window_id, target.pid
),
None,
);
}
WindowServerOwnership::SamePid => {}
}
match facts.element {
ElementAncestry::NotAddressed | ElementAncestry::ProvenDescendant => {}
ElementAncestry::OutsideTargetWindow | ElementAncestry::Unproven => {
return refuse(
refusal_codes::ELEMENT_OUTSIDE_TARGET_WINDOW,
format!(
"the addressed element could not be proven to belong to window {}; \
take a fresh get_window_state snapshot and re-address it",
target.window_id
),
Some("get_window_state"),
);
}
}
if !facts.ax_window_present {
return refuse(
refusal_codes::OFF_SPACE_OR_AX_UNRESOLVED,
format!(
"window {} is not among the process's current AXWindows (another Space, \
or its AX surface is unresolved); background input is refused so a \
sibling window cannot receive it. Observation (capture/list_windows) \
remains available",
target.window_id
),
Some("foreground"),
);
}
// Pointer/keyboard require PROVEN not-minimized and not-hidden: an
// unreadable attribute (`None`) is an unknown fact and unknown facts
// never unlock a route. Semantic AX stays available either way.
let visibility_disproven =
!(facts.target_minimized == Some(false) && facts.app_hidden == Some(false));
match action {
BackgroundAction::AxSemantic => BackgroundInputDecision::Execute {
verification: TargetBoundVerification {
window_id: target.window_id,
},
},
BackgroundAction::WindowPointer => {
if visibility_disproven {
return refuse(
refusal_codes::MINIMIZED_OR_HIDDEN,
format!(
"window {} is minimized or its application is hidden (or that \
state could not be proven); the routed pointer is refused in \
v1. Use an exact element action (click/set_value by element) \
instead",
target.window_id
),
Some("accessibility"),
);
}
BackgroundInputDecision::Execute {
verification: TargetBoundVerification {
window_id: target.window_id,
},
}
}
BackgroundAction::InsertText | BackgroundAction::GenericKey => {
if visibility_disproven {
return refuse(
refusal_codes::MINIMIZED_OR_HIDDEN,
format!(
"window {} is minimized or its application is hidden (or that \
state could not be proven); raw key input is refused in v1. \
Use an exact element action (set_value, click \
action:\"confirm\"/\"press\") instead",
target.window_id
),
Some("accessibility"),
);
}
debug_assert!(action.is_pid_keyboard());
if facts.competing_keyboard_destinations > 0 {
return refuse(
refusal_codes::SAME_PID_KEYBOARD_AMBIGUITY,
format!(
"pid {} owns {} other eligible top-level window(s); process-scoped \
key events cannot be proven to reach window {} and could mutate a \
sibling window. Use an exact element action, the page tool for \
browser content, or delivery_mode:\"foreground\"",
target.pid, facts.competing_keyboard_destinations, target.window_id
),
Some("accessibility"),
);
}
BackgroundInputDecision::Execute {
verification: TargetBoundVerification {
window_id: target.window_id,
},
}
}
}
}
/// The exact-window resolution string reported by the read-only capability
/// section.
fn exact_window_status(facts: &BackgroundTargetFacts) -> &'static str {
match &facts.window_server {
WindowServerOwnership::NotFound => "not_found",
WindowServerOwnership::ForeignPid { .. } => "owner_mismatch",
WindowServerOwnership::SamePid if facts.ax_window_present => "matched",
WindowServerOwnership::SamePid => "ax_unresolved",
}
}
/// Build the additive read-only `background_input` capability report from the
/// same pure facts that gate every action.
///
/// Semantics: `available` means the route's prerequisites are currently
/// proven, not that a future call is guaranteed to succeed; every action
/// revalidates instead of trusting this report. The report contains no window
/// titles, control values, or private symbol addresses.
pub fn background_input_capability_report(
target: ExactWindowTarget,
facts: &BackgroundTargetFacts,
one_shot_capture_available: Option<bool>,
) -> Value {
let route_entry = |route: &str, action: BackgroundAction| -> Value {
match decide_background_input(target, facts, action) {
BackgroundInputDecision::Execute { .. } => {
json!({ "route": route, "status": "available" })
}
BackgroundInputDecision::Refuse(refusal) => {
json!({ "route": route, "status": "refused", "reason": refusal.code })
}
}
};
json!({
"exact_window": {
"status": exact_window_status(facts),
"pid": target.pid,
"window_id": target.window_id,
},
"routes": [
route_entry("accessibility", BackgroundAction::AxSemantic),
route_entry("window_pointer", BackgroundAction::WindowPointer),
route_entry("pid_keyboard", BackgroundAction::GenericKey),
],
"observation": {
"one_shot_capture": match one_shot_capture_available {
Some(true) => "available",
Some(false) => "unavailable",
None => "unknown",
},
"frame_freshness": "unknown",
},
})
}
#[cfg(test)]
mod tests {
use super::*;
const TARGET: ExactWindowTarget = ExactWindowTarget {
pid: 42,
window_id: 700,
};
fn matched_facts() -> BackgroundTargetFacts {
BackgroundTargetFacts {
window_server: WindowServerOwnership::SamePid,
ax_window_present: true,
target_minimized: Some(false),
app_hidden: Some(false),
competing_keyboard_destinations: 0,
element: ElementAncestry::NotAddressed,
}
}
const ALL_ACTIONS: [BackgroundAction; 4] = [
BackgroundAction::AxSemantic,
BackgroundAction::WindowPointer,
BackgroundAction::InsertText,
BackgroundAction::GenericKey,
];
fn code_of(decision: BackgroundInputDecision) -> &'static str {
match decision {
BackgroundInputDecision::Refuse(refusal) => refusal.code,
BackgroundInputDecision::Execute { .. } => panic!("expected a refusal"),
}
}
/// Regression for the proven same-PID wrong-target case: target window A
/// requested while sibling window B is the process's focused/eligible
/// window. Process-scoped keyboard must refuse before dispatch — an
/// explicit `window_id` is an address, not delivery proof.
#[test]
fn two_window_process_refuses_background_keyboard_for_both_text_and_keys() {
let facts = BackgroundTargetFacts {
competing_keyboard_destinations: 1,
..matched_facts()
};
for action in [BackgroundAction::InsertText, BackgroundAction::GenericKey] {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::SAME_PID_KEYBOARD_AMBIGUITY,
"{action:?} must refuse while a sibling keyboard destination exists"
);
}
// Semantic AX and the stamped window-local pointer remain available:
// they are window-addressed, not process-addressed.
for action in [
BackgroundAction::AxSemantic,
BackgroundAction::WindowPointer,
] {
assert!(
decide_background_input(TARGET, &facts, action).is_execute(),
"{action:?} is window-addressed and stays available"
);
}
}
/// A sibling window's state can never satisfy the requested target's
/// verification plan, and unresolvable evidence is not proof.
#[test]
fn sibling_readback_never_confirms_the_requested_window() {
let verification = TargetBoundVerification { window_id: 700 };
assert!(verification.accepts_evidence_from(Some(700)));
assert!(!verification.accepts_evidence_from(Some(701)), "sibling");
assert!(!verification.accepts_evidence_from(None), "unresolvable");
}
#[test]
fn stale_window_id_refuses_every_mutation() {
let facts = BackgroundTargetFacts {
window_server: WindowServerOwnership::NotFound,
ax_window_present: false,
..matched_facts()
};
for action in ALL_ACTIONS {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::WINDOW_NOT_FOUND,
"{action:?}"
);
}
}
#[test]
fn foreign_owner_refuses_every_mutation() {
let facts = BackgroundTargetFacts {
window_server: WindowServerOwnership::ForeignPid { owner_pid: 900 },
..matched_facts()
};
for action in ALL_ACTIONS {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::OWNER_PID_MISMATCH,
"{action:?}"
);
}
}
/// Off-Space / AX-unresolved targets are observation-only: no route may
/// send input, and no sibling window may be used instead.
#[test]
fn ax_unresolved_target_is_observation_only() {
let facts = BackgroundTargetFacts {
ax_window_present: false,
..matched_facts()
};
for action in ALL_ACTIONS {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::OFF_SPACE_OR_AX_UNRESOLVED,
"{action:?}"
);
}
}
/// Minimized/hidden targets retain exact semantic AX actions; raw keys and
/// the routed pointer refuse rather than restoring the window.
#[test]
fn minimized_or_hidden_keeps_semantic_ax_and_refuses_keys_and_pointer() {
for facts in [
BackgroundTargetFacts {
target_minimized: Some(true),
..matched_facts()
},
BackgroundTargetFacts {
app_hidden: Some(true),
..matched_facts()
},
] {
assert!(
decide_background_input(TARGET, &facts, BackgroundAction::AxSemantic).is_execute()
);
for action in [
BackgroundAction::WindowPointer,
BackgroundAction::InsertText,
BackgroundAction::GenericKey,
] {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::MINIMIZED_OR_HIDDEN,
"{action:?}"
);
}
}
}
/// Unknown visibility fails closed: when minimized/hidden state could not
/// be read, the routed pointer and raw keys refuse exactly as if the
/// window were minimized, while exact semantic AX actions remain allowed.
#[test]
fn unknown_visibility_fails_closed_for_pointer_and_keyboard() {
for facts in [
BackgroundTargetFacts {
target_minimized: None,
..matched_facts()
},
BackgroundTargetFacts {
app_hidden: None,
..matched_facts()
},
] {
assert!(
decide_background_input(TARGET, &facts, BackgroundAction::AxSemantic).is_execute()
);
for action in [
BackgroundAction::WindowPointer,
BackgroundAction::InsertText,
BackgroundAction::GenericKey,
] {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::MINIMIZED_OR_HIDDEN,
"{action:?}"
);
}
}
}
/// The singleton rule: with no competing keyboard destination, exact
/// background text/keys are permitted and verification is target-bound.
#[test]
fn singleton_window_permits_background_keyboard_with_target_bound_verification() {
let facts = matched_facts();
for action in [BackgroundAction::InsertText, BackgroundAction::GenericKey] {
match decide_background_input(TARGET, &facts, action) {
BackgroundInputDecision::Execute { verification } => {
assert_eq!(verification.window_id, TARGET.window_id);
}
other => panic!("{action:?} expected Execute, got {other:?}"),
}
}
}
/// An addressed element that cannot be proven to descend from the
/// requested window refuses every route — a cached element under the right
/// cache key is not sufficient after a lifecycle or Space transition.
#[test]
fn unproven_element_ancestry_refuses_all_routes() {
for element in [
ElementAncestry::OutsideTargetWindow,
ElementAncestry::Unproven,
] {
let facts = BackgroundTargetFacts {
element,
..matched_facts()
};
for action in ALL_ACTIONS {
assert_eq!(
code_of(decide_background_input(TARGET, &facts, action)),
refusal_codes::ELEMENT_OUTSIDE_TARGET_WINDOW,
"{action:?} with {element:?}"
);
}
}
}
#[test]
fn proven_element_ancestry_executes_semantic_ax_even_when_minimized() {
let facts = BackgroundTargetFacts {
element: ElementAncestry::ProvenDescendant,
target_minimized: Some(true),
..matched_facts()
};
assert!(decide_background_input(TARGET, &facts, BackgroundAction::AxSemantic).is_execute());
}
/// Refusal precedence: exactness failures are reported before state or
/// cardinality failures so the caller fixes the most fundamental fact.
#[test]
fn refusal_precedence_owner_before_element_before_scope_before_state() {
let worst = BackgroundTargetFacts {
window_server: WindowServerOwnership::ForeignPid { owner_pid: 9 },
ax_window_present: false,
target_minimized: Some(true),
app_hidden: Some(true),
competing_keyboard_destinations: 3,
element: ElementAncestry::OutsideTargetWindow,
};
assert_eq!(
code_of(decide_background_input(
TARGET,
&worst,
BackgroundAction::InsertText
)),
refusal_codes::OWNER_PID_MISMATCH
);
let bad_element = BackgroundTargetFacts {
window_server: WindowServerOwnership::SamePid,
..worst.clone()
};
assert_eq!(
code_of(decide_background_input(
TARGET,
&bad_element,
BackgroundAction::InsertText
)),
refusal_codes::ELEMENT_OUTSIDE_TARGET_WINDOW
);
let unresolved = BackgroundTargetFacts {
element: ElementAncestry::NotAddressed,
..bad_element
};
assert_eq!(
code_of(decide_background_input(
TARGET,
&unresolved,
BackgroundAction::InsertText
)),
refusal_codes::OFF_SPACE_OR_AX_UNRESOLVED
);
let minimized = BackgroundTargetFacts {
ax_window_present: true,
..unresolved
};
assert_eq!(
code_of(decide_background_input(
TARGET,
&minimized,
BackgroundAction::InsertText
)),
refusal_codes::MINIMIZED_OR_HIDDEN
);
}
#[test]
fn capability_report_shape_is_stable() {
let report = background_input_capability_report(TARGET, &matched_facts(), Some(true));
assert_eq!(report["exact_window"]["status"], "matched");
assert_eq!(report["exact_window"]["pid"], 42);
assert_eq!(report["exact_window"]["window_id"], 700);
assert_eq!(report["routes"][0]["route"], "accessibility");
assert_eq!(report["routes"][0]["status"], "available");
assert_eq!(report["routes"][2]["route"], "pid_keyboard");
assert_eq!(report["routes"][2]["status"], "available");
assert_eq!(report["observation"]["one_shot_capture"], "available");
assert_eq!(report["observation"]["frame_freshness"], "unknown");
}
#[test]
fn capability_report_refusals_carry_stable_reason_codes() {
let facts = BackgroundTargetFacts {
ax_window_present: false,
..matched_facts()
};
let report = background_input_capability_report(TARGET, &facts, Some(false));
assert_eq!(report["exact_window"]["status"], "ax_unresolved");
for route in report["routes"].as_array().unwrap() {
assert_eq!(route["status"], "refused");
assert_eq!(route["reason"], refusal_codes::OFF_SPACE_OR_AX_UNRESOLVED);
}
assert_eq!(report["observation"]["one_shot_capture"], "unavailable");
let two_windows = BackgroundTargetFacts {
competing_keyboard_destinations: 1,
..matched_facts()
};
let report = background_input_capability_report(TARGET, &two_windows, None);
assert_eq!(report["routes"][0]["status"], "available");
assert_eq!(report["routes"][2]["status"], "refused");
assert_eq!(
report["routes"][2]["reason"],
refusal_codes::SAME_PID_KEYBOARD_AMBIGUITY
);
assert_eq!(report["observation"]["one_shot_capture"], "unknown");
}
}

299
packages/cua-driver/rust/crates/cua-driver-core/src/browser/approval.rs
View file

@ -1,299 +0,0 @@
//! Explicit approval evidence for mutating browser preparation.
//!
//! MCP-host approval is carried on an internal transport marker. Direct CLI
//! and raw callers instead use a short-lived, single-use artifact minted by
//! the interactive `browser-approve` command. Public tool arguments never
//! treat a Boolean as consent.
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::PathBuf;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use super::platform::{PrepareProfile, PrepareProfileMode};
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
pub const MCP_HOST_APPROVAL_ARG: &str = "_cua_browser_prepare_mcp_host_approved";
pub const MAX_APPROVAL_TTL: Duration = Duration::from_secs(5 * 60);
#[derive(Debug, Serialize, Deserialize)]
struct IsolatedApprovalArtifact {
schema: String,
token: String,
pid: i64,
profile: PrepareProfile,
expires_unix_ms: u128,
}
/// Exact public operation authorized by an interactive existing-profile
/// approval. Transport identity is intentionally absent: the daemon binds it
/// when consuming the artifact and minting its in-memory grant.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ExistingProfileApprovalScope {
pub pid: i64,
pub window_id: u64,
pub session: String,
}
#[derive(Debug, Serialize, Deserialize)]
struct ExistingProfileApprovalArtifact {
schema: String,
token: String,
scope: ExistingProfileApprovalScope,
expires_unix_ms: u128,
}
fn refusal(message: impl Into<String>) -> BrowserRefusal {
BrowserRefusal::new(BrowserRefusalCode::BrowserConsentRequired, message)
}
fn now_unix_ms() -> Result<u128, BrowserRefusal> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_millis())
.map_err(|_| refusal("the system clock cannot validate browser preparation approval"))
}
fn approval_root() -> PathBuf {
std::env::temp_dir().join("cua-driver-browser-approvals-v1")
}
fn artifact_path(token: &str) -> Result<PathBuf, BrowserRefusal> {
let parsed = Uuid::parse_str(token)
.map_err(|_| refusal("browser preparation approval token is malformed or expired"))?;
Ok(approval_root().join(format!("{parsed}.json")))
}
pub fn validate_profile(profile: &PrepareProfile) -> Result<(), BrowserRefusal> {
match profile.mode {
PrepareProfileMode::IsolatedNew => {
if profile.name.is_some() {
return Err(refusal("profile.name is not valid with mode=isolated_new"));
}
}
PrepareProfileMode::IsolatedNamed => {
let Some(name) = profile.name.as_deref() else {
return Err(refusal("profile.name is required with mode=isolated_named"));
};
if name.is_empty()
|| name.len() > 64
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
{
return Err(refusal(
"profile.name must be 1-64 ASCII letters, digits, '-' or '_'",
));
}
}
}
Ok(())
}
/// Mint approval evidence after an interactive caller has confirmed the exact
/// operation. The returned token is the only public value passed to the tool.
pub fn mint_prepare_approval(pid: i64, profile: PrepareProfile) -> Result<String, BrowserRefusal> {
validate_profile(&profile)?;
let token = Uuid::new_v4().to_string();
let root = approval_root();
fs::create_dir_all(&root)
.map_err(|error| refusal(format!("could not create approval directory: {error}")))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700))
.map_err(|error| refusal(format!("could not restrict approval directory: {error}")))?;
}
let artifact = IsolatedApprovalArtifact {
schema: "cua-browser-prepare-approval-v1".to_owned(),
token: token.clone(),
pid,
profile,
expires_unix_ms: now_unix_ms()? + MAX_APPROVAL_TTL.as_millis(),
};
let path = artifact_path(&token)?;
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options
.open(path)
.map_err(|error| refusal(format!("could not create approval artifact: {error}")))?;
let bytes = serde_json::to_vec(&artifact)
.map_err(|error| refusal(format!("could not encode approval artifact: {error}")))?;
file.write_all(&bytes)
.and_then(|_| file.sync_all())
.map_err(|error| refusal(format!("could not persist approval artifact: {error}")))?;
Ok(token)
}
fn persist_artifact<T: Serialize>(token: &str, artifact: &T) -> Result<(), BrowserRefusal> {
let root = approval_root();
fs::create_dir_all(&root)
.map_err(|error| refusal(format!("could not create approval directory: {error}")))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700))
.map_err(|error| refusal(format!("could not restrict approval directory: {error}")))?;
}
let path = artifact_path(token)?;
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options
.open(path)
.map_err(|error| refusal(format!("could not create approval artifact: {error}")))?;
let bytes = serde_json::to_vec(artifact)
.map_err(|error| refusal(format!("could not encode approval artifact: {error}")))?;
file.write_all(&bytes)
.and_then(|_| file.sync_all())
.map_err(|error| refusal(format!("could not persist approval artifact: {error}")))
}
/// Mint a single-use artifact for the exact existing-profile request shown to
/// the person at the interactive approval boundary.
pub fn mint_existing_profile_approval(
scope: ExistingProfileApprovalScope,
) -> Result<String, BrowserRefusal> {
if scope.pid <= 0 || scope.window_id == 0 || scope.session.trim().is_empty() {
return Err(refusal(
"existing-profile approval requires a positive pid/window_id and explicit session",
));
}
let token = Uuid::new_v4().to_string();
let artifact = ExistingProfileApprovalArtifact {
schema: "cua-browser-existing-profile-approval-v1".to_owned(),
token: token.clone(),
scope,
expires_unix_ms: now_unix_ms()? + MAX_APPROVAL_TTL.as_millis(),
};
persist_artifact(&token, &artifact)?;
Ok(token)
}
/// Consume approval evidence. The artifact is removed before any semantic
/// validation so malformed, mismatched, and expired attempts are single-use.
pub fn consume_prepare_approval(
token: &str,
pid: i64,
profile: &PrepareProfile,
) -> Result<(), BrowserRefusal> {
validate_profile(profile)?;
let path = artifact_path(token)?;
let bytes = fs::read(&path)
.map_err(|_| refusal("browser preparation approval token is missing or expired"))?;
fs::remove_file(&path)
.map_err(|error| refusal(format!("could not consume approval artifact: {error}")))?;
let artifact: IsolatedApprovalArtifact = serde_json::from_slice(&bytes)
.map_err(|_| refusal("browser preparation approval artifact is invalid"))?;
if artifact.schema != "cua-browser-prepare-approval-v1"
|| artifact.token != token
|| artifact.pid != pid
|| artifact.profile != *profile
|| artifact.expires_unix_ms < now_unix_ms()?
{
return Err(refusal(
"browser preparation approval does not match this pid/profile request or has expired",
));
}
Ok(())
}
/// Consume an existing-profile artifact before validating it. A mismatch is
/// deliberately destructive so an artifact can never become a probing oracle.
pub fn consume_existing_profile_approval(
token: &str,
scope: &ExistingProfileApprovalScope,
) -> Result<(), BrowserRefusal> {
let path = artifact_path(token)?;
let bytes = fs::read(&path)
.map_err(|_| refusal("browser preparation approval token is missing or expired"))?;
fs::remove_file(&path)
.map_err(|error| refusal(format!("could not consume approval artifact: {error}")))?;
let artifact: ExistingProfileApprovalArtifact = serde_json::from_slice(&bytes)
.map_err(|_| refusal("browser preparation approval artifact is invalid"))?;
if artifact.schema != "cua-browser-existing-profile-approval-v1"
|| artifact.token != token
|| artifact.scope != *scope
|| artifact.expires_unix_ms < now_unix_ms()?
{
return Err(refusal(
"browser preparation approval does not match this pid/window/session request or has expired",
));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn named(name: &str) -> PrepareProfile {
PrepareProfile {
mode: PrepareProfileMode::IsolatedNamed,
name: Some(name.to_owned()),
}
}
#[test]
fn approval_is_bound_and_single_use() {
let profile = named("approval-test");
let token = mint_prepare_approval(42, profile.clone()).unwrap();
consume_prepare_approval(&token, 42, &profile).unwrap();
assert!(consume_prepare_approval(&token, 42, &profile).is_err());
}
#[test]
fn mismatch_consumes_the_artifact() {
let profile = named("approval-mismatch");
let token = mint_prepare_approval(42, profile.clone()).unwrap();
assert!(consume_prepare_approval(&token, 7, &profile).is_err());
assert!(consume_prepare_approval(&token, 42, &profile).is_err());
}
#[test]
fn named_profile_is_path_safe() {
for name in ["", "../Default", "a/b", "with space", "."] {
assert!(validate_profile(&named(name)).is_err(), "accepted {name:?}");
}
validate_profile(&named("research_2026-07")).unwrap();
}
#[test]
fn existing_profile_approval_is_exact_and_single_use() {
let scope = ExistingProfileApprovalScope {
pid: 42,
window_id: 7,
session: "approval-existing".to_owned(),
};
let token = mint_existing_profile_approval(scope.clone()).unwrap();
consume_existing_profile_approval(&token, &scope).unwrap();
assert!(consume_existing_profile_approval(&token, &scope).is_err());
}
#[test]
fn existing_profile_mismatch_consumes_artifact() {
let scope = ExistingProfileApprovalScope {
pid: 42,
window_id: 7,
session: "approval-existing-mismatch".to_owned(),
};
let token = mint_existing_profile_approval(scope.clone()).unwrap();
let mut wrong = scope.clone();
wrong.window_id = 8;
assert!(consume_existing_profile_approval(&token, &wrong).is_err());
assert!(consume_existing_profile_approval(&token, &scope).is_err());
}
}

6
packages/cua-driver/rust/crates/cua-driver-core/src/browser/download.rs
View file

@ -19,6 +19,7 @@ use crate::tool_args::ArgsExt;
use super::cdp_ws::CdpConnection;
use super::engine::BrowserEngine;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::required_session_schema;
use super::store::BrowserActionKind;
/// Injected by an MCP host only after its destructive-tool approval flow.
@ -46,10 +47,7 @@ impl BrowserDownloadTool {
input_schema: json!({
"type": "object",
"properties": {
"session": {
"type": "string",
"description": "Explicit caller session owning the browser capabilities."
},
"session": required_session_schema(),
"target_id": {
"type": "string",
"description": "Opaque exact browser target id from get_browser_state."

20
packages/cua-driver/rust/crates/cua-driver-core/src/browser/engine.rs
View file

@ -96,7 +96,7 @@ fn authorize_live_browser_origin(
let manifest = manifest.ok_or_else(|| {
refuse(
BrowserRefusalCode::BrowserOriginOutsideScope,
"the bounded session policy is unavailable",
"the capability manifest is unavailable",
)
})?;
manifest
@ -1425,15 +1425,13 @@ impl BrowserEngine {
let cdp_session = self.attach(&conn, &tab.cdp_target_id).await?;
let dispatch_context = crate::tool::current_dispatch_authorization_context();
let dispatch_mode = dispatch_context
.as_ref()
.map(|context| context.mode())
.map(Ok)
.unwrap_or_else(crate::authorization::configured_permission_mode);
if dispatch_mode.is_ok_and(|mode| mode == crate::authorization::PermissionMode::Bounded) {
if dispatch_context
.as_deref()
.is_some_and(|context| context.capability_manifest().is_some())
{
let live_url = self.live_top_level_url(&conn, &cdp_session).await?;
// A browser mutation admitted for a delegated bounded session
// must use that exact session's manifest. Falling back to the
// A browser mutation admitted for a delegated session must use
// that exact session's capability manifest. Falling back to the
// process compatibility manifest would let a missing task-local
// context borrow unrelated authority.
let manifest = dispatch_context
@ -1441,10 +1439,10 @@ impl BrowserEngine {
.ok_or_else(|| {
refuse(
BrowserRefusalCode::BrowserOriginOutsideScope,
"the bounded browser authorization context is unavailable",
"the browser authorization context is unavailable",
)
})?
.bounded_manifest();
.capability_manifest();
authorize_live_browser_origin(manifest, &live_url)?;
}
Ok(ValidatedTab {

50
packages/cua-driver/rust/crates/cua-driver-core/src/browser/mod.rs
View file

@ -35,7 +35,6 @@
//! is re-proven before any mutation; frames whose identity or
//! capability cannot be proven are omitted or refused, never guessed.
pub mod approval;
pub mod binding;
pub mod cdp_ws;
pub mod download;
@ -61,8 +60,8 @@ pub use engine::BrowserEngine;
pub use platform::{
BrowserConsentOutcome, BrowserConsentRequest, BrowserPlatform, BrowserVisualAction,
BrowserVisualActionKind, ExistingProfileSetupOutcome, ExistingProfileSetupRequest,
PrepareAction, PrepareAttachment, PrepareAttachmentKind, PrepareAuthorization, PrepareOutcome,
PrepareProfile, PrepareProfileMode, PrepareRequest, PrepareSideEffects, PrepareStrategy,
PrepareAction, PrepareAttachment, PrepareAttachmentKind, PrepareOutcome, PrepareProfile,
PrepareProfileMode, PrepareRequest, PrepareSideEffects, PrepareStrategy,
};
pub use refusal::{BrowserRefusal, BrowserRefusalCode};
pub use setup_descriptor::{
@ -74,3 +73,48 @@ pub use types::{
EndpointOwnershipMethod, EndpointOwnershipProof, NativeOwnershipMethod, NativeOwnershipProof,
NativeWindowInfo, OwnedEndpoint, ProcessFingerprint, Rect,
};
pub(crate) fn session_schema() -> serde_json::Value {
serde_json::json!({
"type": "string",
"description": format!(
"{} Browser targets, tabs, and refs belong to the resolved lifecycle session.",
cua_driver_contract::MULTI_CALL_SESSION_DESCRIPTION
)
})
}
pub(crate) fn required_session_schema() -> serde_json::Value {
serde_json::json!({
"type": "string",
"description": "For multi-call work, prefer a short public session label and repeat it on every call that accepts it. This tool requires the label that owns its browser target, tab, and refs."
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn browser_session_schema_keeps_naming_and_capability_guidance_together() {
let schema = session_schema();
let description = schema["description"]
.as_str()
.expect("browser session description");
assert!(description.contains("prefer a short public session label"));
assert!(description.contains("repeat it on every call that accepts it"));
assert!(description.contains("Browser targets, tabs, and refs"));
}
#[test]
fn required_browser_session_schema_does_not_suggest_omission() {
let schema = required_session_schema();
let description = schema["description"]
.as_str()
.expect("required browser session description");
assert!(description.contains("prefer a short public session label"));
assert!(description.contains("repeat it on every call that accepts it"));
assert!(description.contains("requires the label"));
assert!(!description.contains("Omit it"));
}
}

7
packages/cua-driver/rust/crates/cua-driver-core/src/browser/platform.rs
View file

@ -38,12 +38,6 @@ pub enum PrepareStrategy {
ExistingProfile,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PrepareAuthorization {
McpHost,
ApprovalArtifact(String),
}
/// Caller context for an explicit `browser_prepare` call. Prepare is never
/// implicit: `get_browser_state` must not trigger it.
#[derive(Debug, Clone)]
@ -57,7 +51,6 @@ pub struct PrepareRequest {
/// this independently from the public capability session so either proxy
/// disconnect or explicit `end_session` can reap a spawned browser.
pub transport_session: Option<String>,
pub authorization: Option<PrepareAuthorization>,
/// Omitted for the legacy isolated-profile compatibility form.
pub strategy: Option<PrepareStrategy>,
pub profile: Option<PrepareProfile>,

3
packages/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs
View file

@ -20,6 +20,7 @@ use super::cdp_ws::CdpConnection;
use super::engine::{BrowserEngine, ValidatedTab};
use super::platform::BrowserVisualActionKind;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::required_session_schema;
use super::store::{BrowserActionKind, FrameKind, FrameRef};
use super::tools::{browser_protected_resource_scope, browser_resource_ownership};
@ -299,7 +300,7 @@ impl BrowserPointerTool {
"properties": {
"target_id": { "type": "string", "description": "Opaque target id minted by get_browser_state." },
"tab_id": { "type": "string", "description": "Opaque tab id minted by get_browser_state." },
"session": { "type": "string", "description": "Explicit caller session owning the browser capabilities." },
"session": required_session_schema(),
"action": { "type": "string", "enum": ["hover", "right_click", "double_click", "scroll", "drag"] },
"input_route": { "type": "string", "enum": ["trusted", "dom_event"], "default": "trusted" },
"ref": { "type": "string", "description": "Origin page ref. Alternative to x/y." },

88
packages/cua-driver/rust/crates/cua-driver-core/src/browser/prepare.rs
View file

@ -11,16 +11,12 @@ use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use super::approval::{
consume_existing_profile_approval, consume_prepare_approval, validate_profile,
ExistingProfileApprovalScope,
};
use super::engine::unsupported_engine_refusal;
use super::platform::{
BrowserConsentOutcome, BrowserConsentRequest, ExistingProfileSetupOutcome,
ExistingProfileSetupRequest, PrepareAction, PrepareAttachment, PrepareAttachmentKind,
PrepareAuthorization, PrepareOutcome, PrepareProfile, PrepareProfileMode, PrepareRequest,
PrepareSideEffects, PrepareStrategy,
PrepareOutcome, PrepareProfile, PrepareProfileMode, PrepareRequest, PrepareSideEffects,
PrepareStrategy,
};
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::types::{
@ -31,6 +27,39 @@ use super::BrowserEngine;
const PROFILE_MARKER: &str = ".qwen-cua-driver-owned-profile.json";
const PROFILE_SCHEMA: &str = "qwen-cua-driver-browser-profile-v1";
fn validate_profile(profile: &PrepareProfile) -> Result<(), BrowserRefusal> {
match profile.mode {
PrepareProfileMode::IsolatedNew => {
if profile.name.is_some() {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
"profile.name is not valid with mode=isolated_new",
));
}
}
PrepareProfileMode::IsolatedNamed => {
let Some(name) = profile.name.as_deref() else {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
"profile.name is required with mode=isolated_named",
));
};
if name.is_empty()
|| name.len() > 64
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
{
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
"profile.name must be 1-64 ASCII letters, digits, '-' or '_'",
));
}
}
}
Ok(())
}
async fn claim_with_optional_consent<T, Claim, Consent>(
claim: &mut Pin<Box<Claim>>,
consent: Consent,
@ -670,19 +699,6 @@ impl BrowserEngine {
)
})?;
validate_profile(profile_request)?;
match request.authorization.as_ref() {
Some(PrepareAuthorization::McpHost) => {}
Some(PrepareAuthorization::ApprovalArtifact(token)) => {
consume_prepare_approval(token, request.pid, profile_request)?;
}
None => {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
"browser preparation needs MCP host approval or a fresh browser-approve token",
))
}
}
let classification = self.platform.classify_browser(request.pid).await?;
if !classification.supports_cdp || classification.engine != BrowserEngineFamily::Chromium {
return Err(unsupported_engine_refusal(
@ -769,7 +785,6 @@ impl BrowserEngine {
Protected,
BoundedManifest,
LaunchGrant,
LegacyArtifact,
Unrestricted,
}
@ -779,11 +794,6 @@ impl BrowserEngine {
"strategy=existing_profile requires an exact window_id approval anchor",
)
})?;
let scope = ExistingProfileApprovalScope {
pid: request.pid,
window_id,
session: request.session.clone(),
};
let mode = crate::tool::current_dispatch_authorization_context()
.map(|context| context.mode())
.map(Ok)
@ -803,10 +813,10 @@ impl BrowserEngine {
"bounded existing-profile attachment requires a live session authorization context",
)
})?;
let manifest = context.bounded_manifest().ok_or_else(|| {
let manifest = context.capability_manifest().ok_or_else(|| {
refusal(
BrowserRefusalCode::BrowserConsentRequired,
"bounded existing-profile attachment requires an approved session manifest",
"bounded existing-profile attachment requires an approved capability manifest",
)
})?;
manifest
@ -824,27 +834,6 @@ impl BrowserEngine {
ConsentPath::LaunchGrant
} else if self.approval_broker.provider_id().is_some() {
ConsentPath::Protected
} else if crate::authorization::legacy_existing_profile_approval_enabled() {
match request.authorization.as_ref() {
Some(PrepareAuthorization::ApprovalArtifact(token)) => {
consume_existing_profile_approval(token, &scope)?;
ConsentPath::LegacyArtifact
}
// The ordinary MCP destructive-tool marker proves transport
// provenance, not a person's approval of their authenticated
// profile. It is deliberately insufficient here.
Some(PrepareAuthorization::McpHost) | None => {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
"legacy existing-profile compatibility requires a fresh operation-bound browser-approve artifact",
)
.with_detail(serde_json::json!({
"approval_request_id": uuid::Uuid::new_v4().to_string(),
"approval_command": "qwen-cua-driver browser-approve --strategy existing_profile --pid <pid> --window-id <window_id> --session <session>",
"legacy_approval_enabled": true,
})));
}
}
} else {
return Err(refusal(
BrowserRefusalCode::BrowserConsentRequired,
@ -853,7 +842,6 @@ impl BrowserEngine {
.with_detail(serde_json::json!({
"permission_mode": mode.as_str(),
"authorization_required": true,
"legacy_approval_enabled": false,
"authorization_host": self.approval_broker.provider_id(),
})));
};
@ -985,7 +973,7 @@ impl BrowserEngine {
// Host authorization is requested only after the exact process,
// native window, browser product, and endpoint owner have all been
// proven. Bounded manifests, launch grants, and unrestricted mode
// proven. Bounded capability manifests, launch grants, and unrestricted mode
// never enter this callback path.
let protected_consent = if matches!(consent_path, ConsentPath::Protected) {
let transport_session = request

2
packages/cua-driver/rust/crates/cua-driver-core/src/browser/refusal.rs
View file

@ -62,7 +62,7 @@ pub enum BrowserRefusalCode {
/// browser action.
BrowserActionUnavailable,
/// The live top-level document left the origin set approved in the
/// bounded session manifest. Further browser input is paused.
/// capability manifest. Further browser input is paused.
BrowserOriginOutsideScope,
}

78
packages/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs
View file

@ -14,15 +14,13 @@ use crate::protocol::{Content, ToolResult};
use crate::tool::{ProtectedResourceOwnership, Tool, ToolDef, ToolRegistry};
use crate::tool_args::ArgsExt;
use super::approval::MCP_HOST_APPROVAL_ARG;
use super::cdp_ws::CdpConnection;
use super::download::BrowserDownloadTool;
use super::engine::{BrowserEngine, BrowserTabScreenshot};
use super::platform::{
BrowserVisualActionKind, PrepareAuthorization, PrepareProfile, PrepareRequest, PrepareStrategy,
};
use super::platform::{BrowserVisualActionKind, PrepareProfile, PrepareRequest, PrepareStrategy};
use super::pointer::BrowserPointerTool;
use super::refusal::{BrowserRefusal, BrowserRefusalCode};
use super::session_schema as schema_session;
use super::store::BrowserActionKind;
use super::types::BindingQuality;
@ -87,13 +85,6 @@ fn schema_ref() -> Value {
})
}
fn schema_session() -> Value {
json!({
"type": "string",
"description": "Stable caller-declared session id. Browser targets, tabs, and refs are scoped to this session."
})
}
pub(crate) fn browser_resource_ownership(
engine: &BrowserEngine,
args: &Value,
@ -592,8 +583,8 @@ impl BrowserPrepareTool {
name: "browser_prepare".into(),
description: "Explicitly prepare an owned DevTools endpoint for a browser \
pid. Existing endpoints are detected without side effects. Acting setup \
for an isolated profile requires host approval or a short-lived setup \
token plus allow_launch=true. It launches a separate browser and never \
for an isolated profile follows the runtime permission mode and optional \
capability manifest. It requires allow_launch=true, launches a separate browser, and never \
copies, modifies, or terminates the requested user profile. Existing-profile \
attachment is explicit and follows the runtime's immutable permission mode: \
standard requires an explicit --grant existing-profile launch grant or an \
@ -611,10 +602,6 @@ impl BrowserPrepareTool {
"properties": {
"pid": { "type": "integer", "description": "Browser process id to prepare." },
"window_id": { "type": "integer", "description": "Exact native window approval anchor; required for strategy.kind=existing_profile." },
"approval_token": {
"type": "string",
"description": "Legacy single-use setup token. Existing-profile use is disabled unless a trusted launcher explicitly enables the same-user-writable compatibility path."
},
"allow_launch": {
"type": "boolean",
"description": "Allow a separate driver-owned isolated Chromium process to be launched (default false)."
@ -685,24 +672,11 @@ impl Tool for BrowserPrepareTool {
}
},
};
let approval_token = args.opt_str("approval_token");
let authorization = if strategy == Some(PrepareStrategy::ExistingProfile) {
approval_token.map(PrepareAuthorization::ApprovalArtifact)
} else if args
.get(MCP_HOST_APPROVAL_ARG)
.and_then(Value::as_bool)
.unwrap_or(false)
{
Some(PrepareAuthorization::McpHost)
} else {
approval_token.map(PrepareAuthorization::ApprovalArtifact)
};
let request = PrepareRequest {
pid,
window_id: args.opt_u64("window_id"),
session,
transport_session: args.opt_str("_transport_session_id"),
authorization,
strategy,
profile,
allow_launch: args.opt_bool("allow_launch").unwrap_or(false),
@ -887,7 +861,9 @@ impl BrowserClickTool {
(Input.dispatchMouseEvent), and refuses where that route cannot \
preserve standalone-browser background posture. \
input_route=\"dom_event\" (synthetic \
el.click(), ref required) is used only when explicitly requested. \
el.click(), ref required) is used only when explicitly requested; \
it proves dispatch, not control activation, because trust-gated \
controls may ignore synthetic events. \
Refused for heuristic bindings."
.into(),
input_schema: json!({
@ -905,7 +881,9 @@ impl BrowserClickTool {
"description": "\"trusted\" (default): Input.dispatchMouseEvent. \
It refuses rather than foregrounding a standalone browser. \
\"dom_event\": synthetic full-background DOM click, only \
when explicitly requested."
when explicitly requested. Dispatch does not prove the \
control activated; refresh page state and verify the \
expected postcondition."
},
},
"required": ["target_id", "tab_id"],
@ -1143,16 +1121,23 @@ impl Tool for BrowserClickTool {
.await
{
Ok(_) => ToolResult::text(format!(
"dispatched DOM click on {} in {tab_id}",
"dispatched synthetic DOM click on {} in {tab_id}; application effect not \
verified (trust-gated controls may ignore untrusted events). Refresh page \
state and verify the expected postcondition",
ext_ref.as_deref().unwrap_or("?")
))
.with_structured(json!({
"status": "ok",
"effect": "unverifiable",
"route": "dom_event",
"target_id": target_id,
"tab_id": tab_id,
"ref": ext_ref,
"frame": frame_kind,
"escalation": {
"recommended": "page",
"reason": "synthetic DOM dispatch cannot prove control activation; refresh page state and verify the expected postcondition",
},
})),
Err(e) => ToolResult::error(format!("DOM click failed: {e}")),
};
@ -2612,7 +2597,7 @@ mod tests {
.expect("browser_prepare properties");
assert!(!prepare_properties.contains_key("consent"));
assert!(!prepare_properties.contains_key("allow_restart"));
assert!(!prepare_properties.contains_key(MCP_HOST_APPROVAL_ARG));
assert!(!prepare_properties.contains_key("approval_token"));
let dialog = BrowserDialogTool::new(e.clone());
assert_eq!(
@ -2821,7 +2806,7 @@ mod tests {
}
#[tokio::test]
async fn prepare_requires_non_forgeable_approval_for_acting_setup() {
async fn isolated_prepare_uses_runtime_authorization_without_a_token() {
let tool = BrowserPrepareTool::new(engine());
let result = tool
.invoke(json!({
@ -2833,47 +2818,38 @@ mod tests {
.await;
assert_eq!(
structured(&result)["refusal"]["code"],
"browser_consent_required"
"browser_route_unavailable"
);
}
#[tokio::test]
async fn ordinary_mcp_marker_never_approves_an_existing_profile() {
async fn existing_profile_requires_runtime_authorization() {
let tool = BrowserPrepareTool::new(engine());
let result = tool
.invoke(json!({
"pid": 1,
"window_id": 7,
"strategy": { "kind": "existing_profile" },
"session": "existing-profile-run",
MCP_HOST_APPROVAL_ARG: true
"session": "existing-profile-run"
}))
.await;
let structured = structured(&result);
assert_eq!(structured["refusal"]["code"], "browser_consent_required");
assert!(structured["refusal"]["detail"]["approval_request_id"]
.as_str()
.is_some());
assert_eq!(
structured["refusal"]["detail"]["authorization_required"],
true
);
}
#[tokio::test]
async fn existing_strategy_conflicts_fail_before_platform_setup() {
let tool = BrowserPrepareTool::new(engine());
let token = crate::browser::approval::mint_existing_profile_approval(
crate::browser::approval::ExistingProfileApprovalScope {
pid: 1,
window_id: 7,
session: "existing-conflict".to_owned(),
},
)
.unwrap();
let result = tool
.invoke(json!({
"pid": 1,
"window_id": 7,
"strategy": { "kind": "existing_profile" },
"profile": { "mode": "isolated_new" },
"approval_token": token,
"session": "existing-conflict"
}))
.await;

173
packages/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs
View file

@ -15,6 +15,7 @@ use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use image::{DynamicImage, ImageFormat, Rgba, RgbaImage};
use serde_json::{json, Value};
use crate::action_record::ActionExecutionRecord;
use crate::protocol::{Content, ToolResult};
use crate::tool::Tool;
@ -885,27 +886,6 @@ async fn fixture() -> Fixture {
fixture_with(|_| {}).await
}
async fn existing_profile_only_fixture() -> Fixture {
let state = Arc::new(StdMutex::new(FixtureState::default()));
let server = MockCdpServer::start(fixture_handler(state.clone())).await;
let setup_invoked = Arc::new(AtomicBool::new(false));
let engine = BrowserEngine::new(Arc::new(FixturePlatform {
ws_url: server.ws_url(),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(true)),
setup_invoked: setup_invoked.clone(),
setup_aborted: Arc::new(AtomicBool::new(false)),
stall_consent: false,
}));
Fixture {
state,
_server: server,
engine,
setup_invoked,
}
}
struct FixtureProtectedProvider {
consent_seen: AtomicBool,
}
@ -962,15 +942,20 @@ async fn existing_profile_setup_fixture() -> (Fixture, Arc<AtomicBool>) {
let state = Arc::new(StdMutex::new(FixtureState::default()));
let server = MockCdpServer::start(fixture_handler(state.clone())).await;
let setup_invoked = Arc::new(AtomicBool::new(false));
let engine = BrowserEngine::new(Arc::new(FixturePlatform {
ws_url: server.ws_url(),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: setup_invoked.clone(),
setup_aborted: Arc::new(AtomicBool::new(false)),
stall_consent: false,
}));
let engine = BrowserEngine::new_with_protected_consent_provider(
Arc::new(FixturePlatform {
ws_url: server.ws_url(),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: setup_invoked.clone(),
setup_aborted: Arc::new(AtomicBool::new(false)),
stall_consent: false,
}),
Some(Arc::new(FixtureProtectedProvider {
consent_seen: AtomicBool::new(false),
})),
);
(
Fixture {
state,
@ -1008,23 +993,14 @@ async fn bind(f: &Fixture) -> (String, String) {
async fn approved_existing_profile_attach_claims_then_binds_one_generation() {
// Match Chrome's per-instance toggle: the endpoint is discoverable only
// through the approved existing-profile route, never as driver-managed.
let f = existing_profile_only_fixture().await;
let token = super::approval::mint_existing_profile_approval(
super::approval::ExistingProfileApprovalScope {
pid: 1,
window_id: 7,
session: SESSION.to_owned(),
},
)
.unwrap();
let (f, provider) = protected_existing_profile_fixture().await;
let prepare = BrowserPrepareTool::new(f.engine.clone())
.invoke(json!({
"pid": 1,
"window_id": 7,
"session": SESSION,
"_transport_session_id": "transport-v2-attach",
"strategy": { "kind": "existing_profile" },
"approval_token": token
"strategy": { "kind": "existing_profile" }
}))
.await;
let prepared = structured(&prepare);
@ -1033,6 +1009,7 @@ async fn approved_existing_profile_attach_claims_then_binds_one_generation() {
assert_eq!(prepared["attachment"]["kind"], "existing_profile");
assert_eq!(prepared["attachment"]["capabilities_invalidated"], true);
assert_eq!(prepared["side_effects"]["displayed_consent_prompt"], false);
assert!(provider.consent_seen.load(Ordering::SeqCst));
assert!(!f.setup_invoked.load(Ordering::SeqCst));
let state = GetBrowserStateTool::new(f.engine.clone())
@ -1086,21 +1063,12 @@ async fn protected_provider_accepts_exact_attach_and_session_end_revokes_the_gra
#[tokio::test]
async fn approved_existing_profile_setup_reports_exact_side_effects() {
let (f, setup_invoked) = existing_profile_setup_fixture().await;
let token = super::approval::mint_existing_profile_approval(
super::approval::ExistingProfileApprovalScope {
pid: 1,
window_id: 7,
session: SESSION.to_owned(),
},
)
.unwrap();
let prepare = BrowserPrepareTool::new(f.engine.clone())
.invoke(json!({
"pid": 1,
"window_id": 7,
"session": SESSION,
"strategy": { "kind": "existing_profile" },
"approval_token": token
"strategy": { "kind": "existing_profile" }
}))
.await;
let prepared = structured(&prepare);
@ -1133,31 +1101,27 @@ async fn refused_consent_cancels_stalled_claim_before_revoking_grant() {
let (_stream, _) = listener.accept().await.unwrap();
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
});
let engine = BrowserEngine::new(Arc::new(FixturePlatform {
ws_url: format!("ws://{address}/devtools/browser"),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: Arc::new(AtomicBool::new(false)),
setup_aborted: Arc::new(AtomicBool::new(false)),
stall_consent: false,
}));
let token = super::approval::mint_existing_profile_approval(
super::approval::ExistingProfileApprovalScope {
pid: 1,
window_id: 7,
session: SESSION.to_owned(),
},
)
.unwrap();
let engine = BrowserEngine::new_with_protected_consent_provider(
Arc::new(FixturePlatform {
ws_url: format!("ws://{address}/devtools/browser"),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: Arc::new(AtomicBool::new(false)),
setup_aborted: Arc::new(AtomicBool::new(false)),
stall_consent: false,
}),
Some(Arc::new(FixtureProtectedProvider {
consent_seen: AtomicBool::new(false),
})),
);
let prepared = tokio::time::timeout(
std::time::Duration::from_secs(3),
BrowserPrepareTool::new(engine).invoke(json!({
"pid": 1,
"window_id": 7,
"session": SESSION,
"strategy": { "kind": "existing_profile" },
"approval_token": token
"strategy": { "kind": "existing_profile" }
})),
)
.await
@ -1179,31 +1143,27 @@ async fn cancelled_prepare_aborts_the_exact_pending_setup() {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
});
let setup_aborted = Arc::new(AtomicBool::new(false));
let engine = BrowserEngine::new(Arc::new(FixturePlatform {
ws_url: format!("ws://{address}/devtools/browser"),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: Arc::new(AtomicBool::new(false)),
setup_aborted: setup_aborted.clone(),
stall_consent: true,
}));
let token = super::approval::mint_existing_profile_approval(
super::approval::ExistingProfileApprovalScope {
pid: 1,
window_id: 7,
session: SESSION.to_owned(),
},
)
.unwrap();
let engine = BrowserEngine::new_with_protected_consent_provider(
Arc::new(FixturePlatform {
ws_url: format!("ws://{address}/devtools/browser"),
trusted_input_limited: false,
managed_endpoint_visible: false,
existing_endpoint_visible: Arc::new(AtomicBool::new(false)),
setup_invoked: Arc::new(AtomicBool::new(false)),
setup_aborted: setup_aborted.clone(),
stall_consent: true,
}),
Some(Arc::new(FixtureProtectedProvider {
consent_seen: AtomicBool::new(false),
})),
);
let cancelled = tokio::time::timeout(
std::time::Duration::from_millis(750),
BrowserPrepareTool::new(engine).invoke(json!({
"pid": 1,
"window_id": 7,
"session": SESSION,
"strategy": { "kind": "existing_profile" },
"approval_token": token
"strategy": { "kind": "existing_profile" }
})),
)
.await;
@ -2004,13 +1964,40 @@ async fn trusted_click_refuses_when_standalone_background_posture_is_unavailable
);
assert!(recorded_calls(&f, "Input.dispatchMouseEvent").is_empty());
let synthetic_args = json!({
"target_id": target, "tab_id": tab, "ref": main_ref,
"input_route": "dom_event", "session": SESSION
});
let synthetic = BrowserClickTool::new(f.engine.clone())
.invoke(json!({
"target_id": target, "tab_id": tab, "ref": main_ref,
"input_route": "dom_event", "session": SESSION
}))
.invoke(synthetic_args.clone())
.await;
assert_eq!(structured(&synthetic)["status"], "ok");
assert_eq!(structured(&synthetic)["effect"], "unverifiable");
assert_eq!(structured(&synthetic)["escalation"]["recommended"], "page");
assert!(synthetic.content.iter().any(|content| matches!(
content,
crate::protocol::Content::Text { text, .. }
if text.contains("application effect not verified")
&& text.contains("trust-gated controls")
)));
let public = ActionExecutionRecord::from_legacy(
"browser_click",
&synthetic_args,
structured(&synthetic),
)
.expect("browser click action record")
.public_result()
.expect("public browser click result");
let public = serde_json::to_value(public).expect("serialize public result");
assert_eq!(public["effect"], "unverifiable", "{public}");
assert_eq!(public["route"], "dom", "{public}");
assert_eq!(public["delivery"]["mode"], "background", "{public}");
assert_eq!(public["escalation"]["target"], "page", "{public}");
assert_eq!(
public["escalation"]["reason"], "effect_unconfirmed",
"{public}"
);
assert!(public.get("status").is_none(), "{public}");
assert!(!recorded_calls(&f, "Runtime.callFunctionOn").is_empty());
assert!(recorded_calls(&f, "Page.bringToFront").is_empty());
assert!(recorded_calls(&f, "Target.activateTarget").is_empty());

87
packages/cua-driver/rust/crates/cua-driver-core/src/capture_scope.rs
View file

@ -238,13 +238,18 @@ impl ScopeViolation {
}
}
/// Enforce the modality for a public session. A first non-lifecycle action
/// implicitly starts that session in `auto`, matching the existing cursor and
/// recording lifecycle behavior.
/// Enforce modality only for a session that explicitly opted into the
/// deprecated capture-scope compatibility contract. Lifecycle-only explicit
/// and implicit sessions select modality per call and carry no capture state.
pub fn enforce_tool(tool_name: &str, args: &Value) -> Result<(), ScopeViolation> {
if matches!(
tool_name,
"start_session" | "end_session" | "escalate_session" | "get_session_state"
"start_session"
| "end_session"
| "escalate_session"
| "get_session"
| "list_sessions"
| "get_session_state"
) {
return Ok(());
}
@ -255,11 +260,9 @@ pub fn enforce_tool(tool_name: &str, args: &Value) -> Result<(), ScopeViolation>
else {
return Ok(());
};
let (state, _) = bind_session(session, None).map_err(|_| ScopeViolation {
code: "session_ended",
message: format!("session '{session}' has ended"),
state: SessionCaptureScope::new(CaptureScopePolicy::Auto),
})?;
let Some(state) = get_session(session) else {
return Ok(());
};
match (state.effective_scope(), tool_scope(tool_name, args)) {
(_, ToolScope::Unscoped)
| (EffectiveCaptureScope::Window, ToolScope::Window)
@ -295,13 +298,22 @@ pub fn enforce_tool(tool_name: &str, args: &Value) -> Result<(), ScopeViolation>
),
state,
}),
(EffectiveCaptureScope::Desktop, ToolScope::Window) => Err(ScopeViolation {
code: "window_scope_disabled",
message: format!(
"window-scope tool '{tool_name}' is disabled while session '{session}' is in desktop scope"
),
state,
}),
(EffectiveCaptureScope::Desktop, ToolScope::Window) => {
let message = if state.policy == CaptureScopePolicy::Auto {
format!(
"window-scope tool '{tool_name}' is disabled because escalation to desktop scope is permanent for session '{session}'; to recover, call end_session for session '{session}', then call start_session with a new session id"
)
} else {
format!(
"window-scope tool '{tool_name}' is disabled while session '{session}' is in desktop scope"
)
};
Err(ScopeViolation {
code: "window_scope_disabled",
message,
state,
})
}
}
}
@ -351,6 +363,39 @@ mod tests {
);
}
#[test]
fn escalated_auto_session_reports_permanent_scope_and_recovery() {
let session = fresh("auto-recovery");
bind_session(&session, Some(CaptureScopePolicy::Auto)).unwrap();
escalate_session(
&session,
EscalationReason::ForegroundIneffective,
Some("window ladder exhausted"),
)
.unwrap();
let violation = enforce_tool("get_window_state", &json!({"session": session})).unwrap_err();
assert_eq!(violation.code, "window_scope_disabled");
assert_eq!(
violation.message,
format!(
"window-scope tool 'get_window_state' is disabled because escalation to desktop scope is permanent for session '{session}'; to recover, call end_session for session '{session}', then call start_session with a new session id"
)
);
assert_eq!(
violation.as_json(&session),
json!({
"session": session,
"capture_scope": "auto",
"effective_scope": "desktop",
"desktop_unlocked": true,
"escalation_reason": "foreground_ineffective",
"escalation_detail": "window ladder exhausted",
"code": "window_scope_disabled"
})
);
}
#[test]
fn conflicting_live_rebind_is_rejected() {
let session = fresh("conflict");
@ -495,7 +540,7 @@ mod tests {
}
#[test]
fn end_racing_escalation_never_resurrects_scope_state() {
fn end_racing_escalation_never_resurrects_legacy_scope_state() {
use std::sync::{Arc, Barrier};
let session = fresh("end-escalate-race");
@ -523,11 +568,9 @@ mod tests {
assert!(crate::session::is_session_ended(&session));
assert!(get_session(&session).is_none());
assert_eq!(
enforce_tool("get_window_state", &json!({"session": session}))
.unwrap_err()
.code,
"session_ended"
assert!(
enforce_tool("get_window_state", &json!({"session": session})).is_ok(),
"the compatibility helper must not recreate cleared capture state; the canonical lifecycle gate rejects the ended session"
);
assert!(crate::session::revive_session(&session));

94
packages/cua-driver/rust/crates/cua-driver-core/src/consent.rs
View file

@ -98,7 +98,7 @@ pub enum ConsentError {
DigestMismatch,
#[error("confirmation expired before activation")]
Expired,
#[error("protected resource is outside the bounded session policy: {0}")]
#[error("protected resource is outside the capability manifest: {0}")]
BoundedResource(String),
}
@ -372,7 +372,27 @@ impl ProtectedResourceGrants {
"protected resource adapter '{adapter_id}' is not registered"
))
})?;
match descriptor.behavior_by_mode.for_mode(context.mode()) {
if let Some(manifest) = context.capability_manifest() {
manifest
.authorize_protected_resource(adapter_id, &resource)
.map_err(ConsentError::BoundedResource)?;
}
let profile_behavior = if adapter_id == "process_control"
&& context.mode() == PermissionMode::Standard
&& resource
.get("driver_owned")
.and_then(Value::as_bool)
.unwrap_or(false)
{
// Standard's process-control capability is ownership-dependent:
// an attested runtime-launched process remains available, while a
// foreign process is denied before approval. Manifest scope is an
// additional ceiling and never replaces that ownership proof.
ModeBehavior::Allow
} else {
descriptor.profile_behavior.for_mode(context.mode())
};
match profile_behavior {
ModeBehavior::Allow => return Ok(None),
ModeBehavior::Deny => {
return Err(ConsentError::Provider(format!(
@ -380,15 +400,12 @@ impl ProtectedResourceGrants {
context.mode().as_str()
)))
}
ModeBehavior::Manifest => {
let manifest = context.bounded_manifest().ok_or_else(|| {
ModeBehavior::AllowWithoutGrant => {
context.capability_manifest().ok_or_else(|| {
ConsentError::BoundedResource(
"bounded mode has no approved session manifest".to_owned(),
"bounded mode has no approved capability manifest".to_owned(),
)
})?;
manifest
.authorize_protected_resource(adapter_id, &resource)
.map_err(ConsentError::BoundedResource)?;
return Ok(None);
}
ModeBehavior::RequireGrant => {}
@ -595,6 +612,7 @@ fn digest_request(request: &ConsentRequest) -> String {
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
struct FakeProvider {
action: ConsentAction,
@ -659,6 +677,28 @@ mod tests {
.unwrap()
}
fn authorization_context_with_manifest(
mode: PermissionMode,
) -> Arc<crate::session_authorization::EffectiveAuthorizationContext> {
let mut file = tempfile::NamedTempFile::new().unwrap();
writeln!(
file,
"version: 1\nmode: bounded\nexpires_after: 1h\nidle_timeout: 30m\nresources:\n desktop:\n windows:\n - pid: 42\n window_id: 7\nallow:\n tools: [click]"
)
.unwrap();
let manifest = Arc::new(crate::session_manifest::load_manifest(file.path()).unwrap());
let ceiling = crate::session_authorization::SessionModeCeiling::for_trusted_sessions(
[mode],
mode == PermissionMode::Unrestricted,
Duration::from_secs(60),
Duration::from_secs(30),
)
.unwrap();
crate::session_authorization::SessionAuthorizationRegistry::with_ceiling(ceiling)
.compatibility_context(mode, Some(manifest))
.unwrap()
}
#[test]
fn process_ownership_requires_the_same_attested_process_instance() {
let store = ProtectedResourceOwnershipStore::default();
@ -777,6 +817,44 @@ mod tests {
assert_eq!(provider.requests.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn capability_manifest_resources_narrow_standard_and_unrestricted_without_prompting() {
for mode in [PermissionMode::Standard, PermissionMode::Unrestricted] {
let provider = provider(ConsentAction::Accept);
let grants =
ProtectedResourceGrants::new(Arc::new(ApprovalBroker::new(Some(provider.clone()))));
let context = authorization_context_with_manifest(mode);
assert!(grants
.authorize(
&context,
"desktop_input",
RiskClass::R1,
Some("public-a"),
serde_json::json!({"kind": "window_input", "pid": 42, "window_id": 7}),
"Control the declared window",
Duration::from_secs(30),
Duration::from_secs(60),
)
.await
.is_ok());
assert!(grants
.authorize(
&context,
"desktop_input",
RiskClass::R1,
Some("public-a"),
serde_json::json!({"kind": "window_input", "pid": 42, "window_id": 8}),
"Control an undeclared window",
Duration::from_secs(30),
Duration::from_secs(60),
)
.await
.is_err());
assert_eq!(provider.requests.load(Ordering::SeqCst), 0);
}
}
#[tokio::test]
async fn exact_accept_creates_a_revocable_grant() {
let provider = provider(ConsentAction::Accept);

68
packages/cua-driver/rust/crates/cua-driver-core/src/element_token.rs
View file

@ -195,6 +195,30 @@ impl TokenRegistry {
before - entries.len()
}
/// Invalidate every snapshot owned by a process across runtime lanes.
/// Platform target-lifecycle watchers call this only after the process is
/// confirmed exited, so PID-wide cleanup cannot revoke a live peer.
pub fn clear_pid(&self, pid: i32) -> usize {
let mut entries = self.by_runtime_and_pid.lock().unwrap();
let before = entries.len();
entries.retain(|(_, entry_pid), _| *entry_pid != pid);
before - entries.len()
}
/// Invalidate snapshots for one exact process/window target across runtime
/// lanes while preserving snapshots for the process's other live windows.
pub fn clear_target(&self, pid: i32, window_id: u32) -> usize {
let mut entries = self.by_runtime_and_pid.lock().unwrap();
let before = entries.values().map(Vec::len).sum::<usize>();
entries.retain(|(_, entry_pid), snapshots| {
if *entry_pid == pid {
snapshots.retain(|entry| entry.window_id != window_id);
}
!snapshots.is_empty()
});
before - entries.values().map(Vec::len).sum::<usize>()
}
/// Build the canonical token string for `snapshot_id` / `element_index`.
/// Pure helper, mirrors [`format_token`] but lives on the registry so
/// callers don't have to import the free function.
@ -364,6 +388,24 @@ pub fn resolve_element_args(
}))
};
let resolve_token = |tok: &str| {
if crate::observation_revision::is_revision_token(tok) {
let (window_id, element_index, snapshot_id) =
crate::observation_revision::revision_tokens()
.resolve_current(pid, tok)
.map_err(|error| refusal(error.code(), error.to_string()))?;
let snapshot_token = token_for(snapshot_id, element_index);
let (snapshot_window_id, snapshot_element_index) = global()
.resolve(pid, &snapshot_token)
.map_err(|message| refusal("stale_element_token", message))?;
if snapshot_window_id != window_id || snapshot_element_index != element_index {
return Err(refusal(
"stale_element_token",
"revision element_token no longer matches the current platform snapshot"
.to_owned(),
));
}
return Ok((window_id, element_index));
}
let (wid, idx) = global().resolve(pid, tok).map_err(|message| {
let code = if message.contains("another runtime generation") {
"generation_mismatch"
@ -620,6 +662,32 @@ mod tests {
assert_eq!(reg.snapshot_count(1), 0);
}
#[test]
fn process_exit_cleanup_invalidates_only_that_pid() {
let reg = fresh_registry();
let dead = reg.register_snapshot(41, 1, 1);
let live = reg.register_snapshot(42, 2, 1);
assert_eq!(reg.clear_pid(41), 1);
assert_eq!(
reg.resolve(41, &format_token(dead, 0)).unwrap_err(),
STALE_TOKEN_ERROR
);
assert_eq!(reg.resolve(42, &format_token(live, 0)).unwrap().0, 2);
}
#[test]
fn target_cleanup_preserves_other_windows_for_the_same_pid() {
let reg = fresh_registry();
let closed = reg.register_snapshot(43, 10, 1);
let live = reg.register_snapshot(43, 11, 1);
assert_eq!(reg.clear_target(43, 10), 1);
assert_eq!(
reg.resolve(43, &format_token(closed, 0)).unwrap_err(),
STALE_TOKEN_ERROR
);
assert_eq!(reg.resolve(43, &format_token(live, 0)).unwrap().0, 11);
}
// ── resolve_element_args precedence rule ─────────────────────────
//
// These cover the Surface 6 dispatch contract:

3
packages/cua-driver/rust/crates/cua-driver-core/src/lib.rs
View file

@ -46,7 +46,9 @@ pub fn parent_liveness_stdin_enabled() -> bool {
}
pub mod action_record;
pub mod action_target;
pub mod authorization;
pub mod background_input;
pub mod browser;
pub mod capture_mode;
pub mod capture_scope;
@ -65,6 +67,7 @@ pub mod ffmpeg_install;
pub mod health_report;
pub mod image_utils;
pub mod model_payload;
pub mod observation_revision;
pub mod page;
pub mod pip_hook;
pub mod policy;

1759
packages/cua-driver/rust/crates/cua-driver-core/src/observation_revision.rs Normal file
View file

File diff suppressed because it is too large Load diff

64
packages/cua-driver/rust/crates/cua-driver-core/src/policy.rs
View file

@ -106,6 +106,28 @@ impl PolicyEngine {
Self::Disabled => PolicyDecision::Error("policy support is not enabled".to_owned()),
}
}
/// Returns `true` when the tool should appear in `tools/list`.
///
/// For YAML policies a tool is potentially listable when it is not
/// explicitly denied and has at least one allow path (unconditional or
/// rule-constrained). For Rego, the policy is evaluated with empty
/// arguments as an approximation; Rego rules can implement their own
/// "no-args" sentinel if finer control is needed.
pub fn is_potentially_listable(&self, tool: &str) -> bool {
let tool = canonical_tool_name(tool);
match self {
#[cfg(feature = "yaml")]
Self::Yaml(policy) => policy.is_potentially_listable(tool),
#[cfg(feature = "rego")]
Self::Rego(policy) => {
let empty_args = Value::Object(serde_json::Map::new());
matches!(policy.evaluate(tool, &empty_args), PolicyDecision::Allow)
}
#[cfg(not(any(feature = "yaml", feature = "rego")))]
Self::Disabled => false,
}
}
}
fn canonical_tool_name(tool: &str) -> &str {
@ -288,6 +310,34 @@ pub fn authorize_tool_call(tool: &str, args: &Value) -> Result<(), Authorization
authorize_policy_layers(tool, args, layers)
}
/// Returns `true` when the tool should be advertised in `tools/list`.
///
/// Unlike [`authorize_tool_call`], this does not require a full argument set:
/// it checks only whether the tool has any potential allow path (is not
/// unconditionally denied). Tools that are conditionally allowed via
/// `allow.rules` are still listed so that callers can attempt a constrained
/// invocation. When no policy is configured, all tools are listable.
///
/// On a policy loading error, this returns `true` (fail-open for listing).
/// The error will surface at invocation time through [`authorize_tool_call`],
/// and [`validate_configured_policy`] is expected to have caught it at startup.
pub fn is_tool_listable(tool: &str) -> bool {
let managed = match configured_managed_policy() {
Ok(policy) => policy,
Err(_) => return true,
};
let user = match configured_policy() {
Ok(policy) => policy,
Err(_) => return true,
};
for policy in [managed, user].into_iter().flatten() {
if !policy.is_potentially_listable(tool) {
return false;
}
}
true
}
/// Eagerly validate the immutable process policy before any action endpoint is
/// exposed. This prevents a configured typo from producing a listening daemon
/// that only discovers the error after clients begin issuing calls.
@ -364,6 +414,20 @@ impl YamlPolicy {
failures.join("; ")
))
}
/// Returns `true` when the tool is not explicitly denied and has at least
/// one potential allow path (an unconditional entry in `allow.tools` or an
/// entry in `allow.rules`). This is the correct predicate for advertising
/// a tool in `tools/list`: the tool may be conditionally allowed, so we
/// must not hide it just because a call with empty arguments would be
/// denied by unsatisfied constraints.
fn is_potentially_listable(&self, tool: &str) -> bool {
if self.denied_tools.iter().any(|denied| denied == tool) {
return false;
}
self.allowed_tools.iter().any(|allowed| allowed == tool)
|| self.rules.iter().any(|rule| rule.tool == tool)
}
}
#[cfg(feature = "yaml")]

23
packages/cua-driver/rust/crates/cua-driver-core/src/protocol.rs
View file

@ -406,7 +406,7 @@ Before starting UI work, classify the desired postcondition. For a non-GUI outco
For an app or window outcome, use the narrowest semantic Cua route first: `set_window_frame` plus `list_windows` readback for geometry, typed browser tools for supported page content, and clipboard tools for clipboard state. Then climb through background `element_index` ({tree_kind}), background {coordinate_term}, foreground delivery, and desktop fallback.{coordinate_note} Never advance on transport success alone.
Workflow per turn:
0. `start_session(session)` once; reuse that id and end it when done.
0. `start_session` is optional. For multi-call work, prefer a short `session` label and repeat it on every call that accepts it. Unnamed calls use the transport's implicit session. Only `start_session` revives an ended name; `end_session` explicitly cleans up.
1. `launch_app`, then `get_window_state(pid, window_id)` to refresh element indices.
2. Act with the fresh index.
3. `verify_state(pid, window_id, expect)` checks bounded postconditions. `unknown` is not success; `include_screenshot:true` lets the multimodal agent judge visual evidence.
@ -559,7 +559,7 @@ mod action_record_wire_tests {
#[cfg(test)]
mod agent_instruction_tests {
use super::agent_instructions;
use super::{agent_instructions, initialize_result};
#[test]
fn instructions_route_structured_and_visual_verification_to_the_right_owner() {
@ -584,6 +584,25 @@ mod agent_instruction_tests {
"initialize instructions should stay within the documented context budget"
);
}
#[test]
fn initialize_instructions_describe_implicit_session_lifecycle() {
let result = initialize_result();
let instructions = result["instructions"]
.as_str()
.expect("initialize result should carry agent instructions");
assert!(instructions.contains("`start_session` is optional"));
assert!(instructions.contains("prefer a short `session` label"));
assert!(instructions.contains("repeat it on every call that accepts it"));
assert!(instructions.contains("transport's implicit session"));
assert!(instructions.contains("Only `start_session` revives an ended name"));
assert!(instructions.contains("`end_session` explicitly cleans up"));
assert!(
!instructions.contains("`start_session(session)` once"),
"initialize instructions must not require explicit session setup"
);
}
}
#[cfg(test)]

83
packages/cua-driver/rust/crates/cua-driver-core/src/recording.rs
View file

@ -787,8 +787,16 @@ fn write_turn(
}
};
let click_family = matches!(tool_name.as_str(), "click" | "double_click" | "right_click");
let action_refused = action_record
.is_some_and(|record| record.effect == crate::action_record::ActionEffect::Refused);
let refused_before_target_resolution = click_family && result_is_error && click_point.is_none();
let click_expected = click_family && !refused_before_target_resolution;
// A target may resolve successfully and still be refused before input
// dispatch (for example, a minimized Windows element). Retaining the
// resolved point in action.json is useful diagnostic context, but a
// crosshair would falsely imply that a click was delivered.
let refused_before_dispatch = click_family && result_is_error && action_refused;
let click_expected =
click_family && !refused_before_target_resolution && !refused_before_dispatch;
let mut payload = serde_json::json!({
"tool": tool_name,
@ -820,14 +828,16 @@ fn write_turn(
// the pre-action image. This also keeps modal-dismiss evidence available
// after the modal HWND has closed.
let mut click_captured = false;
if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (
click_point,
before.screenshot.as_deref(),
CLICK_MARKER_FN.get(),
) {
if let Some(click_png) = marker(screenshot, cx, cy) {
std::fs::write(turn_dir.join("click.png"), click_png)?;
click_captured = true;
if click_expected {
if let (Some((cx, cy)), Some(screenshot), Some(marker)) = (
click_point,
before.screenshot.as_deref(),
CLICK_MARKER_FN.get(),
) {
if let Some(click_png) = marker(screenshot, cx, cy) {
std::fs::write(turn_dir.join("click.png"), click_png)?;
click_captured = true;
}
}
}
write_evidence_manifest(
@ -839,6 +849,8 @@ fn write_turn(
click_captured,
if refused_before_target_resolution {
"action_refused_before_target_resolution"
} else if refused_before_dispatch {
"action_refused_before_dispatch"
} else {
"not_a_click_action"
},
@ -1064,6 +1076,49 @@ mod tests {
"action_refused_before_target_resolution"
);
let pending = session
.begin_turn(
"click",
&serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}),
now_ms(),
)
.expect("resolved refusal should reserve an evidence turn");
let refusal_record = crate::action_record::ActionExecutionRecord::builder(
crate::action_record::ActionEffect::Refused,
crate::action_record::ActionTransport::WindowsTargetedInjection,
crate::action_record::RequestedDelivery::Background,
)
.build()
.expect("valid refusal record");
session.finish_turn_with_outcome(
pending,
"refused before dispatch",
Some(&refusal_record),
true,
);
let resolved_refusal_turn = output_dir.join("turn-00004");
let resolved_refusal_action: Value = serde_json::from_slice(
&std::fs::read(resolved_refusal_turn.join("action.json"))
.expect("read resolved refusal action"),
)
.expect("parse resolved refusal action");
assert_eq!(resolved_refusal_action["click_point"]["x"], 3.0);
assert_eq!(resolved_refusal_action["action_truth"]["effect"], "refused");
assert!(!resolved_refusal_turn.join("click.png").exists());
let resolved_refusal_manifest: Value = serde_json::from_slice(
&std::fs::read(resolved_refusal_turn.join("evidence.json"))
.expect("read resolved refusal evidence"),
)
.expect("parse resolved refusal evidence");
assert_eq!(
resolved_refusal_manifest["click"]["status"],
"not_applicable"
);
assert_eq!(
resolved_refusal_manifest["click"]["classification"],
"action_refused_before_dispatch"
);
let files = [
"action.json",
"app_state.json",
@ -1081,11 +1136,13 @@ mod tests {
}
std::fs::remove_dir(directory).expect("remove turn fixture directory");
}
for file in files.into_iter().filter(|file| *file != "click.png") {
std::fs::remove_file(refused_turn.join(file))
.expect("remove refused turn fixture file");
for directory in [&refused_turn, &resolved_refusal_turn] {
for file in files.iter().copied().filter(|file| *file != "click.png") {
std::fs::remove_file(directory.join(file))
.expect("remove refused turn fixture file");
}
std::fs::remove_dir(directory).expect("remove refused turn fixture directory");
}
std::fs::remove_dir(refused_turn).expect("remove refused turn fixture directory");
std::fs::remove_dir(&output_dir).expect("remove recording fixture directory");
}

125
packages/cua-driver/rust/crates/cua-driver-core/src/server.rs
View file

@ -851,6 +851,33 @@ pub async fn handle_request(
req: Request,
id: serde_json::Value,
provider: &dyn ToolProvider,
) -> Response {
handle_request_inner(req, id, provider, None).await
}
/// Dispatch one MCP request with a transport identity proved by the local
/// adapter rather than supplied by the MCP client.
///
/// The ordinary untrusted entry point above must continue to discard every
/// reserved argument. Direct stdio and authenticated HTTP each own a transport
/// lease, so their adapters pass that lease here after parsing the untrusted
/// request. The inner boundary sanitizes caller claims first and then stamps
/// this trusted owner onto both implicit and explicitly named lifecycle
/// sessions.
pub async fn handle_request_with_transport_session(
req: Request,
id: serde_json::Value,
provider: &dyn ToolProvider,
transport_session: &str,
) -> Response {
handle_request_inner(req, id, provider, Some(transport_session)).await
}
async fn handle_request_inner(
req: Request,
id: serde_json::Value,
provider: &dyn ToolProvider,
transport_session: Option<&str>,
) -> Response {
match req.method.as_str() {
"initialize" => Response::ok(id, initialize_result()),
@ -877,23 +904,17 @@ pub async fn handle_request(
.and_then(serde_json::Value::as_str)
.filter(|session| !session.is_empty())
.map(str::to_owned);
if let (Some(arguments), Some(session)) =
(call.args.as_object_mut(), public_session)
{
arguments.insert(
"_session_id".to_owned(),
serde_json::Value::String(session.clone()),
);
arguments.insert(
"_transport_session_id".to_owned(),
serde_json::Value::String(session.clone()),
);
}
if call.name == "browser_prepare" {
if let Some(arguments) = call.args.as_object_mut() {
if let Some(arguments) = call.args.as_object_mut() {
if let Some(session) = public_session.as_deref().or(transport_session) {
arguments.insert(
crate::browser::approval::MCP_HOST_APPROVAL_ARG.to_owned(),
serde_json::Value::Bool(true),
"_session_id".to_owned(),
serde_json::Value::String(session.to_owned()),
);
}
if let Some(owner) = transport_session.or(public_session.as_deref()) {
arguments.insert(
"_transport_session_id".to_owned(),
serde_json::Value::String(owner.to_owned()),
);
}
}
@ -991,6 +1012,75 @@ mod observation_tests {
assert!(arguments.get("_protected_process_fingerprint").is_none());
}
#[tokio::test]
async fn trusted_transport_boundary_keeps_public_label_separate_from_owner() {
let provider = CapturingProvider {
arguments: Mutex::new(None),
};
let request: Request = serde_json::from_value(serde_json::json!({
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
"name": "browser_download",
"arguments": {
"session": "public",
"_session_id": "forged-owner",
"_transport_session_id": "forged-transport",
"_cua_browser_download_mcp_host_approved": false,
"_protected_process_fingerprint": {"pid": 1}
}
}
}))
.unwrap();
let response = handle_request_with_transport_session(
request,
serde_json::json!(1),
&provider,
"mcp-trusted-lease",
)
.await;
assert!(matches!(response.body, ResponseBody::Result { .. }));
let arguments = provider.arguments.lock().unwrap().clone().unwrap();
assert_eq!(arguments["_session_id"], "public");
assert_eq!(arguments["_transport_session_id"], "mcp-trusted-lease");
assert_eq!(arguments["_cua_browser_download_mcp_host_approved"], true);
assert!(arguments.get("_protected_process_fingerprint").is_none());
}
#[tokio::test]
async fn trusted_transport_boundary_mints_implicit_identity_from_lease() {
let provider = CapturingProvider {
arguments: Mutex::new(None),
};
let request: Request = serde_json::from_value(serde_json::json!({
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
"name": "get_config",
"arguments": {
"_session_id": "forged-owner",
"_transport_session_id": "forged-transport"
}
}
}))
.unwrap();
let response = handle_request_with_transport_session(
request,
serde_json::json!(1),
&provider,
"mcp-trusted-lease",
)
.await;
assert!(matches!(response.body, ResponseBody::Result { .. }));
let arguments = provider.arguments.lock().unwrap().clone().unwrap();
assert_eq!(arguments["_session_id"], "mcp-trusted-lease");
assert_eq!(arguments["_transport_session_id"], "mcp-trusted-lease");
}
#[test]
fn successful_mixed_output_is_shape_only() {
let response = Response::ok(
@ -1118,8 +1208,7 @@ mod observation_tests {
(
"browser_prepare",
serde_json::json!({
"strategy": {"kind": "existing_profile"},
"approval_token": "private-token"
"strategy": {"kind": "existing_profile"}
}),
ToolOperation::BrowserPrepareExistingProfile,
),

1356
packages/cua-driver/rust/crates/cua-driver-core/src/session.rs
View file

File diff suppressed because it is too large Load diff

171
packages/cua-driver/rust/crates/cua-driver-core/src/session_authorization.rs
View file

@ -180,7 +180,7 @@ pub struct EffectiveAuthorizationContext {
public_session: Option<String>,
transport_session: Option<String>,
host_lease: Option<HostLeaseId>,
bounded_manifest: Option<Arc<SessionManifest>>,
capability_manifest: Option<Arc<SessionManifest>>,
user_policy_sha256: Option<String>,
managed_policy_sha256: Option<String>,
expires_unix_ms: Option<u128>,
@ -219,15 +219,23 @@ impl EffectiveAuthorizationContext {
}
pub fn bounded_manifest(&self) -> Option<&SessionManifest> {
self.bounded_manifest.as_deref()
self.capability_manifest.as_deref()
}
pub fn capability_manifest(&self) -> Option<&SessionManifest> {
self.capability_manifest.as_deref()
}
pub fn bounded_manifest_sha256(&self) -> Option<&str> {
self.bounded_manifest
self.capability_manifest
.as_deref()
.map(SessionManifest::sha256)
}
pub fn capability_manifest_sha256(&self) -> Option<&str> {
self.bounded_manifest_sha256()
}
pub fn public_session(&self) -> Option<&str> {
self.public_session.as_deref()
}
@ -241,6 +249,15 @@ impl EffectiveAuthorizationContext {
self.transport_session.as_deref()
}
/// Lifecycle idle TTL selected by a trusted delegated-session host. An
/// ordinary compatibility context uses the product default instead.
#[doc(hidden)]
pub fn lifecycle_idle_ttl_override(&self) -> Option<Duration> {
(self.source == AuthorizationContextSource::TrustedHost)
.then_some(self.idle_ttl)
.flatten()
}
#[doc(hidden)]
pub fn user_policy_sha256(&self) -> Option<&str> {
self.user_policy_sha256.as_deref()
@ -278,7 +295,7 @@ impl EffectiveAuthorizationContext {
false
}
pub fn authorize_dispatch(&self) -> Result<(), String> {
fn commit_context_dispatch(&self) -> Result<(), String> {
if self.revoked.load(Ordering::Acquire) {
return Err("authorization context has been revoked".to_owned());
}
@ -301,6 +318,28 @@ impl EffectiveAuthorizationContext {
*previous = now;
Ok(())
}
/// Commit one dispatch after tool admission and every typed-resource
/// check have succeeded. A rejected resource must not refresh either
/// lifetime lease.
pub fn commit_authorized_dispatch(&self) -> Result<(), String> {
if self.is_expired() {
return Err("authorization context expired".to_owned());
}
if let Some(manifest) = self.capability_manifest() {
manifest.check_lifetime()?;
}
self.commit_context_dispatch()?;
if let Some(manifest) = self.capability_manifest() {
manifest.commit_authorized_dispatch()?;
}
Ok(())
}
/// Compatibility alias for callers that commit a fully admitted dispatch.
pub fn authorize_dispatch(&self) -> Result<(), String> {
self.commit_authorized_dispatch()
}
}
pub struct DelegatedSessionRequest {
@ -309,7 +348,7 @@ pub struct DelegatedSessionRequest {
pub mode: PermissionMode,
pub ttl: Duration,
pub idle_ttl: Duration,
pub bounded_manifest: Option<Arc<SessionManifest>>,
pub capability_manifest: Option<Arc<SessionManifest>>,
}
impl std::fmt::Debug for AuthenticatedActionConnection {
@ -338,7 +377,10 @@ impl std::fmt::Debug for EffectiveAuthorizationContext {
.field("mode", &self.mode)
.field("sessions", &"[redacted]")
.field("host_lease", &self.host_lease.is_some())
.field("bounded_manifest_sha256", &self.bounded_manifest_sha256())
.field(
"capability_manifest_sha256",
&self.capability_manifest_sha256(),
)
.field("user_policy_bound", &self.user_policy_sha256.is_some())
.field(
"managed_policy_bound",
@ -358,9 +400,9 @@ impl std::fmt::Debug for DelegatedSessionRequest {
.field("ttl", &self.ttl)
.field("idle_ttl", &self.idle_ttl)
.field(
"bounded_manifest_sha256",
"capability_manifest_sha256",
&self
.bounded_manifest
.capability_manifest
.as_deref()
.map(SessionManifest::sha256),
)
@ -394,10 +436,13 @@ pub enum SessionAuthorizationError {
Expired,
#[error("session and idle TTLs must be non-zero and within the daemon ceiling")]
InvalidTtl,
#[error("bounded mode requires an immutable per-session manifest")]
#[error("bounded mode requires an immutable capability manifest")]
MissingBoundedManifest,
#[error("a bounded manifest is valid only for bounded mode")]
#[deprecated(note = "capability manifests are now valid in every permission profile")]
#[error("the capability manifest is unexpected")]
UnexpectedBoundedManifest,
#[error("the capability manifest is invalid for the selected permission mode")]
InvalidManifest,
}
#[derive(Debug, Default)]
@ -475,14 +520,10 @@ impl SessionAuthorizationRegistry {
pub fn legacy_context(&self) -> Result<Arc<EffectiveAuthorizationContext>, String> {
let mode = crate::authorization::configured_permission_mode()?;
let bounded_manifest = if mode == PermissionMode::Bounded {
crate::session_manifest::configured_session_manifest()?
.cloned()
.map(Arc::new)
} else {
None
};
self.compatibility_context(mode, bounded_manifest)
let capability_manifest = crate::session_manifest::configured_capability_manifest()?
.cloned()
.map(Arc::new);
self.compatibility_context(mode, capability_manifest)
}
/// Construct the compatibility action context from explicit trusted
@ -490,21 +531,16 @@ impl SessionAuthorizationRegistry {
pub fn compatibility_context(
&self,
mode: PermissionMode,
bounded_manifest: Option<Arc<SessionManifest>>,
capability_manifest: Option<Arc<SessionManifest>>,
) -> Result<Arc<EffectiveAuthorizationContext>, String> {
if !self.ceiling.allows(mode) {
return Err("compatibility permission mode is outside the runtime ceiling".to_owned());
}
match (mode, bounded_manifest.as_ref()) {
(PermissionMode::Bounded, None) => {
return Err("bounded compatibility mode requires a session manifest".to_owned())
}
(PermissionMode::Standard | PermissionMode::Unrestricted, Some(_)) => {
return Err(
"a compatibility session manifest is valid only for bounded mode".to_owned(),
)
}
_ => {}
if mode == PermissionMode::Bounded && capability_manifest.is_none() {
return Err("bounded compatibility mode requires a capability manifest".to_owned());
}
if let Some(manifest) = capability_manifest.as_ref() {
manifest.validate_for_mode(mode)?;
}
Ok(Arc::new(EffectiveAuthorizationContext {
daemon_generation: self.daemon_generation,
@ -513,7 +549,7 @@ impl SessionAuthorizationRegistry {
public_session: None,
transport_session: None,
host_lease: None,
bounded_manifest,
capability_manifest,
user_policy_sha256: self.ceiling.user_policy_sha256.clone(),
managed_policy_sha256: self.ceiling.managed_policy_sha256.clone(),
expires_unix_ms: None,
@ -562,14 +598,15 @@ impl SessionAuthorizationRegistry {
{
return Err(SessionAuthorizationError::InvalidTtl);
}
match (request.mode, request.bounded_manifest.as_ref()) {
(PermissionMode::Bounded, None) => {
return Err(SessionAuthorizationError::MissingBoundedManifest)
}
(PermissionMode::Standard | PermissionMode::Unrestricted, Some(_)) => {
return Err(SessionAuthorizationError::UnexpectedBoundedManifest)
}
_ => {}
if request.mode == PermissionMode::Bounded && request.capability_manifest.is_none() {
return Err(SessionAuthorizationError::MissingBoundedManifest);
}
if request
.capability_manifest
.as_ref()
.is_some_and(|manifest| manifest.validate_for_mode(request.mode).is_err())
{
return Err(SessionAuthorizationError::InvalidManifest);
}
let mut state = self.state.lock().unwrap();
state.by_connection.retain(|_, context| {
@ -595,7 +632,7 @@ impl SessionAuthorizationRegistry {
public_session: Some(request.public_session),
transport_session: Some(request.transport_session),
host_lease: Some(host.id),
bounded_manifest: request.bounded_manifest,
capability_manifest: request.capability_manifest,
user_policy_sha256: self.ceiling.user_policy_sha256.clone(),
managed_policy_sha256: self.ceiling.managed_policy_sha256.clone(),
expires_unix_ms: Some(now_unix_ms() + request.ttl.as_millis()),
@ -791,7 +828,7 @@ mod tests {
mode,
ttl: Duration::from_secs(60),
idle_ttl: Duration::from_secs(30),
bounded_manifest: None,
capability_manifest: None,
}
}
@ -1088,7 +1125,7 @@ mod tests {
}
#[test]
fn bounded_manifests_are_stored_per_connection_context() {
fn capability_manifests_are_stored_per_connection_context() {
let registry = SessionAuthorizationRegistry::delegated_for_test(
SessionModeCeiling::delegated([PermissionMode::Bounded], false),
);
@ -1100,7 +1137,7 @@ mod tests {
let hash_b = manifest_b.sha256().to_owned();
let mut request_a = request(PermissionMode::Bounded);
request_a.bounded_manifest = Some(manifest_a);
request_a.capability_manifest = Some(manifest_a);
registry
.bind_delegated_session(&host, &connection_a, request_a)
.unwrap();
@ -1108,7 +1145,7 @@ mod tests {
let mut request_b = request(PermissionMode::Bounded);
request_b.public_session = "public-b".to_owned();
request_b.transport_session = "transport-b".to_owned();
request_b.bounded_manifest = Some(manifest_b);
request_b.capability_manifest = Some(manifest_b);
registry
.bind_delegated_session(&host, &connection_b, request_b)
.unwrap();
@ -1118,24 +1155,28 @@ mod tests {
registry
.resolve_delegated(&connection_a, "public-a", "transport-a")
.unwrap()
.bounded_manifest_sha256(),
.capability_manifest_sha256(),
Some(hash_a.as_str())
);
assert_eq!(
registry
.resolve_delegated(&connection_b, "public-b", "transport-b")
.unwrap()
.bounded_manifest_sha256(),
.capability_manifest_sha256(),
Some(hash_b.as_str())
);
}
#[test]
fn manifest_presence_matches_the_exact_mode() {
fn capability_manifest_is_optional_except_in_bounded_mode() {
let registry =
SessionAuthorizationRegistry::delegated_for_test(SessionModeCeiling::delegated(
[PermissionMode::Standard, PermissionMode::Bounded],
false,
[
PermissionMode::Standard,
PermissionMode::Bounded,
PermissionMode::Unrestricted,
],
true,
));
let (host, bounded_connection) = registry.trusted_pair_for_test(None);
assert_eq!(
@ -1151,13 +1192,33 @@ mod tests {
let (_, standard_connection) = registry.trusted_pair_for_test(Some(&host));
let mut standard = request(PermissionMode::Standard);
standard.bounded_manifest = Some(manifest_with_allowed_tool("click"));
assert_eq!(
registry
.bind_delegated_session(&host, &standard_connection, standard)
.unwrap_err(),
SessionAuthorizationError::UnexpectedBoundedManifest
);
standard.capability_manifest = Some(manifest_with_allowed_tool("click"));
registry
.bind_delegated_session(&host, &standard_connection, standard)
.unwrap();
assert!(registry
.resolve_delegated(&standard_connection, "public-a", "transport-a")
.unwrap()
.capability_manifest()
.is_some());
let (_, unrestricted_connection) = registry.trusted_pair_for_test(Some(&host));
let mut unrestricted = request(PermissionMode::Unrestricted);
unrestricted.public_session = "public-unrestricted".to_owned();
unrestricted.transport_session = "transport-unrestricted".to_owned();
unrestricted.capability_manifest = Some(manifest_with_allowed_tool("click"));
registry
.bind_delegated_session(&host, &unrestricted_connection, unrestricted)
.unwrap();
assert!(registry
.resolve_delegated(
&unrestricted_connection,
"public-unrestricted",
"transport-unrestricted",
)
.unwrap()
.capability_manifest()
.is_some());
}
#[test]

322
packages/cua-driver/rust/crates/cua-driver-core/src/session_manifest.rs
View file

@ -1,4 +1,4 @@
//! Immutable bounded-autonomy manifest loaded at trusted daemon startup.
//! Immutable capability manifest loaded at trusted daemon startup.
//!
//! The agent may propose this file, but selecting and approving it is a
//! launcher responsibility. The manifest only narrows the built-in,
@ -17,6 +17,8 @@ use sha2::{Digest, Sha256};
pub const SESSION_POLICY_FILE_ENV: &str = "CUA_DRIVER_SESSION_POLICY_FILE";
pub const SESSION_POLICY_APPROVED_ENV: &str = "CUA_DRIVER_SESSION_POLICY_APPROVED";
pub const CAPABILITY_MANIFEST_FILE_ENV: &str = "CUA_DRIVER_CAPABILITY_MANIFEST_FILE";
pub const CAPABILITY_MANIFEST_APPROVED_ENV: &str = "CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ManifestDecision {
@ -30,8 +32,8 @@ pub enum ManifestDecision {
pub struct SessionManifest {
version: u32,
sha256: String,
expires_unix_ms: u128,
idle_timeout: Duration,
expires_unix_ms: Option<u128>,
idle_timeout: Option<Duration>,
allow: HashSet<String>,
deny: HashSet<String>,
ask: HashSet<String>,
@ -84,11 +86,11 @@ impl SessionManifest {
&self.sha256
}
pub fn expires_unix_ms(&self) -> u128 {
pub fn expires_unix_ms(&self) -> Option<u128> {
self.expires_unix_ms
}
pub fn idle_timeout(&self) -> Duration {
pub fn idle_timeout(&self) -> Option<Duration> {
self.idle_timeout
}
@ -134,7 +136,7 @@ impl SessionManifest {
&& !self.existing_profiles.contains(&(pid, window_id))
{
return Err(format!(
"browser pid {pid} window {window_id} is outside the bounded session policy"
"browser pid {pid} window {window_id} is outside the capability manifest"
));
}
}
@ -156,12 +158,12 @@ impl SessionManifest {
if self.browser_origins.contains(&origin) {
Ok(())
} else {
Err("the live browser origin is outside the bounded session policy".to_owned())
Err("the live browser origin is outside the capability manifest".to_owned())
}
}
/// Match the implementation-attested resource of an active adapter
/// against the immutable bounded manifest. A tool allow-list entry alone
/// against the immutable capability manifest. A tool allow-list entry alone
/// never widens the resources approved at launch.
pub fn authorize_protected_resource(
&self,
@ -174,7 +176,7 @@ impl SessionManifest {
.ok_or_else(|| format!("{adapter_id} did not attest a resource kind"))?;
let refused = || {
Err(format!(
"{adapter_id} resource kind '{kind}' is outside the bounded session policy"
"{adapter_id} resource kind '{kind}' is outside the capability manifest"
))
};
match adapter_id {
@ -182,7 +184,7 @@ impl SessionManifest {
"window" => self.authorize_desktop_window(resource),
"application" => self.authorize_desktop_application(resource),
"display" => self.desktop_display.then_some(()).ok_or_else(|| {
"desktop display observation is outside the bounded session policy".to_owned()
"desktop display observation is outside the capability manifest".to_owned()
}),
"browser_target" | "authenticated_browser_tab" => {
self.authorize_resource_origin(resource)
@ -193,7 +195,7 @@ impl SessionManifest {
// directly instead of keying enforcement on a field the tool
// can never send.
"session_capture_scope" => self.desktop_display.then_some(()).ok_or_else(|| {
"desktop capture escalation is outside the bounded session policy".to_owned()
"desktop capture escalation is outside the capability manifest".to_owned()
}),
"session_trajectory_recording" => Ok(()),
_ => refused(),
@ -202,7 +204,7 @@ impl SessionManifest {
"window_input" => self.authorize_desktop_window(resource),
"application_input" => self.authorize_desktop_application(resource),
"display_input" => self.desktop_display.then_some(()).ok_or_else(|| {
"desktop-wide input is outside the bounded session policy".to_owned()
"desktop-wide input is outside the capability manifest".to_owned()
}),
_ => refused(),
},
@ -240,7 +242,7 @@ impl SessionManifest {
Ok(())
} else {
Err(format!(
"process pid {pid} is outside the bounded session policy"
"process pid {pid} is outside the capability manifest"
))
}
}
@ -260,7 +262,7 @@ impl SessionManifest {
})
{
return Err(format!(
"driver configuration change '{key}' is outside the bounded session policy"
"driver configuration change '{key}' is outside the capability manifest"
));
}
}
@ -286,7 +288,7 @@ impl SessionManifest {
Ok(())
} else {
Err(format!(
"desktop pid {pid} window {window_id} is outside the bounded session policy"
"desktop pid {pid} window {window_id} is outside the capability manifest"
))
}
}
@ -306,7 +308,7 @@ impl SessionManifest {
Ok(())
} else {
Err(format!(
"desktop application pid {pid} is outside the bounded session policy"
"desktop application pid {pid} is outside the capability manifest"
))
}
}
@ -347,7 +349,7 @@ impl SessionManifest {
if self.browser_origins.contains(origin) {
Ok(())
} else {
Err("the live browser origin is outside the bounded session policy".to_owned())
Err("the live browser origin is outside the capability manifest".to_owned())
}
}
@ -376,10 +378,7 @@ impl SessionManifest {
) {
Ok(())
} else {
Err(
"one or more upload paths are outside the bounded session policy"
.to_owned(),
)
Err("one or more upload paths are outside the capability manifest".to_owned())
}
}
"trajectory_replay" => self.authorize_exact_path(
@ -411,7 +410,7 @@ impl SessionManifest {
// reviewed bounded decision.
"dependency_install" => Ok(()),
_ => Err(format!(
"file resource kind '{kind}' is outside the bounded session policy"
"file resource kind '{kind}' is outside the capability manifest"
)),
}
}
@ -430,34 +429,75 @@ impl SessionManifest {
if path_allowed(path, allowed, roots) {
Ok(())
} else {
Err("the exact path is outside the bounded session policy".to_owned())
Err("the exact path is outside the capability manifest".to_owned())
}
}
pub fn is_expired(&self) -> bool {
self.expires_unix_ms < now_unix_ms()
self.expires_unix_ms
.is_some_and(|expires_unix_ms| expires_unix_ms < now_unix_ms())
}
pub fn is_idle_expired(&self) -> bool {
self.idle_expired.load(Ordering::Acquire)
}
/// Refresh the bounded-mode idle lease only after a call passed every
/// Check the optional idle lease without refreshing it.
pub fn check_lifetime(&self) -> Result<(), String> {
if self.is_expired() {
return Err("capability manifest expired".to_owned());
}
let Some(idle_timeout) = self.idle_timeout else {
return Ok(());
};
if self.is_idle_expired() {
return Err("capability manifest idle timeout exceeded".to_owned());
}
let last = self.last_authorized_dispatch.lock().unwrap();
if Instant::now().duration_since(*last) > idle_timeout {
self.idle_expired.store(true, Ordering::Release);
return Err("capability manifest idle timeout exceeded".to_owned());
}
Ok(())
}
/// Refresh the capability-manifest idle lease only after a call passed every
/// manifest decision and resource check. Once the idle lease expires it is
/// terminal for this daemon instance; a tool call cannot revive it.
pub fn authorize_dispatch(&self) -> Result<(), String> {
if self.is_idle_expired() {
return Err("bounded session policy idle timeout exceeded".to_owned());
}
pub fn commit_authorized_dispatch(&self) -> Result<(), String> {
self.check_lifetime()?;
let Some(idle_timeout) = self.idle_timeout else {
return Ok(());
};
let now = Instant::now();
let mut last = self.last_authorized_dispatch.lock().unwrap();
if now.duration_since(*last) > self.idle_timeout {
if now.duration_since(*last) > idle_timeout {
self.idle_expired.store(true, Ordering::Release);
return Err("bounded session policy idle timeout exceeded".to_owned());
return Err("capability manifest idle timeout exceeded".to_owned());
}
*last = now;
Ok(())
}
/// Compatibility alias retained for downstream callers during migration.
pub fn authorize_dispatch(&self) -> Result<(), String> {
self.commit_authorized_dispatch()
}
pub fn validate_for_mode(
&self,
mode: crate::authorization::PermissionMode,
) -> Result<(), String> {
if mode == crate::authorization::PermissionMode::Bounded
&& (self.expires_unix_ms.is_none() || self.idle_timeout.is_none())
{
return Err(
"bounded mode requires capability manifest expires_after and idle_timeout"
.to_owned(),
);
}
self.check_lifetime()
}
}
#[cfg(feature = "yaml")]
@ -465,9 +505,12 @@ impl SessionManifest {
#[serde(deny_unknown_fields)]
struct RawManifest {
version: u32,
mode: String,
expires_after: String,
idle_timeout: String,
#[serde(default)]
mode: Option<String>,
#[serde(default)]
expires_after: Option<String>,
#[serde(default)]
idle_timeout: Option<String>,
#[serde(default)]
resources: RawResources,
#[serde(default)]
@ -475,7 +518,7 @@ struct RawManifest {
#[serde(default)]
deny: RawToolSet,
#[serde(default)]
ask: RawToolSet,
ask: Option<RawToolSet>,
}
#[cfg(feature = "yaml")]
@ -622,26 +665,66 @@ pub fn configured_session_manifest() -> Result<Option<&'static SessionManifest>,
}
}
pub fn configured_capability_manifest() -> Result<Option<&'static SessionManifest>, String> {
configured_session_manifest()
}
pub fn capability_manifest_configured() -> bool {
std::env::var_os(CAPABILITY_MANIFEST_FILE_ENV).is_some()
|| std::env::var_os(SESSION_POLICY_FILE_ENV).is_some()
}
pub fn capability_manifest_approved() -> bool {
[
CAPABILITY_MANIFEST_APPROVED_ENV,
SESSION_POLICY_APPROVED_ENV,
]
.into_iter()
.any(|name| {
std::env::var(name).is_ok_and(|value| {
matches!(
value.trim().to_ascii_lowercase().as_str(),
"1" | "true" | "yes" | "on"
)
})
})
}
fn load_configured_manifest() -> Result<Option<SessionManifest>, String> {
let Some(path) = std::env::var_os(SESSION_POLICY_FILE_ENV) else {
let capability_path = std::env::var_os(CAPABILITY_MANIFEST_FILE_ENV);
let legacy_path = std::env::var_os(SESSION_POLICY_FILE_ENV);
if capability_path.is_some() && legacy_path.is_some() && capability_path != legacy_path {
return Err(format!(
"{CAPABILITY_MANIFEST_FILE_ENV} conflicts with deprecated {SESSION_POLICY_FILE_ENV}"
));
}
let Some(path) = capability_path.or(legacy_path) else {
return Ok(None);
};
load_manifest(Path::new(&path)).map(Some)
}
/// Load and validate an immutable bounded-session manifest for a trusted
/// Load and validate an immutable capability manifest for a trusted
/// runtime/session constructor.
pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
let bytes = std::fs::read(path)
.map_err(|error| format!("failed to read session policy {}: {error}", path.display()))?;
let bytes = std::fs::read(path).map_err(|error| {
format!(
"failed to read capability manifest {}: {error}",
path.display()
)
})?;
if bytes.is_empty() {
return Err(format!("session policy {} is empty", path.display()));
return Err(format!("capability manifest {} is empty", path.display()));
}
#[cfg(feature = "yaml")]
let raw: RawManifest = serde_yaml_ng::from_slice(&bytes)
.map_err(|error| format!("failed to parse session policy {}: {error}", path.display()))?;
let raw: RawManifest = serde_yaml_ng::from_slice(&bytes).map_err(|error| {
format!(
"failed to parse capability manifest {}: {error}",
path.display()
)
})?;
#[cfg(not(feature = "yaml"))]
return Err("session policy support requires the yaml feature".to_owned());
return Err("capability manifest support requires the yaml feature".to_owned());
#[cfg(feature = "yaml")]
{
@ -684,42 +767,70 @@ pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
changes: raw_configuration_changes,
} = driver_configuration;
if !matches!(version, 1 | 2) {
if !matches!(version, 1..=3) {
return Err(format!(
"unsupported session policy version {}; expected 1 or 2",
"unsupported capability manifest version {}; expected 1, 2, or 3",
version
));
}
if !matches!(mode.as_str(), "bounded" | "autonomous") {
return Err("session policy mode must be bounded".to_owned());
if version < 3 {
if !mode
.as_deref()
.is_some_and(|mode| matches!(mode, "bounded" | "autonomous"))
{
return Err("legacy capability manifest mode must be bounded".to_owned());
}
} else if mode.is_some() {
return Err("capability manifest version 3 must not declare mode".to_owned());
}
let expires_after = parse_duration(&expires_after)?;
let idle_timeout = parse_duration(&idle_timeout)?;
if expires_after.is_zero() || expires_after > Duration::from_secs(24 * 60 * 60) {
return Err("expires_after must be greater than zero and no more than 24h".to_owned());
}
if idle_timeout.is_zero() || idle_timeout > expires_after {
if version < 3 && (expires_after.is_none() || idle_timeout.is_none()) {
return Err(
"idle_timeout must be greater than zero and no more than expires_after".to_owned(),
"legacy capability manifests require expires_after and idle_timeout".to_owned(),
);
}
let expires_after = expires_after.as_deref().map(parse_duration).transpose()?;
let idle_timeout = idle_timeout.as_deref().map(parse_duration).transpose()?;
if expires_after.is_some_and(|duration| {
duration.is_zero() || duration > Duration::from_secs(24 * 60 * 60)
}) {
return Err("expires_after must be greater than zero and no more than 24h".to_owned());
}
if idle_timeout.is_some_and(|idle| {
idle.is_zero() || expires_after.is_some_and(|expires| idle > expires)
}) {
return Err("idle_timeout must be greater than zero and no more than expires_after when both are declared".to_owned());
}
let allow = validate_tools("allow", allow.tools)?;
let deny = validate_tools("deny", deny.tools)?;
let ask = validate_tools("ask", ask.tools)?;
let mut deny = validate_tools("deny", deny.tools)?;
let legacy_ask = if version < 3 {
validate_tools("ask", ask.unwrap_or_default().tools)?
} else if ask.is_some() {
return Err("capability manifest version 3 does not support ask.tools".to_owned());
} else {
HashSet::new()
};
for tool in legacy_ask {
if !deny.insert(tool.clone()) {
return Err(format!(
"capability manifest tool '{tool}' appears in more than one decision set"
));
}
}
let ask = HashSet::new();
if allow.is_empty() {
return Err("session policy allow.tools must not be empty".to_owned());
return Err("capability manifest allow.tools must not be empty".to_owned());
}
for tool in &allow {
if deny.contains(tool) || ask.contains(tool) {
return Err(format!(
"session policy tool '{tool}' appears in more than one decision set"
"capability manifest tool '{tool}' appears in more than one decision set"
));
}
}
for tool in &deny {
if ask.contains(tool) {
return Err(format!(
"session policy tool '{tool}' appears in more than one decision set"
"capability manifest tool '{tool}' appears in more than one decision set"
));
}
}
@ -739,7 +850,9 @@ pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
}
}
if version == 1 && !raw_browser_profiles.is_empty() {
return Err("browser profiles require session policy version 2".to_owned());
return Err(
"browser profiles require capability manifest version 2 or later".to_owned(),
);
}
let mut existing_profile_kind = false;
for profile in raw_browser_profiles {
@ -769,7 +882,8 @@ pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
validate_positive_pids("desktop applications", raw_desktop_applications)?;
if version == 1 && !raw_applications.is_empty() {
return Err(
"application identity resources require session policy version 2".to_owned(),
"application identity resources require capability manifest version 2 or later"
.to_owned(),
);
}
let applications = validate_applications(raw_applications)?;
@ -836,18 +950,23 @@ pub fn load_manifest(path: &Path) -> Result<SessionManifest, String> {
.find(|tool| allow.contains(**tool))
{
return Err(format!(
"origin-scoped bounded manifests cannot allow '{tool}' because it bypasses the typed browser origin adapter"
"origin-scoped capability manifests cannot allow '{tool}' because it bypasses the typed browser origin adapter"
));
}
}
let mut digest = Sha256::new();
digest.update(format!("cua-driver-session-policy-v{version}\0").as_bytes());
let digest_domain = if version == 3 {
"cua-driver-capability-manifest"
} else {
"cua-driver-session-policy"
};
digest.update(format!("{digest_domain}-v{version}\0").as_bytes());
digest.update(&bytes);
Ok(SessionManifest {
version,
sha256: format!("{:x}", digest.finalize()),
expires_unix_ms: now_unix_ms() + expires_after.as_millis(),
expires_unix_ms: expires_after.map(|duration| now_unix_ms() + duration.as_millis()),
idle_timeout,
allow,
deny,
@ -881,12 +1000,12 @@ fn validate_tools(section: &str, tools: Vec<String>) -> Result<HashSet<String>,
== crate::authorization::RiskClass::Unclassified
{
return Err(format!(
"session policy {section}.tools contains unknown or unreviewed tool '{tool}'"
"capability manifest {section}.tools contains unknown or unreviewed tool '{tool}'"
));
}
if !validated.insert(canonical.clone()) {
return Err(format!(
"session policy {section}.tools repeats '{canonical}'"
"capability manifest {section}.tools repeats '{canonical}'"
));
}
}
@ -996,7 +1115,7 @@ fn validate_manifest_path_resources(
RawPathResource::Directory { dir, recursive } => {
if version < 2 {
return Err(format!(
"{section} directory roots require session policy version 2"
"{section} directory roots require capability manifest version 2 or later"
));
}
let canonical = validate_manifest_paths(section, vec![dir])?
@ -1200,7 +1319,7 @@ mod tests {
#[cfg(feature = "yaml")]
#[test]
fn manifest_is_deny_by_default_and_preserves_ask() {
fn legacy_ask_is_deny_and_undeclared_tools_fail_closed() {
let loaded = manifest(
r#"
version: 1
@ -1219,10 +1338,55 @@ ask:
.unwrap();
assert_eq!(loaded.decision("start_session"), ManifestDecision::Allow);
assert_eq!(loaded.decision("page"), ManifestDecision::Deny);
assert_eq!(loaded.decision("browser_download"), ManifestDecision::Ask);
assert_eq!(loaded.decision("browser_download"), ManifestDecision::Deny);
assert_eq!(loaded.decision("click"), ManifestDecision::Undeclared);
}
#[cfg(feature = "yaml")]
#[test]
fn version_three_removes_mode_and_ask_and_allows_optional_lifetimes() {
let loaded =
manifest("version: 3\nallow:\n tools: [get_config]\ndeny:\n tools: [page]\n")
.unwrap();
assert_eq!(loaded.version(), 3);
assert_eq!(loaded.expires_unix_ms(), None);
assert_eq!(loaded.idle_timeout(), None);
assert!(loaded
.validate_for_mode(crate::authorization::PermissionMode::Standard)
.is_ok());
assert!(loaded
.validate_for_mode(crate::authorization::PermissionMode::Unrestricted)
.is_ok());
assert!(loaded
.validate_for_mode(crate::authorization::PermissionMode::Bounded)
.is_err());
assert!(
manifest("version: 3\nmode: bounded\nallow:\n tools: [get_config]\n")
.unwrap_err()
.contains("must not declare mode")
);
assert!(
manifest("version: 3\nallow:\n tools: [get_config]\nask:\n tools: [page]\n")
.unwrap_err()
.contains("does not support ask.tools")
);
}
#[cfg(feature = "yaml")]
#[test]
fn version_three_lifetimes_are_enforced_when_declared() {
let loaded = manifest(
"version: 3\nexpires_after: 1h\nidle_timeout: 5m\nallow:\n tools: [get_config]\n",
)
.unwrap();
assert!(loaded.expires_unix_ms().is_some());
assert_eq!(loaded.idle_timeout(), Some(Duration::from_secs(5 * 60)));
assert!(loaded
.validate_for_mode(crate::authorization::PermissionMode::Bounded)
.is_ok());
}
#[cfg(feature = "yaml")]
#[test]
fn autonomous_manifest_mode_remains_a_compatibility_alias() {
@ -1652,7 +1816,11 @@ allow:
#[cfg(feature = "yaml")]
#[test]
fn manifest_file_resources_reject_filesystem_roots() {
let root = std::path::Path::new(std::path::MAIN_SEPARATOR_STR)
let current_dir = std::env::current_dir().unwrap();
let root = current_dir
.ancestors()
.last()
.expect("an absolute current directory has a filesystem root")
.to_string_lossy()
.into_owned();
let error = validate_manifest_paths("files.write", vec![root]).unwrap_err();
@ -1690,9 +1858,19 @@ allow:
.unwrap();
*loaded.last_authorized_dispatch.lock().unwrap() =
Instant::now().checked_sub(Duration::from_secs(2)).unwrap();
assert!(loaded.authorize_dispatch().is_err());
for mode in [
crate::authorization::PermissionMode::Standard,
crate::authorization::PermissionMode::Bounded,
crate::authorization::PermissionMode::Unrestricted,
] {
assert!(
loaded.validate_for_mode(mode).is_err(),
"declared idle timeout must be enforced in {mode:?}"
);
}
assert!(loaded.commit_authorized_dispatch().is_err());
assert!(loaded.is_idle_expired());
assert!(loaded.authorize_dispatch().is_err());
assert!(loaded.commit_authorized_dispatch().is_err());
for bypass_tool in ["page", "verify_state"] {
let policy = format!(

551
packages/cua-driver/rust/crates/cua-driver-core/src/session_tools.rs
View file

@ -1,11 +1,10 @@
//! Session lifecycle and per-session capture-scope tools.
//!
//! A session is a **caller-declared** identity for an agent run (see
//! [`crate::session`]). It owns the agent cursor and is the key for
//! per-session config + recording. These tools bookend a run's lifetime
//! explicitly, decoupled from the MCP connection: the same `session` id works
//! identically over MCP, the CLI (`--session`), or the raw socket, and a run
//! can span any number of apps/windows.
//! A session is lifecycle identity for an agent run (see [`crate::session`]).
//! It owns the agent cursor, per-run configuration, recording, cleanup, and
//! telemetry. A trusted transport lease receives one implicit session when a
//! session-requiring call omits a public label; explicit labels remain an
//! optional coordination mechanism inside the same trusted namespace.
//!
//! The daemon may mirror an explicit `session` arg into reserved transport
//! fields, but lifecycle and policy tools accept only the public `session`
@ -17,20 +16,95 @@ use crate::tool::{Tool, ToolDef};
use crate::tool_args::parse_typed_input;
use async_trait::async_trait;
use cua_driver_contract::{
CaptureScope, EndSessionInput, EscalateSessionInput, EscalationReason, GetSessionStateInput,
StartSessionInput,
CaptureScope, EndSessionInput, EscalateSessionInput, EscalationReason, GetSessionInput,
GetSessionStateInput, ListSessionsInput, ListSessionsOutput, SessionClientKindOutput,
SessionLifecycleState, SessionOutput, SessionTransportOutput, StartSessionInput,
};
use serde_json::{json, Value};
use std::sync::OnceLock;
/// Read the public, caller-declared session id. Capture-scope policy must never
/// bind to the daemon's reserved `_session_id` transport mirror.
/// Resolve the runtime-private lifecycle id injected at the trusted registry
/// boundary. An explicit public label wins; otherwise the transport lease's
/// implicit id is used.
fn session_id_of(args: &Value) -> Option<String> {
args.as_object()?
.get("session")
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty() && *s != "default")
.map(|s| s.to_owned())
let args = args.as_object()?;
["session", "_session_id"].into_iter().find_map(|key| {
args.get(key)
.and_then(Value::as_str)
.filter(|s| !s.is_empty() && *s != "default")
.map(str::to_owned)
})
}
fn public_label_of(args: &Value) -> Option<String> {
args.get("_public_session_label")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
}
fn response_session_label(args: &Value, session_id: &str) -> String {
public_label_of(args).unwrap_or_else(|| {
if args
.get("session")
.and_then(Value::as_str)
.is_some_and(|value| !value.is_empty() && value != "default")
{
crate::session::public_session_label(session_id).to_owned()
} else {
// Never expose a transport UUID or runtime-private key merely
// because the caller used its unnamed implicit session.
"implicit".to_owned()
}
})
}
fn owner_transport_of(args: &Value, session_id: &str) -> String {
args.get("_transport_session_id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.unwrap_or(session_id)
.to_owned()
}
fn lifecycle_idle_ttl_of(args: &Value) -> Option<std::time::Duration> {
args.get("_session_idle_ttl_ms")
.and_then(Value::as_u64)
.filter(|milliseconds| *milliseconds > 0)
.map(std::time::Duration::from_millis)
}
fn lifecycle_output(snapshot: crate::session::LifecycleSessionSnapshot) -> SessionOutput {
let cursor_visible = crate::session::cursor_visible(&snapshot.runtime_id);
let recording_active = crate::session::recording_active(&snapshot.runtime_id);
SessionOutput {
session: snapshot.public_label,
implicit: snapshot.implicit,
state: if snapshot.ending {
SessionLifecycleState::Ending
} else {
SessionLifecycleState::Active
},
client_kind: match snapshot.client_kind {
crate::session::SessionClientKind::Cli => SessionClientKindOutput::Cli,
crate::session::SessionClientKind::Direct => SessionClientKindOutput::Direct,
crate::session::SessionClientKind::Mcp => SessionClientKindOutput::Mcp,
crate::session::SessionClientKind::PythonSdk => SessionClientKindOutput::PythonSdk,
crate::session::SessionClientKind::TypescriptSdk => {
SessionClientKindOutput::TypescriptSdk
}
},
transport: match snapshot.transport {
crate::session::SessionTransport::Cli => SessionTransportOutput::Cli,
crate::session::SessionTransport::Daemon => SessionTransportOutput::Daemon,
crate::session::SessionTransport::McpStdio => SessionTransportOutput::McpStdio,
crate::session::SessionTransport::McpHttp => SessionTransportOutput::McpHttp,
},
cursor_visible,
recording_active,
idle_seconds: snapshot.idle.as_secs(),
expires_in_seconds: snapshot.expires_in.as_secs(),
}
}
// ── start_session ─────────────────────────────────────────────────────────────
@ -47,7 +121,7 @@ impl Tool for StartSessionTool {
async fn invoke(&self, args: Value) -> ToolResult {
let Some(id) = session_id_of(&args) else {
return ToolResult::error("start_session requires a non-empty `session` id.");
return ToolResult::error("start_session requires an authenticated transport lease or a non-empty `session` label.");
};
if let Some(raw) = args.get("capture_scope") {
let Some(raw) = raw.as_str() else {
@ -66,15 +140,50 @@ impl Tool for StartSessionTool {
}));
}
}
let input = match parse_typed_input::<StartSessionInput>("start_session", args) {
let input = match parse_typed_input::<StartSessionInput>("start_session", args.clone()) {
Ok(input) => input,
Err(result) => return result,
};
let requested = input.capture_scope;
let was_ended = crate::session::is_session_ended(&id);
if !was_ended {
let legacy_capture = requested.is_some();
let public_label = public_label_of(&args);
let owner = owner_transport_of(&args, &id);
let (transport, client_kind) = crate::session::infer_transport_metadata(&owner);
if let Some(requested) = requested {
if let Some(existing) = get_session(&id) {
if existing.policy != requested {
return ToolResult::error(
"session already uses a different deprecated capture policy",
)
.with_structured(json!({
"code": "session_policy_conflict",
"capture_scope": existing.policy.as_str(),
"requested_capture_scope": requested.as_str(),
}));
}
}
}
// Tombstone removal and owner binding are one atomic lifecycle
// transition. A guessed public label cannot be claimed in the gap
// between revival and activation.
let revived = match crate::session::activate_or_revive_session_for_owner(
&id,
public_label.as_deref(),
&owner,
public_label.is_none(),
transport,
client_kind,
lifecycle_idle_ttl_of(&args),
) {
Ok(revived) => revived,
Err(_) => {
return ToolResult::error("session is not available to this transport")
.with_structured(json!({ "code": "session_unavailable" }));
}
};
let scope = if legacy_capture {
match bind_session(&id, requested) {
Ok(_) => {}
Ok((bound, _)) => bound,
Err(BindError::Conflict {
existing,
requested,
@ -90,52 +199,35 @@ impl Tool for StartSessionTool {
}));
}
Err(BindError::Ended) => {
return ToolResult::error(format!(
"session '{id}' ended concurrently; retry start_session to revive it"
))
.with_structured(json!({ "code": "session_ended", "session": id }));
return ToolResult::error(format!("session '{id}' could not be revived"))
.with_structured(json!({ "code": "session_ended", "session": id }));
}
}
}
// Revive a recycled id: if this session was previously ended (explicit
// `end_session` / idle-TTL / connection EOF), clear its tombstone so its
// actions stop being rejected by the daemon's resurrection guard.
// Re-declaring a session is the EXPLICIT, caller-driven way to reuse an
// id — a stray late action still can't silently resurrect a dead one.
let revived = crate::session::revive_session(&id);
let (scope, _) = match bind_session(&id, requested) {
Ok(bound) => bound,
Err(BindError::Conflict {
existing,
requested,
}) => {
return ToolResult::error(format!(
"session '{id}' already uses capture_scope='{existing}'; capture scope is immutable until the session ends"
))
.with_structured(json!({
"code": "session_policy_conflict",
"session": id,
"capture_scope": existing.as_str(),
"requested_capture_scope": requested.as_str(),
}));
}
Err(BindError::Ended) => {
return ToolResult::error(format!("session '{id}' could not be revived"))
.with_structured(json!({ "code": "session_ended", "session": id }));
}
} else {
get_session(&id).unwrap_or_else(|| {
crate::capture_scope::SessionCaptureScope::new(CaptureScope::Auto)
})
};
// Refresh (or begin) the session's idle-TTL clock. The cursor appears on
// the first action carrying this `session`.
crate::session::touch_session(&id);
// Platform overlays keep their own late-command tombstones. Clear
// those only after core revival and capture-policy binding both
// succeeded, so a failed declaration cannot revive render state.
if revived {
crate::session::fire_session_revive_for_owner(&id, &owner);
}
let structured = serde_json::to_value(cua_driver_contract::StartSessionOutput {
state: scope.output(&id),
state: scope.output(&response_session_label(&args, &id)),
active: true,
revived,
})
.expect("start_session output serializes");
ToolResult::text(format!(
"✅ Session '{id}' is active with capture_scope='{}'.",
scope.policy
"✅ Session '{}' is active{}.",
public_label.as_deref().unwrap_or("implicit"),
if legacy_capture {
" with deprecated capture_scope compatibility"
} else {
""
}
))
.with_structured(structured)
}
@ -167,10 +259,11 @@ impl Tool for EscalateSessionTool {
return ToolResult::error("escalate_session.reason is invalid.")
.with_structured(json!({ "code": "invalid_escalation_reason" }));
}
let input = match parse_typed_input::<EscalateSessionInput>("escalate_session", args) {
Ok(input) => input,
Err(result) => return result,
};
let input =
match parse_typed_input::<EscalateSessionInput>("escalate_session", args.clone()) {
Ok(input) => input,
Err(result) => return result,
};
if input
.detail
.as_ref()
@ -179,9 +272,16 @@ impl Tool for EscalateSessionTool {
return ToolResult::error("escalate_session.detail must be at most 200 characters.")
.with_structured(json!({ "code": "invalid_escalation_detail" }));
}
let owner = owner_transport_of(&args, &id);
if crate::session::session_snapshot(&id, &owner, crate::session::DEFAULT_SESSION_IDLE_TTL)
.is_none()
{
return ToolResult::error("session is not visible to this transport")
.with_structured(json!({ "code": "session_not_started" }));
}
match escalate_session(&id, input.reason, input.detail.as_deref()) {
Ok(state) => ToolResult::text(format!("✅ Session '{id}' escalated to desktop scope."))
.with_structured(state.as_json(&id)),
Ok(state) => ToolResult::text("✅ Session escalated to desktop scope.")
.with_structured(state.as_json(&response_session_label(&args, &id))),
Err(error) => {
let (code, message) = match error {
EscalateError::Ended => (
@ -227,19 +327,119 @@ impl Tool for GetSessionStateTool {
return ToolResult::error("get_session_state requires a public `session` id.")
.with_structured(json!({ "code": "session_required" }));
};
if let Err(result) = parse_typed_input::<GetSessionStateInput>("get_session_state", args) {
if let Err(result) =
parse_typed_input::<GetSessionStateInput>("get_session_state", args.clone())
{
return result;
}
let Some(state) = get_session(&id) else {
return ToolResult::error(format!("session '{id}' is not active"))
.with_structured(json!({ "code": "session_not_started", "session": id }));
};
let owner = owner_transport_of(&args, &id);
if crate::session::session_snapshot(&id, &owner, crate::session::DEFAULT_SESSION_IDLE_TTL)
.is_none()
{
return ToolResult::error("session is not visible to this transport")
.with_structured(json!({ "code": "session_not_started" }));
}
let state = get_session(&id)
.unwrap_or_else(|| crate::capture_scope::SessionCaptureScope::new(CaptureScope::Auto));
ToolResult::text(format!(
"Session '{id}' uses capture_scope='{}' (effective_scope='{}').",
state.policy,
state.effective_scope().as_str()
))
.with_structured(state.as_json(&id))
.with_structured(state.as_json(&response_session_label(&args, &id)))
}
}
// ── get_session / list_sessions ─────────────────────────────────────────────
pub struct GetSessionTool;
static GET_DEF: OnceLock<ToolDef> = OnceLock::new();
#[async_trait]
impl Tool for GetSessionTool {
fn def(&self) -> &ToolDef {
GET_DEF.get_or_init(|| session_tool_def("get_session"))
}
async fn invoke(&self, args: Value) -> ToolResult {
if let Err(result) = parse_typed_input::<GetSessionInput>("get_session", args.clone()) {
return result;
}
let Some(id) = session_id_of(&args) else {
return ToolResult::error("no implicit session is attached to this transport")
.with_structured(json!({ "code": "session_not_started" }));
};
let owner = owner_transport_of(&args, &id);
let Some(snapshot) =
crate::session::session_snapshot(&id, &owner, crate::session::DEFAULT_SESSION_IDLE_TTL)
else {
return ToolResult::error("session is not visible to this transport")
.with_structured(json!({ "code": "session_not_started" }));
};
let output = lifecycle_output(snapshot);
ToolResult::text("Session is active.")
.with_structured(serde_json::to_value(output).expect("get_session output serializes"))
}
}
pub struct ListSessionsTool;
static LIST_DEF: OnceLock<ToolDef> = OnceLock::new();
#[async_trait]
impl Tool for ListSessionsTool {
fn def(&self) -> &ToolDef {
LIST_DEF.get_or_init(|| session_tool_def("list_sessions"))
}
async fn invoke(&self, args: Value) -> ToolResult {
let input = match parse_typed_input::<ListSessionsInput>("list_sessions", args.clone()) {
Ok(input) => input,
Err(result) => return result,
};
let Some(owner) = args
.get("_transport_session_id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
else {
return ToolResult::error("list_sessions requires an authenticated transport lease")
.with_structured(json!({ "code": "transport_lease_required" }));
};
let limit = input.limit.unwrap_or(50);
if limit == 0 || limit > 100 {
return ToolResult::error("list_sessions.limit must be from 1 to 100")
.with_structured(json!({ "code": "invalid_limit" }));
}
let offset = match input.cursor.as_deref() {
None => 0,
Some(cursor) => match cursor
.strip_prefix("o:")
.and_then(|v| v.parse::<usize>().ok())
{
Some(offset) => offset,
None => {
return ToolResult::error("list_sessions.cursor is invalid")
.with_structured(json!({ "code": "invalid_cursor" }))
}
},
};
let sessions =
crate::session::list_session_snapshots(owner, crate::session::DEFAULT_SESSION_IDLE_TTL);
let end = (offset + limit as usize).min(sessions.len());
let page = sessions
.get(offset..end)
.unwrap_or_default()
.iter()
.cloned()
.map(lifecycle_output)
.collect::<Vec<_>>();
let output = ListSessionsOutput {
sessions: page,
next_cursor: (end < sessions.len()).then(|| format!("o:{end}")),
};
ToolResult::text(format!("{} visible session(s).", output.sessions.len()))
.with_structured(serde_json::to_value(output).expect("list_sessions output serializes"))
}
}
@ -257,15 +457,51 @@ impl Tool for EndSessionTool {
async fn invoke(&self, args: Value) -> ToolResult {
let Some(id) = session_id_of(&args) else {
return ToolResult::error("end_session requires a non-empty `session` id.");
return ToolResult::error(
"end_session requires an attached implicit session or a non-empty `session` label.",
);
};
if let Err(result) = parse_typed_input::<EndSessionInput>("end_session", args) {
if let Err(result) = parse_typed_input::<EndSessionInput>("end_session", args.clone()) {
return result;
}
crate::session::end_session(&id);
ToolResult::text(format!("✅ Session '{id}' ended.")).with_structured(
let owner = owner_transport_of(&args, &id);
let owned = crate::session::end_session_for_owner(&id, &owner);
if owned {
let cleanup = if crate::session::is_session_ended(&id) {
crate::session::session_cleanup_status(&id)
} else {
crate::session::SessionCleanupReport {
complete: false,
in_progress: true,
failures: Vec::new(),
}
};
if !cleanup.complete {
return ToolResult::error(if cleanup.in_progress {
"Session is ending after its in-flight action completes; retry end_session for final cleanup status."
} else {
"Session ended, but one or more cleanup hooks failed; retry end_session."
})
.with_structured(json!({
"code": if cleanup.in_progress { "session_cleanup_pending" } else { "session_cleanup_partial" },
"session": public_label_of(&args)
.unwrap_or_else(|| crate::session::public_session_label(&id).to_owned()),
"cleanup_complete": false,
"cleanup_in_progress": cleanup.in_progress,
// Hook names are stable, non-sensitive categories. Keep
// implementation errors private because adapters may
// include local paths or platform details in them.
"failed_hooks": cleanup.failures.into_iter().map(|failure| json!({
"hook": failure.hook,
"error": "cleanup failed",
})).collect::<Vec<_>>(),
}));
}
}
let response_label = response_session_label(&args, &id);
ToolResult::text(format!("✅ Session '{response_label}' ended.")).with_structured(
serde_json::to_value(cua_driver_contract::EndSessionOutput {
session: id,
session: response_label,
active: false,
})
.expect("end_session output serializes"),
@ -282,27 +518,57 @@ fn session_tool_def(name: &str) -> ToolDef {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{
atomic::{AtomicUsize, Ordering},
Arc,
};
fn result_code(result: &ToolResult) -> Option<&str> {
result.structured_content.as_ref()?.get("code")?.as_str()
}
#[test]
fn session_id_of_reads_only_public_session() {
fn session_id_of_prefers_public_and_uses_trusted_implicit_fallback() {
assert_eq!(
session_id_of(&json!({ "session": "a" })).as_deref(),
Some("a")
);
assert_eq!(session_id_of(&json!({ "_session_id": "b" })), None);
assert_eq!(
session_id_of(&json!({ "session": "", "_session_id": "c" })),
None
session_id_of(&json!({ "_session_id": "b" })).as_deref(),
Some("b")
);
assert_eq!(
session_id_of(&json!({ "session": "", "_session_id": "c" })).as_deref(),
Some("c")
);
assert_eq!(session_id_of(&json!({})), None);
assert_eq!(session_id_of(&json!({ "session": "" })), None);
assert_eq!(session_id_of(&json!({ "session": "default" })), None);
}
#[tokio::test]
async fn implicit_start_does_not_expose_its_private_transport_identity() {
let id = format!("private-implicit-session-{}", std::process::id());
let owner = format!("private-transport-{}", std::process::id());
let result = StartSessionTool
.invoke(json!({
"_session_id": id.clone(),
"_transport_session_id": owner.clone(),
}))
.await;
assert_ne!(result.is_error, Some(true));
let structured = result.structured_content.as_ref().unwrap();
assert_eq!(structured["session"], "implicit");
let encoded = serde_json::to_string(structured).unwrap();
assert!(!encoded.contains(&owner));
EndSessionTool
.invoke(json!({
"_session_id": id,
"_transport_session_id": owner,
}))
.await;
}
#[tokio::test]
async fn live_policy_is_immutable_and_ended_id_gets_fresh_policy() {
let id = format!("session-tool-policy-{}", std::process::id());
@ -334,10 +600,137 @@ mod tests {
assert_eq!(structured["revived"], true);
}
#[tokio::test]
async fn successful_explicit_revival_notifies_session_owned_subsystems_once() {
let id = format!("session-tool-revive-hook-{}", std::process::id());
let notifications = Arc::new(AtomicUsize::new(0));
let observed_id = id.clone();
let notifications_for_hook = notifications.clone();
let _registration = crate::session::register_scoped_session_revive_hook(move |got| {
if got == observed_id {
notifications_for_hook.fetch_add(1, Ordering::SeqCst);
}
});
let start = StartSessionTool;
let fresh = start
.invoke(json!({"session": id, "capture_scope": "window"}))
.await;
assert_ne!(fresh.is_error, Some(true));
assert_eq!(notifications.load(Ordering::SeqCst), 0);
EndSessionTool.invoke(json!({"session": id})).await;
let revived = start
.invoke(json!({"session": id, "capture_scope": "desktop"}))
.await;
assert_ne!(revived.is_error, Some(true));
assert_eq!(
revived.structured_content.as_ref().unwrap()["revived"],
true
);
assert_eq!(notifications.load(Ordering::SeqCst), 1);
let idempotent = start
.invoke(json!({"session": id, "capture_scope": "desktop"}))
.await;
assert_ne!(idempotent.is_error, Some(true));
assert_eq!(notifications.load(Ordering::SeqCst), 1);
EndSessionTool.invoke(json!({"session": id})).await;
}
#[tokio::test]
async fn cleanup_failure_response_redacts_adapter_error_details() {
let id = format!("session-tool-cleanup-redaction-{}", std::process::id());
let calls = Arc::new(AtomicUsize::new(0));
let calls_for_hook = calls.clone();
let observed_id = id.clone();
let _registration =
crate::session::register_scoped_fallible_session_end_hook("recording", move |ended| {
if ended != observed_id {
return Ok(());
}
let call = calls_for_hook.fetch_add(1, Ordering::SeqCst);
if call < 2 {
Err("failed at /Users/private/recording.mov".into())
} else {
Ok(())
}
});
let started = StartSessionTool.invoke(json!({"session": id})).await;
assert_ne!(started.is_error, Some(true));
let ended = EndSessionTool.invoke(json!({"session": id})).await;
assert_eq!(ended.is_error, Some(true));
assert_eq!(result_code(&ended), Some("session_cleanup_partial"));
let encoded = serde_json::to_string(&ended.structured_content).unwrap();
assert!(encoded.contains("cleanup failed"));
assert!(!encoded.contains("/Users/private"));
let second = EndSessionTool.invoke(json!({"session": id})).await;
assert_eq!(second.is_error, Some(true));
assert_eq!(calls.load(Ordering::SeqCst), 2);
let third = EndSessionTool.invoke(json!({"session": id})).await;
assert_ne!(third.is_error, Some(true));
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn lifecycle_tools_do_not_expose_or_mutate_foreign_transport_sessions() {
let id = format!("session-tool-owner-scope-{}", std::process::id());
let owner_a = format!("http-owner-a-{}", std::process::id());
let owner_b = format!("http-owner-b-{}", std::process::id());
let owned_args = json!({
"session": id,
"_public_session_label": "shared-label",
"_transport_session_id": owner_a,
"capture_scope": "auto"
});
let started = StartSessionTool.invoke(owned_args.clone()).await;
assert_ne!(started.is_error, Some(true));
let foreign = json!({
"session": id,
"_public_session_label": "shared-label",
"_transport_session_id": owner_b
});
let get = GetSessionTool.invoke(foreign.clone()).await;
assert_eq!(get.is_error, Some(true));
assert_eq!(result_code(&get), Some("session_not_started"));
let state = GetSessionStateTool.invoke(foreign.clone()).await;
assert_eq!(state.is_error, Some(true));
assert_eq!(result_code(&state), Some("session_not_started"));
let list = ListSessionsTool
.invoke(json!({"_transport_session_id": owner_b}))
.await;
assert_eq!(
list.structured_content.as_ref().unwrap()["sessions"],
json!([])
);
let ended = EndSessionTool.invoke(foreign.clone()).await;
assert_ne!(ended.is_error, Some(true), "end remains non-enumerating");
assert!(crate::session::session_snapshot(
&id,
&owner_a,
crate::session::DEFAULT_SESSION_IDLE_TTL
)
.is_some());
let stolen = StartSessionTool.invoke(foreign).await;
assert_eq!(stolen.is_error, Some(true));
assert_eq!(result_code(&stolen), Some("session_unavailable"));
EndSessionTool.invoke(owned_args).await;
}
#[tokio::test]
async fn auto_requires_explicit_bounded_escalation() {
let id = format!("session-tool-auto-{}", std::process::id());
let started = StartSessionTool.invoke(json!({"session": id})).await;
let started = StartSessionTool
.invoke(json!({"session": id, "capture_scope": "auto"}))
.await;
assert_eq!(
started.structured_content.as_ref().unwrap()["capture_scope"],
"auto"

905
packages/cua-driver/rust/crates/cua-driver-core/src/tool.rs
View file

File diff suppressed because it is too large Load diff

95
packages/cua-driver/rust/crates/cua-driver-core/src/tool_schema.rs
View file

@ -15,26 +15,34 @@
//! required-set spec. Each platform calls it from a test against its own
//! registry, so CI fails the moment a platform drifts.
//!
//! Descriptions are intentionally NOT compared — prose can legitimately vary
//! Descriptions are generally not compared because prose can legitimately vary
//! per tool (a `delivery_mode` blurb mentioning "PIXEL click" vs "AX insert").
//! The gate enforces *shape* (`type` / `enum` / `items`), which is what governs
//! client compatibility.
//! The `session` guidance is the exception: action tools must tell callers that
//! public labels are preferred for multi-call work and must be repeated on each
//! call. Lifecycle-management tools keep their resource-specific wording.
use serde_json::{json, Value};
// ── Fragments ────────────────────────────────────────────────────────────────
/// `session` — the per-run agent-cursor identity. Uniform across every tool.
/// `session` is an optional public lifecycle label. Uniform across every tool.
pub fn session_schema() -> Value {
json!({
"type": "string",
"description": "Optional session id: declares/uses the agent cursor and \
per-session state for this run. The same id works over MCP, the CLI, \
or the raw socket, and follows the run across apps/windows. Omit to \
run cursor-less."
"description": cua_driver_contract::MULTI_CALL_SESSION_DESCRIPTION
})
}
/// Canonical lifecycle guidance plus a tool-specific ownership or precedence note.
pub fn session_schema_with(note: &str) -> Value {
let mut schema = session_schema();
schema["description"] = Value::String(format!(
"{} {note}",
cua_driver_contract::MULTI_CALL_SESSION_DESCRIPTION
));
schema
}
/// `delivery_mode` — the best-effort-background ladder rung. The prose varies by
/// tool (the surface it injects through differs), so callers may pass their own
/// `description`; the shape is fixed here.
@ -171,6 +179,33 @@ fn structural(schema: &Value) -> Value {
Value::Object(out)
}
fn session_guidance_is_exempt(tool_name: &str) -> bool {
matches!(
tool_name,
"start_session"
| "end_session"
| "get_session"
| "get_session_state"
| "list_sessions"
| "escalate_session"
| "set_agent_cursor_enabled"
| "set_agent_cursor_motion"
| "set_agent_cursor_theme"
| "get_agent_cursor_state"
)
}
fn session_description_has_multi_call_guidance(schema: &Value) -> bool {
schema
.get("description")
.and_then(Value::as_str)
.map(|description| description.split_whitespace().collect::<Vec<_>>().join(" "))
.is_some_and(|description| {
description.contains("prefer a short public session label")
&& description.contains("repeat it on every call that accepts it")
})
}
/// Check one tool's `input_schema` against the shared canon. Returns a list of
/// human-readable violation strings (empty == consistent). Each platform calls
/// this for every tool in its registry from a test.
@ -188,6 +223,15 @@ pub fn shared_schema_violations(tool_name: &str, input_schema: &Value) -> Vec<St
));
}
}
if pname == "session"
&& !session_guidance_is_exempt(tool_name)
&& !session_description_has_multi_call_guidance(pschema)
{
violations.push(format!(
"{tool_name}.session: description must prefer a short public session label \
for multi-call work and tell callers to repeat it on every accepting call"
));
}
}
}
@ -227,6 +271,41 @@ mod tests {
assert_eq!(s, json!({ "type": "string", "enum": ["a", "b"] }));
}
#[test]
fn session_description_prefers_named_multi_call_runs() {
let schema = session_schema();
let description = schema["description"]
.as_str()
.expect("session schema should carry agent guidance");
assert!(description.contains("prefer a short public session label"));
assert!(description.contains("repeat it on every call that accepts it"));
assert!(description.contains("implicit lifecycle session"));
}
#[test]
fn action_session_description_without_repeat_guidance_is_flagged() {
let tool = json!({
"type": "object",
"properties": {
"session": { "type": "string", "description": "Optional session id." }
}
});
let violations = shared_schema_violations("clipboard_write", &tool);
assert_eq!(violations.len(), 1);
assert!(violations[0].contains("repeat it on every accepting call"));
}
#[test]
fn lifecycle_session_tools_keep_resource_specific_descriptions() {
let tool = json!({
"type": "object",
"properties": {
"session": { "type": "string", "description": "Public label to end." }
}
});
assert!(shared_schema_violations("end_session", &tool).is_empty());
}
#[test]
fn matching_param_with_different_prose_passes() {
// Same shape, different description → no violation.

85
packages/cua-driver/rust/crates/cua-driver-core/src/window_inspection.rs
View file

@ -1,39 +1,55 @@
//! Cross-platform window-snapshot coverage signals.
//!
//! A browser permission bubble is browser chrome, not page content. Chromium
//! may composite that chrome in a popup/child surface which is visible in a
//! desktop capture but absent from a capture of the requested native window.
//! A single-window backend therefore cannot prove that no browser-owned blocker
//! exists. Keep that limitation explicit and machine-readable.
//! may composite that chrome in the requested window or in a popup/child
//! surface which is visible only in a desktop capture. A single-window backend
//! therefore cannot always prove that no browser-owned blocker exists. Keep
//! that platform-specific limitation explicit and machine-readable.
use serde_json::{json, Value};
pub const BROWSER_CHROME_COVERAGE_STATUS: &str = "not_observable_in_window_scope";
pub const BROWSER_CHROME_MAY_BE_INCOMPLETE_STATUS: &str = "may_be_incomplete_in_window_scope";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum BrowserChromeCaptureCoverage {
NotObservable,
MayBeIncomplete,
}
impl BrowserChromeCaptureCoverage {
fn status(self) -> &'static str {
match self {
Self::NotObservable => BROWSER_CHROME_COVERAGE_STATUS,
Self::MayBeIncomplete => BROWSER_CHROME_MAY_BE_INCOMPLETE_STATUS,
}
}
}
/// Describe the browser-chrome coverage limit of a Chromium-family window
/// snapshot.
///
/// This deliberately does **not** claim that a prompt is present. Presence is
/// not observable from a single native-window surface on every supported
/// platform. It also carries no prompt text or choices, so routine traces can
/// retain the recovery signal without retaining permission content.
pub fn mark_browser_chrome_capture_coverage(structured: &mut Value, chromium_family_window: bool) {
if !chromium_family_window {
return;
}
/// not reliably complete from a single native-window surface on every
/// supported platform. It also carries no prompt text or choices, so routine
/// traces can retain the recovery signal without retaining permission content.
pub fn mark_browser_chrome_capture_coverage(
structured: &mut Value,
coverage: Option<BrowserChromeCaptureCoverage>,
) {
let Some(coverage) = coverage else { return };
structured["capture_coverage"] = json!({
"browser_chrome": {
"status": BROWSER_CHROME_COVERAGE_STATUS
"status": coverage.status()
},
"recovery": {
"when": "verified_window_action_ineffective",
"escalate": {
"tool": "escalate_session",
"reason": "foreground_ineffective"
},
"inspect": "get_desktop_state",
"act_scope": "desktop",
"act_target": {
"kind": "desktop",
"display_id": "primary"
},
"verify": "get_desktop_state"
}
});
@ -54,8 +70,14 @@ mod tests {
});
let mut after_visible_browser_blocker = before.clone();
mark_browser_chrome_capture_coverage(&mut before, true);
mark_browser_chrome_capture_coverage(&mut after_visible_browser_blocker, true);
mark_browser_chrome_capture_coverage(
&mut before,
Some(BrowserChromeCaptureCoverage::NotObservable),
);
mark_browser_chrome_capture_coverage(
&mut after_visible_browser_blocker,
Some(BrowserChromeCaptureCoverage::NotObservable),
);
// Even when the page-owned state is byte-for-byte unchanged, callers
// receive an explicit recovery branch instead of treating the window
@ -69,17 +91,17 @@ mod tests {
before["capture_coverage"]["recovery"]["when"],
"verified_window_action_ineffective"
);
assert_eq!(
before["capture_coverage"]["recovery"]["escalate"],
json!({
"tool": "escalate_session",
"reason": "foreground_ineffective"
})
);
assert!(before["capture_coverage"]["recovery"]
.get("escalate")
.is_none());
assert_eq!(
before["capture_coverage"]["recovery"]["inspect"],
"get_desktop_state"
);
assert_eq!(
before["capture_coverage"]["recovery"]["act_target"],
json!({"kind": "desktop", "display_id": "primary"})
);
assert!(before.get("browser_chrome_prompt").is_none());
}
@ -87,11 +109,18 @@ mod tests {
fn signal_is_privacy_minimal_and_does_not_change_non_browser_snapshots() {
let mut ordinary = json!({"window_id": 9, "pid": 3});
let unchanged = ordinary.clone();
mark_browser_chrome_capture_coverage(&mut ordinary, false);
mark_browser_chrome_capture_coverage(&mut ordinary, None);
assert_eq!(ordinary, unchanged);
let mut browser = unchanged;
mark_browser_chrome_capture_coverage(&mut browser, true);
mark_browser_chrome_capture_coverage(
&mut browser,
Some(BrowserChromeCaptureCoverage::MayBeIncomplete),
);
assert_eq!(
browser["capture_coverage"]["browser_chrome"]["status"],
BROWSER_CHROME_MAY_BE_INCOMPLETE_STATUS
);
let public = browser.to_string();
for sensitive_key in ["text", "message", "choice", "allow", "deny"] {
assert!(

106
packages/cua-driver/rust/crates/cua-driver-sdk/src/abi.rs
View file

@ -9,11 +9,13 @@ use crate::runtime::{DriverRuntime, RuntimeCreateError, RuntimeOptions, RuntimeS
use crate::{DriverError, DriverMetadata};
use cua_driver_core::{
authorization::{
PermissionMode, DANGEROUS_BYPASS_ENV, DISABLE_UNRESTRICTED_ENV,
LEGACY_EXISTING_PROFILE_APPROVAL_ENV, PERMISSION_MODE_ENV,
PermissionMode, DANGEROUS_BYPASS_ENV, DISABLE_UNRESTRICTED_ENV, PERMISSION_MODE_ENV,
},
session_authorization::{DelegatedSessionRequest, SessionModeCeiling},
session_manifest::{load_manifest, SESSION_POLICY_APPROVED_ENV, SESSION_POLICY_FILE_ENV},
session_manifest::{
load_manifest, CAPABILITY_MANIFEST_APPROVED_ENV, CAPABILITY_MANIFEST_FILE_ENV,
SESSION_POLICY_APPROVED_ENV, SESSION_POLICY_FILE_ENV,
},
};
use serde::Deserialize;
use serde_json::Value;
@ -176,12 +178,34 @@ struct AbiDriverOptions {
struct AbiRuntimeAuthorizationOptions {
allowed_modes: Vec<PermissionMode>,
compatibility_mode: PermissionMode,
#[serde(default)]
compatibility_capability_manifest_path: Option<String>,
#[serde(default)]
compatibility_bounded_manifest_path: Option<String>,
unrestricted_acknowledged: bool,
max_session_ttl_seconds: u64,
max_idle_ttl_seconds: u64,
}
impl AbiRuntimeAuthorizationOptions {
fn capability_manifest_path(&self) -> Result<Option<&str>, AbiFailure> {
if self.compatibility_capability_manifest_path.is_some()
&& self.compatibility_bounded_manifest_path.is_some()
&& self.compatibility_capability_manifest_path
!= self.compatibility_bounded_manifest_path
{
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"compatibility capability manifest path conflicts with its deprecated bounded-manifest alias",
));
}
Ok(self
.compatibility_capability_manifest_path
.as_deref()
.or(self.compatibility_bounded_manifest_path.as_deref()))
}
}
fn validate_explicit_authorization_sources(
authorization: &AbiRuntimeAuthorizationOptions,
) -> Result<(), AbiFailure> {
@ -226,24 +250,28 @@ fn validate_explicit_authorization_sources(
));
}
if environment_flag(LEGACY_EXISTING_PROFILE_APPROVAL_ENV) {
let capability_environment_manifest = std::env::var_os(CAPABILITY_MANIFEST_FILE_ENV);
let legacy_environment_manifest = std::env::var_os(SESSION_POLICY_FILE_ENV);
if capability_environment_manifest.is_some()
&& legacy_environment_manifest.is_some()
&& capability_environment_manifest != legacy_environment_manifest
{
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"explicit runtime authorization conflicts with the legacy existing-profile approval escape hatch",
"capability manifest environment path conflicts with its deprecated session-policy alias",
));
}
let environment_manifest =
std::env::var_os(SESSION_POLICY_FILE_ENV).map(std::path::PathBuf::from);
let environment_manifest = capability_environment_manifest
.or(legacy_environment_manifest)
.map(std::path::PathBuf::from);
if let Some(environment_manifest) = environment_manifest {
let Some(explicit_manifest) = authorization
.compatibility_bounded_manifest_path
.as_deref()
.capability_manifest_path()?
.map(std::path::Path::new)
else {
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"explicit runtime authorization conflicts with a compatibility session policy environment value",
"explicit runtime authorization conflicts with a compatibility capability-manifest environment value",
));
};
let environment_manifest =
@ -251,7 +279,7 @@ fn validate_explicit_authorization_sources(
AbiFailure::new(
CuaDriverStatus::InvalidArgument,
format!(
"canonicalize compatibility session policy {}: {error}",
"canonicalize compatibility capability manifest {}: {error}",
environment_manifest.display()
),
)
@ -260,7 +288,7 @@ fn validate_explicit_authorization_sources(
AbiFailure::new(
CuaDriverStatus::InvalidArgument,
format!(
"canonicalize explicit compatibility session policy {}: {error}",
"canonicalize explicit compatibility capability manifest {}: {error}",
explicit_manifest.display()
),
)
@ -268,15 +296,16 @@ fn validate_explicit_authorization_sources(
if environment_manifest != explicit_manifest {
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"explicit compatibility session policy conflicts with the trusted environment path",
"explicit compatibility capability manifest conflicts with the trusted environment path",
));
}
} else if environment_flag(SESSION_POLICY_APPROVED_ENV)
&& authorization.compatibility_bounded_manifest_path.is_none()
} else if (environment_flag(CAPABILITY_MANIFEST_APPROVED_ENV)
|| environment_flag(SESSION_POLICY_APPROVED_ENV))
&& authorization.capability_manifest_path()?.is_none()
{
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"explicit runtime authorization conflicts with a compatibility session-policy approval",
"explicit runtime authorization conflicts with a compatibility capability-manifest approval",
));
}
Ok(())
@ -294,8 +323,7 @@ fn runtime_options_from_abi(options: AbiDriverOptions) -> Result<RuntimeOptions,
)
.map_err(|error| AbiFailure::new(CuaDriverStatus::InvalidArgument, error))?;
let manifest = authorization
.compatibility_bounded_manifest_path
.as_deref()
.capability_manifest_path()?
.map(std::path::Path::new)
.map(load_manifest)
.transpose()
@ -328,7 +356,33 @@ struct AbiTrustedSessionOptions {
mode: PermissionMode,
ttl_seconds: u64,
idle_ttl_seconds: u64,
#[serde(default)]
capability_manifest_path: Option<String>,
#[serde(default)]
bounded_manifest_path: Option<String>,
/// Host-generated lifecycle lease identity. This field is accepted only
/// by the in-process Rust bridge; it is not part of the generated public
/// TrustedSessionOptions record or any agent-visible tool schema.
#[serde(default)]
transport_session: Option<String>,
}
impl AbiTrustedSessionOptions {
fn capability_manifest_path(&self) -> Result<Option<&str>, AbiFailure> {
if self.capability_manifest_path.is_some()
&& self.bounded_manifest_path.is_some()
&& self.capability_manifest_path != self.bounded_manifest_path
{
return Err(AbiFailure::new(
CuaDriverStatus::InvalidArgument,
"capability manifest path conflicts with its deprecated bounded-manifest alias",
));
}
Ok(self
.capability_manifest_path
.as_deref()
.or(self.bounded_manifest_path.as_deref()))
}
}
fn with_ffi_guard(
@ -758,27 +812,29 @@ pub unsafe extern "C" fn cua_driver_session_create_v1(
"public_session must not be empty",
));
}
let bounded_manifest = options
.bounded_manifest_path
.as_deref()
let capability_manifest = options
.capability_manifest_path()?
.map(|path| {
cua_driver_core::session_manifest::load_manifest(std::path::Path::new(path))
.map(Arc::new)
.map_err(|error| {
AbiFailure::new(
CuaDriverStatus::InvalidArgument,
format!("invalid bounded session manifest: {error}"),
format!("invalid capability manifest: {error}"),
)
})
})
.transpose()?;
let request = DelegatedSessionRequest {
public_session: options.public_session,
transport_session: format!("direct-{}", uuid::Uuid::new_v4()),
transport_session: options
.transport_session
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| format!("direct-{}", uuid::Uuid::new_v4())),
mode: options.mode,
ttl: Duration::from_secs(options.ttl_seconds),
idle_ttl: Duration::from_secs(options.idle_ttl_seconds),
bounded_manifest,
capability_manifest,
};
let session = driver
.runtime

86
packages/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs
View file

@ -43,6 +43,11 @@ pub struct EmbeddedDriverHostOptions {
pub startup_timeout_ms: Option<u64>,
pub shutdown_timeout_ms: Option<u64>,
pub permission_mode: Option<EmbeddedPermissionMode>,
#[uniffi(default = None)]
pub capability_manifest_path: Option<String>,
#[uniffi(default = false)]
pub approve_capability_manifest: bool,
/// Deprecated aliases retained for compatibility.
pub session_policy_path: Option<String>,
pub approve_session_policy: bool,
pub dangerously_bypass_approvals: bool,
@ -251,6 +256,8 @@ impl EmbeddedCuaDriverHost {
startup_timeout_ms: None,
shutdown_timeout_ms: None,
permission_mode: None,
capability_manifest_path: None,
approve_capability_manifest: false,
session_policy_path: None,
approve_session_policy: false,
dangerously_bypass_approvals: false,
@ -633,10 +640,10 @@ impl EmbeddedCuaDriverHost {
.into(),
];
if let Some(path) = &self.options.session_policy_path {
args.extend(["--session-policy".into(), path.clone()]);
args.extend(["--capability-manifest".into(), path.clone()]);
}
if self.options.approve_session_policy {
args.push("--approve-session-policy".into());
args.push("--approve-capability-manifest".into());
}
if self.options.dangerously_bypass_approvals {
args.push("--dangerously-bypass-approvals".into());
@ -732,13 +739,33 @@ fn validate_options(
let permission_mode = options
.permission_mode
.unwrap_or(EmbeddedPermissionMode::Standard);
if options.capability_manifest_path.is_some()
&& options.session_policy_path.is_some()
&& options.capability_manifest_path != options.session_policy_path
{
return configuration_error(
"capability_manifest_path conflicts with deprecated session_policy_path",
);
}
let capability_manifest_path = options
.capability_manifest_path
.clone()
.or_else(|| options.session_policy_path.clone());
let capability_manifest_approved =
options.approve_capability_manifest || options.approve_session_policy;
let manifest_configured = capability_manifest_path
.as_deref()
.is_some_and(|path| !path.trim().is_empty());
if capability_manifest_path.is_some() && !manifest_configured {
return configuration_error("capability manifest path must not be empty");
}
if manifest_configured != capability_manifest_approved {
return configuration_error(
"capability manifest path and approval acknowledgement must be supplied together",
);
}
match permission_mode {
EmbeddedPermissionMode::Standard => {
if options.session_policy_path.is_some() || options.approve_session_policy {
return configuration_error(
"session policy options are valid only in bounded mode",
);
}
if options.dangerously_bypass_approvals {
return configuration_error(
"dangerously_bypass_approvals is valid only in unrestricted mode",
@ -746,15 +773,8 @@ fn validate_options(
}
}
EmbeddedPermissionMode::Bounded => {
if options
.session_policy_path
.as_deref()
.is_none_or(|path| path.trim().is_empty())
{
return configuration_error("bounded mode requires session_policy_path");
}
if !options.approve_session_policy {
return configuration_error("bounded mode requires approve_session_policy=true");
if !manifest_configured {
return configuration_error("bounded mode requires a capability manifest");
}
if options.dangerously_bypass_approvals {
return configuration_error("bounded mode cannot bypass runtime approvals");
@ -766,11 +786,6 @@ fn validate_options(
"unrestricted mode requires dangerously_bypass_approvals=true",
);
}
if options.session_policy_path.is_some() || options.approve_session_policy {
return configuration_error(
"unrestricted mode cannot use a bounded session policy",
);
}
}
}
for variable in &options.environment {
@ -795,8 +810,8 @@ fn validate_options(
startup_timeout: Duration::from_millis(startup_timeout_ms),
shutdown_timeout: Duration::from_millis(shutdown_timeout_ms),
permission_mode,
session_policy_path: options.session_policy_path,
approve_session_policy: options.approve_session_policy,
session_policy_path: capability_manifest_path,
approve_session_policy: capability_manifest_approved,
dangerously_bypass_approvals: options.dangerously_bypass_approvals,
environment: options.environment,
inherit_stderr: options.inherit_stderr,
@ -846,9 +861,10 @@ pub(crate) fn inherited_managed_environment_name(name: &str) -> bool {
"CUA_DRIVER_PERMISSION_MODE"
| "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS"
| "CUA_DRIVER_DISABLE_UNRESTRICTED"
| "CUA_DRIVER_ALLOW_LEGACY_EXISTING_PROFILE_APPROVAL"
| "CUA_DRIVER_SESSION_POLICY_FILE"
| "CUA_DRIVER_SESSION_POLICY_APPROVED"
| "CUA_DRIVER_CAPABILITY_MANIFEST_FILE"
| "CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED"
| "CUA_DRIVER_POLICY_FILE"
| "CUA_DRIVER_MANAGED_POLICY_FILE"
)
@ -1127,6 +1143,8 @@ mod tests {
startup_timeout_ms: None,
shutdown_timeout_ms: None,
permission_mode: Some(mode),
capability_manifest_path: None,
approve_capability_manifest: false,
session_policy_path: None,
approve_session_policy: false,
dangerously_bypass_approvals: false,
@ -1139,6 +1157,11 @@ mod tests {
fn authorization_modes_require_explicit_acknowledgements() {
assert!(validate_options(options(EmbeddedPermissionMode::Standard)).is_ok());
let mut standard_manifest = options(EmbeddedPermissionMode::Standard);
standard_manifest.capability_manifest_path = Some("capabilities.yaml".into());
standard_manifest.approve_capability_manifest = true;
assert!(validate_options(standard_manifest).is_ok());
let bounded = options(EmbeddedPermissionMode::Bounded);
assert!(validate_options(bounded).is_err());
let mut bounded = options(EmbeddedPermissionMode::Bounded);
@ -1151,6 +1174,21 @@ mod tests {
let mut unrestricted = options(EmbeddedPermissionMode::Unrestricted);
unrestricted.dangerously_bypass_approvals = true;
assert!(validate_options(unrestricted).is_ok());
let mut unrestricted_manifest = options(EmbeddedPermissionMode::Unrestricted);
unrestricted_manifest.dangerously_bypass_approvals = true;
unrestricted_manifest.capability_manifest_path = Some("capabilities.yaml".into());
unrestricted_manifest.approve_capability_manifest = true;
assert!(validate_options(unrestricted_manifest).is_ok());
}
#[test]
fn capability_manifest_aliases_must_not_conflict() {
let mut options = options(EmbeddedPermissionMode::Standard);
options.capability_manifest_path = Some("capabilities-v3.yaml".into());
options.session_policy_path = Some("legacy-v2.yaml".into());
options.approve_capability_manifest = true;
assert!(validate_options(options).is_err());
}
#[test]

341
packages/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
View file

@ -6,12 +6,17 @@
//! MCP and daemon transports are downstream adapters rather than peer contracts.
use cua_driver_contract::{
ActionResult, ClickInput, ClipboardReadInput, ClipboardWriteInput, DragInput, EndSessionInput,
EndSessionOutput, EscalateSessionInput, GetAgentCursorStateInput, GetCursorPositionInput,
GetDesktopStateInput, GetScreenSizeInput, GetSessionStateInput, HotkeyInput, InvokeMenuInput,
MoveCursorInput, PressKeyInput, ScrollInput, SessionStateOutput, SetAgentCursorEnabledInput,
SetAgentCursorMotionInput, SetAgentCursorThemeInput, SetWindowFrameInput, StartSessionInput,
ActionResult, ClickInput, ClipboardReadInput, ClipboardWriteInput, DoubleClickInput, DragInput,
EndSessionInput, EndSessionOutput, EscalateSessionInput, GetAgentCursorStateInput,
GetCursorPositionInput, GetDesktopStateInput, GetScreenSizeInput, GetSessionInput,
GetSessionStateInput, GetWindowStateInput, HotkeyInput, InvokeMenuInput, ListAppsInput,
ListSessionsInput, ListSessionsOutput, ListWindowsInput, MoveCursorInput,
PerformSecondaryActionInput, PressKeyInput, RightClickInput, ScrollInput, SessionOutput,
SessionStateOutput, SetAgentCursorEnabledInput, SetAgentCursorMotionInput,
SetAgentCursorThemeInput, SetValueInput, SetWindowFrameInput, StartSessionInput,
StartSessionOutput, ToolInput, TypeTextInput, VerifyStateInput, VerifyStateOutput,
WindowClickInput, WindowDragInput, WindowHotkeyInput, WindowPressKeyInput, WindowScrollInput,
WindowTypeTextInput,
};
use cua_driver_core::daemon::{
is_daemon_listening, request_daemon_metadata, send_request, socket_path_for_namespace,
@ -43,6 +48,43 @@ use runtime::RuntimeOptions;
use service_session::ServiceSessionClient;
use worker::{ActionCompletion, PrivateWorkerClient};
fn host_sessions_json_for_prefix(runtime_prefix: &str) -> Value {
let sessions = cua_driver_core::session::list_session_snapshots_with_prefix(
runtime_prefix,
cua_driver_core::session::DEFAULT_SESSION_IDLE_TTL,
)
.into_iter()
.map(|session| {
let transport = match session.transport {
cua_driver_core::session::SessionTransport::Cli => "cli",
cua_driver_core::session::SessionTransport::Daemon => "daemon",
cua_driver_core::session::SessionTransport::McpStdio => "mcp_stdio",
cua_driver_core::session::SessionTransport::McpHttp => "mcp_http",
};
let client_kind = match session.client_kind {
cua_driver_core::session::SessionClientKind::Cli => "cli",
cua_driver_core::session::SessionClientKind::Direct => "direct",
cua_driver_core::session::SessionClientKind::Mcp => "mcp",
cua_driver_core::session::SessionClientKind::PythonSdk => "python_sdk",
cua_driver_core::session::SessionClientKind::TypescriptSdk => "typescript_sdk",
};
serde_json::json!({
"session": session.public_label.as_deref().and_then(cursor_overlay::sanitize_session_label),
"implicit": session.implicit,
"state": if session.ending { "ending" } else { "active" },
"client_kind": client_kind,
"transport": transport,
"cursor_visible": cua_driver_core::session::cursor_visible(&session.runtime_id),
"recording_active": cua_driver_core::session::recording_active(&session.runtime_id),
"started_seconds_ago": session.started_for.as_secs(),
"idle_seconds": session.idle.as_secs(),
"expires_in_seconds": session.expires_in.as_secs(),
})
})
.collect::<Vec<_>>();
serde_json::json!({"count": sessions.len(), "sessions": sessions})
}
#[derive(Debug, Clone, PartialEq, Eq, uniffi::Record)]
pub struct ImageContent {
pub mime_type: String,
@ -241,11 +283,15 @@ enum DriverBackend {
struct DaemonBackend {
socket_path: String,
transport_session: String,
}
impl DaemonBackend {
fn new(socket_path: String) -> Arc<Self> {
Arc::new(Self { socket_path })
Arc::new(Self {
socket_path,
transport_session: format!("sdk-{}", uuid::Uuid::new_v4()),
})
}
async fn compatible_metadata(&self) -> Result<DaemonMetadata, DriverError> {
@ -268,6 +314,20 @@ impl DaemonBackend {
}
}
impl Drop for DaemonBackend {
fn drop(&mut self) {
let request = DaemonRequest {
method: "session_end".into(),
name: None,
args: None,
session_id: Some(self.transport_session.clone()),
observation_origin: Some(ToolObservationOrigin::Direct),
client_kind: Some(DaemonClientKind::Unknown),
};
let _ = send_request(&self.socket_path, &request);
}
}
fn validate_daemon_metadata(metadata: &DaemonMetadata) -> Result<(), DriverError> {
let mismatch = if metadata.contract_version != cua_driver_contract::CONTRACT_VERSION {
Some(format!(
@ -346,7 +406,10 @@ pub struct RuntimeAuthorizationOptions {
/// Mode inherited by calls made through the released `CuaDriver` object
/// rather than a trusted session-bound action surface.
pub compatibility_mode: SessionPermissionMode,
/// Required only when `compatibility_mode` is bounded.
/// Optional in standard and unrestricted; required when compatibility mode is bounded.
#[uniffi(default = None)]
pub compatibility_capability_manifest_path: Option<String>,
/// Deprecated alias for `compatibility_capability_manifest_path`.
pub compatibility_bounded_manifest_path: Option<String>,
pub unrestricted_acknowledged: bool,
pub max_session_ttl_seconds: u64,
@ -373,6 +436,7 @@ fn configured_driver_options_json(options: &ConfiguredDriverOptions) -> Value {
"authorization": {
"allowed_modes": allowed_modes,
"compatibility_mode": options.authorization.compatibility_mode.as_str(),
"compatibility_capability_manifest_path": options.authorization.compatibility_capability_manifest_path.clone(),
"compatibility_bounded_manifest_path": options.authorization.compatibility_bounded_manifest_path.clone(),
"unrestricted_acknowledged": options.authorization.unrestricted_acknowledged,
"max_session_ttl_seconds": options.authorization.max_session_ttl_seconds,
@ -388,6 +452,9 @@ pub struct TrustedSessionOptions {
pub mode: SessionPermissionMode,
pub ttl_seconds: u64,
pub idle_ttl_seconds: u64,
#[uniffi(default = None)]
pub capability_manifest_path: Option<String>,
/// Deprecated alias for `capability_manifest_path`.
pub bounded_manifest_path: Option<String>,
}
@ -551,7 +618,10 @@ impl From<SdkClientKind> for DaemonClientKind {
macro_rules! desktop_tool_methods {
($callback:ident) => {
$callback! {
list_apps: ListAppsInput,
list_windows: ListWindowsInput,
get_desktop_state: GetDesktopStateInput,
get_window_state: GetWindowStateInput,
get_screen_size: GetScreenSizeInput,
get_cursor_position: GetCursorPositionInput,
verify_state: VerifyStateInput,
@ -559,13 +629,23 @@ macro_rules! desktop_tool_methods {
set_window_frame: SetWindowFrameInput,
invoke_menu: InvokeMenuInput,
click: ClickInput,
window_click: WindowClickInput,
double_click: DoubleClickInput,
right_click: RightClickInput,
drag: DragInput,
window_drag: WindowDragInput,
scroll: ScrollInput,
window_scroll: WindowScrollInput,
clipboard_read: ClipboardReadInput,
clipboard_write: ClipboardWriteInput,
type_text: TypeTextInput,
window_type_text: WindowTypeTextInput,
press_key: PressKeyInput,
window_press_key: WindowPressKeyInput,
hotkey: HotkeyInput,
window_hotkey: WindowHotkeyInput,
set_value: SetValueInput,
perform_secondary_action: PerformSecondaryActionInput,
set_agent_cursor_enabled: SetAgentCursorEnabledInput,
set_agent_cursor_motion: SetAgentCursorMotionInput,
set_agent_cursor_theme: SetAgentCursorThemeInput,
@ -606,6 +686,8 @@ macro_rules! define_exported_tool_names {
const EXPORTED_TOOL_NAMES: &[&str] = &[
<StartSessionInput as ToolInput>::TOOL_NAME,
<EscalateSessionInput as ToolInput>::TOOL_NAME,
<GetSessionInput as ToolInput>::TOOL_NAME,
<ListSessionsInput as ToolInput>::TOOL_NAME,
<GetSessionStateInput as ToolInput>::TOOL_NAME,
<EndSessionInput as ToolInput>::TOOL_NAME,
$(<$input as ToolInput>::TOOL_NAME,)*
@ -961,6 +1043,7 @@ impl CuaDriver {
"mode": options.mode.as_str(),
"ttl_seconds": options.ttl_seconds,
"idle_ttl_seconds": options.idle_ttl_seconds,
"capability_manifest_path": options.capability_manifest_path,
"bounded_manifest_path": options.bounded_manifest_path,
});
Ok(Arc::new(CuaDriverSession {
@ -1000,6 +1083,40 @@ impl CuaDriver {
}
}
/// Daemon-host bridge for reconnecting one trusted host lease without
/// exposing its transport identity through generated bindings.
#[doc(hidden)]
pub fn create_trusted_session_for_transport(
&self,
options: TrustedSessionOptions,
transport_session: &str,
) -> Result<Arc<CuaDriverSession>, DriverError> {
let DriverBackend::Embedded(runtime) = &self.backend else {
return Err(DriverError::Configuration {
reason:
"stable trusted-session transport binding requires an embedded host runtime"
.into(),
});
};
if transport_session.trim().is_empty() {
return Err(DriverError::Configuration {
reason: "trusted-session transport identity must not be empty".into(),
});
}
let native_options = serde_json::json!({
"public_session": options.public_session,
"mode": options.mode.as_str(),
"ttl_seconds": options.ttl_seconds,
"idle_ttl_seconds": options.idle_ttl_seconds,
"capability_manifest_path": options.capability_manifest_path,
"bounded_manifest_path": options.bounded_manifest_path,
"transport_session": transport_session,
});
Ok(Arc::new(CuaDriverSession {
backend: SessionBackend::Embedded(Arc::new(runtime.create_session(native_options)?)),
}))
}
/// Bind a trusted session through an authenticated remote carrier without
/// exposing serialized authority to the caller.
pub async fn create_remote_trusted_session(
@ -1065,14 +1182,10 @@ impl CuaDriver {
reason: format!("authorization configuration is invalid: {error}"),
}
})?;
let manifest = if mode == cua_driver_core::authorization::PermissionMode::Bounded {
cua_driver_core::session_manifest::configured_session_manifest()
.map_err(|error| DriverError::Configuration { reason: error })?
.cloned()
.map(Arc::new)
} else {
None
};
let manifest = cua_driver_core::session_manifest::configured_capability_manifest()
.map_err(|error| DriverError::Configuration { reason: error })?
.cloned()
.map(Arc::new);
let ceiling =
cua_driver_core::session_authorization::SessionModeCeiling::for_trusted_sessions(
[mode],
@ -1121,6 +1234,7 @@ impl CuaDriver {
"authorization": {
"allowed_modes": allowed_modes,
"compatibility_mode": options.authorization.compatibility_mode.as_str(),
"compatibility_capability_manifest_path": options.authorization.compatibility_capability_manifest_path,
"compatibility_bounded_manifest_path": options.authorization.compatibility_bounded_manifest_path,
"unrestricted_acknowledged": options.authorization.unrestricted_acknowledged,
"max_session_ttl_seconds": options.authorization.max_session_ttl_seconds,
@ -1208,7 +1322,7 @@ impl CuaDriver {
method: "list".into(),
name: None,
args: None,
session_id: None,
session_id: Some(daemon.transport_session.clone()),
observation_origin: Some(ToolObservationOrigin::Direct),
client_kind: Some(self.client_kind),
};
@ -1237,6 +1351,54 @@ impl CuaDriver {
})
}
/// Content-free lifecycle summaries for the trusted host's runtime or
/// host-lease namespace. This is deliberately separate from the
/// agent-facing `list_sessions` tool, whose view is transport scoped.
pub async fn list_host_sessions_json(&self) -> Result<String, DriverError> {
let result = match &self.backend {
DriverBackend::Embedded(runtime) => {
let prefix = format!("__cua_runtime_{}:", runtime.runtime_scope_key());
host_sessions_json_for_prefix(&prefix)
}
DriverBackend::PrivateWorker(worker) => worker.list_host_sessions().await?,
DriverBackend::Remote(remote) => remote.list_host_sessions().await?,
DriverBackend::Daemon(daemon) => {
daemon.compatible_metadata().await?;
let socket_path = daemon.socket_path.clone();
let request_path = socket_path.clone();
let request = DaemonRequest {
method: "sessions_list".into(),
name: None,
args: None,
session_id: None,
observation_origin: Some(ToolObservationOrigin::Direct),
client_kind: Some(self.client_kind),
};
let response =
tokio::task::spawn_blocking(move || send_request(&request_path, &request))
.await
.map_err(|error| DriverError::Protocol {
reason: format!("host session listing task failed: {error}"),
})?
.map_err(|error| DriverError::Transport {
socket_path,
reason: error.to_string(),
})?;
if !response.ok {
return Err(DriverError::Protocol {
reason: response
.error
.unwrap_or_else(|| "host session listing failed".into()),
});
}
response.result.unwrap_or(Value::Null)
}
};
serde_json::to_string(&result).map_err(|error| DriverError::Protocol {
reason: error.to_string(),
})
}
pub async fn start_session(
&self,
input: StartSessionInput,
@ -1264,6 +1426,21 @@ impl CuaDriver {
.typed_success(GetSessionStateInput::TOOL_NAME)
}
pub async fn get_session(&self, input: GetSessionInput) -> Result<SessionOutput, DriverError> {
self.invoke_typed(GetSessionInput::TOOL_NAME, input)
.await?
.typed_success(GetSessionInput::TOOL_NAME)
}
pub async fn list_sessions(
&self,
input: ListSessionsInput,
) -> Result<ListSessionsOutput, DriverError> {
self.invoke_typed(ListSessionsInput::TOOL_NAME, input)
.await?
.typed_success(ListSessionsInput::TOOL_NAME)
}
pub async fn end_session(
&self,
input: EndSessionInput,
@ -1327,6 +1504,21 @@ impl CuaDriverSession {
.typed_success(GetSessionStateInput::TOOL_NAME)
}
pub async fn get_session(&self, input: GetSessionInput) -> Result<SessionOutput, DriverError> {
self.invoke_typed(GetSessionInput::TOOL_NAME, input)
.await?
.typed_success(GetSessionInput::TOOL_NAME)
}
pub async fn list_sessions(
&self,
input: ListSessionsInput,
) -> Result<ListSessionsOutput, DriverError> {
self.invoke_typed(ListSessionsInput::TOOL_NAME, input)
.await?
.typed_success(ListSessionsInput::TOOL_NAME)
}
pub async fn end_session(
&self,
input: EndSessionInput,
@ -1461,7 +1653,7 @@ impl CuaDriver {
method: "call".into(),
name: Some(name.into()),
args: Some(arguments),
session_id: None,
session_id: Some(daemon.transport_session.clone()),
observation_origin: Some(ToolObservationOrigin::Direct),
client_kind: Some(self.client_kind),
};
@ -1656,15 +1848,20 @@ mod tests {
#[test]
fn exported_typed_methods_cover_every_published_contract() {
let mut expected = cua_driver_contract::manifest()
let expected = cua_driver_contract::manifest()
.tools
.into_iter()
.map(|tool| tool.name)
.collect::<Vec<_>>();
expected.sort();
let mut exported = EXPORTED_TOOL_NAMES.to_vec();
exported.sort_unstable();
assert_eq!(exported, expected);
.collect::<std::collections::BTreeSet<_>>();
let exported = EXPORTED_TOOL_NAMES
.iter()
.map(|name| (*name).to_owned())
.collect::<std::collections::BTreeSet<_>>();
let missing = expected.difference(&exported).cloned().collect::<Vec<_>>();
assert!(
missing.is_empty(),
"typed SDK methods missing for {missing:?}"
);
}
#[cfg(unix)]
@ -1706,6 +1903,7 @@ mod tests {
authorization: RuntimeAuthorizationOptions {
allowed_modes: vec![SessionPermissionMode::Standard],
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: false,
max_session_ttl_seconds: 60,
@ -1736,6 +1934,7 @@ mod tests {
authorization: RuntimeAuthorizationOptions {
allowed_modes: vec![SessionPermissionMode::Standard],
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: false,
max_session_ttl_seconds: 60,
@ -1767,6 +1966,7 @@ mod tests {
authorization: RuntimeAuthorizationOptions {
allowed_modes: vec![SessionPermissionMode::Unrestricted],
compatibility_mode: SessionPermissionMode::Unrestricted,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: true,
max_session_ttl_seconds: 60,
@ -1874,6 +2074,47 @@ mod tests {
));
}
#[tokio::test]
async fn trusted_host_listing_spans_its_runtime_without_exposing_owner_keys() {
let _runtime_test = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
let driver = CuaDriver::create(None).unwrap();
let public = format!("host-listing-{}", uuid::Uuid::new_v4());
driver
.start_session(StartSessionInput {
session: Some(public.clone()),
capture_scope: None,
cursor_theme: None,
})
.await
.unwrap();
let listed: Value =
serde_json::from_str(&driver.list_host_sessions_json().await.unwrap()).unwrap();
let session = listed["sessions"]
.as_array()
.unwrap()
.iter()
.find(|session| {
session["session"]
.as_str()
.is_some_and(|label| label.starts_with("host-listing-"))
})
.unwrap_or_else(|| {
panic!("host-scoped listing omitted its live named session: {listed}")
});
assert_eq!(session["implicit"], false);
assert!(session.get("owner_transport").is_none());
assert!(session.get("runtime_id").is_none());
driver
.end_session(EndSessionInput {
session: Some(public),
})
.await
.unwrap();
driver.shutdown().await.unwrap();
}
#[tokio::test]
async fn shutdown_drains_an_already_admitted_call() {
let _runtime_test = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
@ -1969,6 +2210,7 @@ mod tests {
SessionPermissionMode::Unrestricted,
],
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: true,
max_session_ttl_seconds: 60,
@ -1982,6 +2224,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -1991,6 +2234,7 @@ mod tests {
mode: SessionPermissionMode::Unrestricted,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2126,6 +2370,45 @@ mod tests {
);
}
#[tokio::test]
async fn trusted_session_idle_ttl_reaches_the_lifecycle_contract() {
let _runtime_test = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
let driver = configured_standard_driver();
let session = driver
.create_trusted_session(TrustedSessionOptions {
public_session: "trusted-lifecycle-ttl".into(),
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 2,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
let started = session
.call_tool("start_session".into(), "{}".into())
.await
.unwrap();
assert!(!started.is_error, "{}", started.text);
let inspected = session
.call_tool("get_session".into(), "{}".into())
.await
.unwrap();
assert!(!inspected.is_error, "{}", inspected.text);
let lifecycle: Value =
serde_json::from_str(inspected.structured_json.as_deref().unwrap()).unwrap();
let expires_in = lifecycle["expires_in_seconds"].as_u64().unwrap();
assert!(
expires_in <= 2,
"trusted lifecycle TTL was not applied: {lifecycle}"
);
assert_eq!(lifecycle["session"], "trusted-lifecycle-ttl");
session.close();
driver.shutdown().await.unwrap();
}
#[tokio::test]
async fn authorization_expiry_in_one_runtime_does_not_affect_another() {
let _runtime_test = crate::runtime::TEST_RUNTIME_LOCK.lock().unwrap();
@ -2134,6 +2417,7 @@ mod tests {
authorization: RuntimeAuthorizationOptions {
allowed_modes: vec![SessionPermissionMode::Standard],
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: false,
max_session_ttl_seconds: 2,
@ -2148,6 +2432,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 1,
idle_ttl_seconds: 1,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2157,6 +2442,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2191,6 +2477,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2235,6 +2522,7 @@ mod tests {
SessionPermissionMode::Unrestricted,
],
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: true,
max_session_ttl_seconds: 60,
@ -2248,6 +2536,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2290,6 +2579,7 @@ mod tests {
mode: SessionPermissionMode::Unrestricted,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
}),
Err(DriverError::Configuration { .. })
@ -2300,6 +2590,7 @@ mod tests {
mode: SessionPermissionMode::Unrestricted,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.unwrap();
@ -2554,7 +2845,7 @@ mod tests {
let driver = CuaDriver::connect(Some(socket)).unwrap();
let output = driver
.start_session(StartSessionInput {
session: "run-2".into(),
session: Some("run-2".into()),
capture_scope: Some(cua_driver_contract::CaptureScope::Auto),
cursor_theme: None,
})
@ -2660,6 +2951,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.await
@ -2862,6 +3154,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
})
.await

4
packages/cua-driver/rust/crates/cua-driver-sdk/src/remote.rs
View file

@ -136,6 +136,10 @@ impl RemoteDriverClient {
exchange(&self.channel, "list", None, None).await
}
pub(crate) async fn list_host_sessions(&self) -> Result<Value, DriverError> {
exchange(&self.channel, "sessions_list", None, None).await
}
pub(crate) async fn invoke(&self, name: &str, arguments: Value) -> Result<Value, DriverError> {
exchange(
&self.channel,

103
packages/cua-driver/rust/crates/cua-driver-sdk/src/runtime.rs
View file

@ -132,7 +132,11 @@ impl RuntimeSession {
.runtime
.invoke_with_context(name, args, self.context.clone())
.await;
if name == "end_session" && result.is_some() {
if name == "end_session"
&& result
.as_ref()
.is_some_and(|result| result.is_error != Some(true))
{
self.authorization_registry
.revoke_connection(&self.connection);
}
@ -556,16 +560,17 @@ fn build_registry(options: &RuntimeOptions) -> ToolRegistry {
register_host_tools(&mut registry);
}
let recording = Arc::downgrade(&registry.recording);
let recording_session_end =
cua_driver_core::session::register_scoped_session_end_hook(move |session| {
let recording_session_end = cua_driver_core::session::register_scoped_fallible_session_end_hook(
"recording",
move |session| {
let Some(recording) = recording.upgrade() else {
return;
return Ok(());
};
let session = session.to_owned();
std::thread::spawn(move || {
let _ = recording.stop_owner(Some(&session));
});
});
recording
.stop_owner(Some(session))
.map_err(|error| error.to_string())
},
);
registry.retain_session_end_hook(recording_session_end);
registry
}
@ -718,7 +723,7 @@ mod tests {
let idle_before_refresh =
cua_driver_core::session::session_idle_duration(&internal).unwrap();
runtime
.invoke("health_report", serde_json::json!({"session": public}))
.invoke("start_session", serde_json::json!({"session": public}))
.await
.unwrap();
let idle_after_refresh =
@ -785,4 +790,82 @@ mod tests {
runtime.shutdown().await;
}
#[tokio::test]
async fn incomplete_end_keeps_trusted_authorization_live_for_cleanup_retry() {
let _runtime_test = TEST_RUNTIME_LOCK.lock().unwrap();
let runtime = DriverRuntime::create(standard_options()).unwrap();
let public_session = "runtime-end-cleanup-retry";
let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let attempts_for_hook = attempts.clone();
let _hook = cua_driver_core::session::register_scoped_fallible_session_end_hook(
"runtime-end-cleanup-retry-test",
move |_| {
if attempts_for_hook.fetch_add(1, Ordering::SeqCst) == 0 {
Err("synthetic first-attempt failure".into())
} else {
Ok(())
}
},
);
let session = runtime
.create_trusted_session(DelegatedSessionRequest {
public_session: public_session.into(),
transport_session: "runtime-end-cleanup-retry-transport".into(),
mode: PermissionMode::Standard,
ttl: Duration::from_secs(60),
idle_ttl: Duration::from_secs(30),
capability_manifest: None,
})
.unwrap();
let started = session
.invoke("start_session", serde_json::json!({}))
.await
.unwrap();
assert_ne!(started.is_error, Some(true));
let runtime_prefix = format!(
"__cua_runtime_{}:",
runtime.compatibility_context.runtime_scope_key()
);
let started_sessions = cua_driver_core::session::list_session_snapshots_with_prefix(
&runtime_prefix,
Duration::from_secs(300),
);
assert_eq!(started_sessions.len(), 1, "{started_sessions:?}");
let first_end = session
.invoke("end_session", serde_json::json!({}))
.await
.unwrap();
assert_eq!(
first_end.is_error,
Some(true),
"result={first_end:?} started={started_sessions:?} attempts={}",
attempts.load(Ordering::SeqCst)
);
assert_eq!(attempts.load(Ordering::SeqCst), 1);
let retried_end = session
.invoke("end_session", serde_json::json!({}))
.await
.unwrap();
assert_ne!(retried_end.is_error, Some(true));
assert_eq!(attempts.load(Ordering::SeqCst), 2);
let after_end = session
.invoke("health_report", serde_json::json!({}))
.await
.unwrap();
assert_eq!(
after_end
.structured_content
.as_ref()
.and_then(|value| value.pointer("/refusal/code"))
.and_then(Value::as_str),
Some("authorization_revoked")
);
runtime.shutdown().await;
}
}

230
packages/cua-driver/rust/crates/cua-driver-sdk/src/service_session.rs
View file

@ -8,11 +8,14 @@ use cua_driver_core::daemon::{DaemonClientKind, DaemonRequest, DaemonResponse};
use serde_json::Value;
use std::io::{BufRead, BufReader, Write};
use std::sync::{Arc, Mutex};
use std::time::Duration;
#[cfg(unix)]
use std::time::{Duration, Instant};
use std::time::Instant;
#[cfg(unix)]
const SERVICE_REQUEST_DEADLINE: Duration = Duration::from_secs(120);
const RESUME_REGISTRATION_RETRIES: usize = 20;
const RESUME_REGISTRATION_DELAY: Duration = Duration::from_millis(10);
#[cfg(unix)]
type ServiceStream = std::os::unix::net::UnixStream;
@ -23,6 +26,7 @@ struct ServiceConnection {
reader: BufReader<ServiceStream>,
writer: ServiceStream,
closed: bool,
resume_credential: String,
}
pub(crate) struct ServiceSessionClient {
@ -55,12 +59,13 @@ impl ServiceSessionClient {
reader: BufReader::new(stream),
writer,
closed: false,
resume_credential: String::new(),
}),
});
let arguments = serde_json::to_value(options).map_err(|error| DriverError::Protocol {
reason: format!("serialize trusted service session options: {error}"),
})?;
client.request(DaemonRequest {
let bound = client.request(DaemonRequest {
method: "trusted_session_begin".into(),
name: None,
args: Some(arguments),
@ -68,6 +73,14 @@ impl ServiceSessionClient {
observation_origin: None,
client_kind: Some(client_kind),
})?;
let resume_credential = bound
.get("resume_credential")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.ok_or_else(|| DriverError::Protocol {
reason: "trusted service bind omitted resume credential".into(),
})?;
client.connection.lock().unwrap().resume_credential = resume_credential.to_owned();
Ok(client)
}
@ -97,7 +110,9 @@ impl ServiceSessionClient {
pub(crate) fn close(&self) {
let mut connection = self.connection.lock().unwrap();
if connection.closed {
return;
if self.resume_connection(&mut connection).is_err() {
return;
}
}
let request = DaemonRequest {
method: "trusted_session_end".into(),
@ -121,7 +136,7 @@ impl ServiceSessionClient {
fn request(&self, request: DaemonRequest) -> Result<Value, DriverError> {
let mut connection = self.connection.lock().unwrap();
if connection.closed {
return Err(DriverError::Shutdown);
self.resume_connection(&mut connection)?;
}
let action_call = request.method == "trusted_session_call";
write_request(&mut connection.writer, &request).map_err(|error| {
@ -172,6 +187,85 @@ impl ServiceSessionClient {
Ok(response.result.unwrap_or(Value::Null))
}
fn resume_connection(&self, connection: &mut ServiceConnection) -> Result<(), DriverError> {
if connection.resume_credential.is_empty() {
return Err(DriverError::Shutdown);
}
let credential = connection.resume_credential.clone();
for attempt in 0..=RESUME_REGISTRATION_RETRIES {
let stream = connect(&self.socket_path)?;
let writer = stream.try_clone().map_err(|error| DriverError::Transport {
socket_path: self.socket_path.clone(),
reason: format!("clone resumed trusted service connection: {error}"),
})?;
// Replace both handles before presenting the credential. Dropping
// the old client handles makes the daemon observe EOF and publish
// the detachable lease on Unix sockets and Windows named pipes.
connection.reader = BufReader::new(stream);
connection.writer = writer;
let request = DaemonRequest {
method: "trusted_session_resume".into(),
name: None,
args: Some(serde_json::json!({
"resume_credential": credential,
})),
session_id: None,
observation_origin: None,
client_kind: None,
};
write_request(&mut connection.writer, &request).map_err(|error| {
DriverError::Transport {
socket_path: self.socket_path.clone(),
reason: format!("write trusted session resume: {error}"),
}
})?;
let line = read_response_line(&mut connection.reader)
.map_err(|error| DriverError::Transport {
socket_path: self.socket_path.clone(),
reason: format!("read trusted session resume: {error}"),
})?
.ok_or_else(|| DriverError::Transport {
socket_path: self.socket_path.clone(),
reason: "trusted service closed during resume".into(),
})?;
let response: DaemonResponse =
serde_json::from_str(&line).map_err(|error| DriverError::Protocol {
reason: format!("parse trusted session resume response: {error}"),
})?;
if response.ok {
let rotated = response
.result
.as_ref()
.and_then(|value| value.get("resume_credential"))
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.ok_or_else(|| DriverError::Protocol {
reason: "trusted session resume omitted rotated credential".into(),
})?;
connection.resume_credential = rotated.to_owned();
connection.closed = false;
return Ok(());
}
let reason = response
.error
.unwrap_or_else(|| "trusted session resume failed".into());
let registration_race = reason.contains("unavailable or already used")
&& attempt < RESUME_REGISTRATION_RETRIES;
if registration_race {
std::thread::sleep(RESUME_REGISTRATION_DELAY);
continue;
}
connection.resume_credential.clear();
return Err(DriverError::Transport {
socket_path: self.socket_path.clone(),
reason,
});
}
unreachable!("bounded resume loop returns on its final attempt")
}
fn connection_failure(&self, action_call: bool, reason: String) -> DriverError {
if action_call {
DriverError::ActionInterrupted {
@ -363,7 +457,10 @@ mod tests {
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({}))).unwrap()
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({
"resume_credential": "resume-test-1"
})))
.unwrap()
)
.unwrap();
@ -382,6 +479,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
},
DaemonClientKind::Unknown,
@ -399,6 +497,122 @@ mod tests {
server.join().unwrap();
}
#[tokio::test]
async fn next_call_resumes_with_single_use_host_credential_after_disconnect() {
let directory = tempfile::tempdir().unwrap();
let socket = directory.path().join("resumable-service.sock");
let listener = UnixListener::bind(&socket).unwrap();
let server = std::thread::spawn(move || {
serve_compatible_metadata(&listener);
{
let (stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut writer = stream;
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let begin: DaemonRequest = serde_json::from_str(&line).unwrap();
assert_eq!(begin.method, "trusted_session_begin");
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({
"resume_credential": "resume-original"
})))
.unwrap()
)
.unwrap();
line.clear();
reader.read_line(&mut line).unwrap();
let call: DaemonRequest = serde_json::from_str(&line).unwrap();
assert_eq!(call.method, "trusted_session_call");
// Drop without a response. The SDK must not retry this action.
}
// The reconnect can beat the daemon task that records EOF from the
// previous connection. A transient unavailable response must keep
// the single-use credential intact and retry on a fresh socket.
{
let (stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut writer = stream;
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let resume: DaemonRequest = serde_json::from_str(&line).unwrap();
assert_eq!(resume.method, "trusted_session_resume");
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::err(
"trusted session resume credential is unavailable or already used",
77,
))
.unwrap()
)
.unwrap();
}
let (stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut writer = stream;
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let resume: DaemonRequest = serde_json::from_str(&line).unwrap();
assert_eq!(resume.method, "trusted_session_resume");
assert_eq!(resume.args.unwrap()["resume_credential"], "resume-original");
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({
"resume_credential": "resume-rotated"
})))
.unwrap()
)
.unwrap();
line.clear();
reader.read_line(&mut line).unwrap();
let call: DaemonRequest = serde_json::from_str(&line).unwrap();
assert_eq!(call.method, "trusted_session_call");
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({
"resumed": true
})))
.unwrap()
)
.unwrap();
});
let client = ServiceSessionClient::connect_and_bind(
socket.to_string_lossy().into_owned(),
TrustedSessionOptions {
public_session: "resume-test".into(),
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
},
DaemonClientKind::Unknown,
)
.unwrap();
assert!(matches!(
client.invoke("click", serde_json::json!({})).await,
Err(DriverError::ActionInterrupted {
completion: crate::worker::ActionCompletion::Unknown,
..
})
));
let resumed = client
.invoke("get_session", serde_json::json!({}))
.await
.unwrap();
assert_eq!(resumed["resumed"], true);
server.join().unwrap();
}
#[tokio::test]
async fn transient_read_timeouts_are_poll_intervals_not_action_deadlines() {
let directory = tempfile::tempdir().unwrap();
@ -415,7 +629,10 @@ mod tests {
writeln!(
writer,
"{}",
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({}))).unwrap()
serde_json::to_string(&DaemonResponse::ok(serde_json::json!({
"resume_credential": "resume-test-2"
})))
.unwrap()
)
.unwrap();
@ -443,6 +660,7 @@ mod tests {
mode: SessionPermissionMode::Standard,
ttl_seconds: 60,
idle_ttl_seconds: 30,
capability_manifest_path: None,
bounded_manifest_path: None,
},
DaemonClientKind::Unknown,

7
packages/cua-driver/rust/crates/cua-driver-sdk/src/worker.rs
View file

@ -233,6 +233,10 @@ impl PrivateWorkerClient {
self.request_async("list", None, None, None).await
}
pub(crate) async fn list_host_sessions(self: &Arc<Self>) -> Result<Value, DriverError> {
self.request_async("sessions_list", None, None, None).await
}
pub(crate) async fn invoke(
self: &Arc<Self>,
name: &str,
@ -550,7 +554,10 @@ mod tests {
for name in [
"CUA_DRIVER_DISABLE_UNRESTRICTED",
"CUA_DRIVER_MANAGED_POLICY_FILE",
"CUA_DRIVER_CAPABILITY_MANIFEST_FILE",
"CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED",
"CUA_DRIVER_SESSION_POLICY_FILE",
"CUA_DRIVER_SESSION_POLICY_APPROVED",
] {
assert!(inherited_managed_environment_name(name));
assert!(

13
packages/cua-driver/rust/crates/cua-driver-sdk/tests/runtime_configuration.rs
View file

@ -12,6 +12,7 @@ fn options(allowed_modes: Vec<SessionPermissionMode>) -> ConfiguredDriverOptions
authorization: RuntimeAuthorizationOptions {
allowed_modes,
compatibility_mode: SessionPermissionMode::Standard,
compatibility_capability_manifest_path: None,
compatibility_bounded_manifest_path: None,
unrestricted_acknowledged: true,
max_session_ttl_seconds: 60,
@ -33,9 +34,6 @@ fn child_configuration_probe() {
SessionPermissionMode::Standard,
SessionPermissionMode::Unrestricted,
])),
"legacy_approval" => {
CuaDriver::create_configured(options(vec![SessionPermissionMode::Standard]))
}
other => panic!("unknown probe case {other}"),
};
@ -45,7 +43,7 @@ fn child_configuration_probe() {
panic!("matching environment was rejected: {error}");
}
}
"conflicting_mode" | "managed_disable" | "legacy_approval" => {
"conflicting_mode" | "managed_disable" => {
assert!(result.is_err(), "contradictory environment was accepted")
}
_ => unreachable!(),
@ -60,7 +58,8 @@ fn run_probe(case: &str, environment: &[(&str, &str)]) {
.env_remove("CUA_DRIVER_PERMISSION_MODE")
.env_remove("CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS")
.env_remove("CUA_DRIVER_DISABLE_UNRESTRICTED")
.env_remove("CUA_DRIVER_ALLOW_LEGACY_EXISTING_PROFILE_APPROVAL")
.env_remove("CUA_DRIVER_CAPABILITY_MANIFEST_FILE")
.env_remove("CUA_DRIVER_CAPABILITY_MANIFEST_APPROVED")
.env_remove("CUA_DRIVER_SESSION_POLICY_FILE")
.env_remove("CUA_DRIVER_SESSION_POLICY_APPROVED");
for (name, value) in environment {
@ -89,8 +88,4 @@ fn explicit_runtime_configuration_rejects_contradictory_environment() {
"managed_disable",
&[("CUA_DRIVER_DISABLE_UNRESTRICTED", "1")],
);
run_probe(
"legacy_approval",
&[("CUA_DRIVER_ALLOW_LEGACY_EXISTING_PROFILE_APPROVAL", "1")],
);
}

3
packages/cua-driver/rust/crates/cua-driver-testkit/Cargo.toml
View file

@ -31,11 +31,14 @@ objc2-app-kit = { version = "0.2", features = [
[target.'cfg(target_os = "windows")'.dependencies]
windows = { version = "0.61", features = [
"Win32_Foundation",
"Win32_Storage_FileSystem",
# Job Object: assign every spawned test child to a KILL_ON_JOB_CLOSE job so
# the OS terminates the whole tree when the test process dies for ANY reason.
"Win32_System_JobObjects",
"Win32_System_Console",
"Win32_System_IO",
"Win32_Security",
"Win32_System_Pipes",
"Win32_System_Threading",
"Win32_UI_Input_KeyboardAndMouse",
"Win32_UI_WindowsAndMessaging",

379
packages/cua-driver/rust/crates/cua-driver-testkit/src/daemon.rs
View file

@ -6,7 +6,7 @@ use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use crate::reaper::ChildReaper;
use crate::reaper::{spawn_in_job, ChildReaper};
#[cfg(not(unix))]
static DAEMON_SEQUENCE: AtomicU64 = AtomicU64::new(1);
@ -14,6 +14,7 @@ static DAEMON_SEQUENCE: AtomicU64 = AtomicU64::new(1);
/// Keeps the temporary socket directory alive for the daemon lifetime.
pub(crate) struct TestDaemon {
pub(crate) socket: String,
pub(crate) pid: u32,
#[cfg(unix)]
_socket_dir: tempfile::TempDir,
}
@ -85,16 +86,35 @@ impl TestDaemon {
for (key, value) in env {
command.env(key, value);
}
reaper
.spawn(&mut command)
// Spawn directly so the `Child` stays reachable for `try_wait` below;
// the reaper adopts it on every exit path. Without the handle, a daemon
// that dies during startup is indistinguishable from one that is merely
// slow, which is what made #2480 undiagnosable from CI logs alone.
let mut child = spawn_in_job(&mut command)
.inspect_err(|error| eprintln!("[testkit] daemon spawn failed: {error}"))
.ok()?;
let pid = child.id();
let deadline = Instant::now() + Duration::from_secs(10);
let deadline = Instant::now() + READINESS_WINDOW;
// Keep the furthest point any probe reached. A late attempt can regress
// (the pipe is torn down as the daemon exits), and the furthest outcome
// is the one that explains where startup actually stopped.
let mut furthest = ProbeOutcome::PipeUnavailable;
let mut exit_status = None;
while Instant::now() < deadline {
if daemon_is_listening(binary, &socket) {
// A daemon that has already exited will never become ready; report
// its status now instead of burning the rest of the window.
if let Ok(Some(status)) = child.try_wait() {
exit_status = Some(status);
break;
}
let outcome = daemon_is_listening(binary, &socket);
furthest = furthest.max(outcome);
if outcome == ProbeOutcome::Ready {
reaper.push(child);
return Some(Self {
socket,
pid,
#[cfg(unix)]
_socket_dir: socket_dir,
});
@ -102,18 +122,118 @@ impl TestDaemon {
std::thread::sleep(Duration::from_millis(50));
}
eprintln!("[testkit] daemon did not become ready on {socket}");
match exit_status {
Some(status) => eprintln!(
"[testkit] daemon exited before becoming ready on {socket}: {status}. \
Furthest readiness probe: {}. Set CUA_TEST_DRIVER_STDERR=1 to inherit \
daemon stderr.",
furthest.describe()
),
None => eprintln!(
"[testkit] daemon did not become ready on {socket} within {}s (process still \
running). Furthest readiness probe: {}. Set CUA_TEST_DRIVER_STDERR=1 to \
inherit daemon stderr.",
READINESS_WINDOW.as_secs(),
furthest.describe()
),
}
reaper.push(child);
None
}
}
/// Total time the daemon has to answer a readiness probe.
const READINESS_WINDOW: Duration = Duration::from_secs(10);
/// Per-attempt bound on a single readiness probe.
///
/// A Windows named-pipe read has no timeout, so a server that accepts the
/// connection but never answers blocks forever. The outer deadline is only
/// consulted *between* attempts, so without an independent per-attempt bound
/// one stalled connection consumes the entire readiness window and the failure
/// surfaces as a bare "did not become ready" (#2480).
#[cfg(target_os = "windows")]
const PROBE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(1);
/// How far a single readiness probe progressed.
///
/// Ordered by progress so `max` across the window keeps the most informative
/// outcome: knowing the pipe was reachable but never answered points at daemon
/// startup, whereas never opening the pipe points at spawn or naming.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum ProbeOutcome {
/// The pipe/socket could not be opened at all.
PipeUnavailable,
/// Opened, but the request could not be written.
WriteFailed,
/// Request written, but no response line arrived before the attempt bound.
NoResponse,
/// A response arrived but was not valid JSON.
MalformedResponse,
/// Valid JSON, but not the `ok: true` the protocol promises.
NotOk,
/// Fully serviced — the daemon is ready.
Ready,
}
impl ProbeOutcome {
pub(crate) fn describe(self) -> &'static str {
match self {
Self::PipeUnavailable => {
"pipe never became connectable (daemon had not created it, or the name differs)"
}
Self::WriteFailed => "pipe opened but the request could not be written",
Self::NoResponse => {
"request written but the daemon sent no response line before the per-attempt \
timeout (accepted-but-stalled connection)"
}
Self::MalformedResponse => "daemon replied but the response was not valid JSON",
Self::NotOk => "daemon replied with valid JSON but without ok:true",
Self::Ready => "ready",
}
}
}
#[cfg(unix)]
fn daemon_is_listening(_binary: &Path, socket: &str) -> bool {
std::os::unix::net::UnixStream::connect(socket).is_ok()
fn daemon_is_listening(_binary: &Path, socket: &str) -> ProbeOutcome {
match std::os::unix::net::UnixStream::connect(socket) {
Ok(_) => ProbeOutcome::Ready,
Err(_) => ProbeOutcome::PipeUnavailable,
}
}
/// Run one named-pipe probe under an independent timeout.
///
/// The probe runs on a worker thread because the blocking read it performs has
/// no native timeout. If the attempt bound expires the thread is abandoned: it
/// is unblocked when the daemon is reaped at teardown (the kill-on-close job
/// object closes the pipe), and the harness is short-lived, so leaking a parked
/// thread is preferable to hanging the whole readiness window.
#[cfg(target_os = "windows")]
fn daemon_is_listening(_binary: &Path, socket: &str) -> ProbeOutcome {
probe_pipe_with_timeout(socket, PROBE_ATTEMPT_TIMEOUT)
}
#[cfg(target_os = "windows")]
fn daemon_is_listening(_binary: &Path, socket: &str) -> bool {
fn probe_pipe_with_timeout(socket: &str, timeout: Duration) -> ProbeOutcome {
use std::sync::mpsc;
let socket = socket.to_string();
let (tx, rx) = mpsc::channel();
if std::thread::Builder::new()
.name("cua-testkit-pipe-probe".into())
.spawn(move || {
let _ = tx.send(probe_pipe_once(&socket));
})
.is_err()
{
return ProbeOutcome::PipeUnavailable;
}
rx.recv_timeout(timeout).unwrap_or(ProbeOutcome::NoResponse)
}
#[cfg(target_os = "windows")]
fn probe_pipe_once(socket: &str) -> ProbeOutcome {
use std::io::{BufRead, BufReader, Write};
// Exercise the real named-pipe protocol instead of spawning `status`.
@ -126,35 +246,254 @@ fn daemon_is_listening(_binary: &Path, socket: &str) -> bool {
.write(true)
.open(socket)
else {
return false;
return ProbeOutcome::PipeUnavailable;
};
let Ok(mut writer) = pipe.try_clone() else {
return false;
return ProbeOutcome::WriteFailed;
};
if writer
.write_all(b"{\"method\":\"list\"}\n")
.and_then(|()| writer.flush())
.is_err()
{
return false;
return ProbeOutcome::WriteFailed;
}
let mut response = String::new();
if BufReader::new(pipe).read_line(&mut response).is_err() {
return false;
return ProbeOutcome::NoResponse;
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(&response) else {
return ProbeOutcome::MalformedResponse;
};
if value.get("ok").and_then(serde_json::Value::as_bool) == Some(true) {
ProbeOutcome::Ready
} else {
ProbeOutcome::NotOk
}
serde_json::from_str::<serde_json::Value>(&response)
.ok()
.and_then(|value| value.get("ok").and_then(serde_json::Value::as_bool))
== Some(true)
}
#[cfg(not(any(unix, target_os = "windows")))]
fn daemon_is_listening(binary: &Path, socket: &str) -> bool {
Command::new(binary)
fn daemon_is_listening(binary: &Path, socket: &str) -> ProbeOutcome {
let ready = Command::new(binary)
.args(["status", "--socket", socket])
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.is_ok_and(|status| status.success())
.is_ok_and(|status| status.success());
if ready {
ProbeOutcome::Ready
} else {
ProbeOutcome::PipeUnavailable
}
}
#[cfg(test)]
mod readiness_probe_tests {
use super::*;
/// The furthest-progress ordering is what makes the expiry message useful,
/// so pin it rather than leaving it to derive order by accident.
#[test]
fn outcomes_are_ordered_by_progress() {
let ordered = [
ProbeOutcome::PipeUnavailable,
ProbeOutcome::WriteFailed,
ProbeOutcome::NoResponse,
ProbeOutcome::MalformedResponse,
ProbeOutcome::NotOk,
ProbeOutcome::Ready,
];
for pair in ordered.windows(2) {
assert!(
pair[0] < pair[1],
"{:?} must rank below {:?}",
pair[0],
pair[1]
);
}
}
/// The reported outcome must survive a late regression: as a daemon exits,
/// its pipe stops opening, and reporting `PipeUnavailable` would point at
/// spawn/naming when the real evidence was an accepted-but-stalled pipe.
#[test]
fn furthest_outcome_survives_a_late_regression() {
let observed = [
ProbeOutcome::PipeUnavailable,
ProbeOutcome::NoResponse,
ProbeOutcome::PipeUnavailable,
];
let furthest = observed
.iter()
.copied()
.fold(ProbeOutcome::PipeUnavailable, ProbeOutcome::max);
assert_eq!(furthest, ProbeOutcome::NoResponse);
}
#[test]
fn ready_outranks_every_failure() {
for outcome in [
ProbeOutcome::PipeUnavailable,
ProbeOutcome::WriteFailed,
ProbeOutcome::NoResponse,
ProbeOutcome::MalformedResponse,
ProbeOutcome::NotOk,
] {
assert!(ProbeOutcome::Ready > outcome);
}
}
/// Every outcome must explain itself; the whole point of #2480 is that the
/// operator can tell a stall from a genuine startup failure.
#[test]
fn every_outcome_has_a_distinct_description() {
let all = [
ProbeOutcome::PipeUnavailable,
ProbeOutcome::WriteFailed,
ProbeOutcome::NoResponse,
ProbeOutcome::MalformedResponse,
ProbeOutcome::NotOk,
ProbeOutcome::Ready,
];
let mut seen = Vec::new();
for outcome in all {
let text = outcome.describe();
assert!(!text.is_empty(), "{outcome:?} needs a description");
assert!(!seen.contains(&text), "duplicate description: {text}");
seen.push(text);
}
}
/// The per-attempt bound must leave room for several attempts inside the
/// window; if one attempt could span it, the flake in #2480 returns.
#[cfg(target_os = "windows")]
#[test]
fn attempt_bound_permits_multiple_attempts_per_window() {
assert!(
PROBE_ATTEMPT_TIMEOUT * 4 <= READINESS_WINDOW,
"one stalled attempt must not consume the readiness window"
);
}
#[cfg(target_os = "windows")]
#[test]
fn stalled_pipe_attempt_is_bounded_and_does_not_block_the_next_probe() {
use std::io::{BufRead, BufReader, Write};
use std::os::windows::io::{AsRawHandle, FromRawHandle};
use std::sync::mpsc;
use windows::core::HSTRING;
use windows::Win32::Foundation::{ERROR_PIPE_CONNECTED, HANDLE};
use windows::Win32::Storage::FileSystem::PIPE_ACCESS_DUPLEX;
use windows::Win32::System::Pipes::{
ConnectNamedPipe, CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT,
};
fn socket_name(label: &str) -> String {
let sequence = DAEMON_SEQUENCE.fetch_add(1, Ordering::Relaxed);
format!(
r"\\.\pipe\cua-driver-test-{label}-{}-{sequence}",
std::process::id()
)
}
fn spawn_server(
socket: String,
response: Option<&'static [u8]>,
release: Option<mpsc::Receiver<()>>,
) -> (
mpsc::Receiver<()>,
mpsc::Receiver<()>,
std::thread::JoinHandle<()>,
) {
let (ready_tx, ready_rx) = mpsc::channel();
let (request_tx, request_rx) = mpsc::channel();
let handle = std::thread::spawn(move || {
let name = HSTRING::from(socket);
let raw = unsafe {
CreateNamedPipeW(
&name,
PIPE_ACCESS_DUPLEX,
PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
1,
4096,
4096,
0,
None,
)
};
assert!(!raw.is_invalid(), "CreateNamedPipeW failed");
let mut pipe = unsafe { std::fs::File::from_raw_handle(raw.0) };
ready_tx.send(()).unwrap();
let raw = HANDLE(pipe.as_raw_handle());
if let Err(error) = unsafe { ConnectNamedPipe(raw, None) } {
assert_eq!(
error.code(),
ERROR_PIPE_CONNECTED.to_hresult(),
"ConnectNamedPipe failed: {error}"
);
}
let mut request = String::new();
BufReader::new(pipe.try_clone().unwrap())
.read_line(&mut request)
.unwrap();
assert_eq!(request, "{\"method\":\"list\"}\n");
request_tx.send(()).unwrap();
if let Some(response) = response {
pipe.write_all(response).unwrap();
pipe.flush().unwrap();
} else {
release
.unwrap()
.recv_timeout(Duration::from_secs(5))
.expect("stalled server was not released");
}
});
(ready_rx, request_rx, handle)
}
let stalled_socket = socket_name("stalled");
let (release_tx, release_rx) = mpsc::channel();
let (stalled_ready, stalled_request, stalled_server) =
spawn_server(stalled_socket.clone(), None, Some(release_rx));
stalled_ready.recv_timeout(Duration::from_secs(2)).unwrap();
let attempt_timeout = Duration::from_millis(250);
let started = Instant::now();
assert_eq!(
probe_pipe_with_timeout(&stalled_socket, attempt_timeout),
ProbeOutcome::NoResponse
);
assert!(
started.elapsed() < attempt_timeout * 4,
"stalled probe exceeded its attempt bound by too much: {:?}",
started.elapsed()
);
stalled_request
.recv_timeout(Duration::from_secs(1))
.unwrap();
// Leave the first probe worker blocked while proving a later probe can
// still complete normally in the same harness process.
let responsive_socket = socket_name("responsive");
let (responsive_ready, responsive_request, responsive_server) =
spawn_server(responsive_socket.clone(), Some(b"{\"ok\":true}\n"), None);
responsive_ready
.recv_timeout(Duration::from_secs(2))
.unwrap();
assert_eq!(
probe_pipe_with_timeout(&responsive_socket, Duration::from_secs(2)),
ProbeOutcome::Ready
);
responsive_request
.recv_timeout(Duration::from_secs(1))
.unwrap();
release_tx.send(()).unwrap();
stalled_server.join().unwrap();
responsive_server.join().unwrap();
}
}

63
packages/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs
View file

@ -315,6 +315,7 @@ pub enum RefusalCode {
BrowserReconnectExhausted,
BrowserInputIncomplete,
BrowserActionUnavailable,
WindowMinimized,
}
impl RefusalCode {
@ -336,6 +337,7 @@ impl RefusalCode {
"browser_reconnect_exhausted" => Some(Self::BrowserReconnectExhausted),
"browser_input_incomplete" => Some(Self::BrowserInputIncomplete),
"browser_action_unavailable" => Some(Self::BrowserActionUnavailable),
"window_minimized" => Some(Self::WindowMinimized),
_ => None,
}
}
@ -1493,6 +1495,12 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
.as_str()
.is_some_and(|summary| summary.starts_with("✅ bring_to_front:"))
});
let refused_minimized_action = action.as_ref().is_some_and(|value| {
value["result_error"].as_bool() == Some(true)
&& value["result_summary"]
.as_str()
.is_some_and(|summary| summary.starts_with("refused (window_minimized):"))
});
let restored_state_captured = manifest
.as_ref()
.is_some_and(|value| value["after"]["state"]["status"].as_str() == Some("captured"));
@ -1506,6 +1514,8 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
&& successful_restore
&& restored_state_captured
&& classification(phase, "screenshot") == Some("capture_failed"))))
|| (refused_minimized_action
&& classification(phase, "screenshot") == Some("target_minimized"))
};
for (phase, kind) in [("before", "screenshot"), ("after", "screenshot")] {
@ -1577,15 +1587,23 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec<String>) {
let refused_before_target_resolution = action.as_ref().is_some_and(|value| {
value["result_error"].as_bool() == Some(true) && value.get("click_point").is_none()
});
if refused_before_target_resolution {
let refused_before_dispatch = action.as_ref().is_some_and(|value| {
value["result_error"].as_bool() == Some(true)
&& value["action_truth"]["effect"].as_str() == Some("refused")
});
if refused_before_target_resolution || refused_before_dispatch {
let click = manifest.as_ref().map(|value| &value["click"]);
let expected_classification = if refused_before_target_resolution {
"action_refused_before_target_resolution"
} else {
"action_refused_before_dispatch"
};
if !click.is_some_and(|value| {
value["status"].as_str() == Some("not_applicable")
&& value["classification"].as_str()
== Some("action_refused_before_target_resolution")
&& value["classification"].as_str() == Some(expected_classification)
}) {
errors.push(format!(
"invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/action_refused_before_target_resolution"
"invalid refused-click evidence for {cell_id}/{turn_name}: expected not_applicable/{expected_classification}"
));
}
return;
@ -2144,7 +2162,6 @@ mod tests {
br#"{
"tool":"browser_prepare",
"arguments":{
"approval_token":"[redacted]",
"pid":1,
"window_id":2,
"strategy":{"kind":"existing_profile"}
@ -2222,6 +2239,42 @@ mod tests {
.expect("a minimized target cannot provide a pre-restore screenshot");
}
#[test]
fn validator_accepts_missing_images_for_typed_minimized_refusal() {
let (root, case, result, turn) = complete_turn_fixture();
std::fs::write(
turn.join("action.json"),
br#"{
"tool":"click",
"arguments":{"pid":1,"window_id":2,"element_index":3},
"result_summary":"refused (window_minimized): restore before retrying",
"result_error":true,
"click_point":{"x":10,"y":20},
"action_truth":{"effect":"refused"}
}"#,
)
.expect("write minimized refusal action");
std::fs::write(
turn.join("evidence.json"),
br#"{
"schema":"cua-turn-evidence/v1",
"before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},
"after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}},
"click":{"status":"not_applicable","classification":"action_refused_before_dispatch"}
}"#,
)
.expect("write minimized refusal evidence manifest");
for file in ["before.png", "after.png", "screenshot.png", "click.png"] {
let path = turn.join(file);
if path.exists() {
std::fs::remove_file(path).expect("remove unavailable image");
}
}
validate_catalog(&[case], &[result], Some(root.path()), true)
.expect("a typed minimized refusal cannot provide target images");
}
#[test]
fn validator_accepts_restored_state_when_host_capture_remains_unavailable() {
let (root, case, result, turn) = complete_turn_fixture();

7
packages/cua-driver/rust/crates/cua-driver-testkit/src/raw.rs
View file

@ -27,6 +27,13 @@ pub struct RawDriver {
}
impl RawDriver {
/// Process id of the daemon that owns platform UI state, when this raw
/// transport is daemon-backed. Linux lifecycle tests use it to inspect
/// the daemon's kernel thread accounting without relying on `ps` races.
pub fn daemon_pid(&self) -> Option<u32> {
self._daemon.as_ref().map(|daemon| daemon.pid)
}
/// Spawn the driver with piped stdio. Returns `None` (with a skip eprintln)
/// if the binary isn't built — callers early-return so an un-built binary
/// skips rather than fails.

3
packages/cua-driver/rust/crates/cua-driver/Cargo.toml
View file

@ -97,3 +97,6 @@ windows = { version = "0.61", features = [
"Win32_Security",
"Win32_System_Threading",
] }
[target.'cfg(target_os = "macos")'.dev-dependencies]
core-foundation = "0.10"

Some files were not shown because too many files have changed in this diff Show more