mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-25 08:28:02 +00:00
refactor(placement): migrate remaining native read callers (#166445)
* refactor(placement): run lifecycle and grant preparation in workers * test(placement): retain lifecycle boundaries in worker fixtures * refactor(placement): prepare native caller reads in workers * test(placement): align startup and recovery owner lifetimes * perf(placement): reuse Stop grants and acknowledged state * test(placement): settle media owners between fixtures * perf(placement): avoid single-query projection transactions * test(placement): fix async caller integration fixtures Match combined placement reads and validate device fixtures with one transport. Wait for native preflight cancellation and exact resource-close settlement while preserving stale-response, no-allocation, and recovery assertions. Reuse the existing runtime fixture to keep the test file within its line budget, and restore sorted worker-test routing. P2 autoreview is scoped-clean. The final callback receiver annotation emits identical JavaScript to the reviewed fixture. Focused boundary, dispatch, model-selection, planner, type, and lint checks passed; remaining repeated attribution and SQL diagnostics are recorded in the PR landing evidence. * perf(placement): reuse reply media custody facts Reuse the workspace already resolved by the media custody authority check for normalization and local roots. Commentary shares the same snapshot across both phases; text-only replies skip unused root preparation. Keep live authority rereads before effects and preserve foreign-commit visibility without adding host SQL, freshness probes, or schema checks. Validation: 78 focused media and dispatch tests passed in 557.58 seconds; production and affected test types, scoped lint, and P2 review passed. The inclusive SQL gate remains pending a fresh alternating comparison. * refactor(placement): require prepared media workspace facts Remove the async workspace fallback after media consumers switch to the custody owner's captured facts. Require the captured scope for path normalization and derive local roots from it without another getter. Preserve live authority checks and requester policy at each effect. Migrate helper tests through canonical session persistence and real scope capture, preserving permission, workspace, cancellation and output checks. No released SDK surface, schema, retention or durability contract changes. Validation: all 95 focused tests passed in 617.26 seconds; production and affected test types, scoped lint, and a fresh P2 review passed. The earlier cf8 head has green CI; final-head SQL qualification is pending.
This commit is contained in:
Peter Steinberger
GitHub
parent
0abba2b160
commit
83a95401fb
69 changed files with 1125 additions and 573 deletions
|
|
@ -27,6 +27,8 @@ describe("runReplyAgent media delivery ownership", () => {
|
|||
async ({ sessionKey, provider, gateway }) => {
|
||||
const { normalizeWebchatReplyMediaPathsForDisplay } =
|
||||
await import("../../gateway/server-methods/chat-reply-media.js");
|
||||
const { seedWebchatReplyMediaScope } =
|
||||
await import("../../gateway/server-methods/chat-reply-media.test-support.js");
|
||||
const { drainGlobalSingletonLifecycleState } =
|
||||
await import("../../shared/global-singleton.js");
|
||||
const { withOpenClawTestState } = await import("../../test-utils/openclaw-test-state.js");
|
||||
|
|
@ -93,7 +95,7 @@ describe("runReplyAgent media delivery ownership", () => {
|
|||
}
|
||||
if (gateway) {
|
||||
expect(result.mediaUrls).toEqual(sources);
|
||||
const [display] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg: config,
|
||||
agentId: "qa",
|
||||
sessionKey,
|
||||
|
|
@ -103,6 +105,9 @@ describe("runReplyAgent media delivery ownership", () => {
|
|||
permissionMode: "workspace",
|
||||
sessionRoot: selected,
|
||||
},
|
||||
});
|
||||
const [display] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: [result],
|
||||
});
|
||||
expect(display?.mediaUrls).toHaveLength(1);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import type { SessionEntry } from "../../config/sessions/types.js";
|
|||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
prepareSessionWorkerPlacementMutationCheckAsync,
|
||||
readSessionWorkerPlacementAsync,
|
||||
resolveWorkerPlacementArchiveRestoreError,
|
||||
type SessionWorkerPlacementContext,
|
||||
} from "../../gateway/worker-environments/session-placement-lifecycle.js";
|
||||
|
|
@ -93,7 +94,10 @@ async function restoreArchivedDispatchSession(params: {
|
|||
}
|
||||
const snapshotSessionId = entry.sessionId;
|
||||
const snapshotArchivedAt = entry.archivedAt;
|
||||
const canRestore = (currentEntry: SessionEntry) => {
|
||||
const canRestore = (
|
||||
currentEntry: SessionEntry,
|
||||
prepared?: { placement: Awaited<ReturnType<typeof readSessionWorkerPlacementAsync>> },
|
||||
) => {
|
||||
if (
|
||||
currentEntry.sessionId !== snapshotSessionId ||
|
||||
currentEntry.archivedAt !== snapshotArchivedAt ||
|
||||
|
|
@ -102,11 +106,13 @@ async function restoreArchivedDispatchSession(params: {
|
|||
return false;
|
||||
}
|
||||
try {
|
||||
const placement = currentEntry.sessionId
|
||||
? placementContext.workerSessionPlacementService
|
||||
?.getMany([currentEntry.sessionId])
|
||||
.get(currentEntry.sessionId)
|
||||
: undefined;
|
||||
const placement = prepared
|
||||
? prepared.placement
|
||||
: currentEntry.sessionId
|
||||
? placementContext.workerSessionPlacementService
|
||||
?.getMany([currentEntry.sessionId])
|
||||
.get(currentEntry.sessionId)
|
||||
: undefined;
|
||||
return !resolveWorkerPlacementArchiveRestoreError({
|
||||
context: placementContext,
|
||||
key: sessionKey,
|
||||
|
|
@ -122,7 +128,15 @@ async function restoreArchivedDispatchSession(params: {
|
|||
run: async () => {
|
||||
const scope = { sessionKey, storePath };
|
||||
const currentEntry = loadSessionEntryReadOnly(scope);
|
||||
if (!currentEntry || !canRestore(currentEntry)) {
|
||||
if (
|
||||
!currentEntry ||
|
||||
!canRestore(currentEntry, {
|
||||
placement: await readSessionWorkerPlacementAsync({
|
||||
context: placementContext,
|
||||
sessionId: currentEntry.sessionId,
|
||||
}),
|
||||
})
|
||||
) {
|
||||
return currentEntry;
|
||||
}
|
||||
let assertCommitAllowed: (() => void) | undefined;
|
||||
|
|
|
|||
|
|
@ -196,7 +196,6 @@ export function createGitHubPublicationCoordinatorMethods(params: {
|
|||
sessionKey: input.sessionKey,
|
||||
agentId: input.agentId,
|
||||
});
|
||||
assertRequester();
|
||||
const loaded = initialAuthority.loaded;
|
||||
const lifecycleRevision = loaded.entry?.lifecycleRevision ?? null;
|
||||
const session = {
|
||||
|
|
@ -205,7 +204,8 @@ export function createGitHubPublicationCoordinatorMethods(params: {
|
|||
agentId: input.agentId,
|
||||
lifecycleRevision,
|
||||
};
|
||||
const placement = params.placements.get(sessionId);
|
||||
const placement = await params.placements.getAsync(sessionId);
|
||||
assertRequester();
|
||||
const validateLocalExecution = () => {
|
||||
const current = params.placements.get(sessionId);
|
||||
return (!current || current.state === "local") && !current?.turnClaim;
|
||||
|
|
@ -360,7 +360,8 @@ export function createGitHubPublicationCoordinatorMethods(params: {
|
|||
worktreeOwner.assertCurrent();
|
||||
return publicationResult(insertSessionRequest());
|
||||
}
|
||||
const current = params.placements.get(sessionId);
|
||||
const current = await params.placements.getAsync(sessionId);
|
||||
assertRequester();
|
||||
if ((current && current.state !== "local") || current?.turnClaim) {
|
||||
throw new Error("GitHub publication session authority changed after verification.");
|
||||
}
|
||||
|
|
@ -402,11 +403,12 @@ export function createGitHubPublicationCoordinatorMethods(params: {
|
|||
);
|
||||
const failures: Error[] = [];
|
||||
const blockedWorktrees = new Set<string>();
|
||||
const placements = await params.placements.getManyAsync(rows.map((row) => row.session_id));
|
||||
for (const row of rows) {
|
||||
if (
|
||||
blockedWorktrees.has(row.worktree_id) ||
|
||||
pending.has(row.session_id) ||
|
||||
params.placements.get(row.session_id)?.turnClaim
|
||||
placements.get(row.session_id)?.turnClaim
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ export function createGitHubPublicationRuntime(params: {
|
|||
}
|
||||
};
|
||||
const publishAcceptedWorkspace = async (claim: WorkerSessionTurnClaim) => {
|
||||
const placement = params.placements.get(claim.sessionId);
|
||||
const placement = await params.placements.getAsync(claim.sessionId);
|
||||
if (!placement) {
|
||||
params.warn(`GitHub publication deferred because placement ${claim.sessionId} disappeared.`);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -147,11 +147,11 @@ export function createGitHubPublicationCoordinator(params: {
|
|||
): Promise<SessionGitHubPublicationResult> => {
|
||||
ensureSchema();
|
||||
const assertRequester = request.requester.assertCurrent;
|
||||
const placement = await params.placements.getAsync(request.claim.sessionId);
|
||||
assertRequester();
|
||||
if (!params.placements.validateTurnClaim(request.claim)) {
|
||||
throw new Error("GitHub publication lost the live session turn claim.");
|
||||
}
|
||||
const placement = params.placements.get(request.claim.sessionId);
|
||||
if (
|
||||
!placement ||
|
||||
placement.sessionKey !== request.sessionKey ||
|
||||
|
|
|
|||
|
|
@ -253,9 +253,14 @@ export function createRepositoryGitHubPublicationRecovery(params: {
|
|||
},
|
||||
async resumeSessionRequests(): Promise<void> {
|
||||
const failures: Error[] = [];
|
||||
for (let row of listRepositoryGitHubPublications({ ownerProfileId: null, pending: true })) {
|
||||
const rows = listRepositoryGitHubPublications({ ownerProfileId: null, pending: true });
|
||||
const currentPlacements = await placements.getManyAsync(rows.map((row) => row.session_id));
|
||||
for (let row of rows) {
|
||||
try {
|
||||
if (placements.get(row.session_id)?.turnClaim || params.isExecuting(row.request_id)) {
|
||||
if (
|
||||
currentPlacements.get(row.session_id)?.turnClaim ||
|
||||
params.isExecuting(row.request_id)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
await placements.withRepositoryWorkspaceReservation(
|
||||
|
|
|
|||
|
|
@ -428,8 +428,9 @@ export function createRepositoryGitHubPublicationCoordinator(params: {
|
|||
}
|
||||
const loaded = loadGatewaySessionEntryReadOnly(input.sessionKey!, { agentId: input.agentId });
|
||||
const placement = loaded.entry?.sessionId
|
||||
? placements.get(loaded.entry.sessionId)
|
||||
? await placements.getAsync(loaded.entry.sessionId)
|
||||
: undefined;
|
||||
input.requester.assertCurrent();
|
||||
const currentClaim = placement ? exactClaimForPlacement(placement) : undefined;
|
||||
if (input.expectedRunId !== undefined && input.expectedRunId !== currentClaim?.runId) {
|
||||
throw new Error("GitHub publication run identity changed.");
|
||||
|
|
@ -439,7 +440,7 @@ export function createRepositoryGitHubPublicationCoordinator(params: {
|
|||
if (
|
||||
terminalRepositoryGitHubPublication(row) ||
|
||||
claim ||
|
||||
placements.get(row.session_id)?.turnClaim
|
||||
(await placements.getAsync(row.session_id))?.turnClaim
|
||||
) {
|
||||
return projectGitHubPublicationResult(row);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import {
|
|||
isCurrentPlacementTurnClaim,
|
||||
type WorkerSessionTurnClaim,
|
||||
} from "./worker-environments/placement-record.js";
|
||||
import { readSessionWorkerPlacementAsync } from "./worker-environments/session-placement-lifecycle.js";
|
||||
|
||||
export type NodeClaudeSkillRuntime = {
|
||||
node: NodeSession;
|
||||
|
|
@ -83,7 +84,10 @@ export async function prepareNodeClaudeSkillRuntime(
|
|||
const caller = getGatewayToolCallerIdentity();
|
||||
const placements = gateway.workerSessionPlacementService;
|
||||
const readPlacement = () => placements?.getMany([run.sessionId]).get(run.sessionId);
|
||||
const placement = readPlacement();
|
||||
const placement = await readSessionWorkerPlacementAsync({
|
||||
context: gateway,
|
||||
sessionId: run.sessionId,
|
||||
});
|
||||
const persistedClaim = placement?.turnClaim;
|
||||
const claim: WorkerSessionTurnClaim | undefined =
|
||||
persistedClaim?.owner === "local"
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import { resolveRequestedSessionAgentId } from "../session-request-agent.js";
|
|||
import { hiddenSessionNotFound } from "../session-sharing-policy.js";
|
||||
import { createSessionListEntryFilter } from "../session-sharing.js";
|
||||
import { readWorkerPlacementIdentity } from "../worker-environments/placement-projector.js";
|
||||
import { readSessionWorkerPlacementAsync } from "../worker-environments/session-placement-lifecycle.js";
|
||||
import { resolveAgentIdOrRespondError } from "./agent-id-shared.js";
|
||||
import type { ChatMetadataReadParams } from "./chat-metadata-contract.js";
|
||||
import { prepareChatMetadataSessionRead } from "./chat-metadata-session-read.js";
|
||||
|
|
@ -96,9 +97,7 @@ export async function resolveChatMetadataReadParams(
|
|||
try {
|
||||
assertVisible();
|
||||
const sessionId = session.entry?.sessionId;
|
||||
const placement = sessionId
|
||||
? context.workerSessionPlacementService?.getMany([sessionId]).get(sessionId)
|
||||
: undefined;
|
||||
const placement = await readSessionWorkerPlacementAsync({ context, sessionId });
|
||||
const workerInference = placement
|
||||
? readWorkerPlacementIdentity(placement, context.workerEnvironmentService)?.inference
|
||||
: undefined;
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ import {
|
|||
} from "../managed-image-attachments.test-support.js";
|
||||
import { makeMockHttpResponse } from "../test-http-response.js";
|
||||
import { buildAssistantReplyContent } from "./chat-assistant-content.js";
|
||||
import { normalizeWebchatReplyMediaPathsForDisplay } from "./chat-reply-media.js";
|
||||
import {
|
||||
captureWebchatReplyMediaScope,
|
||||
normalizeWebchatReplyMediaPathsForDisplay,
|
||||
} from "./chat-reply-media.js";
|
||||
|
||||
const { getRuntimeConfig, readMessages, runFfprobe, runFfmpeg } = vi.hoisted(() => ({
|
||||
getRuntimeConfig: vi.fn<() => OpenClawConfig>(() => ({})),
|
||||
|
|
@ -103,6 +106,12 @@ it("streams a trusted owner webchat video over 16 MiB into managed storage and s
|
|||
}),
|
||||
).rejects.toThrow(/Managed video attachment.*could not be prepared/u);
|
||||
|
||||
const scope = captureWebchatReplyMediaScope({
|
||||
cfg,
|
||||
sessionKey: SESSION_KEY,
|
||||
agentId: "main",
|
||||
sessionLoadOptions: { agentId: "main" },
|
||||
});
|
||||
let largestAllocation = 0;
|
||||
const allocateSafe = Buffer.alloc.bind(Buffer);
|
||||
const allocate = Buffer.allocUnsafe.bind(Buffer);
|
||||
|
|
@ -131,10 +140,7 @@ it("streams a trusted owner webchat video over 16 MiB into managed storage and s
|
|||
});
|
||||
try {
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
cfg,
|
||||
sessionKey: SESSION_KEY,
|
||||
agentId: "main",
|
||||
sessionEntry: undefined,
|
||||
...scope,
|
||||
payloads: [directive],
|
||||
});
|
||||
expect(payloads[0]).toMatchObject({
|
||||
|
|
|
|||
21
src/gateway/server-methods/chat-reply-media.test-support.ts
Normal file
21
src/gateway/server-methods/chat-reply-media.test-support.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import { replaceSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import { loadSessionEntry } from "../session-utils.js";
|
||||
import { captureWebchatReplyMediaScope } from "./chat-reply-media.js";
|
||||
|
||||
export async function seedWebchatReplyMediaScope(
|
||||
params: Parameters<typeof captureWebchatReplyMediaScope>[0] & { sessionEntry: SessionEntry },
|
||||
) {
|
||||
const { sessionEntry, ...captureParams } = params;
|
||||
const sessionLoadOptions = { agentId: params.agentId, ...params.sessionLoadOptions };
|
||||
const target = loadSessionEntry(params.sessionKey, sessionLoadOptions);
|
||||
await replaceSessionEntry(
|
||||
{
|
||||
agentId: target.agentId,
|
||||
sessionKey: target.canonicalKey,
|
||||
storePath: target.storePath,
|
||||
},
|
||||
sessionEntry,
|
||||
);
|
||||
return captureWebchatReplyMediaScope({ ...captureParams, sessionLoadOptions });
|
||||
}
|
||||
|
|
@ -29,7 +29,10 @@ import {
|
|||
buildAssistantReplyContent,
|
||||
buildAssistantReplyContentFromInputs,
|
||||
} from "./chat-assistant-content.js";
|
||||
import { normalizeWebchatReplyMediaPathsForDisplay } from "./chat-reply-media.js";
|
||||
import {
|
||||
captureWebchatReplyMediaScope,
|
||||
normalizeWebchatReplyMediaPathsForDisplay,
|
||||
} from "./chat-reply-media.js";
|
||||
import { buildWebchatAssistantMessageFromReplyPayloads } from "./chat-webchat-media.js";
|
||||
|
||||
const PNG_BYTES = Buffer.from(
|
||||
|
|
@ -135,11 +138,14 @@ describe("normalizeWebchatReplyMediaPathsForDisplay", () => {
|
|||
cfg: OpenClawConfig;
|
||||
payloads: ReplyMediaPayloads;
|
||||
}) {
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = captureWebchatReplyMediaScope({
|
||||
cfg: params.cfg,
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
agentId: "main",
|
||||
sessionEntry: undefined,
|
||||
sessionLoadOptions: { agentId: "main" },
|
||||
});
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: params.payloads,
|
||||
});
|
||||
return payload;
|
||||
|
|
|
|||
|
|
@ -79,30 +79,39 @@ export function captureWebchatReplyMediaScope(
|
|||
sessionKey: string;
|
||||
sessionLoadOptions?: Parameters<typeof loadSessionEntry>[1];
|
||||
},
|
||||
): WebchatReplyMediaScope & { sessionKey: string; assertCurrent: () => void } {
|
||||
): WebchatReplyMediaScope & {
|
||||
sessionKey: string;
|
||||
workspace: ReturnType<typeof resolveWebchatReplyWorkspace>;
|
||||
assertCurrent: () => void;
|
||||
} {
|
||||
const readEntry = () => loadSessionEntry(params.sessionKey, params.sessionLoadOptions).entry;
|
||||
const sessionEntry = readEntry();
|
||||
const scope = { ...params, sessionEntry: sessionEntry ? { ...sessionEntry } : undefined };
|
||||
const authority = (entry: SessionEntry | undefined) => {
|
||||
const currentScope = { ...scope, sessionEntry: entry };
|
||||
const workspace = resolveWebchatReplyWorkspace(currentScope);
|
||||
return JSON.stringify([
|
||||
entry?.sessionId,
|
||||
entry?.lifecycleRevision,
|
||||
entry?.permissionMode,
|
||||
entry?.execNode,
|
||||
entry?.repositoryWorkspaceId,
|
||||
workspace.remote,
|
||||
workspace.workspaceDir,
|
||||
resolveWebchatReplyWorkspaceOnly(currentScope),
|
||||
]);
|
||||
return {
|
||||
workspace,
|
||||
key: JSON.stringify([
|
||||
entry?.sessionId,
|
||||
entry?.lifecycleRevision,
|
||||
entry?.permissionMode,
|
||||
entry?.execNode,
|
||||
entry?.repositoryWorkspaceId,
|
||||
workspace.remote,
|
||||
workspace.workspaceDir,
|
||||
resolveWebchatReplyWorkspaceOnly(currentScope),
|
||||
]),
|
||||
};
|
||||
};
|
||||
const expected = authority(scope.sessionEntry);
|
||||
return {
|
||||
...scope,
|
||||
// The custody fence already read this workspace; preparation consumes that same snapshot.
|
||||
workspace: expected.workspace,
|
||||
assertCurrent: () => {
|
||||
params.assertCurrent?.();
|
||||
if (authority(readEntry()) !== expected) {
|
||||
if (authority(readEntry()).key !== expected.key) {
|
||||
throw new Error("Session media access changed before attachment delivery.");
|
||||
}
|
||||
},
|
||||
|
|
@ -151,7 +160,9 @@ export async function withPreparedWebchatReplyMedia<T>(
|
|||
const payload = payloads[index];
|
||||
return payload ? replaceChatSendReplyPayload(input, payload) : [];
|
||||
});
|
||||
const localRoots = getWebchatReplyMediaLocalRoots({ ...scope, storePath: params.storePath });
|
||||
const localRoots = hasMedia
|
||||
? getWebchatReplyMediaLocalRoots({ ...scope, storePath: params.storePath })
|
||||
: [];
|
||||
return prepare({
|
||||
payloads,
|
||||
inputsByIndex,
|
||||
|
|
@ -230,9 +241,9 @@ function resolveWebchatReplyWorkspaceOnly(params: WebchatReplyMediaScope): boole
|
|||
|
||||
/** Trusted audio bypasses staging, but its reader must use the same session workspace. */
|
||||
export function getWebchatReplyMediaLocalRoots(
|
||||
params: WebchatReplyMediaScope & { storePath?: string },
|
||||
params: ReturnType<typeof captureWebchatReplyMediaScope> & { storePath?: string },
|
||||
): readonly string[] {
|
||||
const { remote, workspaceDir } = resolveWebchatReplyWorkspace(params);
|
||||
const { remote, workspaceDir } = params.workspace;
|
||||
if (remote) {
|
||||
return [getMediaDir()];
|
||||
}
|
||||
|
|
@ -280,9 +291,7 @@ function shouldPreserveDisplayMediaUrl(payload: ReplyPayload, mediaUrl: string):
|
|||
|
||||
/** Normalize reply media paths for webchat display without leaking sensitive media. */
|
||||
export async function normalizeWebchatReplyMediaPathsForDisplay(
|
||||
params: WebchatReplyMediaScope & {
|
||||
sessionKey: string;
|
||||
accountId?: string;
|
||||
params: ReturnType<typeof captureWebchatReplyMediaScope> & {
|
||||
payloads: ReplyPayload[];
|
||||
},
|
||||
): Promise<ReplyPayload[]> {
|
||||
|
|
@ -298,7 +307,7 @@ export async function normalizeWebchatReplyMediaPathsForDisplay(
|
|||
) {
|
||||
return params.payloads;
|
||||
}
|
||||
const { remote, workspaceDir } = resolveWebchatReplyWorkspace(params);
|
||||
const { remote, workspaceDir } = params.workspace;
|
||||
if (!workspaceDir) {
|
||||
return params.payloads;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import {
|
|||
getWebchatReplyMediaLocalRoots,
|
||||
normalizeWebchatReplyMediaPathsForDisplay,
|
||||
} from "./chat-reply-media.js";
|
||||
import { seedWebchatReplyMediaScope } from "./chat-reply-media.test-support.js";
|
||||
|
||||
const PNG_BYTES = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
|
||||
|
|
@ -146,11 +147,14 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
spawnedWorkspaceDir: worktree,
|
||||
spawnedCwd: path.join(worktree, "nested"),
|
||||
};
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionEntry,
|
||||
});
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: [{ mediaUrls: ["./chart.png"] }],
|
||||
});
|
||||
const stagedPath = requireString(payload?.mediaUrls?.[0], "staged workspace image");
|
||||
|
|
@ -159,11 +163,7 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
sessionKey: TEST_SESSION_KEY,
|
||||
agentId: "main",
|
||||
payloads: [{ mediaUrls: [sourcePath], trustedLocalMedia: true }],
|
||||
managedMediaLocalRoots: getWebchatReplyMediaLocalRoots({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionEntry,
|
||||
}),
|
||||
managedMediaLocalRoots: getWebchatReplyMediaLocalRoots(scope),
|
||||
});
|
||||
expect(assistantContent).toEqual([expect.objectContaining({ type: "image" })]);
|
||||
});
|
||||
|
|
@ -207,11 +207,14 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
sessionRoot: selected,
|
||||
permissionMode: mode,
|
||||
};
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionEntry,
|
||||
});
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: imagePaths.map((source) => ({ mediaUrls: [source] })),
|
||||
});
|
||||
for (const [index, payload] of payloads.entries()) {
|
||||
|
|
@ -225,13 +228,10 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
}
|
||||
}
|
||||
const trustedAudioPayloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionEntry,
|
||||
...scope,
|
||||
payloads: audioPaths.map((source) => ({ mediaUrls: [source], trustedLocalMedia: true })),
|
||||
});
|
||||
const localRoots = getWebchatReplyMediaLocalRoots({ cfg, agentId: "main", sessionEntry });
|
||||
const localRoots = getWebchatReplyMediaLocalRoots(scope);
|
||||
const { assistantContent } = await buildAssistantReplyContent({
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
agentId: "main",
|
||||
|
|
@ -255,7 +255,7 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
const source = path.join(selected, "private.png");
|
||||
await fs.mkdir(selected, { recursive: true });
|
||||
await fs.writeFile(source, PNG_BYTES);
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
|
|
@ -265,6 +265,9 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
sessionRoot: selected,
|
||||
permissionMode,
|
||||
},
|
||||
});
|
||||
const [payload] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: [{ mediaUrls: [source] }],
|
||||
});
|
||||
expect(payload?.mediaUrls).toBeUndefined();
|
||||
|
|
@ -304,7 +307,7 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
throw new Error("expected sandbox workspace");
|
||||
}
|
||||
await fs.writeFile(path.join(sandbox.workspaceDir, "sandbox.png"), PNG_BYTES);
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
|
|
@ -314,6 +317,9 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
sessionRoot: selected,
|
||||
permissionMode,
|
||||
},
|
||||
});
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: [{ mediaUrls: [source] }, { mediaUrls: ["./sandbox.png"] }],
|
||||
});
|
||||
expect(payloads[0]?.mediaUrls).toBeUndefined();
|
||||
|
|
@ -364,11 +370,14 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
const remoteUrl = "https://example.test/remote.png";
|
||||
const dataUrl = dataImageUrl();
|
||||
const sources = [...rawImages, ...managedImages, remoteUrl, dataUrl];
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionEntry,
|
||||
});
|
||||
const payloads = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: sources.map((source) => ({ mediaUrls: [source] })),
|
||||
});
|
||||
for (const payload of payloads.slice(0, rawImages.length)) {
|
||||
|
|
@ -391,12 +400,7 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
.path,
|
||||
),
|
||||
);
|
||||
const localRoots = getWebchatReplyMediaLocalRoots({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionEntry,
|
||||
storePath,
|
||||
});
|
||||
const localRoots = getWebchatReplyMediaLocalRoots({ ...scope, storePath });
|
||||
const { assistantContent } = await buildAssistantReplyContent({
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
agentId: "main",
|
||||
|
|
@ -434,11 +438,14 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
execNode: "remote-node",
|
||||
permissionMode: "full",
|
||||
};
|
||||
const [untrusted] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
const scope = await seedWebchatReplyMediaScope({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionEntry,
|
||||
});
|
||||
const [untrusted] = await normalizeWebchatReplyMediaPathsForDisplay({
|
||||
...scope,
|
||||
payloads: [{ mediaUrls: [aliasedSource] }],
|
||||
});
|
||||
expect(untrusted?.mediaUrls).toBeUndefined();
|
||||
|
|
@ -446,11 +453,7 @@ describe("WebChat reply media workspace ownership", () => {
|
|||
sessionKey: TEST_SESSION_KEY,
|
||||
agentId: "main",
|
||||
payloads: [{ mediaUrls: [aliasedSource], trustedLocalMedia: true }],
|
||||
managedMediaLocalRoots: getWebchatReplyMediaLocalRoots({
|
||||
cfg,
|
||||
agentId: "main",
|
||||
sessionEntry,
|
||||
}),
|
||||
managedMediaLocalRoots: getWebchatReplyMediaLocalRoots(scope),
|
||||
});
|
||||
expect(assistantContent).toEqual([expect.objectContaining({ type: "attachment_error" })]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import {
|
|||
onInternalSessionTranscriptUpdate,
|
||||
type InternalSessionTranscriptUpdate,
|
||||
} from "../../sessions/transcript-events.js";
|
||||
import { drainGlobalSingletonLifecycleState } from "../../shared/global-singleton.js";
|
||||
import { withOpenClawTestState } from "../../test-utils/openclaw-test-state.js";
|
||||
import { projectChatDisplayMessages } from "../chat-display-projection.js";
|
||||
import { cleanupManagedOutgoingMediaRecords } from "../managed-image-attachments.js";
|
||||
|
|
@ -713,6 +714,7 @@ describe("webchat commentary media", () => {
|
|||
hookSpy?.mockRestore();
|
||||
publicationSpy?.mockRestore();
|
||||
rewriteSpy?.mockRestore();
|
||||
await drainGlobalSingletonLifecycleState();
|
||||
}
|
||||
});
|
||||
await fixture.track(body);
|
||||
|
|
|
|||
|
|
@ -222,6 +222,7 @@ function observeChatSendCommentaryMedia(params: AssistantCommentaryMediaCustodyP
|
|||
...mediaScope,
|
||||
payloads: mediaUrls.map((url) => ({ mediaUrls: [url] })),
|
||||
});
|
||||
const localRoots = getWebchatReplyMediaLocalRoots(mediaScope);
|
||||
assertCurrent();
|
||||
for (const [index, payload] of payloads.entries()) {
|
||||
// History ownership starts after the rewrite; GC can run during preparation.
|
||||
|
|
@ -229,7 +230,7 @@ function observeChatSendCommentaryMedia(params: AssistantCommentaryMediaCustodyP
|
|||
sessionKey: scope.sessionKey,
|
||||
agentId: scope.agentId,
|
||||
items: prepareOutgoingMediaFromReplyPayload(payload),
|
||||
localRoots: getWebchatReplyMediaLocalRoots(mediaScope),
|
||||
localRoots,
|
||||
continueOnPrepareError: true,
|
||||
assertCurrent: mediaScope.assertCurrent,
|
||||
abortSignal: params.abortSignal,
|
||||
|
|
|
|||
|
|
@ -346,6 +346,7 @@ describe("direct session model catalogs", () => {
|
|||
"catalog owner",
|
||||
] as const)("revalidates the selected model catalog after %s", async (change) => {
|
||||
await withOpenClawTestState(isolated, async (state) => {
|
||||
let closedStorePath: string | undefined;
|
||||
const f = fixture();
|
||||
await state.writeConfig(f.config);
|
||||
const scope = { agentId: "main", sessionKey: "agent:main:held-saved" };
|
||||
|
|
@ -382,6 +383,7 @@ describe("direct session model catalogs", () => {
|
|||
openOpenClawAgentDatabase(scope);
|
||||
} else {
|
||||
closeOpenClawAgentDatabaseByPath(database.path);
|
||||
closedStorePath = database.path;
|
||||
}
|
||||
} else if (change === "profile alias change") {
|
||||
publishUserProfileAliasChange();
|
||||
|
|
@ -417,6 +419,10 @@ describe("direct session model catalogs", () => {
|
|||
retryAfterMs: 0,
|
||||
}),
|
||||
);
|
||||
if (closedStorePath) {
|
||||
// Synchronous revocation precedes the async resource owner's settlement.
|
||||
await closeOpenClawAgentDatabaseByPathAsync(closedStorePath);
|
||||
}
|
||||
}
|
||||
await rotateDatabaseWorkers(projectionLane);
|
||||
expect(hasOpenClawAgentDatabaseAsyncResources()).toBe(false);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
} from "../../worker/workspace-inspection-protocol.js";
|
||||
import { captureSessionMutationRouting } from "../session-sharing-preparation.js";
|
||||
import { loadGatewaySessionEntryReadOnly } from "../session-utils.js";
|
||||
import { readSessionWorkerPlacementAsync } from "../worker-environments/session-placement-lifecycle.js";
|
||||
import type { GatewayRequestContext } from "./types.js";
|
||||
|
||||
type LoadedSession = ReturnType<typeof loadGatewaySessionEntryReadOnly>;
|
||||
|
|
@ -61,10 +62,10 @@ export async function resolveRepositoryWorkspaceAccess(
|
|||
loadGatewaySessionEntryReadOnly(loaded.canonicalKey, { agentId: repository.agentId }).entry,
|
||||
expectedRevision,
|
||||
);
|
||||
assertSession(repository.revision);
|
||||
const placements = context?.workerSessionPlacementService;
|
||||
const environments = context?.workerEnvironmentService;
|
||||
const placement = placements?.getMany([sessionId]).get(sessionId);
|
||||
const placement = await readSessionWorkerPlacementAsync({ context: context ?? {}, sessionId });
|
||||
assertSession(repository.revision);
|
||||
if (placement?.state !== "active" || !environments) {
|
||||
return {
|
||||
kind: "stored" as const,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import {
|
|||
} from "../worker-environments/placement-session-runtime.js";
|
||||
import { isFailedWorkerPlacementEnvironmentGone } from "../worker-environments/placement-target.js";
|
||||
import type { WorkerPlacementDispatchRequest } from "../worker-environments/service-contract.js";
|
||||
import { readSessionWorkerPlacementAsync } from "../worker-environments/session-placement-lifecycle.js";
|
||||
import type { WorkerSessionWorkspace } from "../worker-environments/session-workspace.js";
|
||||
import { listGatewayEnvironments } from "./environments.js";
|
||||
import { emitSessionsChanged } from "./session-change-event.js";
|
||||
|
|
@ -260,7 +261,7 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
if (!resolved) {
|
||||
return;
|
||||
}
|
||||
const { cfg, entry, session, service: dispatchService, reader: placementReader } = resolved;
|
||||
const { cfg, entry, session, service: dispatchService } = resolved;
|
||||
const { sessionId, sessionKey, agentId } = session;
|
||||
let { dispatchTarget } = resolved;
|
||||
const autoDevice = params.autoDevice === true;
|
||||
|
|
@ -355,7 +356,8 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
if (!autoDevice && dispatchTarget && !(await validateExecutionMode(dispatchTarget))) {
|
||||
return;
|
||||
}
|
||||
const existingPlacement = placementReader.getMany([sessionId]).get(sessionId);
|
||||
const existingPlacement = await readSessionWorkerPlacementAsync({ context, sessionId });
|
||||
sessionMutationAuthorization?.assertCurrent();
|
||||
if (
|
||||
existingPlacement?.state === "failed" &&
|
||||
!isFailedWorkerPlacementEnvironmentGone({
|
||||
|
|
@ -503,7 +505,7 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
deviceId: candidateTarget.deviceId,
|
||||
...session,
|
||||
attempted: attemptedPlacement,
|
||||
current: placementReader.getMany([sessionId]).get(sessionId),
|
||||
current: await readSessionWorkerPlacementAsync({ context, sessionId }),
|
||||
environments: context.workerEnvironmentService,
|
||||
})
|
||||
) {
|
||||
|
|
@ -539,13 +541,14 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
if (!resolved) {
|
||||
return;
|
||||
}
|
||||
const { entry, session, service: placementService, reader: placementReader } = resolved;
|
||||
const { entry, session, service: placementService } = resolved;
|
||||
const { sessionId, sessionKey } = session;
|
||||
if (entry.archivedAt !== undefined) {
|
||||
respondInvalidWorkerSession(respond, "cannot move an archived session");
|
||||
return;
|
||||
}
|
||||
const existingPlacement = placementReader.getMany([sessionId]).get(sessionId);
|
||||
const existingPlacement = await readSessionWorkerPlacementAsync({ context, sessionId });
|
||||
sessionMutationAuthorization?.assertCurrent();
|
||||
const retryAbandonment =
|
||||
"abandonSource" in params && isForceAbandonedWorkerPlacement(existingPlacement);
|
||||
if (
|
||||
|
|
@ -619,9 +622,10 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
if (!resolved) {
|
||||
return;
|
||||
}
|
||||
const { entry, session, service: placementService, reader: placementReader } = resolved;
|
||||
const { entry, session, service: placementService } = resolved;
|
||||
const { sessionId, sessionKey } = session;
|
||||
const existingPlacement = placementReader.getMany([sessionId]).get(sessionId);
|
||||
const existingPlacement = await readSessionWorkerPlacementAsync({ context, sessionId });
|
||||
sessionMutationAuthorization?.assertCurrent();
|
||||
const reportPlacementChange = (placement: WorkerSessionPlacementRecord | undefined): void => {
|
||||
if (
|
||||
!placement ||
|
||||
|
|
@ -663,7 +667,7 @@ export const sessionDispatchHandlers: GatewayRequestHandlers = {
|
|||
if (error instanceof SessionMutationAuthorizationChangedError) {
|
||||
throw error;
|
||||
}
|
||||
reportPlacementChange(placementReader.getMany([sessionId]).get(sessionId));
|
||||
reportPlacementChange(await readSessionWorkerPlacementAsync({ context, sessionId }));
|
||||
respondWorkerDispatchError(error, respond);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,12 +37,17 @@ import {
|
|||
type AcceptedWorkerInferenceSessionDrain,
|
||||
type WorkerInferenceSessionDrain,
|
||||
} from "../worker-environments/inference-control-internal.js";
|
||||
import type { WorkerSessionPlacementStore } from "../worker-environments/placement-store.js";
|
||||
import type {
|
||||
WorkerSessionPlacementStore,
|
||||
WorkerSessionPlacementRecord,
|
||||
} from "../worker-environments/placement-store.js";
|
||||
import { isCurrentWorkerWorkspacePendingResultOwner } from "../worker-environments/placement-workspace-result.js";
|
||||
import type { WorkerWorkspacePendingResult } from "../worker-environments/placement-workspace-result.types.js";
|
||||
import {
|
||||
prepareSessionWorkerPlacementArchiveCheckAsync,
|
||||
prepareSessionWorkerPlacementMutationCheckAsync,
|
||||
prepareSessionWorkerPlacementStop,
|
||||
readSessionWorkerPlacementAsync,
|
||||
} from "../worker-environments/session-placement-lifecycle.js";
|
||||
import { hasGatewaySessionAbortOwner } from "./chat-abort-authorization.js";
|
||||
import { abortChatRunsForSessionKeyWithPartials } from "./chat-abort-runtime.js";
|
||||
|
|
@ -241,11 +246,24 @@ export async function prepareSessionLifecycleDrain(
|
|||
}
|
||||
|
||||
params.authorize?.();
|
||||
const placementService: LifecyclePlacementService | undefined =
|
||||
params.context.workerSessionPlacementService;
|
||||
let placement: WorkerSessionPlacementRecord | undefined;
|
||||
if (params.sessionId) {
|
||||
const placements = params.context.workerSessionPlacementService;
|
||||
const pending = (await placements?.listPendingWorkspaceResultsAsync?.(params.sessionId))?.[0];
|
||||
params.authorize?.();
|
||||
const placement = placements?.getMany([params.sessionId]).get(params.sessionId);
|
||||
const preparedPlacement = await placementService?.prepareRuntimeRefresh?.(params.sessionId);
|
||||
let pending: WorkerWorkspacePendingResult | undefined;
|
||||
try {
|
||||
pending = preparedPlacement
|
||||
? preparedPlacement.pendingResult
|
||||
: (await placementService?.listPendingWorkspaceResultsAsync?.(params.sessionId))?.[0];
|
||||
placement = preparedPlacement
|
||||
? preparedPlacement.placement
|
||||
: await readSessionWorkerPlacementAsync(params);
|
||||
params.authorize?.();
|
||||
preparedPlacement?.assertCurrent();
|
||||
} finally {
|
||||
preparedPlacement?.release();
|
||||
}
|
||||
if (
|
||||
pending &&
|
||||
pending.workspaceAcceptedAtMs === null &&
|
||||
|
|
@ -285,11 +303,6 @@ export async function prepareSessionLifecycleDrain(
|
|||
const embeddedWork = params.sessionId
|
||||
? waitForEmbeddedAgentRunEnd(params.sessionId, timeoutMs)
|
||||
: Promise.resolve(true);
|
||||
const placementService: LifecyclePlacementService | undefined =
|
||||
params.context.workerSessionPlacementService;
|
||||
const placement = params.sessionId
|
||||
? placementService?.getMany([params.sessionId]).get(params.sessionId)
|
||||
: undefined;
|
||||
const placementWork = placement?.turnClaim
|
||||
? placementService?.waitForTurnClaimRelease
|
||||
? placementService
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import {
|
|||
import {
|
||||
resolveProtectedSessionVisibilityError,
|
||||
resolveSessionWorkerPlacementPatchError,
|
||||
prepareSessionWorkerPlacementPatchError,
|
||||
sessionLog,
|
||||
} from "./sessions-shared.js";
|
||||
import type { GatewayRequestContext } from "./types.js";
|
||||
|
|
@ -218,7 +219,7 @@ export async function prepareSessionPatchArchive(params: {
|
|||
if (!preview.ok) {
|
||||
return err(preview.error);
|
||||
}
|
||||
const previewPlacementError = resolveSessionWorkerPlacementPatchError({
|
||||
const previewPlacementError = await prepareSessionWorkerPlacementPatchError({
|
||||
agentId: freshResolved.agentId,
|
||||
cfg,
|
||||
context: params.context,
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ import type {
|
|||
MutationTarget,
|
||||
PreparedPatchTarget,
|
||||
} from "./sessions-patch-types.js";
|
||||
import { resolveSessionWorkerPlacementPatchError } from "./sessions-shared.js";
|
||||
import { prepareSessionWorkerPlacementPatchError } from "./sessions-shared.js";
|
||||
import type {
|
||||
GatewayClient,
|
||||
GatewayRequestContext,
|
||||
|
|
@ -159,7 +159,7 @@ export async function executeSessionPatchMutations(params: {
|
|||
}
|
||||
let initialPlacementPatchError: string | undefined;
|
||||
try {
|
||||
initialPlacementPatchError = resolveSessionWorkerPlacementPatchError({
|
||||
initialPlacementPatchError = await prepareSessionWorkerPlacementPatchError({
|
||||
agentId: resolved.agentId,
|
||||
cfg,
|
||||
context: params.context,
|
||||
|
|
|
|||
|
|
@ -18,27 +18,45 @@ import {
|
|||
resolveGatewaySessionStoreTargetWithStore,
|
||||
} from "../session-utils.js";
|
||||
import { resolveWorkerPlacementSessionRuntimeCapabilities } from "../worker-environments/placement-session-runtime.js";
|
||||
import type { SessionWorkerPlacementContext } from "../worker-environments/session-placement-lifecycle.js";
|
||||
import { resolveWorkerPlacementArchiveRestoreError } from "../worker-environments/session-placement-lifecycle.js";
|
||||
import {
|
||||
readSessionWorkerPlacementAsync,
|
||||
resolveWorkerPlacementArchiveRestoreError,
|
||||
type SessionWorkerPlacementContext,
|
||||
} from "../worker-environments/session-placement-lifecycle.js";
|
||||
import type { GatewayRequestContext, RespondFn } from "./types.js";
|
||||
|
||||
export { sessionLog } from "../session-log.js";
|
||||
|
||||
export function resolveSessionWorkerPlacementPatchError(params: {
|
||||
agentId: string;
|
||||
cfg: OpenClawConfig;
|
||||
context: SessionWorkerPlacementContext;
|
||||
entry: SessionEntry | undefined;
|
||||
key: string;
|
||||
patch: SessionsPatchParams;
|
||||
sessionKey: string;
|
||||
validateModelRuntime: boolean;
|
||||
}): string | undefined {
|
||||
const placement = params.entry?.sessionId
|
||||
? params.context.workerSessionPlacementService
|
||||
?.getMany([params.entry.sessionId])
|
||||
.get(params.entry.sessionId)
|
||||
: undefined;
|
||||
export async function prepareSessionWorkerPlacementPatchError(
|
||||
params: Parameters<typeof resolveSessionWorkerPlacementPatchError>[0],
|
||||
) {
|
||||
const placement = await readSessionWorkerPlacementAsync({
|
||||
context: params.context,
|
||||
sessionId: params.entry?.sessionId,
|
||||
});
|
||||
return resolveSessionWorkerPlacementPatchError(params, { placement });
|
||||
}
|
||||
|
||||
export function resolveSessionWorkerPlacementPatchError(
|
||||
params: {
|
||||
agentId: string;
|
||||
cfg: OpenClawConfig;
|
||||
context: SessionWorkerPlacementContext;
|
||||
entry: SessionEntry | undefined;
|
||||
key: string;
|
||||
patch: SessionsPatchParams;
|
||||
sessionKey: string;
|
||||
validateModelRuntime: boolean;
|
||||
},
|
||||
prepared?: { placement: Awaited<ReturnType<typeof readSessionWorkerPlacementAsync>> },
|
||||
): string | undefined {
|
||||
const placement = prepared
|
||||
? prepared.placement
|
||||
: params.entry?.sessionId
|
||||
? params.context.workerSessionPlacementService
|
||||
?.getMany([params.entry.sessionId])
|
||||
.get(params.entry.sessionId)
|
||||
: undefined;
|
||||
if (!placement || placement.state === "local") {
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -855,7 +855,7 @@ describe("sessions.dispatch", () => {
|
|||
const moves = createWorkerPlacementMoveService({
|
||||
placements: {
|
||||
beginPlacementMove: async () => ({ intent, placement: draining, joined }),
|
||||
getAsync: async () => existing,
|
||||
getWithMoveAsync: async () => ({ placement: existing, move: joined ? intent : undefined }),
|
||||
getPlacementMoveAsync: async () => (joined ? intent : undefined),
|
||||
recordPlacementMoveError,
|
||||
} as never,
|
||||
|
|
|
|||
|
|
@ -190,7 +190,11 @@ it.each(["success", "failed-write", "setup-failed-write"] as const)(
|
|||
store: { [target.sessionKey]: entry },
|
||||
};
|
||||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
placements: { get: () => placement as never, waitForTurnClaimRelease: async () => {} },
|
||||
placements: {
|
||||
get: () => placement as never,
|
||||
getAsync: async () => placement as never,
|
||||
waitForTurnClaimRelease: async () => {},
|
||||
},
|
||||
loadSessionRuntime: async () =>
|
||||
({
|
||||
managedWorktrees: {
|
||||
|
|
|
|||
|
|
@ -1,17 +1,42 @@
|
|||
import { afterEach, beforeEach, vi } from "vitest";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { upsertSessionEntryCore } from "../config/sessions/session-accessor.js";
|
||||
import { createDeferredCore } from "../shared/deferred.js";
|
||||
import { closeOpenClawAgentDatabaseByPathAsync } from "../state/openclaw-agent-db-lifecycle.js";
|
||||
import { createTestGatewayScheduler } from "../test-utils/gateway-scheduler-clock.js";
|
||||
import type { WorkerPlacementSessionWorkCancellation } from "./server-worker-placement-cancel.js";
|
||||
import { createGatewayWorkerPlacementRuntime } from "./server-worker-placement-startup.js";
|
||||
import { getWorkerPlacementStartupMocks } from "./server-worker-placement-startup.test-harness.js";
|
||||
import type { WorkerSessionPlacementIdentity } from "./worker-environments/placement-record.js";
|
||||
import type { createWorkerSessionPlacementStore } from "./worker-environments/placement-store.js";
|
||||
import * as support from "./worker-environments/service.test-support.js";
|
||||
|
||||
export function createHeldWorkspacePreflight() {
|
||||
const entered = createDeferredCore();
|
||||
const aborted = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let signal: AbortSignal | undefined;
|
||||
return {
|
||||
entered: entered.promise,
|
||||
aborted: aborted.promise,
|
||||
release: release.resolve,
|
||||
get signal() {
|
||||
return signal;
|
||||
},
|
||||
async run(this: void, request: { signal?: AbortSignal }) {
|
||||
signal = request.signal;
|
||||
signal?.addEventListener("abort", () => aborted.resolve(), { once: true });
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
signal?.throwIfAborted();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createRuntime(
|
||||
placements: ReturnType<typeof createWorkerSessionPlacementStore>,
|
||||
environments: ReturnType<typeof support.createService>,
|
||||
cancelSessionWork: WorkerPlacementSessionWorkCancellation = vi.fn(async () => {}),
|
||||
) {
|
||||
return createGatewayWorkerPlacementRuntime({
|
||||
scheduler: createTestGatewayScheduler(),
|
||||
|
|
@ -20,7 +45,7 @@ export function createRuntime(
|
|||
environments,
|
||||
gatewayNamespace: "gateway-test",
|
||||
warn: vi.fn(),
|
||||
cancelSessionWork: vi.fn(async () => {}),
|
||||
cancelSessionWork,
|
||||
revokeSessionAuthority: vi.fn(),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { setImmediate } from "node:timers/promises";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createTestGatewayScheduler } from "../test-utils/gateway-scheduler-clock.js";
|
||||
import { getWorkerPlacementStartupMocks } from "./server-worker-placement-startup.test-harness.js";
|
||||
import {
|
||||
publishWorkerEnvironmentFixture,
|
||||
|
|
@ -13,7 +12,6 @@ vi.mock("./worker-environments/workspace-sync-preflight.js", () => ({
|
|||
preflightWorkerWorkspace: workspace.preflight,
|
||||
}));
|
||||
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { upsertSessionEntryCore } from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
GatewayDrainingError,
|
||||
|
|
@ -28,12 +26,13 @@ import {
|
|||
} from "../sessions/session-lifecycle-admission.js";
|
||||
import { createDeferredCore } from "../shared/deferred.js";
|
||||
import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js";
|
||||
import type { WorkerPlacementSessionWorkCancellation } from "./server-worker-placement-cancel.js";
|
||||
import {
|
||||
createHeldWorkspacePreflight,
|
||||
createRuntime,
|
||||
setupSessionFixture,
|
||||
} from "./server-worker-placement-provision-cancellation.test-support.js";
|
||||
import { installWorkerPlacementReconcileGuard } from "./server-worker-placement-reconcile-guard.js";
|
||||
import { createGatewayWorkerPlacementRuntime } from "./server-worker-placement-startup.js";
|
||||
import {
|
||||
REQUEST as FIXTURE_REQUEST,
|
||||
seedActivePlacement,
|
||||
|
|
@ -200,13 +199,14 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
);
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const interrupted = createDeferredCore();
|
||||
const destinationAborted = createDeferredCore();
|
||||
let destinationSignal: AbortSignal | undefined;
|
||||
workspace.preflight.mockImplementation(async ({ signal }: { signal?: AbortSignal }) => {
|
||||
if (outcome === "published") {
|
||||
return;
|
||||
}
|
||||
destinationSignal = signal;
|
||||
signal?.addEventListener("abort", () => destinationAborted.resolve(), { once: true });
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
if (outcome === "preflight-error" || outcome === "canceled-preflight-error") {
|
||||
|
|
@ -227,6 +227,7 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
if (outcome === "published") {
|
||||
vi.mocked(harness.environments.createWithRequest).mockImplementation(async ({ signal }) => {
|
||||
destinationSignal = signal;
|
||||
signal?.addEventListener("abort", () => destinationAborted.resolve(), { once: true });
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
signal?.throwIfAborted();
|
||||
|
|
@ -237,19 +238,9 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
...support.createService(support.createProvider()),
|
||||
...harness.environments,
|
||||
};
|
||||
const runtime = createGatewayWorkerPlacementRuntime({
|
||||
scheduler: createTestGatewayScheduler(),
|
||||
getCommittedRuntimeConfig: getRuntimeConfig,
|
||||
placements,
|
||||
environments,
|
||||
gatewayNamespace: "gateway-test",
|
||||
warn: vi.fn(),
|
||||
cancelSessionWork: async (request) => {
|
||||
request.assertCurrent();
|
||||
request.onCancellationStarted?.();
|
||||
interrupted.resolve();
|
||||
},
|
||||
revokeSessionAuthority: vi.fn(),
|
||||
const runtime = createRuntime(placements, environments, async (request) => {
|
||||
request.assertCurrent();
|
||||
request.onCancellationStarted?.();
|
||||
});
|
||||
const transitions: Array<{ state: string; generation: number }> = [];
|
||||
const moving = runtime.dispatchService
|
||||
|
|
@ -287,7 +278,7 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
if (outcome !== "preflight-error") {
|
||||
stopping = runtime.dispatchService.reclaim(REQUEST).catch((error: unknown) => error);
|
||||
await Promise.race([
|
||||
interrupted.promise,
|
||||
destinationAborted.promise,
|
||||
stopping.then((result) => {
|
||||
throw result;
|
||||
}),
|
||||
|
|
@ -351,15 +342,8 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
);
|
||||
|
||||
it("cancels the exact preflight owner without admitting a later provider", async () => {
|
||||
const entered = createDeferredCore();
|
||||
const settled = createDeferredCore();
|
||||
let preflightSignal: AbortSignal | undefined;
|
||||
workspace.preflight.mockImplementation(async ({ signal }: { signal?: AbortSignal }) => {
|
||||
preflightSignal = signal;
|
||||
entered.resolve();
|
||||
await settled.promise;
|
||||
signal?.throwIfAborted();
|
||||
});
|
||||
const preflight = createHeldWorkspacePreflight();
|
||||
workspace.preflight.mockImplementation(preflight.run);
|
||||
const provision = vi.fn(async () => {
|
||||
throw new Error("unexpected provider entry");
|
||||
});
|
||||
|
|
@ -367,7 +351,7 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
const placements = createWorkerSessionPlacementStore({ database: support.testState.stateDb });
|
||||
const runtime = createRuntime(placements, environments);
|
||||
const dispatch = runtime.dispatchService.dispatch(REQUEST).catch((error: unknown) => error);
|
||||
await entered.promise;
|
||||
await preflight.entered;
|
||||
expect(placements.get(REQUEST.sessionId)).toBeUndefined();
|
||||
const stopping = runtime.dispatchService.reclaim(REQUEST);
|
||||
let stopped = false;
|
||||
|
|
@ -378,13 +362,17 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
() => {},
|
||||
);
|
||||
try {
|
||||
await setImmediate();
|
||||
await setImmediate();
|
||||
expect(preflightSignal?.aborted).toBe(true);
|
||||
await Promise.race([
|
||||
preflight.aborted,
|
||||
stopping.then(() => {
|
||||
throw new Error("Stop settled before interrupting the held preflight");
|
||||
}),
|
||||
]);
|
||||
expect(preflight.signal?.aborted).toBe(true);
|
||||
expect(stopped).toBe(false);
|
||||
expect(provision).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
settled.resolve();
|
||||
preflight.release();
|
||||
await dispatch;
|
||||
await stopping.catch(() => undefined);
|
||||
}
|
||||
|
|
@ -392,9 +380,10 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
expect(support.testState.store.list()).toEqual([]);
|
||||
});
|
||||
it.each(["missing", "reclaimed"] as const)(
|
||||
"cancels queued redispatch before the %s placement can allocate",
|
||||
"cancels redispatch from a %s placement before allocation during reconciliation",
|
||||
async (state) => {
|
||||
workspace.preflight.mockResolvedValue(undefined);
|
||||
const preflight = createHeldWorkspacePreflight();
|
||||
workspace.preflight.mockImplementation(preflight.run);
|
||||
const environments = support.createService(support.createProvider());
|
||||
const placements = createWorkerSessionPlacementStore({ database: support.testState.stateDb });
|
||||
if (state === "reclaimed") {
|
||||
|
|
@ -451,22 +440,46 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
() => "active",
|
||||
() => "cancelled",
|
||||
);
|
||||
const stopping = runtime.dispatchService.reclaim(REQUEST).then(
|
||||
(value) => value,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
await setImmediate();
|
||||
release.resolve();
|
||||
await sweep;
|
||||
expect(await dispatch).toBe("cancelled");
|
||||
const result = await stopping;
|
||||
if (state === "reclaimed") {
|
||||
expect(result).toMatchObject({ state: "reclaimed" });
|
||||
} else {
|
||||
expect(result).toBeInstanceOf(Error);
|
||||
let stopping: Promise<unknown> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
preflight.entered,
|
||||
dispatch.then((result) => {
|
||||
throw new Error(`Redispatch ended before preflight (${result})`);
|
||||
}),
|
||||
]);
|
||||
expect(placements.get(REQUEST.sessionId)?.state).toBe(
|
||||
state === "missing" ? undefined : "reclaimed",
|
||||
);
|
||||
stopping = runtime.dispatchService.reclaim(REQUEST).then(
|
||||
(value) => value,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
await Promise.race([
|
||||
preflight.aborted,
|
||||
stopping.then((result) => {
|
||||
throw result;
|
||||
}),
|
||||
]);
|
||||
expect(preflight.signal?.aborted).toBe(true);
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
preflight.release();
|
||||
release.resolve();
|
||||
await sweep;
|
||||
expect(await dispatch).toBe("cancelled");
|
||||
const result = await stopping;
|
||||
if (state === "reclaimed") {
|
||||
expect(result).toMatchObject({ state: "reclaimed" });
|
||||
} else {
|
||||
expect(result).toBeInstanceOf(Error);
|
||||
}
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
expect(support.testState.store.list()).toEqual([]);
|
||||
} finally {
|
||||
preflight.release();
|
||||
release.resolve();
|
||||
await Promise.allSettled([sweep, dispatch, stopping]);
|
||||
}
|
||||
expect(create).not.toHaveBeenCalled();
|
||||
expect(support.testState.store.list()).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -865,16 +878,10 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
executionMode: REQUEST.executionMode,
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "provider_failure" });
|
||||
const cancelSessionWork = vi.fn(
|
||||
async (
|
||||
request: Parameters<
|
||||
Parameters<typeof createGatewayWorkerPlacementRuntime>[0]["cancelSessionWork"]
|
||||
>[0],
|
||||
) => {
|
||||
request.assertCurrent();
|
||||
request.onCancellationStarted?.();
|
||||
},
|
||||
);
|
||||
const cancelSessionWork = vi.fn<WorkerPlacementSessionWorkCancellation>(async (request) => {
|
||||
request.assertCurrent();
|
||||
request.onCancellationStarted?.();
|
||||
});
|
||||
if (mode === "late-sweep") {
|
||||
// Recovery captures service methods when its runtime is created.
|
||||
const reconcileOnce = environments.reconcileOnce.bind(environments);
|
||||
|
|
@ -884,16 +891,7 @@ describe("dispatch Stop before provider allocation", () => {
|
|||
await reconcileOnce();
|
||||
});
|
||||
}
|
||||
const runtime = createGatewayWorkerPlacementRuntime({
|
||||
scheduler: createTestGatewayScheduler(),
|
||||
getCommittedRuntimeConfig: getRuntimeConfig,
|
||||
placements,
|
||||
environments,
|
||||
gatewayNamespace: "gateway-test",
|
||||
warn: vi.fn(),
|
||||
cancelSessionWork,
|
||||
revokeSessionAuthority: vi.fn(),
|
||||
});
|
||||
const runtime = createRuntime(placements, environments, cancelSessionWork);
|
||||
const uninstallGuard = installWorkerPlacementReconcileGuard({
|
||||
placements,
|
||||
environments,
|
||||
|
|
|
|||
198
src/gateway/server-worker-placement-reclaim.barriers.test.ts
Normal file
198
src/gateway/server-worker-placement-reclaim.barriers.test.ts
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
import { expect, it, vi } from "vitest";
|
||||
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
|
||||
import { createDeferredCore } from "../shared/deferred.js";
|
||||
import { createGatewayWorkerPlacementReclaimBarriers } from "./server-worker-placement-reclaim.js";
|
||||
|
||||
vi.mock("../config/config.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../config/config.js")>()),
|
||||
getRuntimeConfig: () => ({}),
|
||||
}));
|
||||
|
||||
function fixture(name: string, state: "active" | "failed" | "local" | "reclaimed" = "active") {
|
||||
const request = {
|
||||
sessionId: `session-${name}`,
|
||||
sessionKey: `agent:main:${name}`,
|
||||
agentId: "main",
|
||||
};
|
||||
const entry = { sessionId: request.sessionId, lifecycleRevision: "original", updatedAt: 1 };
|
||||
const target = {
|
||||
storePath: `/fixture/reclaim-preparation-${name}.sqlite`,
|
||||
canonicalKey: request.sessionKey,
|
||||
storeKeys: [request.sessionKey],
|
||||
agentId: request.agentId,
|
||||
store: { [request.sessionKey]: entry },
|
||||
};
|
||||
const placement = {
|
||||
...request,
|
||||
state,
|
||||
generation: 4,
|
||||
environmentId: "worker",
|
||||
activeOwnerEpoch: 7,
|
||||
};
|
||||
const cancel = vi.fn(async (input: { assertCurrent: () => void }) => input.assertCurrent());
|
||||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
placements: {
|
||||
get: () => ({ ...placement }) as never,
|
||||
getAsync: async () => ({ ...placement }) as never,
|
||||
waitForTurnClaimRelease: async () => {},
|
||||
},
|
||||
loadSessionRuntime: async () => ({
|
||||
managedWorktrees: { findLiveByOwner: async () => undefined },
|
||||
resolveGatewaySessionStoreTargetWithStore: () => target,
|
||||
resolveCanonicalSessionEntryFromStoreKeys: () => entry,
|
||||
}),
|
||||
cancelSessionWork: cancel,
|
||||
revokeSessionAuthority: vi.fn(),
|
||||
});
|
||||
const run = vi.fn(async () => ({ ...placement, state: "reclaimed" as const }) as never);
|
||||
const admit = (onInterrupt?: () => void) =>
|
||||
beginSessionWorkAdmission({
|
||||
scope: target.storePath,
|
||||
identities: [request.sessionKey, request.sessionId],
|
||||
assertAllowed: () => {},
|
||||
onInterrupt,
|
||||
});
|
||||
return {
|
||||
...request,
|
||||
entry,
|
||||
placement,
|
||||
cancel,
|
||||
run,
|
||||
admit,
|
||||
prepare: (options: Partial<Parameters<typeof barriers.runReclaimPreparation>[0]> = {}) =>
|
||||
barriers.runReclaimPreparation({ ...request, run, ...options }),
|
||||
};
|
||||
}
|
||||
|
||||
it("one failed Stop cannot reopen ingress while another Stop still owns its closure", async () => {
|
||||
const f = fixture("overlap");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let calls = 0;
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
assertCurrent();
|
||||
if (++calls === 1) {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
} else {
|
||||
throw new Error("second cancellation failed");
|
||||
}
|
||||
});
|
||||
const first = f.prepare();
|
||||
await entered.promise;
|
||||
try {
|
||||
await expect(f.prepare()).rejects.toThrow("second cancellation failed");
|
||||
await expect(f.admit()).rejects.toThrow();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release.resolve();
|
||||
await first;
|
||||
}
|
||||
(await f.admit()).release();
|
||||
});
|
||||
|
||||
it.each(["authorization", "incarnation"] as const)(
|
||||
"rejects changed %s after cancellation and reopens admission on failure",
|
||||
async (change) => {
|
||||
const f = fixture(`changed-${change}`);
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let authorized = true;
|
||||
const interrupted = vi.fn();
|
||||
const acquired = await f.admit(() => {
|
||||
interrupted();
|
||||
acquired.release();
|
||||
});
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
assertCurrent();
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
});
|
||||
try {
|
||||
const stop = f.prepare({
|
||||
authorize: () => {
|
||||
if (!authorized) {
|
||||
throw new Error("access revoked");
|
||||
}
|
||||
},
|
||||
});
|
||||
const rejected = expect(stop).rejects.toThrow(
|
||||
change === "authorization" ? "access revoked" : "Session",
|
||||
);
|
||||
await entered.promise;
|
||||
if (change === "authorization") {
|
||||
authorized = false;
|
||||
} else {
|
||||
f.entry.lifecycleRevision = "replacement-with-same-session-id";
|
||||
}
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(interrupted).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
(await f.admit()).release();
|
||||
} finally {
|
||||
release.resolve();
|
||||
acquired.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("rechecks the exact worker owner after asynchronous cancellation setup", async () => {
|
||||
const f = fixture("changed-worker");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const signalled = vi.fn();
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
assertCurrent();
|
||||
signalled();
|
||||
});
|
||||
const stop = f.prepare();
|
||||
const rejected = expect(stop).rejects.toThrow("worker changed before cancellation");
|
||||
await entered.promise;
|
||||
// The store returns immutable snapshots; replacement must not mutate the captured owner.
|
||||
const priorGeneration = f.placement.generation;
|
||||
f.placement.generation = priorGeneration + 1;
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(signalled).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("auto-suspend eligibility rejects before closing admission or signalling cancellation", async () => {
|
||||
const f = fixture("auto-suspend");
|
||||
await expect(
|
||||
f.prepare({
|
||||
beforeDrain: () => {
|
||||
throw new Error("session is busy");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("session is busy");
|
||||
expect(f.cancel).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
(await f.admit()).release();
|
||||
});
|
||||
|
||||
it("keeps admissions closed while serialized teardown is queued, then revalidates the incarnation", async () => {
|
||||
const f = fixture("queued-teardown");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const teardown = vi.fn();
|
||||
const stop = f.prepare({
|
||||
run: async (authorize) => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
authorize?.();
|
||||
teardown();
|
||||
return await f.run();
|
||||
},
|
||||
});
|
||||
const rejected = expect(stop).rejects.toThrow("Session");
|
||||
await entered.promise;
|
||||
await expect(f.admit()).rejects.toThrow();
|
||||
f.entry.lifecycleRevision = "new-incarnation";
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(teardown).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -30,6 +30,7 @@ it.each(["active", "failed"] as const)(
|
|||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
placements: {
|
||||
get: () => ({ state, sessionId }) as never,
|
||||
getAsync: async () => ({ state, sessionId }) as never,
|
||||
waitForTurnClaimRelease: vi.fn(),
|
||||
},
|
||||
loadSessionRuntime: async () =>
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import path from "node:path";
|
|||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { replaceSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import { clearAgentRunContext } from "../infra/agent-run-registry.js";
|
||||
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
|
||||
import * as operationAdmission from "../infra/sqlite-worker-operation-admission.js";
|
||||
import { createDeferredCore } from "../shared/deferred.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js";
|
||||
|
|
@ -41,191 +41,6 @@ afterEach(async () => {
|
|||
await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
function fixture(name: string, state: "active" | "failed" | "local" | "reclaimed" = "active") {
|
||||
const request = {
|
||||
sessionId: `session-${name}`,
|
||||
sessionKey: `agent:main:${name}`,
|
||||
agentId: "main",
|
||||
};
|
||||
const entry = { sessionId: request.sessionId, lifecycleRevision: "original", updatedAt: 1 };
|
||||
const target = {
|
||||
storePath: `/fixture/reclaim-preparation-${name}.sqlite`,
|
||||
canonicalKey: request.sessionKey,
|
||||
storeKeys: [request.sessionKey],
|
||||
agentId: request.agentId,
|
||||
store: { [request.sessionKey]: entry },
|
||||
};
|
||||
const placement = {
|
||||
...request,
|
||||
state,
|
||||
generation: 4,
|
||||
environmentId: "worker",
|
||||
activeOwnerEpoch: 7,
|
||||
};
|
||||
const cancel = vi.fn(async (input: { assertCurrent: () => void }) => input.assertCurrent());
|
||||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
placements: { get: () => ({ ...placement }) as never, waitForTurnClaimRelease: async () => {} },
|
||||
loadSessionRuntime: async () => ({
|
||||
managedWorktrees: { findLiveByOwner: async () => undefined },
|
||||
resolveGatewaySessionStoreTargetWithStore: () => target,
|
||||
resolveCanonicalSessionEntryFromStoreKeys: () => entry,
|
||||
}),
|
||||
cancelSessionWork: cancel,
|
||||
revokeSessionAuthority: vi.fn(),
|
||||
});
|
||||
const run = vi.fn(async () => ({ ...placement, state: "reclaimed" as const }) as never);
|
||||
const admit = (onInterrupt?: () => void) =>
|
||||
beginSessionWorkAdmission({
|
||||
scope: target.storePath,
|
||||
identities: [request.sessionKey, request.sessionId],
|
||||
assertAllowed: () => {},
|
||||
onInterrupt,
|
||||
});
|
||||
return {
|
||||
...request,
|
||||
entry,
|
||||
placement,
|
||||
cancel,
|
||||
run,
|
||||
admit,
|
||||
prepare: (options: Partial<Parameters<typeof barriers.runReclaimPreparation>[0]> = {}) =>
|
||||
barriers.runReclaimPreparation({ ...request, run, ...options }),
|
||||
};
|
||||
}
|
||||
|
||||
it("one failed Stop cannot reopen ingress while another Stop still owns its closure", async () => {
|
||||
const f = fixture("overlap");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let calls = 0;
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
assertCurrent();
|
||||
if (++calls === 1) {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
} else {
|
||||
throw new Error("second cancellation failed");
|
||||
}
|
||||
});
|
||||
const first = f.prepare();
|
||||
await entered.promise;
|
||||
try {
|
||||
await expect(f.prepare()).rejects.toThrow("second cancellation failed");
|
||||
await expect(f.admit()).rejects.toThrow();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release.resolve();
|
||||
await first;
|
||||
}
|
||||
(await f.admit()).release();
|
||||
});
|
||||
|
||||
it.each(["authorization", "incarnation"] as const)(
|
||||
"rejects changed %s after cancellation and reopens admission on failure",
|
||||
async (change) => {
|
||||
const f = fixture(`changed-${change}`);
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let authorized = true;
|
||||
const interrupted = vi.fn();
|
||||
const acquired = await f.admit(() => {
|
||||
interrupted();
|
||||
acquired.release();
|
||||
});
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
assertCurrent();
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
});
|
||||
try {
|
||||
const stop = f.prepare({
|
||||
authorize: () => {
|
||||
if (!authorized) {
|
||||
throw new Error("access revoked");
|
||||
}
|
||||
},
|
||||
});
|
||||
const rejected = expect(stop).rejects.toThrow(
|
||||
change === "authorization" ? "access revoked" : "Session",
|
||||
);
|
||||
await entered.promise;
|
||||
if (change === "authorization") {
|
||||
authorized = false;
|
||||
} else {
|
||||
f.entry.lifecycleRevision = "replacement-with-same-session-id";
|
||||
}
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(interrupted).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
(await f.admit()).release();
|
||||
} finally {
|
||||
release.resolve();
|
||||
acquired.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("rechecks the exact worker owner after asynchronous cancellation setup", async () => {
|
||||
const f = fixture("changed-worker");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const signalled = vi.fn();
|
||||
f.cancel.mockImplementation(async ({ assertCurrent }) => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
assertCurrent();
|
||||
signalled();
|
||||
});
|
||||
const stop = f.prepare();
|
||||
const rejected = expect(stop).rejects.toThrow("worker changed before cancellation");
|
||||
await entered.promise;
|
||||
// The store returns immutable snapshots; replacement must not mutate the captured owner.
|
||||
const priorGeneration = f.placement.generation;
|
||||
f.placement.generation = priorGeneration + 1;
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(signalled).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("auto-suspend eligibility rejects before closing admission or signalling cancellation", async () => {
|
||||
const f = fixture("auto-suspend");
|
||||
await expect(
|
||||
f.prepare({
|
||||
beforeDrain: () => {
|
||||
throw new Error("session is busy");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("session is busy");
|
||||
expect(f.cancel).not.toHaveBeenCalled();
|
||||
expect(f.run).not.toHaveBeenCalled();
|
||||
(await f.admit()).release();
|
||||
});
|
||||
|
||||
it("keeps admissions closed while serialized teardown is queued, then revalidates the incarnation", async () => {
|
||||
const f = fixture("queued-teardown");
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const teardown = vi.fn();
|
||||
const stop = f.prepare({
|
||||
run: async (authorize) => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
authorize?.();
|
||||
teardown();
|
||||
return await f.run();
|
||||
},
|
||||
});
|
||||
const rejected = expect(stop).rejects.toThrow("Session");
|
||||
await entered.promise;
|
||||
await expect(f.admit()).rejects.toThrow();
|
||||
f.entry.lifecycleRevision = "new-incarnation";
|
||||
release.resolve();
|
||||
await rejected;
|
||||
expect(teardown).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
async function cancellationLoadFixture(
|
||||
options: NonNullable<Parameters<typeof createHarness>[2]> = {},
|
||||
beforeCancellation?: () => Promise<void>,
|
||||
|
|
@ -970,3 +785,86 @@ it.each([
|
|||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ stage: "transaction", change: "caller" },
|
||||
{ stage: "commit", change: "caller" },
|
||||
{ stage: "transaction", change: "incarnation" },
|
||||
{ stage: "commit", change: "incarnation" },
|
||||
] as const)(
|
||||
"retains the abandoned Stop decision when $change changes at $stage completion admission",
|
||||
async ({ stage, change }) => {
|
||||
let destroyed = false;
|
||||
const f = await cancellationLoadFixture({
|
||||
afterDestroy: () => {
|
||||
destroyed = true;
|
||||
},
|
||||
});
|
||||
const active = await f.coordinated.dispatch(REQUEST);
|
||||
f.harness.markEnvironmentNodeDeviceId("device-1");
|
||||
f.database.db
|
||||
.prepare("UPDATE worker_environments SET profile_id = ? WHERE environment_id = ?")
|
||||
.run("device:device-1", active.environmentId);
|
||||
const begun = await f.placements.beginPlacementMove({
|
||||
sessionId: active.sessionId,
|
||||
source: {
|
||||
generation: active.generation,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
target: { kind: "gateway" },
|
||||
abandonSource: true,
|
||||
});
|
||||
let allowed = true;
|
||||
let admissionReached = false;
|
||||
let connected = true;
|
||||
const assertLifetime = () => {
|
||||
if (!connected) {
|
||||
throw new Error("Stop connection closed");
|
||||
}
|
||||
};
|
||||
const assertCaller = () => {
|
||||
assertLifetime();
|
||||
if (!allowed) {
|
||||
throw new Error("Stop caller revoked at completion");
|
||||
}
|
||||
};
|
||||
const authorize = Object.assign(assertCaller, {
|
||||
assertWorkerLifetime: assertLifetime,
|
||||
assertWorkerGrant: assertCaller,
|
||||
});
|
||||
const createAdmission = operationAdmission.createSqliteWorkerOperationAdmission;
|
||||
const admission = vi
|
||||
.spyOn(operationAdmission, "createSqliteWorkerOperationAdmission")
|
||||
.mockImplementation((admit, attachment) =>
|
||||
createAdmission((request, grant) => {
|
||||
if (destroyed && request.stage === stage) {
|
||||
admissionReached = true;
|
||||
if (change === "caller") {
|
||||
allowed = false;
|
||||
} else {
|
||||
f.entry.lifecycleRevision = "replacement";
|
||||
}
|
||||
}
|
||||
admit(request, grant);
|
||||
}, attachment),
|
||||
);
|
||||
try {
|
||||
await expect(f.coordinated.reclaim(REQUEST, authorize)).rejects.toThrow(
|
||||
change === "caller"
|
||||
? "Stop caller revoked at completion"
|
||||
: "changed before cloud worker stop",
|
||||
);
|
||||
expect(admissionReached).toBe(true);
|
||||
expect(f.harness.environments.destroy).toHaveBeenCalledOnce();
|
||||
expect(f.placements.get(REQUEST.sessionId)).toMatchObject({ state: "failed" });
|
||||
expect(f.placements.getPlacementMove(REQUEST.sessionId)?.operationId).toBe(
|
||||
begun.intent.operationId,
|
||||
);
|
||||
} finally {
|
||||
connected = false;
|
||||
admission.mockRestore();
|
||||
f.context.chatRunState.clear();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import path from "node:path";
|
|||
import { setImmediate } from "node:timers/promises";
|
||||
import { asRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { createDeferred } from "../../test/helpers/promise.js";
|
||||
import { awaitGateBeforeSettlement, createDeferred } from "../../test/helpers/promise.js";
|
||||
import { loadTranscriptEvents, replaceSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import { clearAgentRunContext } from "../infra/agent-run-registry.js";
|
||||
import { runCommandWithTimeout } from "../process/exec.js";
|
||||
|
|
@ -97,6 +97,7 @@ async function scenario(
|
|||
context.cancelRunBoundApprovals = cancelApprovals;
|
||||
const revocations: unknown[] = [];
|
||||
const cancellationLoad = createDeferred();
|
||||
const cancellationEntered = createDeferred();
|
||||
let cancellationLoadEntered = false;
|
||||
let abortedBeforeCancellationLoad = false;
|
||||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
|
|
@ -117,6 +118,7 @@ async function scenario(
|
|||
resolveCanonicalSessionEntryFromStoreKeys: () => entry,
|
||||
}) as never,
|
||||
cancelSessionWork: async (request) => {
|
||||
cancellationEntered.resolve();
|
||||
if (pendingMove) {
|
||||
cancellationLoadEntered = true;
|
||||
await cancellationLoad.promise;
|
||||
|
|
@ -254,6 +256,7 @@ async function scenario(
|
|||
context.chatRunState.getOrCreate(name + "-running").buffer = "partial before queued Move Stop";
|
||||
}
|
||||
let cancellationRecovery: Promise<void> | undefined;
|
||||
const runningAborted = createDeferred();
|
||||
if (running?.ok) {
|
||||
running.value.activeRunAbort.controller.signal.addEventListener("abort", () => {
|
||||
if (cancellationNeedsRecovery) {
|
||||
|
|
@ -264,6 +267,7 @@ async function scenario(
|
|||
} else {
|
||||
running.value.cleanupAdmittedRun();
|
||||
}
|
||||
runningAborted.resolve();
|
||||
});
|
||||
}
|
||||
const inspectionEntered = createDeferred();
|
||||
|
|
@ -349,23 +353,41 @@ async function scenario(
|
|||
},
|
||||
);
|
||||
if (blockedInspection) {
|
||||
await setImmediate();
|
||||
if (pendingMove) {
|
||||
abortedBeforeCancellationLoad = activeController?.controller.signal.aborted === true;
|
||||
try {
|
||||
if (pendingMove) {
|
||||
await awaitGateBeforeSettlement(
|
||||
cancellationEntered.promise,
|
||||
reclaim,
|
||||
"Stop settled before entering cancellation with inspection held",
|
||||
);
|
||||
abortedBeforeCancellationLoad = activeController?.controller.signal.aborted === true;
|
||||
cancellationLoad.resolve();
|
||||
}
|
||||
await awaitGateBeforeSettlement(
|
||||
runningAborted.promise,
|
||||
reclaim,
|
||||
"Stop settled before aborting the running chat with inspection held",
|
||||
);
|
||||
abortedDuringInspection =
|
||||
running?.ok === true && running.value.activeRunAbort.controller.signal.aborted;
|
||||
destroyedDuringInspection = vi.mocked(harness.environments.destroy).mock.calls.length > 0;
|
||||
abortReasonDuringInspection = activeController?.abortStopReason;
|
||||
approvalsCancelledDuringInspection = cancelApprovals.mock.calls.some(
|
||||
([runId]) => runId === name + "-running",
|
||||
);
|
||||
reserveOld();
|
||||
if (moving) {
|
||||
await awaitGateBeforeSettlement(
|
||||
moving,
|
||||
reclaim,
|
||||
"Stop settled before the queued Move with inspection held",
|
||||
);
|
||||
}
|
||||
moveSettledDuringInspection = moveSettled;
|
||||
} finally {
|
||||
cancellationLoad.resolve();
|
||||
await setImmediate();
|
||||
releaseInspection.resolve();
|
||||
}
|
||||
abortedDuringInspection =
|
||||
running?.ok === true && running.value.activeRunAbort.controller.signal.aborted;
|
||||
destroyedDuringInspection = vi.mocked(harness.environments.destroy).mock.calls.length > 0;
|
||||
abortReasonDuringInspection = activeController?.abortStopReason;
|
||||
approvalsCancelledDuringInspection = cancelApprovals.mock.calls.some(
|
||||
([runId]) => runId === name + "-running",
|
||||
);
|
||||
reserveOld();
|
||||
await setImmediate();
|
||||
moveSettledDuringInspection = moveSettled;
|
||||
releaseInspection.resolve();
|
||||
}
|
||||
if (pendingDispatch) {
|
||||
await setImmediate();
|
||||
|
|
@ -542,7 +564,11 @@ it("an idempotent failed-cleanup result does not cancel work already on the loca
|
|||
const local = { state: "local", sessionId: REQUEST.sessionId, generation: 4 };
|
||||
const cancel = vi.fn();
|
||||
const barriers = createGatewayWorkerPlacementReclaimBarriers({
|
||||
placements: { get: () => local as never, waitForTurnClaimRelease: vi.fn() },
|
||||
placements: {
|
||||
get: () => local as never,
|
||||
getAsync: async () => local as never,
|
||||
waitForTurnClaimRelease: vi.fn(),
|
||||
},
|
||||
loadSessionRuntime: async () =>
|
||||
({
|
||||
managedWorktrees: { findLiveByOwner: async () => undefined },
|
||||
|
|
@ -626,7 +652,9 @@ it.each(["missing", "local"] as const)(
|
|||
const cancelApprovals = vi.fn();
|
||||
context.cancelRunBoundApprovals = cancelApprovals;
|
||||
const cancellationLoad = createDeferred();
|
||||
const cancellationEntered = createDeferred();
|
||||
const cancel = vi.fn(async (request: Parameters<typeof cancelGatewayWorkerSessionWork>[1]) => {
|
||||
cancellationEntered.resolve();
|
||||
await cancellationLoad.promise;
|
||||
const runtime = await import("./server-worker-placement-cancel.js");
|
||||
await runtime.cancelGatewayWorkerSessionWork(context, request);
|
||||
|
|
@ -671,6 +699,7 @@ it.each(["missing", "local"] as const)(
|
|||
});
|
||||
const sweep = coordinated.reconcileActive();
|
||||
await entered.promise;
|
||||
expect(placements.get(REQUEST.sessionId)?.state).toBe(state === "local" ? "local" : undefined);
|
||||
let dispatchSettled = false;
|
||||
const dispatch = coordinated
|
||||
.dispatch(REQUEST)
|
||||
|
|
@ -683,15 +712,19 @@ it.each(["missing", "local"] as const)(
|
|||
});
|
||||
const stopping = coordinated.reclaim(REQUEST).catch(() => undefined);
|
||||
try {
|
||||
await setImmediate();
|
||||
await setImmediate();
|
||||
expect(placements.get(REQUEST.sessionId)?.state).toBe(
|
||||
state === "local" ? "local" : undefined,
|
||||
await awaitGateBeforeSettlement(
|
||||
cancellationEntered.promise,
|
||||
stopping,
|
||||
"Stop settled before entering local chat cancellation with inspection held",
|
||||
);
|
||||
expect(cancel).toHaveBeenCalledOnce();
|
||||
expect(controller.controller.signal.aborted).toBe(false);
|
||||
cancellationLoad.resolve();
|
||||
await aborted.promise;
|
||||
await awaitGateBeforeSettlement(
|
||||
aborted.promise,
|
||||
stopping,
|
||||
"Stop settled before aborting local chat with inspection held",
|
||||
);
|
||||
expect(controller.abortStopReason).toBe("rpc");
|
||||
expect(cancelApprovals).toHaveBeenCalledWith(runId);
|
||||
await stopping;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import { matchesWorkerPlacementTarget } from "./worker-environments/placement-ta
|
|||
import type { WorkerPlacementReclaimRequest } from "./worker-environments/service-contract.js";
|
||||
|
||||
type WorkerPlacementReclaimBarrierParams = {
|
||||
placements: Pick<WorkerSessionPlacementStore, "get" | "waitForTurnClaimRelease">;
|
||||
placements: Pick<WorkerSessionPlacementStore, "get" | "getAsync" | "waitForTurnClaimRelease">;
|
||||
loadSessionRuntime: () => Promise<WorkerPlacementSessionRuntime>;
|
||||
cancelSessionWork: WorkerPlacementSessionWorkCancellation;
|
||||
revokeSessionAuthority: (request: { sessionId: string; sessionKeys: readonly string[] }) => void;
|
||||
|
|
@ -131,8 +131,7 @@ export function createGatewayWorkerPlacementReclaimBarriers(
|
|||
target.storeKeys,
|
||||
);
|
||||
const revision = entry?.lifecycleRevision ?? null;
|
||||
const assertCurrent = () => {
|
||||
authorize?.();
|
||||
const assertTargetCurrent = () => {
|
||||
const current = resolveTarget();
|
||||
const currentEntry = sessionRuntime.resolveCanonicalSessionEntryFromStoreKeys(
|
||||
current.store,
|
||||
|
|
@ -150,9 +149,23 @@ export function createGatewayWorkerPlacementReclaimBarriers(
|
|||
);
|
||||
}
|
||||
};
|
||||
const assertCurrent = Object.assign(
|
||||
() => {
|
||||
authorize?.();
|
||||
assertTargetCurrent();
|
||||
},
|
||||
{
|
||||
// Transport waits keep caller custody; both write grants reread the session target.
|
||||
assertWorkerLifetime: () => (authorize?.assertWorkerLifetime ?? authorize)?.(),
|
||||
assertWorkerGrant: () => {
|
||||
(authorize?.assertWorkerGrant ?? authorize)?.();
|
||||
assertTargetCurrent();
|
||||
},
|
||||
},
|
||||
);
|
||||
const placement = await params.placements.getAsync(sessionId);
|
||||
assertCurrent();
|
||||
beforeDrain?.();
|
||||
const placement = params.placements.get(sessionId);
|
||||
const pending = pendingOperations?.isCurrent() ? pendingOperations : undefined;
|
||||
const dispatch = pending?.hasPendingDispatch() === true;
|
||||
if (
|
||||
|
|
@ -238,7 +251,9 @@ export function createGatewayWorkerPlacementReclaimBarriers(
|
|||
expectedTarget: target,
|
||||
errorMessage: `Session ${sessionKey} changed before cloud worker stop. Retry.`,
|
||||
});
|
||||
const placement = params.placements.get(sessionId);
|
||||
const placement = await params.placements.getAsync(sessionId);
|
||||
authorize?.();
|
||||
resolved.assertBindingCurrent(getRuntimeConfig());
|
||||
if (
|
||||
placement?.state !== "active" &&
|
||||
placement?.state !== "draining" &&
|
||||
|
|
@ -319,10 +334,11 @@ export function createGatewayWorkerPlacementReclaimBarriers(
|
|||
scope: target.storePath,
|
||||
identities: lifecycleIdentities,
|
||||
prepare: async (lifecycle) => {
|
||||
assertCurrent();
|
||||
// A preceding failed cleanup may already have returned this placement to local.
|
||||
// Its idempotent result must not cancel work admitted after that completed Stop.
|
||||
if (params.placements.get(sessionId)?.state === "failed") {
|
||||
const placement = await params.placements.getAsync(sessionId);
|
||||
assertCurrent();
|
||||
if (placement?.state === "failed") {
|
||||
await cancelAndDrain(lifecycle.closeWorkAdmissions, assertCurrent);
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ describe("worker placement session events", () => {
|
|||
placements: {
|
||||
workspaceResultInstanceId: () => "gateway-test",
|
||||
getAsync: async () => undefined,
|
||||
getManyAsync: async () => new Map(),
|
||||
listAsync: async () => [],
|
||||
retireSessionPlacementAsync: vi.fn(async () => {}),
|
||||
pruneOrphanedWorkspaceReconciliations: async () => [],
|
||||
|
|
@ -65,6 +66,7 @@ describe("worker placement session events", () => {
|
|||
let scheduledWake: void | Promise<void> = undefined;
|
||||
try {
|
||||
await time.advanceBy(60_000);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
reconcileActive.mockClear();
|
||||
reconcileActive
|
||||
.mockImplementationOnce(() => {
|
||||
|
|
|
|||
|
|
@ -429,11 +429,11 @@ export async function prepareWorkerPlacementRepositoryManifestRefs(
|
|||
}
|
||||
|
||||
export function createWorkerPlacementNodeWorkspaceBindingResolver(options: {
|
||||
placements: Pick<WorkerSessionPlacementStore, "get">;
|
||||
placements: Pick<WorkerSessionPlacementStore, "get" | "getAsync">;
|
||||
resolveWorkspace: (identity: WorkerSessionPlacementIdentity) => Promise<WorkerSessionWorkspace>;
|
||||
}) {
|
||||
return async (binding: { environmentId: string; ownerEpoch: number; sessionId: string }) => {
|
||||
const placement = options.placements.get(binding.sessionId);
|
||||
const placement = await options.placements.getAsync(binding.sessionId);
|
||||
if (
|
||||
!placement ||
|
||||
(placement.state !== "active" &&
|
||||
|
|
|
|||
|
|
@ -681,6 +681,7 @@ describe("worker placement startup recovery authority", () => {
|
|||
placements: {
|
||||
workspaceResultInstanceId: () => "gateway-test",
|
||||
get: () => placement,
|
||||
getAsync: async () => placement,
|
||||
} as never,
|
||||
environments: {} as never,
|
||||
gatewayNamespace: "gateway-test",
|
||||
|
|
|
|||
|
|
@ -246,7 +246,8 @@ export function createGatewayWorkerPlacementRuntime(
|
|||
...identity,
|
||||
action: "recovery",
|
||||
run: async (workspace, assertCurrent) => {
|
||||
const placement = params.placements.get(identity.sessionId);
|
||||
const placement = await params.placements.getAsync(identity.sessionId);
|
||||
assertCurrent();
|
||||
if (
|
||||
placement?.state !== "provisioning" ||
|
||||
placement.generation !== expectedGeneration ||
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ test("sessions.recover settles its active placement before archiving a real sess
|
|||
cancelSessionWork: vi.fn(async () => {}),
|
||||
placements: {
|
||||
get: () => placement,
|
||||
getAsync: async () => placement,
|
||||
waitForTurnClaimRelease: vi.fn(async () => {}),
|
||||
},
|
||||
loadSessionRuntime: async () => ({
|
||||
|
|
@ -348,6 +349,7 @@ test.each(["before-interrupt", "before-drain"] as const)(
|
|||
cancelSessionWork: vi.fn(async () => {}),
|
||||
placements: {
|
||||
get: () => placement,
|
||||
getAsync: async () => placement,
|
||||
waitForTurnClaimRelease: async () => {
|
||||
if (phase === "before-drain") {
|
||||
await wait();
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ async function seedPendingWorkspace(scenario: RecoveryScenario) {
|
|||
providerId: "device",
|
||||
profileId: "device:runner-1",
|
||||
nodeDeviceId: "runner-1",
|
||||
sshEndpoint: null,
|
||||
};
|
||||
writePlacementEnvironmentFixture(database, environment);
|
||||
await seedActivePlacement(placements, environment);
|
||||
|
|
|
|||
|
|
@ -253,6 +253,7 @@ export function createHarness(sharedHost = false, withPolicy = true) {
|
|||
store: { get: () => state.environment },
|
||||
placements: {
|
||||
get: () => state.placement,
|
||||
getAsync: async () => structuredClone(state.placement),
|
||||
validateTurnClaim: (candidate: WorkerSessionTurnClaim) =>
|
||||
isCurrentPlacementTurnClaim(state.placement, candidate),
|
||||
registerTurnClaimClosedHandler(handler: (closed: WorkerSessionTurnClaim) => void) {
|
||||
|
|
|
|||
|
|
@ -131,6 +131,20 @@ describe("session computer transport", () => {
|
|||
resetPluginRuntimeStateForTest();
|
||||
});
|
||||
|
||||
it("refuses computer preparation when the turn closes during its placement read", async () => {
|
||||
const h = createHarness();
|
||||
const placement = structuredClone(h.state.placement);
|
||||
h.options.placements.getAsync = async () => {
|
||||
h.releaseClaim();
|
||||
return placement;
|
||||
};
|
||||
await expect(createWorkerComputerService(h.options).prepare(h.claim)).rejects.toThrow(
|
||||
"Session desktop placement is no longer active",
|
||||
);
|
||||
expect(h.privateInvoke).not.toHaveBeenCalled();
|
||||
expect(h.publicInvoke).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("controls an attached environment without moving the conversation and fences attachment revocation", async () => {
|
||||
const h = createHarness();
|
||||
h.releaseClaim();
|
||||
|
|
|
|||
|
|
@ -628,14 +628,14 @@ export function createEnvironmentComputerTransportOwner(options: WorkerComputerO
|
|||
/** Placement admission retains its exact turn claim; attachments use the same transport owner. */
|
||||
export function createWorkerComputerTransportOwner(
|
||||
options: WorkerComputerOwnerOptions & {
|
||||
placements: Pick<WorkerSessionPlacementStore, "get" | "validateTurnClaim">;
|
||||
placements: Pick<WorkerSessionPlacementStore, "get" | "getAsync" | "validateTurnClaim">;
|
||||
},
|
||||
) {
|
||||
const create = createEnvironmentComputerTransportOwner(options);
|
||||
return (claim: WorkerSessionTurnClaim): Promise<PreparedWorkerComputer | undefined> => {
|
||||
const placement = options.placements.get(claim.sessionId);
|
||||
return async (claim: WorkerSessionTurnClaim): Promise<PreparedWorkerComputer | undefined> => {
|
||||
const placement = await options.placements.getAsync(claim.sessionId);
|
||||
if (placement?.state !== "active" || !options.placements.validateTurnClaim(claim)) {
|
||||
return Promise.reject(new Error("Session desktop placement is no longer active"));
|
||||
throw new Error("Session desktop placement is no longer active");
|
||||
}
|
||||
return create({
|
||||
environmentId: placement.environmentId,
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ export function createNodeWorkerPreparedWorkspaceTransport(options: {
|
|||
},
|
||||
bindPreparedWorkspace: async ({ assertCurrent, signal, ...binding }) => {
|
||||
const record = options.store.get(binding.environmentId);
|
||||
const placement = options.placementStore.get(binding.sessionId);
|
||||
const placement = await options.placementStore.getAsync(binding.sessionId);
|
||||
const assertBindingCurrent = () => {
|
||||
signal?.throwIfAborted();
|
||||
assertCurrent();
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ export type WorkerDispatchPlacementStore = Pick<
|
|||
| "fail"
|
||||
| "get"
|
||||
| "getAsync"
|
||||
| "getWithMoveAsync"
|
||||
| "getPlacementMoveAsync"
|
||||
| "readProjection"
|
||||
| "readRecoveryCandidates"
|
||||
|
|
@ -369,7 +370,7 @@ export function createPlacementFailureActions(deps: {
|
|||
// reconciliation; startup recovery explicitly fences stale claims.
|
||||
return;
|
||||
}
|
||||
const current = placements.get(placement.sessionId);
|
||||
const current = await placements.getAsync(placement.sessionId);
|
||||
if (current?.state !== "draining") {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -681,20 +681,22 @@ export async function recoverPendingWorkspaceResults(
|
|||
});
|
||||
} catch (error) {
|
||||
try {
|
||||
const pendingResults = await placements.listPendingWorkspaceResultsAsync(
|
||||
pending.sessionId,
|
||||
);
|
||||
const current = placements.get(pending.sessionId);
|
||||
const currentPending = pendingResults.find(
|
||||
(candidate) =>
|
||||
candidate.sessionId === pending.sessionId &&
|
||||
candidate.environmentId === pending.environmentId &&
|
||||
candidate.ownerEpoch === pending.ownerEpoch &&
|
||||
candidate.placementGeneration === pending.placementGeneration &&
|
||||
candidate.claimId === pending.claimId &&
|
||||
candidate.runId === pending.runId &&
|
||||
candidate.gatewayInstanceId === pending.gatewayInstanceId,
|
||||
);
|
||||
const currentFacts = await placements.readProjection([pending.sessionId], {
|
||||
current: true,
|
||||
});
|
||||
const current = currentFacts.placements.get(pending.sessionId);
|
||||
const candidate = currentFacts.pendingResults.get(pending.sessionId);
|
||||
const currentPending =
|
||||
candidate &&
|
||||
candidate.sessionId === pending.sessionId &&
|
||||
candidate.environmentId === pending.environmentId &&
|
||||
candidate.ownerEpoch === pending.ownerEpoch &&
|
||||
candidate.placementGeneration === pending.placementGeneration &&
|
||||
candidate.claimId === pending.claimId &&
|
||||
candidate.runId === pending.runId &&
|
||||
candidate.gatewayInstanceId === pending.gatewayInstanceId
|
||||
? candidate
|
||||
: undefined;
|
||||
if (
|
||||
currentPending &&
|
||||
isCurrentWorkerWorkspacePendingResultOwner(current, currentPending)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ export function isPendingProvisioningEnvironment(
|
|||
}
|
||||
|
||||
export function createInterruptedWorkerProvisioningRetainer(options: {
|
||||
placements: Pick<WorkerDispatchPlacementStore, "get">;
|
||||
placements: Pick<WorkerDispatchPlacementStore, "get" | "getAsync">;
|
||||
environments: Pick<WorkerEnvironmentService, "get" | "recordError">;
|
||||
isShuttingDown?: () => boolean;
|
||||
}) {
|
||||
|
|
@ -34,7 +34,7 @@ export function createInterruptedWorkerProvisioningRetainer(options: {
|
|||
owned: WorkerDispatchPlacement,
|
||||
error: unknown,
|
||||
): Promise<WorkerDispatchPlacement | undefined> => {
|
||||
const current = placements.get(owned.sessionId);
|
||||
const current = await placements.getAsync(owned.sessionId);
|
||||
if (
|
||||
error instanceof WorkerPlacementAdmissionTargetError ||
|
||||
error instanceof WorkerDispatchTargetChangedError ||
|
||||
|
|
|
|||
|
|
@ -613,6 +613,12 @@ describe("worker placement restart recovery", () => {
|
|||
let replacement: ReturnType<typeof placements.get>;
|
||||
const observedPlacements = {
|
||||
...placements,
|
||||
getAsync: async (sessionId: string) => {
|
||||
const current = await placements.getAsync(sessionId);
|
||||
return change === "session" && replacement && current
|
||||
? { ...current, sessionKey: "agent:main:replacement-session" }
|
||||
: current;
|
||||
},
|
||||
get: (sessionId: string) => {
|
||||
const current = placements.get(sessionId);
|
||||
return change === "session" && replacement && current
|
||||
|
|
|
|||
|
|
@ -93,14 +93,17 @@ export function createPlacementRecoveryActions(deps: PlacementRecoveryDeps) {
|
|||
try {
|
||||
await environments.stopTunnel(placement.environmentId, placement.activeOwnerEpoch);
|
||||
await placements.closeWorkerTurnToolState(claim);
|
||||
const current = placements.get(placement.sessionId);
|
||||
const currentFacts = await placements.readProjection([placement.sessionId], {
|
||||
current: true,
|
||||
});
|
||||
const current = currentFacts.placements.get(placement.sessionId);
|
||||
const currentEnvironment = environments.get(placement.environmentId);
|
||||
if (
|
||||
current?.state !== "active" ||
|
||||
current.generation !== placement.generation ||
|
||||
currentEnvironment?.nodeDeviceId !== environment.nodeDeviceId ||
|
||||
!isCurrentActiveWorkerEnvironment(current, currentEnvironment) ||
|
||||
placements.getPlacementMove(placement.sessionId)
|
||||
currentFacts.moves.has(placement.sessionId)
|
||||
) {
|
||||
throw new Error("Interrupted worker owner changed while stopping");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
if (placement && (await startup.retainInterruptedProvisioning(placement, error))) {
|
||||
throw error;
|
||||
}
|
||||
const current = placement ? placements.get(request.sessionId) : undefined;
|
||||
const current = placement ? await placements.getAsync(request.sessionId) : undefined;
|
||||
if (current && current.state !== "local" && current.state !== "reclaimed") {
|
||||
if (current.state === "active") {
|
||||
await failure.failActive(current, error);
|
||||
|
|
@ -290,7 +290,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
}
|
||||
}
|
||||
} finally {
|
||||
const finalPlacement = placements.get(request.sessionId);
|
||||
const finalPlacement = await placements.getAsync(request.sessionId);
|
||||
if (finalPlacement) {
|
||||
reportPlacementTransition(onTransition, finalPlacement);
|
||||
}
|
||||
|
|
@ -309,9 +309,32 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
completedOperation?: WorkerPlacementCancellationTarget,
|
||||
onTransition?: (placement: WorkerDispatchPlacement) => void,
|
||||
): Promise<WorkerReclaimPlacement> => {
|
||||
const { placement: current, move } = await placements.getWithMoveAsync(request.sessionId);
|
||||
authorize?.();
|
||||
beforeDrain?.();
|
||||
const current = placements.get(request.sessionId);
|
||||
if (current?.state === "draining" && move?.abandonSource) {
|
||||
if (
|
||||
move.target.kind !== "gateway" ||
|
||||
move.sessionId !== current.sessionId ||
|
||||
current.sessionKey !== request.sessionKey ||
|
||||
current.agentId !== request.agentId ||
|
||||
current.generation !== move.source.generation + 1 ||
|
||||
current.environmentId !== move.source.environmentId ||
|
||||
current.activeOwnerEpoch !== move.source.ownerEpoch
|
||||
) {
|
||||
throw new Error("Abandoned cloud worker move changed before Stop cleanup");
|
||||
}
|
||||
// Stop owns live authority after the canceled Move settles; its committed disposition stays.
|
||||
return await options.runFailedReclaimBarrier({
|
||||
...request,
|
||||
authorize,
|
||||
reclaim: async (reauthorize) => {
|
||||
const local = await abandonment.abandonSource(request, move, reauthorize, current);
|
||||
reportPlacementTransition(onTransition, local);
|
||||
return local;
|
||||
},
|
||||
});
|
||||
}
|
||||
if (current?.state === "reclaimed") {
|
||||
return current;
|
||||
}
|
||||
|
|
@ -329,10 +352,11 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
...request,
|
||||
authorize,
|
||||
reclaim: async (reauthorize) => {
|
||||
let failedPlacement = await placements.getAsync(request.sessionId);
|
||||
reauthorize?.();
|
||||
if (request.recoverToGateway) {
|
||||
beforeDrain?.();
|
||||
}
|
||||
let failedPlacement = placements.get(request.sessionId);
|
||||
if (owned.state === "provisioning") {
|
||||
failedPlacement = await failure.cancelProvisioning(
|
||||
failedPlacement,
|
||||
|
|
@ -355,7 +379,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
throw new Error("Failed cloud worker placement changed during reclaim");
|
||||
}
|
||||
const cleanupError = await failure.retryFailedTeardown(failedPlacement, reauthorize);
|
||||
const failed = placements.get(request.sessionId);
|
||||
const failed = await placements.getAsync(request.sessionId);
|
||||
if (failed?.state !== "failed") {
|
||||
throw new Error("Failed cloud worker placement changed during reclaim");
|
||||
}
|
||||
|
|
@ -404,7 +428,7 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
} catch (error) {
|
||||
// Another teardown path can win after this call has crossed its durable completion fence.
|
||||
// Report the committed terminal state instead of leaking a stale tunnel error to callers.
|
||||
const completed = placements.get(request.sessionId);
|
||||
const completed = await placements.getAsync(request.sessionId);
|
||||
if (error instanceof WorkerTunnelOwnerDisconnectedError && completed?.state === "reclaimed") {
|
||||
return completed;
|
||||
}
|
||||
|
|
@ -422,11 +446,11 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
pendingOperations?: WorkerPlacementPendingOperations,
|
||||
onTransition?: (placement: WorkerDispatchPlacement) => void,
|
||||
): Promise<WorkerReclaimPlacement> => {
|
||||
const assertGatewayRecoverySource = () => {
|
||||
const assertGatewayRecoverySource = (captured?: WorkerDispatchPlacement) => {
|
||||
if (!request.recoverToGateway) {
|
||||
return;
|
||||
}
|
||||
const source = placements.get(request.sessionId);
|
||||
const source = captured ?? placements.get(request.sessionId);
|
||||
if (
|
||||
source?.state !== "failed" ||
|
||||
source.generation !== request.recoverToGateway.expectedGeneration ||
|
||||
|
|
@ -446,10 +470,10 @@ export function createWorkerPlacementDispatchService(options: WorkerPlacementDis
|
|||
);
|
||||
}
|
||||
};
|
||||
const initial = await placements.getAsync(request.sessionId);
|
||||
authorize?.();
|
||||
beforeDrain?.();
|
||||
assertGatewayRecoverySource();
|
||||
const initial = placements.get(request.sessionId);
|
||||
assertGatewayRecoverySource(initial);
|
||||
if (initial) {
|
||||
reportPlacementTransition(onTransition, initial);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,8 +30,9 @@ export async function forceAbandonWorkerEnvironment(
|
|||
environmentId: string;
|
||||
onCleanupError?: (error: unknown) => void;
|
||||
},
|
||||
): Promise<void> {
|
||||
): Promise<ReadonlyMap<string, WorkerSessionPlacementRecord>> {
|
||||
const { environmentId, placements } = params;
|
||||
const failedPlacements = new Map<string, WorkerSessionPlacementRecord>();
|
||||
const recoveryError = FORCED_WORKER_ABANDONMENT_ERROR;
|
||||
const journalOwners = (await params.placements.listWorkspaceReconciliationOwners()).filter(
|
||||
(owner) => owner.environmentId === environmentId,
|
||||
|
|
@ -134,12 +135,13 @@ export async function forceAbandonWorkerEnvironment(
|
|||
});
|
||||
}
|
||||
if (current && (current.state !== "failed" || current.recoveryError !== recoveryError)) {
|
||||
await placements.fail({
|
||||
current = await placements.fail({
|
||||
sessionId: current.sessionId,
|
||||
expectedGeneration: current.generation,
|
||||
recoveryError,
|
||||
});
|
||||
}
|
||||
failedPlacements.set(current.sessionId, current);
|
||||
}
|
||||
|
||||
// The durable fence is now closed. Filesystem rollback and ref cleanup are
|
||||
|
|
@ -191,4 +193,5 @@ export async function forceAbandonWorkerEnvironment(
|
|||
reportWorkerAbandonmentCleanupError(params.onCleanupError, error);
|
||||
}
|
||||
}
|
||||
return failedPlacements;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -330,16 +330,19 @@ describe("worker placement idle suspension", () => {
|
|||
const active = await harness.service.dispatch(REQUEST);
|
||||
nowMs += 60_000;
|
||||
const createAdmission = operationAdmission.createSqliteWorkerOperationAdmission;
|
||||
vi.spyOn(operationAdmission, "createSqliteWorkerOperationAdmission").mockImplementationOnce(
|
||||
let admissionReached = false;
|
||||
vi.spyOn(operationAdmission, "createSqliteWorkerOperationAdmission").mockImplementation(
|
||||
(admit, attachment) =>
|
||||
createAdmission((request, grant) => {
|
||||
if (request.stage === stage) {
|
||||
admissionReached = true;
|
||||
profile.suspendAfter = undefined;
|
||||
}
|
||||
admit(request, grant);
|
||||
}, attachment),
|
||||
);
|
||||
await idleSweep.sweep();
|
||||
expect(admissionReached).toBe(true);
|
||||
expect(placements.get(REQUEST.sessionId)).toMatchObject({
|
||||
state: "active",
|
||||
generation: active.generation,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
createPlacementMoveOps,
|
||||
readWorkerPlacementMovesReadOnly,
|
||||
} from "./placement-move-intent.js";
|
||||
import { readWorkerSessionPlacementProjectionInDatabase } from "./placement-read-projection.js";
|
||||
import {
|
||||
retireWorkerSessionPlacement,
|
||||
type WorkerSessionPlacementRetirement,
|
||||
|
|
@ -71,6 +72,20 @@ export const placementReadOperations = {
|
|||
input: { sessionIds?: readonly string[] },
|
||||
{ open }: WorkerOperationContext,
|
||||
) => readWorkerPlacementsInDatabase(open().db, input.sessionIds),
|
||||
"workerPlacements.readWithMove": (
|
||||
input: { sessionId: string },
|
||||
{ open }: WorkerOperationContext,
|
||||
) => {
|
||||
const { projection } = readWorkerSessionPlacementProjectionInDatabase(
|
||||
open().db,
|
||||
[input.sessionId],
|
||||
[],
|
||||
);
|
||||
return {
|
||||
placement: projection.placements.get(input.sessionId),
|
||||
move: projection.moves.get(input.sessionId),
|
||||
};
|
||||
},
|
||||
"workerPlacements.readMove": (input: { sessionId: string }, { open }: WorkerOperationContext) =>
|
||||
readWorkerPlacementMovesReadOnly(open().db, [input.sessionId]).get(input.sessionId),
|
||||
"workerPlacements.readReconcile": (
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
NODE_WORKER_ENVIRONMENT_STOP_COMMAND,
|
||||
NODE_WORKER_WORKSPACE_EXEC_COMMAND,
|
||||
} from "../../infra/node-commands.js";
|
||||
import { createDeferredCore } from "../../shared/deferred.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseAsync,
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
|
|
@ -28,6 +29,10 @@ import { seedAttachedPlacementEnvironment } from "./placement-test-fixtures.js";
|
|||
import { createWorkerEnvironmentService } from "./service.js";
|
||||
import { BUNDLE_ARTIFACT, createProvider } from "./service.test-support.js";
|
||||
import { createWorkerEnvironmentStore } from "./store.js";
|
||||
import {
|
||||
createWorkerWorkspaceOperationCoordinator,
|
||||
type WorkerWorkspaceOperationCoordinator,
|
||||
} from "./workspace-operation-coordinator.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
|
|
@ -709,6 +714,94 @@ describe("offline device placement abandonment", () => {
|
|||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["intent", "generation", "authority"] as const)(
|
||||
"refuses abandoned Stop cleanup when its %s changes while waiting for the workspace",
|
||||
async (change) => {
|
||||
const coordinator = createWorkerWorkspaceOperationCoordinator();
|
||||
const queued = createDeferredCore();
|
||||
let observeCleanup = false;
|
||||
const workspaceOperations: WorkerWorkspaceOperationCoordinator = {
|
||||
run: (environmentId, operation, signal) => {
|
||||
if (observeCleanup) {
|
||||
queued.resolve();
|
||||
}
|
||||
return coordinator.run(environmentId, operation, signal);
|
||||
},
|
||||
};
|
||||
const harness = createHarness(database, placements, { workspaceOperations });
|
||||
const active = await harness.service.dispatch(REQUEST);
|
||||
harness.markEnvironmentNodeDeviceId("device-1");
|
||||
seedEnvironment(active);
|
||||
const begun = await placements.beginPlacementMove(requestFor(active));
|
||||
const held = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
const holding = coordinator.run(active.environmentId, async () => {
|
||||
held.resolve();
|
||||
await release.promise;
|
||||
});
|
||||
await held.promise;
|
||||
observeCleanup = true;
|
||||
let authorized = true;
|
||||
const revoked = new Error("Stop authority revoked while cleanup waits");
|
||||
const stopping = harness.service
|
||||
.reclaim(REQUEST, () => {
|
||||
if (!authorized) {
|
||||
throw revoked;
|
||||
}
|
||||
})
|
||||
.catch((error: unknown) => error);
|
||||
try {
|
||||
await Promise.race([
|
||||
queued.promise,
|
||||
stopping.then((result) => {
|
||||
throw result instanceof Error
|
||||
? result
|
||||
: new Error("Stop did not wait for the workspace");
|
||||
}),
|
||||
]);
|
||||
expect(placements.get(REQUEST.sessionId)).toMatchObject({
|
||||
state: "draining",
|
||||
generation: begun.placement.generation,
|
||||
turnClaim: null,
|
||||
});
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
if (change === "intent") {
|
||||
await placements.cancelPlacementMove({
|
||||
sessionId: REQUEST.sessionId,
|
||||
operationId: begun.intent.operationId,
|
||||
});
|
||||
} else if (change === "generation") {
|
||||
database.db
|
||||
.prepare(
|
||||
"UPDATE worker_session_placements SET transition_generation = transition_generation + 1 WHERE session_id = ?",
|
||||
)
|
||||
.run(REQUEST.sessionId);
|
||||
} else {
|
||||
authorized = false;
|
||||
}
|
||||
release.resolve();
|
||||
const result = await stopping;
|
||||
if (change === "authority") {
|
||||
expect(result).toBe(revoked);
|
||||
} else {
|
||||
expect(result).toBeInstanceOf(Error);
|
||||
expect(result).toMatchObject({
|
||||
message: expect.stringContaining("abandonment source changed before teardown"),
|
||||
});
|
||||
}
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
expect(placements.get(REQUEST.sessionId)).toMatchObject({
|
||||
state: "draining",
|
||||
turnClaim: null,
|
||||
});
|
||||
expect(await placements.listPendingWorkspaceResultsAsync(REQUEST.sessionId)).toEqual([]);
|
||||
} finally {
|
||||
release.resolve();
|
||||
await Promise.all([holding, stopping]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("retains the durable decision when authorization closes after teardown", async () => {
|
||||
let revoked = false;
|
||||
const harness = createHarness(database, placements, {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ import {
|
|||
isForceAbandonedWorkerPlacement,
|
||||
} from "./placement-record.js";
|
||||
import type { PlacementRecoveryDeps } from "./placement-recovery-contract.js";
|
||||
import {
|
||||
matchesWorkerPlacementTarget,
|
||||
type WorkerPlacementCancellationTarget,
|
||||
} from "./placement-target.js";
|
||||
import type {
|
||||
WorkerPlacementAuthorization,
|
||||
WorkerPlacementMoveRequest,
|
||||
|
|
@ -48,14 +52,14 @@ export function createWorkerPlacementMoveAbandonment(
|
|||
? { sessionId, ownerEpoch: environment.ownerEpoch }
|
||||
: undefined;
|
||||
try {
|
||||
return await environments.destroy(environmentId, abandonment, () =>
|
||||
forceAbandonWorkerEnvironment({
|
||||
return await environments.destroy(environmentId, abandonment, async () => {
|
||||
await forceAbandonWorkerEnvironment({
|
||||
placements,
|
||||
environmentId,
|
||||
resolveWorkspace: options.resolveWorkspace,
|
||||
onCleanupError,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
const current = environments.get(environmentId);
|
||||
if (!current || !isUnavailableEnvironment(current)) {
|
||||
|
|
@ -98,26 +102,37 @@ export function createWorkerPlacementMoveAbandonment(
|
|||
request: WorkerPlacementReclaimRequest,
|
||||
intent: WorkerPlacementMoveIntent,
|
||||
authorize?: WorkerPlacementAuthorization,
|
||||
expectedSource?: WorkerPlacementCancellationTarget,
|
||||
): Promise<Extract<WorkerDispatchPlacement, { state: "local" }>> => {
|
||||
const current = placements.get(request.sessionId);
|
||||
if (
|
||||
!current ||
|
||||
(current.state !== "active" &&
|
||||
current.state !== "draining" &&
|
||||
current.state !== "reconciling" &&
|
||||
current.state !== "failed") ||
|
||||
current.environmentId !== intent.source.environmentId ||
|
||||
current.activeOwnerEpoch !== intent.source.ownerEpoch
|
||||
) {
|
||||
throw new Error(`Session ${request.sessionKey} abandonment source changed before teardown`);
|
||||
}
|
||||
await options.workspaceOperations.run(intent.source.environmentId, async () => {
|
||||
await forceAbandonWorkerEnvironment({
|
||||
const fenced = await options.workspaceOperations.run(intent.source.environmentId, async () => {
|
||||
const { placement: current, move } = await placements.getWithMoveAsync(request.sessionId);
|
||||
authorize?.();
|
||||
if (
|
||||
!current ||
|
||||
(expectedSource !== undefined && !matchesWorkerPlacementTarget(current, expectedSource)) ||
|
||||
(current.state !== "active" &&
|
||||
current.state !== "draining" &&
|
||||
current.state !== "reconciling" &&
|
||||
current.state !== "failed") ||
|
||||
current.sessionKey !== request.sessionKey ||
|
||||
current.agentId !== request.agentId ||
|
||||
current.environmentId !== intent.source.environmentId ||
|
||||
current.activeOwnerEpoch !== intent.source.ownerEpoch ||
|
||||
move?.operationId !== intent.operationId ||
|
||||
!move.abandonSource ||
|
||||
move.target.kind !== "gateway" ||
|
||||
move.source.generation !== intent.source.generation ||
|
||||
move.source.environmentId !== intent.source.environmentId ||
|
||||
move.source.ownerEpoch !== intent.source.ownerEpoch
|
||||
) {
|
||||
throw new Error(`Session ${request.sessionKey} abandonment source changed before teardown`);
|
||||
}
|
||||
const failedPlacements = await forceAbandonWorkerEnvironment({
|
||||
placements,
|
||||
environmentId: intent.source.environmentId,
|
||||
resolveWorkspace: options.resolveWorkspace,
|
||||
});
|
||||
const failed = placements.get(request.sessionId);
|
||||
const failed = failedPlacements.get(request.sessionId);
|
||||
if (!isForceAbandonedWorkerPlacement(failed)) {
|
||||
throw new Error(`Session ${request.sessionKey} abandonment did not fence its remote owner`);
|
||||
}
|
||||
|
|
@ -152,17 +167,18 @@ export function createWorkerPlacementMoveAbandonment(
|
|||
authorize: assertCurrent,
|
||||
});
|
||||
}
|
||||
// The completion transaction compares this acknowledged generation after teardown.
|
||||
return failed;
|
||||
});
|
||||
authorize?.();
|
||||
const failed = placements.get(request.sessionId);
|
||||
if (failed?.state !== "failed") {
|
||||
if (fenced?.state !== "failed") {
|
||||
throw new Error(`Session ${request.sessionKey} abandonment did not fence its remote owner`);
|
||||
}
|
||||
const local = await placements.completeAbandonedPlacementMoveSourceToLocal(
|
||||
{
|
||||
operationId: intent.operationId,
|
||||
sessionId: intent.sessionId,
|
||||
expectedGeneration: failed.generation,
|
||||
expectedGeneration: fenced.generation,
|
||||
expectedRecoveryError: FORCED_WORKER_ABANDONMENT_ERROR,
|
||||
},
|
||||
{ assertCurrent: authorize },
|
||||
|
|
|
|||
|
|
@ -130,22 +130,23 @@ export function createWorkerPlacementMoveService(options: {
|
|||
};
|
||||
// Existing durable decisions own retries. Prepare only a new intent, outside
|
||||
// the synchronous commit, so both branches publish their owner before yielding.
|
||||
if (
|
||||
request.abandonSource &&
|
||||
!(await options.placements.getPlacementMoveAsync(request.sessionId))
|
||||
) {
|
||||
const placement = await options.placements.getAsync(request.sessionId);
|
||||
options.validateAbandonSource(request, placement);
|
||||
const claim = placement ? projectWorkerSessionTurnClaim(placement) : undefined;
|
||||
if (claim && prepareNew) {
|
||||
await prepareNew(claim.runId);
|
||||
const current = await options.placements.getAsync(request.sessionId);
|
||||
if (!current || !isCurrentPlacementTurnClaim(current, claim)) {
|
||||
throw new Error(
|
||||
`Session ${request.sessionKey} abandonment worker turn changed; retry`,
|
||||
);
|
||||
if (request.abandonSource) {
|
||||
const { placement, move: existingMove } = await options.placements.getWithMoveAsync(
|
||||
request.sessionId,
|
||||
);
|
||||
if (!existingMove) {
|
||||
options.validateAbandonSource(request, placement);
|
||||
const claim = placement ? projectWorkerSessionTurnClaim(placement) : undefined;
|
||||
if (claim && prepareNew) {
|
||||
await prepareNew(claim.runId);
|
||||
const current = await options.placements.getAsync(request.sessionId);
|
||||
if (!current || !isCurrentPlacementTurnClaim(current, claim)) {
|
||||
throw new Error(
|
||||
`Session ${request.sessionKey} abandonment worker turn changed; retry`,
|
||||
);
|
||||
}
|
||||
options.validateAbandonSource(request, current);
|
||||
}
|
||||
options.validateAbandonSource(request, current);
|
||||
}
|
||||
}
|
||||
const started = await options.placements.beginPlacementMove(moveRequest, {
|
||||
|
|
|
|||
|
|
@ -36,6 +36,14 @@ export function createPlacementReadStore(params: {
|
|||
const id = required(sessionId, "session id");
|
||||
return (await getManyAsync([id])).get(id);
|
||||
},
|
||||
async getWithMoveAsync(sessionId: string) {
|
||||
const id = required(sessionId, "session id");
|
||||
const result = await runOpenClawStateWorkerOperation(context, (scope) =>
|
||||
scope.execute({ type: "workerPlacements.readWithMove", input: { sessionId: id } }),
|
||||
);
|
||||
const [placement] = decorate(result.placement ? [result.placement] : []);
|
||||
return { ...result, placement };
|
||||
},
|
||||
listAsync: () => read(),
|
||||
async getPlacementMoveAsync(sessionId: string) {
|
||||
const id = required(sessionId, "move session id");
|
||||
|
|
|
|||
|
|
@ -66,7 +66,9 @@ export function createWorkerPlacementReclaim(options: WorkerPlacementReclaimOpti
|
|||
authorize,
|
||||
beforeDrain,
|
||||
begin: async (assertCurrent) => {
|
||||
const current = placements.get(request.sessionId);
|
||||
const current = await placements.getAsync(request.sessionId);
|
||||
assertCurrent?.();
|
||||
beforeDrain?.assertCurrent?.();
|
||||
// A queued stop can observe the previous stop's completion only after
|
||||
// entering the lifecycle fence; joining an outside promise can deadlock it.
|
||||
if (
|
||||
|
|
@ -251,7 +253,6 @@ export function createWorkerPlacementReclaim(options: WorkerPlacementReclaimOpti
|
|||
}
|
||||
};
|
||||
assertCurrent();
|
||||
reauthorize?.();
|
||||
const quiescence = await tunnel.quiesceWorkspace(current.remoteWorkspaceDir);
|
||||
try {
|
||||
assertCurrent();
|
||||
|
|
@ -285,7 +286,6 @@ export function createWorkerPlacementReclaim(options: WorkerPlacementReclaimOpti
|
|||
if (reconciliation.changed && !wasAccepted()) {
|
||||
throw new Error("Cloud worker stop did not commit its reconciled workspace");
|
||||
}
|
||||
reauthorize?.();
|
||||
assertCurrent();
|
||||
await placements.acceptWorkspaceResult(reclaimClaim, reauthorize);
|
||||
const recordedStagedResultRef = (
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ describe("worker session placement moves", () => {
|
|||
};
|
||||
}
|
||||
|
||||
it("mutates native move state without caller-thread SQL", async () => {
|
||||
it("reads and mutates native move state without caller-thread SQL", async () => {
|
||||
const active = await seedActiveEnvironment();
|
||||
const request = {
|
||||
sessionId: ` ${SESSION.sessionId} `,
|
||||
|
|
@ -90,6 +90,18 @@ describe("worker session placement moves", () => {
|
|||
const sql = observeMainThreadSql();
|
||||
try {
|
||||
const begun = await store.beginPlacementMove(request);
|
||||
expect(await store.getWithMoveAsync(request.sessionId)).toMatchObject({
|
||||
placement: {
|
||||
sessionId: SESSION.sessionId,
|
||||
state: "draining",
|
||||
generation: begun.placement.generation,
|
||||
},
|
||||
move: {
|
||||
operationId: begun.intent.operationId,
|
||||
sessionId: SESSION.sessionId,
|
||||
source: request.source,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
await store.recordPlacementMoveError({
|
||||
operationId: begun.intent.operationId,
|
||||
|
|
|
|||
|
|
@ -41,8 +41,8 @@ export function createRepositoryWorkspaceMutationService(options: {
|
|||
mutate: (assertCurrent: () => void) => Promise<{ changed: boolean; value: T }>;
|
||||
},
|
||||
): Promise<T> {
|
||||
const placement = await placements.getAsync(params.sessionId);
|
||||
params.assertCurrent();
|
||||
const placement = placements.get(params.sessionId);
|
||||
if (placement?.state !== "active") {
|
||||
throw new Error("Repository workspace editing requires an active cloud placement");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ function readSessionWorkerPlacement(params: {
|
|||
: undefined;
|
||||
}
|
||||
|
||||
async function readSessionWorkerPlacementAsync(params: {
|
||||
export async function readSessionWorkerPlacementAsync(params: {
|
||||
context: SessionWorkerPlacementContext;
|
||||
sessionId?: string;
|
||||
}): Promise<Placement | undefined> {
|
||||
|
|
@ -296,7 +296,7 @@ export async function prepareSessionWorkerPlacementRetirement(
|
|||
retire: async () => {
|
||||
// Called only after confirmed deletion; orphan reconciliation may have
|
||||
// retired this placement while transcript archive publication awaited.
|
||||
if (!readSessionWorkerPlacement(params)) {
|
||||
if (!(await readSessionWorkerPlacementAsync(params))) {
|
||||
return;
|
||||
}
|
||||
assertCurrent();
|
||||
|
|
|
|||
|
|
@ -282,7 +282,7 @@ export function createWorkerSessionToolExecutor(
|
|||
throw new Error("Cloud child placement does not match its parent profile");
|
||||
}
|
||||
};
|
||||
const childPlacement = params.placements.get(childSessionId);
|
||||
const childPlacement = await params.placements.getAsync(childSessionId);
|
||||
assertSource();
|
||||
if (childPlacement?.state !== "active") {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@ export async function waitForPendingWorkerResult(params: {
|
|||
);
|
||||
// Restart clears local claims without discarding durable results. A claimless result cannot
|
||||
// make progress through this wait; keep its fence and let recovery retain control of the files.
|
||||
const pendingResults = await params.placements.listPendingWorkspaceResultsAsync(params.sessionId);
|
||||
const facts = await params.placements.readProjection([params.sessionId], { current: true });
|
||||
params.signal?.throwIfAborted();
|
||||
if (
|
||||
!params.placements.get(params.sessionId)?.turnClaim &&
|
||||
pendingResults.some((pending) => pending.sessionId === params.sessionId)
|
||||
!facts.placements.get(params.sessionId)?.turnClaim &&
|
||||
facts.pendingResults.has(params.sessionId)
|
||||
) {
|
||||
throw new Error(
|
||||
"Workspace recovery is still pending after its turn ended. " +
|
||||
|
|
@ -124,6 +124,7 @@ export async function waitForInitialWorkerPlacement(params: {
|
|||
const completed = await params.wait(params.placement, params.turn.abortSignal);
|
||||
// Setup completion is a notification, not authority: read the durable owner again.
|
||||
assertSessionCurrent();
|
||||
let placement = completed;
|
||||
const assertCurrent = composeSessionSourceAssertion([assertSessionCurrent], (assertSession) => {
|
||||
assertSession();
|
||||
const current = params.placements.get(identity.sessionId);
|
||||
|
|
@ -136,10 +137,11 @@ export async function waitForInitialWorkerPlacement(params: {
|
|||
) {
|
||||
throw createAbortError("Worker placement changed while waiting for setup");
|
||||
}
|
||||
placement = current;
|
||||
});
|
||||
assertCurrent();
|
||||
return {
|
||||
placement: requireActivePlacement(params.placements.get(identity.sessionId)!),
|
||||
placement: requireActivePlacement(placement),
|
||||
assertCurrent,
|
||||
};
|
||||
}
|
||||
|
|
@ -370,17 +372,17 @@ export async function claimWorkerTurn(params: {
|
|||
) {
|
||||
throw error;
|
||||
}
|
||||
const pendingResults = await params.placements.listPendingWorkspaceResultsAsync(
|
||||
params.identity.sessionId,
|
||||
);
|
||||
const facts = await params.placements.readProjection([params.identity.sessionId], {
|
||||
current: true,
|
||||
});
|
||||
params.signal?.throwIfAborted();
|
||||
params.assertCurrent?.();
|
||||
const activePlacement = params.placements.get(params.identity.sessionId);
|
||||
const activePlacement = facts.placements.get(params.identity.sessionId);
|
||||
const activeClaim = activePlacement?.turnClaim;
|
||||
if (activeClaim?.runId === params.runId) {
|
||||
throw error;
|
||||
}
|
||||
const resultIsReconciling = pendingResults.some(
|
||||
const resultIsReconciling = [...facts.pendingResults.values()].some(
|
||||
(pending) =>
|
||||
activeClaim?.owner === "worker" &&
|
||||
pending.sessionId === params.identity.sessionId &&
|
||||
|
|
@ -397,7 +399,9 @@ export async function claimWorkerTurn(params: {
|
|||
return null;
|
||||
}
|
||||
if (!(cancelledClaim && params.isCancellationRequested(cancelledClaim))) {
|
||||
const refreshed = params.placements.get(params.identity.sessionId);
|
||||
const refreshed = await params.placements.getAsync(params.identity.sessionId);
|
||||
params.signal?.throwIfAborted();
|
||||
params.assertCurrent?.();
|
||||
if (
|
||||
refreshed?.state !== "active" ||
|
||||
!matchesWorkerPlacementTarget(refreshed, params.placement) ||
|
||||
|
|
@ -412,7 +416,9 @@ export async function claimWorkerTurn(params: {
|
|||
timeoutMs: SESSION_WORK_ADMISSION_DRAIN_TIMEOUT_MS,
|
||||
...(params.signal ? { signal: params.signal } : {}),
|
||||
});
|
||||
const refreshed = params.placements.get(params.identity.sessionId);
|
||||
const refreshed = await params.placements.getAsync(params.identity.sessionId);
|
||||
params.signal?.throwIfAborted();
|
||||
params.assertCurrent?.();
|
||||
if (refreshed?.state !== "active" || !matchesWorkerPlacementTarget(refreshed, params.placement)) {
|
||||
throw new Error("Cloud worker placement changed while waiting for the previous turn");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ export async function failHandedOffTurn(params: {
|
|||
if (error instanceof AcceptedWorkspacePublicationIndeterminateError) {
|
||||
throw error;
|
||||
}
|
||||
const current = params.placements.get(params.placement.sessionId);
|
||||
const current = await params.placements.getAsync(params.placement.sessionId);
|
||||
const exactDrainOwner =
|
||||
current?.state === "draining" &&
|
||||
current.generation === params.placement.generation + 1 &&
|
||||
|
|
|
|||
|
|
@ -274,9 +274,14 @@ describe("worker turn launcher claim admission", () => {
|
|||
},
|
||||
});
|
||||
const claimOps = createPlacementTurnClaimFixtureOps(database);
|
||||
vi.spyOn(placements, "listPendingWorkspaceResultsAsync").mockImplementationOnce(async () => {
|
||||
claimOps.releaseTurn(priorClaim);
|
||||
return [];
|
||||
const claimTurn = placements.claimTurn.bind(placements);
|
||||
vi.spyOn(placements, "claimTurn").mockImplementationOnce(async (...args) => {
|
||||
try {
|
||||
return await claimTurn(...args);
|
||||
} catch (error) {
|
||||
claimOps.releaseTurn(priorClaim);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
const provider = createWorkerSessionTurnPlacementProvider({
|
||||
environments: unusedEnvironments(),
|
||||
|
|
@ -311,12 +316,15 @@ describe("worker turn launcher claim admission", () => {
|
|||
runId: "remote-result-run",
|
||||
owner: placementTurnOwner(active),
|
||||
});
|
||||
await placements.markWorkspaceResultPending(priorClaim);
|
||||
const listPendingWorkspaceResultsAsync = vi.spyOn(
|
||||
placements,
|
||||
"listPendingWorkspaceResultsAsync",
|
||||
);
|
||||
listPendingWorkspaceResultsAsync.mockResolvedValueOnce([]);
|
||||
const claimTurn = placements.claimTurn.bind(placements);
|
||||
vi.spyOn(placements, "claimTurn").mockImplementationOnce(async (...args) => {
|
||||
try {
|
||||
return await claimTurn(...args);
|
||||
} catch (error) {
|
||||
await placements.markWorkspaceResultPending(priorClaim);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
const waitForRelease = vi.spyOn(placements, "waitForTurnClaimRelease");
|
||||
const provider = createWorkerSessionTurnPlacementProvider({
|
||||
environments: unusedEnvironments(),
|
||||
|
|
|
|||
|
|
@ -219,13 +219,14 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
if (!current || current.state === "local") {
|
||||
return await runLocalTurn();
|
||||
}
|
||||
const hasPendingWorkspaceResultToSettle = async (sessionId: string, runId: string) =>
|
||||
(await options.placements.listPendingWorkspaceResultsAsync(sessionId)).some(
|
||||
(pending) =>
|
||||
pending.sessionId === sessionId &&
|
||||
// A restarted run has no live claim, even when it reuses the retained run ID.
|
||||
(pending.runId !== runId || !options.placements.get(sessionId)?.turnClaim),
|
||||
const hasPendingWorkspaceResultToSettle = async (sessionId: string, runId: string) => {
|
||||
const facts = await options.placements.readProjection([sessionId], { current: true });
|
||||
const pending = facts.pendingResults.get(sessionId);
|
||||
// A restarted run has no live claim, even when it reuses the retained run ID.
|
||||
return Boolean(
|
||||
pending && (pending.runId !== runId || !facts.placements.get(sessionId)?.turnClaim),
|
||||
);
|
||||
};
|
||||
let identity = resolvePlacementIdentity(claim, current);
|
||||
const reportProvisioning = () =>
|
||||
emitAgentRunStatusEvent({
|
||||
|
|
@ -251,9 +252,9 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
},
|
||||
);
|
||||
// An admission wait ends without authority; retry from the durable placement.
|
||||
const readRoutablePlacement = (message: string, cause?: unknown) => {
|
||||
const readRoutablePlacement = async (message: string, cause?: unknown) => {
|
||||
const refreshed = await options.placements.getAsync(identity.sessionId);
|
||||
assertAdmissionCurrent();
|
||||
const refreshed = options.placements.get(identity.sessionId);
|
||||
if (!refreshed) {
|
||||
throw new Error(message, cause === undefined ? undefined : { cause });
|
||||
}
|
||||
|
|
@ -311,7 +312,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
sessionId: identity.sessionId,
|
||||
...(turn.abortSignal ? { signal: turn.abortSignal } : {}),
|
||||
});
|
||||
routablePlacement = readRoutablePlacement(
|
||||
routablePlacement = await readRoutablePlacement(
|
||||
"Cloud worker placement disappeared after workspace reconciliation",
|
||||
);
|
||||
continue;
|
||||
|
|
@ -333,7 +334,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
reason: "worker:runtime_refresh",
|
||||
}),
|
||||
});
|
||||
routablePlacement = readRoutablePlacement(
|
||||
routablePlacement = await readRoutablePlacement(
|
||||
"Cloud worker placement disappeared while waiting for runtime refresh",
|
||||
);
|
||||
continue;
|
||||
|
|
@ -379,7 +380,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
...(turn.abortSignal ? { signal: turn.abortSignal } : {}),
|
||||
});
|
||||
}
|
||||
routablePlacement = readRoutablePlacement(
|
||||
routablePlacement = await readRoutablePlacement(
|
||||
refreshing
|
||||
? "Cloud worker placement disappeared during runtime refresh admission"
|
||||
: "Cloud worker placement disappeared after workspace reconciliation",
|
||||
|
|
@ -388,7 +389,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
continue;
|
||||
}
|
||||
if (!admitted) {
|
||||
routablePlacement = readRoutablePlacement(
|
||||
routablePlacement = await readRoutablePlacement(
|
||||
"Cloud worker placement disappeared after workspace reconciliation",
|
||||
);
|
||||
continue;
|
||||
|
|
@ -604,7 +605,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
if (!disconnectedBeforeHandoff) {
|
||||
await options.reconcileActivePlacement(placement.environmentId);
|
||||
}
|
||||
const reconciled = options.placements.get(placement.sessionId);
|
||||
const reconciled = await options.placements.getAsync(placement.sessionId);
|
||||
if (canRecoverAdmission) {
|
||||
assertAdmissionCurrent();
|
||||
turn.abortSignal?.throwIfAborted();
|
||||
|
|
@ -678,7 +679,7 @@ export function createWorkerSessionTurnPlacementProvider(options: WorkerTurnLaun
|
|||
await releaseClaimIfOwned(options.placements, turnClaim);
|
||||
throw error;
|
||||
}
|
||||
const settledPlacement = options.placements.get(turnClaim.sessionId);
|
||||
const settledPlacement = await options.placements.getAsync(turnClaim.sessionId);
|
||||
if (
|
||||
(remoteExec || error instanceof WorkerTurnExecutionError) &&
|
||||
settledPlacement?.state === "active" &&
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ export function createModelSelectionMocks() {
|
|||
resolveSessionWorkerPlacementContext: () => ({
|
||||
workerSessionPlacementService: {
|
||||
getMany: placementMocks.getMany,
|
||||
getManyAsync: async (sessionIds: readonly string[]) => placementMocks.getMany(sessionIds),
|
||||
},
|
||||
}),
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import path from "node:path";
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createDeferred } from "../../test/helpers/promise.js";
|
||||
import { awaitGateBeforeSettlement, createDeferred } from "../../test/helpers/promise.js";
|
||||
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
|
||||
import { loadProviderScopedThinkingCatalog } from "../agents/model-catalog.runtime.js";
|
||||
import {
|
||||
|
|
@ -33,7 +33,7 @@ vi.mock("../agents/model-catalog.runtime.js", () => ({
|
|||
loadProviderScopedThinkingCatalog: vi.fn(async () => []),
|
||||
}));
|
||||
|
||||
const { effects, factories, resetMocks } = await vi.hoisted(async () => {
|
||||
const { effects, factories, placementMocks, resetMocks } = await vi.hoisted(async () => {
|
||||
const { createModelSelectionMocks } =
|
||||
await import("./apply-session-model-selection.test-support.js");
|
||||
return createModelSelectionMocks();
|
||||
|
|
@ -516,24 +516,44 @@ describe("applySessionModelSelection", () => {
|
|||
expectNoSelectionEffects();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "locked",
|
||||
concurrent: createEntry({ modelSelectionLocked: true }),
|
||||
outcome: { status: "rejected", reason: "locked" },
|
||||
},
|
||||
{
|
||||
name: "replaced",
|
||||
concurrent: createEntry({ sessionId: "session-2" }),
|
||||
outcome: { status: "conflict" },
|
||||
},
|
||||
])(
|
||||
"preserves an in-memory session $name during metadata preparation",
|
||||
async ({ concurrent, outcome }) => {
|
||||
it.each(
|
||||
[
|
||||
{
|
||||
name: "locked",
|
||||
concurrent: createEntry({ modelSelectionLocked: true }),
|
||||
outcome: { status: "rejected", reason: "locked" },
|
||||
},
|
||||
{
|
||||
name: "replaced",
|
||||
concurrent: createEntry({ sessionId: "session-2" }),
|
||||
outcome: { status: "conflict" },
|
||||
},
|
||||
].flatMap(({ name, concurrent, outcome }) =>
|
||||
["metadata", "placement"].map((phase) => ({ name, concurrent, outcome, phase })),
|
||||
),
|
||||
)(
|
||||
"preserves an in-memory session $name during $phase preparation",
|
||||
async ({ concurrent, outcome, phase }) => {
|
||||
const entered = createDeferred();
|
||||
const metadata = createDeferred<ModelCatalogEntry[]>();
|
||||
vi.mocked(loadProviderScopedThinkingCatalog).mockReturnValueOnce(metadata.promise);
|
||||
const params = createParams();
|
||||
if (phase === "metadata") {
|
||||
vi.mocked(loadProviderScopedThinkingCatalog).mockImplementationOnce(() => {
|
||||
entered.resolve();
|
||||
return metadata.promise;
|
||||
});
|
||||
} else {
|
||||
placementMocks.getMany.mockImplementationOnce(() => {
|
||||
entered.resolve();
|
||||
return new Map();
|
||||
});
|
||||
}
|
||||
const params = createParams({ thinkingCatalog: [] });
|
||||
const pending = applySessionModelSelection(params);
|
||||
await awaitGateBeforeSettlement(
|
||||
entered.promise,
|
||||
pending,
|
||||
`Model selection did not enter ${phase} preparation`,
|
||||
);
|
||||
params.sessionStore[params.sessionKey] = concurrent;
|
||||
metadata.resolve([]);
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
|||
import { triggerSessionPatchHook } from "../gateway/session-patch-hooks.js";
|
||||
import { resolveSessionWorkerPlacementContext } from "../gateway/session-worker-placement-context.js";
|
||||
import { resolveWorkerPlacementSessionRuntimeCapabilities } from "../gateway/worker-environments/placement-session-runtime.js";
|
||||
import { readSessionWorkerPlacementAsync } from "../gateway/worker-environments/session-placement-lifecycle.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { resolveSystemEventQueueKey } from "../infra/system-event-ownership.js";
|
||||
import { enqueueSystemEvent } from "../infra/system-events.js";
|
||||
|
|
@ -128,18 +129,23 @@ function rejectNotAllowed(provider: string, model: string): ApplySessionModelSel
|
|||
* active cloud-worker placement. Mirrors the sessions.patch guard so directive
|
||||
* model changes are validated before they persist.
|
||||
*/
|
||||
function resolveActivePlacementModelSelectionError(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
entry: SessionEntry;
|
||||
}): string | undefined {
|
||||
function resolveActivePlacementModelSelectionError(
|
||||
params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
entry: SessionEntry;
|
||||
},
|
||||
prepared?: { placement: Awaited<ReturnType<typeof readSessionWorkerPlacementAsync>> },
|
||||
): string | undefined {
|
||||
const sessionId = params.entry.sessionId;
|
||||
if (!sessionId) {
|
||||
return undefined;
|
||||
}
|
||||
const placementService = resolveSessionWorkerPlacementContext().workerSessionPlacementService;
|
||||
const placement = placementService?.getMany([sessionId]).get(sessionId);
|
||||
const placement = prepared
|
||||
? prepared.placement
|
||||
: placementService?.getMany([sessionId]).get(sessionId);
|
||||
if (!placement || placement.state === "local") {
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -252,24 +258,6 @@ export async function applySessionModelSelectionInternal(
|
|||
if (authProfileError) {
|
||||
return { status: "rejected", reason: "not-allowed", message: authProfileError };
|
||||
}
|
||||
// Metadata preparation can yield. Memory-only sessions need the same lock and
|
||||
// replacement fence that persisted sessions enforce in their atomic write.
|
||||
const currentEntry = params.storePath
|
||||
? startingEntry
|
||||
: (params.sessionStore[params.sessionKey] ?? params.sessionEntry);
|
||||
if (isModelSelectionLocked(currentEntry)) {
|
||||
return { status: "rejected", reason: "locked", message: MODEL_SELECTION_LOCKED_MESSAGE };
|
||||
}
|
||||
if (
|
||||
!params.storePath &&
|
||||
(params.sessionStore[params.sessionKey] !== startingStoreEntry ||
|
||||
currentEntry.sessionId !== initialEntry.sessionId)
|
||||
) {
|
||||
return {
|
||||
status: "conflict",
|
||||
message: "Model change was not applied because the session changed. Retry.",
|
||||
};
|
||||
}
|
||||
const runtime = prepared.runtime;
|
||||
const thinkingCatalog = prepared.catalog;
|
||||
const selectedCatalogEntry = findSelectedCatalogEntry({ catalog: thinkingCatalog, ...request });
|
||||
|
|
@ -319,12 +307,20 @@ export async function applySessionModelSelectionInternal(
|
|||
};
|
||||
}
|
||||
}
|
||||
const placementError = resolveActivePlacementModelSelectionError({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
entry: nextEntry,
|
||||
});
|
||||
const placementError = resolveActivePlacementModelSelectionError(
|
||||
{
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
entry: nextEntry,
|
||||
},
|
||||
{
|
||||
placement: await readSessionWorkerPlacementAsync({
|
||||
context: resolveSessionWorkerPlacementContext(),
|
||||
sessionId: nextEntry.sessionId,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (placementError) {
|
||||
return { status: "rejected", reason: "invalid-runtime", message: placementError };
|
||||
}
|
||||
|
|
@ -385,6 +381,21 @@ export async function applySessionModelSelectionInternal(
|
|||
if (commitError) {
|
||||
return { status: "rejected", reason: "not-allowed", message: commitError };
|
||||
}
|
||||
// Both metadata and placement preparation can yield. Fence the in-memory
|
||||
// write here, where persisted sessions enforce their lock and identity.
|
||||
const currentEntry = params.sessionStore[params.sessionKey] ?? params.sessionEntry;
|
||||
if (isModelSelectionLocked(currentEntry)) {
|
||||
return { status: "rejected", reason: "locked", message: MODEL_SELECTION_LOCKED_MESSAGE };
|
||||
}
|
||||
if (
|
||||
params.sessionStore[params.sessionKey] !== startingStoreEntry ||
|
||||
currentEntry.sessionId !== initialEntry.sessionId
|
||||
) {
|
||||
return {
|
||||
status: "conflict",
|
||||
message: "Model change was not applied because the session changed. Retry.",
|
||||
};
|
||||
}
|
||||
adoptPersistedSessionSnapshot(params.sessionEntry, nextEntry);
|
||||
params.sessionStore[params.sessionKey] = params.sessionEntry;
|
||||
persistedEntry = params.sessionEntry;
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ export const databaseWorkerCoreTestFiles = [
|
|||
"src/infra/restart-sentinel.update-result.test.ts",
|
||||
"src/auto-reply/reply/conversation-turn-capture.test.ts",
|
||||
"src/auto-reply/reply/agent-runner-result-accounting.persistence.test.ts",
|
||||
"src/auto-reply/reply/agent-runner.media-ownership.test.ts",
|
||||
"src/auto-reply/reply/agent-runner-result-accounting.fallback.test.ts",
|
||||
"src/auto-reply/reply/dispatch-from-config.pending-final.test.ts",
|
||||
"src/auto-reply/reply/followup-turn-execution.worker.test.ts",
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ export const gatewayDatabaseWorkerTestFiles = [
|
|||
"src/gateway/board-http.test.ts",
|
||||
"src/gateway/board-store.test.ts",
|
||||
"src/gateway/call-device-auth.worker.test.ts",
|
||||
"src/gateway/chat-auth-readiness.integration.test.ts",
|
||||
"src/gateway/chat-display-projection.cron.test.ts",
|
||||
"src/gateway/config-reload.activation.integration.test.ts",
|
||||
"src/gateway/config-reload.lease-retry.test.ts",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue