perf(agents): prepare workers before run cleanup

Prepare running task settlement and Git guard reads through existing workers so cleanup avoids cold initialization. Preserve live authority, exact lease deletion, and awaited terminal publication. Release captured opening context after native settlement so cached actors do not retain it. Validated on Blacksmith Testbox with cold-start, contention, lifecycle, GC, typecheck, lint, and regression proof recorded in the PR.
This commit is contained in:
Peter Steinberger 2026-09-23 19:14:08 -07:00 • committed by GitHub
parent 2d6533eac4
commit e789666dc9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 184 additions and 135 deletions

View file

@ -282,10 +282,17 @@ Worktree run-lease cleanup deletes the exact token and reads the Git unlock targ
through the shared-state worker. Failed deletions yield between bounded retries,
retaining the original database admission and Git guard until deletion settles.
Process exit retains its best-effort synchronous deletion because it cannot await
a worker. Deferred context maintenance also awaits task completion and failure
a worker. Git-guard admission reads its registry target through the same retained
worker used by cleanup. Deferred context maintenance publishes its running state
through the task worker before entering the engine, preparing the existing writer
and reader owners for completion. It still awaits task completion and failure
settlement before disposing its engine or releasing its process owner; its progress
timer ends before terminal persistence. Worktree run admission, task creation and
progress, and the remaining native cron transitions still need migration. This
timer ends before terminal persistence. Cold worker startup belongs to admission;
normal idle retirement and memory-pressure eviction remain in effect. Detached
worker opening evaluates live admission guards in their captured caller context,
then releases that capture after native opening settles.
Worktree run admission writes, task creation and progress, and the remaining native
cron transitions still need migration. This
cutover preserves schemas, stored bytes, retention, configuration, and update behavior.
Native cron receipt guards read deletion authority through their transaction's

View file

@ -177,6 +177,8 @@ it.for([
const values: unknown[] = [];
let lateWriteBlocked = false;
let disposalCalls = 0;
let resumedAt = 0;
let disposalLatency = 0;
let physicalDisposals = 0;
let workSignal: AbortSignal | undefined;
let backendSignal: AbortSignal | undefined;
@ -280,6 +282,7 @@ it.for([
: pending;
},
async dispose() {
disposalLatency = performance.now() - resumedAt;
disposalCalls++;
disposalEntered.resolve();
const capturedSignal =
@ -393,6 +396,7 @@ it.for([
if (factory === "none") {
expect.soft(workSignal?.aborted ?? false).toBe(false);
}
resumedAt = performance.now();
resume.resolve();
if (deferred) {
// Worker bookkeeping publishes before engine disposal can start.
@ -402,6 +406,9 @@ it.for([
await Promise.allSettled(work.slice(0, 1));
}
await racePromiseWithAbortSignal(disposalEntered.promise, signal);
if (deferred) {
expect.soft(disposalLatency).toBeLessThan(50);
}
await withTestTimeout(
cleanupTailEntered.promise,
1_000,

View file

@ -1,5 +1,6 @@
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
import { expect, vi } from "vitest";
import { expect } from "vitest";
import { createDeferred } from "../../../test/helpers/promise.js";
import type { ContextEngine } from "../../context-engine/types.js";
import { peekSystemEvents } from "../../infra/system-events.js";
import { createQueuedTaskRunCore as createQueuedTaskRunOrNull } from "../../tasks/task-executor.js";
@ -19,13 +20,18 @@ export function createQueuedTaskRunCore(
export function createBackgroundMaintenanceEngine(
maintain: NonNullable<ContextEngine["maintain"]>,
id = "test",
): ContextEngine {
): ContextEngine & { started: Promise<void> } {
const started = createDeferred();
return {
started: started.promise,
info: { id, name: "Test Engine", turnMaintenanceMode: "background" },
ingest: async () => ({ ingested: true }),
assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }),
compact: async () => ({ ok: true, compacted: false }),
maintain,
maintain(params) {
started.resolve();
return maintain(params);
},
};
}
@ -35,28 +41,6 @@ export async function flushAsyncWork(times = 4): Promise<void> {
}
}
export async function waitForAssertion(
assertion: () => void,
timeoutMs = 2_000,
stepMs = 5,
): Promise<void> {
// Timed polling lets fake-timer tasks advance through queue and delivery
// microtasks without binding assertions to a specific internal await count.
const startedAt = Date.now();
for (;;) {
try {
assertion();
return;
} catch (error) {
if (Date.now() - startedAt >= timeoutMs) {
throw error;
}
await vi.advanceTimersByTimeAsync(stepMs);
await flushAsyncWork();
}
}
}
export const requireRecord = createRequireRecord("record", "expected-label");
export function firstMaintainParams(maintain: {

View file

@ -1,4 +1,5 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createDeferred } from "../../../test/helpers/promise.js";
import { resetAllLanes } from "../../process/command-queue.js";
import { resetCommandQueueStateForTest } from "../../process/command-queue.test-support.js";
import { drainGlobalSingletonLifecycleState } from "../../shared/global-singleton.js";
@ -43,6 +44,7 @@ describe("deferred context-engine maintenance lifecycle", () => {
configureTaskRegistryMaintenance({ runtimeAuthoritative: true });
const sessionKey = "agent:main:context-maintenance-lifecycle";
const maintenanceStarted = createDeferred();
let releaseMaintenance: (() => void) | undefined;
let deferredMaintenance: Promise<void> | undefined;
await runContextEngineMaintenance({
@ -56,6 +58,7 @@ describe("deferred context-engine maintenance lifecycle", () => {
assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }),
compact: async () => ({ ok: true, compacted: false }),
maintain: async () => {
maintenanceStarted.resolve();
await new Promise<void>((resolve) => {
releaseMaintenance = resolve;
});
@ -71,6 +74,7 @@ describe("deferred context-engine maintenance lifecycle", () => {
},
});
await maintenanceStarted.promise;
await vi.advanceTimersByTimeAsync(11_000);
const task = listTasksForOwnerKey(sessionKey).find(
(candidate) =>

View file

@ -32,12 +32,12 @@ const mocks = vi.hoisted(() => ({
vi.mock("../../tasks/detached-task-runtime.js", () => ({
createQueuedTaskRun: mocks.create,
startTaskRunByRunId: mocks.start,
completeTaskRunByRunId: () => [],
failTaskRunByRunId: () => [],
recordTaskRunProgressByRunId: mocks.progress,
}));
vi.mock("../../tasks/detached-task-runtime.async.js", () => ({
startTaskRunByRunIdAsync: mocks.start,
completeTaskRunByRunIdAsync: mocks.complete,
failTaskRunByRunIdAsync: mocks.fail,
}));
@ -238,6 +238,7 @@ let sql: ReturnType<typeof observeMainThreadSql>;
beforeEach(() => {
sql = observeMainThreadSql();
vi.clearAllMocks();
mocks.start.mockResolvedValue([]);
mocks.complete.mockResolvedValue([]);
mocks.fail.mockResolvedValue([]);
resetCommandQueueStateForTest();
@ -255,7 +256,7 @@ afterEach(() => {
});
describe("deferred maintenance synchronous preparation", () => {
it.each(["completion", "failure"] as const)(
it.each(["start", "completion", "failure"] as const)(
"retains maintenance ownership until task %s settles",
async (terminal) => {
vi.useFakeTimers();
@ -263,7 +264,12 @@ describe("deferred maintenance synchronous preparation", () => {
const terminalEntered = createDeferred();
const terminalRelease = createDeferred();
let entered = false;
const transition = terminal === "completion" ? mocks.complete : mocks.fail;
const transition =
terminal === "start"
? mocks.start
: terminal === "completion"
? mocks.complete
: mocks.fail;
transition.mockImplementationOnce(async () => {
entered = true;
terminalEntered.resolve();
@ -278,6 +284,9 @@ describe("deferred maintenance synchronous preparation", () => {
f.workRelease.resolve();
await Promise.race([terminalEntered.promise, f.disposeEntered.promise]);
expect(entered).toBe(true);
if (terminal === "start") {
expect(f.maintain).not.toHaveBeenCalled();
}
const taskId = [...f.rows.keys()][0]!;
expect(isContextEngineMaintenanceTaskOwnerActive(taskId)).toBe(true);
expect(f.dispose).not.toHaveBeenCalled();

View file

@ -38,7 +38,6 @@ import {
firstMaintainParams,
flushAsyncWork,
requireRecord,
waitForAssertion,
} from "./context-engine-maintenance.fixtures.test-support.js";
import { resolveSessionLane } from "./lanes.js";
@ -464,22 +463,22 @@ describe("runContextEngineMaintenance", () => {
});
expect(result).toBeUndefined();
await waitForAssertion(() => expect(maintain).toHaveBeenCalledTimes(1));
await waitForAssertion(() =>
expect(rewriteTranscriptEntriesInSessionManagerMock).toHaveBeenCalledWith({
sessionManager: openedSessionManager,
replacements: [
{
entryId: "entry-1",
message: castAgentMessage({
role: "assistant",
content: [{ type: "text", text: "done" }],
timestamp: 2,
}),
},
],
}),
);
await backgroundEngine.started;
expect(maintain).toHaveBeenCalledTimes(1);
await waitForDeferredTurnMaintenanceForSession(sessionKey);
expect(rewriteTranscriptEntriesInSessionManagerMock).toHaveBeenCalledWith({
sessionManager: openedSessionManager,
replacements: [
{
entryId: "entry-1",
message: castAgentMessage({
role: "assistant",
content: [{ type: "text", text: "done" }],
timestamp: 2,
}),
},
],
});
const queuedTasks = listTasksForOwnerKey(sessionKey).filter(
(task) => task.taskKind === TURN_MAINTENANCE_TASK_KIND,
@ -564,7 +563,8 @@ describe("runContextEngineMaintenance", () => {
sessionFile: "/tmp/session-2.jsonl",
reason: "turn",
});
await waitForAssertion(() => expect(maintain).toHaveBeenCalledTimes(1));
await backgroundEngine.started;
expect(maintain).toHaveBeenCalledTimes(1);
await runContextEngineMaintenance({
contextEngine: backgroundEngine,
sessionId: "session-2",
@ -642,7 +642,8 @@ describe("runContextEngineMaintenance", () => {
},
});
await waitForAssertion(() => expect(maintain).toHaveBeenCalledTimes(1));
await backgroundEngine.started;
expect(maintain).toHaveBeenCalledTimes(1);
await runContextEngineMaintenance({
contextEngine: backgroundEngine,
@ -1132,7 +1133,8 @@ describe("runContextEngineMaintenance", () => {
firstDeferred.push(promise);
},
});
await waitForAssertion(() => expect(maintain).toHaveBeenCalledTimes(1));
await backgroundEngine.started;
expect(maintain).toHaveBeenCalledTimes(1);
const onDeferredMaintenance = vi.fn();
const onDeferredMaintenanceFailure = vi.fn();
@ -1318,9 +1320,8 @@ describe("runContextEngineMaintenance", () => {
events.push("foreground-2-end");
});
await waitForAssertion(() =>
expect(events).toEqual(["foreground-1-start", "maintenance-start"]),
);
await backgroundEngine.started;
expect(events).toEqual(["foreground-1-start", "maintenance-start"]);
expect(maintain).toHaveBeenCalledTimes(1);
await waitForDeferredTurnMaintenanceForSession(sessionKey);
expect(
@ -1333,17 +1334,14 @@ describe("runContextEngineMaintenance", () => {
throw new Error("Expected first foreground release callback to be initialized");
}
releaseFirstForeground();
await waitForAssertion(() =>
expect(events).toEqual([
"foreground-1-start",
"maintenance-start",
"foreground-1-end",
"foreground-2-start",
"foreground-2-end",
]),
);
await Promise.all([firstForeground, secondForeground]);
expect(events).toEqual([
"foreground-1-start",
"maintenance-start",
"foreground-1-end",
"foreground-2-start",
"foreground-2-end",
]);
} finally {
vi.useRealTimers();
}
@ -1409,7 +1407,8 @@ describe("runContextEngineMaintenance", () => {
reason: "turn",
});
await waitForAssertion(() => expect(events).toContain("maintenance-start"));
await backgroundEngine.started;
expect(events).toContain("maintenance-start");
const foregroundTurn = enqueueCommandInLane(sessionLane, async () => {
events.push("foreground-before-read-checkpoint");
@ -1467,7 +1466,8 @@ describe("runContextEngineMaintenance", () => {
sessionFile: "/tmp/session-fast.jsonl",
reason: "turn",
});
await waitForAssertion(() => expect(maintain).toHaveBeenCalledTimes(1));
await backgroundEngine.started;
expect(maintain).toHaveBeenCalledTimes(1);
expect(sendMessageMock).not.toHaveBeenCalled();
expect(peekSystemEvents(sessionKey)).toStrictEqual([]);
@ -1582,26 +1582,25 @@ describe("runContextEngineMaintenance", () => {
resetSystemEventsForTest();
const sessionKey = "agent:main:session-fail";
const backgroundEngine = createBackgroundMaintenanceEngine(
vi.fn(async (rawParams?: unknown) => {
const signal = (rawParams as { abortSignal?: AbortSignal } | undefined)?.abortSignal;
if (!signal) {
throw new Error("expected deferred maintenance abort signal");
const maintain = vi.fn(async (rawParams?: unknown) => {
const signal = (rawParams as { abortSignal?: AbortSignal } | undefined)?.abortSignal;
if (!signal) {
throw new Error("expected deferred maintenance abort signal");
}
await new Promise<void>((_resolve, reject) => {
const onAbort = () => {
signal.removeEventListener("abort", onAbort);
reject(new Error("maintenance cleanup failed"));
};
signal.addEventListener("abort", onAbort, { once: true });
maintenanceStarted.resolve();
if (signal.aborted) {
onAbort();
}
await new Promise<void>((_resolve, reject) => {
const onAbort = () => {
signal.removeEventListener("abort", onAbort);
reject(new Error("maintenance cleanup failed"));
};
signal.addEventListener("abort", onAbort, { once: true });
maintenanceStarted.resolve();
if (signal.aborted) {
onAbort();
}
});
return { changed: false, bytesFreed: 0, rewrittenEntries: 0 };
}),
);
});
return { changed: false, bytesFreed: 0, rewrittenEntries: 0 };
});
const backgroundEngine = createBackgroundMaintenanceEngine(maintain);
await runContextEngineMaintenance({
contextEngine: backgroundEngine,
@ -1617,7 +1616,7 @@ describe("runContextEngineMaintenance", () => {
maintenanceStarted.promise,
expectDefined(deferredMaintenance, "deferred maintenance completion"),
]);
expect(backgroundEngine["maintain"]).toHaveBeenCalledTimes(1);
expect(maintain).toHaveBeenCalledTimes(1);
process.emit("SIGTERM", "SIGTERM");
await expectDefined(deferredMaintenance, "deferred maintenance completion");
await deliveries.settle();

View file

@ -31,11 +31,9 @@ import {
import {
completeTaskRunByRunIdAsync,
failTaskRunByRunIdAsync,
startTaskRunByRunIdAsync,
} from "../../tasks/detached-task-runtime.async.js";
import {
recordTaskRunProgressByRunId,
startTaskRunByRunId,
} from "../../tasks/detached-task-runtime.js";
import { recordTaskRunProgressByRunId } from "../../tasks/detached-task-runtime.js";
import {
cancelTaskByIdForOwner,
findTaskByRunIdForOwner,
@ -320,13 +318,17 @@ async function runDeferredTurnMaintenanceWorker(
try {
const runningAt = Date.now();
startTaskRunByRunId({
...taskRun,
startedAt: runningAt,
lastEventAt: runningAt,
progressSummary: "Running deferred maintenance.",
eventSummary: "Starting deferred maintenance.",
});
// Admit running state through the same retained workers that will settle completion.
await startTaskRunByRunIdAsync(
{
...taskRun,
startedAt: runningAt,
lastEventAt: runningAt,
progressSummary: "Running deferred maintenance.",
eventSummary: "Starting deferred maintenance.",
},
params.assertActive,
);
const longRunningTimer = setTimeout(() => {
try {
makeTaskVisible("state_changes");
@ -405,10 +407,7 @@ async function runDeferredTurnMaintenanceWorker(
function scheduleDeferredTurnMaintenance(
params: DeferredTurnMaintenanceScheduleParams,
): Promise<void> | undefined {
const sessionKey = normalizeOptionalString(params.sessionKey);
if (!sessionKey) {
return undefined;
}
const { sessionKey } = params;
if (isGatewayDraining()) {
params.onScheduleFailure?.(new GatewayDrainingError());
return undefined;

View file

@ -12,7 +12,6 @@ import {
import { observeMainThreadSql } from "../../test-utils/main-thread-sql-spies.js";
import { lockState, unlockWorktree } from "./git-lock.js";
import * as registryRead from "./registry-read.js";
import * as registry from "./registry.js";
import {
admitWorktreeRunLeaseRow,
getRegistryWorktree,
@ -163,7 +162,7 @@ describe("worktree run lease", () => {
it("acquires a Git guard after a previous acquisition failed to read the registry", async () => {
const created = await createSessionWorktree();
const record = getRegistryWorktree(env, created.id)!;
vi.spyOn(registry, "getRegistryWorktree").mockImplementationOnce(() => {
vi.spyOn(registryRead, "readRegistryWorktree").mockImplementationOnce(async () => {
throw new Error("simulated registry read failure");
});

View file

@ -77,7 +77,7 @@ async function withGitLockTransition<T>(id: string, operation: () => Promise<T>)
}
}
async function retainGitLock(env: NodeJS.ProcessEnv, id: string): Promise<void> {
async function retainGitLock(context: OpenClawStateWorkerContext, id: string): Promise<void> {
await withGitLockTransition(id, async () => {
const held = heldGitLocks.get(id) ?? { refcount: 0, gitLocked: false };
const needsLock = held.refcount === 0 && !held.gitLocked;
@ -88,7 +88,7 @@ async function retainGitLock(env: NodeJS.ProcessEnv, id: string): Promise<void>
}
let record: ManagedWorktreeRecord | undefined;
try {
record = getRegistryWorktree(env, id);
record = await readRegistryWorktree(context, id);
if (!record) {
return;
}
@ -281,7 +281,7 @@ export async function acquireWorktreeRunLease(
// Serialize refcount and Git transitions so a cleanup retry cannot unlock a
// newer same-process holder after a prior generation's unlock failed.
try {
await retainGitLock(env, id);
await retainGitLock(context, id);
} catch (error) {
// The failed retain already discarded its in-memory holder; cleanup owns only
// the durable row and keeps it fenced if deletion cannot complete yet.

View file

@ -1,3 +1,4 @@
import { AsyncLocalStorage } from "node:async_hooks";
import type { SqliteWorkerAdmissionFactory } from "../infra/sqlite-worker-operation-admission.js";
import {
runSqliteWorkerStoreOperation,
@ -43,3 +44,34 @@ export function runWithOpenClawStateWorkerStore<T>(
requireStateLifecycle,
);
}
/** Only native opening owns the caller scope; a cached actor must not retain it. */
export function captureOpenClawStateWorkerOpeningGuard(
context: OpenClawStateWorkerContext,
assertCurrent?: () => void,
) {
const admission: { assertCurrent?: () => void; refusal?: { error: unknown } } = {
assertCurrent,
};
let captured: (() => void) | undefined = AsyncLocalStorage.bind(() => {
context.admission.assertCurrent();
try {
assertCurrent?.();
} catch (error) {
admission.refusal = { error };
throw error;
}
});
return {
admission,
assertCurrent: () => {
if (!captured) {
throw new Error("Shared-state worker opening admission is closed");
}
captured();
},
releaseContext() {
captured = undefined;
},
};
}

View file

@ -41,6 +41,7 @@ import type {
} from "./openclaw-state-worker-contract.js";
import { hydrateOpenClawStateWorkerError } from "./openclaw-state-worker-error.js";
import {
captureOpenClawStateWorkerOpeningGuard,
runWithCapturedWorkerContext,
runWithOpenClawStateWorkerStore,
} from "./openclaw-state-worker-operation.js";
@ -59,7 +60,7 @@ function createSharedStateWorkerOwner() {
source: ReturnType<typeof captureRuntimeWorkerSource>;
context: OpenClawStateWorkerContext;
opening: Promise<Store | undefined>;
openingAdmission: { assertCurrent?: () => void; refusal?: { error: unknown } };
openingAdmission: ReturnType<typeof captureOpenClawStateWorkerOpeningGuard>["admission"];
existingOnly: boolean;
store?: Store;
actor?: object;
@ -457,41 +458,36 @@ function createSharedStateWorkerOwner() {
}
}
if (!entry) {
const openingAdmission: Entry["openingAdmission"] = { assertCurrent };
const assertOpeningAdmission = () => {
admission.assertCurrent();
try {
assertCurrent?.();
} catch (error) {
openingAdmission.refusal = { error };
throw error;
}
};
const openingGuard = captureOpenClawStateWorkerOpeningGuard(context, assertCurrent);
const admitted: Entry = {
source,
context,
openingAdmission,
openingAdmission: openingGuard.admission,
existingOnly,
activeOperations: 0,
operationGeneration: 0,
opening: runInDetachedAsyncContext(() =>
openSharedStateSqliteWorkerStore<StoreOperations>(
{
...source,
databasePath: admission.databasePath,
existingOnly,
},
context,
assertOpeningAdmission,
{
maintenanceScope: context.maintenanceScope,
preparation,
retainCleanup: (cleanup) => {
admitted.cleanup = cleanup;
opening: runInDetachedAsyncContext(async () => {
try {
return await openSharedStateSqliteWorkerStore<StoreOperations>(
{
...source,
databasePath: admission.databasePath,
existingOnly,
},
},
),
),
context,
openingGuard.assertCurrent,
{
maintenanceScope: context.maintenanceScope,
preparation,
retainCleanup: (cleanup) => {
admitted.cleanup = cleanup;
},
},
);
} finally {
openingGuard.releaseContext();
}
}),
};
entry = admitted;
admitted.opening = admitted.opening.then((store) => {

View file

@ -36,6 +36,17 @@ async function mutateDetachedTask(
return result;
}
export function startTaskRunByRunIdAsync(
params: Parameters<DetachedTaskLifecycleRuntime["startTaskRunByRunId"]>[0],
assertCurrent?: () => void,
) {
return mutateDetachedTask(
{ kind: "state", params: { ...params, status: "running" } },
(runtime) => runtime.startTaskRunByRunId(params),
assertCurrent,
);
}
export function finalizeTaskRunByRunIdAsync(
params: DetachedTaskFinalizeParams,
assertCurrent?: () => void,

View file

@ -68,6 +68,7 @@ export const databaseWorkerCoreTestFiles = [
"src/agents/tools/gateway.hosted-routing.test.ts",
"src/agents/tools/gateway.runtime-identity.test.ts",
"src/agents/cli-runner.prepare-fixture.test.ts",
"src/agents/cli-runner.context-engine.test.ts",
"src/agents/harness/selection.test.ts",
"src/agents/subagents/announce/subagent-announce.requester-cron-authority.test.ts",
"src/agents/subagents/announce/subagent-announce.requester-settle-results.test.ts",
@ -231,6 +232,7 @@ export const databaseWorkerCoreTestFiles = [
"src/agents/embedded-agent-runner/compact.foreground-resources.test.ts",
"src/agents/embedded-agent-runner/compact.hooks.test.ts",
"src/agents/embedded-agent-runner/compact.queued-successor.test.ts",
"src/agents/embedded-agent-runner/compact.queued-resources.test.ts",
"src/agents/embedded-agent-runner/compact.terminal-metadata.test.ts",
"src/agents/embedded-agent-runner/transcript-rewrite-notifications.test.ts",
"src/agents/embedded-agent-runner/run/attempt-prompt-submit.test.ts",