mirror of
https://github.com/openclaw/openclaw.git
synced 2026-11-22 03:40:05 +00:00
fix(gateway): preserve document context in active-run steering (#138275)
Fixes #138225. Documents attached while an agent run is active lose their extracted contents on the steering path. This repair sends the normal attachment note, safely wrapped document text, and rendered PDF pages into the active run through the shared media owner. Rendering stays read-only so rejected steering can use normal dispatch. Rendering failures preserve the original caption and attachment facts. Existing limits, skipped-file budgets, media ordering, authorization, cancellation, and transcript ownership remain intact. Built on @ruel225's contribution, with the original contributor history retained. Validation: - Real Control UI/Gateway baseline: the active model received the caption but no document contents. - Corrected browser and model-request proof: captioned and captionless documents, scanned PDFs, mixed media, limits, marker budgets, renderer failure/recovery, rejected admission, allowed roots, separate caller authority, slash commands, and cancellation. - Native text-only control: compatible text stayed in the same turn; scanned page images were rejected with `image_input_unsupported` and delivered through one normal follow-up turn. - Independent acceptance passed all 12 original clauses, including verified proof-runtime cleanup. Earlier failed judgments and their corrective evidence remain preserved. - Focused Gateway/media suites, hosted production/test type checks, and fresh source review pass. Required CI is checked against the published head. Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
parent
7f612c6f5d
commit
cb4c06ee78
7 changed files with 644 additions and 20 deletions
|
|
@ -10,15 +10,18 @@ import {
|
|||
type SessionGoalOperationResult,
|
||||
} from "../../config/sessions/goals-operations.js";
|
||||
import type { PrepareAssistantTranscriptMessage } from "../../config/sessions/transcript-assistant-delivery.js";
|
||||
import { logVerbose } from "../../globals.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
import { clearAgentRunContext } from "../../infra/agent-run-registry.js";
|
||||
import { emitDiagnosticsTimelineEvent } from "../../infra/diagnostics-timeline.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import {
|
||||
recordSessionCreated,
|
||||
recordSessionGoalChanged,
|
||||
} from "../../sessions/session-state-events.js";
|
||||
import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
|
||||
import { extractTextFromChatContent } from "../../shared/chat-content.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import type { SkillWorkshopProposalRevisionConstraint } from "../../skills/workshop/types.js";
|
||||
import { isOperatorUiClient } from "../../utils/message-channel.js";
|
||||
import { discardPreparedInboundMedia } from "../chat-attachments.js";
|
||||
|
|
@ -65,6 +68,10 @@ type ChatSendInternalOptions = {
|
|||
skillWorkshopProposalRevision?: SkillWorkshopProposalRevisionConstraint;
|
||||
};
|
||||
|
||||
const mediaDocumentContextLoader = createLazyImportLoader(
|
||||
() => import("../../media-understanding/file-context.js"),
|
||||
);
|
||||
|
||||
async function handleChatSendWithOptions(
|
||||
{
|
||||
params,
|
||||
|
|
@ -390,6 +397,33 @@ async function handleChatSendWithOptions(
|
|||
client?.internal?.syntheticClient ? undefined : client?.authenticatedUserProfile?.profileId,
|
||||
);
|
||||
}
|
||||
// Rendering can fail independently of admission; preserve the raw steer on failure.
|
||||
const steerDocumentContext =
|
||||
messageInjectionTarget && !isInternalTextSlashCommandTurn && ctx.media?.length
|
||||
? await mediaDocumentContextLoader
|
||||
.load()
|
||||
.then(async (runtime) => ({
|
||||
status: "rendered" as const,
|
||||
...(await runtime.renderInboundDocumentContext({
|
||||
ctx,
|
||||
cfg: preparedSession.value.cfg,
|
||||
})),
|
||||
}))
|
||||
.catch((err: unknown) => {
|
||||
// A poisoned lazy import must not be served to later steers.
|
||||
mediaDocumentContextLoader.clear();
|
||||
logVerbose(
|
||||
`steer document render failed, injecting raw content: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
return { status: "failed" as const };
|
||||
})
|
||||
: undefined;
|
||||
if (activeRunAbort.controller.signal.aborted) {
|
||||
return finishAbortedChatSend();
|
||||
}
|
||||
if (sessionRoutingChanged(context.getRuntimeConfig())) {
|
||||
return admitted.value.rejectSessionRoutingChanged();
|
||||
}
|
||||
const beginCapturedMessageInjection = createChatSendMessageInjectionStarter({
|
||||
target: messageInjectionTarget,
|
||||
request: normalizedRequest.value,
|
||||
|
|
@ -397,6 +431,7 @@ async function handleChatSendWithOptions(
|
|||
admittedSessionSettings: admitted.value.admittedSessionSettings,
|
||||
turn: preparedUserTurn,
|
||||
imageOrder,
|
||||
documentContext: steerDocumentContext,
|
||||
userTurnTranscriptRecorder: userTurnRecorder,
|
||||
});
|
||||
const preAckReplyContextPromise =
|
||||
|
|
|
|||
|
|
@ -1,27 +1,43 @@
|
|||
/** Covers steer finalize audit honesty: aborted unconfirmed commits must not audit as completed. */
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { beforeEach, describe, expect, it, onTestFinished, vi } from "vitest";
|
||||
import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js";
|
||||
import {
|
||||
beginReplyMessageInjectionTarget,
|
||||
createReplyOperation,
|
||||
finalizeReplyMessageInjectionAttempt,
|
||||
replyRunRegistry,
|
||||
type ReplyMessageInjectionTarget,
|
||||
} from "../../auto-reply/reply/reply-run-registry.js";
|
||||
import type { RuntimeMsgContext } from "../../auto-reply/templating.js";
|
||||
import {
|
||||
recordSessionParticipant,
|
||||
updateSessionEntry,
|
||||
} from "../../config/sessions/session-accessor.js";
|
||||
import { logMessageProcessed } from "../../logging/diagnostic.js";
|
||||
import { prepareSessionParticipantInput } from "../../sessions/session-participant-input.js";
|
||||
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
|
||||
import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js";
|
||||
import type { ChatImageContent } from "../chat-attachments.js";
|
||||
import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js";
|
||||
import { finalizeAcceptedChatSendMessageInjection } from "./chat-send-message-injection.js";
|
||||
import {
|
||||
createChatSendMessageInjectionStarter,
|
||||
finalizeAcceptedChatSendMessageInjection,
|
||||
} from "./chat-send-message-injection.js";
|
||||
import type { GatewayRequestContext } from "./types.js";
|
||||
|
||||
vi.mock("../../auto-reply/reply/dispatch-from-config.audit.js", () => ({
|
||||
emitInboundMessageAuditTerminal: vi.fn(),
|
||||
}));
|
||||
vi.mock("../../auto-reply/reply/reply-run-registry.js", () => ({
|
||||
beginReplyMessageInjectionTarget: vi.fn(),
|
||||
finalizeReplyMessageInjectionAttempt: vi.fn(),
|
||||
}));
|
||||
vi.mock(import("../../auto-reply/reply/reply-run-registry.js"), async (importOriginal) => {
|
||||
const actual = await importOriginal();
|
||||
return {
|
||||
...actual,
|
||||
beginReplyMessageInjectionTarget: vi.fn<typeof actual.beginReplyMessageInjectionTarget>(),
|
||||
finalizeReplyMessageInjectionAttempt:
|
||||
vi.fn<typeof actual.finalizeReplyMessageInjectionAttempt>(),
|
||||
};
|
||||
});
|
||||
vi.mock("../../auto-reply/reply/message-received-hooks.js", () => ({
|
||||
emitMessageReceivedHooks: vi.fn(),
|
||||
}));
|
||||
|
|
@ -148,3 +164,214 @@ describe("finalizeAcceptedChatSendMessageInjection", () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createChatSendMessageInjectionStarter", () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(beginReplyMessageInjectionTarget).mockImplementation((target) => ({
|
||||
targetRunId: target.runId,
|
||||
acceptance: Promise.resolve(true),
|
||||
outcome: Promise.resolve({ status: "accepted" }),
|
||||
}));
|
||||
});
|
||||
|
||||
function makeStarterParams(params?: {
|
||||
body?: string;
|
||||
rawMessage?: string;
|
||||
media?: RuntimeMsgContext["media"];
|
||||
documentContext?: Parameters<
|
||||
typeof createChatSendMessageInjectionStarter
|
||||
>[0]["documentContext"];
|
||||
replyOptionImages?: ChatImageContent[];
|
||||
isInternalTextSlashCommandTurn?: boolean;
|
||||
}): Parameters<typeof createChatSendMessageInjectionStarter>[0] {
|
||||
const sessionKey = "agent:main:steer-test";
|
||||
const sessionId = "steer-test-session";
|
||||
const rawMessage = params?.rawMessage ?? params?.body ?? "raw steer";
|
||||
const operation = createReplyOperation({ sessionKey, sessionId, resetTriggered: false });
|
||||
onTestFinished(() => operation.complete());
|
||||
operation.setPhase("running");
|
||||
operation.attachBackend({
|
||||
kind: "embedded",
|
||||
runId: "active-run",
|
||||
supportsQueueMessageImages: true,
|
||||
cancel: vi.fn(),
|
||||
messageInjection: {
|
||||
isAvailable: () => true,
|
||||
queueMessage: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const target = expectDefined(
|
||||
replyRunRegistry.resolveCurrentMessageInjectionTarget(sessionKey),
|
||||
"Expected the running test operation to accept steering",
|
||||
);
|
||||
|
||||
return {
|
||||
target,
|
||||
request: {
|
||||
p: { sessionKey, message: rawMessage, idempotencyKey: "steer-input" },
|
||||
rawMessage,
|
||||
supportsTaskSuggestions: false,
|
||||
},
|
||||
session: { cfg: {}, entry: undefined },
|
||||
turn: {
|
||||
discardUnreferencedMedia: async () => {},
|
||||
accountId: undefined,
|
||||
ctx: { Provider: "dashboard", Body: params?.body, media: params?.media },
|
||||
isInternalTextSlashCommandTurn: params?.isInternalTextSlashCommandTurn ?? false,
|
||||
queuedFollowupOwnerKey: undefined,
|
||||
pluginBoundMediaPromise: Promise.resolve([]),
|
||||
replyOptionImages: params?.replyOptionImages ?? [],
|
||||
replyOptionMedia: [],
|
||||
},
|
||||
imageOrder: [],
|
||||
documentContext: params?.documentContext,
|
||||
userTurnTranscriptRecorder: createUserTurnTranscriptRecorder({
|
||||
input: { text: params?.body ?? rawMessage, media: params?.media },
|
||||
target: createTestUserTurnTranscriptTarget({ sessionKey, sessionId }),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
it.each([
|
||||
{
|
||||
caption: "see attached",
|
||||
label: "captioned",
|
||||
expectedText:
|
||||
'[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\nsee attached\n\n<file name="note.txt" mime="text/plain">doc body</file>',
|
||||
},
|
||||
{
|
||||
caption: "",
|
||||
label: "blank-caption",
|
||||
expectedText:
|
||||
'[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\n<file name="note.txt" mime="text/plain">doc body</file>',
|
||||
},
|
||||
])("retains marker and document text for a $label steer", ({ caption, expectedText }) => {
|
||||
const documentText = '<file name="note.txt" mime="text/plain">doc body</file>';
|
||||
const params = makeStarterParams({
|
||||
body: caption,
|
||||
rawMessage: caption,
|
||||
media: [
|
||||
{
|
||||
path: "media://inbound/note.txt",
|
||||
contentType: "text/plain",
|
||||
kind: "document",
|
||||
fileName: "note.txt",
|
||||
},
|
||||
],
|
||||
documentContext: { status: "rendered", text: documentText, images: [] },
|
||||
});
|
||||
const durableContext = structuredClone(params.turn.ctx);
|
||||
|
||||
createChatSendMessageInjectionStarter(params)();
|
||||
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledOnce();
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith(
|
||||
params.target,
|
||||
expectedText,
|
||||
expect.objectContaining({ isInboundUserMessage: true }),
|
||||
);
|
||||
expect(params.turn.ctx).toEqual(durableContext);
|
||||
});
|
||||
|
||||
it("keeps the attachment note when document rendering fails", () => {
|
||||
const params = makeStarterParams({
|
||||
body: "read the attachment",
|
||||
media: [{ path: "media://inbound/note.txt", contentType: "text/plain" }],
|
||||
documentContext: { status: "failed" },
|
||||
});
|
||||
const originalContext = structuredClone(params.turn.ctx);
|
||||
|
||||
createChatSendMessageInjectionStarter(params)();
|
||||
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith(
|
||||
params.target,
|
||||
"[media attached: media://inbound/note.txt (text/plain)]\n\nread the attachment",
|
||||
expect.objectContaining({ isInboundUserMessage: true }),
|
||||
);
|
||||
expect(params.turn.ctx).toEqual(originalContext);
|
||||
});
|
||||
|
||||
it("keeps the base text untouched when no document context was rendered", () => {
|
||||
createChatSendMessageInjectionStarter(makeStarterParams({ body: "plain steer" }))();
|
||||
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"plain steer",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("merges extracted page images after the prepared inbound images", () => {
|
||||
const params = makeStarterParams({
|
||||
body: "see attached",
|
||||
media: [
|
||||
{ path: "media://inbound/photo.png", contentType: "image/png" },
|
||||
{
|
||||
path: "media://inbound/scan.pdf",
|
||||
contentType: "application/pdf",
|
||||
kind: "document",
|
||||
},
|
||||
],
|
||||
replyOptionImages: [
|
||||
{ type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 },
|
||||
],
|
||||
documentContext: {
|
||||
status: "rendered",
|
||||
text: "[PDF content rendered to images]",
|
||||
images: [
|
||||
{ type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 1 },
|
||||
{ type: "image", data: "page-2", mimeType: "image/png", attachmentIndex: 1 },
|
||||
],
|
||||
},
|
||||
});
|
||||
const durableContext = structuredClone(params.turn.ctx);
|
||||
|
||||
createChatSendMessageInjectionStarter(params)();
|
||||
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"[media attached: 2 files]\n" +
|
||||
"[media attached 1/2: media://inbound/photo.png (image/png)]\n" +
|
||||
"[media attached 2/2: media://inbound/scan.pdf (application/pdf)]\n\n" +
|
||||
"see attached\n\n[PDF content rendered to images]",
|
||||
expect.objectContaining({
|
||||
images: [
|
||||
{ type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 },
|
||||
{ type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 1 },
|
||||
{ type: "image", data: "page-2", mimeType: "image/png", sourceIndex: 1 },
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(params.turn.ctx).toEqual(durableContext);
|
||||
});
|
||||
|
||||
it("injects extracted page images when the steer carries no inbound images", () => {
|
||||
createChatSendMessageInjectionStarter(
|
||||
makeStarterParams({
|
||||
body: "scan attached",
|
||||
documentContext: {
|
||||
status: "rendered",
|
||||
text: "[PDF content rendered to images]",
|
||||
images: [{ type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 0 }],
|
||||
},
|
||||
}),
|
||||
)();
|
||||
|
||||
expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"scan attached\n\n[PDF content rendered to images]",
|
||||
expect.objectContaining({
|
||||
images: [{ type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 0 }],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns undefined for internal slash-command turns even with a target", () => {
|
||||
expect(
|
||||
createChatSendMessageInjectionStarter(
|
||||
makeStarterParams({ isInternalTextSlashCommandTurn: true }),
|
||||
)(),
|
||||
).toBeUndefined();
|
||||
expect(beginReplyMessageInjectionTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import { resolveCommandAuthorization } from "../../auto-reply/command-auth.js";
|
||||
import { buildInboundMediaNoteProjection } from "../../auto-reply/media-note.js";
|
||||
import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js";
|
||||
import { finalizeInboundContext } from "../../auto-reply/reply/inbound-context.js";
|
||||
import { hasInboundAudio } from "../../auto-reply/reply/inbound-media.js";
|
||||
|
|
@ -19,9 +20,11 @@ import type { SessionEntry } from "../../config/sessions.js";
|
|||
import { updateSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js";
|
||||
import { logMessageProcessed, logMessageReceived } from "../../logging/diagnostic.js";
|
||||
import type { InboundDocumentContext } from "../../media-understanding/file-context.js";
|
||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||
import { recordAcceptedSessionParticipantInput } from "../../sessions/session-participant-input-recording.js";
|
||||
import { setGatewayDedupeEntry } from "../agent-turn/agent-job.js";
|
||||
import type { ChatImageContent } from "../chat-attachments.js";
|
||||
import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js";
|
||||
import { buildChatSendReplyInjectionText } from "./chat-send-reply-context.js";
|
||||
import type { NormalizedChatSendRequest } from "./chat-send-request.js";
|
||||
|
|
@ -37,6 +40,7 @@ export function createChatSendMessageInjectionStarter(params: {
|
|||
admittedSessionSettings?: Readonly<Pick<SessionEntry, "permissionMode" | "toolOverrides">>;
|
||||
turn: ReturnType<typeof prepareChatSendUserTurn>;
|
||||
imageOrder: ReplyBackendQueueMessageOptions["imageOrder"];
|
||||
documentContext?: ({ status: "rendered" } & InboundDocumentContext) | { status: "failed" };
|
||||
userTurnTranscriptRecorder: NonNullable<
|
||||
ReplyBackendQueueMessageOptions["userTurnTranscriptRecorder"]
|
||||
>;
|
||||
|
|
@ -54,7 +58,32 @@ export function createChatSendMessageInjectionStarter(params: {
|
|||
sessionEntry: entry,
|
||||
inlineMode: p.queueMode,
|
||||
});
|
||||
const text = ctx.BodyForAgent ?? ctx.Body ?? rawMessage;
|
||||
const baseText = ctx.BodyForAgent ?? ctx.Body ?? rawMessage;
|
||||
const rendered =
|
||||
params.documentContext?.status === "rendered" ? params.documentContext : undefined;
|
||||
const documentContext = rendered?.text.trim();
|
||||
let text = baseText;
|
||||
if (documentContext || params.documentContext?.status === "failed") {
|
||||
text = [buildInboundMediaNoteProjection(ctx).text, baseText.trim(), documentContext]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
}
|
||||
const documentImages = rendered?.images ?? [];
|
||||
const injectionImages: ChatImageContent[] | undefined =
|
||||
documentImages.length > 0
|
||||
? [
|
||||
...(replyOptionImages ?? []),
|
||||
// Extracted page images follow the prepared inbound images, the
|
||||
// same inline-then-extracted ordering reply dispatch produces; each
|
||||
// keeps its attachment index as ordering provenance.
|
||||
...documentImages.map((image): ChatImageContent => ({
|
||||
type: "image",
|
||||
data: image.data,
|
||||
mimeType: image.mimeType,
|
||||
sourceIndex: image.attachmentIndex,
|
||||
})),
|
||||
]
|
||||
: replyOptionImages;
|
||||
const authorization = resolveCommandAuthorization({
|
||||
ctx,
|
||||
cfg,
|
||||
|
|
@ -78,7 +107,7 @@ export function createChatSendMessageInjectionStarter(params: {
|
|||
senderIsOwner: authorization.senderIsOwner,
|
||||
disableTools: false,
|
||||
}),
|
||||
...(replyOptionImages?.length ? { images: replyOptionImages } : {}),
|
||||
...(injectionImages?.length ? { images: injectionImages } : {}),
|
||||
...(params.imageOrder?.length ? { imageOrder: params.imageOrder } : {}),
|
||||
...(replyOptionMedia?.length ? { media: replyOptionMedia } : {}),
|
||||
waitForTranscriptCommit: true,
|
||||
|
|
|
|||
|
|
@ -129,6 +129,7 @@ const mockState = vi.hoisted(() => {
|
|||
emittedTranscriptUpdates: [] as TranscriptUpdate[],
|
||||
savedMediaResults: [] as Array<{ id?: string; path: string; contentType?: string }>,
|
||||
saveMediaError: null as Error | null,
|
||||
steerDocumentRenderError: null as Error | null,
|
||||
savedMediaCalls: [] as Array<{ contentType?: string; subdir?: string; size: number }>,
|
||||
saveMediaWait: null as Promise<void> | null,
|
||||
activeSaveMediaCalls: 0,
|
||||
|
|
@ -234,6 +235,23 @@ const OFFLOAD_PNG_BASE64 =
|
|||
const TINY_JPEG_BASE64 =
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEBUQEBAVFRUVFRUVFRUVFRUVFRUVFRUXFhUVFRUYHSggGBolHRUVITEhJSkrLi4uFx8zODMsNygtLisBCgoKDg0OGhAQGi0fICUtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAAEAAQMBEQACEQEDEQH/xAAXAAADAQAAAAAAAAAAAAAAAAAAAQMC/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEAMQAAAB6AAAAP/EABQQAQAAAAAAAAAAAAAAAAAAACD/2gAIAQEAAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQIBAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQMBAT8Af//Z";
|
||||
|
||||
vi.mock("../../media-understanding/file-context.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../media-understanding/file-context.js")>(
|
||||
"../../media-understanding/file-context.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
renderInboundDocumentContext: (
|
||||
params: Parameters<typeof actual.renderInboundDocumentContext>[0],
|
||||
) => {
|
||||
if (mockState.steerDocumentRenderError) {
|
||||
return Promise.reject(mockState.steerDocumentRenderError);
|
||||
}
|
||||
return actual.renderInboundDocumentContext(params);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../session-utils.js", async () => {
|
||||
const original =
|
||||
await vi.importActual<typeof import("../session-utils.js")>("../session-utils.js");
|
||||
|
|
@ -2240,6 +2258,133 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
|
|||
});
|
||||
});
|
||||
|
||||
it("hands steered document attachments to the active run as extracted file context", async () => {
|
||||
const { send } = await createSqliteChatRequest("openclaw-chat-send-steer-document-");
|
||||
mockState.hasMessageReceivedHooks = true;
|
||||
// Under the suite fixture root so the suite cleanup removes it; a bare
|
||||
// tmpdir entry would leak on every run.
|
||||
const documentDir = fs.mkdtempSync(path.join(suiteFixtureRoot, "openclaw-steer-doc-"));
|
||||
const documentPath = path.join(documentDir, "notes.txt");
|
||||
fs.writeFileSync(documentPath, "steered document body");
|
||||
setSavedMediaResults([documentPath, "text/plain"]);
|
||||
const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => {
|
||||
await options?.userTurnTranscriptRecorder?.persistApproved();
|
||||
});
|
||||
const operation = beginMessageInjectionOperation({
|
||||
originatingLeafEntryId: null,
|
||||
runId: "active-run",
|
||||
supportsQueueMessageImages: true,
|
||||
taskSuggestionDeliveryMode: "gateway",
|
||||
queueMessage,
|
||||
});
|
||||
|
||||
try {
|
||||
await send({
|
||||
idempotencyKey: "idem-steer-document",
|
||||
requestParams: {
|
||||
queueMode: "steer",
|
||||
attachments: [
|
||||
createFileAttachment(
|
||||
"notes.txt",
|
||||
"text/plain",
|
||||
Buffer.from("steered document body", "utf8").toString("base64"),
|
||||
),
|
||||
],
|
||||
},
|
||||
client: {
|
||||
connect: {
|
||||
client: {
|
||||
id: GATEWAY_CLIENT_NAMES.CONTROL_UI,
|
||||
mode: GATEWAY_CLIENT_MODES.WEBCHAT,
|
||||
version: "dev",
|
||||
platform: "web",
|
||||
},
|
||||
caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS],
|
||||
scopes: ["operator.admin"],
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
operation.complete();
|
||||
}
|
||||
|
||||
expect(queueMessage).toHaveBeenCalledOnce();
|
||||
const queueCall = vi.mocked(queueMessage).mock.calls[0];
|
||||
if (!queueCall) {
|
||||
throw new Error("expected queueMessage to receive the injected text");
|
||||
}
|
||||
const [injectedText] = queueCall;
|
||||
// The active run never reaches reply dispatch, so the injection text must
|
||||
// already carry the extracted document context.
|
||||
expect(injectedText).toContain('<file name="notes.txt" mime="text/plain">');
|
||||
expect(injectedText).toContain("steered document body");
|
||||
});
|
||||
|
||||
it("keeps the raw steer when the lazy document render fails", async () => {
|
||||
const { respond, send } = await createSqliteChatRequest("openclaw-chat-send-steer-doc-fail-");
|
||||
mockState.hasMessageReceivedHooks = true;
|
||||
const documentPath = path.join(suiteFixtureRoot, "openclaw-steer-doc-fail.txt");
|
||||
fs.writeFileSync(documentPath, "steered document body");
|
||||
setSavedMediaResults([documentPath, "text/plain"]);
|
||||
mockState.steerDocumentRenderError = new Error("lazy media runtime unavailable");
|
||||
const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => {
|
||||
await options?.userTurnTranscriptRecorder?.persistApproved();
|
||||
});
|
||||
const operation = beginMessageInjectionOperation({
|
||||
originatingLeafEntryId: null,
|
||||
runId: "active-run",
|
||||
supportsQueueMessageImages: true,
|
||||
taskSuggestionDeliveryMode: "gateway",
|
||||
queueMessage,
|
||||
});
|
||||
|
||||
try {
|
||||
await send({
|
||||
idempotencyKey: "idem-steer-document-render-failure",
|
||||
requestParams: {
|
||||
queueMode: "steer",
|
||||
attachments: [
|
||||
createFileAttachment(
|
||||
"notes.txt",
|
||||
"text/plain",
|
||||
Buffer.from("steered document body", "utf8").toString("base64"),
|
||||
),
|
||||
],
|
||||
},
|
||||
client: {
|
||||
connect: {
|
||||
client: {
|
||||
id: GATEWAY_CLIENT_NAMES.CONTROL_UI,
|
||||
mode: GATEWAY_CLIENT_MODES.WEBCHAT,
|
||||
version: "dev",
|
||||
platform: "web",
|
||||
},
|
||||
caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS],
|
||||
scopes: ["operator.admin"],
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
operation.complete();
|
||||
mockState.steerDocumentRenderError = null;
|
||||
}
|
||||
|
||||
// A failed render must not abort a steerable message: normal reply
|
||||
// dispatch proceeds with raw content on media-understanding failure, so
|
||||
// the steer injection keeps the same contract.
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({ status: "started" }),
|
||||
undefined,
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(queueMessage).toHaveBeenCalledOnce();
|
||||
const [injectedText] = vi.mocked(queueMessage).mock.calls[0] ?? [];
|
||||
expect(injectedText).toBeDefined();
|
||||
expect(injectedText).not.toContain('<file name="notes.txt"');
|
||||
expect(injectedText).toContain("hello");
|
||||
});
|
||||
|
||||
it("hydrates and accepts reply injection before ACK without waiting for delivery", async () => {
|
||||
const { context, respond, send } = await createSqliteChatRequest(
|
||||
"openclaw-chat-send-reply-steer-",
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn());
|
|||
const runFfmpegMock = vi.hoisted(() => vi.fn());
|
||||
const convertHeicToJpegMock = vi.hoisted(() => vi.fn());
|
||||
const runExecMock = vi.hoisted(() => vi.fn());
|
||||
const extractFileContentFromBufferMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding;
|
||||
let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.test-support.js").clearMediaUnderstandingBinaryCacheForTests;
|
||||
|
|
@ -40,6 +41,10 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock;
|
|||
const mockedRunFfmpeg = runFfmpegMock;
|
||||
const mockedConvertHeicToJpeg = convertHeicToJpegMock;
|
||||
const mockedRunExec = runExecMock;
|
||||
const mockedExtractFileContentFromBuffer = extractFileContentFromBufferMock;
|
||||
let actualExtractFileContentFromBuffer:
|
||||
| typeof import("../media/input-files.js").extractFileContentFromBuffer
|
||||
| undefined;
|
||||
|
||||
const TEMP_MEDIA_PREFIX = "openclaw-media-";
|
||||
const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/;
|
||||
|
|
@ -354,6 +359,15 @@ describe("applyMediaUnderstanding", () => {
|
|||
vi.doMock("../process/exec.js", () => ({
|
||||
runExec: runExecMock,
|
||||
}));
|
||||
vi.doMock("../media/input-files.js", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("../media/input-files.js")>("../media/input-files.js");
|
||||
actualExtractFileContentFromBuffer = actual.extractFileContentFromBuffer;
|
||||
return {
|
||||
...actual,
|
||||
extractFileContentFromBuffer: extractFileContentFromBufferMock,
|
||||
};
|
||||
});
|
||||
vi.doMock("./provider-registry.js", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("./provider-registry.js")>("./provider-registry.js");
|
||||
|
|
@ -405,6 +419,12 @@ describe("applyMediaUnderstanding", () => {
|
|||
mockedConvertHeicToJpeg.mockReset();
|
||||
mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized"));
|
||||
mockedRunExec.mockReset();
|
||||
// Extraction stays real unless a case overrides it; the mock exists so
|
||||
// scanned-PDF render outcomes can be produced without the extract plugin.
|
||||
mockedExtractFileContentFromBuffer.mockReset();
|
||||
if (actualExtractFileContentFromBuffer) {
|
||||
mockedExtractFileContentFromBuffer.mockImplementation(actualExtractFileContentFromBuffer);
|
||||
}
|
||||
mockedReadRemoteMediaBuffer.mockResolvedValue({
|
||||
buffer: createSafeAudioFixtureBuffer(2048),
|
||||
contentType: "audio/ogg",
|
||||
|
|
@ -2624,5 +2644,106 @@ describe("applyMediaUnderstanding", () => {
|
|||
expect(ctx.Body).toContain("<file");
|
||||
expect(ctx.Body).toContain("vendor-json");
|
||||
});
|
||||
|
||||
describe("renderInboundDocumentContext", () => {
|
||||
it("renders a document attachment without mutating ctx", async () => {
|
||||
const { renderInboundDocumentContext } = await import("./file-context.js");
|
||||
const mediaPath = await createTempMediaFile({
|
||||
fileName: "steer-note.txt",
|
||||
content: "document body for the steered run",
|
||||
});
|
||||
const ctx: MsgContext = {
|
||||
Body: "see attached",
|
||||
media: [{ path: mediaPath, contentType: "text/plain" }],
|
||||
};
|
||||
|
||||
const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig });
|
||||
|
||||
expect(context?.text).toContain('<file name="steer-note.txt" mime="text/plain">');
|
||||
expect(context?.text).toContain("document body for the steered run");
|
||||
expect(context?.images).toEqual([]);
|
||||
// Read-only on ctx: a rejected steer falls back to reply dispatch, which
|
||||
// must extract exactly once through the full pipeline.
|
||||
expect(ctx.Body).toBe("see attached");
|
||||
expect(ctx.media?.[0]?.path).toBe(mediaPath);
|
||||
});
|
||||
|
||||
it("returns empty for image attachments owned by the injected images channel", async () => {
|
||||
const { renderInboundDocumentContext } = await import("./file-context.js");
|
||||
const mediaPath = await createTempMediaFile({
|
||||
fileName: "steer.png",
|
||||
content: createSafeAudioFixtureBuffer(16),
|
||||
});
|
||||
const ctx: MsgContext = {
|
||||
Body: "see attached",
|
||||
media: [{ path: mediaPath, contentType: "image/png" }],
|
||||
};
|
||||
|
||||
const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig });
|
||||
|
||||
expect(context?.text).toBe("");
|
||||
expect(context?.images).toEqual([]);
|
||||
expect(ctx.Body).toBe("see attached");
|
||||
});
|
||||
|
||||
it("returns empty context when the steer carries no attachments", async () => {
|
||||
const { renderInboundDocumentContext } = await import("./file-context.js");
|
||||
const context = await renderInboundDocumentContext({
|
||||
ctx: { Body: "plain steer" } as MsgContext,
|
||||
cfg: {} as OpenClawConfig,
|
||||
});
|
||||
expect(context).toEqual({ text: "", images: [] });
|
||||
});
|
||||
|
||||
it("returns rendered PDF page images for a scanned document", async () => {
|
||||
const { renderInboundDocumentContext } = await import("./file-context.js");
|
||||
const mediaPath = await createTempMediaFile({
|
||||
fileName: "scan.pdf",
|
||||
content: Buffer.from("%PDF-1.4\n", "utf8"),
|
||||
});
|
||||
mockedExtractFileContentFromBuffer.mockResolvedValueOnce({
|
||||
text: "",
|
||||
images: [
|
||||
{ type: "image", data: "page-1-bytes", mimeType: "image/png" },
|
||||
{ type: "image", data: "page-2-bytes", mimeType: "image/png" },
|
||||
],
|
||||
});
|
||||
const ctx: MsgContext = {
|
||||
Body: "see attached",
|
||||
media: [{ path: mediaPath, contentType: "application/pdf" }],
|
||||
};
|
||||
|
||||
const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() });
|
||||
|
||||
// The marker alone would tell the model the document exists while the
|
||||
// injected images channel carries nothing; the pages must ride along.
|
||||
expect(context?.text).toContain("[PDF content rendered to images]");
|
||||
expect(context?.images).toEqual([
|
||||
{ type: "image", data: "page-1-bytes", mimeType: "image/png", attachmentIndex: 0 },
|
||||
{ type: "image", data: "page-2-bytes", mimeType: "image/png", attachmentIndex: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("applies the skipped-attachment marker budget to steer blocks", async () => {
|
||||
const { renderInboundDocumentContext } = await import("./file-context.js");
|
||||
const olePayload = Buffer.from("Root Entry WordDocument legacy preview", "utf8");
|
||||
const media: { path: string; contentType: string }[] = [];
|
||||
for (let i = 0; i < 7; i += 1) {
|
||||
const filePath = await createTempMediaFile({
|
||||
fileName: `legacy-${i}.doc`,
|
||||
content: olePayload,
|
||||
});
|
||||
media.push({ path: filePath, contentType: "application/msword" });
|
||||
}
|
||||
const ctx: MsgContext = { Body: "see attached", media };
|
||||
|
||||
const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() });
|
||||
|
||||
const markerCount = context?.text.split("[Unsupported document format").length ?? 0;
|
||||
expect(markerCount - 1).toBe(5);
|
||||
expect(context?.text).toContain("[2 more attachments skipped]");
|
||||
expect(context?.images).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
|
|
|||
|
|
@ -300,33 +300,55 @@ export async function prepareFileContextFromMedia(params: {
|
|||
maxChars: number;
|
||||
assertCurrent: () => void;
|
||||
}) {
|
||||
params.assertCurrent();
|
||||
const ctx: MsgContext = {
|
||||
media: [...params.media],
|
||||
Provider: params.channelId,
|
||||
AccountId: params.accountId,
|
||||
};
|
||||
const limits = resolveFileExtractionLimits(params.config);
|
||||
return await renderInboundDocumentContext({
|
||||
ctx: {
|
||||
media: [...params.media],
|
||||
Provider: params.channelId,
|
||||
AccountId: params.accountId,
|
||||
},
|
||||
cfg: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
maxChars: params.maxChars,
|
||||
assertCurrent: params.assertCurrent,
|
||||
});
|
||||
}
|
||||
|
||||
export type InboundDocumentContext = { text: string; images: ExtractedFileImage[] };
|
||||
|
||||
/** Keep prompt expansion separate from inbound state so rejected steers can dispatch normally. */
|
||||
export async function renderInboundDocumentContext(params: {
|
||||
ctx: MsgContext;
|
||||
cfg: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
maxChars?: number;
|
||||
assertCurrent?: () => void;
|
||||
}): Promise<InboundDocumentContext> {
|
||||
params.assertCurrent?.();
|
||||
const { ctx, cfg } = params;
|
||||
const limits = resolveFileExtractionLimits(cfg);
|
||||
const attachments = normalizeMediaAttachments(ctx);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
localPathRoots: resolveMediaAttachmentLocalRoots({
|
||||
cfg: params.config,
|
||||
cfg,
|
||||
ctx,
|
||||
workspaceDir: params.workspaceDir,
|
||||
}),
|
||||
ssrfPolicy: params.config.tools?.web?.fetch?.ssrfPolicy,
|
||||
ssrfPolicy: cfg.tools?.web?.fetch?.ssrfPolicy,
|
||||
workspaceDir: params.workspaceDir,
|
||||
});
|
||||
try {
|
||||
const context = await extractFileContext({
|
||||
attachments,
|
||||
cache,
|
||||
cfg: params.config,
|
||||
limits: { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) },
|
||||
cfg,
|
||||
limits:
|
||||
params.maxChars === undefined
|
||||
? limits
|
||||
: { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) },
|
||||
selfServePathsEnabled: false,
|
||||
assertCurrent: params.assertCurrent,
|
||||
});
|
||||
params.assertCurrent();
|
||||
params.assertCurrent?.();
|
||||
return {
|
||||
text: applyAttachmentMarkerBudget(context.blocks).join("\n\n"),
|
||||
images: context.images,
|
||||
|
|
|
|||
|
|
@ -988,4 +988,49 @@ describe("Gateway steer FIFO", () => {
|
|||
},
|
||||
TEST_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
it(
|
||||
"hands steered document attachments to the active run as extracted file context",
|
||||
async () => {
|
||||
const fixture = await createGatewayFixture("steer-document-context");
|
||||
const first = await sendHeldTurn(fixture);
|
||||
|
||||
// Real gateway protocol send with a file attachment while the initial
|
||||
// run is still live: admission must steer the active run instead of
|
||||
// dispatching a reply, and the extracted document context must reach
|
||||
// the prompt the run actually sends to the model.
|
||||
const steer = await fixture.diagnosticsClient.request<{ runId?: unknown; status?: unknown }>(
|
||||
"chat.send",
|
||||
{
|
||||
sessionKey: fixture.sessionKey,
|
||||
message: "QUEUED_STEER_DOCUMENT",
|
||||
deliver: false,
|
||||
queueMode: "steer",
|
||||
idempotencyKey: "run-steer-document",
|
||||
attachments: [
|
||||
{
|
||||
type: "file",
|
||||
mimeType: "text/plain",
|
||||
fileName: "notes.txt",
|
||||
content: Buffer.from("steered document body", "utf8").toString("base64"),
|
||||
sizeBytes: "steered document body".length,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
expect(steer).toMatchObject({ status: "started" });
|
||||
|
||||
fixture.modelServer.releaseFirst("final");
|
||||
|
||||
await vi.waitFor(() => expect(fixture.modelServer.requests).toHaveLength(2), WAIT_OPTS);
|
||||
const currentSteer = currentUserInput(fixture.modelServer.requests[1]);
|
||||
expect(currentSteer).toContain("QUEUED_STEER_DOCUMENT");
|
||||
// The media store persists uploads as `<original>---<mediaId><ext>`, so
|
||||
// the extracted block carries the stored name, matching reply dispatch.
|
||||
expect(currentSteer).toMatch(/<file name="notes---[0-9a-f-]+\.txt" mime="text\/plain">/);
|
||||
expect(currentSteer).toContain("steered document body");
|
||||
await waitForRunTerminal(fixture, first.runId);
|
||||
},
|
||||
TEST_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue