From cb4c06ee78fb61891b87ff95b288efb24df55dee Mon Sep 17 00:00:00 2001 From: ruel225 Date: Mon, 7 Sep 2026 03:24:04 +0800 Subject: [PATCH] fix(gateway): preserve document context in active-run steering (#138275) Fixes #138225. Documents attached while an agent run is active lose their extracted contents on the steering path. This repair sends the normal attachment note, safely wrapped document text, and rendered PDF pages into the active run through the shared media owner. Rendering stays read-only so rejected steering can use normal dispatch. Rendering failures preserve the original caption and attachment facts. Existing limits, skipped-file budgets, media ordering, authorization, cancellation, and transcript ownership remain intact. Built on @ruel225's contribution, with the original contributor history retained. Validation: - Real Control UI/Gateway baseline: the active model received the caption but no document contents. - Corrected browser and model-request proof: captioned and captionless documents, scanned PDFs, mixed media, limits, marker budgets, renderer failure/recovery, rejected admission, allowed roots, separate caller authority, slash commands, and cancellation. - Native text-only control: compatible text stayed in the same turn; scanned page images were rejected with `image_input_unsupported` and delivered through one normal follow-up turn. - Independent acceptance passed all 12 original clauses, including verified proof-runtime cleanup. Earlier failed judgments and their corrective evidence remain preserved. - Focused Gateway/media suites, hosted production/test type checks, and fresh source review pass. Required CI is checked against the published head. Co-authored-by: Ayaan Zaidi --- .../server-methods/chat-send-handler.ts | 35 +++ .../chat-send-message-injection.test.ts | 239 +++++++++++++++++- .../chat-send-message-injection.ts | 33 ++- .../chat.directive-tags.test.ts | 145 +++++++++++ src/media-understanding/apply.test.ts | 121 +++++++++ src/media-understanding/file-context.ts | 46 +++- test/gateway-steer-fifo.e2e.test.ts | 45 ++++ 7 files changed, 644 insertions(+), 20 deletions(-) diff --git a/src/gateway/server-methods/chat-send-handler.ts b/src/gateway/server-methods/chat-send-handler.ts index 4715ca4d7cc0..454e9cd0e68e 100644 --- a/src/gateway/server-methods/chat-send-handler.ts +++ b/src/gateway/server-methods/chat-send-handler.ts @@ -10,15 +10,18 @@ import { type SessionGoalOperationResult, } from "../../config/sessions/goals-operations.js"; import type { PrepareAssistantTranscriptMessage } from "../../config/sessions/transcript-assistant-delivery.js"; +import { logVerbose } from "../../globals.js"; import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js"; import { clearAgentRunContext } from "../../infra/agent-run-registry.js"; import { emitDiagnosticsTimelineEvent } from "../../infra/diagnostics-timeline.js"; +import { formatErrorMessage } from "../../infra/errors.js"; import { recordSessionCreated, recordSessionGoalChanged, } from "../../sessions/session-state-events.js"; import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import { extractTextFromChatContent } from "../../shared/chat-content.js"; +import { createLazyImportLoader } from "../../shared/lazy-promise.js"; import type { SkillWorkshopProposalRevisionConstraint } from "../../skills/workshop/types.js"; import { isOperatorUiClient } from "../../utils/message-channel.js"; import { discardPreparedInboundMedia } from "../chat-attachments.js"; @@ -65,6 +68,10 @@ type ChatSendInternalOptions = { skillWorkshopProposalRevision?: SkillWorkshopProposalRevisionConstraint; }; +const mediaDocumentContextLoader = createLazyImportLoader( + () => import("../../media-understanding/file-context.js"), +); + async function handleChatSendWithOptions( { params, @@ -390,6 +397,33 @@ async function handleChatSendWithOptions( client?.internal?.syntheticClient ? undefined : client?.authenticatedUserProfile?.profileId, ); } + // Rendering can fail independently of admission; preserve the raw steer on failure. + const steerDocumentContext = + messageInjectionTarget && !isInternalTextSlashCommandTurn && ctx.media?.length + ? await mediaDocumentContextLoader + .load() + .then(async (runtime) => ({ + status: "rendered" as const, + ...(await runtime.renderInboundDocumentContext({ + ctx, + cfg: preparedSession.value.cfg, + })), + })) + .catch((err: unknown) => { + // A poisoned lazy import must not be served to later steers. + mediaDocumentContextLoader.clear(); + logVerbose( + `steer document render failed, injecting raw content: ${formatErrorMessage(err)}`, + ); + return { status: "failed" as const }; + }) + : undefined; + if (activeRunAbort.controller.signal.aborted) { + return finishAbortedChatSend(); + } + if (sessionRoutingChanged(context.getRuntimeConfig())) { + return admitted.value.rejectSessionRoutingChanged(); + } const beginCapturedMessageInjection = createChatSendMessageInjectionStarter({ target: messageInjectionTarget, request: normalizedRequest.value, @@ -397,6 +431,7 @@ async function handleChatSendWithOptions( admittedSessionSettings: admitted.value.admittedSessionSettings, turn: preparedUserTurn, imageOrder, + documentContext: steerDocumentContext, userTurnTranscriptRecorder: userTurnRecorder, }); const preAckReplyContextPromise = diff --git a/src/gateway/server-methods/chat-send-message-injection.test.ts b/src/gateway/server-methods/chat-send-message-injection.test.ts index 0ec2a1f26016..69e6e2c418b1 100644 --- a/src/gateway/server-methods/chat-send-message-injection.test.ts +++ b/src/gateway/server-methods/chat-send-message-injection.test.ts @@ -1,27 +1,43 @@ /** Covers steer finalize audit honesty: aborted unconfirmed commits must not audit as completed. */ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { expectDefined } from "@openclaw/normalization-core"; +import { beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js"; import { + beginReplyMessageInjectionTarget, + createReplyOperation, finalizeReplyMessageInjectionAttempt, + replyRunRegistry, type ReplyMessageInjectionTarget, } from "../../auto-reply/reply/reply-run-registry.js"; +import type { RuntimeMsgContext } from "../../auto-reply/templating.js"; import { recordSessionParticipant, updateSessionEntry, } from "../../config/sessions/session-accessor.js"; import { logMessageProcessed } from "../../logging/diagnostic.js"; import { prepareSessionParticipantInput } from "../../sessions/session-participant-input.js"; +import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; +import type { ChatImageContent } from "../chat-attachments.js"; import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js"; -import { finalizeAcceptedChatSendMessageInjection } from "./chat-send-message-injection.js"; +import { + createChatSendMessageInjectionStarter, + finalizeAcceptedChatSendMessageInjection, +} from "./chat-send-message-injection.js"; import type { GatewayRequestContext } from "./types.js"; vi.mock("../../auto-reply/reply/dispatch-from-config.audit.js", () => ({ emitInboundMessageAuditTerminal: vi.fn(), })); -vi.mock("../../auto-reply/reply/reply-run-registry.js", () => ({ - beginReplyMessageInjectionTarget: vi.fn(), - finalizeReplyMessageInjectionAttempt: vi.fn(), -})); +vi.mock(import("../../auto-reply/reply/reply-run-registry.js"), async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + beginReplyMessageInjectionTarget: vi.fn(), + finalizeReplyMessageInjectionAttempt: + vi.fn(), + }; +}); vi.mock("../../auto-reply/reply/message-received-hooks.js", () => ({ emitMessageReceivedHooks: vi.fn(), })); @@ -148,3 +164,214 @@ describe("finalizeAcceptedChatSendMessageInjection", () => { ); }); }); + +describe("createChatSendMessageInjectionStarter", () => { + beforeEach(() => { + vi.mocked(beginReplyMessageInjectionTarget).mockImplementation((target) => ({ + targetRunId: target.runId, + acceptance: Promise.resolve(true), + outcome: Promise.resolve({ status: "accepted" }), + })); + }); + + function makeStarterParams(params?: { + body?: string; + rawMessage?: string; + media?: RuntimeMsgContext["media"]; + documentContext?: Parameters< + typeof createChatSendMessageInjectionStarter + >[0]["documentContext"]; + replyOptionImages?: ChatImageContent[]; + isInternalTextSlashCommandTurn?: boolean; + }): Parameters[0] { + const sessionKey = "agent:main:steer-test"; + const sessionId = "steer-test-session"; + const rawMessage = params?.rawMessage ?? params?.body ?? "raw steer"; + const operation = createReplyOperation({ sessionKey, sessionId, resetTriggered: false }); + onTestFinished(() => operation.complete()); + operation.setPhase("running"); + operation.attachBackend({ + kind: "embedded", + runId: "active-run", + supportsQueueMessageImages: true, + cancel: vi.fn(), + messageInjection: { + isAvailable: () => true, + queueMessage: vi.fn(async () => {}), + }, + }); + const target = expectDefined( + replyRunRegistry.resolveCurrentMessageInjectionTarget(sessionKey), + "Expected the running test operation to accept steering", + ); + + return { + target, + request: { + p: { sessionKey, message: rawMessage, idempotencyKey: "steer-input" }, + rawMessage, + supportsTaskSuggestions: false, + }, + session: { cfg: {}, entry: undefined }, + turn: { + discardUnreferencedMedia: async () => {}, + accountId: undefined, + ctx: { Provider: "dashboard", Body: params?.body, media: params?.media }, + isInternalTextSlashCommandTurn: params?.isInternalTextSlashCommandTurn ?? false, + queuedFollowupOwnerKey: undefined, + pluginBoundMediaPromise: Promise.resolve([]), + replyOptionImages: params?.replyOptionImages ?? [], + replyOptionMedia: [], + }, + imageOrder: [], + documentContext: params?.documentContext, + userTurnTranscriptRecorder: createUserTurnTranscriptRecorder({ + input: { text: params?.body ?? rawMessage, media: params?.media }, + target: createTestUserTurnTranscriptTarget({ sessionKey, sessionId }), + }), + }; + } + + it.each([ + { + caption: "see attached", + label: "captioned", + expectedText: + '[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\nsee attached\n\ndoc body', + }, + { + caption: "", + label: "blank-caption", + expectedText: + '[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\ndoc body', + }, + ])("retains marker and document text for a $label steer", ({ caption, expectedText }) => { + const documentText = 'doc body'; + const params = makeStarterParams({ + body: caption, + rawMessage: caption, + media: [ + { + path: "media://inbound/note.txt", + contentType: "text/plain", + kind: "document", + fileName: "note.txt", + }, + ], + documentContext: { status: "rendered", text: documentText, images: [] }, + }); + const durableContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledOnce(); + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + params.target, + expectedText, + expect.objectContaining({ isInboundUserMessage: true }), + ); + expect(params.turn.ctx).toEqual(durableContext); + }); + + it("keeps the attachment note when document rendering fails", () => { + const params = makeStarterParams({ + body: "read the attachment", + media: [{ path: "media://inbound/note.txt", contentType: "text/plain" }], + documentContext: { status: "failed" }, + }); + const originalContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + params.target, + "[media attached: media://inbound/note.txt (text/plain)]\n\nread the attachment", + expect.objectContaining({ isInboundUserMessage: true }), + ); + expect(params.turn.ctx).toEqual(originalContext); + }); + + it("keeps the base text untouched when no document context was rendered", () => { + createChatSendMessageInjectionStarter(makeStarterParams({ body: "plain steer" }))(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "plain steer", + expect.anything(), + ); + }); + + it("merges extracted page images after the prepared inbound images", () => { + const params = makeStarterParams({ + body: "see attached", + media: [ + { path: "media://inbound/photo.png", contentType: "image/png" }, + { + path: "media://inbound/scan.pdf", + contentType: "application/pdf", + kind: "document", + }, + ], + replyOptionImages: [ + { type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 }, + ], + documentContext: { + status: "rendered", + text: "[PDF content rendered to images]", + images: [ + { type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 1 }, + { type: "image", data: "page-2", mimeType: "image/png", attachmentIndex: 1 }, + ], + }, + }); + const durableContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "[media attached: 2 files]\n" + + "[media attached 1/2: media://inbound/photo.png (image/png)]\n" + + "[media attached 2/2: media://inbound/scan.pdf (application/pdf)]\n\n" + + "see attached\n\n[PDF content rendered to images]", + expect.objectContaining({ + images: [ + { type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 }, + { type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 1 }, + { type: "image", data: "page-2", mimeType: "image/png", sourceIndex: 1 }, + ], + }), + ); + expect(params.turn.ctx).toEqual(durableContext); + }); + + it("injects extracted page images when the steer carries no inbound images", () => { + createChatSendMessageInjectionStarter( + makeStarterParams({ + body: "scan attached", + documentContext: { + status: "rendered", + text: "[PDF content rendered to images]", + images: [{ type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 0 }], + }, + }), + )(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "scan attached\n\n[PDF content rendered to images]", + expect.objectContaining({ + images: [{ type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 0 }], + }), + ); + }); + + it("returns undefined for internal slash-command turns even with a target", () => { + expect( + createChatSendMessageInjectionStarter( + makeStarterParams({ isInternalTextSlashCommandTurn: true }), + )(), + ).toBeUndefined(); + expect(beginReplyMessageInjectionTarget).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/server-methods/chat-send-message-injection.ts b/src/gateway/server-methods/chat-send-message-injection.ts index be43ece07498..9b20fad69cde 100644 --- a/src/gateway/server-methods/chat-send-message-injection.ts +++ b/src/gateway/server-methods/chat-send-message-injection.ts @@ -1,6 +1,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js"; import { resolveCommandAuthorization } from "../../auto-reply/command-auth.js"; +import { buildInboundMediaNoteProjection } from "../../auto-reply/media-note.js"; import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js"; import { finalizeInboundContext } from "../../auto-reply/reply/inbound-context.js"; import { hasInboundAudio } from "../../auto-reply/reply/inbound-media.js"; @@ -19,9 +20,11 @@ import type { SessionEntry } from "../../config/sessions.js"; import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { logMessageProcessed, logMessageReceived } from "../../logging/diagnostic.js"; +import type { InboundDocumentContext } from "../../media-understanding/file-context.js"; import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { recordAcceptedSessionParticipantInput } from "../../sessions/session-participant-input-recording.js"; import { setGatewayDedupeEntry } from "../agent-turn/agent-job.js"; +import type { ChatImageContent } from "../chat-attachments.js"; import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js"; import { buildChatSendReplyInjectionText } from "./chat-send-reply-context.js"; import type { NormalizedChatSendRequest } from "./chat-send-request.js"; @@ -37,6 +40,7 @@ export function createChatSendMessageInjectionStarter(params: { admittedSessionSettings?: Readonly>; turn: ReturnType; imageOrder: ReplyBackendQueueMessageOptions["imageOrder"]; + documentContext?: ({ status: "rendered" } & InboundDocumentContext) | { status: "failed" }; userTurnTranscriptRecorder: NonNullable< ReplyBackendQueueMessageOptions["userTurnTranscriptRecorder"] >; @@ -54,7 +58,32 @@ export function createChatSendMessageInjectionStarter(params: { sessionEntry: entry, inlineMode: p.queueMode, }); - const text = ctx.BodyForAgent ?? ctx.Body ?? rawMessage; + const baseText = ctx.BodyForAgent ?? ctx.Body ?? rawMessage; + const rendered = + params.documentContext?.status === "rendered" ? params.documentContext : undefined; + const documentContext = rendered?.text.trim(); + let text = baseText; + if (documentContext || params.documentContext?.status === "failed") { + text = [buildInboundMediaNoteProjection(ctx).text, baseText.trim(), documentContext] + .filter(Boolean) + .join("\n\n"); + } + const documentImages = rendered?.images ?? []; + const injectionImages: ChatImageContent[] | undefined = + documentImages.length > 0 + ? [ + ...(replyOptionImages ?? []), + // Extracted page images follow the prepared inbound images, the + // same inline-then-extracted ordering reply dispatch produces; each + // keeps its attachment index as ordering provenance. + ...documentImages.map((image): ChatImageContent => ({ + type: "image", + data: image.data, + mimeType: image.mimeType, + sourceIndex: image.attachmentIndex, + })), + ] + : replyOptionImages; const authorization = resolveCommandAuthorization({ ctx, cfg, @@ -78,7 +107,7 @@ export function createChatSendMessageInjectionStarter(params: { senderIsOwner: authorization.senderIsOwner, disableTools: false, }), - ...(replyOptionImages?.length ? { images: replyOptionImages } : {}), + ...(injectionImages?.length ? { images: injectionImages } : {}), ...(params.imageOrder?.length ? { imageOrder: params.imageOrder } : {}), ...(replyOptionMedia?.length ? { media: replyOptionMedia } : {}), waitForTranscriptCommit: true, diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index 6975e3518a23..5172a03ef4a4 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -129,6 +129,7 @@ const mockState = vi.hoisted(() => { emittedTranscriptUpdates: [] as TranscriptUpdate[], savedMediaResults: [] as Array<{ id?: string; path: string; contentType?: string }>, saveMediaError: null as Error | null, + steerDocumentRenderError: null as Error | null, savedMediaCalls: [] as Array<{ contentType?: string; subdir?: string; size: number }>, saveMediaWait: null as Promise | null, activeSaveMediaCalls: 0, @@ -234,6 +235,23 @@ const OFFLOAD_PNG_BASE64 = const TINY_JPEG_BASE64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEBUQEBAVFRUVFRUVFRUVFRUVFRUVFRUXFhUVFRUYHSggGBolHRUVITEhJSkrLi4uFx8zODMsNygtLisBCgoKDg0OGhAQGi0fICUtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAAEAAQMBEQACEQEDEQH/xAAXAAADAQAAAAAAAAAAAAAAAAAAAQMC/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEAMQAAAB6AAAAP/EABQQAQAAAAAAAAAAAAAAAAAAACD/2gAIAQEAAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQIBAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQMBAT8Af//Z"; +vi.mock("../../media-understanding/file-context.js", async () => { + const actual = await vi.importActual( + "../../media-understanding/file-context.js", + ); + return { + ...actual, + renderInboundDocumentContext: ( + params: Parameters[0], + ) => { + if (mockState.steerDocumentRenderError) { + return Promise.reject(mockState.steerDocumentRenderError); + } + return actual.renderInboundDocumentContext(params); + }, + }; +}); + vi.mock("../session-utils.js", async () => { const original = await vi.importActual("../session-utils.js"); @@ -2240,6 +2258,133 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); }); + it("hands steered document attachments to the active run as extracted file context", async () => { + const { send } = await createSqliteChatRequest("openclaw-chat-send-steer-document-"); + mockState.hasMessageReceivedHooks = true; + // Under the suite fixture root so the suite cleanup removes it; a bare + // tmpdir entry would leak on every run. + const documentDir = fs.mkdtempSync(path.join(suiteFixtureRoot, "openclaw-steer-doc-")); + const documentPath = path.join(documentDir, "notes.txt"); + fs.writeFileSync(documentPath, "steered document body"); + setSavedMediaResults([documentPath, "text/plain"]); + const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => { + await options?.userTurnTranscriptRecorder?.persistApproved(); + }); + const operation = beginMessageInjectionOperation({ + originatingLeafEntryId: null, + runId: "active-run", + supportsQueueMessageImages: true, + taskSuggestionDeliveryMode: "gateway", + queueMessage, + }); + + try { + await send({ + idempotencyKey: "idem-steer-document", + requestParams: { + queueMode: "steer", + attachments: [ + createFileAttachment( + "notes.txt", + "text/plain", + Buffer.from("steered document body", "utf8").toString("base64"), + ), + ], + }, + client: { + connect: { + client: { + id: GATEWAY_CLIENT_NAMES.CONTROL_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + version: "dev", + platform: "web", + }, + caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS], + scopes: ["operator.admin"], + }, + }, + }); + } finally { + operation.complete(); + } + + expect(queueMessage).toHaveBeenCalledOnce(); + const queueCall = vi.mocked(queueMessage).mock.calls[0]; + if (!queueCall) { + throw new Error("expected queueMessage to receive the injected text"); + } + const [injectedText] = queueCall; + // The active run never reaches reply dispatch, so the injection text must + // already carry the extracted document context. + expect(injectedText).toContain(''); + expect(injectedText).toContain("steered document body"); + }); + + it("keeps the raw steer when the lazy document render fails", async () => { + const { respond, send } = await createSqliteChatRequest("openclaw-chat-send-steer-doc-fail-"); + mockState.hasMessageReceivedHooks = true; + const documentPath = path.join(suiteFixtureRoot, "openclaw-steer-doc-fail.txt"); + fs.writeFileSync(documentPath, "steered document body"); + setSavedMediaResults([documentPath, "text/plain"]); + mockState.steerDocumentRenderError = new Error("lazy media runtime unavailable"); + const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => { + await options?.userTurnTranscriptRecorder?.persistApproved(); + }); + const operation = beginMessageInjectionOperation({ + originatingLeafEntryId: null, + runId: "active-run", + supportsQueueMessageImages: true, + taskSuggestionDeliveryMode: "gateway", + queueMessage, + }); + + try { + await send({ + idempotencyKey: "idem-steer-document-render-failure", + requestParams: { + queueMode: "steer", + attachments: [ + createFileAttachment( + "notes.txt", + "text/plain", + Buffer.from("steered document body", "utf8").toString("base64"), + ), + ], + }, + client: { + connect: { + client: { + id: GATEWAY_CLIENT_NAMES.CONTROL_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + version: "dev", + platform: "web", + }, + caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS], + scopes: ["operator.admin"], + }, + }, + }); + } finally { + operation.complete(); + mockState.steerDocumentRenderError = null; + } + + // A failed render must not abort a steerable message: normal reply + // dispatch proceeds with raw content on media-understanding failure, so + // the steer injection keeps the same contract. + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ status: "started" }), + undefined, + expect.any(Object), + ); + expect(queueMessage).toHaveBeenCalledOnce(); + const [injectedText] = vi.mocked(queueMessage).mock.calls[0] ?? []; + expect(injectedText).toBeDefined(); + expect(injectedText).not.toContain(' { const { context, respond, send } = await createSqliteChatRequest( "openclaw-chat-send-reply-steer-", diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 7bd1b6d0a9ba..4b538a2659fd 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -32,6 +32,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn()); const runFfmpegMock = vi.hoisted(() => vi.fn()); const convertHeicToJpegMock = vi.hoisted(() => vi.fn()); const runExecMock = vi.hoisted(() => vi.fn()); +const extractFileContentFromBufferMock = vi.hoisted(() => vi.fn()); let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding; let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.test-support.js").clearMediaUnderstandingBinaryCacheForTests; @@ -40,6 +41,10 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock; const mockedRunFfmpeg = runFfmpegMock; const mockedConvertHeicToJpeg = convertHeicToJpegMock; const mockedRunExec = runExecMock; +const mockedExtractFileContentFromBuffer = extractFileContentFromBufferMock; +let actualExtractFileContentFromBuffer: + | typeof import("../media/input-files.js").extractFileContentFromBuffer + | undefined; const TEMP_MEDIA_PREFIX = "openclaw-media-"; const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/; @@ -354,6 +359,15 @@ describe("applyMediaUnderstanding", () => { vi.doMock("../process/exec.js", () => ({ runExec: runExecMock, })); + vi.doMock("../media/input-files.js", async () => { + const actual = + await vi.importActual("../media/input-files.js"); + actualExtractFileContentFromBuffer = actual.extractFileContentFromBuffer; + return { + ...actual, + extractFileContentFromBuffer: extractFileContentFromBufferMock, + }; + }); vi.doMock("./provider-registry.js", async () => { const actual = await vi.importActual("./provider-registry.js"); @@ -405,6 +419,12 @@ describe("applyMediaUnderstanding", () => { mockedConvertHeicToJpeg.mockReset(); mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized")); mockedRunExec.mockReset(); + // Extraction stays real unless a case overrides it; the mock exists so + // scanned-PDF render outcomes can be produced without the extract plugin. + mockedExtractFileContentFromBuffer.mockReset(); + if (actualExtractFileContentFromBuffer) { + mockedExtractFileContentFromBuffer.mockImplementation(actualExtractFileContentFromBuffer); + } mockedReadRemoteMediaBuffer.mockResolvedValue({ buffer: createSafeAudioFixtureBuffer(2048), contentType: "audio/ogg", @@ -2624,5 +2644,106 @@ describe("applyMediaUnderstanding", () => { expect(ctx.Body).toContain(" { + it("renders a document attachment without mutating ctx", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "steer-note.txt", + content: "document body for the steered run", + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "text/plain" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig }); + + expect(context?.text).toContain(''); + expect(context?.text).toContain("document body for the steered run"); + expect(context?.images).toEqual([]); + // Read-only on ctx: a rejected steer falls back to reply dispatch, which + // must extract exactly once through the full pipeline. + expect(ctx.Body).toBe("see attached"); + expect(ctx.media?.[0]?.path).toBe(mediaPath); + }); + + it("returns empty for image attachments owned by the injected images channel", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "steer.png", + content: createSafeAudioFixtureBuffer(16), + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "image/png" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig }); + + expect(context?.text).toBe(""); + expect(context?.images).toEqual([]); + expect(ctx.Body).toBe("see attached"); + }); + + it("returns empty context when the steer carries no attachments", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const context = await renderInboundDocumentContext({ + ctx: { Body: "plain steer" } as MsgContext, + cfg: {} as OpenClawConfig, + }); + expect(context).toEqual({ text: "", images: [] }); + }); + + it("returns rendered PDF page images for a scanned document", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "scan.pdf", + content: Buffer.from("%PDF-1.4\n", "utf8"), + }); + mockedExtractFileContentFromBuffer.mockResolvedValueOnce({ + text: "", + images: [ + { type: "image", data: "page-1-bytes", mimeType: "image/png" }, + { type: "image", data: "page-2-bytes", mimeType: "image/png" }, + ], + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "application/pdf" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() }); + + // The marker alone would tell the model the document exists while the + // injected images channel carries nothing; the pages must ride along. + expect(context?.text).toContain("[PDF content rendered to images]"); + expect(context?.images).toEqual([ + { type: "image", data: "page-1-bytes", mimeType: "image/png", attachmentIndex: 0 }, + { type: "image", data: "page-2-bytes", mimeType: "image/png", attachmentIndex: 0 }, + ]); + }); + + it("applies the skipped-attachment marker budget to steer blocks", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const olePayload = Buffer.from("Root Entry WordDocument legacy preview", "utf8"); + const media: { path: string; contentType: string }[] = []; + for (let i = 0; i < 7; i += 1) { + const filePath = await createTempMediaFile({ + fileName: `legacy-${i}.doc`, + content: olePayload, + }); + media.push({ path: filePath, contentType: "application/msword" }); + } + const ctx: MsgContext = { Body: "see attached", media }; + + const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() }); + + const markerCount = context?.text.split("[Unsupported document format").length ?? 0; + expect(markerCount - 1).toBe(5); + expect(context?.text).toContain("[2 more attachments skipped]"); + expect(context?.images).toEqual([]); + }); + }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/media-understanding/file-context.ts b/src/media-understanding/file-context.ts index 6796ea44a094..99f241ff93bf 100644 --- a/src/media-understanding/file-context.ts +++ b/src/media-understanding/file-context.ts @@ -300,33 +300,55 @@ export async function prepareFileContextFromMedia(params: { maxChars: number; assertCurrent: () => void; }) { - params.assertCurrent(); - const ctx: MsgContext = { - media: [...params.media], - Provider: params.channelId, - AccountId: params.accountId, - }; - const limits = resolveFileExtractionLimits(params.config); + return await renderInboundDocumentContext({ + ctx: { + media: [...params.media], + Provider: params.channelId, + AccountId: params.accountId, + }, + cfg: params.config, + workspaceDir: params.workspaceDir, + maxChars: params.maxChars, + assertCurrent: params.assertCurrent, + }); +} + +export type InboundDocumentContext = { text: string; images: ExtractedFileImage[] }; + +/** Keep prompt expansion separate from inbound state so rejected steers can dispatch normally. */ +export async function renderInboundDocumentContext(params: { + ctx: MsgContext; + cfg: OpenClawConfig; + workspaceDir?: string; + maxChars?: number; + assertCurrent?: () => void; +}): Promise { + params.assertCurrent?.(); + const { ctx, cfg } = params; + const limits = resolveFileExtractionLimits(cfg); const attachments = normalizeMediaAttachments(ctx); const cache = createMediaAttachmentCache(attachments, { localPathRoots: resolveMediaAttachmentLocalRoots({ - cfg: params.config, + cfg, ctx, workspaceDir: params.workspaceDir, }), - ssrfPolicy: params.config.tools?.web?.fetch?.ssrfPolicy, + ssrfPolicy: cfg.tools?.web?.fetch?.ssrfPolicy, workspaceDir: params.workspaceDir, }); try { const context = await extractFileContext({ attachments, cache, - cfg: params.config, - limits: { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) }, + cfg, + limits: + params.maxChars === undefined + ? limits + : { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) }, selfServePathsEnabled: false, assertCurrent: params.assertCurrent, }); - params.assertCurrent(); + params.assertCurrent?.(); return { text: applyAttachmentMarkerBudget(context.blocks).join("\n\n"), images: context.images, diff --git a/test/gateway-steer-fifo.e2e.test.ts b/test/gateway-steer-fifo.e2e.test.ts index e74b376ac22c..472cd9ee081d 100644 --- a/test/gateway-steer-fifo.e2e.test.ts +++ b/test/gateway-steer-fifo.e2e.test.ts @@ -988,4 +988,49 @@ describe("Gateway steer FIFO", () => { }, TEST_TIMEOUT_MS, ); + + it( + "hands steered document attachments to the active run as extracted file context", + async () => { + const fixture = await createGatewayFixture("steer-document-context"); + const first = await sendHeldTurn(fixture); + + // Real gateway protocol send with a file attachment while the initial + // run is still live: admission must steer the active run instead of + // dispatching a reply, and the extracted document context must reach + // the prompt the run actually sends to the model. + const steer = await fixture.diagnosticsClient.request<{ runId?: unknown; status?: unknown }>( + "chat.send", + { + sessionKey: fixture.sessionKey, + message: "QUEUED_STEER_DOCUMENT", + deliver: false, + queueMode: "steer", + idempotencyKey: "run-steer-document", + attachments: [ + { + type: "file", + mimeType: "text/plain", + fileName: "notes.txt", + content: Buffer.from("steered document body", "utf8").toString("base64"), + sizeBytes: "steered document body".length, + }, + ], + }, + ); + expect(steer).toMatchObject({ status: "started" }); + + fixture.modelServer.releaseFirst("final"); + + await vi.waitFor(() => expect(fixture.modelServer.requests).toHaveLength(2), WAIT_OPTS); + const currentSteer = currentUserInput(fixture.modelServer.requests[1]); + expect(currentSteer).toContain("QUEUED_STEER_DOCUMENT"); + // The media store persists uploads as `---`, so + // the extracted block carries the stored name, matching reply dispatch. + expect(currentSteer).toMatch(//); + expect(currentSteer).toContain("steered document body"); + await waitForRunTerminal(fixture, first.runId); + }, + TEST_TIMEOUT_MS, + ); });