diff --git a/src/gateway/server-methods/chat-send-handler.ts b/src/gateway/server-methods/chat-send-handler.ts index 4715ca4d7cc0..454e9cd0e68e 100644 --- a/src/gateway/server-methods/chat-send-handler.ts +++ b/src/gateway/server-methods/chat-send-handler.ts @@ -10,15 +10,18 @@ import { type SessionGoalOperationResult, } from "../../config/sessions/goals-operations.js"; import type { PrepareAssistantTranscriptMessage } from "../../config/sessions/transcript-assistant-delivery.js"; +import { logVerbose } from "../../globals.js"; import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js"; import { clearAgentRunContext } from "../../infra/agent-run-registry.js"; import { emitDiagnosticsTimelineEvent } from "../../infra/diagnostics-timeline.js"; +import { formatErrorMessage } from "../../infra/errors.js"; import { recordSessionCreated, recordSessionGoalChanged, } from "../../sessions/session-state-events.js"; import type { UserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; import { extractTextFromChatContent } from "../../shared/chat-content.js"; +import { createLazyImportLoader } from "../../shared/lazy-promise.js"; import type { SkillWorkshopProposalRevisionConstraint } from "../../skills/workshop/types.js"; import { isOperatorUiClient } from "../../utils/message-channel.js"; import { discardPreparedInboundMedia } from "../chat-attachments.js"; @@ -65,6 +68,10 @@ type ChatSendInternalOptions = { skillWorkshopProposalRevision?: SkillWorkshopProposalRevisionConstraint; }; +const mediaDocumentContextLoader = createLazyImportLoader( + () => import("../../media-understanding/file-context.js"), +); + async function handleChatSendWithOptions( { params, @@ -390,6 +397,33 @@ async function handleChatSendWithOptions( client?.internal?.syntheticClient ? undefined : client?.authenticatedUserProfile?.profileId, ); } + // Rendering can fail independently of admission; preserve the raw steer on failure. + const steerDocumentContext = + messageInjectionTarget && !isInternalTextSlashCommandTurn && ctx.media?.length + ? await mediaDocumentContextLoader + .load() + .then(async (runtime) => ({ + status: "rendered" as const, + ...(await runtime.renderInboundDocumentContext({ + ctx, + cfg: preparedSession.value.cfg, + })), + })) + .catch((err: unknown) => { + // A poisoned lazy import must not be served to later steers. + mediaDocumentContextLoader.clear(); + logVerbose( + `steer document render failed, injecting raw content: ${formatErrorMessage(err)}`, + ); + return { status: "failed" as const }; + }) + : undefined; + if (activeRunAbort.controller.signal.aborted) { + return finishAbortedChatSend(); + } + if (sessionRoutingChanged(context.getRuntimeConfig())) { + return admitted.value.rejectSessionRoutingChanged(); + } const beginCapturedMessageInjection = createChatSendMessageInjectionStarter({ target: messageInjectionTarget, request: normalizedRequest.value, @@ -397,6 +431,7 @@ async function handleChatSendWithOptions( admittedSessionSettings: admitted.value.admittedSessionSettings, turn: preparedUserTurn, imageOrder, + documentContext: steerDocumentContext, userTurnTranscriptRecorder: userTurnRecorder, }); const preAckReplyContextPromise = diff --git a/src/gateway/server-methods/chat-send-message-injection.test.ts b/src/gateway/server-methods/chat-send-message-injection.test.ts index 0ec2a1f26016..69e6e2c418b1 100644 --- a/src/gateway/server-methods/chat-send-message-injection.test.ts +++ b/src/gateway/server-methods/chat-send-message-injection.test.ts @@ -1,27 +1,43 @@ /** Covers steer finalize audit honesty: aborted unconfirmed commits must not audit as completed. */ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { expectDefined } from "@openclaw/normalization-core"; +import { beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js"; import { + beginReplyMessageInjectionTarget, + createReplyOperation, finalizeReplyMessageInjectionAttempt, + replyRunRegistry, type ReplyMessageInjectionTarget, } from "../../auto-reply/reply/reply-run-registry.js"; +import type { RuntimeMsgContext } from "../../auto-reply/templating.js"; import { recordSessionParticipant, updateSessionEntry, } from "../../config/sessions/session-accessor.js"; import { logMessageProcessed } from "../../logging/diagnostic.js"; import { prepareSessionParticipantInput } from "../../sessions/session-participant-input.js"; +import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js"; +import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js"; +import type { ChatImageContent } from "../chat-attachments.js"; import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js"; -import { finalizeAcceptedChatSendMessageInjection } from "./chat-send-message-injection.js"; +import { + createChatSendMessageInjectionStarter, + finalizeAcceptedChatSendMessageInjection, +} from "./chat-send-message-injection.js"; import type { GatewayRequestContext } from "./types.js"; vi.mock("../../auto-reply/reply/dispatch-from-config.audit.js", () => ({ emitInboundMessageAuditTerminal: vi.fn(), })); -vi.mock("../../auto-reply/reply/reply-run-registry.js", () => ({ - beginReplyMessageInjectionTarget: vi.fn(), - finalizeReplyMessageInjectionAttempt: vi.fn(), -})); +vi.mock(import("../../auto-reply/reply/reply-run-registry.js"), async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + beginReplyMessageInjectionTarget: vi.fn(), + finalizeReplyMessageInjectionAttempt: + vi.fn(), + }; +}); vi.mock("../../auto-reply/reply/message-received-hooks.js", () => ({ emitMessageReceivedHooks: vi.fn(), })); @@ -148,3 +164,214 @@ describe("finalizeAcceptedChatSendMessageInjection", () => { ); }); }); + +describe("createChatSendMessageInjectionStarter", () => { + beforeEach(() => { + vi.mocked(beginReplyMessageInjectionTarget).mockImplementation((target) => ({ + targetRunId: target.runId, + acceptance: Promise.resolve(true), + outcome: Promise.resolve({ status: "accepted" }), + })); + }); + + function makeStarterParams(params?: { + body?: string; + rawMessage?: string; + media?: RuntimeMsgContext["media"]; + documentContext?: Parameters< + typeof createChatSendMessageInjectionStarter + >[0]["documentContext"]; + replyOptionImages?: ChatImageContent[]; + isInternalTextSlashCommandTurn?: boolean; + }): Parameters[0] { + const sessionKey = "agent:main:steer-test"; + const sessionId = "steer-test-session"; + const rawMessage = params?.rawMessage ?? params?.body ?? "raw steer"; + const operation = createReplyOperation({ sessionKey, sessionId, resetTriggered: false }); + onTestFinished(() => operation.complete()); + operation.setPhase("running"); + operation.attachBackend({ + kind: "embedded", + runId: "active-run", + supportsQueueMessageImages: true, + cancel: vi.fn(), + messageInjection: { + isAvailable: () => true, + queueMessage: vi.fn(async () => {}), + }, + }); + const target = expectDefined( + replyRunRegistry.resolveCurrentMessageInjectionTarget(sessionKey), + "Expected the running test operation to accept steering", + ); + + return { + target, + request: { + p: { sessionKey, message: rawMessage, idempotencyKey: "steer-input" }, + rawMessage, + supportsTaskSuggestions: false, + }, + session: { cfg: {}, entry: undefined }, + turn: { + discardUnreferencedMedia: async () => {}, + accountId: undefined, + ctx: { Provider: "dashboard", Body: params?.body, media: params?.media }, + isInternalTextSlashCommandTurn: params?.isInternalTextSlashCommandTurn ?? false, + queuedFollowupOwnerKey: undefined, + pluginBoundMediaPromise: Promise.resolve([]), + replyOptionImages: params?.replyOptionImages ?? [], + replyOptionMedia: [], + }, + imageOrder: [], + documentContext: params?.documentContext, + userTurnTranscriptRecorder: createUserTurnTranscriptRecorder({ + input: { text: params?.body ?? rawMessage, media: params?.media }, + target: createTestUserTurnTranscriptTarget({ sessionKey, sessionId }), + }), + }; + } + + it.each([ + { + caption: "see attached", + label: "captioned", + expectedText: + '[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\nsee attached\n\ndoc body', + }, + { + caption: "", + label: "blank-caption", + expectedText: + '[media attached: media://inbound/note.txt (text/plain) "note.txt"]\n\ndoc body', + }, + ])("retains marker and document text for a $label steer", ({ caption, expectedText }) => { + const documentText = 'doc body'; + const params = makeStarterParams({ + body: caption, + rawMessage: caption, + media: [ + { + path: "media://inbound/note.txt", + contentType: "text/plain", + kind: "document", + fileName: "note.txt", + }, + ], + documentContext: { status: "rendered", text: documentText, images: [] }, + }); + const durableContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledOnce(); + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + params.target, + expectedText, + expect.objectContaining({ isInboundUserMessage: true }), + ); + expect(params.turn.ctx).toEqual(durableContext); + }); + + it("keeps the attachment note when document rendering fails", () => { + const params = makeStarterParams({ + body: "read the attachment", + media: [{ path: "media://inbound/note.txt", contentType: "text/plain" }], + documentContext: { status: "failed" }, + }); + const originalContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + params.target, + "[media attached: media://inbound/note.txt (text/plain)]\n\nread the attachment", + expect.objectContaining({ isInboundUserMessage: true }), + ); + expect(params.turn.ctx).toEqual(originalContext); + }); + + it("keeps the base text untouched when no document context was rendered", () => { + createChatSendMessageInjectionStarter(makeStarterParams({ body: "plain steer" }))(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "plain steer", + expect.anything(), + ); + }); + + it("merges extracted page images after the prepared inbound images", () => { + const params = makeStarterParams({ + body: "see attached", + media: [ + { path: "media://inbound/photo.png", contentType: "image/png" }, + { + path: "media://inbound/scan.pdf", + contentType: "application/pdf", + kind: "document", + }, + ], + replyOptionImages: [ + { type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 }, + ], + documentContext: { + status: "rendered", + text: "[PDF content rendered to images]", + images: [ + { type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 1 }, + { type: "image", data: "page-2", mimeType: "image/png", attachmentIndex: 1 }, + ], + }, + }); + const durableContext = structuredClone(params.turn.ctx); + + createChatSendMessageInjectionStarter(params)(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "[media attached: 2 files]\n" + + "[media attached 1/2: media://inbound/photo.png (image/png)]\n" + + "[media attached 2/2: media://inbound/scan.pdf (application/pdf)]\n\n" + + "see attached\n\n[PDF content rendered to images]", + expect.objectContaining({ + images: [ + { type: "image", data: "inbound-photo", mimeType: "image/png", sourceIndex: 0 }, + { type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 1 }, + { type: "image", data: "page-2", mimeType: "image/png", sourceIndex: 1 }, + ], + }), + ); + expect(params.turn.ctx).toEqual(durableContext); + }); + + it("injects extracted page images when the steer carries no inbound images", () => { + createChatSendMessageInjectionStarter( + makeStarterParams({ + body: "scan attached", + documentContext: { + status: "rendered", + text: "[PDF content rendered to images]", + images: [{ type: "image", data: "page-1", mimeType: "image/png", attachmentIndex: 0 }], + }, + }), + )(); + + expect(beginReplyMessageInjectionTarget).toHaveBeenCalledWith( + expect.anything(), + "scan attached\n\n[PDF content rendered to images]", + expect.objectContaining({ + images: [{ type: "image", data: "page-1", mimeType: "image/png", sourceIndex: 0 }], + }), + ); + }); + + it("returns undefined for internal slash-command turns even with a target", () => { + expect( + createChatSendMessageInjectionStarter( + makeStarterParams({ isInternalTextSlashCommandTurn: true }), + )(), + ).toBeUndefined(); + expect(beginReplyMessageInjectionTarget).not.toHaveBeenCalled(); + }); +}); diff --git a/src/gateway/server-methods/chat-send-message-injection.ts b/src/gateway/server-methods/chat-send-message-injection.ts index be43ece07498..9b20fad69cde 100644 --- a/src/gateway/server-methods/chat-send-message-injection.ts +++ b/src/gateway/server-methods/chat-send-message-injection.ts @@ -1,6 +1,7 @@ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js"; import { resolveCommandAuthorization } from "../../auto-reply/command-auth.js"; +import { buildInboundMediaNoteProjection } from "../../auto-reply/media-note.js"; import { emitInboundMessageAuditTerminal } from "../../auto-reply/reply/dispatch-from-config.audit.js"; import { finalizeInboundContext } from "../../auto-reply/reply/inbound-context.js"; import { hasInboundAudio } from "../../auto-reply/reply/inbound-media.js"; @@ -19,9 +20,11 @@ import type { SessionEntry } from "../../config/sessions.js"; import { updateSessionEntry } from "../../config/sessions/session-accessor.js"; import { isDiagnosticsEnabled } from "../../infra/diagnostic-events.js"; import { logMessageProcessed, logMessageReceived } from "../../logging/diagnostic.js"; +import type { InboundDocumentContext } from "../../media-understanding/file-context.js"; import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; import { recordAcceptedSessionParticipantInput } from "../../sessions/session-participant-input-recording.js"; import { setGatewayDedupeEntry } from "../agent-turn/agent-job.js"; +import type { ChatImageContent } from "../chat-attachments.js"; import { broadcastChatError, broadcastChatFinal } from "./chat-broadcast.js"; import { buildChatSendReplyInjectionText } from "./chat-send-reply-context.js"; import type { NormalizedChatSendRequest } from "./chat-send-request.js"; @@ -37,6 +40,7 @@ export function createChatSendMessageInjectionStarter(params: { admittedSessionSettings?: Readonly>; turn: ReturnType; imageOrder: ReplyBackendQueueMessageOptions["imageOrder"]; + documentContext?: ({ status: "rendered" } & InboundDocumentContext) | { status: "failed" }; userTurnTranscriptRecorder: NonNullable< ReplyBackendQueueMessageOptions["userTurnTranscriptRecorder"] >; @@ -54,7 +58,32 @@ export function createChatSendMessageInjectionStarter(params: { sessionEntry: entry, inlineMode: p.queueMode, }); - const text = ctx.BodyForAgent ?? ctx.Body ?? rawMessage; + const baseText = ctx.BodyForAgent ?? ctx.Body ?? rawMessage; + const rendered = + params.documentContext?.status === "rendered" ? params.documentContext : undefined; + const documentContext = rendered?.text.trim(); + let text = baseText; + if (documentContext || params.documentContext?.status === "failed") { + text = [buildInboundMediaNoteProjection(ctx).text, baseText.trim(), documentContext] + .filter(Boolean) + .join("\n\n"); + } + const documentImages = rendered?.images ?? []; + const injectionImages: ChatImageContent[] | undefined = + documentImages.length > 0 + ? [ + ...(replyOptionImages ?? []), + // Extracted page images follow the prepared inbound images, the + // same inline-then-extracted ordering reply dispatch produces; each + // keeps its attachment index as ordering provenance. + ...documentImages.map((image): ChatImageContent => ({ + type: "image", + data: image.data, + mimeType: image.mimeType, + sourceIndex: image.attachmentIndex, + })), + ] + : replyOptionImages; const authorization = resolveCommandAuthorization({ ctx, cfg, @@ -78,7 +107,7 @@ export function createChatSendMessageInjectionStarter(params: { senderIsOwner: authorization.senderIsOwner, disableTools: false, }), - ...(replyOptionImages?.length ? { images: replyOptionImages } : {}), + ...(injectionImages?.length ? { images: injectionImages } : {}), ...(params.imageOrder?.length ? { imageOrder: params.imageOrder } : {}), ...(replyOptionMedia?.length ? { media: replyOptionMedia } : {}), waitForTranscriptCommit: true, diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index 6975e3518a23..5172a03ef4a4 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -129,6 +129,7 @@ const mockState = vi.hoisted(() => { emittedTranscriptUpdates: [] as TranscriptUpdate[], savedMediaResults: [] as Array<{ id?: string; path: string; contentType?: string }>, saveMediaError: null as Error | null, + steerDocumentRenderError: null as Error | null, savedMediaCalls: [] as Array<{ contentType?: string; subdir?: string; size: number }>, saveMediaWait: null as Promise | null, activeSaveMediaCalls: 0, @@ -234,6 +235,23 @@ const OFFLOAD_PNG_BASE64 = const TINY_JPEG_BASE64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEBUQEBAVFRUVFRUVFRUVFRUVFRUVFRUXFhUVFRUYHSggGBolHRUVITEhJSkrLi4uFx8zODMsNygtLisBCgoKDg0OGhAQGi0fICUtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLf/AABEIAAEAAQMBEQACEQEDEQH/xAAXAAADAQAAAAAAAAAAAAAAAAAAAQMC/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEAMQAAAB6AAAAP/EABQQAQAAAAAAAAAAAAAAAAAAACD/2gAIAQEAAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQIBAT8Af//EABQRAQAAAAAAAAAAAAAAAAAAACD/2gAIAQMBAT8Af//Z"; +vi.mock("../../media-understanding/file-context.js", async () => { + const actual = await vi.importActual( + "../../media-understanding/file-context.js", + ); + return { + ...actual, + renderInboundDocumentContext: ( + params: Parameters[0], + ) => { + if (mockState.steerDocumentRenderError) { + return Promise.reject(mockState.steerDocumentRenderError); + } + return actual.renderInboundDocumentContext(params); + }, + }; +}); + vi.mock("../session-utils.js", async () => { const original = await vi.importActual("../session-utils.js"); @@ -2240,6 +2258,133 @@ describe("chat directive tag stripping for non-streaming final payloads", () => }); }); + it("hands steered document attachments to the active run as extracted file context", async () => { + const { send } = await createSqliteChatRequest("openclaw-chat-send-steer-document-"); + mockState.hasMessageReceivedHooks = true; + // Under the suite fixture root so the suite cleanup removes it; a bare + // tmpdir entry would leak on every run. + const documentDir = fs.mkdtempSync(path.join(suiteFixtureRoot, "openclaw-steer-doc-")); + const documentPath = path.join(documentDir, "notes.txt"); + fs.writeFileSync(documentPath, "steered document body"); + setSavedMediaResults([documentPath, "text/plain"]); + const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => { + await options?.userTurnTranscriptRecorder?.persistApproved(); + }); + const operation = beginMessageInjectionOperation({ + originatingLeafEntryId: null, + runId: "active-run", + supportsQueueMessageImages: true, + taskSuggestionDeliveryMode: "gateway", + queueMessage, + }); + + try { + await send({ + idempotencyKey: "idem-steer-document", + requestParams: { + queueMode: "steer", + attachments: [ + createFileAttachment( + "notes.txt", + "text/plain", + Buffer.from("steered document body", "utf8").toString("base64"), + ), + ], + }, + client: { + connect: { + client: { + id: GATEWAY_CLIENT_NAMES.CONTROL_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + version: "dev", + platform: "web", + }, + caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS], + scopes: ["operator.admin"], + }, + }, + }); + } finally { + operation.complete(); + } + + expect(queueMessage).toHaveBeenCalledOnce(); + const queueCall = vi.mocked(queueMessage).mock.calls[0]; + if (!queueCall) { + throw new Error("expected queueMessage to receive the injected text"); + } + const [injectedText] = queueCall; + // The active run never reaches reply dispatch, so the injection text must + // already carry the extracted document context. + expect(injectedText).toContain(''); + expect(injectedText).toContain("steered document body"); + }); + + it("keeps the raw steer when the lazy document render fails", async () => { + const { respond, send } = await createSqliteChatRequest("openclaw-chat-send-steer-doc-fail-"); + mockState.hasMessageReceivedHooks = true; + const documentPath = path.join(suiteFixtureRoot, "openclaw-steer-doc-fail.txt"); + fs.writeFileSync(documentPath, "steered document body"); + setSavedMediaResults([documentPath, "text/plain"]); + mockState.steerDocumentRenderError = new Error("lazy media runtime unavailable"); + const queueMessage = vi.fn(async (_text: string, options?: ReplyBackendQueueMessageOptions) => { + await options?.userTurnTranscriptRecorder?.persistApproved(); + }); + const operation = beginMessageInjectionOperation({ + originatingLeafEntryId: null, + runId: "active-run", + supportsQueueMessageImages: true, + taskSuggestionDeliveryMode: "gateway", + queueMessage, + }); + + try { + await send({ + idempotencyKey: "idem-steer-document-render-failure", + requestParams: { + queueMode: "steer", + attachments: [ + createFileAttachment( + "notes.txt", + "text/plain", + Buffer.from("steered document body", "utf8").toString("base64"), + ), + ], + }, + client: { + connect: { + client: { + id: GATEWAY_CLIENT_NAMES.CONTROL_UI, + mode: GATEWAY_CLIENT_MODES.WEBCHAT, + version: "dev", + platform: "web", + }, + caps: [GATEWAY_CLIENT_CAPS.TASK_SUGGESTIONS], + scopes: ["operator.admin"], + }, + }, + }); + } finally { + operation.complete(); + mockState.steerDocumentRenderError = null; + } + + // A failed render must not abort a steerable message: normal reply + // dispatch proceeds with raw content on media-understanding failure, so + // the steer injection keeps the same contract. + expect(respond).toHaveBeenCalledWith( + true, + expect.objectContaining({ status: "started" }), + undefined, + expect.any(Object), + ); + expect(queueMessage).toHaveBeenCalledOnce(); + const [injectedText] = vi.mocked(queueMessage).mock.calls[0] ?? []; + expect(injectedText).toBeDefined(); + expect(injectedText).not.toContain(' { const { context, respond, send } = await createSqliteChatRequest( "openclaw-chat-send-reply-steer-", diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 7bd1b6d0a9ba..4b538a2659fd 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -32,6 +32,7 @@ const readRemoteMediaBufferMock = vi.hoisted(() => vi.fn()); const runFfmpegMock = vi.hoisted(() => vi.fn()); const convertHeicToJpegMock = vi.hoisted(() => vi.fn()); const runExecMock = vi.hoisted(() => vi.fn()); +const extractFileContentFromBufferMock = vi.hoisted(() => vi.fn()); let applyMediaUnderstanding: typeof import("./apply.js").applyMediaUnderstanding; let clearMediaUnderstandingBinaryCacheForTests: typeof import("./runner.test-support.js").clearMediaUnderstandingBinaryCacheForTests; @@ -40,6 +41,10 @@ const mockedReadRemoteMediaBuffer = readRemoteMediaBufferMock; const mockedRunFfmpeg = runFfmpegMock; const mockedConvertHeicToJpeg = convertHeicToJpegMock; const mockedRunExec = runExecMock; +const mockedExtractFileContentFromBuffer = extractFileContentFromBufferMock; +let actualExtractFileContentFromBuffer: + | typeof import("../media/input-files.js").extractFileContentFromBuffer + | undefined; const TEMP_MEDIA_PREFIX = "openclaw-media-"; const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/; @@ -354,6 +359,15 @@ describe("applyMediaUnderstanding", () => { vi.doMock("../process/exec.js", () => ({ runExec: runExecMock, })); + vi.doMock("../media/input-files.js", async () => { + const actual = + await vi.importActual("../media/input-files.js"); + actualExtractFileContentFromBuffer = actual.extractFileContentFromBuffer; + return { + ...actual, + extractFileContentFromBuffer: extractFileContentFromBufferMock, + }; + }); vi.doMock("./provider-registry.js", async () => { const actual = await vi.importActual("./provider-registry.js"); @@ -405,6 +419,12 @@ describe("applyMediaUnderstanding", () => { mockedConvertHeicToJpeg.mockReset(); mockedConvertHeicToJpeg.mockResolvedValue(Buffer.from("jpeg-normalized")); mockedRunExec.mockReset(); + // Extraction stays real unless a case overrides it; the mock exists so + // scanned-PDF render outcomes can be produced without the extract plugin. + mockedExtractFileContentFromBuffer.mockReset(); + if (actualExtractFileContentFromBuffer) { + mockedExtractFileContentFromBuffer.mockImplementation(actualExtractFileContentFromBuffer); + } mockedReadRemoteMediaBuffer.mockResolvedValue({ buffer: createSafeAudioFixtureBuffer(2048), contentType: "audio/ogg", @@ -2624,5 +2644,106 @@ describe("applyMediaUnderstanding", () => { expect(ctx.Body).toContain(" { + it("renders a document attachment without mutating ctx", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "steer-note.txt", + content: "document body for the steered run", + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "text/plain" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig }); + + expect(context?.text).toContain(''); + expect(context?.text).toContain("document body for the steered run"); + expect(context?.images).toEqual([]); + // Read-only on ctx: a rejected steer falls back to reply dispatch, which + // must extract exactly once through the full pipeline. + expect(ctx.Body).toBe("see attached"); + expect(ctx.media?.[0]?.path).toBe(mediaPath); + }); + + it("returns empty for image attachments owned by the injected images channel", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "steer.png", + content: createSafeAudioFixtureBuffer(16), + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "image/png" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: {} as OpenClawConfig }); + + expect(context?.text).toBe(""); + expect(context?.images).toEqual([]); + expect(ctx.Body).toBe("see attached"); + }); + + it("returns empty context when the steer carries no attachments", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const context = await renderInboundDocumentContext({ + ctx: { Body: "plain steer" } as MsgContext, + cfg: {} as OpenClawConfig, + }); + expect(context).toEqual({ text: "", images: [] }); + }); + + it("returns rendered PDF page images for a scanned document", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const mediaPath = await createTempMediaFile({ + fileName: "scan.pdf", + content: Buffer.from("%PDF-1.4\n", "utf8"), + }); + mockedExtractFileContentFromBuffer.mockResolvedValueOnce({ + text: "", + images: [ + { type: "image", data: "page-1-bytes", mimeType: "image/png" }, + { type: "image", data: "page-2-bytes", mimeType: "image/png" }, + ], + }); + const ctx: MsgContext = { + Body: "see attached", + media: [{ path: mediaPath, contentType: "application/pdf" }], + }; + + const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() }); + + // The marker alone would tell the model the document exists while the + // injected images channel carries nothing; the pages must ride along. + expect(context?.text).toContain("[PDF content rendered to images]"); + expect(context?.images).toEqual([ + { type: "image", data: "page-1-bytes", mimeType: "image/png", attachmentIndex: 0 }, + { type: "image", data: "page-2-bytes", mimeType: "image/png", attachmentIndex: 0 }, + ]); + }); + + it("applies the skipped-attachment marker budget to steer blocks", async () => { + const { renderInboundDocumentContext } = await import("./file-context.js"); + const olePayload = Buffer.from("Root Entry WordDocument legacy preview", "utf8"); + const media: { path: string; contentType: string }[] = []; + for (let i = 0; i < 7; i += 1) { + const filePath = await createTempMediaFile({ + fileName: `legacy-${i}.doc`, + content: olePayload, + }); + media.push({ path: filePath, contentType: "application/msword" }); + } + const ctx: MsgContext = { Body: "see attached", media }; + + const context = await renderInboundDocumentContext({ ctx, cfg: createMediaDisabledConfig() }); + + const markerCount = context?.text.split("[Unsupported document format").length ?? 0; + expect(markerCount - 1).toBe(5); + expect(context?.text).toContain("[2 more attachments skipped]"); + expect(context?.images).toEqual([]); + }); + }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/media-understanding/file-context.ts b/src/media-understanding/file-context.ts index 6796ea44a094..99f241ff93bf 100644 --- a/src/media-understanding/file-context.ts +++ b/src/media-understanding/file-context.ts @@ -300,33 +300,55 @@ export async function prepareFileContextFromMedia(params: { maxChars: number; assertCurrent: () => void; }) { - params.assertCurrent(); - const ctx: MsgContext = { - media: [...params.media], - Provider: params.channelId, - AccountId: params.accountId, - }; - const limits = resolveFileExtractionLimits(params.config); + return await renderInboundDocumentContext({ + ctx: { + media: [...params.media], + Provider: params.channelId, + AccountId: params.accountId, + }, + cfg: params.config, + workspaceDir: params.workspaceDir, + maxChars: params.maxChars, + assertCurrent: params.assertCurrent, + }); +} + +export type InboundDocumentContext = { text: string; images: ExtractedFileImage[] }; + +/** Keep prompt expansion separate from inbound state so rejected steers can dispatch normally. */ +export async function renderInboundDocumentContext(params: { + ctx: MsgContext; + cfg: OpenClawConfig; + workspaceDir?: string; + maxChars?: number; + assertCurrent?: () => void; +}): Promise { + params.assertCurrent?.(); + const { ctx, cfg } = params; + const limits = resolveFileExtractionLimits(cfg); const attachments = normalizeMediaAttachments(ctx); const cache = createMediaAttachmentCache(attachments, { localPathRoots: resolveMediaAttachmentLocalRoots({ - cfg: params.config, + cfg, ctx, workspaceDir: params.workspaceDir, }), - ssrfPolicy: params.config.tools?.web?.fetch?.ssrfPolicy, + ssrfPolicy: cfg.tools?.web?.fetch?.ssrfPolicy, workspaceDir: params.workspaceDir, }); try { const context = await extractFileContext({ attachments, cache, - cfg: params.config, - limits: { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) }, + cfg, + limits: + params.maxChars === undefined + ? limits + : { ...limits, maxChars: Math.min(limits.maxChars, params.maxChars) }, selfServePathsEnabled: false, assertCurrent: params.assertCurrent, }); - params.assertCurrent(); + params.assertCurrent?.(); return { text: applyAttachmentMarkerBudget(context.blocks).join("\n\n"), images: context.images, diff --git a/test/gateway-steer-fifo.e2e.test.ts b/test/gateway-steer-fifo.e2e.test.ts index e74b376ac22c..472cd9ee081d 100644 --- a/test/gateway-steer-fifo.e2e.test.ts +++ b/test/gateway-steer-fifo.e2e.test.ts @@ -988,4 +988,49 @@ describe("Gateway steer FIFO", () => { }, TEST_TIMEOUT_MS, ); + + it( + "hands steered document attachments to the active run as extracted file context", + async () => { + const fixture = await createGatewayFixture("steer-document-context"); + const first = await sendHeldTurn(fixture); + + // Real gateway protocol send with a file attachment while the initial + // run is still live: admission must steer the active run instead of + // dispatching a reply, and the extracted document context must reach + // the prompt the run actually sends to the model. + const steer = await fixture.diagnosticsClient.request<{ runId?: unknown; status?: unknown }>( + "chat.send", + { + sessionKey: fixture.sessionKey, + message: "QUEUED_STEER_DOCUMENT", + deliver: false, + queueMode: "steer", + idempotencyKey: "run-steer-document", + attachments: [ + { + type: "file", + mimeType: "text/plain", + fileName: "notes.txt", + content: Buffer.from("steered document body", "utf8").toString("base64"), + sizeBytes: "steered document body".length, + }, + ], + }, + ); + expect(steer).toMatchObject({ status: "started" }); + + fixture.modelServer.releaseFirst("final"); + + await vi.waitFor(() => expect(fixture.modelServer.requests).toHaveLength(2), WAIT_OPTS); + const currentSteer = currentUserInput(fixture.modelServer.requests[1]); + expect(currentSteer).toContain("QUEUED_STEER_DOCUMENT"); + // The media store persists uploads as `---`, so + // the extracted block carries the stored name, matching reply dispatch. + expect(currentSteer).toMatch(//); + expect(currentSteer).toContain("steered document body"); + await waitForRunTerminal(fixture, first.runId); + }, + TEST_TIMEOUT_MS, + ); });