diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d460def92..34e932ea3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -408,9 +408,41 @@ jobs: node -e "const fs = require('node:fs'); for (const key of ['HOME', 'USERPROFILE']) { const dir = process.env[key]; if (dir) fs.mkdirSync(dir, { recursive: true }); }" npm run test:ci - - name: 'Run no-AK integration smoke tests' - if: "${{ needs.classify_pr.outputs.skip_ci != 'true' && steps.ci_profile.outputs.ci_profile == 'full' && github.event_name == 'pull_request' }}" - run: 'npm run test:integration:no-ak:sandbox:none' + - name: 'Run required no-AK integration gate' + if: "${{ needs.classify_pr.outputs.skip_ci != 'true' && steps.ci_profile.outputs.ci_profile == 'full' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" + timeout-minutes: 20 + env: + HOME: '${{ runner.temp }}/qwen-no-ak-home' + USERPROFILE: '${{ runner.temp }}/qwen-no-ak-home' + QWEN_HOME: '${{ runner.temp }}/qwen-no-ak-home/.qwen' + API_KEY: '' + ANTHROPIC_API_KEY: '' + ANTHROPIC_BASE_URL: '' + ANTHROPIC_MODEL: '' + BAILIAN_CODING_PLAN_API_KEY: '' + BAILIAN_TOKEN_PLAN_API_KEY: '' + DEEPSEEK_API_KEY: '' + OPENAI_API_KEY: '' + OPENAI_BASE_URL: '' + OPENAI_MODEL: '' + DASHSCOPE_API_KEY: '' + GOOGLE_API_KEY: '' + GOOGLE_MODEL: '' + QWEN_API_KEY: '' + QWEN_MODEL: '' + GEMINI_API_KEY: '' + GEMINI_MODEL: '' + IDEALAB_API_KEY: '' + MINIMAX_API_KEY: '' + MODELSCOPE_API_KEY: '' + OPENROUTER_API_KEY: '' + REQUESTY_API_KEY: '' + XAI_API_KEY: '' + ZAI_API_KEY: '' + QWEN_DEFAULT_AUTH_TYPE: '' + run: |- + mkdir -p "${HOME}" "${QWEN_HOME}" + npm run test:integration:no-ak:sandbox:none - name: 'Publish Test Report (for non-forks)' if: |- diff --git a/integration-tests/sdk-typescript/permission-control.test.ts b/integration-tests/sdk-typescript/permission-control.test.ts index 25b0f7ff82..5fb1f9e54d 100644 --- a/integration-tests/sdk-typescript/permission-control.test.ts +++ b/integration-tests/sdk-typescript/permission-control.test.ts @@ -15,13 +15,9 @@ import { } from 'vitest'; import { query, - isSDKAssistantMessage, isSDKResultMessage, - isSDKUserMessage, type SDKMessage, type SDKUserMessage, - type ToolUseBlock, - type ContentBlock, } from '@qwen-code/sdk'; import { fakeToolCall, startFakeOpenAIServer } from '../fake-openai-server.js'; import { @@ -36,7 +32,6 @@ import { hasSuccessfulToolResults, hasErrorToolResults, findSystemMessage, - findToolCalls, createResultWaiter, } from './test-helper.js'; @@ -62,6 +57,22 @@ function fakeModelOptions(baseUrl: string) { }; } +function startFakeTextServer() { + return startFakeOpenAIServer( + () => ({ content: 'Done.' }), + FAKE_SERVER_OPTIONS, + ); +} + +function startFakeToolServer(toolName: string, input: Record) { + return startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { toolCalls: [fakeToolCall(toolName, input)] }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); +} + /** * Factory function that creates a streaming input with a control point. * After the first message is yielded, the generator waits for a resume signal, @@ -143,18 +154,26 @@ describe('Permission Control (E2E)', () => { }); describe('canUseTool callback parameter', () => { - it('should invoke canUseTool callback when tool is requested', async () => { + it('should invoke canUseTool callback and deny tool execution', async () => { const toolCalls: Array<{ toolName: string; input: Record; }> = []; + const fileName = 'denied.txt'; + const input = { + file_path: helper.getPath(fileName), + content: 'denied', + }; + const fakeServer = await startFakeToolServer('write_file', input); const q = query({ - prompt: 'Write a js hello world to file.', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'default', cwd: testDir, + coreTools: ['write_file'], canUseTool: async (toolName, input) => { toolCalls.push({ toolName, input }); return { @@ -166,37 +185,36 @@ describe('Permission Control (E2E)', () => { }); try { - let hasToolUse = false; + const messages: SDKMessage[] = []; for await (const message of q) { - if (isSDKAssistantMessage(message)) { - const toolUseBlock = message.message.content.find( - (block: ContentBlock): block is ToolUseBlock => - block.type === 'tool_use', - ); - if (toolUseBlock) { - hasToolUse = true; - } - } + messages.push(message); } - expect(hasToolUse).toBe(true); - expect(toolCalls.length).toBeGreaterThan(0); - expect(toolCalls[0].toolName).toBeDefined(); - expect(toolCalls[0].input).toBeDefined(); + expect(toolCalls).toEqual([{ toolName: 'write_file', input }]); + expect(hasErrorToolResults(messages)).toBe(true); + expect(helper.fileExists(fileName)).toBe(false); } finally { await q.close(); + await fakeServer.close(); } }); it('should allow tool execution when canUseTool returns allow', async () => { let callbackInvoked = false; + const fileName = 'hello.txt'; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath(fileName), + content: 'world', + }); const q = query({ - prompt: 'Create a file named hello.txt with content "world"', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'default', cwd: testDir, + coreTools: ['write_file'], canUseTool: async (toolName, input) => { callbackInvoked = true; return { @@ -208,68 +226,35 @@ describe('Permission Control (E2E)', () => { }); try { - let hasToolResult = false; + const messages: SDKMessage[] = []; for await (const message of q) { - if (isSDKUserMessage(message)) { - if ( - Array.isArray(message.message.content) && - message.message.content.some( - (block) => block.type === 'tool_result', - ) - ) { - hasToolResult = true; - } - } + messages.push(message); } expect(callbackInvoked).toBe(true); - expect(hasToolResult).toBe(true); - } finally { - await q.close(); - } - }); - - it('should deny tool execution when canUseTool returns deny', async () => { - let callbackInvoked = false; - - const q = query({ - prompt: 'Create a file named test.txt', - options: { - ...SHARED_TEST_OPTIONS, - cwd: testDir, - permissionMode: 'default', - canUseTool: async () => { - callbackInvoked = true; - return { - behavior: 'deny', - message: 'Tool execution denied by test', - }; - }, - }, - }); - - try { - for await (const _message of q) { - // Consume all messages - } - - expect(callbackInvoked).toBe(true); - // Tool use might still appear, but execution should be denied - // The exact behavior depends on CLI implementation + expect(hasSuccessfulToolResults(messages)).toBe(true); + await expect(helper.readFile(fileName)).resolves.toBe('world'); } finally { await q.close(); + await fakeServer.close(); } }); it('should pass suggestions to canUseTool callback', async () => { - let receivedSuggestions: unknown = null; + let receivedSuggestions: unknown; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath('data.txt'), + content: 'data', + }); const q = query({ - prompt: 'Create a file named data.txt', + prompt: 'Create data.txt.', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'default', cwd: testDir, + coreTools: ['write_file'], canUseTool: async (toolName, input, options) => { receivedSuggestions = options?.suggestions; return { @@ -282,13 +267,18 @@ describe('Permission Control (E2E)', () => { try { for await (const _message of q) { - // Consume all messages + // Consume all messages. } - // Suggestions may be null or an array, depending on CLI implementation - expect(receivedSuggestions !== undefined).toBe(true); + expect(receivedSuggestions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'allow' }), + expect.objectContaining({ type: 'deny' }), + ]), + ); } finally { await q.close(); + await fakeServer.close(); } }); @@ -299,19 +289,10 @@ describe('Permission Control (E2E)', () => { // write_file call. In default mode write_file requires permission, so // canUseTool always fires — a real model may answer in text and never // invoke the callback, which is the flake this guards against. - const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { - if (requestIndex === 0) { - return { - toolCalls: [ - fakeToolCall('write_file', { - file_path: `${testDir}/signal.txt`, - content: 'signal test', - }), - ], - }; - } - return { content: 'Done.' }; - }, FAKE_SERVER_OPTIONS); + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath('signal.txt'), + content: 'signal test', + }); const q = query({ prompt: 'Create a file named signal.txt', @@ -343,34 +324,11 @@ describe('Permission Control (E2E)', () => { await fakeServer.close(); } }); - - it('should default to deny when canUseTool is not provided', async () => { - const q = query({ - prompt: 'Create a file named default.txt', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'default', - cwd: testDir, - // canUseTool not provided - }, - }); - - try { - // When canUseTool is not provided, tools should be denied by default - // The exact behavior depends on CLI implementation - for await (const _message of q) { - // Consume all messages - } - // Test passes if no errors occur - expect(true).toBe(true); - } finally { - await q.close(); - } - }); }); describe('setPermissionMode API', () => { - it('should change permission mode from default to yolo', async () => { + it('should continue after changing permission mode from default to yolo', async () => { + const fakeServer = await startFakeTextServer(); const resultWaiter = createResultWaiter(2); const { generator, resume } = createStreamingInputWithControlPoint( 'What is 1 + 1?', @@ -382,6 +340,7 @@ describe('Permission Control (E2E)', () => { prompt: generator, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', debug: true, @@ -449,10 +408,12 @@ describe('Permission Control (E2E)', () => { expect(secondResponseReceived).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); - it('should change permission mode from yolo to plan', async () => { + it('should continue after changing permission mode from yolo to plan', async () => { + const fakeServer = await startFakeTextServer(); const resultWaiter = createResultWaiter(2); const { generator, resume } = createStreamingInputWithControlPoint( 'What is 3 + 3?', @@ -464,6 +425,7 @@ describe('Permission Control (E2E)', () => { prompt: generator, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'yolo', }, @@ -530,10 +492,12 @@ describe('Permission Control (E2E)', () => { expect(secondResponseReceived).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); - it('should change permission mode to auto-edit', async () => { + it('should continue after changing permission mode to auto-edit', async () => { + const fakeServer = await startFakeTextServer(); const resultWaiter = createResultWaiter(2); const { generator, resume } = createStreamingInputWithControlPoint( 'What is 5 + 5?', @@ -545,6 +509,7 @@ describe('Permission Control (E2E)', () => { prompt: generator, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'default', }, @@ -611,6 +576,7 @@ describe('Permission Control (E2E)', () => { expect(secondResponseReceived).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }); @@ -644,16 +610,41 @@ describe('Permission Control (E2E)', () => { }); describe('canUseTool and setPermissionMode integration', () => { - it('should work together - canUseTool callback with dynamic permission mode change', async () => { + it('should stop invoking canUseTool after changing to yolo mode', async () => { const toolCalls: Array<{ toolName: string; input: Record; }> = []; + const firstFile = 'first.txt'; + const secondFile = 'second.txt'; + const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { + if (requestIndex === 0) { + return { + toolCalls: [ + fakeToolCall('write_file', { + file_path: helper.getPath(firstFile), + content: 'first', + }), + ], + }; + } + if (requestIndex === 2) { + return { + toolCalls: [ + fakeToolCall('write_file', { + file_path: helper.getPath(secondFile), + content: 'second', + }), + ], + }; + } + return { content: 'Done.' }; + }, FAKE_SERVER_OPTIONS); const resultWaiter = createResultWaiter(2); const { generator, resume } = createStreamingInputWithControlPoint( - 'Create a file named first.txt', - 'Create a file named second.txt', + `Create ${firstFile}.`, + `Create ${secondFile}.`, resultWaiter, ); @@ -661,8 +652,10 @@ describe('Permission Control (E2E)', () => { prompt: generator, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'default', cwd: testDir, + coreTools: ['write_file'], canUseTool: async (toolName, input) => { toolCalls.push({ toolName, input }); return { @@ -714,7 +707,8 @@ describe('Permission Control (E2E)', () => { ]); expect(firstResponseReceived).toBe(true); - expect(toolCalls.length).toBeGreaterThan(0); + expect(toolCalls).toHaveLength(1); + await expect(helper.readFile(firstFile)).resolves.toBe('first'); await q.setPermissionMode('yolo'); @@ -731,8 +725,11 @@ describe('Permission Control (E2E)', () => { ]); expect(secondResponseReceived).toBe(true); + expect(toolCalls).toHaveLength(1); + await expect(helper.readFile(secondFile)).resolves.toBe('second'); } finally { await q.close(); + await fakeServer.close(); } }); }); @@ -742,14 +739,19 @@ describe('Permission Control (E2E)', () => { it( 'should auto-deny tools requiring confirmation without canUseTool callback', async () => { + const fileName = 'test-default-deny.txt'; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath(fileName), + content: 'hello', + }); const q = query({ - prompt: - 'Create a file named test-default-deny.txt with content "hello"', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'default', cwd: testDir, - // No canUseTool callback provided + coreTools: ['write_file'], }, }); @@ -759,48 +761,11 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - // In default mode without canUseTool, tools should be denied - expect(hasAnyToolResults(messages)).toBe(true); expect(hasErrorToolResults(messages)).toBe(true); + expect(helper.fileExists(fileName)).toBe(false); } finally { await q.close(); - } - }, - TEST_TIMEOUT, - ); - - it( - 'should allow tools when canUseTool returns allow', - async () => { - let callbackInvoked = false; - - const q = query({ - prompt: - 'Create a file named test-default-allow.txt with content "world"', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'default', - cwd: testDir, - canUseTool: async (toolName, input) => { - callbackInvoked = true; - return { - behavior: 'allow', - updatedInput: input, - }; - }, - }, - }); - - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - expect(callbackInvoked).toBe(true); - expect(hasSuccessfulToolResults(messages)).toBe(true); - } finally { - await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -809,50 +774,20 @@ describe('Permission Control (E2E)', () => { it( 'should execute read-only tools without confirmation', async () => { - // Create a file so the model has something to read - await helper.createFile( - 'read-only-test.txt', - 'content for read-only test', - ); - - const q = query({ - prompt: - 'Use the read_file tool to read the file read-only-test.txt in the current directory.', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'default', - cwd: testDir, - // No canUseTool callback - read-only tools should still work - }, + const fileName = 'read-only-test.txt'; + await helper.createFile(fileName, 'content for read-only test'); + const fakeServer = await startFakeToolServer('read_file', { + file_path: helper.getPath(fileName), }); - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - expect(hasAnyToolResults(messages)).toBe(true); - } finally { - await q.close(); - } - }, - TEST_TIMEOUT, - ); - }); - - describe('yolo mode', () => { - it( - 'should auto-approve all tools without canUseTool callback', - async () => { const q = query({ - prompt: - 'Create a file named test-yolo.txt with content "yolo mode"', + prompt: `Read ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, - permissionMode: 'yolo', + ...fakeModelOptions(fakeServer.baseUrl), + permissionMode: 'default', cwd: testDir, - // No canUseTool callback - tools should still execute + coreTools: ['read_file'], }, }); @@ -865,22 +800,32 @@ describe('Permission Control (E2E)', () => { expect(hasSuccessfulToolResults(messages)).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, ); + }); + describe('yolo mode', () => { it( - 'should not invoke canUseTool callback in yolo mode', + 'should auto-approve tools without invoking canUseTool callback', async () => { let callbackInvoked = false; + const fileName = 'test-yolo-no-callback.txt'; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath(fileName), + content: 'yolo', + }); const q = query({ - prompt: 'Create a file named test-yolo-no-callback.txt', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'yolo', cwd: testDir, + coreTools: ['write_file'], canUseTool: async (toolName, input) => { callbackInvoked = true; return { @@ -897,11 +842,12 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - expect(hasAnyToolResults(messages)).toBe(true); - // canUseTool should not be invoked in yolo mode + expect(hasSuccessfulToolResults(messages)).toBe(true); expect(callbackInvoked).toBe(false); + await expect(helper.readFile(fileName)).resolves.toBe('yolo'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -935,32 +881,15 @@ describe('Permission Control (E2E)', () => { }); describe('plan mode', () => { - // Write tools that should never be called in plan mode - const WRITE_TOOLS = [ - 'edit', - 'write_file', - 'run_shell_command', - 'delete_file', - 'move_file', - ]; - - // Read tools that should be allowed in plan mode - const READ_TOOLS = [ - 'read_file', - 'read_many_files', - 'grep_search', - 'glob', - 'list_directory', - 'web_fetch', - ]; - it( 'should have permission_mode set to plan in system message', async () => { + const fakeServer = await startFakeTextServer(); const q = query({ prompt: 'List files in the current directory', options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'plan', cwd: testDir, }, @@ -972,33 +901,38 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - // Find the init system message const systemMessage = findSystemMessage(messages, 'init'); expect(systemMessage).not.toBeNull(); expect(systemMessage!.permission_mode).toBe('plan'); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, ); it( - 'should not call any write tools in plan mode', + 'should block write tools in plan mode', async () => { - // Create a test file so the model has something to reference - await helper.createFile( - 'test-plan-file.txt', - 'This is test content for plan mode verification.', - ); + const fileName = 'test-plan-write.txt'; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath(fileName), + content: 'blocked', + }); const q = query({ - prompt: - 'Read the file test-plan-file.txt and suggest how to improve its content.', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'plan', cwd: testDir, + coreTools: ['write_file'], + canUseTool: async (_toolName, input) => ({ + behavior: 'allow', + updatedInput: input, + }), }, }); @@ -1008,88 +942,34 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - // Verify permission_mode is 'plan' - const systemMessage = findSystemMessage(messages, 'init'); - expect(systemMessage!.permission_mode).toBe('plan'); - - // Find all tool calls and verify none are write tools - const allToolCalls = findToolCalls(messages); - const writeToolCalls = allToolCalls.filter((tc) => - WRITE_TOOLS.includes(tc.toolUse.name), - ); - - // No write tools should be called in plan mode - expect(writeToolCalls.length).toBe(0); - } finally { - await q.close(); - } - }, - TEST_TIMEOUT, - ); - - // FIXME: This test is flaky and sometimes fails with no tool calls. - it.skip( - 'should allow read-only tools without restrictions', - async () => { - // Create test files for the model to read - await helper.createFile('test-read-1.txt', 'Content of file 1'); - await helper.createFile('test-read-2.txt', 'Content of file 2'); - - const q = query({ - prompt: - 'Read the contents of test-read-1.txt and test-read-2.txt files, then list files in the current directory.', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'plan', - cwd: testDir, - }, - }); - - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - // Verify permission_mode is 'plan' - const systemMessage = findSystemMessage(messages, 'init'); - expect(systemMessage!.permission_mode).toBe('plan'); - - // Find all tool calls - const allToolCalls = findToolCalls(messages); - - // Verify read tools were called (at least one) - const readToolCalls = allToolCalls.filter((tc) => - READ_TOOLS.includes(tc.toolUse.name), - ); - expect(readToolCalls.length).toBeGreaterThan(0); - - // Verify tool results are successful (not blocked) - expect(hasSuccessfulToolResults(messages)).toBe(true); + expect(hasErrorToolResults(messages)).toBe(true); + expect(helper.fileExists(fileName)).toBe(false); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, ); it( - 'should not invoke canUseTool callback in plan mode since no permission approval is expected', + 'should allow read tools without invoking canUseTool callback', async () => { let callbackInvoked = false; - - // Create a test file for reading - await helper.createFile( - 'test-plan-callback.txt', - 'Content for callback test', - ); + const fileName = 'test-plan-read.txt'; + await helper.createFile(fileName, 'plan read'); + const fakeServer = await startFakeToolServer('read_file', { + file_path: helper.getPath(fileName), + }); const q = query({ - prompt: 'Read the file test-plan-callback.txt', + prompt: `Read ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), permissionMode: 'plan', cwd: testDir, + coreTools: ['read_file'], canUseTool: async (toolName, input) => { callbackInvoked = true; return { @@ -1106,63 +986,11 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - // Verify permission_mode is 'plan' - const systemMessage = findSystemMessage(messages, 'init'); - expect(systemMessage!.permission_mode).toBe('plan'); - - // Read tools should work without invoking canUseTool - // In plan mode, no permission approval is expected from user expect(hasSuccessfulToolResults(messages)).toBe(true); - - // canUseTool should not be invoked in plan mode - // since plan mode is for research only, no permission interaction needed expect(callbackInvoked).toBe(false); } finally { await q.close(); - } - }, - TEST_TIMEOUT, - ); - - it( - 'should only output research and plan as text, no actual changes', - async () => { - // Create a test file - const originalContent = 'Original content for plan mode test'; - await helper.createFile('test-no-changes.txt', originalContent); - - const q = query({ - prompt: - 'Read test-no-changes.txt and plan how you would modify it to add a header. Do not actually make any changes.', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'plan', - cwd: testDir, - }, - }); - - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - // Verify permission_mode is 'plan' - const systemMessage = findSystemMessage(messages, 'init'); - expect(systemMessage!.permission_mode).toBe('plan'); - - // Verify the file was not modified - const fileContent = await helper.readFile('test-no-changes.txt'); - expect(fileContent).toBe(originalContent); - - // Verify no write tools were called - const allToolCalls = findToolCalls(messages); - const writeToolCalls = allToolCalls.filter((tc) => - WRITE_TOOLS.includes(tc.toolUse.name), - ); - expect(writeToolCalls.length).toBe(0); - } finally { - await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, @@ -1171,54 +999,17 @@ describe('Permission Control (E2E)', () => { describe('auto-edit mode', () => { it( - 'should auto-approve write/edit tools without canUseTool callback', - async () => { - const q = query({ - prompt: - 'Create a file named test-auto-edit.txt with content "auto-edit test"', - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: 'auto-edit', - cwd: testDir, - // No canUseTool callback - write/edit tools should still execute - }, - }); - - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - // auto-edit mode should auto-approve write/edit tools - expect(hasSuccessfulToolResults(messages)).toBe(true); - } finally { - await q.close(); - } - }, - TEST_TIMEOUT, - ); - - it( - 'should not invoke canUseTool callback for write/edit tools', + 'should auto-approve write tools without invoking canUseTool callback', async () => { let callbackInvoked = false; - const fakeServer = await startFakeOpenAIServer(({ requestIndex }) => { - if (requestIndex === 0) { - return { - toolCalls: [ - fakeToolCall('write_file', { - file_path: `${testDir}/test-auto-edit-no-callback.txt`, - content: 'auto-edit callback test', - }), - ], - }; - } - return { content: 'Done.' }; - }, FAKE_SERVER_OPTIONS); + const fileName = 'test-auto-edit-no-callback.txt'; + const fakeServer = await startFakeToolServer('write_file', { + file_path: helper.getPath(fileName), + content: 'auto-edit callback test', + }); const q = query({ - prompt: 'Create test-auto-edit-no-callback.txt.', + prompt: `Create ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, ...fakeModelOptions(fakeServer.baseUrl), @@ -1241,9 +1032,11 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - // auto-edit mode should auto-approve write/edit tools without invoking callback expect(hasSuccessfulToolResults(messages)).toBe(true); expect(callbackInvoked).toBe(false); + await expect(helper.readFile(fileName)).resolves.toBe( + 'auto-edit callback test', + ); } finally { await q.close(); await fakeServer.close(); @@ -1255,19 +1048,23 @@ describe('Permission Control (E2E)', () => { it( 'should execute read-only tools without confirmation', async () => { - // Create a test file in the test directory for the model to read + const fileName = 'test-read-file.txt'; await helper.createFile( - 'test-read-file.txt', + fileName, 'This is a test file for read-only tool verification.', ); + const fakeServer = await startFakeToolServer('read_file', { + file_path: helper.getPath(fileName), + }); const q = query({ - prompt: 'Read the contents of test-read-file.txt file', + prompt: `Read ${fileName}.`, options: { ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), cwd: testDir, permissionMode: 'auto-edit', - // No canUseTool callback - read-only tools should still work + coreTools: ['read_file'], }, }); @@ -1277,65 +1074,14 @@ describe('Permission Control (E2E)', () => { messages.push(message); } - expect(hasAnyToolResults(messages)).toBe(true); + expect(hasSuccessfulToolResults(messages)).toBe(true); } finally { await q.close(); + await fakeServer.close(); } }, TEST_TIMEOUT, ); }); - - describe('mode comparison tests', () => { - it.skip( - 'should demonstrate different behaviors across all modes for write operations', - async () => { - const modes: Array<'default' | 'auto-edit' | 'yolo'> = [ - 'default', - 'auto-edit', - 'yolo', - ]; - const results: Record = {}; - - for (const mode of modes) { - const q = query({ - prompt: `Create a file named test-${mode}.txt`, - options: { - ...SHARED_TEST_OPTIONS, - permissionMode: mode, - cwd: testDir, - canUseTool: - mode === 'yolo' || mode === 'auto-edit' - ? undefined - : async (toolName, input) => { - return { - behavior: 'allow', - updatedInput: input, - }; - }, - }, - }); - - try { - const messages: SDKMessage[] = []; - for await (const message of q) { - messages.push(message); - } - - results[mode] = hasSuccessfulToolResults(messages); - } finally { - await q.close(); - } - } - - // Verify expected behaviors - expect(results['default']).toBe(true); // Allowed via canUseTool - // expect(results['plan']).toBe(false); // Blocked by plan mode - expect(results['auto-edit']).toBe(true); // Auto-approved for write/edit tools - expect(results['yolo']).toBe(true); // Auto-approved for all tools - }, - TEST_TIMEOUT * 4, - ); - }); }); }); diff --git a/integration-tests/sdk-typescript/system-control.test.ts b/integration-tests/sdk-typescript/system-control.test.ts index 41ff86bf4e..5ed8803388 100644 --- a/integration-tests/sdk-typescript/system-control.test.ts +++ b/integration-tests/sdk-typescript/system-control.test.ts @@ -203,6 +203,31 @@ describe('System Control (E2E)', () => { await fakeServer.close(); } }); + + it('should throw error when setModel is called on closed query', async () => { + const fakeServer = await startFakeOpenAIServer( + () => ({ content: 'Done.' }), + FAKE_SERVER_OPTIONS, + ); + try { + const q = query({ + prompt: 'Hello', + options: { + ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), + cwd: testDir, + }, + }); + + await q.close(); + + await expect(q.setModel('fake-model-2')).rejects.toThrow( + 'Query is closed', + ); + } finally { + await fakeServer.close(); + } + }); }); describe('supportedCommands API', () => { @@ -284,5 +309,28 @@ describe('System Control (E2E)', () => { await fakeServer.close(); } }); + + it('should throw error when supportedCommands is called on closed query', async () => { + const fakeServer = await startFakeOpenAIServer( + () => ({ content: 'Done.' }), + FAKE_SERVER_OPTIONS, + ); + try { + const q = query({ + prompt: 'Hello', + options: { + ...SHARED_TEST_OPTIONS, + ...fakeModelOptions(fakeServer.baseUrl), + cwd: testDir, + }, + }); + + await q.close(); + + await expect(q.supportedCommands()).rejects.toThrow('Query is closed'); + } finally { + await fakeServer.close(); + } + }); }); }); diff --git a/package.json b/package.json index 7904e9958f..8a903bdaeb 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "test:integration:sandbox:none": "cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests", "test:integration:sandbox:docker": "cross-env QWEN_SANDBOX=docker npm run build:sandbox && QWEN_SANDBOX=docker vitest run --root ./integration-tests", "test:integration:sandbox:podman": "cross-env QWEN_SANDBOX=podman vitest run --root ./integration-tests", - "test:integration:no-ak:sandbox:none": "cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests ./fake-openai-server.test.ts ./cli/qwen-serve-routes.test.ts ./cli/qwen-serve-streaming.test.ts ./cli/daemon-invocation-context.test.ts", + "test:integration:no-ak:sandbox:none": "cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests --maxWorkers 2 ./fake-openai-server.test.ts ./cli/daemon-invocation-context.test.ts ./cli/list_directory.test.ts ./cli/qwen-serve-routes.test.ts ./cli/qwen-serve-streaming.test.ts ./sdk-typescript/abort-and-lifecycle.test.ts ./sdk-typescript/permission-control.test.ts ./sdk-typescript/sdk-mcp-server.test.ts ./sdk-typescript/subagents.test.ts ./sdk-typescript/system-control.test.ts ./sdk-typescript/tool-control.test.ts", "test:integration:sdk:sandbox:none": "cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests --poolOptions.threads.maxThreads 2 sdk-typescript", "test:integration:sdk:sandbox:docker": "cross-env QWEN_SANDBOX=docker npm run build:sandbox && QWEN_SANDBOX=docker vitest run --root ./integration-tests --poolOptions.threads.maxThreads 2 sdk-typescript", "test:sdk:python": "python3 -m pytest -c packages/sdk-python/pyproject.toml packages/sdk-python/tests -q", diff --git a/packages/core/src/providers/all-providers.ts b/packages/core/src/providers/all-providers.ts index 6ca881e819..aa5aad3dba 100644 --- a/packages/core/src/providers/all-providers.ts +++ b/packages/core/src/providers/all-providers.ts @@ -47,6 +47,9 @@ export { // --------------------------------------------------------------------------- /** All known providers, in display order. */ +// Adding a provider that reads credentials from a new env key? Also add that +// key to the no-AK gate's cleared-env list in .github/workflows/ci.yml and to +// scripts/tests/no-ak-integration-ci.test.js, or the gate can leak runner creds. export const ALL_PROVIDERS: readonly ProviderConfig[] = [ codingPlanProvider, tokenPlanProvider, diff --git a/scripts/tests/no-ak-integration-ci.test.js b/scripts/tests/no-ak-integration-ci.test.js index 8e7d6bba68..843785aaf3 100644 --- a/scripts/tests/no-ak-integration-ci.test.js +++ b/scripts/tests/no-ak-integration-ci.test.js @@ -33,16 +33,23 @@ describe('no-AK integration CI wiring', () => { expect(packageJson.scripts[NO_AK_SCRIPT]).toBe( [ - 'cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests', + 'cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests --maxWorkers 2', './fake-openai-server.test.ts', + './cli/daemon-invocation-context.test.ts', + './cli/list_directory.test.ts', './cli/qwen-serve-routes.test.ts', './cli/qwen-serve-streaming.test.ts', - './cli/daemon-invocation-context.test.ts', + './sdk-typescript/abort-and-lifecycle.test.ts', + './sdk-typescript/permission-control.test.ts', + './sdk-typescript/sdk-mcp-server.test.ts', + './sdk-typescript/subagents.test.ts', + './sdk-typescript/system-control.test.ts', + './sdk-typescript/tool-control.test.ts', ].join(' '), ); }); - it('runs the no-AK integration script in the Ubuntu gate only', () => { + it('runs the no-AK integration script in the required Linux gate only', () => { const workflow = readFileSync( path.join(ROOT, '.github/workflows/ci.yml'), 'utf8', @@ -50,13 +57,68 @@ describe('no-AK integration CI wiring', () => { const ubuntuJob = getWorkflowJob(workflow, 'test'); const macosJob = getWorkflowJob(workflow, 'test_macos'); const windowsJob = getWorkflowJob(workflow, 'test_windows'); + const permissionsIndex = workflow.indexOf('\npermissions:'); + expect(permissionsIndex).toBeGreaterThan(0); + const workflowTriggers = workflow.slice(0, permissionsIndex); + const gateStepMarker = + " - name: 'Run required no-AK integration gate'"; + const gateStepStart = ubuntuJob.indexOf(gateStepMarker); + expect(gateStepStart).toBeGreaterThanOrEqual(0); + const nextStepIndex = ubuntuJob.indexOf( + '\n - name:', + gateStepStart + gateStepMarker.length, + ); + expect(nextStepIndex).toBeGreaterThan(0); + const gateStep = ubuntuJob.slice(gateStepStart, nextStepIndex); expect(workflow).not.toContain(' integration_no_ak:'); expect(workflow.split(`npm run ${NO_AK_SCRIPT}`).length - 1).toBe(1); + expect(workflowTriggers).toContain('\n pull_request:\n'); + expect(workflowTriggers).toContain('\n merge_group:\n'); - expect(ubuntuJob).toContain("name: 'Run no-AK integration smoke tests'"); - expect(ubuntuJob).toContain("github.event_name == 'pull_request'"); - expect(ubuntuJob).toContain(`npm run ${NO_AK_SCRIPT}`); + expect(gateStep).toContain( + "(github.event_name == 'pull_request' || github.event_name == 'merge_group')", + ); + expect(gateStep).toContain(`npm run ${NO_AK_SCRIPT}`); + expect(gateStep).toContain( + "QWEN_HOME: '${{ runner.temp }}/qwen-no-ak-home/.qwen'", + ); + expect(gateStep).toContain('timeout-minutes: 20'); + expect(gateStep).toContain( + "\n HOME: '${{ runner.temp }}/qwen-no-ak-home'", + ); + expect(gateStep).toContain( + "\n USERPROFILE: '${{ runner.temp }}/qwen-no-ak-home'", + ); + for (const key of [ + 'API_KEY', + 'ANTHROPIC_API_KEY', + 'ANTHROPIC_BASE_URL', + 'ANTHROPIC_MODEL', + 'BAILIAN_CODING_PLAN_API_KEY', + 'BAILIAN_TOKEN_PLAN_API_KEY', + 'DASHSCOPE_API_KEY', + 'DEEPSEEK_API_KEY', + 'GEMINI_API_KEY', + 'GEMINI_MODEL', + 'GOOGLE_API_KEY', + 'GOOGLE_MODEL', + 'IDEALAB_API_KEY', + 'MINIMAX_API_KEY', + 'MODELSCOPE_API_KEY', + 'OPENAI_API_KEY', + 'OPENAI_BASE_URL', + 'OPENAI_MODEL', + 'OPENROUTER_API_KEY', + 'QWEN_API_KEY', + 'QWEN_DEFAULT_AUTH_TYPE', + 'QWEN_MODEL', + 'REQUESTY_API_KEY', + 'XAI_API_KEY', + 'ZAI_API_KEY', + ]) { + expect(gateStep).toContain(`\n ${key}: ''`); + } expect(ubuntuJob).not.toContain('secrets.OPENAI_API_KEY'); expect(ubuntuJob).not.toContain('secrets.OPENAI_BASE_URL'); expect(ubuntuJob).not.toContain('secrets.OPENAI_MODEL');