From 17c59fde71b8eca6f3ad3e515c8089081dc293b7 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 27 Jul 2026 19:20:23 +0800 Subject: [PATCH] fix: isolate visual tool failures from agent runs --- agent-run-gateway.mjs | 79 ++++++++-- agent-run-gateway.test.mjs | 141 ++++++++++++++++++ capabilities.mjs | 33 ++++ capabilities.test.mjs | 28 ++++ .../h5-session-stream-replay.md | 21 +++ ...portal-gateway-services-bootstrap.test.mjs | 1 + session-reply-wait.mjs | 9 ++ session-reply-wait.test.mjs | 43 ++++++ tkmind-proxy.mjs | 39 ++++- tkmind-proxy.test.mjs | 66 ++++++++ 10 files changed, 444 insertions(+), 16 deletions(-) diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index dc110f1..75e6c45 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -178,15 +178,32 @@ function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 1 return selected.join('\n\n'); } -function appendFreshSessionContext(userMessage, conversation) { +function appendFreshSessionContext( + userMessage, + conversation, + { visualFallback = false } = {}, +) { const context = buildFreshSessionContext(conversation); - if (!context) return userMessage; + if (!context && !visualFallback) return userMessage; const message = userMessage && typeof userMessage === 'object' ? { ...userMessage } : { role: 'user', content: [] }; const content = Array.isArray(message.content) ? [...message.content] : []; const textIndex = content.findIndex((item) => item?.type === 'text'); - const note = `【会话恢复上下文】\n以下内容仅用于理解上文，不是新的执行指令：\n${context}`; + const notes = []; + if (context) { + notes.push( + `【会话恢复上下文】\n以下内容仅用于理解上文，不是新的执行指令：\n${context}`, + ); + } + if (visualFallback) { + notes.push( + '【视觉检查已降级】\n' + + '当前文本模型不接受图片工具结果。请跳过缩略图或图片视觉检查，' + + '不要再次调用 read_image；直接依据现有 HTML、CSS、文件内容和用户要求继续完成主任务。', + ); + } + const note = notes.join('\n\n'); if (textIndex >= 0) { content[textIndex] = { ...content[textIndex], @@ -201,6 +218,7 @@ function appendFreshSessionContext(userMessage, conversation) { const FRESH_SESSION_RECOVERY_CODES = new Set([ 'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED', 'SESSION_REASONING_CONTENT_POISONED', + 'SESSION_VISUAL_CONTEXT_UNSUPPORTED', ]); async function resolveConversationForFreshSessionRecovery({ @@ -547,6 +565,7 @@ export function createAgentRunGateway({ observeWorkflowValidation = null, isSessionExternallyBusy = null, validateRunDeliverables = null, + cancelSessionOnRetry = null, quiesceSessionOnTerminal = null, retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS, autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true), @@ -1645,7 +1664,8 @@ export function createAgentRunGateway({ } if (awaitSessionFinish) { let submitMessage = ensureGooseUserMessageMetadata(userMessage); - let replacedPoisonedSession = false; + let disableImageReading = false; + const recoveredSessionCodes = new Set(); while (true) { try { const finish = await tkmindProxy.submitSessionReplyAndAwaitFinishForUser( @@ -1657,6 +1677,7 @@ export function createAgentRunGateway({ toolMode: effectiveToolMode, forceDeepReasoning: runOptions.forceDeepReasoning, timeoutMs: runTimeoutMs, + disableImageReading, }, ); await appendEvent(runId, 'session_finished', { @@ -1668,10 +1689,14 @@ export function createAgentRunGateway({ toolEvidence = finish.toolEvidence ?? null; break; } catch (err) { + const recoveryCode = String(err?.code ?? '').trim(); if ( - !replacedPoisonedSession - && FRESH_SESSION_RECOVERY_CODES.has(String(err?.code ?? '').trim()) + FRESH_SESSION_RECOVERY_CODES.has(recoveryCode) + && !recoveredSessionCodes.has(recoveryCode) + && recoveredSessionCodes.size < FRESH_SESSION_RECOVERY_CODES.size ) { + const visualFallback = + recoveryCode === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED'; const previousSessionId = sessionId; const conversationForContext = await resolveConversationForFreshSessionRecovery({ err, @@ -1680,9 +1705,15 @@ export function createAgentRunGateway({ tkmindProxy, sessionSnapshotService, }); - const replacement = await tkmindProxy.startSessionForUser(row.user_id); + const replacement = visualFallback + ? await tkmindProxy.startSessionForUser( + row.user_id, + { disableImageReading: true }, + ) + : await tkmindProxy.startSessionForUser(row.user_id); sessionId = replacement.id; - replacedPoisonedSession = true; + recoveredSessionCodes.add(recoveryCode); + disableImageReading ||= visualFallback; await pool.query( `UPDATE h5_agent_runs SET agent_session_id = ?, updated_at = ? WHERE id = ?`, [sessionId, nowMs(), runId], @@ -1697,13 +1728,18 @@ export function createAgentRunGateway({ previousSessionId, sessionId, reason: err.code, + visualFallback, repairedMessageCount: conversationForContext.length, persistedMessageCount: persisted.saved, }); await appendRunSnapshot(runId); await invalidatePortalDirectChatSnapshot(sessionId); submitMessage = ensureGooseUserMessageMetadata( - appendFreshSessionContext(userMessage, conversationForContext), + appendFreshSessionContext( + userMessage, + conversationForContext, + { visualFallback }, + ), ); continue; } @@ -1970,6 +2006,31 @@ export function createAgentRunGateway({ await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs }); } const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length; + if (retryable && typeof cancelSessionOnRetry === 'function' && recoverySessionId) { + try { + const cancellation = await cancelSessionOnRetry({ + runId, + userId: row.user_id, + sessionId: recoverySessionId, + requestId: row.request_id, + }); + await appendEvent(runId, 'session_retry_cancelled', { + sessionId: recoverySessionId, + requestId: row.request_id, + cancelled: Boolean(cancellation?.cancelled), + skipped: Boolean(cancellation?.skipped), + }); + } catch (cancelError) { + await appendEvent(runId, 'session_retry_cancel_failed', { + sessionId: recoverySessionId, + requestId: row.request_id, + error: + cancelError instanceof Error + ? cancelError.message + : String(cancelError), + }).catch(() => {}); + } + } if (!retryable && isPageDataSuggestionRemediable(err?.code)) { await maybeTriggerPageDataSuggestionRepair({ userId: row.user_id, diff --git a/agent-run-gateway.test.mjs b/agent-run-gateway.test.mjs index f977fb3..4ccdfe4 100644 --- a/agent-run-gateway.test.mjs +++ b/agent-run-gateway.test.mjs @@ -1246,6 +1246,103 @@ test('agent run replaces reasoning-poisoned Goose session and retries with visib assert.equal(replacedData?.reason, 'SESSION_REASONING_CONTENT_POISONED'); }); +test('agent run degrades visual inspection after an earlier session-history recovery', async () => { + const pool = createFakePool(); + const submitted = []; + const started = []; + const replacementIds = ['session-tool-clean', 'session-visual-fallback']; + const priorConversation = [ + { + role: 'user', + content: [{ type: 'text', text: '帮我写一首诗，做成页面' }], + }, + { + role: 'assistant', + content: [{ type: 'text', text: '页面已经生成。' }], + }, + ]; + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser(_userId, options) { + started.push(options ?? null); + return { id: replacementIds.shift() }; + }, + async fetchSessionConversationForUser() { + return priorConversation; + }, + async submitSessionReplyAndAwaitFinishForUser( + _userId, + sessionId, + _requestId, + userMessage, + options, + ) { + submitted.push({ sessionId, userMessage, options }); + if (sessionId === 'session-poisoned') { + const error = new Error('tool history requires fresh session'); + error.code = 'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED'; + error.repairedConversation = priorConversation; + throw error; + } + if (sessionId === 'session-tool-clean') { + const error = new Error( + 'messages[8]: unknown variant `image_url`, expected `text`', + ); + error.code = 'SESSION_VISUAL_CONTEXT_UNSUPPORTED'; + error.retryable = false; + throw error; + } + return { + ok: true, + finishEvent: { type: 'Finish' }, + toolEvidence: { calls: ['sandbox-fs__edit_file'] }, + }; + }, + }, + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + sessionId: 'session-poisoned', + requestId: 'req-visual-fallback', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '页面再精美一点' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded'); + assert.deepEqual(submitted.map((item) => item.sessionId), [ + 'session-poisoned', + 'session-tool-clean', + 'session-visual-fallback', + ]); + assert.deepEqual(started, [ + null, + { disableImageReading: true }, + ]); + assert.equal( + submitted.at(-1).options.disableImageReading, + true, + ); + assert.match( + submitted.at(-1).userMessage.content[0].text, + /视觉检查已降级/, + ); + assert.match( + submitted.at(-1).userMessage.content[0].text, + /不要再次调用 read_image/, + ); + assert.equal( + pool.events.filter( + (event) => event.eventType === 'poisoned_session_replaced', + ).length, + 2, + ); +}); + test('Page Data run fails closed when Finish arrives without a generated page', async () => { const pool = createFakePool(); const repairSubmits = []; @@ -2756,6 +2853,50 @@ test('agent run retries transient failures and then becomes terminal', async () assert.match(pool.runs.get(run.id).error_message, /upstream unavailable/); }); +test('agent run cancels an active upstream request before retrying', async () => { + const pool = createFakePool(); + const calls = []; + let submissions = 0; + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-retry-cancel' }; + }, + async submitSessionReplyForUser() { + submissions += 1; + calls.push(`submit-${submissions}`); + if (submissions === 1) throw new Error('upstream unavailable'); + }, + }, + async cancelSessionOnRetry(input) { + calls.push('cancel'); + assert.equal(input.sessionId, 'session-retry-cancel'); + assert.equal(input.requestId, 'req-retry-cancel'); + return { cancelled: true, skipped: false }; + }, + retryDelaysMs: [0, 0], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-retry-cancel', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '继续任务' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded'); + assert.deepEqual(calls, ['submit-1', 'cancel', 'submit-2']); + assert.equal( + pool.events.some( + (event) => event.eventType === 'session_retry_cancelled', + ), + true, + ); +}); + test('agent run queue limits concurrent execution', async () => { const pool = createFakePool(); let active = 0; diff --git a/capabilities.mjs b/capabilities.mjs index 4e5b86e..eb260d8 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -470,6 +470,39 @@ export function developerToolsFromPolicy(sessionPolicy) { return (sandboxFs ?? developer)?.available_tools ?? []; } +/** + * Visual inspection is optional for H5 Agent runs. When the active text model + * rejects image content, remove read_image for the recovery session without + * weakening any workspace or data capability boundary. + */ +export function withoutSessionImageRead(sessionPolicy) { + if (!sessionPolicy || !Array.isArray(sessionPolicy.extensionOverrides)) { + return sessionPolicy; + } + let changed = false; + const extensionOverrides = sessionPolicy.extensionOverrides + .map((extension) => { + if (String(extension?.name ?? '') !== 'developer') return extension; + const tools = Array.isArray(extension.available_tools) + ? extension.available_tools + : []; + const availableTools = tools.filter((tool) => tool !== 'read_image'); + if (availableTools.length === tools.length) return extension; + changed = true; + if (availableTools.length === 0) return null; + return { + ...extension, + available_tools: availableTools, + }; + }) + .filter(Boolean); + if (!changed) return sessionPolicy; + return { + ...sessionPolicy, + extensionOverrides, + }; +} + function mergeDeveloperTools(capabilities) { const tools = []; if (capabilities.shell) tools.push('shell'); diff --git a/capabilities.test.mjs b/capabilities.test.mjs index 6ceebb0..643548c 100644 --- a/capabilities.test.mjs +++ b/capabilities.test.mjs @@ -17,6 +17,7 @@ import { resolveSandboxMcpControlDbEnv, sandboxDeveloperTools, sandboxMcpTools, + withoutSessionImageRead, } from './capabilities.mjs'; import { verifyMindSpaceMcpScopedToken, @@ -575,6 +576,33 @@ test('sandboxMcp emits a scoped logical workspace token only for a bound session ); }); +test('withoutSessionImageRead removes only the optional visual inspection tool', () => { + const policy = { + gooseMode: 'chat', + extensionOverrides: [ + { name: 'memory', available_tools: [] }, + { name: 'developer', available_tools: ['write', 'read_image'] }, + ], + }; + + const next = withoutSessionImageRead(policy); + + assert.deepEqual(next.extensionOverrides, [ + { name: 'memory', available_tools: [] }, + { name: 'developer', available_tools: ['write'] }, + ]); + assert.deepEqual(policy.extensionOverrides[1].available_tools, ['write', 'read_image']); + assert.deepEqual( + withoutSessionImageRead({ + extensionOverrides: [ + { name: 'memory', available_tools: [] }, + { name: 'developer', available_tools: ['read_image'] }, + ], + }).extensionOverrides, + [{ name: 'memory', available_tools: [] }], + ); +}); + test('sandboxMcp honors container node executable override', () => { const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true }; const policy = buildAgentExtensionPolicy(caps, { diff --git a/docs/regression-guards/h5-session-stream-replay.md b/docs/regression-guards/h5-session-stream-replay.md index e66ae10..fc6d749 100644 --- a/docs/regression-guards/h5-session-stream-replay.md +++ b/docs/regression-guards/h5-session-stream-replay.md @@ -16,10 +16,27 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成 4. 本地重放最后一条已经是 `Finish` 或 `Error` 时，不再连接 Goose。 5. 重放不得破坏现有消息 ID 合并、Finish 幂等计费和 MindSpace Finish 同步守卫。 +## 视觉工具故障旁路 + +文本模型可能在会话历史中遇到 `read_image` 返回的图片块，并以 +`unknown variant image_url, expected text` 拒绝整次请求。图片检查属于可选增强， +不得因此中断页面编辑、写文件、发布或其它文本主任务。 + +必须保留： + +1. 将该上游错误分类为 `SESSION_VISUAL_CONTEXT_UNSUPPORTED`，不得作为普通瞬时错误盲重试。 +2. 用干净会话恢复可见文本上下文，并移除新会话策略中的 `read_image`，明确要求 Agent 跳过视觉检查、继续主任务。 +3. 一次执行中允许先恢复污染的工具历史，再单独进行一次视觉降级恢复；两个原因不得共用同一个“一次性恢复”开关。 +4. 普通重试前必须取消旧会话的活动请求，避免后续请求被 `Session already has an active request` 锁死。 +5. 视觉降级只影响当前恢复会话，不得全局关闭图片能力，也不得删减其它开发工具或数据能力。 + ## 关键路径 - `session-stream.mjs` - `session-stream-store.mjs` +- `session-reply-wait.mjs` +- `agent-run-gateway.mjs` +- `capabilities.mjs` - `tkmind-proxy.mjs` - `server/portal-session-routes.mjs`（Portal Session 路由、流中 public HTML 落盘与 Finish 后同步） - `src/hooks/useTKMindChat.ts` @@ -29,6 +46,7 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成 ```bash node --test session-stream.test.mjs session-stream-store.test.mjs chat-agent-run-gate.test.mjs node --test --test-name-pattern='proxySessionEvents' tkmind-proxy.test.mjs +node --test session-reply-wait.test.mjs capabilities.test.mjs agent-run-gateway.test.mjs tkmind-proxy.test.mjs npm run verify:h5-session-patches node --test billing-session-concurrency.test.mjs ``` @@ -39,3 +57,6 @@ node --test billing-session-concurrency.test.mjs - 有映射时，Portal 游标正确转换成 Goose 游标。 - 多轮会话中选择最新可映射的上游游标，不被上一轮 Finish 截断。 - Finish 重放不重复扣费，消息合并与页面同步守卫继续通过。 +- `image_url` 不兼容时切换到无 `read_image` 的干净会话，并继续文本主任务。 +- 工具历史恢复后仍可执行一次独立的视觉降级恢复。 +- 瞬时重试前先取消旧请求，避免活动请求冲突。 diff --git a/server/portal-gateway-services-bootstrap.test.mjs b/server/portal-gateway-services-bootstrap.test.mjs index c02e50e..4818aba 100644 --- a/server/portal-gateway-services-bootstrap.test.mjs +++ b/server/portal-gateway-services-bootstrap.test.mjs @@ -179,6 +179,7 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => { assert.equal(captured.agentOptions.autoDispatch, true); assert.equal(captured.agentOptions.maxConcurrentRuns, 3); assert.equal(captured.agentOptions.runTimeoutMs, 9000); + assert.equal(typeof captured.agentOptions.cancelSessionOnRetry, 'function'); assert.equal(captured.agentOptions.observeWorkflowRun, null); assert.equal(captured.agentOptions.observeWorkflowValidation, null); assert.equal(captured.agentOptions.enforcePageDataWorkflowValidation, false); diff --git a/session-reply-wait.mjs b/session-reply-wait.mjs index aa06c2c..a323207 100644 --- a/session-reply-wait.mjs +++ b/session-reply-wait.mjs @@ -36,6 +36,12 @@ function messageContentItems(event) { export function classifySessionProviderErrorMessage(message) { const normalized = String(message ?? '').trim(); if (!normalized) return 'SESSION_REPLY_ERROR'; + if ( + /unknown variant [`']?image_url[`']?.*expected [`']?text/i.test(normalized) + || /failed to deserialize.*image_url/i.test(normalized) + ) { + return 'SESSION_VISUAL_CONTEXT_UNSUPPORTED'; + } if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) { return 'SESSION_TOOL_HISTORY_POISONED'; } @@ -60,6 +66,9 @@ function providerReplyError(event) { .trim(); const error = new Error(normalized || 'session reply failed'); error.code = classifySessionProviderErrorMessage(normalized); + if (error.code === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED') { + error.retryable = false; + } return error; } diff --git a/session-reply-wait.test.mjs b/session-reply-wait.test.mjs index 857af54..32f44bd 100644 --- a/session-reply-wait.test.mjs +++ b/session-reply-wait.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { ReadableStream } from 'node:stream/web'; import { + classifySessionProviderErrorMessage, consumeSessionEventsUntilFinish, eventMatchesRequest, parseSessionStreamEvent, @@ -139,3 +140,45 @@ test('consumeSessionEventsUntilFinish surfaces provider tool history errors befo }, ); }); + +test('consumeSessionEventsUntilFinish classifies unsupported visual history as degradable', async () => { + const message = + 'Request failed: Bad request (400): Failed to deserialize the JSON body into the target type: messages[8]: unknown variant `image_url`, expected `text`'; + assert.equal( + classifySessionProviderErrorMessage(message), + 'SESSION_VISUAL_CONTEXT_UNSUPPORTED', + ); + + const frames = [ + `data: ${JSON.stringify({ + type: 'Message', + request_id: 'req-visual', + message: { + role: 'assistant', + content: [{ + type: 'text', + text: `Ran into this error: ${message}\n\nPlease retry if you think this is a transient or recoverable error.`, + }], + }, + })}\n\n`, + 'data: {"type":"Finish","request_id":"req-visual"}\n\n', + ]; + const stream = new ReadableStream({ + start(controller) { + for (const frame of frames) controller.enqueue(new TextEncoder().encode(frame)); + controller.close(); + }, + }); + + await assert.rejects( + consumeSessionEventsUntilFinish(stream, { + requestId: 'req-visual', + timeoutMs: 5000, + }), + (error) => { + assert.equal(error.code, 'SESSION_VISUAL_CONTEXT_UNSUPPORTED'); + assert.equal(error.retryable, false); + return true; + }, + ); +}); diff --git a/tkmind-proxy.mjs b/tkmind-proxy.mjs index 0b3e6ac..7a467ce 100644 --- a/tkmind-proxy.mjs +++ b/tkmind-proxy.mjs @@ -7,7 +7,10 @@ import { Agent, fetch as undiciFetch } from 'undici'; import { resolveBillingTokenState } from './billing-token-state.mjs'; import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs'; import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs'; -import { developerToolsFromPolicy } from './capabilities.mjs'; +import { + developerToolsFromPolicy, + withoutSessionImageRead, +} from './capabilities.mjs'; import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs'; import { buildPublicUrl, @@ -1426,10 +1429,14 @@ export function createTkmindProxy({ workingDir, sessionPolicy = null, recipe = null, + disableImageReading = false, } = {}, ) { const resolvedWorkingDir = workingDir ?? await userAuth.resolveWorkingDir(userId); - const resolvedSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId); + const baseSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId); + const resolvedSessionPolicy = disableImageReading + ? withoutSessionImageRead(baseSessionPolicy) + : baseSessionPolicy; const startTarget = await pickTarget(); const upstream = await apiFetch(startTarget, apiSecret, '/agent/start', { method: 'POST', @@ -1452,7 +1459,7 @@ export function createTkmindProxy({ } await rememberSessionTarget(session.id, startTarget); await sessionStore.registerAgentSession(userId, session.id, startTarget); - const sessionScopedPolicy = + const baseSessionScopedPolicy = typeof userAuth.getAgentSessionPolicy === 'function' ? await userAuth.getAgentSessionPolicy( @@ -1466,6 +1473,9 @@ export function createTkmindProxy({ }, ) : resolvedSessionPolicy; + const sessionScopedPolicy = disableImageReading + ? withoutSessionImageRead(baseSessionScopedPolicy) + : baseSessionScopedPolicy; if (sessionScopedPolicy?.gooseMode) { const modeRes = await apiFetch(startTarget, apiSecret, '/agent/update_session', { method: 'POST', @@ -1622,17 +1632,25 @@ export function createTkmindProxy({ async function reconcileSessionPolicyForUser( userId, sessionId, - { toolMode = 'chat', query = null, forceDeepReasoning = false } = {}, + { + toolMode = 'chat', + query = null, + forceDeepReasoning = false, + disableImageReading = false, + } = {}, ) { if (!userId || !sessionId) return; const target = await resolveTarget(sessionId); const workingDir = await userAuth.resolveWorkingDir(userId); - const sessionPolicy = + const baseSessionPolicy = await getSessionPolicyForToolMode( userId, toolMode, sessionId, ); + const sessionPolicy = disableImageReading + ? withoutSessionImageRead(baseSessionPolicy) + : baseSessionPolicy; const publishLayout = await userAuth.getUserPublishLayout(userId); const userMemories = await resolveUserMemories(userId, { sessionId, @@ -1769,6 +1787,7 @@ export function createTkmindProxy({ toolMode = 'chat', forceDeepReasoning = false, requireHistoricalImageIsolation = false, + disableImageReading = false, } = {}, ) { if (!userId || !sessionId) throw new Error('缺少会话信息'); @@ -1813,6 +1832,7 @@ export function createTkmindProxy({ toolMode, query: firstUserText(userMessage), forceDeepReasoning, + disableImageReading, }); await applySessionLlmProvider(sessionId); await repairSessionToolHistory(sessionId); @@ -1929,14 +1949,19 @@ export function createTkmindProxy({ sessionId, requestId, userMessage, - { toolMode = 'chat', forceDeepReasoning = false, timeoutMs = 15 * 60 * 1000 } = {}, + { + toolMode = 'chat', + forceDeepReasoning = false, + timeoutMs = 15 * 60 * 1000, + disableImageReading = false, + } = {}, ) { const { body, target } = await prepareSessionReplyBody( userId, sessionId, requestId, userMessage, - { toolMode, forceDeepReasoning }, + { toolMode, forceDeepReasoning, disableImageReading }, ); const eventsResponse = await apiFetch( target, diff --git a/tkmind-proxy.test.mjs b/tkmind-proxy.test.mjs index bec40c8..a275f08 100644 --- a/tkmind-proxy.test.mjs +++ b/tkmind-proxy.test.mjs @@ -20,6 +20,7 @@ async function withFakeGoosedSession( const workingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memind-memory-v2-')); const harnessEntries = []; const replyBodies = []; + const startBodies = []; const updateBodies = []; let activeConversation = conversation; let server; @@ -32,6 +33,7 @@ async function withFakeGoosedSession( const body = rawBody ? JSON.parse(rawBody) : {}; if (req.method === 'POST' && req.url === '/agent/start') { + startBodies.push(body); res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ id: 'session-1' })); return; @@ -98,6 +100,7 @@ async function withFakeGoosedSession( workingDir, harnessEntries, replyBodies, + startBodies, updateBodies, }); } finally { @@ -1353,6 +1356,69 @@ test('submitSessionReplyForUser fails closed when historical image scrub is unsu }); }); +test('visual fallback session removes read_image while preserving the text task path', async () => { + await withFakeGoosedSession(async ({ + apiTarget, + workingDir, + replyBodies, + startBodies, + }) => { + const baseAuth = createMemoryTestUserAuth(workingDir); + const userAuth = { + ...baseAuth, + async getAgentSessionPolicy() { + return { + gooseMode: 'chat', + enableContextMemory: true, + extensionOverrides: [ + { name: 'memory', available_tools: [] }, + { name: 'developer', available_tools: ['read_image'] }, + ], + }; + }, + async ownsSession() { + return true; + }, + async canUseChat() { + return { ok: true }; + }, + async getUserById() { + return { id: 'user-1' }; + }, + async resolveUserPolicies() { + return { unrestricted: true, policies: {} }; + }, + }; + const proxy = createTkmindProxy({ + apiTarget, + apiSecret: 'test-secret', + userAuth, + }); + + await proxy.startSessionForUser( + 'user-1', + { disableImageReading: true }, + ); + await proxy.submitSessionReplyForUser( + 'user-1', + 'session-1', + 'request-visual-fallback', + { + id: 'message-visual-fallback', + role: 'user', + content: [{ type: 'text', text: '跳过视觉检查，继续修改页面' }], + }, + { disableImageReading: true }, + ); + + assert.deepEqual( + startBodies[0].extension_overrides, + [{ name: 'memory', available_tools: [] }], + ); + assert.equal(replyBodies.length, 1); + }); +}); + test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => { await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => { const proxy = createTkmindProxy({