fix: isolate visual tool failures from agent runs

This commit is contained in:
john
2026-07-27 19:20:23 +08:00
parent 7002007128
commit 17c59fde71
10 changed files with 444 additions and 16 deletions
+70 -9
View File
@@ -178,15 +178,32 @@ function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 1
return selected.join('\n\n');
}
function appendFreshSessionContext(userMessage, conversation) {
function appendFreshSessionContext(
userMessage,
conversation,
{ visualFallback = false } = {},
) {
const context = buildFreshSessionContext(conversation);
if (!context) return userMessage;
if (!context && !visualFallback) return userMessage;
const message = userMessage && typeof userMessage === 'object'
? { ...userMessage }
: { role: 'user', content: [] };
const content = Array.isArray(message.content) ? [...message.content] : [];
const textIndex = content.findIndex((item) => item?.type === 'text');
const note = `【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`;
const notes = [];
if (context) {
notes.push(
`【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`,
);
}
if (visualFallback) {
notes.push(
'【视觉检查已降级】\n'
+ '当前文本模型不接受图片工具结果。请跳过缩略图或图片视觉检查,'
+ '不要再次调用 read_image;直接依据现有 HTML、CSS、文件内容和用户要求继续完成主任务。',
);
}
const note = notes.join('\n\n');
if (textIndex >= 0) {
content[textIndex] = {
...content[textIndex],
@@ -201,6 +218,7 @@ function appendFreshSessionContext(userMessage, conversation) {
const FRESH_SESSION_RECOVERY_CODES = new Set([
'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED',
'SESSION_REASONING_CONTENT_POISONED',
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
]);
async function resolveConversationForFreshSessionRecovery({
@@ -547,6 +565,7 @@ export function createAgentRunGateway({
observeWorkflowValidation = null,
isSessionExternallyBusy = null,
validateRunDeliverables = null,
cancelSessionOnRetry = null,
quiesceSessionOnTerminal = null,
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
@@ -1645,7 +1664,8 @@ export function createAgentRunGateway({
}
if (awaitSessionFinish) {
let submitMessage = ensureGooseUserMessageMetadata(userMessage);
let replacedPoisonedSession = false;
let disableImageReading = false;
const recoveredSessionCodes = new Set();
while (true) {
try {
const finish = await tkmindProxy.submitSessionReplyAndAwaitFinishForUser(
@@ -1657,6 +1677,7 @@ export function createAgentRunGateway({
toolMode: effectiveToolMode,
forceDeepReasoning: runOptions.forceDeepReasoning,
timeoutMs: runTimeoutMs,
disableImageReading,
},
);
await appendEvent(runId, 'session_finished', {
@@ -1668,10 +1689,14 @@ export function createAgentRunGateway({
toolEvidence = finish.toolEvidence ?? null;
break;
} catch (err) {
const recoveryCode = String(err?.code ?? '').trim();
if (
!replacedPoisonedSession
&& FRESH_SESSION_RECOVERY_CODES.has(String(err?.code ?? '').trim())
FRESH_SESSION_RECOVERY_CODES.has(recoveryCode)
&& !recoveredSessionCodes.has(recoveryCode)
&& recoveredSessionCodes.size < FRESH_SESSION_RECOVERY_CODES.size
) {
const visualFallback =
recoveryCode === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
const previousSessionId = sessionId;
const conversationForContext = await resolveConversationForFreshSessionRecovery({
err,
@@ -1680,9 +1705,15 @@ export function createAgentRunGateway({
tkmindProxy,
sessionSnapshotService,
});
const replacement = await tkmindProxy.startSessionForUser(row.user_id);
const replacement = visualFallback
? await tkmindProxy.startSessionForUser(
row.user_id,
{ disableImageReading: true },
)
: await tkmindProxy.startSessionForUser(row.user_id);
sessionId = replacement.id;
replacedPoisonedSession = true;
recoveredSessionCodes.add(recoveryCode);
disableImageReading ||= visualFallback;
await pool.query(
`UPDATE h5_agent_runs SET agent_session_id = ?, updated_at = ? WHERE id = ?`,
[sessionId, nowMs(), runId],
@@ -1697,13 +1728,18 @@ export function createAgentRunGateway({
previousSessionId,
sessionId,
reason: err.code,
visualFallback,
repairedMessageCount: conversationForContext.length,
persistedMessageCount: persisted.saved,
});
await appendRunSnapshot(runId);
await invalidatePortalDirectChatSnapshot(sessionId);
submitMessage = ensureGooseUserMessageMetadata(
appendFreshSessionContext(userMessage, conversationForContext),
appendFreshSessionContext(
userMessage,
conversationForContext,
{ visualFallback },
),
);
continue;
}
@@ -1970,6 +2006,31 @@ export function createAgentRunGateway({
await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs });
}
const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length;
if (retryable && typeof cancelSessionOnRetry === 'function' && recoverySessionId) {
try {
const cancellation = await cancelSessionOnRetry({
runId,
userId: row.user_id,
sessionId: recoverySessionId,
requestId: row.request_id,
});
await appendEvent(runId, 'session_retry_cancelled', {
sessionId: recoverySessionId,
requestId: row.request_id,
cancelled: Boolean(cancellation?.cancelled),
skipped: Boolean(cancellation?.skipped),
});
} catch (cancelError) {
await appendEvent(runId, 'session_retry_cancel_failed', {
sessionId: recoverySessionId,
requestId: row.request_id,
error:
cancelError instanceof Error
? cancelError.message
: String(cancelError),
}).catch(() => {});
}
}
if (!retryable && isPageDataSuggestionRemediable(err?.code)) {
await maybeTriggerPageDataSuggestionRepair({
userId: row.user_id,
+141
View File
@@ -1246,6 +1246,103 @@ test('agent run replaces reasoning-poisoned Goose session and retries with visib
assert.equal(replacedData?.reason, 'SESSION_REASONING_CONTENT_POISONED');
});
test('agent run degrades visual inspection after an earlier session-history recovery', async () => {
const pool = createFakePool();
const submitted = [];
const started = [];
const replacementIds = ['session-tool-clean', 'session-visual-fallback'];
const priorConversation = [
{
role: 'user',
content: [{ type: 'text', text: '帮我写一首诗,做成页面' }],
},
{
role: 'assistant',
content: [{ type: 'text', text: '页面已经生成。' }],
},
];
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser(_userId, options) {
started.push(options ?? null);
return { id: replacementIds.shift() };
},
async fetchSessionConversationForUser() {
return priorConversation;
},
async submitSessionReplyAndAwaitFinishForUser(
_userId,
sessionId,
_requestId,
userMessage,
options,
) {
submitted.push({ sessionId, userMessage, options });
if (sessionId === 'session-poisoned') {
const error = new Error('tool history requires fresh session');
error.code = 'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED';
error.repairedConversation = priorConversation;
throw error;
}
if (sessionId === 'session-tool-clean') {
const error = new Error(
'messages[8]: unknown variant `image_url`, expected `text`',
);
error.code = 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
error.retryable = false;
throw error;
}
return {
ok: true,
finishEvent: { type: 'Finish' },
toolEvidence: { calls: ['sandbox-fs__edit_file'] },
};
},
},
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
sessionId: 'session-poisoned',
requestId: 'req-visual-fallback',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '页面再精美一点' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
assert.deepEqual(submitted.map((item) => item.sessionId), [
'session-poisoned',
'session-tool-clean',
'session-visual-fallback',
]);
assert.deepEqual(started, [
null,
{ disableImageReading: true },
]);
assert.equal(
submitted.at(-1).options.disableImageReading,
true,
);
assert.match(
submitted.at(-1).userMessage.content[0].text,
/视觉检查已降级/,
);
assert.match(
submitted.at(-1).userMessage.content[0].text,
/不要再次调用 read_image/,
);
assert.equal(
pool.events.filter(
(event) => event.eventType === 'poisoned_session_replaced',
).length,
2,
);
});
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
const pool = createFakePool();
const repairSubmits = [];
@@ -2756,6 +2853,50 @@ test('agent run retries transient failures and then becomes terminal', async ()
assert.match(pool.runs.get(run.id).error_message, /upstream unavailable/);
});
test('agent run cancels an active upstream request before retrying', async () => {
const pool = createFakePool();
const calls = [];
let submissions = 0;
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-retry-cancel' };
},
async submitSessionReplyForUser() {
submissions += 1;
calls.push(`submit-${submissions}`);
if (submissions === 1) throw new Error('upstream unavailable');
},
},
async cancelSessionOnRetry(input) {
calls.push('cancel');
assert.equal(input.sessionId, 'session-retry-cancel');
assert.equal(input.requestId, 'req-retry-cancel');
return { cancelled: true, skipped: false };
},
retryDelaysMs: [0, 0],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-retry-cancel',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '继续任务' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
assert.deepEqual(calls, ['submit-1', 'cancel', 'submit-2']);
assert.equal(
pool.events.some(
(event) => event.eventType === 'session_retry_cancelled',
),
true,
);
});
test('agent run queue limits concurrent execution', async () => {
const pool = createFakePool();
let active = 0;
+33
View File
@@ -470,6 +470,39 @@ export function developerToolsFromPolicy(sessionPolicy) {
return (sandboxFs ?? developer)?.available_tools ?? [];
}
/**
* Visual inspection is optional for H5 Agent runs. When the active text model
* rejects image content, remove read_image for the recovery session without
* weakening any workspace or data capability boundary.
*/
export function withoutSessionImageRead(sessionPolicy) {
if (!sessionPolicy || !Array.isArray(sessionPolicy.extensionOverrides)) {
return sessionPolicy;
}
let changed = false;
const extensionOverrides = sessionPolicy.extensionOverrides
.map((extension) => {
if (String(extension?.name ?? '') !== 'developer') return extension;
const tools = Array.isArray(extension.available_tools)
? extension.available_tools
: [];
const availableTools = tools.filter((tool) => tool !== 'read_image');
if (availableTools.length === tools.length) return extension;
changed = true;
if (availableTools.length === 0) return null;
return {
...extension,
available_tools: availableTools,
};
})
.filter(Boolean);
if (!changed) return sessionPolicy;
return {
...sessionPolicy,
extensionOverrides,
};
}
function mergeDeveloperTools(capabilities) {
const tools = [];
if (capabilities.shell) tools.push('shell');
+28
View File
@@ -17,6 +17,7 @@ import {
resolveSandboxMcpControlDbEnv,
sandboxDeveloperTools,
sandboxMcpTools,
withoutSessionImageRead,
} from './capabilities.mjs';
import {
verifyMindSpaceMcpScopedToken,
@@ -575,6 +576,33 @@ test('sandboxMcp emits a scoped logical workspace token only for a bound session
);
});
test('withoutSessionImageRead removes only the optional visual inspection tool', () => {
const policy = {
gooseMode: 'chat',
extensionOverrides: [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['write', 'read_image'] },
],
};
const next = withoutSessionImageRead(policy);
assert.deepEqual(next.extensionOverrides, [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['write'] },
]);
assert.deepEqual(policy.extensionOverrides[1].available_tools, ['write', 'read_image']);
assert.deepEqual(
withoutSessionImageRead({
extensionOverrides: [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['read_image'] },
],
}).extensionOverrides,
[{ name: 'memory', available_tools: [] }],
);
});
test('sandboxMcp honors container node executable override', () => {
const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true };
const policy = buildAgentExtensionPolicy(caps, {
@@ -16,10 +16,27 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
4. 本地重放最后一条已经是 `Finish``Error` 时,不再连接 Goose。
5. 重放不得破坏现有消息 ID 合并、Finish 幂等计费和 MindSpace Finish 同步守卫。
## 视觉工具故障旁路
文本模型可能在会话历史中遇到 `read_image` 返回的图片块,并以
`unknown variant image_url, expected text` 拒绝整次请求。图片检查属于可选增强,
不得因此中断页面编辑、写文件、发布或其它文本主任务。
必须保留:
1. 将该上游错误分类为 `SESSION_VISUAL_CONTEXT_UNSUPPORTED`,不得作为普通瞬时错误盲重试。
2. 用干净会话恢复可见文本上下文,并移除新会话策略中的 `read_image`,明确要求 Agent 跳过视觉检查、继续主任务。
3. 一次执行中允许先恢复污染的工具历史,再单独进行一次视觉降级恢复;两个原因不得共用同一个“一次性恢复”开关。
4. 普通重试前必须取消旧会话的活动请求,避免后续请求被 `Session already has an active request` 锁死。
5. 视觉降级只影响当前恢复会话,不得全局关闭图片能力,也不得删减其它开发工具或数据能力。
## 关键路径
- `session-stream.mjs`
- `session-stream-store.mjs`
- `session-reply-wait.mjs`
- `agent-run-gateway.mjs`
- `capabilities.mjs`
- `tkmind-proxy.mjs`
- `server/portal-session-routes.mjs`Portal Session 路由、流中 public HTML 落盘与 Finish 后同步)
- `src/hooks/useTKMindChat.ts`
@@ -29,6 +46,7 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
```bash
node --test session-stream.test.mjs session-stream-store.test.mjs chat-agent-run-gate.test.mjs
node --test --test-name-pattern='proxySessionEvents' tkmind-proxy.test.mjs
node --test session-reply-wait.test.mjs capabilities.test.mjs agent-run-gateway.test.mjs tkmind-proxy.test.mjs
npm run verify:h5-session-patches
node --test billing-session-concurrency.test.mjs
```
@@ -39,3 +57,6 @@ node --test billing-session-concurrency.test.mjs
- 有映射时,Portal 游标正确转换成 Goose 游标。
- 多轮会话中选择最新可映射的上游游标,不被上一轮 Finish 截断。
- Finish 重放不重复扣费,消息合并与页面同步守卫继续通过。
- `image_url` 不兼容时切换到无 `read_image` 的干净会话,并继续文本主任务。
- 工具历史恢复后仍可执行一次独立的视觉降级恢复。
- 瞬时重试前先取消旧请求,避免活动请求冲突。
@@ -179,6 +179,7 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
assert.equal(captured.agentOptions.autoDispatch, true);
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
assert.equal(typeof captured.agentOptions.cancelSessionOnRetry, 'function');
assert.equal(captured.agentOptions.observeWorkflowRun, null);
assert.equal(captured.agentOptions.observeWorkflowValidation, null);
assert.equal(captured.agentOptions.enforcePageDataWorkflowValidation, false);
+9
View File
@@ -36,6 +36,12 @@ function messageContentItems(event) {
export function classifySessionProviderErrorMessage(message) {
const normalized = String(message ?? '').trim();
if (!normalized) return 'SESSION_REPLY_ERROR';
if (
/unknown variant [`']?image_url[`']?.*expected [`']?text/i.test(normalized)
|| /failed to deserialize.*image_url/i.test(normalized)
) {
return 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
}
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) {
return 'SESSION_TOOL_HISTORY_POISONED';
}
@@ -60,6 +66,9 @@ function providerReplyError(event) {
.trim();
const error = new Error(normalized || 'session reply failed');
error.code = classifySessionProviderErrorMessage(normalized);
if (error.code === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED') {
error.retryable = false;
}
return error;
}
+43
View File
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
import test from 'node:test';
import { ReadableStream } from 'node:stream/web';
import {
classifySessionProviderErrorMessage,
consumeSessionEventsUntilFinish,
eventMatchesRequest,
parseSessionStreamEvent,
@@ -139,3 +140,45 @@ test('consumeSessionEventsUntilFinish surfaces provider tool history errors befo
},
);
});
test('consumeSessionEventsUntilFinish classifies unsupported visual history as degradable', async () => {
const message =
'Request failed: Bad request (400): Failed to deserialize the JSON body into the target type: messages[8]: unknown variant `image_url`, expected `text`';
assert.equal(
classifySessionProviderErrorMessage(message),
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
);
const frames = [
`data: ${JSON.stringify({
type: 'Message',
request_id: 'req-visual',
message: {
role: 'assistant',
content: [{
type: 'text',
text: `Ran into this error: ${message}\n\nPlease retry if you think this is a transient or recoverable error.`,
}],
},
})}\n\n`,
'data: {"type":"Finish","request_id":"req-visual"}\n\n',
];
const stream = new ReadableStream({
start(controller) {
for (const frame of frames) controller.enqueue(new TextEncoder().encode(frame));
controller.close();
},
});
await assert.rejects(
consumeSessionEventsUntilFinish(stream, {
requestId: 'req-visual',
timeoutMs: 5000,
}),
(error) => {
assert.equal(error.code, 'SESSION_VISUAL_CONTEXT_UNSUPPORTED');
assert.equal(error.retryable, false);
return true;
},
);
});
+32 -7
View File
@@ -7,7 +7,10 @@ import { Agent, fetch as undiciFetch } from 'undici';
import { resolveBillingTokenState } from './billing-token-state.mjs';
import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs';
import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs';
import { developerToolsFromPolicy } from './capabilities.mjs';
import {
developerToolsFromPolicy,
withoutSessionImageRead,
} from './capabilities.mjs';
import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs';
import {
buildPublicUrl,
@@ -1426,10 +1429,14 @@ export function createTkmindProxy({
workingDir,
sessionPolicy = null,
recipe = null,
disableImageReading = false,
} = {},
) {
const resolvedWorkingDir = workingDir ?? await userAuth.resolveWorkingDir(userId);
const resolvedSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
const baseSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
const resolvedSessionPolicy = disableImageReading
? withoutSessionImageRead(baseSessionPolicy)
: baseSessionPolicy;
const startTarget = await pickTarget();
const upstream = await apiFetch(startTarget, apiSecret, '/agent/start', {
method: 'POST',
@@ -1452,7 +1459,7 @@ export function createTkmindProxy({
}
await rememberSessionTarget(session.id, startTarget);
await sessionStore.registerAgentSession(userId, session.id, startTarget);
const sessionScopedPolicy =
const baseSessionScopedPolicy =
typeof userAuth.getAgentSessionPolicy ===
'function'
? await userAuth.getAgentSessionPolicy(
@@ -1466,6 +1473,9 @@ export function createTkmindProxy({
},
)
: resolvedSessionPolicy;
const sessionScopedPolicy = disableImageReading
? withoutSessionImageRead(baseSessionScopedPolicy)
: baseSessionScopedPolicy;
if (sessionScopedPolicy?.gooseMode) {
const modeRes = await apiFetch(startTarget, apiSecret, '/agent/update_session', {
method: 'POST',
@@ -1622,17 +1632,25 @@ export function createTkmindProxy({
async function reconcileSessionPolicyForUser(
userId,
sessionId,
{ toolMode = 'chat', query = null, forceDeepReasoning = false } = {},
{
toolMode = 'chat',
query = null,
forceDeepReasoning = false,
disableImageReading = false,
} = {},
) {
if (!userId || !sessionId) return;
const target = await resolveTarget(sessionId);
const workingDir = await userAuth.resolveWorkingDir(userId);
const sessionPolicy =
const baseSessionPolicy =
await getSessionPolicyForToolMode(
userId,
toolMode,
sessionId,
);
const sessionPolicy = disableImageReading
? withoutSessionImageRead(baseSessionPolicy)
: baseSessionPolicy;
const publishLayout = await userAuth.getUserPublishLayout(userId);
const userMemories = await resolveUserMemories(userId, {
sessionId,
@@ -1769,6 +1787,7 @@ export function createTkmindProxy({
toolMode = 'chat',
forceDeepReasoning = false,
requireHistoricalImageIsolation = false,
disableImageReading = false,
} = {},
) {
if (!userId || !sessionId) throw new Error('缺少会话信息');
@@ -1813,6 +1832,7 @@ export function createTkmindProxy({
toolMode,
query: firstUserText(userMessage),
forceDeepReasoning,
disableImageReading,
});
await applySessionLlmProvider(sessionId);
await repairSessionToolHistory(sessionId);
@@ -1929,14 +1949,19 @@ export function createTkmindProxy({
sessionId,
requestId,
userMessage,
{ toolMode = 'chat', forceDeepReasoning = false, timeoutMs = 15 * 60 * 1000 } = {},
{
toolMode = 'chat',
forceDeepReasoning = false,
timeoutMs = 15 * 60 * 1000,
disableImageReading = false,
} = {},
) {
const { body, target } = await prepareSessionReplyBody(
userId,
sessionId,
requestId,
userMessage,
{ toolMode, forceDeepReasoning },
{ toolMode, forceDeepReasoning, disableImageReading },
);
const eventsResponse = await apiFetch(
target,
+66
View File
@@ -20,6 +20,7 @@ async function withFakeGoosedSession(
const workingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memind-memory-v2-'));
const harnessEntries = [];
const replyBodies = [];
const startBodies = [];
const updateBodies = [];
let activeConversation = conversation;
let server;
@@ -32,6 +33,7 @@ async function withFakeGoosedSession(
const body = rawBody ? JSON.parse(rawBody) : {};
if (req.method === 'POST' && req.url === '/agent/start') {
startBodies.push(body);
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ id: 'session-1' }));
return;
@@ -98,6 +100,7 @@ async function withFakeGoosedSession(
workingDir,
harnessEntries,
replyBodies,
startBodies,
updateBodies,
});
} finally {
@@ -1353,6 +1356,69 @@ test('submitSessionReplyForUser fails closed when historical image scrub is unsu
});
});
test('visual fallback session removes read_image while preserving the text task path', async () => {
await withFakeGoosedSession(async ({
apiTarget,
workingDir,
replyBodies,
startBodies,
}) => {
const baseAuth = createMemoryTestUserAuth(workingDir);
const userAuth = {
...baseAuth,
async getAgentSessionPolicy() {
return {
gooseMode: 'chat',
enableContextMemory: true,
extensionOverrides: [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['read_image'] },
],
};
},
async ownsSession() {
return true;
},
async canUseChat() {
return { ok: true };
},
async getUserById() {
return { id: 'user-1' };
},
async resolveUserPolicies() {
return { unrestricted: true, policies: {} };
},
};
const proxy = createTkmindProxy({
apiTarget,
apiSecret: 'test-secret',
userAuth,
});
await proxy.startSessionForUser(
'user-1',
{ disableImageReading: true },
);
await proxy.submitSessionReplyForUser(
'user-1',
'session-1',
'request-visual-fallback',
{
id: 'message-visual-fallback',
role: 'user',
content: [{ type: 'text', text: '跳过视觉检查,继续修改页面' }],
},
{ disableImageReading: true },
);
assert.deepEqual(
startBodies[0].extension_overrides,
[{ name: 'memory', available_tools: [] }],
);
assert.equal(replyBodies.length, 1);
});
});
test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => {
await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => {
const proxy = createTkmindProxy({