fix: isolate visual tool failures from agent runs
This commit is contained in:
+70
-9
@@ -178,15 +178,32 @@ function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 1
|
||||
return selected.join('\n\n');
|
||||
}
|
||||
|
||||
function appendFreshSessionContext(userMessage, conversation) {
|
||||
function appendFreshSessionContext(
|
||||
userMessage,
|
||||
conversation,
|
||||
{ visualFallback = false } = {},
|
||||
) {
|
||||
const context = buildFreshSessionContext(conversation);
|
||||
if (!context) return userMessage;
|
||||
if (!context && !visualFallback) return userMessage;
|
||||
const message = userMessage && typeof userMessage === 'object'
|
||||
? { ...userMessage }
|
||||
: { role: 'user', content: [] };
|
||||
const content = Array.isArray(message.content) ? [...message.content] : [];
|
||||
const textIndex = content.findIndex((item) => item?.type === 'text');
|
||||
const note = `【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`;
|
||||
const notes = [];
|
||||
if (context) {
|
||||
notes.push(
|
||||
`【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`,
|
||||
);
|
||||
}
|
||||
if (visualFallback) {
|
||||
notes.push(
|
||||
'【视觉检查已降级】\n'
|
||||
+ '当前文本模型不接受图片工具结果。请跳过缩略图或图片视觉检查,'
|
||||
+ '不要再次调用 read_image;直接依据现有 HTML、CSS、文件内容和用户要求继续完成主任务。',
|
||||
);
|
||||
}
|
||||
const note = notes.join('\n\n');
|
||||
if (textIndex >= 0) {
|
||||
content[textIndex] = {
|
||||
...content[textIndex],
|
||||
@@ -201,6 +218,7 @@ function appendFreshSessionContext(userMessage, conversation) {
|
||||
const FRESH_SESSION_RECOVERY_CODES = new Set([
|
||||
'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED',
|
||||
'SESSION_REASONING_CONTENT_POISONED',
|
||||
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
|
||||
]);
|
||||
|
||||
async function resolveConversationForFreshSessionRecovery({
|
||||
@@ -547,6 +565,7 @@ export function createAgentRunGateway({
|
||||
observeWorkflowValidation = null,
|
||||
isSessionExternallyBusy = null,
|
||||
validateRunDeliverables = null,
|
||||
cancelSessionOnRetry = null,
|
||||
quiesceSessionOnTerminal = null,
|
||||
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
|
||||
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
|
||||
@@ -1645,7 +1664,8 @@ export function createAgentRunGateway({
|
||||
}
|
||||
if (awaitSessionFinish) {
|
||||
let submitMessage = ensureGooseUserMessageMetadata(userMessage);
|
||||
let replacedPoisonedSession = false;
|
||||
let disableImageReading = false;
|
||||
const recoveredSessionCodes = new Set();
|
||||
while (true) {
|
||||
try {
|
||||
const finish = await tkmindProxy.submitSessionReplyAndAwaitFinishForUser(
|
||||
@@ -1657,6 +1677,7 @@ export function createAgentRunGateway({
|
||||
toolMode: effectiveToolMode,
|
||||
forceDeepReasoning: runOptions.forceDeepReasoning,
|
||||
timeoutMs: runTimeoutMs,
|
||||
disableImageReading,
|
||||
},
|
||||
);
|
||||
await appendEvent(runId, 'session_finished', {
|
||||
@@ -1668,10 +1689,14 @@ export function createAgentRunGateway({
|
||||
toolEvidence = finish.toolEvidence ?? null;
|
||||
break;
|
||||
} catch (err) {
|
||||
const recoveryCode = String(err?.code ?? '').trim();
|
||||
if (
|
||||
!replacedPoisonedSession
|
||||
&& FRESH_SESSION_RECOVERY_CODES.has(String(err?.code ?? '').trim())
|
||||
FRESH_SESSION_RECOVERY_CODES.has(recoveryCode)
|
||||
&& !recoveredSessionCodes.has(recoveryCode)
|
||||
&& recoveredSessionCodes.size < FRESH_SESSION_RECOVERY_CODES.size
|
||||
) {
|
||||
const visualFallback =
|
||||
recoveryCode === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||
const previousSessionId = sessionId;
|
||||
const conversationForContext = await resolveConversationForFreshSessionRecovery({
|
||||
err,
|
||||
@@ -1680,9 +1705,15 @@ export function createAgentRunGateway({
|
||||
tkmindProxy,
|
||||
sessionSnapshotService,
|
||||
});
|
||||
const replacement = await tkmindProxy.startSessionForUser(row.user_id);
|
||||
const replacement = visualFallback
|
||||
? await tkmindProxy.startSessionForUser(
|
||||
row.user_id,
|
||||
{ disableImageReading: true },
|
||||
)
|
||||
: await tkmindProxy.startSessionForUser(row.user_id);
|
||||
sessionId = replacement.id;
|
||||
replacedPoisonedSession = true;
|
||||
recoveredSessionCodes.add(recoveryCode);
|
||||
disableImageReading ||= visualFallback;
|
||||
await pool.query(
|
||||
`UPDATE h5_agent_runs SET agent_session_id = ?, updated_at = ? WHERE id = ?`,
|
||||
[sessionId, nowMs(), runId],
|
||||
@@ -1697,13 +1728,18 @@ export function createAgentRunGateway({
|
||||
previousSessionId,
|
||||
sessionId,
|
||||
reason: err.code,
|
||||
visualFallback,
|
||||
repairedMessageCount: conversationForContext.length,
|
||||
persistedMessageCount: persisted.saved,
|
||||
});
|
||||
await appendRunSnapshot(runId);
|
||||
await invalidatePortalDirectChatSnapshot(sessionId);
|
||||
submitMessage = ensureGooseUserMessageMetadata(
|
||||
appendFreshSessionContext(userMessage, conversationForContext),
|
||||
appendFreshSessionContext(
|
||||
userMessage,
|
||||
conversationForContext,
|
||||
{ visualFallback },
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
@@ -1970,6 +2006,31 @@ export function createAgentRunGateway({
|
||||
await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs });
|
||||
}
|
||||
const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length;
|
||||
if (retryable && typeof cancelSessionOnRetry === 'function' && recoverySessionId) {
|
||||
try {
|
||||
const cancellation = await cancelSessionOnRetry({
|
||||
runId,
|
||||
userId: row.user_id,
|
||||
sessionId: recoverySessionId,
|
||||
requestId: row.request_id,
|
||||
});
|
||||
await appendEvent(runId, 'session_retry_cancelled', {
|
||||
sessionId: recoverySessionId,
|
||||
requestId: row.request_id,
|
||||
cancelled: Boolean(cancellation?.cancelled),
|
||||
skipped: Boolean(cancellation?.skipped),
|
||||
});
|
||||
} catch (cancelError) {
|
||||
await appendEvent(runId, 'session_retry_cancel_failed', {
|
||||
sessionId: recoverySessionId,
|
||||
requestId: row.request_id,
|
||||
error:
|
||||
cancelError instanceof Error
|
||||
? cancelError.message
|
||||
: String(cancelError),
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
if (!retryable && isPageDataSuggestionRemediable(err?.code)) {
|
||||
await maybeTriggerPageDataSuggestionRepair({
|
||||
userId: row.user_id,
|
||||
|
||||
@@ -1246,6 +1246,103 @@ test('agent run replaces reasoning-poisoned Goose session and retries with visib
|
||||
assert.equal(replacedData?.reason, 'SESSION_REASONING_CONTENT_POISONED');
|
||||
});
|
||||
|
||||
test('agent run degrades visual inspection after an earlier session-history recovery', async () => {
|
||||
const pool = createFakePool();
|
||||
const submitted = [];
|
||||
const started = [];
|
||||
const replacementIds = ['session-tool-clean', 'session-visual-fallback'];
|
||||
const priorConversation = [
|
||||
{
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我写一首诗,做成页面' }],
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: '页面已经生成。' }],
|
||||
},
|
||||
];
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser(_userId, options) {
|
||||
started.push(options ?? null);
|
||||
return { id: replacementIds.shift() };
|
||||
},
|
||||
async fetchSessionConversationForUser() {
|
||||
return priorConversation;
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser(
|
||||
_userId,
|
||||
sessionId,
|
||||
_requestId,
|
||||
userMessage,
|
||||
options,
|
||||
) {
|
||||
submitted.push({ sessionId, userMessage, options });
|
||||
if (sessionId === 'session-poisoned') {
|
||||
const error = new Error('tool history requires fresh session');
|
||||
error.code = 'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED';
|
||||
error.repairedConversation = priorConversation;
|
||||
throw error;
|
||||
}
|
||||
if (sessionId === 'session-tool-clean') {
|
||||
const error = new Error(
|
||||
'messages[8]: unknown variant `image_url`, expected `text`',
|
||||
);
|
||||
error.code = 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||
error.retryable = false;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
finishEvent: { type: 'Finish' },
|
||||
toolEvidence: { calls: ['sandbox-fs__edit_file'] },
|
||||
};
|
||||
},
|
||||
},
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
sessionId: 'session-poisoned',
|
||||
requestId: 'req-visual-fallback',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '页面再精美一点' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||
assert.deepEqual(submitted.map((item) => item.sessionId), [
|
||||
'session-poisoned',
|
||||
'session-tool-clean',
|
||||
'session-visual-fallback',
|
||||
]);
|
||||
assert.deepEqual(started, [
|
||||
null,
|
||||
{ disableImageReading: true },
|
||||
]);
|
||||
assert.equal(
|
||||
submitted.at(-1).options.disableImageReading,
|
||||
true,
|
||||
);
|
||||
assert.match(
|
||||
submitted.at(-1).userMessage.content[0].text,
|
||||
/视觉检查已降级/,
|
||||
);
|
||||
assert.match(
|
||||
submitted.at(-1).userMessage.content[0].text,
|
||||
/不要再次调用 read_image/,
|
||||
);
|
||||
assert.equal(
|
||||
pool.events.filter(
|
||||
(event) => event.eventType === 'poisoned_session_replaced',
|
||||
).length,
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
|
||||
const pool = createFakePool();
|
||||
const repairSubmits = [];
|
||||
@@ -2756,6 +2853,50 @@ test('agent run retries transient failures and then becomes terminal', async ()
|
||||
assert.match(pool.runs.get(run.id).error_message, /upstream unavailable/);
|
||||
});
|
||||
|
||||
test('agent run cancels an active upstream request before retrying', async () => {
|
||||
const pool = createFakePool();
|
||||
const calls = [];
|
||||
let submissions = 0;
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-retry-cancel' };
|
||||
},
|
||||
async submitSessionReplyForUser() {
|
||||
submissions += 1;
|
||||
calls.push(`submit-${submissions}`);
|
||||
if (submissions === 1) throw new Error('upstream unavailable');
|
||||
},
|
||||
},
|
||||
async cancelSessionOnRetry(input) {
|
||||
calls.push('cancel');
|
||||
assert.equal(input.sessionId, 'session-retry-cancel');
|
||||
assert.equal(input.requestId, 'req-retry-cancel');
|
||||
return { cancelled: true, skipped: false };
|
||||
},
|
||||
retryDelaysMs: [0, 0],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-retry-cancel',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '继续任务' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||
assert.deepEqual(calls, ['submit-1', 'cancel', 'submit-2']);
|
||||
assert.equal(
|
||||
pool.events.some(
|
||||
(event) => event.eventType === 'session_retry_cancelled',
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('agent run queue limits concurrent execution', async () => {
|
||||
const pool = createFakePool();
|
||||
let active = 0;
|
||||
|
||||
@@ -470,6 +470,39 @@ export function developerToolsFromPolicy(sessionPolicy) {
|
||||
return (sandboxFs ?? developer)?.available_tools ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Visual inspection is optional for H5 Agent runs. When the active text model
|
||||
* rejects image content, remove read_image for the recovery session without
|
||||
* weakening any workspace or data capability boundary.
|
||||
*/
|
||||
export function withoutSessionImageRead(sessionPolicy) {
|
||||
if (!sessionPolicy || !Array.isArray(sessionPolicy.extensionOverrides)) {
|
||||
return sessionPolicy;
|
||||
}
|
||||
let changed = false;
|
||||
const extensionOverrides = sessionPolicy.extensionOverrides
|
||||
.map((extension) => {
|
||||
if (String(extension?.name ?? '') !== 'developer') return extension;
|
||||
const tools = Array.isArray(extension.available_tools)
|
||||
? extension.available_tools
|
||||
: [];
|
||||
const availableTools = tools.filter((tool) => tool !== 'read_image');
|
||||
if (availableTools.length === tools.length) return extension;
|
||||
changed = true;
|
||||
if (availableTools.length === 0) return null;
|
||||
return {
|
||||
...extension,
|
||||
available_tools: availableTools,
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
if (!changed) return sessionPolicy;
|
||||
return {
|
||||
...sessionPolicy,
|
||||
extensionOverrides,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeDeveloperTools(capabilities) {
|
||||
const tools = [];
|
||||
if (capabilities.shell) tools.push('shell');
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
resolveSandboxMcpControlDbEnv,
|
||||
sandboxDeveloperTools,
|
||||
sandboxMcpTools,
|
||||
withoutSessionImageRead,
|
||||
} from './capabilities.mjs';
|
||||
import {
|
||||
verifyMindSpaceMcpScopedToken,
|
||||
@@ -575,6 +576,33 @@ test('sandboxMcp emits a scoped logical workspace token only for a bound session
|
||||
);
|
||||
});
|
||||
|
||||
test('withoutSessionImageRead removes only the optional visual inspection tool', () => {
|
||||
const policy = {
|
||||
gooseMode: 'chat',
|
||||
extensionOverrides: [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['write', 'read_image'] },
|
||||
],
|
||||
};
|
||||
|
||||
const next = withoutSessionImageRead(policy);
|
||||
|
||||
assert.deepEqual(next.extensionOverrides, [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['write'] },
|
||||
]);
|
||||
assert.deepEqual(policy.extensionOverrides[1].available_tools, ['write', 'read_image']);
|
||||
assert.deepEqual(
|
||||
withoutSessionImageRead({
|
||||
extensionOverrides: [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['read_image'] },
|
||||
],
|
||||
}).extensionOverrides,
|
||||
[{ name: 'memory', available_tools: [] }],
|
||||
);
|
||||
});
|
||||
|
||||
test('sandboxMcp honors container node executable override', () => {
|
||||
const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true };
|
||||
const policy = buildAgentExtensionPolicy(caps, {
|
||||
|
||||
@@ -16,10 +16,27 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
|
||||
4. 本地重放最后一条已经是 `Finish` 或 `Error` 时,不再连接 Goose。
|
||||
5. 重放不得破坏现有消息 ID 合并、Finish 幂等计费和 MindSpace Finish 同步守卫。
|
||||
|
||||
## 视觉工具故障旁路
|
||||
|
||||
文本模型可能在会话历史中遇到 `read_image` 返回的图片块,并以
|
||||
`unknown variant image_url, expected text` 拒绝整次请求。图片检查属于可选增强,
|
||||
不得因此中断页面编辑、写文件、发布或其它文本主任务。
|
||||
|
||||
必须保留:
|
||||
|
||||
1. 将该上游错误分类为 `SESSION_VISUAL_CONTEXT_UNSUPPORTED`,不得作为普通瞬时错误盲重试。
|
||||
2. 用干净会话恢复可见文本上下文,并移除新会话策略中的 `read_image`,明确要求 Agent 跳过视觉检查、继续主任务。
|
||||
3. 一次执行中允许先恢复污染的工具历史,再单独进行一次视觉降级恢复;两个原因不得共用同一个“一次性恢复”开关。
|
||||
4. 普通重试前必须取消旧会话的活动请求,避免后续请求被 `Session already has an active request` 锁死。
|
||||
5. 视觉降级只影响当前恢复会话,不得全局关闭图片能力,也不得删减其它开发工具或数据能力。
|
||||
|
||||
## 关键路径
|
||||
|
||||
- `session-stream.mjs`
|
||||
- `session-stream-store.mjs`
|
||||
- `session-reply-wait.mjs`
|
||||
- `agent-run-gateway.mjs`
|
||||
- `capabilities.mjs`
|
||||
- `tkmind-proxy.mjs`
|
||||
- `server/portal-session-routes.mjs`(Portal Session 路由、流中 public HTML 落盘与 Finish 后同步)
|
||||
- `src/hooks/useTKMindChat.ts`
|
||||
@@ -29,6 +46,7 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
|
||||
```bash
|
||||
node --test session-stream.test.mjs session-stream-store.test.mjs chat-agent-run-gate.test.mjs
|
||||
node --test --test-name-pattern='proxySessionEvents' tkmind-proxy.test.mjs
|
||||
node --test session-reply-wait.test.mjs capabilities.test.mjs agent-run-gateway.test.mjs tkmind-proxy.test.mjs
|
||||
npm run verify:h5-session-patches
|
||||
node --test billing-session-concurrency.test.mjs
|
||||
```
|
||||
@@ -39,3 +57,6 @@ node --test billing-session-concurrency.test.mjs
|
||||
- 有映射时,Portal 游标正确转换成 Goose 游标。
|
||||
- 多轮会话中选择最新可映射的上游游标,不被上一轮 Finish 截断。
|
||||
- Finish 重放不重复扣费,消息合并与页面同步守卫继续通过。
|
||||
- `image_url` 不兼容时切换到无 `read_image` 的干净会话,并继续文本主任务。
|
||||
- 工具历史恢复后仍可执行一次独立的视觉降级恢复。
|
||||
- 瞬时重试前先取消旧请求,避免活动请求冲突。
|
||||
|
||||
@@ -179,6 +179,7 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
|
||||
assert.equal(captured.agentOptions.autoDispatch, true);
|
||||
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
||||
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
||||
assert.equal(typeof captured.agentOptions.cancelSessionOnRetry, 'function');
|
||||
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
||||
assert.equal(captured.agentOptions.observeWorkflowValidation, null);
|
||||
assert.equal(captured.agentOptions.enforcePageDataWorkflowValidation, false);
|
||||
|
||||
@@ -36,6 +36,12 @@ function messageContentItems(event) {
|
||||
export function classifySessionProviderErrorMessage(message) {
|
||||
const normalized = String(message ?? '').trim();
|
||||
if (!normalized) return 'SESSION_REPLY_ERROR';
|
||||
if (
|
||||
/unknown variant [`']?image_url[`']?.*expected [`']?text/i.test(normalized)
|
||||
|| /failed to deserialize.*image_url/i.test(normalized)
|
||||
) {
|
||||
return 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||
}
|
||||
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) {
|
||||
return 'SESSION_TOOL_HISTORY_POISONED';
|
||||
}
|
||||
@@ -60,6 +66,9 @@ function providerReplyError(event) {
|
||||
.trim();
|
||||
const error = new Error(normalized || 'session reply failed');
|
||||
error.code = classifySessionProviderErrorMessage(normalized);
|
||||
if (error.code === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED') {
|
||||
error.retryable = false;
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { ReadableStream } from 'node:stream/web';
|
||||
import {
|
||||
classifySessionProviderErrorMessage,
|
||||
consumeSessionEventsUntilFinish,
|
||||
eventMatchesRequest,
|
||||
parseSessionStreamEvent,
|
||||
@@ -139,3 +140,45 @@ test('consumeSessionEventsUntilFinish surfaces provider tool history errors befo
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('consumeSessionEventsUntilFinish classifies unsupported visual history as degradable', async () => {
|
||||
const message =
|
||||
'Request failed: Bad request (400): Failed to deserialize the JSON body into the target type: messages[8]: unknown variant `image_url`, expected `text`';
|
||||
assert.equal(
|
||||
classifySessionProviderErrorMessage(message),
|
||||
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
|
||||
);
|
||||
|
||||
const frames = [
|
||||
`data: ${JSON.stringify({
|
||||
type: 'Message',
|
||||
request_id: 'req-visual',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Ran into this error: ${message}\n\nPlease retry if you think this is a transient or recoverable error.`,
|
||||
}],
|
||||
},
|
||||
})}\n\n`,
|
||||
'data: {"type":"Finish","request_id":"req-visual"}\n\n',
|
||||
];
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
for (const frame of frames) controller.enqueue(new TextEncoder().encode(frame));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
consumeSessionEventsUntilFinish(stream, {
|
||||
requestId: 'req-visual',
|
||||
timeoutMs: 5000,
|
||||
}),
|
||||
(error) => {
|
||||
assert.equal(error.code, 'SESSION_VISUAL_CONTEXT_UNSUPPORTED');
|
||||
assert.equal(error.retryable, false);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
+32
-7
@@ -7,7 +7,10 @@ import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { resolveBillingTokenState } from './billing-token-state.mjs';
|
||||
import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs';
|
||||
import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs';
|
||||
import { developerToolsFromPolicy } from './capabilities.mjs';
|
||||
import {
|
||||
developerToolsFromPolicy,
|
||||
withoutSessionImageRead,
|
||||
} from './capabilities.mjs';
|
||||
import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs';
|
||||
import {
|
||||
buildPublicUrl,
|
||||
@@ -1426,10 +1429,14 @@ export function createTkmindProxy({
|
||||
workingDir,
|
||||
sessionPolicy = null,
|
||||
recipe = null,
|
||||
disableImageReading = false,
|
||||
} = {},
|
||||
) {
|
||||
const resolvedWorkingDir = workingDir ?? await userAuth.resolveWorkingDir(userId);
|
||||
const resolvedSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
|
||||
const baseSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
|
||||
const resolvedSessionPolicy = disableImageReading
|
||||
? withoutSessionImageRead(baseSessionPolicy)
|
||||
: baseSessionPolicy;
|
||||
const startTarget = await pickTarget();
|
||||
const upstream = await apiFetch(startTarget, apiSecret, '/agent/start', {
|
||||
method: 'POST',
|
||||
@@ -1452,7 +1459,7 @@ export function createTkmindProxy({
|
||||
}
|
||||
await rememberSessionTarget(session.id, startTarget);
|
||||
await sessionStore.registerAgentSession(userId, session.id, startTarget);
|
||||
const sessionScopedPolicy =
|
||||
const baseSessionScopedPolicy =
|
||||
typeof userAuth.getAgentSessionPolicy ===
|
||||
'function'
|
||||
? await userAuth.getAgentSessionPolicy(
|
||||
@@ -1466,6 +1473,9 @@ export function createTkmindProxy({
|
||||
},
|
||||
)
|
||||
: resolvedSessionPolicy;
|
||||
const sessionScopedPolicy = disableImageReading
|
||||
? withoutSessionImageRead(baseSessionScopedPolicy)
|
||||
: baseSessionScopedPolicy;
|
||||
if (sessionScopedPolicy?.gooseMode) {
|
||||
const modeRes = await apiFetch(startTarget, apiSecret, '/agent/update_session', {
|
||||
method: 'POST',
|
||||
@@ -1622,17 +1632,25 @@ export function createTkmindProxy({
|
||||
async function reconcileSessionPolicyForUser(
|
||||
userId,
|
||||
sessionId,
|
||||
{ toolMode = 'chat', query = null, forceDeepReasoning = false } = {},
|
||||
{
|
||||
toolMode = 'chat',
|
||||
query = null,
|
||||
forceDeepReasoning = false,
|
||||
disableImageReading = false,
|
||||
} = {},
|
||||
) {
|
||||
if (!userId || !sessionId) return;
|
||||
const target = await resolveTarget(sessionId);
|
||||
const workingDir = await userAuth.resolveWorkingDir(userId);
|
||||
const sessionPolicy =
|
||||
const baseSessionPolicy =
|
||||
await getSessionPolicyForToolMode(
|
||||
userId,
|
||||
toolMode,
|
||||
sessionId,
|
||||
);
|
||||
const sessionPolicy = disableImageReading
|
||||
? withoutSessionImageRead(baseSessionPolicy)
|
||||
: baseSessionPolicy;
|
||||
const publishLayout = await userAuth.getUserPublishLayout(userId);
|
||||
const userMemories = await resolveUserMemories(userId, {
|
||||
sessionId,
|
||||
@@ -1769,6 +1787,7 @@ export function createTkmindProxy({
|
||||
toolMode = 'chat',
|
||||
forceDeepReasoning = false,
|
||||
requireHistoricalImageIsolation = false,
|
||||
disableImageReading = false,
|
||||
} = {},
|
||||
) {
|
||||
if (!userId || !sessionId) throw new Error('缺少会话信息');
|
||||
@@ -1813,6 +1832,7 @@ export function createTkmindProxy({
|
||||
toolMode,
|
||||
query: firstUserText(userMessage),
|
||||
forceDeepReasoning,
|
||||
disableImageReading,
|
||||
});
|
||||
await applySessionLlmProvider(sessionId);
|
||||
await repairSessionToolHistory(sessionId);
|
||||
@@ -1929,14 +1949,19 @@ export function createTkmindProxy({
|
||||
sessionId,
|
||||
requestId,
|
||||
userMessage,
|
||||
{ toolMode = 'chat', forceDeepReasoning = false, timeoutMs = 15 * 60 * 1000 } = {},
|
||||
{
|
||||
toolMode = 'chat',
|
||||
forceDeepReasoning = false,
|
||||
timeoutMs = 15 * 60 * 1000,
|
||||
disableImageReading = false,
|
||||
} = {},
|
||||
) {
|
||||
const { body, target } = await prepareSessionReplyBody(
|
||||
userId,
|
||||
sessionId,
|
||||
requestId,
|
||||
userMessage,
|
||||
{ toolMode, forceDeepReasoning },
|
||||
{ toolMode, forceDeepReasoning, disableImageReading },
|
||||
);
|
||||
const eventsResponse = await apiFetch(
|
||||
target,
|
||||
|
||||
@@ -20,6 +20,7 @@ async function withFakeGoosedSession(
|
||||
const workingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memind-memory-v2-'));
|
||||
const harnessEntries = [];
|
||||
const replyBodies = [];
|
||||
const startBodies = [];
|
||||
const updateBodies = [];
|
||||
let activeConversation = conversation;
|
||||
let server;
|
||||
@@ -32,6 +33,7 @@ async function withFakeGoosedSession(
|
||||
const body = rawBody ? JSON.parse(rawBody) : {};
|
||||
|
||||
if (req.method === 'POST' && req.url === '/agent/start') {
|
||||
startBodies.push(body);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ id: 'session-1' }));
|
||||
return;
|
||||
@@ -98,6 +100,7 @@ async function withFakeGoosedSession(
|
||||
workingDir,
|
||||
harnessEntries,
|
||||
replyBodies,
|
||||
startBodies,
|
||||
updateBodies,
|
||||
});
|
||||
} finally {
|
||||
@@ -1353,6 +1356,69 @@ test('submitSessionReplyForUser fails closed when historical image scrub is unsu
|
||||
});
|
||||
});
|
||||
|
||||
test('visual fallback session removes read_image while preserving the text task path', async () => {
|
||||
await withFakeGoosedSession(async ({
|
||||
apiTarget,
|
||||
workingDir,
|
||||
replyBodies,
|
||||
startBodies,
|
||||
}) => {
|
||||
const baseAuth = createMemoryTestUserAuth(workingDir);
|
||||
const userAuth = {
|
||||
...baseAuth,
|
||||
async getAgentSessionPolicy() {
|
||||
return {
|
||||
gooseMode: 'chat',
|
||||
enableContextMemory: true,
|
||||
extensionOverrides: [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['read_image'] },
|
||||
],
|
||||
};
|
||||
},
|
||||
async ownsSession() {
|
||||
return true;
|
||||
},
|
||||
async canUseChat() {
|
||||
return { ok: true };
|
||||
},
|
||||
async getUserById() {
|
||||
return { id: 'user-1' };
|
||||
},
|
||||
async resolveUserPolicies() {
|
||||
return { unrestricted: true, policies: {} };
|
||||
},
|
||||
};
|
||||
const proxy = createTkmindProxy({
|
||||
apiTarget,
|
||||
apiSecret: 'test-secret',
|
||||
userAuth,
|
||||
});
|
||||
|
||||
await proxy.startSessionForUser(
|
||||
'user-1',
|
||||
{ disableImageReading: true },
|
||||
);
|
||||
await proxy.submitSessionReplyForUser(
|
||||
'user-1',
|
||||
'session-1',
|
||||
'request-visual-fallback',
|
||||
{
|
||||
id: 'message-visual-fallback',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '跳过视觉检查,继续修改页面' }],
|
||||
},
|
||||
{ disableImageReading: true },
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
startBodies[0].extension_overrides,
|
||||
[{ name: 'memory', available_tools: [] }],
|
||||
);
|
||||
assert.equal(replyBodies.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => {
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => {
|
||||
const proxy = createTkmindProxy({
|
||||
|
||||
Reference in New Issue
Block a user