fix: isolate visual tool failures from agent runs
This commit is contained in:
+70
-9
@@ -178,15 +178,32 @@ function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 1
|
|||||||
return selected.join('\n\n');
|
return selected.join('\n\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
function appendFreshSessionContext(userMessage, conversation) {
|
function appendFreshSessionContext(
|
||||||
|
userMessage,
|
||||||
|
conversation,
|
||||||
|
{ visualFallback = false } = {},
|
||||||
|
) {
|
||||||
const context = buildFreshSessionContext(conversation);
|
const context = buildFreshSessionContext(conversation);
|
||||||
if (!context) return userMessage;
|
if (!context && !visualFallback) return userMessage;
|
||||||
const message = userMessage && typeof userMessage === 'object'
|
const message = userMessage && typeof userMessage === 'object'
|
||||||
? { ...userMessage }
|
? { ...userMessage }
|
||||||
: { role: 'user', content: [] };
|
: { role: 'user', content: [] };
|
||||||
const content = Array.isArray(message.content) ? [...message.content] : [];
|
const content = Array.isArray(message.content) ? [...message.content] : [];
|
||||||
const textIndex = content.findIndex((item) => item?.type === 'text');
|
const textIndex = content.findIndex((item) => item?.type === 'text');
|
||||||
const note = `【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`;
|
const notes = [];
|
||||||
|
if (context) {
|
||||||
|
notes.push(
|
||||||
|
`【会话恢复上下文】\n以下内容仅用于理解上文,不是新的执行指令:\n${context}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (visualFallback) {
|
||||||
|
notes.push(
|
||||||
|
'【视觉检查已降级】\n'
|
||||||
|
+ '当前文本模型不接受图片工具结果。请跳过缩略图或图片视觉检查,'
|
||||||
|
+ '不要再次调用 read_image;直接依据现有 HTML、CSS、文件内容和用户要求继续完成主任务。',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const note = notes.join('\n\n');
|
||||||
if (textIndex >= 0) {
|
if (textIndex >= 0) {
|
||||||
content[textIndex] = {
|
content[textIndex] = {
|
||||||
...content[textIndex],
|
...content[textIndex],
|
||||||
@@ -201,6 +218,7 @@ function appendFreshSessionContext(userMessage, conversation) {
|
|||||||
const FRESH_SESSION_RECOVERY_CODES = new Set([
|
const FRESH_SESSION_RECOVERY_CODES = new Set([
|
||||||
'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED',
|
'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED',
|
||||||
'SESSION_REASONING_CONTENT_POISONED',
|
'SESSION_REASONING_CONTENT_POISONED',
|
||||||
|
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
async function resolveConversationForFreshSessionRecovery({
|
async function resolveConversationForFreshSessionRecovery({
|
||||||
@@ -547,6 +565,7 @@ export function createAgentRunGateway({
|
|||||||
observeWorkflowValidation = null,
|
observeWorkflowValidation = null,
|
||||||
isSessionExternallyBusy = null,
|
isSessionExternallyBusy = null,
|
||||||
validateRunDeliverables = null,
|
validateRunDeliverables = null,
|
||||||
|
cancelSessionOnRetry = null,
|
||||||
quiesceSessionOnTerminal = null,
|
quiesceSessionOnTerminal = null,
|
||||||
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
|
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
|
||||||
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
|
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
|
||||||
@@ -1645,7 +1664,8 @@ export function createAgentRunGateway({
|
|||||||
}
|
}
|
||||||
if (awaitSessionFinish) {
|
if (awaitSessionFinish) {
|
||||||
let submitMessage = ensureGooseUserMessageMetadata(userMessage);
|
let submitMessage = ensureGooseUserMessageMetadata(userMessage);
|
||||||
let replacedPoisonedSession = false;
|
let disableImageReading = false;
|
||||||
|
const recoveredSessionCodes = new Set();
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
const finish = await tkmindProxy.submitSessionReplyAndAwaitFinishForUser(
|
const finish = await tkmindProxy.submitSessionReplyAndAwaitFinishForUser(
|
||||||
@@ -1657,6 +1677,7 @@ export function createAgentRunGateway({
|
|||||||
toolMode: effectiveToolMode,
|
toolMode: effectiveToolMode,
|
||||||
forceDeepReasoning: runOptions.forceDeepReasoning,
|
forceDeepReasoning: runOptions.forceDeepReasoning,
|
||||||
timeoutMs: runTimeoutMs,
|
timeoutMs: runTimeoutMs,
|
||||||
|
disableImageReading,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
await appendEvent(runId, 'session_finished', {
|
await appendEvent(runId, 'session_finished', {
|
||||||
@@ -1668,10 +1689,14 @@ export function createAgentRunGateway({
|
|||||||
toolEvidence = finish.toolEvidence ?? null;
|
toolEvidence = finish.toolEvidence ?? null;
|
||||||
break;
|
break;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
const recoveryCode = String(err?.code ?? '').trim();
|
||||||
if (
|
if (
|
||||||
!replacedPoisonedSession
|
FRESH_SESSION_RECOVERY_CODES.has(recoveryCode)
|
||||||
&& FRESH_SESSION_RECOVERY_CODES.has(String(err?.code ?? '').trim())
|
&& !recoveredSessionCodes.has(recoveryCode)
|
||||||
|
&& recoveredSessionCodes.size < FRESH_SESSION_RECOVERY_CODES.size
|
||||||
) {
|
) {
|
||||||
|
const visualFallback =
|
||||||
|
recoveryCode === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||||
const previousSessionId = sessionId;
|
const previousSessionId = sessionId;
|
||||||
const conversationForContext = await resolveConversationForFreshSessionRecovery({
|
const conversationForContext = await resolveConversationForFreshSessionRecovery({
|
||||||
err,
|
err,
|
||||||
@@ -1680,9 +1705,15 @@ export function createAgentRunGateway({
|
|||||||
tkmindProxy,
|
tkmindProxy,
|
||||||
sessionSnapshotService,
|
sessionSnapshotService,
|
||||||
});
|
});
|
||||||
const replacement = await tkmindProxy.startSessionForUser(row.user_id);
|
const replacement = visualFallback
|
||||||
|
? await tkmindProxy.startSessionForUser(
|
||||||
|
row.user_id,
|
||||||
|
{ disableImageReading: true },
|
||||||
|
)
|
||||||
|
: await tkmindProxy.startSessionForUser(row.user_id);
|
||||||
sessionId = replacement.id;
|
sessionId = replacement.id;
|
||||||
replacedPoisonedSession = true;
|
recoveredSessionCodes.add(recoveryCode);
|
||||||
|
disableImageReading ||= visualFallback;
|
||||||
await pool.query(
|
await pool.query(
|
||||||
`UPDATE h5_agent_runs SET agent_session_id = ?, updated_at = ? WHERE id = ?`,
|
`UPDATE h5_agent_runs SET agent_session_id = ?, updated_at = ? WHERE id = ?`,
|
||||||
[sessionId, nowMs(), runId],
|
[sessionId, nowMs(), runId],
|
||||||
@@ -1697,13 +1728,18 @@ export function createAgentRunGateway({
|
|||||||
previousSessionId,
|
previousSessionId,
|
||||||
sessionId,
|
sessionId,
|
||||||
reason: err.code,
|
reason: err.code,
|
||||||
|
visualFallback,
|
||||||
repairedMessageCount: conversationForContext.length,
|
repairedMessageCount: conversationForContext.length,
|
||||||
persistedMessageCount: persisted.saved,
|
persistedMessageCount: persisted.saved,
|
||||||
});
|
});
|
||||||
await appendRunSnapshot(runId);
|
await appendRunSnapshot(runId);
|
||||||
await invalidatePortalDirectChatSnapshot(sessionId);
|
await invalidatePortalDirectChatSnapshot(sessionId);
|
||||||
submitMessage = ensureGooseUserMessageMetadata(
|
submitMessage = ensureGooseUserMessageMetadata(
|
||||||
appendFreshSessionContext(userMessage, conversationForContext),
|
appendFreshSessionContext(
|
||||||
|
userMessage,
|
||||||
|
conversationForContext,
|
||||||
|
{ visualFallback },
|
||||||
|
),
|
||||||
);
|
);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1970,6 +2006,31 @@ export function createAgentRunGateway({
|
|||||||
await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs });
|
await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs });
|
||||||
}
|
}
|
||||||
const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length;
|
const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length;
|
||||||
|
if (retryable && typeof cancelSessionOnRetry === 'function' && recoverySessionId) {
|
||||||
|
try {
|
||||||
|
const cancellation = await cancelSessionOnRetry({
|
||||||
|
runId,
|
||||||
|
userId: row.user_id,
|
||||||
|
sessionId: recoverySessionId,
|
||||||
|
requestId: row.request_id,
|
||||||
|
});
|
||||||
|
await appendEvent(runId, 'session_retry_cancelled', {
|
||||||
|
sessionId: recoverySessionId,
|
||||||
|
requestId: row.request_id,
|
||||||
|
cancelled: Boolean(cancellation?.cancelled),
|
||||||
|
skipped: Boolean(cancellation?.skipped),
|
||||||
|
});
|
||||||
|
} catch (cancelError) {
|
||||||
|
await appendEvent(runId, 'session_retry_cancel_failed', {
|
||||||
|
sessionId: recoverySessionId,
|
||||||
|
requestId: row.request_id,
|
||||||
|
error:
|
||||||
|
cancelError instanceof Error
|
||||||
|
? cancelError.message
|
||||||
|
: String(cancelError),
|
||||||
|
}).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!retryable && isPageDataSuggestionRemediable(err?.code)) {
|
if (!retryable && isPageDataSuggestionRemediable(err?.code)) {
|
||||||
await maybeTriggerPageDataSuggestionRepair({
|
await maybeTriggerPageDataSuggestionRepair({
|
||||||
userId: row.user_id,
|
userId: row.user_id,
|
||||||
|
|||||||
@@ -1246,6 +1246,103 @@ test('agent run replaces reasoning-poisoned Goose session and retries with visib
|
|||||||
assert.equal(replacedData?.reason, 'SESSION_REASONING_CONTENT_POISONED');
|
assert.equal(replacedData?.reason, 'SESSION_REASONING_CONTENT_POISONED');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('agent run degrades visual inspection after an earlier session-history recovery', async () => {
|
||||||
|
const pool = createFakePool();
|
||||||
|
const submitted = [];
|
||||||
|
const started = [];
|
||||||
|
const replacementIds = ['session-tool-clean', 'session-visual-fallback'];
|
||||||
|
const priorConversation = [
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '帮我写一首诗,做成页面' }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{ type: 'text', text: '页面已经生成。' }],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const gateway = createAgentRunGateway({
|
||||||
|
pool,
|
||||||
|
userAuth: {},
|
||||||
|
tkmindProxy: {
|
||||||
|
async startSessionForUser(_userId, options) {
|
||||||
|
started.push(options ?? null);
|
||||||
|
return { id: replacementIds.shift() };
|
||||||
|
},
|
||||||
|
async fetchSessionConversationForUser() {
|
||||||
|
return priorConversation;
|
||||||
|
},
|
||||||
|
async submitSessionReplyAndAwaitFinishForUser(
|
||||||
|
_userId,
|
||||||
|
sessionId,
|
||||||
|
_requestId,
|
||||||
|
userMessage,
|
||||||
|
options,
|
||||||
|
) {
|
||||||
|
submitted.push({ sessionId, userMessage, options });
|
||||||
|
if (sessionId === 'session-poisoned') {
|
||||||
|
const error = new Error('tool history requires fresh session');
|
||||||
|
error.code = 'SESSION_TOOL_HISTORY_FRESH_SESSION_REQUIRED';
|
||||||
|
error.repairedConversation = priorConversation;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (sessionId === 'session-tool-clean') {
|
||||||
|
const error = new Error(
|
||||||
|
'messages[8]: unknown variant `image_url`, expected `text`',
|
||||||
|
);
|
||||||
|
error.code = 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||||
|
error.retryable = false;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
finishEvent: { type: 'Finish' },
|
||||||
|
toolEvidence: { calls: ['sandbox-fs__edit_file'] },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
retryDelaysMs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const run = await gateway.createRun('user-1', {
|
||||||
|
sessionId: 'session-poisoned',
|
||||||
|
requestId: 'req-visual-fallback',
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '页面再精美一点' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||||
|
assert.deepEqual(submitted.map((item) => item.sessionId), [
|
||||||
|
'session-poisoned',
|
||||||
|
'session-tool-clean',
|
||||||
|
'session-visual-fallback',
|
||||||
|
]);
|
||||||
|
assert.deepEqual(started, [
|
||||||
|
null,
|
||||||
|
{ disableImageReading: true },
|
||||||
|
]);
|
||||||
|
assert.equal(
|
||||||
|
submitted.at(-1).options.disableImageReading,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
submitted.at(-1).userMessage.content[0].text,
|
||||||
|
/视觉检查已降级/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
submitted.at(-1).userMessage.content[0].text,
|
||||||
|
/不要再次调用 read_image/,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
pool.events.filter(
|
||||||
|
(event) => event.eventType === 'poisoned_session_replaced',
|
||||||
|
).length,
|
||||||
|
2,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
|
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
|
||||||
const pool = createFakePool();
|
const pool = createFakePool();
|
||||||
const repairSubmits = [];
|
const repairSubmits = [];
|
||||||
@@ -2756,6 +2853,50 @@ test('agent run retries transient failures and then becomes terminal', async ()
|
|||||||
assert.match(pool.runs.get(run.id).error_message, /upstream unavailable/);
|
assert.match(pool.runs.get(run.id).error_message, /upstream unavailable/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('agent run cancels an active upstream request before retrying', async () => {
|
||||||
|
const pool = createFakePool();
|
||||||
|
const calls = [];
|
||||||
|
let submissions = 0;
|
||||||
|
const gateway = createAgentRunGateway({
|
||||||
|
pool,
|
||||||
|
userAuth: {},
|
||||||
|
tkmindProxy: {
|
||||||
|
async startSessionForUser() {
|
||||||
|
return { id: 'session-retry-cancel' };
|
||||||
|
},
|
||||||
|
async submitSessionReplyForUser() {
|
||||||
|
submissions += 1;
|
||||||
|
calls.push(`submit-${submissions}`);
|
||||||
|
if (submissions === 1) throw new Error('upstream unavailable');
|
||||||
|
},
|
||||||
|
},
|
||||||
|
async cancelSessionOnRetry(input) {
|
||||||
|
calls.push('cancel');
|
||||||
|
assert.equal(input.sessionId, 'session-retry-cancel');
|
||||||
|
assert.equal(input.requestId, 'req-retry-cancel');
|
||||||
|
return { cancelled: true, skipped: false };
|
||||||
|
},
|
||||||
|
retryDelaysMs: [0, 0],
|
||||||
|
});
|
||||||
|
|
||||||
|
const run = await gateway.createRun('user-1', {
|
||||||
|
requestId: 'req-retry-cancel',
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '继续任务' }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||||
|
assert.deepEqual(calls, ['submit-1', 'cancel', 'submit-2']);
|
||||||
|
assert.equal(
|
||||||
|
pool.events.some(
|
||||||
|
(event) => event.eventType === 'session_retry_cancelled',
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test('agent run queue limits concurrent execution', async () => {
|
test('agent run queue limits concurrent execution', async () => {
|
||||||
const pool = createFakePool();
|
const pool = createFakePool();
|
||||||
let active = 0;
|
let active = 0;
|
||||||
|
|||||||
@@ -470,6 +470,39 @@ export function developerToolsFromPolicy(sessionPolicy) {
|
|||||||
return (sandboxFs ?? developer)?.available_tools ?? [];
|
return (sandboxFs ?? developer)?.available_tools ?? [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Visual inspection is optional for H5 Agent runs. When the active text model
|
||||||
|
* rejects image content, remove read_image for the recovery session without
|
||||||
|
* weakening any workspace or data capability boundary.
|
||||||
|
*/
|
||||||
|
export function withoutSessionImageRead(sessionPolicy) {
|
||||||
|
if (!sessionPolicy || !Array.isArray(sessionPolicy.extensionOverrides)) {
|
||||||
|
return sessionPolicy;
|
||||||
|
}
|
||||||
|
let changed = false;
|
||||||
|
const extensionOverrides = sessionPolicy.extensionOverrides
|
||||||
|
.map((extension) => {
|
||||||
|
if (String(extension?.name ?? '') !== 'developer') return extension;
|
||||||
|
const tools = Array.isArray(extension.available_tools)
|
||||||
|
? extension.available_tools
|
||||||
|
: [];
|
||||||
|
const availableTools = tools.filter((tool) => tool !== 'read_image');
|
||||||
|
if (availableTools.length === tools.length) return extension;
|
||||||
|
changed = true;
|
||||||
|
if (availableTools.length === 0) return null;
|
||||||
|
return {
|
||||||
|
...extension,
|
||||||
|
available_tools: availableTools,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
if (!changed) return sessionPolicy;
|
||||||
|
return {
|
||||||
|
...sessionPolicy,
|
||||||
|
extensionOverrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function mergeDeveloperTools(capabilities) {
|
function mergeDeveloperTools(capabilities) {
|
||||||
const tools = [];
|
const tools = [];
|
||||||
if (capabilities.shell) tools.push('shell');
|
if (capabilities.shell) tools.push('shell');
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
resolveSandboxMcpControlDbEnv,
|
resolveSandboxMcpControlDbEnv,
|
||||||
sandboxDeveloperTools,
|
sandboxDeveloperTools,
|
||||||
sandboxMcpTools,
|
sandboxMcpTools,
|
||||||
|
withoutSessionImageRead,
|
||||||
} from './capabilities.mjs';
|
} from './capabilities.mjs';
|
||||||
import {
|
import {
|
||||||
verifyMindSpaceMcpScopedToken,
|
verifyMindSpaceMcpScopedToken,
|
||||||
@@ -575,6 +576,33 @@ test('sandboxMcp emits a scoped logical workspace token only for a bound session
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('withoutSessionImageRead removes only the optional visual inspection tool', () => {
|
||||||
|
const policy = {
|
||||||
|
gooseMode: 'chat',
|
||||||
|
extensionOverrides: [
|
||||||
|
{ name: 'memory', available_tools: [] },
|
||||||
|
{ name: 'developer', available_tools: ['write', 'read_image'] },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
const next = withoutSessionImageRead(policy);
|
||||||
|
|
||||||
|
assert.deepEqual(next.extensionOverrides, [
|
||||||
|
{ name: 'memory', available_tools: [] },
|
||||||
|
{ name: 'developer', available_tools: ['write'] },
|
||||||
|
]);
|
||||||
|
assert.deepEqual(policy.extensionOverrides[1].available_tools, ['write', 'read_image']);
|
||||||
|
assert.deepEqual(
|
||||||
|
withoutSessionImageRead({
|
||||||
|
extensionOverrides: [
|
||||||
|
{ name: 'memory', available_tools: [] },
|
||||||
|
{ name: 'developer', available_tools: ['read_image'] },
|
||||||
|
],
|
||||||
|
}).extensionOverrides,
|
||||||
|
[{ name: 'memory', available_tools: [] }],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test('sandboxMcp honors container node executable override', () => {
|
test('sandboxMcp honors container node executable override', () => {
|
||||||
const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true };
|
const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true };
|
||||||
const policy = buildAgentExtensionPolicy(caps, {
|
const policy = buildAgentExtensionPolicy(caps, {
|
||||||
|
|||||||
@@ -16,10 +16,27 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
|
|||||||
4. 本地重放最后一条已经是 `Finish` 或 `Error` 时,不再连接 Goose。
|
4. 本地重放最后一条已经是 `Finish` 或 `Error` 时,不再连接 Goose。
|
||||||
5. 重放不得破坏现有消息 ID 合并、Finish 幂等计费和 MindSpace Finish 同步守卫。
|
5. 重放不得破坏现有消息 ID 合并、Finish 幂等计费和 MindSpace Finish 同步守卫。
|
||||||
|
|
||||||
|
## 视觉工具故障旁路
|
||||||
|
|
||||||
|
文本模型可能在会话历史中遇到 `read_image` 返回的图片块,并以
|
||||||
|
`unknown variant image_url, expected text` 拒绝整次请求。图片检查属于可选增强,
|
||||||
|
不得因此中断页面编辑、写文件、发布或其它文本主任务。
|
||||||
|
|
||||||
|
必须保留:
|
||||||
|
|
||||||
|
1. 将该上游错误分类为 `SESSION_VISUAL_CONTEXT_UNSUPPORTED`,不得作为普通瞬时错误盲重试。
|
||||||
|
2. 用干净会话恢复可见文本上下文,并移除新会话策略中的 `read_image`,明确要求 Agent 跳过视觉检查、继续主任务。
|
||||||
|
3. 一次执行中允许先恢复污染的工具历史,再单独进行一次视觉降级恢复;两个原因不得共用同一个“一次性恢复”开关。
|
||||||
|
4. 普通重试前必须取消旧会话的活动请求,避免后续请求被 `Session already has an active request` 锁死。
|
||||||
|
5. 视觉降级只影响当前恢复会话,不得全局关闭图片能力,也不得删减其它开发工具或数据能力。
|
||||||
|
|
||||||
## 关键路径
|
## 关键路径
|
||||||
|
|
||||||
- `session-stream.mjs`
|
- `session-stream.mjs`
|
||||||
- `session-stream-store.mjs`
|
- `session-stream-store.mjs`
|
||||||
|
- `session-reply-wait.mjs`
|
||||||
|
- `agent-run-gateway.mjs`
|
||||||
|
- `capabilities.mjs`
|
||||||
- `tkmind-proxy.mjs`
|
- `tkmind-proxy.mjs`
|
||||||
- `server/portal-session-routes.mjs`(Portal Session 路由、流中 public HTML 落盘与 Finish 后同步)
|
- `server/portal-session-routes.mjs`(Portal Session 路由、流中 public HTML 落盘与 Finish 后同步)
|
||||||
- `src/hooks/useTKMindChat.ts`
|
- `src/hooks/useTKMindChat.ts`
|
||||||
@@ -29,6 +46,7 @@ Portal 对 SSE 帧使用本地持久化游标。该游标可能是 Portal 生成
|
|||||||
```bash
|
```bash
|
||||||
node --test session-stream.test.mjs session-stream-store.test.mjs chat-agent-run-gate.test.mjs
|
node --test session-stream.test.mjs session-stream-store.test.mjs chat-agent-run-gate.test.mjs
|
||||||
node --test --test-name-pattern='proxySessionEvents' tkmind-proxy.test.mjs
|
node --test --test-name-pattern='proxySessionEvents' tkmind-proxy.test.mjs
|
||||||
|
node --test session-reply-wait.test.mjs capabilities.test.mjs agent-run-gateway.test.mjs tkmind-proxy.test.mjs
|
||||||
npm run verify:h5-session-patches
|
npm run verify:h5-session-patches
|
||||||
node --test billing-session-concurrency.test.mjs
|
node --test billing-session-concurrency.test.mjs
|
||||||
```
|
```
|
||||||
@@ -39,3 +57,6 @@ node --test billing-session-concurrency.test.mjs
|
|||||||
- 有映射时,Portal 游标正确转换成 Goose 游标。
|
- 有映射时,Portal 游标正确转换成 Goose 游标。
|
||||||
- 多轮会话中选择最新可映射的上游游标,不被上一轮 Finish 截断。
|
- 多轮会话中选择最新可映射的上游游标,不被上一轮 Finish 截断。
|
||||||
- Finish 重放不重复扣费,消息合并与页面同步守卫继续通过。
|
- Finish 重放不重复扣费,消息合并与页面同步守卫继续通过。
|
||||||
|
- `image_url` 不兼容时切换到无 `read_image` 的干净会话,并继续文本主任务。
|
||||||
|
- 工具历史恢复后仍可执行一次独立的视觉降级恢复。
|
||||||
|
- 瞬时重试前先取消旧请求,避免活动请求冲突。
|
||||||
|
|||||||
@@ -179,6 +179,7 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
|
|||||||
assert.equal(captured.agentOptions.autoDispatch, true);
|
assert.equal(captured.agentOptions.autoDispatch, true);
|
||||||
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
||||||
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
||||||
|
assert.equal(typeof captured.agentOptions.cancelSessionOnRetry, 'function');
|
||||||
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
||||||
assert.equal(captured.agentOptions.observeWorkflowValidation, null);
|
assert.equal(captured.agentOptions.observeWorkflowValidation, null);
|
||||||
assert.equal(captured.agentOptions.enforcePageDataWorkflowValidation, false);
|
assert.equal(captured.agentOptions.enforcePageDataWorkflowValidation, false);
|
||||||
|
|||||||
@@ -36,6 +36,12 @@ function messageContentItems(event) {
|
|||||||
export function classifySessionProviderErrorMessage(message) {
|
export function classifySessionProviderErrorMessage(message) {
|
||||||
const normalized = String(message ?? '').trim();
|
const normalized = String(message ?? '').trim();
|
||||||
if (!normalized) return 'SESSION_REPLY_ERROR';
|
if (!normalized) return 'SESSION_REPLY_ERROR';
|
||||||
|
if (
|
||||||
|
/unknown variant [`']?image_url[`']?.*expected [`']?text/i.test(normalized)
|
||||||
|
|| /failed to deserialize.*image_url/i.test(normalized)
|
||||||
|
) {
|
||||||
|
return 'SESSION_VISUAL_CONTEXT_UNSUPPORTED';
|
||||||
|
}
|
||||||
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) {
|
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) {
|
||||||
return 'SESSION_TOOL_HISTORY_POISONED';
|
return 'SESSION_TOOL_HISTORY_POISONED';
|
||||||
}
|
}
|
||||||
@@ -60,6 +66,9 @@ function providerReplyError(event) {
|
|||||||
.trim();
|
.trim();
|
||||||
const error = new Error(normalized || 'session reply failed');
|
const error = new Error(normalized || 'session reply failed');
|
||||||
error.code = classifySessionProviderErrorMessage(normalized);
|
error.code = classifySessionProviderErrorMessage(normalized);
|
||||||
|
if (error.code === 'SESSION_VISUAL_CONTEXT_UNSUPPORTED') {
|
||||||
|
error.retryable = false;
|
||||||
|
}
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import { ReadableStream } from 'node:stream/web';
|
import { ReadableStream } from 'node:stream/web';
|
||||||
import {
|
import {
|
||||||
|
classifySessionProviderErrorMessage,
|
||||||
consumeSessionEventsUntilFinish,
|
consumeSessionEventsUntilFinish,
|
||||||
eventMatchesRequest,
|
eventMatchesRequest,
|
||||||
parseSessionStreamEvent,
|
parseSessionStreamEvent,
|
||||||
@@ -139,3 +140,45 @@ test('consumeSessionEventsUntilFinish surfaces provider tool history errors befo
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('consumeSessionEventsUntilFinish classifies unsupported visual history as degradable', async () => {
|
||||||
|
const message =
|
||||||
|
'Request failed: Bad request (400): Failed to deserialize the JSON body into the target type: messages[8]: unknown variant `image_url`, expected `text`';
|
||||||
|
assert.equal(
|
||||||
|
classifySessionProviderErrorMessage(message),
|
||||||
|
'SESSION_VISUAL_CONTEXT_UNSUPPORTED',
|
||||||
|
);
|
||||||
|
|
||||||
|
const frames = [
|
||||||
|
`data: ${JSON.stringify({
|
||||||
|
type: 'Message',
|
||||||
|
request_id: 'req-visual',
|
||||||
|
message: {
|
||||||
|
role: 'assistant',
|
||||||
|
content: [{
|
||||||
|
type: 'text',
|
||||||
|
text: `Ran into this error: ${message}\n\nPlease retry if you think this is a transient or recoverable error.`,
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
})}\n\n`,
|
||||||
|
'data: {"type":"Finish","request_id":"req-visual"}\n\n',
|
||||||
|
];
|
||||||
|
const stream = new ReadableStream({
|
||||||
|
start(controller) {
|
||||||
|
for (const frame of frames) controller.enqueue(new TextEncoder().encode(frame));
|
||||||
|
controller.close();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
consumeSessionEventsUntilFinish(stream, {
|
||||||
|
requestId: 'req-visual',
|
||||||
|
timeoutMs: 5000,
|
||||||
|
}),
|
||||||
|
(error) => {
|
||||||
|
assert.equal(error.code, 'SESSION_VISUAL_CONTEXT_UNSUPPORTED');
|
||||||
|
assert.equal(error.retryable, false);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|||||||
+32
-7
@@ -7,7 +7,10 @@ import { Agent, fetch as undiciFetch } from 'undici';
|
|||||||
import { resolveBillingTokenState } from './billing-token-state.mjs';
|
import { resolveBillingTokenState } from './billing-token-state.mjs';
|
||||||
import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs';
|
import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs';
|
||||||
import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs';
|
import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs';
|
||||||
import { developerToolsFromPolicy } from './capabilities.mjs';
|
import {
|
||||||
|
developerToolsFromPolicy,
|
||||||
|
withoutSessionImageRead,
|
||||||
|
} from './capabilities.mjs';
|
||||||
import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs';
|
import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs';
|
||||||
import {
|
import {
|
||||||
buildPublicUrl,
|
buildPublicUrl,
|
||||||
@@ -1426,10 +1429,14 @@ export function createTkmindProxy({
|
|||||||
workingDir,
|
workingDir,
|
||||||
sessionPolicy = null,
|
sessionPolicy = null,
|
||||||
recipe = null,
|
recipe = null,
|
||||||
|
disableImageReading = false,
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
const resolvedWorkingDir = workingDir ?? await userAuth.resolveWorkingDir(userId);
|
const resolvedWorkingDir = workingDir ?? await userAuth.resolveWorkingDir(userId);
|
||||||
const resolvedSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
|
const baseSessionPolicy = sessionPolicy ?? await userAuth.getAgentSessionPolicy(userId);
|
||||||
|
const resolvedSessionPolicy = disableImageReading
|
||||||
|
? withoutSessionImageRead(baseSessionPolicy)
|
||||||
|
: baseSessionPolicy;
|
||||||
const startTarget = await pickTarget();
|
const startTarget = await pickTarget();
|
||||||
const upstream = await apiFetch(startTarget, apiSecret, '/agent/start', {
|
const upstream = await apiFetch(startTarget, apiSecret, '/agent/start', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@@ -1452,7 +1459,7 @@ export function createTkmindProxy({
|
|||||||
}
|
}
|
||||||
await rememberSessionTarget(session.id, startTarget);
|
await rememberSessionTarget(session.id, startTarget);
|
||||||
await sessionStore.registerAgentSession(userId, session.id, startTarget);
|
await sessionStore.registerAgentSession(userId, session.id, startTarget);
|
||||||
const sessionScopedPolicy =
|
const baseSessionScopedPolicy =
|
||||||
typeof userAuth.getAgentSessionPolicy ===
|
typeof userAuth.getAgentSessionPolicy ===
|
||||||
'function'
|
'function'
|
||||||
? await userAuth.getAgentSessionPolicy(
|
? await userAuth.getAgentSessionPolicy(
|
||||||
@@ -1466,6 +1473,9 @@ export function createTkmindProxy({
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
: resolvedSessionPolicy;
|
: resolvedSessionPolicy;
|
||||||
|
const sessionScopedPolicy = disableImageReading
|
||||||
|
? withoutSessionImageRead(baseSessionScopedPolicy)
|
||||||
|
: baseSessionScopedPolicy;
|
||||||
if (sessionScopedPolicy?.gooseMode) {
|
if (sessionScopedPolicy?.gooseMode) {
|
||||||
const modeRes = await apiFetch(startTarget, apiSecret, '/agent/update_session', {
|
const modeRes = await apiFetch(startTarget, apiSecret, '/agent/update_session', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@@ -1622,17 +1632,25 @@ export function createTkmindProxy({
|
|||||||
async function reconcileSessionPolicyForUser(
|
async function reconcileSessionPolicyForUser(
|
||||||
userId,
|
userId,
|
||||||
sessionId,
|
sessionId,
|
||||||
{ toolMode = 'chat', query = null, forceDeepReasoning = false } = {},
|
{
|
||||||
|
toolMode = 'chat',
|
||||||
|
query = null,
|
||||||
|
forceDeepReasoning = false,
|
||||||
|
disableImageReading = false,
|
||||||
|
} = {},
|
||||||
) {
|
) {
|
||||||
if (!userId || !sessionId) return;
|
if (!userId || !sessionId) return;
|
||||||
const target = await resolveTarget(sessionId);
|
const target = await resolveTarget(sessionId);
|
||||||
const workingDir = await userAuth.resolveWorkingDir(userId);
|
const workingDir = await userAuth.resolveWorkingDir(userId);
|
||||||
const sessionPolicy =
|
const baseSessionPolicy =
|
||||||
await getSessionPolicyForToolMode(
|
await getSessionPolicyForToolMode(
|
||||||
userId,
|
userId,
|
||||||
toolMode,
|
toolMode,
|
||||||
sessionId,
|
sessionId,
|
||||||
);
|
);
|
||||||
|
const sessionPolicy = disableImageReading
|
||||||
|
? withoutSessionImageRead(baseSessionPolicy)
|
||||||
|
: baseSessionPolicy;
|
||||||
const publishLayout = await userAuth.getUserPublishLayout(userId);
|
const publishLayout = await userAuth.getUserPublishLayout(userId);
|
||||||
const userMemories = await resolveUserMemories(userId, {
|
const userMemories = await resolveUserMemories(userId, {
|
||||||
sessionId,
|
sessionId,
|
||||||
@@ -1769,6 +1787,7 @@ export function createTkmindProxy({
|
|||||||
toolMode = 'chat',
|
toolMode = 'chat',
|
||||||
forceDeepReasoning = false,
|
forceDeepReasoning = false,
|
||||||
requireHistoricalImageIsolation = false,
|
requireHistoricalImageIsolation = false,
|
||||||
|
disableImageReading = false,
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
if (!userId || !sessionId) throw new Error('缺少会话信息');
|
if (!userId || !sessionId) throw new Error('缺少会话信息');
|
||||||
@@ -1813,6 +1832,7 @@ export function createTkmindProxy({
|
|||||||
toolMode,
|
toolMode,
|
||||||
query: firstUserText(userMessage),
|
query: firstUserText(userMessage),
|
||||||
forceDeepReasoning,
|
forceDeepReasoning,
|
||||||
|
disableImageReading,
|
||||||
});
|
});
|
||||||
await applySessionLlmProvider(sessionId);
|
await applySessionLlmProvider(sessionId);
|
||||||
await repairSessionToolHistory(sessionId);
|
await repairSessionToolHistory(sessionId);
|
||||||
@@ -1929,14 +1949,19 @@ export function createTkmindProxy({
|
|||||||
sessionId,
|
sessionId,
|
||||||
requestId,
|
requestId,
|
||||||
userMessage,
|
userMessage,
|
||||||
{ toolMode = 'chat', forceDeepReasoning = false, timeoutMs = 15 * 60 * 1000 } = {},
|
{
|
||||||
|
toolMode = 'chat',
|
||||||
|
forceDeepReasoning = false,
|
||||||
|
timeoutMs = 15 * 60 * 1000,
|
||||||
|
disableImageReading = false,
|
||||||
|
} = {},
|
||||||
) {
|
) {
|
||||||
const { body, target } = await prepareSessionReplyBody(
|
const { body, target } = await prepareSessionReplyBody(
|
||||||
userId,
|
userId,
|
||||||
sessionId,
|
sessionId,
|
||||||
requestId,
|
requestId,
|
||||||
userMessage,
|
userMessage,
|
||||||
{ toolMode, forceDeepReasoning },
|
{ toolMode, forceDeepReasoning, disableImageReading },
|
||||||
);
|
);
|
||||||
const eventsResponse = await apiFetch(
|
const eventsResponse = await apiFetch(
|
||||||
target,
|
target,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ async function withFakeGoosedSession(
|
|||||||
const workingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memind-memory-v2-'));
|
const workingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memind-memory-v2-'));
|
||||||
const harnessEntries = [];
|
const harnessEntries = [];
|
||||||
const replyBodies = [];
|
const replyBodies = [];
|
||||||
|
const startBodies = [];
|
||||||
const updateBodies = [];
|
const updateBodies = [];
|
||||||
let activeConversation = conversation;
|
let activeConversation = conversation;
|
||||||
let server;
|
let server;
|
||||||
@@ -32,6 +33,7 @@ async function withFakeGoosedSession(
|
|||||||
const body = rawBody ? JSON.parse(rawBody) : {};
|
const body = rawBody ? JSON.parse(rawBody) : {};
|
||||||
|
|
||||||
if (req.method === 'POST' && req.url === '/agent/start') {
|
if (req.method === 'POST' && req.url === '/agent/start') {
|
||||||
|
startBodies.push(body);
|
||||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||||
res.end(JSON.stringify({ id: 'session-1' }));
|
res.end(JSON.stringify({ id: 'session-1' }));
|
||||||
return;
|
return;
|
||||||
@@ -98,6 +100,7 @@ async function withFakeGoosedSession(
|
|||||||
workingDir,
|
workingDir,
|
||||||
harnessEntries,
|
harnessEntries,
|
||||||
replyBodies,
|
replyBodies,
|
||||||
|
startBodies,
|
||||||
updateBodies,
|
updateBodies,
|
||||||
});
|
});
|
||||||
} finally {
|
} finally {
|
||||||
@@ -1353,6 +1356,69 @@ test('submitSessionReplyForUser fails closed when historical image scrub is unsu
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('visual fallback session removes read_image while preserving the text task path', async () => {
|
||||||
|
await withFakeGoosedSession(async ({
|
||||||
|
apiTarget,
|
||||||
|
workingDir,
|
||||||
|
replyBodies,
|
||||||
|
startBodies,
|
||||||
|
}) => {
|
||||||
|
const baseAuth = createMemoryTestUserAuth(workingDir);
|
||||||
|
const userAuth = {
|
||||||
|
...baseAuth,
|
||||||
|
async getAgentSessionPolicy() {
|
||||||
|
return {
|
||||||
|
gooseMode: 'chat',
|
||||||
|
enableContextMemory: true,
|
||||||
|
extensionOverrides: [
|
||||||
|
{ name: 'memory', available_tools: [] },
|
||||||
|
{ name: 'developer', available_tools: ['read_image'] },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async ownsSession() {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
async canUseChat() {
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
async getUserById() {
|
||||||
|
return { id: 'user-1' };
|
||||||
|
},
|
||||||
|
async resolveUserPolicies() {
|
||||||
|
return { unrestricted: true, policies: {} };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const proxy = createTkmindProxy({
|
||||||
|
apiTarget,
|
||||||
|
apiSecret: 'test-secret',
|
||||||
|
userAuth,
|
||||||
|
});
|
||||||
|
|
||||||
|
await proxy.startSessionForUser(
|
||||||
|
'user-1',
|
||||||
|
{ disableImageReading: true },
|
||||||
|
);
|
||||||
|
await proxy.submitSessionReplyForUser(
|
||||||
|
'user-1',
|
||||||
|
'session-1',
|
||||||
|
'request-visual-fallback',
|
||||||
|
{
|
||||||
|
id: 'message-visual-fallback',
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '跳过视觉检查,继续修改页面' }],
|
||||||
|
},
|
||||||
|
{ disableImageReading: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
startBodies[0].extension_overrides,
|
||||||
|
[{ name: 'memory', available_tools: [] }],
|
||||||
|
);
|
||||||
|
assert.equal(replyBodies.length, 1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => {
|
test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => {
|
||||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => {
|
await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => {
|
||||||
const proxy = createTkmindProxy({
|
const proxy = createTkmindProxy({
|
||||||
|
|||||||
Reference in New Issue
Block a user