fix(vision): stop read_image from poisoning text-provider sessions

Image turns already get a vision-model description injected into the prompt,
but the agent kept calling read_image to "confirm" the pictures. Those tool
results carry base64 image parts that Goose persists, so every later turn
against the text-only chat provider failed with `unknown variant image_url`
before the agent could write the page. WeChat page requests therefore fell
through to the fail-closed delivery message.

Drop read_image for the turn whenever a vision model handles the images, say
so explicitly in the injected prompt, and teach the poison scan to recognise
tool image parts so already-polluted sessions rotate instead of failing again.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-22 10:40:12 +08:00
parent 4d12ea438b
commit ef4ce12bbf
9 changed files with 457 additions and 6 deletions
+2
View File
@@ -52,6 +52,7 @@ bash scripts/check-release-ready.sh
| H5 SSE 断线续播、Portal/Goose 游标映射与 Finish 终态恢复 | [docs/regression-guards/h5-session-stream-replay.md](docs/regression-guards/h5-session-stream-replay.md) |
| Memory V2 候选表初始化与生命周期灰度作用域 | [docs/regression-guards/memory-v2-candidate-and-lifecycle.md](docs/regression-guards/memory-v2-candidate-and-lifecycle.md) |
| MindSpace SEO/GEO 收录策略与私有页 noindex | [docs/regression-guards/mindspace-seo-geo.md](docs/regression-guards/mindspace-seo-geo.md) |
| 图片轮次禁用 `read_image` 与工具图片污染会话轮换 | [docs/regression-guards/vision-turn-read-image-isolation.md](docs/regression-guards/vision-turn-read-image-isolation.md) |
索引:[docs/regression-guards/README.md](docs/regression-guards/README.md)
@@ -81,6 +82,7 @@ npm run verify:seo-discovery
- `mindspace-page-sync-service.mjs` + `server.mjs` - remote 模式也必须 sync public HTML
- `session-stream.mjs` + `session-stream-store.mjs` + `tkmind-proxy.mjs` - Portal replay ID 不得直接作为 Goose `Last-Event-ID`
- `mindspace-index-policy.mjs` + `mindspace-seo-geo-delivery.mjs` - 私有页必须 noindex;仅 confirmed public 可收录
- `tkmind-proxy.mjs` + `chat-image-turn-scope.mjs` + `wechat-mp.mjs` - 带图轮次必须摘掉 `read_image`;工具图片污染必须触发会话轮换
代码内搜索 `REGRESSION GUARD` 可定位所有内联说明。
+29
View File
@@ -186,6 +186,35 @@ export function messageContentHasImageUrl(content) {
return content.some((item) => item?.type === 'image_url' && item?.image_url?.url);
}
function toolResultParts(item) {
const result = item?.toolResult ?? item?.tool_result ?? item?.toolResponse ?? item?.tool_response;
const value = result?.value ?? result;
if (Array.isArray(value)) return value;
if (Array.isArray(value?.content)) return value.content;
return [];
}
/**
* REGRESSION GUARD: vision-turn-read-image-isolation
* `read_image` answers with base64 image parts nested in the tool response.
* Those parts never appear as `image_url`, but once Goose persists them a
* text-only chat provider rejects the whole history with
* `unknown variant image_url, expected text`, so they poison the session too.
*/
export function messageContentHasToolImagePart(content) {
if (!Array.isArray(content)) return false;
return content.some((item) => {
const type = String(item?.type ?? '');
if (type !== 'toolResponse' && type !== 'tool_response') return false;
return toolResultParts(item).some((part) => String(part?.type ?? '') === 'image');
});
}
export function conversationHasToolImageContent(conversation) {
if (!Array.isArray(conversation)) return false;
return conversation.some((message) => messageContentHasToolImagePart(message?.content));
}
/**
* Any persisted image_url part will break DeepSeek / other text-only providers.
* User metadata.imageUrls alone is not enough — Goose may have expanded them into
+59
View File
@@ -3,6 +3,7 @@ import test from 'node:test';
import {
buildCurrentTurnImageScopeNote,
conversationHasImageUrlContent,
conversationHasToolImageContent,
dedupeImageUrlsByAssetKey,
extractCurrentTurnImageUrls,
scrubConversationHistoricalImageAttachments,
@@ -183,6 +184,64 @@ test('conversationHasImageUrlContent detects historical poison and ignores activ
);
});
test('conversationHasToolImageContent detects read_image base64 poison', () => {
const readImageTurn = [
{
id: 'assistant-read',
role: 'assistant',
content: [
{
type: 'toolRequest',
id: 'call_1',
toolCall: { status: 'success', value: { name: 'read_image', arguments: { source: 'a.jpg' } } },
},
],
},
{
id: 'user-read-result',
role: 'user',
content: [
{
type: 'toolResponse',
id: 'call_1',
toolResult: {
status: 'success',
value: {
content: [
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
],
},
},
},
],
},
];
assert.equal(conversationHasToolImageContent(readImageTurn), true);
// image_url scanning alone cannot see this poison, which is why it needs its own check.
assert.equal(conversationHasImageUrlContent(readImageTurn), false);
assert.equal(
conversationHasToolImageContent([
{
id: 'user-text-tool',
role: 'user',
content: [
{
type: 'toolResponse',
id: 'call_2',
toolResult: { status: 'success', value: { content: [{ type: 'text', text: '[文件] a.jpg' }] } },
},
],
},
]),
false,
);
assert.equal(conversationHasToolImageContent([]), false);
assert.equal(conversationHasToolImageContent(null), false);
});
test('detachCurrentTurnImagesForTextProvider archives urls and keeps the VL note', () => {
const detached = detachCurrentTurnImagesForTextProvider(
{
+1
View File
@@ -15,6 +15,7 @@
| [memory-v2-candidate-and-lifecycle.md](./memory-v2-candidate-and-lifecycle.md) | 候选记忆表幂等初始化、Portal fail-open、生命周期 off/canary/active 作用域 |
| [episodic-history-recall.md](./episodic-history-recall.md) | 历史会话召回、用户隔离、旧快照回退、提示注入与 off/canary/active 灰度 |
| [mindspace-seo-geo.md](./mindspace-seo-geo.md) | 公开页 SEO/GEO 注入、私有页 noindex、sitemap/llms.txt 与百度推送开关 |
| [vision-turn-read-image-isolation.md](./vision-turn-read-image-isolation.md) | 图片轮次禁用 `read_image`、视觉提示硬约束、`read_image` 工具图片污染检测与会话轮换 |
## 自动化
@@ -0,0 +1,69 @@
# 图片轮次读图工具隔离守卫
## 已知故障
2026-08-22 服务号用户唐连发 4 张图后要求「把那几张图片做成主题页面」,连续三次没有拿到页面。
链路是这样断的:
1. `attachRecentMediaForFollowup` 正确把 4 张图挂到了这一轮,Qwen VL 也成功产出了图片描述,
并注入进 Agent 提示。
2. Agent 拿到描述后,仍然逐张调用 `read_image` 去「确认图片内容」。
3. `read_image` 的工具结果不是 `image_url`,而是嵌在 `toolResponse.toolResult.value.content[]`
里的 base64 `image` 块,4 张图累计 1.5MB 以上。Goose 会把它们持久化进会话历史。
4. 下一次请求把整段历史发给纯文本聊天模型(DeepSeek),上游直接返回
`unknown variant image_url, expected text`Agent 还没走到 `write_file` 就中断。
5. 页面交付是 fail-closed 的:没有新落盘的 HTML,就不会发占位链接,用户只看到
「这次页面没有按服务号页面技能真正生成成功」。
原有的两道防线都没拦住:
- `conversationHasImageUrlContent` 只扫 `image_url`,看不见 `read_image` 留下的工具图片块,
所以微信侧的会话轮换不会触发,同一个被污染的会话被反复复用。
- `agent-run-gateway``SESSION_VISUAL_CONTEXT_UNSUPPORTED` 视觉降级是**事后**补救,
只覆盖 H5 Agent Run,服务号回复路径没有对应保护。
## 必须保留的行为
1. 本轮消息带图且已配置图片模型(`llmProviderService.hasVisionKey()`)时,
`prepareSessionReplyBody` 必须以 `disableImageReading` 下发会话策略,
使这一轮的 `developer` 扩展不含 `read_image`。视觉理解由图片模型独占,
聊天模型不承担读图。
2. 不带图的轮次必须保留 `read_image`。禁用是按轮次生效的,不能把会话永久降级。
3. `buildVisionPayload` 注入的提示必须明确写出「本轮不会再提供读图工具 + 禁止调用
`read_image`」,并说明强行读图会让本轮及后续请求全部失败。
4. 图片模型没能产出描述时,提示必须改口为「不要臆造画面细节」,不能继续声称
「依据上面的描述写文案」。此时仍然禁用读图 —— 纯文本模型看不到图,
放开 `read_image` 只会让整轮崩掉,而图片嵌入路径依然可用。
5. `conversationHasToolImageContent` 必须能识别 `toolResponse` / `tool_response`
`toolResult.value.content[].type === 'image'` 的工具图片块,
并覆盖 `toolResult` / `tool_result` / 直接数组等结构变体。
6. `rotateWechatSessionIfImagePolluted` 必须同时检查 `image_url` 污染和工具图片污染,
任一命中都要在回复前换掉会话。历史遗留的被污染会话靠这条自愈。
7. `agent-run-gateway` 的事后视觉降级(`SESSION_VISUAL_CONTEXT_UNSUPPORTED`
新会话 + `disableImageReading: true`)保留为兜底,不得因为新增事前预防而删除。
## 改动前必跑
```bash
node --test tkmind-proxy.test.mjs chat-image-turn-scope.test.mjs wechat-mp.test.mjs agent-run-gateway.test.mjs
npm run verify:h5-session-patches
```
## 相关代码与用例
| 位置 | 作用 |
|------|------|
| `tkmind-proxy.mjs` · `prepareSessionReplyBody` | 事前禁用本轮 `read_image` |
| `tkmind-proxy.mjs` · `buildVisionPayload` | 注入禁止读图的硬性提示 |
| `chat-image-turn-scope.mjs` · `conversationHasToolImageContent` | 识别 `read_image` 工具图片污染 |
| `wechat-mp.mjs` · `rotateWechatSessionIfImagePolluted` | 回复前轮换被污染会话 |
| `capabilities.mjs` · `withoutSessionImageRead` | 从会话策略里摘掉 `read_image` |
| 用例 | 覆盖 |
|------|------|
| `tkmind-proxy.test.mjs` · `image turns drop read_image so vision results cannot poison the text provider` | 带图轮次下发的 `developer` 工具不含 `read_image` |
| `tkmind-proxy.test.mjs` · `text-only turns keep read_image available` | 纯文本轮次不降级 |
| `tkmind-proxy.test.mjs` · `submitSessionReplyForUser applies the shared Qwen vision preprocessing path` | 提示中包含禁止读图约束 |
| `chat-image-turn-scope.test.mjs` · `conversationHasToolImageContent detects read_image base64 poison` | 工具图片块识别,且 `image_url` 扫描确实看不见它 |
| `wechat-mp.test.mjs` · `wechat mp rotates a session poisoned by read_image tool results` | 服务号在工具图片污染时换会话 |
+16 -1
View File
@@ -966,6 +966,12 @@ export async function buildVisionPayload({
(visionDescription
? `Qwen VL 图片描述:\n${visionDescription}\n\n`
: '') +
'视觉检查已由图片模型完成,本轮不会再提供读图工具。'
+ '禁止调用 read_image 或任何读图工具去「确认图片内容」:当前聊天模型不接受图片工具结果,'
+ '强行读图会让本轮及后续请求全部失败。'
+ (visionDescription
? '直接依据上面的描述写文案。\n'
: '本轮没有拿到图片描述时,不要臆造画面细节,按用户文字要求组织内容,并把下面的图片路径原样嵌进页面。\n') +
'写作约束:不得改写图片里人物的年龄、性别、人数或主体关系;如果用户明确要求儿童语气或童趣风格,也只能调整表达方式,不能把图片主体改写成儿童场景。\n' +
`图片 HTML 嵌入路径(直接写入 <img> 标签;以下都是无需 cookie 的公开压缩标准图片,禁止使用 local://、/users/ 私有路径、原图地址或需要登录态的下载链接):\n${pathList}\n` +
'执行要求:必须先调用 load_skill → static-page-publish(每次生成页面都要调用,不可省略),' +
@@ -1979,11 +1985,20 @@ export function createTkmindProxy({
throw err;
}
// REGRESSION GUARD: vision-turn-read-image-isolation — the vision provider
// already describes this turn's images in the injected prompt. Leaving
// read_image available lets the agent re-read the originals and persist
// base64 image parts that the text-only chat provider then rejects for
// every later turn, so drop the tool before the turn starts.
const visionHandlesThisTurn = Boolean(llmProviderService)
&& messageHasImages(userMessage)
&& await llmProviderService.hasVisionKey().catch(() => false);
await reconcileSessionPolicyForUser(userId, sessionId, {
toolMode,
query: firstUserText(userMessage),
forceDeepReasoning,
disableImageReading,
disableImageReading: disableImageReading || visionHandlesThisTurn,
});
await applySessionLlmProvider(sessionId);
await repairSessionToolHistory(sessionId);
+140 -3
View File
@@ -22,6 +22,8 @@ async function withFakeGoosedSession(
const replyBodies = [];
const startBodies = [];
const updateBodies = [];
const addedExtensions = [];
let activeExtensions = [{ name: 'memory', available_tools: [] }];
let activeConversation = conversation;
let server;
@@ -67,9 +69,7 @@ async function withFakeGoosedSession(
}
if (req.method === 'GET' && req.url === '/sessions/session-1/extensions') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
extensions: [{ name: 'memory', available_tools: [] }],
}));
res.end(JSON.stringify({ extensions: activeExtensions }));
return;
}
if (req.method === 'POST' && req.url === '/sessions/session-1/reply') {
@@ -78,6 +78,23 @@ async function withFakeGoosedSession(
res.end(JSON.stringify({ ok: true }));
return;
}
if (
req.method === 'POST'
&& (req.url === '/agent/add_extension'
|| req.url === '/agent/remove_extension'
|| req.url === '/agent/restart')
) {
if (req.url === '/agent/add_extension') {
addedExtensions.push(body);
activeExtensions = [...activeExtensions, body.config];
}
if (req.url === '/agent/remove_extension') {
activeExtensions = activeExtensions.filter((ext) => ext?.name !== body.name);
}
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: true }));
return;
}
if (req.method === 'POST' && req.url === '/agent/harness_remember') {
harnessEntries.push(body);
res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -102,6 +119,7 @@ async function withFakeGoosedSession(
replyBodies,
startBodies,
updateBodies,
addedExtensions,
});
} finally {
if (server?.listening) {
@@ -1707,11 +1725,130 @@ test('submitSessionReplyForUser applies the shared Qwen vision preprocessing pat
const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? '';
assert.match(forwardedText, /Qwen VL 图片描述/);
assert.match(forwardedText, /一件蓝色产品/);
assert.match(forwardedText, /禁止调用 read_image/);
assert.equal(replyBodies[0]?.user_message?.metadata?.imageUrls, undefined);
assert.ok(Array.isArray(replyBodies[0]?.user_message?.metadata?.archivedImageUrls));
});
});
function createReadImagePolicyUserAuth(workingDir) {
return {
...createMemoryTestUserAuth(workingDir),
async getAgentSessionPolicy() {
return {
gooseMode: 'chat',
enableContextMemory: true,
extensionOverrides: [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['write', 'edit', 'read_image'] },
],
};
},
async ownsSession() {
return true;
},
async canUseChat() {
return { ok: true };
},
async getUserById() {
return { id: 'user-1' };
},
async getUserPublishLayout() {
return { publicUrl: 'https://example.com/MindSpace/user-1' };
},
async resolveUserPolicies() {
return { unrestricted: true, policies: {} };
},
};
}
function developerToolsFromAddedExtensions(addedExtensions) {
const developer = addedExtensions
.map((body) => body?.config)
.find((config) => config?.name === 'developer');
return developer?.available_tools ?? null;
}
test('image turns drop read_image so vision results cannot poison the text provider', async () => {
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
const proxy = createTkmindProxy({
apiTarget,
apiSecret: 'test-secret',
userAuth: createReadImagePolicyUserAuth(workingDir),
localFetchAsset: async () => ({
buffer: Buffer.from('fake-image'),
mimeType: 'image/jpeg',
}),
llmProviderService: {
async applyBestProviderForSession() {
return { ok: true };
},
async hasVisionKey() {
return true;
},
async analyzeImagesWithVision() {
return '四张汉服写真。';
},
},
});
await proxy.submitSessionReplyForUser(
'user-1',
'session-1',
'request-image-turn',
{
id: 'message-image-turn',
role: 'user',
content: [{ type: 'text', text: '用刚才发的4张图片做成主题页面' }],
metadata: {
imageUrls: [
'/api/mindspace/v1/assets/asset-1/download?inline=1',
'/api/mindspace/v1/assets/asset-2/download?inline=1',
],
},
},
);
const developerTools = developerToolsFromAddedExtensions(addedExtensions);
assert.ok(developerTools, 'developer extension should still be provisioned');
assert.deepEqual(developerTools, ['write', 'edit']);
});
});
test('text-only turns keep read_image available', async () => {
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
const proxy = createTkmindProxy({
apiTarget,
apiSecret: 'test-secret',
userAuth: createReadImagePolicyUserAuth(workingDir),
llmProviderService: {
async applyBestProviderForSession() {
return { ok: true };
},
async hasVisionKey() {
return true;
},
},
});
await proxy.submitSessionReplyForUser(
'user-1',
'session-1',
'request-text-turn',
{
id: 'message-text-turn',
role: 'user',
content: [{ type: 'text', text: '把页面标题改成星尘之门' }],
},
);
assert.deepEqual(
developerToolsFromAddedExtensions(addedExtensions),
['write', 'edit', 'read_image'],
);
});
});
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
let resolveInput = null;
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {
+13 -2
View File
@@ -85,7 +85,10 @@ import { resolveMindSpaceUserPublishDir } from './mindspace-runtime-config.mjs';
import {
buildPageDataCollectFailureText,
} from './mindspace-page-data-finish-guard.mjs';
import { conversationHasImageUrlContent } from './chat-image-turn-scope.mjs';
import {
conversationHasImageUrlContent,
conversationHasToolImageContent,
} from './chat-image-turn-scope.mjs';
export { buildWechatAgentPrompt };
@@ -2450,12 +2453,20 @@ export function createWechatMpService({
}
const payload = await readJsonResponse(response);
const conversation = Array.isArray(payload?.conversation) ? payload.conversation : [];
if (!conversationHasImageUrlContent(conversation)) {
const hasImageUrlContent = conversationHasImageUrlContent(conversation);
// REGRESSION GUARD: vision-turn-read-image-isolation — read_image leaves
// base64 image parts inside tool responses instead of image_url. Goose
// keeps them, so the next turn against a text-only chat provider fails
// before the agent can write any page.
const hasToolImageContent = conversationHasToolImageContent(conversation);
if (!hasImageUrlContent && !hasToolImageContent) {
return { sessionId, carriedSessionContent, rotated: false };
}
logger.warn?.('WeChat MP rotating image-polluted agent session before reply:', {
agentSessionId: sessionId,
conversationLength: conversation.length,
hasImageUrlContent,
hasToolImageContent,
});
const nextRoute = await ensureWechatAgentSession({
userId,
+128
View File
@@ -5692,6 +5692,134 @@ test('wechat mp rotates polluted image session before a content-edit follow-up',
assert.equal(activeSessionId, submitCalls[0].sessionId);
});
test('wechat mp rotates a session poisoned by read_image tool results', async () => {
const token = 'token';
const timestamp = '1710000000';
const nonce = 'nonce';
const testUserId = 'test-user-read-image-rotate';
const submitCalls = [];
let activeSessionId = 'session-1';
let nextSessionId = 2;
let routeCleared = false;
// read_image hides base64 payloads inside the tool response instead of image_url,
// so the older image_url-only scan let this session keep failing every turn.
const pollutedConversation = [
{
id: 'assistant-read',
role: 'assistant',
content: [
{
type: 'toolRequest',
id: 'call_1',
toolCall: { status: 'success', value: { name: 'read_image', arguments: { path: 'a.jpg' } } },
},
],
},
{
id: 'user-read-result',
role: 'user',
content: [
{
type: 'toolResponse',
id: 'call_1',
toolResult: {
status: 'success',
value: {
content: [
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
],
},
},
},
],
},
];
const service = createBoundWechatService({
token,
config: {
mediaAnalysisGrayUsers: [testUserId],
},
userAuth: {
async findWechatUserByOpenid() {
return { userId: testUserId, status: 'active', nickname: '唐' };
},
async getWechatAgentRoute() {
if (routeCleared || !activeSessionId) return null;
return { agentSessionId: activeSessionId, status: 'active', updatedAt: Date.now() };
},
async upsertWechatAgentRoute({ agentSessionId }) {
activeSessionId = agentSessionId;
routeCleared = false;
},
async clearWechatAgentRoute() {
routeCleared = true;
},
},
startAgentSession: async () => ({ id: `session-${nextSessionId++}` }),
sessionApiFetch: async (sessionId, pathname) => {
if (pathname === `/sessions/${sessionId}`) {
const body = sessionId === 'session-1'
? { conversation: pollutedConversation }
: { conversation: [] };
return new Response(JSON.stringify(body), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
if (pathname === `/sessions/${sessionId}/events`) {
return new Response(
[
'data: {"type":"Message","request_id":"req-read-image","message":{"id":"assistant-2","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"页面已生成。"}]}}\n\n',
'data: {"type":"Finish","request_id":"req-read-image","token_state":{"inputTokens":1,"outputTokens":2}}\n\n',
].join(''),
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
);
}
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
}
throw new Error(`unexpected api path: ${sessionId} ${pathname}`);
},
submitSessionReply: async (input) => {
submitCalls.push(input);
return { ok: true };
},
wechatFetch: async (url) => {
if (String(url).includes('/cgi-bin/stable_token')) {
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
if (String(url).includes('/cgi-bin/message/custom/send')) {
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
});
}
throw new Error(`unexpected wechat url: ${url}`);
},
});
const originalRandomUuid = crypto.randomUUID;
crypto.randomUUID = () => 'req-read-image';
try {
const result = await service.handleInboundMessage(
inboundXml({ msgType: 'text', content: '帮我把内容再详细一点' }),
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
);
await result.task;
} finally {
crypto.randomUUID = originalRandomUuid;
}
assert.equal(submitCalls.length, 1);
assert.notEqual(submitCalls[0].sessionId, 'session-1');
assert.equal(activeSessionId, submitCalls[0].sessionId);
});
test('wechat mp simulates tang image report page and desensitized follow-up', async () => {
const token = 'token';
const timestamp = '1710000000';