fix(vision): stop read_image from poisoning text-provider sessions
Image turns already get a vision-model description injected into the prompt, but the agent kept calling read_image to "confirm" the pictures. Those tool results carry base64 image parts that Goose persists, so every later turn against the text-only chat provider failed with `unknown variant image_url` before the agent could write the page. WeChat page requests therefore fell through to the fail-closed delivery message. Drop read_image for the turn whenever a vision model handles the images, say so explicitly in the injected prompt, and teach the poison scan to recognise tool image parts so already-polluted sessions rotate instead of failing again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -52,6 +52,7 @@ bash scripts/check-release-ready.sh
|
||||
| H5 SSE 断线续播、Portal/Goose 游标映射与 Finish 终态恢复 | [docs/regression-guards/h5-session-stream-replay.md](docs/regression-guards/h5-session-stream-replay.md) |
|
||||
| Memory V2 候选表初始化与生命周期灰度作用域 | [docs/regression-guards/memory-v2-candidate-and-lifecycle.md](docs/regression-guards/memory-v2-candidate-and-lifecycle.md) |
|
||||
| MindSpace SEO/GEO 收录策略与私有页 noindex | [docs/regression-guards/mindspace-seo-geo.md](docs/regression-guards/mindspace-seo-geo.md) |
|
||||
| 图片轮次禁用 `read_image` 与工具图片污染会话轮换 | [docs/regression-guards/vision-turn-read-image-isolation.md](docs/regression-guards/vision-turn-read-image-isolation.md) |
|
||||
|
||||
索引:[docs/regression-guards/README.md](docs/regression-guards/README.md)
|
||||
|
||||
@@ -81,6 +82,7 @@ npm run verify:seo-discovery
|
||||
- `mindspace-page-sync-service.mjs` + `server.mjs` - remote 模式也必须 sync public HTML
|
||||
- `session-stream.mjs` + `session-stream-store.mjs` + `tkmind-proxy.mjs` - Portal replay ID 不得直接作为 Goose `Last-Event-ID`
|
||||
- `mindspace-index-policy.mjs` + `mindspace-seo-geo-delivery.mjs` - 私有页必须 noindex;仅 confirmed public 可收录
|
||||
- `tkmind-proxy.mjs` + `chat-image-turn-scope.mjs` + `wechat-mp.mjs` - 带图轮次必须摘掉 `read_image`;工具图片污染必须触发会话轮换
|
||||
|
||||
代码内搜索 `REGRESSION GUARD` 可定位所有内联说明。
|
||||
|
||||
|
||||
@@ -186,6 +186,35 @@ export function messageContentHasImageUrl(content) {
|
||||
return content.some((item) => item?.type === 'image_url' && item?.image_url?.url);
|
||||
}
|
||||
|
||||
function toolResultParts(item) {
|
||||
const result = item?.toolResult ?? item?.tool_result ?? item?.toolResponse ?? item?.tool_response;
|
||||
const value = result?.value ?? result;
|
||||
if (Array.isArray(value)) return value;
|
||||
if (Array.isArray(value?.content)) return value.content;
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* REGRESSION GUARD: vision-turn-read-image-isolation
|
||||
* `read_image` answers with base64 image parts nested in the tool response.
|
||||
* Those parts never appear as `image_url`, but once Goose persists them a
|
||||
* text-only chat provider rejects the whole history with
|
||||
* `unknown variant image_url, expected text`, so they poison the session too.
|
||||
*/
|
||||
export function messageContentHasToolImagePart(content) {
|
||||
if (!Array.isArray(content)) return false;
|
||||
return content.some((item) => {
|
||||
const type = String(item?.type ?? '');
|
||||
if (type !== 'toolResponse' && type !== 'tool_response') return false;
|
||||
return toolResultParts(item).some((part) => String(part?.type ?? '') === 'image');
|
||||
});
|
||||
}
|
||||
|
||||
export function conversationHasToolImageContent(conversation) {
|
||||
if (!Array.isArray(conversation)) return false;
|
||||
return conversation.some((message) => messageContentHasToolImagePart(message?.content));
|
||||
}
|
||||
|
||||
/**
|
||||
* Any persisted image_url part will break DeepSeek / other text-only providers.
|
||||
* User metadata.imageUrls alone is not enough — Goose may have expanded them into
|
||||
|
||||
@@ -3,6 +3,7 @@ import test from 'node:test';
|
||||
import {
|
||||
buildCurrentTurnImageScopeNote,
|
||||
conversationHasImageUrlContent,
|
||||
conversationHasToolImageContent,
|
||||
dedupeImageUrlsByAssetKey,
|
||||
extractCurrentTurnImageUrls,
|
||||
scrubConversationHistoricalImageAttachments,
|
||||
@@ -183,6 +184,64 @@ test('conversationHasImageUrlContent detects historical poison and ignores activ
|
||||
);
|
||||
});
|
||||
|
||||
test('conversationHasToolImageContent detects read_image base64 poison', () => {
|
||||
const readImageTurn = [
|
||||
{
|
||||
id: 'assistant-read',
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'toolRequest',
|
||||
id: 'call_1',
|
||||
toolCall: { status: 'success', value: { name: 'read_image', arguments: { source: 'a.jpg' } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'user-read-result',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_1',
|
||||
toolResult: {
|
||||
status: 'success',
|
||||
value: {
|
||||
content: [
|
||||
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
|
||||
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
assert.equal(conversationHasToolImageContent(readImageTurn), true);
|
||||
// image_url scanning alone cannot see this poison, which is why it needs its own check.
|
||||
assert.equal(conversationHasImageUrlContent(readImageTurn), false);
|
||||
|
||||
assert.equal(
|
||||
conversationHasToolImageContent([
|
||||
{
|
||||
id: 'user-text-tool',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_2',
|
||||
toolResult: { status: 'success', value: { content: [{ type: 'text', text: '[文件] a.jpg' }] } },
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
false,
|
||||
);
|
||||
assert.equal(conversationHasToolImageContent([]), false);
|
||||
assert.equal(conversationHasToolImageContent(null), false);
|
||||
});
|
||||
|
||||
test('detachCurrentTurnImagesForTextProvider archives urls and keeps the VL note', () => {
|
||||
const detached = detachCurrentTurnImagesForTextProvider(
|
||||
{
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
| [memory-v2-candidate-and-lifecycle.md](./memory-v2-candidate-and-lifecycle.md) | 候选记忆表幂等初始化、Portal fail-open、生命周期 off/canary/active 作用域 |
|
||||
| [episodic-history-recall.md](./episodic-history-recall.md) | 历史会话召回、用户隔离、旧快照回退、提示注入与 off/canary/active 灰度 |
|
||||
| [mindspace-seo-geo.md](./mindspace-seo-geo.md) | 公开页 SEO/GEO 注入、私有页 noindex、sitemap/llms.txt 与百度推送开关 |
|
||||
| [vision-turn-read-image-isolation.md](./vision-turn-read-image-isolation.md) | 图片轮次禁用 `read_image`、视觉提示硬约束、`read_image` 工具图片污染检测与会话轮换 |
|
||||
|
||||
## 自动化
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# 图片轮次读图工具隔离守卫
|
||||
|
||||
## 已知故障
|
||||
|
||||
2026-08-22 服务号用户唐连发 4 张图后要求「把那几张图片做成主题页面」,连续三次没有拿到页面。
|
||||
|
||||
链路是这样断的:
|
||||
|
||||
1. `attachRecentMediaForFollowup` 正确把 4 张图挂到了这一轮,Qwen VL 也成功产出了图片描述,
|
||||
并注入进 Agent 提示。
|
||||
2. Agent 拿到描述后,仍然逐张调用 `read_image` 去「确认图片内容」。
|
||||
3. `read_image` 的工具结果不是 `image_url`,而是嵌在 `toolResponse.toolResult.value.content[]`
|
||||
里的 base64 `image` 块,4 张图累计 1.5MB 以上。Goose 会把它们持久化进会话历史。
|
||||
4. 下一次请求把整段历史发给纯文本聊天模型(DeepSeek),上游直接返回
|
||||
`unknown variant image_url, expected text`,Agent 还没走到 `write_file` 就中断。
|
||||
5. 页面交付是 fail-closed 的:没有新落盘的 HTML,就不会发占位链接,用户只看到
|
||||
「这次页面没有按服务号页面技能真正生成成功」。
|
||||
|
||||
原有的两道防线都没拦住:
|
||||
|
||||
- `conversationHasImageUrlContent` 只扫 `image_url`,看不见 `read_image` 留下的工具图片块,
|
||||
所以微信侧的会话轮换不会触发,同一个被污染的会话被反复复用。
|
||||
- `agent-run-gateway` 的 `SESSION_VISUAL_CONTEXT_UNSUPPORTED` 视觉降级是**事后**补救,
|
||||
只覆盖 H5 Agent Run,服务号回复路径没有对应保护。
|
||||
|
||||
## 必须保留的行为
|
||||
|
||||
1. 本轮消息带图且已配置图片模型(`llmProviderService.hasVisionKey()`)时,
|
||||
`prepareSessionReplyBody` 必须以 `disableImageReading` 下发会话策略,
|
||||
使这一轮的 `developer` 扩展不含 `read_image`。视觉理解由图片模型独占,
|
||||
聊天模型不承担读图。
|
||||
2. 不带图的轮次必须保留 `read_image`。禁用是按轮次生效的,不能把会话永久降级。
|
||||
3. `buildVisionPayload` 注入的提示必须明确写出「本轮不会再提供读图工具 + 禁止调用
|
||||
`read_image`」,并说明强行读图会让本轮及后续请求全部失败。
|
||||
4. 图片模型没能产出描述时,提示必须改口为「不要臆造画面细节」,不能继续声称
|
||||
「依据上面的描述写文案」。此时仍然禁用读图 —— 纯文本模型看不到图,
|
||||
放开 `read_image` 只会让整轮崩掉,而图片嵌入路径依然可用。
|
||||
5. `conversationHasToolImageContent` 必须能识别 `toolResponse` / `tool_response` 里
|
||||
`toolResult.value.content[].type === 'image'` 的工具图片块,
|
||||
并覆盖 `toolResult` / `tool_result` / 直接数组等结构变体。
|
||||
6. `rotateWechatSessionIfImagePolluted` 必须同时检查 `image_url` 污染和工具图片污染,
|
||||
任一命中都要在回复前换掉会话。历史遗留的被污染会话靠这条自愈。
|
||||
7. `agent-run-gateway` 的事后视觉降级(`SESSION_VISUAL_CONTEXT_UNSUPPORTED` →
|
||||
新会话 + `disableImageReading: true`)保留为兜底,不得因为新增事前预防而删除。
|
||||
|
||||
## 改动前必跑
|
||||
|
||||
```bash
|
||||
node --test tkmind-proxy.test.mjs chat-image-turn-scope.test.mjs wechat-mp.test.mjs agent-run-gateway.test.mjs
|
||||
npm run verify:h5-session-patches
|
||||
```
|
||||
|
||||
## 相关代码与用例
|
||||
|
||||
| 位置 | 作用 |
|
||||
|------|------|
|
||||
| `tkmind-proxy.mjs` · `prepareSessionReplyBody` | 事前禁用本轮 `read_image` |
|
||||
| `tkmind-proxy.mjs` · `buildVisionPayload` | 注入禁止读图的硬性提示 |
|
||||
| `chat-image-turn-scope.mjs` · `conversationHasToolImageContent` | 识别 `read_image` 工具图片污染 |
|
||||
| `wechat-mp.mjs` · `rotateWechatSessionIfImagePolluted` | 回复前轮换被污染会话 |
|
||||
| `capabilities.mjs` · `withoutSessionImageRead` | 从会话策略里摘掉 `read_image` |
|
||||
|
||||
| 用例 | 覆盖 |
|
||||
|------|------|
|
||||
| `tkmind-proxy.test.mjs` · `image turns drop read_image so vision results cannot poison the text provider` | 带图轮次下发的 `developer` 工具不含 `read_image` |
|
||||
| `tkmind-proxy.test.mjs` · `text-only turns keep read_image available` | 纯文本轮次不降级 |
|
||||
| `tkmind-proxy.test.mjs` · `submitSessionReplyForUser applies the shared Qwen vision preprocessing path` | 提示中包含禁止读图约束 |
|
||||
| `chat-image-turn-scope.test.mjs` · `conversationHasToolImageContent detects read_image base64 poison` | 工具图片块识别,且 `image_url` 扫描确实看不见它 |
|
||||
| `wechat-mp.test.mjs` · `wechat mp rotates a session poisoned by read_image tool results` | 服务号在工具图片污染时换会话 |
|
||||
+16
-1
@@ -966,6 +966,12 @@ export async function buildVisionPayload({
|
||||
(visionDescription
|
||||
? `Qwen VL 图片描述:\n${visionDescription}\n\n`
|
||||
: '') +
|
||||
'视觉检查已由图片模型完成,本轮不会再提供读图工具。'
|
||||
+ '禁止调用 read_image 或任何读图工具去「确认图片内容」:当前聊天模型不接受图片工具结果,'
|
||||
+ '强行读图会让本轮及后续请求全部失败。'
|
||||
+ (visionDescription
|
||||
? '直接依据上面的描述写文案。\n'
|
||||
: '本轮没有拿到图片描述时,不要臆造画面细节,按用户文字要求组织内容,并把下面的图片路径原样嵌进页面。\n') +
|
||||
'写作约束:不得改写图片里人物的年龄、性别、人数或主体关系;如果用户明确要求儿童语气或童趣风格,也只能调整表达方式,不能把图片主体改写成儿童场景。\n' +
|
||||
`图片 HTML 嵌入路径(直接写入 <img> 标签;以下都是无需 cookie 的公开压缩标准图片,禁止使用 local://、/users/ 私有路径、原图地址或需要登录态的下载链接):\n${pathList}\n` +
|
||||
'执行要求:必须先调用 load_skill → static-page-publish(每次生成页面都要调用,不可省略),' +
|
||||
@@ -1979,11 +1985,20 @@ export function createTkmindProxy({
|
||||
throw err;
|
||||
}
|
||||
|
||||
// REGRESSION GUARD: vision-turn-read-image-isolation — the vision provider
|
||||
// already describes this turn's images in the injected prompt. Leaving
|
||||
// read_image available lets the agent re-read the originals and persist
|
||||
// base64 image parts that the text-only chat provider then rejects for
|
||||
// every later turn, so drop the tool before the turn starts.
|
||||
const visionHandlesThisTurn = Boolean(llmProviderService)
|
||||
&& messageHasImages(userMessage)
|
||||
&& await llmProviderService.hasVisionKey().catch(() => false);
|
||||
|
||||
await reconcileSessionPolicyForUser(userId, sessionId, {
|
||||
toolMode,
|
||||
query: firstUserText(userMessage),
|
||||
forceDeepReasoning,
|
||||
disableImageReading,
|
||||
disableImageReading: disableImageReading || visionHandlesThisTurn,
|
||||
});
|
||||
await applySessionLlmProvider(sessionId);
|
||||
await repairSessionToolHistory(sessionId);
|
||||
|
||||
+140
-3
@@ -22,6 +22,8 @@ async function withFakeGoosedSession(
|
||||
const replyBodies = [];
|
||||
const startBodies = [];
|
||||
const updateBodies = [];
|
||||
const addedExtensions = [];
|
||||
let activeExtensions = [{ name: 'memory', available_tools: [] }];
|
||||
let activeConversation = conversation;
|
||||
let server;
|
||||
|
||||
@@ -67,9 +69,7 @@ async function withFakeGoosedSession(
|
||||
}
|
||||
if (req.method === 'GET' && req.url === '/sessions/session-1/extensions') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
extensions: [{ name: 'memory', available_tools: [] }],
|
||||
}));
|
||||
res.end(JSON.stringify({ extensions: activeExtensions }));
|
||||
return;
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/sessions/session-1/reply') {
|
||||
@@ -78,6 +78,23 @@ async function withFakeGoosedSession(
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
if (
|
||||
req.method === 'POST'
|
||||
&& (req.url === '/agent/add_extension'
|
||||
|| req.url === '/agent/remove_extension'
|
||||
|| req.url === '/agent/restart')
|
||||
) {
|
||||
if (req.url === '/agent/add_extension') {
|
||||
addedExtensions.push(body);
|
||||
activeExtensions = [...activeExtensions, body.config];
|
||||
}
|
||||
if (req.url === '/agent/remove_extension') {
|
||||
activeExtensions = activeExtensions.filter((ext) => ext?.name !== body.name);
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/agent/harness_remember') {
|
||||
harnessEntries.push(body);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
@@ -102,6 +119,7 @@ async function withFakeGoosedSession(
|
||||
replyBodies,
|
||||
startBodies,
|
||||
updateBodies,
|
||||
addedExtensions,
|
||||
});
|
||||
} finally {
|
||||
if (server?.listening) {
|
||||
@@ -1707,11 +1725,130 @@ test('submitSessionReplyForUser applies the shared Qwen vision preprocessing pat
|
||||
const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? '';
|
||||
assert.match(forwardedText, /Qwen VL 图片描述/);
|
||||
assert.match(forwardedText, /一件蓝色产品/);
|
||||
assert.match(forwardedText, /禁止调用 read_image/);
|
||||
assert.equal(replyBodies[0]?.user_message?.metadata?.imageUrls, undefined);
|
||||
assert.ok(Array.isArray(replyBodies[0]?.user_message?.metadata?.archivedImageUrls));
|
||||
});
|
||||
});
|
||||
|
||||
function createReadImagePolicyUserAuth(workingDir) {
|
||||
return {
|
||||
...createMemoryTestUserAuth(workingDir),
|
||||
async getAgentSessionPolicy() {
|
||||
return {
|
||||
gooseMode: 'chat',
|
||||
enableContextMemory: true,
|
||||
extensionOverrides: [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['write', 'edit', 'read_image'] },
|
||||
],
|
||||
};
|
||||
},
|
||||
async ownsSession() {
|
||||
return true;
|
||||
},
|
||||
async canUseChat() {
|
||||
return { ok: true };
|
||||
},
|
||||
async getUserById() {
|
||||
return { id: 'user-1' };
|
||||
},
|
||||
async getUserPublishLayout() {
|
||||
return { publicUrl: 'https://example.com/MindSpace/user-1' };
|
||||
},
|
||||
async resolveUserPolicies() {
|
||||
return { unrestricted: true, policies: {} };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function developerToolsFromAddedExtensions(addedExtensions) {
|
||||
const developer = addedExtensions
|
||||
.map((body) => body?.config)
|
||||
.find((config) => config?.name === 'developer');
|
||||
return developer?.available_tools ?? null;
|
||||
}
|
||||
|
||||
test('image turns drop read_image so vision results cannot poison the text provider', async () => {
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
|
||||
const proxy = createTkmindProxy({
|
||||
apiTarget,
|
||||
apiSecret: 'test-secret',
|
||||
userAuth: createReadImagePolicyUserAuth(workingDir),
|
||||
localFetchAsset: async () => ({
|
||||
buffer: Buffer.from('fake-image'),
|
||||
mimeType: 'image/jpeg',
|
||||
}),
|
||||
llmProviderService: {
|
||||
async applyBestProviderForSession() {
|
||||
return { ok: true };
|
||||
},
|
||||
async hasVisionKey() {
|
||||
return true;
|
||||
},
|
||||
async analyzeImagesWithVision() {
|
||||
return '四张汉服写真。';
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await proxy.submitSessionReplyForUser(
|
||||
'user-1',
|
||||
'session-1',
|
||||
'request-image-turn',
|
||||
{
|
||||
id: 'message-image-turn',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '用刚才发的4张图片做成主题页面' }],
|
||||
metadata: {
|
||||
imageUrls: [
|
||||
'/api/mindspace/v1/assets/asset-1/download?inline=1',
|
||||
'/api/mindspace/v1/assets/asset-2/download?inline=1',
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const developerTools = developerToolsFromAddedExtensions(addedExtensions);
|
||||
assert.ok(developerTools, 'developer extension should still be provisioned');
|
||||
assert.deepEqual(developerTools, ['write', 'edit']);
|
||||
});
|
||||
});
|
||||
|
||||
test('text-only turns keep read_image available', async () => {
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
|
||||
const proxy = createTkmindProxy({
|
||||
apiTarget,
|
||||
apiSecret: 'test-secret',
|
||||
userAuth: createReadImagePolicyUserAuth(workingDir),
|
||||
llmProviderService: {
|
||||
async applyBestProviderForSession() {
|
||||
return { ok: true };
|
||||
},
|
||||
async hasVisionKey() {
|
||||
return true;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await proxy.submitSessionReplyForUser(
|
||||
'user-1',
|
||||
'session-1',
|
||||
'request-text-turn',
|
||||
{
|
||||
id: 'message-text-turn',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '把页面标题改成星尘之门' }],
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
developerToolsFromAddedExtensions(addedExtensions),
|
||||
['write', 'edit', 'read_image'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
|
||||
let resolveInput = null;
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {
|
||||
|
||||
+13
-2
@@ -85,7 +85,10 @@ import { resolveMindSpaceUserPublishDir } from './mindspace-runtime-config.mjs';
|
||||
import {
|
||||
buildPageDataCollectFailureText,
|
||||
} from './mindspace-page-data-finish-guard.mjs';
|
||||
import { conversationHasImageUrlContent } from './chat-image-turn-scope.mjs';
|
||||
import {
|
||||
conversationHasImageUrlContent,
|
||||
conversationHasToolImageContent,
|
||||
} from './chat-image-turn-scope.mjs';
|
||||
|
||||
export { buildWechatAgentPrompt };
|
||||
|
||||
@@ -2450,12 +2453,20 @@ export function createWechatMpService({
|
||||
}
|
||||
const payload = await readJsonResponse(response);
|
||||
const conversation = Array.isArray(payload?.conversation) ? payload.conversation : [];
|
||||
if (!conversationHasImageUrlContent(conversation)) {
|
||||
const hasImageUrlContent = conversationHasImageUrlContent(conversation);
|
||||
// REGRESSION GUARD: vision-turn-read-image-isolation — read_image leaves
|
||||
// base64 image parts inside tool responses instead of image_url. Goose
|
||||
// keeps them, so the next turn against a text-only chat provider fails
|
||||
// before the agent can write any page.
|
||||
const hasToolImageContent = conversationHasToolImageContent(conversation);
|
||||
if (!hasImageUrlContent && !hasToolImageContent) {
|
||||
return { sessionId, carriedSessionContent, rotated: false };
|
||||
}
|
||||
logger.warn?.('WeChat MP rotating image-polluted agent session before reply:', {
|
||||
agentSessionId: sessionId,
|
||||
conversationLength: conversation.length,
|
||||
hasImageUrlContent,
|
||||
hasToolImageContent,
|
||||
});
|
||||
const nextRoute = await ensureWechatAgentSession({
|
||||
userId,
|
||||
|
||||
@@ -5692,6 +5692,134 @@ test('wechat mp rotates polluted image session before a content-edit follow-up',
|
||||
assert.equal(activeSessionId, submitCalls[0].sessionId);
|
||||
});
|
||||
|
||||
test('wechat mp rotates a session poisoned by read_image tool results', async () => {
|
||||
const token = 'token';
|
||||
const timestamp = '1710000000';
|
||||
const nonce = 'nonce';
|
||||
const testUserId = 'test-user-read-image-rotate';
|
||||
const submitCalls = [];
|
||||
let activeSessionId = 'session-1';
|
||||
let nextSessionId = 2;
|
||||
let routeCleared = false;
|
||||
// read_image hides base64 payloads inside the tool response instead of image_url,
|
||||
// so the older image_url-only scan let this session keep failing every turn.
|
||||
const pollutedConversation = [
|
||||
{
|
||||
id: 'assistant-read',
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'toolRequest',
|
||||
id: 'call_1',
|
||||
toolCall: { status: 'success', value: { name: 'read_image', arguments: { path: 'a.jpg' } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'user-read-result',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_1',
|
||||
toolResult: {
|
||||
status: 'success',
|
||||
value: {
|
||||
content: [
|
||||
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
|
||||
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const service = createBoundWechatService({
|
||||
token,
|
||||
config: {
|
||||
mediaAnalysisGrayUsers: [testUserId],
|
||||
},
|
||||
userAuth: {
|
||||
async findWechatUserByOpenid() {
|
||||
return { userId: testUserId, status: 'active', nickname: '唐' };
|
||||
},
|
||||
async getWechatAgentRoute() {
|
||||
if (routeCleared || !activeSessionId) return null;
|
||||
return { agentSessionId: activeSessionId, status: 'active', updatedAt: Date.now() };
|
||||
},
|
||||
async upsertWechatAgentRoute({ agentSessionId }) {
|
||||
activeSessionId = agentSessionId;
|
||||
routeCleared = false;
|
||||
},
|
||||
async clearWechatAgentRoute() {
|
||||
routeCleared = true;
|
||||
},
|
||||
},
|
||||
startAgentSession: async () => ({ id: `session-${nextSessionId++}` }),
|
||||
sessionApiFetch: async (sessionId, pathname) => {
|
||||
if (pathname === `/sessions/${sessionId}`) {
|
||||
const body = sessionId === 'session-1'
|
||||
? { conversation: pollutedConversation }
|
||||
: { conversation: [] };
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (pathname === `/sessions/${sessionId}/events`) {
|
||||
return new Response(
|
||||
[
|
||||
'data: {"type":"Message","request_id":"req-read-image","message":{"id":"assistant-2","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"页面已生成。"}]}}\n\n',
|
||||
'data: {"type":"Finish","request_id":"req-read-image","token_state":{"inputTokens":1,"outputTokens":2}}\n\n',
|
||||
].join(''),
|
||||
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||
);
|
||||
}
|
||||
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||
}
|
||||
throw new Error(`unexpected api path: ${sessionId} ${pathname}`);
|
||||
},
|
||||
submitSessionReply: async (input) => {
|
||||
submitCalls.push(input);
|
||||
return { ok: true };
|
||||
},
|
||||
wechatFetch: async (url) => {
|
||||
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected wechat url: ${url}`);
|
||||
},
|
||||
});
|
||||
|
||||
const originalRandomUuid = crypto.randomUUID;
|
||||
crypto.randomUUID = () => 'req-read-image';
|
||||
try {
|
||||
const result = await service.handleInboundMessage(
|
||||
inboundXml({ msgType: 'text', content: '帮我把内容再详细一点' }),
|
||||
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||
);
|
||||
await result.task;
|
||||
} finally {
|
||||
crypto.randomUUID = originalRandomUuid;
|
||||
}
|
||||
|
||||
assert.equal(submitCalls.length, 1);
|
||||
assert.notEqual(submitCalls[0].sessionId, 'session-1');
|
||||
assert.equal(activeSessionId, submitCalls[0].sessionId);
|
||||
});
|
||||
|
||||
test('wechat mp simulates tang image report page and desensitized follow-up', async () => {
|
||||
const token = 'token';
|
||||
const timestamp = '1710000000';
|
||||
|
||||
Reference in New Issue
Block a user