fix(vision): stop read_image from poisoning text-provider sessions

Image turns already get a vision-model description injected into the prompt,
but the agent kept calling read_image to "confirm" the pictures. Those tool
results carry base64 image parts that Goose persists, so every later turn
against the text-only chat provider failed with `unknown variant image_url`
before the agent could write the page. WeChat page requests therefore fell
through to the fail-closed delivery message.

Drop read_image for the turn whenever a vision model handles the images, say
so explicitly in the injected prompt, and teach the poison scan to recognise
tool image parts so already-polluted sessions rotate instead of failing again.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-22 10:40:12 +08:00
parent 4d12ea438b
commit ef4ce12bbf
9 changed files with 457 additions and 6 deletions
+140 -3
View File
@@ -22,6 +22,8 @@ async function withFakeGoosedSession(
const replyBodies = [];
const startBodies = [];
const updateBodies = [];
const addedExtensions = [];
let activeExtensions = [{ name: 'memory', available_tools: [] }];
let activeConversation = conversation;
let server;
@@ -67,9 +69,7 @@ async function withFakeGoosedSession(
}
if (req.method === 'GET' && req.url === '/sessions/session-1/extensions') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
extensions: [{ name: 'memory', available_tools: [] }],
}));
res.end(JSON.stringify({ extensions: activeExtensions }));
return;
}
if (req.method === 'POST' && req.url === '/sessions/session-1/reply') {
@@ -78,6 +78,23 @@ async function withFakeGoosedSession(
res.end(JSON.stringify({ ok: true }));
return;
}
if (
req.method === 'POST'
&& (req.url === '/agent/add_extension'
|| req.url === '/agent/remove_extension'
|| req.url === '/agent/restart')
) {
if (req.url === '/agent/add_extension') {
addedExtensions.push(body);
activeExtensions = [...activeExtensions, body.config];
}
if (req.url === '/agent/remove_extension') {
activeExtensions = activeExtensions.filter((ext) => ext?.name !== body.name);
}
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: true }));
return;
}
if (req.method === 'POST' && req.url === '/agent/harness_remember') {
harnessEntries.push(body);
res.writeHead(200, { 'Content-Type': 'application/json' });
@@ -102,6 +119,7 @@ async function withFakeGoosedSession(
replyBodies,
startBodies,
updateBodies,
addedExtensions,
});
} finally {
if (server?.listening) {
@@ -1707,11 +1725,130 @@ test('submitSessionReplyForUser applies the shared Qwen vision preprocessing pat
const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? '';
assert.match(forwardedText, /Qwen VL 图片描述/);
assert.match(forwardedText, /一件蓝色产品/);
assert.match(forwardedText, /禁止调用 read_image/);
assert.equal(replyBodies[0]?.user_message?.metadata?.imageUrls, undefined);
assert.ok(Array.isArray(replyBodies[0]?.user_message?.metadata?.archivedImageUrls));
});
});
function createReadImagePolicyUserAuth(workingDir) {
return {
...createMemoryTestUserAuth(workingDir),
async getAgentSessionPolicy() {
return {
gooseMode: 'chat',
enableContextMemory: true,
extensionOverrides: [
{ name: 'memory', available_tools: [] },
{ name: 'developer', available_tools: ['write', 'edit', 'read_image'] },
],
};
},
async ownsSession() {
return true;
},
async canUseChat() {
return { ok: true };
},
async getUserById() {
return { id: 'user-1' };
},
async getUserPublishLayout() {
return { publicUrl: 'https://example.com/MindSpace/user-1' };
},
async resolveUserPolicies() {
return { unrestricted: true, policies: {} };
},
};
}
function developerToolsFromAddedExtensions(addedExtensions) {
const developer = addedExtensions
.map((body) => body?.config)
.find((config) => config?.name === 'developer');
return developer?.available_tools ?? null;
}
test('image turns drop read_image so vision results cannot poison the text provider', async () => {
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
const proxy = createTkmindProxy({
apiTarget,
apiSecret: 'test-secret',
userAuth: createReadImagePolicyUserAuth(workingDir),
localFetchAsset: async () => ({
buffer: Buffer.from('fake-image'),
mimeType: 'image/jpeg',
}),
llmProviderService: {
async applyBestProviderForSession() {
return { ok: true };
},
async hasVisionKey() {
return true;
},
async analyzeImagesWithVision() {
return '四张汉服写真。';
},
},
});
await proxy.submitSessionReplyForUser(
'user-1',
'session-1',
'request-image-turn',
{
id: 'message-image-turn',
role: 'user',
content: [{ type: 'text', text: '用刚才发的4张图片做成主题页面' }],
metadata: {
imageUrls: [
'/api/mindspace/v1/assets/asset-1/download?inline=1',
'/api/mindspace/v1/assets/asset-2/download?inline=1',
],
},
},
);
const developerTools = developerToolsFromAddedExtensions(addedExtensions);
assert.ok(developerTools, 'developer extension should still be provisioned');
assert.deepEqual(developerTools, ['write', 'edit']);
});
});
test('text-only turns keep read_image available', async () => {
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
const proxy = createTkmindProxy({
apiTarget,
apiSecret: 'test-secret',
userAuth: createReadImagePolicyUserAuth(workingDir),
llmProviderService: {
async applyBestProviderForSession() {
return { ok: true };
},
async hasVisionKey() {
return true;
},
},
});
await proxy.submitSessionReplyForUser(
'user-1',
'session-1',
'request-text-turn',
{
id: 'message-text-turn',
role: 'user',
content: [{ type: 'text', text: '把页面标题改成星尘之门' }],
},
);
assert.deepEqual(
developerToolsFromAddedExtensions(addedExtensions),
['write', 'edit', 'read_image'],
);
});
});
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
let resolveInput = null;
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {