fix(vision): stop read_image from poisoning text-provider sessions

Image turns already get a vision-model description injected into the prompt,
but the agent kept calling read_image to "confirm" the pictures. Those tool
results carry base64 image parts that Goose persists, so every later turn
against the text-only chat provider failed with `unknown variant image_url`
before the agent could write the page. WeChat page requests therefore fell
through to the fail-closed delivery message.

Drop read_image for the turn whenever a vision model handles the images, say
so explicitly in the injected prompt, and teach the poison scan to recognise
tool image parts so already-polluted sessions rotate instead of failing again.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-22 10:40:12 +08:00
parent 4d12ea438b
commit ef4ce12bbf
9 changed files with 457 additions and 6 deletions
+59
View File
@@ -3,6 +3,7 @@ import test from 'node:test';
import {
buildCurrentTurnImageScopeNote,
conversationHasImageUrlContent,
conversationHasToolImageContent,
dedupeImageUrlsByAssetKey,
extractCurrentTurnImageUrls,
scrubConversationHistoricalImageAttachments,
@@ -183,6 +184,64 @@ test('conversationHasImageUrlContent detects historical poison and ignores activ
);
});
test('conversationHasToolImageContent detects read_image base64 poison', () => {
const readImageTurn = [
{
id: 'assistant-read',
role: 'assistant',
content: [
{
type: 'toolRequest',
id: 'call_1',
toolCall: { status: 'success', value: { name: 'read_image', arguments: { source: 'a.jpg' } } },
},
],
},
{
id: 'user-read-result',
role: 'user',
content: [
{
type: 'toolResponse',
id: 'call_1',
toolResult: {
status: 'success',
value: {
content: [
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
],
},
},
},
],
},
];
assert.equal(conversationHasToolImageContent(readImageTurn), true);
// image_url scanning alone cannot see this poison, which is why it needs its own check.
assert.equal(conversationHasImageUrlContent(readImageTurn), false);
assert.equal(
conversationHasToolImageContent([
{
id: 'user-text-tool',
role: 'user',
content: [
{
type: 'toolResponse',
id: 'call_2',
toolResult: { status: 'success', value: { content: [{ type: 'text', text: '[文件] a.jpg' }] } },
},
],
},
]),
false,
);
assert.equal(conversationHasToolImageContent([]), false);
assert.equal(conversationHasToolImageContent(null), false);
});
test('detachCurrentTurnImagesForTextProvider archives urls and keeps the VL note', () => {
const detached = detachCurrentTurnImagesForTextProvider(
{