fix(vision): stop read_image from poisoning text-provider sessions
Image turns already get a vision-model description injected into the prompt, but the agent kept calling read_image to "confirm" the pictures. Those tool results carry base64 image parts that Goose persists, so every later turn against the text-only chat provider failed with `unknown variant image_url` before the agent could write the page. WeChat page requests therefore fell through to the fail-closed delivery message. Drop read_image for the turn whenever a vision model handles the images, say so explicitly in the injected prompt, and teach the poison scan to recognise tool image parts so already-polluted sessions rotate instead of failing again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -3,6 +3,7 @@ import test from 'node:test';
|
||||
import {
|
||||
buildCurrentTurnImageScopeNote,
|
||||
conversationHasImageUrlContent,
|
||||
conversationHasToolImageContent,
|
||||
dedupeImageUrlsByAssetKey,
|
||||
extractCurrentTurnImageUrls,
|
||||
scrubConversationHistoricalImageAttachments,
|
||||
@@ -183,6 +184,64 @@ test('conversationHasImageUrlContent detects historical poison and ignores activ
|
||||
);
|
||||
});
|
||||
|
||||
test('conversationHasToolImageContent detects read_image base64 poison', () => {
|
||||
const readImageTurn = [
|
||||
{
|
||||
id: 'assistant-read',
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'toolRequest',
|
||||
id: 'call_1',
|
||||
toolCall: { status: 'success', value: { name: 'read_image', arguments: { source: 'a.jpg' } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'user-read-result',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_1',
|
||||
toolResult: {
|
||||
status: 'success',
|
||||
value: {
|
||||
content: [
|
||||
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
|
||||
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
assert.equal(conversationHasToolImageContent(readImageTurn), true);
|
||||
// image_url scanning alone cannot see this poison, which is why it needs its own check.
|
||||
assert.equal(conversationHasImageUrlContent(readImageTurn), false);
|
||||
|
||||
assert.equal(
|
||||
conversationHasToolImageContent([
|
||||
{
|
||||
id: 'user-text-tool',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_2',
|
||||
toolResult: { status: 'success', value: { content: [{ type: 'text', text: '[文件] a.jpg' }] } },
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
false,
|
||||
);
|
||||
assert.equal(conversationHasToolImageContent([]), false);
|
||||
assert.equal(conversationHasToolImageContent(null), false);
|
||||
});
|
||||
|
||||
test('detachCurrentTurnImagesForTextProvider archives urls and keeps the VL note', () => {
|
||||
const detached = detachCurrentTurnImagesForTextProvider(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user