fix(vision): stop read_image from poisoning text-provider sessions
Image turns already get a vision-model description injected into the prompt, but the agent kept calling read_image to "confirm" the pictures. Those tool results carry base64 image parts that Goose persists, so every later turn against the text-only chat provider failed with `unknown variant image_url` before the agent could write the page. WeChat page requests therefore fell through to the fail-closed delivery message. Drop read_image for the turn whenever a vision model handles the images, say so explicitly in the injected prompt, and teach the poison scan to recognise tool image parts so already-polluted sessions rotate instead of failing again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+140
-3
@@ -22,6 +22,8 @@ async function withFakeGoosedSession(
|
||||
const replyBodies = [];
|
||||
const startBodies = [];
|
||||
const updateBodies = [];
|
||||
const addedExtensions = [];
|
||||
let activeExtensions = [{ name: 'memory', available_tools: [] }];
|
||||
let activeConversation = conversation;
|
||||
let server;
|
||||
|
||||
@@ -67,9 +69,7 @@ async function withFakeGoosedSession(
|
||||
}
|
||||
if (req.method === 'GET' && req.url === '/sessions/session-1/extensions') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
extensions: [{ name: 'memory', available_tools: [] }],
|
||||
}));
|
||||
res.end(JSON.stringify({ extensions: activeExtensions }));
|
||||
return;
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/sessions/session-1/reply') {
|
||||
@@ -78,6 +78,23 @@ async function withFakeGoosedSession(
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
if (
|
||||
req.method === 'POST'
|
||||
&& (req.url === '/agent/add_extension'
|
||||
|| req.url === '/agent/remove_extension'
|
||||
|| req.url === '/agent/restart')
|
||||
) {
|
||||
if (req.url === '/agent/add_extension') {
|
||||
addedExtensions.push(body);
|
||||
activeExtensions = [...activeExtensions, body.config];
|
||||
}
|
||||
if (req.url === '/agent/remove_extension') {
|
||||
activeExtensions = activeExtensions.filter((ext) => ext?.name !== body.name);
|
||||
}
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
if (req.method === 'POST' && req.url === '/agent/harness_remember') {
|
||||
harnessEntries.push(body);
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
@@ -102,6 +119,7 @@ async function withFakeGoosedSession(
|
||||
replyBodies,
|
||||
startBodies,
|
||||
updateBodies,
|
||||
addedExtensions,
|
||||
});
|
||||
} finally {
|
||||
if (server?.listening) {
|
||||
@@ -1707,11 +1725,130 @@ test('submitSessionReplyForUser applies the shared Qwen vision preprocessing pat
|
||||
const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? '';
|
||||
assert.match(forwardedText, /Qwen VL 图片描述/);
|
||||
assert.match(forwardedText, /一件蓝色产品/);
|
||||
assert.match(forwardedText, /禁止调用 read_image/);
|
||||
assert.equal(replyBodies[0]?.user_message?.metadata?.imageUrls, undefined);
|
||||
assert.ok(Array.isArray(replyBodies[0]?.user_message?.metadata?.archivedImageUrls));
|
||||
});
|
||||
});
|
||||
|
||||
function createReadImagePolicyUserAuth(workingDir) {
|
||||
return {
|
||||
...createMemoryTestUserAuth(workingDir),
|
||||
async getAgentSessionPolicy() {
|
||||
return {
|
||||
gooseMode: 'chat',
|
||||
enableContextMemory: true,
|
||||
extensionOverrides: [
|
||||
{ name: 'memory', available_tools: [] },
|
||||
{ name: 'developer', available_tools: ['write', 'edit', 'read_image'] },
|
||||
],
|
||||
};
|
||||
},
|
||||
async ownsSession() {
|
||||
return true;
|
||||
},
|
||||
async canUseChat() {
|
||||
return { ok: true };
|
||||
},
|
||||
async getUserById() {
|
||||
return { id: 'user-1' };
|
||||
},
|
||||
async getUserPublishLayout() {
|
||||
return { publicUrl: 'https://example.com/MindSpace/user-1' };
|
||||
},
|
||||
async resolveUserPolicies() {
|
||||
return { unrestricted: true, policies: {} };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function developerToolsFromAddedExtensions(addedExtensions) {
|
||||
const developer = addedExtensions
|
||||
.map((body) => body?.config)
|
||||
.find((config) => config?.name === 'developer');
|
||||
return developer?.available_tools ?? null;
|
||||
}
|
||||
|
||||
test('image turns drop read_image so vision results cannot poison the text provider', async () => {
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
|
||||
const proxy = createTkmindProxy({
|
||||
apiTarget,
|
||||
apiSecret: 'test-secret',
|
||||
userAuth: createReadImagePolicyUserAuth(workingDir),
|
||||
localFetchAsset: async () => ({
|
||||
buffer: Buffer.from('fake-image'),
|
||||
mimeType: 'image/jpeg',
|
||||
}),
|
||||
llmProviderService: {
|
||||
async applyBestProviderForSession() {
|
||||
return { ok: true };
|
||||
},
|
||||
async hasVisionKey() {
|
||||
return true;
|
||||
},
|
||||
async analyzeImagesWithVision() {
|
||||
return '四张汉服写真。';
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await proxy.submitSessionReplyForUser(
|
||||
'user-1',
|
||||
'session-1',
|
||||
'request-image-turn',
|
||||
{
|
||||
id: 'message-image-turn',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '用刚才发的4张图片做成主题页面' }],
|
||||
metadata: {
|
||||
imageUrls: [
|
||||
'/api/mindspace/v1/assets/asset-1/download?inline=1',
|
||||
'/api/mindspace/v1/assets/asset-2/download?inline=1',
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const developerTools = developerToolsFromAddedExtensions(addedExtensions);
|
||||
assert.ok(developerTools, 'developer extension should still be provisioned');
|
||||
assert.deepEqual(developerTools, ['write', 'edit']);
|
||||
});
|
||||
});
|
||||
|
||||
test('text-only turns keep read_image available', async () => {
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir, addedExtensions }) => {
|
||||
const proxy = createTkmindProxy({
|
||||
apiTarget,
|
||||
apiSecret: 'test-secret',
|
||||
userAuth: createReadImagePolicyUserAuth(workingDir),
|
||||
llmProviderService: {
|
||||
async applyBestProviderForSession() {
|
||||
return { ok: true };
|
||||
},
|
||||
async hasVisionKey() {
|
||||
return true;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await proxy.submitSessionReplyForUser(
|
||||
'user-1',
|
||||
'session-1',
|
||||
'request-text-turn',
|
||||
{
|
||||
id: 'message-text-turn',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '把页面标题改成星尘之门' }],
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
developerToolsFromAddedExtensions(addedExtensions),
|
||||
['write', 'edit', 'read_image'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
|
||||
let resolveInput = null;
|
||||
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {
|
||||
|
||||
Reference in New Issue
Block a user