fix(vision): stop read_image from poisoning text-provider sessions
Image turns already get a vision-model description injected into the prompt, but the agent kept calling read_image to "confirm" the pictures. Those tool results carry base64 image parts that Goose persists, so every later turn against the text-only chat provider failed with `unknown variant image_url` before the agent could write the page. WeChat page requests therefore fell through to the fail-closed delivery message. Drop read_image for the turn whenever a vision model handles the images, say so explicitly in the injected prompt, and teach the poison scan to recognise tool image parts so already-polluted sessions rotate instead of failing again. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -5692,6 +5692,134 @@ test('wechat mp rotates polluted image session before a content-edit follow-up',
|
||||
assert.equal(activeSessionId, submitCalls[0].sessionId);
|
||||
});
|
||||
|
||||
test('wechat mp rotates a session poisoned by read_image tool results', async () => {
|
||||
const token = 'token';
|
||||
const timestamp = '1710000000';
|
||||
const nonce = 'nonce';
|
||||
const testUserId = 'test-user-read-image-rotate';
|
||||
const submitCalls = [];
|
||||
let activeSessionId = 'session-1';
|
||||
let nextSessionId = 2;
|
||||
let routeCleared = false;
|
||||
// read_image hides base64 payloads inside the tool response instead of image_url,
|
||||
// so the older image_url-only scan let this session keep failing every turn.
|
||||
const pollutedConversation = [
|
||||
{
|
||||
id: 'assistant-read',
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'toolRequest',
|
||||
id: 'call_1',
|
||||
toolCall: { status: 'success', value: { name: 'read_image', arguments: { path: 'a.jpg' } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'user-read-result',
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'toolResponse',
|
||||
id: 'call_1',
|
||||
toolResult: {
|
||||
status: 'success',
|
||||
value: {
|
||||
content: [
|
||||
{ type: 'text', text: 'Loaded image from a.jpg (202672 bytes, image/jpeg, 1280x1707).' },
|
||||
{ type: 'image', data: '/9j/4AAQSkZJRg==', mimeType: 'image/jpeg' },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const service = createBoundWechatService({
|
||||
token,
|
||||
config: {
|
||||
mediaAnalysisGrayUsers: [testUserId],
|
||||
},
|
||||
userAuth: {
|
||||
async findWechatUserByOpenid() {
|
||||
return { userId: testUserId, status: 'active', nickname: '唐' };
|
||||
},
|
||||
async getWechatAgentRoute() {
|
||||
if (routeCleared || !activeSessionId) return null;
|
||||
return { agentSessionId: activeSessionId, status: 'active', updatedAt: Date.now() };
|
||||
},
|
||||
async upsertWechatAgentRoute({ agentSessionId }) {
|
||||
activeSessionId = agentSessionId;
|
||||
routeCleared = false;
|
||||
},
|
||||
async clearWechatAgentRoute() {
|
||||
routeCleared = true;
|
||||
},
|
||||
},
|
||||
startAgentSession: async () => ({ id: `session-${nextSessionId++}` }),
|
||||
sessionApiFetch: async (sessionId, pathname) => {
|
||||
if (pathname === `/sessions/${sessionId}`) {
|
||||
const body = sessionId === 'session-1'
|
||||
? { conversation: pollutedConversation }
|
||||
: { conversation: [] };
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (pathname === `/sessions/${sessionId}/events`) {
|
||||
return new Response(
|
||||
[
|
||||
'data: {"type":"Message","request_id":"req-read-image","message":{"id":"assistant-2","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"页面已生成。"}]}}\n\n',
|
||||
'data: {"type":"Finish","request_id":"req-read-image","token_state":{"inputTokens":1,"outputTokens":2}}\n\n',
|
||||
].join(''),
|
||||
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||
);
|
||||
}
|
||||
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||
}
|
||||
throw new Error(`unexpected api path: ${sessionId} ${pathname}`);
|
||||
},
|
||||
submitSessionReply: async (input) => {
|
||||
submitCalls.push(input);
|
||||
return { ok: true };
|
||||
},
|
||||
wechatFetch: async (url) => {
|
||||
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected wechat url: ${url}`);
|
||||
},
|
||||
});
|
||||
|
||||
const originalRandomUuid = crypto.randomUUID;
|
||||
crypto.randomUUID = () => 'req-read-image';
|
||||
try {
|
||||
const result = await service.handleInboundMessage(
|
||||
inboundXml({ msgType: 'text', content: '帮我把内容再详细一点' }),
|
||||
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||
);
|
||||
await result.task;
|
||||
} finally {
|
||||
crypto.randomUUID = originalRandomUuid;
|
||||
}
|
||||
|
||||
assert.equal(submitCalls.length, 1);
|
||||
assert.notEqual(submitCalls[0].sessionId, 'session-1');
|
||||
assert.equal(activeSessionId, submitCalls[0].sessionId);
|
||||
});
|
||||
|
||||
test('wechat mp simulates tang image report page and desensitized follow-up', async () => {
|
||||
const token = 'token';
|
||||
const timestamp = '1710000000';
|
||||
|
||||
Reference in New Issue
Block a user