From 41f245c88ac1e19b64b3e8cba0eba21454de88d3 Mon Sep 17 00:00:00 2001 From: john Date: Fri, 14 Aug 2026 21:45:02 +0800 Subject: [PATCH] fix(wechat): rotate image_url-poisoned sessions for report follow-ups Goose cannot persist historical image scrub (PUT 405), so DeepSeek rejects image_url on the next turn. Reattach the recent report image and rotate to a fresh session instead of treating the follow-up as a missing page source. Co-authored-by: Cursor --- chat-image-turn-scope.mjs | 46 +++++++ chat-image-turn-scope.test.mjs | 38 ++++++ tkmind-proxy-vision.test.mjs | 9 +- tkmind-proxy.mjs | 20 +-- tkmind-proxy.test.mjs | 2 + wechat-mp.mjs | 36 +++-- wechat-mp.test.mjs | 238 +++++++++++++++++++++++++++++++++ 7 files changed, 370 insertions(+), 19 deletions(-) diff --git a/chat-image-turn-scope.mjs b/chat-image-turn-scope.mjs index fd207cb..30e0baf 100644 --- a/chat-image-turn-scope.mjs +++ b/chat-image-turn-scope.mjs @@ -75,6 +75,52 @@ function stripAgentImageText(text) { .trim(); } +function stripImageUrlLines(text) { + return String(text ?? '').replace(IMAGE_URL_LINES_RE, '').trim(); +} + +/** + * After Qwen VL analysis, Goose must receive text only. DeepSeek rejects + * `image_url` parts, and native Goose may expand metadata.imageUrls into them. + * Keep archived/preview URLs for UI history; leave the VL note in the text. + */ +export function detachCurrentTurnImagesForTextProvider(message, canonicalImageUrls = []) { + if (!message) return message; + const metadata = + message.metadata && typeof message.metadata === 'object' && !Array.isArray(message.metadata) + ? { ...message.metadata } + : {}; + const urls = (Array.isArray(canonicalImageUrls) && canonicalImageUrls.length > 0 + ? canonicalImageUrls + : Array.isArray(metadata.imageUrls) ? metadata.imageUrls : [] + ).filter((url) => typeof url === 'string' && url.trim()); + if (urls.length > 0) { + metadata.archivedImageUrls = urls; + if (!Array.isArray(metadata.previewImageUrls) || metadata.previewImageUrls.length === 0) { + metadata.previewImageUrls = urls; + } + } + delete metadata.imageUrls; + + const content = Array.isArray(message.content) + ? message.content + .map((item) => { + if (item?.type === 'image_url') return null; + if (item?.type !== 'text' || typeof item.text !== 'string') return item; + const nextText = stripImageUrlLines(item.text); + if (!nextText) return null; + return nextText === item.text ? item : { ...item, text: nextText }; + }) + .filter(Boolean) + : message.content; + + return { + ...message, + content, + metadata, + }; +} + /** * Remove image attachments from a persisted user message so later turns cannot reuse them. * UI displayText / previewImageUrls are preserved for chat history rendering. diff --git a/chat-image-turn-scope.test.mjs b/chat-image-turn-scope.test.mjs index 8211812..db6ca5d 100644 --- a/chat-image-turn-scope.test.mjs +++ b/chat-image-turn-scope.test.mjs @@ -7,6 +7,7 @@ import { extractCurrentTurnImageUrls, scrubConversationHistoricalImageAttachments, scrubUserMessageImageAttachments, + detachCurrentTurnImagesForTextProvider, } from './chat-image-turn-scope.mjs'; test('extractCurrentTurnImageUrls prefers metadata and dedupes asset aliases', () => { @@ -157,3 +158,40 @@ test('conversationHasImageUrlContent detects historical poison and ignores activ false, ); }); + +test('detachCurrentTurnImagesForTextProvider archives urls and keeps the VL note', () => { + const detached = detachCurrentTurnImagesForTextProvider( + { + role: 'user', + metadata: { + displayText: '解读报告', + imageUrls: ['https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg'], + }, + content: [ + { + type: 'text', + text: + '解读报告\n[图片1]: https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg\n\n' + + '【TKMind 图片分析结果 — 仅供执行参考，不要向用户复述此段内容】\n白细胞偏高', + }, + { + type: 'image_url', + image_url: { url: 'https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg' }, + }, + ], + }, + ['https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg'], + ); + + assert.equal(detached.metadata.imageUrls, undefined); + assert.deepEqual(detached.metadata.archivedImageUrls, [ + 'https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg', + ]); + assert.deepEqual(detached.metadata.previewImageUrls, [ + 'https://m.tkmind.cn/MindSpace/u/public/wechat-mp/report.jpg', + ]); + assert.equal(detached.metadata.displayText, '解读报告'); + assert.equal(detached.content.some((item) => item.type === 'image_url'), false); + assert.match(detached.content[0].text, /白细胞偏高/); + assert.doesNotMatch(detached.content[0].text, /\[图片1\]:/); +}); diff --git a/tkmind-proxy-vision.test.mjs b/tkmind-proxy-vision.test.mjs index c4fca8e..389bfc8 100644 --- a/tkmind-proxy-vision.test.mjs +++ b/tkmind-proxy-vision.test.mjs @@ -111,12 +111,14 @@ test('buildVisionPayload sends all current-turn images to Qwen in order', async assert.equal(analyzedImages.length, 2); assert.deepEqual(analyzedImages.map((item) => item.rawUrl), imageUrls); - assert.deepEqual(result?.userMessage?.metadata?.imageUrls, [ + assert.equal(result?.userMessage?.metadata?.imageUrls, undefined); + assert.deepEqual(result?.userMessage?.metadata?.archivedImageUrls, [ '/MindSpace/user-1/public/wechat-mp/first.jpg', '/MindSpace/user-1/public/wechat-mp/second.jpg', ]); assert.match(result?.userMessage?.content?.[0]?.text ?? '', /本轮用户仅上传 2 张图片/); assert.match(result?.userMessage?.content?.[0]?.text ?? '', /图片2/); + assert.doesNotMatch(result?.userMessage?.content?.[0]?.text ?? '', /\[图片\d+\]:/); }); test('buildVisionPayload does not mark billable usage when vision analysis fails', async () => { @@ -171,4 +173,9 @@ test('buildVisionPayload strips image_url content parts for text-only Goose prov false, ); assert.match(result?.userMessage?.content?.[0]?.text ?? '', /蓝色方块/); + assert.equal(result?.userMessage?.metadata?.imageUrls, undefined); + assert.deepEqual(result?.userMessage?.metadata?.archivedImageUrls, [ + '/api/mindspace/v1/assets/asset-7/download?inline=1', + ]); + assert.doesNotMatch(result?.userMessage?.content?.[0]?.text ?? '', /\[图片\d+\]:/); }); diff --git a/tkmind-proxy.mjs b/tkmind-proxy.mjs index 93dad49..b723567 100644 --- a/tkmind-proxy.mjs +++ b/tkmind-proxy.mjs @@ -35,6 +35,7 @@ import { extractAttachmentText } from './mindspace-attachment-text.mjs'; import { buildCurrentTurnImageScopeNote, conversationHasImageUrlContent, + detachCurrentTurnImagesForTextProvider, extractCurrentTurnImageUrls, scrubConversationHistoricalImageAttachments, } from './chat-image-turn-scope.mjs'; @@ -1004,15 +1005,18 @@ export async function buildVisionPayload({ .filter((url) => typeof url === 'string' && url.trim()); return { - userMessage: { - ...userMessage, - content: updatedContent, - metadata: { - ...(userMessage.metadata ?? {}), - ...(originalDisplayText ? { displayText: originalDisplayText } : {}), - ...(canonicalImageUrls.length ? { imageUrls: canonicalImageUrls } : {}), + userMessage: detachCurrentTurnImagesForTextProvider( + { + ...userMessage, + content: updatedContent, + metadata: { + ...(userMessage.metadata ?? {}), + ...(originalDisplayText ? { displayText: originalDisplayText } : {}), + ...(canonicalImageUrls.length ? { imageUrls: canonicalImageUrls } : {}), + }, }, - }, + canonicalImageUrls, + ), billableImageCount: visionDescription ? 1 : 0, }; } diff --git a/tkmind-proxy.test.mjs b/tkmind-proxy.test.mjs index 04b12fa..ba745eb 100644 --- a/tkmind-proxy.test.mjs +++ b/tkmind-proxy.test.mjs @@ -1707,6 +1707,8 @@ test('submitSessionReplyForUser applies the shared Qwen vision preprocessing pat const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? ''; assert.match(forwardedText, /Qwen VL 图片描述/); assert.match(forwardedText, /一件蓝色产品/); + assert.equal(replyBodies[0]?.user_message?.metadata?.imageUrls, undefined); + assert.ok(Array.isArray(replyBodies[0]?.user_message?.metadata?.archivedImageUrls)); }); }); diff --git a/wechat-mp.mjs b/wechat-mp.mjs index 5d6a0a0..ab52b0f 100644 --- a/wechat-mp.mjs +++ b/wechat-mp.mjs @@ -1088,6 +1088,12 @@ export function isWechatPageDataTask(text) { return isPageDataIntent(text) || isPageDataDevIntent(text); } +export function isWechatHistoricalImageSessionError(message) { + return /historical_image_session_update_unsupported|unknown variant [`']?image_url/i.test( + String(message ?? '').trim(), + ); +} + export function isRecoverableWechatAgentSessionError(message) { const normalized = String(message ?? '').trim(); if (!normalized) return false; @@ -1095,9 +1101,7 @@ export function isRecoverableWechatAgentSessionError(message) { if (/wechat_page_fresh_thumbnail_required:/i.test(normalized)) return true; if (/403|404|not found|无权访问/i.test(normalized)) return true; if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) return true; - if (/historical_image_session_update_unsupported|unknown variant [`']?image_url/i.test(normalized)) { - return true; - } + if (isWechatHistoricalImageSessionError(normalized)) return true; if (/session already has an active request|active request.*cancel/i.test(normalized)) return true; if (/wechat_agent_incomplete_reply/i.test(normalized)) return true; return false; @@ -1712,7 +1716,13 @@ export function createWechatMpService({ const attachRecentMediaForFollowup = (openid, intent, mediaAnalysisEnabled) => { if (!mediaAnalysisEnabled || intent?.msgType !== 'text' || intent?.media?.publicUrl) return; const text = String(intent?.agentText ?? ''); - if (!/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word)/iu.test(text)) return; + if ( + !/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word|报告|解读|化验|检验|做成页面|做个页面|做页面|生成页面)/iu.test( + text, + ) + ) { + return; + } const key = String(openid ?? '').trim(); const recent = recentMediaByOpenid.get(key); if ( @@ -3099,8 +3109,13 @@ export function createWechatMpService({ const mayBeStaleSession = sessionId && (isRecoverableWechatAgentSessionError(message) || isWechatAgentApiErrorText(message)); + const historicalImageError = isWechatHistoricalImageSessionError(message); if (mayBeStaleSession) { - if (sessionPageContinuation && isWechatPageContinuationRepairableError(message)) { + if ( + sessionPageContinuation + && !historicalImageError + && isWechatPageContinuationRepairableError(message) + ) { const text = buildPagePublishFailureText(); await userAuth.clearWechatAgentRoute(config.appId, inbound.fromUserName).catch((clearErr) => { logger.warn?.('WeChat MP page continuation repair route clear failed:', clearErr); @@ -3116,7 +3131,7 @@ export function createWechatMpService({ repairError.wechatAgentSessionId = sessionId; throw repairError; } - if (sessionPageContinuation) { + if (sessionPageContinuation && !historicalImageError) { await userAuth.clearWechatAgentRoute(config.appId, inbound.fromUserName).catch((clearErr) => { logger.warn?.('WeChat MP contextual follow-up route clear failed:', clearErr); }); @@ -3144,17 +3159,18 @@ export function createWechatMpService({ await rememberWechatUserContext(sessionId, user, { forceBootstrap: route.isNewSession }); const retryId = crypto.randomUUID(); const retryStartedAt = Date.now(); + const retryPageContinuation = sessionPageContinuation && !historicalImageError; const retryPrompt = wechatIntent.kind === 'page.generate' ? buildPageGenerateAgentPrompt(intent, { wantsDocx: wechatIntent.wantsDocx, imagePolicy, - preferImmediateContext: sessionPageContinuation, - carriedSessionContent, + preferImmediateContext: retryPageContinuation, + carriedSessionContent: retryPageContinuation ? carriedSessionContent : '', }) : buildWechatAgentPrompt(intent, { imagePolicy, - preferSessionPageContinuation: sessionPageContinuation, + preferSessionPageContinuation: retryPageContinuation, }); const reply = await executeSessionReply( (pathname, init) => fetchForSession(sessionId, pathname, init), @@ -3171,7 +3187,7 @@ export function createWechatMpService({ userId: user.userId, sessionId, userMessage, - preserveAgentPrompt: sessionPageContinuation, + preserveAgentPrompt: retryPageContinuation, }), submitReply: submitSessionReply ? ({ requestId: replyRequestId, userMessage }) => diff --git a/wechat-mp.test.mjs b/wechat-mp.test.mjs index 00fded1..e2f9f39 100644 --- a/wechat-mp.test.mjs +++ b/wechat-mp.test.mjs @@ -13,6 +13,7 @@ import { findRecoverableWechatAgentErrorInReply, isRecoverableWechatAgentSessionError, isWechatAgentApiErrorText, + isWechatHistoricalImageSessionError, sanitizeWechatAgentOutboundText, loadWechatMpConfig, maybeAttachPublishedHtmlLink, @@ -385,6 +386,13 @@ test('WeChat session page continuation covers retry, edit, and poem edits', () = shouldDeliverWechatHtmlArtifacts(poemEdit, { agentText: '把诗里第三段改长一点' }), true, ); + assert.equal( + isWechatSessionPageContinuation( + classifyWechatIntent({ msgType: 'text', agentText: '解读详细报告，做成页面' }), + '解读详细报告，做成页面', + ), + false, + ); }); test('WeChat immediate-context page skips fresh thumbnail requirement', () => { @@ -3477,6 +3485,16 @@ test('isRecoverableWechatAgentSessionError detects poisoned tool_calls history', isRecoverableWechatAgentSessionError('historical_image_session_update_unsupported:405'), true, ); + assert.equal( + isWechatHistoricalImageSessionError('historical_image_session_update_unsupported:405'), + true, + ); + assert.equal( + isWechatHistoricalImageSessionError( + 'Request failed: Bad request (400): messages[74]: unknown variant `image_url`, expected `text`', + ), + true, + ); assert.equal(isRecoverableWechatAgentSessionError('network timeout'), false); }); @@ -3572,6 +3590,113 @@ test('wechat mp rotates and retries when historical image isolation is unsupport assert.equal(sentPayloads[0].text.content, '新会话已恢复，可以继续。'); }); +test('wechat mp rotates page continuation instead of dropping image_url session errors', async () => { + const token = 'token'; + const timestamp = '1710000000'; + const nonce = 'nonce'; + const submittedSessions = []; + const sentPayloads = []; + let activeSessionId = 'session-1'; + let routeCleared = false; + const poem = `《临江仙·秋思》${'昨夜西风凋碧树，独上高楼，望尽天涯路。'.repeat(3)}`; + + const service = createBoundWechatService({ + token, + startAgentSession: async () => ({ id: 'session-2' }), + userAuth: { + async getWechatAgentRoute() { + return routeCleared ? null : { agentSessionId: activeSessionId, status: 'active' }; + }, + async clearWechatAgentRoute() { + routeCleared = true; + }, + async upsertWechatAgentRoute({ agentSessionId }) { + activeSessionId = agentSessionId; + routeCleared = false; + }, + }, + submitSessionReply: async ({ sessionId, options }) => { + submittedSessions.push(sessionId); + assert.equal(options?.requireHistoricalImageIsolation, true); + if (sessionId === 'session-1') { + throw new Error( + 'Ran into this error: Request failed: Bad request (400): Failed to deserialize the JSON body into the target type: messages[74]: unknown variant `image_url`, expected `text`', + ); + } + return { ok: true }; + }, + sessionApiFetch: async (sessionId, pathname) => { + if (pathname === `/sessions/${sessionId}`) { + return new Response(JSON.stringify({ + conversation: sessionId === 'session-1' + ? [{ role: 'assistant', content: [{ type: 'text', text: poem }] }] + : [], + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (pathname === `/sessions/${sessionId}/events`) { + if (sessionId === 'session-1') { + return new Response('', { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }); + } + return new Response( + [ + 'data: {"type":"Message","request_id":"req-page-retry","message":{"id":"assistant-2","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"新会话已恢复，可以继续。"}]}}\n\n', + 'data: {"type":"Finish","request_id":"req-page-retry","token_state":{"inputTokens":1,"outputTokens":2}}\n\n', + ].join(''), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') { + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }); + } + throw new Error(`unexpected api path: ${sessionId} ${pathname}`); + }, + wechatFetch: async (url, init = {}) => { + if (String(url).includes('/cgi-bin/stable_token')) { + return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/cgi-bin/message/custom/send')) { + sentPayloads.push(JSON.parse(init.body)); + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected wechat url: ${url}`); + }, + }); + + const originalRandomUuid = crypto.randomUUID; + crypto.randomUUID = (() => { + const ids = ['req-page-first', 'req-page-retry']; + return () => ids.shift() ?? 'req-page-retry'; + })(); + try { + const result = await service.handleInboundMessage( + inboundXml({ content: '把刚才的诗做成页面' }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + await result.task; + } finally { + crypto.randomUUID = originalRandomUuid; + } + + assert.deepEqual(submittedSessions, ['session-1', 'session-2']); + assert.equal(activeSessionId, 'session-2'); + assert.equal( + sentPayloads.some((payload) => /没能可靠确认/.test(String(payload?.text?.content ?? ''))), + false, + ); +}); + test('findRecoverableWechatAgentErrorInReply scans all assistant messages', () => { const toolCallsError = "Request failed: Bad request (400): An assistant message with 'tool_calls' must be followed by tool messages responding to each 'tool_call_id'. (insufficient tool messages following tool_calls message)."; @@ -5318,6 +5443,119 @@ test('wechat mp serializes image and follow-up text and reattaches recent image' assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text'); }); +test('wechat mp reattaches recent image for report interpretation follow-up', async () => { + const token = 'token'; + const timestamp = '1710000000'; + const nonce = 'nonce'; + const testUserId = 'test-user-report-followup'; + const submitCalls = []; + let eventCall = 0; + let releaseFirst = null; + const service = createBoundWechatService({ + token, + config: { + mediaAnalysisGrayUsers: [testUserId], + }, + userAuth: { + async findWechatUserByOpenid() { + return { userId: testUserId, status: 'active', nickname: '唐' }; + }, + }, + sessionApiFetch: async (_sessionId, pathname) => { + if (pathname === '/sessions/session-1/events') { + eventCall += 1; + if (eventCall === 1) { + return new Response( + new ReadableStream({ + start(controller) { + releaseFirst = () => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"Message","message":{"id":"assistant-image","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"图片已识别。"}]}}\n\n' + + 'data: {"type":"Finish"}\n\n', + ), + ); + controller.close(); + }; + }, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + return new Response( + [ + 'data: {"type":"Message","message":{"id":"assistant-followup","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"已解读报告。"}]}}\n\n', + 'data: {"type":"Finish"}\n\n', + ].join(''), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') { + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }); + } + throw new Error(`unexpected api path: ${pathname}`); + }, + submitSessionReply: async (input) => { + submitCalls.push(input); + return { ok: true }; + }, + wechatFetch: async (url) => { + if (String(url).includes('/cgi-bin/stable_token')) { + return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/cgi-bin/media/get')) { + return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), { + status: 200, + headers: { 'Content-Type': 'image/png' }, + }); + } + if (String(url).includes('/cgi-bin/message/custom/send')) { + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected wechat url: ${url}`); + }, + }); + + try { + const imageResult = await service.handleInboundMessage( + inboundXml({ + msgType: 'image', + content: '', + extraFields: { MediaId: 'media-report', PicUrl: 'https://wx.example.com/report.png' }, + }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0)); + + const followupResult = await service.handleInboundMessage( + inboundXml({ msgType: 'text', content: '解读详细报告，做成页面' }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(submitCalls.length, 1); + + releaseFirst(); + await imageResult.task; + await followupResult.task; + } finally { + fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true }); + } + + assert.equal(submitCalls.length, 2); + assert.deepEqual( + submitCalls[1].userMessage.metadata.imageUrls, + submitCalls[0].userMessage.metadata.imageUrls, + ); + assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text'); + assert.equal(submitCalls[1].userMessage.metadata.displayText, '解读详细报告，做成页面'); +}); + test('wechat mp groups consecutive images for one follow-up and clears the consumed batch', async () => { const token = 'token'; const timestamp = '1710000000';