diff --git a/tkmind-proxy-vision.test.mjs b/tkmind-proxy-vision.test.mjs index 6526343..068d665 100644 --- a/tkmind-proxy-vision.test.mjs +++ b/tkmind-proxy-vision.test.mjs @@ -83,6 +83,42 @@ test('buildVisionPayload marks one billable image analysis when vision succeeds' ); }); +test('buildVisionPayload sends all current-turn images to Qwen in order', async () => { + const imageUrls = [ + 'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/first.jpg', + 'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/second.jpg', + ]; + let analyzedImages = []; + const result = await buildVisionPayload({ + userId: 'user-1', + publishLayout: { publicUrl: 'https://m.tkmind.cn/MindSpace/user-1' }, + userMessage: { + content: [{ type: 'text', text: '请结合两张图片生成页面' }], + metadata: { imageUrls }, + }, + fetchImpl: async () => + new Response(Buffer.from('fake-image'), { + status: 200, + headers: { 'Content-Type': 'image/jpeg' }, + }), + llmProviderService: { + analyzeImagesWithVision: async (images) => { + analyzedImages = images; + return '第一张是宝塔，第二张是晚霞。'; + }, + }, + }); + + assert.equal(analyzedImages.length, 2); + assert.deepEqual(analyzedImages.map((item) => item.rawUrl), imageUrls); + assert.deepEqual(result?.userMessage?.metadata?.imageUrls, [ + '/MindSpace/user-1/public/wechat-mp/first.jpg', + '/MindSpace/user-1/public/wechat-mp/second.jpg', + ]); + assert.match(result?.userMessage?.content?.[0]?.text ?? '', /本轮用户仅上传 2 张图片/); + assert.match(result?.userMessage?.content?.[0]?.text ?? '', /图片2/); +}); + test('buildVisionPayload does not mark billable usage when vision analysis fails', async () => { const result = await buildVisionPayload({ userId: 'user-1', diff --git a/wechat-mp.mjs b/wechat-mp.mjs index 413027a..5a386e8 100644 --- a/wechat-mp.mjs +++ b/wechat-mp.mjs @@ -50,6 +50,8 @@ const DEFAULT_WECHAT_CUSTOMER_SERVICE_URL = 'https://api.weixin.qq.com/cgi-bin/message/custom/send'; const DEFAULT_WECHAT_JSAPI_TICKET_URL = 'https://api.weixin.qq.com/cgi-bin/ticket/getticket'; const DEFAULT_ASR_TARGET = process.env.H5_ASR_TARGET ?? 'https://asr.tkmind.cn'; +const WECHAT_RECENT_MEDIA_TTL_MS = 15 * 60 * 1000; +const WECHAT_RECENT_IMAGE_MAX_COUNT = 10; export { loadWechatMpConfig }; const PUBLIC_HTML_LINK_PATTERN = /https?:\/\/[^\s<>"')\]]+\/MindSpace\/([0-9a-f-]{36}|[a-z0-9._-]+)\/public\/([^\s<>"')\]]+\.html)/gi; @@ -1538,10 +1540,31 @@ export function createWechatMpService({ const rememberRecentMedia = (openid, intent) => { const publicUrl = String(intent?.media?.publicUrl ?? '').trim(); if (!publicUrl) return; - recentMediaByOpenid.set(String(openid ?? '').trim(), { + const key = String(openid ?? '').trim(); + const now = Date.now(); + const item = { media: { ...(intent.media ?? {}) }, attachment: intent.attachment ? { ...intent.attachment } : null, - rememberedAt: Date.now(), + }; + const current = recentMediaByOpenid.get(key); + const canAppendImage = + intent?.msgType === 'image' && + current && + !current.claimed && + now - current.rememberedAt <= WECHAT_RECENT_MEDIA_TTL_MS && + current.items.every((recentItem) => !recentItem.attachment); + const candidates = canAppendImage ? [...current.items, item] : [item]; + const deduped = candidates.filter( + (candidate, index, values) => + values.findIndex( + (value) => String(value?.media?.publicUrl ?? '') === String(candidate?.media?.publicUrl ?? ''), + ) === index, + ); + recentMediaByOpenid.set(key, { + items: deduped.slice(-WECHAT_RECENT_IMAGE_MAX_COUNT), + rememberedAt: now, + batchId: crypto.randomUUID(), + claimed: false, }); }; @@ -1549,10 +1572,36 @@ export function createWechatMpService({ if (!mediaAnalysisEnabled || intent?.msgType !== 'text' || intent?.media?.publicUrl) return; const text = String(intent?.agentText ?? ''); if (!/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word)/iu.test(text)) return; - const recent = recentMediaByOpenid.get(String(openid ?? '').trim()); - if (!recent || Date.now() - recent.rememberedAt > 15 * 60 * 1000) return; - intent.media = { ...recent.media, source: 'wechat_recent_media' }; - if (recent.attachment) intent.attachment = { ...recent.attachment }; + const key = String(openid ?? '').trim(); + const recent = recentMediaByOpenid.get(key); + if ( + !recent || + recent.claimed || + Date.now() - recent.rememberedAt > WECHAT_RECENT_MEDIA_TTL_MS || + recent.items.length === 0 + ) { + return; + } + const items = recent.items.map((item) => ({ + media: { ...(item.media ?? {}), source: 'wechat_recent_media' }, + attachment: item.attachment ? { ...item.attachment } : null, + })); + const primary = items.at(-1); + intent.media = { ...(primary?.media ?? {}) }; + if (primary?.attachment) intent.attachment = { ...primary.attachment }; + intent.recentMediaItems = items; + intent.recentMediaBatchId = recent.batchId; + recent.claimed = true; + }; + + const settleRecentMediaBatch = (openid, intent, { succeeded }) => { + const batchId = String(intent?.recentMediaBatchId ?? '').trim(); + if (!batchId) return; + const key = String(openid ?? '').trim(); + const recent = recentMediaByOpenid.get(key); + if (!recent || recent.batchId !== batchId) return; + if (succeeded) recentMediaByOpenid.delete(key); + else recent.claimed = false; }; const resolveWechatBillingTokenState = (sessionId, tokenState) => @@ -1975,25 +2024,33 @@ export function createWechatMpService({ const buildIntentMetadata = (intent, { mediaAnalysisEnabled = false } = {}) => { const mediaPublicUrl = intent.media?.publicUrl || null; - const fileAttachment = - mediaPublicUrl && intent.attachment?.filename - ? { - assetId: '', - downloadUrl: mediaPublicUrl, - filename: intent.attachment.filename, - mimeType: intent.attachment.mimeType || 'application/octet-stream', - } - : null; + const mediaItems = Array.isArray(intent.recentMediaItems) && intent.recentMediaItems.length > 0 + ? intent.recentMediaItems + : mediaPublicUrl + ? [{ media: intent.media, attachment: intent.attachment ?? null }] + : []; + const imageUrls = mediaItems + .filter((item) => !item.attachment) + .map((item) => String(item?.media?.publicUrl ?? '').trim()) + .filter((url, index, values) => url && values.indexOf(url) === index); + const fileAttachments = mediaItems + .filter((item) => item.attachment?.filename && item.media?.publicUrl) + .map((item) => ({ + assetId: '', + downloadUrl: item.media.publicUrl, + filename: item.attachment.filename, + mimeType: item.attachment.mimeType || 'application/octet-stream', + })); return { source: 'wechat_mp', msgType: intent.msgType, originalMsgId: intent.msgId || null, displayText: intent.displayText || '', mediaPublicUrl, - ...(mediaAnalysisEnabled && mediaPublicUrl && !fileAttachment - ? { imageUrls: [mediaPublicUrl] } + ...(mediaAnalysisEnabled && imageUrls.length > 0 + ? { imageUrls } : {}), - ...(mediaAnalysisEnabled && fileAttachment ? { fileAttachments: [fileAttachment] } : {}), + ...(mediaAnalysisEnabled && fileAttachments.length > 0 ? { fileAttachments } : {}), recognition: intent.msgType === 'voice' ? intent.agentText || null : null, location: intent.location || null, link: intent.link || null, @@ -2776,6 +2833,7 @@ export function createWechatMpService({ }), ) .then(async ({ sessionId } = {}) => { + settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: true }); if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') { await userAuth.finishWechatMpMessage({ appId: config.appId, @@ -2787,6 +2845,7 @@ export function createWechatMpService({ } }) .catch(async (err) => { + settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: false }); if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') { await userAuth.finishWechatMpMessage({ appId: config.appId, diff --git a/wechat-mp.test.mjs b/wechat-mp.test.mjs index cb96f50..bdf9693 100644 --- a/wechat-mp.test.mjs +++ b/wechat-mp.test.mjs @@ -4029,6 +4029,150 @@ test('wechat mp serializes image and follow-up text and reattaches recent image' assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text'); }); +test('wechat mp groups consecutive images for one follow-up and clears the consumed batch', async () => { + const token = 'token'; + const timestamp = '1710000000'; + const nonce = 'nonce'; + const testUserId = 'test-user-multi-image-followup'; + const submitCalls = []; + let eventCall = 0; + let releaseFirst = null; + const service = createBoundWechatService({ + token, + config: { + mediaAnalysisGrayUsers: [testUserId], + }, + userAuth: { + async findWechatUserByOpenid() { + return { userId: testUserId, status: 'active', nickname: '唐' }; + }, + }, + sessionApiFetch: async (_sessionId, pathname) => { + if (pathname === '/sessions/session-1/events') { + eventCall += 1; + if (eventCall === 1) { + return new Response( + new ReadableStream({ + start(controller) { + releaseFirst = () => { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"Message","message":{"id":"assistant-first","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"第一张处理完成。"}]}}\n\n' + + 'data: {"type":"Finish"}\n\n', + ), + ); + controller.close(); + }; + }, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + return new Response( + [ + 'data: {"type":"Message","message":{"id":"assistant-ok","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"处理完成。"}]}}\n\n', + 'data: {"type":"Finish"}\n\n', + ].join(''), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') { + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }); + } + throw new Error(`unexpected api path: ${pathname}`); + }, + submitSessionReply: async (input) => { + submitCalls.push(input); + return { ok: true }; + }, + wechatFetch: async (url) => { + if (String(url).includes('/cgi-bin/stable_token')) { + return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/cgi-bin/media/get')) { + return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), { + status: 200, + headers: { 'Content-Type': 'image/png' }, + }); + } + if (String(url).includes('/cgi-bin/message/custom/send')) { + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected wechat url: ${url}`); + }, + }); + + try { + const firstImageResult = await service.handleInboundMessage( + inboundXml({ + msgType: 'image', + content: '', + extraFields: { + MsgId: '10011', + MediaId: 'media-first', + PicUrl: 'https://wx.example.com/media-first.png', + }, + }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0)); + + const secondImageResult = await service.handleInboundMessage( + inboundXml({ + msgType: 'image', + content: '', + extraFields: { + MsgId: '10012', + MediaId: 'media-second', + PicUrl: 'https://wx.example.com/media-second.png', + }, + }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + + const followupResult = await service.handleInboundMessage( + inboundXml({ + content: '请结合刚才两张图片分析主题', + extraFields: { MsgId: '10013' }, + }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(submitCalls.length, 1); + + releaseFirst(); + await firstImageResult.task; + await secondImageResult.task; + await followupResult.task; + + const laterResult = await service.handleInboundMessage( + inboundXml({ + content: '请再次分析刚才图片', + extraFields: { MsgId: '10014' }, + }), + { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) }, + ); + await laterResult.task; + } finally { + fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true }); + } + + assert.equal(submitCalls.length, 4); + assert.equal(submitCalls[0].userMessage.metadata.imageUrls.length, 1); + assert.equal(submitCalls[1].userMessage.metadata.imageUrls.length, 1); + assert.equal(submitCalls[2].userMessage.metadata.imageUrls.length, 2); + assert.match(submitCalls[2].userMessage.metadata.imageUrls[0], /media-first/); + assert.match(submitCalls[2].userMessage.metadata.imageUrls[1], /media-second/); + assert.equal(submitCalls[2].userMessage.metadata.msgType, 'text'); + assert.equal(submitCalls[3].userMessage.metadata.imageUrls, undefined); +}); + test('wechat mp service persists Word and Excel files in user public area and reuses H5 attachment metadata', async () => { const token = 'token'; const timestamp = '1710000000';