merge: fix WeChat multi-image gray flow

2026-07-18 19:53:21 +08:00
parent 341cf5ae89 711a7d2061
commit 1161292482
3 changed files with 257 additions and 18 deletions
@@ -83,6 +83,42 @@ test('buildVisionPayload marks one billable image analysis when vision succeeds'
  );
 });

+test('buildVisionPayload sends all current-turn images to Qwen in order', async () => {
+  const imageUrls = [
+    'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/first.jpg',
+    'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/second.jpg',
+  ];
+  let analyzedImages = [];
+  const result = await buildVisionPayload({
+    userId: 'user-1',
+    publishLayout: { publicUrl: 'https://m.tkmind.cn/MindSpace/user-1' },
+    userMessage: {
+      content: [{ type: 'text', text: '请结合两张图片生成页面' }],
+      metadata: { imageUrls },
+    },
+    fetchImpl: async () =>
+      new Response(Buffer.from('fake-image'), {
+        status: 200,
+        headers: { 'Content-Type': 'image/jpeg' },
+      }),
+    llmProviderService: {
+      analyzeImagesWithVision: async (images) => {
+        analyzedImages = images;
+        return '第一张是宝塔，第二张是晚霞。';
+      },
+    },
+  });
+
+  assert.equal(analyzedImages.length, 2);
+  assert.deepEqual(analyzedImages.map((item) => item.rawUrl), imageUrls);
+  assert.deepEqual(result?.userMessage?.metadata?.imageUrls, [
+    '/MindSpace/user-1/public/wechat-mp/first.jpg',
+    '/MindSpace/user-1/public/wechat-mp/second.jpg',
+  ]);
+  assert.match(result?.userMessage?.content?.[0]?.text ?? '', /本轮用户仅上传 2 张图片/);
+  assert.match(result?.userMessage?.content?.[0]?.text ?? '', /图片2/);
+});
+
 test('buildVisionPayload does not mark billable usage when vision analysis fails', async () => {
  const result = await buildVisionPayload({
    userId: 'user-1',
@@ -50,6 +50,8 @@ const DEFAULT_WECHAT_CUSTOMER_SERVICE_URL =
  'https://api.weixin.qq.com/cgi-bin/message/custom/send';
 const DEFAULT_WECHAT_JSAPI_TICKET_URL = 'https://api.weixin.qq.com/cgi-bin/ticket/getticket';
 const DEFAULT_ASR_TARGET = process.env.H5_ASR_TARGET ?? 'https://asr.tkmind.cn';
+const WECHAT_RECENT_MEDIA_TTL_MS = 15 * 60 * 1000;
+const WECHAT_RECENT_IMAGE_MAX_COUNT = 10;
 export { loadWechatMpConfig };
 const PUBLIC_HTML_LINK_PATTERN =
  /https?:\/\/[^\s<>"')\]]+\/MindSpace\/([0-9a-f-]{36}|[a-z0-9._-]+)\/public\/([^\s<>"')\]]+\.html)/gi;
@@ -1538,10 +1540,31 @@ export function createWechatMpService({
  const rememberRecentMedia = (openid, intent) => {
    const publicUrl = String(intent?.media?.publicUrl ?? '').trim();
    if (!publicUrl) return;
-    recentMediaByOpenid.set(String(openid ?? '').trim(), {
+    const key = String(openid ?? '').trim();
+    const now = Date.now();
+    const item = {
      media: { ...(intent.media ?? {}) },
      attachment: intent.attachment ? { ...intent.attachment } : null,
-      rememberedAt: Date.now(),
+    };
+    const current = recentMediaByOpenid.get(key);
+    const canAppendImage =
+      intent?.msgType === 'image' &&
+      current &&
+      !current.claimed &&
+      now - current.rememberedAt <= WECHAT_RECENT_MEDIA_TTL_MS &&
+      current.items.every((recentItem) => !recentItem.attachment);
+    const candidates = canAppendImage ? [...current.items, item] : [item];
+    const deduped = candidates.filter(
+      (candidate, index, values) =>
+        values.findIndex(
+          (value) => String(value?.media?.publicUrl ?? '') === String(candidate?.media?.publicUrl ?? ''),
+        ) === index,
+    );
+    recentMediaByOpenid.set(key, {
+      items: deduped.slice(-WECHAT_RECENT_IMAGE_MAX_COUNT),
+      rememberedAt: now,
+      batchId: crypto.randomUUID(),
+      claimed: false,
    });
  };

@@ -1549,10 +1572,36 @@ export function createWechatMpService({
    if (!mediaAnalysisEnabled || intent?.msgType !== 'text' || intent?.media?.publicUrl) return;
    const text = String(intent?.agentText ?? '');
    if (!/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word)/iu.test(text)) return;
-    const recent = recentMediaByOpenid.get(String(openid ?? '').trim());
-    if (!recent || Date.now() - recent.rememberedAt > 15 * 60 * 1000) return;
-    intent.media = { ...recent.media, source: 'wechat_recent_media' };
-    if (recent.attachment) intent.attachment = { ...recent.attachment };
+    const key = String(openid ?? '').trim();
+    const recent = recentMediaByOpenid.get(key);
+    if (
+      !recent ||
+      recent.claimed ||
+      Date.now() - recent.rememberedAt > WECHAT_RECENT_MEDIA_TTL_MS ||
+      recent.items.length === 0
+    ) {
+      return;
+    }
+    const items = recent.items.map((item) => ({
+      media: { ...(item.media ?? {}), source: 'wechat_recent_media' },
+      attachment: item.attachment ? { ...item.attachment } : null,
+    }));
+    const primary = items.at(-1);
+    intent.media = { ...(primary?.media ?? {}) };
+    if (primary?.attachment) intent.attachment = { ...primary.attachment };
+    intent.recentMediaItems = items;
+    intent.recentMediaBatchId = recent.batchId;
+    recent.claimed = true;
+  };
+
+  const settleRecentMediaBatch = (openid, intent, { succeeded }) => {
+    const batchId = String(intent?.recentMediaBatchId ?? '').trim();
+    if (!batchId) return;
+    const key = String(openid ?? '').trim();
+    const recent = recentMediaByOpenid.get(key);
+    if (!recent || recent.batchId !== batchId) return;
+    if (succeeded) recentMediaByOpenid.delete(key);
+    else recent.claimed = false;
  };

  const resolveWechatBillingTokenState = (sessionId, tokenState) =>
@@ -1975,25 +2024,33 @@ export function createWechatMpService({

  const buildIntentMetadata = (intent, { mediaAnalysisEnabled = false } = {}) => {
    const mediaPublicUrl = intent.media?.publicUrl || null;
-    const fileAttachment =
-      mediaPublicUrl && intent.attachment?.filename
-        ? {
-            assetId: '',
-            downloadUrl: mediaPublicUrl,
-            filename: intent.attachment.filename,
-            mimeType: intent.attachment.mimeType || 'application/octet-stream',
-          }
-        : null;
+    const mediaItems = Array.isArray(intent.recentMediaItems) && intent.recentMediaItems.length > 0
+      ? intent.recentMediaItems
+      : mediaPublicUrl
+        ? [{ media: intent.media, attachment: intent.attachment ?? null }]
+        : [];
+    const imageUrls = mediaItems
+      .filter((item) => !item.attachment)
+      .map((item) => String(item?.media?.publicUrl ?? '').trim())
+      .filter((url, index, values) => url && values.indexOf(url) === index);
+    const fileAttachments = mediaItems
+      .filter((item) => item.attachment?.filename && item.media?.publicUrl)
+      .map((item) => ({
+        assetId: '',
+        downloadUrl: item.media.publicUrl,
+        filename: item.attachment.filename,
+        mimeType: item.attachment.mimeType || 'application/octet-stream',
+      }));
    return {
      source: 'wechat_mp',
      msgType: intent.msgType,
      originalMsgId: intent.msgId || null,
      displayText: intent.displayText || '',
      mediaPublicUrl,
-      ...(mediaAnalysisEnabled && mediaPublicUrl && !fileAttachment
-        ? { imageUrls: [mediaPublicUrl] }
+      ...(mediaAnalysisEnabled && imageUrls.length > 0
+        ? { imageUrls }
        : {}),
-      ...(mediaAnalysisEnabled && fileAttachment ? { fileAttachments: [fileAttachment] } : {}),
+      ...(mediaAnalysisEnabled && fileAttachments.length > 0 ? { fileAttachments } : {}),
      recognition: intent.msgType === 'voice' ? intent.agentText || null : null,
      location: intent.location || null,
      link: intent.link || null,
@@ -2776,6 +2833,7 @@ export function createWechatMpService({
      }),
    )
      .then(async ({ sessionId } = {}) => {
+        settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: true });
        if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
          await userAuth.finishWechatMpMessage({
            appId: config.appId,
@@ -2787,6 +2845,7 @@ export function createWechatMpService({
        }
      })
      .catch(async (err) => {
+        settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: false });
        if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
          await userAuth.finishWechatMpMessage({
            appId: config.appId,
@@ -4029,6 +4029,150 @@ test('wechat mp serializes image and follow-up text and reattaches recent image'
  assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text');
 });

+test('wechat mp groups consecutive images for one follow-up and clears the consumed batch', async () => {
+  const token = 'token';
+  const timestamp = '1710000000';
+  const nonce = 'nonce';
+  const testUserId = 'test-user-multi-image-followup';
+  const submitCalls = [];
+  let eventCall = 0;
+  let releaseFirst = null;
+  const service = createBoundWechatService({
+    token,
+    config: {
+      mediaAnalysisGrayUsers: [testUserId],
+    },
+    userAuth: {
+      async findWechatUserByOpenid() {
+        return { userId: testUserId, status: 'active', nickname: '唐' };
+      },
+    },
+    sessionApiFetch: async (_sessionId, pathname) => {
+      if (pathname === '/sessions/session-1/events') {
+        eventCall += 1;
+        if (eventCall === 1) {
+          return new Response(
+            new ReadableStream({
+              start(controller) {
+                releaseFirst = () => {
+                  controller.enqueue(
+                    new TextEncoder().encode(
+                      'data: {"type":"Message","message":{"id":"assistant-first","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"第一张处理完成。"}]}}\n\n' +
+                        'data: {"type":"Finish"}\n\n',
+                    ),
+                  );
+                  controller.close();
+                };
+              },
+            }),
+            { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+          );
+        }
+        return new Response(
+          [
+            'data: {"type":"Message","message":{"id":"assistant-ok","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"处理完成。"}]}}\n\n',
+            'data: {"type":"Finish"}\n\n',
+          ].join(''),
+          { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+        );
+      }
+      if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
+        return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
+      }
+      throw new Error(`unexpected api path: ${pathname}`);
+    },
+    submitSessionReply: async (input) => {
+      submitCalls.push(input);
+      return { ok: true };
+    },
+    wechatFetch: async (url) => {
+      if (String(url).includes('/cgi-bin/stable_token')) {
+        return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/media/get')) {
+        return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
+          status: 200,
+          headers: { 'Content-Type': 'image/png' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/message/custom/send')) {
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected wechat url: ${url}`);
+    },
+  });
+
+  try {
+    const firstImageResult = await service.handleInboundMessage(
+      inboundXml({
+        msgType: 'image',
+        content: '',
+        extraFields: {
+          MsgId: '10011',
+          MediaId: 'media-first',
+          PicUrl: 'https://wx.example.com/media-first.png',
+        },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0));
+
+    const secondImageResult = await service.handleInboundMessage(
+      inboundXml({
+        msgType: 'image',
+        content: '',
+        extraFields: {
+          MsgId: '10012',
+          MediaId: 'media-second',
+          PicUrl: 'https://wx.example.com/media-second.png',
+        },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+
+    const followupResult = await service.handleInboundMessage(
+      inboundXml({
+        content: '请结合刚才两张图片分析主题',
+        extraFields: { MsgId: '10013' },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    assert.equal(submitCalls.length, 1);
+
+    releaseFirst();
+    await firstImageResult.task;
+    await secondImageResult.task;
+    await followupResult.task;
+
+    const laterResult = await service.handleInboundMessage(
+      inboundXml({
+        content: '请再次分析刚才图片',
+        extraFields: { MsgId: '10014' },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    await laterResult.task;
+  } finally {
+    fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
+  }
+
+  assert.equal(submitCalls.length, 4);
+  assert.equal(submitCalls[0].userMessage.metadata.imageUrls.length, 1);
+  assert.equal(submitCalls[1].userMessage.metadata.imageUrls.length, 1);
+  assert.equal(submitCalls[2].userMessage.metadata.imageUrls.length, 2);
+  assert.match(submitCalls[2].userMessage.metadata.imageUrls[0], /media-first/);
+  assert.match(submitCalls[2].userMessage.metadata.imageUrls[1], /media-second/);
+  assert.equal(submitCalls[2].userMessage.metadata.msgType, 'text');
+  assert.equal(submitCalls[3].userMessage.metadata.imageUrls, undefined);
+});
+
 test('wechat mp service persists Word and Excel files in user public area and reuses H5 attachment metadata', async () => {
  const token = 'token';
  const timestamp = '1710000000';