fix: route WeChat media through shared vision pipeline

2026-07-18 18:15:22 +08:00
parent 8ccaf3b39d
commit 8eaf4d23f3
4 changed files with 369 additions and 19 deletions
@@ -79,6 +79,7 @@ function createBoundWechatService({
  config = {},
  scheduleService = null,
  applySessionLlmProvider = null,
+  submitSessionReply = null,
 }) {
  return createWechatMpService({
    config: {
@@ -128,6 +129,7 @@ function createBoundWechatService({
    },
    startAgentSession,
    sessionApiFetch,
+    submitSessionReply,
    scheduleService,
    applySessionLlmProvider,
    wechatFetch,
@@ -2752,6 +2754,17 @@ test('sanitizeWechatAgentOutboundText replaces raw api errors with friendly text
  );
 });

+test('assertWechatAgentReplyIsSendable rejects image inspection placeholders', () => {
+  assert.throws(
+    () => assertWechatAgentReplyIsSendable({ text: 'Let me look at the image first.' }),
+    /wechat_agent_incomplete_reply/,
+  );
+  assert.throws(
+    () => assertWechatAgentReplyIsSendable({ text: '我先看一下这张图片。' }),
+    /wechat_agent_incomplete_reply/,
+  );
+});
+
 test('wechat mp service recreates dedicated session when tool_calls error arrives via Finish assistant text', async () => {
  const token = 'token';
  const timestamp = '1710000000';
@@ -3821,6 +3834,197 @@ test('wechat mp service persists image and routes image url into agent prompt',
  assert.match(detailCalls[0].mediaPublicUrl, /\/wechat-mp\//);
 });

+test('wechat mp image submission reuses the H5 prepared reply path', async () => {
+  const token = 'token';
+  const timestamp = '1710000000';
+  const nonce = 'nonce';
+  const testUserId = 'test-user-image-prepared';
+  const submitCalls = [];
+  const service = createBoundWechatService({
+    token,
+    config: {
+      mediaAnalysisGrayUsers: [testUserId],
+    },
+    userAuth: {
+      async findWechatUserByOpenid() {
+        return { userId: testUserId, status: 'active', nickname: '唐' };
+      },
+    },
+    sessionApiFetch: async (_sessionId, pathname) => {
+      if (pathname === '/sessions/session-1/events') {
+        return new Response(
+          [
+            'data: {"type":"Message","message":{"id":"assistant-1","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"图片内容已识别。"}]}}\n\n',
+            'data: {"type":"Finish"}\n\n',
+          ].join(''),
+          { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+        );
+      }
+      if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
+        return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
+      }
+      throw new Error(`unexpected api path: ${pathname}`);
+    },
+    submitSessionReply: async (input) => {
+      submitCalls.push(input);
+      return { ok: true };
+    },
+    wechatFetch: async (url) => {
+      if (String(url).includes('/cgi-bin/stable_token')) {
+        return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/media/get')) {
+        return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
+          status: 200,
+          headers: { 'Content-Type': 'image/png' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/message/custom/send')) {
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected wechat url: ${url}`);
+    },
+  });
+
+  try {
+    const result = await service.handleInboundMessage(
+      inboundXml({
+        msgType: 'image',
+        content: '',
+        extraFields: { MediaId: 'media-prepared', PicUrl: 'https://wx.example.com/image.png' },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    await result.task;
+  } finally {
+    fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
+  }
+
+  assert.equal(submitCalls.length, 1);
+  assert.equal(submitCalls[0].userId, testUserId);
+  assert.equal(submitCalls[0].sessionId, 'session-1');
+  assert.equal(submitCalls[0].userMessage.metadata.imageUrls.length, 1);
+  assert.match(submitCalls[0].userMessage.metadata.imageUrls[0], /\/public\/wechat-mp\//);
+});
+
+test('wechat mp serializes image and follow-up text and reattaches recent image', async () => {
+  const token = 'token';
+  const timestamp = '1710000000';
+  const nonce = 'nonce';
+  const testUserId = 'test-user-image-followup';
+  const submitCalls = [];
+  let eventCall = 0;
+  let releaseFirst = null;
+  const service = createBoundWechatService({
+    token,
+    config: {
+      mediaAnalysisGrayUsers: [testUserId],
+    },
+    userAuth: {
+      async findWechatUserByOpenid() {
+        return { userId: testUserId, status: 'active', nickname: '唐' };
+      },
+    },
+    sessionApiFetch: async (_sessionId, pathname) => {
+      if (pathname === '/sessions/session-1/events') {
+        eventCall += 1;
+        if (eventCall === 1) {
+          return new Response(
+            new ReadableStream({
+              start(controller) {
+                releaseFirst = () => {
+                  controller.enqueue(
+                    new TextEncoder().encode(
+                      'data: {"type":"Message","message":{"id":"assistant-image","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"图片已识别。"}]}}\n\n' +
+                        'data: {"type":"Finish"}\n\n',
+                    ),
+                  );
+                  controller.close();
+                };
+              },
+            }),
+            { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+          );
+        }
+        return new Response(
+          [
+            'data: {"type":"Message","message":{"id":"assistant-followup","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"已结合刚才图片分析主题。"}]}}\n\n',
+            'data: {"type":"Finish"}\n\n',
+          ].join(''),
+          { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+        );
+      }
+      if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
+        return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
+      }
+      throw new Error(`unexpected api path: ${pathname}`);
+    },
+    submitSessionReply: async (input) => {
+      submitCalls.push(input);
+      return { ok: true };
+    },
+    wechatFetch: async (url) => {
+      if (String(url).includes('/cgi-bin/stable_token')) {
+        return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/media/get')) {
+        return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
+          status: 200,
+          headers: { 'Content-Type': 'image/png' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/message/custom/send')) {
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected wechat url: ${url}`);
+    },
+  });
+
+  try {
+    const imageResult = await service.handleInboundMessage(
+      inboundXml({
+        msgType: 'image',
+        content: '',
+        extraFields: { MediaId: 'media-followup', PicUrl: 'https://wx.example.com/image.png' },
+      }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0));
+
+    const followupResult = await service.handleInboundMessage(
+      inboundXml({ msgType: 'text', content: '请根据刚才图片分析主题' }),
+      { timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
+    );
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    assert.equal(submitCalls.length, 1);
+
+    releaseFirst();
+    await imageResult.task;
+    await followupResult.task;
+  } finally {
+    fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
+  }
+
+  assert.equal(submitCalls.length, 2);
+  assert.deepEqual(
+    submitCalls[1].userMessage.metadata.imageUrls,
+    submitCalls[0].userMessage.metadata.imageUrls,
+  );
+  assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text');
+});
+
 test('wechat mp service persists Word and Excel files in user public area and reuses H5 attachment metadata', async () => {
  const token = 'token';
  const timestamp = '1710000000';