feat(wechat): add WeChat voice reco API fallback for service account

When passive Recognition is empty, convert AMR to mp3 and call WeChat
addvoicetorecofortext before the existing asr.tkmind.cn fallback.

Co-authored-by: Cursor <cursoragent@cursor.com>

.env.example

@@ -338,6 +338,11 @@ H5_ACCESS_PASSWORD=change-me
 # H5_ASR_TARGET=https://asr.tkmind.cn
 # H5_ASR_MAX_BYTES=5242880
 # H5_ASR_TIMEOUT_MS=45000
+# 微信服务号语音：Recognition 为空时优先走微信 addvoicetorecofortext（需 ffmpeg 转 mp3）；失败再回落 H5_ASR
+# H5_WECHAT_MP_VOICE_RECO_API=1
+# H5_WECHAT_MP_VOICE_RECO_LANG=zh_CN
+# H5_WECHAT_MP_VOICE_RECO_API_BASE=https://api.weixin.qq.com
+# H5_FFMPEG_PATH=ffmpeg
 
 # 前端构建时注入（Vite，需 VITE_ 前缀）
 # 工作目录：新建会话时使用，必填

wechat-mp-config.mjs

@@ -82,6 +82,11 @@ export function loadWechatMpConfig(env = process.env) {
     maxImageBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_IMAGE_BYTES ?? 10 * 1024 * 1024)),
     maxFileBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_FILE_BYTES ?? 30 * 1024 * 1024)),
     acceptVoice: env.H5_WECHAT_MP_ACCEPT_VOICE !== '0',
+    wechatVoiceRecoApiEnabled: env.H5_WECHAT_MP_VOICE_RECO_API !== '0',
+    wechatVoiceRecoLang: env.H5_WECHAT_MP_VOICE_RECO_LANG?.trim() || 'zh_CN',
+    wechatVoiceRecoApiBase:
+      env.H5_WECHAT_MP_VOICE_RECO_API_BASE?.trim()?.replace(/\/$/, '')
+      || 'https://api.weixin.qq.com',
     acceptImage: env.H5_WECHAT_MP_ACCEPT_IMAGE !== '0',
     acceptFile: env.H5_WECHAT_MP_ACCEPT_FILE !== '0',
     acceptLocation: env.H5_WECHAT_MP_ACCEPT_LOCATION !== '0',

wechat-mp.mjs

@@ -12,6 +12,10 @@ import {
   persistWechatImage,
   uploadWechatGeneratedImage,
 } from './wechat-media.mjs';
+import {
+  buildWechatVoiceRecoVoiceId,
+  transcribeWechatVoiceViaRecoApi,
+} from './wechat-voice-reco.mjs';
 import { normalizeWechatName, resolveWechatAddressName } from './wechat/user/display-name.mjs';
 import { buildAckText } from './wechat/ack/ack-provider.mjs';
 import {
@@ -1624,6 +1628,9 @@ export function createWechatMpService({
     requireFreshPageThumbnail,
     repairFreshPageThumbnail,
     asrTarget: config.asrTarget || DEFAULT_ASR_TARGET,
+    wechatVoiceRecoApiEnabled: config.wechatVoiceRecoApiEnabled !== false,
+    wechatVoiceRecoLang: config.wechatVoiceRecoLang || 'zh_CN',
+    wechatVoiceRecoApiBase: config.wechatVoiceRecoApiBase || 'https://api.weixin.qq.com',
   };
   const deferredStore = createWechatCustomerServiceDeferredStore({ mysqlPool, logger });
 
@@ -2091,12 +2098,30 @@ export function createWechatMpService({
     }
   };
 
-  const transcribeWechatVoiceMedia = async (mediaId, format) => {
+  const transcribeWechatVoiceMedia = async (mediaId, format, { msgId } = {}) => {
     if (!mediaId) return '';
     const accessToken = await getStableAccessToken();
     const downloaded = await downloadTemporaryMedia(accessToken, mediaId, { wechatFetch });
     if (!downloaded.buffer?.length) return '';
 
+    if (config.wechatVoiceRecoApiEnabled) {
+      try {
+        const recoText = await transcribeWechatVoiceViaRecoApi({
+          accessToken,
+          voiceBuffer: downloaded.buffer,
+          format,
+          voiceId: buildWechatVoiceRecoVoiceId({ msgId, mediaId }),
+          lang: config.wechatVoiceRecoLang,
+          apiBase: config.wechatVoiceRecoApiBase,
+          wechatFetch,
+          convertToMp3: config.wechatVoiceRecoConvertToMp3,
+        });
+        if (recoText) return recoText;
+      } catch (err) {
+        logger.warn?.('WeChat MP voice reco API failed:', err);
+      }
+    }
+
     const extension = String(format ?? '').trim().toLowerCase() || 'amr';
     const form = new FormData();
     form.append(
@@ -3577,7 +3602,11 @@ export function createWechatMpService({
 
     if (intent.msgType === 'voice' && !intent.agentText.trim() && intent.media?.mediaId) {
       try {
-        const fallbackText = await transcribeWechatVoiceMedia(intent.media.mediaId, intent.media.format);
+        const fallbackText = await transcribeWechatVoiceMedia(
+          intent.media.mediaId,
+          intent.media.format,
+          { msgId: intent.msgId },
+        );
         if (fallbackText) {
           intent.agentText = fallbackText;
           intent.displayText = `语音：${fallbackText}`;

wechat-mp.test.mjs

@@ -4444,6 +4444,7 @@ test('wechat mp service falls back to ASR when voice recognition is empty', asyn
       unsupportedText: 'unsupported',
       unboundTextPrefix: '请先绑定',
       asrTarget: 'https://asr.example.com',
+      wechatVoiceRecoApiEnabled: false,
     },
     userAuth: {
       async findWechatUserByOpenid() {
@@ -4548,6 +4549,145 @@ test('wechat mp service falls back to ASR when voice recognition is empty', asyn
   }
 });
 
+test('wechat mp service uses WeChat voice reco API before legacy ASR fallback', async () => {
+  const token = 'token';
+  const timestamp = '1710000000';
+  const nonce = 'nonce';
+  let replyCalled = false;
+  let asrCalled = false;
+  const prompts = [];
+  const service = createWechatMpService({
+    config: {
+      enabled: true,
+      appId: 'wx123',
+      appSecret: 'secret',
+      token,
+      publicBaseUrl: 'https://example.com',
+      bindPath: '/auth/wechat/authorize?intent=login',
+      ackText: 'ack',
+      unsupportedText: 'unsupported',
+      unboundTextPrefix: '请先绑定',
+      asrTarget: 'https://asr.example.com',
+      wechatVoiceRecoApiEnabled: true,
+      wechatVoiceRecoConvertToMp3: async () => Buffer.from('fake-mp3'),
+    },
+    userAuth: {
+      async findWechatUserByOpenid() {
+        return { userId: 'user-1', status: 'active', nickname: '唐' };
+      },
+      async getWechatAgentRoute() {
+        return { agentSessionId: 'session-1' };
+      },
+      async clearWechatAgentRoute() {},
+      async canUseChat() {
+        return { ok: true };
+      },
+      async resolveWorkingDir() {
+        return '/tmp/user-1';
+      },
+      async getAgentSessionPolicy() {
+        return { enableContextMemory: false, extensionOverrides: [], unrestricted: true };
+      },
+      async getUserPublishLayout() {
+        return { displayName: '唐', username: 'wx_ul610et8', slug: 'wx_ul610et8', constraints: null };
+      },
+      async registerAgentSession() {},
+      async upsertWechatAgentRoute() {},
+      async billSessionUsage() {},
+      async insertWechatMpMessageDetail() {},
+    },
+    sessionApiFetch: async (sessionId, pathname, init = {}) => {
+      assert.equal(sessionId, 'session-1');
+      if (pathname === '/sessions/session-1/events') {
+        return new Response(
+          [
+            'data: {"type":"Message","request_id":"req-voice-reco","message":{"id":"assistant-1","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"收到。"}]}}\n\n',
+            'data: {"type":"Finish","request_id":"req-voice-reco","token_state":{"inputTokens":1,"outputTokens":2}}\n\n',
+          ].join(''),
+          { status: 200, headers: { 'Content-Type': 'text/event-stream' } },
+        );
+      }
+      if (pathname === '/sessions/session-1/reply') {
+        replyCalled = true;
+        const body = JSON.parse(init.body);
+        prompts.push(body.user_message.content[0].text);
+        return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
+      }
+      if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
+        return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
+      }
+      throw new Error(`unexpected api path: ${pathname}`);
+    },
+    wechatFetch: async (url, init = {}) => {
+      if (String(url).includes('/cgi-bin/stable_token')) {
+        return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/media/get')) {
+        return new Response(Buffer.from('fake-amr-audio'), {
+          status: 200,
+          headers: { 'Content-Type': 'audio/amr' },
+        });
+      }
+      if (String(url).includes('/addvoicetorecofortext')) {
+        assert.equal(init.method, 'POST');
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/queryrecoresultfortext')) {
+        return new Response(
+          JSON.stringify({ errcode: 0, errmsg: 'ok', result: '帮我看看仙居最近的天气情况' }),
+          { status: 200, headers: { 'Content-Type': 'application/json' } },
+        );
+      }
+      if (String(url).includes('https://asr.example.com/asr/oneshot')) {
+        asrCalled = true;
+        return new Response(JSON.stringify({ code: 200, data: { text: 'legacy-asr' } }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/cgi-bin/message/custom/send')) {
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected wechat url: ${url}`);
+    },
+  });
+
+  const originalRandomUuid = crypto.randomUUID;
+  crypto.randomUUID = () => 'req-voice-reco';
+  try {
+    const result = await service.handleInboundMessage(
+      inboundXml({
+        msgType: 'voice',
+        content: '',
+        extraFields: { MediaId: 'media-1', Format: 'amr', MsgId: '7670473258902224896' },
+      }),
+      {
+        timestamp,
+        nonce,
+        signature: signatureFor(token, timestamp, nonce),
+      },
+    );
+
+    assert.equal(result.status, 200);
+    assert.doesNotMatch(result.body, /未识别到语音文字/);
+    await result.task;
+    assert.equal(replyCalled, true);
+    assert.equal(asrCalled, false);
+    assert.match(prompts[0], /帮我看看仙居最近的天气情况/);
+  } finally {
+    crypto.randomUUID = originalRandomUuid;
+  }
+});
+
 test('wechat mp wildcard media access persists image and routes image url into agent prompt', async () => {
   const token = 'token';
   const timestamp = '1710000000';

wechat-voice-reco.mjs

@@ -0,0 +1,209 @@
+import crypto from 'node:crypto';
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+export const DEFAULT_WECHAT_VOICE_RECO_API_BASE = 'https://api.weixin.qq.com';
+export const DEFAULT_WECHAT_VOICE_RECO_LANG = 'zh_CN';
+export const WECHAT_VOICE_RECO_MAX_MP3_BYTES = 1024 * 1024;
+export const WECHAT_VOICE_RECO_DEFAULT_POLL_INTERVAL_MS = 300;
+export const WECHAT_VOICE_RECO_DEFAULT_POLL_TIMEOUT_MS = 8000;
+
+export function buildWechatVoiceRecoVoiceId({ msgId = '', mediaId = '' } = {}) {
+  const raw = String(msgId || mediaId || '').trim();
+  if (raw) return raw.slice(0, 64);
+  return crypto.randomUUID().replace(/-/g, '');
+}
+
+function resolveFfmpegPath(explicitPath = '') {
+  const configured = String(explicitPath ?? process.env.H5_FFMPEG_PATH ?? '').trim();
+  return configured || 'ffmpeg';
+}
+
+function runFfmpeg(ffmpegPath, args) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(ffmpegPath, args, { stdio: ['ignore', 'ignore', 'pipe'] });
+    let stderr = '';
+    proc.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    proc.on('error', reject);
+    proc.on('close', (code) => {
+      if (code === 0) {
+        resolve(undefined);
+        return;
+      }
+      reject(new Error(stderr.trim() || `ffmpeg exit ${code}`));
+    });
+  });
+}
+
+export async function convertWechatVoiceToMp3(
+  buffer,
+  {
+    format = 'amr',
+    ffmpegPath = resolveFfmpegPath(),
+  } = {},
+) {
+  if (!Buffer.isBuffer(buffer) || buffer.length === 0) return null;
+
+  const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-wechat-voice-'));
+  const extension = String(format ?? '').trim().toLowerCase() || 'amr';
+  const inputPath = path.join(tmpDir, `input.${extension}`);
+  const outputPath = path.join(tmpDir, 'output.mp3');
+
+  try {
+    await fs.writeFile(inputPath, buffer);
+    await runFfmpeg(ffmpegPath, [
+      '-y',
+      '-i',
+      inputPath,
+      '-ar',
+      '16000',
+      '-ac',
+      '1',
+      '-f',
+      'mp3',
+      outputPath,
+    ]);
+    const mp3Buffer = await fs.readFile(outputPath);
+    if (!mp3Buffer.length || mp3Buffer.length > WECHAT_VOICE_RECO_MAX_MP3_BYTES) {
+      return null;
+    }
+    return mp3Buffer;
+  } catch {
+    return null;
+  } finally {
+    await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+async function readWechatApiPayload(response) {
+  const text = await response.text();
+  if (!text) return { payload: null, text: '' };
+  try {
+    return { payload: JSON.parse(text), text };
+  } catch {
+    return { payload: null, text };
+  }
+}
+
+function assertWechatApiOk(payload, fallbackText, httpStatus) {
+  const errcode = Number(payload?.errcode ?? 0);
+  if (errcode !== 0) {
+    throw new Error(String(payload?.errmsg ?? fallbackText ?? `WeChat API error ${errcode}`));
+  }
+  if (!httpStatus || httpStatus < 200 || httpStatus >= 300) {
+    throw new Error(fallbackText || `WeChat HTTP ${httpStatus}`);
+  }
+}
+
+export function isRetryableWechatVoiceRecoQueryError(payload) {
+  const errcode = Number(payload?.errcode ?? 0);
+  return errcode === -1 || errcode === 87009;
+}
+
+async function queryWechatVoiceRecoResultOnce({
+  apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE,
+  accessToken,
+  voiceId,
+  lang = DEFAULT_WECHAT_VOICE_RECO_LANG,
+  wechatFetch,
+}) {
+  if (!accessToken) throw new Error('缺少微信 access_token');
+  if (!voiceId) throw new Error('缺少 voice_id');
+
+  const url = new URL('/cgi-bin/media/voice/queryrecoresultfortext', apiBase);
+  url.searchParams.set('access_token', accessToken);
+  url.searchParams.set('voice_id', voiceId);
+  url.searchParams.set('lang', lang);
+
+  const response = await wechatFetch(url.toString(), { method: 'POST' });
+  const { payload, text } = await readWechatApiPayload(response);
+  if (isRetryableWechatVoiceRecoQueryError(payload)) {
+    return '';
+  }
+  assertWechatApiOk(payload, text, response.status);
+  return String(payload?.result ?? '').trim();
+}
+
+export async function uploadWechatVoiceForReco({
+  apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE,
+  accessToken,
+  voiceId,
+  mp3Buffer,
+  lang = DEFAULT_WECHAT_VOICE_RECO_LANG,
+  wechatFetch,
+}) {
+  if (!accessToken) throw new Error('缺少微信 access_token');
+  if (!voiceId) throw new Error('缺少 voice_id');
+  if (!Buffer.isBuffer(mp3Buffer) || !mp3Buffer.length) {
+    throw new Error('语音内容为空');
+  }
+
+  const url = new URL('/cgi-bin/media/voice/addvoicetorecofortext', apiBase);
+  url.searchParams.set('access_token', accessToken);
+  url.searchParams.set('format', 'mp3');
+  url.searchParams.set('voice_id', voiceId);
+  url.searchParams.set('lang', lang);
+
+  const form = new FormData();
+  form.append('media', new Blob([mp3Buffer], { type: 'audio/mpeg' }), 'voice.mp3');
+
+  const response = await wechatFetch(url.toString(), { method: 'POST', body: form });
+  const { payload, text } = await readWechatApiPayload(response);
+  assertWechatApiOk(payload, text, response.status);
+  return payload;
+}
+
+export async function queryWechatVoiceRecoResult(options) {
+  return queryWechatVoiceRecoResultOnce(options);
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+export async function transcribeWechatVoiceViaRecoApi({
+  accessToken,
+  voiceBuffer,
+  format = 'amr',
+  voiceId,
+  lang = DEFAULT_WECHAT_VOICE_RECO_LANG,
+  apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE,
+  wechatFetch,
+  pollIntervalMs = WECHAT_VOICE_RECO_DEFAULT_POLL_INTERVAL_MS,
+  pollTimeoutMs = WECHAT_VOICE_RECO_DEFAULT_POLL_TIMEOUT_MS,
+  convertToMp3 = convertWechatVoiceToMp3,
+  now = Date.now,
+}) {
+  const mp3Buffer = await convertToMp3(voiceBuffer, { format });
+  if (!mp3Buffer?.length) return '';
+
+  await uploadWechatVoiceForReco({
+    apiBase,
+    accessToken,
+    voiceId,
+    mp3Buffer,
+    lang,
+    wechatFetch,
+  });
+
+  const deadline = now() + Math.max(0, pollTimeoutMs);
+  while (now() < deadline) {
+    const result = await queryWechatVoiceRecoResultOnce({
+      apiBase,
+      accessToken,
+      voiceId,
+      lang,
+      wechatFetch,
+    });
+    if (result) return result;
+    await sleep(Math.max(1, pollIntervalMs));
+  }
+
+  return '';
+}

wechat-voice-reco.test.mjs

@@ -0,0 +1,138 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  buildWechatVoiceRecoVoiceId,
+  queryWechatVoiceRecoResult,
+  transcribeWechatVoiceViaRecoApi,
+  uploadWechatVoiceForReco,
+} from './wechat-voice-reco.mjs';
+
+test('buildWechatVoiceRecoVoiceId prefers msgId', () => {
+  assert.equal(
+    buildWechatVoiceRecoVoiceId({ msgId: '7670473258902224896', mediaId: 'media-1' }),
+    '7670473258902224896',
+  );
+});
+
+test('transcribeWechatVoiceViaRecoApi uploads mp3 and polls reco result', async () => {
+  const calls = [];
+  let queryCount = 0;
+  const mp3Buffer = Buffer.from('fake-mp3');
+
+  const text = await transcribeWechatVoiceViaRecoApi({
+    accessToken: 'token-1',
+    voiceBuffer: Buffer.from('fake-amr'),
+    format: 'amr',
+    voiceId: 'voice-1',
+    wechatFetch: async (url, init = {}) => {
+      calls.push([String(url), init.method ?? 'GET']);
+      if (String(url).includes('/addvoicetorecofortext')) {
+        assert.equal(init.method, 'POST');
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/queryrecoresultfortext')) {
+        queryCount += 1;
+        const result = queryCount >= 2 ? '帮我看看最近的天气' : '';
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected url: ${url}`);
+    },
+    convertToMp3: async () => mp3Buffer,
+    pollIntervalMs: 1,
+    pollTimeoutMs: 50,
+    now: () => Date.now(),
+  });
+
+  assert.equal(text, '帮我看看最近的天气');
+  assert.equal(calls.some(([url]) => url.includes('/addvoicetorecofortext')), true);
+  assert.ok(queryCount >= 2);
+});
+
+test('transcribeWechatVoiceViaRecoApi returns empty when mp3 conversion fails', async () => {
+  let fetchCalled = false;
+  const text = await transcribeWechatVoiceViaRecoApi({
+    accessToken: 'token-1',
+    voiceBuffer: Buffer.from('fake-amr'),
+    voiceId: 'voice-1',
+    wechatFetch: async () => {
+      fetchCalled = true;
+      return new Response('{}', { status: 200 });
+    },
+    convertToMp3: async () => null,
+  });
+
+  assert.equal(text, '');
+  assert.equal(fetchCalled, false);
+});
+
+test('uploadWechatVoiceForReco throws on WeChat business error', async () => {
+  await assert.rejects(
+    () =>
+      uploadWechatVoiceForReco({
+        accessToken: 'token-1',
+        voiceId: 'voice-1',
+        mp3Buffer: Buffer.from('fake-mp3'),
+        wechatFetch: async () =>
+          new Response(JSON.stringify({ errcode: 40010, errmsg: 'invalid voice size' }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' },
+          }),
+      }),
+    /invalid voice size/,
+  );
+});
+
+test('queryWechatVoiceRecoResult retries not-ready as empty result during polling', async () => {
+  let queryCount = 0;
+  const text = await transcribeWechatVoiceViaRecoApi({
+    accessToken: 'token-1',
+    voiceBuffer: Buffer.from('fake-amr'),
+    voiceId: 'voice-1',
+    wechatFetch: async (url) => {
+      if (String(url).includes('/addvoicetorecofortext')) {
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      if (String(url).includes('/queryrecoresultfortext')) {
+        queryCount += 1;
+        if (queryCount === 1) {
+          return new Response(JSON.stringify({ errcode: -1, errmsg: 'system error' }), {
+            status: 200,
+            headers: { 'Content-Type': 'application/json' },
+          });
+        }
+        return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result: '识别完成' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+      throw new Error(`unexpected url: ${url}`);
+    },
+    convertToMp3: async () => Buffer.from('fake-mp3'),
+    pollIntervalMs: 1,
+    pollTimeoutMs: 200,
+  });
+  assert.equal(text, '识别完成');
+  assert.ok(queryCount >= 2);
+});
+
+test('queryWechatVoiceRecoResult returns trimmed result', async () => {
+  const result = await queryWechatVoiceRecoResult({
+    accessToken: 'token-1',
+    voiceId: 'voice-1',
+    wechatFetch: async () =>
+      new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result: '  你好  ' }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      }),
+  });
+  assert.equal(result, '你好');
+});