From 26846a027440b5359d8b6e68f7d7977d30bd9eb3 Mon Sep 17 00:00:00 2001 From: john Date: Wed, 5 Aug 2026 18:01:50 +0800 Subject: [PATCH] feat(wechat): add WeChat voice reco API fallback for service account When passive Recognition is empty, convert AMR to mp3 and call WeChat addvoicetorecofortext before the existing asr.tkmind.cn fallback. Co-authored-by: Cursor --- .env.example | 5 + package.json | 2 +- wechat-mp-config.mjs | 5 + wechat-mp.mjs | 33 +++++- wechat-mp.test.mjs | 140 +++++++++++++++++++++++++ wechat-voice-reco.mjs | 209 +++++++++++++++++++++++++++++++++++++ wechat-voice-reco.test.mjs | 138 ++++++++++++++++++++++++ 7 files changed, 529 insertions(+), 3 deletions(-) create mode 100644 wechat-voice-reco.mjs create mode 100644 wechat-voice-reco.test.mjs diff --git a/.env.example b/.env.example index a5f0b19..26fce87 100644 --- a/.env.example +++ b/.env.example @@ -338,6 +338,11 @@ H5_ACCESS_PASSWORD=change-me # H5_ASR_TARGET=https://asr.tkmind.cn # H5_ASR_MAX_BYTES=5242880 # H5_ASR_TIMEOUT_MS=45000 +# 微信服务号语音：Recognition 为空时优先走微信 addvoicetorecofortext（需 ffmpeg 转 mp3）；失败再回落 H5_ASR +# H5_WECHAT_MP_VOICE_RECO_API=1 +# H5_WECHAT_MP_VOICE_RECO_LANG=zh_CN +# H5_WECHAT_MP_VOICE_RECO_API_BASE=https://api.weixin.qq.com +# H5_FFMPEG_PATH=ffmpeg # 前端构建时注入（Vite，需 VITE_ 前缀） # 工作目录：新建会话时使用，必填 diff --git a/package.json b/package.json index 506611a..78614f2 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "ci:page-data-dev-loop-smoke": "node scripts/ci-page-data-dev-loop-smoke.mjs", "migrate:agent-code-run-config": "node scripts/migrate-agent-code-run-config-from-env.mjs", "test:scenario:john4-diet": "node scripts/run-scenario-test.mjs --scenario john4-children-hobby-diet-update", - "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs scheduled-task-intent.test.mjs scheduled-task-service.test.mjs scheduled-task-executor.test.mjs scheduled-task-worker.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-long-image.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-save-service.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-wechat-html-delivery.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-artifact-service.test.mjs mindspace-conversation-package-audit.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-rybbit.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-workspace-publication-delivery-service.test.mjs mindspace-workspace-tool-service.test.mjs mindspace-mcp-scoped-token.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", + "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-voice-reco.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs scheduled-task-intent.test.mjs scheduled-task-service.test.mjs scheduled-task-executor.test.mjs scheduled-task-worker.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-long-image.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-save-service.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-wechat-html-delivery.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-artifact-service.test.mjs mindspace-conversation-package-audit.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-rybbit.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-workspace-publication-delivery-service.test.mjs mindspace-workspace-tool-service.test.mjs mindspace-mcp-scoped-token.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", diff --git a/wechat-mp-config.mjs b/wechat-mp-config.mjs index ec47e52..6e1c43b 100644 --- a/wechat-mp-config.mjs +++ b/wechat-mp-config.mjs @@ -82,6 +82,11 @@ export function loadWechatMpConfig(env = process.env) { maxImageBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_IMAGE_BYTES ?? 10 * 1024 * 1024)), maxFileBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_FILE_BYTES ?? 30 * 1024 * 1024)), acceptVoice: env.H5_WECHAT_MP_ACCEPT_VOICE !== '0', + wechatVoiceRecoApiEnabled: env.H5_WECHAT_MP_VOICE_RECO_API !== '0', + wechatVoiceRecoLang: env.H5_WECHAT_MP_VOICE_RECO_LANG?.trim() || 'zh_CN', + wechatVoiceRecoApiBase: + env.H5_WECHAT_MP_VOICE_RECO_API_BASE?.trim()?.replace(/\/$/, '') + || 'https://api.weixin.qq.com', acceptImage: env.H5_WECHAT_MP_ACCEPT_IMAGE !== '0', acceptFile: env.H5_WECHAT_MP_ACCEPT_FILE !== '0', acceptLocation: env.H5_WECHAT_MP_ACCEPT_LOCATION !== '0', diff --git a/wechat-mp.mjs b/wechat-mp.mjs index ca8d90c..9f4fba1 100644 --- a/wechat-mp.mjs +++ b/wechat-mp.mjs @@ -12,6 +12,10 @@ import { persistWechatImage, uploadWechatGeneratedImage, } from './wechat-media.mjs'; +import { + buildWechatVoiceRecoVoiceId, + transcribeWechatVoiceViaRecoApi, +} from './wechat-voice-reco.mjs'; import { normalizeWechatName, resolveWechatAddressName } from './wechat/user/display-name.mjs'; import { buildAckText } from './wechat/ack/ack-provider.mjs'; import { @@ -1624,6 +1628,9 @@ export function createWechatMpService({ requireFreshPageThumbnail, repairFreshPageThumbnail, asrTarget: config.asrTarget || DEFAULT_ASR_TARGET, + wechatVoiceRecoApiEnabled: config.wechatVoiceRecoApiEnabled !== false, + wechatVoiceRecoLang: config.wechatVoiceRecoLang || 'zh_CN', + wechatVoiceRecoApiBase: config.wechatVoiceRecoApiBase || 'https://api.weixin.qq.com', }; const deferredStore = createWechatCustomerServiceDeferredStore({ mysqlPool, logger }); @@ -2091,12 +2098,30 @@ export function createWechatMpService({ } }; - const transcribeWechatVoiceMedia = async (mediaId, format) => { + const transcribeWechatVoiceMedia = async (mediaId, format, { msgId } = {}) => { if (!mediaId) return ''; const accessToken = await getStableAccessToken(); const downloaded = await downloadTemporaryMedia(accessToken, mediaId, { wechatFetch }); if (!downloaded.buffer?.length) return ''; + if (config.wechatVoiceRecoApiEnabled) { + try { + const recoText = await transcribeWechatVoiceViaRecoApi({ + accessToken, + voiceBuffer: downloaded.buffer, + format, + voiceId: buildWechatVoiceRecoVoiceId({ msgId, mediaId }), + lang: config.wechatVoiceRecoLang, + apiBase: config.wechatVoiceRecoApiBase, + wechatFetch, + convertToMp3: config.wechatVoiceRecoConvertToMp3, + }); + if (recoText) return recoText; + } catch (err) { + logger.warn?.('WeChat MP voice reco API failed:', err); + } + } + const extension = String(format ?? '').trim().toLowerCase() || 'amr'; const form = new FormData(); form.append( @@ -3577,7 +3602,11 @@ export function createWechatMpService({ if (intent.msgType === 'voice' && !intent.agentText.trim() && intent.media?.mediaId) { try { - const fallbackText = await transcribeWechatVoiceMedia(intent.media.mediaId, intent.media.format); + const fallbackText = await transcribeWechatVoiceMedia( + intent.media.mediaId, + intent.media.format, + { msgId: intent.msgId }, + ); if (fallbackText) { intent.agentText = fallbackText; intent.displayText = `语音：${fallbackText}`; diff --git a/wechat-mp.test.mjs b/wechat-mp.test.mjs index f211bb4..846310f 100644 --- a/wechat-mp.test.mjs +++ b/wechat-mp.test.mjs @@ -4444,6 +4444,7 @@ test('wechat mp service falls back to ASR when voice recognition is empty', asyn unsupportedText: 'unsupported', unboundTextPrefix: '请先绑定', asrTarget: 'https://asr.example.com', + wechatVoiceRecoApiEnabled: false, }, userAuth: { async findWechatUserByOpenid() { @@ -4548,6 +4549,145 @@ test('wechat mp service falls back to ASR when voice recognition is empty', asyn } }); +test('wechat mp service uses WeChat voice reco API before legacy ASR fallback', async () => { + const token = 'token'; + const timestamp = '1710000000'; + const nonce = 'nonce'; + let replyCalled = false; + let asrCalled = false; + const prompts = []; + const service = createWechatMpService({ + config: { + enabled: true, + appId: 'wx123', + appSecret: 'secret', + token, + publicBaseUrl: 'https://example.com', + bindPath: '/auth/wechat/authorize?intent=login', + ackText: 'ack', + unsupportedText: 'unsupported', + unboundTextPrefix: '请先绑定', + asrTarget: 'https://asr.example.com', + wechatVoiceRecoApiEnabled: true, + wechatVoiceRecoConvertToMp3: async () => Buffer.from('fake-mp3'), + }, + userAuth: { + async findWechatUserByOpenid() { + return { userId: 'user-1', status: 'active', nickname: '唐' }; + }, + async getWechatAgentRoute() { + return { agentSessionId: 'session-1' }; + }, + async clearWechatAgentRoute() {}, + async canUseChat() { + return { ok: true }; + }, + async resolveWorkingDir() { + return '/tmp/user-1'; + }, + async getAgentSessionPolicy() { + return { enableContextMemory: false, extensionOverrides: [], unrestricted: true }; + }, + async getUserPublishLayout() { + return { displayName: '唐', username: 'wx_ul610et8', slug: 'wx_ul610et8', constraints: null }; + }, + async registerAgentSession() {}, + async upsertWechatAgentRoute() {}, + async billSessionUsage() {}, + async insertWechatMpMessageDetail() {}, + }, + sessionApiFetch: async (sessionId, pathname, init = {}) => { + assert.equal(sessionId, 'session-1'); + if (pathname === '/sessions/session-1/events') { + return new Response( + [ + 'data: {"type":"Message","request_id":"req-voice-reco","message":{"id":"assistant-1","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"收到。"}]}}\n\n', + 'data: {"type":"Finish","request_id":"req-voice-reco","token_state":{"inputTokens":1,"outputTokens":2}}\n\n', + ].join(''), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + if (pathname === '/sessions/session-1/reply') { + replyCalled = true; + const body = JSON.parse(init.body); + prompts.push(body.user_message.content[0].text); + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }); + } + if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') { + return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } }); + } + throw new Error(`unexpected api path: ${pathname}`); + }, + wechatFetch: async (url, init = {}) => { + if (String(url).includes('/cgi-bin/stable_token')) { + return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/cgi-bin/media/get')) { + return new Response(Buffer.from('fake-amr-audio'), { + status: 200, + headers: { 'Content-Type': 'audio/amr' }, + }); + } + if (String(url).includes('/addvoicetorecofortext')) { + assert.equal(init.method, 'POST'); + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/queryrecoresultfortext')) { + return new Response( + JSON.stringify({ errcode: 0, errmsg: 'ok', result: '帮我看看仙居最近的天气情况' }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ); + } + if (String(url).includes('https://asr.example.com/asr/oneshot')) { + asrCalled = true; + return new Response(JSON.stringify({ code: 200, data: { text: 'legacy-asr' } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/cgi-bin/message/custom/send')) { + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected wechat url: ${url}`); + }, + }); + + const originalRandomUuid = crypto.randomUUID; + crypto.randomUUID = () => 'req-voice-reco'; + try { + const result = await service.handleInboundMessage( + inboundXml({ + msgType: 'voice', + content: '', + extraFields: { MediaId: 'media-1', Format: 'amr', MsgId: '7670473258902224896' }, + }), + { + timestamp, + nonce, + signature: signatureFor(token, timestamp, nonce), + }, + ); + + assert.equal(result.status, 200); + assert.doesNotMatch(result.body, /未识别到语音文字/); + await result.task; + assert.equal(replyCalled, true); + assert.equal(asrCalled, false); + assert.match(prompts[0], /帮我看看仙居最近的天气情况/); + } finally { + crypto.randomUUID = originalRandomUuid; + } +}); + test('wechat mp wildcard media access persists image and routes image url into agent prompt', async () => { const token = 'token'; const timestamp = '1710000000'; diff --git a/wechat-voice-reco.mjs b/wechat-voice-reco.mjs new file mode 100644 index 0000000..5963ca0 --- /dev/null +++ b/wechat-voice-reco.mjs @@ -0,0 +1,209 @@ +import crypto from 'node:crypto'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +export const DEFAULT_WECHAT_VOICE_RECO_API_BASE = 'https://api.weixin.qq.com'; +export const DEFAULT_WECHAT_VOICE_RECO_LANG = 'zh_CN'; +export const WECHAT_VOICE_RECO_MAX_MP3_BYTES = 1024 * 1024; +export const WECHAT_VOICE_RECO_DEFAULT_POLL_INTERVAL_MS = 300; +export const WECHAT_VOICE_RECO_DEFAULT_POLL_TIMEOUT_MS = 8000; + +export function buildWechatVoiceRecoVoiceId({ msgId = '', mediaId = '' } = {}) { + const raw = String(msgId || mediaId || '').trim(); + if (raw) return raw.slice(0, 64); + return crypto.randomUUID().replace(/-/g, ''); +} + +function resolveFfmpegPath(explicitPath = '') { + const configured = String(explicitPath ?? process.env.H5_FFMPEG_PATH ?? '').trim(); + return configured || 'ffmpeg'; +} + +function runFfmpeg(ffmpegPath, args) { + return new Promise((resolve, reject) => { + const proc = spawn(ffmpegPath, args, { stdio: ['ignore', 'ignore', 'pipe'] }); + let stderr = ''; + proc.stderr.on('data', (chunk) => { + stderr += chunk; + }); + proc.on('error', reject); + proc.on('close', (code) => { + if (code === 0) { + resolve(undefined); + return; + } + reject(new Error(stderr.trim() || `ffmpeg exit ${code}`)); + }); + }); +} + +export async function convertWechatVoiceToMp3( + buffer, + { + format = 'amr', + ffmpegPath = resolveFfmpegPath(), + } = {}, +) { + if (!Buffer.isBuffer(buffer) || buffer.length === 0) return null; + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-wechat-voice-')); + const extension = String(format ?? '').trim().toLowerCase() || 'amr'; + const inputPath = path.join(tmpDir, `input.${extension}`); + const outputPath = path.join(tmpDir, 'output.mp3'); + + try { + await fs.writeFile(inputPath, buffer); + await runFfmpeg(ffmpegPath, [ + '-y', + '-i', + inputPath, + '-ar', + '16000', + '-ac', + '1', + '-f', + 'mp3', + outputPath, + ]); + const mp3Buffer = await fs.readFile(outputPath); + if (!mp3Buffer.length || mp3Buffer.length > WECHAT_VOICE_RECO_MAX_MP3_BYTES) { + return null; + } + return mp3Buffer; + } catch { + return null; + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } +} + +async function readWechatApiPayload(response) { + const text = await response.text(); + if (!text) return { payload: null, text: '' }; + try { + return { payload: JSON.parse(text), text }; + } catch { + return { payload: null, text }; + } +} + +function assertWechatApiOk(payload, fallbackText, httpStatus) { + const errcode = Number(payload?.errcode ?? 0); + if (errcode !== 0) { + throw new Error(String(payload?.errmsg ?? fallbackText ?? `WeChat API error ${errcode}`)); + } + if (!httpStatus || httpStatus < 200 || httpStatus >= 300) { + throw new Error(fallbackText || `WeChat HTTP ${httpStatus}`); + } +} + +export function isRetryableWechatVoiceRecoQueryError(payload) { + const errcode = Number(payload?.errcode ?? 0); + return errcode === -1 || errcode === 87009; +} + +async function queryWechatVoiceRecoResultOnce({ + apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE, + accessToken, + voiceId, + lang = DEFAULT_WECHAT_VOICE_RECO_LANG, + wechatFetch, +}) { + if (!accessToken) throw new Error('缺少微信 access_token'); + if (!voiceId) throw new Error('缺少 voice_id'); + + const url = new URL('/cgi-bin/media/voice/queryrecoresultfortext', apiBase); + url.searchParams.set('access_token', accessToken); + url.searchParams.set('voice_id', voiceId); + url.searchParams.set('lang', lang); + + const response = await wechatFetch(url.toString(), { method: 'POST' }); + const { payload, text } = await readWechatApiPayload(response); + if (isRetryableWechatVoiceRecoQueryError(payload)) { + return ''; + } + assertWechatApiOk(payload, text, response.status); + return String(payload?.result ?? '').trim(); +} + +export async function uploadWechatVoiceForReco({ + apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE, + accessToken, + voiceId, + mp3Buffer, + lang = DEFAULT_WECHAT_VOICE_RECO_LANG, + wechatFetch, +}) { + if (!accessToken) throw new Error('缺少微信 access_token'); + if (!voiceId) throw new Error('缺少 voice_id'); + if (!Buffer.isBuffer(mp3Buffer) || !mp3Buffer.length) { + throw new Error('语音内容为空'); + } + + const url = new URL('/cgi-bin/media/voice/addvoicetorecofortext', apiBase); + url.searchParams.set('access_token', accessToken); + url.searchParams.set('format', 'mp3'); + url.searchParams.set('voice_id', voiceId); + url.searchParams.set('lang', lang); + + const form = new FormData(); + form.append('media', new Blob([mp3Buffer], { type: 'audio/mpeg' }), 'voice.mp3'); + + const response = await wechatFetch(url.toString(), { method: 'POST', body: form }); + const { payload, text } = await readWechatApiPayload(response); + assertWechatApiOk(payload, text, response.status); + return payload; +} + +export async function queryWechatVoiceRecoResult(options) { + return queryWechatVoiceRecoResultOnce(options); +} + +function sleep(ms) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +export async function transcribeWechatVoiceViaRecoApi({ + accessToken, + voiceBuffer, + format = 'amr', + voiceId, + lang = DEFAULT_WECHAT_VOICE_RECO_LANG, + apiBase = DEFAULT_WECHAT_VOICE_RECO_API_BASE, + wechatFetch, + pollIntervalMs = WECHAT_VOICE_RECO_DEFAULT_POLL_INTERVAL_MS, + pollTimeoutMs = WECHAT_VOICE_RECO_DEFAULT_POLL_TIMEOUT_MS, + convertToMp3 = convertWechatVoiceToMp3, + now = Date.now, +}) { + const mp3Buffer = await convertToMp3(voiceBuffer, { format }); + if (!mp3Buffer?.length) return ''; + + await uploadWechatVoiceForReco({ + apiBase, + accessToken, + voiceId, + mp3Buffer, + lang, + wechatFetch, + }); + + const deadline = now() + Math.max(0, pollTimeoutMs); + while (now() < deadline) { + const result = await queryWechatVoiceRecoResultOnce({ + apiBase, + accessToken, + voiceId, + lang, + wechatFetch, + }); + if (result) return result; + await sleep(Math.max(1, pollIntervalMs)); + } + + return ''; +} diff --git a/wechat-voice-reco.test.mjs b/wechat-voice-reco.test.mjs new file mode 100644 index 0000000..f5b18b4 --- /dev/null +++ b/wechat-voice-reco.test.mjs @@ -0,0 +1,138 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildWechatVoiceRecoVoiceId, + queryWechatVoiceRecoResult, + transcribeWechatVoiceViaRecoApi, + uploadWechatVoiceForReco, +} from './wechat-voice-reco.mjs'; + +test('buildWechatVoiceRecoVoiceId prefers msgId', () => { + assert.equal( + buildWechatVoiceRecoVoiceId({ msgId: '7670473258902224896', mediaId: 'media-1' }), + '7670473258902224896', + ); +}); + +test('transcribeWechatVoiceViaRecoApi uploads mp3 and polls reco result', async () => { + const calls = []; + let queryCount = 0; + const mp3Buffer = Buffer.from('fake-mp3'); + + const text = await transcribeWechatVoiceViaRecoApi({ + accessToken: 'token-1', + voiceBuffer: Buffer.from('fake-amr'), + format: 'amr', + voiceId: 'voice-1', + wechatFetch: async (url, init = {}) => { + calls.push([String(url), init.method ?? 'GET']); + if (String(url).includes('/addvoicetorecofortext')) { + assert.equal(init.method, 'POST'); + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/queryrecoresultfortext')) { + queryCount += 1; + const result = queryCount >= 2 ? '帮我看看最近的天气' : ''; + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected url: ${url}`); + }, + convertToMp3: async () => mp3Buffer, + pollIntervalMs: 1, + pollTimeoutMs: 50, + now: () => Date.now(), + }); + + assert.equal(text, '帮我看看最近的天气'); + assert.equal(calls.some(([url]) => url.includes('/addvoicetorecofortext')), true); + assert.ok(queryCount >= 2); +}); + +test('transcribeWechatVoiceViaRecoApi returns empty when mp3 conversion fails', async () => { + let fetchCalled = false; + const text = await transcribeWechatVoiceViaRecoApi({ + accessToken: 'token-1', + voiceBuffer: Buffer.from('fake-amr'), + voiceId: 'voice-1', + wechatFetch: async () => { + fetchCalled = true; + return new Response('{}', { status: 200 }); + }, + convertToMp3: async () => null, + }); + + assert.equal(text, ''); + assert.equal(fetchCalled, false); +}); + +test('uploadWechatVoiceForReco throws on WeChat business error', async () => { + await assert.rejects( + () => + uploadWechatVoiceForReco({ + accessToken: 'token-1', + voiceId: 'voice-1', + mp3Buffer: Buffer.from('fake-mp3'), + wechatFetch: async () => + new Response(JSON.stringify({ errcode: 40010, errmsg: 'invalid voice size' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + }), + /invalid voice size/, + ); +}); + +test('queryWechatVoiceRecoResult retries not-ready as empty result during polling', async () => { + let queryCount = 0; + const text = await transcribeWechatVoiceViaRecoApi({ + accessToken: 'token-1', + voiceBuffer: Buffer.from('fake-amr'), + voiceId: 'voice-1', + wechatFetch: async (url) => { + if (String(url).includes('/addvoicetorecofortext')) { + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + if (String(url).includes('/queryrecoresultfortext')) { + queryCount += 1; + if (queryCount === 1) { + return new Response(JSON.stringify({ errcode: -1, errmsg: 'system error' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result: '识别完成' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + } + throw new Error(`unexpected url: ${url}`); + }, + convertToMp3: async () => Buffer.from('fake-mp3'), + pollIntervalMs: 1, + pollTimeoutMs: 200, + }); + assert.equal(text, '识别完成'); + assert.ok(queryCount >= 2); +}); + +test('queryWechatVoiceRecoResult returns trimmed result', async () => { + const result = await queryWechatVoiceRecoResult({ + accessToken: 'token-1', + voiceId: 'voice-1', + wechatFetch: async () => + new Response(JSON.stringify({ errcode: 0, errmsg: 'ok', result: ' 你好 ' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + }); + assert.equal(result, '你好'); +});