Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f4366e882 | |||
| 2ce1527edd | |||
| 1161292482 | |||
| 711a7d2061 | |||
| 341cf5ae89 | |||
| ae3e1ba3fa | |||
| 8eaf4d23f3 | |||
| 8ccaf3b39d | |||
| 94f347398d | |||
| fd3904fdee | |||
| 055d53c58b |
@@ -820,6 +820,8 @@ async function bootstrapUserAuth() {
|
|||||||
const target = await tkmindProxy.resolveTarget(sessionId);
|
const target = await tkmindProxy.resolveTarget(sessionId);
|
||||||
return tkmindProxy.apiFetchTo(target, pathname, init);
|
return tkmindProxy.apiFetchTo(target, pathname, init);
|
||||||
},
|
},
|
||||||
|
submitSessionReply: ({ userId, sessionId, requestId, userMessage }) =>
|
||||||
|
tkmindProxy.submitSessionReplyForUser(userId, sessionId, requestId, userMessage),
|
||||||
scheduleService: process.env.H5_SCHEDULE_ENABLED === '1' ? scheduleService : null,
|
scheduleService: process.env.H5_SCHEDULE_ENABLED === '1' ? scheduleService : null,
|
||||||
wechatScheduleLlmConfigService,
|
wechatScheduleLlmConfigService,
|
||||||
llmProviderService,
|
llmProviderService,
|
||||||
|
|||||||
@@ -83,6 +83,42 @@ test('buildVisionPayload marks one billable image analysis when vision succeeds'
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('buildVisionPayload sends all current-turn images to Qwen in order', async () => {
|
||||||
|
const imageUrls = [
|
||||||
|
'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/first.jpg',
|
||||||
|
'https://m.tkmind.cn/MindSpace/user-1/public/wechat-mp/second.jpg',
|
||||||
|
];
|
||||||
|
let analyzedImages = [];
|
||||||
|
const result = await buildVisionPayload({
|
||||||
|
userId: 'user-1',
|
||||||
|
publishLayout: { publicUrl: 'https://m.tkmind.cn/MindSpace/user-1' },
|
||||||
|
userMessage: {
|
||||||
|
content: [{ type: 'text', text: '请结合两张图片生成页面' }],
|
||||||
|
metadata: { imageUrls },
|
||||||
|
},
|
||||||
|
fetchImpl: async () =>
|
||||||
|
new Response(Buffer.from('fake-image'), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'image/jpeg' },
|
||||||
|
}),
|
||||||
|
llmProviderService: {
|
||||||
|
analyzeImagesWithVision: async (images) => {
|
||||||
|
analyzedImages = images;
|
||||||
|
return '第一张是宝塔,第二张是晚霞。';
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(analyzedImages.length, 2);
|
||||||
|
assert.deepEqual(analyzedImages.map((item) => item.rawUrl), imageUrls);
|
||||||
|
assert.deepEqual(result?.userMessage?.metadata?.imageUrls, [
|
||||||
|
'/MindSpace/user-1/public/wechat-mp/first.jpg',
|
||||||
|
'/MindSpace/user-1/public/wechat-mp/second.jpg',
|
||||||
|
]);
|
||||||
|
assert.match(result?.userMessage?.content?.[0]?.text ?? '', /本轮用户仅上传 2 张图片/);
|
||||||
|
assert.match(result?.userMessage?.content?.[0]?.text ?? '', /图片2/);
|
||||||
|
});
|
||||||
|
|
||||||
test('buildVisionPayload does not mark billable usage when vision analysis fails', async () => {
|
test('buildVisionPayload does not mark billable usage when vision analysis fails', async () => {
|
||||||
const result = await buildVisionPayload({
|
const result = await buildVisionPayload({
|
||||||
userId: 'user-1',
|
userId: 'user-1',
|
||||||
|
|||||||
@@ -898,6 +898,67 @@ test('submitSessionReplyForUser adds goose metadata visibility flags before repl
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('submitSessionReplyForUser applies the shared Qwen vision preprocessing path', async () => {
|
||||||
|
await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => {
|
||||||
|
const proxy = createTkmindProxy({
|
||||||
|
apiTarget,
|
||||||
|
apiSecret: 'test-secret',
|
||||||
|
userAuth: {
|
||||||
|
...createMemoryTestUserAuth(workingDir),
|
||||||
|
async ownsSession() {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
async canUseChat() {
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
async getUserById() {
|
||||||
|
return { id: 'user-1' };
|
||||||
|
},
|
||||||
|
async getUserPublishLayout() {
|
||||||
|
return { publicUrl: 'https://example.com/MindSpace/user-1' };
|
||||||
|
},
|
||||||
|
async resolveUserPolicies() {
|
||||||
|
return { unrestricted: true, policies: {} };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
localFetchAsset: async () => ({
|
||||||
|
buffer: Buffer.from('fake-image'),
|
||||||
|
mimeType: 'image/jpeg',
|
||||||
|
}),
|
||||||
|
llmProviderService: {
|
||||||
|
async applyBestProviderForSession() {
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
async hasVisionKey() {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
async analyzeImagesWithVision() {
|
||||||
|
return '一件蓝色产品,白色背景,竖版构图。';
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await proxy.submitSessionReplyForUser(
|
||||||
|
'user-1',
|
||||||
|
'session-1',
|
||||||
|
'request-qwen-vision',
|
||||||
|
{
|
||||||
|
id: 'message-qwen-vision',
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '请分析这张图片' }],
|
||||||
|
metadata: {
|
||||||
|
imageUrls: ['/api/mindspace/v1/assets/asset-1/download?inline=1'],
|
||||||
|
displayText: '请分析这张图片',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const forwardedText = replyBodies[0]?.user_message?.content?.[0]?.text ?? '';
|
||||||
|
assert.match(forwardedText, /Qwen VL 图片描述/);
|
||||||
|
assert.match(forwardedText, /一件蓝色产品/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
|
test('submitSessionReplyForUser passes current prompt to Memory V2 resolve before existing reply path', async () => {
|
||||||
let resolveInput = null;
|
let resolveInput = null;
|
||||||
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {
|
await withFakeGoosedSession(async ({ apiTarget, workingDir }) => {
|
||||||
|
|||||||
@@ -9,12 +9,19 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|||||||
|
|
||||||
const DEFAULT_WECHAT_MEDIA_URL = 'https://api.weixin.qq.com/cgi-bin/media/get';
|
const DEFAULT_WECHAT_MEDIA_URL = 'https://api.weixin.qq.com/cgi-bin/media/get';
|
||||||
const DEFAULT_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
const DEFAULT_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||||
|
const DEFAULT_MAX_ATTACHMENT_BYTES = 30 * 1024 * 1024;
|
||||||
const ALLOWED_IMAGE_MIME_TYPES = new Map([
|
const ALLOWED_IMAGE_MIME_TYPES = new Map([
|
||||||
['image/jpeg', 'jpg'],
|
['image/jpeg', 'jpg'],
|
||||||
['image/png', 'png'],
|
['image/png', 'png'],
|
||||||
['image/webp', 'webp'],
|
['image/webp', 'webp'],
|
||||||
['image/gif', 'gif'],
|
['image/gif', 'gif'],
|
||||||
]);
|
]);
|
||||||
|
const ALLOWED_ATTACHMENT_EXTENSIONS = new Map([
|
||||||
|
['.doc', 'application/msword'],
|
||||||
|
['.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
||||||
|
['.xls', 'application/vnd.ms-excel'],
|
||||||
|
['.xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'],
|
||||||
|
]);
|
||||||
|
|
||||||
function resolveImageExtension(contentType = '', fallbackUrl = '') {
|
function resolveImageExtension(contentType = '', fallbackUrl = '') {
|
||||||
const normalized = String(contentType ?? '')
|
const normalized = String(contentType ?? '')
|
||||||
@@ -44,6 +51,32 @@ function ensureImageWithinLimit(buffer, maxBytes) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sanitizeAttachmentFilename(filename = '') {
|
||||||
|
const basename = path.basename(String(filename ?? '').trim()).replace(/[\u0000-\u001f\u007f]/g, '');
|
||||||
|
if (!basename || basename === '.' || basename === '..') {
|
||||||
|
throw new Error('微信文件缺少有效文件名');
|
||||||
|
}
|
||||||
|
const extension = path.extname(basename).toLowerCase();
|
||||||
|
const mimeType = ALLOWED_ATTACHMENT_EXTENSIONS.get(extension);
|
||||||
|
if (!mimeType) {
|
||||||
|
throw new Error('当前服务号文件仅支持 Word(doc/docx)和 Excel(xls/xlsx)');
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
filename: basename.slice(0, 160),
|
||||||
|
extension,
|
||||||
|
mimeType,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureAttachmentWithinLimit(buffer, maxBytes) {
|
||||||
|
if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
|
||||||
|
throw new Error('微信文件内容为空');
|
||||||
|
}
|
||||||
|
if (buffer.length > maxBytes) {
|
||||||
|
throw new Error(`文件超过大小限制(${maxBytes} bytes)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function downloadTemporaryMedia(accessToken, mediaId, { wechatFetch = undiciFetch } = {}) {
|
export async function downloadTemporaryMedia(accessToken, mediaId, { wechatFetch = undiciFetch } = {}) {
|
||||||
if (!accessToken) throw new Error('缺少微信 access_token');
|
if (!accessToken) throw new Error('缺少微信 access_token');
|
||||||
if (!mediaId) throw new Error('缺少微信 mediaId');
|
if (!mediaId) throw new Error('缺少微信 mediaId');
|
||||||
@@ -154,3 +187,58 @@ export async function persistWechatImage(
|
|||||||
source,
|
source,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function persistWechatAttachment(
|
||||||
|
{
|
||||||
|
userId,
|
||||||
|
appId,
|
||||||
|
openid,
|
||||||
|
msgId,
|
||||||
|
mediaId,
|
||||||
|
filename,
|
||||||
|
publicBaseUrl,
|
||||||
|
maxFileBytes = DEFAULT_MAX_ATTACHMENT_BYTES,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
wechatFetch = undiciFetch,
|
||||||
|
accessToken,
|
||||||
|
h5Root = __dirname,
|
||||||
|
} = {},
|
||||||
|
) {
|
||||||
|
if (!userId) throw new Error('缺少 userId');
|
||||||
|
const resolved = sanitizeAttachmentFilename(filename);
|
||||||
|
const downloaded = await downloadTemporaryMedia(accessToken, mediaId, { wechatFetch });
|
||||||
|
ensureAttachmentWithinLimit(downloaded.buffer, maxFileBytes);
|
||||||
|
|
||||||
|
const publishDir = path.join(h5Root, PUBLISH_ROOT_DIR, String(userId), PUBLIC_ZONE_DIR, 'wechat-mp');
|
||||||
|
fs.mkdirSync(publishDir, { recursive: true });
|
||||||
|
|
||||||
|
const timestamp = Date.now();
|
||||||
|
const hash = crypto.createHash('sha1').update(downloaded.buffer).digest('hex').slice(0, 12);
|
||||||
|
const originalStem = path.basename(resolved.filename, resolved.extension)
|
||||||
|
.replace(/[^\p{L}\p{N}._-]+/gu, '_')
|
||||||
|
.replace(/^_+|_+$/g, '')
|
||||||
|
.slice(0, 80) || 'attachment';
|
||||||
|
const identity = [appId || 'wx', openid || 'openid', msgId || timestamp, mediaId || hash]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join('-')
|
||||||
|
.replace(/[^a-zA-Z0-9._-]+/g, '_')
|
||||||
|
.slice(0, 100);
|
||||||
|
const publicFilename = `${originalStem}-${identity}-${hash}${resolved.extension}`;
|
||||||
|
const absolutePath = path.join(publishDir, publicFilename);
|
||||||
|
fs.writeFileSync(absolutePath, downloaded.buffer);
|
||||||
|
|
||||||
|
return {
|
||||||
|
absolutePath,
|
||||||
|
bytes: downloaded.buffer.length,
|
||||||
|
contentType: resolved.mimeType,
|
||||||
|
filename: resolved.filename,
|
||||||
|
publicFilename,
|
||||||
|
publicUrl: buildWechatImagePublicUrl({
|
||||||
|
publicBaseUrl,
|
||||||
|
publishKey: String(userId),
|
||||||
|
filename: publicFilename,
|
||||||
|
}),
|
||||||
|
source: 'wechat_media',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,13 @@ function deriveWechatEndpointFromUrl(baseUrl, suffix) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseCsvList(value = '') {
|
||||||
|
return String(value ?? '')
|
||||||
|
.split(',')
|
||||||
|
.map((item) => item.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
export function loadWechatMpConfig(env = process.env) {
|
export function loadWechatMpConfig(env = process.env) {
|
||||||
const appId = env.H5_WECHAT_MP_APP_ID?.trim() ?? env.H5_WECHAT_APP_ID?.trim() ?? '';
|
const appId = env.H5_WECHAT_MP_APP_ID?.trim() ?? env.H5_WECHAT_APP_ID?.trim() ?? '';
|
||||||
const appSecret =
|
const appSecret =
|
||||||
@@ -67,10 +74,13 @@ export function loadWechatMpConfig(env = process.env) {
|
|||||||
mediaPublicBaseUrl:
|
mediaPublicBaseUrl:
|
||||||
env.H5_WECHAT_MP_MEDIA_PUBLIC_BASE_URL?.trim()?.replace(/\/$/, '') || publicBaseUrl,
|
env.H5_WECHAT_MP_MEDIA_PUBLIC_BASE_URL?.trim()?.replace(/\/$/, '') || publicBaseUrl,
|
||||||
maxImageBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_IMAGE_BYTES ?? 10 * 1024 * 1024)),
|
maxImageBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_IMAGE_BYTES ?? 10 * 1024 * 1024)),
|
||||||
|
maxFileBytes: Math.max(1, Number(env.H5_WECHAT_MP_MAX_FILE_BYTES ?? 30 * 1024 * 1024)),
|
||||||
acceptVoice: env.H5_WECHAT_MP_ACCEPT_VOICE !== '0',
|
acceptVoice: env.H5_WECHAT_MP_ACCEPT_VOICE !== '0',
|
||||||
acceptImage: env.H5_WECHAT_MP_ACCEPT_IMAGE !== '0',
|
acceptImage: env.H5_WECHAT_MP_ACCEPT_IMAGE !== '0',
|
||||||
|
acceptFile: env.H5_WECHAT_MP_ACCEPT_FILE !== '0',
|
||||||
acceptLocation: env.H5_WECHAT_MP_ACCEPT_LOCATION !== '0',
|
acceptLocation: env.H5_WECHAT_MP_ACCEPT_LOCATION !== '0',
|
||||||
acceptLink: env.H5_WECHAT_MP_ACCEPT_LINK !== '0',
|
acceptLink: env.H5_WECHAT_MP_ACCEPT_LINK !== '0',
|
||||||
|
mediaAnalysisGrayUsers: parseCsvList(env.H5_WECHAT_MP_MEDIA_GRAY_USERS),
|
||||||
encodingAesKey: env.H5_WECHAT_MP_ENCODING_AES_KEY?.trim() ?? '',
|
encodingAesKey: env.H5_WECHAT_MP_ENCODING_AES_KEY?.trim() ?? '',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
+305
-30
@@ -9,7 +9,11 @@ import { resolveSessionAccess } from './session-broker.mjs';
|
|||||||
import { isStubPublicHtmlContent, materializeMissingPublicHtmlWrites } from './mindspace-public-finish-sync.mjs';
|
import { isStubPublicHtmlContent, materializeMissingPublicHtmlWrites } from './mindspace-public-finish-sync.mjs';
|
||||||
import { loadWechatMpConfig } from './wechat-mp-config.mjs';
|
import { loadWechatMpConfig } from './wechat-mp-config.mjs';
|
||||||
import { buildPublicUrl, PUBLISH_ROOT_DIR } from './user-publish.mjs';
|
import { buildPublicUrl, PUBLISH_ROOT_DIR } from './user-publish.mjs';
|
||||||
import { downloadTemporaryMedia, persistWechatImage } from './wechat-media.mjs';
|
import {
|
||||||
|
downloadTemporaryMedia,
|
||||||
|
persistWechatAttachment,
|
||||||
|
persistWechatImage,
|
||||||
|
} from './wechat-media.mjs';
|
||||||
import { normalizeWechatName, resolveWechatAddressName } from './wechat/user/display-name.mjs';
|
import { normalizeWechatName, resolveWechatAddressName } from './wechat/user/display-name.mjs';
|
||||||
import { buildAckText } from './wechat/ack/ack-provider.mjs';
|
import { buildAckText } from './wechat/ack/ack-provider.mjs';
|
||||||
import { guardScheduleConfirmationReply } from './wechat/handlers/schedule-guard.mjs';
|
import { guardScheduleConfirmationReply } from './wechat/handlers/schedule-guard.mjs';
|
||||||
@@ -46,6 +50,8 @@ const DEFAULT_WECHAT_CUSTOMER_SERVICE_URL =
|
|||||||
'https://api.weixin.qq.com/cgi-bin/message/custom/send';
|
'https://api.weixin.qq.com/cgi-bin/message/custom/send';
|
||||||
const DEFAULT_WECHAT_JSAPI_TICKET_URL = 'https://api.weixin.qq.com/cgi-bin/ticket/getticket';
|
const DEFAULT_WECHAT_JSAPI_TICKET_URL = 'https://api.weixin.qq.com/cgi-bin/ticket/getticket';
|
||||||
const DEFAULT_ASR_TARGET = process.env.H5_ASR_TARGET ?? 'https://asr.tkmind.cn';
|
const DEFAULT_ASR_TARGET = process.env.H5_ASR_TARGET ?? 'https://asr.tkmind.cn';
|
||||||
|
const WECHAT_RECENT_MEDIA_TTL_MS = 15 * 60 * 1000;
|
||||||
|
const WECHAT_RECENT_IMAGE_MAX_COUNT = 10;
|
||||||
export { loadWechatMpConfig };
|
export { loadWechatMpConfig };
|
||||||
const PUBLIC_HTML_LINK_PATTERN =
|
const PUBLIC_HTML_LINK_PATTERN =
|
||||||
/https?:\/\/[^\s<>"')\]]+\/MindSpace\/([0-9a-f-]{36}|[a-z0-9._-]+)\/public\/([^\s<>"')\]]+\.html)/gi;
|
/https?:\/\/[^\s<>"')\]]+\/MindSpace\/([0-9a-f-]{36}|[a-z0-9._-]+)\/public\/([^\s<>"')\]]+\.html)/gi;
|
||||||
@@ -100,6 +106,8 @@ function parseWechatMessage(xml) {
|
|||||||
description: parseXmlField(xml, 'Description'),
|
description: parseXmlField(xml, 'Description'),
|
||||||
url: parseXmlField(xml, 'Url'),
|
url: parseXmlField(xml, 'Url'),
|
||||||
thumbMediaId: parseXmlField(xml, 'ThumbMediaId'),
|
thumbMediaId: parseXmlField(xml, 'ThumbMediaId'),
|
||||||
|
fileName: parseXmlField(xml, 'FileName') || parseXmlField(xml, 'Filename'),
|
||||||
|
fileSize: parseXmlField(xml, 'FileSize'),
|
||||||
event: parseXmlField(xml, 'Event').toLowerCase(),
|
event: parseXmlField(xml, 'Event').toLowerCase(),
|
||||||
eventKey: parseXmlField(xml, 'EventKey'),
|
eventKey: parseXmlField(xml, 'EventKey'),
|
||||||
latitude: parseXmlField(xml, 'Latitude'),
|
latitude: parseXmlField(xml, 'Latitude'),
|
||||||
@@ -204,7 +212,14 @@ function pushMessage(messages, incoming) {
|
|||||||
return [...messages, incoming];
|
return [...messages, incoming];
|
||||||
}
|
}
|
||||||
|
|
||||||
async function executeSessionReply(apiFetch, sessionId, requestId, prompt, metadata = {}) {
|
async function executeSessionReply(
|
||||||
|
apiFetch,
|
||||||
|
sessionId,
|
||||||
|
requestId,
|
||||||
|
prompt,
|
||||||
|
metadata = {},
|
||||||
|
{ submitReply = null } = {},
|
||||||
|
) {
|
||||||
const eventsResponse = await apiFetch(`/sessions/${sessionId}/events`, {
|
const eventsResponse = await apiFetch(`/sessions/${sessionId}/events`, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: { Accept: 'text/event-stream' },
|
headers: { Accept: 'text/event-stream' },
|
||||||
@@ -214,18 +229,23 @@ async function executeSessionReply(apiFetch, sessionId, requestId, prompt, metad
|
|||||||
throw new Error(text || '无法建立公众号消息事件流');
|
throw new Error(text || '无法建立公众号消息事件流');
|
||||||
}
|
}
|
||||||
|
|
||||||
const replyResponse = await apiFetch(`/sessions/${sessionId}/reply`, {
|
const userMessage = createUserMessage(prompt, metadata);
|
||||||
method: 'POST',
|
if (submitReply) {
|
||||||
body: JSON.stringify({
|
await submitReply({ sessionId, requestId, userMessage });
|
||||||
request_id: requestId,
|
} else {
|
||||||
user_message: createUserMessage(prompt, metadata),
|
const replyResponse = await apiFetch(`/sessions/${sessionId}/reply`, {
|
||||||
}),
|
method: 'POST',
|
||||||
});
|
body: JSON.stringify({
|
||||||
if (!replyResponse.ok) {
|
request_id: requestId,
|
||||||
const text = await replyResponse.text().catch(() => '');
|
user_message: userMessage,
|
||||||
throw new Error(text || 'Agent reply 请求失败');
|
}),
|
||||||
|
});
|
||||||
|
if (!replyResponse.ok) {
|
||||||
|
const text = await replyResponse.text().catch(() => '');
|
||||||
|
throw new Error(text || 'Agent reply 请求失败');
|
||||||
|
}
|
||||||
|
replyResponse.body?.cancel().catch?.(() => {});
|
||||||
}
|
}
|
||||||
replyResponse.body?.cancel().catch?.(() => {});
|
|
||||||
|
|
||||||
const reader = eventsResponse.body.getReader();
|
const reader = eventsResponse.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
@@ -994,6 +1014,8 @@ export function isRecoverableWechatAgentSessionError(message) {
|
|||||||
if (/stale_session_poisoned_completion/i.test(normalized)) return true;
|
if (/stale_session_poisoned_completion/i.test(normalized)) return true;
|
||||||
if (/403|404|not found|无权访问/i.test(normalized)) return true;
|
if (/403|404|not found|无权访问/i.test(normalized)) return true;
|
||||||
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) return true;
|
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) return true;
|
||||||
|
if (/session already has an active request|active request.*cancel/i.test(normalized)) return true;
|
||||||
|
if (/wechat_agent_incomplete_reply/i.test(normalized)) return true;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1024,6 +1046,13 @@ export function findRecoverableWechatAgentErrorInReply(reply) {
|
|||||||
export function assertWechatAgentReplyIsSendable(reply) {
|
export function assertWechatAgentReplyIsSendable(reply) {
|
||||||
const recoverable = findRecoverableWechatAgentErrorInReply(reply);
|
const recoverable = findRecoverableWechatAgentErrorInReply(reply);
|
||||||
if (recoverable) throw new Error(recoverable);
|
if (recoverable) throw new Error(recoverable);
|
||||||
|
const text = String(reply?.text ?? '').trim();
|
||||||
|
if (
|
||||||
|
/^(?:let me|i(?:'ll| will))\s+(?:first\s+)?(?:look|check|inspect|analy[sz]e)(?:\s+at)?\s+(?:the\s+)?image(?:\s+first)?[.!]?$/iu.test(text) ||
|
||||||
|
/^(?:让我|我先)(?:先)?(?:看|查看|检查|分析)(?:一下)?(?:这张|该张|这个)?图片[。!!]?$/u.test(text)
|
||||||
|
) {
|
||||||
|
throw new Error('wechat_agent_incomplete_reply');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isWechatAgentApiErrorText(message) {
|
export function isWechatAgentApiErrorText(message) {
|
||||||
@@ -1164,6 +1193,24 @@ function normalizeNumber(value) {
|
|||||||
return Number.isFinite(num) ? num : null;
|
return Number.isFinite(num) ? num : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isWechatMediaGrayUser(user, configuredUsers = []) {
|
||||||
|
const allowlist = Array.isArray(configuredUsers)
|
||||||
|
? configuredUsers.map((value) => String(value ?? '').trim().toLowerCase()).filter(Boolean)
|
||||||
|
: [];
|
||||||
|
if (allowlist.length === 0) return false;
|
||||||
|
if (allowlist.includes('*')) return true;
|
||||||
|
const identities = [
|
||||||
|
user?.userId,
|
||||||
|
user?.username,
|
||||||
|
user?.slug,
|
||||||
|
user?.displayName,
|
||||||
|
user?.nickname,
|
||||||
|
]
|
||||||
|
.map((value) => String(value ?? '').trim().toLowerCase())
|
||||||
|
.filter(Boolean);
|
||||||
|
return identities.some((identity) => allowlist.includes(identity));
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeWechatInboundIntent(inbound) {
|
function normalizeWechatInboundIntent(inbound) {
|
||||||
const msgType = String(inbound?.msgType ?? '').toLowerCase();
|
const msgType = String(inbound?.msgType ?? '').toLowerCase();
|
||||||
const base = {
|
const base = {
|
||||||
@@ -1210,6 +1257,22 @@ function normalizeWechatInboundIntent(inbound) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (msgType === 'file') {
|
||||||
|
const filename = String(inbound?.fileName ?? '').trim();
|
||||||
|
return {
|
||||||
|
...base,
|
||||||
|
displayText: filename ? `收到文件:${filename}` : '收到文件',
|
||||||
|
agentText: '',
|
||||||
|
media: {
|
||||||
|
mediaId: inbound?.mediaId || '',
|
||||||
|
},
|
||||||
|
attachment: {
|
||||||
|
filename,
|
||||||
|
sizeBytes: normalizeNumber(inbound?.fileSize),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (msgType === 'location') {
|
if (msgType === 'location') {
|
||||||
const latitude = normalizeNumber(inbound?.locationX);
|
const latitude = normalizeNumber(inbound?.locationX);
|
||||||
const longitude = normalizeNumber(inbound?.locationY);
|
const longitude = normalizeNumber(inbound?.locationY);
|
||||||
@@ -1409,6 +1472,7 @@ export function createWechatMpService({
|
|||||||
apiFetch,
|
apiFetch,
|
||||||
startAgentSession = null,
|
startAgentSession = null,
|
||||||
sessionApiFetch = null,
|
sessionApiFetch = null,
|
||||||
|
submitSessionReply = null,
|
||||||
scheduleService = null,
|
scheduleService = null,
|
||||||
wechatScheduleLlmConfigService = null,
|
wechatScheduleLlmConfigService = null,
|
||||||
llmProviderService = null,
|
llmProviderService = null,
|
||||||
@@ -1434,10 +1498,15 @@ export function createWechatMpService({
|
|||||||
jsapiTicketUrl: config.jsapiTicketUrl || DEFAULT_WECHAT_JSAPI_TICKET_URL,
|
jsapiTicketUrl: config.jsapiTicketUrl || DEFAULT_WECHAT_JSAPI_TICKET_URL,
|
||||||
mediaPublicBaseUrl: config.mediaPublicBaseUrl || config.publicBaseUrl,
|
mediaPublicBaseUrl: config.mediaPublicBaseUrl || config.publicBaseUrl,
|
||||||
maxImageBytes: Math.max(1, Number(config.maxImageBytes ?? 10 * 1024 * 1024)),
|
maxImageBytes: Math.max(1, Number(config.maxImageBytes ?? 10 * 1024 * 1024)),
|
||||||
|
maxFileBytes: Math.max(1, Number(config.maxFileBytes ?? 30 * 1024 * 1024)),
|
||||||
acceptVoice: config.acceptVoice !== false,
|
acceptVoice: config.acceptVoice !== false,
|
||||||
acceptImage: config.acceptImage !== false,
|
acceptImage: config.acceptImage !== false,
|
||||||
|
acceptFile: config.acceptFile !== false,
|
||||||
acceptLocation: config.acceptLocation !== false,
|
acceptLocation: config.acceptLocation !== false,
|
||||||
acceptLink: config.acceptLink !== false,
|
acceptLink: config.acceptLink !== false,
|
||||||
|
mediaAnalysisGrayUsers: Array.isArray(config.mediaAnalysisGrayUsers)
|
||||||
|
? config.mediaAnalysisGrayUsers
|
||||||
|
: [],
|
||||||
asrTarget: config.asrTarget || DEFAULT_ASR_TARGET,
|
asrTarget: config.asrTarget || DEFAULT_ASR_TARGET,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1446,6 +1515,8 @@ export function createWechatMpService({
|
|||||||
expiresAt: 0,
|
expiresAt: 0,
|
||||||
};
|
};
|
||||||
const rememberedWechatContexts = new Map();
|
const rememberedWechatContexts = new Map();
|
||||||
|
const messageTasksByOpenid = new Map();
|
||||||
|
const recentMediaByOpenid = new Map();
|
||||||
let jsapiTicketCache = {
|
let jsapiTicketCache = {
|
||||||
ticket: null,
|
ticket: null,
|
||||||
expiresAt: 0,
|
expiresAt: 0,
|
||||||
@@ -1454,6 +1525,86 @@ export function createWechatMpService({
|
|||||||
const fetchForSession = (sessionId, pathname, init) =>
|
const fetchForSession = (sessionId, pathname, init) =>
|
||||||
sessionApiFetch ? sessionApiFetch(sessionId, pathname, init) : apiFetch(pathname, init);
|
sessionApiFetch ? sessionApiFetch(sessionId, pathname, init) : apiFetch(pathname, init);
|
||||||
|
|
||||||
|
const enqueueMessageTask = (openid, taskFactory) => {
|
||||||
|
const key = String(openid ?? '').trim();
|
||||||
|
const previous = messageTasksByOpenid.get(key) ?? Promise.resolve();
|
||||||
|
const task = previous.catch(() => undefined).then(taskFactory);
|
||||||
|
messageTasksByOpenid.set(key, task);
|
||||||
|
void task
|
||||||
|
.finally(() => {
|
||||||
|
if (messageTasksByOpenid.get(key) === task) messageTasksByOpenid.delete(key);
|
||||||
|
})
|
||||||
|
.catch(() => {});
|
||||||
|
return task;
|
||||||
|
};
|
||||||
|
|
||||||
|
const rememberRecentMedia = (openid, intent) => {
|
||||||
|
const publicUrl = String(intent?.media?.publicUrl ?? '').trim();
|
||||||
|
if (!publicUrl) return;
|
||||||
|
const key = String(openid ?? '').trim();
|
||||||
|
const now = Date.now();
|
||||||
|
const item = {
|
||||||
|
media: { ...(intent.media ?? {}) },
|
||||||
|
attachment: intent.attachment ? { ...intent.attachment } : null,
|
||||||
|
};
|
||||||
|
const current = recentMediaByOpenid.get(key);
|
||||||
|
const canAppendImage =
|
||||||
|
intent?.msgType === 'image' &&
|
||||||
|
current &&
|
||||||
|
!current.claimed &&
|
||||||
|
now - current.rememberedAt <= WECHAT_RECENT_MEDIA_TTL_MS &&
|
||||||
|
current.items.every((recentItem) => !recentItem.attachment);
|
||||||
|
const candidates = canAppendImage ? [...current.items, item] : [item];
|
||||||
|
const deduped = candidates.filter(
|
||||||
|
(candidate, index, values) =>
|
||||||
|
values.findIndex(
|
||||||
|
(value) => String(value?.media?.publicUrl ?? '') === String(candidate?.media?.publicUrl ?? ''),
|
||||||
|
) === index,
|
||||||
|
);
|
||||||
|
recentMediaByOpenid.set(key, {
|
||||||
|
items: deduped.slice(-WECHAT_RECENT_IMAGE_MAX_COUNT),
|
||||||
|
rememberedAt: now,
|
||||||
|
batchId: crypto.randomUUID(),
|
||||||
|
claimed: false,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const attachRecentMediaForFollowup = (openid, intent, mediaAnalysisEnabled) => {
|
||||||
|
if (!mediaAnalysisEnabled || intent?.msgType !== 'text' || intent?.media?.publicUrl) return;
|
||||||
|
const text = String(intent?.agentText ?? '');
|
||||||
|
if (!/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word)/iu.test(text)) return;
|
||||||
|
const key = String(openid ?? '').trim();
|
||||||
|
const recent = recentMediaByOpenid.get(key);
|
||||||
|
if (
|
||||||
|
!recent ||
|
||||||
|
recent.claimed ||
|
||||||
|
Date.now() - recent.rememberedAt > WECHAT_RECENT_MEDIA_TTL_MS ||
|
||||||
|
recent.items.length === 0
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const items = recent.items.map((item) => ({
|
||||||
|
media: { ...(item.media ?? {}), source: 'wechat_recent_media' },
|
||||||
|
attachment: item.attachment ? { ...item.attachment } : null,
|
||||||
|
}));
|
||||||
|
const primary = items.at(-1);
|
||||||
|
intent.media = { ...(primary?.media ?? {}) };
|
||||||
|
if (primary?.attachment) intent.attachment = { ...primary.attachment };
|
||||||
|
intent.recentMediaItems = items;
|
||||||
|
intent.recentMediaBatchId = recent.batchId;
|
||||||
|
recent.claimed = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
const settleRecentMediaBatch = (openid, intent, { succeeded }) => {
|
||||||
|
const batchId = String(intent?.recentMediaBatchId ?? '').trim();
|
||||||
|
if (!batchId) return;
|
||||||
|
const key = String(openid ?? '').trim();
|
||||||
|
const recent = recentMediaByOpenid.get(key);
|
||||||
|
if (!recent || recent.batchId !== batchId) return;
|
||||||
|
if (succeeded) recentMediaByOpenid.delete(key);
|
||||||
|
else recent.claimed = false;
|
||||||
|
};
|
||||||
|
|
||||||
const resolveWechatBillingTokenState = (sessionId, tokenState) =>
|
const resolveWechatBillingTokenState = (sessionId, tokenState) =>
|
||||||
resolveBillingTokenState(tokenState, {
|
resolveBillingTokenState(tokenState, {
|
||||||
sessionId,
|
sessionId,
|
||||||
@@ -1872,16 +2023,40 @@ export function createWechatMpService({
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const buildIntentMetadata = (intent) => ({
|
const buildIntentMetadata = (intent, { mediaAnalysisEnabled = false } = {}) => {
|
||||||
source: 'wechat_mp',
|
const mediaPublicUrl = intent.media?.publicUrl || null;
|
||||||
msgType: intent.msgType,
|
const mediaItems = Array.isArray(intent.recentMediaItems) && intent.recentMediaItems.length > 0
|
||||||
originalMsgId: intent.msgId || null,
|
? intent.recentMediaItems
|
||||||
displayText: intent.displayText || '',
|
: mediaPublicUrl
|
||||||
mediaPublicUrl: intent.media?.publicUrl || null,
|
? [{ media: intent.media, attachment: intent.attachment ?? null }]
|
||||||
recognition: intent.msgType === 'voice' ? intent.agentText || null : null,
|
: [];
|
||||||
location: intent.location || null,
|
const imageUrls = mediaItems
|
||||||
link: intent.link || null,
|
.filter((item) => !item.attachment)
|
||||||
});
|
.map((item) => String(item?.media?.publicUrl ?? '').trim())
|
||||||
|
.filter((url, index, values) => url && values.indexOf(url) === index);
|
||||||
|
const fileAttachments = mediaItems
|
||||||
|
.filter((item) => item.attachment?.filename && item.media?.publicUrl)
|
||||||
|
.map((item) => ({
|
||||||
|
assetId: '',
|
||||||
|
downloadUrl: item.media.publicUrl,
|
||||||
|
filename: item.attachment.filename,
|
||||||
|
mimeType: item.attachment.mimeType || 'application/octet-stream',
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
source: 'wechat_mp',
|
||||||
|
msgType: intent.msgType,
|
||||||
|
originalMsgId: intent.msgId || null,
|
||||||
|
displayText: intent.displayText || '',
|
||||||
|
mediaPublicUrl,
|
||||||
|
...(mediaAnalysisEnabled && imageUrls.length > 0
|
||||||
|
? { imageUrls }
|
||||||
|
: {}),
|
||||||
|
...(mediaAnalysisEnabled && fileAttachments.length > 0 ? { fileAttachments } : {}),
|
||||||
|
recognition: intent.msgType === 'voice' ? intent.agentText || null : null,
|
||||||
|
location: intent.location || null,
|
||||||
|
link: intent.link || null,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
const persistIntentDetail = async ({ intent, userId = null, rawXmlHash = '' }) => {
|
const persistIntentDetail = async ({ intent, userId = null, rawXmlHash = '' }) => {
|
||||||
if (typeof userAuth.insertWechatMpMessageDetail !== 'function') return;
|
if (typeof userAuth.insertWechatMpMessageDetail !== 'function') return;
|
||||||
@@ -1921,6 +2096,7 @@ export function createWechatMpService({
|
|||||||
|
|
||||||
const runIntentMessage = async ({ inbound, intent, user }) => {
|
const runIntentMessage = async ({ inbound, intent, user }) => {
|
||||||
const wechatIntent = classifyWechatIntent(intent);
|
const wechatIntent = classifyWechatIntent(intent);
|
||||||
|
const mediaAnalysisEnabled = isWechatMediaGrayUser(user, config.mediaAnalysisGrayUsers);
|
||||||
const resetCandidate =
|
const resetCandidate =
|
||||||
intent.msgType === 'text' || intent.msgType === 'voice' ? intent.agentText : '';
|
intent.msgType === 'text' || intent.msgType === 'voice' ? intent.agentText : '';
|
||||||
// Page Data delivery owns persistent files, datasets and two publication
|
// Page Data delivery owns persistent files, datasets and two publication
|
||||||
@@ -1970,7 +2146,18 @@ export function createWechatMpService({
|
|||||||
sessionId,
|
sessionId,
|
||||||
requestId,
|
requestId,
|
||||||
agentPrompt,
|
agentPrompt,
|
||||||
buildIntentMetadata(intent),
|
buildIntentMetadata(intent, { mediaAnalysisEnabled }),
|
||||||
|
{
|
||||||
|
submitReply: submitSessionReply
|
||||||
|
? ({ requestId: replyRequestId, userMessage }) =>
|
||||||
|
submitSessionReply({
|
||||||
|
userId: user.userId,
|
||||||
|
sessionId,
|
||||||
|
requestId: replyRequestId,
|
||||||
|
userMessage,
|
||||||
|
})
|
||||||
|
: null,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
|
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
|
||||||
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
|
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
|
||||||
@@ -2129,7 +2316,18 @@ export function createWechatMpService({
|
|||||||
sessionId,
|
sessionId,
|
||||||
retryId,
|
retryId,
|
||||||
retryPrompt,
|
retryPrompt,
|
||||||
buildIntentMetadata(intent),
|
buildIntentMetadata(intent, { mediaAnalysisEnabled }),
|
||||||
|
{
|
||||||
|
submitReply: submitSessionReply
|
||||||
|
? ({ requestId: replyRequestId, userMessage }) =>
|
||||||
|
submitSessionReply({
|
||||||
|
userId: user.userId,
|
||||||
|
sessionId,
|
||||||
|
requestId: replyRequestId,
|
||||||
|
userMessage,
|
||||||
|
})
|
||||||
|
: null,
|
||||||
|
},
|
||||||
);
|
);
|
||||||
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
|
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
|
||||||
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
|
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
|
||||||
@@ -2302,6 +2500,7 @@ export function createWechatMpService({
|
|||||||
const supportedByConfig =
|
const supportedByConfig =
|
||||||
(intent.msgType === 'voice' && config.acceptVoice) ||
|
(intent.msgType === 'voice' && config.acceptVoice) ||
|
||||||
(intent.msgType === 'image' && config.acceptImage) ||
|
(intent.msgType === 'image' && config.acceptImage) ||
|
||||||
|
(intent.msgType === 'file' && config.acceptFile) ||
|
||||||
(intent.msgType === 'location' && config.acceptLocation) ||
|
(intent.msgType === 'location' && config.acceptLocation) ||
|
||||||
(intent.msgType === 'link' && config.acceptLink) ||
|
(intent.msgType === 'link' && config.acceptLink) ||
|
||||||
intent.msgType === 'text' ||
|
intent.msgType === 'text' ||
|
||||||
@@ -2350,6 +2549,26 @@ export function createWechatMpService({
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const mediaAnalysisEnabled = isWechatMediaGrayUser(
|
||||||
|
boundUser,
|
||||||
|
config.mediaAnalysisGrayUsers,
|
||||||
|
);
|
||||||
|
attachRecentMediaForFollowup(inbound.fromUserName, intent, mediaAnalysisEnabled);
|
||||||
|
|
||||||
|
if (intent.msgType === 'file' && !mediaAnalysisEnabled) {
|
||||||
|
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
contentType: 'application/xml; charset=utf-8',
|
||||||
|
body: buildWechatTextReply({
|
||||||
|
toUserName: inbound.fromUserName,
|
||||||
|
fromUserName: inbound.toUserName,
|
||||||
|
content: '当前账号尚未开启服务号文件分析灰度,请先通过 H5 上传文件。',
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (intent.msgType === 'voice' && !intent.agentText.trim() && intent.media?.mediaId) {
|
if (intent.msgType === 'voice' && !intent.agentText.trim() && intent.media?.mediaId) {
|
||||||
try {
|
try {
|
||||||
const fallbackText = await transcribeWechatVoiceMedia(intent.media.mediaId, intent.media.format);
|
const fallbackText = await transcribeWechatVoiceMedia(intent.media.mediaId, intent.media.format);
|
||||||
@@ -2418,6 +2637,7 @@ export function createWechatMpService({
|
|||||||
source: persisted.source,
|
source: persisted.source,
|
||||||
};
|
};
|
||||||
intent.agentText = `[图片1]: ${persisted.publicUrl}`;
|
intent.agentText = `[图片1]: ${persisted.publicUrl}`;
|
||||||
|
rememberRecentMedia(inbound.fromUserName, intent);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
|
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
|
||||||
return {
|
return {
|
||||||
@@ -2433,6 +2653,57 @@ export function createWechatMpService({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (intent.msgType === 'file') {
|
||||||
|
try {
|
||||||
|
const accessToken = await getStableAccessToken();
|
||||||
|
const persisted = await persistWechatAttachment(
|
||||||
|
{
|
||||||
|
userId: boundUser.userId,
|
||||||
|
appId: config.appId,
|
||||||
|
openid: inbound.fromUserName,
|
||||||
|
msgId: inbound.msgId,
|
||||||
|
mediaId: inbound.mediaId,
|
||||||
|
filename: inbound.fileName,
|
||||||
|
publicBaseUrl: config.mediaPublicBaseUrl,
|
||||||
|
maxFileBytes: config.maxFileBytes,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
wechatFetch,
|
||||||
|
accessToken,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
intent.media = {
|
||||||
|
...intent.media,
|
||||||
|
mediaId: inbound.mediaId || intent.media?.mediaId || '',
|
||||||
|
publicUrl: persisted.publicUrl,
|
||||||
|
format: persisted.contentType,
|
||||||
|
source: persisted.source,
|
||||||
|
};
|
||||||
|
intent.attachment = {
|
||||||
|
...intent.attachment,
|
||||||
|
filename: persisted.filename,
|
||||||
|
publicUrl: persisted.publicUrl,
|
||||||
|
mimeType: persisted.contentType,
|
||||||
|
sizeBytes: persisted.bytes,
|
||||||
|
};
|
||||||
|
intent.agentText = `[文件1: ${persisted.filename}]: ${persisted.publicUrl}`;
|
||||||
|
intent.displayText = `文件:${persisted.filename}`;
|
||||||
|
rememberRecentMedia(inbound.fromUserName, intent);
|
||||||
|
} catch (error) {
|
||||||
|
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
contentType: 'application/xml; charset=utf-8',
|
||||||
|
body: buildWechatTextReply({
|
||||||
|
toUserName: inbound.fromUserName,
|
||||||
|
fromUserName: inbound.toUserName,
|
||||||
|
content: error instanceof Error ? error.message : '文件处理失败,请稍后重试。',
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
intent.msgType === 'location' &&
|
intent.msgType === 'location' &&
|
||||||
(intent.location?.latitude == null || intent.location?.longitude == null)
|
(intent.location?.latitude == null || intent.location?.longitude == null)
|
||||||
@@ -2555,12 +2826,15 @@ export function createWechatMpService({
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const task = runIntentMessage({
|
const task = enqueueMessageTask(inbound.fromUserName, () =>
|
||||||
inbound,
|
runIntentMessage({
|
||||||
intent,
|
inbound,
|
||||||
user: boundUser,
|
intent,
|
||||||
})
|
user: boundUser,
|
||||||
|
}),
|
||||||
|
)
|
||||||
.then(async ({ sessionId } = {}) => {
|
.then(async ({ sessionId } = {}) => {
|
||||||
|
settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: true });
|
||||||
if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
|
if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
|
||||||
await userAuth.finishWechatMpMessage({
|
await userAuth.finishWechatMpMessage({
|
||||||
appId: config.appId,
|
appId: config.appId,
|
||||||
@@ -2572,6 +2846,7 @@ export function createWechatMpService({
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch(async (err) => {
|
.catch(async (err) => {
|
||||||
|
settleRecentMediaBatch(inbound.fromUserName, intent, { succeeded: false });
|
||||||
if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
|
if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
|
||||||
await userAuth.finishWechatMpMessage({
|
await userAuth.finishWechatMpMessage({
|
||||||
appId: config.appId,
|
appId: config.appId,
|
||||||
|
|||||||
+554
-1
@@ -79,6 +79,7 @@ function createBoundWechatService({
|
|||||||
config = {},
|
config = {},
|
||||||
scheduleService = null,
|
scheduleService = null,
|
||||||
applySessionLlmProvider = null,
|
applySessionLlmProvider = null,
|
||||||
|
submitSessionReply = null,
|
||||||
}) {
|
}) {
|
||||||
return createWechatMpService({
|
return createWechatMpService({
|
||||||
config: {
|
config: {
|
||||||
@@ -128,6 +129,7 @@ function createBoundWechatService({
|
|||||||
},
|
},
|
||||||
startAgentSession,
|
startAgentSession,
|
||||||
sessionApiFetch,
|
sessionApiFetch,
|
||||||
|
submitSessionReply,
|
||||||
scheduleService,
|
scheduleService,
|
||||||
applySessionLlmProvider,
|
applySessionLlmProvider,
|
||||||
wechatFetch,
|
wechatFetch,
|
||||||
@@ -2712,6 +2714,10 @@ test('isRecoverableWechatAgentSessionError detects poisoned tool_calls history',
|
|||||||
);
|
);
|
||||||
assert.equal(isRecoverableWechatAgentSessionError('stale_session_poisoned_completion'), true);
|
assert.equal(isRecoverableWechatAgentSessionError('stale_session_poisoned_completion'), true);
|
||||||
assert.equal(isRecoverableWechatAgentSessionError('无权访问该会话'), true);
|
assert.equal(isRecoverableWechatAgentSessionError('无权访问该会话'), true);
|
||||||
|
assert.equal(
|
||||||
|
isRecoverableWechatAgentSessionError('Session already has an active request. Cancel it first.'),
|
||||||
|
true,
|
||||||
|
);
|
||||||
assert.equal(isRecoverableWechatAgentSessionError('network timeout'), false);
|
assert.equal(isRecoverableWechatAgentSessionError('network timeout'), false);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2752,6 +2758,17 @@ test('sanitizeWechatAgentOutboundText replaces raw api errors with friendly text
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('assertWechatAgentReplyIsSendable rejects image inspection placeholders', () => {
|
||||||
|
assert.throws(
|
||||||
|
() => assertWechatAgentReplyIsSendable({ text: 'Let me look at the image first.' }),
|
||||||
|
/wechat_agent_incomplete_reply/,
|
||||||
|
);
|
||||||
|
assert.throws(
|
||||||
|
() => assertWechatAgentReplyIsSendable({ text: '我先看一下这张图片。' }),
|
||||||
|
/wechat_agent_incomplete_reply/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test('wechat mp service recreates dedicated session when tool_calls error arrives via Finish assistant text', async () => {
|
test('wechat mp service recreates dedicated session when tool_calls error arrives via Finish assistant text', async () => {
|
||||||
const token = 'token';
|
const token = 'token';
|
||||||
const timestamp = '1710000000';
|
const timestamp = '1710000000';
|
||||||
@@ -3680,12 +3697,13 @@ test('wechat mp service falls back to ASR when voice recognition is empty', asyn
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test('wechat mp service persists image and routes image url into agent prompt', async () => {
|
test('wechat mp wildcard media access persists image and routes image url into agent prompt', async () => {
|
||||||
const token = 'token';
|
const token = 'token';
|
||||||
const timestamp = '1710000000';
|
const timestamp = '1710000000';
|
||||||
const nonce = 'nonce';
|
const nonce = 'nonce';
|
||||||
const testUserId = 'test-user-image';
|
const testUserId = 'test-user-image';
|
||||||
const prompts = [];
|
const prompts = [];
|
||||||
|
const metadataCalls = [];
|
||||||
const detailCalls = [];
|
const detailCalls = [];
|
||||||
const service = createWechatMpService({
|
const service = createWechatMpService({
|
||||||
config: {
|
config: {
|
||||||
@@ -3701,6 +3719,7 @@ test('wechat mp service persists image and routes image url into agent prompt',
|
|||||||
unboundTextPrefix: '请先绑定',
|
unboundTextPrefix: '请先绑定',
|
||||||
progressDelayMs: 0,
|
progressDelayMs: 0,
|
||||||
maxImageBytes: 1024 * 1024,
|
maxImageBytes: 1024 * 1024,
|
||||||
|
mediaAnalysisGrayUsers: ['*'],
|
||||||
},
|
},
|
||||||
userAuth: {
|
userAuth: {
|
||||||
async findWechatUserByOpenid() {
|
async findWechatUserByOpenid() {
|
||||||
@@ -3747,6 +3766,7 @@ test('wechat mp service persists image and routes image url into agent prompt',
|
|||||||
if (pathname === '/sessions/session-1/reply') {
|
if (pathname === '/sessions/session-1/reply') {
|
||||||
const body = JSON.parse(init.body);
|
const body = JSON.parse(init.body);
|
||||||
prompts.push(body.user_message.content[0].text);
|
prompts.push(body.user_message.content[0].text);
|
||||||
|
metadataCalls.push(body.user_message.metadata);
|
||||||
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
}
|
}
|
||||||
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||||
@@ -3809,10 +3829,543 @@ test('wechat mp service persists image and routes image url into agent prompt',
|
|||||||
prompts[0],
|
prompts[0],
|
||||||
/\[图片1\]: https:\/\/example\.com\/MindSpace\/test-user-image\/public\/wechat-mp\//,
|
/\[图片1\]: https:\/\/example\.com\/MindSpace\/test-user-image\/public\/wechat-mp\//,
|
||||||
);
|
);
|
||||||
|
assert.equal(metadataCalls.length, 1);
|
||||||
|
assert.equal(metadataCalls[0].source, 'wechat_mp');
|
||||||
|
assert.equal(metadataCalls[0].msgType, 'image');
|
||||||
|
assert.equal(metadataCalls[0].imageUrls.length, 1);
|
||||||
|
assert.match(metadataCalls[0].imageUrls[0], /\/public\/wechat-mp\//);
|
||||||
assert.equal(detailCalls.length, 1);
|
assert.equal(detailCalls.length, 1);
|
||||||
assert.match(detailCalls[0].mediaPublicUrl, /\/wechat-mp\//);
|
assert.match(detailCalls[0].mediaPublicUrl, /\/wechat-mp\//);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('wechat mp image submission reuses the H5 prepared reply path', async () => {
|
||||||
|
const token = 'token';
|
||||||
|
const timestamp = '1710000000';
|
||||||
|
const nonce = 'nonce';
|
||||||
|
const testUserId = 'test-user-image-prepared';
|
||||||
|
const submitCalls = [];
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
token,
|
||||||
|
config: {
|
||||||
|
mediaAnalysisGrayUsers: [testUserId],
|
||||||
|
},
|
||||||
|
userAuth: {
|
||||||
|
async findWechatUserByOpenid() {
|
||||||
|
return { userId: testUserId, status: 'active', nickname: '唐' };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sessionApiFetch: async (_sessionId, pathname) => {
|
||||||
|
if (pathname === '/sessions/session-1/events') {
|
||||||
|
return new Response(
|
||||||
|
[
|
||||||
|
'data: {"type":"Message","message":{"id":"assistant-1","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"图片内容已识别。"}]}}\n\n',
|
||||||
|
'data: {"type":"Finish"}\n\n',
|
||||||
|
].join(''),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||||
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected api path: ${pathname}`);
|
||||||
|
},
|
||||||
|
submitSessionReply: async (input) => {
|
||||||
|
submitCalls.push(input);
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||||
|
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) {
|
||||||
|
return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'image/png' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||||
|
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'image',
|
||||||
|
content: '',
|
||||||
|
extraFields: { MediaId: 'media-prepared', PicUrl: 'https://wx.example.com/image.png' },
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
await result.task;
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(submitCalls.length, 1);
|
||||||
|
assert.equal(submitCalls[0].userId, testUserId);
|
||||||
|
assert.equal(submitCalls[0].sessionId, 'session-1');
|
||||||
|
assert.equal(submitCalls[0].userMessage.metadata.imageUrls.length, 1);
|
||||||
|
assert.match(submitCalls[0].userMessage.metadata.imageUrls[0], /\/public\/wechat-mp\//);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wechat mp serializes image and follow-up text and reattaches recent image', async () => {
|
||||||
|
const token = 'token';
|
||||||
|
const timestamp = '1710000000';
|
||||||
|
const nonce = 'nonce';
|
||||||
|
const testUserId = 'test-user-image-followup';
|
||||||
|
const submitCalls = [];
|
||||||
|
let eventCall = 0;
|
||||||
|
let releaseFirst = null;
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
token,
|
||||||
|
config: {
|
||||||
|
mediaAnalysisGrayUsers: [testUserId],
|
||||||
|
},
|
||||||
|
userAuth: {
|
||||||
|
async findWechatUserByOpenid() {
|
||||||
|
return { userId: testUserId, status: 'active', nickname: '唐' };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sessionApiFetch: async (_sessionId, pathname) => {
|
||||||
|
if (pathname === '/sessions/session-1/events') {
|
||||||
|
eventCall += 1;
|
||||||
|
if (eventCall === 1) {
|
||||||
|
return new Response(
|
||||||
|
new ReadableStream({
|
||||||
|
start(controller) {
|
||||||
|
releaseFirst = () => {
|
||||||
|
controller.enqueue(
|
||||||
|
new TextEncoder().encode(
|
||||||
|
'data: {"type":"Message","message":{"id":"assistant-image","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"图片已识别。"}]}}\n\n' +
|
||||||
|
'data: {"type":"Finish"}\n\n',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
controller.close();
|
||||||
|
};
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new Response(
|
||||||
|
[
|
||||||
|
'data: {"type":"Message","message":{"id":"assistant-followup","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"已结合刚才图片分析主题。"}]}}\n\n',
|
||||||
|
'data: {"type":"Finish"}\n\n',
|
||||||
|
].join(''),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||||
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected api path: ${pathname}`);
|
||||||
|
},
|
||||||
|
submitSessionReply: async (input) => {
|
||||||
|
submitCalls.push(input);
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||||
|
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) {
|
||||||
|
return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'image/png' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||||
|
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const imageResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'image',
|
||||||
|
content: '',
|
||||||
|
extraFields: { MediaId: 'media-followup', PicUrl: 'https://wx.example.com/image.png' },
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|
||||||
|
const followupResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({ msgType: 'text', content: '请根据刚才图片分析主题' }),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
assert.equal(submitCalls.length, 1);
|
||||||
|
|
||||||
|
releaseFirst();
|
||||||
|
await imageResult.task;
|
||||||
|
await followupResult.task;
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(submitCalls.length, 2);
|
||||||
|
assert.deepEqual(
|
||||||
|
submitCalls[1].userMessage.metadata.imageUrls,
|
||||||
|
submitCalls[0].userMessage.metadata.imageUrls,
|
||||||
|
);
|
||||||
|
assert.equal(submitCalls[1].userMessage.metadata.msgType, 'text');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wechat mp groups consecutive images for one follow-up and clears the consumed batch', async () => {
|
||||||
|
const token = 'token';
|
||||||
|
const timestamp = '1710000000';
|
||||||
|
const nonce = 'nonce';
|
||||||
|
const testUserId = 'test-user-multi-image-followup';
|
||||||
|
const submitCalls = [];
|
||||||
|
let eventCall = 0;
|
||||||
|
let releaseFirst = null;
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
token,
|
||||||
|
config: {
|
||||||
|
mediaAnalysisGrayUsers: [testUserId],
|
||||||
|
},
|
||||||
|
userAuth: {
|
||||||
|
async findWechatUserByOpenid() {
|
||||||
|
return { userId: testUserId, status: 'active', nickname: '唐' };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sessionApiFetch: async (_sessionId, pathname) => {
|
||||||
|
if (pathname === '/sessions/session-1/events') {
|
||||||
|
eventCall += 1;
|
||||||
|
if (eventCall === 1) {
|
||||||
|
return new Response(
|
||||||
|
new ReadableStream({
|
||||||
|
start(controller) {
|
||||||
|
releaseFirst = () => {
|
||||||
|
controller.enqueue(
|
||||||
|
new TextEncoder().encode(
|
||||||
|
'data: {"type":"Message","message":{"id":"assistant-first","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"第一张处理完成。"}]}}\n\n' +
|
||||||
|
'data: {"type":"Finish"}\n\n',
|
||||||
|
),
|
||||||
|
);
|
||||||
|
controller.close();
|
||||||
|
};
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return new Response(
|
||||||
|
[
|
||||||
|
'data: {"type":"Message","message":{"id":"assistant-ok","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"处理完成。"}]}}\n\n',
|
||||||
|
'data: {"type":"Finish"}\n\n',
|
||||||
|
].join(''),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||||
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected api path: ${pathname}`);
|
||||||
|
},
|
||||||
|
submitSessionReply: async (input) => {
|
||||||
|
submitCalls.push(input);
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||||
|
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) {
|
||||||
|
return new Response(Buffer.from([0x89, 0x50, 0x4e, 0x47]), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'image/png' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||||
|
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const firstImageResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'image',
|
||||||
|
content: '',
|
||||||
|
extraFields: {
|
||||||
|
MsgId: '10011',
|
||||||
|
MediaId: 'media-first',
|
||||||
|
PicUrl: 'https://wx.example.com/media-first.png',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
while (submitCalls.length === 0) await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
|
||||||
|
const secondImageResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'image',
|
||||||
|
content: '',
|
||||||
|
extraFields: {
|
||||||
|
MsgId: '10012',
|
||||||
|
MediaId: 'media-second',
|
||||||
|
PicUrl: 'https://wx.example.com/media-second.png',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
|
||||||
|
const followupResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
content: '请结合刚才两张图片分析主题',
|
||||||
|
extraFields: { MsgId: '10013' },
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
assert.equal(submitCalls.length, 1);
|
||||||
|
|
||||||
|
releaseFirst();
|
||||||
|
await firstImageResult.task;
|
||||||
|
await secondImageResult.task;
|
||||||
|
await followupResult.task;
|
||||||
|
|
||||||
|
const laterResult = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
content: '请再次分析刚才图片',
|
||||||
|
extraFields: { MsgId: '10014' },
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
await laterResult.task;
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(submitCalls.length, 4);
|
||||||
|
assert.equal(submitCalls[0].userMessage.metadata.imageUrls.length, 1);
|
||||||
|
assert.equal(submitCalls[1].userMessage.metadata.imageUrls.length, 1);
|
||||||
|
assert.equal(submitCalls[2].userMessage.metadata.imageUrls.length, 2);
|
||||||
|
assert.match(submitCalls[2].userMessage.metadata.imageUrls[0], /media-first/);
|
||||||
|
assert.match(submitCalls[2].userMessage.metadata.imageUrls[1], /media-second/);
|
||||||
|
assert.equal(submitCalls[2].userMessage.metadata.msgType, 'text');
|
||||||
|
assert.equal(submitCalls[3].userMessage.metadata.imageUrls, undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wechat mp service persists Word and Excel files in user public area and reuses H5 attachment metadata', async () => {
|
||||||
|
const token = 'token';
|
||||||
|
const timestamp = '1710000000';
|
||||||
|
const nonce = 'nonce';
|
||||||
|
const testUserId = 'test-user-wechat-file';
|
||||||
|
const userMessages = [];
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
token,
|
||||||
|
config: {
|
||||||
|
maxFileBytes: 1024 * 1024,
|
||||||
|
acceptFile: true,
|
||||||
|
mediaAnalysisGrayUsers: [testUserId],
|
||||||
|
},
|
||||||
|
userAuth: {
|
||||||
|
async findWechatUserByOpenid() {
|
||||||
|
return { userId: testUserId, status: 'active', nickname: '毕升' };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sessionApiFetch: async (_sessionId, pathname, init = {}) => {
|
||||||
|
if (pathname === '/sessions/session-1/events') {
|
||||||
|
return new Response(
|
||||||
|
[
|
||||||
|
'data: {"type":"Message","request_id":"req-file","message":{"id":"assistant-1","role":"assistant","metadata":{"userVisible":true},"content":[{"type":"text","text":"文件已解析。"}]}}\n\n',
|
||||||
|
'data: {"type":"Finish","request_id":"req-file","token_state":{"inputTokens":1,"outputTokens":2}}\n\n',
|
||||||
|
].join(''),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'text/event-stream' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (pathname === '/sessions/session-1/reply') {
|
||||||
|
const body = JSON.parse(init.body);
|
||||||
|
userMessages.push(body.user_message);
|
||||||
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
|
}
|
||||||
|
if (pathname === '/agent/harness_remember' || pathname === '/agent/harness_bootstrap') {
|
||||||
|
return new Response('{}', { status: 200, headers: { 'Content-Type': 'application/json' } });
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected api path: ${pathname}`);
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||||
|
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) {
|
||||||
|
return new Response(Buffer.from('fake-docx-content'), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/octet-stream' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/message/custom/send')) {
|
||||||
|
return new Response(JSON.stringify({ errcode: 0, errmsg: 'ok' }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const originalRandomUuid = crypto.randomUUID;
|
||||||
|
crypto.randomUUID = () => 'req-file';
|
||||||
|
try {
|
||||||
|
for (const [index, filename] of ['季度分析.docx', '销售数据.xlsx'].entries()) {
|
||||||
|
const result = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'file',
|
||||||
|
content: '',
|
||||||
|
extraFields: {
|
||||||
|
MediaId: `file-media-${index + 1}`,
|
||||||
|
FileName: filename,
|
||||||
|
FileSize: 17,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
{ timestamp, nonce, signature: signatureFor(token, timestamp, nonce) },
|
||||||
|
);
|
||||||
|
assert.equal(result.status, 200);
|
||||||
|
await result.task;
|
||||||
|
const attachment = userMessages.at(-1)?.metadata?.fileAttachments?.[0];
|
||||||
|
const publicFilename = decodeURIComponent(new URL(attachment.downloadUrl).pathname.split('/').at(-1));
|
||||||
|
assert.equal(
|
||||||
|
fs.existsSync(
|
||||||
|
path.join(process.cwd(), 'MindSpace', testUserId, 'public', 'wechat-mp', publicFilename),
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
crypto.randomUUID = originalRandomUuid;
|
||||||
|
fs.rmSync(path.join(process.cwd(), 'MindSpace', testUserId), {
|
||||||
|
recursive: true,
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.equal(userMessages.length, 2);
|
||||||
|
assert.match(userMessages[0].content[0].text, /【微信服务号文件消息】/);
|
||||||
|
assert.match(userMessages[0].content[0].text, /\[文件1: 季度分析\.docx\]:/);
|
||||||
|
assert.equal(userMessages[0].metadata.msgType, 'file');
|
||||||
|
assert.equal(userMessages[0].metadata.fileAttachments.length, 1);
|
||||||
|
assert.equal(userMessages[0].metadata.fileAttachments[0].filename, '季度分析.docx');
|
||||||
|
assert.equal(
|
||||||
|
userMessages[0].metadata.fileAttachments[0].mimeType,
|
||||||
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||||
|
);
|
||||||
|
assert.match(userMessages[0].metadata.fileAttachments[0].downloadUrl, /\/public\/wechat-mp\//);
|
||||||
|
assert.match(userMessages[1].content[0].text, /\[文件1: 销售数据\.xlsx\]:/);
|
||||||
|
assert.equal(userMessages[1].metadata.fileAttachments[0].filename, '销售数据.xlsx');
|
||||||
|
assert.equal(
|
||||||
|
userMessages[1].metadata.fileAttachments[0].mimeType,
|
||||||
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
);
|
||||||
|
assert.match(userMessages[1].metadata.fileAttachments[0].downloadUrl, /\/public\/wechat-mp\//);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wechat mp service rejects unsupported public file types without entering the agent flow', async () => {
|
||||||
|
let mediaDownloads = 0;
|
||||||
|
let sessionCalls = 0;
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
config: { acceptFile: true, mediaAnalysisGrayUsers: ['user-1'] },
|
||||||
|
sessionApiFetch: async () => {
|
||||||
|
sessionCalls += 1;
|
||||||
|
throw new Error('session should not be called');
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/stable_token')) {
|
||||||
|
return new Response(JSON.stringify({ access_token: 'access-1', expires_in: 7200 }), {
|
||||||
|
status: 200,
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) mediaDownloads += 1;
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'file',
|
||||||
|
content: '',
|
||||||
|
extraFields: { MediaId: 'file-media-2', FileName: 'payload.exe' },
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
timestamp: '1710000000',
|
||||||
|
nonce: 'nonce',
|
||||||
|
signature: signatureFor('token', '1710000000', 'nonce'),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(result.status, 200);
|
||||||
|
assert.match(result.body, /仅支持 Word/);
|
||||||
|
assert.equal(mediaDownloads, 0);
|
||||||
|
assert.equal(sessionCalls, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wechat mp service keeps file analysis disabled outside the media gray allowlist', async () => {
|
||||||
|
let mediaDownloads = 0;
|
||||||
|
let sessionCalls = 0;
|
||||||
|
const service = createBoundWechatService({
|
||||||
|
config: {
|
||||||
|
acceptFile: true,
|
||||||
|
mediaAnalysisGrayUsers: ['唐'],
|
||||||
|
},
|
||||||
|
sessionApiFetch: async () => {
|
||||||
|
sessionCalls += 1;
|
||||||
|
throw new Error('session should not be called');
|
||||||
|
},
|
||||||
|
wechatFetch: async (url) => {
|
||||||
|
if (String(url).includes('/cgi-bin/media/get')) mediaDownloads += 1;
|
||||||
|
throw new Error(`unexpected wechat url: ${url}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await service.handleInboundMessage(
|
||||||
|
inboundXml({
|
||||||
|
msgType: 'file',
|
||||||
|
content: '',
|
||||||
|
extraFields: { MediaId: 'file-media-gray', FileName: '测试.docx' },
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
timestamp: '1710000000',
|
||||||
|
nonce: 'nonce',
|
||||||
|
signature: signatureFor('token', '1710000000', 'nonce'),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.equal(result.status, 200);
|
||||||
|
assert.match(result.body, /尚未开启服务号文件分析灰度/);
|
||||||
|
assert.equal(mediaDownloads, 0);
|
||||||
|
assert.equal(sessionCalls, 0);
|
||||||
|
});
|
||||||
|
|
||||||
test('wechat mp service accepts full voice xml payload and routes recognition text into agent', async () => {
|
test('wechat mp service accepts full voice xml payload and routes recognition text into agent', async () => {
|
||||||
const token = 'token';
|
const token = 'token';
|
||||||
const timestamp = '1710000000';
|
const timestamp = '1710000000';
|
||||||
|
|||||||
@@ -21,6 +21,12 @@ const ImageBuilder = {
|
|||||||
build: (ctx) => buildText(TEMPLATES.image, ctx),
|
build: (ctx) => buildText(TEMPLATES.image, ctx),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const FileBuilder = {
|
||||||
|
priority: 80,
|
||||||
|
support: (ctx) => ctx.msgType === 'file',
|
||||||
|
build: (ctx) => buildText(TEMPLATES.file, ctx),
|
||||||
|
};
|
||||||
|
|
||||||
const VoiceBuilder = {
|
const VoiceBuilder = {
|
||||||
priority: 80,
|
priority: 80,
|
||||||
support: (ctx) => ctx.msgType === 'voice',
|
support: (ctx) => ctx.msgType === 'voice',
|
||||||
@@ -54,7 +60,7 @@ const DefaultBuilder = {
|
|||||||
build: (ctx) => buildText(TEMPLATES.default, ctx),
|
build: (ctx) => buildText(TEMPLATES.default, ctx),
|
||||||
};
|
};
|
||||||
|
|
||||||
const BUILDERS = [ImageBuilder, VoiceBuilder, LocationBuilder, LinkBuilder, IntentBuilder, DefaultBuilder]
|
const BUILDERS = [ImageBuilder, FileBuilder, VoiceBuilder, LocationBuilder, LinkBuilder, IntentBuilder, DefaultBuilder]
|
||||||
.sort((a, b) => b.priority - a.priority);
|
.sort((a, b) => b.priority - a.priority);
|
||||||
|
|
||||||
export function selectBuilder(ctx) {
|
export function selectBuilder(ctx) {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ const INTENT_RULES = [
|
|||||||
|
|
||||||
export function resolveIntent(msgType, text) {
|
export function resolveIntent(msgType, text) {
|
||||||
if (msgType === 'image') return { task: 'image_analysis' };
|
if (msgType === 'image') return { task: 'image_analysis' };
|
||||||
|
if (msgType === 'file') return { task: 'file_analysis' };
|
||||||
if (msgType === 'voice') return { task: 'voice_analysis' };
|
if (msgType === 'voice') return { task: 'voice_analysis' };
|
||||||
if (msgType === 'location') return { task: 'location' };
|
if (msgType === 'location') return { task: 'location' };
|
||||||
if (msgType === 'link') return { task: 'link' };
|
if (msgType === 'link') return { task: 'link' };
|
||||||
|
|||||||
@@ -26,6 +26,11 @@ describe('buildAckText', () => {
|
|||||||
assert.equal(result, '图片收到,我先看看。');
|
assert.equal(result, '图片收到,我先看看。');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('file → file template', () => {
|
||||||
|
const result = buildAckText({ intent: intent('file'), nickname: '', config: cfg, fallbackText: '' });
|
||||||
|
assert.equal(result, '文件收到,我先读取分析。');
|
||||||
|
});
|
||||||
|
|
||||||
it('voice → voice template', () => {
|
it('voice → voice template', () => {
|
||||||
const result = buildAckText({ intent: intent('voice'), nickname: '', config: cfg, fallbackText: '' });
|
const result = buildAckText({ intent: intent('voice'), nickname: '', config: cfg, fallbackText: '' });
|
||||||
assert.equal(result, '收到语音,我先听一下。');
|
assert.equal(result, '收到语音,我先听一下。');
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export const TEMPLATES = {
|
|||||||
'让我看看这张图片。',
|
'让我看看这张图片。',
|
||||||
'图片已收到,我来处理。',
|
'图片已收到,我来处理。',
|
||||||
],
|
],
|
||||||
|
file: ['文件收到,我先读取分析。'],
|
||||||
voice: [
|
voice: [
|
||||||
'收到语音,我先听一下。',
|
'收到语音,我先听一下。',
|
||||||
'听到啦,我马上处理。',
|
'听到啦,我马上处理。',
|
||||||
|
|||||||
@@ -81,6 +81,20 @@ export function buildWechatAgentPrompt(intent, { grantedSkills = [] } = {}) {
|
|||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join('\n');
|
.join('\n');
|
||||||
}
|
}
|
||||||
|
if (msgType === 'file') {
|
||||||
|
const attachment = intent?.attachment ?? {};
|
||||||
|
return [
|
||||||
|
currentTimeHint,
|
||||||
|
'【微信服务号文件消息】用户发送了需要解析的 Office 文件。',
|
||||||
|
attachment.filename ? `文件名:${attachment.filename}` : '',
|
||||||
|
attachment.publicUrl ? `文件链接:${attachment.publicUrl}` : '',
|
||||||
|
'请复用 H5 附件分析结果回答;Excel 在专用工具可用时必须优先使用 Excel 工具读取完整工作簿。',
|
||||||
|
'',
|
||||||
|
String(agentText).trim(),
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join('\n');
|
||||||
|
}
|
||||||
if (msgType === 'location') {
|
if (msgType === 'location') {
|
||||||
const location = intent?.location ?? {};
|
const location = intent?.location ?? {};
|
||||||
return [
|
return [
|
||||||
|
|||||||
Reference in New Issue
Block a user