tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix: route WeChat media through shared vision pipeline

Browse Source
This commit is contained in:
john
2026-07-18 18:15:22 +08:00
parent 8ccaf3b39d
commit 8eaf4d23f3
4 changed files with 369 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wechat-mp.mjs
+102 -19
View File
@@ -210,7 +210,14 @@ function pushMessage(messages, incoming) {
return [...messages, incoming];
}
async function executeSessionReply(apiFetch, sessionId, requestId, prompt, metadata = {}) {
async function executeSessionReply(
apiFetch,
sessionId,
requestId,
prompt,
metadata = {},
{ submitReply = null } = {},
) {
const eventsResponse = await apiFetch(`/sessions/${sessionId}/events`, {
method: 'GET',
headers: { Accept: 'text/event-stream' },
@@ -220,18 +227,23 @@ async function executeSessionReply(apiFetch, sessionId, requestId, prompt, metad
throw new Error(text || '无法建立公众号消息事件流');
}
const replyResponse = await apiFetch(`/sessions/${sessionId}/reply`, {
method: 'POST',
body: JSON.stringify({
request_id: requestId,
user_message: createUserMessage(prompt, metadata),
}),
});
if (!replyResponse.ok) {
const text = await replyResponse.text().catch(() => '');
throw new Error(text || 'Agent reply 请求失败');
const userMessage = createUserMessage(prompt, metadata);
if (submitReply) {
await submitReply({ sessionId, requestId, userMessage });
} else {
const replyResponse = await apiFetch(`/sessions/${sessionId}/reply`, {
method: 'POST',
body: JSON.stringify({
request_id: requestId,
user_message: userMessage,
}),
});
if (!replyResponse.ok) {
const text = await replyResponse.text().catch(() => '');
throw new Error(text || 'Agent reply 请求失败');
}
replyResponse.body?.cancel().catch?.(() => {});
}
replyResponse.body?.cancel().catch?.(() => {});
const reader = eventsResponse.body.getReader();
const decoder = new TextDecoder();
@@ -1000,6 +1012,7 @@ export function isRecoverableWechatAgentSessionError(message) {
if (/stale_session_poisoned_completion/i.test(normalized)) return true;
if (/403|404|not found|无权访问/i.test(normalized)) return true;
if (/tool_calls|tool_call_id|insufficient tool messages/i.test(normalized)) return true;
if (/wechat_agent_incomplete_reply/i.test(normalized)) return true;
return false;
}
@@ -1030,6 +1043,13 @@ export function findRecoverableWechatAgentErrorInReply(reply) {
export function assertWechatAgentReplyIsSendable(reply) {
const recoverable = findRecoverableWechatAgentErrorInReply(reply);
if (recoverable) throw new Error(recoverable);
const text = String(reply?.text ?? '').trim();
if (
/^(?:let me|i(?:'ll| will))\s+(?:first\s+)?(?:look|check|inspect|analy[sz]e)(?:\s+at)?\s+(?:the\s+)?image(?:\s+first)?[.!]?$/iu.test(text) ||
/^(?:让我|我先)(?:先)?(?:看|查看|检查|分析)(?:一下)?(?:这张|该张|这个)?图片[。！!]?$/u.test(text)
) {
throw new Error('wechat_agent_incomplete_reply');
}
}
export function isWechatAgentApiErrorText(message) {
@@ -1448,6 +1468,7 @@ export function createWechatMpService({
apiFetch,
startAgentSession = null,
sessionApiFetch = null,
submitSessionReply = null,
scheduleService = null,
wechatScheduleLlmConfigService = null,
llmProviderService = null,
@@ -1490,6 +1511,8 @@ export function createWechatMpService({
expiresAt: 0,
};
const rememberedWechatContexts = new Map();
const messageTasksByOpenid = new Map();
const recentMediaByOpenid = new Map();
let jsapiTicketCache = {
ticket: null,
expiresAt: 0,
@@ -1498,6 +1521,39 @@ export function createWechatMpService({
const fetchForSession = (sessionId, pathname, init) =>
sessionApiFetch ? sessionApiFetch(sessionId, pathname, init) : apiFetch(pathname, init);
const enqueueMessageTask = (openid, taskFactory) => {
const key = String(openid ?? '').trim();
const previous = messageTasksByOpenid.get(key) ?? Promise.resolve();
const task = previous.catch(() => undefined).then(taskFactory);
messageTasksByOpenid.set(key, task);
void task
.finally(() => {
if (messageTasksByOpenid.get(key) === task) messageTasksByOpenid.delete(key);
})
.catch(() => {});
return task;
};
const rememberRecentMedia = (openid, intent) => {
const publicUrl = String(intent?.media?.publicUrl ?? '').trim();
if (!publicUrl) return;
recentMediaByOpenid.set(String(openid ?? '').trim(), {
media: { ...(intent.media ?? {}) },
attachment: intent.attachment ? { ...intent.attachment } : null,
rememberedAt: Date.now(),
});
};
const attachRecentMediaForFollowup = (openid, intent, mediaAnalysisEnabled) => {
if (!mediaAnalysisEnabled || intent?.msgType !== 'text' || intent?.media?.publicUrl) return;
const text = String(intent?.agentText ?? '');
if (!/(?:刚才|之前|上一|这张|这份|图片|图像|照片|文件|文档|表格|excel|word)/iu.test(text)) return;
const recent = recentMediaByOpenid.get(String(openid ?? '').trim());
if (!recent || Date.now() - recent.rememberedAt > 15 * 60 * 1000) return;
intent.media = { ...recent.media, source: 'wechat_recent_media' };
if (recent.attachment) intent.attachment = { ...recent.attachment };
};
const resolveWechatBillingTokenState = (sessionId, tokenState) =>
resolveBillingTokenState(tokenState, {
sessionId,
@@ -1919,7 +1975,7 @@ export function createWechatMpService({
const buildIntentMetadata = (intent, { mediaAnalysisEnabled = false } = {}) => {
const mediaPublicUrl = intent.media?.publicUrl || null;
const fileAttachment =
intent.msgType === 'file' && mediaPublicUrl && intent.attachment?.filename
mediaPublicUrl && intent.attachment?.filename
? {
assetId: '',
downloadUrl: mediaPublicUrl,
@@ -1933,7 +1989,7 @@ export function createWechatMpService({
originalMsgId: intent.msgId || null,
displayText: intent.displayText || '',
mediaPublicUrl,
...(mediaAnalysisEnabled && intent.msgType === 'image' && mediaPublicUrl
...(mediaAnalysisEnabled && mediaPublicUrl && !fileAttachment
? { imageUrls: [mediaPublicUrl] }
: {}),
...(mediaAnalysisEnabled && fileAttachment ? { fileAttachments: [fileAttachment] } : {}),
@@ -2032,6 +2088,17 @@ export function createWechatMpService({
requestId,
agentPrompt,
buildIntentMetadata(intent, { mediaAnalysisEnabled }),
{
submitReply: submitSessionReply
? ({ requestId: replyRequestId, userMessage }) =>
submitSessionReply({
userId: user.userId,
sessionId,
requestId: replyRequestId,
userMessage,
})
: null,
},
);
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
@@ -2191,6 +2258,17 @@ export function createWechatMpService({
retryId,
retryPrompt,
buildIntentMetadata(intent, { mediaAnalysisEnabled }),
{
submitReply: submitSessionReply
? ({ requestId: replyRequestId, userMessage }) =>
submitSessionReply({
userId: user.userId,
sessionId,
requestId: replyRequestId,
userMessage,
})
: null,
},
);
const workingDir = publishLayout?.publishDir ?? (await userAuth.resolveWorkingDir(user.userId));
const linkExistsForRequest = resolveLinkExistsForWorkingDir(workingDir, linkExists);
@@ -2416,6 +2494,7 @@ export function createWechatMpService({
boundUser,
config.mediaAnalysisGrayUsers,
);
attachRecentMediaForFollowup(inbound.fromUserName, intent, mediaAnalysisEnabled);
if (intent.msgType === 'file' && !mediaAnalysisEnabled) {
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
@@ -2499,6 +2578,7 @@ export function createWechatMpService({
source: persisted.source,
};
intent.agentText = `[图片1]: ${persisted.publicUrl}`;
rememberRecentMedia(inbound.fromUserName, intent);
} catch (error) {
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
return {
@@ -2549,6 +2629,7 @@ export function createWechatMpService({
};
intent.agentText = `[文件1: ${persisted.filename}]: ${persisted.publicUrl}`;
intent.displayText = `文件：${persisted.filename}`;
rememberRecentMedia(inbound.fromUserName, intent);
} catch (error) {
await persistIntentDetail({ intent, userId: boundUser.userId, rawXmlHash });
return {
@@ -2686,11 +2767,13 @@ export function createWechatMpService({
};
}
const task = runIntentMessage({
inbound,
intent,
user: boundUser,
})
const task = enqueueMessageTask(inbound.fromUserName, () =>
runIntentMessage({
inbound,
intent,
user: boundUser,
}),
)
.then(async ({ sessionId } = {}) => {
if (inbound.msgId && typeof userAuth.finishWechatMpMessage === 'function') {
await userAuth.finishWechatMpMessage({