Add smart ACK provider for WeChat MP replies

Replace fixed ackText with a rule-based AckProvider that picks response templates by message type and intent (translate, summary, rewrite, poster, ppt, mindmap, code, search, schedule). Pure sync, zero I/O, auto-falls back to config.ackText on any error. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-26 15:19:03 +08:00
parent 9ed4fd48d7
commit 9b4a25799f
162 changed files with 17276 additions and 2054 deletions
@@ -4,6 +4,7 @@ import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs
 import { developerToolsFromPolicy } from './capabilities.mjs';
 import { evaluateProxyRequest, isNativeH5ApiPath } from './policies.mjs';
 import { buildSandboxSessionConstraints } from './user-publish.mjs';
+import { buildTaskRoutingAgentText } from './user-memory-profile.mjs';
 import { reconcileAgentSession } from './session-reconcile.mjs';

 const insecureDispatcher = new Agent({
@@ -79,7 +80,162 @@ function extractSessionId(req, body) {
  return null;
 }

-export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth, llmProviderService }) {
+function firstUserText(message) {
+  return (
+    message?.content?.find?.((item) => item?.type === 'text' && typeof item.text === 'string')?.text ??
+    ''
+  );
+}
+
+const IMAGE_URL_LINE_RE = /^\[图片\d+]:\s*(\S.+)$/;
+
+export function extractImageUrlsFromMessage(userMessage) {
+  const urls = [];
+  const imageUrls = userMessage?.metadata?.imageUrls;
+  if (Array.isArray(imageUrls)) {
+    urls.push(...imageUrls);
+  }
+
+  const content = userMessage?.content;
+  if (Array.isArray(content)) {
+    for (const item of content) {
+      if (item?.type === 'image_url' && item.image_url?.url) {
+        urls.push(item.image_url.url);
+        continue;
+      }
+      if (item?.type !== 'text' || typeof item.text !== 'string') continue;
+      for (const line of item.text.split('\n')) {
+        const match = line.trim().match(IMAGE_URL_LINE_RE);
+        if (match?.[1]) urls.push(match[1].trim());
+      }
+    }
+  }
+
+  return [...new Set(urls.filter((url) => typeof url === 'string' && url.trim()))];
+}
+
+function messageHasImages(userMessage) {
+  return extractImageUrlsFromMessage(userMessage).length > 0;
+}
+
+function injectTaskRoutingHint(body, sessionPolicy) {
+  const originalText = firstUserText(body?.user_message);
+  if (!originalText?.trim()) return body;
+
+  const agentText = buildTaskRoutingAgentText(originalText, sessionPolicy);
+  if (!agentText || agentText === originalText) return body;
+
+  const userMessage = body.user_message ?? {};
+  const content = Array.isArray(userMessage.content) ? [...userMessage.content] : [];
+  const firstTextIndex = content.findIndex(
+    (item) => item?.type === 'text' && typeof item.text === 'string',
+  );
+  if (firstTextIndex >= 0) {
+    content[firstTextIndex] = { ...content[firstTextIndex], text: agentText };
+  } else {
+    content.unshift({ type: 'text', text: agentText });
+  }
+
+  return {
+    ...body,
+    user_message: {
+      ...userMessage,
+      content,
+      metadata: {
+        ...(userMessage.metadata ?? {}),
+        displayText:
+          userMessage.metadata?.displayText && String(userMessage.metadata.displayText).trim()
+            ? userMessage.metadata.displayText
+            : originalText,
+      },
+    },
+  };
+}
+
+export async function buildVisionPayload({
+  userMessage,
+  userId,
+  publishLayout,
+  localFetchAsset,
+  llmProviderService,
+}) {
+  if (!localFetchAsset || !llmProviderService || !userId) return null;
+  const rawImageUrls = extractImageUrlsFromMessage(userMessage);
+  if (rawImageUrls.length === 0) return null;
+
+  const imageItems = [];
+  for (const rawUrl of rawImageUrls) {
+    try {
+      const match = rawUrl.match(/\/mindspace\/v1\/assets\/([^/?#]+)\/download/);
+      if (!match) continue;
+      const assetId = decodeURIComponent(match[1]);
+      const { buffer, mimeType } = await localFetchAsset(userId, assetId);
+      let relativePath = rawUrl;
+      try {
+        const parsed = new URL(rawUrl);
+        relativePath = parsed.pathname + parsed.search;
+      } catch { /* keep rawUrl */ }
+      imageItems.push({ mimeType, data: buffer.toString('base64'), relativePath, rawUrl });
+    } catch (err) {
+      console.warn('Vision image fetch skipped:', err instanceof Error ? err.message : err);
+    }
+  }
+  if (imageItems.length === 0) return null;
+
+  const visionDescription = await llmProviderService
+    .analyzeImagesWithVision(imageItems, '请详细描述图片的视觉内容：主体、颜色、风格、构图，不要生成代码或页面方案')
+    .catch(() => null);
+
+  const pathList = imageItems
+    .map((item, i) => `图片${i + 1}: <img src="${item.relativePath}" alt="图片${i + 1}">`)
+    .join('  ');
+
+  const publicUrlPrefix = publishLayout?.publicUrl
+    ? `${String(publishLayout.publicUrl).replace(/\/$/, '')}/public/`
+    : null;
+
+  const injectedNote =
+    '\n\n【TKMind 图片分析结果 — 仅供执行参考，不要向用户复述此段内容】\n' +
+    (visionDescription
+      ? `Qwen VL 图片描述：\n${visionDescription}\n\n`
+      : '') +
+    `图片 HTML 嵌入路径（直接写入 <img> 标签，浏览器有 cookie 可直接加载，无需 fetch）：\n${pathList}\n` +
+    '执行要求：必须先调用 load_skill → static-page-publish（每次生成页面都要调用，不可省略），' +
+    '再用 write_file 写入 public/页面.html；' +
+    (publicUrlPrefix
+      ? `完成后向用户给出 Markdown 可点击链接，格式：[页面标题](${publicUrlPrefix}<文件名>.html)；`
+      : '完成后按技能说明里的链接格式给用户一个 Markdown 可点击链接；') +
+    '不要向用户展示 HTML 代码块。';
+
+  let updatedContent = Array.isArray(userMessage?.content) ? [...userMessage.content] : [];
+  for (const item of imageItems) {
+    updatedContent = updatedContent.map((c) => {
+      if (c?.type !== 'text' || typeof c.text !== 'string') return c;
+      const updated = c.text.replaceAll(item.rawUrl, item.relativePath);
+      return updated === c.text ? c : { ...c, text: updated };
+    });
+  }
+
+  const lastTextIdx = updatedContent.reduceRight(
+    (found, item, i) => (found === -1 && item?.type === 'text' ? i : found),
+    -1,
+  );
+  if (lastTextIdx >= 0) {
+    updatedContent[lastTextIdx] = {
+      ...updatedContent[lastTextIdx],
+      text: updatedContent[lastTextIdx].text + injectedNote,
+    };
+  } else {
+    updatedContent.push({ type: 'text', text: injectedNote });
+  }
+
+  return {
+    userMessage: { ...userMessage, content: updatedContent },
+    billableImageCount: visionDescription ? 1 : 0,
+  };
+}
+
+export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth, llmProviderService, localFetchAsset, subscriptionService }) {
  const targets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : [];
  const primaryTarget = targets[0] ?? apiTarget ?? '';
  let rrIdx = 0;
@@ -142,6 +298,60 @@ export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth,
    );
  }

+  // Tracks which sessions are currently using the vision provider so we only
+  // issue a switch-back call when the session was actually on vision.
+  const visionActiveSessions = new Set();
+
+  async function applyVisionProviderForSession(sessionId) {
+    if (!llmProviderService || !sessionId) return null;
+    const target = await resolveTarget(sessionId);
+    return llmProviderService.applyVisionProviderForSession(
+      sessionId,
+      (url, init) => apiFetch(target, apiSecret, `${url.pathname}${url.search}`, init),
+    );
+  }
+
+  // Two-step vision approach:
+  //   Step 1 — Call Qwen VL directly (not through Goose) to analyze the image.
+  //   Step 2 — Inject Qwen's text description + server-relative image paths into the
+  //             user_message that goes to DeepSeek via Goose.  DeepSeek retains full
+  //             tool-calling capability (write_file, etc.) and creates the page properly.
+  async function buildVisionBody(userMessage, userId, publishLayout) {
+    return buildVisionPayload({
+      userMessage,
+      userId,
+      publishLayout,
+      localFetchAsset,
+      llmProviderService,
+    });
+  }
+
+  async function reconcileSessionPolicyForUser(userId, sessionId) {
+    if (!userId || !sessionId) return;
+    const target = await resolveTarget(sessionId);
+    const workingDir = await userAuth.resolveWorkingDir(userId);
+    const sessionPolicy = await userAuth.getAgentSessionPolicy(userId);
+    const publishLayout = await userAuth.getUserPublishLayout(userId);
+    await reconcileAgentSession(
+      (pathname, init) => apiFetch(target, apiSecret, pathname, init),
+      sessionId,
+      {
+        workingDir,
+        sessionPolicy,
+        sandboxConstraints: publishLayout?.constraints ?? null,
+        tolerateInvalidWorkingDir: true,
+        userContext: publishLayout
+          ? {
+              userId,
+              displayName: publishLayout.displayName,
+              username: publishLayout.username,
+              slug: publishLayout.slug,
+            }
+          : null,
+      },
+    );
+  }
+
  const requireUser = async (req, res, next) => {
    try {
      const session = req.userSession;
@@ -415,7 +625,7 @@ export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth,
    },
  ];

-  const proxySessionEvents = async (req, res, sessionId) => {
+  const proxySessionEvents = async (req, res, sessionId, { onAfterFinish } = {}) => {
    try {
      const pathname = `/sessions/${sessionId}/events`;
      const sessionTarget = await resolveTarget(sessionId);
@@ -458,6 +668,10 @@ export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth,
              },
            };
          }
+          // Fire-and-forget snapshot refresh after conversation finishes.
+          if (typeof onAfterFinish === 'function') {
+            void Promise.resolve().then(() => onAfterFinish(sessionId, req.currentUser.id)).catch(() => {});
+          }
        },
      });

@@ -510,14 +724,59 @@ export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth,
        res.status(403).json({ message: '当前账户未开通项目记忆，无法访问该 API' });
        return;
      }
+      const isReplyPath = pathname.match(/^\/sessions\/[^/]+\/reply$/) && req.method === 'POST';
+      const sessionMatch = pathname.match(/^\/sessions\/([^/]+)/);
+
+      let baseBody = req.body;
+      if (isReplyPath && !policyState.unrestricted) {
+        baseBody = injectTaskRoutingHint(
+          req.body,
+          await userAuth.getAgentSessionPolicy(req.currentUser.id),
+        );
+      }
+
+      // Vision pre-processing: call Qwen VL directly for image analysis, then inject
+      // the description + image paths into the message text for DeepSeek/Goose.
+      // We do NOT switch the Goose session provider — DeepSeek retains full tool-calling
+      // capability so it can write files and return real links.
+      if (isReplyPath && llmProviderService) {
+        const sessionId = sessionMatch?.[1];
+        if (sessionId) {
+          const hasImages = messageHasImages(req.body?.user_message);
+          if (hasImages && await llmProviderService.hasVisionKey()) {
+            const publishLayout = await userAuth
+              .getUserPublishLayout(req.currentUser?.id)
+              .catch(() => null);
+            const visionResult = await buildVisionBody(
+              baseBody?.user_message ?? req.body?.user_message,
+              req.currentUser?.id,
+              publishLayout,
+            ).catch(() => null);
+            if (visionResult?.userMessage) {
+              baseBody = { ...(baseBody ?? req.body), user_message: visionResult.userMessage };
+            }
+            if (visionResult?.billableImageCount > 0 && subscriptionService) {
+              await subscriptionService.consumeImageQuota(
+                req.currentUser.id,
+                visionResult.billableImageCount,
+              ).catch((err) => {
+                console.warn(
+                  'Subscription image quota consume skipped:',
+                  err instanceof Error ? err.message : err,
+                );
+              });
+            }
+          }
+        }
+      }
+
      const body =
        req.method === 'GET' || req.method === 'HEAD'
          ? undefined
-          : req.body
-            ? JSON.stringify(req.body)
+          : baseBody
+            ? JSON.stringify(baseBody)
            : undefined;

-      const sessionMatch = pathname.match(/^\/sessions\/([^/]+)/);
      const fallbackTarget =
        req.goosedTarget ??
        (sessionMatch ? await resolveTarget(sessionMatch[1]) : primaryTarget);
@@ -546,6 +805,8 @@ export function createTkmindProxy({ apiTarget, apiTargets, apiSecret, userAuth,
    ensureChatAllowed,
    applySessionLlmProvider,
    applyLocalFallbackForSession,
+    applyVisionProviderForSession,
+    reconcileSessionPolicyForUser,
    handlers,
    sessionScoped,
    proxyFallback,