feat(h5): add LLM intent router admin controls and shadow verification.

Expose shadow/canary router policy in ops admin, add FAQ rule fast-path, and tighten router defaults (1200ms timeout, 0.65 confidence). Includes verify-h5-llm-router-shadow for production canary rollout. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-01 16:48:01 +08:00
parent 005612029f
commit 43bc8bbc2b
14 changed files with 1391 additions and 80 deletions
@@ -26,9 +26,14 @@ import {
  IMAGE_GENERATION_MODE,
  resolveImageGenerationDecision,
  coerceRealtimeWebSkill,
+  isChatLlmRouterEligible,
+  logChatLlmRouterShadow,
  resolveChatIntentRouterPolicy,
 } from './chat-intent-router.mjs';

+/** Ambiguous user text that should miss FAQ/rules and exercise LLM router paths in tests. */
+const AMBIGUOUS_LLM_ROUTER_QUERY = '周末想放松一下，有什么活动建议？';
+
 test('image generation intent requires a real image for explicit background requests', () => {
  const decision = resolveImageGenerationDecision({
    text: '帮我生成一首唐诗页面，背景是大唐盛景图片，页面要精美',
@@ -568,11 +573,11 @@ test('createChatIntentRouter keeps continued sessions on agent without llm route
  assert.match(result.reason, /延续已有 Agent 会话/);
 });

-test('createChatIntentRouter uses agent fallback for general conversational analysis (skill-only)', async () => {
+test('createChatIntentRouter uses agent fallback for general conversational analysis when llm router disabled', async () => {
  const llmCalls = [];
  const resolveCalls = [];
  const router = createChatIntentRouter({
-    enabled: true,
+    enabled: false,
    modelProviderKeyId: 'key-router',
    model: 'deepseek-chat',
    llmProviderService: {
@@ -694,7 +699,7 @@ test('createChatIntentRouter routes memory recall through rules when memory reso
  assert.equal(result.source, 'rule');
 });

-test('createManagedChatIntentRouter hot-loads admin config and stays skill-only', async () => {
+test('createManagedChatIntentRouter hot-loads admin config and activates llm routing', async () => {
  const states = [
    {
      fingerprint: 'off',
@@ -743,16 +748,16 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only'
  const result = await router.classify({
    userMessage: {
      role: 'user',
-      content: [{ type: 'text', text: '帮我概括一下这段材料的要点' }],
-      metadata: { displayText: '帮我概括一下这段材料的要点' },
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
    },
  });
-  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
-  assert.equal(result.source, 'fallback');
-  assert.equal(llmCalls.length, 0);
+  assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
+  assert.equal(result.source, 'llm');
+  assert.equal(llmCalls.length, 1);
 });

-test('createChatIntentRouter uses agent fallback when rules miss (skill-only)', async () => {
+test('createChatIntentRouter rejects low-confidence llm direct_chat and falls back to agent', async () => {
  const llmCalls = [];
  const router = createChatIntentRouter({
    enabled: true,
@@ -777,17 +782,17 @@ test('createChatIntentRouter uses agent fallback when rules miss (skill-only)',
  const result = await router.classify({
    userMessage: {
      role: 'user',
-      content: [{ type: 'text', text: '帮我整理一下这段材料' }],
-      metadata: { displayText: '帮我整理一下这段材料' },
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
    },
  });

  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
  assert.equal(result.source, 'fallback');
-  assert.equal(llmCalls.length, 0);
+  assert.equal(llmCalls.length, 1);
 });

-test('createChatIntentRouter falls back to agent when rules miss (skill-only)', async () => {
+test('createChatIntentRouter falls back to agent when llm router fails', async () => {
  const llmCalls = [];
  const router = createChatIntentRouter({
    enabled: true,
@@ -802,17 +807,53 @@ test('createChatIntentRouter falls back to agent when rules miss (skill-only)',
  const result = await router.classify({
    userMessage: {
      role: 'user',
-      content: [{ type: 'text', text: '帮我概括一下这段材料的要点' }],
-      metadata: { displayText: '帮我概括一下这段材料的要点' },
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
    },
    grantedSkills: ['web'],
  });

+  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
+  assert.equal(result.source, 'fallback');
+  assert.equal(llmCalls.length, 1);
+});
+
+test('createChatIntentRouter uses agent fallback when llm router disabled', async () => {
+  const llmCalls = [];
+  const router = createChatIntentRouter({
+    enabled: false,
+    llmProviderService: {
+      async createChatCompletion() {
+        llmCalls.push(1);
+        return { ok: false, message: 'router model unavailable' };
+      },
+    },
+  });
+
+  const result = await router.classify({
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
+    },
+  });
+
  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
  assert.equal(result.source, 'fallback');
  assert.equal(llmCalls.length, 0);
 });

+test('classifyWithRules fast-paths FAQ concept questions to direct chat without LLM', () => {
+  const result = classifyWithRules({
+    text: '什么是 SSE？',
+    sessionId: null,
+    sessionMessageCount: 0,
+  });
+  assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
+  assert.equal(result.source, 'rule');
+  assert.match(result.reason, /FAQ/);
+});
+
 test('classifyWithRules routes memory recall to direct chat on fresh session', () => {
  const result = classifyWithRules({
    text: '你记得我说想去哪儿吗',
@@ -932,7 +973,7 @@ test('buildRouterNormalizedDecision maps legacy routes and session hints', () =>
      sessionMessageCount: 0,
    },
  );
-  assert.equal(freshAgentSession.session_hint, ROUTER_DECISION_SESSION_HINT.NEW);
+  assert.equal(freshAgentSession.session_hint, ROUTER_DECISION_SESSION_HINT.REUSE);

  const memoryRecall = buildRouterNormalizedDecision(
    { route: CHAT_INTENT_ROUTE.DIRECT_CHAT },
@@ -940,6 +981,12 @@ test('buildRouterNormalizedDecision maps legacy routes and session hints', () =>
  );
  assert.equal(memoryRecall.route, ROUTER_DECISION_ROUTE.CHAT);
  assert.ok(memoryRecall.flags.includes('memory_recall'));
+
+  const goalRun = buildRouterNormalizedDecision(
+    { route: CHAT_INTENT_ROUTE.AGENT },
+    { text: '帮我分阶段完成下季度产品规划' },
+  );
+  assert.ok(goalRun.flags.includes('goal_run'));
 });

 test('resolveLegacyRouteFromClassification prefers decision when flag enabled', () => {
@@ -985,7 +1032,7 @@ test('resolveGatewayAgentSessionId honors session_hint without breaking hard rul
          decision: { session_hint: ROUTER_DECISION_SESSION_HINT.NEW },
        },
      }),
-      null,
+      'sess-1',
    );
    assert.equal(
      resolveGatewayAgentSessionId({
@@ -1370,31 +1417,6 @@ test('logRouterDecisionShadow emits payload only in shadow mode', () => {
  }
 });

-test('createChatIntentRouter uses agent fallback for unmatched general questions (skill-only)', async () => {
-  const llmCalls = [];
-  const router = createChatIntentRouter({
-    enabled: true,
-    llmProviderService: {
-      async createChatCompletion() {
-        llmCalls.push(1);
-        return { ok: false, message: 'router disabled' };
-      },
-    },
-  });
-
-  const result = await router.classify({
-    userMessage: {
-      role: 'user',
-      content: [{ type: 'text', text: '什么是 SSE？' }],
-      metadata: { displayText: '什么是 SSE？' },
-    },
-  });
-
-  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
-  assert.equal(result.source, 'fallback');
-  assert.equal(llmCalls.length, 0);
-});
-
 test('classifyWithRules routes page data dev repair before new page-data collect', () => {
  const result = classifyWithRules({
    text: '修复问卷 insert 403，columns_not_allowed',
@@ -1405,3 +1427,135 @@ test('classifyWithRules routes page data dev repair before new page-data collect
  assert.match(result.reason, /Page Data 开发修复/);
  assert.equal(result.suggested_skill, undefined);
 });
+
+test('isChatLlmRouterEligible respects enabled flag and canary user ids', () => {
+  assert.equal(isChatLlmRouterEligible({ enabled: false, canaryUserIds: ['john'], userId: 'john' }), false);
+  assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: [], userId: 'john' }), true);
+  assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: 'john' }), true);
+  assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: 'other' }), false);
+  assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: null }), false);
+});
+
+test('createChatIntentRouter limits llm routing to canary users', async () => {
+  const llmCalls = [];
+  const router = createChatIntentRouter({
+    enabled: true,
+    canaryUserIds: ['user-canary'],
+    llmProviderService: {
+      async createChatCompletion() {
+        llmCalls.push(1);
+        return {
+          ok: true,
+          reply: JSON.stringify({
+            route: 'direct_chat',
+            confidence: 0.91,
+            reason: '纯文字问答',
+            suggested_skill: null,
+            agent_brief: '',
+          }),
+        };
+      },
+    },
+  });
+
+  const control = await router.classify({
+    userId: 'user-control',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
+    },
+  });
+  const canary = await router.classify({
+    userId: 'user-canary',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
+    },
+  });
+
+  assert.equal(control.route, CHAT_INTENT_ROUTE.AGENT);
+  assert.equal(control.source, 'fallback');
+  assert.equal(canary.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
+  assert.equal(canary.source, 'llm');
+  assert.equal(llmCalls.length, 1);
+});
+
+test('createChatIntentRouter shadow mode logs llm suggestion without changing route', async () => {
+  const lines = [];
+  const logger = { warn(line) { lines.push(line); }, info(line) { lines.push(line); } };
+  const router = createChatIntentRouter({
+    enabled: true,
+    shadowMode: true,
+    llmProviderService: {
+      async createChatCompletion() {
+        return {
+          ok: true,
+          reply: JSON.stringify({
+            route: 'direct_chat',
+            confidence: 0.95,
+            reason: '纯文字问答',
+            suggested_skill: null,
+            agent_brief: '',
+          }),
+        };
+      },
+    },
+    logger,
+  });
+
+  const result = await router.classify({
+    userId: 'user-john',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
+      metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
+    },
+  });
+
+  assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
+  assert.equal(result.source, 'fallback');
+  assert.equal(result.llmShadow?.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
+  assert.equal(result.llmShadow?.wouldChangeRoute, true);
+  assert.equal(lines.length, 1);
+  assert.match(lines[0], /\[chat-llm-router-shadow\]/);
+  assert.match(lines[0], /wouldChangeRoute":true/);
+});
+
+test('logChatLlmRouterShadow emits structured payload', () => {
+  const lines = [];
+  logChatLlmRouterShadow({
+    baseline: { route: CHAT_INTENT_ROUTE.AGENT, source: 'fallback' },
+    llmResult: { route: CHAT_INTENT_ROUTE.DIRECT_CHAT, confidence: 0.9, reason: '闲聊' },
+    userId: 'user-1',
+    sessionId: 'sess-1',
+  }, { logger: { info(line) { lines.push(line); } } });
+  assert.equal(lines.length, 1);
+  assert.match(lines[0], /baselineRoute":"agent_orchestration"/);
+  assert.match(lines[0], /llmRoute":"direct_chat"/);
+});
+
+test('resolveChatIntentRouterPolicy parses llm router shadow and canary env', () => {
+  const policy = resolveChatIntentRouterPolicy({
+    env: {
+      MEMIND_CHAT_LLM_ROUTER_ENABLED: '1',
+      MEMIND_CHAT_LLM_ROUTER_SHADOW: '1',
+      MEMIND_CHAT_ROUTER_CANARY_USER_IDS: 'john, testuser1',
+    },
+  });
+  assert.equal(policy.enabled, true);
+  assert.equal(policy.shadowMode, true);
+  assert.deepEqual(policy.canaryUserIds, ['john', 'testuser1']);
+});
+
+test('createChatIntentRouter getStatus reflects llm router modes', () => {
+  const active = createChatIntentRouter({ enabled: true });
+  const shadow = createChatIntentRouter({ enabled: true, shadowMode: true });
+  const off = createChatIntentRouter({ enabled: false });
+  assert.equal(active.getStatus().llmRoutingEnabled, true);
+  assert.equal(active.getStatus().llmRoutingShadow, false);
+  assert.equal(shadow.getStatus().llmRoutingEnabled, false);
+  assert.equal(shadow.getStatus().llmRoutingShadow, true);
+  assert.equal(off.getStatus().configuredLlmEnabled, false);
+});