feat(h5): add LLM intent router admin controls and shadow verification.

Expose shadow/canary router policy in ops admin, add FAQ rule fast-path,
and tighten router defaults (1200ms timeout, 0.65 confidence).
Includes verify-h5-llm-router-shadow for production canary rollout.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-01 16:48:01 +08:00
parent 005612029f
commit 43bc8bbc2b
14 changed files with 1391 additions and 80 deletions
+56 -2
View File
@@ -141,6 +141,33 @@ H5_PUBLIC_BASE_URL=http://127.0.0.1:5173
# 1/ongateway 优先读 classification.decision.route / session_hint
# MEMIND_ROUTER_NORMALIZED_DECISION=0
# H5 Chat LLM intent router(本地开发建议见下方「本地 LLM Router」块)
# 0:仅规则路由;规则未命中 fallback Agent
# 1 + SHADOW=1:调用轻量 LLM 观测分歧,行为仍 fallback Agent
# 1 + CANARY_USER_IDS:仅名单内用户在规则未命中时走 LLM 路由;留空=全员
# MEMIND_CHAT_LLM_ROUTER_ENABLED=0
# MEMIND_CHAT_LLM_ROUTER_SHADOW=0
# MEMIND_CHAT_ROUTER_CANARY_USER_IDS=
# MEMIND_CHAT_ROUTER_MODEL_PROVIDER_KEY_ID=
# MEMIND_CHAT_ROUTER_MODEL=
# MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65
# MEMIND_CHAT_ROUTER_TIMEOUT_MS=1200
# MEMIND_CHAT_ROUTER_MEMORY_ENABLED=1
# MEMIND_CHAT_ROUTER_FALLBACK_ROUTE=agent_orchestration
# ----- 本地 LLM Routerpnpm dev,全员 Active,无 Shadow-----
# Router 模型建议(分类 JSON,短 prompt,要快不要大):
# 本机 Ollamaqwen2.5:3b / qwen2.5:1.5b / llama3.2:3b(见 H5_LOCAL_LLM_*
# 云端 relaygpt-4o-mini / deepseek-chat / qwen-turbo / glm-4-flash
# Direct Chat 仍用全局选中模型;Router 单独绑小模型即可。
# MEMIND_CHAT_LLM_ROUTER_ENABLED=1
# MEMIND_CHAT_LLM_ROUTER_SHADOW=0
# MEMIND_CHAT_ROUTER_CANARY_USER_IDS=
# MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65
# MEMIND_CHAT_ROUTER_TIMEOUT_MS=1200
# MEMIND_CHAT_ROUTER_MODEL_PROVIDER_KEY_ID=<h5_llm_provider_keys.id>
# MEMIND_CHAT_ROUTER_MODEL=qwen2.5:3b
# H5 Session stream replaydocs/h5-session-architecture-20260706.md Patch 4c
# 默认 0session SSE 仍纯透传 goosed;设 1 时 Portal 持久化 session 事件并支持 Last-Event-ID 重连补发。
# MEMIND_SESSION_STREAM_REPLAY=0
@@ -195,7 +222,8 @@ H5_ADMIN_PASSWORD=change-me-admin
# H5_ADMIN_WORKSPACE_ROOT=/Users/john/PycharmProjects/tkmind
# LLM Key 数据库加密(可选;默认派生自 TKMIND_SERVER__SECRET_KEY
# H5_SETTINGS_ENCRYPTION_KEY=change-me-encryption-key
# 启动时自动导入 Relay(若不存在同名配置)
# 启动时自动导入 Relay(若不存在同名配置);改用手动 Provider 时请设为 1
# H5_RELAY_BOOTSTRAP_DISABLED=1
# H5_RELAY_BOOTSTRAP_NAME=Local Goose Relay DeepSeek
# H5_RELAY_BOOTSTRAP_URL=http://127.0.0.1:18300/relay/buyer/v1/chat/completions
# H5_RELAY_BOOTSTRAP_API_KEY=your-bearer-token
@@ -305,7 +333,33 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind
# VITE_TKMIND_MODEL=gpt-4o
# 可选:harness recall 的默认查询
# VITE_TKMIND_MEMORY_QUERY=project architecture conventions current goals decisions risks and recent work
# Memory V2 Phase A canary (local dev)
# Full template: docs/memory-v2/phase-a-canary.env.example
# Readiness check: npm run check:memory-v2-phase-a
#
# Recommended local canary users (explicit-memory testers):
# 32035858-9a20-425b-89da-c118ef0779aa
# 1c99b83b-0454-474f-a5d2-129d34506a32
# MEMORY_ENABLED=1
# MEMORY_CANDIDATE_ENABLED=1
# MEMORY_CANDIDATE_MODE=canary
# MEMORY_CANDIDATE_PERSISTENCE_ENABLED=1
# MEMORY_CANDIDATE_AUTO_ACCEPT_ALL=0
# MEMORY_AGENT_RESOLVE_ENABLED=1
# MEMORY_AGENT_INJECTION_MODE=canary
# MEMORY_AGENT_CANARY_USER_IDS=32035858-9a20-425b-89da-c118ef0779aa,1c99b83b-0454-474f-a5d2-129d34506a32
# MEMORY_LIFECYCLE_ENABLED=1
# MEMORY_LIFECYCLE_ROLLOUT_MODE=canary
# MEMORY_LIFECYCLE_ROLLOUT_USER_IDS=32035858-9a20-425b-89da-c118ef0779aa,1c99b83b-0454-474f-a5d2-129d34506a32
# MEMORY_PROMOTION_ENABLED=1
# MEMORY_LIFECYCLE_FORGETTING_ENABLED=0
# Goal Run MVP(默认关闭;canary 用户名单见 docs/architecture/goal-run-data-model-draft.md
# GOAL_RUN_ENABLED=0
# GOAL_RUN_CANARY_USER_IDS=
# GOAL_RUN_REQUIRE_APPROVAL=0
# MEMORY_LIFECYCLE_DECAY_ENABLED=0
# 历史列表每次展示/加载条数(默认 25,与 desktop 一致)
# VITE_SESSION_PAGE_SIZE=25
+124
View File
@@ -0,0 +1,124 @@
/**
* H5 Chat Intent Router — FAQ / fast-path rule library.
*
* Add patterns here for messages that should bypass LLM routing latency.
* Consumed by chat-intent-router.mjs → classifyWithRules().
*
* Keep rules conservative: task execution, page generation, and realtime lookup
* must stay on Agent paths (checked via FAQ_EXCLUSION_PATTERNS).
*/
/** Block FAQ match when the message clearly needs Agent / tools. */
export const FAQ_EXCLUSION_PATTERNS = [
/\bpublic\/[^\s"'<>]+\.html\b/i,
/MindSpace\/[^/\s]+\/public\/[^\s"'<>]+\.html/i,
/(?:生成|创建|制作|做|写|设计|发布|搭建).{0,24}(?:H5|h5|HTML|html|网页|页面|活动页|宣传页|落地页|分享页|链接)/u,
/(?:H5|h5|HTML|html|网页|页面).{0,24}(?:生成|创建|制作|做|写|设计|发布|链接)/u,
/(?:写|做|生成|制作|设计).{0,16}主题(?:页面|文章)/u,
/(?:服务号|公众号).{0,16}(?:页面|文章|推文|图文)/u,
/(?:今天|今日|现在|最新|当前|实时).{0,12}(?:新闻|赛况|比分|天气|行情|股价|汇率|热点)/u,
/(?:查(?:一下|查)?|搜索|打听).{0,16}(?:今天|今日|现在|最新|当前)/u,
/(?:世界杯|欧冠|NBA|英超).{0,16}(?:赛况|赛程|比分)/u,
/(?:load_skill|page-data|sqlite|express|后端服务)/iu,
/(?:修|改|测试|回归|verify).{0,12}(?:bug|问卷|policy|public)/iu,
];
/**
* @typedef {{ id: string, reason: string, patterns: RegExp[] }} DirectChatFaqRule
*/
/** @type {DirectChatFaqRule[]} */
export const DIRECT_CHAT_FAQ_RULES = [
{
id: 'concept_what_is',
reason: '概念解释类问答(FAQ',
patterns: [
/^什么是[\s\S]{1,48}[?]?$/u,
/^何为[\s\S]{1,48}[?]?$/u,
/^什么叫[\s\S]{1,48}[?]?$/u,
/^[\s\S]{1,40}是什么[?]?$/u,
/^[\s\S]{1,40}是什么意思[?]?$/u,
/^[\s\S]{1,40}(?:是什么东西|指什么|啥意思)[?]?$/u,
],
},
{
id: 'concept_explain',
reason: '解释说明类问答(FAQ',
patterns: [
/^(?:请)?(?:解释|说明|介绍|讲解|阐述)(?:一下|下)?[\s\S]{1,48}[?]?$/u,
/^帮我(?:解释|说明|介绍|讲解)(?:一下|下)?[\s\S]{1,48}[?]?$/u,
/^能(?:不能|否)(?:解释|说明|介绍)(?:一下|下)?[\s\S]{1,40}[?]?$/u,
],
},
{
id: 'concept_compare',
reason: '概念对比类问答(FAQ',
patterns: [
/^[\s\S]{1,32}(?:和|与|跟)[\s\S]{1,32}(?:的)?(?:区别|差异|不同|对比|比较)[是什么]?[?]?$/u,
/^(?:对比|比较)(?:一下|下)?[\s\S]{1,48}[?]?$/u,
/^[\s\S]{1,24}(?:vs|VS|Vs)[\s\S]{1,24}(?:区别|差异|不同)?[?]?$/u,
],
},
{
id: 'concept_how_works',
reason: '原理/机制类问答(FAQ',
patterns: [
/^[\s\S]{1,40}(?:是怎么|是如何)(?:工作|运行|实现|运作)的[?]?$/u,
/^[\s\S]{1,40}(?:原理|机制|流程)(?:是什么|是怎样的)[?]?$/u,
/^为什么[\s\S]{1,48}[?]?$/u,
],
},
{
id: 'text_transform',
reason: '纯文字处理(总结/翻译/改写)',
patterns: [
/^(?:请)?(?:帮我|帮忙)?(?:总结|概括|归纳|提炼)(?:一下|下)?[\s\S]{1,80}[?]?$/u,
/^(?:请)?(?:帮我|帮忙)?(?:翻译|改写|润色|扩写|缩写|校对)(?:一下|下)?[\s\S]{1,80}[?]?$/u,
],
},
{
id: 'tech_acronym',
reason: '常见技术名词解释(FAQ',
patterns: [
/^(?:什么是|解释一下|介绍一下)?\s*(?:SSE|WebSocket|REST(?:ful)?|GraphQL|JWT|OAuth|RAG|Embedding|向量数据库|Server-Sent Events)[\s\S]{0,24}[?]?$/iu,
/^(?:SSE|WebSocket|REST(?:ful)?|GraphQL|JWT|OAuth|RAG)(?:是什么|什么意思|怎么用)[?]?$/iu,
],
},
{
id: 'learning_advice',
reason: '学习/选型建议(纯文字)',
patterns: [
/^(?:学|学习|入门).{0,24}(?:应该|该|推荐|建议).{0,32}(?:先|从哪|哪个)[?]?$/u,
/^(?:初学者|新手|入门).{0,24}(?:应该|该|推荐|建议).{0,32}[?]?$/u,
/^.{0,24}(?:先学|从哪学|怎么学).{0,32}[?]?$/u,
],
},
{
id: 'casual_opinion',
reason: '主观建议/闲聊(无任务执行)',
patterns: [
/^你觉得.{1,48}[?]?$/u,
/^你认为.{1,48}[?]?$/u,
/^(?:随便|随意)(?:聊|说说|谈谈).{0,24}[?]?$/u,
/^.{0,16}(?:有什么建议|给点建议|怎么看)[?]?$/u,
],
},
];
const MAX_FAQ_TEXT_LENGTH = 200;
/**
* @param {string} text
* @returns {{ id: string, reason: string } | null}
*/
export function matchDirectChatFaqRule(text) {
const normalized = String(text ?? '').trim();
if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null;
if (FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))) return null;
for (const rule of DIRECT_CHAT_FAQ_RULES) {
if (rule.patterns.some((pattern) => pattern.test(normalized))) {
return { id: rule.id, reason: rule.reason };
}
}
return null;
}
+221 -30
View File
@@ -15,6 +15,10 @@ import {
resolveMemoryInterventionMode,
} from './memory-intervention.mjs';
import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs';
import { matchDirectChatFaqRule } from './chat-intent-router-rules.mjs';
import { isGoalRunIntent } from './goal-run-intent.mjs';
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS } from './chat-intent-router-rules.mjs';
export const CHAT_INTENT_ROUTE = {
DIRECT_CHAT: 'direct_chat',
@@ -42,6 +46,7 @@ const ALLOWED_ROUTER_DECISION_FLAGS = new Set([
'force_deep_reasoning',
'code_task',
'long_running',
'goal_run',
]);
const AGENT_ORCHESTRATION_HEADER = '【Memind 任务编排】';
@@ -304,9 +309,9 @@ function resolveRouterFallbackRoute(env, overrides = {}) {
return raw === CHAT_INTENT_ROUTE.DIRECT_CHAT ? CHAT_INTENT_ROUTE.AGENT : raw;
}
const DEFAULT_ROUTER_TIMEOUT_MS = 2500;
const DEFAULT_ROUTER_TIMEOUT_MS = 1200;
const DEFAULT_ROUTER_MEMORY_LIMIT = 8;
const DEFAULT_ROUTER_MIN_CONFIDENCE = 0.55;
const DEFAULT_ROUTER_MIN_CONFIDENCE = 0.65;
const REALTIME_WEB_AGENT_BRIEF =
'先 load_skill → web;获取实时信息时同一轮并行调用 tkmind_search(专用联网搜索)和 web_search(内置联网搜索),合并去重并保留来源。向用户只称「联网搜索」,不要提具体搜索引擎或中间件名称。任一侧不可用时继续使用另一侧,必要时再用 fetch_url 读取可靠来源;禁止使用 search 技能查工作区,不要反复 scrape 同一站点。';
@@ -623,22 +628,17 @@ export function buildRouterNormalizedDecision(
const mode = ROUTER_DECISION_MODE.SSE;
let sessionHint = ROUTER_DECISION_SESSION_HINT.NEW;
if (sessionId) {
// Portal /agent/start already binds a Goose session before the first user turn.
// Reuse it on agent fallback (e.g. direct_chat_failed) instead of orphaning it.
sessionHint = ROUTER_DECISION_SESSION_HINT.REUSE;
}
if (
sessionId &&
!isDirectChatSessionId(sessionId) &&
sessionMessageCount != null &&
Number(sessionMessageCount) === 0
) {
sessionHint = ROUTER_DECISION_SESSION_HINT.NEW;
}
const flags = [];
if (isMemoryRecallQuestion(text)) flags.push('memory_recall');
if (forceDeepReasoning) flags.push('force_deep_reasoning');
if (toolMode === 'code') flags.push('code_task');
if (classification?.suggestedSkill) flags.push('selected_skill');
if (isGoalRunIntent(text)) flags.push('goal_run');
const normalizedFlags = [...new Set(flags)].filter((flag) => ALLOWED_ROUTER_DECISION_FLAGS.has(flag));
return {
@@ -693,8 +693,9 @@ export function logRouterDecisionShadow(
}
/**
* Apply router session_hint for goosed agent path without overriding hard rules.
* Returns null when gateway should start a fresh goosed session.
* Resolve the Goose session id for agent-run gateway execution.
* When Portal already bound a session, always reuse it; session_hint only
* requests a fresh Goose session when no agentSessionId is present yet.
*/
export function resolveGatewayAgentSessionId({
agentSessionId = null,
@@ -725,16 +726,63 @@ export function resolveGatewayAgentSessionId({
return sessionId;
}
if (decision.session_hint === ROUTER_DECISION_SESSION_HINT.NEW) {
return null;
}
return sessionId;
}
export function normalizeChatRouterCanaryUserIds(value) {
return normalizeAgentCanaryUserIds(value);
}
export function isChatLlmRouterEligible({
enabled = false,
canaryUserIds = [],
userId = null,
} = {}) {
if (!enabled) return false;
if (!Array.isArray(canaryUserIds) || canaryUserIds.length === 0) return true;
const normalizedUserId = String(userId ?? '').trim();
if (!normalizedUserId) return false;
return canaryUserIds.includes(normalizedUserId);
}
export function logChatLlmRouterShadow(
{
baseline = null,
llmResult = null,
userId = null,
sessionId = null,
routerMemoryItemsUsed = null,
} = {},
{ logger = console } = {},
) {
const payload = {
userId: userId ?? null,
sessionId: sessionId ?? null,
baselineRoute: baseline?.route ?? null,
baselineSource: baseline?.source ?? null,
llmRoute: llmResult?.route ?? null,
llmConfidence: llmResult?.confidence ?? null,
llmReason: llmResult?.reason ?? null,
routerMemoryItemsUsed,
wouldChangeRoute: Boolean(
llmResult?.route != null
&& baseline?.route != null
&& llmResult.route !== baseline.route,
),
};
const line = `[chat-llm-router-shadow] ${JSON.stringify(payload)}`;
if (typeof logger.warn === 'function') logger.warn(line);
else if (typeof logger.info === 'function') logger.info(line);
else logger.log(line);
}
export function resolveChatIntentRouterPolicy({ env = process.env, overrides = {} } = {}) {
return {
enabled: envFlag(env?.MEMIND_CHAT_LLM_ROUTER_ENABLED, false),
shadowMode: envFlag(env?.MEMIND_CHAT_LLM_ROUTER_SHADOW, false),
canaryUserIds: normalizeChatRouterCanaryUserIds(
overrides.canaryUserIds ?? env?.MEMIND_CHAT_ROUTER_CANARY_USER_IDS,
),
modelProviderKeyId: String(env?.MEMIND_CHAT_ROUTER_MODEL_PROVIDER_KEY_ID ?? '').trim() || null,
model: String(env?.MEMIND_CHAT_ROUTER_MODEL ?? '').trim() || null,
modelApiType: String(env?.MEMIND_CHAT_ROUTER_MODEL_API ?? '').trim() || 'chat',
@@ -811,6 +859,7 @@ export function buildAgentOrchestrationAgentText({
classification,
skillPrompt = '',
memoryContext = null,
goalContext = null,
}) {
const taskBody = String(displayText ?? '').trim();
const memoryLines = memoryContext?.injectionEnabled
@@ -823,6 +872,9 @@ export function buildAgentOrchestrationAgentText({
.filter(Boolean)
.slice(0, 8)
: [];
const goalEnvelope = goalContext?.injectionEnabled
? String(goalContext.envelope ?? '').trim()
: '';
const lines = [
`${AGENT_ORCHESTRATION_HEADER}以下为用户任务,请使用工具与技能实际执行并产出结果,不要只做文字描述。`,
`路由判定:${classification.reason}`,
@@ -830,6 +882,7 @@ export function buildAgentOrchestrationAgentText({
classification.suggestedSkill ? `建议 skill${classification.suggestedSkill}` : '',
buildImageGenerationInstruction(classification.imageGeneration),
skillPrompt,
goalEnvelope,
memoryLines.length
? [
'[Memory Context]',
@@ -847,7 +900,7 @@ export function buildAgentOrchestrationAgentText({
export function applyAgentOrchestrationToUserMessage(
userMessage,
classification,
{ grantedSkills = [], memoryContext = null } = {},
{ grantedSkills = [], memoryContext = null, goalContext = null } = {},
) {
const displayText = messageDisplayText(userMessage);
const skillPrompt = resolveSkillPrompt(classification?.suggestedSkill, grantedSkills, displayText);
@@ -856,6 +909,7 @@ export function applyAgentOrchestrationToUserMessage(
classification,
skillPrompt,
memoryContext,
goalContext,
});
const content = Array.isArray(userMessage?.content)
? userMessage.content.map((item, index) => {
@@ -894,7 +948,6 @@ export function classifyWithRules({
sessionMessageCount = null,
userMessage = null,
includeIntentPatterns = true,
llmRouterEnabled = false,
} = {}) {
const decisionContext = {
text,
@@ -992,6 +1045,16 @@ export function classifyWithRules({
if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) {
return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext);
}
if (includeIntentPatterns && normalized) {
const faqMatch = matchDirectChatFaqRule(normalized);
if (faqMatch) {
return finalizeRouterClassification(normalizeClassification({
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
confidence: 0.94,
reason: faqMatch.reason,
}, { source: 'rule' }), decisionContext);
}
}
if (
includeIntentPatterns &&
normalized &&
@@ -1030,6 +1093,8 @@ export function createChatIntentRouter(options = {}) {
env,
overrides: pickDefined(options, [
'enabled',
'shadowMode',
'canaryUserIds',
'modelProviderKeyId',
'model',
'modelApiType',
@@ -1041,6 +1106,68 @@ export function createChatIntentRouter(options = {}) {
'fallbackRoute',
]),
});
function isLlmRouterEligibleForUser(userId) {
return isChatLlmRouterEligible({
enabled: policy.enabled,
canaryUserIds: policy.canaryUserIds,
userId,
});
}
function buildFallbackClassification(reason = '规则未命中,走 skill 默认 Agent 通道') {
return normalizeClassification({
route: policy.fallbackRoute,
confidence: 0.5,
reason,
}, { source: 'fallback', fallbackRoute: policy.fallbackRoute });
}
async function classifyWithLlm({
text,
grantedSkills = [],
routerContext = null,
} = {}) {
if (typeof llmProviderService?.createChatCompletion !== 'function') return null;
try {
const completion = await withTimeout(
llmProviderService.createChatCompletion({
providerKeyId: policy.modelProviderKeyId || undefined,
model: policy.model || undefined,
temperature: policy.temperature ?? 0,
messages: [
{ role: 'system', content: buildRouterSystemPrompt(grantedSkills) },
{
role: 'user',
content: buildRouterUserPrompt({
text,
routerContext: routerContext?.content ?? null,
}),
},
],
}),
policy.timeoutMs,
'Chat intent router',
);
if (!completion?.ok) return null;
const parsed = parseRouterJson(completion.reply);
if (!parsed) return null;
return normalizeClassification({
route: parsed.route,
confidence: parsed.confidence,
reason: parsed.reason,
suggested_skill: parsed.suggested_skill ?? parsed.suggestedSkill,
agent_brief: parsed.agent_brief ?? parsed.agentBrief,
providerKeyId: completion.providerKeyId ?? policy.modelProviderKeyId,
model: completion.model ?? policy.model,
}, { source: 'llm', fallbackRoute: policy.fallbackRoute });
} catch (err) {
logger?.warn?.(
`[chat-intent-router] LLM classify skipped: ${err instanceof Error ? err.message : err}`,
);
return null;
}
}
const agentMemoryPolicy = {
enabled: envFlag(env?.MEMORY_AGENT_RESOLVE_ENABLED, false),
mode: normalizeAgentInjectionMode(env?.MEMORY_AGENT_INJECTION_MODE),
@@ -1062,8 +1189,10 @@ export function createChatIntentRouter(options = {}) {
return {
enabled: true,
ruleRoutingEnabled: true,
llmRoutingEnabled: false,
llmRoutingEnabled: Boolean(policy.enabled && !policy.shadowMode),
llmRoutingShadow: Boolean(policy.enabled && policy.shadowMode),
configuredLlmEnabled: Boolean(policy.enabled),
llmRoutingCanaryUserCount: policy.canaryUserIds.length,
modelProviderKeyId: policy.modelProviderKeyId ?? null,
model: policy.model ?? null,
modelApiType: policy.modelApiType ?? 'chat',
@@ -1083,8 +1212,8 @@ export function createChatIntentRouter(options = {}) {
}
function isEnabled() {
// Rule + skill routing is always active. The LLM classifier stays disabled
// independently; returning false here would make the gateway skip rules too.
// Rule + skill routing is always active. LLM routing is separately gated by
// MEMIND_CHAT_LLM_ROUTER_ENABLED; returning false here would skip rules too.
return true;
}
@@ -1296,6 +1425,7 @@ export function createChatIntentRouter(options = {}) {
}),
}, decisionContext);
};
const llmRouterEligible = isLlmRouterEligibleForUser(userId);
const ruleResult = classifyWithRules({
text,
forceDeepReasoning,
@@ -1304,20 +1434,81 @@ export function createChatIntentRouter(options = {}) {
sessionMessageCount,
userMessage,
includeIntentPatterns: true,
llmRouterEnabled: false,
});
if (ruleResult) return finalizeWithCoercion(ruleResult);
return finalizeWithCoercion(normalizeClassification({
route: policy.fallbackRoute,
confidence: 0.5,
reason: '规则未命中,走 skill 默认 Agent 通道',
}, { source: 'fallback' }));
const baseline = buildFallbackClassification();
if (!llmRouterEligible) {
return finalizeWithCoercion(baseline);
}
const memoryPromise = policy.memoryResolveEnabled && userId
? resolveRouterContext({
userId,
sessionId,
text,
forceDeepReasoning,
})
.then((result) => buildRouterContext(result))
.catch(() => buildRouterContext(null))
: Promise.resolve(buildRouterContext(null));
// Memory is optional routing hint; run LLM in parallel to cut serial latency.
const [routerContext, llmResult] = await Promise.all([
memoryPromise,
classifyWithLlm({
text,
grantedSkills,
routerContext: null,
}),
]);
if (policy.shadowMode) {
logChatLlmRouterShadow({
baseline,
llmResult,
userId,
sessionId,
routerMemoryItemsUsed: routerContext?.itemsUsed ?? 0,
}, { logger });
return finalizeWithCoercion({
...baseline,
llmShadow: llmResult
? {
route: llmResult.route,
confidence: llmResult.confidence,
reason: llmResult.reason,
source: llmResult.source ?? 'llm',
wouldChangeRoute: llmResult.route !== baseline.route,
}
: {
skipped: true,
reason: 'llm_unavailable_or_low_signal',
wouldChangeRoute: false,
},
});
}
if (!llmResult || llmResult.confidence < policy.minConfidence) {
return finalizeWithCoercion({
...baseline,
llmSuggestion: llmResult
? {
route: llmResult.route,
confidence: llmResult.confidence,
reason: llmResult.reason,
source: llmResult.source ?? 'llm',
}
: null,
});
}
return finalizeWithCoercion(llmResult);
}
async function classifySessionAction({ text } = {}) {
// The general H5 router intentionally remains rule/skill-only. Session
// action classification is separately opt-in because it runs before every
// WeChat Agent request that contains a context hint.
// WeChat session action classification is separately opt-in because it runs
// before every WeChat Agent request that contains a context hint.
if (!policy.enabled || typeof llmProviderService?.createChatCompletion !== 'function') return null;
try {
const completion = await withTimeout(
+196 -42
View File
@@ -26,9 +26,14 @@ import {
IMAGE_GENERATION_MODE,
resolveImageGenerationDecision,
coerceRealtimeWebSkill,
isChatLlmRouterEligible,
logChatLlmRouterShadow,
resolveChatIntentRouterPolicy,
} from './chat-intent-router.mjs';
/** Ambiguous user text that should miss FAQ/rules and exercise LLM router paths in tests. */
const AMBIGUOUS_LLM_ROUTER_QUERY = '周末想放松一下,有什么活动建议?';
test('image generation intent requires a real image for explicit background requests', () => {
const decision = resolveImageGenerationDecision({
text: '帮我生成一首唐诗页面,背景是大唐盛景图片,页面要精美',
@@ -568,11 +573,11 @@ test('createChatIntentRouter keeps continued sessions on agent without llm route
assert.match(result.reason, /延续已有 Agent 会话/);
});
test('createChatIntentRouter uses agent fallback for general conversational analysis (skill-only)', async () => {
test('createChatIntentRouter uses agent fallback for general conversational analysis when llm router disabled', async () => {
const llmCalls = [];
const resolveCalls = [];
const router = createChatIntentRouter({
enabled: true,
enabled: false,
modelProviderKeyId: 'key-router',
model: 'deepseek-chat',
llmProviderService: {
@@ -694,7 +699,7 @@ test('createChatIntentRouter routes memory recall through rules when memory reso
assert.equal(result.source, 'rule');
});
test('createManagedChatIntentRouter hot-loads admin config and stays skill-only', async () => {
test('createManagedChatIntentRouter hot-loads admin config and activates llm routing', async () => {
const states = [
{
fingerprint: 'off',
@@ -743,16 +748,16 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only'
const result = await router.classify({
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我概括一下这段材料的要点' }],
metadata: { displayText: '帮我概括一下这段材料的要点' },
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(llmCalls.length, 0);
assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
assert.equal(result.source, 'llm');
assert.equal(llmCalls.length, 1);
});
test('createChatIntentRouter uses agent fallback when rules miss (skill-only)', async () => {
test('createChatIntentRouter rejects low-confidence llm direct_chat and falls back to agent', async () => {
const llmCalls = [];
const router = createChatIntentRouter({
enabled: true,
@@ -777,17 +782,17 @@ test('createChatIntentRouter uses agent fallback when rules miss (skill-only)',
const result = await router.classify({
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我整理一下这段材料' }],
metadata: { displayText: '帮我整理一下这段材料' },
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(llmCalls.length, 0);
assert.equal(llmCalls.length, 1);
});
test('createChatIntentRouter falls back to agent when rules miss (skill-only)', async () => {
test('createChatIntentRouter falls back to agent when llm router fails', async () => {
const llmCalls = [];
const router = createChatIntentRouter({
enabled: true,
@@ -802,17 +807,53 @@ test('createChatIntentRouter falls back to agent when rules miss (skill-only)',
const result = await router.classify({
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我概括一下这段材料的要点' }],
metadata: { displayText: '帮我概括一下这段材料的要点' },
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
grantedSkills: ['web'],
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(llmCalls.length, 1);
});
test('createChatIntentRouter uses agent fallback when llm router disabled', async () => {
const llmCalls = [];
const router = createChatIntentRouter({
enabled: false,
llmProviderService: {
async createChatCompletion() {
llmCalls.push(1);
return { ok: false, message: 'router model unavailable' };
},
},
});
const result = await router.classify({
userMessage: {
role: 'user',
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(llmCalls.length, 0);
});
test('classifyWithRules fast-paths FAQ concept questions to direct chat without LLM', () => {
const result = classifyWithRules({
text: '什么是 SSE',
sessionId: null,
sessionMessageCount: 0,
});
assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
assert.equal(result.source, 'rule');
assert.match(result.reason, /FAQ/);
});
test('classifyWithRules routes memory recall to direct chat on fresh session', () => {
const result = classifyWithRules({
text: '你记得我说想去哪儿吗',
@@ -932,7 +973,7 @@ test('buildRouterNormalizedDecision maps legacy routes and session hints', () =>
sessionMessageCount: 0,
},
);
assert.equal(freshAgentSession.session_hint, ROUTER_DECISION_SESSION_HINT.NEW);
assert.equal(freshAgentSession.session_hint, ROUTER_DECISION_SESSION_HINT.REUSE);
const memoryRecall = buildRouterNormalizedDecision(
{ route: CHAT_INTENT_ROUTE.DIRECT_CHAT },
@@ -940,6 +981,12 @@ test('buildRouterNormalizedDecision maps legacy routes and session hints', () =>
);
assert.equal(memoryRecall.route, ROUTER_DECISION_ROUTE.CHAT);
assert.ok(memoryRecall.flags.includes('memory_recall'));
const goalRun = buildRouterNormalizedDecision(
{ route: CHAT_INTENT_ROUTE.AGENT },
{ text: '帮我分阶段完成下季度产品规划' },
);
assert.ok(goalRun.flags.includes('goal_run'));
});
test('resolveLegacyRouteFromClassification prefers decision when flag enabled', () => {
@@ -985,7 +1032,7 @@ test('resolveGatewayAgentSessionId honors session_hint without breaking hard rul
decision: { session_hint: ROUTER_DECISION_SESSION_HINT.NEW },
},
}),
null,
'sess-1',
);
assert.equal(
resolveGatewayAgentSessionId({
@@ -1370,31 +1417,6 @@ test('logRouterDecisionShadow emits payload only in shadow mode', () => {
}
});
test('createChatIntentRouter uses agent fallback for unmatched general questions (skill-only)', async () => {
const llmCalls = [];
const router = createChatIntentRouter({
enabled: true,
llmProviderService: {
async createChatCompletion() {
llmCalls.push(1);
return { ok: false, message: 'router disabled' };
},
},
});
const result = await router.classify({
userMessage: {
role: 'user',
content: [{ type: 'text', text: '什么是 SSE' }],
metadata: { displayText: '什么是 SSE' },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(llmCalls.length, 0);
});
test('classifyWithRules routes page data dev repair before new page-data collect', () => {
const result = classifyWithRules({
text: '修复问卷 insert 403columns_not_allowed',
@@ -1405,3 +1427,135 @@ test('classifyWithRules routes page data dev repair before new page-data collect
assert.match(result.reason, /Page Data 开发修复/);
assert.equal(result.suggested_skill, undefined);
});
test('isChatLlmRouterEligible respects enabled flag and canary user ids', () => {
assert.equal(isChatLlmRouterEligible({ enabled: false, canaryUserIds: ['john'], userId: 'john' }), false);
assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: [], userId: 'john' }), true);
assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: 'john' }), true);
assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: 'other' }), false);
assert.equal(isChatLlmRouterEligible({ enabled: true, canaryUserIds: ['john'], userId: null }), false);
});
test('createChatIntentRouter limits llm routing to canary users', async () => {
const llmCalls = [];
const router = createChatIntentRouter({
enabled: true,
canaryUserIds: ['user-canary'],
llmProviderService: {
async createChatCompletion() {
llmCalls.push(1);
return {
ok: true,
reply: JSON.stringify({
route: 'direct_chat',
confidence: 0.91,
reason: '纯文字问答',
suggested_skill: null,
agent_brief: '',
}),
};
},
},
});
const control = await router.classify({
userId: 'user-control',
userMessage: {
role: 'user',
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
const canary = await router.classify({
userId: 'user-canary',
userMessage: {
role: 'user',
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
assert.equal(control.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(control.source, 'fallback');
assert.equal(canary.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
assert.equal(canary.source, 'llm');
assert.equal(llmCalls.length, 1);
});
test('createChatIntentRouter shadow mode logs llm suggestion without changing route', async () => {
const lines = [];
const logger = { warn(line) { lines.push(line); }, info(line) { lines.push(line); } };
const router = createChatIntentRouter({
enabled: true,
shadowMode: true,
llmProviderService: {
async createChatCompletion() {
return {
ok: true,
reply: JSON.stringify({
route: 'direct_chat',
confidence: 0.95,
reason: '纯文字问答',
suggested_skill: null,
agent_brief: '',
}),
};
},
},
logger,
});
const result = await router.classify({
userId: 'user-john',
userMessage: {
role: 'user',
content: [{ type: 'text', text: AMBIGUOUS_LLM_ROUTER_QUERY }],
metadata: { displayText: AMBIGUOUS_LLM_ROUTER_QUERY },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.source, 'fallback');
assert.equal(result.llmShadow?.route, CHAT_INTENT_ROUTE.DIRECT_CHAT);
assert.equal(result.llmShadow?.wouldChangeRoute, true);
assert.equal(lines.length, 1);
assert.match(lines[0], /\[chat-llm-router-shadow\]/);
assert.match(lines[0], /wouldChangeRoute":true/);
});
test('logChatLlmRouterShadow emits structured payload', () => {
const lines = [];
logChatLlmRouterShadow({
baseline: { route: CHAT_INTENT_ROUTE.AGENT, source: 'fallback' },
llmResult: { route: CHAT_INTENT_ROUTE.DIRECT_CHAT, confidence: 0.9, reason: '闲聊' },
userId: 'user-1',
sessionId: 'sess-1',
}, { logger: { info(line) { lines.push(line); } } });
assert.equal(lines.length, 1);
assert.match(lines[0], /baselineRoute":"agent_orchestration"/);
assert.match(lines[0], /llmRoute":"direct_chat"/);
});
test('resolveChatIntentRouterPolicy parses llm router shadow and canary env', () => {
const policy = resolveChatIntentRouterPolicy({
env: {
MEMIND_CHAT_LLM_ROUTER_ENABLED: '1',
MEMIND_CHAT_LLM_ROUTER_SHADOW: '1',
MEMIND_CHAT_ROUTER_CANARY_USER_IDS: 'john, testuser1',
},
});
assert.equal(policy.enabled, true);
assert.equal(policy.shadowMode, true);
assert.deepEqual(policy.canaryUserIds, ['john', 'testuser1']);
});
test('createChatIntentRouter getStatus reflects llm router modes', () => {
const active = createChatIntentRouter({ enabled: true });
const shadow = createChatIntentRouter({ enabled: true, shadowMode: true });
const off = createChatIntentRouter({ enabled: false });
assert.equal(active.getStatus().llmRoutingEnabled, true);
assert.equal(active.getStatus().llmRoutingShadow, false);
assert.equal(shadow.getStatus().llmRoutingEnabled, false);
assert.equal(shadow.getStatus().llmRoutingShadow, true);
assert.equal(off.getStatus().configuredLlmEnabled, false);
});
+76
View File
@@ -0,0 +1,76 @@
export const GOAL_RUN_INTENT_PATTERNS = [
/(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i,
/(?:分阶段|多步骤|长期任务|持续跟进|帮我规划并执行)/i,
];
export function isGoalRunIntent(text) {
const normalized = String(text ?? '').trim();
if (!normalized) return false;
return GOAL_RUN_INTENT_PATTERNS.some((pattern) => pattern.test(normalized));
}
function envFlag(value, fallback = false) {
const raw = String(value ?? '').trim().toLowerCase();
if (!raw) return fallback;
return ['1', 'true', 'yes', 'on'].includes(raw);
}
function parseUserIdSet(raw) {
return new Set(
String(raw ?? '')
.split(/[,;\s]+/)
.map((item) => item.trim())
.filter(Boolean),
);
}
export function isGoalRunEnabledForUser(userId, env = process.env) {
if (!envFlag(env.GOAL_RUN_ENABLED, false)) return false;
const canaryUserIds = parseUserIdSet(env.GOAL_RUN_CANARY_USER_IDS);
if (!canaryUserIds.size) return true;
return canaryUserIds.has(String(userId ?? '').trim());
}
export function extractUserMessageDisplayText(userMessage) {
if (!userMessage) return '';
if (typeof userMessage === 'string') return userMessage.trim();
const metadataText = String(userMessage?.metadata?.displayText ?? '').trim();
if (metadataText) return metadataText;
const content = userMessage?.content;
if (typeof content === 'string') return content.trim();
if (Array.isArray(content)) {
for (const item of content) {
if (typeof item === 'string' && item.trim()) return item.trim();
if (item?.type === 'text' && String(item.text ?? '').trim()) {
return String(item.text).trim();
}
}
}
return '';
}
export function deriveGoalTitleFromText(text, maxLength = 80) {
const normalized = String(text ?? '').replace(/\s+/g, ' ').trim();
if (!normalized) return '长期任务';
return normalized.length <= maxLength
? normalized
: `${normalized.slice(0, Math.max(0, maxLength - 1))}`;
}
export function buildGoalContinueUserMessage(feedback = null, { id = null } = {}) {
const displayText = String(feedback ?? '').trim()
? `继续下一阶段。补充说明:${String(feedback).trim()}`
: '继续执行下一阶段';
const messageId = id ?? (typeof crypto !== 'undefined' && crypto.randomUUID
? crypto.randomUUID()
: `goal-continue-${Date.now()}`);
return {
id: messageId,
role: 'user',
content: [{ type: 'text', text: displayText }],
metadata: {
userVisible: true,
displayText,
},
};
}
+60
View File
@@ -0,0 +1,60 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { buildGoalContextEnvelope } from './goal-run-context.mjs';
import {
buildGoalContinueUserMessage,
deriveGoalTitleFromText,
isGoalRunEnabledForUser,
isGoalRunIntent,
} from './goal-run-intent.mjs';
test('isGoalRunIntent detects long-running task phrases', () => {
assert.equal(isGoalRunIntent('帮我分阶段完成下季度产品规划'), true);
assert.equal(isGoalRunIntent('你好'), false);
});
test('isGoalRunEnabledForUser respects feature flag and canary list', () => {
assert.equal(isGoalRunEnabledForUser('user-a', { GOAL_RUN_ENABLED: '0' }), false);
assert.equal(isGoalRunEnabledForUser('user-a', { GOAL_RUN_ENABLED: '1' }), true);
assert.equal(
isGoalRunEnabledForUser('user-a', {
GOAL_RUN_ENABLED: '1',
GOAL_RUN_CANARY_USER_IDS: 'user-b',
}),
false,
);
assert.equal(
isGoalRunEnabledForUser('user-a', {
GOAL_RUN_ENABLED: '1',
GOAL_RUN_CANARY_USER_IDS: 'user-a,user-b',
}),
true,
);
});
test('buildGoalContextEnvelope renders friendly goal summary', () => {
const envelope = buildGoalContextEnvelope({
title: '准备下季度产品规划',
currentCheckpointId: 'cp-1',
checkpoints: [
{ id: 'cp-1', title: '启动', status: 'running', outputSummary: null },
{ id: 'cp-0', title: '调研', status: 'approved', outputSummary: '完成竞品列表' },
],
});
assert.match(envelope, /【Goal Context】/);
assert.match(envelope, /准备下季度产品规划/);
assert.match(envelope, /完成竞品列表/);
});
test('deriveGoalTitleFromText truncates long titles', () => {
const title = deriveGoalTitleFromText('a'.repeat(120), 20);
assert.equal(title.length, 20);
assert.match(title, /…$/);
});
test('buildGoalContinueUserMessage renders continue text', () => {
const message = buildGoalContinueUserMessage('先聚焦竞品', { id: 'msg-1' });
assert.equal(message.id, 'msg-1');
assert.match(message.metadata.displayText, /继续下一阶段/);
assert.match(message.metadata.displayText, /竞品/);
});
+1
View File
@@ -70,6 +70,7 @@ export const GOOSED_BOUNDARY_SCAN_IGNORE = [
/^scripts\//,
/^mindspace-service\//,
/^\.runtime\//,
/^\.release-gate\//,
/^node_modules\//,
/\.test\.mjs$/,
/\.test\.ts$/,
+4 -1
View File
@@ -13,6 +13,8 @@ const FIELD_SPECS = [
{ env: 'MEMORY_FAIL_OPEN', group: 'global', field: 'failOpen', type: 'boolean' },
{ env: 'MEMIND_CHAT_LLM_ROUTER_ENABLED', group: 'chatIntentRouter', field: 'enabled', type: 'boolean' },
{ env: 'MEMIND_CHAT_LLM_ROUTER_SHADOW', group: 'chatIntentRouter', field: 'shadowMode', type: 'boolean' },
{ env: 'MEMIND_CHAT_ROUTER_CANARY_USER_IDS', group: 'chatIntentRouter', field: 'canaryUserIds', type: 'string' },
{ env: 'MEMIND_CHAT_ROUTER_MODEL_PROVIDER_KEY_ID', group: 'chatIntentRouter', field: 'modelProviderKeyId', type: 'string' },
{ env: 'MEMIND_CHAT_ROUTER_MODEL', group: 'chatIntentRouter', field: 'model', type: 'string' },
{ env: 'MEMIND_CHAT_ROUTER_MODEL_API', group: 'chatIntentRouter', field: 'modelApiType', type: 'string' },
@@ -28,6 +30,7 @@ const FIELD_SPECS = [
{ env: 'MEMORY_CANDIDATE_MIN_CONFIDENCE', group: 'candidateMemory', field: 'minConfidence', type: 'number' },
{ env: 'MEMORY_CANDIDATE_MAX_PENDING', group: 'candidateMemory', field: 'maxPending', type: 'number' },
{ env: 'MEMORY_CANDIDATE_PERSISTENCE_ENABLED', group: 'candidateMemory', field: 'persistenceEnabled', type: 'boolean' },
{ env: 'MEMORY_CANDIDATE_AUTO_ACCEPT_ALL', group: 'candidateMemory', field: 'autoAcceptAll', type: 'boolean' },
{ env: 'MEMORY_AGENT_RESOLVE_ENABLED', group: 'runtimeControl', field: 'agentResolveEnabled', type: 'boolean' },
{ env: 'MEMORY_AGENT_INJECTION_MODE', group: 'runtimeControl', field: 'agentInjectionMode', type: 'string' },
@@ -283,7 +286,7 @@ function applyEnv(config, secrets, env = process.env) {
config.chatIntentRouter.memoryResolveLimit = '8';
}
if (!normalizeString(config.chatIntentRouter.timeoutMs)) {
config.chatIntentRouter.timeoutMs = '2500';
config.chatIntentRouter.timeoutMs = '1200';
}
if (config.chatIntentRouter.enabled === undefined) {
config.chatIntentRouter.enabled = false;
+2
View File
@@ -15,6 +15,7 @@ import { AgentCodeRunPage } from './pages/admin/AgentCodeRunPage';
import { WechatPage } from './pages/admin/WechatPage';
import { SystemPolicyPage } from './pages/admin/SystemPolicyPage';
import { OrchestratorPage } from './pages/admin/OrchestratorPage';
import { LlmProvidersPage } from './pages/admin/LlmProvidersPage';
export function App() {
return (
@@ -50,6 +51,7 @@ export function App() {
<Route path="wechat" element={<WechatPage />} />
<Route path="policy" element={<SystemPolicyPage />} />
<Route path="orchestrator" element={<OrchestratorPage />} />
<Route path="llm" element={<LlmProvidersPage />} />
<Route path="*" element={<Navigate to="/admin" replace />} />
</Route>
+78
View File
@@ -815,3 +815,81 @@ export async function fetchAgentCodeRunHistory(limit = 50) {
`/admin-api/agent-code-run/runs?limit=${limit}`,
);
}
// ─── Unified LLM Providers ────────────────────────────────────────────────────
export type LlmProviderKeyRow = {
id: string;
providerId: string;
providerKind: 'builtin' | 'custom';
providerLabel: string;
name: string;
defaultModel: string;
models: string[];
apiUrl: string | null;
status: 'active' | 'disabled';
isSelected: boolean;
apiKeyMasked: string;
};
export type LlmGlobalSettings = {
keyId: string | null;
keyName: string | null;
providerLabel: string | null;
globalModel: string | null;
availableModels: string[];
};
export type ChatIntentRouterAdminConfig = {
enabled?: boolean;
shadowMode?: boolean;
canaryUserIds?: string;
modelProviderKeyId?: string;
model?: string;
modelApiType?: string;
minConfidence?: string;
memoryResolveEnabled?: boolean;
memoryResolveLimit?: string;
timeoutMs?: string;
fallbackRoute?: string;
};
export type MemoryV2AdminConfigState = {
config: {
chatIntentRouter?: ChatIntentRouterAdminConfig;
[key: string]: unknown;
};
updatedAt: number | null;
updatedBy: string | null;
};
export type MemoryV2RuntimeState = {
source: string;
updatedAt: number | null;
updatedBy: string | null;
fingerprint?: string;
overrides: Record<string, string>;
};
export async function fetchLlmProviderKeys() {
return adminFetch<{ keys: LlmProviderKeyRow[] }>('/admin-api/llm-providers/keys');
}
export async function fetchLlmGlobalSettings() {
return adminFetch<{ global: LlmGlobalSettings }>('/admin-api/llm-providers/global');
}
export async function fetchMemoryV2Config() {
return adminFetch<MemoryV2AdminConfigState>('/admin-api/memory-v2/config');
}
export async function patchMemoryV2Config(patch: Record<string, unknown>) {
return adminFetch<MemoryV2AdminConfigState>('/admin-api/memory-v2/config', {
method: 'PATCH',
body: JSON.stringify(patch),
});
}
export async function fetchMemoryV2Runtime() {
return adminFetch<MemoryV2RuntimeState>('/admin-api/memory-v2/runtime');
}
+1
View File
@@ -8,6 +8,7 @@ const links = [
{ to: '/admin/agent-code-run', label: 'Code Run' },
{ to: '/admin/policy', label: '策略中心' },
{ to: '/admin/orchestrator', label: '任务编排' },
{ to: '/admin/llm', label: '统一大模型' },
];
export function AdminLayout() {
+367
View File
@@ -0,0 +1,367 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import {
fetchLlmGlobalSettings,
fetchLlmProviderKeys,
fetchMemoryV2Config,
fetchMemoryV2Runtime,
patchMemoryV2Config,
type ChatIntentRouterAdminConfig,
type LlmGlobalSettings,
type LlmProviderKeyRow,
type MemoryV2RuntimeState,
} from '../../api/admin';
type RouterForm = {
enabled: boolean;
shadowMode: boolean;
modelProviderKeyId: string;
model: string;
canaryUserIds: string;
};
function formatTime(value: number | null | undefined) {
if (!value) return '—';
return new Date(value).toLocaleString('zh-CN', { hour12: false });
}
function routerConfigToForm(
config: ChatIntentRouterAdminConfig | undefined,
keys: LlmProviderKeyRow[],
): RouterForm {
const keyId = String(config?.modelProviderKeyId ?? '').trim();
const selectedKey = keys.find((item) => item.id === keyId) ?? null;
const model = String(config?.model ?? '').trim()
|| selectedKey?.defaultModel
|| selectedKey?.models?.[0]
|| '';
return {
enabled: Boolean(config?.enabled),
shadowMode: Boolean(config?.shadowMode),
modelProviderKeyId: keyId,
model,
canaryUserIds: String(config?.canaryUserIds ?? '').trim(),
};
}
function formToPatch(form: RouterForm) {
return {
chatIntentRouter: {
enabled: form.enabled,
shadowMode: form.shadowMode,
modelProviderKeyId: form.modelProviderKeyId || '',
model: form.model || '',
canaryUserIds: form.canaryUserIds,
},
};
}
function ToggleRow({
label,
hint,
checked,
onChange,
disabled,
}: {
label: string;
hint?: string;
checked: boolean;
onChange: (next: boolean) => void;
disabled?: boolean;
}) {
return (
<label style={{ display: 'grid', gridTemplateColumns: '1fr auto', gap: 12, alignItems: 'start' }}>
<span>
<strong>{label}</strong>
{hint ? <p style={{ margin: '4px 0 0', color: '#68716c', fontSize: 13 }}>{hint}</p> : null}
</span>
<input
type="checkbox"
checked={checked}
disabled={disabled}
onChange={(event) => onChange(event.target.checked)}
style={{ width: 'auto', marginTop: 4 }}
/>
</label>
);
}
function runtimeMode(overrides: Record<string, string>) {
const enabled = ['1', 'true', 'yes', 'on'].includes(String(overrides.MEMIND_CHAT_LLM_ROUTER_ENABLED ?? '').toLowerCase());
const shadow = ['1', 'true', 'yes', 'on'].includes(String(overrides.MEMIND_CHAT_LLM_ROUTER_SHADOW ?? '').toLowerCase());
if (!enabled) return '关闭(仅规则路由)';
if (shadow) return 'Shadow 观测(行为不变)';
return '已激活(规则未命中时走 LLM 路由)';
}
export function LlmProvidersPage() {
const [keys, setKeys] = useState<LlmProviderKeyRow[]>([]);
const [globalSettings, setGlobalSettings] = useState<LlmGlobalSettings | null>(null);
const [form, setForm] = useState<RouterForm | null>(null);
const [savedForm, setSavedForm] = useState<RouterForm | null>(null);
const [runtime, setRuntime] = useState<MemoryV2RuntimeState | null>(null);
const [updatedAt, setUpdatedAt] = useState<number | null>(null);
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [notice, setNotice] = useState<string | null>(null);
const activeKeys = useMemo(
() => keys.filter((item) => item.status === 'active'),
[keys],
);
const selectedRouterKey = useMemo(
() => activeKeys.find((item) => item.id === form?.modelProviderKeyId) ?? null,
[activeKeys, form?.modelProviderKeyId],
);
const modelOptions = useMemo(
() => selectedRouterKey?.models ?? [],
[selectedRouterKey],
);
const dirty = useMemo(() => {
if (!form || !savedForm) return false;
return JSON.stringify(form) !== JSON.stringify(savedForm);
}, [form, savedForm]);
const load = useCallback(async () => {
setLoading(true);
setError(null);
try {
const [keysResult, globalResult, configResult, runtimeResult] = await Promise.all([
fetchLlmProviderKeys(),
fetchLlmGlobalSettings(),
fetchMemoryV2Config(),
fetchMemoryV2Runtime(),
]);
const nextKeys = keysResult.keys ?? [];
const nextForm = routerConfigToForm(configResult.config?.chatIntentRouter, nextKeys);
setKeys(nextKeys);
setGlobalSettings(globalResult.global ?? null);
setForm(nextForm);
setSavedForm(nextForm);
setRuntime(runtimeResult);
setUpdatedAt(configResult.updatedAt ?? null);
} catch (err) {
setError(err instanceof Error ? err.message : '加载失败');
} finally {
setLoading(false);
}
}, []);
useEffect(() => {
void load();
}, [load]);
const updateForm = (patch: Partial<RouterForm>) => {
setForm((current) => {
if (!current) return current;
const next = { ...current, ...patch };
if ('modelProviderKeyId' in patch) {
const key = activeKeys.find((item) => item.id === next.modelProviderKeyId);
const models = key?.models ?? [];
if (!models.includes(next.model)) {
next.model = key?.defaultModel ?? models[0] ?? '';
}
}
return next;
});
};
const save = async () => {
if (!form) return;
setSaving(true);
setError(null);
setNotice(null);
try {
const result = await patchMemoryV2Config(formToPatch(form));
const nextForm = routerConfigToForm(result.config?.chatIntentRouter, keys);
setForm(nextForm);
setSavedForm(nextForm);
setUpdatedAt(result.updatedAt ?? null);
const runtimeResult = await fetchMemoryV2Runtime();
setRuntime(runtimeResult);
setNotice('已保存。Portal 将在下一次 H5 路由请求时热加载新配置,无需重启。');
} catch (err) {
setError(err instanceof Error ? err.message : '保存失败');
} finally {
setSaving(false);
}
};
if (loading) return <p></p>;
if (error && !form) return <p className="alert">{error}</p>;
if (!form) return <p className="alert"></p>;
return (
<div className="grid">
<div className="card">
<h2 style={{ marginTop: 0 }}></h2>
<p style={{ color: '#68716c', marginTop: 0 }}>
LLM H5
</p>
{globalSettings ? (
<p style={{ marginBottom: 0 }}>
{' '}
<strong>{globalSettings.keyName ?? '(未选择密钥)'}</strong>
{' / '}
<strong>{globalSettings.globalModel ?? '(未设置模型)'}</strong>
</p>
) : null}
</div>
<div className="card">
<h3 style={{ marginTop: 0 }}></h3>
{activeKeys.length === 0 ? (
<p style={{ color: '#68716c' }}> Provider Key</p>
) : (
<table className="table">
<thead>
<tr>
<th></th>
<th>Provider</th>
<th></th>
<th></th>
</tr>
</thead>
<tbody>
{activeKeys.map((item) => (
<tr key={item.id}>
<td>
{item.name}
{item.isSelected ? <span style={{ marginLeft: 8, color: '#2f7d32' }}></span> : null}
</td>
<td>{item.providerLabel}</td>
<td>{item.defaultModel}</td>
<td>{item.status}</td>
</tr>
))}
</tbody>
</table>
)}
</div>
<div className="card">
<h3 style={{ marginTop: 0 }}>H5 LLM Router</h3>
<p style={{ color: '#68716c', marginTop: 0 }}>
LLM Shadow `[chat-llm-router-shadow]` `intent_routed.llmShadow`
</p>
<div style={{ display: 'grid', gap: 16, maxWidth: 720 }}>
<ToggleRow
label="启用 H5 LLM 路由"
hint="关闭时与现网一致:规则未命中直接 fallback 到 Goose。"
checked={form.enabled}
onChange={(enabled) => updateForm({ enabled })}
/>
<ToggleRow
label="Shadow 观测模式"
hint="开启后仍走 fallback Agent,仅记录 LLM 建议路由,适合灰度观测。"
checked={form.shadowMode}
disabled={!form.enabled}
onChange={(shadowMode) => updateForm({ shadowMode })}
/>
<label style={{ display: 'grid', gap: 6 }}>
<strong></strong>
<span style={{ color: '#68716c', fontSize: 13 }}>使 env </span>
<select
value={form.modelProviderKeyId}
disabled={!form.enabled}
onChange={(event) => updateForm({ modelProviderKeyId: event.target.value })}
>
<option value=""></option>
{activeKeys.map((item) => (
<option key={item.id} value={item.id}>
{item.name}
{' '}
(
{item.providerLabel}
)
</option>
))}
</select>
</label>
<label style={{ display: 'grid', gap: 6 }}>
<strong></strong>
<select
value={form.model}
disabled={!form.enabled || !form.modelProviderKeyId || modelOptions.length === 0}
onChange={(event) => updateForm({ model: event.target.value })}
>
{modelOptions.length === 0 ? (
<option value=""></option>
) : (
modelOptions.map((model) => (
<option key={model} value={model}>{model}</option>
))
)}
</select>
</label>
<label style={{ display: 'grid', gap: 6 }}>
<strong> ID</strong>
<span style={{ color: '#68716c', fontSize: 13 }}>
LLM
</span>
<input
value={form.canaryUserIds}
disabled={!form.enabled}
onChange={(event) => updateForm({ canaryUserIds: event.target.value })}
placeholder="user-id-1, user-id-2"
/>
</label>
</div>
<div style={{ display: 'flex', gap: 12, marginTop: 20, alignItems: 'center', flexWrap: 'wrap' }}>
<button className="btn" type="button" disabled={!dirty || saving} onClick={() => void save()}>
{saving ? '保存中…' : '保存 H5 路由配置'}
</button>
<button className="btn secondary" type="button" disabled={loading || saving} onClick={() => void load()}>
</button>
<span style={{ color: '#68716c', fontSize: 13 }}>
{formatTime(updatedAt)}
</span>
</div>
{notice ? <p style={{ color: '#2f7d32', marginBottom: 0 }}>{notice}</p> : null}
{error ? <p className="alert" style={{ marginBottom: 0 }}>{error}</p> : null}
</div>
{runtime ? (
<div className="card">
<h3 style={{ marginTop: 0 }}></h3>
<p style={{ marginTop: 0 }}>
<strong>{runtime.source}</strong>
{' · '}
<strong>{runtimeMode(runtime.overrides ?? {})}</strong>
</p>
<div style={{ display: 'grid', gap: 8, fontSize: 14 }}>
<div>
<code>{runtime.overrides.MEMIND_CHAT_ROUTER_MODEL_PROVIDER_KEY_ID || '(未设置)'}</code>
</div>
<div>
<code>{runtime.overrides.MEMIND_CHAT_ROUTER_MODEL || '(未设置)'}</code>
</div>
<div>
<code>{runtime.overrides.MEMIND_CHAT_ROUTER_CANARY_USER_IDS || '(全部用户)'}</code>
</div>
<div style={{ color: '#68716c' }}>
fingerprint
{runtime.fingerprint ?? '—'}
</div>
</div>
</div>
) : null}
</div>
);
}
+4 -5
View File
@@ -68,11 +68,10 @@ export function SummaryPage() {
<Link className="btn secondary" to="/admin/agent-code-run" style={{ textAlign: 'center', textDecoration: 'none' }}>
Code Run
</Link>
<Link
className="btn secondary"
to="/admin/users"
style={{ textAlign: 'center', textDecoration: 'none' }}
>
<Link className="btn secondary" to="/admin/llm" style={{ textAlign: 'center', textDecoration: 'none' }}>
/ H5
</Link>
<Link className="btn secondary" to="/admin/users" style={{ textAlign: 'center', textDecoration: 'none' }}>
</Link>
</div>
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env node
/**
* Local smoke: verify ambiguous H5 chat triggers LLM router shadow logging.
* Usage: node scripts/verify-h5-llm-router-shadow.mjs
*/
import crypto from 'node:crypto';
import { loadH5Environment } from './load-env.mjs';
import { createDbPool } from '../db.mjs';
import { createUserAuth } from '../user-auth.mjs';
import { resolveChatIntentRouterPolicy } from '../chat-intent-router.mjs';
loadH5Environment(import.meta.dirname);
const PORTAL = `http://127.0.0.1:${process.env.H5_PORT ?? 8081}`;
const USERNAME = process.env.VERIFY_LLM_ROUTER_USER ?? 'john2';
const PASSWORD = process.env.JOHN_PASSWORD ?? process.env.H5_ACCESS_PASSWORD ?? '888888';
const QUERY = '周末想放松一下,有什么活动建议?';
const MAX_WAIT_MS = 120_000;
const LOG_PATH = process.env.MEMIND_LLM_ROUTER_SHADOW_LOG
?? '/Users/john/Library/Logs/memind-h5-local.log';
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function pass(label, detail = '') {
console.log(`PASS ${label}${detail ? `: ${detail}` : ''}`);
}
function fail(label, detail = '') {
console.error(`FAIL ${label}${detail ? `: ${detail}` : ''}`);
process.exitCode = 1;
}
async function login() {
const pool = await createDbPool();
const auth = createUserAuth(pool);
const result = await auth.login({ username: USERNAME, password: PASSWORD, ip: '127.0.0.1' });
if (!result.ok) {
throw new Error(`john 登录失败: ${result.message ?? 'unknown'}`);
}
await pool.end();
return result.token;
}
async function createRun(token) {
const requestId = crypto.randomUUID();
const response = await fetch(`${PORTAL}/api/agent/runs`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Cookie: `tkmind_user_session=${token}`,
},
body: JSON.stringify({
request_id: requestId,
user_message: {
id: crypto.randomUUID(),
role: 'user',
content: [{ type: 'text', text: QUERY }],
metadata: { userVisible: true, displayText: QUERY },
},
}),
});
const body = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(`POST /agent/runs ${response.status}: ${JSON.stringify(body)}`);
}
const run = body.run ?? body;
return { runId: run.id, requestId, status: run.status };
}
async function waitForRun(token, runId) {
const started = Date.now();
while (Date.now() - started < MAX_WAIT_MS) {
const response = await fetch(`${PORTAL}/api/agent/runs/${runId}`, {
headers: { Cookie: `tkmind_user_session=${token}` },
});
const payload = await response.json().catch(() => ({}));
const body = payload.run ?? payload;
if (!response.ok) {
throw new Error(`GET /agent/runs/${runId} ${response.status}: ${JSON.stringify(payload)}`);
}
if (['succeeded', 'failed'].includes(body.status)) {
return body;
}
await sleep(1000);
}
throw new Error(`run ${runId} 未在 ${MAX_WAIT_MS}ms 内完成`);
}
async function loadRunEvents(runId) {
const pool = await createDbPool();
const [rows] = await pool.query(
`SELECT event_type, data_json FROM h5_agent_run_events WHERE run_id = ? ORDER BY id ASC`,
[runId],
);
await pool.end();
return rows.map((row) => ({
event_type: row.event_type,
data: typeof row.data_json === 'string' ? JSON.parse(row.data_json) : row.data_json,
}));
}
async function waitForPortal() {
const started = Date.now();
while (Date.now() - started < 60_000) {
try {
const response = await fetch(`${PORTAL}/api/status`);
if (response.ok) return;
} catch {
// retry
}
await sleep(1000);
}
throw new Error(`Portal ${PORTAL} 未在 60s 内就绪`);
}
async function readShadowLogLines() {
try {
const fs = await import('node:fs/promises');
const text = await fs.readFile(LOG_PATH, 'utf8');
return text.split('\n').filter((line) => line.includes('[chat-llm-router-shadow]'));
} catch {
return [];
}
}
async function main() {
const policy = resolveChatIntentRouterPolicy();
console.log('--- router policy ---');
console.log(JSON.stringify({
enabled: policy.enabled,
shadowMode: policy.shadowMode,
canaryUserIds: policy.canaryUserIds,
timeoutMs: policy.timeoutMs,
}, null, 2));
if (!policy.enabled || !policy.shadowMode) {
fail('policy', '需要 MEMIND_CHAT_LLM_ROUTER_ENABLED=1 且 MEMIND_CHAT_LLM_ROUTER_SHADOW=1');
return;
}
pass('policy', 'shadow 模式已开启');
await waitForPortal();
pass('portal', PORTAL);
const token = await login();
pass('login', USERNAME);
const beforeShadowLines = await readShadowLogLines();
const { runId } = await createRun(token);
pass('agent run', runId);
await waitForRun(token, runId);
pass('run finished');
const events = await loadRunEvents(runId);
const routed = events.find((row) => row.event_type === 'intent_routed');
if (!routed?.data) {
fail('intent_routed', '未找到路由事件');
} else {
const { route, source, reason, llmShadow } = routed.data;
console.log('\n--- intent_routed ---');
console.log(JSON.stringify(routed.data, null, 2));
if (source === 'fallback' && route === 'agent_orchestration') {
pass('shadow routing', '规则未命中,Shadow 仍 fallback Agent');
} else if (source === 'rule') {
fail('shadow routing', `意外命中规则: ${reason ?? route}`);
} else {
fail('shadow routing', `source=${source}, route=${route}`);
}
if (llmShadow) {
pass('llm shadow payload', JSON.stringify(llmShadow));
} else {
fail('llm shadow payload', 'intent_routed 缺少 llmShadowLLM 路由可能未启用)');
}
}
await sleep(500);
const afterShadowLines = await readShadowLogLines();
const newShadowLines = afterShadowLines.slice(beforeShadowLines.length);
if (newShadowLines.length > 0) {
pass('shadow log', `${newShadowLines.length} 条 [chat-llm-router-shadow]`);
for (const line of newShadowLines.slice(-3)) {
console.log(line.trim());
}
} else if (!process.exitCode) {
console.log('NOTE shadow log 未写入 LaunchAgent 日志,但 intent_routed.llmShadow 已记录观测结果');
}
if (process.exitCode && process.exitCode !== 0) {
console.error('\n验证未完全通过。');
} else {
console.log('\n验证通过:Shadow 模式已观测 LLM 路由建议,用户侧仍走 Agent fallback。');
}
}
main().catch((err) => {
console.error(err instanceof Error ? err.stack ?? err.message : err);
process.exit(1);
});