feat(h5): add LLM intent router admin controls and shadow verification.
Expose shadow/canary router policy in ops admin, add FAQ rule fast-path, and tighten router defaults (1200ms timeout, 0.65 confidence). Includes verify-h5-llm-router-shadow for production canary rollout. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Local smoke: verify ambiguous H5 chat triggers LLM router shadow logging.
|
||||
* Usage: node scripts/verify-h5-llm-router-shadow.mjs
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { loadH5Environment } from './load-env.mjs';
|
||||
import { createDbPool } from '../db.mjs';
|
||||
import { createUserAuth } from '../user-auth.mjs';
|
||||
import { resolveChatIntentRouterPolicy } from '../chat-intent-router.mjs';
|
||||
|
||||
loadH5Environment(import.meta.dirname);
|
||||
|
||||
const PORTAL = `http://127.0.0.1:${process.env.H5_PORT ?? 8081}`;
|
||||
const USERNAME = process.env.VERIFY_LLM_ROUTER_USER ?? 'john2';
|
||||
const PASSWORD = process.env.JOHN_PASSWORD ?? process.env.H5_ACCESS_PASSWORD ?? '888888';
|
||||
const QUERY = '周末想放松一下，有什么活动建议？';
|
||||
const MAX_WAIT_MS = 120_000;
|
||||
const LOG_PATH = process.env.MEMIND_LLM_ROUTER_SHADOW_LOG
|
||||
?? '/Users/john/Library/Logs/memind-h5-local.log';
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function pass(label, detail = '') {
|
||||
console.log(`PASS ${label}${detail ? `: ${detail}` : ''}`);
|
||||
}
|
||||
|
||||
function fail(label, detail = '') {
|
||||
console.error(`FAIL ${label}${detail ? `: ${detail}` : ''}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function login() {
|
||||
const pool = await createDbPool();
|
||||
const auth = createUserAuth(pool);
|
||||
const result = await auth.login({ username: USERNAME, password: PASSWORD, ip: '127.0.0.1' });
|
||||
if (!result.ok) {
|
||||
throw new Error(`john 登录失败: ${result.message ?? 'unknown'}`);
|
||||
}
|
||||
await pool.end();
|
||||
return result.token;
|
||||
}
|
||||
|
||||
async function createRun(token) {
|
||||
const requestId = crypto.randomUUID();
|
||||
const response = await fetch(`${PORTAL}/api/agent/runs`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Cookie: `tkmind_user_session=${token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
request_id: requestId,
|
||||
user_message: {
|
||||
id: crypto.randomUUID(),
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: QUERY }],
|
||||
metadata: { userVisible: true, displayText: QUERY },
|
||||
},
|
||||
}),
|
||||
});
|
||||
const body = await response.json().catch(() => ({}));
|
||||
if (!response.ok) {
|
||||
throw new Error(`POST /agent/runs ${response.status}: ${JSON.stringify(body)}`);
|
||||
}
|
||||
const run = body.run ?? body;
|
||||
return { runId: run.id, requestId, status: run.status };
|
||||
}
|
||||
|
||||
async function waitForRun(token, runId) {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < MAX_WAIT_MS) {
|
||||
const response = await fetch(`${PORTAL}/api/agent/runs/${runId}`, {
|
||||
headers: { Cookie: `tkmind_user_session=${token}` },
|
||||
});
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
const body = payload.run ?? payload;
|
||||
if (!response.ok) {
|
||||
throw new Error(`GET /agent/runs/${runId} ${response.status}: ${JSON.stringify(payload)}`);
|
||||
}
|
||||
if (['succeeded', 'failed'].includes(body.status)) {
|
||||
return body;
|
||||
}
|
||||
await sleep(1000);
|
||||
}
|
||||
throw new Error(`run ${runId} 未在 ${MAX_WAIT_MS}ms 内完成`);
|
||||
}
|
||||
|
||||
async function loadRunEvents(runId) {
|
||||
const pool = await createDbPool();
|
||||
const [rows] = await pool.query(
|
||||
`SELECT event_type, data_json FROM h5_agent_run_events WHERE run_id = ? ORDER BY id ASC`,
|
||||
[runId],
|
||||
);
|
||||
await pool.end();
|
||||
return rows.map((row) => ({
|
||||
event_type: row.event_type,
|
||||
data: typeof row.data_json === 'string' ? JSON.parse(row.data_json) : row.data_json,
|
||||
}));
|
||||
}
|
||||
|
||||
async function waitForPortal() {
|
||||
const started = Date.now();
|
||||
while (Date.now() - started < 60_000) {
|
||||
try {
|
||||
const response = await fetch(`${PORTAL}/api/status`);
|
||||
if (response.ok) return;
|
||||
} catch {
|
||||
// retry
|
||||
}
|
||||
await sleep(1000);
|
||||
}
|
||||
throw new Error(`Portal ${PORTAL} 未在 60s 内就绪`);
|
||||
}
|
||||
|
||||
async function readShadowLogLines() {
|
||||
try {
|
||||
const fs = await import('node:fs/promises');
|
||||
const text = await fs.readFile(LOG_PATH, 'utf8');
|
||||
return text.split('\n').filter((line) => line.includes('[chat-llm-router-shadow]'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const policy = resolveChatIntentRouterPolicy();
|
||||
console.log('--- router policy ---');
|
||||
console.log(JSON.stringify({
|
||||
enabled: policy.enabled,
|
||||
shadowMode: policy.shadowMode,
|
||||
canaryUserIds: policy.canaryUserIds,
|
||||
timeoutMs: policy.timeoutMs,
|
||||
}, null, 2));
|
||||
|
||||
if (!policy.enabled || !policy.shadowMode) {
|
||||
fail('policy', '需要 MEMIND_CHAT_LLM_ROUTER_ENABLED=1 且 MEMIND_CHAT_LLM_ROUTER_SHADOW=1');
|
||||
return;
|
||||
}
|
||||
pass('policy', 'shadow 模式已开启');
|
||||
|
||||
await waitForPortal();
|
||||
pass('portal', PORTAL);
|
||||
|
||||
const token = await login();
|
||||
pass('login', USERNAME);
|
||||
|
||||
const beforeShadowLines = await readShadowLogLines();
|
||||
const { runId } = await createRun(token);
|
||||
pass('agent run', runId);
|
||||
|
||||
await waitForRun(token, runId);
|
||||
pass('run finished');
|
||||
|
||||
const events = await loadRunEvents(runId);
|
||||
const routed = events.find((row) => row.event_type === 'intent_routed');
|
||||
if (!routed?.data) {
|
||||
fail('intent_routed', '未找到路由事件');
|
||||
} else {
|
||||
const { route, source, reason, llmShadow } = routed.data;
|
||||
console.log('\n--- intent_routed ---');
|
||||
console.log(JSON.stringify(routed.data, null, 2));
|
||||
if (source === 'fallback' && route === 'agent_orchestration') {
|
||||
pass('shadow routing', '规则未命中，Shadow 仍 fallback Agent');
|
||||
} else if (source === 'rule') {
|
||||
fail('shadow routing', `意外命中规则: ${reason ?? route}`);
|
||||
} else {
|
||||
fail('shadow routing', `source=${source}, route=${route}`);
|
||||
}
|
||||
if (llmShadow) {
|
||||
pass('llm shadow payload', JSON.stringify(llmShadow));
|
||||
} else {
|
||||
fail('llm shadow payload', 'intent_routed 缺少 llmShadow（LLM 路由可能未启用）');
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(500);
|
||||
const afterShadowLines = await readShadowLogLines();
|
||||
const newShadowLines = afterShadowLines.slice(beforeShadowLines.length);
|
||||
if (newShadowLines.length > 0) {
|
||||
pass('shadow log', `${newShadowLines.length} 条 [chat-llm-router-shadow]`);
|
||||
for (const line of newShadowLines.slice(-3)) {
|
||||
console.log(line.trim());
|
||||
}
|
||||
} else if (!process.exitCode) {
|
||||
console.log('NOTE shadow log 未写入 LaunchAgent 日志，但 intent_routed.llmShadow 已记录观测结果');
|
||||
}
|
||||
|
||||
if (process.exitCode && process.exitCode !== 0) {
|
||||
console.error('\n验证未完全通过。');
|
||||
} else {
|
||||
console.log('\n验证通过：Shadow 模式已观测 LLM 路由建议，用户侧仍走 Agent fallback。');
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err instanceof Error ? err.stack ?? err.message : err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user