#!/usr/bin/env node /** * Local smoke: verify ambiguous H5 chat triggers LLM router shadow logging. * Usage: node scripts/verify-h5-llm-router-shadow.mjs */ import crypto from 'node:crypto'; import { loadH5Environment } from './load-env.mjs'; import { createDbPool } from '../db.mjs'; import { createUserAuth } from '../user-auth.mjs'; import { resolveChatIntentRouterPolicy } from '../chat-intent-router.mjs'; loadH5Environment(import.meta.dirname); const PORTAL = `http://127.0.0.1:${process.env.H5_PORT ?? 8081}`; const USERNAME = process.env.VERIFY_LLM_ROUTER_USER ?? 'john2'; const PASSWORD = process.env.JOHN_PASSWORD ?? process.env.H5_ACCESS_PASSWORD ?? '888888'; const QUERY = '周末想放松一下,有什么活动建议?'; const MAX_WAIT_MS = 120_000; const LOG_PATH = process.env.MEMIND_LLM_ROUTER_SHADOW_LOG ?? '/Users/john/Library/Logs/memind-h5-local.log'; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function pass(label, detail = '') { console.log(`PASS ${label}${detail ? `: ${detail}` : ''}`); } function fail(label, detail = '') { console.error(`FAIL ${label}${detail ? `: ${detail}` : ''}`); process.exitCode = 1; } async function login() { const pool = await createDbPool(); const auth = createUserAuth(pool); const result = await auth.login({ username: USERNAME, password: PASSWORD, ip: '127.0.0.1' }); if (!result.ok) { throw new Error(`john 登录失败: ${result.message ?? 'unknown'}`); } await pool.end(); return result.token; } async function createRun(token) { const requestId = crypto.randomUUID(); const response = await fetch(`${PORTAL}/api/agent/runs`, { method: 'POST', headers: { 'Content-Type': 'application/json', Cookie: `tkmind_user_session=${token}`, }, body: JSON.stringify({ request_id: requestId, user_message: { id: crypto.randomUUID(), role: 'user', content: [{ type: 'text', text: QUERY }], metadata: { userVisible: true, displayText: QUERY }, }, }), }); const body = await response.json().catch(() => ({})); if (!response.ok) { throw new Error(`POST /agent/runs ${response.status}: ${JSON.stringify(body)}`); } const run = body.run ?? body; return { runId: run.id, requestId, status: run.status }; } async function waitForRun(token, runId) { const started = Date.now(); while (Date.now() - started < MAX_WAIT_MS) { const response = await fetch(`${PORTAL}/api/agent/runs/${runId}`, { headers: { Cookie: `tkmind_user_session=${token}` }, }); const payload = await response.json().catch(() => ({})); const body = payload.run ?? payload; if (!response.ok) { throw new Error(`GET /agent/runs/${runId} ${response.status}: ${JSON.stringify(payload)}`); } if (['succeeded', 'failed'].includes(body.status)) { return body; } await sleep(1000); } throw new Error(`run ${runId} 未在 ${MAX_WAIT_MS}ms 内完成`); } async function loadRunEvents(runId) { const pool = await createDbPool(); const [rows] = await pool.query( `SELECT event_type, data_json FROM h5_agent_run_events WHERE run_id = ? ORDER BY id ASC`, [runId], ); await pool.end(); return rows.map((row) => ({ event_type: row.event_type, data: typeof row.data_json === 'string' ? JSON.parse(row.data_json) : row.data_json, })); } async function waitForPortal() { const started = Date.now(); while (Date.now() - started < 60_000) { try { const response = await fetch(`${PORTAL}/api/status`); if (response.ok) return; } catch { // retry } await sleep(1000); } throw new Error(`Portal ${PORTAL} 未在 60s 内就绪`); } async function readShadowLogLines() { try { const fs = await import('node:fs/promises'); const text = await fs.readFile(LOG_PATH, 'utf8'); return text.split('\n').filter((line) => line.includes('[chat-llm-router-shadow]')); } catch { return []; } } async function main() { const policy = resolveChatIntentRouterPolicy(); console.log('--- router policy ---'); console.log(JSON.stringify({ enabled: policy.enabled, shadowMode: policy.shadowMode, canaryUserIds: policy.canaryUserIds, timeoutMs: policy.timeoutMs, }, null, 2)); if (!policy.enabled || !policy.shadowMode) { fail('policy', '需要 MEMIND_CHAT_LLM_ROUTER_ENABLED=1 且 MEMIND_CHAT_LLM_ROUTER_SHADOW=1'); return; } pass('policy', 'shadow 模式已开启'); await waitForPortal(); pass('portal', PORTAL); const token = await login(); pass('login', USERNAME); const beforeShadowLines = await readShadowLogLines(); const { runId } = await createRun(token); pass('agent run', runId); await waitForRun(token, runId); pass('run finished'); const events = await loadRunEvents(runId); const routed = events.find((row) => row.event_type === 'intent_routed'); if (!routed?.data) { fail('intent_routed', '未找到路由事件'); } else { const { route, source, reason, llmShadow } = routed.data; console.log('\n--- intent_routed ---'); console.log(JSON.stringify(routed.data, null, 2)); if (source === 'fallback' && route === 'agent_orchestration') { pass('shadow routing', '规则未命中,Shadow 仍 fallback Agent'); } else if (source === 'rule') { fail('shadow routing', `意外命中规则: ${reason ?? route}`); } else { fail('shadow routing', `source=${source}, route=${route}`); } if (llmShadow) { pass('llm shadow payload', JSON.stringify(llmShadow)); } else { fail('llm shadow payload', 'intent_routed 缺少 llmShadow(LLM 路由可能未启用)'); } } await sleep(500); const afterShadowLines = await readShadowLogLines(); const newShadowLines = afterShadowLines.slice(beforeShadowLines.length); if (newShadowLines.length > 0) { pass('shadow log', `${newShadowLines.length} 条 [chat-llm-router-shadow]`); for (const line of newShadowLines.slice(-3)) { console.log(line.trim()); } } else if (!process.exitCode) { console.log('NOTE shadow log 未写入 LaunchAgent 日志,但 intent_routed.llmShadow 已记录观测结果'); } if (process.exitCode && process.exitCode !== 0) { console.error('\n验证未完全通过。'); } else { console.log('\n验证通过:Shadow 模式已观测 LLM 路由建议,用户侧仍走 Agent fallback。'); } } main().catch((err) => { console.error(err instanceof Error ? err.stack ?? err.message : err); process.exit(1); });