#!/usr/bin/env node /** * Simulate multi-persona chat utterances against the live chat intent router. * * Usage: * node scripts/simulate-persona-intent-routing.mjs * node scripts/run-scenario-test.mjs --scenario persona-intent-routing */ import fs from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { createDbPool } from '../db.mjs'; import { createLlmProviderService } from '../llm-providers.mjs'; import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs'; import { createChatIntentRouter, createManagedChatIntentRouter, } from '../chat-intent-router.mjs'; const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const JOHN_USER_ID = 'a6fb1e97-2b0f-447b-b138-4561d8e5c53e'; function sourceMatches(actual, expected) { if (!expected) return true; return String(actual ?? '') === String(expected); } export async function loadPersonaIntentCases(scenarioId = 'persona-intent-routing') { const scenarioPath = path.join(repoRoot, 'scenarios', `${scenarioId}.json`); const scenario = JSON.parse(await fs.readFile(scenarioPath, 'utf8')); return { scenario, cases: scenario.cases ?? [] }; } export async function runPersonaIntentRouting({ userId = JOHN_USER_ID, scenarioId = 'persona-intent-routing', reporter, env = process.env, } = {}) { const { scenario, cases } = await loadPersonaIntentCases(scenarioId); if (!cases.length) { throw new Error(`场景 ${scenarioId} 没有 cases`); } const pool = createDbPool(env); const llmProviderService = createLlmProviderService(pool, { apiTarget: env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006', apiSecret: env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret', }); const configService = createMemoryV2AdminConfigService(pool, { env }); const adminRouter = createManagedChatIntentRouter({ llmProviderService, configService, env, logger: { warn() {}, log() {}, info() {} }, }); const adminStatus = await adminRouter.getStatus().catch(() => null); // Local simulation follows .env (defer + LLM router). Admin-db currently // disables LLM routing and would swallow ambiguous tasks into fallback chat. const router = createChatIntentRouter({ llmProviderService, env, logger: console, }); try { const status = router.getStatus(); reporter?.pass?.( 'router 状态', `env llm=${status.llmRoutingEnabled ? 'on' : 'off'} shadow=${status.llmRoutingShadow ? 'on' : 'off'};admin-db llm=${adminStatus?.llmRoutingEnabled ? 'on' : 'off'}`, ); if (!status.llmRoutingEnabled) { reporter?.fail?.('LLM router', '本机 .env 未进入生效模式,歧义句无法精准识别'); } else if (adminStatus && !adminStatus.llmRoutingEnabled) { reporter?.pass?.( 'admin-db 对照', '管理后台当前关闭 LLM router,Portal 热路径仍会把歧义句打成 fallback 直聊;本模拟按 .env 验证识别精度', ); } const rows = []; for (const item of cases) { const classification = await router.classify({ userId, sessionId: 'h5direct_persona_sim', sessionMessageCount: 3, toolMode: 'chat', userMessage: { role: 'user', content: [{ type: 'text', text: item.text }], metadata: { displayText: item.text, userVisible: true }, }, }); const routeOk = classification?.route === item.expectRoute; const sourceOk = item.expectSource === 'llm' ? classification?.source === 'llm' || classification?.source === 'rule' : sourceMatches(classification?.source, item.expectSource); // Rule-expected cases must keep rule source. LLM-expected cases may still // be caught by a stronger rule (page/news); that is acceptable if route matches. const sourceStrictOk = item.expectSource === 'rule' ? classification?.source === 'rule' : true; const ok = Boolean(routeOk && sourceStrictOk); const label = `${item.persona} · ${item.intent}`; const detail = `${classification?.route ?? 'null'}/${classification?.source ?? 'none'} conf=${classification?.confidence ?? '-'} · ${classification?.reason ?? ''}`; if (ok) reporter?.pass?.(label, `${item.text} → ${detail}`); else { reporter?.fail?.( label, `期望 ${item.expectRoute}/${item.expectSource ?? '*'},实际 ${detail};话术:${item.text}`, ); } rows.push({ ...item, ok, routeOk, sourceOk, actualRoute: classification?.route ?? null, actualSource: classification?.source ?? null, confidence: classification?.confidence ?? null, reason: classification?.reason ?? null, }); } const failed = rows.filter((row) => !row.ok); console.log('\n--- 人群 × 意图 ---'); for (const row of rows) { const mark = row.ok ? '✔' : '✘'; console.log( `${mark} [${row.persona}] ${row.intent}: ${row.actualRoute} (${row.actualSource}) <- ${row.text}`, ); } return { scenario, rows, failed, ok: failed.length === 0, }; } finally { await pool.end?.(); } } async function main() { const { createReporter } = await import('./scenario-test-lib.mjs'); const { loadH5Environment } = await import('./load-env.mjs'); loadH5Environment(import.meta.dirname); const reporter = createReporter(); const result = await runPersonaIntentRouting({ reporter, userId: process.env.MEMIND_E2E_USER_ID ?? JOHN_USER_ID, }); const code = reporter.summary(); process.exit(result.ok ? code : 1); } const isDirectRun = process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); if (isDirectRun) { main().catch((error) => { console.error(error instanceof Error ? error.stack ?? error.message : error); process.exit(1); }); }