feat(intent): 歧义任务走 LLM 路由,并在选中 agent 时离开直聊会话
Memind CI / Test, build, and release guards (push) Has been cancelled
Memind CI / Test, build, and release guards (push) Has been cancelled
对比/清单/行程等口吻按人群场景交给分类器;本机 .env 可覆盖 admin-db 的 router 开关。 顺带在 MindSpace MCP token 过期时轮换 Goose 会话,避免旧会话空 Finish。
This commit is contained in:
@@ -122,6 +122,25 @@ async function runScenario(scenario, port) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (step.action === 'classify_intents') {
|
||||
if (!auth) {
|
||||
throw new Error('classify_intents 步骤前必须先 login');
|
||||
}
|
||||
const { runPersonaIntentRouting } = await import('./simulate-persona-intent-routing.mjs');
|
||||
const result = await runPersonaIntentRouting({
|
||||
userId: auth.user?.id ?? auth.userId ?? null,
|
||||
scenarioId: scenario.id ?? 'persona-intent-routing',
|
||||
reporter,
|
||||
});
|
||||
if (!result.ok) {
|
||||
reporter.fail(
|
||||
'人群意图识别',
|
||||
`${result.failed.length}/${result.rows.length} 条与期望不符`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (step.action === 'upload_images') {
|
||||
if (!auth) {
|
||||
throw new Error('upload_images 步骤前必须先 login');
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Simulate multi-persona chat utterances against the live chat intent router.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/simulate-persona-intent-routing.mjs
|
||||
* node scripts/run-scenario-test.mjs --scenario persona-intent-routing
|
||||
*/
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createDbPool } from '../db.mjs';
|
||||
import { createLlmProviderService } from '../llm-providers.mjs';
|
||||
import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs';
|
||||
import {
|
||||
createChatIntentRouter,
|
||||
createManagedChatIntentRouter,
|
||||
} from '../chat-intent-router.mjs';
|
||||
|
||||
const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const JOHN_USER_ID = 'a6fb1e97-2b0f-447b-b138-4561d8e5c53e';
|
||||
|
||||
function sourceMatches(actual, expected) {
|
||||
if (!expected) return true;
|
||||
return String(actual ?? '') === String(expected);
|
||||
}
|
||||
|
||||
export async function loadPersonaIntentCases(scenarioId = 'persona-intent-routing') {
|
||||
const scenarioPath = path.join(repoRoot, 'scenarios', `${scenarioId}.json`);
|
||||
const scenario = JSON.parse(await fs.readFile(scenarioPath, 'utf8'));
|
||||
return { scenario, cases: scenario.cases ?? [] };
|
||||
}
|
||||
|
||||
export async function runPersonaIntentRouting({
|
||||
userId = JOHN_USER_ID,
|
||||
scenarioId = 'persona-intent-routing',
|
||||
reporter,
|
||||
env = process.env,
|
||||
} = {}) {
|
||||
const { scenario, cases } = await loadPersonaIntentCases(scenarioId);
|
||||
if (!cases.length) {
|
||||
throw new Error(`场景 ${scenarioId} 没有 cases`);
|
||||
}
|
||||
|
||||
const pool = createDbPool(env);
|
||||
const llmProviderService = createLlmProviderService(pool, {
|
||||
apiTarget: env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
|
||||
apiSecret: env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const configService = createMemoryV2AdminConfigService(pool, { env });
|
||||
const adminRouter = createManagedChatIntentRouter({
|
||||
llmProviderService,
|
||||
configService,
|
||||
env,
|
||||
logger: { warn() {}, log() {}, info() {} },
|
||||
});
|
||||
const adminStatus = await adminRouter.getStatus().catch(() => null);
|
||||
// Local simulation follows .env (defer + LLM router). Admin-db currently
|
||||
// disables LLM routing and would swallow ambiguous tasks into fallback chat.
|
||||
const router = createChatIntentRouter({
|
||||
llmProviderService,
|
||||
env,
|
||||
logger: console,
|
||||
});
|
||||
|
||||
try {
|
||||
const status = router.getStatus();
|
||||
reporter?.pass?.(
|
||||
'router 状态',
|
||||
`env llm=${status.llmRoutingEnabled ? 'on' : 'off'} shadow=${status.llmRoutingShadow ? 'on' : 'off'};admin-db llm=${adminStatus?.llmRoutingEnabled ? 'on' : 'off'}`,
|
||||
);
|
||||
if (!status.llmRoutingEnabled) {
|
||||
reporter?.fail?.('LLM router', '本机 .env 未进入生效模式,歧义句无法精准识别');
|
||||
} else if (adminStatus && !adminStatus.llmRoutingEnabled) {
|
||||
reporter?.pass?.(
|
||||
'admin-db 对照',
|
||||
'管理后台当前关闭 LLM router,Portal 热路径仍会把歧义句打成 fallback 直聊;本模拟按 .env 验证识别精度',
|
||||
);
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
for (const item of cases) {
|
||||
const classification = await router.classify({
|
||||
userId,
|
||||
sessionId: 'h5direct_persona_sim',
|
||||
sessionMessageCount: 3,
|
||||
toolMode: 'chat',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: item.text }],
|
||||
metadata: { displayText: item.text, userVisible: true },
|
||||
},
|
||||
});
|
||||
const routeOk = classification?.route === item.expectRoute;
|
||||
const sourceOk = item.expectSource === 'llm'
|
||||
? classification?.source === 'llm' || classification?.source === 'rule'
|
||||
: sourceMatches(classification?.source, item.expectSource);
|
||||
// Rule-expected cases must keep rule source. LLM-expected cases may still
|
||||
// be caught by a stronger rule (page/news); that is acceptable if route matches.
|
||||
const sourceStrictOk = item.expectSource === 'rule'
|
||||
? classification?.source === 'rule'
|
||||
: true;
|
||||
const ok = Boolean(routeOk && sourceStrictOk);
|
||||
const label = `${item.persona} · ${item.intent}`;
|
||||
const detail = `${classification?.route ?? 'null'}/${classification?.source ?? 'none'} conf=${classification?.confidence ?? '-'} · ${classification?.reason ?? ''}`;
|
||||
if (ok) reporter?.pass?.(label, `${item.text} → ${detail}`);
|
||||
else {
|
||||
reporter?.fail?.(
|
||||
label,
|
||||
`期望 ${item.expectRoute}/${item.expectSource ?? '*'},实际 ${detail};话术:${item.text}`,
|
||||
);
|
||||
}
|
||||
rows.push({
|
||||
...item,
|
||||
ok,
|
||||
routeOk,
|
||||
sourceOk,
|
||||
actualRoute: classification?.route ?? null,
|
||||
actualSource: classification?.source ?? null,
|
||||
confidence: classification?.confidence ?? null,
|
||||
reason: classification?.reason ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
const failed = rows.filter((row) => !row.ok);
|
||||
console.log('\n--- 人群 × 意图 ---');
|
||||
for (const row of rows) {
|
||||
const mark = row.ok ? '✔' : '✘';
|
||||
console.log(
|
||||
`${mark} [${row.persona}] ${row.intent}: ${row.actualRoute} (${row.actualSource}) <- ${row.text}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
scenario,
|
||||
rows,
|
||||
failed,
|
||||
ok: failed.length === 0,
|
||||
};
|
||||
} finally {
|
||||
await pool.end?.();
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { createReporter } = await import('./scenario-test-lib.mjs');
|
||||
const { loadH5Environment } = await import('./load-env.mjs');
|
||||
loadH5Environment(import.meta.dirname);
|
||||
const reporter = createReporter();
|
||||
const result = await runPersonaIntentRouting({
|
||||
reporter,
|
||||
userId: process.env.MEMIND_E2E_USER_ID ?? JOHN_USER_ID,
|
||||
});
|
||||
const code = reporter.summary();
|
||||
process.exit(result.ok ? code : 1);
|
||||
}
|
||||
|
||||
const isDirectRun = process.argv[1]
|
||||
&& fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
||||
if (isDirectRun) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user