john
5f3980e0df
Phase B appends snapshot context when leaving h5direct sessions so Goose no longer starts blind. Phase C lets the 0.72 fallback return null on gated chat text so the existing LLM router can run, flags default off. Co-authored-by: Cursor <cursoragent@cursor.com>
365 lines
13 KiB
JavaScript
365 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Local acceptance for chat-task-intent-layer (docs/architecture/chat-task-intent-layer-review-20260828.md).
|
|
*
|
|
* Phase B: direct→agent escalation injects snapshot context (direct_escalation_context_injected).
|
|
* Phase C: optional shadow observation when chat session defer + LLM router shadow are enabled.
|
|
*
|
|
* Usage:
|
|
* MEMIND_DIRECT_ESCALATION_CONTEXT_ENABLED=1 node scripts/verify-chat-task-intent-layer.mjs
|
|
*
|
|
* Requires Portal on 8081, agent-run-worker, goosed, and DB from .env.
|
|
*/
|
|
import crypto from 'node:crypto';
|
|
import { loadH5Environment } from './load-env.mjs';
|
|
import { createDbPool } from '../db.mjs';
|
|
import { createUserAuth, USER_COOKIE } from '../user-auth.mjs';
|
|
import {
|
|
CHAT_INTENT_ROUTE,
|
|
classifyWithRules,
|
|
} from '../chat-intent-router.mjs';
|
|
import {
|
|
isDirectEscalationContextEnabledForUser,
|
|
resolveDirectEscalationContextPolicy,
|
|
resolveChatSessionDeferPolicy,
|
|
shouldDeferChatSessionRoutingToLlm,
|
|
} from '../chat-task-intent-config.mjs';
|
|
|
|
loadH5Environment(import.meta.dirname);
|
|
|
|
const PORTAL = `http://127.0.0.1:${process.env.H5_PORT ?? 8081}`;
|
|
const USERNAME = process.env.MEMIND_E2E_USERNAME ?? process.env.VERIFY_LLM_ROUTER_USER ?? 'john';
|
|
const PASSWORD = process.env.JOHN_PASSWORD ?? process.env.H5_ACCESS_PASSWORD ?? '888888';
|
|
const DIRECT_TURN_MESSAGE = '我想了解车载冰箱，MPV 用，宽度不超过 60cm，先聊聊选购要点';
|
|
const AGENT_TURN_MESSAGE = '帮我生成对比页面 public/fridge-compare-test.html';
|
|
const MAX_RUN_WAIT_MS = Number(process.env.E2E_RUN_WAIT_MS ?? 300_000);
|
|
const MAX_REPLY_WAIT_MS = 120_000;
|
|
|
|
const issues = [];
|
|
const checks = [];
|
|
|
|
function pass(label, detail = '') {
|
|
checks.push({ ok: true, label, detail });
|
|
console.log(`✔ ${label}${detail ? `: ${detail}` : ''}`);
|
|
}
|
|
|
|
function fail(label, detail = '') {
|
|
issues.push({ label, detail });
|
|
checks.push({ ok: false, label, detail });
|
|
console.error(`✘ ${label}${detail ? `: ${detail}` : ''}`);
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function parseEventData(raw) {
|
|
if (!raw) return null;
|
|
if (typeof raw === 'object') return raw;
|
|
try {
|
|
return JSON.parse(String(raw));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function loginViaApi() {
|
|
const response = await fetch(`${PORTAL}/auth/login`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ username: USERNAME, password: PASSWORD }),
|
|
});
|
|
const body = await response.json().catch(() => ({}));
|
|
if (response.ok && body?.authenticated) {
|
|
const setCookie = response.headers.getSetCookie?.() ?? [];
|
|
const cookieLine = setCookie.find((line) => line.startsWith(`${USER_COOKIE}=`))
|
|
?? response.headers.get('set-cookie');
|
|
const match = String(cookieLine ?? '').match(new RegExp(`${USER_COOKIE}=([^;]+)`));
|
|
if (match?.[1]) {
|
|
return {
|
|
token: decodeURIComponent(match[1]),
|
|
userId: body.user?.id ?? null,
|
|
};
|
|
}
|
|
}
|
|
|
|
const pool = await createDbPool();
|
|
const auth = createUserAuth(pool);
|
|
const result = await auth.login({ username: USERNAME, password: PASSWORD, ip: '127.0.0.1' });
|
|
await pool.end();
|
|
if (!result.ok || !result.token) {
|
|
throw new Error(`登录失败: ${result.message ?? 'unknown'}`);
|
|
}
|
|
return {
|
|
token: result.token,
|
|
userId: result.user?.id ?? null,
|
|
};
|
|
}
|
|
|
|
async function createRun(token, { message, sessionId = null, forceDeepReasoning = false } = {}) {
|
|
const requestId = crypto.randomUUID();
|
|
const response = await fetch(`${PORTAL}/api/agent/runs`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
Cookie: `${USER_COOKIE}=${token}`,
|
|
},
|
|
body: JSON.stringify({
|
|
request_id: requestId,
|
|
session_id: sessionId,
|
|
force_deep_reasoning: forceDeepReasoning,
|
|
user_message: {
|
|
id: crypto.randomUUID(),
|
|
role: 'user',
|
|
content: [{ type: 'text', text: message }],
|
|
metadata: { displayText: message, userVisible: true },
|
|
},
|
|
}),
|
|
});
|
|
const payload = await response.json().catch(() => ({}));
|
|
if (!response.ok) {
|
|
throw new Error(`POST /api/agent/runs ${response.status}: ${JSON.stringify(payload)}`);
|
|
}
|
|
const run = payload.run ?? payload;
|
|
return {
|
|
runId: run.id,
|
|
sessionId: resolveRunSessionId(run, sessionId),
|
|
status: run.status,
|
|
};
|
|
}
|
|
|
|
function resolveRunSessionId(run, fallbackSessionId = null) {
|
|
return run?.sessionId ?? run?.agent_session_id ?? fallbackSessionId ?? null;
|
|
}
|
|
|
|
async function getRun(token, runId) {
|
|
const response = await fetch(`${PORTAL}/api/agent/runs/${runId}`, {
|
|
headers: { Cookie: `${USER_COOKIE}=${token}` },
|
|
});
|
|
const payload = await response.json().catch(() => ({}));
|
|
if (!response.ok) {
|
|
throw new Error(`GET run ${response.status}: ${JSON.stringify(payload)}`);
|
|
}
|
|
return payload.run ?? payload;
|
|
}
|
|
|
|
async function readRunEvents(runId) {
|
|
const pool = await createDbPool();
|
|
const [rows] = await pool.query(
|
|
`SELECT event_type, data_json, created_at
|
|
FROM h5_agent_run_events
|
|
WHERE run_id = ?
|
|
ORDER BY created_at ASC`,
|
|
[runId],
|
|
);
|
|
await pool.end();
|
|
return rows;
|
|
}
|
|
|
|
async function waitForRunTerminal(token, runId) {
|
|
const started = Date.now();
|
|
let last = null;
|
|
while (Date.now() - started < MAX_RUN_WAIT_MS) {
|
|
last = await getRun(token, runId);
|
|
if (['succeeded', 'failed'].includes(last.status)) return last;
|
|
await sleep(2000);
|
|
}
|
|
throw new Error(`run ${runId} 超时，最后状态 ${last?.status ?? 'unknown'}`);
|
|
}
|
|
|
|
async function waitForDirectReply(token, sessionId) {
|
|
const started = Date.now();
|
|
while (Date.now() - started < MAX_REPLY_WAIT_MS) {
|
|
const response = await fetch(`${PORTAL}/api/sessions/${encodeURIComponent(sessionId)}`, {
|
|
headers: { Cookie: `${USER_COOKIE}=${token}` },
|
|
});
|
|
const payload = await response.json().catch(() => ({}));
|
|
if (response.ok) {
|
|
const messages = payload?.messages ?? payload?.conversation ?? [];
|
|
const assistant = [...messages].reverse().find((item) =>
|
|
item?.role === 'assistant' && item?.metadata?.source === 'portal-direct-chat');
|
|
if (assistant) {
|
|
const text = Array.isArray(assistant.content)
|
|
? assistant.content.map((part) => part?.text ?? '').join('')
|
|
: String(assistant.content ?? '');
|
|
if (text.trim().length > 10) {
|
|
return text.trim();
|
|
}
|
|
}
|
|
}
|
|
await sleep(1500);
|
|
}
|
|
throw new Error(`direct chat 回复超时 session=${sessionId}`);
|
|
}
|
|
|
|
function verifyStaticGates(userId) {
|
|
const escalationPolicy = resolveDirectEscalationContextPolicy(process.env);
|
|
if (!escalationPolicy.enabled) {
|
|
fail('阶段 B flag', 'MEMIND_DIRECT_ESCALATION_CONTEXT_ENABLED 未开启，E2E 无法验收注入');
|
|
} else if (!isDirectEscalationContextEnabledForUser(userId, escalationPolicy)) {
|
|
fail('阶段 B canary', `用户 ${userId} 不在 MEMIND_DIRECT_ESCALATION_CONTEXT_CANARY_USER_IDS`);
|
|
} else {
|
|
pass('阶段 B flag', 'direct escalation context enabled');
|
|
}
|
|
|
|
const deferPolicy = resolveChatSessionDeferPolicy(process.env);
|
|
const deferSample = classifyWithRules({
|
|
text: '帮我比较一下英得尔和美的车载冰箱参数',
|
|
sessionId: 'h5direct_verify',
|
|
sessionMessageCount: 2,
|
|
chatSessionDeferEnabled: deferPolicy.enabled,
|
|
chatSessionDeferMinTextLength: deferPolicy.minTextLength,
|
|
});
|
|
if (deferPolicy.enabled) {
|
|
if (deferSample !== null) {
|
|
fail('阶段 C defer 规则', '歧义文本应返回 null 交给 LLM router');
|
|
} else {
|
|
pass('阶段 C defer 规则', '歧义文本已 defer');
|
|
}
|
|
} else {
|
|
pass('阶段 C defer 规则', 'flag 未开，跳过（仅静态检查 defer 函数）');
|
|
if (!shouldDeferChatSessionRoutingToLlm({
|
|
enabled: true,
|
|
text: '帮我比较一下英得尔和美的车载冰箱参数',
|
|
minTextLength: 12,
|
|
isExplicitDirectChatOnlyText: () => false,
|
|
})) {
|
|
fail('阶段 C defer 函数', 'shouldDeferChatSessionRoutingToLlm 预期 true');
|
|
}
|
|
}
|
|
|
|
const recall = classifyWithRules({
|
|
text: '你记得我说想去哪儿吗',
|
|
sessionId: 'h5direct_verify',
|
|
chatSessionDeferEnabled: true,
|
|
});
|
|
if (recall?.route !== CHAT_INTENT_ROUTE.DIRECT_CHAT) {
|
|
fail('记忆召回硬规则', `期望 direct_chat，实际 ${recall?.route ?? 'null'}`);
|
|
} else {
|
|
pass('记忆召回硬规则', recall.reason);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const health = await fetch(`${PORTAL}/auth/status`).catch(() => null);
|
|
if (!health?.ok) {
|
|
throw new Error(`Portal 不可用: ${PORTAL}/auth/status`);
|
|
}
|
|
pass('Portal 健康', PORTAL);
|
|
|
|
const { token, userId } = await loginViaApi();
|
|
pass('登录', `${USERNAME} (${userId ?? 'unknown-id'})`);
|
|
|
|
verifyStaticGates(userId);
|
|
|
|
const runTag = crypto.randomUUID().slice(0, 8);
|
|
const directMessage = `${DIRECT_TURN_MESSAGE} [verify-${runTag}]`;
|
|
const agentMessage = `${AGENT_TURN_MESSAGE.replace('.html', `-${runTag}.html`)}`;
|
|
|
|
const directRun = await createRun(token, { message: directMessage });
|
|
pass('direct 轮次提交', `run=${directRun.runId} session=${directRun.sessionId ?? 'pending'}`);
|
|
|
|
const directTerminal = await waitForRunTerminal(token, directRun.runId);
|
|
const directEvents = await readRunEvents(directRun.runId);
|
|
const directRouted = directEvents.find((row) => row.event_type === 'intent_routed');
|
|
const routedData = parseEventData(directRouted?.data_json);
|
|
if (routedData?.route !== CHAT_INTENT_ROUTE.DIRECT_CHAT) {
|
|
fail('direct 路由', `期望 direct_chat，实际 ${routedData?.route ?? 'missing'}`);
|
|
} else {
|
|
pass('direct 路由', routedData.reason ?? 'direct_chat');
|
|
}
|
|
if (directTerminal.status !== 'succeeded') {
|
|
fail('direct run 终态', directTerminal.status);
|
|
} else {
|
|
pass('direct run 终态', 'succeeded');
|
|
}
|
|
|
|
let sessionId = resolveRunSessionId(directTerminal, directRun.sessionId);
|
|
if (!sessionId) {
|
|
const directEventsEarly = await readRunEvents(directRun.runId);
|
|
const completed = directEventsEarly.find((row) => row.event_type === 'direct_chat_completed');
|
|
sessionId = parseEventData(completed?.data_json)?.sessionId ?? null;
|
|
}
|
|
if (!sessionId) {
|
|
fail('direct session', '未获得 sessionId');
|
|
} else {
|
|
pass('direct session', sessionId);
|
|
}
|
|
|
|
let directReply = '';
|
|
try {
|
|
directReply = await waitForDirectReply(token, sessionId);
|
|
pass('direct 回复', `${directReply.length} chars`);
|
|
if (/已交由后台任务处理/.test(directReply)) {
|
|
fail('direct 文案', '仍包含「已交由后台任务处理」');
|
|
} else {
|
|
pass('direct 文案', '未谎报后台执行');
|
|
}
|
|
} catch (err) {
|
|
fail('direct 回复', err instanceof Error ? err.message : String(err));
|
|
}
|
|
|
|
const agentRun = await createRun(token, {
|
|
message: agentMessage,
|
|
sessionId,
|
|
forceDeepReasoning: true,
|
|
});
|
|
pass('agent 升级提交', `run=${agentRun.runId}`);
|
|
|
|
const agentTerminal = await waitForRunTerminal(token, agentRun.runId);
|
|
const agentEvents = await readRunEvents(agentRun.runId);
|
|
if (agentTerminal.status !== 'succeeded') {
|
|
fail('agent run 终态', `${agentTerminal.status} ${agentTerminal.error_message ?? ''}`);
|
|
} else {
|
|
pass('agent run 终态', 'succeeded');
|
|
}
|
|
|
|
const escalated = agentEvents.some((row) => row.event_type === 'direct_session_escalated_to_deep_reasoning');
|
|
if (escalated) {
|
|
pass('升级事件', 'direct_session_escalated_to_deep_reasoning');
|
|
} else {
|
|
pass('升级事件', '同 session agent（非 h5direct 升级路径）');
|
|
}
|
|
|
|
const contextInjected = agentEvents.find((row) => row.event_type === 'direct_escalation_context_injected');
|
|
if (resolveDirectEscalationContextPolicy(process.env).enabled) {
|
|
if (!contextInjected) {
|
|
fail('阶段 B 注入', '缺少 direct_escalation_context_injected 事件');
|
|
} else {
|
|
const data = parseEventData(contextInjected.data_json);
|
|
pass('阶段 B 注入', `messages=${data?.messageCount ?? '?'}`);
|
|
}
|
|
}
|
|
|
|
const transcriptPersisted = agentEvents.some((row) =>
|
|
row.event_type === 'direct_session_transcript_persisted'
|
|
|| row.event_type === 'portal_direct_transcript_persisted');
|
|
if (transcriptPersisted) {
|
|
pass('transcript 持久化', 'ok');
|
|
} else {
|
|
fail('transcript 持久化', '未观察到 transcript 事件');
|
|
}
|
|
|
|
const shadowEvent = agentEvents.find((row) => row.event_type === 'intent_routed');
|
|
const shadowData = parseEventData(shadowEvent?.data_json);
|
|
if (shadowData?.llmShadow?.wouldChangeRoute != null) {
|
|
pass('阶段 C shadow', `wouldChangeRoute=${shadowData.llmShadow.wouldChangeRoute}`);
|
|
}
|
|
|
|
console.log('\n=== 汇总 ===');
|
|
console.log(`通过: ${checks.filter((item) => item.ok).length}/${checks.length}`);
|
|
if (issues.length) {
|
|
console.log('失败项:');
|
|
for (const item of issues) {
|
|
console.log(` - ${item.label}: ${item.detail}`);
|
|
}
|
|
process.exit(1);
|
|
}
|
|
console.log('chat-task-intent-layer 本地验收通过');
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(err instanceof Error ? err.stack ?? err.message : err);
|
|
process.exit(1);
|
|
});
|