john
d58dc2a251
Introduce Draft → Confirm → Commit flow for WeChat schedule intents behind feature flags, plus h5_tasks dual-write/read aggregation and rollout scripts so reminders and automations get explicit user confirmation before persisting. Co-authored-by: Cursor <cursoragent@cursor.com>
334 lines
13 KiB
JavaScript
334 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* Simulate ~1000 user utterances across reminder / schedule / automation tiers.
|
||
* Reports routing accuracy, ambiguity, and agent fallthrough rate.
|
||
*/
|
||
import {
|
||
parseScheduleIntent,
|
||
shouldUseScheduleAssistant,
|
||
} from '../schedule-intent.mjs';
|
||
import {
|
||
isScheduledTaskIntent,
|
||
parseScheduledTaskIntent,
|
||
shouldUseScheduledTaskAutomation,
|
||
} from '../scheduled-task-intent.mjs';
|
||
|
||
const TZ = 'Asia/Shanghai';
|
||
const NOW = Date.UTC(2026, 7, 24, 4, 0, 0); // 2026-08-24 12:00 CST
|
||
|
||
const TIME_VARIANTS = [
|
||
'早上6点', '上午9点', '中午12点', '下午2点', '下午14:30', '14：30', '晚上8点', '今晚9点半',
|
||
'明天早上7点', '明天下午3点', '后天上午10点', '今天15点', '明早6点半', '下午两点半',
|
||
];
|
||
const REMINDER_VERBS = ['提醒我', '设置提醒', '设个提醒', '闹钟', '叫我', '到点提醒'];
|
||
const REMINDER_TOPICS = [
|
||
'开会', '项目计划例会', '吃药', '接孩子', '交周报', '还信用卡', 'Standup',
|
||
'给老板发邮件', '健身', '订外卖', '复盘', '周会', '1对1',
|
||
];
|
||
const TODO_VERBS = ['帮我记一下', '记个待办', '添加待办', '设置一个代办', '先记一下'];
|
||
const TODO_TOPICS = ['跟进合同', '买牛奶', '回复张三', '整理发票', '还书', '修空调'];
|
||
const AUTO_TOPICS = [
|
||
'做今日新闻页面', '生成天气预报页面', '整理待办摘要页面', '推送销售日报',
|
||
'汇总行业资讯', '更新数据看板页面', '生成诗词页面',
|
||
];
|
||
const AUTO_RECURRENCE = ['每天', '每日', '每周一', '每周五', '定时'];
|
||
|
||
function cartesian(parts) {
|
||
return parts.reduce(
|
||
(acc, group) => acc.flatMap((prefix) => group.map((item) => [...prefix, item])),
|
||
[[]],
|
||
);
|
||
}
|
||
|
||
function buildCorpus() {
|
||
const buckets = {
|
||
reminder: [],
|
||
schedule: [],
|
||
automation: [],
|
||
edge: [],
|
||
};
|
||
|
||
for (const time of TIME_VARIANTS) {
|
||
for (const verb of REMINDER_VERBS) {
|
||
for (const topic of REMINDER_TOPICS.slice(0, 6)) {
|
||
buckets.reminder.push({
|
||
text: `${time}${verb}${topic}`,
|
||
expectedTier: 'reminder',
|
||
expectedPath: 'preflight_or_agent',
|
||
});
|
||
buckets.reminder.push({
|
||
text: `帮我设置提醒，${time}，${topic}`,
|
||
expectedTier: 'reminder',
|
||
expectedPath: 'preflight_or_agent',
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
for (const verb of TODO_VERBS) {
|
||
for (const topic of TODO_TOPICS) {
|
||
buckets.schedule.push({
|
||
text: `${verb} ${topic}`,
|
||
expectedTier: 'schedule',
|
||
expectedPath: 'preflight',
|
||
});
|
||
buckets.schedule.push({
|
||
text: `${verb}「${topic}」`,
|
||
expectedTier: 'schedule',
|
||
expectedPath: 'preflight',
|
||
});
|
||
}
|
||
}
|
||
|
||
for (const time of ['早上7点', '每天8点', '每日6点半', '每天早上7点', '每天7点']) {
|
||
buckets.schedule.push({
|
||
text: `${time}把当天待办发给我`,
|
||
expectedTier: 'schedule',
|
||
expectedPath: 'preflight',
|
||
});
|
||
}
|
||
|
||
for (const threshold of [10, 20, 50, 100, 200]) {
|
||
buckets.schedule.push({
|
||
text: `余额低于${threshold}元提醒我`,
|
||
expectedTier: 'schedule',
|
||
expectedPath: 'preflight',
|
||
});
|
||
}
|
||
|
||
for (const rec of AUTO_RECURRENCE) {
|
||
for (const time of ['5点', '6点', '7点半', '8:00', '18点', '8点30分']) {
|
||
for (const topic of AUTO_TOPICS) {
|
||
buckets.automation.push({
|
||
text: `${rec}${time}帮我${topic}`,
|
||
expectedTier: 'automation',
|
||
expectedPath: 'preflight_or_agent',
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
buckets.edge.push(
|
||
{ text: '明天早上六点去跑步，五点半提醒我', expectedTier: 'reminder', expectedPath: 'agent', note: '双时间点' },
|
||
{ text: '每天6点帮我做今日新闻页面', expectedTier: 'automation', expectedPath: 'preflight_or_agent' },
|
||
{ text: '每天6点提醒我开会', expectedTier: 'reminder', expectedPath: 'agent', note: '循环提醒' },
|
||
{ text: '帮我生成一个唐诗页面', expectedTier: 'none', expectedPath: 'general' },
|
||
{ text: '取消我的定时任务', expectedTier: 'automation', expectedPath: 'preflight_or_agent' },
|
||
{ text: '看看我的待办', expectedTier: 'schedule', expectedPath: 'preflight' },
|
||
{ text: '设置提醒', expectedTier: 'reminder', expectedPath: 'clarify', note: '缺时间' },
|
||
{ text: '下午3点提醒我', expectedTier: 'reminder', expectedPath: 'clarify', note: '缺标题' },
|
||
{ text: '不是待办，下午2点提醒我交报告', expectedTier: 'reminder', expectedPath: 'preflight_or_agent' },
|
||
{ text: '帮我设置一个代办明天早上六点去跑步记得在五点半的时候提醒我', expectedTier: 'reminder', expectedPath: 'agent' },
|
||
{ text: '每周一7点整理待办摘要', expectedTier: 'automation', expectedPath: 'preflight_or_agent' },
|
||
{ text: '今晚8点推送今日待办清单', expectedTier: 'schedule', expectedPath: 'agent_or_clarify', note: '待办清单' },
|
||
{ text: '定时任务：每天8点生成天气页面', expectedTier: 'automation', expectedPath: 'preflight_or_agent' },
|
||
{ text: '3小时后提醒我开会', expectedTier: 'reminder', expectedPath: 'agent', note: '相对时间' },
|
||
{ text: '半小时后叫我', expectedTier: 'reminder', expectedPath: 'agent', note: '相对时间' },
|
||
{ text: '周五下午3点项目评审提醒我', expectedTier: 'reminder', expectedPath: 'agent', note: '周几' },
|
||
{ text: '8月30号下午2点提醒我续费', expectedTier: 'reminder', expectedPath: 'agent', note: '具体日期' },
|
||
{ text: '提前15分钟提醒我开会', expectedTier: 'reminder', expectedPath: 'agent', note: '偏移提醒' },
|
||
{ text: '会议是3点，提前10分钟提醒我', expectedTier: 'reminder', expectedPath: 'agent', note: '事件+偏移' },
|
||
{ text: '查一下是否有执行的新闻任务', expectedTier: 'none', expectedPath: 'general', note: '误触' },
|
||
{ text: '设个提醒下午3点开会', expectedTier: 'reminder', expectedPath: 'preflight_or_agent' },
|
||
{ text: '到点提醒下午2点交报告', expectedTier: 'reminder', expectedPath: 'preflight_or_agent' },
|
||
{ text: '帮我设置提醒，下午 14：30 分开会，项目计划例会', expectedTier: 'reminder', expectedPath: 'preflight' },
|
||
// --- Adversarial / ambiguous (expect clarify or correct layer, never wrong commit) ---
|
||
{ text: '明天提醒我自动生成日报', expectedTier: 'ambiguous', expectedPath: 'clarify', note: 'adv:提醒+生成' },
|
||
{ text: '帮我每天看看有没有新的招聘信息', expectedTier: 'automation', expectedPath: 'preflight_or_agent', note: 'adv:监控型自动化' },
|
||
{ text: '设置一个任务，如果余额低于100提醒我', expectedTier: 'schedule', expectedPath: 'preflight_or_agent', note: 'adv:condition trigger' },
|
||
{ text: '有没有我的新闻定时任务', expectedTier: 'none', expectedPath: 'general', note: 'adv:query guard' },
|
||
{ text: '每天8点提醒我跑步', expectedTier: 'reminder', expectedPath: 'agent', note: 'adv:recurring reminder' },
|
||
{ text: '每天8点帮我生成跑步报告', expectedTier: 'automation', expectedPath: 'preflight_or_agent', note: 'adv:notify vs act' },
|
||
);
|
||
|
||
const noisePrefixes = ['那个', '嗯', '麻烦', '请', '能不能'];
|
||
for (const prefix of noisePrefixes) {
|
||
for (const base of buckets.reminder.slice(0, 30)) {
|
||
buckets.edge.push({
|
||
text: `${prefix}${base.text}`,
|
||
expectedTier: 'reminder',
|
||
expectedPath: 'preflight_or_agent',
|
||
});
|
||
}
|
||
}
|
||
|
||
function sample(arr, n) {
|
||
if (arr.length <= n) return arr;
|
||
const step = arr.length / n;
|
||
const out = [];
|
||
for (let i = 0; i < n; i += 1) out.push(arr[Math.floor(i * step)]);
|
||
return out;
|
||
}
|
||
|
||
const target = {
|
||
reminder: 400,
|
||
schedule: 250,
|
||
automation: 280,
|
||
edge: 70,
|
||
};
|
||
|
||
const cases = [];
|
||
for (const [bucket, count] of Object.entries(target)) {
|
||
for (const [index, item] of sample(buckets[bucket], count).entries()) {
|
||
cases.push({ id: `${bucket}-${index}`, ...item });
|
||
}
|
||
}
|
||
return cases;
|
||
}
|
||
|
||
function classifyRoute(text) {
|
||
const schedTask = parseScheduledTaskIntent(text, { now: NOW, timezone: TZ });
|
||
const sched = parseScheduleIntent(text, { timezone: TZ, now: NOW });
|
||
|
||
if (isScheduledTaskIntent(schedTask)) {
|
||
if (schedTask.action === 'create_scheduled_task') {
|
||
return {
|
||
tier: 'automation',
|
||
path: schedTask.needsClarification?.length ? 'clarify' : 'preflight',
|
||
action: schedTask.action,
|
||
detail: schedTask,
|
||
};
|
||
}
|
||
return {
|
||
tier: 'automation',
|
||
path: 'preflight',
|
||
action: schedTask.action,
|
||
detail: schedTask,
|
||
};
|
||
}
|
||
|
||
if (sched.action === 'create_timed_reminder') {
|
||
return {
|
||
tier: 'reminder',
|
||
path: sched.needsClarification?.length ? 'clarify' : 'preflight',
|
||
action: sched.action,
|
||
detail: sched,
|
||
};
|
||
}
|
||
|
||
if (['create_todo', 'create_daily_todo_digest', 'create_balance_alert', 'query_schedule'].includes(sched.action)) {
|
||
return {
|
||
tier: 'schedule',
|
||
path: sched.needsClarification?.length ? 'clarify' : 'preflight',
|
||
action: sched.action,
|
||
detail: sched,
|
||
};
|
||
}
|
||
|
||
if (sched.action === 'schedule_agent') {
|
||
return { tier: 'reminder', path: 'agent', action: sched.action, detail: sched };
|
||
}
|
||
|
||
if (shouldUseScheduledTaskAutomation(text)) {
|
||
return { tier: 'automation', path: 'agent', action: 'none', detail: schedTask };
|
||
}
|
||
|
||
if (shouldUseScheduleAssistant(text)) {
|
||
return { tier: 'reminder', path: 'agent', action: sched.action, detail: sched };
|
||
}
|
||
|
||
return { tier: 'none', path: 'general', action: 'none', detail: sched };
|
||
}
|
||
|
||
function tierMatches(expectedTier, actualTier, text) {
|
||
if (expectedTier === 'ambiguous') return actualTier === 'reminder' || actualTier === 'automation';
|
||
if (expectedTier === actualTier) return true;
|
||
if (expectedTier === 'none' && actualTier === 'none') return true;
|
||
return false;
|
||
}
|
||
|
||
function pathMatches(expectedPath, actualPath) {
|
||
if (expectedPath === actualPath) return true;
|
||
if (expectedPath === 'preflight_or_agent' && (actualPath === 'preflight' || actualPath === 'agent')) return true;
|
||
if (expectedPath === 'agent_or_clarify' && (actualPath === 'agent' || actualPath === 'clarify')) return true;
|
||
if (expectedPath === 'clarify' && (actualPath === 'clarify' || actualPath === 'agent')) return true;
|
||
if (expectedPath === 'general' && actualPath === 'general') return true;
|
||
return false;
|
||
}
|
||
|
||
function main() {
|
||
const corpus = buildCorpus();
|
||
const stats = {
|
||
total: corpus.length,
|
||
tierCorrect: 0,
|
||
pathOk: 0,
|
||
byTier: {},
|
||
byPath: {},
|
||
misroutes: [],
|
||
clarifyCases: [],
|
||
agentFallback: [],
|
||
conflicts: [],
|
||
};
|
||
|
||
for (const item of corpus) {
|
||
const route = classifyRoute(item.text);
|
||
stats.byTier[route.tier] = (stats.byTier[route.tier] ?? 0) + 1;
|
||
stats.byPath[route.path] = (stats.byPath[route.path] ?? 0) + 1;
|
||
|
||
const tierOk = tierMatches(item.expectedTier, route.tier, item.text);
|
||
const pathOk = pathMatches(item.expectedPath ?? 'preflight_or_agent', route.path);
|
||
|
||
if (tierOk) stats.tierCorrect += 1;
|
||
if (pathOk) stats.pathOk += 1;
|
||
|
||
if (!tierOk) {
|
||
stats.misroutes.push({
|
||
text: item.text,
|
||
expectedTier: item.expectedTier,
|
||
actualTier: route.tier,
|
||
path: route.path,
|
||
action: route.action,
|
||
note: item.note ?? null,
|
||
});
|
||
}
|
||
|
||
if (route.path === 'clarify') {
|
||
stats.clarifyCases.push({ text: item.text, action: route.action, detail: route.detail });
|
||
}
|
||
if (route.path === 'agent') {
|
||
stats.agentFallback.push({ text: item.text, expectedTier: item.expectedTier, action: route.action });
|
||
}
|
||
if (shouldUseScheduledTaskAutomation(item.text) && shouldUseScheduleAssistant(item.text)) {
|
||
stats.conflicts.push(item.text);
|
||
}
|
||
}
|
||
|
||
const tierAccuracy = ((stats.tierCorrect / stats.total) * 100).toFixed(1);
|
||
const pathAccuracy = ((stats.pathOk / stats.total) * 100).toFixed(1);
|
||
const preflightRate = (
|
||
((stats.byPath.preflight ?? 0) / stats.total) * 100
|
||
).toFixed(1);
|
||
const agentRate = (
|
||
((stats.byPath.agent ?? 0) / stats.total) * 100
|
||
).toFixed(1);
|
||
const clarifyRate = (
|
||
((stats.byPath.clarify ?? 0) / stats.total) * 100
|
||
).toFixed(1);
|
||
|
||
const misrouteByReason = {};
|
||
for (const row of stats.misroutes) {
|
||
const key = `${row.expectedTier}->${row.actualTier}`;
|
||
misrouteByReason[key] = (misrouteByReason[key] ?? 0) + 1;
|
||
}
|
||
|
||
console.log(JSON.stringify({
|
||
summary: {
|
||
total: stats.total,
|
||
tierAccuracyPct: Number(tierAccuracy),
|
||
pathAccuracyPct: Number(pathAccuracy),
|
||
preflightRatePct: Number(preflightRate),
|
||
agentFallbackRatePct: Number(agentRate),
|
||
clarifyRatePct: Number(clarifyRate),
|
||
skillConflicts: stats.conflicts.length,
|
||
},
|
||
routeDistribution: stats.byTier,
|
||
pathDistribution: stats.byPath,
|
||
misroutePatterns: misrouteByReason,
|
||
topMisroutes: stats.misroutes.slice(0, 25),
|
||
sampleAgentFallback: stats.agentFallback.slice(0, 15),
|
||
sampleClarify: stats.clarifyCases.slice(0, 10),
|
||
}, null, 2));
|
||
}
|
||
|
||
main();
|