feat(router): improve direct_chat fast-path and add 103 token env sync
Memind CI / Test, build, and release guards (push) Failing after 12m44s
Memind CI / Test, build, and release guards (push) Failing after 12m44s
Route explicit text-only Q&A away from Agent before page-generation rules, and add a script to enable Router + tighter memory budgets on 103 without changing the global deepseek-v4-pro model. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -103,6 +103,25 @@ export const DIRECT_CHAT_FAQ_RULES = [
|
|||||||
/^.{0,16}(?:有什么建议|给点建议|怎么看)[??]?$/u,
|
/^.{0,16}(?:有什么建议|给点建议|怎么看)[??]?$/u,
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: 'text_only_declined_page',
|
||||||
|
reason: '明确要求纯文字、不生成页面/链接',
|
||||||
|
patterns: [
|
||||||
|
/(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u,
|
||||||
|
/(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'brief_intro',
|
||||||
|
reason: '短篇幅介绍/说明(纯文字)',
|
||||||
|
patterns: [
|
||||||
|
/(?:用|以).{0,6}(?:一|两|二|三|四|五|几|\d+).{0,4}(?:句话|句).{0,32}(?:介绍|说明|描述|讲讲|推荐)/u,
|
||||||
|
/^(?:简单|简要|大概|概括)(?:介绍|说明|描述|讲讲).{0,48}[??]?$/u,
|
||||||
|
/^简要介绍.{0,48}(?:区别|差异|不同|对比|比较)[??]?$/u,
|
||||||
|
/^(?:推荐|介绍)(?:一个|一下).{0,32}(?:理由|原因)[,,]?一句话/u,
|
||||||
|
/(?:推荐|介绍).{0,32}一句话.{0,16}(?:说明|讲讲).{0,12}(?:理由|原因)/u,
|
||||||
|
],
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
const MAX_FAQ_TEXT_LENGTH = 200;
|
const MAX_FAQ_TEXT_LENGTH = 200;
|
||||||
@@ -111,10 +130,26 @@ const MAX_FAQ_TEXT_LENGTH = 200;
|
|||||||
* @param {string} text
|
* @param {string} text
|
||||||
* @returns {{ id: string, reason: string } | null}
|
* @returns {{ id: string, reason: string } | null}
|
||||||
*/
|
*/
|
||||||
|
const EXPLICIT_TEXT_ONLY_PAGE_DECLINE = /(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u;
|
||||||
|
|
||||||
|
export function isExplicitTextOnlyRequest(text) {
|
||||||
|
const normalized = String(text ?? '').trim();
|
||||||
|
if (!normalized) return false;
|
||||||
|
return EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|
||||||
|
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
export function matchDirectChatFaqRule(text) {
|
export function matchDirectChatFaqRule(text) {
|
||||||
const normalized = String(text ?? '').trim();
|
const normalized = String(text ?? '').trim();
|
||||||
if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null;
|
if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null;
|
||||||
if (FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))) return null;
|
const declinedPage = EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|
||||||
|
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
|
||||||
|
if (
|
||||||
|
!declinedPage
|
||||||
|
&& FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
for (const rule of DIRECT_CHAT_FAQ_RULES) {
|
for (const rule of DIRECT_CHAT_FAQ_RULES) {
|
||||||
if (rule.patterns.some((pattern) => pattern.test(normalized))) {
|
if (rule.patterns.some((pattern) => pattern.test(normalized))) {
|
||||||
return { id: rule.id, reason: rule.reason };
|
return { id: rule.id, reason: rule.reason };
|
||||||
|
|||||||
+20
-12
@@ -15,11 +15,11 @@ import {
|
|||||||
resolveMemoryInterventionMode,
|
resolveMemoryInterventionMode,
|
||||||
} from './memory-intervention.mjs';
|
} from './memory-intervention.mjs';
|
||||||
import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs';
|
import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs';
|
||||||
import { matchDirectChatFaqRule } from './chat-intent-router-rules.mjs';
|
import { matchDirectChatFaqRule, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
|
||||||
import { isGoalRunIntent } from './goal-run-intent.mjs';
|
import { isGoalRunIntent } from './goal-run-intent.mjs';
|
||||||
import { pgvectorMemoryBackendInternals } from './memory-v2-pgvector.mjs';
|
import { pgvectorMemoryBackendInternals } from './memory-v2-pgvector.mjs';
|
||||||
|
|
||||||
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS } from './chat-intent-router-rules.mjs';
|
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
|
||||||
|
|
||||||
export const CHAT_INTENT_ROUTE = {
|
export const CHAT_INTENT_ROUTE = {
|
||||||
DIRECT_CHAT: 'direct_chat',
|
DIRECT_CHAT: 'direct_chat',
|
||||||
@@ -1083,6 +1083,14 @@ export function classifyWithRules({
|
|||||||
reason: '用户在询问个人记忆或历史对话',
|
reason: '用户在询问个人记忆或历史对话',
|
||||||
}, { source: 'rule' }), decisionContext);
|
}, { source: 'rule' }), decisionContext);
|
||||||
}
|
}
|
||||||
|
if (includeIntentPatterns && normalized && isExplicitTextOnlyRequest(normalized)) {
|
||||||
|
const faqMatch = matchDirectChatFaqRule(normalized);
|
||||||
|
return finalizeRouterClassification(normalizeClassification({
|
||||||
|
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
||||||
|
confidence: 0.96,
|
||||||
|
reason: faqMatch?.reason ?? '用户明确要求纯文字、不生成页面',
|
||||||
|
}, { source: 'rule' }), decisionContext);
|
||||||
|
}
|
||||||
if (hasPriorAgentConversation(sessionId, sessionMessageCount)) {
|
if (hasPriorAgentConversation(sessionId, sessionMessageCount)) {
|
||||||
const reason = isAgentSessionContinueText(normalized)
|
const reason = isAgentSessionContinueText(normalized)
|
||||||
? 'Agent 会话确认/续聊'
|
? 'Agent 会话确认/续聊'
|
||||||
@@ -1093,6 +1101,16 @@ export function classifyWithRules({
|
|||||||
reason,
|
reason,
|
||||||
}, { source: 'rule' }), decisionContext);
|
}, { source: 'rule' }), decisionContext);
|
||||||
}
|
}
|
||||||
|
if (includeIntentPatterns && normalized) {
|
||||||
|
const faqMatch = matchDirectChatFaqRule(normalized);
|
||||||
|
if (faqMatch) {
|
||||||
|
return finalizeRouterClassification(normalizeClassification({
|
||||||
|
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
||||||
|
confidence: 0.94,
|
||||||
|
reason: faqMatch.reason,
|
||||||
|
}, { source: 'rule' }), decisionContext);
|
||||||
|
}
|
||||||
|
}
|
||||||
if (includeIntentPatterns && normalized && isPageDataDevIntent(normalized)) {
|
if (includeIntentPatterns && normalized && isPageDataDevIntent(normalized)) {
|
||||||
return finalizeRouterClassification(normalizeClassification({
|
return finalizeRouterClassification(normalizeClassification({
|
||||||
route: CHAT_INTENT_ROUTE.AGENT,
|
route: CHAT_INTENT_ROUTE.AGENT,
|
||||||
@@ -1142,16 +1160,6 @@ export function classifyWithRules({
|
|||||||
if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) {
|
if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) {
|
||||||
return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext);
|
return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext);
|
||||||
}
|
}
|
||||||
if (includeIntentPatterns && normalized) {
|
|
||||||
const faqMatch = matchDirectChatFaqRule(normalized);
|
|
||||||
if (faqMatch) {
|
|
||||||
return finalizeRouterClassification(normalizeClassification({
|
|
||||||
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
|
||||||
confidence: 0.94,
|
|
||||||
reason: faqMatch.reason,
|
|
||||||
}, { source: 'rule' }), decisionContext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (
|
if (
|
||||||
includeIntentPatterns &&
|
includeIntentPatterns &&
|
||||||
normalized &&
|
normalized &&
|
||||||
|
|||||||
@@ -109,6 +109,25 @@ test('classifyWithRules routes greetings to direct chat', () => {
|
|||||||
assert.equal(result.source, 'rule');
|
assert.equal(result.source, 'rule');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('classifyWithRules routes explicit text-only Q&A to direct chat', () => {
|
||||||
|
for (const text of [
|
||||||
|
'用三句话介绍苏州园林,不要生成页面',
|
||||||
|
'简要介绍一下 Rust 和 Go 的区别',
|
||||||
|
'推荐一个最适合第一次去的,一句话说明理由',
|
||||||
|
]) {
|
||||||
|
const result = classifyWithRules({
|
||||||
|
text,
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text }],
|
||||||
|
metadata: { displayText: text },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT, text);
|
||||||
|
assert.equal(result.source, 'rule', text);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('classifyWithRules routes page generation to agent orchestration', () => {
|
test('classifyWithRules routes page generation to agent orchestration', () => {
|
||||||
const result = classifyWithRules({
|
const result = classifyWithRules({
|
||||||
text: '帮我做一个秋夜诗的 H5 页面',
|
text: '帮我做一个秋夜诗的 H5 页面',
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"id": "token-benchmark-chat",
|
||||||
|
"name": "Token 基准:Direct Chat 三轮",
|
||||||
|
"description": "简单对话,不走做页面 Agent,对比 pro vs flash",
|
||||||
|
"account": {
|
||||||
|
"username": "john2",
|
||||||
|
"password": "888888"
|
||||||
|
},
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"action": "chat",
|
||||||
|
"label": "打招呼",
|
||||||
|
"message": "hi",
|
||||||
|
"expect": {
|
||||||
|
"assistantMinChars": 1,
|
||||||
|
"timeoutMs": 120000
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "chat",
|
||||||
|
"label": "苏州简介",
|
||||||
|
"message": "用三句话介绍苏州园林,不要生成页面",
|
||||||
|
"expect": {
|
||||||
|
"assistantMinChars": 20,
|
||||||
|
"timeoutMs": 120000,
|
||||||
|
"replyKeywords": ["苏州"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "chat",
|
||||||
|
"label": "追问",
|
||||||
|
"message": "推荐一个最适合第一次去的,一句话说明理由",
|
||||||
|
"expect": {
|
||||||
|
"assistantMinChars": 10,
|
||||||
|
"timeoutMs": 120000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"id": "token-benchmark-page",
|
||||||
|
"name": "Token 基准:独立苏州页面",
|
||||||
|
"description": "每次使用唯一文件名,避免复用已有 HTML 污染上下文",
|
||||||
|
"account": {
|
||||||
|
"username": "john2",
|
||||||
|
"password": "888888"
|
||||||
|
},
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"action": "chat",
|
||||||
|
"label": "生成独立苏州页面",
|
||||||
|
"message": "请帮我做一个全新的苏州一日游攻略页面,保存为 public/token-bench-{{PROFILE}}-{{RUN_ID}}.html,不要修改或复用已有页面,做完直接给我链接。",
|
||||||
|
"expect": {
|
||||||
|
"assistantMinChars": 20,
|
||||||
|
"timeoutMs": 600000,
|
||||||
|
"replyKeywords": ["苏州"],
|
||||||
|
"page": {
|
||||||
|
"keywords": ["苏州"],
|
||||||
|
"requirePublicLink": true,
|
||||||
|
"requireHttp200": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Executable
+542
@@ -0,0 +1,542 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Local A/B benchmark: DeepSeek token usage vs scenario quality.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node scripts/benchmark-token-optimization.mjs
|
||||||
|
* node scripts/benchmark-token-optimization.mjs --scenario john2-suzhou-page
|
||||||
|
* node scripts/benchmark-token-optimization.mjs --phase baseline
|
||||||
|
* node scripts/benchmark-token-optimization.mjs --phase optimized
|
||||||
|
* node scripts/benchmark-token-optimization.mjs --restore
|
||||||
|
*
|
||||||
|
* Does NOT touch production. Restores model + memory admin config on exit.
|
||||||
|
*/
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { createDbPool } from '../db.mjs';
|
||||||
|
import { createLlmProviderService } from '../llm-providers.mjs';
|
||||||
|
import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs';
|
||||||
|
import { loadH5Environment } from './load-env.mjs';
|
||||||
|
import {
|
||||||
|
createReporter,
|
||||||
|
createAgentRun,
|
||||||
|
extractAssistantTexts,
|
||||||
|
extractPublicLinks,
|
||||||
|
getSession,
|
||||||
|
loadScenario,
|
||||||
|
loginViaApi,
|
||||||
|
resolvePortalBase,
|
||||||
|
snapshotPublicHtml,
|
||||||
|
verifyPageAccess,
|
||||||
|
waitForAssistantGrowth,
|
||||||
|
waitForRunTerminal,
|
||||||
|
} from './scenario-test-lib.mjs';
|
||||||
|
|
||||||
|
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||||
|
const root = path.join(scriptDir, '..');
|
||||||
|
const statePath = path.join(root, '.token-benchmark-state.json');
|
||||||
|
|
||||||
|
loadH5Environment(scriptDir);
|
||||||
|
|
||||||
|
const PROFILES = {
|
||||||
|
baseline: {
|
||||||
|
label: 'baseline (deepseek-v4-pro + 默认记忆预算)',
|
||||||
|
model: 'deepseek-v4-pro',
|
||||||
|
memoryPatch: null,
|
||||||
|
},
|
||||||
|
optimized: {
|
||||||
|
label: 'optimized (deepseek-v4-flash + 收紧记忆/Router)',
|
||||||
|
model: 'deepseek-v4-flash',
|
||||||
|
memoryPatch: {
|
||||||
|
runtimeControl: {
|
||||||
|
agentResolveLimit: '3',
|
||||||
|
},
|
||||||
|
retriever: {
|
||||||
|
limit: '4',
|
||||||
|
tokenBudget: '900',
|
||||||
|
},
|
||||||
|
persona: {
|
||||||
|
maxTokens: '300',
|
||||||
|
},
|
||||||
|
chatIntentRouter: {
|
||||||
|
enabled: true,
|
||||||
|
shadowMode: false,
|
||||||
|
model: 'deepseek-chat',
|
||||||
|
memoryResolveLimit: '2',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
let scenarioId = 'token-benchmark-page';
|
||||||
|
let phase = 'both';
|
||||||
|
let port = Number(process.env.H5_PORT ?? 8081);
|
||||||
|
for (let i = 2; i < argv.length; i += 1) {
|
||||||
|
const arg = argv[i];
|
||||||
|
if (arg === '--scenario' && argv[i + 1]) scenarioId = argv[++i];
|
||||||
|
else if (arg === '--phase' && argv[i + 1]) phase = argv[++i];
|
||||||
|
else if (arg === '--port' && argv[i + 1]) port = Number(argv[++i]);
|
||||||
|
else if (arg === '--restore') phase = 'restore';
|
||||||
|
else if (arg === '-h' || arg === '--help') {
|
||||||
|
console.log(`Usage:
|
||||||
|
node scripts/benchmark-token-optimization.mjs [--scenario <id>] [--phase baseline|optimized|both|restore]`);
|
||||||
|
process.exit(0);
|
||||||
|
} else {
|
||||||
|
throw new Error(`Unknown argument: ${arg}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!['baseline', 'optimized', 'both', 'restore'].includes(phase)) {
|
||||||
|
throw new Error(`Invalid --phase: ${phase}`);
|
||||||
|
}
|
||||||
|
return { scenarioId, phase, port };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensurePortalReady(baseUrl) {
|
||||||
|
const response = await fetch(`${baseUrl}/auth/status`);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`Portal 未就绪: ${baseUrl}/auth/status -> ${response.status}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readCurrentModel(pool) {
|
||||||
|
const [rows] = await pool.query(
|
||||||
|
`SELECT default_model FROM h5_llm_provider_keys WHERE is_selected = 1 AND status = 'active' LIMIT 1`,
|
||||||
|
);
|
||||||
|
return rows[0]?.default_model ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveState(pool, memoryConfigService) {
|
||||||
|
const model = await readCurrentModel(pool);
|
||||||
|
const memoryAdmin = await memoryConfigService.getAdminConfig().catch(() => null);
|
||||||
|
const payload = {
|
||||||
|
savedAt: Date.now(),
|
||||||
|
model,
|
||||||
|
memoryConfig: memoryAdmin?.config ?? null,
|
||||||
|
};
|
||||||
|
fs.writeFileSync(statePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadSavedState() {
|
||||||
|
if (!fs.existsSync(statePath)) return null;
|
||||||
|
return JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyProfile(pool, llmProviderService, memoryConfigService, profileKey) {
|
||||||
|
const profile = PROFILES[profileKey];
|
||||||
|
if (!profile) throw new Error(`Unknown profile: ${profileKey}`);
|
||||||
|
|
||||||
|
const modelResult = await llmProviderService.setGlobalModel(profile.model);
|
||||||
|
if (!modelResult?.ok) {
|
||||||
|
throw new Error(`切换模型失败 (${profile.model}): ${modelResult?.message ?? 'unknown'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (profile.memoryPatch) {
|
||||||
|
await memoryConfigService.updateAdminConfig(profile.memoryPatch, {
|
||||||
|
updatedBy: 'token-benchmark',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentModel = await readCurrentModel(pool);
|
||||||
|
console.log(`\n==> 已应用 ${profile.label}`);
|
||||||
|
console.log(` 全局模型: ${currentModel}`);
|
||||||
|
if (profile.memoryPatch) {
|
||||||
|
console.log(' 记忆/Router 预算: 已写入 h5_memory_v2_admin_config');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restoreSavedState(pool, llmProviderService, memoryConfigService) {
|
||||||
|
const saved = loadSavedState();
|
||||||
|
if (!saved) {
|
||||||
|
console.log('无 .token-benchmark-state.json,跳过恢复');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (saved.model) {
|
||||||
|
const result = await llmProviderService.setGlobalModel(saved.model);
|
||||||
|
if (!result?.ok) {
|
||||||
|
console.warn(`恢复模型失败: ${result?.message ?? 'unknown'}`);
|
||||||
|
} else {
|
||||||
|
console.log(`已恢复全局模型: ${saved.model}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (saved.memoryConfig) {
|
||||||
|
await memoryConfigService.updateAdminConfig(saved.memoryConfig, {
|
||||||
|
updatedBy: 'token-benchmark-restore',
|
||||||
|
});
|
||||||
|
console.log('已恢复 memory v2 admin config');
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.unlinkSync(statePath);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function collectUsageMetrics(pool, requestIds) {
|
||||||
|
if (!requestIds.length) {
|
||||||
|
return { inputTokens: 0, outputTokens: 0, costCents: 0, records: [] };
|
||||||
|
}
|
||||||
|
const placeholders = requestIds.map(() => '?').join(', ');
|
||||||
|
const [rows] = await pool.query(
|
||||||
|
`SELECT request_id, input_tokens, output_tokens, cost_cents, billing_source, created_at
|
||||||
|
FROM h5_usage_records
|
||||||
|
WHERE request_id IN (${placeholders})
|
||||||
|
ORDER BY created_at ASC`,
|
||||||
|
requestIds,
|
||||||
|
);
|
||||||
|
let inputTokens = 0;
|
||||||
|
let outputTokens = 0;
|
||||||
|
let costCents = 0;
|
||||||
|
for (const row of rows) {
|
||||||
|
inputTokens += Number(row.input_tokens ?? 0);
|
||||||
|
outputTokens += Number(row.output_tokens ?? 0);
|
||||||
|
costCents += Number(row.cost_cents ?? 0);
|
||||||
|
}
|
||||||
|
return { inputTokens, outputTokens, costCents, records: rows };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function collectRunEventTokens(pool, runIds) {
|
||||||
|
if (!runIds.length) return { inputTokens: 0, outputTokens: 0, events: [] };
|
||||||
|
const placeholders = runIds.map(() => '?').join(', ');
|
||||||
|
const [rows] = await pool.query(
|
||||||
|
`SELECT run_id, data_json, created_at
|
||||||
|
FROM h5_agent_run_events
|
||||||
|
WHERE run_id IN (${placeholders}) AND event_type = 'session_finished'
|
||||||
|
ORDER BY created_at ASC`,
|
||||||
|
runIds,
|
||||||
|
);
|
||||||
|
let inputTokens = 0;
|
||||||
|
let outputTokens = 0;
|
||||||
|
const events = [];
|
||||||
|
for (const row of rows) {
|
||||||
|
const data = typeof row.data_json === 'string'
|
||||||
|
? JSON.parse(row.data_json)
|
||||||
|
: row.data_json;
|
||||||
|
const tokenState = data?.tokenState ?? data?.token_state ?? null;
|
||||||
|
if (!tokenState) continue;
|
||||||
|
const input = Number(
|
||||||
|
tokenState.accumulatedInputTokens
|
||||||
|
?? tokenState.accumulated_input_tokens
|
||||||
|
?? tokenState.inputTokens
|
||||||
|
?? tokenState.input_tokens
|
||||||
|
?? 0,
|
||||||
|
);
|
||||||
|
const output = Number(
|
||||||
|
tokenState.accumulatedOutputTokens
|
||||||
|
?? tokenState.accumulated_output_tokens
|
||||||
|
?? tokenState.outputTokens
|
||||||
|
?? tokenState.output_tokens
|
||||||
|
?? 0,
|
||||||
|
);
|
||||||
|
inputTokens += input;
|
||||||
|
outputTokens += output;
|
||||||
|
events.push({ runId: row.run_id, input, output, tokenState });
|
||||||
|
}
|
||||||
|
return { inputTokens, outputTokens, events };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runScenarioBenchmark(scenario, port, { profileKey = 'run', runId = Date.now() } = {}) {
|
||||||
|
const reporter = createReporter();
|
||||||
|
const baseUrl = resolvePortalBase(port);
|
||||||
|
const account = {
|
||||||
|
username: scenario.account?.username ?? 'john2',
|
||||||
|
password: process.env.JOHN_PASSWORD
|
||||||
|
?? process.env.H5_ACCESS_PASSWORD
|
||||||
|
?? scenario.account?.password
|
||||||
|
?? '888888',
|
||||||
|
};
|
||||||
|
|
||||||
|
await ensurePortalReady(baseUrl);
|
||||||
|
|
||||||
|
const auth = await loginViaApi(baseUrl, account, reporter);
|
||||||
|
const publishKey = auth.user?.id ?? auth.user?.publishSlug ?? account.username;
|
||||||
|
|
||||||
|
let sessionId = null;
|
||||||
|
let assistantCount = 0;
|
||||||
|
let assistantCombinedLength = 0;
|
||||||
|
const tracked = {
|
||||||
|
requestIds: [],
|
||||||
|
runIds: [],
|
||||||
|
sessionIds: [],
|
||||||
|
replies: [],
|
||||||
|
pageLinks: [],
|
||||||
|
elapsedMs: 0,
|
||||||
|
profileKey,
|
||||||
|
runId,
|
||||||
|
};
|
||||||
|
const started = Date.now();
|
||||||
|
|
||||||
|
for (const step of scenario.steps ?? []) {
|
||||||
|
if (step.action !== 'chat') continue;
|
||||||
|
|
||||||
|
const message = String(step.message ?? '')
|
||||||
|
.replaceAll('{{PROFILE}}', profileKey)
|
||||||
|
.replaceAll('{{RUN_ID}}', String(runId));
|
||||||
|
|
||||||
|
let htmlBefore = [];
|
||||||
|
if (step.expect?.page) {
|
||||||
|
htmlBefore = await snapshotPublicHtml(publishKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = await createAgentRun(baseUrl, auth.cookie, {
|
||||||
|
message,
|
||||||
|
sessionId: null,
|
||||||
|
selectedChatSkill: step.selectedChatSkill ?? null,
|
||||||
|
});
|
||||||
|
tracked.requestIds.push(run.requestId);
|
||||||
|
tracked.runIds.push(run.runId);
|
||||||
|
|
||||||
|
const terminal = await waitForRunTerminal(
|
||||||
|
baseUrl,
|
||||||
|
auth.cookie,
|
||||||
|
run.runId,
|
||||||
|
step.expect?.timeoutMs ?? 600_000,
|
||||||
|
);
|
||||||
|
sessionId = terminal.sessionId ?? terminal.agent_session_id ?? run.sessionId ?? sessionId;
|
||||||
|
if (sessionId) tracked.sessionIds.push(sessionId);
|
||||||
|
|
||||||
|
if (terminal.status === 'failed') {
|
||||||
|
reporter.fail('run 终态', terminal.error ?? 'failed');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
reporter.pass('run 终态', terminal.status);
|
||||||
|
|
||||||
|
const reply = await waitForAssistantGrowth(baseUrl, auth.cookie, sessionId, {
|
||||||
|
previousCount: assistantCount,
|
||||||
|
previousCombinedLength: assistantCombinedLength,
|
||||||
|
minChars: step.expect?.assistantMinChars ?? 1,
|
||||||
|
timeoutMs: step.expect?.timeoutMs ?? 120_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!reply) {
|
||||||
|
reporter.fail('assistant 回复', '超时未收到新回复');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
assistantCount = reply.count;
|
||||||
|
assistantCombinedLength = reply.combined.length;
|
||||||
|
tracked.replies.push({
|
||||||
|
label: step.label ?? step.message,
|
||||||
|
chars: reply.combined.length,
|
||||||
|
elapsedMs: reply.elapsedMs,
|
||||||
|
text: reply.combined,
|
||||||
|
});
|
||||||
|
reporter.pass('assistant 回复', `${reply.combined.length} 字 / ${reply.elapsedMs}ms`);
|
||||||
|
|
||||||
|
const keywords = step.expect?.replyKeywords ?? [];
|
||||||
|
if (keywords.length) {
|
||||||
|
const hit = keywords.filter((word) => reply.combined.includes(word));
|
||||||
|
if (hit.length === 0) reporter.fail('回复关键词', `未命中: ${keywords.join(', ')}`);
|
||||||
|
else reporter.pass('回复关键词', hit.join(', '));
|
||||||
|
}
|
||||||
|
|
||||||
|
tracked.pageLinks.push(...extractPublicLinks(reply.combined, baseUrl));
|
||||||
|
|
||||||
|
if (step.expect?.page) {
|
||||||
|
await verifyPageAccess({
|
||||||
|
baseUrl,
|
||||||
|
cookie: auth.cookie,
|
||||||
|
publishKey,
|
||||||
|
replyText: reply.combined,
|
||||||
|
htmlBefore,
|
||||||
|
expect: step.expect.page,
|
||||||
|
uploadedAssetIds: [],
|
||||||
|
reporter,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tracked.elapsedMs = Date.now() - started;
|
||||||
|
return {
|
||||||
|
reporter,
|
||||||
|
tracked,
|
||||||
|
passed: reporter.issues.length === 0,
|
||||||
|
issueCount: reporter.issues.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pctDelta(before, after) {
|
||||||
|
if (!before) return after ? 100 : 0;
|
||||||
|
return Math.round(((after - before) / before) * 1000) / 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
function printComparison(baseline, optimized) {
|
||||||
|
console.log('\n========================================');
|
||||||
|
console.log('Token 优化 A/B 对比(本地)');
|
||||||
|
console.log('========================================\n');
|
||||||
|
|
||||||
|
const rows = [
|
||||||
|
['指标', 'Baseline', 'Optimized', '变化'],
|
||||||
|
[
|
||||||
|
'输入 Token (usage)',
|
||||||
|
String(baseline.usage.inputTokens),
|
||||||
|
String(optimized.usage.inputTokens),
|
||||||
|
`${pctDelta(baseline.usage.inputTokens, optimized.usage.inputTokens)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'输出 Token (usage)',
|
||||||
|
String(baseline.usage.outputTokens),
|
||||||
|
String(optimized.usage.outputTokens),
|
||||||
|
`${pctDelta(baseline.usage.outputTokens, optimized.usage.outputTokens)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'总 Token (usage)',
|
||||||
|
String(baseline.usage.inputTokens + baseline.usage.outputTokens),
|
||||||
|
String(optimized.usage.inputTokens + optimized.usage.outputTokens),
|
||||||
|
`${pctDelta(
|
||||||
|
baseline.usage.inputTokens + baseline.usage.outputTokens,
|
||||||
|
optimized.usage.inputTokens + optimized.usage.outputTokens,
|
||||||
|
)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'扣费 (分)',
|
||||||
|
String(baseline.usage.costCents),
|
||||||
|
String(optimized.usage.costCents),
|
||||||
|
`${pctDelta(baseline.usage.costCents, optimized.usage.costCents)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'场景耗时 (s)',
|
||||||
|
String(Math.round(baseline.tracked.elapsedMs / 1000)),
|
||||||
|
String(Math.round(optimized.tracked.elapsedMs / 1000)),
|
||||||
|
`${pctDelta(baseline.tracked.elapsedMs, optimized.tracked.elapsedMs)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'回复字数',
|
||||||
|
String(baseline.tracked.replies.at(-1)?.chars ?? 0),
|
||||||
|
String(optimized.tracked.replies.at(-1)?.chars ?? 0),
|
||||||
|
`${pctDelta(
|
||||||
|
baseline.tracked.replies.at(-1)?.chars ?? 0,
|
||||||
|
optimized.tracked.replies.at(-1)?.chars ?? 0,
|
||||||
|
)}%`,
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'质量检查',
|
||||||
|
baseline.passed ? '通过' : `失败 ${baseline.issueCount} 项`,
|
||||||
|
optimized.passed ? '通过' : `失败 ${optimized.issueCount} 项`,
|
||||||
|
optimized.passed === baseline.passed ? '持平' : (optimized.passed ? '改善' : '下降'),
|
||||||
|
],
|
||||||
|
];
|
||||||
|
|
||||||
|
const widths = [0, 1, 2, 3].map((col) => Math.max(...rows.map((row) => row[col].length)));
|
||||||
|
for (const row of rows) {
|
||||||
|
console.log(row.map((cell, i) => cell.padEnd(widths[i] + 2)).join(''));
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n--- Baseline 页面链接 ---');
|
||||||
|
for (const link of baseline.tracked.pageLinks) console.log(link);
|
||||||
|
console.log('\n--- Optimized 页面链接 ---');
|
||||||
|
for (const link of optimized.tracked.pageLinks) console.log(link);
|
||||||
|
|
||||||
|
if (baseline.passed && optimized.passed) {
|
||||||
|
const totalBefore = baseline.usage.inputTokens + baseline.usage.outputTokens;
|
||||||
|
const totalAfter = optimized.usage.inputTokens + optimized.usage.outputTokens;
|
||||||
|
const saved = totalBefore - totalAfter;
|
||||||
|
console.log(`\n结论: 质量检查均通过。Token 总量 ${totalBefore} → ${totalAfter}(${saved >= 0 ? '节省' : '增加'} ${Math.abs(saved)})`);
|
||||||
|
if (saved > 0 && optimized.passed) {
|
||||||
|
console.log('效果未打折,可继续观察更多场景。');
|
||||||
|
} else if (saved <= 0) {
|
||||||
|
console.log('Token 未下降,需检查模型/Router/记忆配置是否生效。');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log('\n结论: 存在质量回归,不建议直接上生产。');
|
||||||
|
if (!optimized.passed) {
|
||||||
|
console.log('Optimized 失败项:');
|
||||||
|
for (const issue of optimized.reporter.issues) {
|
||||||
|
console.log(` - ${issue.label}: ${issue.detail}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runPhase(pool, llmProviderService, memoryConfigService, profileKey, scenario, port) {
|
||||||
|
await applyProfile(pool, llmProviderService, memoryConfigService, profileKey);
|
||||||
|
const runId = `${profileKey}-${Date.now()}`;
|
||||||
|
console.log(`\n>>> 开始 ${profileKey} 场景跑分 (runId=${runId})...`);
|
||||||
|
const result = await runScenarioBenchmark(scenario, port, { profileKey, runId });
|
||||||
|
const usage = await collectUsageMetrics(pool, result.tracked.requestIds);
|
||||||
|
const eventTokens = await collectRunEventTokens(pool, result.tracked.runIds);
|
||||||
|
console.log(`\n--- ${profileKey} token 汇总 ---`);
|
||||||
|
console.log(`usage records: in=${usage.inputTokens} out=${usage.outputTokens} cost=${usage.costCents}分`);
|
||||||
|
console.log(`run events: in=${eventTokens.inputTokens} out=${eventTokens.outputTokens}`);
|
||||||
|
return { ...result, usage, eventTokens, profileKey };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const { scenarioId, phase, port } = parseArgs(process.argv);
|
||||||
|
const pool = createDbPool(process.env);
|
||||||
|
const llmProviderService = createLlmProviderService(pool, {
|
||||||
|
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
|
||||||
|
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||||
|
});
|
||||||
|
const memoryConfigService = createMemoryV2AdminConfigService(pool);
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (phase === 'restore') {
|
||||||
|
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const scenario = await loadScenario(scenarioId);
|
||||||
|
console.log(`场景: ${scenario.name ?? scenario.id}`);
|
||||||
|
console.log(`Portal: ${resolvePortalBase(port)}`);
|
||||||
|
|
||||||
|
if (phase === 'both') {
|
||||||
|
await saveState(pool, memoryConfigService);
|
||||||
|
}
|
||||||
|
|
||||||
|
let baselineResult = null;
|
||||||
|
let optimizedResult = null;
|
||||||
|
|
||||||
|
if (phase === 'both' || phase === 'baseline') {
|
||||||
|
baselineResult = await runPhase(
|
||||||
|
pool,
|
||||||
|
llmProviderService,
|
||||||
|
memoryConfigService,
|
||||||
|
'baseline',
|
||||||
|
scenario,
|
||||||
|
port,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (phase === 'both' || phase === 'optimized') {
|
||||||
|
optimizedResult = await runPhase(
|
||||||
|
pool,
|
||||||
|
llmProviderService,
|
||||||
|
memoryConfigService,
|
||||||
|
'optimized',
|
||||||
|
scenario,
|
||||||
|
port,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (baselineResult && optimizedResult) {
|
||||||
|
printComparison(baselineResult, optimizedResult);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (phase === 'both') {
|
||||||
|
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||||
|
console.log('\n已恢复 benchmark 前的模型与 memory 配置。');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await pool.end?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(async (error) => {
|
||||||
|
console.error(error instanceof Error ? error.stack ?? error.message : error);
|
||||||
|
try {
|
||||||
|
const pool = createDbPool(process.env);
|
||||||
|
const llmProviderService = createLlmProviderService(pool, {
|
||||||
|
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
|
||||||
|
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||||
|
});
|
||||||
|
const memoryConfigService = createMemoryV2AdminConfigService(pool);
|
||||||
|
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||||
|
await pool.end?.();
|
||||||
|
} catch {
|
||||||
|
// best effort restore
|
||||||
|
}
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
Executable
+76
@@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Sync token-optimization Router / memory env to 103 Portal (.env only).
|
||||||
|
# Does NOT change global LLM model (keep deepseek-v4-pro for Agent quality).
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# bash scripts/sync-token-optimization-env-103.sh # dry-run
|
||||||
|
# bash scripts/sync-token-optimization-env-103.sh --apply # write + restart portal
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
HOST="${MEMIND_103_HOST:-john@58.38.22.103}"
|
||||||
|
REMOTE_ENV="/Users/john/Project/Memind/.env"
|
||||||
|
APPLY=0
|
||||||
|
if [[ "${1:-}" == "--apply" ]]; then
|
||||||
|
APPLY=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PATCH_LINES=(
|
||||||
|
"MEMIND_CHAT_LLM_ROUTER_ENABLED=1"
|
||||||
|
"MEMIND_CHAT_LLM_ROUTER_SHADOW=0"
|
||||||
|
"MEMIND_CHAT_ROUTER_CANARY_USER_IDS="
|
||||||
|
"MEMIND_CHAT_ROUTER_MODEL=deepseek-chat"
|
||||||
|
"MEMIND_CHAT_ROUTER_MEMORY_LIMIT=2"
|
||||||
|
"MEMIND_CHAT_ROUTER_TIMEOUT_MS=2500"
|
||||||
|
"MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65"
|
||||||
|
"MEMORY_AGENT_RESOLVE_LIMIT=3"
|
||||||
|
"MEMORY_RETRIEVER_LIMIT=4"
|
||||||
|
"MEMORY_RETRIEVER_TOKEN_BUDGET=900"
|
||||||
|
"MEMORY_PERSONA_MAX_TOKENS=300"
|
||||||
|
)
|
||||||
|
|
||||||
|
echo "==> Target: ${HOST}:${REMOTE_ENV}"
|
||||||
|
echo "==> Patch:"
|
||||||
|
printf ' %s\n' "${PATCH_LINES[@]}"
|
||||||
|
echo
|
||||||
|
|
||||||
|
if [[ "${APPLY}" -ne 1 ]]; then
|
||||||
|
echo "Dry-run only. Re-run with --apply to write and restart Portal."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
PATCH_B64=$(printf '%s\n' "${PATCH_LINES[@]}" | base64 | tr -d '\n')
|
||||||
|
|
||||||
|
ssh "${HOST}" "bash -s" <<REMOTE
|
||||||
|
set -euo pipefail
|
||||||
|
ENV_FILE="${REMOTE_ENV}"
|
||||||
|
BACKUP="\${ENV_FILE}.bak-token-opt-\$(date +%Y%m%d-%H%M%S)"
|
||||||
|
cp "\${ENV_FILE}" "\${BACKUP}"
|
||||||
|
echo "Backed up to \${BACKUP}"
|
||||||
|
|
||||||
|
upsert() {
|
||||||
|
local key="\$1" val="\$2"
|
||||||
|
if grep -q "^\${key}=" "\${ENV_FILE}"; then
|
||||||
|
sed -i '' "s|^\${key}=.*|\${key}=\${val}|" "\${ENV_FILE}"
|
||||||
|
else
|
||||||
|
printf '\n%s=%s\n' "\${key}" "\${val}" >> "\${ENV_FILE}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
while IFS='=' read -r key val; do
|
||||||
|
[[ -z "\${key}" ]] && continue
|
||||||
|
upsert "\${key}" "\${val}"
|
||||||
|
done <<'ENVPATCH'
|
||||||
|
$(printf '%s\n' "${PATCH_LINES[@]}")
|
||||||
|
ENVPATCH
|
||||||
|
|
||||||
|
echo "==> Updated keys:"
|
||||||
|
grep -E '^(MEMIND_CHAT_|MEMORY_AGENT_RESOLVE|MEMORY_RETRIEVER|MEMORY_PERSONA_MAX)' "\${ENV_FILE}" || true
|
||||||
|
|
||||||
|
GUI="\$(/usr/bin/stat -f %u /dev/console)"
|
||||||
|
launchctl kickstart -k "gui/\${GUI}/cn.tkmind.memind-portal"
|
||||||
|
sleep 3
|
||||||
|
curl -sf http://127.0.0.1:8081/api/status >/dev/null
|
||||||
|
echo "Portal restarted and /api/status OK"
|
||||||
|
REMOTE
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
Reference in New Issue
Block a user