feat(router): improve direct_chat fast-path and add 103 token env sync
Memind CI / Test, build, and release guards (push) Failing after 12m44s
Memind CI / Test, build, and release guards (push) Failing after 12m44s
Route explicit text-only Q&A away from Agent before page-generation rules, and add a script to enable Router + tighter memory budgets on 103 without changing the global deepseek-v4-pro model. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -103,6 +103,25 @@ export const DIRECT_CHAT_FAQ_RULES = [
|
||||
/^.{0,16}(?:有什么建议|给点建议|怎么看)[??]?$/u,
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'text_only_declined_page',
|
||||
reason: '明确要求纯文字、不生成页面/链接',
|
||||
patterns: [
|
||||
/(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u,
|
||||
/(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u,
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'brief_intro',
|
||||
reason: '短篇幅介绍/说明(纯文字)',
|
||||
patterns: [
|
||||
/(?:用|以).{0,6}(?:一|两|二|三|四|五|几|\d+).{0,4}(?:句话|句).{0,32}(?:介绍|说明|描述|讲讲|推荐)/u,
|
||||
/^(?:简单|简要|大概|概括)(?:介绍|说明|描述|讲讲).{0,48}[??]?$/u,
|
||||
/^简要介绍.{0,48}(?:区别|差异|不同|对比|比较)[??]?$/u,
|
||||
/^(?:推荐|介绍)(?:一个|一下).{0,32}(?:理由|原因)[,,]?一句话/u,
|
||||
/(?:推荐|介绍).{0,32}一句话.{0,16}(?:说明|讲讲).{0,12}(?:理由|原因)/u,
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const MAX_FAQ_TEXT_LENGTH = 200;
|
||||
@@ -111,10 +130,26 @@ const MAX_FAQ_TEXT_LENGTH = 200;
|
||||
* @param {string} text
|
||||
* @returns {{ id: string, reason: string } | null}
|
||||
*/
|
||||
const EXPLICIT_TEXT_ONLY_PAGE_DECLINE = /(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u;
|
||||
|
||||
export function isExplicitTextOnlyRequest(text) {
|
||||
const normalized = String(text ?? '').trim();
|
||||
if (!normalized) return false;
|
||||
return EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|
||||
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
|
||||
}
|
||||
|
||||
export function matchDirectChatFaqRule(text) {
|
||||
const normalized = String(text ?? '').trim();
|
||||
if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null;
|
||||
if (FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))) return null;
|
||||
const declinedPage = EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|
||||
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
|
||||
if (
|
||||
!declinedPage
|
||||
&& FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
for (const rule of DIRECT_CHAT_FAQ_RULES) {
|
||||
if (rule.patterns.some((pattern) => pattern.test(normalized))) {
|
||||
return { id: rule.id, reason: rule.reason };
|
||||
|
||||
+20
-12
@@ -15,11 +15,11 @@ import {
|
||||
resolveMemoryInterventionMode,
|
||||
} from './memory-intervention.mjs';
|
||||
import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs';
|
||||
import { matchDirectChatFaqRule } from './chat-intent-router-rules.mjs';
|
||||
import { matchDirectChatFaqRule, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
|
||||
import { isGoalRunIntent } from './goal-run-intent.mjs';
|
||||
import { pgvectorMemoryBackendInternals } from './memory-v2-pgvector.mjs';
|
||||
|
||||
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS } from './chat-intent-router-rules.mjs';
|
||||
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
|
||||
|
||||
export const CHAT_INTENT_ROUTE = {
|
||||
DIRECT_CHAT: 'direct_chat',
|
||||
@@ -1083,6 +1083,14 @@ export function classifyWithRules({
|
||||
reason: '用户在询问个人记忆或历史对话',
|
||||
}, { source: 'rule' }), decisionContext);
|
||||
}
|
||||
if (includeIntentPatterns && normalized && isExplicitTextOnlyRequest(normalized)) {
|
||||
const faqMatch = matchDirectChatFaqRule(normalized);
|
||||
return finalizeRouterClassification(normalizeClassification({
|
||||
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
||||
confidence: 0.96,
|
||||
reason: faqMatch?.reason ?? '用户明确要求纯文字、不生成页面',
|
||||
}, { source: 'rule' }), decisionContext);
|
||||
}
|
||||
if (hasPriorAgentConversation(sessionId, sessionMessageCount)) {
|
||||
const reason = isAgentSessionContinueText(normalized)
|
||||
? 'Agent 会话确认/续聊'
|
||||
@@ -1093,6 +1101,16 @@ export function classifyWithRules({
|
||||
reason,
|
||||
}, { source: 'rule' }), decisionContext);
|
||||
}
|
||||
if (includeIntentPatterns && normalized) {
|
||||
const faqMatch = matchDirectChatFaqRule(normalized);
|
||||
if (faqMatch) {
|
||||
return finalizeRouterClassification(normalizeClassification({
|
||||
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
||||
confidence: 0.94,
|
||||
reason: faqMatch.reason,
|
||||
}, { source: 'rule' }), decisionContext);
|
||||
}
|
||||
}
|
||||
if (includeIntentPatterns && normalized && isPageDataDevIntent(normalized)) {
|
||||
return finalizeRouterClassification(normalizeClassification({
|
||||
route: CHAT_INTENT_ROUTE.AGENT,
|
||||
@@ -1142,16 +1160,6 @@ export function classifyWithRules({
|
||||
if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) {
|
||||
return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext);
|
||||
}
|
||||
if (includeIntentPatterns && normalized) {
|
||||
const faqMatch = matchDirectChatFaqRule(normalized);
|
||||
if (faqMatch) {
|
||||
return finalizeRouterClassification(normalizeClassification({
|
||||
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
|
||||
confidence: 0.94,
|
||||
reason: faqMatch.reason,
|
||||
}, { source: 'rule' }), decisionContext);
|
||||
}
|
||||
}
|
||||
if (
|
||||
includeIntentPatterns &&
|
||||
normalized &&
|
||||
|
||||
@@ -109,6 +109,25 @@ test('classifyWithRules routes greetings to direct chat', () => {
|
||||
assert.equal(result.source, 'rule');
|
||||
});
|
||||
|
||||
test('classifyWithRules routes explicit text-only Q&A to direct chat', () => {
|
||||
for (const text of [
|
||||
'用三句话介绍苏州园林,不要生成页面',
|
||||
'简要介绍一下 Rust 和 Go 的区别',
|
||||
'推荐一个最适合第一次去的,一句话说明理由',
|
||||
]) {
|
||||
const result = classifyWithRules({
|
||||
text,
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text }],
|
||||
metadata: { displayText: text },
|
||||
},
|
||||
});
|
||||
assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT, text);
|
||||
assert.equal(result.source, 'rule', text);
|
||||
}
|
||||
});
|
||||
|
||||
test('classifyWithRules routes page generation to agent orchestration', () => {
|
||||
const result = classifyWithRules({
|
||||
text: '帮我做一个秋夜诗的 H5 页面',
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"id": "token-benchmark-chat",
|
||||
"name": "Token 基准:Direct Chat 三轮",
|
||||
"description": "简单对话,不走做页面 Agent,对比 pro vs flash",
|
||||
"account": {
|
||||
"username": "john2",
|
||||
"password": "888888"
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"action": "chat",
|
||||
"label": "打招呼",
|
||||
"message": "hi",
|
||||
"expect": {
|
||||
"assistantMinChars": 1,
|
||||
"timeoutMs": 120000
|
||||
}
|
||||
},
|
||||
{
|
||||
"action": "chat",
|
||||
"label": "苏州简介",
|
||||
"message": "用三句话介绍苏州园林,不要生成页面",
|
||||
"expect": {
|
||||
"assistantMinChars": 20,
|
||||
"timeoutMs": 120000,
|
||||
"replyKeywords": ["苏州"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"action": "chat",
|
||||
"label": "追问",
|
||||
"message": "推荐一个最适合第一次去的,一句话说明理由",
|
||||
"expect": {
|
||||
"assistantMinChars": 10,
|
||||
"timeoutMs": 120000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"id": "token-benchmark-page",
|
||||
"name": "Token 基准:独立苏州页面",
|
||||
"description": "每次使用唯一文件名,避免复用已有 HTML 污染上下文",
|
||||
"account": {
|
||||
"username": "john2",
|
||||
"password": "888888"
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"action": "chat",
|
||||
"label": "生成独立苏州页面",
|
||||
"message": "请帮我做一个全新的苏州一日游攻略页面,保存为 public/token-bench-{{PROFILE}}-{{RUN_ID}}.html,不要修改或复用已有页面,做完直接给我链接。",
|
||||
"expect": {
|
||||
"assistantMinChars": 20,
|
||||
"timeoutMs": 600000,
|
||||
"replyKeywords": ["苏州"],
|
||||
"page": {
|
||||
"keywords": ["苏州"],
|
||||
"requirePublicLink": true,
|
||||
"requireHttp200": true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+542
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Local A/B benchmark: DeepSeek token usage vs scenario quality.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/benchmark-token-optimization.mjs
|
||||
* node scripts/benchmark-token-optimization.mjs --scenario john2-suzhou-page
|
||||
* node scripts/benchmark-token-optimization.mjs --phase baseline
|
||||
* node scripts/benchmark-token-optimization.mjs --phase optimized
|
||||
* node scripts/benchmark-token-optimization.mjs --restore
|
||||
*
|
||||
* Does NOT touch production. Restores model + memory admin config on exit.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { createDbPool } from '../db.mjs';
|
||||
import { createLlmProviderService } from '../llm-providers.mjs';
|
||||
import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs';
|
||||
import { loadH5Environment } from './load-env.mjs';
|
||||
import {
|
||||
createReporter,
|
||||
createAgentRun,
|
||||
extractAssistantTexts,
|
||||
extractPublicLinks,
|
||||
getSession,
|
||||
loadScenario,
|
||||
loginViaApi,
|
||||
resolvePortalBase,
|
||||
snapshotPublicHtml,
|
||||
verifyPageAccess,
|
||||
waitForAssistantGrowth,
|
||||
waitForRunTerminal,
|
||||
} from './scenario-test-lib.mjs';
|
||||
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.join(scriptDir, '..');
|
||||
const statePath = path.join(root, '.token-benchmark-state.json');
|
||||
|
||||
loadH5Environment(scriptDir);
|
||||
|
||||
const PROFILES = {
|
||||
baseline: {
|
||||
label: 'baseline (deepseek-v4-pro + 默认记忆预算)',
|
||||
model: 'deepseek-v4-pro',
|
||||
memoryPatch: null,
|
||||
},
|
||||
optimized: {
|
||||
label: 'optimized (deepseek-v4-flash + 收紧记忆/Router)',
|
||||
model: 'deepseek-v4-flash',
|
||||
memoryPatch: {
|
||||
runtimeControl: {
|
||||
agentResolveLimit: '3',
|
||||
},
|
||||
retriever: {
|
||||
limit: '4',
|
||||
tokenBudget: '900',
|
||||
},
|
||||
persona: {
|
||||
maxTokens: '300',
|
||||
},
|
||||
chatIntentRouter: {
|
||||
enabled: true,
|
||||
shadowMode: false,
|
||||
model: 'deepseek-chat',
|
||||
memoryResolveLimit: '2',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
let scenarioId = 'token-benchmark-page';
|
||||
let phase = 'both';
|
||||
let port = Number(process.env.H5_PORT ?? 8081);
|
||||
for (let i = 2; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (arg === '--scenario' && argv[i + 1]) scenarioId = argv[++i];
|
||||
else if (arg === '--phase' && argv[i + 1]) phase = argv[++i];
|
||||
else if (arg === '--port' && argv[i + 1]) port = Number(argv[++i]);
|
||||
else if (arg === '--restore') phase = 'restore';
|
||||
else if (arg === '-h' || arg === '--help') {
|
||||
console.log(`Usage:
|
||||
node scripts/benchmark-token-optimization.mjs [--scenario <id>] [--phase baseline|optimized|both|restore]`);
|
||||
process.exit(0);
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
if (!['baseline', 'optimized', 'both', 'restore'].includes(phase)) {
|
||||
throw new Error(`Invalid --phase: ${phase}`);
|
||||
}
|
||||
return { scenarioId, phase, port };
|
||||
}
|
||||
|
||||
async function ensurePortalReady(baseUrl) {
|
||||
const response = await fetch(`${baseUrl}/auth/status`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Portal 未就绪: ${baseUrl}/auth/status -> ${response.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function readCurrentModel(pool) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT default_model FROM h5_llm_provider_keys WHERE is_selected = 1 AND status = 'active' LIMIT 1`,
|
||||
);
|
||||
return rows[0]?.default_model ?? null;
|
||||
}
|
||||
|
||||
async function saveState(pool, memoryConfigService) {
|
||||
const model = await readCurrentModel(pool);
|
||||
const memoryAdmin = await memoryConfigService.getAdminConfig().catch(() => null);
|
||||
const payload = {
|
||||
savedAt: Date.now(),
|
||||
model,
|
||||
memoryConfig: memoryAdmin?.config ?? null,
|
||||
};
|
||||
fs.writeFileSync(statePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||||
return payload;
|
||||
}
|
||||
|
||||
function loadSavedState() {
|
||||
if (!fs.existsSync(statePath)) return null;
|
||||
return JSON.parse(fs.readFileSync(statePath, 'utf8'));
|
||||
}
|
||||
|
||||
async function applyProfile(pool, llmProviderService, memoryConfigService, profileKey) {
|
||||
const profile = PROFILES[profileKey];
|
||||
if (!profile) throw new Error(`Unknown profile: ${profileKey}`);
|
||||
|
||||
const modelResult = await llmProviderService.setGlobalModel(profile.model);
|
||||
if (!modelResult?.ok) {
|
||||
throw new Error(`切换模型失败 (${profile.model}): ${modelResult?.message ?? 'unknown'}`);
|
||||
}
|
||||
|
||||
if (profile.memoryPatch) {
|
||||
await memoryConfigService.updateAdminConfig(profile.memoryPatch, {
|
||||
updatedBy: 'token-benchmark',
|
||||
});
|
||||
}
|
||||
|
||||
const currentModel = await readCurrentModel(pool);
|
||||
console.log(`\n==> 已应用 ${profile.label}`);
|
||||
console.log(` 全局模型: ${currentModel}`);
|
||||
if (profile.memoryPatch) {
|
||||
console.log(' 记忆/Router 预算: 已写入 h5_memory_v2_admin_config');
|
||||
}
|
||||
}
|
||||
|
||||
async function restoreSavedState(pool, llmProviderService, memoryConfigService) {
|
||||
const saved = loadSavedState();
|
||||
if (!saved) {
|
||||
console.log('无 .token-benchmark-state.json,跳过恢复');
|
||||
return false;
|
||||
}
|
||||
|
||||
if (saved.model) {
|
||||
const result = await llmProviderService.setGlobalModel(saved.model);
|
||||
if (!result?.ok) {
|
||||
console.warn(`恢复模型失败: ${result?.message ?? 'unknown'}`);
|
||||
} else {
|
||||
console.log(`已恢复全局模型: ${saved.model}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (saved.memoryConfig) {
|
||||
await memoryConfigService.updateAdminConfig(saved.memoryConfig, {
|
||||
updatedBy: 'token-benchmark-restore',
|
||||
});
|
||||
console.log('已恢复 memory v2 admin config');
|
||||
}
|
||||
|
||||
fs.unlinkSync(statePath);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function collectUsageMetrics(pool, requestIds) {
|
||||
if (!requestIds.length) {
|
||||
return { inputTokens: 0, outputTokens: 0, costCents: 0, records: [] };
|
||||
}
|
||||
const placeholders = requestIds.map(() => '?').join(', ');
|
||||
const [rows] = await pool.query(
|
||||
`SELECT request_id, input_tokens, output_tokens, cost_cents, billing_source, created_at
|
||||
FROM h5_usage_records
|
||||
WHERE request_id IN (${placeholders})
|
||||
ORDER BY created_at ASC`,
|
||||
requestIds,
|
||||
);
|
||||
let inputTokens = 0;
|
||||
let outputTokens = 0;
|
||||
let costCents = 0;
|
||||
for (const row of rows) {
|
||||
inputTokens += Number(row.input_tokens ?? 0);
|
||||
outputTokens += Number(row.output_tokens ?? 0);
|
||||
costCents += Number(row.cost_cents ?? 0);
|
||||
}
|
||||
return { inputTokens, outputTokens, costCents, records: rows };
|
||||
}
|
||||
|
||||
async function collectRunEventTokens(pool, runIds) {
|
||||
if (!runIds.length) return { inputTokens: 0, outputTokens: 0, events: [] };
|
||||
const placeholders = runIds.map(() => '?').join(', ');
|
||||
const [rows] = await pool.query(
|
||||
`SELECT run_id, data_json, created_at
|
||||
FROM h5_agent_run_events
|
||||
WHERE run_id IN (${placeholders}) AND event_type = 'session_finished'
|
||||
ORDER BY created_at ASC`,
|
||||
runIds,
|
||||
);
|
||||
let inputTokens = 0;
|
||||
let outputTokens = 0;
|
||||
const events = [];
|
||||
for (const row of rows) {
|
||||
const data = typeof row.data_json === 'string'
|
||||
? JSON.parse(row.data_json)
|
||||
: row.data_json;
|
||||
const tokenState = data?.tokenState ?? data?.token_state ?? null;
|
||||
if (!tokenState) continue;
|
||||
const input = Number(
|
||||
tokenState.accumulatedInputTokens
|
||||
?? tokenState.accumulated_input_tokens
|
||||
?? tokenState.inputTokens
|
||||
?? tokenState.input_tokens
|
||||
?? 0,
|
||||
);
|
||||
const output = Number(
|
||||
tokenState.accumulatedOutputTokens
|
||||
?? tokenState.accumulated_output_tokens
|
||||
?? tokenState.outputTokens
|
||||
?? tokenState.output_tokens
|
||||
?? 0,
|
||||
);
|
||||
inputTokens += input;
|
||||
outputTokens += output;
|
||||
events.push({ runId: row.run_id, input, output, tokenState });
|
||||
}
|
||||
return { inputTokens, outputTokens, events };
|
||||
}
|
||||
|
||||
async function runScenarioBenchmark(scenario, port, { profileKey = 'run', runId = Date.now() } = {}) {
|
||||
const reporter = createReporter();
|
||||
const baseUrl = resolvePortalBase(port);
|
||||
const account = {
|
||||
username: scenario.account?.username ?? 'john2',
|
||||
password: process.env.JOHN_PASSWORD
|
||||
?? process.env.H5_ACCESS_PASSWORD
|
||||
?? scenario.account?.password
|
||||
?? '888888',
|
||||
};
|
||||
|
||||
await ensurePortalReady(baseUrl);
|
||||
|
||||
const auth = await loginViaApi(baseUrl, account, reporter);
|
||||
const publishKey = auth.user?.id ?? auth.user?.publishSlug ?? account.username;
|
||||
|
||||
let sessionId = null;
|
||||
let assistantCount = 0;
|
||||
let assistantCombinedLength = 0;
|
||||
const tracked = {
|
||||
requestIds: [],
|
||||
runIds: [],
|
||||
sessionIds: [],
|
||||
replies: [],
|
||||
pageLinks: [],
|
||||
elapsedMs: 0,
|
||||
profileKey,
|
||||
runId,
|
||||
};
|
||||
const started = Date.now();
|
||||
|
||||
for (const step of scenario.steps ?? []) {
|
||||
if (step.action !== 'chat') continue;
|
||||
|
||||
const message = String(step.message ?? '')
|
||||
.replaceAll('{{PROFILE}}', profileKey)
|
||||
.replaceAll('{{RUN_ID}}', String(runId));
|
||||
|
||||
let htmlBefore = [];
|
||||
if (step.expect?.page) {
|
||||
htmlBefore = await snapshotPublicHtml(publishKey);
|
||||
}
|
||||
|
||||
const run = await createAgentRun(baseUrl, auth.cookie, {
|
||||
message,
|
||||
sessionId: null,
|
||||
selectedChatSkill: step.selectedChatSkill ?? null,
|
||||
});
|
||||
tracked.requestIds.push(run.requestId);
|
||||
tracked.runIds.push(run.runId);
|
||||
|
||||
const terminal = await waitForRunTerminal(
|
||||
baseUrl,
|
||||
auth.cookie,
|
||||
run.runId,
|
||||
step.expect?.timeoutMs ?? 600_000,
|
||||
);
|
||||
sessionId = terminal.sessionId ?? terminal.agent_session_id ?? run.sessionId ?? sessionId;
|
||||
if (sessionId) tracked.sessionIds.push(sessionId);
|
||||
|
||||
if (terminal.status === 'failed') {
|
||||
reporter.fail('run 终态', terminal.error ?? 'failed');
|
||||
continue;
|
||||
}
|
||||
reporter.pass('run 终态', terminal.status);
|
||||
|
||||
const reply = await waitForAssistantGrowth(baseUrl, auth.cookie, sessionId, {
|
||||
previousCount: assistantCount,
|
||||
previousCombinedLength: assistantCombinedLength,
|
||||
minChars: step.expect?.assistantMinChars ?? 1,
|
||||
timeoutMs: step.expect?.timeoutMs ?? 120_000,
|
||||
});
|
||||
|
||||
if (!reply) {
|
||||
reporter.fail('assistant 回复', '超时未收到新回复');
|
||||
continue;
|
||||
}
|
||||
|
||||
assistantCount = reply.count;
|
||||
assistantCombinedLength = reply.combined.length;
|
||||
tracked.replies.push({
|
||||
label: step.label ?? step.message,
|
||||
chars: reply.combined.length,
|
||||
elapsedMs: reply.elapsedMs,
|
||||
text: reply.combined,
|
||||
});
|
||||
reporter.pass('assistant 回复', `${reply.combined.length} 字 / ${reply.elapsedMs}ms`);
|
||||
|
||||
const keywords = step.expect?.replyKeywords ?? [];
|
||||
if (keywords.length) {
|
||||
const hit = keywords.filter((word) => reply.combined.includes(word));
|
||||
if (hit.length === 0) reporter.fail('回复关键词', `未命中: ${keywords.join(', ')}`);
|
||||
else reporter.pass('回复关键词', hit.join(', '));
|
||||
}
|
||||
|
||||
tracked.pageLinks.push(...extractPublicLinks(reply.combined, baseUrl));
|
||||
|
||||
if (step.expect?.page) {
|
||||
await verifyPageAccess({
|
||||
baseUrl,
|
||||
cookie: auth.cookie,
|
||||
publishKey,
|
||||
replyText: reply.combined,
|
||||
htmlBefore,
|
||||
expect: step.expect.page,
|
||||
uploadedAssetIds: [],
|
||||
reporter,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
tracked.elapsedMs = Date.now() - started;
|
||||
return {
|
||||
reporter,
|
||||
tracked,
|
||||
passed: reporter.issues.length === 0,
|
||||
issueCount: reporter.issues.length,
|
||||
};
|
||||
}
|
||||
|
||||
function pctDelta(before, after) {
|
||||
if (!before) return after ? 100 : 0;
|
||||
return Math.round(((after - before) / before) * 1000) / 10;
|
||||
}
|
||||
|
||||
function printComparison(baseline, optimized) {
|
||||
console.log('\n========================================');
|
||||
console.log('Token 优化 A/B 对比(本地)');
|
||||
console.log('========================================\n');
|
||||
|
||||
const rows = [
|
||||
['指标', 'Baseline', 'Optimized', '变化'],
|
||||
[
|
||||
'输入 Token (usage)',
|
||||
String(baseline.usage.inputTokens),
|
||||
String(optimized.usage.inputTokens),
|
||||
`${pctDelta(baseline.usage.inputTokens, optimized.usage.inputTokens)}%`,
|
||||
],
|
||||
[
|
||||
'输出 Token (usage)',
|
||||
String(baseline.usage.outputTokens),
|
||||
String(optimized.usage.outputTokens),
|
||||
`${pctDelta(baseline.usage.outputTokens, optimized.usage.outputTokens)}%`,
|
||||
],
|
||||
[
|
||||
'总 Token (usage)',
|
||||
String(baseline.usage.inputTokens + baseline.usage.outputTokens),
|
||||
String(optimized.usage.inputTokens + optimized.usage.outputTokens),
|
||||
`${pctDelta(
|
||||
baseline.usage.inputTokens + baseline.usage.outputTokens,
|
||||
optimized.usage.inputTokens + optimized.usage.outputTokens,
|
||||
)}%`,
|
||||
],
|
||||
[
|
||||
'扣费 (分)',
|
||||
String(baseline.usage.costCents),
|
||||
String(optimized.usage.costCents),
|
||||
`${pctDelta(baseline.usage.costCents, optimized.usage.costCents)}%`,
|
||||
],
|
||||
[
|
||||
'场景耗时 (s)',
|
||||
String(Math.round(baseline.tracked.elapsedMs / 1000)),
|
||||
String(Math.round(optimized.tracked.elapsedMs / 1000)),
|
||||
`${pctDelta(baseline.tracked.elapsedMs, optimized.tracked.elapsedMs)}%`,
|
||||
],
|
||||
[
|
||||
'回复字数',
|
||||
String(baseline.tracked.replies.at(-1)?.chars ?? 0),
|
||||
String(optimized.tracked.replies.at(-1)?.chars ?? 0),
|
||||
`${pctDelta(
|
||||
baseline.tracked.replies.at(-1)?.chars ?? 0,
|
||||
optimized.tracked.replies.at(-1)?.chars ?? 0,
|
||||
)}%`,
|
||||
],
|
||||
[
|
||||
'质量检查',
|
||||
baseline.passed ? '通过' : `失败 ${baseline.issueCount} 项`,
|
||||
optimized.passed ? '通过' : `失败 ${optimized.issueCount} 项`,
|
||||
optimized.passed === baseline.passed ? '持平' : (optimized.passed ? '改善' : '下降'),
|
||||
],
|
||||
];
|
||||
|
||||
const widths = [0, 1, 2, 3].map((col) => Math.max(...rows.map((row) => row[col].length)));
|
||||
for (const row of rows) {
|
||||
console.log(row.map((cell, i) => cell.padEnd(widths[i] + 2)).join(''));
|
||||
}
|
||||
|
||||
console.log('\n--- Baseline 页面链接 ---');
|
||||
for (const link of baseline.tracked.pageLinks) console.log(link);
|
||||
console.log('\n--- Optimized 页面链接 ---');
|
||||
for (const link of optimized.tracked.pageLinks) console.log(link);
|
||||
|
||||
if (baseline.passed && optimized.passed) {
|
||||
const totalBefore = baseline.usage.inputTokens + baseline.usage.outputTokens;
|
||||
const totalAfter = optimized.usage.inputTokens + optimized.usage.outputTokens;
|
||||
const saved = totalBefore - totalAfter;
|
||||
console.log(`\n结论: 质量检查均通过。Token 总量 ${totalBefore} → ${totalAfter}(${saved >= 0 ? '节省' : '增加'} ${Math.abs(saved)})`);
|
||||
if (saved > 0 && optimized.passed) {
|
||||
console.log('效果未打折,可继续观察更多场景。');
|
||||
} else if (saved <= 0) {
|
||||
console.log('Token 未下降,需检查模型/Router/记忆配置是否生效。');
|
||||
}
|
||||
} else {
|
||||
console.log('\n结论: 存在质量回归,不建议直接上生产。');
|
||||
if (!optimized.passed) {
|
||||
console.log('Optimized 失败项:');
|
||||
for (const issue of optimized.reporter.issues) {
|
||||
console.log(` - ${issue.label}: ${issue.detail}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runPhase(pool, llmProviderService, memoryConfigService, profileKey, scenario, port) {
|
||||
await applyProfile(pool, llmProviderService, memoryConfigService, profileKey);
|
||||
const runId = `${profileKey}-${Date.now()}`;
|
||||
console.log(`\n>>> 开始 ${profileKey} 场景跑分 (runId=${runId})...`);
|
||||
const result = await runScenarioBenchmark(scenario, port, { profileKey, runId });
|
||||
const usage = await collectUsageMetrics(pool, result.tracked.requestIds);
|
||||
const eventTokens = await collectRunEventTokens(pool, result.tracked.runIds);
|
||||
console.log(`\n--- ${profileKey} token 汇总 ---`);
|
||||
console.log(`usage records: in=${usage.inputTokens} out=${usage.outputTokens} cost=${usage.costCents}分`);
|
||||
console.log(`run events: in=${eventTokens.inputTokens} out=${eventTokens.outputTokens}`);
|
||||
return { ...result, usage, eventTokens, profileKey };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { scenarioId, phase, port } = parseArgs(process.argv);
|
||||
const pool = createDbPool(process.env);
|
||||
const llmProviderService = createLlmProviderService(pool, {
|
||||
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
|
||||
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const memoryConfigService = createMemoryV2AdminConfigService(pool);
|
||||
|
||||
try {
|
||||
if (phase === 'restore') {
|
||||
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||
return;
|
||||
}
|
||||
|
||||
const scenario = await loadScenario(scenarioId);
|
||||
console.log(`场景: ${scenario.name ?? scenario.id}`);
|
||||
console.log(`Portal: ${resolvePortalBase(port)}`);
|
||||
|
||||
if (phase === 'both') {
|
||||
await saveState(pool, memoryConfigService);
|
||||
}
|
||||
|
||||
let baselineResult = null;
|
||||
let optimizedResult = null;
|
||||
|
||||
if (phase === 'both' || phase === 'baseline') {
|
||||
baselineResult = await runPhase(
|
||||
pool,
|
||||
llmProviderService,
|
||||
memoryConfigService,
|
||||
'baseline',
|
||||
scenario,
|
||||
port,
|
||||
);
|
||||
}
|
||||
|
||||
if (phase === 'both' || phase === 'optimized') {
|
||||
optimizedResult = await runPhase(
|
||||
pool,
|
||||
llmProviderService,
|
||||
memoryConfigService,
|
||||
'optimized',
|
||||
scenario,
|
||||
port,
|
||||
);
|
||||
}
|
||||
|
||||
if (baselineResult && optimizedResult) {
|
||||
printComparison(baselineResult, optimizedResult);
|
||||
}
|
||||
|
||||
if (phase === 'both') {
|
||||
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||
console.log('\n已恢复 benchmark 前的模型与 memory 配置。');
|
||||
}
|
||||
} finally {
|
||||
await pool.end?.();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(async (error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : error);
|
||||
try {
|
||||
const pool = createDbPool(process.env);
|
||||
const llmProviderService = createLlmProviderService(pool, {
|
||||
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
|
||||
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const memoryConfigService = createMemoryV2AdminConfigService(pool);
|
||||
await restoreSavedState(pool, llmProviderService, memoryConfigService);
|
||||
await pool.end?.();
|
||||
} catch {
|
||||
// best effort restore
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sync token-optimization Router / memory env to 103 Portal (.env only).
|
||||
# Does NOT change global LLM model (keep deepseek-v4-pro for Agent quality).
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/sync-token-optimization-env-103.sh # dry-run
|
||||
# bash scripts/sync-token-optimization-env-103.sh --apply # write + restart portal
|
||||
set -euo pipefail
|
||||
|
||||
HOST="${MEMIND_103_HOST:-john@58.38.22.103}"
|
||||
REMOTE_ENV="/Users/john/Project/Memind/.env"
|
||||
APPLY=0
|
||||
if [[ "${1:-}" == "--apply" ]]; then
|
||||
APPLY=1
|
||||
fi
|
||||
|
||||
PATCH_LINES=(
|
||||
"MEMIND_CHAT_LLM_ROUTER_ENABLED=1"
|
||||
"MEMIND_CHAT_LLM_ROUTER_SHADOW=0"
|
||||
"MEMIND_CHAT_ROUTER_CANARY_USER_IDS="
|
||||
"MEMIND_CHAT_ROUTER_MODEL=deepseek-chat"
|
||||
"MEMIND_CHAT_ROUTER_MEMORY_LIMIT=2"
|
||||
"MEMIND_CHAT_ROUTER_TIMEOUT_MS=2500"
|
||||
"MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65"
|
||||
"MEMORY_AGENT_RESOLVE_LIMIT=3"
|
||||
"MEMORY_RETRIEVER_LIMIT=4"
|
||||
"MEMORY_RETRIEVER_TOKEN_BUDGET=900"
|
||||
"MEMORY_PERSONA_MAX_TOKENS=300"
|
||||
)
|
||||
|
||||
echo "==> Target: ${HOST}:${REMOTE_ENV}"
|
||||
echo "==> Patch:"
|
||||
printf ' %s\n' "${PATCH_LINES[@]}"
|
||||
echo
|
||||
|
||||
if [[ "${APPLY}" -ne 1 ]]; then
|
||||
echo "Dry-run only. Re-run with --apply to write and restart Portal."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PATCH_B64=$(printf '%s\n' "${PATCH_LINES[@]}" | base64 | tr -d '\n')
|
||||
|
||||
ssh "${HOST}" "bash -s" <<REMOTE
|
||||
set -euo pipefail
|
||||
ENV_FILE="${REMOTE_ENV}"
|
||||
BACKUP="\${ENV_FILE}.bak-token-opt-\$(date +%Y%m%d-%H%M%S)"
|
||||
cp "\${ENV_FILE}" "\${BACKUP}"
|
||||
echo "Backed up to \${BACKUP}"
|
||||
|
||||
upsert() {
|
||||
local key="\$1" val="\$2"
|
||||
if grep -q "^\${key}=" "\${ENV_FILE}"; then
|
||||
sed -i '' "s|^\${key}=.*|\${key}=\${val}|" "\${ENV_FILE}"
|
||||
else
|
||||
printf '\n%s=%s\n' "\${key}" "\${val}" >> "\${ENV_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
while IFS='=' read -r key val; do
|
||||
[[ -z "\${key}" ]] && continue
|
||||
upsert "\${key}" "\${val}"
|
||||
done <<'ENVPATCH'
|
||||
$(printf '%s\n' "${PATCH_LINES[@]}")
|
||||
ENVPATCH
|
||||
|
||||
echo "==> Updated keys:"
|
||||
grep -E '^(MEMIND_CHAT_|MEMORY_AGENT_RESOLVE|MEMORY_RETRIEVER|MEMORY_PERSONA_MAX)' "\${ENV_FILE}" || true
|
||||
|
||||
GUI="\$(/usr/bin/stat -f %u /dev/console)"
|
||||
launchctl kickstart -k "gui/\${GUI}/cn.tkmind.memind-portal"
|
||||
sleep 3
|
||||
curl -sf http://127.0.0.1:8081/api/status >/dev/null
|
||||
echo "Portal restarted and /api/status OK"
|
||||
REMOTE
|
||||
|
||||
echo "Done."
|
||||
Reference in New Issue
Block a user