feat(router): improve direct_chat fast-path and add 103 token env sync
Memind CI / Test, build, and release guards (push) Failing after 12m44s

Route explicit text-only Q&A away from Agent before page-generation rules,
and add a script to enable Router + tighter memory budgets on 103 without
changing the global deepseek-v4-pro model.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-25 15:13:22 +08:00
parent 9dfafb99ca
commit 9332bacabd
7 changed files with 758 additions and 13 deletions
+36 -1
View File
@@ -103,6 +103,25 @@ export const DIRECT_CHAT_FAQ_RULES = [
/^.{0,16}(?:有什么建议|给点建议|怎么看)[?]?$/u,
],
},
{
id: 'text_only_declined_page',
reason: '明确要求纯文字、不生成页面/链接',
patterns: [
/(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u,
/(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u,
],
},
{
id: 'brief_intro',
reason: '短篇幅介绍/说明(纯文字)',
patterns: [
/(?:用|以).{0,6}(?:一|两|二|三|四|五|几|\d+).{0,4}(?:句话|句).{0,32}(?:介绍|说明|描述|讲讲|推荐)/u,
/^(?:简单|简要|大概|概括)(?:介绍|说明|描述|讲讲).{0,48}[?]?$/u,
/^简要介绍.{0,48}(?:区别|差异|不同|对比|比较)[?]?$/u,
/^(?:推荐|介绍)(?:一个|一下).{0,32}(?:理由|原因)[,]?一句话/u,
/(?:推荐|介绍).{0,32}一句话.{0,16}(?:说明|讲讲).{0,12}(?:理由|原因)/u,
],
},
];
const MAX_FAQ_TEXT_LENGTH = 200;
@@ -111,10 +130,26 @@ const MAX_FAQ_TEXT_LENGTH = 200;
* @param {string} text
* @returns {{ id: string, reason: string } | null}
*/
const EXPLICIT_TEXT_ONLY_PAGE_DECLINE = /(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u;
export function isExplicitTextOnlyRequest(text) {
const normalized = String(text ?? '').trim();
if (!normalized) return false;
return EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
}
export function matchDirectChatFaqRule(text) {
const normalized = String(text ?? '').trim();
if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null;
if (FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))) return null;
const declinedPage = EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized)
|| /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized);
if (
!declinedPage
&& FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))
) {
return null;
}
for (const rule of DIRECT_CHAT_FAQ_RULES) {
if (rule.patterns.some((pattern) => pattern.test(normalized))) {
return { id: rule.id, reason: rule.reason };
+20 -12
View File
@@ -15,11 +15,11 @@ import {
resolveMemoryInterventionMode,
} from './memory-intervention.mjs';
import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs';
import { matchDirectChatFaqRule } from './chat-intent-router-rules.mjs';
import { matchDirectChatFaqRule, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
import { isGoalRunIntent } from './goal-run-intent.mjs';
import { pgvectorMemoryBackendInternals } from './memory-v2-pgvector.mjs';
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS } from './chat-intent-router-rules.mjs';
export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs';
export const CHAT_INTENT_ROUTE = {
DIRECT_CHAT: 'direct_chat',
@@ -1083,6 +1083,14 @@ export function classifyWithRules({
reason: '用户在询问个人记忆或历史对话',
}, { source: 'rule' }), decisionContext);
}
if (includeIntentPatterns && normalized && isExplicitTextOnlyRequest(normalized)) {
const faqMatch = matchDirectChatFaqRule(normalized);
return finalizeRouterClassification(normalizeClassification({
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
confidence: 0.96,
reason: faqMatch?.reason ?? '用户明确要求纯文字、不生成页面',
}, { source: 'rule' }), decisionContext);
}
if (hasPriorAgentConversation(sessionId, sessionMessageCount)) {
const reason = isAgentSessionContinueText(normalized)
? 'Agent 会话确认/续聊'
@@ -1093,6 +1101,16 @@ export function classifyWithRules({
reason,
}, { source: 'rule' }), decisionContext);
}
if (includeIntentPatterns && normalized) {
const faqMatch = matchDirectChatFaqRule(normalized);
if (faqMatch) {
return finalizeRouterClassification(normalizeClassification({
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
confidence: 0.94,
reason: faqMatch.reason,
}, { source: 'rule' }), decisionContext);
}
}
if (includeIntentPatterns && normalized && isPageDataDevIntent(normalized)) {
return finalizeRouterClassification(normalizeClassification({
route: CHAT_INTENT_ROUTE.AGENT,
@@ -1142,16 +1160,6 @@ export function classifyWithRules({
if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) {
return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext);
}
if (includeIntentPatterns && normalized) {
const faqMatch = matchDirectChatFaqRule(normalized);
if (faqMatch) {
return finalizeRouterClassification(normalizeClassification({
route: CHAT_INTENT_ROUTE.DIRECT_CHAT,
confidence: 0.94,
reason: faqMatch.reason,
}, { source: 'rule' }), decisionContext);
}
}
if (
includeIntentPatterns &&
normalized &&
+19
View File
@@ -109,6 +109,25 @@ test('classifyWithRules routes greetings to direct chat', () => {
assert.equal(result.source, 'rule');
});
test('classifyWithRules routes explicit text-only Q&A to direct chat', () => {
for (const text of [
'用三句话介绍苏州园林,不要生成页面',
'简要介绍一下 Rust 和 Go 的区别',
'推荐一个最适合第一次去的,一句话说明理由',
]) {
const result = classifyWithRules({
text,
userMessage: {
role: 'user',
content: [{ type: 'text', text }],
metadata: { displayText: text },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT, text);
assert.equal(result.source, 'rule', text);
}
});
test('classifyWithRules routes page generation to agent orchestration', () => {
const result = classifyWithRules({
text: '帮我做一个秋夜诗的 H5 页面',
+39
View File
@@ -0,0 +1,39 @@
{
"id": "token-benchmark-chat",
"name": "Token 基准:Direct Chat 三轮",
"description": "简单对话,不走做页面 Agent,对比 pro vs flash",
"account": {
"username": "john2",
"password": "888888"
},
"steps": [
{
"action": "chat",
"label": "打招呼",
"message": "hi",
"expect": {
"assistantMinChars": 1,
"timeoutMs": 120000
}
},
{
"action": "chat",
"label": "苏州简介",
"message": "用三句话介绍苏州园林,不要生成页面",
"expect": {
"assistantMinChars": 20,
"timeoutMs": 120000,
"replyKeywords": ["苏州"]
}
},
{
"action": "chat",
"label": "追问",
"message": "推荐一个最适合第一次去的,一句话说明理由",
"expect": {
"assistantMinChars": 10,
"timeoutMs": 120000
}
}
]
}
+26
View File
@@ -0,0 +1,26 @@
{
"id": "token-benchmark-page",
"name": "Token 基准:独立苏州页面",
"description": "每次使用唯一文件名,避免复用已有 HTML 污染上下文",
"account": {
"username": "john2",
"password": "888888"
},
"steps": [
{
"action": "chat",
"label": "生成独立苏州页面",
"message": "请帮我做一个全新的苏州一日游攻略页面,保存为 public/token-bench-{{PROFILE}}-{{RUN_ID}}.html,不要修改或复用已有页面,做完直接给我链接。",
"expect": {
"assistantMinChars": 20,
"timeoutMs": 600000,
"replyKeywords": ["苏州"],
"page": {
"keywords": ["苏州"],
"requirePublicLink": true,
"requireHttp200": true
}
}
}
]
}
+542
View File
@@ -0,0 +1,542 @@
#!/usr/bin/env node
/**
* Local A/B benchmark: DeepSeek token usage vs scenario quality.
*
* Usage:
* node scripts/benchmark-token-optimization.mjs
* node scripts/benchmark-token-optimization.mjs --scenario john2-suzhou-page
* node scripts/benchmark-token-optimization.mjs --phase baseline
* node scripts/benchmark-token-optimization.mjs --phase optimized
* node scripts/benchmark-token-optimization.mjs --restore
*
* Does NOT touch production. Restores model + memory admin config on exit.
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { createDbPool } from '../db.mjs';
import { createLlmProviderService } from '../llm-providers.mjs';
import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs';
import { loadH5Environment } from './load-env.mjs';
import {
createReporter,
createAgentRun,
extractAssistantTexts,
extractPublicLinks,
getSession,
loadScenario,
loginViaApi,
resolvePortalBase,
snapshotPublicHtml,
verifyPageAccess,
waitForAssistantGrowth,
waitForRunTerminal,
} from './scenario-test-lib.mjs';
const scriptDir = path.dirname(fileURLToPath(import.meta.url));
const root = path.join(scriptDir, '..');
const statePath = path.join(root, '.token-benchmark-state.json');
loadH5Environment(scriptDir);
const PROFILES = {
baseline: {
label: 'baseline (deepseek-v4-pro + 默认记忆预算)',
model: 'deepseek-v4-pro',
memoryPatch: null,
},
optimized: {
label: 'optimized (deepseek-v4-flash + 收紧记忆/Router)',
model: 'deepseek-v4-flash',
memoryPatch: {
runtimeControl: {
agentResolveLimit: '3',
},
retriever: {
limit: '4',
tokenBudget: '900',
},
persona: {
maxTokens: '300',
},
chatIntentRouter: {
enabled: true,
shadowMode: false,
model: 'deepseek-chat',
memoryResolveLimit: '2',
},
},
},
};
function parseArgs(argv) {
let scenarioId = 'token-benchmark-page';
let phase = 'both';
let port = Number(process.env.H5_PORT ?? 8081);
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--scenario' && argv[i + 1]) scenarioId = argv[++i];
else if (arg === '--phase' && argv[i + 1]) phase = argv[++i];
else if (arg === '--port' && argv[i + 1]) port = Number(argv[++i]);
else if (arg === '--restore') phase = 'restore';
else if (arg === '-h' || arg === '--help') {
console.log(`Usage:
node scripts/benchmark-token-optimization.mjs [--scenario <id>] [--phase baseline|optimized|both|restore]`);
process.exit(0);
} else {
throw new Error(`Unknown argument: ${arg}`);
}
}
if (!['baseline', 'optimized', 'both', 'restore'].includes(phase)) {
throw new Error(`Invalid --phase: ${phase}`);
}
return { scenarioId, phase, port };
}
async function ensurePortalReady(baseUrl) {
const response = await fetch(`${baseUrl}/auth/status`);
if (!response.ok) {
throw new Error(`Portal 未就绪: ${baseUrl}/auth/status -> ${response.status}`);
}
}
async function readCurrentModel(pool) {
const [rows] = await pool.query(
`SELECT default_model FROM h5_llm_provider_keys WHERE is_selected = 1 AND status = 'active' LIMIT 1`,
);
return rows[0]?.default_model ?? null;
}
async function saveState(pool, memoryConfigService) {
const model = await readCurrentModel(pool);
const memoryAdmin = await memoryConfigService.getAdminConfig().catch(() => null);
const payload = {
savedAt: Date.now(),
model,
memoryConfig: memoryAdmin?.config ?? null,
};
fs.writeFileSync(statePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
return payload;
}
function loadSavedState() {
if (!fs.existsSync(statePath)) return null;
return JSON.parse(fs.readFileSync(statePath, 'utf8'));
}
async function applyProfile(pool, llmProviderService, memoryConfigService, profileKey) {
const profile = PROFILES[profileKey];
if (!profile) throw new Error(`Unknown profile: ${profileKey}`);
const modelResult = await llmProviderService.setGlobalModel(profile.model);
if (!modelResult?.ok) {
throw new Error(`切换模型失败 (${profile.model}): ${modelResult?.message ?? 'unknown'}`);
}
if (profile.memoryPatch) {
await memoryConfigService.updateAdminConfig(profile.memoryPatch, {
updatedBy: 'token-benchmark',
});
}
const currentModel = await readCurrentModel(pool);
console.log(`\n==> 已应用 ${profile.label}`);
console.log(` 全局模型: ${currentModel}`);
if (profile.memoryPatch) {
console.log(' 记忆/Router 预算: 已写入 h5_memory_v2_admin_config');
}
}
async function restoreSavedState(pool, llmProviderService, memoryConfigService) {
const saved = loadSavedState();
if (!saved) {
console.log('无 .token-benchmark-state.json,跳过恢复');
return false;
}
if (saved.model) {
const result = await llmProviderService.setGlobalModel(saved.model);
if (!result?.ok) {
console.warn(`恢复模型失败: ${result?.message ?? 'unknown'}`);
} else {
console.log(`已恢复全局模型: ${saved.model}`);
}
}
if (saved.memoryConfig) {
await memoryConfigService.updateAdminConfig(saved.memoryConfig, {
updatedBy: 'token-benchmark-restore',
});
console.log('已恢复 memory v2 admin config');
}
fs.unlinkSync(statePath);
return true;
}
async function collectUsageMetrics(pool, requestIds) {
if (!requestIds.length) {
return { inputTokens: 0, outputTokens: 0, costCents: 0, records: [] };
}
const placeholders = requestIds.map(() => '?').join(', ');
const [rows] = await pool.query(
`SELECT request_id, input_tokens, output_tokens, cost_cents, billing_source, created_at
FROM h5_usage_records
WHERE request_id IN (${placeholders})
ORDER BY created_at ASC`,
requestIds,
);
let inputTokens = 0;
let outputTokens = 0;
let costCents = 0;
for (const row of rows) {
inputTokens += Number(row.input_tokens ?? 0);
outputTokens += Number(row.output_tokens ?? 0);
costCents += Number(row.cost_cents ?? 0);
}
return { inputTokens, outputTokens, costCents, records: rows };
}
async function collectRunEventTokens(pool, runIds) {
if (!runIds.length) return { inputTokens: 0, outputTokens: 0, events: [] };
const placeholders = runIds.map(() => '?').join(', ');
const [rows] = await pool.query(
`SELECT run_id, data_json, created_at
FROM h5_agent_run_events
WHERE run_id IN (${placeholders}) AND event_type = 'session_finished'
ORDER BY created_at ASC`,
runIds,
);
let inputTokens = 0;
let outputTokens = 0;
const events = [];
for (const row of rows) {
const data = typeof row.data_json === 'string'
? JSON.parse(row.data_json)
: row.data_json;
const tokenState = data?.tokenState ?? data?.token_state ?? null;
if (!tokenState) continue;
const input = Number(
tokenState.accumulatedInputTokens
?? tokenState.accumulated_input_tokens
?? tokenState.inputTokens
?? tokenState.input_tokens
?? 0,
);
const output = Number(
tokenState.accumulatedOutputTokens
?? tokenState.accumulated_output_tokens
?? tokenState.outputTokens
?? tokenState.output_tokens
?? 0,
);
inputTokens += input;
outputTokens += output;
events.push({ runId: row.run_id, input, output, tokenState });
}
return { inputTokens, outputTokens, events };
}
async function runScenarioBenchmark(scenario, port, { profileKey = 'run', runId = Date.now() } = {}) {
const reporter = createReporter();
const baseUrl = resolvePortalBase(port);
const account = {
username: scenario.account?.username ?? 'john2',
password: process.env.JOHN_PASSWORD
?? process.env.H5_ACCESS_PASSWORD
?? scenario.account?.password
?? '888888',
};
await ensurePortalReady(baseUrl);
const auth = await loginViaApi(baseUrl, account, reporter);
const publishKey = auth.user?.id ?? auth.user?.publishSlug ?? account.username;
let sessionId = null;
let assistantCount = 0;
let assistantCombinedLength = 0;
const tracked = {
requestIds: [],
runIds: [],
sessionIds: [],
replies: [],
pageLinks: [],
elapsedMs: 0,
profileKey,
runId,
};
const started = Date.now();
for (const step of scenario.steps ?? []) {
if (step.action !== 'chat') continue;
const message = String(step.message ?? '')
.replaceAll('{{PROFILE}}', profileKey)
.replaceAll('{{RUN_ID}}', String(runId));
let htmlBefore = [];
if (step.expect?.page) {
htmlBefore = await snapshotPublicHtml(publishKey);
}
const run = await createAgentRun(baseUrl, auth.cookie, {
message,
sessionId: null,
selectedChatSkill: step.selectedChatSkill ?? null,
});
tracked.requestIds.push(run.requestId);
tracked.runIds.push(run.runId);
const terminal = await waitForRunTerminal(
baseUrl,
auth.cookie,
run.runId,
step.expect?.timeoutMs ?? 600_000,
);
sessionId = terminal.sessionId ?? terminal.agent_session_id ?? run.sessionId ?? sessionId;
if (sessionId) tracked.sessionIds.push(sessionId);
if (terminal.status === 'failed') {
reporter.fail('run 终态', terminal.error ?? 'failed');
continue;
}
reporter.pass('run 终态', terminal.status);
const reply = await waitForAssistantGrowth(baseUrl, auth.cookie, sessionId, {
previousCount: assistantCount,
previousCombinedLength: assistantCombinedLength,
minChars: step.expect?.assistantMinChars ?? 1,
timeoutMs: step.expect?.timeoutMs ?? 120_000,
});
if (!reply) {
reporter.fail('assistant 回复', '超时未收到新回复');
continue;
}
assistantCount = reply.count;
assistantCombinedLength = reply.combined.length;
tracked.replies.push({
label: step.label ?? step.message,
chars: reply.combined.length,
elapsedMs: reply.elapsedMs,
text: reply.combined,
});
reporter.pass('assistant 回复', `${reply.combined.length} 字 / ${reply.elapsedMs}ms`);
const keywords = step.expect?.replyKeywords ?? [];
if (keywords.length) {
const hit = keywords.filter((word) => reply.combined.includes(word));
if (hit.length === 0) reporter.fail('回复关键词', `未命中: ${keywords.join(', ')}`);
else reporter.pass('回复关键词', hit.join(', '));
}
tracked.pageLinks.push(...extractPublicLinks(reply.combined, baseUrl));
if (step.expect?.page) {
await verifyPageAccess({
baseUrl,
cookie: auth.cookie,
publishKey,
replyText: reply.combined,
htmlBefore,
expect: step.expect.page,
uploadedAssetIds: [],
reporter,
});
}
}
tracked.elapsedMs = Date.now() - started;
return {
reporter,
tracked,
passed: reporter.issues.length === 0,
issueCount: reporter.issues.length,
};
}
function pctDelta(before, after) {
if (!before) return after ? 100 : 0;
return Math.round(((after - before) / before) * 1000) / 10;
}
function printComparison(baseline, optimized) {
console.log('\n========================================');
console.log('Token 优化 A/B 对比(本地)');
console.log('========================================\n');
const rows = [
['指标', 'Baseline', 'Optimized', '变化'],
[
'输入 Token (usage)',
String(baseline.usage.inputTokens),
String(optimized.usage.inputTokens),
`${pctDelta(baseline.usage.inputTokens, optimized.usage.inputTokens)}%`,
],
[
'输出 Token (usage)',
String(baseline.usage.outputTokens),
String(optimized.usage.outputTokens),
`${pctDelta(baseline.usage.outputTokens, optimized.usage.outputTokens)}%`,
],
[
'总 Token (usage)',
String(baseline.usage.inputTokens + baseline.usage.outputTokens),
String(optimized.usage.inputTokens + optimized.usage.outputTokens),
`${pctDelta(
baseline.usage.inputTokens + baseline.usage.outputTokens,
optimized.usage.inputTokens + optimized.usage.outputTokens,
)}%`,
],
[
'扣费 (分)',
String(baseline.usage.costCents),
String(optimized.usage.costCents),
`${pctDelta(baseline.usage.costCents, optimized.usage.costCents)}%`,
],
[
'场景耗时 (s)',
String(Math.round(baseline.tracked.elapsedMs / 1000)),
String(Math.round(optimized.tracked.elapsedMs / 1000)),
`${pctDelta(baseline.tracked.elapsedMs, optimized.tracked.elapsedMs)}%`,
],
[
'回复字数',
String(baseline.tracked.replies.at(-1)?.chars ?? 0),
String(optimized.tracked.replies.at(-1)?.chars ?? 0),
`${pctDelta(
baseline.tracked.replies.at(-1)?.chars ?? 0,
optimized.tracked.replies.at(-1)?.chars ?? 0,
)}%`,
],
[
'质量检查',
baseline.passed ? '通过' : `失败 ${baseline.issueCount}`,
optimized.passed ? '通过' : `失败 ${optimized.issueCount}`,
optimized.passed === baseline.passed ? '持平' : (optimized.passed ? '改善' : '下降'),
],
];
const widths = [0, 1, 2, 3].map((col) => Math.max(...rows.map((row) => row[col].length)));
for (const row of rows) {
console.log(row.map((cell, i) => cell.padEnd(widths[i] + 2)).join(''));
}
console.log('\n--- Baseline 页面链接 ---');
for (const link of baseline.tracked.pageLinks) console.log(link);
console.log('\n--- Optimized 页面链接 ---');
for (const link of optimized.tracked.pageLinks) console.log(link);
if (baseline.passed && optimized.passed) {
const totalBefore = baseline.usage.inputTokens + baseline.usage.outputTokens;
const totalAfter = optimized.usage.inputTokens + optimized.usage.outputTokens;
const saved = totalBefore - totalAfter;
console.log(`\n结论: 质量检查均通过。Token 总量 ${totalBefore}${totalAfter}${saved >= 0 ? '节省' : '增加'} ${Math.abs(saved)}`);
if (saved > 0 && optimized.passed) {
console.log('效果未打折,可继续观察更多场景。');
} else if (saved <= 0) {
console.log('Token 未下降,需检查模型/Router/记忆配置是否生效。');
}
} else {
console.log('\n结论: 存在质量回归,不建议直接上生产。');
if (!optimized.passed) {
console.log('Optimized 失败项:');
for (const issue of optimized.reporter.issues) {
console.log(` - ${issue.label}: ${issue.detail}`);
}
}
}
}
async function runPhase(pool, llmProviderService, memoryConfigService, profileKey, scenario, port) {
await applyProfile(pool, llmProviderService, memoryConfigService, profileKey);
const runId = `${profileKey}-${Date.now()}`;
console.log(`\n>>> 开始 ${profileKey} 场景跑分 (runId=${runId})...`);
const result = await runScenarioBenchmark(scenario, port, { profileKey, runId });
const usage = await collectUsageMetrics(pool, result.tracked.requestIds);
const eventTokens = await collectRunEventTokens(pool, result.tracked.runIds);
console.log(`\n--- ${profileKey} token 汇总 ---`);
console.log(`usage records: in=${usage.inputTokens} out=${usage.outputTokens} cost=${usage.costCents}`);
console.log(`run events: in=${eventTokens.inputTokens} out=${eventTokens.outputTokens}`);
return { ...result, usage, eventTokens, profileKey };
}
async function main() {
const { scenarioId, phase, port } = parseArgs(process.argv);
const pool = createDbPool(process.env);
const llmProviderService = createLlmProviderService(pool, {
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
});
const memoryConfigService = createMemoryV2AdminConfigService(pool);
try {
if (phase === 'restore') {
await restoreSavedState(pool, llmProviderService, memoryConfigService);
return;
}
const scenario = await loadScenario(scenarioId);
console.log(`场景: ${scenario.name ?? scenario.id}`);
console.log(`Portal: ${resolvePortalBase(port)}`);
if (phase === 'both') {
await saveState(pool, memoryConfigService);
}
let baselineResult = null;
let optimizedResult = null;
if (phase === 'both' || phase === 'baseline') {
baselineResult = await runPhase(
pool,
llmProviderService,
memoryConfigService,
'baseline',
scenario,
port,
);
}
if (phase === 'both' || phase === 'optimized') {
optimizedResult = await runPhase(
pool,
llmProviderService,
memoryConfigService,
'optimized',
scenario,
port,
);
}
if (baselineResult && optimizedResult) {
printComparison(baselineResult, optimizedResult);
}
if (phase === 'both') {
await restoreSavedState(pool, llmProviderService, memoryConfigService);
console.log('\n已恢复 benchmark 前的模型与 memory 配置。');
}
} finally {
await pool.end?.();
}
}
main().catch(async (error) => {
console.error(error instanceof Error ? error.stack ?? error.message : error);
try {
const pool = createDbPool(process.env);
const llmProviderService = createLlmProviderService(pool, {
apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006',
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
});
const memoryConfigService = createMemoryV2AdminConfigService(pool);
await restoreSavedState(pool, llmProviderService, memoryConfigService);
await pool.end?.();
} catch {
// best effort restore
}
process.exit(1);
});
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
# Sync token-optimization Router / memory env to 103 Portal (.env only).
# Does NOT change global LLM model (keep deepseek-v4-pro for Agent quality).
#
# Usage:
# bash scripts/sync-token-optimization-env-103.sh # dry-run
# bash scripts/sync-token-optimization-env-103.sh --apply # write + restart portal
set -euo pipefail
HOST="${MEMIND_103_HOST:-john@58.38.22.103}"
REMOTE_ENV="/Users/john/Project/Memind/.env"
APPLY=0
if [[ "${1:-}" == "--apply" ]]; then
APPLY=1
fi
PATCH_LINES=(
"MEMIND_CHAT_LLM_ROUTER_ENABLED=1"
"MEMIND_CHAT_LLM_ROUTER_SHADOW=0"
"MEMIND_CHAT_ROUTER_CANARY_USER_IDS="
"MEMIND_CHAT_ROUTER_MODEL=deepseek-chat"
"MEMIND_CHAT_ROUTER_MEMORY_LIMIT=2"
"MEMIND_CHAT_ROUTER_TIMEOUT_MS=2500"
"MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65"
"MEMORY_AGENT_RESOLVE_LIMIT=3"
"MEMORY_RETRIEVER_LIMIT=4"
"MEMORY_RETRIEVER_TOKEN_BUDGET=900"
"MEMORY_PERSONA_MAX_TOKENS=300"
)
echo "==> Target: ${HOST}:${REMOTE_ENV}"
echo "==> Patch:"
printf ' %s\n' "${PATCH_LINES[@]}"
echo
if [[ "${APPLY}" -ne 1 ]]; then
echo "Dry-run only. Re-run with --apply to write and restart Portal."
exit 0
fi
PATCH_B64=$(printf '%s\n' "${PATCH_LINES[@]}" | base64 | tr -d '\n')
ssh "${HOST}" "bash -s" <<REMOTE
set -euo pipefail
ENV_FILE="${REMOTE_ENV}"
BACKUP="\${ENV_FILE}.bak-token-opt-\$(date +%Y%m%d-%H%M%S)"
cp "\${ENV_FILE}" "\${BACKUP}"
echo "Backed up to \${BACKUP}"
upsert() {
local key="\$1" val="\$2"
if grep -q "^\${key}=" "\${ENV_FILE}"; then
sed -i '' "s|^\${key}=.*|\${key}=\${val}|" "\${ENV_FILE}"
else
printf '\n%s=%s\n' "\${key}" "\${val}" >> "\${ENV_FILE}"
fi
}
while IFS='=' read -r key val; do
[[ -z "\${key}" ]] && continue
upsert "\${key}" "\${val}"
done <<'ENVPATCH'
$(printf '%s\n' "${PATCH_LINES[@]}")
ENVPATCH
echo "==> Updated keys:"
grep -E '^(MEMIND_CHAT_|MEMORY_AGENT_RESOLVE|MEMORY_RETRIEVER|MEMORY_PERSONA_MAX)' "\${ENV_FILE}" || true
GUI="\$(/usr/bin/stat -f %u /dev/console)"
launchctl kickstart -k "gui/\${GUI}/cn.tkmind.memind-portal"
sleep 3
curl -sf http://127.0.0.1:8081/api/status >/dev/null
echo "Portal restarted and /api/status OK"
REMOTE
echo "Done."