From 9332bacabdaf99d777589ddb0d39ee06853bbf93 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 25 Aug 2026 15:13:22 +0800 Subject: [PATCH] feat(router): improve direct_chat fast-path and add 103 token env sync Route explicit text-only Q&A away from Agent before page-generation rules, and add a script to enable Router + tighter memory budgets on 103 without changing the global deepseek-v4-pro model. Co-authored-by: Cursor --- chat-intent-router-rules.mjs | 37 +- chat-intent-router.mjs | 32 +- chat-intent-router.test.mjs | 19 + scenarios/token-benchmark-chat.json | 39 ++ scenarios/token-benchmark-page.json | 26 + scripts/benchmark-token-optimization.mjs | 542 +++++++++++++++++++++ scripts/sync-token-optimization-env-103.sh | 76 +++ 7 files changed, 758 insertions(+), 13 deletions(-) create mode 100644 scenarios/token-benchmark-chat.json create mode 100644 scenarios/token-benchmark-page.json create mode 100755 scripts/benchmark-token-optimization.mjs create mode 100755 scripts/sync-token-optimization-env-103.sh diff --git a/chat-intent-router-rules.mjs b/chat-intent-router-rules.mjs index 2575b07..fb6d05e 100644 --- a/chat-intent-router-rules.mjs +++ b/chat-intent-router-rules.mjs @@ -103,6 +103,25 @@ export const DIRECT_CHAT_FAQ_RULES = [ /^.{0,16}(?:有什么建议|给点建议|怎么看)[？?]?$/u, ], }, + { + id: 'text_only_declined_page', + reason: '明确要求纯文字、不生成页面/链接', + patterns: [ + /(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u, + /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u, + ], + }, + { + id: 'brief_intro', + reason: '短篇幅介绍/说明（纯文字）', + patterns: [ + /(?:用|以).{0,6}(?:一|两|二|三|四|五|几|\d+).{0,4}(?:句话|句).{0,32}(?:介绍|说明|描述|讲讲|推荐)/u, + /^(?:简单|简要|大概|概括)(?:介绍|说明|描述|讲讲).{0,48}[？?]?$/u, + /^简要介绍.{0,48}(?:区别|差异|不同|对比|比较)[？?]?$/u, + /^(?:推荐|介绍)(?:一个|一下).{0,32}(?:理由|原因)[，,]?一句话/u, + /(?:推荐|介绍).{0,32}一句话.{0,16}(?:说明|讲讲).{0,12}(?:理由|原因)/u, + ], + }, ]; const MAX_FAQ_TEXT_LENGTH = 200; @@ -111,10 +130,26 @@ const MAX_FAQ_TEXT_LENGTH = 200; * @param {string} text * @returns {{ id: string, reason: string } | null} */ +const EXPLICIT_TEXT_ONLY_PAGE_DECLINE = /(?:不要|别|无需|不需要|禁止).{0,12}(?:生成|做|制作|创建|发布).{0,8}(?:页面|网页|HTML|html|H5|h5|链接)/u; + +export function isExplicitTextOnlyRequest(text) { + const normalized = String(text ?? '').trim(); + if (!normalized) return false; + return EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized) + || /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized); +} + export function matchDirectChatFaqRule(text) { const normalized = String(text ?? '').trim(); if (!normalized || normalized.length > MAX_FAQ_TEXT_LENGTH) return null; - if (FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized))) return null; + const declinedPage = EXPLICIT_TEXT_ONLY_PAGE_DECLINE.test(normalized) + || /(?:纯文字|只要文字|文字回答|不要页面|不用页面|别做页面)/u.test(normalized); + if ( + !declinedPage + && FAQ_EXCLUSION_PATTERNS.some((pattern) => pattern.test(normalized)) + ) { + return null; + } for (const rule of DIRECT_CHAT_FAQ_RULES) { if (rule.patterns.some((pattern) => pattern.test(normalized))) { return { id: rule.id, reason: rule.reason }; diff --git a/chat-intent-router.mjs b/chat-intent-router.mjs index d1c4773..4a403c8 100644 --- a/chat-intent-router.mjs +++ b/chat-intent-router.mjs @@ -15,11 +15,11 @@ import { resolveMemoryInterventionMode, } from './memory-intervention.mjs'; import { filterMemoriesByQuery } from './memory-legacy-fallback.mjs'; -import { matchDirectChatFaqRule } from './chat-intent-router-rules.mjs'; +import { matchDirectChatFaqRule, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs'; import { isGoalRunIntent } from './goal-run-intent.mjs'; import { pgvectorMemoryBackendInternals } from './memory-v2-pgvector.mjs'; -export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS } from './chat-intent-router-rules.mjs'; +export { matchDirectChatFaqRule, DIRECT_CHAT_FAQ_RULES, FAQ_EXCLUSION_PATTERNS, isExplicitTextOnlyRequest } from './chat-intent-router-rules.mjs'; export const CHAT_INTENT_ROUTE = { DIRECT_CHAT: 'direct_chat', @@ -1083,6 +1083,14 @@ export function classifyWithRules({ reason: '用户在询问个人记忆或历史对话', }, { source: 'rule' }), decisionContext); } + if (includeIntentPatterns && normalized && isExplicitTextOnlyRequest(normalized)) { + const faqMatch = matchDirectChatFaqRule(normalized); + return finalizeRouterClassification(normalizeClassification({ + route: CHAT_INTENT_ROUTE.DIRECT_CHAT, + confidence: 0.96, + reason: faqMatch?.reason ?? '用户明确要求纯文字、不生成页面', + }, { source: 'rule' }), decisionContext); + } if (hasPriorAgentConversation(sessionId, sessionMessageCount)) { const reason = isAgentSessionContinueText(normalized) ? 'Agent 会话确认/续聊' @@ -1093,6 +1101,16 @@ export function classifyWithRules({ reason, }, { source: 'rule' }), decisionContext); } + if (includeIntentPatterns && normalized) { + const faqMatch = matchDirectChatFaqRule(normalized); + if (faqMatch) { + return finalizeRouterClassification(normalizeClassification({ + route: CHAT_INTENT_ROUTE.DIRECT_CHAT, + confidence: 0.94, + reason: faqMatch.reason, + }, { source: 'rule' }), decisionContext); + } + } if (includeIntentPatterns && normalized && isPageDataDevIntent(normalized)) { return finalizeRouterClassification(normalizeClassification({ route: CHAT_INTENT_ROUTE.AGENT, @@ -1142,16 +1160,6 @@ export function classifyWithRules({ if (includeIntentPatterns && normalized && isRealtimeInfoQuestion(normalized)) { return finalizeRouterClassification(buildRealtimeInfoClassification(), decisionContext); } - if (includeIntentPatterns && normalized) { - const faqMatch = matchDirectChatFaqRule(normalized); - if (faqMatch) { - return finalizeRouterClassification(normalizeClassification({ - route: CHAT_INTENT_ROUTE.DIRECT_CHAT, - confidence: 0.94, - reason: faqMatch.reason, - }, { source: 'rule' }), decisionContext); - } - } if ( includeIntentPatterns && normalized && diff --git a/chat-intent-router.test.mjs b/chat-intent-router.test.mjs index 071f746..5fd0000 100644 --- a/chat-intent-router.test.mjs +++ b/chat-intent-router.test.mjs @@ -109,6 +109,25 @@ test('classifyWithRules routes greetings to direct chat', () => { assert.equal(result.source, 'rule'); }); +test('classifyWithRules routes explicit text-only Q&A to direct chat', () => { + for (const text of [ + '用三句话介绍苏州园林，不要生成页面', + '简要介绍一下 Rust 和 Go 的区别', + '推荐一个最适合第一次去的，一句话说明理由', + ]) { + const result = classifyWithRules({ + text, + userMessage: { + role: 'user', + content: [{ type: 'text', text }], + metadata: { displayText: text }, + }, + }); + assert.equal(result.route, CHAT_INTENT_ROUTE.DIRECT_CHAT, text); + assert.equal(result.source, 'rule', text); + } +}); + test('classifyWithRules routes page generation to agent orchestration', () => { const result = classifyWithRules({ text: '帮我做一个秋夜诗的 H5 页面', diff --git a/scenarios/token-benchmark-chat.json b/scenarios/token-benchmark-chat.json new file mode 100644 index 0000000..220e7d4 --- /dev/null +++ b/scenarios/token-benchmark-chat.json @@ -0,0 +1,39 @@ +{ + "id": "token-benchmark-chat", + "name": "Token 基准：Direct Chat 三轮", + "description": "简单对话，不走做页面 Agent，对比 pro vs flash", + "account": { + "username": "john2", + "password": "888888" + }, + "steps": [ + { + "action": "chat", + "label": "打招呼", + "message": "hi", + "expect": { + "assistantMinChars": 1, + "timeoutMs": 120000 + } + }, + { + "action": "chat", + "label": "苏州简介", + "message": "用三句话介绍苏州园林，不要生成页面", + "expect": { + "assistantMinChars": 20, + "timeoutMs": 120000, + "replyKeywords": ["苏州"] + } + }, + { + "action": "chat", + "label": "追问", + "message": "推荐一个最适合第一次去的，一句话说明理由", + "expect": { + "assistantMinChars": 10, + "timeoutMs": 120000 + } + } + ] +} diff --git a/scenarios/token-benchmark-page.json b/scenarios/token-benchmark-page.json new file mode 100644 index 0000000..7086dba --- /dev/null +++ b/scenarios/token-benchmark-page.json @@ -0,0 +1,26 @@ +{ + "id": "token-benchmark-page", + "name": "Token 基准：独立苏州页面", + "description": "每次使用唯一文件名，避免复用已有 HTML 污染上下文", + "account": { + "username": "john2", + "password": "888888" + }, + "steps": [ + { + "action": "chat", + "label": "生成独立苏州页面", + "message": "请帮我做一个全新的苏州一日游攻略页面，保存为 public/token-bench-{{PROFILE}}-{{RUN_ID}}.html，不要修改或复用已有页面，做完直接给我链接。", + "expect": { + "assistantMinChars": 20, + "timeoutMs": 600000, + "replyKeywords": ["苏州"], + "page": { + "keywords": ["苏州"], + "requirePublicLink": true, + "requireHttp200": true + } + } + } + ] +} diff --git a/scripts/benchmark-token-optimization.mjs b/scripts/benchmark-token-optimization.mjs new file mode 100755 index 0000000..aa66f5a --- /dev/null +++ b/scripts/benchmark-token-optimization.mjs @@ -0,0 +1,542 @@ +#!/usr/bin/env node +/** + * Local A/B benchmark: DeepSeek token usage vs scenario quality. + * + * Usage: + * node scripts/benchmark-token-optimization.mjs + * node scripts/benchmark-token-optimization.mjs --scenario john2-suzhou-page + * node scripts/benchmark-token-optimization.mjs --phase baseline + * node scripts/benchmark-token-optimization.mjs --phase optimized + * node scripts/benchmark-token-optimization.mjs --restore + * + * Does NOT touch production. Restores model + memory admin config on exit. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createDbPool } from '../db.mjs'; +import { createLlmProviderService } from '../llm-providers.mjs'; +import { createMemoryV2AdminConfigService } from '../memory-v2-admin-config.mjs'; +import { loadH5Environment } from './load-env.mjs'; +import { + createReporter, + createAgentRun, + extractAssistantTexts, + extractPublicLinks, + getSession, + loadScenario, + loginViaApi, + resolvePortalBase, + snapshotPublicHtml, + verifyPageAccess, + waitForAssistantGrowth, + waitForRunTerminal, +} from './scenario-test-lib.mjs'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const root = path.join(scriptDir, '..'); +const statePath = path.join(root, '.token-benchmark-state.json'); + +loadH5Environment(scriptDir); + +const PROFILES = { + baseline: { + label: 'baseline (deepseek-v4-pro + 默认记忆预算)', + model: 'deepseek-v4-pro', + memoryPatch: null, + }, + optimized: { + label: 'optimized (deepseek-v4-flash + 收紧记忆/Router)', + model: 'deepseek-v4-flash', + memoryPatch: { + runtimeControl: { + agentResolveLimit: '3', + }, + retriever: { + limit: '4', + tokenBudget: '900', + }, + persona: { + maxTokens: '300', + }, + chatIntentRouter: { + enabled: true, + shadowMode: false, + model: 'deepseek-chat', + memoryResolveLimit: '2', + }, + }, + }, +}; + +function parseArgs(argv) { + let scenarioId = 'token-benchmark-page'; + let phase = 'both'; + let port = Number(process.env.H5_PORT ?? 8081); + for (let i = 2; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--scenario' && argv[i + 1]) scenarioId = argv[++i]; + else if (arg === '--phase' && argv[i + 1]) phase = argv[++i]; + else if (arg === '--port' && argv[i + 1]) port = Number(argv[++i]); + else if (arg === '--restore') phase = 'restore'; + else if (arg === '-h' || arg === '--help') { + console.log(`Usage: + node scripts/benchmark-token-optimization.mjs [--scenario ] [--phase baseline|optimized|both|restore]`); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!['baseline', 'optimized', 'both', 'restore'].includes(phase)) { + throw new Error(`Invalid --phase: ${phase}`); + } + return { scenarioId, phase, port }; +} + +async function ensurePortalReady(baseUrl) { + const response = await fetch(`${baseUrl}/auth/status`); + if (!response.ok) { + throw new Error(`Portal 未就绪: ${baseUrl}/auth/status -> ${response.status}`); + } +} + +async function readCurrentModel(pool) { + const [rows] = await pool.query( + `SELECT default_model FROM h5_llm_provider_keys WHERE is_selected = 1 AND status = 'active' LIMIT 1`, + ); + return rows[0]?.default_model ?? null; +} + +async function saveState(pool, memoryConfigService) { + const model = await readCurrentModel(pool); + const memoryAdmin = await memoryConfigService.getAdminConfig().catch(() => null); + const payload = { + savedAt: Date.now(), + model, + memoryConfig: memoryAdmin?.config ?? null, + }; + fs.writeFileSync(statePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); + return payload; +} + +function loadSavedState() { + if (!fs.existsSync(statePath)) return null; + return JSON.parse(fs.readFileSync(statePath, 'utf8')); +} + +async function applyProfile(pool, llmProviderService, memoryConfigService, profileKey) { + const profile = PROFILES[profileKey]; + if (!profile) throw new Error(`Unknown profile: ${profileKey}`); + + const modelResult = await llmProviderService.setGlobalModel(profile.model); + if (!modelResult?.ok) { + throw new Error(`切换模型失败 (${profile.model}): ${modelResult?.message ?? 'unknown'}`); + } + + if (profile.memoryPatch) { + await memoryConfigService.updateAdminConfig(profile.memoryPatch, { + updatedBy: 'token-benchmark', + }); + } + + const currentModel = await readCurrentModel(pool); + console.log(`\n==> 已应用 ${profile.label}`); + console.log(` 全局模型: ${currentModel}`); + if (profile.memoryPatch) { + console.log(' 记忆/Router 预算: 已写入 h5_memory_v2_admin_config'); + } +} + +async function restoreSavedState(pool, llmProviderService, memoryConfigService) { + const saved = loadSavedState(); + if (!saved) { + console.log('无 .token-benchmark-state.json，跳过恢复'); + return false; + } + + if (saved.model) { + const result = await llmProviderService.setGlobalModel(saved.model); + if (!result?.ok) { + console.warn(`恢复模型失败: ${result?.message ?? 'unknown'}`); + } else { + console.log(`已恢复全局模型: ${saved.model}`); + } + } + + if (saved.memoryConfig) { + await memoryConfigService.updateAdminConfig(saved.memoryConfig, { + updatedBy: 'token-benchmark-restore', + }); + console.log('已恢复 memory v2 admin config'); + } + + fs.unlinkSync(statePath); + return true; +} + +async function collectUsageMetrics(pool, requestIds) { + if (!requestIds.length) { + return { inputTokens: 0, outputTokens: 0, costCents: 0, records: [] }; + } + const placeholders = requestIds.map(() => '?').join(', '); + const [rows] = await pool.query( + `SELECT request_id, input_tokens, output_tokens, cost_cents, billing_source, created_at + FROM h5_usage_records + WHERE request_id IN (${placeholders}) + ORDER BY created_at ASC`, + requestIds, + ); + let inputTokens = 0; + let outputTokens = 0; + let costCents = 0; + for (const row of rows) { + inputTokens += Number(row.input_tokens ?? 0); + outputTokens += Number(row.output_tokens ?? 0); + costCents += Number(row.cost_cents ?? 0); + } + return { inputTokens, outputTokens, costCents, records: rows }; +} + +async function collectRunEventTokens(pool, runIds) { + if (!runIds.length) return { inputTokens: 0, outputTokens: 0, events: [] }; + const placeholders = runIds.map(() => '?').join(', '); + const [rows] = await pool.query( + `SELECT run_id, data_json, created_at + FROM h5_agent_run_events + WHERE run_id IN (${placeholders}) AND event_type = 'session_finished' + ORDER BY created_at ASC`, + runIds, + ); + let inputTokens = 0; + let outputTokens = 0; + const events = []; + for (const row of rows) { + const data = typeof row.data_json === 'string' + ? JSON.parse(row.data_json) + : row.data_json; + const tokenState = data?.tokenState ?? data?.token_state ?? null; + if (!tokenState) continue; + const input = Number( + tokenState.accumulatedInputTokens + ?? tokenState.accumulated_input_tokens + ?? tokenState.inputTokens + ?? tokenState.input_tokens + ?? 0, + ); + const output = Number( + tokenState.accumulatedOutputTokens + ?? tokenState.accumulated_output_tokens + ?? tokenState.outputTokens + ?? tokenState.output_tokens + ?? 0, + ); + inputTokens += input; + outputTokens += output; + events.push({ runId: row.run_id, input, output, tokenState }); + } + return { inputTokens, outputTokens, events }; +} + +async function runScenarioBenchmark(scenario, port, { profileKey = 'run', runId = Date.now() } = {}) { + const reporter = createReporter(); + const baseUrl = resolvePortalBase(port); + const account = { + username: scenario.account?.username ?? 'john2', + password: process.env.JOHN_PASSWORD + ?? process.env.H5_ACCESS_PASSWORD + ?? scenario.account?.password + ?? '888888', + }; + + await ensurePortalReady(baseUrl); + + const auth = await loginViaApi(baseUrl, account, reporter); + const publishKey = auth.user?.id ?? auth.user?.publishSlug ?? account.username; + + let sessionId = null; + let assistantCount = 0; + let assistantCombinedLength = 0; + const tracked = { + requestIds: [], + runIds: [], + sessionIds: [], + replies: [], + pageLinks: [], + elapsedMs: 0, + profileKey, + runId, + }; + const started = Date.now(); + + for (const step of scenario.steps ?? []) { + if (step.action !== 'chat') continue; + + const message = String(step.message ?? '') + .replaceAll('{{PROFILE}}', profileKey) + .replaceAll('{{RUN_ID}}', String(runId)); + + let htmlBefore = []; + if (step.expect?.page) { + htmlBefore = await snapshotPublicHtml(publishKey); + } + + const run = await createAgentRun(baseUrl, auth.cookie, { + message, + sessionId: null, + selectedChatSkill: step.selectedChatSkill ?? null, + }); + tracked.requestIds.push(run.requestId); + tracked.runIds.push(run.runId); + + const terminal = await waitForRunTerminal( + baseUrl, + auth.cookie, + run.runId, + step.expect?.timeoutMs ?? 600_000, + ); + sessionId = terminal.sessionId ?? terminal.agent_session_id ?? run.sessionId ?? sessionId; + if (sessionId) tracked.sessionIds.push(sessionId); + + if (terminal.status === 'failed') { + reporter.fail('run 终态', terminal.error ?? 'failed'); + continue; + } + reporter.pass('run 终态', terminal.status); + + const reply = await waitForAssistantGrowth(baseUrl, auth.cookie, sessionId, { + previousCount: assistantCount, + previousCombinedLength: assistantCombinedLength, + minChars: step.expect?.assistantMinChars ?? 1, + timeoutMs: step.expect?.timeoutMs ?? 120_000, + }); + + if (!reply) { + reporter.fail('assistant 回复', '超时未收到新回复'); + continue; + } + + assistantCount = reply.count; + assistantCombinedLength = reply.combined.length; + tracked.replies.push({ + label: step.label ?? step.message, + chars: reply.combined.length, + elapsedMs: reply.elapsedMs, + text: reply.combined, + }); + reporter.pass('assistant 回复', `${reply.combined.length} 字 / ${reply.elapsedMs}ms`); + + const keywords = step.expect?.replyKeywords ?? []; + if (keywords.length) { + const hit = keywords.filter((word) => reply.combined.includes(word)); + if (hit.length === 0) reporter.fail('回复关键词', `未命中: ${keywords.join(', ')}`); + else reporter.pass('回复关键词', hit.join(', ')); + } + + tracked.pageLinks.push(...extractPublicLinks(reply.combined, baseUrl)); + + if (step.expect?.page) { + await verifyPageAccess({ + baseUrl, + cookie: auth.cookie, + publishKey, + replyText: reply.combined, + htmlBefore, + expect: step.expect.page, + uploadedAssetIds: [], + reporter, + }); + } + } + + tracked.elapsedMs = Date.now() - started; + return { + reporter, + tracked, + passed: reporter.issues.length === 0, + issueCount: reporter.issues.length, + }; +} + +function pctDelta(before, after) { + if (!before) return after ? 100 : 0; + return Math.round(((after - before) / before) * 1000) / 10; +} + +function printComparison(baseline, optimized) { + console.log('\n========================================'); + console.log('Token 优化 A/B 对比（本地）'); + console.log('========================================\n'); + + const rows = [ + ['指标', 'Baseline', 'Optimized', '变化'], + [ + '输入 Token (usage)', + String(baseline.usage.inputTokens), + String(optimized.usage.inputTokens), + `${pctDelta(baseline.usage.inputTokens, optimized.usage.inputTokens)}%`, + ], + [ + '输出 Token (usage)', + String(baseline.usage.outputTokens), + String(optimized.usage.outputTokens), + `${pctDelta(baseline.usage.outputTokens, optimized.usage.outputTokens)}%`, + ], + [ + '总 Token (usage)', + String(baseline.usage.inputTokens + baseline.usage.outputTokens), + String(optimized.usage.inputTokens + optimized.usage.outputTokens), + `${pctDelta( + baseline.usage.inputTokens + baseline.usage.outputTokens, + optimized.usage.inputTokens + optimized.usage.outputTokens, + )}%`, + ], + [ + '扣费 (分)', + String(baseline.usage.costCents), + String(optimized.usage.costCents), + `${pctDelta(baseline.usage.costCents, optimized.usage.costCents)}%`, + ], + [ + '场景耗时 (s)', + String(Math.round(baseline.tracked.elapsedMs / 1000)), + String(Math.round(optimized.tracked.elapsedMs / 1000)), + `${pctDelta(baseline.tracked.elapsedMs, optimized.tracked.elapsedMs)}%`, + ], + [ + '回复字数', + String(baseline.tracked.replies.at(-1)?.chars ?? 0), + String(optimized.tracked.replies.at(-1)?.chars ?? 0), + `${pctDelta( + baseline.tracked.replies.at(-1)?.chars ?? 0, + optimized.tracked.replies.at(-1)?.chars ?? 0, + )}%`, + ], + [ + '质量检查', + baseline.passed ? '通过' : `失败 ${baseline.issueCount} 项`, + optimized.passed ? '通过' : `失败 ${optimized.issueCount} 项`, + optimized.passed === baseline.passed ? '持平' : (optimized.passed ? '改善' : '下降'), + ], + ]; + + const widths = [0, 1, 2, 3].map((col) => Math.max(...rows.map((row) => row[col].length))); + for (const row of rows) { + console.log(row.map((cell, i) => cell.padEnd(widths[i] + 2)).join('')); + } + + console.log('\n--- Baseline 页面链接 ---'); + for (const link of baseline.tracked.pageLinks) console.log(link); + console.log('\n--- Optimized 页面链接 ---'); + for (const link of optimized.tracked.pageLinks) console.log(link); + + if (baseline.passed && optimized.passed) { + const totalBefore = baseline.usage.inputTokens + baseline.usage.outputTokens; + const totalAfter = optimized.usage.inputTokens + optimized.usage.outputTokens; + const saved = totalBefore - totalAfter; + console.log(`\n结论: 质量检查均通过。Token 总量 ${totalBefore} → ${totalAfter}（${saved >= 0 ? '节省' : '增加'} ${Math.abs(saved)}）`); + if (saved > 0 && optimized.passed) { + console.log('效果未打折，可继续观察更多场景。'); + } else if (saved <= 0) { + console.log('Token 未下降，需检查模型/Router/记忆配置是否生效。'); + } + } else { + console.log('\n结论: 存在质量回归，不建议直接上生产。'); + if (!optimized.passed) { + console.log('Optimized 失败项:'); + for (const issue of optimized.reporter.issues) { + console.log(` - ${issue.label}: ${issue.detail}`); + } + } + } +} + +async function runPhase(pool, llmProviderService, memoryConfigService, profileKey, scenario, port) { + await applyProfile(pool, llmProviderService, memoryConfigService, profileKey); + const runId = `${profileKey}-${Date.now()}`; + console.log(`\n>>> 开始 ${profileKey} 场景跑分 (runId=${runId})...`); + const result = await runScenarioBenchmark(scenario, port, { profileKey, runId }); + const usage = await collectUsageMetrics(pool, result.tracked.requestIds); + const eventTokens = await collectRunEventTokens(pool, result.tracked.runIds); + console.log(`\n--- ${profileKey} token 汇总 ---`); + console.log(`usage records: in=${usage.inputTokens} out=${usage.outputTokens} cost=${usage.costCents}分`); + console.log(`run events: in=${eventTokens.inputTokens} out=${eventTokens.outputTokens}`); + return { ...result, usage, eventTokens, profileKey }; +} + +async function main() { + const { scenarioId, phase, port } = parseArgs(process.argv); + const pool = createDbPool(process.env); + const llmProviderService = createLlmProviderService(pool, { + apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006', + apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret', + }); + const memoryConfigService = createMemoryV2AdminConfigService(pool); + + try { + if (phase === 'restore') { + await restoreSavedState(pool, llmProviderService, memoryConfigService); + return; + } + + const scenario = await loadScenario(scenarioId); + console.log(`场景: ${scenario.name ?? scenario.id}`); + console.log(`Portal: ${resolvePortalBase(port)}`); + + if (phase === 'both') { + await saveState(pool, memoryConfigService); + } + + let baselineResult = null; + let optimizedResult = null; + + if (phase === 'both' || phase === 'baseline') { + baselineResult = await runPhase( + pool, + llmProviderService, + memoryConfigService, + 'baseline', + scenario, + port, + ); + } + + if (phase === 'both' || phase === 'optimized') { + optimizedResult = await runPhase( + pool, + llmProviderService, + memoryConfigService, + 'optimized', + scenario, + port, + ); + } + + if (baselineResult && optimizedResult) { + printComparison(baselineResult, optimizedResult); + } + + if (phase === 'both') { + await restoreSavedState(pool, llmProviderService, memoryConfigService); + console.log('\n已恢复 benchmark 前的模型与 memory 配置。'); + } + } finally { + await pool.end?.(); + } +} + +main().catch(async (error) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + try { + const pool = createDbPool(process.env); + const llmProviderService = createLlmProviderService(pool, { + apiTarget: process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006', + apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret', + }); + const memoryConfigService = createMemoryV2AdminConfigService(pool); + await restoreSavedState(pool, llmProviderService, memoryConfigService); + await pool.end?.(); + } catch { + // best effort restore + } + process.exit(1); +}); diff --git a/scripts/sync-token-optimization-env-103.sh b/scripts/sync-token-optimization-env-103.sh new file mode 100755 index 0000000..7b5add4 --- /dev/null +++ b/scripts/sync-token-optimization-env-103.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Sync token-optimization Router / memory env to 103 Portal (.env only). +# Does NOT change global LLM model (keep deepseek-v4-pro for Agent quality). +# +# Usage: +# bash scripts/sync-token-optimization-env-103.sh # dry-run +# bash scripts/sync-token-optimization-env-103.sh --apply # write + restart portal +set -euo pipefail + +HOST="${MEMIND_103_HOST:-john@58.38.22.103}" +REMOTE_ENV="/Users/john/Project/Memind/.env" +APPLY=0 +if [[ "${1:-}" == "--apply" ]]; then + APPLY=1 +fi + +PATCH_LINES=( + "MEMIND_CHAT_LLM_ROUTER_ENABLED=1" + "MEMIND_CHAT_LLM_ROUTER_SHADOW=0" + "MEMIND_CHAT_ROUTER_CANARY_USER_IDS=" + "MEMIND_CHAT_ROUTER_MODEL=deepseek-chat" + "MEMIND_CHAT_ROUTER_MEMORY_LIMIT=2" + "MEMIND_CHAT_ROUTER_TIMEOUT_MS=2500" + "MEMIND_CHAT_ROUTER_MIN_CONFIDENCE=0.65" + "MEMORY_AGENT_RESOLVE_LIMIT=3" + "MEMORY_RETRIEVER_LIMIT=4" + "MEMORY_RETRIEVER_TOKEN_BUDGET=900" + "MEMORY_PERSONA_MAX_TOKENS=300" +) + +echo "==> Target: ${HOST}:${REMOTE_ENV}" +echo "==> Patch:" +printf ' %s\n' "${PATCH_LINES[@]}" +echo + +if [[ "${APPLY}" -ne 1 ]]; then + echo "Dry-run only. Re-run with --apply to write and restart Portal." + exit 0 +fi + +PATCH_B64=$(printf '%s\n' "${PATCH_LINES[@]}" | base64 | tr -d '\n') + +ssh "${HOST}" "bash -s" <> "\${ENV_FILE}" + fi +} + +while IFS='=' read -r key val; do + [[ -z "\${key}" ]] && continue + upsert "\${key}" "\${val}" +done <<'ENVPATCH' +$(printf '%s\n' "${PATCH_LINES[@]}") +ENVPATCH + +echo "==> Updated keys:" +grep -E '^(MEMIND_CHAT_|MEMORY_AGENT_RESOLVE|MEMORY_RETRIEVER|MEMORY_PERSONA_MAX)' "\${ENV_FILE}" || true + +GUI="\$(/usr/bin/stat -f %u /dev/console)" +launchctl kickstart -k "gui/\${GUI}/cn.tkmind.memind-portal" +sleep 3 +curl -sf http://127.0.0.1:8081/api/status >/dev/null +echo "Portal restarted and /api/status OK" +REMOTE + +echo "Done."