From 614837e1998589cf77acc68862e241146d724a9a Mon Sep 17 00:00:00 2001 From: john Date: Sun, 5 Jul 2026 11:04:25 +0800 Subject: [PATCH] fix: repair chat history from DB when Goose session has empty placeholders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Portal direct chat → goosed escalation can leave hollow Goose messages while h5_conversation_messages retains the full dialogue. Backfill session detail from DB, extend snapshot list fallback, and keep stored session id on transient boot failures so refresh no longer looks like a blank new chat. Co-authored-by: Cursor --- conversation-repair.mjs | 126 +++++++++++++++++++++++ conversation-repair.test.mjs | 86 ++++++++++++++++ scripts/memind-portal-tunnel.sh | 1 + scripts/verify-session-repair-local.mjs | 128 ++++++++++++++++++++++++ server.mjs | 30 +++++- src/hooks/useTKMindChat.ts | 3 +- tkmind-proxy.mjs | 6 +- 7 files changed, 371 insertions(+), 9 deletions(-) create mode 100644 conversation-repair.mjs create mode 100644 conversation-repair.test.mjs create mode 100644 scripts/verify-session-repair-local.mjs diff --git a/conversation-repair.mjs b/conversation-repair.mjs new file mode 100644 index 0000000..735af20 --- /dev/null +++ b/conversation-repair.mjs @@ -0,0 +1,126 @@ +/** + * Repair Goose conversations from h5_conversation_messages when agent sync + * left empty placeholders (portal direct chat → goosed escalation). + */ + +import { extractConversationMessageText } from './conversation-memory.mjs'; + +export function parseStoredConversationRow(row) { + if (!row) return null; + const rawJson = row.raw_json ?? row.rawJson; + if (typeof rawJson === 'string' && rawJson.trim()) { + try { + const parsed = JSON.parse(rawJson); + if (parsed && typeof parsed === 'object') { + return { + ...parsed, + metadata: { + userVisible: true, + agentVisible: true, + ...(parsed.metadata && typeof parsed.metadata === 'object' ? parsed.metadata : {}), + }, + }; + } + } catch { + // fall through to text reconstruction + } + } + const text = String(row.text ?? '').trim(); + if (!text) return null; + const messageKey = String(row.message_key ?? row.messageKey ?? '').trim(); + const createdMs = Number(row.created_at ?? row.createdAt ?? 0) || Date.now(); + return { + id: messageKey || undefined, + role: String(row.role ?? 'user').trim() || 'user', + created: Math.floor(createdMs / 1000), + content: [{ type: 'text', text }], + metadata: { + userVisible: true, + agentVisible: true, + ...(String(row.role ?? '') === 'assistant' ? { source: 'conversation-db-repair' } : {}), + }, + }; +} + +export function countNonEmptyConversationMessages(messages) { + if (!Array.isArray(messages)) return 0; + return messages.filter((message) => { + if (message?.metadata?.userVisible === false) return false; + return Boolean(extractConversationMessageText(message)); + }).length; +} + +export function buildConversationFromDbRows(dbRows) { + if (!Array.isArray(dbRows) || dbRows.length === 0) return []; + const byKey = new Map(); + for (const row of dbRows) { + const parsed = parseStoredConversationRow(row); + if (!parsed) continue; + const key = + String(row.message_key ?? row.messageKey ?? parsed.id ?? '').trim() || + `${parsed.role}:${extractConversationMessageText(parsed).slice(0, 48)}`; + byKey.set(key, { + message: parsed, + sequenceNo: Number(row.sequence_no ?? row.sequenceNo ?? 0), + createdAt: Number(row.created_at ?? row.createdAt ?? 0), + }); + } + return [...byKey.values()] + .sort((left, right) => { + if (left.sequenceNo !== right.sequenceNo) return left.sequenceNo - right.sequenceNo; + if (left.createdAt !== right.createdAt) return left.createdAt - right.createdAt; + return String(left.message.id ?? '').localeCompare(String(right.message.id ?? '')); + }) + .map((entry) => entry.message); +} + +export function shouldRepairConversationFromDb(gooseMessages, dbRows) { + const dbBuilt = buildConversationFromDbRows(dbRows); + const dbCount = countNonEmptyConversationMessages(dbBuilt); + if (dbCount === 0) return false; + + const gooseVisible = Array.isArray(gooseMessages) + ? gooseMessages.filter((message) => message?.metadata?.userVisible !== false) + : []; + const gooseNonEmpty = countNonEmptyConversationMessages(gooseVisible); + const gooseEmptyVisible = gooseVisible.some( + (message) => !extractConversationMessageText(message), + ); + + if (gooseNonEmpty === 0) return true; + if (dbCount > gooseNonEmpty) return true; + if (gooseEmptyVisible && dbCount >= gooseNonEmpty) return true; + return false; +} + +export function repairConversationFromDbRows(gooseMessages, dbRows) { + const dbBuilt = buildConversationFromDbRows(dbRows); + if (!shouldRepairConversationFromDb(gooseMessages, dbRows)) { + return Array.isArray(gooseMessages) ? gooseMessages : []; + } + if (dbBuilt.length === 0) return Array.isArray(gooseMessages) ? gooseMessages : []; + return dbBuilt; +} + +export async function loadSessionConversationRows(pool, sessionId, userId) { + if (!pool || !sessionId || !userId) return []; + const [rows] = await pool.query( + `SELECT message_key, sequence_no, role, text, raw_json, created_at + FROM h5_conversation_messages + WHERE agent_session_id = ? AND user_id = ? + ORDER BY sequence_no ASC, created_at ASC`, + [sessionId, userId], + ); + return rows; +} + +export async function repairSessionConversationFromDb(pool, gooseSession, sessionId, userId) { + if (!gooseSession || !pool || !sessionId || !userId) return gooseSession; + const dbRows = await loadSessionConversationRows(pool, sessionId, userId); + const conversation = Array.isArray(gooseSession.conversation) ? gooseSession.conversation : []; + if (!shouldRepairConversationFromDb(conversation, dbRows)) return gooseSession; + return { + ...gooseSession, + conversation: repairConversationFromDbRows(conversation, dbRows), + }; +} diff --git a/conversation-repair.test.mjs b/conversation-repair.test.mjs new file mode 100644 index 0000000..60f4ea8 --- /dev/null +++ b/conversation-repair.test.mjs @@ -0,0 +1,86 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + buildConversationFromDbRows, + countNonEmptyConversationMessages, + parseStoredConversationRow, + repairConversationFromDbRows, + shouldRepairConversationFromDb, +} from './conversation-repair.mjs'; + +const gooseMsg = (id, role, text) => ({ + id, + role, + metadata: { userVisible: true }, + content: text ? [{ type: 'text', text }] : [], +}); + +test('parseStoredConversationRow prefers raw_json', () => { + const row = { + message_key: 'abc', + role: 'user', + text: 'fallback', + raw_json: JSON.stringify({ + id: 'abc', + role: 'user', + content: [{ type: 'text', text: 'from raw' }], + metadata: { userVisible: true, displayText: '可见文案' }, + }), + created_at: 1_700_000_000_000, + }; + const parsed = parseStoredConversationRow(row); + assert.equal(parsed.content[0].text, 'from raw'); + assert.equal(parsed.metadata.displayText, '可见文案'); +}); + +test('shouldRepairConversationFromDb when goose has empty placeholders', () => { + const goose = [ + gooseMsg('u1', 'user', '编排前缀'), + gooseMsg('a1', 'assistant', '好的'), + gooseMsg('msg_empty1', 'assistant', ''), + gooseMsg('msg_empty2', 'user', ''), + ]; + const dbRows = [ + { + message_key: 'u-real', + role: 'user', + text: '小朋友在上海读初中', + created_at: 1, + sequence_no: 0, + }, + { + message_key: 'a-real', + role: 'assistant', + text: '关于中考选择的分析…', + created_at: 2, + sequence_no: 1, + }, + ]; + assert.equal(shouldRepairConversationFromDb(goose, dbRows), true); + const repaired = repairConversationFromDbRows(goose, dbRows); + assert.equal(countNonEmptyConversationMessages(repaired), 2); + assert.match(repaired[0].content[0].text, /上海读初中/); + assert.match(repaired[1].content[0].text, /中考/); +}); + +test('shouldRepairConversationFromDb returns false when goose is complete', () => { + const goose = [ + gooseMsg('u1', 'user', 'hello'), + gooseMsg('a1', 'assistant', 'hi there'), + ]; + const dbRows = [ + { message_key: 'u1', role: 'user', text: 'hello', created_at: 1, sequence_no: 0 }, + ]; + assert.equal(shouldRepairConversationFromDb(goose, dbRows), false); +}); + +test('buildConversationFromDbRows dedupes by message_key', () => { + const rows = [ + { message_key: 'same', role: 'user', text: 'first', created_at: 1, sequence_no: 0 }, + { message_key: 'same', role: 'user', text: 'second', created_at: 2, sequence_no: 0 }, + ]; + const built = buildConversationFromDbRows(rows); + assert.equal(built.length, 1); + assert.equal(built[0].content[0].text, 'second'); +}); diff --git a/scripts/memind-portal-tunnel.sh b/scripts/memind-portal-tunnel.sh index 3d543ea..f2c55ae 100644 --- a/scripts/memind-portal-tunnel.sh +++ b/scripts/memind-portal-tunnel.sh @@ -2,6 +2,7 @@ # 反向 SSH 隧道：105 通过 127.0.0.1:19081 访问本机 Portal (:8081) set -euo pipefail +# 103 上由 john 用户的 LaunchAgent 执行；SSH 目标为 105 的 root（~/.ssh/config Host ssh105-public） HOST="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}" LOCAL_PORT="${MEMIND_PORTAL_TUNNEL_LOCAL_PORT:-8081}" REMOTE_PORT="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}" diff --git a/scripts/verify-session-repair-local.mjs b/scripts/verify-session-repair-local.mjs new file mode 100644 index 0000000..bd632ba --- /dev/null +++ b/scripts/verify-session-repair-local.mjs @@ -0,0 +1,128 @@ +#!/usr/bin/env node +/** + * Simulate GET /api/sessions/:id repair for a session using live 103 data. + * Usage: node scripts/verify-session-repair-local.mjs [sessionId] [userId] + */ +import { createConnection } from 'mysql2/promise'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + countNonEmptyConversationMessages, + loadSessionConversationRows, + repairConversationFromDbRows, + shouldRepairConversationFromDb, +} from '../conversation-repair.mjs'; +import { extractConversationMessageText } from '../conversation-memory.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = join(__dirname, '..'); + +const sessionId = process.argv[2] ?? '20260705_2'; +const userId = process.argv[3] ?? 'a6fb1e97-2b0f-447b-b138-4561d8e5c53e'; + +function previewMessage(message, index) { + const text = extractConversationMessageText(message).replace(/\s+/g, ' ').slice(0, 90); + return `${index}. [${message.role}] ${text || '(empty)'}`; +} + +async function load103Env() { + const { execSync } = await import('node:child_process'); + const envText = execSync('ssh -o BatchMode=yes john@58.38.22.103 cat /Users/john/Project/Memind/.env', { + encoding: 'utf8', + }); + const dbLine = envText.match(/^DATABASE_URL=(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, ''); + const secret = envText.match(/^TKMIND_SERVER__SECRET_KEY=(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, ''); + if (!dbLine || !secret) throw new Error('missing DATABASE_URL or TKMIND_SERVER__SECRET_KEY on 103'); + const url = new URL(dbLine); + return { + pool: await createConnection({ + host: url.hostname, + port: Number(url.port || 3306), + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + database: url.pathname.replace(/^\//, ''), + }), + secret, + }; +} + +async function fetchGooseSession(secret, sid) { + const { execSync } = await import('node:child_process'); + const json = execSync( + `ssh -o BatchMode=yes john@58.38.22.103 'SECRET=$(grep ^TKMIND_SERVER__SECRET_KEY= /Users/john/Project/Memind/.env | cut -d= -f2- | tr -d "\\""); curl -k -sf "https://127.0.0.1:18006/sessions/${sid}" -H "X-Secret-Key: $SECRET"'`, + { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }, + ); + return JSON.parse(json); +} + +async function fetchProductionApi(sessionId) { + const { execSync } = await import('node:child_process'); + const login = execSync( + `curl -sk -c /tmp/verify-session-repair-cj.txt -X POST https://m.tkmind.cn/auth/login -H 'Content-Type: application/json' -d '{"username":"john","password":"981122tj"}'`, + { encoding: 'utf8' }, + ); + const auth = JSON.parse(login); + if (!auth.authenticated) throw new Error('login failed'); + const detail = execSync( + `curl -sk -b /tmp/verify-session-repair-cj.txt 'https://m.tkmind.cn/api/sessions/${encodeURIComponent(sessionId)}'`, + { encoding: 'utf8' }, + ); + return JSON.parse(detail); +} + +async function main() { + console.log(`\n=== 模拟验证 session ${sessionId} (user ${userId}) ===\n`); + + const { pool, secret } = await load103Env(); + const gooseSession = await fetchGooseSession(secret, sessionId); + const dbRows = await loadSessionConversationRows(pool, sessionId, userId); + await pool.end(); + + const gooseConversation = (gooseSession.conversation ?? []).filter( + (m) => m?.metadata?.userVisible !== false, + ); + const repairedConversation = repairConversationFromDbRows(gooseConversation, dbRows); + + console.log('--- Goose 原始（API 发版前会返回这类数据）---'); + console.log(`消息数: ${gooseConversation.length}, 非空: ${countNonEmptyConversationMessages(gooseConversation)}`); + gooseConversation.forEach((m, i) => console.log(previewMessage(m, i))); + + console.log('\n--- DB h5_conversation_messages ---'); + console.log(`行数: ${dbRows.length}`); + dbRows.forEach((row, i) => { + const text = String(row.text ?? '').replace(/\s+/g, ' ').slice(0, 90); + console.log(`${i}. [${row.role}] seq=${row.sequence_no} ${text || '(empty)'}`); + }); + + console.log(`\nshouldRepair: ${shouldRepairConversationFromDb(gooseConversation, dbRows)}`); + + console.log('\n--- 本地 repair 后（发版后 API 应接近此结果）---'); + console.log(`消息数: ${repairedConversation.length}, 非空: ${countNonEmptyConversationMessages(repairedConversation)}`); + repairedConversation.forEach((m, i) => console.log(previewMessage(m, i))); + + console.log('\n--- 当前生产 API（未发版，无 repair）---'); + try { + const prod = await fetchProductionApi(sessionId); + const prodConv = (prod.conversation ?? []).filter((m) => m?.metadata?.userVisible !== false); + console.log(`name: ${prod.name}`); + console.log(`消息数: ${prodConv.length}, 非空: ${countNonEmptyConversationMessages(prodConv)}`); + prodConv.forEach((m, i) => console.log(previewMessage(m, i))); + } catch (err) { + console.log(`生产 API 拉取失败: ${err instanceof Error ? err.message : err}`); + } + + console.log('\n=== 结论 ==='); + const repairedNonEmpty = countNonEmptyConversationMessages(repairedConversation); + const gooseNonEmpty = countNonEmptyConversationMessages(gooseConversation); + if (repairedNonEmpty > gooseNonEmpty) { + console.log(`✔ repair 后非空消息 ${gooseNonEmpty} → ${repairedNonEmpty}，发版后 john 应能看到完整历史`); + } else { + console.log(`✖ repair 未改善非空消息数 (${gooseNonEmpty} → ${repairedNonEmpty})`); + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/server.mjs b/server.mjs index 2f38dbd..c650fe5 100644 --- a/server.mjs +++ b/server.mjs @@ -164,6 +164,7 @@ import { createFeedbackService } from './user-feedback.mjs'; import { startScheduleReminderWorker } from './schedule-reminder-worker.mjs'; import { createLlmProviderService, RELAY_BOOTSTRAP } from './llm-providers.mjs'; import { createDirectChatService, isDirectChatSessionId, isPortalDirectChatSnapshot, sendDirectChatSessionEvents, shouldExpirePortalDirectChatSnapshot } from './direct-chat-service.mjs'; +import { repairSessionConversationFromDb } from './conversation-repair.mjs'; import { createManagedChatIntentRouter } from './chat-intent-router.mjs'; import { createSessionSnapshotService } from './session-snapshot.mjs'; import { createConversationMemoryService } from './conversation-memory.mjs'; @@ -1888,7 +1889,7 @@ async function ensureUserMemoryCapability(req, res) { return capabilityState; } -async function loadUserVisibleConversation(sessionId) { +async function loadUserVisibleConversation(sessionId, userId) { const target = await tkmindProxy.resolveTarget(sessionId); const upstream = await tkmindProxy.apiFetchTo(target, `/sessions/${encodeURIComponent(sessionId)}`, { method: 'GET', @@ -1897,7 +1898,10 @@ async function loadUserVisibleConversation(sessionId) { const message = await upstream.text().catch(() => ''); throw new Error(message || '读取会话失败'); } - const session = await upstream.json(); + let session = await upstream.json(); + if (authPool && userId) { + session = await repairSessionConversationFromDb(authPool, session, sessionId, userId); + } return (session?.conversation ?? []).filter((message) => message?.metadata?.userVisible); } @@ -1928,7 +1932,7 @@ api.post('/user-memory/v1/remember-recent', async (req, res) => { } try { - const messages = await loadUserVisibleConversation(sessionId); + const messages = await loadUserVisibleConversation(sessionId, req.currentUser.id); const result = await memoryV2.write({ userId: req.currentUser.id, sessionId, @@ -4469,11 +4473,19 @@ api.get('/sessions/:sessionId', async (req, res, next) => { req.currentUser, ); // Cache hit — reconstruct a Goose-compatible session response. - const cachedGooseSession = { + let cachedGooseSession = { ...snapshot.session, // Embed only userVisible messages so getSession callers still work. conversation: sanitizedMessages, }; + if (authPool) { + cachedGooseSession = await repairSessionConversationFromDb( + authPool, + cachedGooseSession, + sessionId, + req.currentUser.id, + ); + } return res.json(cachedGooseSession); } } @@ -4494,13 +4506,21 @@ api.get('/sessions/:sessionId', async (req, res, next) => { const text = await upstream.text().catch(() => ''); return res.status(upstream.status).send(text); } - const gooseSession = await upstream.json(); + let gooseSession = await upstream.json(); if (Array.isArray(gooseSession.conversation)) { gooseSession.conversation = sanitizeSessionConversationPublicHtmlLinks( gooseSession.conversation, req.currentUser, ); } + if (authPool) { + gooseSession = await repairSessionConversationFromDb( + authPool, + gooseSession, + sessionId, + req.currentUser.id, + ); + } // Write-through: persist snapshot async, don't block the response. if (sessionSnapshotService?.isEnabled()) { const messages = (gooseSession.conversation ?? []) diff --git a/src/hooks/useTKMindChat.ts b/src/hooks/useTKMindChat.ts index 680556b..cb60848 100644 --- a/src/hooks/useTKMindChat.ts +++ b/src/hooks/useTKMindChat.ts @@ -1125,7 +1125,6 @@ export function useTKMindChat( setMessages([]); setChatState('idle'); } else if (isTransientConnectError(err)) { - clearStoredSessionId(userRef.current?.id); setSession(null); messageHistoryLoadedCountRef.current = 0; messageHistoryTotalRef.current = 0; @@ -1134,7 +1133,7 @@ export function useTKMindChat( setMessageHistoryTotal(0); setMessages([]); setChatState('idle'); - setNotice('上次会话恢复超时，已为你准备新对话，可直接发送消息'); + setNotice('上次会话恢复超时，请点左上角 ☰ 选择历史对话，或稍后刷新重试'); } else { throw err; } diff --git a/tkmind-proxy.mjs b/tkmind-proxy.mjs index 68223f0..def7f13 100644 --- a/tkmind-proxy.mjs +++ b/tkmind-proxy.mjs @@ -1611,8 +1611,10 @@ export function createTkmindProxy({ } const directSessionIds = [...owned].filter(isDirectChatSessionId); - if (directSessionIds.length > 0 && sessionSnapshotService?.isEnabled?.()) { - for (const sessionId of directSessionIds) { + const ownedMissingFromGoose = [...owned].filter((sessionId) => !sessionsById.has(sessionId)); + const snapshotFallbackIds = [...new Set([...directSessionIds, ...ownedMissingFromGoose])]; + if (snapshotFallbackIds.length > 0 && sessionSnapshotService?.isEnabled?.()) { + for (const sessionId of snapshotFallbackIds) { if (sessionsById.has(sessionId)) continue; const snapshot = await sessionSnapshotService.get(sessionId).catch(() => null); if (!snapshot?.session) continue;