diff --git a/docs/regression-guards/memory-v2-candidate-and-lifecycle.md b/docs/regression-guards/memory-v2-candidate-and-lifecycle.md index db7a5c6..b8f97a0 100644 --- a/docs/regression-guards/memory-v2-candidate-and-lifecycle.md +++ b/docs/regression-guards/memory-v2-candidate-and-lifecycle.md @@ -17,6 +17,10 @@ 包含已注入的 `[Memory Context]`。提取器因此可能把旧记忆再次沉淀,而忽略用户本轮明确 要求保存的内容,形成旧记忆自我复制。 +当前生产使用的本地 hash embedding 只有 3 维,且连续中文可能被视为单个 token。即使正确 +记忆已进入 pgvector,它也可能排在向量 Top-K 之外。因此 pgvector 读取必须合并有界的 +向量候选与最近候选,再以中文字符 n-gram 查询覆盖率重排;无词法重合时保持向量排序。 + ## 必须保留的行为 1. `MEMORY_CANDIDATE_PERSISTENCE_ENABLED=1` 且 MySQL 可用时,Portal 必须先执行 @@ -35,6 +39,8 @@ 7. 显式记忆提取必须优先使用同一用户、同一会话中已成功 `h5_agent_runs.user_message_json` 保存的原始用户消息。不得把 Agent 编排提示或 `[Memory Context]` 当作用户的新记忆; 原始消息查询失败时必须 fail-open 到现有可见会话,不能阻塞保存接口。 +8. pgvector 召回必须同时覆盖有界向量候选和有界最近候选,去重后只返回请求的 limit; + 中文词法重排用于弥补低维本地 hash 的排序缺陷,且候选池上限不得超过 100。 ## 回归检查 diff --git a/memory-v2-pgvector.mjs b/memory-v2-pgvector.mjs index c253983..535276b 100644 --- a/memory-v2-pgvector.mjs +++ b/memory-v2-pgvector.mjs @@ -24,6 +24,43 @@ function vectorLiteral(embedding) { return `[${embedding.join(',')}]`; } +function normalizeSearchText(value) { + return String(value ?? '') + .normalize('NFKC') + .toLowerCase() + .replace(/[^a-z0-9\u4e00-\u9fff]+/gu, ''); +} + +function buildCharacterNgrams(value, size = 2) { + const text = normalizeSearchText(value); + if (!text) return new Set(); + if (text.length <= size) return new Set([text]); + const grams = new Set(); + for (let index = 0; index <= text.length - size; index += 1) { + grams.add(text.slice(index, index + size)); + } + return grams; +} + +function lexicalQueryCoverage(query, text) { + const queryGrams = buildCharacterNgrams(query); + if (queryGrams.size === 0) return 0; + const textGrams = buildCharacterNgrams(text); + let overlap = 0; + for (const gram of queryGrams) { + if (textGrams.has(gram)) overlap += 1; + } + return overlap / queryGrams.size; +} + +function timestampValue(value) { + if (value == null) return 0; + const numeric = Number(value); + if (Number.isFinite(numeric)) return numeric; + const parsed = Date.parse(String(value)); + return Number.isFinite(parsed) ? parsed : 0; +} + function normalizeRow(row) { const text = String(row?.content ?? row?.memory_text ?? row?.text ?? '').trim(); if (!text) return null; @@ -33,9 +70,38 @@ function normalizeRow(row) { text, score: row?.score == null ? null : Number(row.score), createdAt: row?.created_at ?? row?.createdAt ?? null, + updatedAt: row?.updated_at ?? row?.updatedAt ?? row?.created_at ?? row?.createdAt ?? null, }; } +function rankHybridCandidates(rows, query, limit) { + const byId = new Map(); + for (const row of rows ?? []) { + const memory = normalizeRow(row); + if (!memory) continue; + const key = memory.id ?? `${memory.label}:${memory.text}`; + if (!byId.has(key)) byId.set(key, memory); + } + return [...byId.values()] + .map((memory) => ({ + memory, + lexicalScore: lexicalQueryCoverage(query, memory.text), + vectorScore: Number.isFinite(memory.score) ? memory.score : -1, + updatedAt: timestampValue(memory.updatedAt), + })) + .sort((left, right) => { + if (left.lexicalScore !== right.lexicalScore) { + return right.lexicalScore - left.lexicalScore; + } + if (left.lexicalScore > 0 && left.updatedAt !== right.updatedAt) { + return right.updatedAt - left.updatedAt; + } + return right.vectorScore - left.vectorScore; + }) + .slice(0, limit) + .map(({ memory }) => memory); +} + export function createPgvectorMemoryBackend({ pool = null, enabled = false, @@ -75,15 +141,38 @@ export function createPgvectorMemoryBackend({ const embedding = await resolveEmbedding(input); if (!embedding) return { memories: [], semanticMemories: [] }; const limit = Math.max(1, Math.min(50, Number(input.limit ?? defaultLimit) || defaultLimit)); + const candidateLimit = Math.max( + limit, + Math.min(100, Number(input.candidateLimit ?? 50) || 50), + ); const sql = ` - SELECT id, content, type, created_at, 1 - (embedding <=> $2::vector) AS score - FROM ${resolvedTableName} - WHERE user_id = $1 - ORDER BY embedding <=> $2::vector - LIMIT $3 + WITH vector_candidates AS ( + SELECT id, content, type, created_at, updated_at, + 1 - (embedding <=> $2::vector) AS score, + 0 AS source_priority + FROM ${resolvedTableName} + WHERE user_id = $1 + ORDER BY embedding <=> $2::vector + LIMIT $3 + ), recent_candidates AS ( + SELECT id, content, type, created_at, updated_at, + 1 - (embedding <=> $2::vector) AS score, + 1 AS source_priority + FROM ${resolvedTableName} + WHERE user_id = $1 + ORDER BY updated_at DESC + LIMIT $3 + ) + SELECT DISTINCT ON (id) id, content, type, created_at, updated_at, score + FROM ( + SELECT * FROM vector_candidates + UNION ALL + SELECT * FROM recent_candidates + ) AS candidates + ORDER BY id, source_priority `; - const result = await pool.query(sql, [userId, vectorLiteral(embedding), limit]); - const memories = (result?.rows ?? []).map((row) => normalizeRow(row)).filter(Boolean); + const result = await pool.query(sql, [userId, vectorLiteral(embedding), candidateLimit]); + const memories = rankHybridCandidates(result?.rows ?? [], input.query, limit); return { semanticMemories: memories.map((item) => item.text), memories, @@ -91,3 +180,8 @@ export function createPgvectorMemoryBackend({ }, }; } + +export const pgvectorMemoryBackendInternals = { + lexicalQueryCoverage, + rankHybridCandidates, +}; diff --git a/memory-v2-pgvector.test.mjs b/memory-v2-pgvector.test.mjs index e29713f..ae4434f 100644 --- a/memory-v2-pgvector.test.mjs +++ b/memory-v2-pgvector.test.mjs @@ -1,7 +1,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { createMemoryV2 } from './memory-v2.mjs'; -import { createPgvectorMemoryBackend } from './memory-v2-pgvector.mjs'; +import { + createPgvectorMemoryBackend, + pgvectorMemoryBackendInternals, +} from './memory-v2-pgvector.mjs'; test('pgvector backend is disabled by default and does not query storage', async () => { let queried = false; @@ -85,7 +88,9 @@ test('pgvector backend performs parameterized vector lookup when explicitly enab assert.equal(queries.length, 1); assert.match(queries[0].sql, /FROM memory_embeddings/); - assert.deepEqual(queries[0].params, ['user-1', '[0.25,0.5,0.75]', 5]); + assert.match(queries[0].sql, /WITH vector_candidates/); + assert.match(queries[0].sql, /recent_candidates/); + assert.deepEqual(queries[0].params, ['user-1', '[0.25,0.5,0.75]', 50]); assert.deepEqual(result.semanticMemories, ['用户关注 Memory V2 的 facade 边界']); assert.deepEqual(result.memories, [ { @@ -94,10 +99,68 @@ test('pgvector backend performs parameterized vector lookup when explicitly enab text: '用户关注 Memory V2 的 facade 边界', score: 0.87, createdAt: '2026-07-02T00:00:00.000Z', + updatedAt: '2026-07-02T00:00:00.000Z', }, ]); }); +test('pgvector hybrid ranking recovers a recent Chinese memory missed by vector top-k', async () => { + const marker = 'MEM-RECALL-NEW'; + const backend = createPgvectorMemoryBackend({ + enabled: true, + pool: { + async query() { + return { + rows: [ + { + id: 1, + content: '用户以前关注贵州旅游攻略', + type: 'interest', + score: 0.91, + created_at: '2026-06-01T00:00:00.000Z', + updated_at: '2026-06-01T00:00:00.000Z', + }, + { + id: 2, + content: `用户的记忆召回灰度测试代号是 ${marker}`, + type: 'fact', + score: -0.49, + created_at: '2026-07-22T00:00:00.000Z', + updated_at: '2026-07-22T00:00:00.000Z', + }, + { + id: 3, + content: '用户偏好简洁回答', + type: 'preference', + score: 0.3, + created_at: '2026-07-21T00:00:00.000Z', + updated_at: '2026-07-21T00:00:00.000Z', + }, + ], + }; + }, + }, + embedQuery: async () => [0.25, 0.5, 0.75], + }); + + const result = await backend.resolve({ + userId: 'user-1', + query: '我之前让你记住的记忆召回灰度测试代号是什么?请只回答完整代号。', + limit: 3, + }); + + assert.match(result.memories[0].text, new RegExp(marker)); +}); + +test('pgvector hybrid ranking keeps vector order when query has no lexical overlap', () => { + const ranked = pgvectorMemoryBackendInternals.rankHybridCandidates([ + { id: 1, content: 'alpha', score: 0.2 }, + { id: 2, content: 'beta', score: 0.8 }, + ], '完全无关的中文查询', 2); + assert.equal(ranked[0].id, '2'); + assert.equal(ranked[1].id, '1'); +}); + test('pgvector backend validates table names before building SQL', () => { assert.throws( () => createPgvectorMemoryBackend({ tableName: 'memory_embeddings;DROP TABLE users' }), diff --git a/memory-v2-runtime.test.mjs b/memory-v2-runtime.test.mjs index 1128adc..a9bafac 100644 --- a/memory-v2-runtime.test.mjs +++ b/memory-v2-runtime.test.mjs @@ -385,7 +385,8 @@ test('createMemoryV2Runtime selects pgvector only when pool and embedding are co assert.equal(queries.length, 1); assert.equal(queries[0].options.connectionString, 'postgresql://local/memory'); assert.equal(queries[0].options.max, 2); - assert.deepEqual(queries[0].params, ['u1', '[0.1,0.2,0.3]', 8]); + assert.match(queries[0].sql, /recent_candidates/); + assert.deepEqual(queries[0].params, ['u1', '[0.1,0.2,0.3]', 50]); await memory.close(); assert.equal(poolEnded, true);