From 6d48480b1ce5775871c71e5b39d8ba83546baa9f Mon Sep 17 00:00:00 2001 From: john Date: Wed, 2 Sep 2026 14:12:48 +0800 Subject: [PATCH] Improve cross_device MemFuse recall with source tags and recallContext. Auto-enable device/location prefixes for cross_device scenarios, thread question_device into keyword and embedding paths, and boost lexical scores from source-tag tokens. Co-authored-by: Cursor --- docs/architecture/memfuse-bench-baseline.md | 23 ++++++ memory-v2-memfuse-bench.mjs | 8 +- memory-v2-pgvector.mjs | 88 ++++++++++++++++++++- memory-v2-pgvector.test.mjs | 20 +++++ 4 files changed, 134 insertions(+), 5 deletions(-) diff --git a/docs/architecture/memfuse-bench-baseline.md b/docs/architecture/memfuse-bench-baseline.md index ecbb390..310b150 100644 --- a/docs/architecture/memfuse-bench-baseline.md +++ b/docs/architecture/memfuse-bench-baseline.md @@ -167,6 +167,29 @@ DashScope 指标与门控前持平；lexical-hash 路径 recall +0.7pp。deep ra 相对初始 DashScope 基线（~23.7% recall），累计 recall **+5.1pp**、hitAny **+10pp**、MRR **+0.07**。 +### cross_device 弱维优化（k=20，commit 待填） + +`cross_device_*` 题全带 `question_device`（如 `phone_sarah`），证据跨 watch/phone/car 等设备，且 fusion 维平均 **15.2** 条 gold/题（recall@k 天然偏低，hitAny 更关键）。 + +改动： +- 含 `cross_device_*` 题的 scenario **自动启用 `[device · location]` source-tags**（写入语料前缀，供嵌入/lexical 共用） +- `resolve({ recallContext: { device, user } })`：device 拆词优先 keyword（`phone`/`sarah`），嵌入 query 追加 `device context:` 行 +- `latinWordQueryCoverage` 对 source-tag 前缀 token（含 `watch_ethan` → `ethan`）额外 +35% 权重上限 + +| 维度 | 指标 | 优化前 | 优化后 | Δ | +|---|---|---|---|---| +| information_fusion | candidateRecall | 70.1% | **78.6%** | +8.5pp | +| information_fusion | hitAny@k | 62.0% | **69.0%** | +7.0pp | +| information_fusion | recall@k | 11.0% | 11.8% | +0.8pp | +| causal_reasoning | candidateRecall | 66.0% | **82.3%** | +16.3pp | +| causal_reasoning | hitAny@k | 81.0% | 79.4% | −1.6pp | +| causal_reasoning | recall@k | 22.8% | 21.7% | −1.1pp | +| **全量 357** | recall@k | 28.8% | **28.6%** | ≈0 | +| **全量 357** | hitAny@k | 74.8% | **75.6%** | +0.8pp | +| **全量 357** | MRR | 0.392 | **0.397** | +0.005 | + +fusion 仍是最弱维，但候选/ hitAny 明显提升；全量指标基本持平略升。生产侧需在写入记忆时 materialize 与 bench 一致的 device 前缀（或等价 metadata）。 + ### 语义嵌入（DashScope / Qwen，推荐） memind_adm 后台 Providers 里配置的 **DashScope Qwen 密钥**存在 MySQL `h5_llm_provider_keys`，MemFuse bench 可直接复用，**不需要 OpenAI**： diff --git a/memory-v2-memfuse-bench.mjs b/memory-v2-memfuse-bench.mjs index bfca68d..22e17d8 100644 --- a/memory-v2-memfuse-bench.mjs +++ b/memory-v2-memfuse-bench.mjs @@ -450,6 +450,10 @@ export async function runMemFuseBenchCase({ query: testCase.question, limit, candidateLimit, + recallContext: { + device: testCase.questionDevice || null, + user: testCase.questionUser || null, + }, }); const retrieved = result.memories.map((memory) => memory.id).filter(Boolean); @@ -548,7 +552,9 @@ export async function runMemFuseBench({ questionIds, }); if (cases.length === 0) continue; - const corpus = buildScenarioCorpus(scenario, { includeSourceTags }); + const needsSourceTags = includeSourceTags + || cases.some((testCase) => String(testCase.dimension).startsWith('cross_device_')); + const corpus = buildScenarioCorpus(scenario, { includeSourceTags: needsSourceTags }); if (typeof prefetchEmbedTexts === 'function') { await prefetchEmbedTexts([ ...corpus.rows.map((row) => row.content), diff --git a/memory-v2-pgvector.mjs b/memory-v2-pgvector.mjs index ea04fe8..bf4095b 100644 --- a/memory-v2-pgvector.mjs +++ b/memory-v2-pgvector.mjs @@ -88,6 +88,7 @@ const ENGLISH_KEYWORD_STOP_TERMS = new Set([ function latinWordQueryCoverage(query, text) { const stop = ENGLISH_KEYWORD_STOP_TERMS; + const { body, tagTokens } = parseSourceTagPrefix(text); let queryTokens = [...latinWordTokens(query)].filter( (token) => token.length >= 3 && !stop.has(token), ); @@ -95,12 +96,84 @@ function latinWordQueryCoverage(query, text) { queryTokens = [...latinWordTokens(query)].filter((token) => token.length >= 3); } if (queryTokens.length === 0) return 0; - const textTokens = latinWordTokens(text); + const textTokens = latinWordTokens(body); let overlap = 0; for (const token of queryTokens) { if (textTokens.has(token)) overlap += 1; } - return overlap / queryTokens.length; + let coverage = overlap / queryTokens.length; + if (tagTokens.size > 0) { + let tagHits = 0; + for (const token of queryTokens) { + if (tagTokens.has(token)) tagHits += 1; + } + if (tagHits > 0) { + coverage = Math.min(1, coverage + (tagHits / queryTokens.length) * 0.35); + } + } + return coverage; +} + +function tokenizeSourceTagPart(tagPart) { + const tokens = new Set(); + for (const piece of String(tagPart ?? '').toLowerCase().split(/[^a-z0-9]+/)) { + if (!piece) continue; + for (const sub of piece.split('_')) { + if (sub.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(sub)) tokens.add(sub); + } + if (piece.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(piece)) tokens.add(piece); + } + return tokens; +} + +function parseSourceTagPrefix(text) { + const raw = String(text ?? ''); + const match = raw.match(/^\[([^\]]+)\]\s*/); + if (!match) return { body: raw, tagTokens: new Set() }; + const tagTokens = tokenizeSourceTagPart(match[1]); + return { body: raw.slice(match[0].length), tagTokens }; +} + +export function extractRecallContextTerms(recallContext) { + const terms = new Set(); + const device = String(recallContext?.device ?? '').trim().toLowerCase(); + if (device) { + terms.add(device); + for (const part of device.split('_')) { + if (part.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(part)) terms.add(part); + } + } + const user = String(recallContext?.user ?? '').trim().toLowerCase(); + if (user.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(user)) terms.add(user); + return [...terms]; +} + +export function mergeKeywordTerms(query, recallContext, options = {}) { + const contextTerms = extractRecallContextTerms(recallContext); + const queryTerms = extractKeywordTerms(query, options); + const properNouns = new Set(); + for (const match of String(query ?? '').matchAll(/\b[A-Z][a-z]{2,}\b/g)) { + properNouns.add(match[0].toLowerCase()); + } + const contextSet = new Set(contextTerms); + return [...new Set([...contextTerms, ...queryTerms])] + .sort((left, right) => { + const leftContext = contextSet.has(left) ? 1 : 0; + const rightContext = contextSet.has(right) ? 1 : 0; + if (leftContext !== rightContext) return rightContext - leftContext; + const leftProper = properNouns.has(left) ? 1 : 0; + const rightProper = properNouns.has(right) ? 1 : 0; + if (leftProper !== rightProper) return rightProper - leftProper; + return right.length - left.length; + }) + .slice(0, options.maxTerms ?? 12); +} + +export function buildEmbeddingQuery(query, recallContext) { + const parts = [String(query ?? '').trim()]; + const device = String(recallContext?.device ?? '').replace(/_/g, ' ').trim(); + if (device) parts.push(`device context: ${device}`); + return parts.filter(Boolean).join('\n'); } function queryScriptProfile(query) { @@ -276,8 +349,9 @@ async function fetchKeywordCandidates(pool, { tableName, limit = 20, maxTerms = 12, + recallContext = null, } = {}) { - const terms = extractKeywordTerms(query, { maxTerms }); + const terms = mergeKeywordTerms(query, recallContext, { maxTerms }); if (!terms.length) return []; const safeLimit = Math.max(1, Math.min(KEYWORD_RETURN_CAP, Number(limit) || 20)); const fetchCap = Math.min(KEYWORD_FETCH_CAP, Math.max(safeLimit, safeLimit * 10)); @@ -506,7 +580,8 @@ export function createPgvectorMemoryBackend({ const explicit = normalizeEmbedding(input?.embedding); if (explicit) return explicit; if (typeof embedQuery !== 'function' || !input?.query) return null; - return normalizeEmbedding(await embedQuery(input.query, input)); + const embeddingQuery = buildEmbeddingQuery(input.query, input.recallContext); + return normalizeEmbedding(await embedQuery(embeddingQuery, input)); } return { @@ -582,6 +657,7 @@ export function createPgvectorMemoryBackend({ query: input.query, tableName: resolvedTableName, limit: Math.max(limit, candidateLimit), + recallContext: input.recallContext ?? null, }).catch(() => []), ]); const memories = rankHybridCandidates( @@ -604,6 +680,10 @@ export const pgvectorMemoryBackendInternals = { recallRankingWeights, rankHybridCandidates, extractKeywordTerms, + extractRecallContextTerms, + mergeKeywordTerms, + buildEmbeddingQuery, + parseSourceTagPrefix, selectKeywordCandidateRows, selectVectorCandidateRows, dedupeContentPrefix, diff --git a/memory-v2-pgvector.test.mjs b/memory-v2-pgvector.test.mjs index 136bb5b..b436a93 100644 --- a/memory-v2-pgvector.test.mjs +++ b/memory-v2-pgvector.test.mjs @@ -353,6 +353,26 @@ test('extractKeywordTerms prioritizes proper nouns over generic English terms', assert.equal(terms.includes('soccer'), true); }); +test('mergeKeywordTerms prioritizes recallContext device tokens', () => { + const { mergeKeywordTerms } = pgvectorMemoryBackendInternals; + const terms = mergeKeywordTerms( + 'What happened this morning?', + { device: 'phone_sarah', user: 'Sarah' }, + { maxTerms: 8 }, + ); + assert.equal(terms.includes('sarah'), true); + assert.equal(terms.includes('phone'), true); + assert.ok(terms.indexOf('phone') < terms.indexOf('morning')); +}); + +test('latinWordQueryCoverage boosts tagged device tokens in source prefix', () => { + const { latinWordQueryCoverage } = pgvectorMemoryBackendInternals; + const query = 'What did Ethan do at soccer practice?'; + const tagged = '[watch_ethan · field] Ethan fell during soccer practice'; + const untagged = 'Ethan fell during soccer practice'; + assert.ok(latinWordQueryCoverage(query, tagged) >= latinWordQueryCoverage(query, untagged)); +}); + test('selectKeywordCandidateRows keeps priority-term gold under broad OR truncation', () => { const { selectKeywordCandidateRows, extractKeywordTerms } = pgvectorMemoryBackendInternals; const query = 'Why did David promise Ethan extra LEGO time on weekends?';