Improve cross_device MemFuse recall with source tags and recallContext.
Auto-enable device/location prefixes for cross_device scenarios, thread question_device into keyword and embedding paths, and boost lexical scores from source-tag tokens. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+84
-4
@@ -88,6 +88,7 @@ const ENGLISH_KEYWORD_STOP_TERMS = new Set([
|
||||
|
||||
function latinWordQueryCoverage(query, text) {
|
||||
const stop = ENGLISH_KEYWORD_STOP_TERMS;
|
||||
const { body, tagTokens } = parseSourceTagPrefix(text);
|
||||
let queryTokens = [...latinWordTokens(query)].filter(
|
||||
(token) => token.length >= 3 && !stop.has(token),
|
||||
);
|
||||
@@ -95,12 +96,84 @@ function latinWordQueryCoverage(query, text) {
|
||||
queryTokens = [...latinWordTokens(query)].filter((token) => token.length >= 3);
|
||||
}
|
||||
if (queryTokens.length === 0) return 0;
|
||||
const textTokens = latinWordTokens(text);
|
||||
const textTokens = latinWordTokens(body);
|
||||
let overlap = 0;
|
||||
for (const token of queryTokens) {
|
||||
if (textTokens.has(token)) overlap += 1;
|
||||
}
|
||||
return overlap / queryTokens.length;
|
||||
let coverage = overlap / queryTokens.length;
|
||||
if (tagTokens.size > 0) {
|
||||
let tagHits = 0;
|
||||
for (const token of queryTokens) {
|
||||
if (tagTokens.has(token)) tagHits += 1;
|
||||
}
|
||||
if (tagHits > 0) {
|
||||
coverage = Math.min(1, coverage + (tagHits / queryTokens.length) * 0.35);
|
||||
}
|
||||
}
|
||||
return coverage;
|
||||
}
|
||||
|
||||
function tokenizeSourceTagPart(tagPart) {
|
||||
const tokens = new Set();
|
||||
for (const piece of String(tagPart ?? '').toLowerCase().split(/[^a-z0-9]+/)) {
|
||||
if (!piece) continue;
|
||||
for (const sub of piece.split('_')) {
|
||||
if (sub.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(sub)) tokens.add(sub);
|
||||
}
|
||||
if (piece.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(piece)) tokens.add(piece);
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
function parseSourceTagPrefix(text) {
|
||||
const raw = String(text ?? '');
|
||||
const match = raw.match(/^\[([^\]]+)\]\s*/);
|
||||
if (!match) return { body: raw, tagTokens: new Set() };
|
||||
const tagTokens = tokenizeSourceTagPart(match[1]);
|
||||
return { body: raw.slice(match[0].length), tagTokens };
|
||||
}
|
||||
|
||||
export function extractRecallContextTerms(recallContext) {
|
||||
const terms = new Set();
|
||||
const device = String(recallContext?.device ?? '').trim().toLowerCase();
|
||||
if (device) {
|
||||
terms.add(device);
|
||||
for (const part of device.split('_')) {
|
||||
if (part.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(part)) terms.add(part);
|
||||
}
|
||||
}
|
||||
const user = String(recallContext?.user ?? '').trim().toLowerCase();
|
||||
if (user.length >= 3 && !ENGLISH_KEYWORD_STOP_TERMS.has(user)) terms.add(user);
|
||||
return [...terms];
|
||||
}
|
||||
|
||||
export function mergeKeywordTerms(query, recallContext, options = {}) {
|
||||
const contextTerms = extractRecallContextTerms(recallContext);
|
||||
const queryTerms = extractKeywordTerms(query, options);
|
||||
const properNouns = new Set();
|
||||
for (const match of String(query ?? '').matchAll(/\b[A-Z][a-z]{2,}\b/g)) {
|
||||
properNouns.add(match[0].toLowerCase());
|
||||
}
|
||||
const contextSet = new Set(contextTerms);
|
||||
return [...new Set([...contextTerms, ...queryTerms])]
|
||||
.sort((left, right) => {
|
||||
const leftContext = contextSet.has(left) ? 1 : 0;
|
||||
const rightContext = contextSet.has(right) ? 1 : 0;
|
||||
if (leftContext !== rightContext) return rightContext - leftContext;
|
||||
const leftProper = properNouns.has(left) ? 1 : 0;
|
||||
const rightProper = properNouns.has(right) ? 1 : 0;
|
||||
if (leftProper !== rightProper) return rightProper - leftProper;
|
||||
return right.length - left.length;
|
||||
})
|
||||
.slice(0, options.maxTerms ?? 12);
|
||||
}
|
||||
|
||||
export function buildEmbeddingQuery(query, recallContext) {
|
||||
const parts = [String(query ?? '').trim()];
|
||||
const device = String(recallContext?.device ?? '').replace(/_/g, ' ').trim();
|
||||
if (device) parts.push(`device context: ${device}`);
|
||||
return parts.filter(Boolean).join('\n');
|
||||
}
|
||||
|
||||
function queryScriptProfile(query) {
|
||||
@@ -276,8 +349,9 @@ async function fetchKeywordCandidates(pool, {
|
||||
tableName,
|
||||
limit = 20,
|
||||
maxTerms = 12,
|
||||
recallContext = null,
|
||||
} = {}) {
|
||||
const terms = extractKeywordTerms(query, { maxTerms });
|
||||
const terms = mergeKeywordTerms(query, recallContext, { maxTerms });
|
||||
if (!terms.length) return [];
|
||||
const safeLimit = Math.max(1, Math.min(KEYWORD_RETURN_CAP, Number(limit) || 20));
|
||||
const fetchCap = Math.min(KEYWORD_FETCH_CAP, Math.max(safeLimit, safeLimit * 10));
|
||||
@@ -506,7 +580,8 @@ export function createPgvectorMemoryBackend({
|
||||
const explicit = normalizeEmbedding(input?.embedding);
|
||||
if (explicit) return explicit;
|
||||
if (typeof embedQuery !== 'function' || !input?.query) return null;
|
||||
return normalizeEmbedding(await embedQuery(input.query, input));
|
||||
const embeddingQuery = buildEmbeddingQuery(input.query, input.recallContext);
|
||||
return normalizeEmbedding(await embedQuery(embeddingQuery, input));
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -582,6 +657,7 @@ export function createPgvectorMemoryBackend({
|
||||
query: input.query,
|
||||
tableName: resolvedTableName,
|
||||
limit: Math.max(limit, candidateLimit),
|
||||
recallContext: input.recallContext ?? null,
|
||||
}).catch(() => []),
|
||||
]);
|
||||
const memories = rankHybridCandidates(
|
||||
@@ -604,6 +680,10 @@ export const pgvectorMemoryBackendInternals = {
|
||||
recallRankingWeights,
|
||||
rankHybridCandidates,
|
||||
extractKeywordTerms,
|
||||
extractRecallContextTerms,
|
||||
mergeKeywordTerms,
|
||||
buildEmbeddingQuery,
|
||||
parseSourceTagPrefix,
|
||||
selectKeywordCandidateRows,
|
||||
selectVectorCandidateRows,
|
||||
dedupeContentPrefix,
|
||||
|
||||
Reference in New Issue
Block a user