john
79 lines
2.8 KiB
JavaScript
79 lines
2.8 KiB
JavaScript
const CJK_QUERY_STOP_WORDS = new Set([
|
||
'之前', '什么', '哪个', '多少', '记得', '记住', '有没有', '是否', '我的', '你是',
|
||
'我们', '一下', '可以', '这个', '那个', '请问', '告诉', '说过', '提到', '聊过',
|
||
]);
|
||
|
||
function extractQueryTokens(query) {
|
||
const q = String(query ?? '').trim();
|
||
if (!q) return [];
|
||
const tokens = new Set();
|
||
for (const part of q.replace(/[^\p{L}\p{N}]+/gu, ' ').split(/\s+/)) {
|
||
const token = part.trim();
|
||
if (token.length >= 2 && !CJK_QUERY_STOP_WORDS.has(token)) tokens.add(token);
|
||
}
|
||
for (const run of q.match(/[\p{Script=Han}]{2,}/gu) ?? []) {
|
||
if (run.length <= 6 && !CJK_QUERY_STOP_WORDS.has(run)) tokens.add(run);
|
||
for (let i = 0; i < run.length - 1; i += 1) {
|
||
const bigram = run.slice(i, i + 2);
|
||
if (!CJK_QUERY_STOP_WORDS.has(bigram)) tokens.add(bigram);
|
||
}
|
||
}
|
||
return [...tokens];
|
||
}
|
||
|
||
function scoreMemoryForRecall(text, tokens) {
|
||
let score = 0;
|
||
for (const token of tokens) {
|
||
if (text.includes(token)) score += 1;
|
||
}
|
||
if (/别名.{0,8}是/u.test(text)) score += 5;
|
||
if (/[？?]\s*$/.test(text)) score -= 3;
|
||
if (/^用户记住/u.test(text)) score -= 2;
|
||
return score;
|
||
}
|
||
|
||
export function filterMemoriesByQuery(memories, query) {
|
||
const items = Array.isArray(memories) ? memories : [];
|
||
const tokens = extractQueryTokens(query);
|
||
if (!tokens.length || !items.length) return items.slice(0, Math.min(items.length, 5));
|
||
const matched = items
|
||
.map((item) => {
|
||
const text = String(item?.text ?? item?.memory_text ?? '').trim();
|
||
return { item, text, score: scoreMemoryForRecall(text, tokens) };
|
||
})
|
||
.filter(({ text, score }) => text && score > 0)
|
||
.sort((a, b) => b.score - a.score)
|
||
.map(({ item }) => item);
|
||
return matched.length ? matched.slice(0, 5) : items.slice(0, 5);
|
||
}
|
||
|
||
export async function resolveMemoriesWithLegacyFallback({
|
||
memoryV2 = null,
|
||
conversationMemoryService = null,
|
||
userId,
|
||
sessionId = null,
|
||
query = null,
|
||
limit = 20,
|
||
recallQuestion = false,
|
||
} = {}) {
|
||
if (!userId || limit <= 0) return [];
|
||
if (recallQuestion && conversationMemoryService?.listMemories) {
|
||
const legacy = await conversationMemoryService.listMemories(userId, { limit }).catch(() => []);
|
||
const filtered = filterMemoriesByQuery(legacy, query);
|
||
if (filtered.length) return filtered;
|
||
}
|
||
if (memoryV2?.resolve) {
|
||
const resolved = await memoryV2.resolve({
|
||
userId,
|
||
sessionId,
|
||
query,
|
||
limit,
|
||
}).catch(() => null);
|
||
const memories = Array.isArray(resolved?.memories) ? resolved.memories : [];
|
||
if (memories.length) return memories;
|
||
}
|
||
if (!conversationMemoryService?.listMemories) return [];
|
||
const legacy = await conversationMemoryService.listMemories(userId, { limit }).catch(() => []);
|
||
return filterMemoriesByQuery(legacy, query);
|
||
}
|