memind/temporal-recall-service/dedupe.mjs

function normalizeTitle(text) {
  return String(text ?? '')
    .toLowerCase()
    .replace(/\s+/g, '')
    .slice(0, 64);
}

function overlapScore(a, b) {
  const ta = normalizeTitle(a);
  const tb = normalizeTitle(b);
  if (!ta || !tb) return 0;
  if (ta === tb) return 1;
  if (ta.includes(tb) || tb.includes(ta)) return 0.85;
  const shorter = ta.length < tb.length ? ta : tb;
  const longer = ta.length < tb.length ? tb : ta;
  let common = 0;
  for (let i = 0; i < shorter.length; i++) {
    if (longer.includes(shorter[i])) common++;
  }
  return common / Math.max(longer.length, 1);
}

function timeClose(a, b, windowMs = 15 * 60_000) {
  const ma = a ? new Date(a).getTime() : null;
  const mb = b ? new Date(b).getTime() : null;
  if (ma === null || mb === null) return false;
  return Math.abs(ma - mb) <= windowMs;
}

/**
 * @param {object[]} items
 */
export function dedupeTimelineItems(items) {
  const kept = [];
  for (const item of items) {
    let merged = false;
    for (let i = 0; i < kept.length; i++) {
      const existing = kept[i];
      const titleSim = overlapScore(existing.title, item.title);
      const sameDayObserved =
        existing.observed_time?.slice(0, 10) === item.observed_time?.slice(0, 10);
      const close =
        timeClose(existing.event_time, item.event_time) ||
        (sameDayObserved && titleSim > 0.65);
      if (titleSim >= 0.7 && close) {
        const mergedFrom = [
          ...(existing.merged_from ?? [existing.source_ref]),
          item.source_ref,
        ];
        kept[i] = {
          ...existing,
          recall_score: Math.max(existing.recall_score ?? 0, item.recall_score ?? 0),
          importance: Math.max(existing.importance ?? 0, item.importance ?? 0),
          confidence: Math.max(existing.confidence ?? 0, item.confidence ?? 0),
          merged_from: mergedFrom,
          content:
            (existing.content?.length ?? 0) >= (item.content?.length ?? 0)
              ? existing.content
              : item.content,
        };
        merged = true;
        break;
      }
    }
    if (!merged) kept.push({ ...item });
  }
  return kept;
}