memind/memory-v2-recall-benchmark.mjs

import { createPgvectorMemoryBackend } from './memory-v2-pgvector.mjs';
import path from 'node:path';
import { pathToFileURL } from 'node:url';

export function resolveEmbeddingModuleSpecifier(specifier, baseDir = process.cwd()) {
  const value = String(specifier ?? '').trim();
  if (!value) return null;
  if (value.startsWith('.') || value.startsWith('/')) {
    return pathToFileURL(path.resolve(baseDir, value)).href;
  }
  return value;
}

export const RECALL_BENCHMARK_CASES = [
  {
    id: 'chinese-paraphrase-code',
    memory: '用户的记忆召回灰度测试代号是 MEM-RECALL-NEW',
    query: '我之前让你记住的记忆召回灰度测试代号是什么？请只回答完整代号。',
    expectMatch: /MEM-RECALL-NEW/,
  },
  {
    id: 'explicit-weekly-review',
    memory: '用户每周三下午做代码评审',
    query: '我什么时候做代码评审？',
    expectMatch: /周三/,
  },
  {
    id: 'preference-pgvector',
    memory: '用户偏好使用 pgvector 做语义检索',
    query: '我之前说过向量检索偏好用什么？',
    expectMatch: /pgvector/,
  },
  {
    id: 'goal-long-term',
    memory: '用户的长期目标是建设一个持续成长的 Personal Agent',
    query: '我的长期目标是什么？',
    expectMatch: /Personal Agent/,
  },
];

function buildDistractorRows(targetMemory, marker) {
  return [
    {
      id: 901,
      content: '用户以前关注贵州旅游攻略',
      type: 'interest',
      score: 0.91,
      created_at: '2026-06-01T00:00:00.000Z',
      updated_at: '2026-06-01T00:00:00.000Z',
    },
    {
      id: 902,
      content: targetMemory,
      type: 'fact',
      score: -0.49,
      created_at: '2026-07-22T00:00:00.000Z',
      updated_at: '2026-07-22T00:00:00.000Z',
    },
    {
      id: 903,
      content: `无关占位记忆 ${marker}`,
      type: 'fact',
      score: 0.3,
      created_at: '2026-07-21T00:00:00.000Z',
      updated_at: '2026-07-21T00:00:00.000Z',
    },
  ];
}

export async function probeEmbeddingModule({
  moduleSpecifier,
  importModule = (specifier) => import(specifier),
  sampleText = 'memory recall benchmark probe',
} = {}) {
  const modulePath = String(moduleSpecifier ?? '').trim();
  if (!modulePath) {
    return { configured: false, reason: 'embedding_module_not_configured' };
  }
  try {
    const resolved = resolveEmbeddingModuleSpecifier(modulePath);
    if (!resolved) {
      return { configured: false, reason: 'embedding_module_not_configured' };
    }
    const imported = await importModule(resolved);
    const embedQuery = imported?.embedQuery ?? imported?.default;
    if (typeof embedQuery !== 'function') {
      return { configured: false, modulePath, reason: 'embed_query_export_missing' };
    }
    const vector = await embedQuery(sampleText);
    if (!Array.isArray(vector) || vector.length === 0) {
      return { configured: false, modulePath, reason: 'empty_embedding_vector' };
    }
    return {
      configured: true,
      modulePath,
      dimensions: vector.length,
      samplePreview: vector.slice(0, 5),
    };
  } catch (err) {
    return {
      configured: false,
      modulePath,
      reason: 'embedding_module_load_failed',
      error: err instanceof Error ? err.message : String(err),
    };
  }
}

export async function runRecallBenchmarkCase({
  testCase,
  embedQuery,
  limit = 5,
}) {
  const marker = `CASE-${testCase.id}`;
  const backend = createPgvectorMemoryBackend({
    enabled: true,
    embedQuery,
    pool: {
      async query() {
        return { rows: buildDistractorRows(testCase.memory, marker) };
      },
    },
  });
  const result = await backend.resolve({
    userId: 'benchmark-user',
    query: testCase.query,
    limit,
  });
  const texts = result.memories.map((item) => item.text);
  const hitIndex = texts.findIndex((text) => testCase.expectMatch.test(text));
  return {
    id: testCase.id,
    hit: hitIndex >= 0,
    hitRank: hitIndex >= 0 ? hitIndex + 1 : null,
    topText: texts[0] ?? null,
    returned: texts.length,
  };
}

export async function runRecallBenchmark({
  cases = RECALL_BENCHMARK_CASES,
  embedQuery,
  limit = 5,
} = {}) {
  if (typeof embedQuery !== 'function') {
    throw new Error('runRecallBenchmark requires embedQuery');
  }
  const results = [];
  for (const testCase of cases) {
    results.push(await runRecallBenchmarkCase({ testCase, embedQuery, limit }));
  }
  const hits = results.filter((item) => item.hit).length;
  return {
    limit,
    caseCount: cases.length,
    recallAtK: cases.length > 0 ? hits / cases.length : 0,
    hits,
    misses: cases.length - hits,
    results,
  };
}

export function summarizeRecallBenchmark(report) {
  return {
    recallAt5: report.recallAtK,
    hits: report.hits,
    misses: report.misses,
    caseCount: report.caseCount,
    failedCases: report.results.filter((item) => !item.hit).map((item) => item.id),
  };
}