memind/scripts/benchmark-memory-v2-recall.mjs

#!/usr/bin/env node
import fs from 'node:fs/promises';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import {
  probeEmbeddingModule,
  resolveEmbeddingModuleSpecifier,
  runRecallBenchmark,
  summarizeRecallBenchmark,
} from '../memory-v2-recall-benchmark.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';

function usage() {
  return [
    'Usage: node scripts/benchmark-memory-v2-recall.mjs [--json] [--output <path>]',
    '',
    'Reports MEMORY_PGVECTOR_EMBEDDING_MODULE dimensions and offline recall@5 baseline',
    'against the fixed Memory V2 hybrid-ranking benchmark suite.',
  ].join('\n');
}

function parseArgs(argv) {
  const args = { json: false, output: '' };
  for (let i = 2; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--json') args.json = true;
    else if (arg === '--output' && argv[i + 1]) args.output = argv[++i];
    else if (arg === '--help' || arg === '-h') {
      console.log(usage());
      process.exit(0);
    } else {
      console.error(`Unknown argument: ${arg}`);
      console.error(usage());
      process.exit(2);
    }
  }
  return args;
}

async function loadEmbedQuery(moduleSpecifier) {
  const resolved = resolveEmbeddingModuleSpecifier(moduleSpecifier);
  if (!resolved) return null;
  const imported = await import(resolved);
  const fn = imported?.embedQuery ?? imported?.default;
  return typeof fn === 'function' ? fn : null;
}

async function writeOutput(outputPath, payload) {
  if (!outputPath) return;
  const resolved = path.resolve(outputPath);
  await fs.mkdir(path.dirname(resolved), { recursive: true });
  await fs.writeFile(resolved, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}

const args = parseArgs(process.argv);
loadMemindEnvFiles(process.cwd());

const embeddingModule = String(process.env.MEMORY_PGVECTOR_EMBEDDING_MODULE ?? '').trim()
  || './scripts/embed-memory-v2-local-hash.mjs';
const probe = await probeEmbeddingModule({
  moduleSpecifier: embeddingModule,
  importModule: (specifier) => import(specifier),
});
const embedQuery = await loadEmbedQuery(probe.configured ? embeddingModule : null);
const benchmark = embedQuery
  ? await runRecallBenchmark({ embedQuery, limit: 5 })
  : null;

const payload = {
  generatedAt: new Date().toISOString(),
  embedding: probe,
  benchmark: benchmark ? {
    ...summarizeRecallBenchmark(benchmark),
    results: benchmark.results,
  } : null,
  env: {
    MEMORY_PGVECTOR_EMBEDDING_MODULE: embeddingModule,
    MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS: process.env.MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS ?? null,
    MEMORY_PGVECTOR_ENABLED: process.env.MEMORY_PGVECTOR_ENABLED ?? null,
    MEMORY_PGVECTOR_DATABASE_URL: process.env.MEMORY_PGVECTOR_DATABASE_URL ? '[configured]' : null,
  },
};

await writeOutput(args.output, payload);

if (args.json) {
  console.log(JSON.stringify(payload, null, 2));
} else {
  console.log('memory v2 recall benchmark');
  console.log(`embedding module: ${probe.modulePath ?? embeddingModule}`);
  console.log(`configured: ${probe.configured}`);
  if (probe.configured) {
    console.log(`dimensions: ${probe.dimensions}`);
  } else {
    console.log(`reason: ${probe.reason}`);
  }
  if (benchmark) {
    const summary = summarizeRecallBenchmark(benchmark);
    console.log(`recall@5: ${(summary.recallAt5 * 100).toFixed(1)}% (${summary.hits}/${summary.caseCount})`);
    if (summary.failedCases.length > 0) {
      console.log(`failed cases: ${summary.failedCases.join(', ')}`);
    }
    for (const item of benchmark.results) {
      console.log(`- ${item.id}: ${item.hit ? `hit@${item.hitRank}` : 'miss'} top="${item.topText ?? ''}"`);
    }
  }
  if (args.output) {
    console.log(`\nreport written: ${path.resolve(args.output)}`);
  }
}

process.exit(benchmark && summarizeRecallBenchmark(benchmark).recallAt5 >= 0.75 ? 0 : 1);