memind/scripts/run-memory-v2-memfuse-bench.mjs

#!/usr/bin/env node
/**
 * Run the MemFuseBench retrieval benchmark against the Memory V2 pgvector
 * ranking path.
 *
 * The dataset lives outside this repository (MIT, Mi-Memory / Darwin Agent
 * Team). Point at it with --dataset or MEMFUSE_BENCH_DATASET; without it the
 * run reports `skipped` and exits 0 unless --strict is passed.
 *
 * Usage:
 *   node scripts/run-memory-v2-memfuse-bench.mjs
 *   node scripts/run-memory-v2-memfuse-bench.mjs --scenario sc1 --limit 20
 *   node scripts/run-memory-v2-memfuse-bench.mjs --dimension multi_source_conflict_arbitration
 *   node scripts/run-memory-v2-memfuse-bench.mjs --embedding-module ./scripts/my-embedder.mjs
 *   node scripts/run-memory-v2-memfuse-bench.mjs --json --out .release-gate/memfuse.json
 */

import fs from 'node:fs/promises';
import path from 'node:path';

import {
  MEMFUSE_DIMENSIONS,
  loadMemFuseDataset,
  runMemFuseBench,
  summarizeMemFuseBench,
} from '../memory-v2-memfuse-bench.mjs';
import { resolveEmbeddingModuleSpecifier } from '../memory-v2-recall-benchmark.mjs';

function parseArgs(argv) {
  const options = {
    dataset: null,
    scenarios: null,
    dimensions: null,
    questionIds: null,
    limit: 20,
    candidateLimit: 100,
    maxQuestions: null,
    sourceTags: false,
    embeddingModule: null,
    json: false,
    out: null,
    strict: false,
    quiet: false,
  };
  const list = (value) =>
    String(value ?? '')
      .split(',')
      .map((item) => item.trim())
      .filter(Boolean);

  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    const next = () => argv[(index += 1)];
    switch (arg) {
      case '--dataset': options.dataset = next(); break;
      case '--scenario': options.scenarios = list(next()); break;
      case '--dimension': options.dimensions = list(next()); break;
      case '--question': options.questionIds = list(next()); break;
      case '--limit': options.limit = Number(next()); break;
      case '--candidate-limit': options.candidateLimit = Number(next()); break;
      case '--max-questions': options.maxQuestions = Number(next()); break;
      case '--source-tags': options.sourceTags = true; break;
      case '--embedding-module': options.embeddingModule = next(); break;
      case '--json': options.json = true; break;
      case '--out': options.out = next(); break;
      case '--strict': options.strict = true; break;
      case '--quiet': options.quiet = true; break;
      case '-h':
      case '--help': options.help = true; break;
      default:
        if (arg.startsWith('--')) throw new Error(`Unknown flag: ${arg}`);
    }
  }
  return options;
}

function usage() {
  process.stdout.write(`Usage: node scripts/run-memory-v2-memfuse-bench.mjs [options]

  --dataset <path>          MemFuseBench dataset JSON (default: $MEMFUSE_BENCH_DATASET)
  --scenario <ids>          Comma-separated scenario ids, e.g. sc1,sc2
  --dimension <names>       Comma-separated dimensions. Known values:
                            ${MEMFUSE_DIMENSIONS.join('\n                            ')}
  --question <ids>          Comma-separated question ids
  --limit <n>               Retrieved rows per question (clamped to 50)
  --candidate-limit <n>     Candidate pool size before ranking (clamped to 100)
  --max-questions <n>       Cap questions per scenario (useful for smoke runs)
  --source-tags             Prepend "[device · location]" to event text
  --embedding-module <spec> Module exporting embedQuery/embedText/default
  --json                    Print the full report as JSON
  --out <path>              Write the JSON report to a file
  --strict                  Exit non-zero when the dataset is unavailable
  --quiet                   Suppress per-scenario progress
`);
}

async function loadEmbedder(specifier) {
  const resolved = resolveEmbeddingModuleSpecifier(specifier);
  if (!resolved) throw new Error(`Cannot resolve embedding module: ${specifier}`);
  const imported = await import(resolved);
  const embed = imported?.embedText ?? imported?.embedQuery ?? imported?.default;
  if (typeof embed !== 'function') {
    throw new Error(`Embedding module must export embedText, embedQuery or default: ${specifier}`);
  }
  return embed;
}

function formatPercent(value) {
  return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : 'n/a';
}

function formatRow(label, metrics, width) {
  return [
    label.padEnd(width),
    String(metrics.caseCount).padStart(5),
    formatPercent(metrics.candidateRecall).padStart(9),
    formatPercent(metrics.recallAtK).padStart(8),
    formatPercent(metrics.rankingLoss).padStart(8),
    formatPercent(metrics.hitAnyRate).padStart(8),
    formatPercent(metrics.checklistCoverage).padStart(10),
    (Number.isFinite(metrics.mrr) ? metrics.mrr.toFixed(3) : 'n/a').padStart(7),
    formatPercent(metrics.distractorRate).padStart(11),
  ].join('  ');
}

function printReport(report) {
  const labels = [
    ...report.byScenario.map((item) => item.scenarioId),
    ...Object.keys(report.byDimension),
    'OVERALL',
  ];
  const width = Math.max(12, ...labels.map((label) => label.length));
  const header = [
    'group'.padEnd(width),
    'cases'.padStart(5),
    'cand.rec'.padStart(9),
    'recall'.padStart(8),
    'rankLoss'.padStart(8),
    'hit@k'.padStart(8),
    'checklist'.padStart(10),
    'mrr'.padStart(7),
    'distractor'.padStart(11),
  ].join('  ');

  process.stdout.write(`\nMemFuseBench retrieval @k=${report.limit}`);
  process.stdout.write(
    ` · embedding=${report.embeddingMode}` +
      ` · sourceTags=${report.includeSourceTags ? 'on' : 'off'}\n`,
  );
  if (report.embeddingMode === 'lexical-hash') {
    process.stdout.write(
      'NOTE: lexical-hash is a deterministic offline stand-in, not a semantic embedder.\n' +
        '      Use --embedding-module to measure real semantic recall.\n',
    );
  }
  process.stdout.write(`\n${header}\n${'-'.repeat(header.length)}\n`);
  for (const scenario of report.byScenario) {
    process.stdout.write(`${formatRow(scenario.scenarioId, scenario, width)}\n`);
  }
  process.stdout.write(`${'-'.repeat(header.length)}\n`);
  for (const [dimension, metrics] of Object.entries(report.byDimension).sort(
    (left, right) => left[1].recallAtK - right[1].recallAtK,
  )) {
    process.stdout.write(`${formatRow(dimension, metrics, width)}\n`);
  }
  process.stdout.write(`${'-'.repeat(header.length)}\n`);
  process.stdout.write(`${formatRow('OVERALL', report.overall, width)}\n\n`);
}

async function main() {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    usage();
    return 0;
  }

  const loaded = await loadMemFuseDataset({ datasetPath: options.dataset });
  if (!loaded.available) {
    const message =
      `MemFuseBench dataset unavailable (${loaded.reason}) at ${loaded.path}\n` +
      'Clone it outside this repo, then set MEMFUSE_BENCH_DATASET or pass --dataset:\n' +
      '  git clone https://github.com/Darwin-Agent/Mi-Memory ~/Project/mi-memory\n';
    process.stderr.write(message);
    return options.strict ? 1 : 0;
  }

  if (!options.quiet) {
    process.stdout.write(
      `dataset: ${loaded.path}\n` +
        `scenarios=${loaded.stats.scenarioCount} ` +
        `events=${loaded.stats.eventCount} questions=${loaded.stats.questionCount}\n`,
    );
  }

  const embedText = options.embeddingModule
    ? await loadEmbedder(options.embeddingModule)
    : null;

  let lastScenario = null;
  const report = await runMemFuseBench({
    dataset: loaded,
    scenarioIds: options.scenarios,
    dimensions: options.dimensions,
    questionIds: options.questionIds,
    maxQuestionsPerScenario: Number.isFinite(options.maxQuestions) ? options.maxQuestions : null,
    limit: options.limit,
    candidateLimit: options.candidateLimit,
    includeSourceTags: options.sourceTags,
    embedText,
    onProgress: options.quiet
      ? null
      : (event) => {
          if (event.scenarioId !== lastScenario) {
            lastScenario = event.scenarioId;
            process.stderr.write(`  running ${event.scenarioId}...\n`);
          }
        },
  });

  if (options.json) {
    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
  } else {
    printReport(report);
    process.stdout.write(`${JSON.stringify(summarizeMemFuseBench(report), null, 2)}\n`);
  }

  if (options.out) {
    const target = path.resolve(options.out);
    await fs.mkdir(path.dirname(target), { recursive: true });
    await fs.writeFile(target, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
    process.stderr.write(`report written to ${target}\n`);
  }

  return 0;
}

main()
  .then((code) => process.exit(code))
  .catch((err) => {
    process.stderr.write(`${err instanceof Error ? err.stack : String(err)}\n`);
    process.exit(1);
  });