Add MemFuseBench retrieval harness and first baseline report.
Wire an offline benchmark over the production pgvector ranking path so Memory V2 recall can be measured with candidate/ranking loss split before lifecycle work lands. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Run the MemFuseBench retrieval benchmark against the Memory V2 pgvector
|
||||
* ranking path.
|
||||
*
|
||||
* The dataset lives outside this repository (MIT, Mi-Memory / Darwin Agent
|
||||
* Team). Point at it with --dataset or MEMFUSE_BENCH_DATASET; without it the
|
||||
* run reports `skipped` and exits 0 unless --strict is passed.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/run-memory-v2-memfuse-bench.mjs
|
||||
* node scripts/run-memory-v2-memfuse-bench.mjs --scenario sc1 --limit 20
|
||||
* node scripts/run-memory-v2-memfuse-bench.mjs --dimension multi_source_conflict_arbitration
|
||||
* node scripts/run-memory-v2-memfuse-bench.mjs --embedding-module ./scripts/my-embedder.mjs
|
||||
* node scripts/run-memory-v2-memfuse-bench.mjs --json --out .release-gate/memfuse.json
|
||||
*/
|
||||
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
MEMFUSE_DIMENSIONS,
|
||||
loadMemFuseDataset,
|
||||
runMemFuseBench,
|
||||
summarizeMemFuseBench,
|
||||
} from '../memory-v2-memfuse-bench.mjs';
|
||||
import { resolveEmbeddingModuleSpecifier } from '../memory-v2-recall-benchmark.mjs';
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
dataset: null,
|
||||
scenarios: null,
|
||||
dimensions: null,
|
||||
questionIds: null,
|
||||
limit: 20,
|
||||
candidateLimit: 100,
|
||||
maxQuestions: null,
|
||||
sourceTags: false,
|
||||
embeddingModule: null,
|
||||
json: false,
|
||||
out: null,
|
||||
strict: false,
|
||||
quiet: false,
|
||||
};
|
||||
const list = (value) =>
|
||||
String(value ?? '')
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const next = () => argv[(index += 1)];
|
||||
switch (arg) {
|
||||
case '--dataset': options.dataset = next(); break;
|
||||
case '--scenario': options.scenarios = list(next()); break;
|
||||
case '--dimension': options.dimensions = list(next()); break;
|
||||
case '--question': options.questionIds = list(next()); break;
|
||||
case '--limit': options.limit = Number(next()); break;
|
||||
case '--candidate-limit': options.candidateLimit = Number(next()); break;
|
||||
case '--max-questions': options.maxQuestions = Number(next()); break;
|
||||
case '--source-tags': options.sourceTags = true; break;
|
||||
case '--embedding-module': options.embeddingModule = next(); break;
|
||||
case '--json': options.json = true; break;
|
||||
case '--out': options.out = next(); break;
|
||||
case '--strict': options.strict = true; break;
|
||||
case '--quiet': options.quiet = true; break;
|
||||
case '-h':
|
||||
case '--help': options.help = true; break;
|
||||
default:
|
||||
if (arg.startsWith('--')) throw new Error(`Unknown flag: ${arg}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
process.stdout.write(`Usage: node scripts/run-memory-v2-memfuse-bench.mjs [options]
|
||||
|
||||
--dataset <path> MemFuseBench dataset JSON (default: $MEMFUSE_BENCH_DATASET)
|
||||
--scenario <ids> Comma-separated scenario ids, e.g. sc1,sc2
|
||||
--dimension <names> Comma-separated dimensions. Known values:
|
||||
${MEMFUSE_DIMENSIONS.join('\n ')}
|
||||
--question <ids> Comma-separated question ids
|
||||
--limit <n> Retrieved rows per question (clamped to 50)
|
||||
--candidate-limit <n> Candidate pool size before ranking (clamped to 100)
|
||||
--max-questions <n> Cap questions per scenario (useful for smoke runs)
|
||||
--source-tags Prepend "[device · location]" to event text
|
||||
--embedding-module <spec> Module exporting embedQuery/embedText/default
|
||||
--json Print the full report as JSON
|
||||
--out <path> Write the JSON report to a file
|
||||
--strict Exit non-zero when the dataset is unavailable
|
||||
--quiet Suppress per-scenario progress
|
||||
`);
|
||||
}
|
||||
|
||||
async function loadEmbedder(specifier) {
|
||||
const resolved = resolveEmbeddingModuleSpecifier(specifier);
|
||||
if (!resolved) throw new Error(`Cannot resolve embedding module: ${specifier}`);
|
||||
const imported = await import(resolved);
|
||||
const embed = imported?.embedText ?? imported?.embedQuery ?? imported?.default;
|
||||
if (typeof embed !== 'function') {
|
||||
throw new Error(`Embedding module must export embedText, embedQuery or default: ${specifier}`);
|
||||
}
|
||||
return embed;
|
||||
}
|
||||
|
||||
function formatPercent(value) {
|
||||
return Number.isFinite(value) ? `${(value * 100).toFixed(1)}%` : 'n/a';
|
||||
}
|
||||
|
||||
function formatRow(label, metrics, width) {
|
||||
return [
|
||||
label.padEnd(width),
|
||||
String(metrics.caseCount).padStart(5),
|
||||
formatPercent(metrics.candidateRecall).padStart(9),
|
||||
formatPercent(metrics.recallAtK).padStart(8),
|
||||
formatPercent(metrics.rankingLoss).padStart(8),
|
||||
formatPercent(metrics.hitAnyRate).padStart(8),
|
||||
formatPercent(metrics.checklistCoverage).padStart(10),
|
||||
(Number.isFinite(metrics.mrr) ? metrics.mrr.toFixed(3) : 'n/a').padStart(7),
|
||||
formatPercent(metrics.distractorRate).padStart(11),
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
function printReport(report) {
|
||||
const labels = [
|
||||
...report.byScenario.map((item) => item.scenarioId),
|
||||
...Object.keys(report.byDimension),
|
||||
'OVERALL',
|
||||
];
|
||||
const width = Math.max(12, ...labels.map((label) => label.length));
|
||||
const header = [
|
||||
'group'.padEnd(width),
|
||||
'cases'.padStart(5),
|
||||
'cand.rec'.padStart(9),
|
||||
'recall'.padStart(8),
|
||||
'rankLoss'.padStart(8),
|
||||
'hit@k'.padStart(8),
|
||||
'checklist'.padStart(10),
|
||||
'mrr'.padStart(7),
|
||||
'distractor'.padStart(11),
|
||||
].join(' ');
|
||||
|
||||
process.stdout.write(`\nMemFuseBench retrieval @k=${report.limit}`);
|
||||
process.stdout.write(
|
||||
` · embedding=${report.embeddingMode}` +
|
||||
` · sourceTags=${report.includeSourceTags ? 'on' : 'off'}\n`,
|
||||
);
|
||||
if (report.embeddingMode === 'lexical-hash') {
|
||||
process.stdout.write(
|
||||
'NOTE: lexical-hash is a deterministic offline stand-in, not a semantic embedder.\n' +
|
||||
' Use --embedding-module to measure real semantic recall.\n',
|
||||
);
|
||||
}
|
||||
process.stdout.write(`\n${header}\n${'-'.repeat(header.length)}\n`);
|
||||
for (const scenario of report.byScenario) {
|
||||
process.stdout.write(`${formatRow(scenario.scenarioId, scenario, width)}\n`);
|
||||
}
|
||||
process.stdout.write(`${'-'.repeat(header.length)}\n`);
|
||||
for (const [dimension, metrics] of Object.entries(report.byDimension).sort(
|
||||
(left, right) => left[1].recallAtK - right[1].recallAtK,
|
||||
)) {
|
||||
process.stdout.write(`${formatRow(dimension, metrics, width)}\n`);
|
||||
}
|
||||
process.stdout.write(`${'-'.repeat(header.length)}\n`);
|
||||
process.stdout.write(`${formatRow('OVERALL', report.overall, width)}\n\n`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const loaded = await loadMemFuseDataset({ datasetPath: options.dataset });
|
||||
if (!loaded.available) {
|
||||
const message =
|
||||
`MemFuseBench dataset unavailable (${loaded.reason}) at ${loaded.path}\n` +
|
||||
'Clone it outside this repo, then set MEMFUSE_BENCH_DATASET or pass --dataset:\n' +
|
||||
' git clone https://github.com/Darwin-Agent/Mi-Memory ~/Project/mi-memory\n';
|
||||
process.stderr.write(message);
|
||||
return options.strict ? 1 : 0;
|
||||
}
|
||||
|
||||
if (!options.quiet) {
|
||||
process.stdout.write(
|
||||
`dataset: ${loaded.path}\n` +
|
||||
`scenarios=${loaded.stats.scenarioCount} ` +
|
||||
`events=${loaded.stats.eventCount} questions=${loaded.stats.questionCount}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const embedText = options.embeddingModule
|
||||
? await loadEmbedder(options.embeddingModule)
|
||||
: null;
|
||||
|
||||
let lastScenario = null;
|
||||
const report = await runMemFuseBench({
|
||||
dataset: loaded,
|
||||
scenarioIds: options.scenarios,
|
||||
dimensions: options.dimensions,
|
||||
questionIds: options.questionIds,
|
||||
maxQuestionsPerScenario: Number.isFinite(options.maxQuestions) ? options.maxQuestions : null,
|
||||
limit: options.limit,
|
||||
candidateLimit: options.candidateLimit,
|
||||
includeSourceTags: options.sourceTags,
|
||||
embedText,
|
||||
onProgress: options.quiet
|
||||
? null
|
||||
: (event) => {
|
||||
if (event.scenarioId !== lastScenario) {
|
||||
lastScenario = event.scenarioId;
|
||||
process.stderr.write(` running ${event.scenarioId}...\n`);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (options.json) {
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
} else {
|
||||
printReport(report);
|
||||
process.stdout.write(`${JSON.stringify(summarizeMemFuseBench(report), null, 2)}\n`);
|
||||
}
|
||||
|
||||
if (options.out) {
|
||||
const target = path.resolve(options.out);
|
||||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||||
await fs.writeFile(target, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
process.stderr.write(`report written to ${target}\n`);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
main()
|
||||
.then((code) => process.exit(code))
|
||||
.catch((err) => {
|
||||
process.stderr.write(`${err instanceof Error ? err.stack : String(err)}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user