feat(memory-v2): close Phase A with auto-review, product events, and H5 recall UI.

Add candidate auto-review pipeline, shadow audit tooling, admin metrics page,
and user-visible memory recall hints in chat with phase-a readiness checks.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-08-01 17:14:06 +08:00
parent 666db0b939
commit 6f3e53a56a
50 changed files with 3730 additions and 38 deletions
+282
View File
@@ -0,0 +1,282 @@
#!/usr/bin/env node
import fs from 'node:fs/promises';
import path from 'node:path';
import mysql from 'mysql2/promise';
import {
auditMemoryV2Shadow,
formatMemoryV2ShadowAuditReport,
parseSinceArg,
} from '../memory-v2-shadow-audit.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/audit-memory-v2-shadow.mjs [--since 7d] [--user-id <id>] [--output <path>] [--json]',
'',
'Audits Memory V2 shadow/canary data quality:',
'- candidate status and policy reason distribution',
'- suspicious false-store samples',
'- active memories missing from pgvector',
'- agent_memory_resolved hit rate',
].join('\n');
}
function parseArgs(argv) {
const args = {
since: '7d',
userId: '',
output: '',
json: false,
};
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--since' && argv[i + 1]) {
args.since = argv[++i];
} else if (arg === '--user-id' && argv[i + 1]) {
args.userId = argv[++i];
} else if (arg === '--output' && argv[i + 1]) {
args.output = argv[++i];
} else if (arg === '--json') {
args.json = true;
} else if (arg === '--help' || arg === '-h') {
console.log(usage());
process.exit(0);
} else {
console.error(`Unknown argument: ${arg}`);
console.error(usage());
process.exit(2);
}
}
return args;
}
function createMysqlPoolFromEnv() {
loadMemindEnvFiles(process.cwd());
const poolOptions = { connectionLimit: 3 };
if (process.env.DATABASE_URL) {
return mysql.createPool({
uri: process.env.DATABASE_URL,
...poolOptions,
});
}
if (!process.env.MYSQL_HOST && !process.env.MYSQL_DATABASE) {
throw new Error('Memory V2 shadow audit requires DATABASE_URL or MYSQL_* configuration');
}
return mysql.createPool({
host: process.env.MYSQL_HOST ?? 'localhost',
port: Number(process.env.MYSQL_PORT ?? 3306),
user: process.env.MYSQL_USER ?? 'boot',
password: process.env.MYSQL_PASSWORD ?? '',
database: process.env.MYSQL_DATABASE ?? 'tkmind',
...poolOptions,
});
}
async function createPgPoolFromEnv() {
const connectionString = String(process.env.MEMORY_PGVECTOR_DATABASE_URL ?? '').trim();
if (!connectionString) return null;
const imported = await import('pg');
const PgPool = imported?.Pool ?? imported?.default?.Pool;
if (typeof PgPool !== 'function') {
throw new Error('pg module does not export Pool');
}
return new PgPool({
connectionString,
max: Math.max(1, Number(process.env.MEMORY_PGVECTOR_POOL_MAX ?? 3) || 3),
});
}
function isSafeIdentifier(value) {
return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(String(value ?? ''));
}
async function tableExists(pool, tableName) {
const [rows] = await pool.query(
`SELECT 1
FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = ?
LIMIT 1`,
[tableName],
);
return rows.length > 0;
}
async function loadCandidates(pool, { sinceMs, userId }) {
if (!(await tableExists(pool, 'h5_memory_v2_candidates'))) {
return { rows: [], tableMissing: true };
}
const clauses = ['created_at >= ?'];
const params = [sinceMs];
if (userId) {
clauses.push('user_id = ?');
params.push(userId);
}
const [rows] = await pool.query(
`SELECT id, user_id, session_id, memory_type, content, status,
policy_reason, confidence, importance, created_at, updated_at
FROM h5_memory_v2_candidates
WHERE ${clauses.join(' AND ')}
ORDER BY created_at DESC
LIMIT 5000`,
params,
);
return { rows, tableMissing: false };
}
async function loadMemoryItems(pool, { sinceMs, userId }) {
const clauses = ['updated_at >= ?'];
const params = [sinceMs];
if (userId) {
clauses.push('user_id = ?');
params.push(userId);
}
const [rows] = await pool.query(
`SELECT id, user_id, label, memory_text, status, created_at, updated_at
FROM h5_user_memory_items
WHERE ${clauses.join(' AND ')}
ORDER BY updated_at DESC
LIMIT 5000`,
params,
);
const activeClauses = ["status = 'active'"];
const activeParams = [];
if (userId) {
activeClauses.push('user_id = ?');
activeParams.push(userId);
}
const [activeRows] = await pool.query(
`SELECT id, user_id, label, memory_text, status, created_at, updated_at
FROM h5_user_memory_items
WHERE ${activeClauses.join(' AND ')}
ORDER BY updated_at DESC
LIMIT 10000`,
activeParams,
);
return {
updatedRows: rows,
activeRows,
};
}
async function loadAgentMemoryEvents(pool, { sinceMs, userId }) {
const clauses = ["e.event_type = 'agent_memory_resolved'", 'e.created_at >= ?'];
const params = [sinceMs];
if (userId) {
clauses.push('r.user_id = ?');
params.push(userId);
}
const [rows] = await pool.query(
`SELECT e.event_type, e.data_json, e.created_at, r.user_id
FROM h5_agent_run_events e
INNER JOIN h5_agent_runs r ON r.id = e.run_id
WHERE ${clauses.join(' AND ')}
ORDER BY e.created_at DESC
LIMIT 5000`,
params,
);
return rows;
}
async function loadPgvectorMemoryIds(pgPool, { userId, tableName }) {
if (!pgPool) return new Set();
if (!isSafeIdentifier(tableName)) {
throw new Error(`Invalid pgvector table name: ${tableName}`);
}
const clauses = ['source_memory_id IS NOT NULL'];
const params = [];
if (userId) {
clauses.push('user_id = $1');
params.push(userId);
}
try {
const result = await pgPool.query(
`SELECT source_memory_id
FROM "${tableName}"
WHERE ${clauses.join(' AND ')}`,
params,
);
return new Set(
result.rows
.map((row) => String(row.source_memory_id ?? '').trim())
.filter(Boolean),
);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new Error(`Failed to query pgvector table "${tableName}": ${message}`);
}
}
async function writeOutput(outputPath, payload) {
if (!outputPath) return;
const resolved = path.resolve(outputPath);
await fs.mkdir(path.dirname(resolved), { recursive: true });
await fs.writeFile(resolved, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}
const args = parseArgs(process.argv);
const nowMs = Date.now();
const { sinceMs, label: sinceLabel } = parseSinceArg(args.since, nowMs);
const mysqlPool = createMysqlPoolFromEnv();
let pgPool = null;
try {
pgPool = await createPgPoolFromEnv();
const [candidateResult, memoryResult, agentMemoryEvents] = await Promise.all([
loadCandidates(mysqlPool, { sinceMs, userId: args.userId }),
loadMemoryItems(mysqlPool, { sinceMs, userId: args.userId }),
loadAgentMemoryEvents(mysqlPool, { sinceMs, userId: args.userId }),
]);
const pgvectorTable = String(process.env.MEMORY_PGVECTOR_TABLE ?? 'memory_embeddings').trim()
|| 'memory_embeddings';
const pgvectorMemoryIds = await loadPgvectorMemoryIds(pgPool, {
userId: args.userId,
tableName: pgvectorTable,
});
const report = auditMemoryV2Shadow({
candidates: candidateResult.rows,
memoryItems: memoryResult.activeRows,
pgvectorMemoryIds,
pgvectorConfigured: Boolean(pgPool),
agentMemoryEvents,
sinceMs,
nowMs,
});
const payload = {
...report,
meta: {
since: sinceLabel,
userId: args.userId || null,
candidateTableMissing: candidateResult.tableMissing,
pgvectorConfigured: Boolean(pgPool),
pgvectorTable,
},
};
await writeOutput(args.output, payload);
if (args.json) {
console.log(JSON.stringify(payload, null, 2));
} else {
console.log(formatMemoryV2ShadowAuditReport(payload));
if (candidateResult.tableMissing) {
console.log('\nwarning: h5_memory_v2_candidates table is missing');
}
if (!pgPool) {
console.log('\nwarning: MEMORY_PGVECTOR_DATABASE_URL not configured; pgvector lag check skipped');
}
if (args.output) {
console.log(`\nreport written: ${path.resolve(args.output)}`);
}
}
process.exit(report.ok || !pgPool || candidateResult.tableMissing ? 0 : 1);
} finally {
await mysqlPool.end();
await pgPool?.end?.();
}
@@ -0,0 +1,72 @@
#!/usr/bin/env node
import { pathToFileURL } from 'node:url';
import { createDbPool, isDatabaseConfigured } from '../db.mjs';
import { runCandidateAutoReviewBatch } from '../memory-v2-candidate-auto-review.mjs';
function usage() {
return [
'Usage: node scripts/auto-review-memory-v2-candidates.mjs [--apply] [--user-id <id>] [--limit <n>] [--json]',
'',
'Default is dry-run summary against pending candidates.',
].join('\n');
}
function parseArgs(argv) {
const options = { apply: false, userId: null, limit: 200, json: false, help: false };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--help' || arg === '-h') options.help = true;
else if (arg === '--apply') options.apply = true;
else if (arg === '--json') options.json = true;
else if (arg === '--user-id') options.userId = String(argv[++index] ?? '').trim() || null;
else if (arg === '--limit') options.limit = Number(argv[++index]);
else throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
export async function runAutoReviewMemoryV2CandidatesCli(
argv = process.argv.slice(2),
env = process.env,
) {
const options = parseArgs(argv);
if (options.help) {
console.log(usage());
return { ok: true, mode: 'help' };
}
if (!isDatabaseConfigured()) {
throw new Error('DATABASE_URL or MYSQL_* must be configured');
}
const pool = createDbPool();
try {
const result = await runCandidateAutoReviewBatch(pool, {
env,
userId: options.userId,
limit: options.limit,
reviewedBy: options.apply ? 'system:auto-review-cli' : 'system:auto-review-dry-run',
dryRun: !options.apply,
});
const payload = {
...result,
mode: options.apply ? 'apply' : 'dry-run',
note: options.apply
? 'Auto-review applied to pending candidates'
: 'Dry-run only; pass --apply to mutate candidate statuses',
};
if (options.json) console.log(JSON.stringify(payload, null, 2));
else {
console.log(`${payload.mode}: scanned=${payload.scanned ?? 0} accept=${payload.accepted ?? 0} reject=${payload.rejected ?? 0} pending=${payload.pending ?? 0}`);
if (payload.samples?.length) console.log(JSON.stringify(payload.samples, null, 2));
}
return payload;
} finally {
await pool.end?.();
}
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
runAutoReviewMemoryV2CandidatesCli().catch((err) => {
console.error(err instanceof Error ? err.message : err);
process.exit(1);
});
}
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env node
import fs from 'node:fs/promises';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import {
probeEmbeddingModule,
resolveEmbeddingModuleSpecifier,
runRecallBenchmark,
summarizeRecallBenchmark,
} from '../memory-v2-recall-benchmark.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/benchmark-memory-v2-recall.mjs [--json] [--output <path>]',
'',
'Reports MEMORY_PGVECTOR_EMBEDDING_MODULE dimensions and offline recall@5 baseline',
'against the fixed Memory V2 hybrid-ranking benchmark suite.',
].join('\n');
}
function parseArgs(argv) {
const args = { json: false, output: '' };
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--json') args.json = true;
else if (arg === '--output' && argv[i + 1]) args.output = argv[++i];
else if (arg === '--help' || arg === '-h') {
console.log(usage());
process.exit(0);
} else {
console.error(`Unknown argument: ${arg}`);
console.error(usage());
process.exit(2);
}
}
return args;
}
async function loadEmbedQuery(moduleSpecifier) {
const resolved = resolveEmbeddingModuleSpecifier(moduleSpecifier);
if (!resolved) return null;
const imported = await import(resolved);
const fn = imported?.embedQuery ?? imported?.default;
return typeof fn === 'function' ? fn : null;
}
async function writeOutput(outputPath, payload) {
if (!outputPath) return;
const resolved = path.resolve(outputPath);
await fs.mkdir(path.dirname(resolved), { recursive: true });
await fs.writeFile(resolved, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}
const args = parseArgs(process.argv);
loadMemindEnvFiles(process.cwd());
const embeddingModule = String(process.env.MEMORY_PGVECTOR_EMBEDDING_MODULE ?? '').trim()
|| './scripts/embed-memory-v2-local-hash.mjs';
const probe = await probeEmbeddingModule({
moduleSpecifier: embeddingModule,
importModule: (specifier) => import(specifier),
});
const embedQuery = await loadEmbedQuery(probe.configured ? embeddingModule : null);
const benchmark = embedQuery
? await runRecallBenchmark({ embedQuery, limit: 5 })
: null;
const payload = {
generatedAt: new Date().toISOString(),
embedding: probe,
benchmark: benchmark ? {
...summarizeRecallBenchmark(benchmark),
results: benchmark.results,
} : null,
env: {
MEMORY_PGVECTOR_EMBEDDING_MODULE: embeddingModule,
MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS: process.env.MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS ?? null,
MEMORY_PGVECTOR_ENABLED: process.env.MEMORY_PGVECTOR_ENABLED ?? null,
MEMORY_PGVECTOR_DATABASE_URL: process.env.MEMORY_PGVECTOR_DATABASE_URL ? '[configured]' : null,
},
};
await writeOutput(args.output, payload);
if (args.json) {
console.log(JSON.stringify(payload, null, 2));
} else {
console.log('memory v2 recall benchmark');
console.log(`embedding module: ${probe.modulePath ?? embeddingModule}`);
console.log(`configured: ${probe.configured}`);
if (probe.configured) {
console.log(`dimensions: ${probe.dimensions}`);
} else {
console.log(`reason: ${probe.reason}`);
}
if (benchmark) {
const summary = summarizeRecallBenchmark(benchmark);
console.log(`recall@5: ${(summary.recallAt5 * 100).toFixed(1)}% (${summary.hits}/${summary.caseCount})`);
if (summary.failedCases.length > 0) {
console.log(`failed cases: ${summary.failedCases.join(', ')}`);
}
for (const item of benchmark.results) {
console.log(`- ${item.id}: ${item.hit ? `hit@${item.hitRank}` : 'miss'} top="${item.topText ?? ''}"`);
}
}
if (args.output) {
console.log(`\nreport written: ${path.resolve(args.output)}`);
}
}
process.exit(benchmark && summarizeRecallBenchmark(benchmark).recallAt5 >= 0.75 ? 0 : 1);
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env node
import { pathToFileURL } from 'node:url';
import { evaluateMemoryV2PhaseAReadiness } from '../memory-v2-phase-a-ready.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/check-memory-v2-phase-a-ready.mjs [--json]',
'',
'Validates local/admin env for Memory V2 Phase A canary rollout.',
].join('\n');
}
function parseArgs(argv) {
const args = { json: false };
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--json') args.json = true;
else if (arg === '--help' || arg === '-h') {
console.log(usage());
process.exit(0);
} else {
console.error(`Unknown argument: ${arg}`);
console.error(usage());
process.exit(2);
}
}
return args;
}
function printReport(report, { json = false } = {}) {
if (json) {
console.log(JSON.stringify(report, null, 2));
return;
}
console.log(`memory v2 phase-a readiness: ${report.ok ? 'ready' : 'blocked'}`);
console.log(JSON.stringify(report.summary, null, 2));
for (const item of report.issues) {
console.log(`- issue ${item.code}: ${item.message}`);
}
for (const item of report.warnings) {
console.log(`- warning ${item.code}: ${item.message}`);
}
}
async function main() {
const args = parseArgs(process.argv);
loadMemindEnvFiles(process.cwd());
const report = evaluateMemoryV2PhaseAReadiness(process.env);
printReport(report, { json: args.json });
process.exit(report.ok ? 0 : 1);
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
void main();
}
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env node
import mysql from 'mysql2/promise';
import { pathToFileURL } from 'node:url';
import { detectFalseStoreCandidate } from '../memory-v2-shadow-audit.mjs';
import { inspectMemoryContentDurability } from '../memory-v2-personal-shadow.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/repair-memory-v2-candidates.mjs [--apply] [--user-id <id>] [--limit <n>] [--json]',
'',
'Finds low-quality accepted/candidate rows and rejects them.',
'Default mode is dry-run. --apply performs UPDATE status=rejected.',
].join('\n');
}
function parseArgs(argv) {
const args = {
apply: false,
userId: '',
limit: 500,
json: false,
};
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--apply') args.apply = true;
else if (arg === '--user-id' && argv[i + 1]) args.userId = argv[++i];
else if (arg === '--limit' && argv[i + 1]) {
args.limit = Math.max(1, Math.min(5000, Number(argv[++i]) || 500));
} else if (arg === '--json') args.json = true;
else if (arg === '--help' || arg === '-h') {
console.log(usage());
process.exit(0);
} else {
console.error(`Unknown argument: ${arg}`);
console.error(usage());
process.exit(2);
}
}
return args;
}
function createPoolFromEnv() {
loadMemindEnvFiles(process.cwd());
const poolOptions = { connectionLimit: 3 };
if (process.env.DATABASE_URL) {
return mysql.createPool({ uri: process.env.DATABASE_URL, ...poolOptions });
}
return mysql.createPool({
host: process.env.MYSQL_HOST ?? 'localhost',
port: Number(process.env.MYSQL_PORT ?? 3306),
user: process.env.MYSQL_USER ?? 'boot',
password: process.env.MYSQL_PASSWORD ?? '',
database: process.env.MYSQL_DATABASE ?? 'tkmind',
...poolOptions,
});
}
export function evaluateRepairCandidate(row) {
const content = String(row.content ?? '');
const durabilityReason = inspectMemoryContentDurability(content);
if (durabilityReason) {
return { repair: true, code: durabilityReason, message: `Non-durable content (${durabilityReason})` };
}
const falseStore = detectFalseStoreCandidate(content);
if (falseStore.suspicious) {
return { repair: true, code: falseStore.code, message: falseStore.message };
}
if (String(row.policy_reason ?? '') === 'decision_signal' && /(?:public\/|page data|durability-)/iu.test(content)) {
return { repair: true, code: 'decision_signal_task_leak', message: 'Decision signal on agent task content' };
}
return { repair: false };
}
async function loadCandidates(pool, { userId, limit }) {
const clauses = ["status IN ('candidate', 'accepted')"];
const params = [];
if (userId) {
clauses.push('user_id = ?');
params.push(userId);
}
params.push(limit);
const [rows] = await pool.query(
`SELECT id, user_id, session_id, memory_type, content, status, policy_reason,
confidence, importance, created_at, updated_at
FROM h5_memory_v2_candidates
WHERE ${clauses.join(' AND ')}
ORDER BY updated_at DESC
LIMIT ?`,
params,
);
return rows;
}
async function applyRepairs(pool, repairs, nowMs) {
let updated = 0;
for (const item of repairs) {
const [result] = await pool.query(
`UPDATE h5_memory_v2_candidates
SET status = 'rejected',
reviewed_by = 'system:repair',
reviewed_at = ?,
updated_at = ?
WHERE id = ?
AND status IN ('candidate', 'accepted')`,
[nowMs, nowMs, item.id],
);
updated += Number(result?.affectedRows ?? 0);
}
return updated;
}
function printTextReport(report) {
console.log(`memory v2 candidate repair: ${report.apply ? 'apply' : 'dry-run'}`);
console.log(`scanned: ${report.scanned}`);
console.log(`repairable: ${report.repairable}`);
if (report.repairs.length > 0) {
for (const item of report.repairs.slice(0, 20)) {
console.log(`- [${item.code}] ${item.id} (${item.status}/${item.policyReason}): ${item.contentPreview}`);
}
if (report.repairs.length > 20) {
console.log(`... ${report.repairs.length - 20} more`);
}
}
if (report.apply) {
console.log(`updated: ${report.updated}`);
}
}
export async function runMemoryV2CandidateRepair(args, { pool: injectedPool, nowMs = Date.now() } = {}) {
const ownsPool = !injectedPool;
const activePool = injectedPool ?? createPoolFromEnv();
try {
const rows = await loadCandidates(activePool, args);
const repairs = rows
.map((row) => {
const verdict = evaluateRepairCandidate(row);
if (!verdict.repair) return null;
return {
id: String(row.id),
userId: String(row.user_id),
status: String(row.status),
policyReason: String(row.policy_reason ?? ''),
code: verdict.code,
message: verdict.message,
contentPreview: String(row.content ?? '').slice(0, 120),
};
})
.filter(Boolean);
const report = {
ok: true,
apply: args.apply,
scanned: rows.length,
repairable: repairs.length,
repairs,
updated: 0,
};
if (args.apply && repairs.length > 0) {
report.updated = await applyRepairs(activePool, repairs, nowMs);
}
return report;
} finally {
if (ownsPool) {
await activePool.end();
}
}
}
async function main() {
const args = parseArgs(process.argv);
try {
const report = await runMemoryV2CandidateRepair(args);
if (args.json) {
console.log(JSON.stringify(report, null, 2));
} else {
printTextReport(report);
}
process.exit(0);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error(`repair failed: ${message}`);
process.exit(1);
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
void main();
}
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env node
import path from 'node:path';
import { pathToFileURL } from 'node:url';
import { createDbPool, isDatabaseConfigured } from '../db.mjs';
import { createMemoryV2LifecycleService } from '../memory-v2-lifecycle.mjs';
import { resolveCandidateAutoReviewScopes, runCandidateAutoReviewBatch } from '../memory-v2-candidate-auto-review.mjs';
import { syncLegacyUserMemoriesToPgvector } from '../memory-v2-pgvector-backfill.mjs';
import { evaluateMemoryV2PhaseAReadiness } from '../memory-v2-phase-a-ready.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/run-memory-v2-phase-a-closure.mjs [--json] [--skip-auto-review] [--skip-promote] [--skip-pgvector]',
'',
'Runs canary closure: auto-review pending candidates → promote accepted → pgvector user sync.',
].join('\n');
}
function parseArgs(argv) {
const options = {
json: false,
skipAutoReview: false,
skipPromote: false,
skipPgvector: false,
help: false,
};
for (const arg of argv) {
if (arg === '--json') options.json = true;
else if (arg === '--skip-auto-review') options.skipAutoReview = true;
else if (arg === '--skip-promote') options.skipPromote = true;
else if (arg === '--skip-pgvector') options.skipPgvector = true;
else if (arg === '--help' || arg === '-h') options.help = true;
else throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
async function loadEmbedMemory(env) {
const modulePath = env.MEMORY_PGVECTOR_EMBEDDING_MODULE;
if (!modulePath) throw new Error('MEMORY_PGVECTOR_EMBEDDING_MODULE is required for pgvector sync');
const resolved = path.isAbsolute(modulePath)
? modulePath
: path.resolve(process.cwd(), modulePath);
const mod = await import(pathToFileURL(resolved).href);
const embedMemory = mod.embedMemory ?? mod.default;
if (typeof embedMemory !== 'function') {
throw new Error('embedding module must export embedMemory(memory) or default');
}
return embedMemory;
}
async function countCandidatesByStatus(pool) {
const [rows] = await pool.query(
`SELECT status, COUNT(*) AS count FROM h5_memory_v2_candidates GROUP BY status`,
);
return Object.fromEntries(rows.map((row) => [String(row.status), Number(row.count)]));
}
async function countActiveMemoriesForUsers(pool, userIds) {
if (!userIds.length) return 0;
const placeholders = userIds.map(() => '?').join(', ');
const [rows] = await pool.query(
`SELECT COUNT(*) AS count FROM h5_user_memory_items
WHERE status = 'active' AND user_id IN (${placeholders})`,
userIds,
);
return Number(rows[0]?.count ?? 0);
}
async function countPgvectorForUsers(pgPool, userIds) {
if (!pgPool?.query || !userIds.length) return null;
const { rows } = await pgPool.query(
`SELECT COUNT(*)::int AS count FROM memory_embeddings WHERE user_id = ANY($1::text[])`,
[userIds],
);
return Number(rows[0]?.count ?? 0);
}
export async function runMemoryV2PhaseAClosure(env = process.env, options = {}) {
if (!isDatabaseConfigured()) {
throw new Error('DATABASE_URL or MYSQL_* must be configured');
}
const readiness = evaluateMemoryV2PhaseAReadiness(env);
const canaryUserIds = resolveCandidateAutoReviewScopes(env);
const mysqlPool = createDbPool();
let pgPool = null;
const report = {
ok: false,
readiness: readiness.summary,
canaryUserIds,
before: {},
after: {},
steps: {},
};
try {
report.before.candidateCounts = await countCandidatesByStatus(mysqlPool);
report.before.activeMemories = await countActiveMemoriesForUsers(mysqlPool, canaryUserIds);
if (env.MEMORY_PGVECTOR_DATABASE_URL) {
const { default: pg } = await import('pg');
pgPool = new pg.Pool({ connectionString: env.MEMORY_PGVECTOR_DATABASE_URL, max: 1 });
report.before.pgvectorMemories = await countPgvectorForUsers(pgPool, canaryUserIds);
}
if (!options.skipAutoReview) {
report.steps.autoReview = await runCandidateAutoReviewBatch(mysqlPool, {
env,
limit: 500,
reviewedBy: 'system:phase-a-closure',
});
} else {
report.steps.autoReview = { skipped: true };
}
const lifecycle = createMemoryV2LifecycleService({ pool: mysqlPool, env });
const promoteResults = [];
if (!options.skipPromote) {
for (const userId of canaryUserIds.length ? canaryUserIds : [null]) {
const input = userId ? { userId, limit: 200 } : { limit: 200 };
promoteResults.push({
userId,
result: await lifecycle.promote(input),
});
}
report.steps.promote = promoteResults;
} else {
report.steps.promote = { skipped: true };
}
if (!options.skipPgvector && env.MEMORY_PGVECTOR_DATABASE_URL) {
if (!pgPool) {
const { default: pg } = await import('pg');
pgPool = new pg.Pool({ connectionString: env.MEMORY_PGVECTOR_DATABASE_URL, max: 1 });
}
const embedMemory = await loadEmbedMemory(env);
const syncResults = [];
for (const userId of canaryUserIds) {
syncResults.push(await syncLegacyUserMemoriesToPgvector({
mysqlPool,
pgPool,
embedMemory,
tableName: env.MEMORY_PGVECTOR_TABLE ?? 'memory_embeddings',
userId,
limit: Number(env.MEMORY_PGVECTOR_SYNC_USER_LIMIT ?? 50) || 50,
}));
}
report.steps.pgvectorSync = syncResults;
report.after.pgvectorMemories = await countPgvectorForUsers(pgPool, canaryUserIds);
} else {
report.steps.pgvectorSync = { skipped: true, reason: 'pgvector not configured' };
}
report.after.candidateCounts = await countCandidatesByStatus(mysqlPool);
report.after.activeMemories = await countActiveMemoriesForUsers(mysqlPool, canaryUserIds);
const promotedTotal = Array.isArray(report.steps.promote)
? report.steps.promote.reduce((sum, item) => sum + Number(item.result?.promoted ?? 0), 0)
: 0;
const pendingAfter = Number(report.after.candidateCounts?.candidate ?? 0);
const acceptedAfter = Number(report.after.candidateCounts?.accepted ?? 0);
report.ok = readiness.ok
&& pendingAfter === 0
&& report.after.activeMemories > 0
&& (acceptedAfter === 0 || promotedTotal > 0 || report.after.activeMemories >= report.before.activeMemories)
&& (options.skipPgvector || !env.MEMORY_PGVECTOR_DATABASE_URL
|| Number(report.after.pgvectorMemories ?? 0) >= Math.max(1, report.after.activeMemories - 5));
report.summary = {
pendingCandidates: Number(report.after.candidateCounts?.candidate ?? 0),
acceptedCandidates: Number(report.after.candidateCounts?.accepted ?? 0),
promotedThisRun: promotedTotal,
activeMemoriesCanary: report.after.activeMemories,
pgvectorMemoriesCanary: report.after.pgvectorMemories ?? null,
};
return report;
} finally {
await pgPool?.end?.();
await mysqlPool.end?.();
}
}
async function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
console.log(usage());
return;
}
loadMemindEnvFiles(process.cwd());
const report = await runMemoryV2PhaseAClosure(process.env, options);
if (options.json) {
console.log(JSON.stringify(report, null, 2));
} else {
console.log(`memory v2 phase-a closure: ${report.ok ? 'ok' : 'check required'}`);
console.log(JSON.stringify(report.summary, null, 2));
if (report.steps.autoReview && !report.steps.autoReview.skipped) {
console.log(`auto-review: scanned=${report.steps.autoReview.scanned} accept=${report.steps.autoReview.accepted} reject=${report.steps.autoReview.rejected}`);
}
if (Array.isArray(report.steps.promote)) {
for (const item of report.steps.promote) {
console.log(`promote user=${item.userId ?? 'all'} promoted=${item.result?.promoted ?? 0}`);
}
}
}
process.exit(report.ok ? 0 : 1);
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
void main().catch((err) => {
console.error(err instanceof Error ? err.message : err);
process.exit(1);
});
}
@@ -0,0 +1,76 @@
#!/usr/bin/env node
import { pathToFileURL } from 'node:url';
import { ensurePgvectorMemorySchema } from '../memory-v2-pgvector-schema.mjs';
import { runPgvectorMemorySmoke } from '../memory-v2-pgvector-smoke.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
function usage() {
return [
'Usage: node scripts/setup-memory-v2-pgvector-local.mjs [--dimensions 3] [--skip-smoke]',
'',
'Ensures local pgvector schema exists and runs a read/write smoke test.',
].join('\n');
}
function parseArgs(argv) {
const args = { dimensions: 3, skipSmoke: false };
for (let i = 2; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--dimensions' && argv[i + 1]) args.dimensions = Number(argv[++i]);
else if (arg === '--skip-smoke') args.skipSmoke = true;
else if (arg === '--help' || arg === '-h') {
console.log(usage());
process.exit(0);
} else {
console.error(`Unknown argument: ${arg}`);
console.error(usage());
process.exit(2);
}
}
return args;
}
async function createPgPool(connectionString) {
const imported = await import('pg');
const PgPool = imported?.Pool ?? imported?.default?.Pool;
if (typeof PgPool !== 'function') throw new Error('pg module does not export Pool');
return new PgPool({ connectionString, max: 3 });
}
async function main() {
const args = parseArgs(process.argv);
loadMemindEnvFiles(process.cwd());
const connectionString = String(process.env.MEMORY_PGVECTOR_DATABASE_URL ?? '').trim()
|| 'postgresql://john@127.0.0.1:5432/memind_memory';
const tableName = String(process.env.MEMORY_PGVECTOR_TABLE ?? 'memory_embeddings').trim()
|| 'memory_embeddings';
const pool = await createPgPool(connectionString);
try {
const schema = await ensurePgvectorMemorySchema(pool, {
tableName,
dimensions: args.dimensions,
createExtension: true,
});
console.log(`pgvector schema ready: table=${schema.tableName} dimensions=${schema.dimensions}`);
if (!args.skipSmoke) {
const smoke = await runPgvectorMemorySmoke({
pool,
tableName,
dimensions: args.dimensions,
cleanup: true,
embedQuery: async () => Array.from({ length: args.dimensions }, (_, index) => (index + 1) * 0.1),
});
console.log(`pgvector smoke: ${smoke.ok ? 'ok' : 'failed'} inserted=${smoke.inserted} queried=${smoke.queried}`);
if (!smoke.ok) process.exit(1);
}
} finally {
await pool.end();
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((err) => {
console.error(err instanceof Error ? err.message : err);
process.exit(1);
});
}