feat(mindspace): enforce PostgreSQL user data delivery

This commit is contained in:
john
2026-07-13 15:29:29 +08:00
parent b33c943b69
commit a6620fb719
57 changed files with 2779 additions and 164 deletions
+9 -16
View File
@@ -8,6 +8,11 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
applyMemindRuntimeProfile,
describeMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const opsDir = path.join(root, 'ops');
@@ -23,22 +28,6 @@ const plazaPublicBase = (
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
).replace(/\/$/, '');
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(root, '../../.env.local'));
loadEnvFile(path.join(root, '.env'));
const logFile =
process.env.MEMIND_DEV_LOG ?? path.join(os.homedir(), 'Library/Logs/memind-dev.log');
@@ -51,6 +40,10 @@ function log(message) {
}
}
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
log(`Runtime profile: ${describeMemindRuntimeProfile()}`);
function freePort(port) {
try {
execSync(`lsof -ti TCP:${port} -sTCP:LISTEN | xargs kill -9`, { stdio: 'ignore' });
+8 -15
View File
@@ -3,6 +3,11 @@ import { spawn, execSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
applyMemindRuntimeProfile,
describeMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const opsDir = path.join(root, 'ops');
@@ -18,21 +23,9 @@ const plazaPublicBase = (
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
).replace(/\/$/, '');
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(root, '../../.env.local'));
loadEnvFile(path.join(root, '.env'));
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
console.log(`==> Runtime profile: ${describeMemindRuntimeProfile()}`);
function freePort(port) {
try {
+7 -16
View File
@@ -1,20 +1,11 @@
import fs from 'node:fs';
import path from 'node:path';
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const separator = trimmed.indexOf('=');
if (separator < 0) continue;
const key = trimmed.slice(0, separator).trim();
const value = trimmed.slice(separator + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
import {
applyMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
export function loadH5Environment(scriptDirectory) {
loadEnvFile(path.join(scriptDirectory, '../../../.env.local'));
loadEnvFile(path.join(scriptDirectory, '../.env'));
const root = path.join(scriptDirectory, '..');
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
}
+119
View File
@@ -0,0 +1,119 @@
import fs from 'node:fs';
import path from 'node:path';
export const MEMIND_RUNTIME_PROFILES = new Set(['local', 'split-service', 'production']);
/**
* Load Memind env files in standard order. Later files only fill unset keys.
*/
export function loadMemindEnvFiles(rootDir, env = process.env) {
const root = path.resolve(rootDir);
for (const relativePath of ['../../.env.local', '.env']) {
loadEnvFile(path.join(root, relativePath), env);
}
}
function loadEnvFile(filePath, env = process.env) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const separator = trimmed.indexOf('=');
if (separator < 0) continue;
const key = trimmed.slice(0, separator).trim();
const value = trimmed.slice(separator + 1).trim();
if (!env[key]) env[key] = value;
}
}
function expandEnvReference(value, env = process.env) {
const raw = String(value ?? '').trim();
const match = raw.match(/^\$\{([A-Z0-9_]+)\}$/);
if (!match) return raw;
return String(env[match[1]] ?? '').trim() || raw;
}
export function resolveMemindRuntimeProfile(env = process.env) {
const explicit = String(env.MEMIND_RUNTIME_PROFILE ?? '').trim().toLowerCase();
if (explicit) {
if (!MEMIND_RUNTIME_PROFILES.has(explicit)) {
throw new Error(
`Unsupported MEMIND_RUNTIME_PROFILE "${explicit}". Expected: ${[...MEMIND_RUNTIME_PROFILES].join(', ')}`,
);
}
return explicit;
}
if (String(env.NODE_ENV ?? '').trim().toLowerCase() === 'production') {
return 'production';
}
return 'local';
}
/**
* Apply environment-specific defaults after .env files are loaded.
*
* - local: monolith MindSpace adapter, repo-local storage (pnpm dev default)
* - split-service: remote adapter against standalone MindSpace on 8082
* - production: no overrides; 103 .env is source of truth
*/
export function applyMemindRuntimeProfile({
rootDir = process.cwd(),
env = process.env,
profile,
} = {}) {
const resolvedProfile = profile ?? resolveMemindRuntimeProfile(env);
env.MEMIND_RUNTIME_PROFILE = resolvedProfile;
if (resolvedProfile === 'local') {
env.MINDSPACE_SERVER_ADAPTER = 'local';
if (!env.MINDSPACE_STORAGE_ROOT) {
env.MINDSPACE_STORAGE_ROOT = path.join(path.resolve(rootDir), 'data', 'mindspace');
}
return {
profile: resolvedProfile,
mindspaceAdapter: 'local',
storageRoot: env.MINDSPACE_STORAGE_ROOT,
};
}
if (resolvedProfile === 'split-service') {
env.MINDSPACE_SERVER_ADAPTER = 'remote';
env.MINDSPACE_REMOTE_BASE_URL = String(
env.MINDSPACE_REMOTE_BASE_URL ?? 'http://127.0.0.1:8082',
).trim();
const expandedToken = expandEnvReference(env.MINDSPACE_REMOTE_AUTH_TOKEN, env);
if (expandedToken) {
env.MINDSPACE_REMOTE_AUTH_TOKEN = expandedToken;
} else if (env.TKMIND_SERVER__SECRET_KEY) {
env.MINDSPACE_REMOTE_AUTH_TOKEN = String(env.TKMIND_SERVER__SECRET_KEY).trim();
}
if (!env.MINDSPACE_STORAGE_ROOT) {
env.MINDSPACE_STORAGE_ROOT = '/Users/john/MindSpace/data/mindspace';
}
return {
profile: resolvedProfile,
mindspaceAdapter: 'remote',
remoteBaseUrl: env.MINDSPACE_REMOTE_BASE_URL,
storageRoot: env.MINDSPACE_STORAGE_ROOT,
};
}
return {
profile: resolvedProfile,
mindspaceAdapter: String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local',
storageRoot: env.MINDSPACE_STORAGE_ROOT ?? null,
};
}
export function describeMemindRuntimeProfile(env = process.env) {
const profile = resolveMemindRuntimeProfile(env);
const adapter = String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local';
const parts = [`MEMIND_RUNTIME_PROFILE=${profile}`, `MINDSPACE_SERVER_ADAPTER=${adapter}`];
if (adapter === 'remote') {
parts.push(`MINDSPACE_REMOTE_BASE_URL=${env.MINDSPACE_REMOTE_BASE_URL ?? '(unset)'}`);
}
if (env.MINDSPACE_STORAGE_ROOT) {
parts.push(`MINDSPACE_STORAGE_ROOT=${env.MINDSPACE_STORAGE_ROOT}`);
}
return parts.join(', ');
}
+54
View File
@@ -0,0 +1,54 @@
import assert from 'node:assert/strict';
import path from 'node:path';
import test from 'node:test';
import {
applyMemindRuntimeProfile,
resolveMemindRuntimeProfile,
} from './memind-runtime-profile.mjs';
test('resolveMemindRuntimeProfile defaults to local for dev', () => {
assert.equal(resolveMemindRuntimeProfile({}), 'local');
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'development' }), 'local');
});
test('resolveMemindRuntimeProfile honors explicit profile and production NODE_ENV', () => {
assert.equal(resolveMemindRuntimeProfile({ MEMIND_RUNTIME_PROFILE: 'split-service' }), 'split-service');
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'production' }), 'production');
});
test('applyMemindRuntimeProfile local forces monolith adapter and repo storage', () => {
const env = {
MINDSPACE_SERVER_ADAPTER: 'remote',
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
};
const result = applyMemindRuntimeProfile({
rootDir: '/tmp/memind',
env,
profile: 'local',
});
assert.equal(result.profile, 'local');
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'local');
assert.match(env.MINDSPACE_STORAGE_ROOT, /\/tmp\/memind\/data\/mindspace$/);
});
test('applyMemindRuntimeProfile split-service expands auth token reference', () => {
const env = {
TKMIND_SERVER__SECRET_KEY: 'local-dev-secret',
MINDSPACE_REMOTE_AUTH_TOKEN: '${TKMIND_SERVER__SECRET_KEY}',
};
applyMemindRuntimeProfile({ env, profile: 'split-service' });
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'local-dev-secret');
});
test('applyMemindRuntimeProfile production does not override adapter', () => {
const env = {
MINDSPACE_SERVER_ADAPTER: 'remote',
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
MINDSPACE_REMOTE_AUTH_TOKEN: 'prod-token',
};
const result = applyMemindRuntimeProfile({ env, profile: 'production' });
assert.equal(result.profile, 'production');
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'prod-token');
});
@@ -0,0 +1,85 @@
#!/usr/bin/env node
import fs from 'node:fs';
import {
buildControlSchemaSql,
createSqliteSnapshot,
inspectSqliteDatabase,
migrateSqliteSnapshot,
provisionUserSpace,
} from '../mindspace-userdata-postgres.mjs';
function usage() {
return `
Usage:
node scripts/migrate-mindspace-sqlite-to-postgres.mjs --source <sqlite> --user-id <uuid> [--apply] [--replace-shadow]
Default is read-only dry-run. --apply requires MINDSPACE_USERDATA_PG_URL.
The command creates a consistent SQLite snapshot and never modifies the source SQLite.
`.trim();
}
export function parseArgs(argv = []) {
const options = { apply: false, replaceShadow: false, source: '', userId: '', help: false };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--apply') options.apply = true;
else if (arg === '--replace-shadow') options.replaceShadow = true;
else if (arg === '--source') options.source = argv[++index] ?? '';
else if (arg === '--user-id') options.userId = argv[++index] ?? '';
else if (arg === '--help' || arg === '-h') options.help = true;
else throw new Error(`未知参数:${arg}`);
}
if (!options.help && (!options.source || !options.userId)) throw new Error('--source 和 --user-id 必填');
return options;
}
export async function run(argv = process.argv.slice(2), env = process.env) {
const options = parseArgs(argv);
if (options.help) {
console.log(usage());
return { mode: 'help' };
}
const inspected = inspectSqliteDatabase(options.source);
const sourceRows = inspected.tables.reduce((total, table) => total + table.rows.length, 0);
if (!options.apply) {
console.log(JSON.stringify({
mode: 'dry-run',
userId: options.userId,
source: inspected.path,
sizeBytes: inspected.sizeBytes,
tables: inspected.tables.map((table) => ({ name: table.name, rows: table.rows.length })),
sourceRows,
note: '未修改 SQLite 或 PostgreSQL;使用 --apply 才会创建影子空间。',
}, null, 2));
return { mode: 'dry-run', tableCount: inspected.tables.length, sourceRows };
}
if (!env.MINDSPACE_USERDATA_PG_URL) throw new Error('--apply requires MINDSPACE_USERDATA_PG_URL');
const snapshotPath = createSqliteSnapshot(options.source);
const snapshot = inspectSqliteDatabase(snapshotPath);
const { default: pg } = await import('pg');
const client = new pg.Client({ connectionString: env.MINDSPACE_USERDATA_PG_URL });
try {
await client.connect();
await client.query(buildControlSchemaSql());
await provisionUserSpace(client, options.userId, {
sourceSqlitePath: inspected.path,
});
const result = await migrateSqliteSnapshot(client, snapshot, options.userId, {
replaceShadow: options.replaceShadow,
sourcePath: inspected.path,
});
if (result.sourceRows !== result.targetRows) throw new Error('迁移行数校验失败');
console.log(JSON.stringify({ mode: 'shadow', ...result }, null, 2));
return { mode: 'shadow', ...result };
} finally {
await client.end().catch(() => {});
fs.rmSync(snapshotPath, { force: true });
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
run().catch((error) => {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 1;
});
}
@@ -0,0 +1,25 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { parseArgs } from './migrate-mindspace-sqlite-to-postgres.mjs';
test('migration CLI defaults to dry-run', () => {
assert.deepEqual(parseArgs(['--source', '/tmp/a.sqlite', '--user-id', 'user-id']), {
apply: false,
replaceShadow: false,
source: '/tmp/a.sqlite',
userId: 'user-id',
help: false,
});
});
test('migration CLI requires explicit source and user', () => {
assert.throws(() => parseArgs([]), /--source 和 --user-id 必填/);
});
test('migration CLI recognizes explicit apply', () => {
assert.equal(parseArgs(['--apply', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).apply, true);
});
test('migration CLI requires explicit replace-shadow for destructive reruns', () => {
assert.equal(parseArgs(['--replace-shadow', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).replaceShadow, true);
});
@@ -0,0 +1,198 @@
#!/usr/bin/env node
/**
* Local-only E2E proof for the questionnaire -> Page Data API -> PostgreSQL path.
*
* The probe uses an existing local user space, creates a uniquely named survey
* table/dataset and page policy, submits one answer through the public HTTP API,
* verifies the row directly in PostgreSQL, and removes all probe objects.
*/
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import express from 'express';
import pg from 'pg';
import { attachPageDataRoutes } from '../page-data-routes.mjs';
import { createPageDataService } from '../page-data-service.mjs';
import { createPageDataPublicService } from '../page-data-public-service.mjs';
import { deletePageAccessPolicy, writePageAccessPolicy } from '../page-data-policy-store.mjs';
import { createUserDataSpaceService } from '../user-data-space-service.mjs';
import { deriveUserSpaceNames, quotePgIdentifier } from '../mindspace-userdata-postgres.mjs';
import { loadH5Environment } from './load-env.mjs';
loadH5Environment(import.meta.dirname);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const userId = process.argv[2] ?? '1c99b83b-0454-474f-a5d2-129d34506a32';
const workspaceRoot = path.join(repoRoot, 'MindSpace', userId);
const suffix = `${Date.now()}_${process.pid}`;
const tableName = `survey_e2e_${suffix}`;
const datasetName = tableName;
const pageId = `page-survey-e2e-${suffix}`;
const names = deriveUserSpaceNames(userId);
const policyPath = path.join(workspaceRoot, '.mindspace', 'page-data-policies', `${pageId}.json`);
const logPath = path.join(workspaceRoot, '.mindspace', 'page-data-logs', `${pageId}.jsonl`);
assert.equal(process.env.MINDSPACE_USERDATA_BACKEND, 'postgres', '本机未启用 PostgreSQL 用户数据后端');
assert.ok(fs.existsSync(workspaceRoot), `用户工作区不存在: ${workspaceRoot}`);
const pgPool = new pg.Pool({
host: process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp',
port: Number(process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433),
database: process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev',
user: process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER,
max: 2,
});
function publicationPool() {
return {
async query(sql) {
if (sql.includes('FROM h5_publish_records')) {
return [[{
id: `publication-${suffix}`,
user_id: userId,
page_id: pageId,
access_mode: 'public',
password_hash: null,
status: 'online',
expires_at: null,
}]];
}
return [[]];
},
};
}
function buildApp() {
const fakePublicationPool = publicationPool();
const publicService = createPageDataPublicService({
getPool: () => fakePublicationPool,
resolveWorkspaceRootForOwner: () => workspaceRoot,
});
const api = express.Router();
api.use(express.json());
attachPageDataRoutes(api, {
sendData: (res, _req, data, status = 200) => res.status(status).json({ data }),
sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }),
getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }),
getPageDataPublicService: () => publicService,
});
const app = express();
app.set('trust proxy', true);
app.use('/api', api);
return app;
}
async function submitQuestionnaire(app, row) {
const server = app.listen(0, '127.0.0.1');
await new Promise((resolve, reject) => {
server.once('listening', resolve);
server.once('error', reject);
});
try {
const { port } = server.address();
const apiPath = `/api/public/pages/${pageId}/data/${datasetName}/rows`;
const response = await fetch(`http://127.0.0.1:${port}${apiPath}`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(row),
});
return { apiPath, status: response.status, body: await response.json() };
} finally {
await new Promise((resolve) => server.close(resolve));
}
}
async function main() {
const ownerService = createUserDataSpaceService({ workspaceRoot, userId });
let tableCreated = false;
try {
// Equivalent to the Agent's private_data_execute + register_dataset + bind policy flow.
await ownerService.executeSql(`CREATE TABLE ${tableName} (
id INTEGER PRIMARY KEY AUTOINCREMENT,
satisfaction TEXT NOT NULL,
favorite_feature TEXT NOT NULL,
suggestion TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now', 'localtime'))
)`);
tableCreated = true;
const dataset = await ownerService.upsertDataset({
name: datasetName,
table: tableName,
description: '本机 PG 问卷端到端验证',
actions: ['read', 'insert'],
columns: {
read: ['id', 'satisfaction', 'favorite_feature', 'suggestion', 'created_at'],
insert: ['satisfaction', 'favorite_feature', 'suggestion'],
},
});
writePageAccessPolicy(workspaceRoot, {
pageId,
ownerUserId: userId,
accessMode: 'public',
datasets: {
[datasetName]: {
insert: true,
read: false,
columns: { insert: ['satisfaction', 'favorite_feature', 'suggestion'] },
},
},
});
const answer = {
satisfaction: '非常满意',
favorite_feature: '个人数据报表',
suggestion: '增加按月趋势分析',
};
const submitted = await submitQuestionnaire(buildApp(), answer);
assert.equal(submitted.status, 201, JSON.stringify(submitted.body));
assert.equal(submitted.body.data.dataset, datasetName);
assert.equal(submitted.body.data.row.satisfaction, answer.satisfaction);
const direct = await pgPool.query(
`SELECT id,satisfaction,favorite_feature,suggestion,created_at
FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(tableName)}
ORDER BY id DESC LIMIT 1`,
);
assert.equal(direct.rowCount, 1);
assert.equal(direct.rows[0].satisfaction, answer.satisfaction);
assert.equal(direct.rows[0].favorite_feature, answer.favorite_feature);
assert.equal(direct.rows[0].suggestion, answer.suggestion);
const registered = await pgPool.query(
`SELECT name,table_name FROM ${quotePgIdentifier(names.schemaName)}.__page_data_datasets WHERE name=$1`,
[datasetName],
);
assert.equal(registered.rowCount, 1);
console.log(JSON.stringify({
result: 'PASS',
localOnly: true,
userId,
schema: names.schemaName,
dataset: { name: dataset.name, table: dataset.table, actions: dataset.actions },
pageDataApi: { method: 'POST', path: submitted.apiPath, status: submitted.status },
postgresRow: direct.rows[0],
checks: [
'Agent-style table creation succeeded in the user schema',
'Dataset registry was stored in PostgreSQL',
'Questionnaire submission used the public Page Data HTTP API',
'Submitted answers matched a direct PostgreSQL query',
],
}, null, 2));
} finally {
if (tableCreated) {
await ownerService.executeSql(`DELETE FROM __page_data_datasets WHERE name='${datasetName}'`).catch(() => {});
await ownerService.executeSql(`DROP TABLE IF EXISTS ${tableName}`).catch(() => {});
}
deletePageAccessPolicy(workspaceRoot, pageId);
fs.rmSync(logPath, { force: true });
fs.rmSync(policyPath, { force: true });
await pgPool.end();
}
}
main().catch((error) => {
console.error(error instanceof Error ? error.stack ?? error.message : error);
process.exitCode = 1;
});