feat(mindspace): enforce PostgreSQL user data delivery

This commit is contained in:
john
2026-07-13 15:29:29 +08:00
parent b33c943b69
commit a6620fb719
57 changed files with 2779 additions and 164 deletions
+28 -1
View File
@@ -242,4 +242,31 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind
# MINDSPACE_AGENT_JOBS_ENABLED=true
# MINDSPACE_INTERNAL_AGENT_SECRET=local-dev-secret
# MINDSPACE_FREE_PUBLIC_PAGE_LIMIT=10
# MINDSPACE_STORAGE_ROOT=/path/to/mindspace-storage
# ---------------------------------------------------------------------------
# Memind runtime profile(本地 vs 生产 必须区分)
# 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。
#
# | profile | 场景 | MindSpace 数据路径 |
# |----------------|-------------------|--------------------|
# | local | 本机日常开发 | <repo>/data/mindspace |
# | split-service | 本机验证 103 拆分 | /Users/john/MindSpace/data/mindspace |
# | production | 103 生产服务器 | 103 .env 为准,勿提交 Git |
# ---------------------------------------------------------------------------
# 【本地开发 — 复制到 .env 的默认值】
MEMIND_RUNTIME_PROFILE=local
# MINDSPACE_STORAGE_ROOT 留空即可,自动使用 <repo>/data/mindspace
# 【本机 split-service 联调 — 需要独立 MindSpace LaunchAgent 已启动】
# MEMIND_RUNTIME_PROFILE=split-service
# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
# MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret
# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace
# 【103 生产 — 仅在服务器维护,示例勿照抄密钥】
# MEMIND_RUNTIME_PROFILE=production
# MINDSPACE_SERVER_ADAPTER=remote
# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
# MINDSPACE_REMOTE_AUTH_TOKEN=<与 cn.tkmind.mindspace-service 一致>
# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace
+34 -7
View File
@@ -26,6 +26,7 @@ function mapDeliverableRows(rows) {
publicationId: row.publication_id ? String(row.publication_id) : null,
publicationStatus: row.publication_status ? String(row.publication_status) : null,
publicUrl: row.public_url ? String(row.public_url) : null,
workspaceRelativePath: row.workspace_relative_path ? String(row.workspace_relative_path) : null,
})).filter((row) => row.pageId);
const publicationCount = pages.filter((row) => row.publicationId).length;
return {
@@ -51,13 +52,16 @@ export function mergeDeliverableSummaries(...summaries) {
};
}
export async function detectSessionDeliverables(pool, userId, sessionId) {
export async function detectSessionDeliverables(pool, userId, sessionId, { sinceMs = null } = {}) {
if (!pool?.query || !userId || !sessionId) {
return { pageCount: 0, publicationCount: 0, pages: [] };
}
const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : '';
const params = sinceMs != null ? [userId, sessionId, sinceMs] : [userId, sessionId];
const [rows] = await pool.query(
`SELECT p.id AS page_id,
p.title,
p.workspace_relative_path,
pr.id AS publication_id,
pr.status AS publication_status,
pr.public_url
@@ -68,22 +72,39 @@ export async function detectSessionDeliverables(pool, userId, sessionId) {
AND pr.status = 'online'
WHERE p.user_id = ?
AND p.source_session_id = ?
${sinceClause}
AND p.status <> 'deleted'
ORDER BY p.created_at ASC`,
[userId, sessionId],
params,
);
return mapDeliverableRows(rows);
}
export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs = null } = {}) {
export async function detectWorkspaceSyncedDeliverables(
pool,
userId,
{ sinceMs = null, onlyRelativePaths = null } = {},
) {
if (!pool?.query || !userId) {
return { pageCount: 0, publicationCount: 0, pages: [] };
}
const normalizedPaths = onlyRelativePaths == null
? null
: [...new Set(onlyRelativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))];
if (normalizedPaths?.length === 0) {
return { pageCount: 0, publicationCount: 0, pages: [] };
}
const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : '';
const params = sinceMs != null ? [userId, sinceMs] : [userId];
const pathClause = normalizedPaths == null
? ''
: `AND p.workspace_relative_path IN (${normalizedPaths.map(() => '?').join(', ')})`;
const params = [userId];
if (sinceMs != null) params.push(sinceMs);
if (normalizedPaths != null) params.push(...normalizedPaths);
const [rows] = await pool.query(
`SELECT p.id AS page_id,
p.title,
p.workspace_relative_path,
pr.id AS publication_id,
pr.status AS publication_status,
pr.public_url
@@ -96,6 +117,7 @@ export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs
WHERE p.user_id = ?
AND p.status <> 'deleted'
${sinceClause}
${pathClause}
AND (
p.workspace_relative_path LIKE 'public/%.html'
OR JSON_UNQUOTE(JSON_EXTRACT(pv.source_snapshot_json, '$.relative_path')) LIKE 'public/%.html'
@@ -116,15 +138,20 @@ export async function prepareAndDetectSessionDeliverables({
sessionId,
runStartedAtMs = null,
prepareDeliverables = null,
workspaceRelativePaths = null,
} = {}) {
let prepared = null;
if (typeof prepareDeliverables === 'function') {
await prepareDeliverables({ userId, sessionId }).catch(() => {});
prepared = await prepareDeliverables({ userId, sessionId }).catch(() => null);
}
const sinceMs = runStartedAtMs == null
? null
: Math.max(0, Number(runStartedAtMs) - WORKSPACE_DELIVERABLE_LOOKBACK_MS);
const bySession = await detectSessionDeliverables(pool, userId, sessionId);
const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs });
const bySession = await detectSessionDeliverables(pool, userId, sessionId, { sinceMs });
const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, {
sinceMs,
onlyRelativePaths: workspaceRelativePaths ?? prepared?.pageDataRelativePaths ?? null,
});
return mergeDeliverableSummaries(bySession, byWorkspace);
}
+34
View File
@@ -98,6 +98,40 @@ test('detectWorkspaceSyncedDeliverables filters auto-synced public html pages',
assert.equal(summary.pageCount, 1);
});
test('detectWorkspaceSyncedDeliverables stays inside current run artifact paths', async () => {
const pool = {
async query(sql, params) {
assert.match(sql, /workspace_relative_path IN \(\?, \?\)/);
assert.deepEqual(params, [
'user-1',
1000,
'public/current.html',
'public/current-admin.html',
]);
return [[{
page_id: 'page-current',
title: 'Current',
workspace_relative_path: 'public/current.html',
}]];
},
};
const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', {
sinceMs: 1000,
onlyRelativePaths: ['public/current.html', 'public/current-admin.html'],
});
assert.equal(summary.pageCount, 1);
assert.equal(summary.pages[0].workspaceRelativePath, 'public/current.html');
});
test('detectWorkspaceSyncedDeliverables skips workspace sweep for an empty run scope', async () => {
const pool = { async query() { throw new Error('must not query'); } };
const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', {
sinceMs: 1000,
onlyRelativePaths: [],
});
assert.equal(summary.pageCount, 0);
});
test('prepareAndDetectSessionDeliverables merges session and workspace pages after prepare hook', async () => {
const prepareCalls = [];
const pool = {
+47 -14
View File
@@ -16,7 +16,7 @@ import {
SESSION_FINISHED_STALE_GRACE_MS,
tryRecoverRunFromDeliverables,
} from './agent-run-deliverable-check.mjs';
import { isPageDataIntent } from './chat-skills.mjs';
import { isPageDataIntent, isPageGenerationIntent } from './chat-skills.mjs';
const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000];
const TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
@@ -262,6 +262,7 @@ export function createAgentRunGateway({
syncUserPagesOnSuccess = null,
observePersonalMemoryOnSuccess = null,
isSessionExternallyBusy = null,
validateRunDeliverables = null,
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
maxConcurrentRuns = positiveInteger(
@@ -420,7 +421,7 @@ export function createAgentRunGateway({
async function prepareRunDeliverables({ userId, sessionId, runId, runStartedAtMs = null }) {
if (typeof syncUserPagesOnSuccess !== 'function') return;
await syncUserPagesOnSuccess({
return await syncUserPagesOnSuccess({
userId,
sessionId,
runId,
@@ -619,7 +620,7 @@ export function createAgentRunGateway({
billed: Boolean(result.billing?.ok),
});
await appendRunSnapshot(runId);
return { sessionId: result.sessionId };
return { sessionId: result.sessionId, routing };
} catch (err) {
await appendEvent(runId, 'direct_chat_failed', {
code: err?.code ?? null,
@@ -673,7 +674,7 @@ export function createAgentRunGateway({
});
throw err;
}
return { sessionId: row.agent_session_id ?? null };
return { sessionId: row.agent_session_id ?? null, routing };
}
let sessionId = resolveGatewayAgentSessionId({
@@ -769,7 +770,7 @@ export function createAgentRunGateway({
sessionId,
err,
})) {
return { sessionId };
return { sessionId, routing };
}
throw err;
}
@@ -785,16 +786,17 @@ export function createAgentRunGateway({
},
);
}
return { sessionId };
return { sessionId, routing };
}
async function finalizeSuccessfulRun(runId, row, sessionId) {
async function finalizeSuccessfulRun(runId, row, sessionId, { routing = null } = {}) {
let deliveryResult = null;
if (typeof syncUserPagesOnSuccess === 'function') {
deliveryResult = await syncUserPagesOnSuccess({
userId: row.user_id,
sessionId,
runId,
runStartedAtMs: row.started_at ?? null,
});
}
const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors)
@@ -806,17 +808,48 @@ export function createAgentRunGateway({
error.retryable = false;
throw error;
}
if (isPageDataIntent(extractRunMessageText(row))) {
const runMessageText = extractRunMessageText(row);
const pageDataIntent = isPageDataIntent(runMessageText)
|| routing?.suggestedSkill === 'page-data-collect';
const pageGenerationIntent = isPageGenerationIntent(runMessageText)
|| routing?.suggestedSkill === 'static-page-publish';
const requiresPageDeliverable = pageDataIntent || pageGenerationIntent;
let deliverables = null;
if (requiresPageDeliverable || typeof validateRunDeliverables === 'function') {
const latest = await getRunById(runId);
const deliverables = await prepareAndDetectSessionDeliverables({
deliverables = await prepareAndDetectSessionDeliverables({
pool,
userId: row.user_id,
sessionId,
runStartedAtMs: latest?.started_at ?? row.started_at ?? null,
workspaceRelativePaths: deliveryResult?.pageDataRelativePaths ?? [],
});
if (deliverables.pageCount < 1) {
const error = new Error('Page Data 任务未生成可交付页面,不能标记成功');
error.code = 'PAGE_DATA_DELIVERABLE_MISSING';
if (requiresPageDeliverable && deliverables.pageCount < 1) {
const error = new Error(
pageDataIntent
? 'Page Data 任务未生成可交付页面,不能标记成功'
: '页面任务未生成 public HTML 交付物,不能标记成功',
);
error.code = pageDataIntent
? 'PAGE_DATA_DELIVERABLE_MISSING'
: 'PUBLIC_PAGE_DELIVERABLE_MISSING';
error.retryable = false;
throw error;
}
}
if (typeof validateRunDeliverables === 'function') {
const validation = await validateRunDeliverables({
userId: row.user_id,
sessionId,
runId,
deliverables: deliverables ?? { pageCount: 0, publicationCount: 0, pages: [] },
});
const errors = Array.isArray(validation?.errors) ? validation.errors : [];
if (errors.length > 0) {
const error = new Error(
`页面交付违反数据存储策略:${errors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`,
);
error.code = 'DELIVERABLE_DATA_STORAGE_FORBIDDEN';
error.retryable = false;
throw error;
}
@@ -856,8 +889,8 @@ export function createAgentRunGateway({
const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt });
try {
const { sessionId } = await runWithTimeout(runId, () => executeRun(row, runId));
await finalizeSuccessfulRun(runId, row, sessionId);
const { sessionId, routing } = await runWithTimeout(runId, () => executeRun(row, runId));
await finalizeSuccessfulRun(runId, row, sessionId, { routing });
} catch (err) {
const latest = await getRunById(runId);
const recoverySessionId = latest?.agent_session_id ?? row.agent_session_id ?? null;
+261 -5
View File
@@ -387,6 +387,191 @@ test('agent run awaits session Finish before succeeding when proxy supports it',
assert.equal(finishEvents.length, 1);
});
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
const pool = createFakePool();
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-page-data-missing' };
},
async submitSessionReplyAndAwaitFinishForUser() {
return { ok: true, finishEvent: { type: 'Finish' } };
},
},
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-page-data-missing',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我创建调查问卷,保存提交记录并发布页面' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
assert.match(pool.runs.get(run.id).error_message, /未生成可交付页面/);
});
test('implicit sticky-note app run fails closed when Apps returns Finish without a public page', async () => {
const pool = createFakePool();
const gateway = createAgentRunGateway({
pool,
userAuth: {},
chatIntentRouter: {
isEnabled() {
return true;
},
async classify() {
return {
route: 'agent_orchestration',
confidence: 0.96,
reason: '页面需要数据交互与持久化',
suggestedSkill: 'page-data-collect',
source: 'rule',
};
},
applyAgentOrchestration(message) {
return message;
},
},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-sticky-note-missing' };
},
async submitSessionReplyAndAwaitFinishForUser() {
return { ok: true, finishEvent: { type: 'Finish' } };
},
},
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-sticky-note-missing',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我设计一个便签提醒,可以写便签提交,时间轴来显示' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
assert.match(pool.runs.get(run.id).error_message, /Page Data 任务未生成可交付页面/);
assert.ok(pool.events.some((event) => event.eventType === 'intent_routed'));
});
test('static page run fails closed when Finish arrives without public HTML', async () => {
const pool = createFakePool();
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-public-page-missing' };
},
async submitSessionReplyAndAwaitFinishForUser() {
return { ok: true, finishEvent: { type: 'Finish' } };
},
},
syncUserPagesOnSuccess: async () => ({}),
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-public-page-missing',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我做一个秋夜诗的 H5 页面' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
assert.match(pool.runs.get(run.id).error_message, /public HTML 交付物/);
});
test('Page Data run succeeds only after a generated session page is detected', async () => {
const pool = createFakePool({
sessionDeliverables: {
'user-1:session-page-data-ready': [{
page_id: 'page-ready',
title: '问卷',
publication_id: 'pub-ready',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/u/john/pages/page-ready',
}],
},
});
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-page-data-ready' };
},
async submitSessionReplyAndAwaitFinishForUser() {
return { ok: true, finishEvent: { type: 'Finish' } };
},
},
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-page-data-ready',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '创建一个可以保存提交记录的问卷' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
});
test('agent run fails closed when a generated page violates browser storage policy', async () => {
const pool = createFakePool({
sessionDeliverables: {
'user-1:session-browser-storage': [{
page_id: 'page-storage',
title: '页面',
workspace_relative_path: 'public/page.html',
}],
},
});
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {
async startSessionForUser() {
return { id: 'session-browser-storage' };
},
async submitSessionReplyAndAwaitFinishForUser() {
return { ok: true, finishEvent: { type: 'Finish' } };
},
},
validateRunDeliverables: async ({ deliverables }) => ({
errors: deliverables.pages.some((page) => page.workspaceRelativePath === 'public/page.html')
? [{ code: 'browser_storage_forbidden', message: 'public/page.html 使用 localStorage' }]
: [],
}),
retryDelaysMs: [],
});
const run = await gateway.createRun('user-1', {
requestId: 'req-browser-storage',
userMessage: {
role: 'user',
content: [{ type: 'text', text: '帮我做一个展示页面' }],
},
});
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
assert.match(pool.runs.get(run.id).error_message, /页面交付违反数据存储策略/);
assert.match(pool.runs.get(run.id).error_message, /localStorage/);
});
test('agent run succeeds when Finish is missing but session pages were already created', async () => {
const pool = createFakePool({
sessionDeliverables: {
@@ -563,7 +748,17 @@ test('agent run uses direct chat on regular agent sessions when llm routes direc
});
test('agent run invalidates portal direct chat snapshot before submitting to goosed', async () => {
const pool = createFakePool();
const pool = createFakePool({
sessionDeliverables: {
'user-1:20260704_31': [{
page_id: 'page-essay',
title: '散文页面',
publication_id: 'pub-essay',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/essay.html',
}],
},
});
const submitted = [];
const invalidated = [];
const gateway = createAgentRunGateway({
@@ -723,7 +918,17 @@ test('agent run falls back to backend session when router is disabled', async ()
});
test('agent run uses chat intent router to enrich agent orchestration messages', async () => {
const pool = createFakePool();
const pool = createFakePool({
sessionDeliverables: {
'user-1:agent-session-1': [{
page_id: 'page-router-agent',
title: '路由页面',
publication_id: 'pub-router-agent',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/router.html',
}],
},
});
const submitted = [];
const gateway = createAgentRunGateway({
pool,
@@ -790,7 +995,17 @@ test('agent run uses chat intent router to enrich agent orchestration messages',
});
test('agent run escalates direct sessions to a new backend session when forced', async () => {
const pool = createFakePool();
const pool = createFakePool({
sessionDeliverables: {
'user-1:deep-session-1': [{
page_id: 'page-deep',
title: '深度页面',
publication_id: 'pub-deep',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html',
}],
},
});
const submitted = [];
const gateway = createAgentRunGateway({
pool,
@@ -820,7 +1035,17 @@ test('agent run escalates direct sessions to a new backend session when forced',
});
test('agent run persists direct session transcript before escalating to goosed', async () => {
const pool = createFakePool();
const pool = createFakePool({
sessionDeliverables: {
'user-1:deep-session-1': [{
page_id: 'page-deep-transcript',
title: '深度页面',
publication_id: 'pub-deep-transcript',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html',
}],
},
});
const submitted = [];
const saved = [];
const removed = [];
@@ -914,7 +1139,17 @@ test('agent run rejects reused goosed session when broker ownership check fails'
});
test('agent run persists portal direct snapshot before goosed submit on same session', async () => {
const pool = createFakePool();
const pool = createFakePool({
sessionDeliverables: {
'user-1:20260705_2': [{
page_id: 'page-report',
title: '报告页面',
publication_id: 'pub-report',
publication_status: 'online',
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/report.html',
}],
},
});
const submitted = [];
const saved = [];
const gateway = createAgentRunGateway({
@@ -1792,6 +2027,27 @@ test('createRun rejects with SESSION_RUN_CONFLICT when same session already has
assert.equal(run3.requestId, 'req-conflict-3');
});
test('createRun rejects while the same session is finishing page delivery', async () => {
const pool = createFakePool();
const gateway = createAgentRunGateway({
pool,
userAuth: {},
tkmindProxy: {},
autoDispatch: false,
isSessionExternallyBusy: ({ sessionId }) => sessionId === 'sess-repairing',
});
await assert.rejects(
gateway.createRun('user-1', {
sessionId: 'sess-repairing',
requestId: 'req-during-repair',
userMessage: { role: 'user', content: [{ type: 'text', text: '继续' }] },
}),
(err) => err?.code === 'SESSION_RUN_CONFLICT' && err?.status === 409 && /自动修复/.test(err.message),
);
assert.equal(pool.runs.size, 0);
});
test('createRun does not apply per-session conflict check for direct-chat sessions', async () => {
const pool = createFakePool();
const gateway = createAgentRunGateway({
+1 -1
View File
@@ -161,7 +161,7 @@ export const CAPABILITY_CATALOG = [
];
/** Capabilities that must never be enabled for regular users, even via DB overrides. */
export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin']);
export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin', 'apps']);
export function clampUserCapabilities(capabilities) {
const clamped = { ...capabilities };
+10
View File
@@ -164,6 +164,16 @@ test('clampUserCapabilities blocks extension_admin even when stored as true', ()
assert.equal(clamped.shell, true);
});
test('clampUserCapabilities blocks Apps for regular H5 users even when granted by role', () => {
const clamped = clampUserCapabilities({
...DEFAULT_USER_CAPABILITIES,
apps: true,
});
assert.equal(clamped.apps, false);
const policy = buildAgentExtensionPolicy(clamped);
assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'apps'), false);
});
test('catalog keys are unique', () => {
const keys = CAPABILITY_CATALOG.map((item) => item.key);
assert.equal(keys.length, new Set(keys).size);
+7 -3
View File
@@ -888,7 +888,10 @@ export function createChatIntentRouter(options = {}) {
function getStatus() {
return {
enabled: Boolean(policy.enabled),
enabled: true,
ruleRoutingEnabled: true,
llmRoutingEnabled: false,
configuredLlmEnabled: Boolean(policy.enabled),
modelProviderKeyId: policy.modelProviderKeyId ?? null,
model: policy.model ?? null,
modelApiType: policy.modelApiType ?? 'chat',
@@ -903,8 +906,9 @@ export function createChatIntentRouter(options = {}) {
}
function isEnabled() {
// Skill + rule routing only; LLM intent router is intentionally disabled.
return false;
// Rule + skill routing is always active. The LLM classifier stays disabled
// independently; returning false here would make the gateway skip rules too.
return true;
}
async function resolveRouterContext({ userId, sessionId, text, forceDeepReasoning = false }) {
+30 -2
View File
@@ -67,6 +67,34 @@ test('classifyWithRules routes page data collection to page-data-collect', () =>
assert.equal(result.suggestedSkill, 'page-data-collect');
});
test('classifyWithRules routes ordinary household ledger language to page-data-collect', () => {
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
const result = classifyWithRules({
text,
userMessage: {
role: 'user',
content: [{ type: 'text', text }],
metadata: { displayText: text },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.suggestedSkill, 'page-data-collect');
});
test('classifyWithRules routes implicit sticky-note timeline apps to page-data-collect', () => {
const text = '帮我设计一个便签提醒,可以写便签提交,时间轴来显示';
const result = classifyWithRules({
text,
userMessage: {
role: 'user',
content: [{ type: 'text', text }],
metadata: { displayText: text },
},
});
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
assert.equal(result.suggestedSkill, 'page-data-collect');
});
test('coercePageDataSkill overrides static-page-publish when data intent is present', () => {
const text = '帮我在这个页面增加调查问卷,密码 888 查看提交记录';
const coerced = coercePageDataSkill(
@@ -579,9 +607,9 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only'
},
});
assert.equal(await router.isEnabled(), false);
assert.equal(await router.isEnabled(), true);
states.shift();
assert.equal(await router.isEnabled(), false);
assert.equal(await router.isEnabled(), true);
const result = await router.classify({
userMessage: {
role: 'user',
+23 -2
View File
@@ -43,10 +43,31 @@ const PAGE_DATA_INTENT_PATTERNS = [
/(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u,
];
const INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN = /(?:便签|便利贴|备忘录|待办|清单)/u;
const INTERACTIVE_PAGE_DATA_SURFACE_PATTERN = /(?:应用|app|页面|网页|H5|h5|HTML|html|时间轴|看板)/u;
const INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS = [
/(?:写|填写|提交|保存)/u,
/(?:添加|新增|创建)/u,
/(?:编辑|修改)/u,
/(?:删除|完成)/u,
/(?:筛选|统计|汇总|管理)/u,
/(?:提醒时间|优先级|分类标签|时间轴)/u,
];
function isInteractivePageDataIntent(text) {
if (!INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN.test(text)) return false;
const featureCount = INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS.reduce(
(count, pattern) => count + (pattern.test(text) ? 1 : 0),
0,
);
return INTERACTIVE_PAGE_DATA_SURFACE_PATTERN.test(text) || featureCount >= 2;
}
export function isPageDataIntent(text) {
const normalized = String(text ?? '').trim();
if (!normalized) return false;
return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized));
return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized))
|| isInteractivePageDataIntent(normalized);
}
export function isPageGenerationIntent(text) {
@@ -161,7 +182,7 @@ export function buildChatSkillPrompt(promptKey, skillName) {
`请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能:在 MindSpace 页面中实现可提交、可持久化的数据收集(问卷/报名/台账等),必须使用 Page Data API。` +
'先匹配技能内能力分支(默认 A:匿名前台 public insert + 独立后台 password read,口令默认 88888888);“帮我做/创建”本身就是创建授权,展示简短方案摘要后必须同一轮继续执行,只有互斥需求或非法口令才追问。' +
'流程:load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html(含 /assets/page-data-client.js)→ private_data_bind_workspace_page 发布并写策略。' +
'禁止 localStorage / 浏览器本地存储 fallback;禁止自建 Express/独立端口(如 8899)、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl(/MindSpace/<用户ID>/public/...),禁止给用户 /u/用户名/pages/... 链接,并说明后台入口与口令。'
'当前 session 的 private_data_* 已绑定当前用户专属 PostgreSQL schema;所有需保存或再次读取的数据必须走 PG + Page Data API。禁止 localStorage、sessionStorage、IndexedDB、SQLite、静态 JSON/JS 或内存持久化 fallback;禁止自建 Express/独立端口(如 8899)、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl(/MindSpace/<用户ID>/public/...),禁止给用户 /u/用户名/pages/... 链接,并说明后台入口与口令。'
);
case 'service-integration-smoke':
return `请使用 ${skillName ?? 'service-integration-smoke'} 技能:按标准联调流程检查当前服务,覆盖身份、普通聊天、记忆读取,以及我本轮明确要求验证的技能/发布链路,并输出通过项、失败项、待确认项:`;
+27
View File
@@ -107,9 +107,36 @@ test('buildAutoChatSkillPrefix prefers page-data-collect for survey requests', (
test('isPageDataIntent matches survey and backend keywords', () => {
assert.equal(isPageDataIntent('增加调查问卷和后台入口,密码 888'), true);
assert.equal(
isPageDataIntent('帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。'),
true,
);
assert.equal(isPageDataIntent('帮我做一个苏州攻略页面'), false);
});
test('isPageDataIntent matches implicit interactive sticky-note app requests', () => {
const text = '帮我设计一个便签提醒,可以写便签提交,时间轴来显示';
assert.equal(isPageDataIntent(text), true);
assert.match(
buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']),
/page-data-collect/,
);
});
test('isPageDataIntent does not steal a simple schedule reminder', () => {
assert.equal(isPageDataIntent('明天上午9点提醒我开会'), false);
assert.equal(isPageDataIntent('创建一个待办'), false);
});
test('buildAutoChatSkillPrefix routes ordinary household ledger language to Page Data', () => {
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
const prefix = buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']);
assert.match(prefix, /page-data-collect/);
assert.match(prefix, /同一轮继续执行/);
assert.match(prefix, /禁止 localStorage/);
assert.doesNotMatch(prefix, /static-page-publish 技能/);
});
test('buildAutoChatSkillPrefix enables publish for implicit page requests', () => {
const prefix = buildAutoChatSkillPrefix('苏州攻略页面', ['static-page-publish']);
assert.match(prefix, /static-page-publish/);
+30
View File
@@ -63,6 +63,36 @@ pnpm dev:all
| `PLAZA_APP_DIR` | ../memind_plaza | Plaza 专用脚本的源码路径(`pnpm dev:plaza` / `pnpm start:plaza` / `pnpm dev:all`) |
| `MEMIND_SESSION_BROKER_ENABLED` | `0`(未设置) | Session Broker 灰度开关;Patch 2+ 本地验证时可设 `1`,见 [H5 Session 架构](./h5-session-architecture-20260706.md) |
### MindSpace:本地 vs 生产(必须区分)
`pnpm dev` / `server.mjs` 启动时会读取 `MEMIND_RUNTIME_PROFILE`(见 `scripts/memind-runtime-profile.mjs`):
| `MEMIND_RUNTIME_PROFILE` | 用途 | MindSpace 实现 |
|--------------------------|------|----------------|
| `local`(**默认**) | 本机日常开发 | Portal 内置 `MINDSPACE_SERVER_ADAPTER=local` |
| `split-service` | 本机验证 103 拆分架构 | `remote` → `http://127.0.0.1:8082` 独立 MindSpace 服务 |
| `production` | **仅 103 服务器** | 不覆盖 `.env`,以服务器配置为准 |
**本地 `.env` 推荐:**
```bash
MEMIND_RUNTIME_PROFILE=local
```
**本机要测 split-service 时:**
```bash
MEMIND_RUNTIME_PROFILE=split-service
MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret # 必须与 MindSpace LaunchAgent 一致
launchctl kickstart -k "gui/$(id -u)/cn.tkmind.mindspace-service"
pnpm dev
```
启动后 Portal 日志应出现:`[Portal] Runtime profile: MEMIND_RUNTIME_PROFILE=local, MINDSPACE_SERVER_ADAPTER=local, ...`
**禁止:** 把 103 生产 `.env`、RDS 连接串、`MINDSPACE_REMOTE_AUTH_TOKEN` 生产值复制进本机 Git 仓库。
### H5 Session 架构本地全量联调(`0706-bug干净` 分支)
在 **本机 `.env`**(勿提交 Git)打开下列开关后重启 `pnpm dev`:
@@ -0,0 +1,40 @@
# MindSpace 浏览器存储迁移清单(仅本机)
更新时间:2026-07-13
## 当前策略
- 新建或修改的 Agent 页面禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 持久化数据。
- 新 session 明确绑定当前用户的独立 PostgreSQL schema;持久化数据必须经 `private_data_*` 和 Page Data API。
- 历史页面迁移期间继续可访问,不做全局 409 阻断。
- 页面迁移并验证完成后,才考虑启用历史页面全局阻断。
## 待迁移页面
| 用户 ID | 页面 | 初步类型 | 状态 |
|---|---|---|---|
| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment.html` | 预约业务数据 | 待迁移 |
| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment-admin.html` | 预约管理数据 | 待迁移 |
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger.html` | 家庭记账数据 | 待迁移 |
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger-manage.html` | 家庭记账管理 | 待迁移 |
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/index.html` | 工作区首页状态 | 待分类 |
| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/oa-system.html` | OA 业务数据 | 待迁移 |
| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/decision-comparison.html` | 决策工具状态 | 待分类 |
| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/daily-admin.html` | 日报管理数据 | 待迁移 |
| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/sales-register.html` | 销售登记数据 | 待迁移 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey.html` | 调查数据 | 待迁移 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey-admin.html` | 调查管理数据 | 待迁移 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/survey-youth.html` | 调查数据 | 待迁移 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/daily-report-system.html` | 日报业务数据 | 待迁移 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/weather-calendar.html` | 工具偏好或业务数据 | 待分类 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/runner-game.html` | 游戏存档 | 待决定 PG 或取消持久化 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/tetris-game.html` | 游戏存档 | 待决定 PG 或取消持久化 |
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/bird-key-fly.html` | 游戏存档 | 待决定 PG 或取消持久化 |
## 单页迁移完成条件
1. 建表位于页面所有者的独立 PG schema。
2. dataset 注册完成,页面 policy 权限符合前台/管理页用途。
3. HTML 只通过 Page Data API 访问数据,不包含浏览器持久化 fallback。
4. 实际提交、刷新后读取、PG 直接查询三者结果一致。
5. 原页面链接继续可用,管理页鉴权和跨用户隔离验证通过。
+60
View File
@@ -0,0 +1,60 @@
import fs from 'node:fs';
import path from 'node:path';
const AGENT_PAGE_CODE_EXTENSIONS = new Set(['.html', '.htm', '.js', '.mjs', '.jsx', '.ts', '.tsx']);
const PROHIBITED_BROWSER_STORAGE_PATTERNS = [
{ api: 'localStorage', pattern: /\blocalStorage\b/i },
{ api: 'sessionStorage', pattern: /\bsessionStorage\b/i },
{ api: 'IndexedDB', pattern: /\b(?:indexedDB|IDBDatabase|IDBObjectStore|IDBTransaction)\b/i },
];
export function isAgentPageCodePath(relativePath) {
return AGENT_PAGE_CODE_EXTENSIONS.has(path.extname(String(relativePath ?? '')).toLowerCase());
}
export function detectProhibitedBrowserStorage(content, { relativePath = '' } = {}) {
if (!isAgentPageCodePath(relativePath)) return [];
let source = String(content ?? '');
// Fold adjacent string literals so computed-property obfuscation such as
// window['local' + 'Storage'] cannot bypass the delivery policy.
for (let pass = 0; pass < 8; pass += 1) {
const folded = source.replace(
/(['"])([^'"\\]*)\1\s*\+\s*(['"])([^'"\\]*)\3/g,
(_match, quote, left, _rightQuote, right) => `${quote}${left}${right}${quote}`,
);
if (folded === source) break;
source = folded;
}
return PROHIBITED_BROWSER_STORAGE_PATTERNS
.filter(({ pattern }) => pattern.test(source))
.map(({ api }) => api);
}
export function assertNoProhibitedBrowserStorage(content, { relativePath = '' } = {}) {
const APIs = detectProhibitedBrowserStorage(content, { relativePath });
if (APIs.length === 0) return;
const error = new Error(
`${relativePath || '页面代码'} 禁止使用 ${APIs.join(', ')} 保存数据。`
+ '用户业务数据必须通过 private_data_* 工具写入该用户专属 PostgreSQL schema,'
+ 'HTML 必须通过 Page Data API 访问;请改写后再保存。',
);
error.code = 'BROWSER_STORAGE_FORBIDDEN';
error.apis = APIs;
error.relativePath = relativePath || null;
throw error;
}
export function scanWorkspaceFilesForProhibitedBrowserStorage({ publishDir, relativePaths = [] } = {}) {
const root = path.resolve(String(publishDir ?? ''));
if (!root) return [];
const violations = [];
for (const relativePath of new Set(relativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))) {
if (!isAgentPageCodePath(relativePath)) continue;
const absolutePath = path.resolve(root, relativePath);
if (absolutePath !== root && !absolutePath.startsWith(`${root}${path.sep}`)) continue;
if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) continue;
const apis = detectProhibitedBrowserStorage(fs.readFileSync(absolutePath, 'utf8'), { relativePath });
if (apis.length > 0) violations.push({ relativePath, apis });
}
return violations;
}
+41 -1
View File
@@ -10,6 +10,7 @@ import {
isStubPublicHtmlContent,
materializeMissingPublicHtmlWrites,
} from './mindspace-public-finish-sync.mjs';
import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
const PUBLIC_HTML_PATH_PATTERN_GLOBAL = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/gi;
const PUBLIC_ASSET_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi;
@@ -106,6 +107,26 @@ export function collectMissingPublicHtmlPaths(messages, currentUser, publishDir)
);
}
export function collectTouchedPublicCodePaths(messages, publishDir, syncResult = null) {
const paths = new Set(
(syncResult?.publicHtmlRelativePaths ?? [])
.map((item) => normalizePublicRelativePath(item, { publishDir }))
.filter(Boolean),
);
for (const message of Array.isArray(messages) ? messages : []) {
for (const item of message?.content ?? []) {
if (item?.type !== 'toolRequest') continue;
const toolCall = item.toolCall?.value;
const name = String(toolCall?.name ?? '').trim().split('__').at(-1);
if (!['write_file', 'edit_file', 'write', 'edit'].includes(name)) continue;
const args = toolCall?.arguments ?? {};
const relativePath = normalizePublicRelativePath(args.path ?? args.file_path, { publishDir });
if (relativePath) paths.add(relativePath);
}
}
return [...paths];
}
export function collectMissingPublicAssetReferences(publishDir) {
const root = path.resolve(String(publishDir ?? ''));
const publicDir = path.join(root, 'public');
@@ -135,6 +156,10 @@ export function evaluateH5HtmlFinishGuard({
materializeMissingPublicHtmlWrites({ messages, publishDir });
const missingHtml = collectMissingPublicHtmlPaths(messages, currentUser, publishDir);
const missingAssets = collectMissingPublicAssetReferences(publishDir);
const browserStorageViolations = scanWorkspaceFilesForProhibitedBrowserStorage({
publishDir,
relativePaths: collectTouchedPublicCodePaths(messages, publishDir, syncResult),
});
const lastAssistant = [...(Array.isArray(messages) ? messages : [])]
.reverse()
@@ -159,18 +184,24 @@ export function evaluateH5HtmlFinishGuard({
const needsRepair =
missingHtml.length > 0 ||
missingAssets.length > 0 ||
browserStorageViolations.length > 0 ||
htmlGenerationNeedsRetry;
return {
needsRepair,
missingHtml,
missingAssets,
browserStorageViolations,
htmlGenerationNeedsRetry,
linkExists,
};
}
export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } = {}) {
export function buildH5HtmlRepairPrompt({
missingHtml = [],
missingAssets = [],
browserStorageViolations = [],
} = {}) {
const lines = [
'【系统补盘请求】检测到 public 页面交付不完整。请立即使用 write_file 写入缺失文件(完整内容),并在完成后确认磁盘存在。',
'要求:先 load_skill static-page-publish,再逐个 write_file;不要用空回复或仅 edit_file 改 HTML 代替资源落盘。',
@@ -185,6 +216,15 @@ export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] }
lines.push(`- ${item.relativePath}(来自 ${item.htmlRelativePath})`);
}
}
if (browserStorageViolations.length) {
lines.push(
'',
'以下页面违反数据存储硬约束:',
...browserStorageViolations.map((item) => `- ${item.relativePath}:禁止 ${item.apis.join(', ')}`),
'',
'必须删除浏览器存储代码,load_skill → page-data-collect,使用 private_data_* 在当前用户专属 PostgreSQL schema 建表和注册 dataset,HTML 只能通过 Page Data API 读写。禁止 SQLite 或任何本地 fallback。',
);
}
lines.push('', '写完后请 list_dir public/assets 或对应目录,确认文件真实存在。');
return lines.join('\n');
}
+55
View File
@@ -72,6 +72,61 @@ test('evaluateH5HtmlFinishGuard flags missing assets after html exists', () => {
}
});
test('evaluateH5HtmlFinishGuard blocks browser storage in any touched public page', () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-ban-'));
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(
path.join(publishDir, 'public', 'ledger.html'),
'<html><script>sessionStorage.setItem("ledger", "[]"); indexedDB.open("ledger")</script></html>',
'utf8',
);
const evaluation = evaluateH5HtmlFinishGuard({
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
name: 'write_file',
arguments: { path: 'public/ledger.html' },
} } }] }],
currentUser: USER,
publishDir,
});
assert.equal(evaluation.needsRepair, true);
assert.deepEqual(evaluation.browserStorageViolations, [{
relativePath: 'public/ledger.html',
apis: ['sessionStorage', 'IndexedDB'],
}]);
assert.match(buildH5HtmlRepairPrompt(evaluation), /当前用户专属 PostgreSQL schema/);
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
});
test('evaluateH5HtmlFinishGuard blocks computed localStorage obfuscation', () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-obfuscation-'));
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(
path.join(publishDir, 'public', 'bypass.html'),
`<script>window['local' + 'Sto' + 'rage'].setItem('data', '[]')</script>`,
'utf8',
);
const evaluation = evaluateH5HtmlFinishGuard({
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
name: 'write_file',
arguments: { path: 'public/bypass.html' },
} } }] }],
currentUser: USER,
publishDir,
});
assert.equal(evaluation.needsRepair, true);
assert.deepEqual(evaluation.browserStorageViolations, [{
relativePath: 'public/bypass.html',
apis: ['localStorage'],
}]);
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
});
test('buildH5HtmlRepairPrompt lists html and asset targets', () => {
const prompt = buildH5HtmlRepairPrompt({
missingHtml: ['public/page.html'],
+37 -8
View File
@@ -15,7 +15,9 @@ import {
normalizePageDataApiBase,
verifyPageDataDeliveryArtifacts,
} from './page-data-delivery-assess.mjs';
import { listPageAccessPolicies } from './page-data-policy-store.mjs';
import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs';
import { detectProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
export { inferPageDataBindAccessMode };
@@ -23,7 +25,7 @@ const PUBLICATION_ROUTE_LINK_PATTERN =
/https?:\/\/[^\s<>"')\]]+\/u\/([0-9a-f-]{36}|[a-z0-9._-]+)\/pages\/([^\s<>"')\]]+)/gi;
const PAGE_DATA_CLIENT_SCRIPT_PATTERN = /\/assets\/page-data-client\.js/i;
const LOCAL_STORAGE_DATA_PATTERN = /localStorage\.(?:getItem|setItem)\s*\(/i;
const LEGACY_PAGE_DATA_OWNER_API_PATTERN = /['"`]\/api\/page-data(?:\/|['"`])/i;
const LOCAL_STORAGE_FALLBACK_HINT_PATTERN = /fallback\s*到\s*localStorage|localStorage\s*fallback/i;
const repairAttemptsBySession = new Map();
@@ -109,10 +111,14 @@ export function buildPageDataDeliveryArtifactsFromBindResult(autoBind, publishDi
export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) {
const content = String(html ?? '');
const usage = detectPageDataDatasetUsageFromHtml(content);
const prohibitedBrowserStorage = detectProhibitedBrowserStorage(content, { relativePath });
const usesPageDataApi =
usage.size > 0 ||
PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content) ||
/\bMindSpacePageData\b/.test(content);
/\bMindSpacePageData\b/.test(content) ||
LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content) ||
prohibitedBrowserStorage.length > 0 ||
LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content);
if (!usesPageDataApi) {
return { usesPageDataApi: false, issues: [] };
@@ -122,9 +128,15 @@ export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) {
if (!PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content)) {
issues.push('missing_page_data_client_script');
}
if (LOCAL_STORAGE_DATA_PATTERN.test(content)) {
if (LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content)) {
issues.push('forbidden_legacy_owner_api');
}
if (prohibitedBrowserStorage.includes('localStorage')) {
issues.push('forbidden_local_storage');
}
if (prohibitedBrowserStorage.some((api) => api !== 'localStorage')) {
issues.push('forbidden_browser_storage');
}
if (LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content)) {
issues.push('forbidden_local_storage_fallback');
}
@@ -156,7 +168,22 @@ export function collectPageDataPublicHtmlFiles(publishDir) {
}
function isPageDataHtmlWorkspaceReady({ publishDir, relativePath, html }) {
return assessWorkspacePageDataReadiness({ publishDir, relativePath, html }).ready;
const usage = detectPageDataDatasetUsageFromHtml(html);
if (!usage.size) return true;
return Boolean(findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }));
}
function findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }) {
const usage = detectPageDataDatasetUsageFromHtml(html);
const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html);
for (const policy of listPageAccessPolicies(publishDir)) {
if (String(policy?.accessMode ?? '') !== expectedAccessMode) continue;
if ([...usage.entries()].every(([name, perms]) => {
const granted = policy?.datasets?.[name];
return Boolean(granted && (!perms.read || granted.read) && (!perms.insert || granted.insert));
})) return policy;
}
return null;
}
async function collectUnboundPageDataFiles({
@@ -186,11 +213,13 @@ async function collectUnboundPageDataFiles({
}
continue;
}
if (!isPageDataHtmlWorkspaceReady({
const workspace = await assessWorkspacePageDataReadiness({
userId,
publishDir,
relativePath: file.relativePath,
html: file.content,
})) {
});
if (!workspace.ready) {
unboundFiles.push(file);
}
}
@@ -320,7 +349,7 @@ export function evaluatePageDataFinishGuard({
);
const needsRepair =
pageDataIntent &&
(pageDataIntent || structuralFiles.length > 0) &&
(htmlIssues.length > 0 ||
unboundFiles.length > 0 ||
(relevantFiles.length === 0 &&
@@ -382,7 +411,7 @@ export function buildPageDataCollectFailureText() {
'这次问卷/数据收集页面没有完成 Page Data API 绑定,所以我先不发链接。',
'请直接重发一次完整需求(例如:调查问卷 + 后台查看),我会按 page-data-collect 技能:',
'建表 → 注册 dataset → 写含 page-data-client.js 的 HTML → private_data_bind_workspace_page 发布。',
'数据必须走平台 API 写入 SQLite,禁止 localStorage 或自建后端。',
'数据必须走平台 API 写入用户私有数据空间,禁止 localStorage 或自建后端。',
].join('');
}
+50 -2
View File
@@ -8,6 +8,7 @@ import {
buildPageDataCollectRepairPrompt,
collectPageDataDeliveryArtifacts,
evaluatePageDataFinishGuard,
evaluatePageDataFinishGuardAsync,
evaluatePageDataHtmlContent,
extractRecentPageDataBindTargets,
extractRecentPageDataHtmlWrites,
@@ -42,6 +43,23 @@ test('evaluatePageDataHtmlContent flags missing client script and localStorage f
const bad = evaluatePageDataHtmlContent(BAD_SURVEY_HTML, { relativePath: 'public/children-diet-survey.html' });
assert.ok(bad.issues.includes('missing_page_data_client_script'));
assert.ok(bad.issues.includes('forbidden_local_storage'));
const indexed = evaluatePageDataHtmlContent(
'<script>sessionStorage.setItem("draft", "x"); indexedDB.open("drafts")</script>',
{ relativePath: 'public/draft.html' },
);
assert.ok(indexed.issues.includes('forbidden_browser_storage'));
const legacyOwnerApi = evaluatePageDataHtmlContent(
`<script>
const PG_API = '/api/page-data';
fetch(PG_API + '/query', { method: 'POST' });
</script>`,
{ relativePath: 'public/sticky-notes.html' },
);
assert.equal(legacyOwnerApi.usesPageDataApi, true);
assert.ok(legacyOwnerApi.issues.includes('missing_page_data_client_script'));
assert.ok(legacyOwnerApi.issues.includes('forbidden_legacy_owner_api'));
});
test('inferPageDataBindAccessMode chooses password for admin html', () => {
@@ -70,6 +88,35 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => {
}
});
test('finish guard blocks localStorage ledger generated from ordinary user language', () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-ledger-'));
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(path.join(publishDir, 'public', 'family-ledger.html'), `<!doctype html><script>
localStorage.setItem('tk_family_ledger', JSON.stringify([]));
</script>`, 'utf8');
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: text,
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
name: 'write_file',
arguments: { path: 'public/family-ledger.html' },
} } }] }],
});
assert.equal(evaluation.pageDataIntent, true);
assert.equal(evaluation.needsRepair, true);
assert.equal(
evaluation.htmlIssues.some((item) => (
item.issue === 'forbidden_local_storage' && item.relativePath === 'public/family-ledger.html'
)),
true,
);
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
});
test('finish guard ignores unrelated historical broken Page Data html', async () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-history-'));
try {
@@ -162,7 +209,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => {
}
});
test('evaluatePageDataFinishGuard flags html when dataset is not registered', () => {
test('evaluatePageDataFinishGuard flags html when dataset is not registered', async () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-bound-'));
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
@@ -179,8 +226,9 @@ test('evaluatePageDataFinishGuard flags html when dataset is not registered', ()
},
},
});
const evaluation = evaluatePageDataFinishGuard({
const evaluation = await evaluatePageDataFinishGuardAsync({
publishDir,
userId: 'user-1',
agentText: '调查问卷和后台',
messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }],
});
+10 -5
View File
@@ -11,12 +11,16 @@ export function createPageSyncService({
} = {}) {
const syncInFlight = new Map();
async function syncUserGeneratedPages(userId) {
async function syncUserGeneratedPages(userId, { onlyRelativePaths = null } = {}) {
if (!pool || !pageService || !userId) {
return { created: 0, skipped: 0, updated: 0 };
}
let inFlight = syncInFlight.get(userId);
const scopeKey = onlyRelativePaths == null
? '*'
: [...new Set(onlyRelativePaths)].sort().join('|');
const inFlightKey = `${userId}:${scopeKey}`;
let inFlight = syncInFlight.get(inFlightKey);
if (!inFlight) {
inFlight = syncGeneratedPagesFromPublicAssets({
pool,
@@ -25,17 +29,18 @@ export function createPageSyncService({
userId,
publishDir: h5Root ? resolvePublishDir(h5Root, { id: userId }) : null,
syncWorkspaceAssets,
onlyRelativePaths,
})
.catch((error) => {
logger.warn?.('[MindSpace] page sync failed:', error?.message ?? error);
return { created: 0, skipped: 0, updated: 0 };
})
.finally(() => {
if (syncInFlight.get(userId) === inFlight) {
syncInFlight.delete(userId);
if (syncInFlight.get(inFlightKey) === inFlight) {
syncInFlight.delete(inFlightKey);
}
});
syncInFlight.set(userId, inFlight);
syncInFlight.set(inFlightKey, inFlight);
}
return inFlight;
}
+10 -1
View File
@@ -213,16 +213,23 @@ export async function syncGeneratedPagesFromPublicAssets({
userId,
publishDir = null,
syncWorkspaceAssets = null,
onlyRelativePaths = null,
} = {}) {
if (!pool || !pageService || !userId) {
return { created: 0, skipped: 0, updated: 0 };
}
if (typeof syncWorkspaceAssets === 'function') {
await syncWorkspaceAssets(userId, { categoryCode: 'public' }).catch(() => {});
await syncWorkspaceAssets(userId, {
categoryCode: 'public',
onlyRelativePaths,
}).catch(() => {});
}
const indexedPages = await loadIndexedWorkspacePages(pool, userId);
const allowList = onlyRelativePaths == null
? null
: new Set(onlyRelativePaths.map((item) => normalizePublicHtmlPath(item)).filter(Boolean));
let created = 0;
let updated = 0;
let skipped = 0;
@@ -230,6 +237,7 @@ export async function syncGeneratedPagesFromPublicAssets({
if (publishDir) {
const htmlFiles = await listPublishPublicHtmlFiles(publishDir);
for (const file of htmlFiles) {
if (allowList && !allowList.has(file.relativePath)) continue;
try {
const content = await fs.readFile(file.absolutePath, 'utf8');
if (!content.trim()) {
@@ -285,6 +293,7 @@ export async function syncGeneratedPagesFromPublicAssets({
? asset.original_filename
: `public/${asset.original_filename}`,
);
if (allowList && !allowList.has(relativePath)) continue;
if (indexedPages.has(relativePath)) {
skipped += 1;
continue;
+34
View File
@@ -47,6 +47,40 @@ test('syncGeneratedPagesFromPublicAssets registers html files from publish publi
assert.equal(created[0].source.snapshot.relative_path, 'public/demo.html');
});
test('syncGeneratedPagesFromPublicAssets scopes agent-run sync to current session html', async () => {
const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-scoped-'));
const publicDir = path.join(publishDir, 'public');
await fs.mkdir(publicDir, { recursive: true });
await fs.writeFile(path.join(publicDir, 'current.html'), '<title>Current</title>', 'utf8');
await fs.writeFile(path.join(publicDir, 'historical.html'), '<title>Historical</title>', 'utf8');
const created = [];
const pool = {
async query(sql) {
if (sql.includes('FROM h5_page_records p')) return [[]];
if (sql.includes('FROM h5_assets a')) return [[]];
return [[]];
},
};
const pageService = {
async createFromChat(_userId, input, source) {
created.push({ input, source });
return { id: `page-${created.length}` };
},
};
await syncGeneratedPagesFromPublicAssets({
pool,
pageService,
userId: 'user-1',
publishDir,
onlyRelativePaths: ['public/current.html'],
});
assert.deepEqual(created.map((item) => item.source.snapshot.relative_path), ['public/current.html']);
await fs.rm(publishDir, { recursive: true, force: true });
});
test('syncGeneratedPagesFromPublicAssets uses file mtime as createdAt', async () => {
const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-mtime-'));
const publicDir = path.join(publishDir, 'public');
+1
View File
@@ -516,6 +516,7 @@ export async function syncPublicHtmlAfterFinish({
categoryCode: 'public',
sourceSessionId: sessionId,
sourceMessageId,
onlyRelativePaths: publicHtmlRelativePaths,
});
synced = true;
}
+1
View File
@@ -196,6 +196,7 @@ test('syncPublicHtmlAfterFinish forwards session source to workspace sync', asyn
categoryCode: 'public',
sourceSessionId: 'session-1',
sourceMessageId: 'msg-public-1',
onlyRelativePaths: ['public/session-page.html'],
},
]]);
} finally {
+29
View File
@@ -121,6 +121,35 @@ test('resolveMindSpacePublicRequest coordinates canonical redirect, serve, and n
});
});
test('resolveMindSpacePublicRequest keeps legacy browser-storage pages available during migration', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-storage-ban-'));
const ownerKey = '123e4567-e89b-12d3-a456-426614174000';
const targetDir = path.join(root, 'MindSpace', ownerKey, 'public');
fs.mkdirSync(targetDir, { recursive: true });
fs.writeFileSync(
path.join(targetDir, 'ledger.html'),
'<script>localStorage.setItem("ledger", "[]")</script>',
);
fs.writeFileSync(
path.join(targetDir, 'ledger.js'),
'sessionStorage.setItem("ledger", "[]")',
);
const result = await resolveMindSpacePublicRequest({
h5Root: root,
requestPath: `/${ownerKey}/public/ledger.html`,
});
assert.equal(result.action, 'serve');
assert.equal(result.filePath, path.join(targetDir, 'ledger.html'));
const scriptResult = await resolveMindSpacePublicRequest({
h5Root: root,
requestPath: `/${ownerKey}/public/ledger.js`,
});
assert.equal(scriptResult.action, 'serve');
assert.equal(scriptResult.filePath, path.join(targetDir, 'ledger.js'));
});
test('resolveMindSpacePublicRequest serves missing thumbnail png when svg sidecar exists', async () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-thumb-'));
const ownerKey = '123e4567-e89b-12d3-a456-426614174000';
+10 -3
View File
@@ -24,6 +24,7 @@ import { closePolicyDataset, normalizePageAccessPolicy } from './page-access-pol
import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs';
import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs';
import { resolveMindSpaceStorageRoot } from './mindspace-runtime-config.mjs';
import { assertNoProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
const SANDBOX_ROOT = process.argv[2]?.trim() || process.env.SANDBOX_ROOT?.trim();
if (!SANDBOX_ROOT) {
@@ -215,7 +216,7 @@ const ALL_TOOLS = [
{
name: 'private_data_info',
description:
'查看当前用户“用户私有数据空间”的状态。每个用户只有一个私有 SQLite 数据库,适合保存问卷、表单、清单、调研数据和分析中间表;不要创建额外 SQLite 文件。',
'查看当前用户“用户私有数据空间”的状态。每个用户只有一个由平台分配和隔离的数据空间,适合保存问卷、表单、清单、调研数据和分析中间表;不要创建额外数据库文件。',
inputSchema: { type: 'object', properties: {} },
},
{
@@ -504,6 +505,7 @@ async function callTool(name, args) {
}
case 'write_file': {
const abs = resolveSandboxed(args.path);
assertNoProhibitedBrowserStorage(args.content ?? '', { relativePath: args.path });
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, args.content ?? '', 'utf8');
return [{ type: 'text', text: `已写入 ${args.path}(${(args.content ?? '').length} 字节)` }];
@@ -516,6 +518,7 @@ async function callTool(name, args) {
throw new Error('edit_file: old_str 在文件中不存在,替换失败');
}
const updated = oldStr ? original.replace(oldStr, args.new_str ?? '') : (args.new_str ?? '');
assertNoProhibitedBrowserStorage(updated, { relativePath: args.path });
fs.writeFileSync(abs, updated, 'utf8');
return [{ type: 'text', text: `已编辑 ${args.path}` }];
}
@@ -578,7 +581,9 @@ async function callTool(name, args) {
{
...info,
rules: [
'每个用户只能使用这个唯一 SQLite 数据库',
info.backend === 'postgres'
? '每个用户只能使用新 PG 中分配给自己的独立 schema'
: '每个用户只能使用这个唯一 SQLite 数据库',
'适合问卷、表单、清单、调研数据和分析中间表',
'不要存账号、计费、权限、审计、公开平台数据或跨用户数据',
'HTML 页面通过 Page Data API 访问 dataset,不要直接暴露 SQL',
@@ -603,7 +608,9 @@ async function callTool(name, args) {
return [
{
type: 'text',
text: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `,已同步空间占用 ${result.deltaBytes} 字节` : ''}`,
text: result.backend === 'postgres'
? `已在用户专属 PG schema 中执行。命令 ${result.command ?? 'OK'},影响 ${result.affectedRows ?? 0} 行`
: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `,已同步空间占用 ${result.deltaBytes} 字节` : ''}`,
},
];
}
+35
View File
@@ -85,6 +85,41 @@ function startSandbox(root, envOverrides = {}) {
return { child, request };
}
test('sandbox MCP rejects browser storage in generated page code', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-storage-ban-'));
const server = startSandbox(root, { ALLOWED_TOOLS: 'write_file,edit_file' });
t.after(() => server.child.kill());
await server.request('initialize');
const rejectedWrite = await server.request('tools/call', {
name: 'write_file',
arguments: {
path: 'public/ledger.html',
content: '<script>localStorage.setItem("ledger", "[]")</script>',
},
});
assert.equal(rejectedWrite.result.isError, true);
assert.match(rejectedWrite.result.content[0].text, /BROWSER_STORAGE_FORBIDDEN|禁止使用 localStorage/);
assert.equal(fs.existsSync(path.join(root, 'public', 'ledger.html')), false);
const acceptedWrite = await server.request('tools/call', {
name: 'write_file',
arguments: { path: 'public/ledger.html', content: '<script>client.insertRow("ledger", row)</script>' },
});
assert.equal(acceptedWrite.result.isError, false);
const rejectedEdit = await server.request('tools/call', {
name: 'edit_file',
arguments: {
path: 'public/ledger.html',
old_str: 'client.insertRow("ledger", row)',
new_str: 'indexedDB.open("ledger")',
},
});
assert.equal(rejectedEdit.result.isError, true);
assert.doesNotMatch(fs.readFileSync(path.join(root, 'public', 'ledger.html'), 'utf8'), /indexedDB/);
});
test('sandbox MCP exposes and protects the user private data space', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-'));
const server = startSandbox(root);
+439
View File
@@ -0,0 +1,439 @@
import { execFileSync } from 'node:child_process';
import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
export const CONTROL_SCHEMA = 'mindspace_control';
export function assertUserId(value) {
const userId = String(value ?? '').trim().toLowerCase();
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(userId)) {
throw new Error('user id 必须是标准 UUID');
}
return userId;
}
export function quotePgIdentifier(value) {
return `"${String(value).replaceAll('"', '""')}"`;
}
export function deriveUserSpaceNames(value) {
const userId = assertUserId(value);
const compact = userId.replaceAll('-', '');
const short = compact.slice(0, 12);
return {
userId,
schemaName: `u_${compact}`,
ownerRole: `ms_u_${short}_owner`,
agentRole: `ms_u_${short}_agent`,
};
}
export function buildControlSchemaSql() {
return `
CREATE SCHEMA IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)};
REVOKE ALL ON SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC;
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces (
user_id uuid PRIMARY KEY,
schema_name name NOT NULL UNIQUE,
owner_role name NOT NULL UNIQUE,
agent_role name NOT NULL UNIQUE,
backend_type text NOT NULL DEFAULT 'postgres' CHECK (backend_type IN ('sqlite', 'postgres')),
source_sqlite_path text,
migration_state text NOT NULL DEFAULT 'pending' CHECK (
migration_state IN ('pending', 'snapshot_created', 'schema_converted', 'data_imported', 'verified', 'shadow', 'cutover', 'rolled_back', 'failed')
),
schema_version integer NOT NULL DEFAULT 1,
quota_bytes bigint NOT NULL DEFAULT 104857600 CHECK (quota_bytes > 0),
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
cutover_at timestamptz,
rollback_until timestamptz
);
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs (
id uuid PRIMARY KEY,
user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE,
source_path text NOT NULL,
source_size_bytes bigint NOT NULL DEFAULT 0,
source_sha256 text,
status text NOT NULL,
table_count integer NOT NULL DEFAULT 0,
source_row_count bigint NOT NULL DEFAULT 0,
target_row_count bigint NOT NULL DEFAULT 0,
detail_json jsonb NOT NULL DEFAULT '{}'::jsonb,
started_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
finished_at timestamptz
);
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog (
user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE,
dataset_name text NOT NULL,
physical_table name NOT NULL,
config_json jsonb NOT NULL,
sensitivity text NOT NULL DEFAULT 'normal' CHECK (sensitivity IN ('normal', 'personal', 'credential', 'health')),
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (user_id, dataset_name)
);
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.audit_events (
id bigserial PRIMARY KEY,
user_id uuid,
actor_type text NOT NULL,
actor_id text,
action text NOT NULL,
object_type text,
object_name text,
statement_hash text,
affected_rows bigint,
duration_ms integer,
result text NOT NULL,
detail_json jsonb NOT NULL DEFAULT '{}'::jsonb,
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
);
REVOKE ALL ON ALL TABLES IN SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC;
`.trim();
}
export function buildProvisionUserSql(userId, { sourceSqlitePath = null, quotaBytes = 104857600 } = {}) {
const names = deriveUserSpaceNames(userId);
const schema = quotePgIdentifier(names.schemaName);
const owner = quotePgIdentifier(names.ownerRole);
const agent = quotePgIdentifier(names.agentRole);
return {
...names,
statements: [
`DO $$ BEGIN CREATE ROLE ${owner} NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN CREATE ROLE ${agent} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
`DO $$ BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_auth_members am
JOIN pg_roles r ON r.oid = am.roleid
JOIN pg_roles m ON m.oid = am.member
WHERE r.rolname = '${names.ownerRole}' AND m.rolname = CURRENT_USER AND am.set_option
) THEN
EXECUTE format('GRANT %I TO %I', '${names.ownerRole}', CURRENT_USER);
END IF;
END $$`,
`CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION ${owner}`,
`REVOKE ALL ON SCHEMA ${schema} FROM PUBLIC`,
`GRANT USAGE, CREATE ON SCHEMA ${schema} TO ${agent}`,
`ALTER ROLE ${agent} SET search_path = ${schema}, pg_catalog`,
`ALTER ROLE ${agent} SET statement_timeout = '30s'`,
`ALTER ROLE ${agent} SET lock_timeout = '5s'`,
`ALTER ROLE ${agent} SET idle_in_transaction_session_timeout = '60s'`,
`ALTER ROLE ${agent} SET work_mem = '4MB'`,
`ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${agent}`,
`ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO ${agent}`,
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces
(user_id, schema_name, owner_role, agent_role, source_sqlite_path, quota_bytes, updated_at)
VALUES ($1::uuid, $2::name, $3::name, $4::name, $5, $6, CURRENT_TIMESTAMP)
ON CONFLICT (user_id) DO UPDATE SET
source_sqlite_path = EXCLUDED.source_sqlite_path,
quota_bytes = EXCLUDED.quota_bytes,
updated_at = CURRENT_TIMESTAMP`,
],
params: [names.userId, names.schemaName, names.ownerRole, names.agentRole, sourceSqlitePath, quotaBytes],
};
}
export function mapSqliteTypeToPostgres(column) {
const declared = String(column.type ?? '').trim().toUpperCase();
if (Number(column.pk) === 1 && declared.includes('INT')) return 'bigint GENERATED BY DEFAULT AS IDENTITY';
if (declared.includes('TEXT') && isSqliteTimestampDefault(column.dflt_value)) return 'timestamptz';
if (declared.includes('INT')) return 'bigint';
if (declared.includes('REAL') || declared.includes('FLOA') || declared.includes('DOUB')) return 'double precision';
if (declared.includes('BLOB')) return 'bytea';
if (declared.includes('NUM') || declared.includes('DEC')) return 'numeric';
if (declared.includes('BOOL')) return 'boolean';
return 'text';
}
export function isSqliteTimestampDefault(value) {
return /^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(String(value ?? '').trim());
}
export function translateSqliteDefault(value) {
if (value == null) return null;
const input = String(value).trim();
if (/^NULL$/i.test(input)) return 'NULL';
if (/^[+-]?\d+(?:\.\d+)?$/.test(input)) return input;
if (/^CURRENT_TIMESTAMP$/i.test(input)) return 'CURRENT_TIMESTAMP';
if (/^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(input)) return 'CURRENT_TIMESTAMP';
if (/^".*"$/.test(input)) return `'${input.slice(1, -1).replaceAll("'", "''")}'`;
if (/^'.*'$/.test(input)) return input;
return null;
}
export function buildPostgresTableSql(schemaName, tableName, columns) {
if (!Array.isArray(columns) || columns.length === 0) throw new Error(`表 ${tableName} 没有字段`);
const primaryKeys = columns.filter((column) => Number(column.pk) > 0).sort((a, b) => Number(a.pk) - Number(b.pk));
const definitions = columns.map((column) => {
const parts = [quotePgIdentifier(column.name), mapSqliteTypeToPostgres(column)];
if (Number(column.notnull) === 1) parts.push('NOT NULL');
const translatedDefault = translateSqliteDefault(column.dflt_value);
if (translatedDefault != null) parts.push(`DEFAULT ${translatedDefault}`);
if (primaryKeys.length === 1 && primaryKeys[0].name === column.name) parts.push('PRIMARY KEY');
return parts.join(' ');
});
if (primaryKeys.length > 1) {
definitions.push(`PRIMARY KEY (${primaryKeys.map((column) => quotePgIdentifier(column.name)).join(', ')})`);
}
return `CREATE TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (\n ${definitions.join(',\n ')}\n)`;
}
function stableObjectName(prefix, ...parts) {
const readable = `${prefix}_${parts.join('_')}`.replace(/[^a-zA-Z0-9_]+/g, '_').toLowerCase();
const hash = createHash('sha256').update(readable).digest('hex').slice(0, 8);
return `${readable.slice(0, 50)}_${hash}`;
}
export function extractSqliteChecks(createSql) {
const sql = String(createSql ?? '');
const checks = [];
const pattern = /\bCHECK\s*\(/ig;
let match;
while ((match = pattern.exec(sql))) {
const start = pattern.lastIndex;
let depth = 1;
let quote = '';
let index = start;
for (; index < sql.length && depth > 0; index += 1) {
const char = sql[index];
if (quote) {
if (char === quote && sql[index + 1] === quote) index += 1;
else if (char === quote) quote = '';
continue;
}
if (char === "'" || char === '"') quote = char;
else if (char === '(') depth += 1;
else if (char === ')') depth -= 1;
}
if (depth === 0) checks.push(sql.slice(start, index - 1).trim());
pattern.lastIndex = index;
}
return checks;
}
export function buildPostgresIndexSql(schemaName, tableName, index) {
if (!index.columns?.length) return null;
if (index.partial) return null;
const prefix = index.unique ? 'ux' : 'ix';
const name = stableObjectName(prefix, tableName, ...index.columns);
return `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX ${quotePgIdentifier(name)} ON ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (${index.columns.map(quotePgIdentifier).join(', ')})`;
}
export function buildPostgresForeignKeySql(schemaName, tableName, foreignKey) {
const name = stableObjectName('fk', tableName, foreignKey.id, foreignKey.table);
const onUpdate = String(foreignKey.onUpdate ?? 'NO ACTION').toUpperCase();
const onDelete = String(foreignKey.onDelete ?? 'NO ACTION').toUpperCase();
const allowedActions = new Set(['NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT']);
if (!allowedActions.has(onUpdate) || !allowedActions.has(onDelete)) throw new Error('SQLite 外键动作不受支持');
return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} FOREIGN KEY (${foreignKey.from.map(quotePgIdentifier).join(', ')}) REFERENCES ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(foreignKey.table)} (${foreignKey.to.map(quotePgIdentifier).join(', ')}) ON UPDATE ${onUpdate} ON DELETE ${onDelete}`;
}
export function buildPostgresCheckSql(schemaName, tableName, expression, index) {
if (/;|--|\/\*/.test(expression)) throw new Error(`CHECK 表达式包含不安全内容:${tableName}`);
const name = stableObjectName('ck', tableName, index);
return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} CHECK (${expression})`;
}
export function convertSqliteValueForPostgres(value, column) {
if (value == null) return null;
if (isSqliteTimestampDefault(column.dflt_value) && typeof value === 'string') {
const trimmed = value.trim();
if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(trimmed)) {
return `${trimmed.replace(' ', 'T')}+08:00`;
}
}
return value;
}
function sqliteJson(sqliteBin, databasePath, sql) {
const output = execFileSync(sqliteBin, ['-readonly', '-json', databasePath, sql], {
encoding: 'utf8',
maxBuffer: 32 * 1024 * 1024,
}).trim();
return output ? JSON.parse(output) : [];
}
function quoteSqliteIdentifier(value) {
return `"${String(value).replaceAll('"', '""')}"`;
}
export function inspectSqliteDatabase(databasePath, { sqliteBin = 'sqlite3' } = {}) {
const resolved = path.resolve(databasePath);
if (!fs.statSync(resolved).isFile()) throw new Error(`SQLite 文件不存在:${resolved}`);
const integrity = sqliteJson(sqliteBin, resolved, 'PRAGMA integrity_check;');
if (integrity[0]?.integrity_check !== 'ok') throw new Error(`SQLite integrity_check 未通过:${resolved}`);
const tables = sqliteJson(
sqliteBin,
resolved,
"SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;",
).map((table) => {
const columns = sqliteJson(sqliteBin, resolved, `PRAGMA table_info(${quoteSqliteIdentifier(table.name)});`);
const rows = sqliteJson(sqliteBin, resolved, `SELECT * FROM ${quoteSqliteIdentifier(table.name)};`);
const indexes = sqliteJson(sqliteBin, resolved, `PRAGMA index_list(${quoteSqliteIdentifier(table.name)});`).map((index) => ({
name: index.name,
unique: Boolean(index.unique),
origin: index.origin,
partial: Boolean(index.partial),
columns: sqliteJson(sqliteBin, resolved, `PRAGMA index_info(${quoteSqliteIdentifier(index.name)});`).map((item) => item.name),
})).filter((index) => index.origin !== 'pk');
const foreignKeyRows = sqliteJson(sqliteBin, resolved, `PRAGMA foreign_key_list(${quoteSqliteIdentifier(table.name)});`);
const foreignKeys = [...new Set(foreignKeyRows.map((item) => item.id))].map((id) => {
const items = foreignKeyRows.filter((item) => item.id === id).sort((a, b) => a.seq - b.seq);
return {
id,
table: items[0].table,
from: items.map((item) => item.from),
to: items.map((item) => item.to),
onUpdate: items[0].on_update,
onDelete: items[0].on_delete,
};
});
return { ...table, columns, rows, indexes, foreignKeys, checks: extractSqliteChecks(table.sql) };
});
return { path: resolved, sizeBytes: fs.statSync(resolved).size, tables };
}
export function createSqliteSnapshot(sourcePath, { sqliteBin = 'sqlite3' } = {}) {
const target = path.join(os.tmpdir(), `mindspace-userdata-${Date.now()}-${process.pid}.sqlite`);
if (/\s/.test(target)) throw new Error('SQLite snapshot 临时路径不能包含空格');
execFileSync(sqliteBin, [path.resolve(sourcePath), `.backup ${target}`], { stdio: 'pipe' });
// A standalone snapshot must not depend on the source WAL/SHM files.
execFileSync(sqliteBin, [target, 'PRAGMA journal_mode=DELETE;'], { stdio: 'pipe' });
return target;
}
export async function provisionUserSpace(client, userId, options = {}) {
const provision = buildProvisionUserSql(userId, options);
await client.query('BEGIN');
try {
for (let index = 0; index < provision.statements.length; index += 1) {
const params = index === provision.statements.length - 1 ? provision.params : [];
await client.query(provision.statements[index], params);
}
await client.query('COMMIT');
return provision;
} catch (error) {
await client.query('ROLLBACK');
throw error;
}
}
export async function migrateSqliteSnapshot(client, snapshot, userId, { replaceShadow = false, sourcePath = snapshot.path } = {}) {
const names = deriveUserSpaceNames(userId);
const owner = quotePgIdentifier(names.ownerRole);
const agent = quotePgIdentifier(names.agentRole);
const migrationId = randomUUID();
let sourceRows = 0;
let targetRows = 0;
await client.query('BEGIN');
try {
const stateResult = await client.query(
`SELECT migration_state FROM ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces WHERE user_id = $1::uuid FOR UPDATE`,
[names.userId],
);
const migrationState = stateResult.rows[0]?.migration_state;
if (migrationState === 'cutover') throw new Error('用户空间已经 cutover,禁止迁移器覆盖');
const existingResult = await client.query(
`SELECT count(*)::integer AS count FROM pg_catalog.pg_tables WHERE schemaname = $1`,
[names.schemaName],
);
if (Number(existingResult.rows[0]?.count ?? 0) > 0 && !replaceShadow) {
throw new Error('目标影子空间已有表;如确认重建,显式使用 --replace-shadow');
}
await client.query(
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs
(id, user_id, source_path, source_size_bytes, status, started_at)
VALUES ($1::uuid, $2::uuid, $3, $4, 'importing', CURRENT_TIMESTAMP)`,
[migrationId, names.userId, sourcePath, snapshot.sizeBytes],
);
for (const table of snapshot.tables) {
await client.query(`DROP TABLE IF EXISTS ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} CASCADE`);
await client.query(buildPostgresTableSql(names.schemaName, table.name, table.columns));
await client.query(`ALTER TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} OWNER TO ${owner}`);
await client.query(
`GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} TO ${agent}`,
);
for (const row of table.rows) {
const columns = Object.keys(row);
const params = columns.map((_, index) => `$${index + 1}`);
await client.query(
`INSERT INTO ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} (${columns.map(quotePgIdentifier).join(', ')}) VALUES (${params.join(', ')})`,
columns.map((column) => convertSqliteValueForPostgres(
row[column],
table.columns.find((item) => item.name === column) ?? {},
)),
);
}
for (const column of table.columns.filter(
(item) => Number(item.pk) === 1 && String(item.type ?? '').toUpperCase().includes('INT'),
)) {
const qualifiedTable = `${names.schemaName}.${table.name}`;
await client.query(
`SELECT setval(
pg_get_serial_sequence($1, $2),
GREATEST(COALESCE(MAX(${quotePgIdentifier(column.name)}), 0), 1),
COALESCE(MAX(${quotePgIdentifier(column.name)}), 0) > 0
)
FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`,
[qualifiedTable, column.name],
);
}
sourceRows += table.rows.length;
const count = await client.query(
`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`,
);
targetRows += Number(count.rows[0].count);
}
for (const table of snapshot.tables) {
for (const index of table.indexes ?? []) {
const sql = buildPostgresIndexSql(names.schemaName, table.name, index);
if (sql) await client.query(sql);
}
for (const foreignKey of table.foreignKeys ?? []) {
await client.query(buildPostgresForeignKeySql(names.schemaName, table.name, foreignKey));
}
for (let index = 0; index < (table.checks ?? []).length; index += 1) {
await client.query(buildPostgresCheckSql(names.schemaName, table.name, table.checks[index], index));
}
}
await client.query(`GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA ${quotePgIdentifier(names.schemaName)} TO ${agent}`);
const registry = snapshot.tables.find((table) => table.name === '__page_data_datasets');
for (const row of registry?.rows ?? []) {
await client.query(
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog
(user_id, dataset_name, physical_table, config_json, updated_at)
VALUES ($1::uuid, $2, $3::name, $4::jsonb, CURRENT_TIMESTAMP)
ON CONFLICT (user_id, dataset_name) DO UPDATE SET
physical_table = EXCLUDED.physical_table,
config_json = EXCLUDED.config_json,
updated_at = CURRENT_TIMESTAMP`,
[names.userId, row.name, row.table_name, row.config_json],
);
}
await client.query(
`UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces
SET migration_state = 'shadow', updated_at = CURRENT_TIMESTAMP
WHERE user_id = $1::uuid`,
[names.userId],
);
await client.query(
`UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs
SET status = 'verified', table_count = $2, source_row_count = $3,
target_row_count = $4, finished_at = CURRENT_TIMESTAMP
WHERE id = $1::uuid`,
[migrationId, snapshot.tables.length, sourceRows, targetRows],
);
await client.query('COMMIT');
return { migrationId, ...names, tableCount: snapshot.tables.length, sourceRows, targetRows };
} catch (error) {
await client.query('ROLLBACK');
throw error;
}
}
+106
View File
@@ -0,0 +1,106 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
buildControlSchemaSql,
buildPostgresTableSql,
buildPostgresCheckSql,
buildPostgresForeignKeySql,
buildPostgresIndexSql,
buildProvisionUserSql,
createSqliteSnapshot,
convertSqliteValueForPostgres,
deriveUserSpaceNames,
extractSqliteChecks,
mapSqliteTypeToPostgres,
translateSqliteDefault,
} from './mindspace-userdata-postgres.mjs';
import { execFileSync } from 'node:child_process';
const USER_ID = 'ecc1c649-fff7-4361-a243-a69b460cc407';
test('deriveUserSpaceNames creates stable UUID based identifiers', () => {
assert.deepEqual(deriveUserSpaceNames(USER_ID), {
userId: USER_ID,
schemaName: 'u_ecc1c649fff74361a243a69b460cc407',
ownerRole: 'ms_u_ecc1c649fff7_owner',
agentRole: 'ms_u_ecc1c649fff7_agent',
});
});
test('control schema is private and tracks migration lifecycle', () => {
const sql = buildControlSchemaSql();
assert.match(sql, /REVOKE ALL ON SCHEMA "mindspace_control" FROM PUBLIC/);
assert.match(sql, /CREATE TABLE IF NOT EXISTS "mindspace_control"\.user_spaces/);
assert.match(sql, /migration_runs/);
assert.match(sql, /audit_events/);
});
test('provision SQL creates isolated no-login roles and safety limits', () => {
const provision = buildProvisionUserSql(USER_ID, { sourceSqlitePath: '/tmp/user.sqlite' });
const sql = provision.statements.join('\n');
assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_owner" NOLOGIN/);
assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_agent" NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE/);
assert.match(sql, /pg_auth_members/);
assert.match(sql, /REVOKE ALL ON SCHEMA "u_ecc1c649fff74361a243a69b460cc407" FROM PUBLIC/);
assert.match(sql, /statement_timeout = '30s'/);
assert.match(sql, /NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION/);
assert.doesNotMatch(sql, /temp_file_limit/);
});
test('SQLite types and defaults map to conservative PostgreSQL equivalents', () => {
assert.equal(mapSqliteTypeToPostgres({ type: 'INTEGER', pk: 1 }), 'bigint GENERATED BY DEFAULT AS IDENTITY');
assert.equal(mapSqliteTypeToPostgres({ type: 'REAL', pk: 0 }), 'double precision');
assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0 }), 'text');
assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0, dflt_value: "datetime('now', 'localtime')" }), 'timestamptz');
assert.equal(translateSqliteDefault("(datetime('now', '+8 hours'))"), 'CURRENT_TIMESTAMP');
assert.equal(translateSqliteDefault('""'), "''");
assert.equal(translateSqliteDefault('dangerous_function()'), null);
});
test('SQLite checks, indexes and foreign keys become scoped PostgreSQL objects', () => {
assert.deepEqual(extractSqliteChecks("CREATE TABLE x(status TEXT CHECK(status IN ('a','b')), score INT CHECK(score > 0))"), [
"status IN ('a','b')",
'score > 0',
]);
assert.match(buildPostgresIndexSql('u_test', 'users', { unique: true, columns: ['username'] }), /CREATE UNIQUE INDEX/);
assert.match(buildPostgresForeignKeySql('u_test', 'child', {
id: 0, table: 'parent', from: ['parent_id'], to: ['id'], onUpdate: 'NO ACTION', onDelete: 'CASCADE',
}), /REFERENCES "u_test"\."parent" \("id"\).*ON DELETE CASCADE/);
assert.match(buildPostgresCheckSql('u_test', 'users', "status IN ('a','b')", 0), /CHECK \(status IN \('a','b'\)\)/);
});
test('SQLite local timestamps are made explicit for PostgreSQL', () => {
assert.equal(
convertSqliteValueForPostgres('2026-07-13 10:20:30', { dflt_value: "datetime('now', '+8 hours')" }),
'2026-07-13T10:20:30+08:00',
);
assert.equal(convertSqliteValueForPostgres('plain', { dflt_value: null }), 'plain');
});
test('table builder preserves primary key, not-null and safe defaults', () => {
const sql = buildPostgresTableSql('u_example', 'survey', [
{ name: 'id', type: 'INTEGER', pk: 1, notnull: 0, dflt_value: null },
{ name: 'answer', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "''" },
{ name: 'created_at', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "datetime('now', 'localtime')" },
]);
assert.match(sql, /"id" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY/);
assert.match(sql, /"answer" text NOT NULL DEFAULT ''/);
assert.match(sql, /"created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP/);
});
test('SQLite snapshot includes committed rows without changing source', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-pg-test-'));
const source = path.join(root, 'source.sqlite');
execFileSync('sqlite3', [source, 'CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO sample VALUES (1, \'ok\');']);
const snapshot = createSqliteSnapshot(source);
try {
const count = execFileSync('sqlite3', ['-readonly', snapshot, 'SELECT count(*) FROM sample;'], { encoding: 'utf8' }).trim();
assert.equal(count, '1');
} finally {
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(snapshot, { force: true });
}
});
@@ -56,8 +56,8 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page
},
},
pageDataEnsure: {
async ensurePageDataHtmlPagesBound() {
bound.push(true);
async ensurePageDataHtmlPagesBound(options) {
bound.push(options);
return { bound: [{ relativePath: 'public/survey.html' }], skipped: [], errors: [] };
},
},
@@ -65,8 +65,11 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page
storageRoot: null,
});
const result = await service.syncAndDeliver(userId);
const result = await service.syncAndDeliver(userId, {
pageDataRelativePaths: ['public/survey.html'],
});
assert.equal(bound.length, 1);
assert.deepEqual(bound[0].onlyRelativePaths, ['public/survey.html']);
assert.equal(result.publish.published, 0);
assert.equal(result.publish.skipped, 1);
assert.equal(published.length, 0);
+10 -4
View File
@@ -95,7 +95,7 @@ export function createWorkspacePageDeliverService({
return resolvePublishDir(h5Root, { id: userId });
}
async function ensurePageDataBindings(userId) {
async function ensurePageDataBindings(userId, { onlyRelativePaths = null } = {}) {
if (!pageDataEnsure?.ensurePageDataHtmlPagesBound || !pool || !userId) {
return { bound: [], skipped: [], errors: [] };
}
@@ -110,6 +110,7 @@ export function createWorkspacePageDeliverService({
userId,
workspaceRoot,
findPageByRelativePath,
onlyRelativePaths,
logger,
});
}
@@ -189,15 +190,20 @@ export function createWorkspacePageDeliverService({
return { published, skipped, errors };
}
async function syncAndDeliver(userId) {
async function syncAndDeliver(userId, { pageDataRelativePaths = null } = {}) {
let syncResult = { created: 0, updated: 0, skipped: 0 };
if (pageSyncService?.syncUserGeneratedPages) {
syncResult = await pageSyncService.syncUserGeneratedPages(userId);
syncResult = await pageSyncService.syncUserGeneratedPages(userId, {
onlyRelativePaths: pageDataRelativePaths,
});
}
const pageDataBindResult = await ensurePageDataBindings(userId);
const pageDataBindResult = await ensurePageDataBindings(userId, {
onlyRelativePaths: pageDataRelativePaths,
});
const publishResult = await ensureWorkspaceHtmlPublications(userId);
const refreshResult = await refreshOnlineWorkspacePublications(userId);
return {
pageDataRelativePaths: pageDataRelativePaths == null ? null : [...pageDataRelativePaths],
sync: syncResult,
pageDataBind: pageDataBindResult,
publish: publishResult,
+17 -4
View File
@@ -436,10 +436,20 @@ export function createWorkspaceAssetSync({
}
};
const syncCategory = async (userId, categoryCode, source = {}) => {
const syncCategory = async (userId, categoryCode, source = {}, { onlyRelativePaths = null } = {}) => {
if (!h5Root) return { imported: 0, updated: 0, skipped: 0 };
const workspaceRoot = resolveUserWorkspaceRoot(h5Root, { id: userId });
const files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode);
let files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode);
if (onlyRelativePaths != null) {
const allowed = new Set(
onlyRelativePaths
.map((item) => normalizeWorkspaceRelativePath(item))
.filter(Boolean),
);
files = files.filter((file) =>
allowed.has(normalizeWorkspaceRelativePath(`${categoryCode}/${file.filename}`)),
);
}
const [categories] = await pool.query(
`SELECT c.id, c.space_id, c.category_code
FROM h5_space_categories c
@@ -487,14 +497,17 @@ export function createWorkspaceAssetSync({
return { imported, updated, skipped };
};
const syncUserWorkspace = async (userId, { categoryCode, sourceSessionId, sourceMessageId, title } = {}) => {
const syncUserWorkspace = async (
userId,
{ categoryCode, sourceSessionId, sourceMessageId, title, onlyRelativePaths = null } = {},
) => {
const codes = categoryCode ? [categoryCode] : UPLOAD_ZONE_CODES;
let imported = 0;
let updated = 0;
let skipped = 0;
const source = { sessionId: sourceSessionId, messageId: sourceMessageId, title };
for (const code of codes) {
const result = await syncCategory(userId, code, source);
const result = await syncCategory(userId, code, source, { onlyRelativePaths });
imported += result.imported;
updated += result.updated;
skipped += result.skipped;
+5 -1
View File
@@ -316,6 +316,7 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn
const workspace = resolveUserWorkspaceRoot(h5Root, { id: 'user-1', username: 'john' });
await fs.mkdir(path.join(workspace, 'oa', '暑假实习报告'), { recursive: true });
await fs.writeFile(path.join(workspace, 'oa', '暑假实习报告', 'report.csv'), 'a,b\n1,2\n');
await fs.writeFile(path.join(workspace, 'oa', 'historical.csv'), 'old,data\n3,4\n');
const state = {
categories: [
@@ -417,7 +418,10 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn
idFactory: () => `id-${++nextId}`,
});
const result = await sync.syncUserWorkspace('user-1', { categoryCode: 'oa' });
const result = await sync.syncUserWorkspace('user-1', {
categoryCode: 'oa',
onlyRelativePaths: ['oa/暑假实习报告/report.csv'],
});
assert.equal(result.imported, 1);
assert.equal(state.assets[0].original_filename, '暑假实习报告/report.csv');
});
+45 -9
View File
@@ -40,6 +40,33 @@ export function buildSmokeInsertRow(policy, datasetName) {
return row;
}
export function assessPageDataAuthenticationContract({ policy, html } = {}) {
const accessMode = String(policy?.accessMode ?? '').trim();
const content = String(html ?? '');
const usesServerAuthentication = /\.\s*authenticate\s*\(/.test(content);
const passwordValue = String.raw`\b(?:pwd|pass(?:word)?|passwordInput|adminPassword)\w*(?:\.value)?`;
const passwordComparison = String.raw`(?:===?|!==?)`;
const passwordOperand = String.raw`(?:[A-Za-z_$][\w$]*|['"\x60][^'"\x60]*['"\x60])`;
const usesClientSidePasswordCheck = new RegExp(
`(?:${passwordValue}\\s*${passwordComparison}\\s*${passwordOperand}|${passwordOperand}\\s*${passwordComparison}\\s*${passwordValue})`,
'i',
).test(content);
const reasons = [];
if (accessMode === 'password' && !usesServerAuthentication) {
reasons.push('password_page_missing_server_authentication');
}
if (accessMode === 'public' && usesServerAuthentication) {
reasons.push('public_page_cannot_use_password_authentication');
}
if (accessMode === 'public' && usesClientSidePasswordCheck) {
reasons.push('public_page_uses_client_side_password_gate');
}
if (accessMode === 'password' && usesClientSidePasswordCheck) {
reasons.push('password_page_uses_client_side_password_gate');
}
return reasons;
}
export async function queryOnlinePublication(pool, pageId) {
if (!pool || !pageId) return null;
const [rows] = await pool.query(
@@ -76,20 +103,13 @@ function findWorkspacePolicyForHtml({ publishDir, relativePath, html }) {
return null;
}
export function assessWorkspacePageDataReadiness({ publishDir, relativePath, html }) {
function assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }) {
const usage = detectPageDataDatasetUsageFromHtml(html);
if (!usage.size) {
return { ready: true, reasons: [] };
}
const reasons = [];
const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir });
for (const datasetName of usage.keys()) {
if (!dataSpace.getDataset(datasetName)) {
reasons.push(`dataset_not_registered:${datasetName}`);
}
}
const policy = findWorkspacePolicyForHtml({ publishDir, relativePath, html });
if (!policy) {
reasons.push('missing_workspace_policy');
@@ -98,6 +118,21 @@ export function assessWorkspacePageDataReadiness({ publishDir, relativePath, htm
return { ready: reasons.length === 0, reasons, policy };
}
export async function assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }) {
const usage = detectPageDataDatasetUsageFromHtml(html);
const policyAssessment = assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html });
if (!usage.size) return policyAssessment;
const reasons = [...policyAssessment.reasons];
const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir, userId });
for (const datasetName of usage.keys()) {
if (!(await dataSpace.getDataset(datasetName))) {
reasons.push(`dataset_not_registered:${datasetName}`);
}
}
return { ...policyAssessment, ready: reasons.length === 0, reasons };
}
export async function assessPageDataHtmlBinding({
pool,
userId,
@@ -111,7 +146,7 @@ export async function assessPageDataHtmlBinding({
return { bound: true, reasons: [], pageId: null };
}
const workspace = assessWorkspacePageDataReadiness({ publishDir, relativePath, html });
const workspace = await assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html });
const reasons = [...workspace.reasons];
let pageId = null;
@@ -139,6 +174,7 @@ export async function assessPageDataHtmlBinding({
if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) {
reasons.push('policy_access_mode_mismatch');
}
reasons.push(...assessPageDataAuthenticationContract({ policy, html }));
for (const [datasetName, perms] of usage) {
const policyDataset = policy.datasets?.[datasetName];
if (!policyDataset) continue;
+38
View File
@@ -1,11 +1,49 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
assessPageDataAuthenticationContract,
buildSmokeInsertRow,
extractInjectedPageIdFromHtml,
normalizePageDataApiBase,
} from './page-data-delivery-assess.mjs';
test('password pages must authenticate with the server before data reads', () => {
assert.deepEqual(
assessPageDataAuthenticationContract({
policy: { accessMode: 'password' },
html: '<script>if (pwd === ADMIN_PASSWORD) loadData()</script>',
}),
[
'password_page_missing_server_authentication',
'password_page_uses_client_side_password_gate',
],
);
assert.deepEqual(
assessPageDataAuthenticationContract({
policy: { accessMode: 'password' },
html: '<script>client.authenticate(password).then(loadData)</script>',
}),
[],
);
assert.deepEqual(
assessPageDataAuthenticationContract({
policy: { accessMode: 'password' },
html: '<script>if (passwordInput.value !== "88888888") return; client.authenticate(passwordInput.value)</script>',
}),
['password_page_uses_client_side_password_gate'],
);
});
test('public pages cannot use a client-side password gate as authorization', () => {
assert.deepEqual(
assessPageDataAuthenticationContract({
policy: { accessMode: 'public' },
html: '<script>if (pwd === DIARY_PASSWORD) client.insertRow("diary_entries", row)</script>',
}),
['public_page_uses_client_side_password_gate'],
);
});
test('extractInjectedPageIdFromHtml reads injected pageId', () => {
const html = `<html><head>
<meta name="mindspace-page-data-page-id" content="page-abc">
+19 -19
View File
@@ -171,13 +171,13 @@ export function createPageDataPublicService(deps = {}) {
});
}
function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) {
async function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) {
const policyDataset = policy.datasets?.[datasetName];
if (!policyDataset) {
throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' });
}
const ownerService = createOwnerService(ownerUserId);
const registryDataset = ownerService.getDataset(datasetName);
const registryDataset = await ownerService.getDataset(datasetName);
if (!registryDataset) {
throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
}
@@ -190,9 +190,9 @@ export function createPageDataPublicService(deps = {}) {
return { ownerService, effective, policyDataset };
}
function resolveRowScope(ownerService, effective, policyDataset, visitorContext) {
async function resolveRowScope(ownerService, effective, policyDataset, visitorContext) {
if (!policyDataset?.rowPolicy || !visitorContext) return null;
const tableColumns = ownerService.listTableColumns(effective.table);
const tableColumns = await ownerService.listTableColumns(effective.table);
return resolveRowAccessScope(policyDataset.rowPolicy, {
visitorRole: visitorContext.visitorRole,
visitorUserId: visitorContext.visitorUserId,
@@ -303,13 +303,13 @@ export function createPageDataPublicService(deps = {}) {
const policy = await loadPolicy(publication);
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
enforceRateLimit(req, publication, 'read');
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
access.rolePermissions ?? null,
);
const rowScope = resolveRowScope(
const rowScope = await resolveRowScope(
ownerService,
effective,
policyDataset,
@@ -330,7 +330,7 @@ export function createPageDataPublicService(deps = {}) {
const publication = await queryPublication(pageId);
const policy = await loadPolicy(publication);
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
const { ownerService, effective } = resolveEffectiveDataset(
const { ownerService, effective } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
@@ -345,13 +345,13 @@ export function createPageDataPublicService(deps = {}) {
const publication = await queryPublication(pageId);
const policy = await loadPolicy(publication);
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
access.rolePermissions ?? null,
);
const rowScope = resolveRowScope(
const rowScope = await resolveRowScope(
ownerService,
effective,
policyDataset,
@@ -415,13 +415,13 @@ export function createPageDataPublicService(deps = {}) {
}
const payload = stripPageDataMetaFields(payloadInput);
enforceRateLimit(req, publication, 'insert');
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
access.rolePermissions ?? null,
);
const rowScope = resolveRowScope(
const rowScope = await resolveRowScope(
ownerService,
effective,
policyDataset,
@@ -465,13 +465,13 @@ export function createPageDataPublicService(deps = {}) {
datasetName,
});
enforceRateLimit(req, publication, 'update');
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
access.rolePermissions ?? null,
);
const rowScope = resolveRowScope(
const rowScope = await resolveRowScope(
ownerService,
effective,
policyDataset,
@@ -513,13 +513,13 @@ export function createPageDataPublicService(deps = {}) {
datasetName,
});
enforceRateLimit(req, publication, 'soft_delete');
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
publication.user_id,
policy,
datasetName,
access.rolePermissions ?? null,
);
const rowScope = resolveRowScope(
const rowScope = await resolveRowScope(
ownerService,
effective,
policyDataset,
@@ -598,7 +598,7 @@ export function createPageDataPublicService(deps = {}) {
throw Object.assign(new Error('缺少 dataset 名称'), { code: 'invalid_request', status: 400 });
}
const ownerService = createOwnerService(user.id);
const registryDataset = ownerService.getDataset(normalizedDatasetName);
const registryDataset = await ownerService.getDataset(normalizedDatasetName);
if (!registryDataset) {
throw Object.assign(new Error('dataset 未注册'), { code: 'dataset_not_found', status: 404 });
}
@@ -639,8 +639,8 @@ export function createPageDataPublicService(deps = {}) {
for (const [name, config] of Object.entries(policy.datasets ?? {})) {
let stats = null;
try {
if (ownerService.getDataset(name)) {
stats = ownerService.getDatasetStats(name);
if (await ownerService.getDataset(name)) {
stats = await ownerService.getDatasetStats(name);
}
} catch {
stats = null;
@@ -660,7 +660,7 @@ export function createPageDataPublicService(deps = {}) {
const recentSubmissions = logs.filter((entry) => entry.action === 'insert').slice(0, 10);
const activeSessions = sessionStore.listByPageId(pageId);
const index = await getPageDataPolicyIndex(getPool(), pageId);
const registeredDatasets = ownerService.listDatasets().map((dataset) => dataset.name);
const registeredDatasets = (await ownerService.listDatasets()).map((dataset) => dataset.name);
const configuredNames = new Set(Object.keys(policy.datasets ?? {}));
const unconfiguredDatasets = registeredDatasets.filter((name) => !configuredNames.has(name));
return {
+5 -4
View File
@@ -58,7 +58,7 @@ export function createPageDataService(deps = {}) {
async function listRows(user, datasetName, query = {}) {
const { service } = await createServiceForUser(user);
try {
return service.readDatasetRows(datasetName, {
return await service.readDatasetRows(datasetName, {
limit: query.limit,
offset: query.offset,
orderBy: query.orderBy ?? query.order_by,
@@ -73,7 +73,7 @@ export function createPageDataService(deps = {}) {
async function getSchema(user, datasetName) {
const { service } = await createServiceForUser(user);
try {
return service.getDatasetSchema(datasetName);
return await service.getDatasetSchema(datasetName);
} catch (error) {
throw mapServiceError(error);
}
@@ -82,7 +82,7 @@ export function createPageDataService(deps = {}) {
async function getStats(user, datasetName) {
const { service } = await createServiceForUser(user);
try {
return service.getDatasetStats(datasetName);
return await service.getDatasetStats(datasetName);
} catch (error) {
throw mapServiceError(error);
}
@@ -139,7 +139,8 @@ export function createPageDataService(deps = {}) {
async function listDatasets(user) {
const { service } = await createServiceForUser(user);
return service.listDatasets().map((dataset) => ({
const datasets = await service.listDatasets();
return datasets.map((dataset) => ({
name: dataset.name,
table: dataset.table,
description: dataset.description,
+33 -2
View File
@@ -171,6 +171,16 @@ function assertRegisteredDatasetTables(userDataSpace, registryDatasets, usage) {
}
}
async function assertRegisteredDatasetTablesAsync(userDataSpace, registryDatasets, usage) {
for (const dataset of registryDatasets) {
const adapter = {
listTableColumns: () => userDataSpace.listTableColumns(dataset.table),
};
const columns = await adapter.listTableColumns();
assertRegisteredDatasetTables({ listTableColumns: () => columns }, [dataset], usage);
}
}
/**
* Page Data policy must be derived from a registered SQLite dataset, never
* accepted solely from an Agent-supplied policy. This runs before page or
@@ -213,6 +223,27 @@ export function resolveRegisteredPageDataPolicy({
};
}
export async function resolveRegisteredPageDataPolicyAsync(options = {}) {
const { html, userId, accessMode, pageDataPolicy = null, userDataSpace } = options;
const usage = detectPageDataDatasetUsageFromHtml(html);
if (!usage.size) return null;
if (!userDataSpace) throw new Error('缺少 Page Data 数据空间');
if (pageDataPolicy?.datasets) assertPolicyMatchesHtmlDatasets(html, pageDataPolicy.datasets);
const registryDatasets = await userDataSpace.listDatasets();
const datasets = buildPageDataPolicyDatasetsFromRegistry({ html, registryDatasets, usage });
await assertRegisteredDatasetTablesAsync(
userDataSpace,
registryDatasets.filter((dataset) => datasets[dataset.name]),
usage,
);
return {
...pageDataPolicy,
ownerUserId: String(pageDataPolicy?.ownerUserId ?? userId).trim(),
accessMode: pageDataPolicy?.accessMode ?? accessMode,
datasets,
};
}
export async function bindWorkspaceHtmlForPageData({
pool,
h5Root,
@@ -236,8 +267,8 @@ export async function bindWorkspaceHtmlForPageData({
const normalizedAccessMode = resolvePageDataBindAccess(accessMode);
const resolvedPassword = resolvePageDataBindPassword(normalizedAccessMode, password);
const userDataSpace = createUserDataSpaceService({ workspaceRoot });
const resolvedPageDataPolicy = resolveRegisteredPageDataPolicy({
const userDataSpace = createUserDataSpaceService({ workspaceRoot, userId, query: pool });
const resolvedPageDataPolicy = await resolveRegisteredPageDataPolicyAsync({
html: content,
userId,
accessMode: normalizedAccessMode,
+1 -1
View File
@@ -66,7 +66,7 @@ export async function ensureRegisteredDatasetFromHtml({
datasetName,
}) {
const dataSpace = createUserDataSpaceService({ workspaceRoot, userId, query });
const existing = dataSpace.getDataset(datasetName);
const existing = await dataSpace.getDataset(datasetName);
if (existing) return existing;
const insertColumns = inferInsertColumnsFromHtml(html, datasetName);
+7 -1
View File
@@ -57,7 +57,13 @@ test('evaluatePageDataFinishGuard detects unbound homework html even when user o
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: '确认发布',
messages: [],
messages: [{
createdAt: Date.now(),
content: [{
type: 'toolRequest',
toolCall: { value: { name: 'write_file', arguments: { path: 'public/homework.html' } } },
}],
}],
});
assert.equal(evaluation.pageDataIntent, false);
assert.equal(evaluation.structuralPageData, true);
+391
View File
@@ -0,0 +1,391 @@
import pg from 'pg';
import {
buildControlSchemaSql,
deriveUserSpaceNames,
provisionUserSpace,
quotePgIdentifier,
} from './mindspace-userdata-postgres.mjs';
const pools = new Map();
const provisionedUsers = new Set();
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
function ident(value, label = '标识符') {
const text = String(value ?? '').trim();
if (!IDENTIFIER.test(text)) throw Object.assign(new Error(`${label} 格式无效`), { code: 'invalid_identifier' });
return text;
}
function normalizeDataset(raw, fallbackName = null) {
const value = typeof raw === 'string' ? JSON.parse(raw) : raw;
if (!value || typeof value !== 'object') throw Object.assign(new Error('dataset 配置无效'), { code: 'invalid_dataset_config' });
const name = ident(value.name ?? fallbackName, 'dataset 名称');
const table = ident(value.table ?? value.table_name, 'dataset 表名');
const columns = value.columns && typeof value.columns === 'object' ? value.columns : {};
for (const [action, fields] of Object.entries(columns)) {
if (!Array.isArray(fields)) throw Object.assign(new Error(`dataset 字段白名单无效:${action}`), { code: 'invalid_dataset_config' });
columns[action] = fields.map((field) => ident(field, `${action} 字段`));
}
return {
name,
table,
description: String(value.description ?? ''),
actions: Array.isArray(value.actions) ? value.actions.map(String) : [],
columns,
limits: {
maxRowsPerRead: Number(value.limits?.maxRowsPerRead ?? 200),
maxInsertBytes: Number(value.limits?.maxInsertBytes ?? 8192),
},
};
}
function poolConfig(options) {
if (options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL) {
return { connectionString: options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL, max: 5 };
}
return {
host: options.pgHost ?? process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp',
port: Number(options.pgPort ?? process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433),
database: options.pgDatabase ?? process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev',
user: options.pgUser ?? process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER,
max: 5,
};
}
function sharedPool(options) {
if (options.pgPool) return options.pgPool;
const config = poolConfig(options);
const key = JSON.stringify(config);
if (!pools.has(key)) pools.set(key, new pg.Pool(config));
return pools.get(key);
}
function rejectSql(sql, { readonly = false } = {}) {
const cleaned = String(sql ?? '').replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trim();
if (!cleaned || cleaned.length > 20000) throw new Error('SQL 为空或过长');
if (readonly && !/^(SELECT|WITH)\b/i.test(cleaned)) throw new Error('private_data_query 只允许 SELECT/WITH');
if (/\b(CREATE|ALTER|DROP)\s+(ROLE|USER|DATABASE|TABLESPACE|EXTENSION)|ALTER\s+SYSTEM|COPY[\s\S]+PROGRAM|SECURITY\s+DEFINER|pg_(read|write)_file|lo_import|dblink/i.test(cleaned)) {
throw new Error('SQL 包含用户空间不允许的操作');
}
if (/\bSET\s+(ROLE|SESSION_AUTHORIZATION|search_path)\b/i.test(cleaned)) throw new Error('SQL 不允许改变安全上下文');
return cleaned;
}
function translateSqliteAgentDdl(sql) {
return sql
.replace(/INTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT/gi, 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY')
.replace(/TEXT\s+NOT\s+NULL\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP')
.replace(/TEXT\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP');
}
export function createPostgresUserDataSpaceService(options = {}) {
const names = deriveUserSpaceNames(options.userId);
const pool = sharedPool(options);
const schema = quotePgIdentifier(names.schemaName);
const role = quotePgIdentifier(names.agentRole);
const maxRows = Number(options.maxRows ?? 200);
async function ensureProvisioned({ force = false } = {}) {
if (provisionedUsers.has(names.userId)) return;
const client = await pool.connect();
try {
await client.query(buildControlSchemaSql());
const existing = await client.query('SELECT migration_state FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]);
if (!existing.rows[0]) {
if (!force && String(options.autoProvision ?? process.env.MINDSPACE_USERDATA_AUTO_PROVISION ?? '0') !== '1') {
throw Object.assign(new Error('用户 PG 空间尚未分配'), { code: 'user_space_not_provisioned' });
}
await provisionUserSpace(client, names.userId, { sourceSqlitePath: options.workspaceRoot ? `${options.workspaceRoot}/.mindspace/private-data.sqlite` : null });
await client.query('BEGIN');
try {
await client.query(`CREATE TABLE IF NOT EXISTS ${schema}.__page_data_datasets (name text PRIMARY KEY, table_name text NOT NULL, config_json text NOT NULL, created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)`);
await client.query(`ALTER TABLE ${schema}.__page_data_datasets OWNER TO ${quotePgIdentifier(names.ownerRole)}`);
await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON ${schema}.__page_data_datasets TO ${role}`);
await client.query("UPDATE mindspace_control.user_spaces SET migration_state='cutover',cutover_at=CURRENT_TIMESTAMP,rollback_until=CURRENT_TIMESTAMP + INTERVAL '30 days',updated_at=CURRENT_TIMESTAMP WHERE user_id=$1::uuid", [names.userId]);
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK');
throw error;
}
}
provisionedUsers.add(names.userId);
} finally {
client.release();
}
}
async function tenant(fn, { write = false } = {}) {
await ensureProvisioned();
const client = await pool.connect();
let quotaBytes = null;
try {
await client.query('BEGIN');
if (write) {
const quota = await client.query('SELECT quota_bytes FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]);
quotaBytes = Number(quota.rows[0]?.quota_bytes ?? 0);
}
await client.query(`SET LOCAL ROLE ${role}`);
await client.query(`SET LOCAL search_path = ${schema}, pg_catalog`);
await client.query(`SET LOCAL statement_timeout = '${write ? 30 : 10}s'`);
const result = await fn(client);
if (write && quotaBytes > 0) {
const usage = await client.query(
`SELECT COALESCE(sum(pg_total_relation_size(c.oid)),0)::bigint AS bytes
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
WHERE n.nspname=$1 AND c.relkind IN ('r','p','m')`,
[names.schemaName],
);
if (Number(usage.rows[0].bytes) > quotaBytes) {
throw Object.assign(new Error(`用户 PG 空间超过配额:${usage.rows[0].bytes}/${quotaBytes}`), { code: 'quota_exceeded' });
}
}
await client.query(write ? 'COMMIT' : 'ROLLBACK');
if (write) {
await client.query(
`INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json)
VALUES($1::uuid,'agent',$2,'postgres_write','success',$3::jsonb)`,
[names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName })],
);
}
return result;
} catch (error) {
await client.query('ROLLBACK').catch(() => {});
if (write) {
await client.query(
`INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json)
VALUES($1::uuid,'agent',$2,'postgres_write','failed',$3::jsonb)`,
[names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName, code: error?.code ?? null })],
).catch(() => {});
}
throw error;
} finally {
client.release();
}
}
async function listTableColumns(tableName) {
const table = ident(tableName, '表名');
return tenant(async (client) => {
const result = await client.query(
`SELECT column_name AS name, data_type AS type,
(is_nullable = 'NO') AS "notNull",
(SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) WHERE i.indrelid=($1||'.'||$2)::regclass AND i.indisprimary AND a.attname=c.column_name)) AS pk
FROM information_schema.columns c WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`,
[names.schemaName, table],
);
return result.rows;
});
}
async function getSchema() {
return tenant(async (client) => (await client.query(
`SELECT table_name, column_name, data_type AS type, (is_nullable='NO') AS not_null
FROM information_schema.columns WHERE table_schema=$1 ORDER BY table_name,ordinal_position`,
[names.schemaName],
)).rows);
}
async function querySql(sql) {
const cleaned = rejectSql(sql, { readonly: true }).replace(/;\s*$/, '');
return tenant(async (client) => (await client.query(`SELECT * FROM (${cleaned}) AS q LIMIT ${maxRows}`)).rows);
}
async function executeSql(sql) {
const cleaned = translateSqliteAgentDdl(rejectSql(sql));
return tenant(async (client) => {
const result = await client.query(cleaned);
return { backend: 'postgres', command: result.command, affectedRows: result.rowCount ?? 0, sizeBytes: null, deltaBytes: null, quotaSynced: false };
}, { write: true });
}
async function getDataset(name) {
const datasetName = ident(name, 'dataset 名称');
return tenant(async (client) => {
const result = await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets WHERE name=$1 LIMIT 1', [datasetName]);
if (!result.rows[0]) return null;
return { ...normalizeDataset(result.rows[0].config_json, result.rows[0].name), createdAt: result.rows[0].created_at, updatedAt: result.rows[0].updated_at };
});
}
async function listDatasets() {
return tenant(async (client) => (await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets ORDER BY name')).rows.map((row) => ({
...normalizeDataset(row.config_json, row.name), createdAt: row.created_at, updatedAt: row.updated_at,
})));
}
async function upsertDataset(input) {
const dataset = normalizeDataset(input);
if (!(await listTableColumns(dataset.table)).length) throw Object.assign(new Error(`dataset 对应表不存在:${dataset.table}`), { code: 'table_not_found' });
await tenant((client) => client.query(
`INSERT INTO __page_data_datasets(name,table_name,config_json,updated_at) VALUES($1,$2,$3,CURRENT_TIMESTAMP)
ON CONFLICT(name) DO UPDATE SET table_name=EXCLUDED.table_name,config_json=EXCLUDED.config_json,updated_at=CURRENT_TIMESTAMP`,
[dataset.name, dataset.table, JSON.stringify(dataset)],
), { write: true });
return getDataset(dataset.name);
}
function assertAction(dataset, action) {
if (!dataset.actions.includes(action)) throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' });
}
async function readRowsForDataset(datasetInput, options = {}) {
const dataset = await Promise.resolve(datasetInput);
assertAction(dataset, 'read');
const columns = dataset.columns.read?.map((item) => ident(item, '字段')) ?? [];
if (!columns.length) throw Object.assign(new Error('dataset 未配置可读字段'), { code: 'columns_not_allowed' });
const limit = Math.min(Math.max(1, Number(options.limit ?? dataset.limits.maxRowsPerRead)), maxRows, dataset.limits.maxRowsPerRead);
const offset = Math.max(0, Number(options.offset ?? 0));
const order = options.orderBy ? ident(options.orderBy, '排序字段') : columns.includes('id') ? 'id' : null;
if (order && !columns.includes(order)) throw Object.assign(new Error('排序字段未授权'), { code: 'columns_not_allowed' });
return tenant(async (client) => {
const filters = [];
if ((await listTableColumns(dataset.table)).some((column) => column.name === 'deleted_at') && !options.includeDeleted) filters.push('deleted_at IS NULL');
if (options.rowScope?.whereClause) filters.push(`(${options.rowScope.whereClause})`);
const where = filters.length ? ` WHERE ${filters.join(' AND ')}` : '';
const result = await client.query(
`SELECT ${columns.map(quotePgIdentifier).join(',')} FROM ${quotePgIdentifier(dataset.table)}${where}${order ? ` ORDER BY ${quotePgIdentifier(order)} ${String(options.orderDir).toLowerCase() === 'asc' ? 'ASC' : 'DESC'}` : ''} LIMIT $1 OFFSET $2`,
[limit, offset],
);
return { dataset, rows: result.rows, limit, offset };
});
}
async function readDatasetRows(name, options) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return readRowsForDataset(dataset, options);
}
async function getDatasetStats(name) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
assertAction(dataset, 'read');
return tenant(async (client) => ({ dataset: { name: dataset.name, table: dataset.table }, total: Number((await client.query(`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(dataset.table)}`)).rows[0].count) }));
}
async function insertWithDataset(dataset, payload, meta = {}) {
assertAction(dataset, 'insert');
const allowed = new Set(dataset.columns.insert ?? []);
const keys = Object.keys(payload ?? {});
if (keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('提交包含未授权字段'), { code: 'columns_not_allowed' });
return tenant(async (client) => {
const columns = await listTableColumns(dataset.table);
const values = { ...payload };
if (meta.rowScope?.ownerColumn && columns.some((c) => c.name === meta.rowScope.ownerColumn)) values[meta.rowScope.ownerColumn] ??= meta.rowScope.visitorUserId;
const insertKeys = Object.keys(values).map((key) => ident(key, '字段'));
const result = await client.query(
`INSERT INTO ${quotePgIdentifier(dataset.table)} (${insertKeys.map(quotePgIdentifier).join(',')}) VALUES (${insertKeys.map((_, i) => `$${i + 1}`).join(',')}) RETURNING *`,
insertKeys.map((key) => values[key]),
);
return { dataset, row: result.rows[0] };
}, { write: true });
}
async function insertDatasetRow(name, payload, meta = {}) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return insertWithDataset(dataset, payload, meta);
}
async function insertRowForDataset(datasetInput, payload, meta = {}) {
const dataset = await Promise.resolve(datasetInput);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return insertWithDataset(dataset, payload, meta);
}
async function updateRowForDataset(datasetInput, rowId, payload, meta = {}) {
const dataset = await Promise.resolve(datasetInput);
assertAction(dataset, 'update');
const id = Number(rowId);
if (!Number.isFinite(id) || id <= 0) throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' });
const allowed = new Set(dataset.columns.update ?? []);
const keys = Object.keys(payload ?? {});
if (!keys.length || keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('更新包含未授权字段'), { code: 'columns_not_allowed' });
return tenant(async (client) => {
const result = await client.query(
`UPDATE ${quotePgIdentifier(dataset.table)} SET ${keys.map((key, i) => `${quotePgIdentifier(ident(key, '字段'))}=$${i + 1}`).join(',')} WHERE id=$${keys.length + 1}${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING *`,
[...keys.map((key) => payload[key]), id],
);
if (!result.rows[0]) throw Object.assign(new Error('无权更新该行或行不存在'), { code: 'row_not_allowed' });
return { dataset, row: result.rows[0] };
}, { write: true });
}
async function updateDatasetRow(name, rowId, payload, meta) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return updateRowForDataset(dataset, rowId, payload, meta);
}
async function softDeleteRowForDataset(datasetInput, rowId, meta = {}) {
const dataset = await Promise.resolve(datasetInput);
assertAction(dataset, 'soft_delete');
const columns = await listTableColumns(dataset.table);
if (!columns.some((item) => item.name === 'deleted_at')) throw Object.assign(new Error('目标表不支持软删除'), { code: 'soft_delete_unsupported' });
const id = Number(rowId);
return tenant(async (client) => {
const hasDeletedBy = columns.some((item) => item.name === 'deleted_by');
const result = await client.query(
`UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=CURRENT_TIMESTAMP${hasDeletedBy ? ',deleted_by=$2' : ''} WHERE id=$1 AND deleted_at IS NULL${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING id,deleted_at`,
hasDeletedBy ? [id, meta.deletedBy ?? meta.updatedByLabel ?? 'public'] : [id],
);
if (!result.rows[0]) throw Object.assign(new Error('无权删除该行或行不存在'), { code: 'row_not_allowed' });
return { dataset, id, deleted: true };
}, { write: true });
}
async function softDeleteDatasetRow(name, rowId, meta) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return softDeleteRowForDataset(dataset, rowId, meta);
}
async function restoreSoftDeletedRow(name, rowId) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
const columns = await listTableColumns(dataset.table);
return tenant(async (client) => {
const result = await client.query(
`UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=NULL${columns.some((item) => item.name === 'deleted_by') ? ',deleted_by=NULL' : ''} WHERE id=$1 AND deleted_at IS NOT NULL RETURNING *`,
[Number(rowId)],
);
if (!result.rows[0]) throw Object.assign(new Error('行不存在或未删除'), { code: 'row_not_allowed' });
return { dataset, row: result.rows[0], restored: true };
}, { write: true });
}
async function exportDatasetRows(name, { format = 'json', includeDeleted = false, limit = 1000 } = {}) {
const result = await readDatasetRows(name, { includeDeleted, limit: Math.min(Number(limit), 5000) });
if (String(format).toLowerCase() !== 'csv') return { dataset: name, format: 'json', rows: result.rows, rowCount: result.rows.length };
const columns = result.dataset.columns.read ?? [];
const escape = (value) => {
const text = value == null ? '' : String(value);
return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
};
return { dataset: name, format: 'csv', content: `${columns.join(',')}\n${result.rows.map((row) => columns.map((column) => escape(row[column])).join(',')).join('\n')}\n`, rowCount: result.rows.length };
}
async function getDatasetSchema(name) {
const dataset = await getDataset(name);
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
return { dataset, tableColumns: await listTableColumns(dataset.table) };
}
async function getInfo() {
return { name: '用户私有数据空间', backend: 'postgres', database: poolConfig(options).database, schema: names.schemaName, userId: names.userId };
}
async function ensureReady() {
await ensureProvisioned({ force: true });
return getInfo();
}
return {
backend: 'postgres', workspaceRoot: options.workspaceRoot, privateDataDb: null,
ensureReady, getInfo, getSchema, querySql, executeSql, listTableColumns, getDataset, listDatasets, upsertDataset,
readRowsForDataset, readDatasetRows, getDatasetStats, getStatsForDataset: async (dataset) => getDatasetStats((await Promise.resolve(dataset)).name),
getDatasetSchema, getSchemaForDataset: async (dataset) => ({ dataset: await Promise.resolve(dataset), tableColumns: await listTableColumns((await Promise.resolve(dataset)).table) }),
insertDatasetRow, insertRowForDataset, updateDatasetRow, updateRowForDataset,
softDeleteDatasetRow, softDeleteRowForDataset, restoreSoftDeletedRow, exportDatasetRows,
};
}
+9 -16
View File
@@ -8,6 +8,11 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
applyMemindRuntimeProfile,
describeMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const opsDir = path.join(root, 'ops');
@@ -23,22 +28,6 @@ const plazaPublicBase = (
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
).replace(/\/$/, '');
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(root, '../../.env.local'));
loadEnvFile(path.join(root, '.env'));
const logFile =
process.env.MEMIND_DEV_LOG ?? path.join(os.homedir(), 'Library/Logs/memind-dev.log');
@@ -51,6 +40,10 @@ function log(message) {
}
}
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
log(`Runtime profile: ${describeMemindRuntimeProfile()}`);
function freePort(port) {
try {
execSync(`lsof -ti TCP:${port} -sTCP:LISTEN | xargs kill -9`, { stdio: 'ignore' });
+8 -15
View File
@@ -3,6 +3,11 @@ import { spawn, execSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
applyMemindRuntimeProfile,
describeMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
const opsDir = path.join(root, 'ops');
@@ -18,21 +23,9 @@ const plazaPublicBase = (
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
).replace(/\/$/, '');
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(root, '../../.env.local'));
loadEnvFile(path.join(root, '.env'));
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
console.log(`==> Runtime profile: ${describeMemindRuntimeProfile()}`);
function freePort(port) {
try {
+7 -16
View File
@@ -1,20 +1,11 @@
import fs from 'node:fs';
import path from 'node:path';
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const separator = trimmed.indexOf('=');
if (separator < 0) continue;
const key = trimmed.slice(0, separator).trim();
const value = trimmed.slice(separator + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
import {
applyMemindRuntimeProfile,
loadMemindEnvFiles,
} from './memind-runtime-profile.mjs';
export function loadH5Environment(scriptDirectory) {
loadEnvFile(path.join(scriptDirectory, '../../../.env.local'));
loadEnvFile(path.join(scriptDirectory, '../.env'));
const root = path.join(scriptDirectory, '..');
loadMemindEnvFiles(root);
applyMemindRuntimeProfile({ rootDir: root });
}
+119
View File
@@ -0,0 +1,119 @@
import fs from 'node:fs';
import path from 'node:path';
export const MEMIND_RUNTIME_PROFILES = new Set(['local', 'split-service', 'production']);
/**
* Load Memind env files in standard order. Later files only fill unset keys.
*/
export function loadMemindEnvFiles(rootDir, env = process.env) {
const root = path.resolve(rootDir);
for (const relativePath of ['../../.env.local', '.env']) {
loadEnvFile(path.join(root, relativePath), env);
}
}
function loadEnvFile(filePath, env = process.env) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const separator = trimmed.indexOf('=');
if (separator < 0) continue;
const key = trimmed.slice(0, separator).trim();
const value = trimmed.slice(separator + 1).trim();
if (!env[key]) env[key] = value;
}
}
function expandEnvReference(value, env = process.env) {
const raw = String(value ?? '').trim();
const match = raw.match(/^\$\{([A-Z0-9_]+)\}$/);
if (!match) return raw;
return String(env[match[1]] ?? '').trim() || raw;
}
export function resolveMemindRuntimeProfile(env = process.env) {
const explicit = String(env.MEMIND_RUNTIME_PROFILE ?? '').trim().toLowerCase();
if (explicit) {
if (!MEMIND_RUNTIME_PROFILES.has(explicit)) {
throw new Error(
`Unsupported MEMIND_RUNTIME_PROFILE "${explicit}". Expected: ${[...MEMIND_RUNTIME_PROFILES].join(', ')}`,
);
}
return explicit;
}
if (String(env.NODE_ENV ?? '').trim().toLowerCase() === 'production') {
return 'production';
}
return 'local';
}
/**
* Apply environment-specific defaults after .env files are loaded.
*
* - local: monolith MindSpace adapter, repo-local storage (pnpm dev default)
* - split-service: remote adapter against standalone MindSpace on 8082
* - production: no overrides; 103 .env is source of truth
*/
export function applyMemindRuntimeProfile({
rootDir = process.cwd(),
env = process.env,
profile,
} = {}) {
const resolvedProfile = profile ?? resolveMemindRuntimeProfile(env);
env.MEMIND_RUNTIME_PROFILE = resolvedProfile;
if (resolvedProfile === 'local') {
env.MINDSPACE_SERVER_ADAPTER = 'local';
if (!env.MINDSPACE_STORAGE_ROOT) {
env.MINDSPACE_STORAGE_ROOT = path.join(path.resolve(rootDir), 'data', 'mindspace');
}
return {
profile: resolvedProfile,
mindspaceAdapter: 'local',
storageRoot: env.MINDSPACE_STORAGE_ROOT,
};
}
if (resolvedProfile === 'split-service') {
env.MINDSPACE_SERVER_ADAPTER = 'remote';
env.MINDSPACE_REMOTE_BASE_URL = String(
env.MINDSPACE_REMOTE_BASE_URL ?? 'http://127.0.0.1:8082',
).trim();
const expandedToken = expandEnvReference(env.MINDSPACE_REMOTE_AUTH_TOKEN, env);
if (expandedToken) {
env.MINDSPACE_REMOTE_AUTH_TOKEN = expandedToken;
} else if (env.TKMIND_SERVER__SECRET_KEY) {
env.MINDSPACE_REMOTE_AUTH_TOKEN = String(env.TKMIND_SERVER__SECRET_KEY).trim();
}
if (!env.MINDSPACE_STORAGE_ROOT) {
env.MINDSPACE_STORAGE_ROOT = '/Users/john/MindSpace/data/mindspace';
}
return {
profile: resolvedProfile,
mindspaceAdapter: 'remote',
remoteBaseUrl: env.MINDSPACE_REMOTE_BASE_URL,
storageRoot: env.MINDSPACE_STORAGE_ROOT,
};
}
return {
profile: resolvedProfile,
mindspaceAdapter: String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local',
storageRoot: env.MINDSPACE_STORAGE_ROOT ?? null,
};
}
export function describeMemindRuntimeProfile(env = process.env) {
const profile = resolveMemindRuntimeProfile(env);
const adapter = String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local';
const parts = [`MEMIND_RUNTIME_PROFILE=${profile}`, `MINDSPACE_SERVER_ADAPTER=${adapter}`];
if (adapter === 'remote') {
parts.push(`MINDSPACE_REMOTE_BASE_URL=${env.MINDSPACE_REMOTE_BASE_URL ?? '(unset)'}`);
}
if (env.MINDSPACE_STORAGE_ROOT) {
parts.push(`MINDSPACE_STORAGE_ROOT=${env.MINDSPACE_STORAGE_ROOT}`);
}
return parts.join(', ');
}
+54
View File
@@ -0,0 +1,54 @@
import assert from 'node:assert/strict';
import path from 'node:path';
import test from 'node:test';
import {
applyMemindRuntimeProfile,
resolveMemindRuntimeProfile,
} from './memind-runtime-profile.mjs';
test('resolveMemindRuntimeProfile defaults to local for dev', () => {
assert.equal(resolveMemindRuntimeProfile({}), 'local');
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'development' }), 'local');
});
test('resolveMemindRuntimeProfile honors explicit profile and production NODE_ENV', () => {
assert.equal(resolveMemindRuntimeProfile({ MEMIND_RUNTIME_PROFILE: 'split-service' }), 'split-service');
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'production' }), 'production');
});
test('applyMemindRuntimeProfile local forces monolith adapter and repo storage', () => {
const env = {
MINDSPACE_SERVER_ADAPTER: 'remote',
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
};
const result = applyMemindRuntimeProfile({
rootDir: '/tmp/memind',
env,
profile: 'local',
});
assert.equal(result.profile, 'local');
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'local');
assert.match(env.MINDSPACE_STORAGE_ROOT, /\/tmp\/memind\/data\/mindspace$/);
});
test('applyMemindRuntimeProfile split-service expands auth token reference', () => {
const env = {
TKMIND_SERVER__SECRET_KEY: 'local-dev-secret',
MINDSPACE_REMOTE_AUTH_TOKEN: '${TKMIND_SERVER__SECRET_KEY}',
};
applyMemindRuntimeProfile({ env, profile: 'split-service' });
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'local-dev-secret');
});
test('applyMemindRuntimeProfile production does not override adapter', () => {
const env = {
MINDSPACE_SERVER_ADAPTER: 'remote',
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
MINDSPACE_REMOTE_AUTH_TOKEN: 'prod-token',
};
const result = applyMemindRuntimeProfile({ env, profile: 'production' });
assert.equal(result.profile, 'production');
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'prod-token');
});
@@ -0,0 +1,85 @@
#!/usr/bin/env node
import fs from 'node:fs';
import {
buildControlSchemaSql,
createSqliteSnapshot,
inspectSqliteDatabase,
migrateSqliteSnapshot,
provisionUserSpace,
} from '../mindspace-userdata-postgres.mjs';
function usage() {
return `
Usage:
node scripts/migrate-mindspace-sqlite-to-postgres.mjs --source <sqlite> --user-id <uuid> [--apply] [--replace-shadow]
Default is read-only dry-run. --apply requires MINDSPACE_USERDATA_PG_URL.
The command creates a consistent SQLite snapshot and never modifies the source SQLite.
`.trim();
}
export function parseArgs(argv = []) {
const options = { apply: false, replaceShadow: false, source: '', userId: '', help: false };
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === '--apply') options.apply = true;
else if (arg === '--replace-shadow') options.replaceShadow = true;
else if (arg === '--source') options.source = argv[++index] ?? '';
else if (arg === '--user-id') options.userId = argv[++index] ?? '';
else if (arg === '--help' || arg === '-h') options.help = true;
else throw new Error(`未知参数:${arg}`);
}
if (!options.help && (!options.source || !options.userId)) throw new Error('--source 和 --user-id 必填');
return options;
}
export async function run(argv = process.argv.slice(2), env = process.env) {
const options = parseArgs(argv);
if (options.help) {
console.log(usage());
return { mode: 'help' };
}
const inspected = inspectSqliteDatabase(options.source);
const sourceRows = inspected.tables.reduce((total, table) => total + table.rows.length, 0);
if (!options.apply) {
console.log(JSON.stringify({
mode: 'dry-run',
userId: options.userId,
source: inspected.path,
sizeBytes: inspected.sizeBytes,
tables: inspected.tables.map((table) => ({ name: table.name, rows: table.rows.length })),
sourceRows,
note: '未修改 SQLite 或 PostgreSQL;使用 --apply 才会创建影子空间。',
}, null, 2));
return { mode: 'dry-run', tableCount: inspected.tables.length, sourceRows };
}
if (!env.MINDSPACE_USERDATA_PG_URL) throw new Error('--apply requires MINDSPACE_USERDATA_PG_URL');
const snapshotPath = createSqliteSnapshot(options.source);
const snapshot = inspectSqliteDatabase(snapshotPath);
const { default: pg } = await import('pg');
const client = new pg.Client({ connectionString: env.MINDSPACE_USERDATA_PG_URL });
try {
await client.connect();
await client.query(buildControlSchemaSql());
await provisionUserSpace(client, options.userId, {
sourceSqlitePath: inspected.path,
});
const result = await migrateSqliteSnapshot(client, snapshot, options.userId, {
replaceShadow: options.replaceShadow,
sourcePath: inspected.path,
});
if (result.sourceRows !== result.targetRows) throw new Error('迁移行数校验失败');
console.log(JSON.stringify({ mode: 'shadow', ...result }, null, 2));
return { mode: 'shadow', ...result };
} finally {
await client.end().catch(() => {});
fs.rmSync(snapshotPath, { force: true });
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
run().catch((error) => {
console.error(error instanceof Error ? error.message : error);
process.exitCode = 1;
});
}
@@ -0,0 +1,25 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { parseArgs } from './migrate-mindspace-sqlite-to-postgres.mjs';
test('migration CLI defaults to dry-run', () => {
assert.deepEqual(parseArgs(['--source', '/tmp/a.sqlite', '--user-id', 'user-id']), {
apply: false,
replaceShadow: false,
source: '/tmp/a.sqlite',
userId: 'user-id',
help: false,
});
});
test('migration CLI requires explicit source and user', () => {
assert.throws(() => parseArgs([]), /--source 和 --user-id 必填/);
});
test('migration CLI recognizes explicit apply', () => {
assert.equal(parseArgs(['--apply', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).apply, true);
});
test('migration CLI requires explicit replace-shadow for destructive reruns', () => {
assert.equal(parseArgs(['--replace-shadow', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).replaceShadow, true);
});
@@ -0,0 +1,198 @@
#!/usr/bin/env node
/**
* Local-only E2E proof for the questionnaire -> Page Data API -> PostgreSQL path.
*
* The probe uses an existing local user space, creates a uniquely named survey
* table/dataset and page policy, submits one answer through the public HTTP API,
* verifies the row directly in PostgreSQL, and removes all probe objects.
*/
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import express from 'express';
import pg from 'pg';
import { attachPageDataRoutes } from '../page-data-routes.mjs';
import { createPageDataService } from '../page-data-service.mjs';
import { createPageDataPublicService } from '../page-data-public-service.mjs';
import { deletePageAccessPolicy, writePageAccessPolicy } from '../page-data-policy-store.mjs';
import { createUserDataSpaceService } from '../user-data-space-service.mjs';
import { deriveUserSpaceNames, quotePgIdentifier } from '../mindspace-userdata-postgres.mjs';
import { loadH5Environment } from './load-env.mjs';
loadH5Environment(import.meta.dirname);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const userId = process.argv[2] ?? '1c99b83b-0454-474f-a5d2-129d34506a32';
const workspaceRoot = path.join(repoRoot, 'MindSpace', userId);
const suffix = `${Date.now()}_${process.pid}`;
const tableName = `survey_e2e_${suffix}`;
const datasetName = tableName;
const pageId = `page-survey-e2e-${suffix}`;
const names = deriveUserSpaceNames(userId);
const policyPath = path.join(workspaceRoot, '.mindspace', 'page-data-policies', `${pageId}.json`);
const logPath = path.join(workspaceRoot, '.mindspace', 'page-data-logs', `${pageId}.jsonl`);
assert.equal(process.env.MINDSPACE_USERDATA_BACKEND, 'postgres', '本机未启用 PostgreSQL 用户数据后端');
assert.ok(fs.existsSync(workspaceRoot), `用户工作区不存在: ${workspaceRoot}`);
const pgPool = new pg.Pool({
host: process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp',
port: Number(process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433),
database: process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev',
user: process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER,
max: 2,
});
function publicationPool() {
return {
async query(sql) {
if (sql.includes('FROM h5_publish_records')) {
return [[{
id: `publication-${suffix}`,
user_id: userId,
page_id: pageId,
access_mode: 'public',
password_hash: null,
status: 'online',
expires_at: null,
}]];
}
return [[]];
},
};
}
function buildApp() {
const fakePublicationPool = publicationPool();
const publicService = createPageDataPublicService({
getPool: () => fakePublicationPool,
resolveWorkspaceRootForOwner: () => workspaceRoot,
});
const api = express.Router();
api.use(express.json());
attachPageDataRoutes(api, {
sendData: (res, _req, data, status = 200) => res.status(status).json({ data }),
sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }),
getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }),
getPageDataPublicService: () => publicService,
});
const app = express();
app.set('trust proxy', true);
app.use('/api', api);
return app;
}
async function submitQuestionnaire(app, row) {
const server = app.listen(0, '127.0.0.1');
await new Promise((resolve, reject) => {
server.once('listening', resolve);
server.once('error', reject);
});
try {
const { port } = server.address();
const apiPath = `/api/public/pages/${pageId}/data/${datasetName}/rows`;
const response = await fetch(`http://127.0.0.1:${port}${apiPath}`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(row),
});
return { apiPath, status: response.status, body: await response.json() };
} finally {
await new Promise((resolve) => server.close(resolve));
}
}
async function main() {
const ownerService = createUserDataSpaceService({ workspaceRoot, userId });
let tableCreated = false;
try {
// Equivalent to the Agent's private_data_execute + register_dataset + bind policy flow.
await ownerService.executeSql(`CREATE TABLE ${tableName} (
id INTEGER PRIMARY KEY AUTOINCREMENT,
satisfaction TEXT NOT NULL,
favorite_feature TEXT NOT NULL,
suggestion TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now', 'localtime'))
)`);
tableCreated = true;
const dataset = await ownerService.upsertDataset({
name: datasetName,
table: tableName,
description: '本机 PG 问卷端到端验证',
actions: ['read', 'insert'],
columns: {
read: ['id', 'satisfaction', 'favorite_feature', 'suggestion', 'created_at'],
insert: ['satisfaction', 'favorite_feature', 'suggestion'],
},
});
writePageAccessPolicy(workspaceRoot, {
pageId,
ownerUserId: userId,
accessMode: 'public',
datasets: {
[datasetName]: {
insert: true,
read: false,
columns: { insert: ['satisfaction', 'favorite_feature', 'suggestion'] },
},
},
});
const answer = {
satisfaction: '非常满意',
favorite_feature: '个人数据报表',
suggestion: '增加按月趋势分析',
};
const submitted = await submitQuestionnaire(buildApp(), answer);
assert.equal(submitted.status, 201, JSON.stringify(submitted.body));
assert.equal(submitted.body.data.dataset, datasetName);
assert.equal(submitted.body.data.row.satisfaction, answer.satisfaction);
const direct = await pgPool.query(
`SELECT id,satisfaction,favorite_feature,suggestion,created_at
FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(tableName)}
ORDER BY id DESC LIMIT 1`,
);
assert.equal(direct.rowCount, 1);
assert.equal(direct.rows[0].satisfaction, answer.satisfaction);
assert.equal(direct.rows[0].favorite_feature, answer.favorite_feature);
assert.equal(direct.rows[0].suggestion, answer.suggestion);
const registered = await pgPool.query(
`SELECT name,table_name FROM ${quotePgIdentifier(names.schemaName)}.__page_data_datasets WHERE name=$1`,
[datasetName],
);
assert.equal(registered.rowCount, 1);
console.log(JSON.stringify({
result: 'PASS',
localOnly: true,
userId,
schema: names.schemaName,
dataset: { name: dataset.name, table: dataset.table, actions: dataset.actions },
pageDataApi: { method: 'POST', path: submitted.apiPath, status: submitted.status },
postgresRow: direct.rows[0],
checks: [
'Agent-style table creation succeeded in the user schema',
'Dataset registry was stored in PostgreSQL',
'Questionnaire submission used the public Page Data HTTP API',
'Submitted answers matched a direct PostgreSQL query',
],
}, null, 2));
} finally {
if (tableCreated) {
await ownerService.executeSql(`DELETE FROM __page_data_datasets WHERE name='${datasetName}'`).catch(() => {});
await ownerService.executeSql(`DROP TABLE IF EXISTS ${tableName}`).catch(() => {});
}
deletePageAccessPolicy(workspaceRoot, pageId);
fs.rmSync(logPath, { force: true });
fs.rmSync(policyPath, { force: true });
await pgPool.end();
}
}
main().catch((error) => {
console.error(error instanceof Error ? error.stack ?? error.message : error);
process.exitCode = 1;
});
+58 -4
View File
@@ -14,6 +14,7 @@ import {
} from './auth.mjs';
import { createDbPool, initSchema, isDatabaseConfigured } from './db.mjs';
import { createWorkspacePageDeliverService } from './mindspace-workspace-page-deliver.mjs';
import { createUserDataSpaceService } from './user-data-space-service.mjs';
import { createToolGateway } from './tool-gateway.mjs';
import { createAgentRunGateway } from './agent-run-gateway.mjs';
import {
@@ -164,6 +165,7 @@ import {
registerChatDocxArtifactForConversation,
} from './mindspace-chat-docx-package.mjs';
import { scanContent } from './mindspace-content-scan.mjs';
import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
import { renderImageAssetViewerHtml, wantsInlineImageViewer } from './mindspace-asset-preview.mjs';
import { DEFAULT_IMAGE_UPLOAD_MAX_BYTES } from './user-image-normalize.mjs';
import { createRechargeService } from './billing-recharge.mjs';
@@ -535,6 +537,10 @@ async function bootstrapUserAuth() {
h5Root: __dirname,
defaultSignupBalanceCents: Number(process.env.H5_SIGNUP_BALANCE_CENTS ?? 500),
subscriptionService,
provisionUserDataSpace: async ({ userId, workspaceRoot }) => {
const service = createUserDataSpaceService({ workspaceRoot, userId, query: pool });
return service.ensureReady();
},
});
sessionAccess = createSessionAccess({ userAuth, enabled: isSessionBrokerEnabled() });
if (sessionAccess.enabled) {
@@ -622,6 +628,12 @@ async function bootstrapUserAuth() {
expireStaleUploadsIntervalMs: 5 * 60 * 1000,
});
await userAuth.ensureAdminUser();
const userDataSpaceBackfill = await userAuth.ensureAllUserDataSpaces();
if (userDataSpaceBackfill.errors.length > 0) {
console.warn(
`[PageData] PG space backfill incomplete: ${userDataSpaceBackfill.errors.length} user(s) failed`,
);
}
llmProviderService = createLlmProviderService(pool, {
apiTarget: API_TARGET,
apiTargets: API_TARGETS,
@@ -717,8 +729,24 @@ async function bootstrapUserAuth() {
messages: [userMessage],
});
},
syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId),
syncUserPagesOnSuccess: async ({ userId, sessionId, runStartedAtMs }) =>
syncUserGeneratedPages(userId, { sessionId, sinceMs: runStartedAtMs }),
isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0,
validateRunDeliverables: async ({ userId, deliverables }) => {
const publishDir = resolveMindSpaceUserPublishDir(__dirname, { id: userId });
const violations = scanWorkspaceFilesForProhibitedBrowserStorage({
publishDir,
relativePaths: (deliverables?.pages ?? [])
.map((page) => page.workspaceRelativePath)
.filter(Boolean),
});
return {
errors: violations.map((violation) => ({
code: 'browser_storage_forbidden',
message: `${violation.relativePath} 使用 ${violation.apis.join(', ')}`,
})),
};
},
autoDispatch: ['1', 'true', 'yes', 'on'].includes(
String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(),
),
@@ -3231,10 +3259,36 @@ async function resolveExistingSavedPage(userId, { sessionId, messageId, relative
const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']);
// REGRESSION GUARD: mindspace-page-sync-thumbnail — remote 也经 pageSyncService RPC 同步 public HTML
async function syncUserGeneratedPages(userId) {
async function listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs = null } = {}) {
if (!authPool || !userId || !sessionId) return [];
const sinceClause = sinceMs == null ? '' : 'AND ca.created_at >= ?';
const params = sinceMs == null ? [userId, sessionId] : [userId, sessionId, sinceMs];
const [rows] = await authPool.query(
`SELECT ca.display_name
FROM h5_conversation_artifacts ca
JOIN h5_conversation_packages cp ON cp.id = ca.package_id
WHERE cp.user_id = ?
AND cp.session_id = ?
AND ca.artifact_kind = 'public_html'
${sinceClause}
ORDER BY ca.sort_order ASC, ca.created_at ASC`,
params,
);
return [...new Set((rows ?? [])
.map((row) => normalizeWorkspaceRelativePath(`public/${String(row.display_name ?? '').trim()}`))
.filter((relativePath) => relativePath?.startsWith('public/') && relativePath.toLowerCase().endsWith('.html')))];
}
async function syncUserGeneratedPages(userId, { sessionId = null, sinceMs = null } = {}) {
if (!userId) return;
// Agent-run/Finish delivery must stay scoped to the current conversation.
// A stale Page Data page elsewhere in the user's workspace must not turn a
// successfully completed current task into a failed run.
const pageDataRelativePaths = sessionId
? await listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs })
: null;
if (workspacePageDeliver?.syncAndDeliver) {
return await workspacePageDeliver.syncAndDeliver(userId);
return await workspacePageDeliver.syncAndDeliver(userId, { pageDataRelativePaths });
}
if (!mindSpacePageSync) return;
return await mindSpacePageSync.syncUserGeneratedPages(userId);
@@ -5099,7 +5153,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
.filter(Boolean)
.join('\n')
: '';
await syncUserGeneratedPages(uid);
await syncUserGeneratedPages(uid, { sessionId: sid });
await maybeRepairPageDataAfterFinish({
sessionId: sid,
userId: uid,
+13 -10
View File
@@ -1,11 +1,11 @@
---
name: page-data-collect
description: 在 MindSpace 页面中收集、存储并管理结构化数据(问卷、报名、台账、后台查看),基于 Page Data API 与用户私有 SQLite
description: 在 MindSpace 页面中收集、存储并管理结构化数据(问卷、报名、台账、后台查看),基于 Page Data API 与用户隔离的 PostgreSQL 空间
---
# 页面数据收集(Page Data Collect)
在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互,并将提交持久化到当前用户的 `.mindspace/private-data.sqlite`,通过平台 **Page Data API** 受控读写。
在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互,并将提交持久化到当前用户隔离的 PostgreSQL schema,通过平台 **Page Data API** 受控读写。运行时禁止创建、读取或回退到用户 SQLite。
详细 API 说明见工作区外文档 `docs/page-data-api-usage.md`(Memind 仓库)。
@@ -25,7 +25,7 @@ description: 在 MindSpace 页面中收集、存储并管理结构化数据(
```text
Agent 可以建模 SQL 并注册 dataset;
HTML 页面只能调用 Page Data API(page-data-client.js);
禁止自建 Express / 独立端口 / 直接暴露 SQLite 文件路径。
禁止自建 Express / 独立端口 / 直接暴露数据库连接;禁止 SQLite / localStorage 数据回退。
```
## 方案选择:默认快车道 + 能力分支(必做)
@@ -36,7 +36,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
用户描述需求
→ 匹配能力分支(下表 A/B/C/D,默认 A)
→ 仅冲突或无法匹配时,用 1 题让用户选分支或改口令
→ 输出「方案摘要」供确认(用户可只改口令/选项)
→ 输出简短「方案摘要」;用户已明确要求直接创建/完成/发布时视为已确认并立即执行
→ 再 load_skill / 建表 / write_file / bind
```
@@ -50,7 +50,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
| 提交后修改 | 不支持(一次性) |
| 页面 | `public/*-survey.html` + `public/*-admin.html`(两文件各 bind 一次) |
用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A,先展示摘要再开工**。
用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A**。若用户已说「直接做」「完整创建并发布」「不用询问」「持续推进」等终态指令,摘要后必须在同一轮立即调用工具开工,禁止停下来等待再次确认。
口令规则:
@@ -83,6 +83,9 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
```text
- public:可匿名 insert;服务端禁止 update / softDelete
- password:所有 API 须先 authenticate;口令 ≥8 位且须写入发布记录
- `public` 策略中的 `insert: true` 等价于任何访客都能直接调用 API 写入;前端口令、隐藏按钮和 JavaScript 判断都不是权限控制
- password 页面必须调用 `client.authenticate(password)` 获取服务端 token;禁止在 HTML 中硬编码口令或只做 `password === "..."` 比较
- 同一公开页若要求「所有人可评价、只有所有者可写正文」:正文 dataset 在公开页必须 `insert: false`,评价 dataset 才能 `insert: true`;所有者写正文必须使用独立 `password` 作者页或 `login_required` 页面
- 同页不能同时「匿名 insert」+「口令 read 全量」→ 须拆前台 + 后台(方案 A/E)
- password 无法区分访客身份;「只改自己的」须 login_required + own_rows(方案 B)
- 匿名提交后「凭链接改自己的」无内置能力,须改 B 或接受一次性提交
@@ -101,7 +104,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
- 页面:public/xxx.html + public/xxx-admin.html
- 内容:(由你的描述生成题目/字段/报表)
确认后开始建表与写页面。若需登录提交或团队共改,请说明,我换成 B/C 分支。
若用户尚未授权创建,确认后开始建表与写页面;若用户已明确要求创建/完成/发布,此摘要仅用于告知,后面必须紧接工具调用,不得结束回复等待确认。
```
### 分支切换示例
@@ -121,8 +124,8 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
```text
load_skill → page-data-collect
→ 匹配分支(默认 A)→ 方案摘要 → 用户确认或仅改口令/选项
→ 禁止未确认就 bind;禁止空转计划不调用工具
→ 匹配分支(默认 A)→ 方案摘要 → 用户确认,或从明确的创建/完成/发布指令判定已确认
→ 禁止未获创建授权就 bind;已获授权时禁止空转计划、重复确认或不调用工具
```
### 2. 数据层:建表 + 注册 dataset
@@ -262,7 +265,7 @@ CREATE TABLE IF NOT EXISTS survey_responses (
## 严格禁止
1. **禁止**在方案摘要未确认时建表 / bind / 发布(默认 A 也须展示摘要;用户明确「按默认做」视为确认)
1. **禁止**在尚未获得创建授权时建表 / bind / 发布;用户明确要求「创建、完成、发布、直接做、不用询问、持续推进」均视为确认,禁止再次停下来询问
2. **禁止**让 LLM 自行发明 accessMode/拆页/口令;必须落在分支 A~E 与默认口令规则内
3. **禁止**连续两轮仅输出计划、不调用 `load_skill` / `list_dir` / `write_file` / `private_data_execute` / `bind`
4. **禁止**接受 <8 位发布口令;用户未指定口令时后台默认 **`88888888`**
@@ -274,7 +277,7 @@ CREATE TABLE IF NOT EXISTS survey_responses (
10. **禁止**只 `CREATE TABLE` 而不 `private_data_register_dataset`
11. **禁止**未配置 `private_data_set_page_policy` 就让页面调用公开 API
12. **禁止**先发布占位页(如 `<p>问卷页面</p>`)再让用户访问 `/u/.../pages/...`
13. **禁止**在 Page Data 页面中使用 `localStorage` / `sessionStorage` 存提交记录或做 API fallback
13. **禁止**在 Agent 生成的 HTML/JavaScript 中使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据或做 API fallback;所有需持久化的数据必须进入当前用户专属 PostgreSQL schema,禁止 SQLite、静态 JSON/JS 文件和内存 fallback
## 交付前自检
+1
View File
@@ -27,6 +27,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报
10. **禁止**在 HTML 中用 CDN `https://...` 引用 `<script src>`;MindSpace 发布页 CSP 仅允许同源脚本。常用库优先用平台路径(如 Chart.js → `/assets/chart.umd.min.js`),或下载到 `public/assets/` 后相对引用
11. **纯静态页(无 Page Data 问卷/后台)禁止调用 `private_data_bind_workspace_page`**;`write_file` / `edit_file` 落盘 `public/*.html` 后,直接按下方「回复格式」交付 **MindSpace 路径**(`/MindSpace/<用户ID>/public/...`),无需 bind、无需口令
12. **微信分享必须使用 MindSpace 路径**:该路由会在服务端注入 `og:site_name=TKMind 智趣`、封面缩略图(`*.thumbnail.png`)与微信 JS-SDK 分享桥;禁止交付 `/u/用户名/pages/...` 作为用户可见链接
13. **只要页面接收用户输入并要求以后查看、汇总、统计、修改或删除,就不是纯静态页**。禁止使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据;必须立即 `load_skill` → `page-data-collect`,通过 Page Data API 写入当前用户专属 PostgreSQL schema。禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化。记账、日记、打卡、清单、台账、问卷、报名及其管理页都适用。
详细约束以工作区内的 `.goosehints` 与 `.agents/skills/static-page-publish/SKILL.md` 为准。
+37
View File
@@ -138,6 +138,9 @@ export function createUserAuth(pool, options = {}) {
let rechargeNotifier =
typeof options.onRechargeNotification === 'function' ? options.onRechargeNotification : null;
const subscriptionService = options.subscriptionService ?? null;
const provisionUserDataSpace = typeof options.provisionUserDataSpace === 'function'
? options.provisionUserDataSpace
: null;
const sessions = new Map();
const loginFailures = new Map();
@@ -277,6 +280,14 @@ export function createUserAuth(pool, options = {}) {
};
};
const ensureUserDataSpaceForUser = async (user, workspaceRoot = null) => {
if (!provisionUserDataSpace || !user?.id || user.role === 'admin') return null;
return provisionUserDataSpace({
userId: user.id,
workspaceRoot: workspaceRoot ?? user.workspaceRoot ?? user.workspace_root,
});
};
const listSkillGrants = async (subjectType, subjectId) => {
const [rows] = await pool.query(
`SELECT skill_name, enabled
@@ -463,6 +474,7 @@ export function createUserAuth(pool, options = {}) {
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
now,
});
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, workspaceRoot);
await conn.commit();
ensureWorkspace(workspaceRoot);
ensureUserMemoryProfile(workspaceRoot, {
@@ -697,6 +709,28 @@ export function createUserAuth(pool, options = {}) {
}
};
const ensureAllUserDataSpaces = async () => {
if (!provisionUserDataSpace) return { provisioned: 0, errors: [] };
const [rows] = await pool.query(
`SELECT id, workspace_root FROM h5_users WHERE role = 'user' AND status = 'active'`,
);
let provisioned = 0;
const errors = [];
for (const row of rows) {
try {
await ensureUserDataSpaceForUser({ id: row.id, role: 'user' }, row.workspace_root);
provisioned += 1;
} catch (error) {
errors.push({
userId: row.id,
code: error?.code ?? null,
message: error instanceof Error ? error.message : String(error),
});
}
}
return { provisioned, errors };
};
const seedRoleSkillDefaults = async () => {
const now = Date.now();
for (const [name, enabled] of Object.entries(DEFAULT_USER_SKILLS)) {
@@ -941,6 +975,7 @@ export function createUserAuth(pool, options = {}) {
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
now,
});
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, root);
}
await conn.commit();
ensureWorkspace(root);
@@ -2634,6 +2669,7 @@ export function createUserAuth(pool, options = {}) {
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
now,
});
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, workspaceRoot);
await conn.commit();
ensureWorkspace(workspaceRoot);
ensureUserMemoryProfile(workspaceRoot, {
@@ -2940,6 +2976,7 @@ export function createUserAuth(pool, options = {}) {
getUserPublishLayout,
isPathAllowed,
repairAllUserPublishDirs,
ensureAllUserDataSpaces,
registerAgentSession,
getSessionNode,
getSessionTarget,
+26
View File
@@ -7,6 +7,32 @@ import test from 'node:test';
import { Algorithm as Argon2Algorithm, hashRawSync as argon2HashRawSync } from '@node-rs/argon2';
import { createUserAuth, clearUserLogoutCookies, userLoginCookies } from './user-auth.mjs';
test('ensureAllUserDataSpaces provisions every active non-admin user', async () => {
const calls = [];
const pool = {
async query(sql) {
if (sql.includes("WHERE role = 'user' AND status = 'active'")) {
return [[
{ id: 'user-1', workspace_root: '/tmp/user-1' },
{ id: 'user-2', workspace_root: '/tmp/user-2' },
]];
}
return [[]];
},
};
const auth = createUserAuth(pool, {
persistSessions: false,
provisionUserDataSpace: async (input) => calls.push(input),
});
const result = await auth.ensureAllUserDataSpaces();
assert.equal(result.provisioned, 2);
assert.deepEqual(result.errors, []);
assert.deepEqual(calls, [
{ userId: 'user-1', workspaceRoot: '/tmp/user-1' },
{ userId: 'user-2', workspaceRoot: '/tmp/user-2' },
]);
});
function hashPasswordPbkdf2(password, salt) {
return crypto.pbkdf2Sync(password, salt, 100000, 64, 'sha512').toString('hex');
}
+24
View File
@@ -1,6 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createPostgresUserDataSpaceService } from './postgres-user-data-space-service.mjs';
export const DATASET_REGISTRY_TABLE = '__page_data_datasets';
@@ -92,6 +93,23 @@ export function createUserDataSpaceService(options = {}) {
}
const userId = options.userId ? String(options.userId).trim() : null;
const isNodeTest = Boolean(process.env.NODE_TEST_CONTEXT);
const backend = String(
options.backend ?? process.env.MINDSPACE_USERDATA_BACKEND ?? (isNodeTest ? 'sqlite' : 'postgres'),
).trim().toLowerCase();
if (backend === 'sqlite' && !isNodeTest) {
throw Object.assign(
new Error('用户私有数据运行时已禁用 SQLite,只允许使用 PostgreSQL'),
{ code: 'sqlite_runtime_disabled' },
);
}
if (backend !== 'postgres' && backend !== 'sqlite') {
throw Object.assign(new Error(`不支持的用户数据后端:${backend}`), { code: 'invalid_userdata_backend' });
}
if (backend === 'postgres') {
if (!userId) throw new Error('PostgreSQL 用户数据空间要求 userId');
return createPostgresUserDataSpaceService({ ...options, userId });
}
const query = typeof options.query === 'function' ? options.query : null;
const sqliteBin = options.sqliteBin?.trim() || process.env.SQLITE_BIN?.trim() || 'sqlite3';
const maxBytes = Number(options.maxBytes ?? process.env.PRIVATE_DATA_MAX_BYTES ?? DEFAULT_MAX_BYTES);
@@ -111,6 +129,11 @@ export function createUserDataSpaceService(options = {}) {
return privateDataDb;
}
function ensureReady() {
ensurePrivateDataDb();
return getInfo();
}
function privateDataSize() {
let total = 0;
for (const file of fs.existsSync(privateDataDir) ? fs.readdirSync(privateDataDir) : []) {
@@ -750,6 +773,7 @@ export function createUserDataSpaceService(options = {}) {
workspaceRoot,
privateDataDir,
privateDataDb,
ensureReady,
ensurePrivateDataDb,
privateDataSize,
getInfo,
+21
View File
@@ -3,8 +3,29 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { spawnSync } from 'node:child_process';
import { createUserDataSpaceService } from './user-data-space-service.mjs';
test('runtime rejects SQLite user data backend outside node tests', () => {
const script = `
import { createUserDataSpaceService } from './user-data-space-service.mjs';
try {
createUserDataSpaceService({ workspaceRoot: '/tmp/runtime-sqlite-denied', backend: 'sqlite' });
process.exit(0);
} catch (error) {
console.error(error.code);
process.exit(7);
}
`;
const result = spawnSync(process.execPath, ['--input-type=module', '--eval', script], {
cwd: process.cwd(),
encoding: 'utf8',
env: { ...process.env, NODE_TEST_CONTEXT: '' },
});
assert.equal(result.status, 7);
assert.match(result.stderr, /sqlite_runtime_disabled/);
});
function createTempWorkspace() {
return fs.mkdtempSync(path.join(os.tmpdir(), 'user-data-space-'));
}
+16
View File
@@ -152,6 +152,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报
6. 生成 HTML 后,向用户提供可访问链接,格式:
\`${publicBaseUrl}/${PUBLISH_ROOT_DIR}/${slug}/public/<文件名>\`
(写入 \`public/页面.html\` 时 URL **必须**含 \`public/\`;仅当 HTML 直接写在 workspace 根目录时才省略)
7. **只要页面接收用户输入并要求以后查看、汇总、统计、修改或删除,就不是纯静态页**。禁止使用 \`localStorage\` / \`sessionStorage\` / \`IndexedDB\` 保存任何数据;必须立即 \`load_skill\` → \`page-data-collect\`,通过 Page Data API 写入当前用户专属 PostgreSQL schema。禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化。记账、日记、打卡、清单、台账、问卷、报名及其管理页都适用。
## 推荐工作流
@@ -298,6 +299,13 @@ ${renderBrandingBlock(addressName)}
- 有 \`shell\` 时用 \`ls oa/\`、\`find .\`、\`cat oa/file.csv\`;有 \`tree\` 时用 \`tree oa/\`
- 不要说「我没有 shell」——若系统提示已列出 shell,就必须用 shell 在工作区内操作
## 生成 / 发布 HTML 页面
- 用户要网页、交互应用或分享链接时,必须使用 对应 skill 并将最终 HTML 写入 \`public/*.html\`
- 若页面接收用户输入并需要保存、查询、统计、修改或删除,必须使用 \`page-data-collect\` + Page Data API,禁止 localStorage / sessionStorage / IndexedDB
- \`apps__create_app\` 只能用于设计预览,不是 H5 交付物;使用后仍必须写入 \`public/*.html\` 并返回真实可访问链接
- 未验证文件已落盘且链接可访问前,禁止声称「已创建」「已打开」或「可在 Apps 标签找到」
## Word 下载页
- 若用户要求页面里可下载 Word / docx:必须先 \`load_skill\` → \`docx-generate\`
@@ -322,6 +330,12 @@ export function buildSandboxSessionConstraints({ baseConstraints, developerTools
const lines = [
baseConstraints,
'',
'## 当前用户专属 PostgreSQL 数据空间(硬性)',
'- 当前 session 已绑定当前用户身份;`private_data_*` 工具只连接该用户在新 PG 中的独立 schema,禁止跨用户访问',
'- 任何需要保存、再次读取、统计、汇总、筛选、修改或删除的数据都必须使用 `private_data_*` 建表/注册 dataset,并由 HTML 通过 Page Data API 访问',
'- Agent 生成的 HTML/JavaScript 禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 保存数据;write_file/edit_file 会直接拒绝此类代码',
'- 不允许 SQLite,也不允许在浏览器存储、静态 JSON/JS 文件中伪装持久化业务数据;失败时必须修复 PG/Page Data 链路,不能降级',
'',
'## 当前会话可用文件工具',
`- ${tools.join(', ')}`,
hasSandboxMcp
@@ -385,6 +399,8 @@ export function buildPublishConstraints({ slug, username, publicBaseUrl, publish
'- **查找文件**:在工作区内对应分区搜索(如 `oa/2025-12-06T13-34_export.csv`),不要搜其它用户目录或公网 URL',
'- **禁止**:访问 `assets/` 内部路径、其它用户目录、主机绝对路径;禁止用公网 URL 列目录或读 CSV',
'- **路径规则**:只用相对路径;禁止 `../`;工作区外的路径会被系统拒绝(OS 层强制,非软约束)',
'- **当前用户专属 PG**:本 session 的 `private_data_*` 工具已绑定当前用户,只能访问该用户在新 PostgreSQL 中的独立 schema;任何需持久化、再次读取、统计、汇总、修改或删除的数据都必须走 PG + Page Data API',
'- **浏览器存储硬禁令**:Agent 生成的 HTML/JavaScript 禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 保存数据;禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化',
'- **生成页面(必须亲自完成)**:先 `load_skill` → `static-page-publish`,再用 `write_file`/`edit_file` 写入 `public/页面.html`',
'- **默认只生成 HTML**:不要在没有明确需求时强制生成 Word、PDF、长图等伴生文件',
'- **Word 下载页(按需亲自完成)**:若用户明确要求 Word / docx 下载,必须先 `load_skill` → `docx-generate`,生成并确认 `public/*.docx` 已存在,再在 HTML 中用相对路径链接',
+11
View File
@@ -85,6 +85,10 @@ test('publish dir and public url use stable user id', () => {
assert.match(skillText, /docx-generate/);
assert.match(skillText, /long-image-download/);
assert.match(skillText, /generate_long_image/);
assert.match(skillText, /禁止使用 `localStorage`/);
assert.match(skillText, /IndexedDB/);
assert.match(skillText, /PostgreSQL schema/);
assert.match(skillText, /page-data-collect/);
assert.match(docxSkillText, /public\/文件名\.docx/);
assert.match(longImageSkillText, /public\/report\.long\.png/);
assert.match(skillText, /report\.docx|伴生下载/);
@@ -137,6 +141,10 @@ test('buildSandboxSessionConstraints documents shell and forbids public url brow
assert.match(text, /docx-generate/);
assert.match(text, /public\/\*\.docx/);
assert.match(text, /\/public\//);
assert.match(text, /当前用户专属 PostgreSQL 数据空间/);
assert.match(text, /localStorage/);
assert.match(text, /IndexedDB/);
assert.match(text, /write_file\/edit_file 会直接拒绝/);
});
test('renderWorkspaceHints includes docx download guidance for published pages', () => {
@@ -150,6 +158,9 @@ test('renderWorkspaceHints includes docx download guidance for published pages',
assert.match(text, /public\/\*\.docx/);
assert.match(text, /long-image-download/);
assert.match(text, /generate_long_image/);
assert.match(text, /apps__create_app/);
assert.match(text, /Page Data API/);
assert.match(text, /未验证文件已落盘/);
});
test('isPathInsidePublishDir blocks escape', () => {