From a6620fb7193d818f748da83698abf471fbbda41e Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 15:29:29 +0800 Subject: [PATCH] feat(mindspace): enforce PostgreSQL user data delivery --- .env.example | 29 +- agent-run-deliverable-check.mjs | 41 +- agent-run-deliverable-check.test.mjs | 34 ++ agent-run-gateway.mjs | 61 ++- agent-run-gateway.test.mjs | 266 ++++++++++- capabilities.mjs | 2 +- capabilities.test.mjs | 10 + chat-intent-router.mjs | 10 +- chat-intent-router.test.mjs | 32 +- chat-skills.mjs | 25 +- chat-skills.test.mjs | 27 ++ docs/local-dev.md | 30 ++ ...pace-browser-storage-pg-migration-local.md | 40 ++ mindspace-browser-storage-policy.mjs | 60 +++ mindspace-h5-html-finish-guard.mjs | 42 +- mindspace-h5-html-finish-guard.test.mjs | 55 +++ mindspace-page-data-finish-guard.mjs | 45 +- mindspace-page-data-finish-guard.test.mjs | 52 ++- mindspace-page-sync-service.mjs | 15 +- mindspace-page-sync.mjs | 11 +- mindspace-page-sync.test.mjs | 34 ++ mindspace-public-finish-sync.mjs | 1 + mindspace-public-finish-sync.test.mjs | 1 + mindspace-public-route.test.mjs | 29 ++ mindspace-sandbox-mcp.mjs | 13 +- mindspace-sandbox-mcp.test.mjs | 35 ++ mindspace-userdata-postgres.mjs | 439 ++++++++++++++++++ mindspace-userdata-postgres.test.mjs | 106 +++++ ...-workspace-page-deliver-page-data.test.mjs | 9 +- mindspace-workspace-page-deliver.mjs | 14 +- mindspace-workspace-sync.mjs | 21 +- mindspace-workspace-sync.test.mjs | 6 +- page-data-delivery-assess.mjs | 54 ++- page-data-delivery-assess.test.mjs | 38 ++ page-data-public-service.mjs | 38 +- page-data-service.mjs | 9 +- page-data-workspace-bind.mjs | 35 +- page-data-workspace-ensure.mjs | 2 +- page-data-workspace-ensure.test.mjs | 8 +- postgres-user-data-space-service.mjs | 391 ++++++++++++++++ scripts/dev-core-daemon.mjs | 25 +- scripts/dev-core.mjs | 23 +- scripts/load-env.mjs | 23 +- scripts/memind-runtime-profile.mjs | 119 +++++ scripts/memind-runtime-profile.test.mjs | 54 +++ .../migrate-mindspace-sqlite-to-postgres.mjs | 85 ++++ ...rate-mindspace-sqlite-to-postgres.test.mjs | 25 + scripts/verify-local-pg-questionnaire-e2e.mjs | 198 ++++++++ server.mjs | 62 ++- skills/page-data-collect/SKILL.md | 23 +- skills/static-page-publish/SKILL.md | 1 + user-auth.mjs | 37 ++ user-auth.test.mjs | 26 ++ user-data-space-service.mjs | 24 + user-data-space-service.test.mjs | 21 + user-publish.mjs | 16 + user-publish.test.mjs | 11 + 57 files changed, 2779 insertions(+), 164 deletions(-) create mode 100644 docs/mindspace-browser-storage-pg-migration-local.md create mode 100644 mindspace-browser-storage-policy.mjs create mode 100644 mindspace-userdata-postgres.mjs create mode 100644 mindspace-userdata-postgres.test.mjs create mode 100644 postgres-user-data-space-service.mjs create mode 100644 scripts/memind-runtime-profile.mjs create mode 100644 scripts/memind-runtime-profile.test.mjs create mode 100644 scripts/migrate-mindspace-sqlite-to-postgres.mjs create mode 100644 scripts/migrate-mindspace-sqlite-to-postgres.test.mjs create mode 100644 scripts/verify-local-pg-questionnaire-e2e.mjs diff --git a/.env.example b/.env.example index 27f31d1..6d9c654 100644 --- a/.env.example +++ b/.env.example @@ -242,4 +242,31 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind # MINDSPACE_AGENT_JOBS_ENABLED=true # MINDSPACE_INTERNAL_AGENT_SECRET=local-dev-secret # MINDSPACE_FREE_PUBLIC_PAGE_LIMIT=10 -# MINDSPACE_STORAGE_ROOT=/path/to/mindspace-storage + +# --------------------------------------------------------------------------- +# Memind runtime profile（本地 vs 生产 必须区分） +# 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。 +# +# | profile | 场景 | MindSpace 数据路径 | +# |----------------|-------------------|--------------------| +# | local | 本机日常开发 | /data/mindspace | +# | split-service | 本机验证 103 拆分 | /Users/john/MindSpace/data/mindspace | +# | production | 103 生产服务器 | 103 .env 为准，勿提交 Git | +# --------------------------------------------------------------------------- + +# 【本地开发 — 复制到 .env 的默认值】 +MEMIND_RUNTIME_PROFILE=local +# MINDSPACE_STORAGE_ROOT 留空即可，自动使用 /data/mindspace + +# 【本机 split-service 联调 — 需要独立 MindSpace LaunchAgent 已启动】 +# MEMIND_RUNTIME_PROFILE=split-service +# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +# MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret +# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace + +# 【103 生产 — 仅在服务器维护，示例勿照抄密钥】 +# MEMIND_RUNTIME_PROFILE=production +# MINDSPACE_SERVER_ADAPTER=remote +# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +# MINDSPACE_REMOTE_AUTH_TOKEN=<与 cn.tkmind.mindspace-service 一致> +# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace diff --git a/agent-run-deliverable-check.mjs b/agent-run-deliverable-check.mjs index dfe5685..eeba16c 100644 --- a/agent-run-deliverable-check.mjs +++ b/agent-run-deliverable-check.mjs @@ -26,6 +26,7 @@ function mapDeliverableRows(rows) { publicationId: row.publication_id ? String(row.publication_id) : null, publicationStatus: row.publication_status ? String(row.publication_status) : null, publicUrl: row.public_url ? String(row.public_url) : null, + workspaceRelativePath: row.workspace_relative_path ? String(row.workspace_relative_path) : null, })).filter((row) => row.pageId); const publicationCount = pages.filter((row) => row.publicationId).length; return { @@ -51,13 +52,16 @@ export function mergeDeliverableSummaries(...summaries) { }; } -export async function detectSessionDeliverables(pool, userId, sessionId) { +export async function detectSessionDeliverables(pool, userId, sessionId, { sinceMs = null } = {}) { if (!pool?.query || !userId || !sessionId) { return { pageCount: 0, publicationCount: 0, pages: [] }; } + const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : ''; + const params = sinceMs != null ? [userId, sessionId, sinceMs] : [userId, sessionId]; const [rows] = await pool.query( `SELECT p.id AS page_id, p.title, + p.workspace_relative_path, pr.id AS publication_id, pr.status AS publication_status, pr.public_url @@ -68,22 +72,39 @@ export async function detectSessionDeliverables(pool, userId, sessionId) { AND pr.status = 'online' WHERE p.user_id = ? AND p.source_session_id = ? + ${sinceClause} AND p.status <> 'deleted' ORDER BY p.created_at ASC`, - [userId, sessionId], + params, ); return mapDeliverableRows(rows); } -export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs = null } = {}) { +export async function detectWorkspaceSyncedDeliverables( + pool, + userId, + { sinceMs = null, onlyRelativePaths = null } = {}, +) { if (!pool?.query || !userId) { return { pageCount: 0, publicationCount: 0, pages: [] }; } + const normalizedPaths = onlyRelativePaths == null + ? null + : [...new Set(onlyRelativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))]; + if (normalizedPaths?.length === 0) { + return { pageCount: 0, publicationCount: 0, pages: [] }; + } const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : ''; - const params = sinceMs != null ? [userId, sinceMs] : [userId]; + const pathClause = normalizedPaths == null + ? '' + : `AND p.workspace_relative_path IN (${normalizedPaths.map(() => '?').join(', ')})`; + const params = [userId]; + if (sinceMs != null) params.push(sinceMs); + if (normalizedPaths != null) params.push(...normalizedPaths); const [rows] = await pool.query( `SELECT p.id AS page_id, p.title, + p.workspace_relative_path, pr.id AS publication_id, pr.status AS publication_status, pr.public_url @@ -96,6 +117,7 @@ export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs WHERE p.user_id = ? AND p.status <> 'deleted' ${sinceClause} + ${pathClause} AND ( p.workspace_relative_path LIKE 'public/%.html' OR JSON_UNQUOTE(JSON_EXTRACT(pv.source_snapshot_json, '$.relative_path')) LIKE 'public/%.html' @@ -116,15 +138,20 @@ export async function prepareAndDetectSessionDeliverables({ sessionId, runStartedAtMs = null, prepareDeliverables = null, + workspaceRelativePaths = null, } = {}) { + let prepared = null; if (typeof prepareDeliverables === 'function') { - await prepareDeliverables({ userId, sessionId }).catch(() => {}); + prepared = await prepareDeliverables({ userId, sessionId }).catch(() => null); } const sinceMs = runStartedAtMs == null ? null : Math.max(0, Number(runStartedAtMs) - WORKSPACE_DELIVERABLE_LOOKBACK_MS); - const bySession = await detectSessionDeliverables(pool, userId, sessionId); - const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs }); + const bySession = await detectSessionDeliverables(pool, userId, sessionId, { sinceMs }); + const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { + sinceMs, + onlyRelativePaths: workspaceRelativePaths ?? prepared?.pageDataRelativePaths ?? null, + }); return mergeDeliverableSummaries(bySession, byWorkspace); } diff --git a/agent-run-deliverable-check.test.mjs b/agent-run-deliverable-check.test.mjs index 8d3c03d..b335e63 100644 --- a/agent-run-deliverable-check.test.mjs +++ b/agent-run-deliverable-check.test.mjs @@ -98,6 +98,40 @@ test('detectWorkspaceSyncedDeliverables filters auto-synced public html pages', assert.equal(summary.pageCount, 1); }); +test('detectWorkspaceSyncedDeliverables stays inside current run artifact paths', async () => { + const pool = { + async query(sql, params) { + assert.match(sql, /workspace_relative_path IN \(\?, \?\)/); + assert.deepEqual(params, [ + 'user-1', + 1000, + 'public/current.html', + 'public/current-admin.html', + ]); + return [[{ + page_id: 'page-current', + title: 'Current', + workspace_relative_path: 'public/current.html', + }]]; + }, + }; + const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', { + sinceMs: 1000, + onlyRelativePaths: ['public/current.html', 'public/current-admin.html'], + }); + assert.equal(summary.pageCount, 1); + assert.equal(summary.pages[0].workspaceRelativePath, 'public/current.html'); +}); + +test('detectWorkspaceSyncedDeliverables skips workspace sweep for an empty run scope', async () => { + const pool = { async query() { throw new Error('must not query'); } }; + const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', { + sinceMs: 1000, + onlyRelativePaths: [], + }); + assert.equal(summary.pageCount, 0); +}); + test('prepareAndDetectSessionDeliverables merges session and workspace pages after prepare hook', async () => { const prepareCalls = []; const pool = { diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index b7d6fe3..e3a01dd 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -16,7 +16,7 @@ import { SESSION_FINISHED_STALE_GRACE_MS, tryRecoverRunFromDeliverables, } from './agent-run-deliverable-check.mjs'; -import { isPageDataIntent } from './chat-skills.mjs'; +import { isPageDataIntent, isPageGenerationIntent } from './chat-skills.mjs'; const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000]; const TERMINAL_STATUSES = new Set(['succeeded', 'failed']); @@ -262,6 +262,7 @@ export function createAgentRunGateway({ syncUserPagesOnSuccess = null, observePersonalMemoryOnSuccess = null, isSessionExternallyBusy = null, + validateRunDeliverables = null, retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS, autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true), maxConcurrentRuns = positiveInteger( @@ -420,7 +421,7 @@ export function createAgentRunGateway({ async function prepareRunDeliverables({ userId, sessionId, runId, runStartedAtMs = null }) { if (typeof syncUserPagesOnSuccess !== 'function') return; - await syncUserPagesOnSuccess({ + return await syncUserPagesOnSuccess({ userId, sessionId, runId, @@ -619,7 +620,7 @@ export function createAgentRunGateway({ billed: Boolean(result.billing?.ok), }); await appendRunSnapshot(runId); - return { sessionId: result.sessionId }; + return { sessionId: result.sessionId, routing }; } catch (err) { await appendEvent(runId, 'direct_chat_failed', { code: err?.code ?? null, @@ -673,7 +674,7 @@ export function createAgentRunGateway({ }); throw err; } - return { sessionId: row.agent_session_id ?? null }; + return { sessionId: row.agent_session_id ?? null, routing }; } let sessionId = resolveGatewayAgentSessionId({ @@ -769,7 +770,7 @@ export function createAgentRunGateway({ sessionId, err, })) { - return { sessionId }; + return { sessionId, routing }; } throw err; } @@ -785,16 +786,17 @@ export function createAgentRunGateway({ }, ); } - return { sessionId }; + return { sessionId, routing }; } - async function finalizeSuccessfulRun(runId, row, sessionId) { + async function finalizeSuccessfulRun(runId, row, sessionId, { routing = null } = {}) { let deliveryResult = null; if (typeof syncUserPagesOnSuccess === 'function') { deliveryResult = await syncUserPagesOnSuccess({ userId: row.user_id, sessionId, runId, + runStartedAtMs: row.started_at ?? null, }); } const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors) @@ -806,17 +808,48 @@ export function createAgentRunGateway({ error.retryable = false; throw error; } - if (isPageDataIntent(extractRunMessageText(row))) { + const runMessageText = extractRunMessageText(row); + const pageDataIntent = isPageDataIntent(runMessageText) + || routing?.suggestedSkill === 'page-data-collect'; + const pageGenerationIntent = isPageGenerationIntent(runMessageText) + || routing?.suggestedSkill === 'static-page-publish'; + const requiresPageDeliverable = pageDataIntent || pageGenerationIntent; + let deliverables = null; + if (requiresPageDeliverable || typeof validateRunDeliverables === 'function') { const latest = await getRunById(runId); - const deliverables = await prepareAndDetectSessionDeliverables({ + deliverables = await prepareAndDetectSessionDeliverables({ pool, userId: row.user_id, sessionId, runStartedAtMs: latest?.started_at ?? row.started_at ?? null, + workspaceRelativePaths: deliveryResult?.pageDataRelativePaths ?? [], }); - if (deliverables.pageCount < 1) { - const error = new Error('Page Data 任务未生成可交付页面，不能标记成功'); - error.code = 'PAGE_DATA_DELIVERABLE_MISSING'; + if (requiresPageDeliverable && deliverables.pageCount < 1) { + const error = new Error( + pageDataIntent + ? 'Page Data 任务未生成可交付页面，不能标记成功' + : '页面任务未生成 public HTML 交付物，不能标记成功', + ); + error.code = pageDataIntent + ? 'PAGE_DATA_DELIVERABLE_MISSING' + : 'PUBLIC_PAGE_DELIVERABLE_MISSING'; + error.retryable = false; + throw error; + } + } + if (typeof validateRunDeliverables === 'function') { + const validation = await validateRunDeliverables({ + userId: row.user_id, + sessionId, + runId, + deliverables: deliverables ?? { pageCount: 0, publicationCount: 0, pages: [] }, + }); + const errors = Array.isArray(validation?.errors) ? validation.errors : []; + if (errors.length > 0) { + const error = new Error( + `页面交付违反数据存储策略：${errors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`, + ); + error.code = 'DELIVERABLE_DATA_STORAGE_FORBIDDEN'; error.retryable = false; throw error; } @@ -856,8 +889,8 @@ export function createAgentRunGateway({ const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt }); try { - const { sessionId } = await runWithTimeout(runId, () => executeRun(row, runId)); - await finalizeSuccessfulRun(runId, row, sessionId); + const { sessionId, routing } = await runWithTimeout(runId, () => executeRun(row, runId)); + await finalizeSuccessfulRun(runId, row, sessionId, { routing }); } catch (err) { const latest = await getRunById(runId); const recoverySessionId = latest?.agent_session_id ?? row.agent_session_id ?? null; diff --git a/agent-run-gateway.test.mjs b/agent-run-gateway.test.mjs index dc4872b..d426ee7 100644 --- a/agent-run-gateway.test.mjs +++ b/agent-run-gateway.test.mjs @@ -387,6 +387,191 @@ test('agent run awaits session Finish before succeeding when proxy supports it', assert.equal(finishEvents.length, 1); }); +test('Page Data run fails closed when Finish arrives without a generated page', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-page-data-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-page-data-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我创建调查问卷，保存提交记录并发布页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /未生成可交付页面/); +}); + +test('implicit sticky-note app run fails closed when Apps returns Finish without a public page', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + chatIntentRouter: { + isEnabled() { + return true; + }, + async classify() { + return { + route: 'agent_orchestration', + confidence: 0.96, + reason: '页面需要数据交互与持久化', + suggestedSkill: 'page-data-collect', + source: 'rule', + }; + }, + applyAgentOrchestration(message) { + return message; + }, + }, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-sticky-note-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-sticky-note-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我设计一个便签提醒，可以写便签提交，时间轴来显示' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /Page Data 任务未生成可交付页面/); + assert.ok(pool.events.some((event) => event.eventType === 'intent_routed')); +}); + +test('static page run fails closed when Finish arrives without public HTML', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-public-page-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({}), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-public-page-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我做一个秋夜诗的 H5 页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /public HTML 交付物/); +}); + +test('Page Data run succeeds only after a generated session page is detected', async () => { + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:session-page-data-ready': [{ + page_id: 'page-ready', + title: '问卷', + publication_id: 'pub-ready', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/u/john/pages/page-ready', + }], + }, + }); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-page-data-ready' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-page-data-ready', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '创建一个可以保存提交记录的问卷' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded'); +}); + +test('agent run fails closed when a generated page violates browser storage policy', async () => { + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:session-browser-storage': [{ + page_id: 'page-storage', + title: '页面', + workspace_relative_path: 'public/page.html', + }], + }, + }); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-browser-storage' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + validateRunDeliverables: async ({ deliverables }) => ({ + errors: deliverables.pages.some((page) => page.workspaceRelativePath === 'public/page.html') + ? [{ code: 'browser_storage_forbidden', message: 'public/page.html 使用 localStorage' }] + : [], + }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-browser-storage', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我做一个展示页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /页面交付违反数据存储策略/); + assert.match(pool.runs.get(run.id).error_message, /localStorage/); +}); + test('agent run succeeds when Finish is missing but session pages were already created', async () => { const pool = createFakePool({ sessionDeliverables: { @@ -563,7 +748,17 @@ test('agent run uses direct chat on regular agent sessions when llm routes direc }); test('agent run invalidates portal direct chat snapshot before submitting to goosed', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:20260704_31': [{ + page_id: 'page-essay', + title: '散文页面', + publication_id: 'pub-essay', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/essay.html', + }], + }, + }); const submitted = []; const invalidated = []; const gateway = createAgentRunGateway({ @@ -723,7 +918,17 @@ test('agent run falls back to backend session when router is disabled', async () }); test('agent run uses chat intent router to enrich agent orchestration messages', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:agent-session-1': [{ + page_id: 'page-router-agent', + title: '路由页面', + publication_id: 'pub-router-agent', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/router.html', + }], + }, + }); const submitted = []; const gateway = createAgentRunGateway({ pool, @@ -790,7 +995,17 @@ test('agent run uses chat intent router to enrich agent orchestration messages', }); test('agent run escalates direct sessions to a new backend session when forced', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:deep-session-1': [{ + page_id: 'page-deep', + title: '深度页面', + publication_id: 'pub-deep', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html', + }], + }, + }); const submitted = []; const gateway = createAgentRunGateway({ pool, @@ -820,7 +1035,17 @@ test('agent run escalates direct sessions to a new backend session when forced', }); test('agent run persists direct session transcript before escalating to goosed', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:deep-session-1': [{ + page_id: 'page-deep-transcript', + title: '深度页面', + publication_id: 'pub-deep-transcript', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html', + }], + }, + }); const submitted = []; const saved = []; const removed = []; @@ -914,7 +1139,17 @@ test('agent run rejects reused goosed session when broker ownership check fails' }); test('agent run persists portal direct snapshot before goosed submit on same session', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:20260705_2': [{ + page_id: 'page-report', + title: '报告页面', + publication_id: 'pub-report', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/report.html', + }], + }, + }); const submitted = []; const saved = []; const gateway = createAgentRunGateway({ @@ -1792,6 +2027,27 @@ test('createRun rejects with SESSION_RUN_CONFLICT when same session already has assert.equal(run3.requestId, 'req-conflict-3'); }); +test('createRun rejects while the same session is finishing page delivery', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: {}, + autoDispatch: false, + isSessionExternallyBusy: ({ sessionId }) => sessionId === 'sess-repairing', + }); + + await assert.rejects( + gateway.createRun('user-1', { + sessionId: 'sess-repairing', + requestId: 'req-during-repair', + userMessage: { role: 'user', content: [{ type: 'text', text: '继续' }] }, + }), + (err) => err?.code === 'SESSION_RUN_CONFLICT' && err?.status === 409 && /自动修复/.test(err.message), + ); + assert.equal(pool.runs.size, 0); +}); + test('createRun does not apply per-session conflict check for direct-chat sessions', async () => { const pool = createFakePool(); const gateway = createAgentRunGateway({ diff --git a/capabilities.mjs b/capabilities.mjs index f3c722a..d400b2e 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -161,7 +161,7 @@ export const CAPABILITY_CATALOG = [ ]; /** Capabilities that must never be enabled for regular users, even via DB overrides. */ -export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin']); +export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin', 'apps']); export function clampUserCapabilities(capabilities) { const clamped = { ...capabilities }; diff --git a/capabilities.test.mjs b/capabilities.test.mjs index 2b1dd1d..1e4966a 100644 --- a/capabilities.test.mjs +++ b/capabilities.test.mjs @@ -164,6 +164,16 @@ test('clampUserCapabilities blocks extension_admin even when stored as true', () assert.equal(clamped.shell, true); }); +test('clampUserCapabilities blocks Apps for regular H5 users even when granted by role', () => { + const clamped = clampUserCapabilities({ + ...DEFAULT_USER_CAPABILITIES, + apps: true, + }); + assert.equal(clamped.apps, false); + const policy = buildAgentExtensionPolicy(clamped); + assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'apps'), false); +}); + test('catalog keys are unique', () => { const keys = CAPABILITY_CATALOG.map((item) => item.key); assert.equal(keys.length, new Set(keys).size); diff --git a/chat-intent-router.mjs b/chat-intent-router.mjs index d8e9a8b..e9acdff 100644 --- a/chat-intent-router.mjs +++ b/chat-intent-router.mjs @@ -888,7 +888,10 @@ export function createChatIntentRouter(options = {}) { function getStatus() { return { - enabled: Boolean(policy.enabled), + enabled: true, + ruleRoutingEnabled: true, + llmRoutingEnabled: false, + configuredLlmEnabled: Boolean(policy.enabled), modelProviderKeyId: policy.modelProviderKeyId ?? null, model: policy.model ?? null, modelApiType: policy.modelApiType ?? 'chat', @@ -903,8 +906,9 @@ export function createChatIntentRouter(options = {}) { } function isEnabled() { - // Skill + rule routing only; LLM intent router is intentionally disabled. - return false; + // Rule + skill routing is always active. The LLM classifier stays disabled + // independently; returning false here would make the gateway skip rules too. + return true; } async function resolveRouterContext({ userId, sessionId, text, forceDeepReasoning = false }) { diff --git a/chat-intent-router.test.mjs b/chat-intent-router.test.mjs index 374cea7..1cbdc30 100644 --- a/chat-intent-router.test.mjs +++ b/chat-intent-router.test.mjs @@ -67,6 +67,34 @@ test('classifyWithRules routes page data collection to page-data-collect', () => assert.equal(result.suggestedSkill, 'page-data-collect'); }); +test('classifyWithRules routes ordinary household ledger language to page-data-collect', () => { + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + const result = classifyWithRules({ + text, + userMessage: { + role: 'user', + content: [{ type: 'text', text }], + metadata: { displayText: text }, + }, + }); + assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT); + assert.equal(result.suggestedSkill, 'page-data-collect'); +}); + +test('classifyWithRules routes implicit sticky-note timeline apps to page-data-collect', () => { + const text = '帮我设计一个便签提醒，可以写便签提交，时间轴来显示'; + const result = classifyWithRules({ + text, + userMessage: { + role: 'user', + content: [{ type: 'text', text }], + metadata: { displayText: text }, + }, + }); + assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT); + assert.equal(result.suggestedSkill, 'page-data-collect'); +}); + test('coercePageDataSkill overrides static-page-publish when data intent is present', () => { const text = '帮我在这个页面增加调查问卷，密码 888 查看提交记录'; const coerced = coercePageDataSkill( @@ -579,9 +607,9 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only' }, }); - assert.equal(await router.isEnabled(), false); + assert.equal(await router.isEnabled(), true); states.shift(); - assert.equal(await router.isEnabled(), false); + assert.equal(await router.isEnabled(), true); const result = await router.classify({ userMessage: { role: 'user', diff --git a/chat-skills.mjs b/chat-skills.mjs index 7d49846..4fa3bd3 100644 --- a/chat-skills.mjs +++ b/chat-skills.mjs @@ -43,10 +43,31 @@ const PAGE_DATA_INTENT_PATTERNS = [ /(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u, ]; +const INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN = /(?:便签|便利贴|备忘录|待办|清单)/u; +const INTERACTIVE_PAGE_DATA_SURFACE_PATTERN = /(?:应用|app|页面|网页|H5|h5|HTML|html|时间轴|看板)/u; +const INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS = [ + /(?:写|填写|提交|保存)/u, + /(?:添加|新增|创建)/u, + /(?:编辑|修改)/u, + /(?:删除|完成)/u, + /(?:筛选|统计|汇总|管理)/u, + /(?:提醒时间|优先级|分类标签|时间轴)/u, +]; + +function isInteractivePageDataIntent(text) { + if (!INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN.test(text)) return false; + const featureCount = INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS.reduce( + (count, pattern) => count + (pattern.test(text) ? 1 : 0), + 0, + ); + return INTERACTIVE_PAGE_DATA_SURFACE_PATTERN.test(text) || featureCount >= 2; +} + export function isPageDataIntent(text) { const normalized = String(text ?? '').trim(); if (!normalized) return false; - return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized)); + return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized)) + || isInteractivePageDataIntent(normalized); } export function isPageGenerationIntent(text) { @@ -161,7 +182,7 @@ export function buildChatSkillPrompt(promptKey, skillName) { `请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能：在 MindSpace 页面中实现可提交、可持久化的数据收集（问卷/报名/台账等），必须使用 Page Data API。` + '先匹配技能内能力分支（默认 A：匿名前台 public insert + 独立后台 password read，口令默认 88888888）；“帮我做/创建”本身就是创建授权，展示简短方案摘要后必须同一轮继续执行，只有互斥需求或非法口令才追问。' + '流程：load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html（含 /assets/page-data-client.js）→ private_data_bind_workspace_page 发布并写策略。' + - '禁止 localStorage / 浏览器本地存储 fallback；禁止自建 Express/独立端口（如 8899）、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl（/MindSpace/<用户ID>/public/...），禁止给用户 /u/用户名/pages/... 链接，并说明后台入口与口令。' + '当前 session 的 private_data_* 已绑定当前用户专属 PostgreSQL schema；所有需保存或再次读取的数据必须走 PG + Page Data API。禁止 localStorage、sessionStorage、IndexedDB、SQLite、静态 JSON/JS 或内存持久化 fallback；禁止自建 Express/独立端口（如 8899）、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl（/MindSpace/<用户ID>/public/...），禁止给用户 /u/用户名/pages/... 链接，并说明后台入口与口令。' ); case 'service-integration-smoke': return `请使用 ${skillName ?? 'service-integration-smoke'} 技能：按标准联调流程检查当前服务，覆盖身份、普通聊天、记忆读取，以及我本轮明确要求验证的技能/发布链路，并输出通过项、失败项、待确认项：`; diff --git a/chat-skills.test.mjs b/chat-skills.test.mjs index 4ad38f6..8dfe9ca 100644 --- a/chat-skills.test.mjs +++ b/chat-skills.test.mjs @@ -107,9 +107,36 @@ test('buildAutoChatSkillPrefix prefers page-data-collect for survey requests', ( test('isPageDataIntent matches survey and backend keywords', () => { assert.equal(isPageDataIntent('增加调查问卷和后台入口，密码 888'), true); + assert.equal( + isPageDataIntent('帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'), + true, + ); assert.equal(isPageDataIntent('帮我做一个苏州攻略页面'), false); }); +test('isPageDataIntent matches implicit interactive sticky-note app requests', () => { + const text = '帮我设计一个便签提醒，可以写便签提交，时间轴来显示'; + assert.equal(isPageDataIntent(text), true); + assert.match( + buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']), + /page-data-collect/, + ); +}); + +test('isPageDataIntent does not steal a simple schedule reminder', () => { + assert.equal(isPageDataIntent('明天上午9点提醒我开会'), false); + assert.equal(isPageDataIntent('创建一个待办'), false); +}); + +test('buildAutoChatSkillPrefix routes ordinary household ledger language to Page Data', () => { + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + const prefix = buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']); + assert.match(prefix, /page-data-collect/); + assert.match(prefix, /同一轮继续执行/); + assert.match(prefix, /禁止 localStorage/); + assert.doesNotMatch(prefix, /static-page-publish 技能/); +}); + test('buildAutoChatSkillPrefix enables publish for implicit page requests', () => { const prefix = buildAutoChatSkillPrefix('苏州攻略页面', ['static-page-publish']); assert.match(prefix, /static-page-publish/); diff --git a/docs/local-dev.md b/docs/local-dev.md index 6016038..b712856 100644 --- a/docs/local-dev.md +++ b/docs/local-dev.md @@ -63,6 +63,36 @@ pnpm dev:all | `PLAZA_APP_DIR` | ../memind_plaza | Plaza 专用脚本的源码路径（`pnpm dev:plaza` / `pnpm start:plaza` / `pnpm dev:all`） | | `MEMIND_SESSION_BROKER_ENABLED` | `0`（未设置） | Session Broker 灰度开关；Patch 2+ 本地验证时可设 `1`，见 [H5 Session 架构](./h5-session-architecture-20260706.md) | +### MindSpace：本地 vs 生产（必须区分） + +`pnpm dev` / `server.mjs` 启动时会读取 `MEMIND_RUNTIME_PROFILE`（见 `scripts/memind-runtime-profile.mjs`）： + +| `MEMIND_RUNTIME_PROFILE` | 用途 | MindSpace 实现 | +|--------------------------|------|----------------| +| `local`（**默认**） | 本机日常开发 | Portal 内置 `MINDSPACE_SERVER_ADAPTER=local` | +| `split-service` | 本机验证 103 拆分架构 | `remote` → `http://127.0.0.1:8082` 独立 MindSpace 服务 | +| `production` | **仅 103 服务器** | 不覆盖 `.env`，以服务器配置为准 | + +**本地 `.env` 推荐：** + +```bash +MEMIND_RUNTIME_PROFILE=local +``` + +**本机要测 split-service 时：** + +```bash +MEMIND_RUNTIME_PROFILE=split-service +MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret # 必须与 MindSpace LaunchAgent 一致 +launchctl kickstart -k "gui/$(id -u)/cn.tkmind.mindspace-service" +pnpm dev +``` + +启动后 Portal 日志应出现：`[Portal] Runtime profile: MEMIND_RUNTIME_PROFILE=local, MINDSPACE_SERVER_ADAPTER=local, ...` + +**禁止：** 把 103 生产 `.env`、RDS 连接串、`MINDSPACE_REMOTE_AUTH_TOKEN` 生产值复制进本机 Git 仓库。 + ### H5 Session 架构本地全量联调（`0706-bug干净` 分支） 在 **本机 `.env`**（勿提交 Git）打开下列开关后重启 `pnpm dev`： diff --git a/docs/mindspace-browser-storage-pg-migration-local.md b/docs/mindspace-browser-storage-pg-migration-local.md new file mode 100644 index 0000000..7bb6853 --- /dev/null +++ b/docs/mindspace-browser-storage-pg-migration-local.md @@ -0,0 +1,40 @@ +# MindSpace 浏览器存储迁移清单（仅本机） + +更新时间：2026-07-13 + +## 当前策略 + +- 新建或修改的 Agent 页面禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 持久化数据。 +- 新 session 明确绑定当前用户的独立 PostgreSQL schema；持久化数据必须经 `private_data_*` 和 Page Data API。 +- 历史页面迁移期间继续可访问，不做全局 409 阻断。 +- 页面迁移并验证完成后，才考虑启用历史页面全局阻断。 + +## 待迁移页面 + +| 用户 ID | 页面 | 初步类型 | 状态 | +|---|---|---|---| +| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment.html` | 预约业务数据 | 待迁移 | +| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment-admin.html` | 预约管理数据 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger.html` | 家庭记账数据 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger-manage.html` | 家庭记账管理 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/index.html` | 工作区首页状态 | 待分类 | +| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/oa-system.html` | OA 业务数据 | 待迁移 | +| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/decision-comparison.html` | 决策工具状态 | 待分类 | +| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/daily-admin.html` | 日报管理数据 | 待迁移 | +| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/sales-register.html` | 销售登记数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey.html` | 调查数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey-admin.html` | 调查管理数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/survey-youth.html` | 调查数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/daily-report-system.html` | 日报业务数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/weather-calendar.html` | 工具偏好或业务数据 | 待分类 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/runner-game.html` | 游戏存档 | 待决定 PG 或取消持久化 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/tetris-game.html` | 游戏存档 | 待决定 PG 或取消持久化 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/bird-key-fly.html` | 游戏存档 | 待决定 PG 或取消持久化 | + +## 单页迁移完成条件 + +1. 建表位于页面所有者的独立 PG schema。 +2. dataset 注册完成，页面 policy 权限符合前台/管理页用途。 +3. HTML 只通过 Page Data API 访问数据，不包含浏览器持久化 fallback。 +4. 实际提交、刷新后读取、PG 直接查询三者结果一致。 +5. 原页面链接继续可用，管理页鉴权和跨用户隔离验证通过。 diff --git a/mindspace-browser-storage-policy.mjs b/mindspace-browser-storage-policy.mjs new file mode 100644 index 0000000..0c528fb --- /dev/null +++ b/mindspace-browser-storage-policy.mjs @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const AGENT_PAGE_CODE_EXTENSIONS = new Set(['.html', '.htm', '.js', '.mjs', '.jsx', '.ts', '.tsx']); +const PROHIBITED_BROWSER_STORAGE_PATTERNS = [ + { api: 'localStorage', pattern: /\blocalStorage\b/i }, + { api: 'sessionStorage', pattern: /\bsessionStorage\b/i }, + { api: 'IndexedDB', pattern: /\b(?:indexedDB|IDBDatabase|IDBObjectStore|IDBTransaction)\b/i }, +]; + +export function isAgentPageCodePath(relativePath) { + return AGENT_PAGE_CODE_EXTENSIONS.has(path.extname(String(relativePath ?? '')).toLowerCase()); +} + +export function detectProhibitedBrowserStorage(content, { relativePath = '' } = {}) { + if (!isAgentPageCodePath(relativePath)) return []; + let source = String(content ?? ''); + // Fold adjacent string literals so computed-property obfuscation such as + // window['local' + 'Storage'] cannot bypass the delivery policy. + for (let pass = 0; pass < 8; pass += 1) { + const folded = source.replace( + /(['"])([^'"\\]*)\1\s*\+\s*(['"])([^'"\\]*)\3/g, + (_match, quote, left, _rightQuote, right) => `${quote}${left}${right}${quote}`, + ); + if (folded === source) break; + source = folded; + } + return PROHIBITED_BROWSER_STORAGE_PATTERNS + .filter(({ pattern }) => pattern.test(source)) + .map(({ api }) => api); +} + +export function assertNoProhibitedBrowserStorage(content, { relativePath = '' } = {}) { + const APIs = detectProhibitedBrowserStorage(content, { relativePath }); + if (APIs.length === 0) return; + const error = new Error( + `${relativePath || '页面代码'} 禁止使用 ${APIs.join(', ')} 保存数据。` + + '用户业务数据必须通过 private_data_* 工具写入该用户专属 PostgreSQL schema，' + + 'HTML 必须通过 Page Data API 访问；请改写后再保存。', + ); + error.code = 'BROWSER_STORAGE_FORBIDDEN'; + error.apis = APIs; + error.relativePath = relativePath || null; + throw error; +} + +export function scanWorkspaceFilesForProhibitedBrowserStorage({ publishDir, relativePaths = [] } = {}) { + const root = path.resolve(String(publishDir ?? '')); + if (!root) return []; + const violations = []; + for (const relativePath of new Set(relativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))) { + if (!isAgentPageCodePath(relativePath)) continue; + const absolutePath = path.resolve(root, relativePath); + if (absolutePath !== root && !absolutePath.startsWith(`${root}${path.sep}`)) continue; + if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) continue; + const apis = detectProhibitedBrowserStorage(fs.readFileSync(absolutePath, 'utf8'), { relativePath }); + if (apis.length > 0) violations.push({ relativePath, apis }); + } + return violations; +} diff --git a/mindspace-h5-html-finish-guard.mjs b/mindspace-h5-html-finish-guard.mjs index c6a8e55..666872a 100644 --- a/mindspace-h5-html-finish-guard.mjs +++ b/mindspace-h5-html-finish-guard.mjs @@ -10,6 +10,7 @@ import { isStubPublicHtmlContent, materializeMissingPublicHtmlWrites, } from './mindspace-public-finish-sync.mjs'; +import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; const PUBLIC_HTML_PATH_PATTERN_GLOBAL = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/gi; const PUBLIC_ASSET_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi; @@ -106,6 +107,26 @@ export function collectMissingPublicHtmlPaths(messages, currentUser, publishDir) ); } +export function collectTouchedPublicCodePaths(messages, publishDir, syncResult = null) { + const paths = new Set( + (syncResult?.publicHtmlRelativePaths ?? []) + .map((item) => normalizePublicRelativePath(item, { publishDir })) + .filter(Boolean), + ); + for (const message of Array.isArray(messages) ? messages : []) { + for (const item of message?.content ?? []) { + if (item?.type !== 'toolRequest') continue; + const toolCall = item.toolCall?.value; + const name = String(toolCall?.name ?? '').trim().split('__').at(-1); + if (!['write_file', 'edit_file', 'write', 'edit'].includes(name)) continue; + const args = toolCall?.arguments ?? {}; + const relativePath = normalizePublicRelativePath(args.path ?? args.file_path, { publishDir }); + if (relativePath) paths.add(relativePath); + } + } + return [...paths]; +} + export function collectMissingPublicAssetReferences(publishDir) { const root = path.resolve(String(publishDir ?? '')); const publicDir = path.join(root, 'public'); @@ -135,6 +156,10 @@ export function evaluateH5HtmlFinishGuard({ materializeMissingPublicHtmlWrites({ messages, publishDir }); const missingHtml = collectMissingPublicHtmlPaths(messages, currentUser, publishDir); const missingAssets = collectMissingPublicAssetReferences(publishDir); + const browserStorageViolations = scanWorkspaceFilesForProhibitedBrowserStorage({ + publishDir, + relativePaths: collectTouchedPublicCodePaths(messages, publishDir, syncResult), + }); const lastAssistant = [...(Array.isArray(messages) ? messages : [])] .reverse() @@ -159,18 +184,24 @@ export function evaluateH5HtmlFinishGuard({ const needsRepair = missingHtml.length > 0 || missingAssets.length > 0 || + browserStorageViolations.length > 0 || htmlGenerationNeedsRetry; return { needsRepair, missingHtml, missingAssets, + browserStorageViolations, htmlGenerationNeedsRetry, linkExists, }; } -export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } = {}) { +export function buildH5HtmlRepairPrompt({ + missingHtml = [], + missingAssets = [], + browserStorageViolations = [], +} = {}) { const lines = [ '【系统补盘请求】检测到 public 页面交付不完整。请立即使用 write_file 写入缺失文件（完整内容），并在完成后确认磁盘存在。', '要求：先 load_skill static-page-publish，再逐个 write_file；不要用空回复或仅 edit_file 改 HTML 代替资源落盘。', @@ -185,6 +216,15 @@ export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } lines.push(`- ${item.relativePath}（来自 ${item.htmlRelativePath}）`); } } + if (browserStorageViolations.length) { + lines.push( + '', + '以下页面违反数据存储硬约束：', + ...browserStorageViolations.map((item) => `- ${item.relativePath}：禁止 ${item.apis.join(', ')}`), + '', + '必须删除浏览器存储代码，load_skill → page-data-collect，使用 private_data_* 在当前用户专属 PostgreSQL schema 建表和注册 dataset，HTML 只能通过 Page Data API 读写。禁止 SQLite 或任何本地 fallback。', + ); + } lines.push('', '写完后请 list_dir public/assets 或对应目录，确认文件真实存在。'); return lines.join('\n'); } diff --git a/mindspace-h5-html-finish-guard.test.mjs b/mindspace-h5-html-finish-guard.test.mjs index 64dee35..681b8bb 100644 --- a/mindspace-h5-html-finish-guard.test.mjs +++ b/mindspace-h5-html-finish-guard.test.mjs @@ -72,6 +72,61 @@ test('evaluateH5HtmlFinishGuard flags missing assets after html exists', () => { } }); +test('evaluateH5HtmlFinishGuard blocks browser storage in any touched public page', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-ban-')); + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync( + path.join(publishDir, 'public', 'ledger.html'), + '', + 'utf8', + ); + const evaluation = evaluateH5HtmlFinishGuard({ + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/ledger.html' }, + } } }] }], + currentUser: USER, + publishDir, + }); + assert.equal(evaluation.needsRepair, true); + assert.deepEqual(evaluation.browserStorageViolations, [{ + relativePath: 'public/ledger.html', + apis: ['sessionStorage', 'IndexedDB'], + }]); + assert.match(buildH5HtmlRepairPrompt(evaluation), /当前用户专属 PostgreSQL schema/); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + +test('evaluateH5HtmlFinishGuard blocks computed localStorage obfuscation', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-obfuscation-')); + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync( + path.join(publishDir, 'public', 'bypass.html'), + ``, + 'utf8', + ); + const evaluation = evaluateH5HtmlFinishGuard({ + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/bypass.html' }, + } } }] }], + currentUser: USER, + publishDir, + }); + assert.equal(evaluation.needsRepair, true); + assert.deepEqual(evaluation.browserStorageViolations, [{ + relativePath: 'public/bypass.html', + apis: ['localStorage'], + }]); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + test('buildH5HtmlRepairPrompt lists html and asset targets', () => { const prompt = buildH5HtmlRepairPrompt({ missingHtml: ['public/page.html'], diff --git a/mindspace-page-data-finish-guard.mjs b/mindspace-page-data-finish-guard.mjs index edd78f3..f76a6f1 100644 --- a/mindspace-page-data-finish-guard.mjs +++ b/mindspace-page-data-finish-guard.mjs @@ -15,7 +15,9 @@ import { normalizePageDataApiBase, verifyPageDataDeliveryArtifacts, } from './page-data-delivery-assess.mjs'; +import { listPageAccessPolicies } from './page-data-policy-store.mjs'; import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs'; +import { detectProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; export { inferPageDataBindAccessMode }; @@ -23,7 +25,7 @@ const PUBLICATION_ROUTE_LINK_PATTERN = /https?:\/\/[^\s<>"')\]]+\/u\/([0-9a-f-]{36}|[a-z0-9._-]+)\/pages\/([^\s<>"')\]]+)/gi; const PAGE_DATA_CLIENT_SCRIPT_PATTERN = /\/assets\/page-data-client\.js/i; -const LOCAL_STORAGE_DATA_PATTERN = /localStorage\.(?:getItem|setItem)\s*\(/i; +const LEGACY_PAGE_DATA_OWNER_API_PATTERN = /['"`]\/api\/page-data(?:\/|['"`])/i; const LOCAL_STORAGE_FALLBACK_HINT_PATTERN = /fallback\s*到\s*localStorage|localStorage\s*fallback/i; const repairAttemptsBySession = new Map(); @@ -109,10 +111,14 @@ export function buildPageDataDeliveryArtifactsFromBindResult(autoBind, publishDi export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) { const content = String(html ?? ''); const usage = detectPageDataDatasetUsageFromHtml(content); + const prohibitedBrowserStorage = detectProhibitedBrowserStorage(content, { relativePath }); const usesPageDataApi = usage.size > 0 || PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content) || - /\bMindSpacePageData\b/.test(content); + /\bMindSpacePageData\b/.test(content) || + LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content) || + prohibitedBrowserStorage.length > 0 || + LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content); if (!usesPageDataApi) { return { usesPageDataApi: false, issues: [] }; @@ -122,9 +128,15 @@ export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) { if (!PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content)) { issues.push('missing_page_data_client_script'); } - if (LOCAL_STORAGE_DATA_PATTERN.test(content)) { + if (LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content)) { + issues.push('forbidden_legacy_owner_api'); + } + if (prohibitedBrowserStorage.includes('localStorage')) { issues.push('forbidden_local_storage'); } + if (prohibitedBrowserStorage.some((api) => api !== 'localStorage')) { + issues.push('forbidden_browser_storage'); + } if (LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content)) { issues.push('forbidden_local_storage_fallback'); } @@ -156,7 +168,22 @@ export function collectPageDataPublicHtmlFiles(publishDir) { } function isPageDataHtmlWorkspaceReady({ publishDir, relativePath, html }) { - return assessWorkspacePageDataReadiness({ publishDir, relativePath, html }).ready; + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) return true; + return Boolean(findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html })); +} + +function findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html); + for (const policy of listPageAccessPolicies(publishDir)) { + if (String(policy?.accessMode ?? '') !== expectedAccessMode) continue; + if ([...usage.entries()].every(([name, perms]) => { + const granted = policy?.datasets?.[name]; + return Boolean(granted && (!perms.read || granted.read) && (!perms.insert || granted.insert)); + })) return policy; + } + return null; } async function collectUnboundPageDataFiles({ @@ -186,11 +213,13 @@ async function collectUnboundPageDataFiles({ } continue; } - if (!isPageDataHtmlWorkspaceReady({ + const workspace = await assessWorkspacePageDataReadiness({ + userId, publishDir, relativePath: file.relativePath, html: file.content, - })) { + }); + if (!workspace.ready) { unboundFiles.push(file); } } @@ -320,7 +349,7 @@ export function evaluatePageDataFinishGuard({ ); const needsRepair = - pageDataIntent && + (pageDataIntent || structuralFiles.length > 0) && (htmlIssues.length > 0 || unboundFiles.length > 0 || (relevantFiles.length === 0 && @@ -382,7 +411,7 @@ export function buildPageDataCollectFailureText() { '这次问卷/数据收集页面没有完成 Page Data API 绑定，所以我先不发链接。', '请直接重发一次完整需求（例如：调查问卷 + 后台查看），我会按 page-data-collect 技能：', '建表 → 注册 dataset → 写含 page-data-client.js 的 HTML → private_data_bind_workspace_page 发布。', - '数据必须走平台 API 写入 SQLite，禁止 localStorage 或自建后端。', + '数据必须走平台 API 写入用户私有数据空间，禁止 localStorage 或自建后端。', ].join(''); } diff --git a/mindspace-page-data-finish-guard.test.mjs b/mindspace-page-data-finish-guard.test.mjs index 335b3ee..64bf312 100644 --- a/mindspace-page-data-finish-guard.test.mjs +++ b/mindspace-page-data-finish-guard.test.mjs @@ -8,6 +8,7 @@ import { buildPageDataCollectRepairPrompt, collectPageDataDeliveryArtifacts, evaluatePageDataFinishGuard, + evaluatePageDataFinishGuardAsync, evaluatePageDataHtmlContent, extractRecentPageDataBindTargets, extractRecentPageDataHtmlWrites, @@ -42,6 +43,23 @@ test('evaluatePageDataHtmlContent flags missing client script and localStorage f const bad = evaluatePageDataHtmlContent(BAD_SURVEY_HTML, { relativePath: 'public/children-diet-survey.html' }); assert.ok(bad.issues.includes('missing_page_data_client_script')); assert.ok(bad.issues.includes('forbidden_local_storage')); + + const indexed = evaluatePageDataHtmlContent( + '', + { relativePath: 'public/draft.html' }, + ); + assert.ok(indexed.issues.includes('forbidden_browser_storage')); + + const legacyOwnerApi = evaluatePageDataHtmlContent( + ``, + { relativePath: 'public/sticky-notes.html' }, + ); + assert.equal(legacyOwnerApi.usesPageDataApi, true); + assert.ok(legacyOwnerApi.issues.includes('missing_page_data_client_script')); + assert.ok(legacyOwnerApi.issues.includes('forbidden_legacy_owner_api')); }); test('inferPageDataBindAccessMode chooses password for admin html', () => { @@ -70,6 +88,35 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => { } }); +test('finish guard blocks localStorage ledger generated from ordinary user language', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-ledger-')); + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync(path.join(publishDir, 'public', 'family-ledger.html'), ``, 'utf8'); + const evaluation = evaluatePageDataFinishGuard({ + publishDir, + agentText: text, + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/family-ledger.html' }, + } } }] }], + }); + assert.equal(evaluation.pageDataIntent, true); + assert.equal(evaluation.needsRepair, true); + assert.equal( + evaluation.htmlIssues.some((item) => ( + item.issue === 'forbidden_local_storage' && item.relativePath === 'public/family-ledger.html' + )), + true, + ); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + test('finish guard ignores unrelated historical broken Page Data html', async () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-history-')); try { @@ -162,7 +209,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => { } }); -test('evaluatePageDataFinishGuard flags html when dataset is not registered', () => { +test('evaluatePageDataFinishGuard flags html when dataset is not registered', async () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-bound-')); try { fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); @@ -179,8 +226,9 @@ test('evaluatePageDataFinishGuard flags html when dataset is not registered', () }, }, }); - const evaluation = evaluatePageDataFinishGuard({ + const evaluation = await evaluatePageDataFinishGuardAsync({ publishDir, + userId: 'user-1', agentText: '调查问卷和后台', messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }], }); diff --git a/mindspace-page-sync-service.mjs b/mindspace-page-sync-service.mjs index 3f3b6b3..c5a996f 100644 --- a/mindspace-page-sync-service.mjs +++ b/mindspace-page-sync-service.mjs @@ -11,12 +11,16 @@ export function createPageSyncService({ } = {}) { const syncInFlight = new Map(); - async function syncUserGeneratedPages(userId) { + async function syncUserGeneratedPages(userId, { onlyRelativePaths = null } = {}) { if (!pool || !pageService || !userId) { return { created: 0, skipped: 0, updated: 0 }; } - let inFlight = syncInFlight.get(userId); + const scopeKey = onlyRelativePaths == null + ? '*' + : [...new Set(onlyRelativePaths)].sort().join('|'); + const inFlightKey = `${userId}:${scopeKey}`; + let inFlight = syncInFlight.get(inFlightKey); if (!inFlight) { inFlight = syncGeneratedPagesFromPublicAssets({ pool, @@ -25,17 +29,18 @@ export function createPageSyncService({ userId, publishDir: h5Root ? resolvePublishDir(h5Root, { id: userId }) : null, syncWorkspaceAssets, + onlyRelativePaths, }) .catch((error) => { logger.warn?.('[MindSpace] page sync failed:', error?.message ?? error); return { created: 0, skipped: 0, updated: 0 }; }) .finally(() => { - if (syncInFlight.get(userId) === inFlight) { - syncInFlight.delete(userId); + if (syncInFlight.get(inFlightKey) === inFlight) { + syncInFlight.delete(inFlightKey); } }); - syncInFlight.set(userId, inFlight); + syncInFlight.set(inFlightKey, inFlight); } return inFlight; } diff --git a/mindspace-page-sync.mjs b/mindspace-page-sync.mjs index 5159b48..fcdfcbe 100644 --- a/mindspace-page-sync.mjs +++ b/mindspace-page-sync.mjs @@ -213,16 +213,23 @@ export async function syncGeneratedPagesFromPublicAssets({ userId, publishDir = null, syncWorkspaceAssets = null, + onlyRelativePaths = null, } = {}) { if (!pool || !pageService || !userId) { return { created: 0, skipped: 0, updated: 0 }; } if (typeof syncWorkspaceAssets === 'function') { - await syncWorkspaceAssets(userId, { categoryCode: 'public' }).catch(() => {}); + await syncWorkspaceAssets(userId, { + categoryCode: 'public', + onlyRelativePaths, + }).catch(() => {}); } const indexedPages = await loadIndexedWorkspacePages(pool, userId); + const allowList = onlyRelativePaths == null + ? null + : new Set(onlyRelativePaths.map((item) => normalizePublicHtmlPath(item)).filter(Boolean)); let created = 0; let updated = 0; let skipped = 0; @@ -230,6 +237,7 @@ export async function syncGeneratedPagesFromPublicAssets({ if (publishDir) { const htmlFiles = await listPublishPublicHtmlFiles(publishDir); for (const file of htmlFiles) { + if (allowList && !allowList.has(file.relativePath)) continue; try { const content = await fs.readFile(file.absolutePath, 'utf8'); if (!content.trim()) { @@ -285,6 +293,7 @@ export async function syncGeneratedPagesFromPublicAssets({ ? asset.original_filename : `public/${asset.original_filename}`, ); + if (allowList && !allowList.has(relativePath)) continue; if (indexedPages.has(relativePath)) { skipped += 1; continue; diff --git a/mindspace-page-sync.test.mjs b/mindspace-page-sync.test.mjs index 92593a5..8c97bcc 100644 --- a/mindspace-page-sync.test.mjs +++ b/mindspace-page-sync.test.mjs @@ -47,6 +47,40 @@ test('syncGeneratedPagesFromPublicAssets registers html files from publish publi assert.equal(created[0].source.snapshot.relative_path, 'public/demo.html'); }); +test('syncGeneratedPagesFromPublicAssets scopes agent-run sync to current session html', async () => { + const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-scoped-')); + const publicDir = path.join(publishDir, 'public'); + await fs.mkdir(publicDir, { recursive: true }); + await fs.writeFile(path.join(publicDir, 'current.html'), 'Current', 'utf8'); + await fs.writeFile(path.join(publicDir, 'historical.html'), 'Historical', 'utf8'); + + const created = []; + const pool = { + async query(sql) { + if (sql.includes('FROM h5_page_records p')) return [[]]; + if (sql.includes('FROM h5_assets a')) return [[]]; + return [[]]; + }, + }; + const pageService = { + async createFromChat(_userId, input, source) { + created.push({ input, source }); + return { id: `page-${created.length}` }; + }, + }; + + await syncGeneratedPagesFromPublicAssets({ + pool, + pageService, + userId: 'user-1', + publishDir, + onlyRelativePaths: ['public/current.html'], + }); + + assert.deepEqual(created.map((item) => item.source.snapshot.relative_path), ['public/current.html']); + await fs.rm(publishDir, { recursive: true, force: true }); +}); + test('syncGeneratedPagesFromPublicAssets uses file mtime as createdAt', async () => { const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-mtime-')); const publicDir = path.join(publishDir, 'public'); diff --git a/mindspace-public-finish-sync.mjs b/mindspace-public-finish-sync.mjs index 3d5c295..3e340e2 100644 --- a/mindspace-public-finish-sync.mjs +++ b/mindspace-public-finish-sync.mjs @@ -516,6 +516,7 @@ export async function syncPublicHtmlAfterFinish({ categoryCode: 'public', sourceSessionId: sessionId, sourceMessageId, + onlyRelativePaths: publicHtmlRelativePaths, }); synced = true; } diff --git a/mindspace-public-finish-sync.test.mjs b/mindspace-public-finish-sync.test.mjs index 90751c5..c5a4906 100644 --- a/mindspace-public-finish-sync.test.mjs +++ b/mindspace-public-finish-sync.test.mjs @@ -196,6 +196,7 @@ test('syncPublicHtmlAfterFinish forwards session source to workspace sync', asyn categoryCode: 'public', sourceSessionId: 'session-1', sourceMessageId: 'msg-public-1', + onlyRelativePaths: ['public/session-page.html'], }, ]]); } finally { diff --git a/mindspace-public-route.test.mjs b/mindspace-public-route.test.mjs index 05853aa..fcda274 100644 --- a/mindspace-public-route.test.mjs +++ b/mindspace-public-route.test.mjs @@ -121,6 +121,35 @@ test('resolveMindSpacePublicRequest coordinates canonical redirect, serve, and n }); }); +test('resolveMindSpacePublicRequest keeps legacy browser-storage pages available during migration', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-storage-ban-')); + const ownerKey = '123e4567-e89b-12d3-a456-426614174000'; + const targetDir = path.join(root, 'MindSpace', ownerKey, 'public'); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync( + path.join(targetDir, 'ledger.html'), + '', + ); + fs.writeFileSync( + path.join(targetDir, 'ledger.js'), + 'sessionStorage.setItem("ledger", "[]")', + ); + + const result = await resolveMindSpacePublicRequest({ + h5Root: root, + requestPath: `/${ownerKey}/public/ledger.html`, + }); + assert.equal(result.action, 'serve'); + assert.equal(result.filePath, path.join(targetDir, 'ledger.html')); + + const scriptResult = await resolveMindSpacePublicRequest({ + h5Root: root, + requestPath: `/${ownerKey}/public/ledger.js`, + }); + assert.equal(scriptResult.action, 'serve'); + assert.equal(scriptResult.filePath, path.join(targetDir, 'ledger.js')); +}); + test('resolveMindSpacePublicRequest serves missing thumbnail png when svg sidecar exists', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-thumb-')); const ownerKey = '123e4567-e89b-12d3-a456-426614174000'; diff --git a/mindspace-sandbox-mcp.mjs b/mindspace-sandbox-mcp.mjs index 4cef617..f49753c 100644 --- a/mindspace-sandbox-mcp.mjs +++ b/mindspace-sandbox-mcp.mjs @@ -24,6 +24,7 @@ import { closePolicyDataset, normalizePageAccessPolicy } from './page-access-pol import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs'; import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs'; import { resolveMindSpaceStorageRoot } from './mindspace-runtime-config.mjs'; +import { assertNoProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; const SANDBOX_ROOT = process.argv[2]?.trim() || process.env.SANDBOX_ROOT?.trim(); if (!SANDBOX_ROOT) { @@ -215,7 +216,7 @@ const ALL_TOOLS = [ { name: 'private_data_info', description: - '查看当前用户“用户私有数据空间”的状态。每个用户只有一个私有 SQLite 数据库，适合保存问卷、表单、清单、调研数据和分析中间表；不要创建额外 SQLite 文件。', + '查看当前用户“用户私有数据空间”的状态。每个用户只有一个由平台分配和隔离的数据空间，适合保存问卷、表单、清单、调研数据和分析中间表；不要创建额外数据库文件。', inputSchema: { type: 'object', properties: {} }, }, { @@ -504,6 +505,7 @@ async function callTool(name, args) { } case 'write_file': { const abs = resolveSandboxed(args.path); + assertNoProhibitedBrowserStorage(args.content ?? '', { relativePath: args.path }); fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, args.content ?? '', 'utf8'); return [{ type: 'text', text: `已写入 ${args.path}（${(args.content ?? '').length} 字节）` }]; @@ -516,6 +518,7 @@ async function callTool(name, args) { throw new Error('edit_file: old_str 在文件中不存在，替换失败'); } const updated = oldStr ? original.replace(oldStr, args.new_str ?? '') : (args.new_str ?? ''); + assertNoProhibitedBrowserStorage(updated, { relativePath: args.path }); fs.writeFileSync(abs, updated, 'utf8'); return [{ type: 'text', text: `已编辑 ${args.path}` }]; } @@ -578,7 +581,9 @@ async function callTool(name, args) { { ...info, rules: [ - '每个用户只能使用这个唯一 SQLite 数据库', + info.backend === 'postgres' + ? '每个用户只能使用新 PG 中分配给自己的独立 schema' + : '每个用户只能使用这个唯一 SQLite 数据库', '适合问卷、表单、清单、调研数据和分析中间表', '不要存账号、计费、权限、审计、公开平台数据或跨用户数据', 'HTML 页面通过 Page Data API 访问 dataset，不要直接暴露 SQL', @@ -603,7 +608,9 @@ async function callTool(name, args) { return [ { type: 'text', - text: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `，已同步空间占用 ${result.deltaBytes} 字节` : ''}`, + text: result.backend === 'postgres' + ? `已在用户专属 PG schema 中执行。命令 ${result.command ?? 'OK'}，影响 ${result.affectedRows ?? 0} 行` + : `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `，已同步空间占用 ${result.deltaBytes} 字节` : ''}`, }, ]; } diff --git a/mindspace-sandbox-mcp.test.mjs b/mindspace-sandbox-mcp.test.mjs index 9833e7a..6bcb917 100644 --- a/mindspace-sandbox-mcp.test.mjs +++ b/mindspace-sandbox-mcp.test.mjs @@ -85,6 +85,41 @@ function startSandbox(root, envOverrides = {}) { return { child, request }; } +test('sandbox MCP rejects browser storage in generated page code', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-storage-ban-')); + const server = startSandbox(root, { ALLOWED_TOOLS: 'write_file,edit_file' }); + t.after(() => server.child.kill()); + + await server.request('initialize'); + const rejectedWrite = await server.request('tools/call', { + name: 'write_file', + arguments: { + path: 'public/ledger.html', + content: '', + }, + }); + assert.equal(rejectedWrite.result.isError, true); + assert.match(rejectedWrite.result.content[0].text, /BROWSER_STORAGE_FORBIDDEN|禁止使用 localStorage/); + assert.equal(fs.existsSync(path.join(root, 'public', 'ledger.html')), false); + + const acceptedWrite = await server.request('tools/call', { + name: 'write_file', + arguments: { path: 'public/ledger.html', content: '' }, + }); + assert.equal(acceptedWrite.result.isError, false); + + const rejectedEdit = await server.request('tools/call', { + name: 'edit_file', + arguments: { + path: 'public/ledger.html', + old_str: 'client.insertRow("ledger", row)', + new_str: 'indexedDB.open("ledger")', + }, + }); + assert.equal(rejectedEdit.result.isError, true); + assert.doesNotMatch(fs.readFileSync(path.join(root, 'public', 'ledger.html'), 'utf8'), /indexedDB/); +}); + test('sandbox MCP exposes and protects the user private data space', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-')); const server = startSandbox(root); diff --git a/mindspace-userdata-postgres.mjs b/mindspace-userdata-postgres.mjs new file mode 100644 index 0000000..39347a2 --- /dev/null +++ b/mindspace-userdata-postgres.mjs @@ -0,0 +1,439 @@ +import { execFileSync } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export const CONTROL_SCHEMA = 'mindspace_control'; + +export function assertUserId(value) { + const userId = String(value ?? '').trim().toLowerCase(); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(userId)) { + throw new Error('user id 必须是标准 UUID'); + } + return userId; +} + +export function quotePgIdentifier(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function deriveUserSpaceNames(value) { + const userId = assertUserId(value); + const compact = userId.replaceAll('-', ''); + const short = compact.slice(0, 12); + return { + userId, + schemaName: `u_${compact}`, + ownerRole: `ms_u_${short}_owner`, + agentRole: `ms_u_${short}_agent`, + }; +} + +export function buildControlSchemaSql() { + return ` +CREATE SCHEMA IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}; +REVOKE ALL ON SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC; + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces ( + user_id uuid PRIMARY KEY, + schema_name name NOT NULL UNIQUE, + owner_role name NOT NULL UNIQUE, + agent_role name NOT NULL UNIQUE, + backend_type text NOT NULL DEFAULT 'postgres' CHECK (backend_type IN ('sqlite', 'postgres')), + source_sqlite_path text, + migration_state text NOT NULL DEFAULT 'pending' CHECK ( + migration_state IN ('pending', 'snapshot_created', 'schema_converted', 'data_imported', 'verified', 'shadow', 'cutover', 'rolled_back', 'failed') + ), + schema_version integer NOT NULL DEFAULT 1, + quota_bytes bigint NOT NULL DEFAULT 104857600 CHECK (quota_bytes > 0), + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + cutover_at timestamptz, + rollback_until timestamptz +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE, + source_path text NOT NULL, + source_size_bytes bigint NOT NULL DEFAULT 0, + source_sha256 text, + status text NOT NULL, + table_count integer NOT NULL DEFAULT 0, + source_row_count bigint NOT NULL DEFAULT 0, + target_row_count bigint NOT NULL DEFAULT 0, + detail_json jsonb NOT NULL DEFAULT '{}'::jsonb, + started_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at timestamptz +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog ( + user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE, + dataset_name text NOT NULL, + physical_table name NOT NULL, + config_json jsonb NOT NULL, + sensitivity text NOT NULL DEFAULT 'normal' CHECK (sensitivity IN ('normal', 'personal', 'credential', 'health')), + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (user_id, dataset_name) +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.audit_events ( + id bigserial PRIMARY KEY, + user_id uuid, + actor_type text NOT NULL, + actor_id text, + action text NOT NULL, + object_type text, + object_name text, + statement_hash text, + affected_rows bigint, + duration_ms integer, + result text NOT NULL, + detail_json jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); +REVOKE ALL ON ALL TABLES IN SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC; +`.trim(); +} + +export function buildProvisionUserSql(userId, { sourceSqlitePath = null, quotaBytes = 104857600 } = {}) { + const names = deriveUserSpaceNames(userId); + const schema = quotePgIdentifier(names.schemaName); + const owner = quotePgIdentifier(names.ownerRole); + const agent = quotePgIdentifier(names.agentRole); + return { + ...names, + statements: [ + `DO $$ BEGIN CREATE ROLE ${owner} NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN CREATE ROLE ${agent} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid + JOIN pg_roles m ON m.oid = am.member + WHERE r.rolname = '${names.ownerRole}' AND m.rolname = CURRENT_USER AND am.set_option + ) THEN + EXECUTE format('GRANT %I TO %I', '${names.ownerRole}', CURRENT_USER); + END IF; + END $$`, + `CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION ${owner}`, + `REVOKE ALL ON SCHEMA ${schema} FROM PUBLIC`, + `GRANT USAGE, CREATE ON SCHEMA ${schema} TO ${agent}`, + `ALTER ROLE ${agent} SET search_path = ${schema}, pg_catalog`, + `ALTER ROLE ${agent} SET statement_timeout = '30s'`, + `ALTER ROLE ${agent} SET lock_timeout = '5s'`, + `ALTER ROLE ${agent} SET idle_in_transaction_session_timeout = '60s'`, + `ALTER ROLE ${agent} SET work_mem = '4MB'`, + `ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${agent}`, + `ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO ${agent}`, + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces + (user_id, schema_name, owner_role, agent_role, source_sqlite_path, quota_bytes, updated_at) + VALUES ($1::uuid, $2::name, $3::name, $4::name, $5, $6, CURRENT_TIMESTAMP) + ON CONFLICT (user_id) DO UPDATE SET + source_sqlite_path = EXCLUDED.source_sqlite_path, + quota_bytes = EXCLUDED.quota_bytes, + updated_at = CURRENT_TIMESTAMP`, + ], + params: [names.userId, names.schemaName, names.ownerRole, names.agentRole, sourceSqlitePath, quotaBytes], + }; +} + +export function mapSqliteTypeToPostgres(column) { + const declared = String(column.type ?? '').trim().toUpperCase(); + if (Number(column.pk) === 1 && declared.includes('INT')) return 'bigint GENERATED BY DEFAULT AS IDENTITY'; + if (declared.includes('TEXT') && isSqliteTimestampDefault(column.dflt_value)) return 'timestamptz'; + if (declared.includes('INT')) return 'bigint'; + if (declared.includes('REAL') || declared.includes('FLOA') || declared.includes('DOUB')) return 'double precision'; + if (declared.includes('BLOB')) return 'bytea'; + if (declared.includes('NUM') || declared.includes('DEC')) return 'numeric'; + if (declared.includes('BOOL')) return 'boolean'; + return 'text'; +} + +export function isSqliteTimestampDefault(value) { + return /^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(String(value ?? '').trim()); +} + +export function translateSqliteDefault(value) { + if (value == null) return null; + const input = String(value).trim(); + if (/^NULL$/i.test(input)) return 'NULL'; + if (/^[+-]?\d+(?:\.\d+)?$/.test(input)) return input; + if (/^CURRENT_TIMESTAMP$/i.test(input)) return 'CURRENT_TIMESTAMP'; + if (/^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(input)) return 'CURRENT_TIMESTAMP'; + if (/^".*"$/.test(input)) return `'${input.slice(1, -1).replaceAll("'", "''")}'`; + if (/^'.*'$/.test(input)) return input; + return null; +} + +export function buildPostgresTableSql(schemaName, tableName, columns) { + if (!Array.isArray(columns) || columns.length === 0) throw new Error(`表 ${tableName} 没有字段`); + const primaryKeys = columns.filter((column) => Number(column.pk) > 0).sort((a, b) => Number(a.pk) - Number(b.pk)); + const definitions = columns.map((column) => { + const parts = [quotePgIdentifier(column.name), mapSqliteTypeToPostgres(column)]; + if (Number(column.notnull) === 1) parts.push('NOT NULL'); + const translatedDefault = translateSqliteDefault(column.dflt_value); + if (translatedDefault != null) parts.push(`DEFAULT ${translatedDefault}`); + if (primaryKeys.length === 1 && primaryKeys[0].name === column.name) parts.push('PRIMARY KEY'); + return parts.join(' '); + }); + if (primaryKeys.length > 1) { + definitions.push(`PRIMARY KEY (${primaryKeys.map((column) => quotePgIdentifier(column.name)).join(', ')})`); + } + return `CREATE TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (\n ${definitions.join(',\n ')}\n)`; +} + +function stableObjectName(prefix, ...parts) { + const readable = `${prefix}_${parts.join('_')}`.replace(/[^a-zA-Z0-9_]+/g, '_').toLowerCase(); + const hash = createHash('sha256').update(readable).digest('hex').slice(0, 8); + return `${readable.slice(0, 50)}_${hash}`; +} + +export function extractSqliteChecks(createSql) { + const sql = String(createSql ?? ''); + const checks = []; + const pattern = /\bCHECK\s*\(/ig; + let match; + while ((match = pattern.exec(sql))) { + const start = pattern.lastIndex; + let depth = 1; + let quote = ''; + let index = start; + for (; index < sql.length && depth > 0; index += 1) { + const char = sql[index]; + if (quote) { + if (char === quote && sql[index + 1] === quote) index += 1; + else if (char === quote) quote = ''; + continue; + } + if (char === "'" || char === '"') quote = char; + else if (char === '(') depth += 1; + else if (char === ')') depth -= 1; + } + if (depth === 0) checks.push(sql.slice(start, index - 1).trim()); + pattern.lastIndex = index; + } + return checks; +} + +export function buildPostgresIndexSql(schemaName, tableName, index) { + if (!index.columns?.length) return null; + if (index.partial) return null; + const prefix = index.unique ? 'ux' : 'ix'; + const name = stableObjectName(prefix, tableName, ...index.columns); + return `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX ${quotePgIdentifier(name)} ON ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (${index.columns.map(quotePgIdentifier).join(', ')})`; +} + +export function buildPostgresForeignKeySql(schemaName, tableName, foreignKey) { + const name = stableObjectName('fk', tableName, foreignKey.id, foreignKey.table); + const onUpdate = String(foreignKey.onUpdate ?? 'NO ACTION').toUpperCase(); + const onDelete = String(foreignKey.onDelete ?? 'NO ACTION').toUpperCase(); + const allowedActions = new Set(['NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT']); + if (!allowedActions.has(onUpdate) || !allowedActions.has(onDelete)) throw new Error('SQLite 外键动作不受支持'); + return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} FOREIGN KEY (${foreignKey.from.map(quotePgIdentifier).join(', ')}) REFERENCES ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(foreignKey.table)} (${foreignKey.to.map(quotePgIdentifier).join(', ')}) ON UPDATE ${onUpdate} ON DELETE ${onDelete}`; +} + +export function buildPostgresCheckSql(schemaName, tableName, expression, index) { + if (/;|--|\/\*/.test(expression)) throw new Error(`CHECK 表达式包含不安全内容：${tableName}`); + const name = stableObjectName('ck', tableName, index); + return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} CHECK (${expression})`; +} + +export function convertSqliteValueForPostgres(value, column) { + if (value == null) return null; + if (isSqliteTimestampDefault(column.dflt_value) && typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(trimmed)) { + return `${trimmed.replace(' ', 'T')}+08:00`; + } + } + return value; +} + +function sqliteJson(sqliteBin, databasePath, sql) { + const output = execFileSync(sqliteBin, ['-readonly', '-json', databasePath, sql], { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }).trim(); + return output ? JSON.parse(output) : []; +} + +function quoteSqliteIdentifier(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function inspectSqliteDatabase(databasePath, { sqliteBin = 'sqlite3' } = {}) { + const resolved = path.resolve(databasePath); + if (!fs.statSync(resolved).isFile()) throw new Error(`SQLite 文件不存在：${resolved}`); + const integrity = sqliteJson(sqliteBin, resolved, 'PRAGMA integrity_check;'); + if (integrity[0]?.integrity_check !== 'ok') throw new Error(`SQLite integrity_check 未通过：${resolved}`); + const tables = sqliteJson( + sqliteBin, + resolved, + "SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;", + ).map((table) => { + const columns = sqliteJson(sqliteBin, resolved, `PRAGMA table_info(${quoteSqliteIdentifier(table.name)});`); + const rows = sqliteJson(sqliteBin, resolved, `SELECT * FROM ${quoteSqliteIdentifier(table.name)};`); + const indexes = sqliteJson(sqliteBin, resolved, `PRAGMA index_list(${quoteSqliteIdentifier(table.name)});`).map((index) => ({ + name: index.name, + unique: Boolean(index.unique), + origin: index.origin, + partial: Boolean(index.partial), + columns: sqliteJson(sqliteBin, resolved, `PRAGMA index_info(${quoteSqliteIdentifier(index.name)});`).map((item) => item.name), + })).filter((index) => index.origin !== 'pk'); + const foreignKeyRows = sqliteJson(sqliteBin, resolved, `PRAGMA foreign_key_list(${quoteSqliteIdentifier(table.name)});`); + const foreignKeys = [...new Set(foreignKeyRows.map((item) => item.id))].map((id) => { + const items = foreignKeyRows.filter((item) => item.id === id).sort((a, b) => a.seq - b.seq); + return { + id, + table: items[0].table, + from: items.map((item) => item.from), + to: items.map((item) => item.to), + onUpdate: items[0].on_update, + onDelete: items[0].on_delete, + }; + }); + return { ...table, columns, rows, indexes, foreignKeys, checks: extractSqliteChecks(table.sql) }; + }); + return { path: resolved, sizeBytes: fs.statSync(resolved).size, tables }; +} + +export function createSqliteSnapshot(sourcePath, { sqliteBin = 'sqlite3' } = {}) { + const target = path.join(os.tmpdir(), `mindspace-userdata-${Date.now()}-${process.pid}.sqlite`); + if (/\s/.test(target)) throw new Error('SQLite snapshot 临时路径不能包含空格'); + execFileSync(sqliteBin, [path.resolve(sourcePath), `.backup ${target}`], { stdio: 'pipe' }); + // A standalone snapshot must not depend on the source WAL/SHM files. + execFileSync(sqliteBin, [target, 'PRAGMA journal_mode=DELETE;'], { stdio: 'pipe' }); + return target; +} + +export async function provisionUserSpace(client, userId, options = {}) { + const provision = buildProvisionUserSql(userId, options); + await client.query('BEGIN'); + try { + for (let index = 0; index < provision.statements.length; index += 1) { + const params = index === provision.statements.length - 1 ? provision.params : []; + await client.query(provision.statements[index], params); + } + await client.query('COMMIT'); + return provision; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} + +export async function migrateSqliteSnapshot(client, snapshot, userId, { replaceShadow = false, sourcePath = snapshot.path } = {}) { + const names = deriveUserSpaceNames(userId); + const owner = quotePgIdentifier(names.ownerRole); + const agent = quotePgIdentifier(names.agentRole); + const migrationId = randomUUID(); + let sourceRows = 0; + let targetRows = 0; + await client.query('BEGIN'); + try { + const stateResult = await client.query( + `SELECT migration_state FROM ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces WHERE user_id = $1::uuid FOR UPDATE`, + [names.userId], + ); + const migrationState = stateResult.rows[0]?.migration_state; + if (migrationState === 'cutover') throw new Error('用户空间已经 cutover，禁止迁移器覆盖'); + const existingResult = await client.query( + `SELECT count(*)::integer AS count FROM pg_catalog.pg_tables WHERE schemaname = $1`, + [names.schemaName], + ); + if (Number(existingResult.rows[0]?.count ?? 0) > 0 && !replaceShadow) { + throw new Error('目标影子空间已有表；如确认重建，显式使用 --replace-shadow'); + } + await client.query( + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs + (id, user_id, source_path, source_size_bytes, status, started_at) + VALUES ($1::uuid, $2::uuid, $3, $4, 'importing', CURRENT_TIMESTAMP)`, + [migrationId, names.userId, sourcePath, snapshot.sizeBytes], + ); + for (const table of snapshot.tables) { + await client.query(`DROP TABLE IF EXISTS ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} CASCADE`); + await client.query(buildPostgresTableSql(names.schemaName, table.name, table.columns)); + await client.query(`ALTER TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} OWNER TO ${owner}`); + await client.query( + `GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} TO ${agent}`, + ); + for (const row of table.rows) { + const columns = Object.keys(row); + const params = columns.map((_, index) => `$${index + 1}`); + await client.query( + `INSERT INTO ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} (${columns.map(quotePgIdentifier).join(', ')}) VALUES (${params.join(', ')})`, + columns.map((column) => convertSqliteValueForPostgres( + row[column], + table.columns.find((item) => item.name === column) ?? {}, + )), + ); + } + for (const column of table.columns.filter( + (item) => Number(item.pk) === 1 && String(item.type ?? '').toUpperCase().includes('INT'), + )) { + const qualifiedTable = `${names.schemaName}.${table.name}`; + await client.query( + `SELECT setval( + pg_get_serial_sequence($1, $2), + GREATEST(COALESCE(MAX(${quotePgIdentifier(column.name)}), 0), 1), + COALESCE(MAX(${quotePgIdentifier(column.name)}), 0) > 0 + ) + FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`, + [qualifiedTable, column.name], + ); + } + sourceRows += table.rows.length; + const count = await client.query( + `SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`, + ); + targetRows += Number(count.rows[0].count); + } + for (const table of snapshot.tables) { + for (const index of table.indexes ?? []) { + const sql = buildPostgresIndexSql(names.schemaName, table.name, index); + if (sql) await client.query(sql); + } + for (const foreignKey of table.foreignKeys ?? []) { + await client.query(buildPostgresForeignKeySql(names.schemaName, table.name, foreignKey)); + } + for (let index = 0; index < (table.checks ?? []).length; index += 1) { + await client.query(buildPostgresCheckSql(names.schemaName, table.name, table.checks[index], index)); + } + } + await client.query(`GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA ${quotePgIdentifier(names.schemaName)} TO ${agent}`); + const registry = snapshot.tables.find((table) => table.name === '__page_data_datasets'); + for (const row of registry?.rows ?? []) { + await client.query( + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog + (user_id, dataset_name, physical_table, config_json, updated_at) + VALUES ($1::uuid, $2, $3::name, $4::jsonb, CURRENT_TIMESTAMP) + ON CONFLICT (user_id, dataset_name) DO UPDATE SET + physical_table = EXCLUDED.physical_table, + config_json = EXCLUDED.config_json, + updated_at = CURRENT_TIMESTAMP`, + [names.userId, row.name, row.table_name, row.config_json], + ); + } + await client.query( + `UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces + SET migration_state = 'shadow', updated_at = CURRENT_TIMESTAMP + WHERE user_id = $1::uuid`, + [names.userId], + ); + await client.query( + `UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs + SET status = 'verified', table_count = $2, source_row_count = $3, + target_row_count = $4, finished_at = CURRENT_TIMESTAMP + WHERE id = $1::uuid`, + [migrationId, snapshot.tables.length, sourceRows, targetRows], + ); + await client.query('COMMIT'); + return { migrationId, ...names, tableCount: snapshot.tables.length, sourceRows, targetRows }; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} diff --git a/mindspace-userdata-postgres.test.mjs b/mindspace-userdata-postgres.test.mjs new file mode 100644 index 0000000..569fe38 --- /dev/null +++ b/mindspace-userdata-postgres.test.mjs @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + buildControlSchemaSql, + buildPostgresTableSql, + buildPostgresCheckSql, + buildPostgresForeignKeySql, + buildPostgresIndexSql, + buildProvisionUserSql, + createSqliteSnapshot, + convertSqliteValueForPostgres, + deriveUserSpaceNames, + extractSqliteChecks, + mapSqliteTypeToPostgres, + translateSqliteDefault, +} from './mindspace-userdata-postgres.mjs'; +import { execFileSync } from 'node:child_process'; + +const USER_ID = 'ecc1c649-fff7-4361-a243-a69b460cc407'; + +test('deriveUserSpaceNames creates stable UUID based identifiers', () => { + assert.deepEqual(deriveUserSpaceNames(USER_ID), { + userId: USER_ID, + schemaName: 'u_ecc1c649fff74361a243a69b460cc407', + ownerRole: 'ms_u_ecc1c649fff7_owner', + agentRole: 'ms_u_ecc1c649fff7_agent', + }); +}); + +test('control schema is private and tracks migration lifecycle', () => { + const sql = buildControlSchemaSql(); + assert.match(sql, /REVOKE ALL ON SCHEMA "mindspace_control" FROM PUBLIC/); + assert.match(sql, /CREATE TABLE IF NOT EXISTS "mindspace_control"\.user_spaces/); + assert.match(sql, /migration_runs/); + assert.match(sql, /audit_events/); +}); + +test('provision SQL creates isolated no-login roles and safety limits', () => { + const provision = buildProvisionUserSql(USER_ID, { sourceSqlitePath: '/tmp/user.sqlite' }); + const sql = provision.statements.join('\n'); + assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_owner" NOLOGIN/); + assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_agent" NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE/); + assert.match(sql, /pg_auth_members/); + assert.match(sql, /REVOKE ALL ON SCHEMA "u_ecc1c649fff74361a243a69b460cc407" FROM PUBLIC/); + assert.match(sql, /statement_timeout = '30s'/); + assert.match(sql, /NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION/); + assert.doesNotMatch(sql, /temp_file_limit/); +}); + +test('SQLite types and defaults map to conservative PostgreSQL equivalents', () => { + assert.equal(mapSqliteTypeToPostgres({ type: 'INTEGER', pk: 1 }), 'bigint GENERATED BY DEFAULT AS IDENTITY'); + assert.equal(mapSqliteTypeToPostgres({ type: 'REAL', pk: 0 }), 'double precision'); + assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0 }), 'text'); + assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0, dflt_value: "datetime('now', 'localtime')" }), 'timestamptz'); + assert.equal(translateSqliteDefault("(datetime('now', '+8 hours'))"), 'CURRENT_TIMESTAMP'); + assert.equal(translateSqliteDefault('""'), "''"); + assert.equal(translateSqliteDefault('dangerous_function()'), null); +}); + +test('SQLite checks, indexes and foreign keys become scoped PostgreSQL objects', () => { + assert.deepEqual(extractSqliteChecks("CREATE TABLE x(status TEXT CHECK(status IN ('a','b')), score INT CHECK(score > 0))"), [ + "status IN ('a','b')", + 'score > 0', + ]); + assert.match(buildPostgresIndexSql('u_test', 'users', { unique: true, columns: ['username'] }), /CREATE UNIQUE INDEX/); + assert.match(buildPostgresForeignKeySql('u_test', 'child', { + id: 0, table: 'parent', from: ['parent_id'], to: ['id'], onUpdate: 'NO ACTION', onDelete: 'CASCADE', + }), /REFERENCES "u_test"\."parent" \("id"\).*ON DELETE CASCADE/); + assert.match(buildPostgresCheckSql('u_test', 'users', "status IN ('a','b')", 0), /CHECK \(status IN \('a','b'\)\)/); +}); + +test('SQLite local timestamps are made explicit for PostgreSQL', () => { + assert.equal( + convertSqliteValueForPostgres('2026-07-13 10:20:30', { dflt_value: "datetime('now', '+8 hours')" }), + '2026-07-13T10:20:30+08:00', + ); + assert.equal(convertSqliteValueForPostgres('plain', { dflt_value: null }), 'plain'); +}); + +test('table builder preserves primary key, not-null and safe defaults', () => { + const sql = buildPostgresTableSql('u_example', 'survey', [ + { name: 'id', type: 'INTEGER', pk: 1, notnull: 0, dflt_value: null }, + { name: 'answer', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "''" }, + { name: 'created_at', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "datetime('now', 'localtime')" }, + ]); + assert.match(sql, /"id" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY/); + assert.match(sql, /"answer" text NOT NULL DEFAULT ''/); + assert.match(sql, /"created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP/); +}); + +test('SQLite snapshot includes committed rows without changing source', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-pg-test-')); + const source = path.join(root, 'source.sqlite'); + execFileSync('sqlite3', [source, 'CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO sample VALUES (1, \'ok\');']); + const snapshot = createSqliteSnapshot(source); + try { + const count = execFileSync('sqlite3', ['-readonly', snapshot, 'SELECT count(*) FROM sample;'], { encoding: 'utf8' }).trim(); + assert.equal(count, '1'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(snapshot, { force: true }); + } +}); diff --git a/mindspace-workspace-page-deliver-page-data.test.mjs b/mindspace-workspace-page-deliver-page-data.test.mjs index 5cc7f5e..6614ae0 100644 --- a/mindspace-workspace-page-deliver-page-data.test.mjs +++ b/mindspace-workspace-page-deliver-page-data.test.mjs @@ -56,8 +56,8 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page }, }, pageDataEnsure: { - async ensurePageDataHtmlPagesBound() { - bound.push(true); + async ensurePageDataHtmlPagesBound(options) { + bound.push(options); return { bound: [{ relativePath: 'public/survey.html' }], skipped: [], errors: [] }; }, }, @@ -65,8 +65,11 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page storageRoot: null, }); - const result = await service.syncAndDeliver(userId); + const result = await service.syncAndDeliver(userId, { + pageDataRelativePaths: ['public/survey.html'], + }); assert.equal(bound.length, 1); + assert.deepEqual(bound[0].onlyRelativePaths, ['public/survey.html']); assert.equal(result.publish.published, 0); assert.equal(result.publish.skipped, 1); assert.equal(published.length, 0); diff --git a/mindspace-workspace-page-deliver.mjs b/mindspace-workspace-page-deliver.mjs index b849ba2..36781bd 100644 --- a/mindspace-workspace-page-deliver.mjs +++ b/mindspace-workspace-page-deliver.mjs @@ -95,7 +95,7 @@ export function createWorkspacePageDeliverService({ return resolvePublishDir(h5Root, { id: userId }); } - async function ensurePageDataBindings(userId) { + async function ensurePageDataBindings(userId, { onlyRelativePaths = null } = {}) { if (!pageDataEnsure?.ensurePageDataHtmlPagesBound || !pool || !userId) { return { bound: [], skipped: [], errors: [] }; } @@ -110,6 +110,7 @@ export function createWorkspacePageDeliverService({ userId, workspaceRoot, findPageByRelativePath, + onlyRelativePaths, logger, }); } @@ -189,15 +190,20 @@ export function createWorkspacePageDeliverService({ return { published, skipped, errors }; } - async function syncAndDeliver(userId) { + async function syncAndDeliver(userId, { pageDataRelativePaths = null } = {}) { let syncResult = { created: 0, updated: 0, skipped: 0 }; if (pageSyncService?.syncUserGeneratedPages) { - syncResult = await pageSyncService.syncUserGeneratedPages(userId); + syncResult = await pageSyncService.syncUserGeneratedPages(userId, { + onlyRelativePaths: pageDataRelativePaths, + }); } - const pageDataBindResult = await ensurePageDataBindings(userId); + const pageDataBindResult = await ensurePageDataBindings(userId, { + onlyRelativePaths: pageDataRelativePaths, + }); const publishResult = await ensureWorkspaceHtmlPublications(userId); const refreshResult = await refreshOnlineWorkspacePublications(userId); return { + pageDataRelativePaths: pageDataRelativePaths == null ? null : [...pageDataRelativePaths], sync: syncResult, pageDataBind: pageDataBindResult, publish: publishResult, diff --git a/mindspace-workspace-sync.mjs b/mindspace-workspace-sync.mjs index 1a47a54..603708a 100644 --- a/mindspace-workspace-sync.mjs +++ b/mindspace-workspace-sync.mjs @@ -436,10 +436,20 @@ export function createWorkspaceAssetSync({ } }; - const syncCategory = async (userId, categoryCode, source = {}) => { + const syncCategory = async (userId, categoryCode, source = {}, { onlyRelativePaths = null } = {}) => { if (!h5Root) return { imported: 0, updated: 0, skipped: 0 }; const workspaceRoot = resolveUserWorkspaceRoot(h5Root, { id: userId }); - const files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode); + let files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode); + if (onlyRelativePaths != null) { + const allowed = new Set( + onlyRelativePaths + .map((item) => normalizeWorkspaceRelativePath(item)) + .filter(Boolean), + ); + files = files.filter((file) => + allowed.has(normalizeWorkspaceRelativePath(`${categoryCode}/${file.filename}`)), + ); + } const [categories] = await pool.query( `SELECT c.id, c.space_id, c.category_code FROM h5_space_categories c @@ -487,14 +497,17 @@ export function createWorkspaceAssetSync({ return { imported, updated, skipped }; }; - const syncUserWorkspace = async (userId, { categoryCode, sourceSessionId, sourceMessageId, title } = {}) => { + const syncUserWorkspace = async ( + userId, + { categoryCode, sourceSessionId, sourceMessageId, title, onlyRelativePaths = null } = {}, + ) => { const codes = categoryCode ? [categoryCode] : UPLOAD_ZONE_CODES; let imported = 0; let updated = 0; let skipped = 0; const source = { sessionId: sourceSessionId, messageId: sourceMessageId, title }; for (const code of codes) { - const result = await syncCategory(userId, code, source); + const result = await syncCategory(userId, code, source, { onlyRelativePaths }); imported += result.imported; updated += result.updated; skipped += result.skipped; diff --git a/mindspace-workspace-sync.test.mjs b/mindspace-workspace-sync.test.mjs index 65f340f..b15da83 100644 --- a/mindspace-workspace-sync.test.mjs +++ b/mindspace-workspace-sync.test.mjs @@ -316,6 +316,7 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn const workspace = resolveUserWorkspaceRoot(h5Root, { id: 'user-1', username: 'john' }); await fs.mkdir(path.join(workspace, 'oa', '暑假实习报告'), { recursive: true }); await fs.writeFile(path.join(workspace, 'oa', '暑假实习报告', 'report.csv'), 'a,b\n1,2\n'); + await fs.writeFile(path.join(workspace, 'oa', 'historical.csv'), 'old,data\n3,4\n'); const state = { categories: [ @@ -417,7 +418,10 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn idFactory: () => `id-${++nextId}`, }); - const result = await sync.syncUserWorkspace('user-1', { categoryCode: 'oa' }); + const result = await sync.syncUserWorkspace('user-1', { + categoryCode: 'oa', + onlyRelativePaths: ['oa/暑假实习报告/report.csv'], + }); assert.equal(result.imported, 1); assert.equal(state.assets[0].original_filename, '暑假实习报告/report.csv'); }); diff --git a/page-data-delivery-assess.mjs b/page-data-delivery-assess.mjs index df0c39d..3bd3b3c 100644 --- a/page-data-delivery-assess.mjs +++ b/page-data-delivery-assess.mjs @@ -40,6 +40,33 @@ export function buildSmokeInsertRow(policy, datasetName) { return row; } +export function assessPageDataAuthenticationContract({ policy, html } = {}) { + const accessMode = String(policy?.accessMode ?? '').trim(); + const content = String(html ?? ''); + const usesServerAuthentication = /\.\s*authenticate\s*\(/.test(content); + const passwordValue = String.raw`\b(?:pwd|pass(?:word)?|passwordInput|adminPassword)\w*(?:\.value)?`; + const passwordComparison = String.raw`(?:===?|!==?)`; + const passwordOperand = String.raw`(?:[A-Za-z_$][\w$]*|['"\x60][^'"\x60]*['"\x60])`; + const usesClientSidePasswordCheck = new RegExp( + `(?:${passwordValue}\\s*${passwordComparison}\\s*${passwordOperand}|${passwordOperand}\\s*${passwordComparison}\\s*${passwordValue})`, + 'i', + ).test(content); + const reasons = []; + if (accessMode === 'password' && !usesServerAuthentication) { + reasons.push('password_page_missing_server_authentication'); + } + if (accessMode === 'public' && usesServerAuthentication) { + reasons.push('public_page_cannot_use_password_authentication'); + } + if (accessMode === 'public' && usesClientSidePasswordCheck) { + reasons.push('public_page_uses_client_side_password_gate'); + } + if (accessMode === 'password' && usesClientSidePasswordCheck) { + reasons.push('password_page_uses_client_side_password_gate'); + } + return reasons; +} + export async function queryOnlinePublication(pool, pageId) { if (!pool || !pageId) return null; const [rows] = await pool.query( @@ -76,20 +103,13 @@ function findWorkspacePolicyForHtml({ publishDir, relativePath, html }) { return null; } -export function assessWorkspacePageDataReadiness({ publishDir, relativePath, html }) { +function assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }) { const usage = detectPageDataDatasetUsageFromHtml(html); if (!usage.size) { return { ready: true, reasons: [] }; } const reasons = []; - const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir }); - for (const datasetName of usage.keys()) { - if (!dataSpace.getDataset(datasetName)) { - reasons.push(`dataset_not_registered:${datasetName}`); - } - } - const policy = findWorkspacePolicyForHtml({ publishDir, relativePath, html }); if (!policy) { reasons.push('missing_workspace_policy'); @@ -98,6 +118,21 @@ export function assessWorkspacePageDataReadiness({ publishDir, relativePath, htm return { ready: reasons.length === 0, reasons, policy }; } +export async function assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const policyAssessment = assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }); + if (!usage.size) return policyAssessment; + + const reasons = [...policyAssessment.reasons]; + const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir, userId }); + for (const datasetName of usage.keys()) { + if (!(await dataSpace.getDataset(datasetName))) { + reasons.push(`dataset_not_registered:${datasetName}`); + } + } + return { ...policyAssessment, ready: reasons.length === 0, reasons }; +} + export async function assessPageDataHtmlBinding({ pool, userId, @@ -111,7 +146,7 @@ export async function assessPageDataHtmlBinding({ return { bound: true, reasons: [], pageId: null }; } - const workspace = assessWorkspacePageDataReadiness({ publishDir, relativePath, html }); + const workspace = await assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }); const reasons = [...workspace.reasons]; let pageId = null; @@ -139,6 +174,7 @@ export async function assessPageDataHtmlBinding({ if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) { reasons.push('policy_access_mode_mismatch'); } + reasons.push(...assessPageDataAuthenticationContract({ policy, html })); for (const [datasetName, perms] of usage) { const policyDataset = policy.datasets?.[datasetName]; if (!policyDataset) continue; diff --git a/page-data-delivery-assess.test.mjs b/page-data-delivery-assess.test.mjs index 0c0498e..b0db110 100644 --- a/page-data-delivery-assess.test.mjs +++ b/page-data-delivery-assess.test.mjs @@ -1,11 +1,49 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + assessPageDataAuthenticationContract, buildSmokeInsertRow, extractInjectedPageIdFromHtml, normalizePageDataApiBase, } from './page-data-delivery-assess.mjs'; +test('password pages must authenticate with the server before data reads', () => { + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + [ + 'password_page_missing_server_authentication', + 'password_page_uses_client_side_password_gate', + ], + ); + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + [], + ); + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + ['password_page_uses_client_side_password_gate'], + ); +}); + +test('public pages cannot use a client-side password gate as authorization', () => { + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'public' }, + html: '', + }), + ['public_page_uses_client_side_password_gate'], + ); +}); + test('extractInjectedPageIdFromHtml reads injected pageId', () => { const html = ` diff --git a/page-data-public-service.mjs b/page-data-public-service.mjs index f474f59..3ef15c0 100644 --- a/page-data-public-service.mjs +++ b/page-data-public-service.mjs @@ -171,13 +171,13 @@ export function createPageDataPublicService(deps = {}) { }); } - function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) { + async function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) { const policyDataset = policy.datasets?.[datasetName]; if (!policyDataset) { throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' }); } const ownerService = createOwnerService(ownerUserId); - const registryDataset = ownerService.getDataset(datasetName); + const registryDataset = await ownerService.getDataset(datasetName); if (!registryDataset) { throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); } @@ -190,9 +190,9 @@ export function createPageDataPublicService(deps = {}) { return { ownerService, effective, policyDataset }; } - function resolveRowScope(ownerService, effective, policyDataset, visitorContext) { + async function resolveRowScope(ownerService, effective, policyDataset, visitorContext) { if (!policyDataset?.rowPolicy || !visitorContext) return null; - const tableColumns = ownerService.listTableColumns(effective.table); + const tableColumns = await ownerService.listTableColumns(effective.table); return resolveRowAccessScope(policyDataset.rowPolicy, { visitorRole: visitorContext.visitorRole, visitorUserId: visitorContext.visitorUserId, @@ -303,13 +303,13 @@ export function createPageDataPublicService(deps = {}) { const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); enforceRateLimit(req, publication, 'read'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -330,7 +330,7 @@ export function createPageDataPublicService(deps = {}) { const publication = await queryPublication(pageId); const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); - const { ownerService, effective } = resolveEffectiveDataset( + const { ownerService, effective } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, @@ -345,13 +345,13 @@ export function createPageDataPublicService(deps = {}) { const publication = await queryPublication(pageId); const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -415,13 +415,13 @@ export function createPageDataPublicService(deps = {}) { } const payload = stripPageDataMetaFields(payloadInput); enforceRateLimit(req, publication, 'insert'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -465,13 +465,13 @@ export function createPageDataPublicService(deps = {}) { datasetName, }); enforceRateLimit(req, publication, 'update'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -513,13 +513,13 @@ export function createPageDataPublicService(deps = {}) { datasetName, }); enforceRateLimit(req, publication, 'soft_delete'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -598,7 +598,7 @@ export function createPageDataPublicService(deps = {}) { throw Object.assign(new Error('缺少 dataset 名称'), { code: 'invalid_request', status: 400 }); } const ownerService = createOwnerService(user.id); - const registryDataset = ownerService.getDataset(normalizedDatasetName); + const registryDataset = await ownerService.getDataset(normalizedDatasetName); if (!registryDataset) { throw Object.assign(new Error('dataset 未注册'), { code: 'dataset_not_found', status: 404 }); } @@ -639,8 +639,8 @@ export function createPageDataPublicService(deps = {}) { for (const [name, config] of Object.entries(policy.datasets ?? {})) { let stats = null; try { - if (ownerService.getDataset(name)) { - stats = ownerService.getDatasetStats(name); + if (await ownerService.getDataset(name)) { + stats = await ownerService.getDatasetStats(name); } } catch { stats = null; @@ -660,7 +660,7 @@ export function createPageDataPublicService(deps = {}) { const recentSubmissions = logs.filter((entry) => entry.action === 'insert').slice(0, 10); const activeSessions = sessionStore.listByPageId(pageId); const index = await getPageDataPolicyIndex(getPool(), pageId); - const registeredDatasets = ownerService.listDatasets().map((dataset) => dataset.name); + const registeredDatasets = (await ownerService.listDatasets()).map((dataset) => dataset.name); const configuredNames = new Set(Object.keys(policy.datasets ?? {})); const unconfiguredDatasets = registeredDatasets.filter((name) => !configuredNames.has(name)); return { diff --git a/page-data-service.mjs b/page-data-service.mjs index bd6a812..9124624 100644 --- a/page-data-service.mjs +++ b/page-data-service.mjs @@ -58,7 +58,7 @@ export function createPageDataService(deps = {}) { async function listRows(user, datasetName, query = {}) { const { service } = await createServiceForUser(user); try { - return service.readDatasetRows(datasetName, { + return await service.readDatasetRows(datasetName, { limit: query.limit, offset: query.offset, orderBy: query.orderBy ?? query.order_by, @@ -73,7 +73,7 @@ export function createPageDataService(deps = {}) { async function getSchema(user, datasetName) { const { service } = await createServiceForUser(user); try { - return service.getDatasetSchema(datasetName); + return await service.getDatasetSchema(datasetName); } catch (error) { throw mapServiceError(error); } @@ -82,7 +82,7 @@ export function createPageDataService(deps = {}) { async function getStats(user, datasetName) { const { service } = await createServiceForUser(user); try { - return service.getDatasetStats(datasetName); + return await service.getDatasetStats(datasetName); } catch (error) { throw mapServiceError(error); } @@ -139,7 +139,8 @@ export function createPageDataService(deps = {}) { async function listDatasets(user) { const { service } = await createServiceForUser(user); - return service.listDatasets().map((dataset) => ({ + const datasets = await service.listDatasets(); + return datasets.map((dataset) => ({ name: dataset.name, table: dataset.table, description: dataset.description, diff --git a/page-data-workspace-bind.mjs b/page-data-workspace-bind.mjs index c9f24b6..9d2ea45 100644 --- a/page-data-workspace-bind.mjs +++ b/page-data-workspace-bind.mjs @@ -171,6 +171,16 @@ function assertRegisteredDatasetTables(userDataSpace, registryDatasets, usage) { } } +async function assertRegisteredDatasetTablesAsync(userDataSpace, registryDatasets, usage) { + for (const dataset of registryDatasets) { + const adapter = { + listTableColumns: () => userDataSpace.listTableColumns(dataset.table), + }; + const columns = await adapter.listTableColumns(); + assertRegisteredDatasetTables({ listTableColumns: () => columns }, [dataset], usage); + } +} + /** * Page Data policy must be derived from a registered SQLite dataset, never * accepted solely from an Agent-supplied policy. This runs before page or @@ -213,6 +223,27 @@ export function resolveRegisteredPageDataPolicy({ }; } +export async function resolveRegisteredPageDataPolicyAsync(options = {}) { + const { html, userId, accessMode, pageDataPolicy = null, userDataSpace } = options; + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) return null; + if (!userDataSpace) throw new Error('缺少 Page Data 数据空间'); + if (pageDataPolicy?.datasets) assertPolicyMatchesHtmlDatasets(html, pageDataPolicy.datasets); + const registryDatasets = await userDataSpace.listDatasets(); + const datasets = buildPageDataPolicyDatasetsFromRegistry({ html, registryDatasets, usage }); + await assertRegisteredDatasetTablesAsync( + userDataSpace, + registryDatasets.filter((dataset) => datasets[dataset.name]), + usage, + ); + return { + ...pageDataPolicy, + ownerUserId: String(pageDataPolicy?.ownerUserId ?? userId).trim(), + accessMode: pageDataPolicy?.accessMode ?? accessMode, + datasets, + }; +} + export async function bindWorkspaceHtmlForPageData({ pool, h5Root, @@ -236,8 +267,8 @@ export async function bindWorkspaceHtmlForPageData({ const normalizedAccessMode = resolvePageDataBindAccess(accessMode); const resolvedPassword = resolvePageDataBindPassword(normalizedAccessMode, password); - const userDataSpace = createUserDataSpaceService({ workspaceRoot }); - const resolvedPageDataPolicy = resolveRegisteredPageDataPolicy({ + const userDataSpace = createUserDataSpaceService({ workspaceRoot, userId, query: pool }); + const resolvedPageDataPolicy = await resolveRegisteredPageDataPolicyAsync({ html: content, userId, accessMode: normalizedAccessMode, diff --git a/page-data-workspace-ensure.mjs b/page-data-workspace-ensure.mjs index 55e36c1..25643d6 100644 --- a/page-data-workspace-ensure.mjs +++ b/page-data-workspace-ensure.mjs @@ -66,7 +66,7 @@ export async function ensureRegisteredDatasetFromHtml({ datasetName, }) { const dataSpace = createUserDataSpaceService({ workspaceRoot, userId, query }); - const existing = dataSpace.getDataset(datasetName); + const existing = await dataSpace.getDataset(datasetName); if (existing) return existing; const insertColumns = inferInsertColumnsFromHtml(html, datasetName); diff --git a/page-data-workspace-ensure.test.mjs b/page-data-workspace-ensure.test.mjs index 8b58c76..bffd100 100644 --- a/page-data-workspace-ensure.test.mjs +++ b/page-data-workspace-ensure.test.mjs @@ -57,7 +57,13 @@ test('evaluatePageDataFinishGuard detects unbound homework html even when user o const evaluation = evaluatePageDataFinishGuard({ publishDir, agentText: '确认发布', - messages: [], + messages: [{ + createdAt: Date.now(), + content: [{ + type: 'toolRequest', + toolCall: { value: { name: 'write_file', arguments: { path: 'public/homework.html' } } }, + }], + }], }); assert.equal(evaluation.pageDataIntent, false); assert.equal(evaluation.structuralPageData, true); diff --git a/postgres-user-data-space-service.mjs b/postgres-user-data-space-service.mjs new file mode 100644 index 0000000..33ccdb4 --- /dev/null +++ b/postgres-user-data-space-service.mjs @@ -0,0 +1,391 @@ +import pg from 'pg'; +import { + buildControlSchemaSql, + deriveUserSpaceNames, + provisionUserSpace, + quotePgIdentifier, +} from './mindspace-userdata-postgres.mjs'; + +const pools = new Map(); +const provisionedUsers = new Set(); +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function ident(value, label = '标识符') { + const text = String(value ?? '').trim(); + if (!IDENTIFIER.test(text)) throw Object.assign(new Error(`${label} 格式无效`), { code: 'invalid_identifier' }); + return text; +} + +function normalizeDataset(raw, fallbackName = null) { + const value = typeof raw === 'string' ? JSON.parse(raw) : raw; + if (!value || typeof value !== 'object') throw Object.assign(new Error('dataset 配置无效'), { code: 'invalid_dataset_config' }); + const name = ident(value.name ?? fallbackName, 'dataset 名称'); + const table = ident(value.table ?? value.table_name, 'dataset 表名'); + const columns = value.columns && typeof value.columns === 'object' ? value.columns : {}; + for (const [action, fields] of Object.entries(columns)) { + if (!Array.isArray(fields)) throw Object.assign(new Error(`dataset 字段白名单无效：${action}`), { code: 'invalid_dataset_config' }); + columns[action] = fields.map((field) => ident(field, `${action} 字段`)); + } + return { + name, + table, + description: String(value.description ?? ''), + actions: Array.isArray(value.actions) ? value.actions.map(String) : [], + columns, + limits: { + maxRowsPerRead: Number(value.limits?.maxRowsPerRead ?? 200), + maxInsertBytes: Number(value.limits?.maxInsertBytes ?? 8192), + }, + }; +} + +function poolConfig(options) { + if (options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL) { + return { connectionString: options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL, max: 5 }; + } + return { + host: options.pgHost ?? process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp', + port: Number(options.pgPort ?? process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433), + database: options.pgDatabase ?? process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev', + user: options.pgUser ?? process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER, + max: 5, + }; +} + +function sharedPool(options) { + if (options.pgPool) return options.pgPool; + const config = poolConfig(options); + const key = JSON.stringify(config); + if (!pools.has(key)) pools.set(key, new pg.Pool(config)); + return pools.get(key); +} + +function rejectSql(sql, { readonly = false } = {}) { + const cleaned = String(sql ?? '').replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trim(); + if (!cleaned || cleaned.length > 20000) throw new Error('SQL 为空或过长'); + if (readonly && !/^(SELECT|WITH)\b/i.test(cleaned)) throw new Error('private_data_query 只允许 SELECT/WITH'); + if (/\b(CREATE|ALTER|DROP)\s+(ROLE|USER|DATABASE|TABLESPACE|EXTENSION)|ALTER\s+SYSTEM|COPY[\s\S]+PROGRAM|SECURITY\s+DEFINER|pg_(read|write)_file|lo_import|dblink/i.test(cleaned)) { + throw new Error('SQL 包含用户空间不允许的操作'); + } + if (/\bSET\s+(ROLE|SESSION_AUTHORIZATION|search_path)\b/i.test(cleaned)) throw new Error('SQL 不允许改变安全上下文'); + return cleaned; +} + +function translateSqliteAgentDdl(sql) { + return sql + .replace(/INTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT/gi, 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY') + .replace(/TEXT\s+NOT\s+NULL\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP') + .replace(/TEXT\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP'); +} + +export function createPostgresUserDataSpaceService(options = {}) { + const names = deriveUserSpaceNames(options.userId); + const pool = sharedPool(options); + const schema = quotePgIdentifier(names.schemaName); + const role = quotePgIdentifier(names.agentRole); + const maxRows = Number(options.maxRows ?? 200); + + async function ensureProvisioned({ force = false } = {}) { + if (provisionedUsers.has(names.userId)) return; + const client = await pool.connect(); + try { + await client.query(buildControlSchemaSql()); + const existing = await client.query('SELECT migration_state FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]); + if (!existing.rows[0]) { + if (!force && String(options.autoProvision ?? process.env.MINDSPACE_USERDATA_AUTO_PROVISION ?? '0') !== '1') { + throw Object.assign(new Error('用户 PG 空间尚未分配'), { code: 'user_space_not_provisioned' }); + } + await provisionUserSpace(client, names.userId, { sourceSqlitePath: options.workspaceRoot ? `${options.workspaceRoot}/.mindspace/private-data.sqlite` : null }); + await client.query('BEGIN'); + try { + await client.query(`CREATE TABLE IF NOT EXISTS ${schema}.__page_data_datasets (name text PRIMARY KEY, table_name text NOT NULL, config_json text NOT NULL, created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)`); + await client.query(`ALTER TABLE ${schema}.__page_data_datasets OWNER TO ${quotePgIdentifier(names.ownerRole)}`); + await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON ${schema}.__page_data_datasets TO ${role}`); + await client.query("UPDATE mindspace_control.user_spaces SET migration_state='cutover',cutover_at=CURRENT_TIMESTAMP,rollback_until=CURRENT_TIMESTAMP + INTERVAL '30 days',updated_at=CURRENT_TIMESTAMP WHERE user_id=$1::uuid", [names.userId]); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } + } + provisionedUsers.add(names.userId); + } finally { + client.release(); + } + } + + async function tenant(fn, { write = false } = {}) { + await ensureProvisioned(); + const client = await pool.connect(); + let quotaBytes = null; + try { + await client.query('BEGIN'); + if (write) { + const quota = await client.query('SELECT quota_bytes FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]); + quotaBytes = Number(quota.rows[0]?.quota_bytes ?? 0); + } + await client.query(`SET LOCAL ROLE ${role}`); + await client.query(`SET LOCAL search_path = ${schema}, pg_catalog`); + await client.query(`SET LOCAL statement_timeout = '${write ? 30 : 10}s'`); + const result = await fn(client); + if (write && quotaBytes > 0) { + const usage = await client.query( + `SELECT COALESCE(sum(pg_total_relation_size(c.oid)),0)::bigint AS bytes + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=$1 AND c.relkind IN ('r','p','m')`, + [names.schemaName], + ); + if (Number(usage.rows[0].bytes) > quotaBytes) { + throw Object.assign(new Error(`用户 PG 空间超过配额：${usage.rows[0].bytes}/${quotaBytes}`), { code: 'quota_exceeded' }); + } + } + await client.query(write ? 'COMMIT' : 'ROLLBACK'); + if (write) { + await client.query( + `INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json) + VALUES($1::uuid,'agent',$2,'postgres_write','success',$3::jsonb)`, + [names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName })], + ); + } + return result; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + if (write) { + await client.query( + `INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json) + VALUES($1::uuid,'agent',$2,'postgres_write','failed',$3::jsonb)`, + [names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName, code: error?.code ?? null })], + ).catch(() => {}); + } + throw error; + } finally { + client.release(); + } + } + + async function listTableColumns(tableName) { + const table = ident(tableName, '表名'); + return tenant(async (client) => { + const result = await client.query( + `SELECT column_name AS name, data_type AS type, + (is_nullable = 'NO') AS "notNull", + (SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) WHERE i.indrelid=($1||'.'||$2)::regclass AND i.indisprimary AND a.attname=c.column_name)) AS pk + FROM information_schema.columns c WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`, + [names.schemaName, table], + ); + return result.rows; + }); + } + + async function getSchema() { + return tenant(async (client) => (await client.query( + `SELECT table_name, column_name, data_type AS type, (is_nullable='NO') AS not_null + FROM information_schema.columns WHERE table_schema=$1 ORDER BY table_name,ordinal_position`, + [names.schemaName], + )).rows); + } + + async function querySql(sql) { + const cleaned = rejectSql(sql, { readonly: true }).replace(/;\s*$/, ''); + return tenant(async (client) => (await client.query(`SELECT * FROM (${cleaned}) AS q LIMIT ${maxRows}`)).rows); + } + + async function executeSql(sql) { + const cleaned = translateSqliteAgentDdl(rejectSql(sql)); + return tenant(async (client) => { + const result = await client.query(cleaned); + return { backend: 'postgres', command: result.command, affectedRows: result.rowCount ?? 0, sizeBytes: null, deltaBytes: null, quotaSynced: false }; + }, { write: true }); + } + + async function getDataset(name) { + const datasetName = ident(name, 'dataset 名称'); + return tenant(async (client) => { + const result = await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets WHERE name=$1 LIMIT 1', [datasetName]); + if (!result.rows[0]) return null; + return { ...normalizeDataset(result.rows[0].config_json, result.rows[0].name), createdAt: result.rows[0].created_at, updatedAt: result.rows[0].updated_at }; + }); + } + + async function listDatasets() { + return tenant(async (client) => (await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets ORDER BY name')).rows.map((row) => ({ + ...normalizeDataset(row.config_json, row.name), createdAt: row.created_at, updatedAt: row.updated_at, + }))); + } + + async function upsertDataset(input) { + const dataset = normalizeDataset(input); + if (!(await listTableColumns(dataset.table)).length) throw Object.assign(new Error(`dataset 对应表不存在：${dataset.table}`), { code: 'table_not_found' }); + await tenant((client) => client.query( + `INSERT INTO __page_data_datasets(name,table_name,config_json,updated_at) VALUES($1,$2,$3,CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET table_name=EXCLUDED.table_name,config_json=EXCLUDED.config_json,updated_at=CURRENT_TIMESTAMP`, + [dataset.name, dataset.table, JSON.stringify(dataset)], + ), { write: true }); + return getDataset(dataset.name); + } + + function assertAction(dataset, action) { + if (!dataset.actions.includes(action)) throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + + async function readRowsForDataset(datasetInput, options = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'read'); + const columns = dataset.columns.read?.map((item) => ident(item, '字段')) ?? []; + if (!columns.length) throw Object.assign(new Error('dataset 未配置可读字段'), { code: 'columns_not_allowed' }); + const limit = Math.min(Math.max(1, Number(options.limit ?? dataset.limits.maxRowsPerRead)), maxRows, dataset.limits.maxRowsPerRead); + const offset = Math.max(0, Number(options.offset ?? 0)); + const order = options.orderBy ? ident(options.orderBy, '排序字段') : columns.includes('id') ? 'id' : null; + if (order && !columns.includes(order)) throw Object.assign(new Error('排序字段未授权'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const filters = []; + if ((await listTableColumns(dataset.table)).some((column) => column.name === 'deleted_at') && !options.includeDeleted) filters.push('deleted_at IS NULL'); + if (options.rowScope?.whereClause) filters.push(`(${options.rowScope.whereClause})`); + const where = filters.length ? ` WHERE ${filters.join(' AND ')}` : ''; + const result = await client.query( + `SELECT ${columns.map(quotePgIdentifier).join(',')} FROM ${quotePgIdentifier(dataset.table)}${where}${order ? ` ORDER BY ${quotePgIdentifier(order)} ${String(options.orderDir).toLowerCase() === 'asc' ? 'ASC' : 'DESC'}` : ''} LIMIT $1 OFFSET $2`, + [limit, offset], + ); + return { dataset, rows: result.rows, limit, offset }; + }); + } + + async function readDatasetRows(name, options) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return readRowsForDataset(dataset, options); + } + + async function getDatasetStats(name) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + assertAction(dataset, 'read'); + return tenant(async (client) => ({ dataset: { name: dataset.name, table: dataset.table }, total: Number((await client.query(`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(dataset.table)}`)).rows[0].count) })); + } + + async function insertWithDataset(dataset, payload, meta = {}) { + assertAction(dataset, 'insert'); + const allowed = new Set(dataset.columns.insert ?? []); + const keys = Object.keys(payload ?? {}); + if (keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('提交包含未授权字段'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const columns = await listTableColumns(dataset.table); + const values = { ...payload }; + if (meta.rowScope?.ownerColumn && columns.some((c) => c.name === meta.rowScope.ownerColumn)) values[meta.rowScope.ownerColumn] ??= meta.rowScope.visitorUserId; + const insertKeys = Object.keys(values).map((key) => ident(key, '字段')); + const result = await client.query( + `INSERT INTO ${quotePgIdentifier(dataset.table)} (${insertKeys.map(quotePgIdentifier).join(',')}) VALUES (${insertKeys.map((_, i) => `$${i + 1}`).join(',')}) RETURNING *`, + insertKeys.map((key) => values[key]), + ); + return { dataset, row: result.rows[0] }; + }, { write: true }); + } + + async function insertDatasetRow(name, payload, meta = {}) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return insertWithDataset(dataset, payload, meta); + } + + async function insertRowForDataset(datasetInput, payload, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return insertWithDataset(dataset, payload, meta); + } + + async function updateRowForDataset(datasetInput, rowId, payload, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'update'); + const id = Number(rowId); + if (!Number.isFinite(id) || id <= 0) throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' }); + const allowed = new Set(dataset.columns.update ?? []); + const keys = Object.keys(payload ?? {}); + if (!keys.length || keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('更新包含未授权字段'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET ${keys.map((key, i) => `${quotePgIdentifier(ident(key, '字段'))}=$${i + 1}`).join(',')} WHERE id=$${keys.length + 1}${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING *`, + [...keys.map((key) => payload[key]), id], + ); + if (!result.rows[0]) throw Object.assign(new Error('无权更新该行或行不存在'), { code: 'row_not_allowed' }); + return { dataset, row: result.rows[0] }; + }, { write: true }); + } + + async function updateDatasetRow(name, rowId, payload, meta) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return updateRowForDataset(dataset, rowId, payload, meta); + } + + async function softDeleteRowForDataset(datasetInput, rowId, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'soft_delete'); + const columns = await listTableColumns(dataset.table); + if (!columns.some((item) => item.name === 'deleted_at')) throw Object.assign(new Error('目标表不支持软删除'), { code: 'soft_delete_unsupported' }); + const id = Number(rowId); + return tenant(async (client) => { + const hasDeletedBy = columns.some((item) => item.name === 'deleted_by'); + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=CURRENT_TIMESTAMP${hasDeletedBy ? ',deleted_by=$2' : ''} WHERE id=$1 AND deleted_at IS NULL${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING id,deleted_at`, + hasDeletedBy ? [id, meta.deletedBy ?? meta.updatedByLabel ?? 'public'] : [id], + ); + if (!result.rows[0]) throw Object.assign(new Error('无权删除该行或行不存在'), { code: 'row_not_allowed' }); + return { dataset, id, deleted: true }; + }, { write: true }); + } + + async function softDeleteDatasetRow(name, rowId, meta) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return softDeleteRowForDataset(dataset, rowId, meta); + } + + async function restoreSoftDeletedRow(name, rowId) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + const columns = await listTableColumns(dataset.table); + return tenant(async (client) => { + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=NULL${columns.some((item) => item.name === 'deleted_by') ? ',deleted_by=NULL' : ''} WHERE id=$1 AND deleted_at IS NOT NULL RETURNING *`, + [Number(rowId)], + ); + if (!result.rows[0]) throw Object.assign(new Error('行不存在或未删除'), { code: 'row_not_allowed' }); + return { dataset, row: result.rows[0], restored: true }; + }, { write: true }); + } + + async function exportDatasetRows(name, { format = 'json', includeDeleted = false, limit = 1000 } = {}) { + const result = await readDatasetRows(name, { includeDeleted, limit: Math.min(Number(limit), 5000) }); + if (String(format).toLowerCase() !== 'csv') return { dataset: name, format: 'json', rows: result.rows, rowCount: result.rows.length }; + const columns = result.dataset.columns.read ?? []; + const escape = (value) => { + const text = value == null ? '' : String(value); + return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; + }; + return { dataset: name, format: 'csv', content: `${columns.join(',')}\n${result.rows.map((row) => columns.map((column) => escape(row[column])).join(',')).join('\n')}\n`, rowCount: result.rows.length }; + } + + async function getDatasetSchema(name) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return { dataset, tableColumns: await listTableColumns(dataset.table) }; + } + + async function getInfo() { + return { name: '用户私有数据空间', backend: 'postgres', database: poolConfig(options).database, schema: names.schemaName, userId: names.userId }; + } + + async function ensureReady() { + await ensureProvisioned({ force: true }); + return getInfo(); + } + + return { + backend: 'postgres', workspaceRoot: options.workspaceRoot, privateDataDb: null, + ensureReady, getInfo, getSchema, querySql, executeSql, listTableColumns, getDataset, listDatasets, upsertDataset, + readRowsForDataset, readDatasetRows, getDatasetStats, getStatsForDataset: async (dataset) => getDatasetStats((await Promise.resolve(dataset)).name), + getDatasetSchema, getSchemaForDataset: async (dataset) => ({ dataset: await Promise.resolve(dataset), tableColumns: await listTableColumns((await Promise.resolve(dataset)).table) }), + insertDatasetRow, insertRowForDataset, updateDatasetRow, updateRowForDataset, + softDeleteDatasetRow, softDeleteRowForDataset, restoreSoftDeletedRow, exportDatasetRows, + }; +} diff --git a/scripts/dev-core-daemon.mjs b/scripts/dev-core-daemon.mjs index 64231f3..9303c9c 100644 --- a/scripts/dev-core-daemon.mjs +++ b/scripts/dev-core-daemon.mjs @@ -8,6 +8,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + applyMemindRuntimeProfile, + describeMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const opsDir = path.join(root, 'ops'); @@ -23,22 +28,6 @@ const plazaPublicBase = ( process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}` ).replace(/\/$/, ''); -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq < 0) continue; - const key = trimmed.slice(0, eq).trim(); - const value = trimmed.slice(eq + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} - -loadEnvFile(path.join(root, '../../.env.local')); -loadEnvFile(path.join(root, '.env')); - const logFile = process.env.MEMIND_DEV_LOG ?? path.join(os.homedir(), 'Library/Logs/memind-dev.log'); @@ -51,6 +40,10 @@ function log(message) { } } +loadMemindEnvFiles(root); +applyMemindRuntimeProfile({ rootDir: root }); +log(`Runtime profile: ${describeMemindRuntimeProfile()}`); + function freePort(port) { try { execSync(`lsof -ti TCP:${port} -sTCP:LISTEN | xargs kill -9`, { stdio: 'ignore' }); diff --git a/scripts/dev-core.mjs b/scripts/dev-core.mjs index ff54454..e00455d 100644 --- a/scripts/dev-core.mjs +++ b/scripts/dev-core.mjs @@ -3,6 +3,11 @@ import { spawn, execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + applyMemindRuntimeProfile, + describeMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const opsDir = path.join(root, 'ops'); @@ -18,21 +23,9 @@ const plazaPublicBase = ( process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}` ).replace(/\/$/, ''); -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq < 0) continue; - const key = trimmed.slice(0, eq).trim(); - const value = trimmed.slice(eq + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} - -loadEnvFile(path.join(root, '../../.env.local')); -loadEnvFile(path.join(root, '.env')); +loadMemindEnvFiles(root); +applyMemindRuntimeProfile({ rootDir: root }); +console.log(`==> Runtime profile: ${describeMemindRuntimeProfile()}`); function freePort(port) { try { diff --git a/scripts/load-env.mjs b/scripts/load-env.mjs index beeecce..d9eb1b6 100644 --- a/scripts/load-env.mjs +++ b/scripts/load-env.mjs @@ -1,20 +1,11 @@ -import fs from 'node:fs'; import path from 'node:path'; - -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const separator = trimmed.indexOf('='); - if (separator < 0) continue; - const key = trimmed.slice(0, separator).trim(); - const value = trimmed.slice(separator + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} +import { + applyMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; export function loadH5Environment(scriptDirectory) { - loadEnvFile(path.join(scriptDirectory, '../../../.env.local')); - loadEnvFile(path.join(scriptDirectory, '../.env')); + const root = path.join(scriptDirectory, '..'); + loadMemindEnvFiles(root); + applyMemindRuntimeProfile({ rootDir: root }); } diff --git a/scripts/memind-runtime-profile.mjs b/scripts/memind-runtime-profile.mjs new file mode 100644 index 0000000..5d7e744 --- /dev/null +++ b/scripts/memind-runtime-profile.mjs @@ -0,0 +1,119 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const MEMIND_RUNTIME_PROFILES = new Set(['local', 'split-service', 'production']); + +/** + * Load Memind env files in standard order. Later files only fill unset keys. + */ +export function loadMemindEnvFiles(rootDir, env = process.env) { + const root = path.resolve(rootDir); + for (const relativePath of ['../../.env.local', '.env']) { + loadEnvFile(path.join(root, relativePath), env); + } +} + +function loadEnvFile(filePath, env = process.env) { + if (!fs.existsSync(filePath)) return; + for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const separator = trimmed.indexOf('='); + if (separator < 0) continue; + const key = trimmed.slice(0, separator).trim(); + const value = trimmed.slice(separator + 1).trim(); + if (!env[key]) env[key] = value; + } +} + +function expandEnvReference(value, env = process.env) { + const raw = String(value ?? '').trim(); + const match = raw.match(/^\$\{([A-Z0-9_]+)\}$/); + if (!match) return raw; + return String(env[match[1]] ?? '').trim() || raw; +} + +export function resolveMemindRuntimeProfile(env = process.env) { + const explicit = String(env.MEMIND_RUNTIME_PROFILE ?? '').trim().toLowerCase(); + if (explicit) { + if (!MEMIND_RUNTIME_PROFILES.has(explicit)) { + throw new Error( + `Unsupported MEMIND_RUNTIME_PROFILE "${explicit}". Expected: ${[...MEMIND_RUNTIME_PROFILES].join(', ')}`, + ); + } + return explicit; + } + if (String(env.NODE_ENV ?? '').trim().toLowerCase() === 'production') { + return 'production'; + } + return 'local'; +} + +/** + * Apply environment-specific defaults after .env files are loaded. + * + * - local: monolith MindSpace adapter, repo-local storage (pnpm dev default) + * - split-service: remote adapter against standalone MindSpace on 8082 + * - production: no overrides; 103 .env is source of truth + */ +export function applyMemindRuntimeProfile({ + rootDir = process.cwd(), + env = process.env, + profile, +} = {}) { + const resolvedProfile = profile ?? resolveMemindRuntimeProfile(env); + env.MEMIND_RUNTIME_PROFILE = resolvedProfile; + + if (resolvedProfile === 'local') { + env.MINDSPACE_SERVER_ADAPTER = 'local'; + if (!env.MINDSPACE_STORAGE_ROOT) { + env.MINDSPACE_STORAGE_ROOT = path.join(path.resolve(rootDir), 'data', 'mindspace'); + } + return { + profile: resolvedProfile, + mindspaceAdapter: 'local', + storageRoot: env.MINDSPACE_STORAGE_ROOT, + }; + } + + if (resolvedProfile === 'split-service') { + env.MINDSPACE_SERVER_ADAPTER = 'remote'; + env.MINDSPACE_REMOTE_BASE_URL = String( + env.MINDSPACE_REMOTE_BASE_URL ?? 'http://127.0.0.1:8082', + ).trim(); + const expandedToken = expandEnvReference(env.MINDSPACE_REMOTE_AUTH_TOKEN, env); + if (expandedToken) { + env.MINDSPACE_REMOTE_AUTH_TOKEN = expandedToken; + } else if (env.TKMIND_SERVER__SECRET_KEY) { + env.MINDSPACE_REMOTE_AUTH_TOKEN = String(env.TKMIND_SERVER__SECRET_KEY).trim(); + } + if (!env.MINDSPACE_STORAGE_ROOT) { + env.MINDSPACE_STORAGE_ROOT = '/Users/john/MindSpace/data/mindspace'; + } + return { + profile: resolvedProfile, + mindspaceAdapter: 'remote', + remoteBaseUrl: env.MINDSPACE_REMOTE_BASE_URL, + storageRoot: env.MINDSPACE_STORAGE_ROOT, + }; + } + + return { + profile: resolvedProfile, + mindspaceAdapter: String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local', + storageRoot: env.MINDSPACE_STORAGE_ROOT ?? null, + }; +} + +export function describeMemindRuntimeProfile(env = process.env) { + const profile = resolveMemindRuntimeProfile(env); + const adapter = String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local'; + const parts = [`MEMIND_RUNTIME_PROFILE=${profile}`, `MINDSPACE_SERVER_ADAPTER=${adapter}`]; + if (adapter === 'remote') { + parts.push(`MINDSPACE_REMOTE_BASE_URL=${env.MINDSPACE_REMOTE_BASE_URL ?? '(unset)'}`); + } + if (env.MINDSPACE_STORAGE_ROOT) { + parts.push(`MINDSPACE_STORAGE_ROOT=${env.MINDSPACE_STORAGE_ROOT}`); + } + return parts.join(', '); +} diff --git a/scripts/memind-runtime-profile.test.mjs b/scripts/memind-runtime-profile.test.mjs new file mode 100644 index 0000000..6e7b73d --- /dev/null +++ b/scripts/memind-runtime-profile.test.mjs @@ -0,0 +1,54 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; +import { + applyMemindRuntimeProfile, + resolveMemindRuntimeProfile, +} from './memind-runtime-profile.mjs'; + +test('resolveMemindRuntimeProfile defaults to local for dev', () => { + assert.equal(resolveMemindRuntimeProfile({}), 'local'); + assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'development' }), 'local'); +}); + +test('resolveMemindRuntimeProfile honors explicit profile and production NODE_ENV', () => { + assert.equal(resolveMemindRuntimeProfile({ MEMIND_RUNTIME_PROFILE: 'split-service' }), 'split-service'); + assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'production' }), 'production'); +}); + +test('applyMemindRuntimeProfile local forces monolith adapter and repo storage', () => { + const env = { + MINDSPACE_SERVER_ADAPTER: 'remote', + MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082', + }; + const result = applyMemindRuntimeProfile({ + rootDir: '/tmp/memind', + env, + profile: 'local', + }); + assert.equal(result.profile, 'local'); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'local'); + assert.match(env.MINDSPACE_STORAGE_ROOT, /\/tmp\/memind\/data\/mindspace$/); +}); + +test('applyMemindRuntimeProfile split-service expands auth token reference', () => { + const env = { + TKMIND_SERVER__SECRET_KEY: 'local-dev-secret', + MINDSPACE_REMOTE_AUTH_TOKEN: '${TKMIND_SERVER__SECRET_KEY}', + }; + applyMemindRuntimeProfile({ env, profile: 'split-service' }); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote'); + assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'local-dev-secret'); +}); + +test('applyMemindRuntimeProfile production does not override adapter', () => { + const env = { + MINDSPACE_SERVER_ADAPTER: 'remote', + MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082', + MINDSPACE_REMOTE_AUTH_TOKEN: 'prod-token', + }; + const result = applyMemindRuntimeProfile({ env, profile: 'production' }); + assert.equal(result.profile, 'production'); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote'); + assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'prod-token'); +}); diff --git a/scripts/migrate-mindspace-sqlite-to-postgres.mjs b/scripts/migrate-mindspace-sqlite-to-postgres.mjs new file mode 100644 index 0000000..7fb698f --- /dev/null +++ b/scripts/migrate-mindspace-sqlite-to-postgres.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import { + buildControlSchemaSql, + createSqliteSnapshot, + inspectSqliteDatabase, + migrateSqliteSnapshot, + provisionUserSpace, +} from '../mindspace-userdata-postgres.mjs'; + +function usage() { + return ` +Usage: + node scripts/migrate-mindspace-sqlite-to-postgres.mjs --source --user-id [--apply] [--replace-shadow] + +Default is read-only dry-run. --apply requires MINDSPACE_USERDATA_PG_URL. +The command creates a consistent SQLite snapshot and never modifies the source SQLite. +`.trim(); +} + +export function parseArgs(argv = []) { + const options = { apply: false, replaceShadow: false, source: '', userId: '', help: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--apply') options.apply = true; + else if (arg === '--replace-shadow') options.replaceShadow = true; + else if (arg === '--source') options.source = argv[++index] ?? ''; + else if (arg === '--user-id') options.userId = argv[++index] ?? ''; + else if (arg === '--help' || arg === '-h') options.help = true; + else throw new Error(`未知参数：${arg}`); + } + if (!options.help && (!options.source || !options.userId)) throw new Error('--source 和 --user-id 必填'); + return options; +} + +export async function run(argv = process.argv.slice(2), env = process.env) { + const options = parseArgs(argv); + if (options.help) { + console.log(usage()); + return { mode: 'help' }; + } + const inspected = inspectSqliteDatabase(options.source); + const sourceRows = inspected.tables.reduce((total, table) => total + table.rows.length, 0); + if (!options.apply) { + console.log(JSON.stringify({ + mode: 'dry-run', + userId: options.userId, + source: inspected.path, + sizeBytes: inspected.sizeBytes, + tables: inspected.tables.map((table) => ({ name: table.name, rows: table.rows.length })), + sourceRows, + note: '未修改 SQLite 或 PostgreSQL；使用 --apply 才会创建影子空间。', + }, null, 2)); + return { mode: 'dry-run', tableCount: inspected.tables.length, sourceRows }; + } + if (!env.MINDSPACE_USERDATA_PG_URL) throw new Error('--apply requires MINDSPACE_USERDATA_PG_URL'); + const snapshotPath = createSqliteSnapshot(options.source); + const snapshot = inspectSqliteDatabase(snapshotPath); + const { default: pg } = await import('pg'); + const client = new pg.Client({ connectionString: env.MINDSPACE_USERDATA_PG_URL }); + try { + await client.connect(); + await client.query(buildControlSchemaSql()); + await provisionUserSpace(client, options.userId, { + sourceSqlitePath: inspected.path, + }); + const result = await migrateSqliteSnapshot(client, snapshot, options.userId, { + replaceShadow: options.replaceShadow, + sourcePath: inspected.path, + }); + if (result.sourceRows !== result.targetRows) throw new Error('迁移行数校验失败'); + console.log(JSON.stringify({ mode: 'shadow', ...result }, null, 2)); + return { mode: 'shadow', ...result }; + } finally { + await client.end().catch(() => {}); + fs.rmSync(snapshotPath, { force: true }); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + run().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs b/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs new file mode 100644 index 0000000..063561c --- /dev/null +++ b/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { parseArgs } from './migrate-mindspace-sqlite-to-postgres.mjs'; + +test('migration CLI defaults to dry-run', () => { + assert.deepEqual(parseArgs(['--source', '/tmp/a.sqlite', '--user-id', 'user-id']), { + apply: false, + replaceShadow: false, + source: '/tmp/a.sqlite', + userId: 'user-id', + help: false, + }); +}); + +test('migration CLI requires explicit source and user', () => { + assert.throws(() => parseArgs([]), /--source 和 --user-id 必填/); +}); + +test('migration CLI recognizes explicit apply', () => { + assert.equal(parseArgs(['--apply', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).apply, true); +}); + +test('migration CLI requires explicit replace-shadow for destructive reruns', () => { + assert.equal(parseArgs(['--replace-shadow', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).replaceShadow, true); +}); diff --git a/scripts/verify-local-pg-questionnaire-e2e.mjs b/scripts/verify-local-pg-questionnaire-e2e.mjs new file mode 100644 index 0000000..0582a3c --- /dev/null +++ b/scripts/verify-local-pg-questionnaire-e2e.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +/** + * Local-only E2E proof for the questionnaire -> Page Data API -> PostgreSQL path. + * + * The probe uses an existing local user space, creates a uniquely named survey + * table/dataset and page policy, submits one answer through the public HTTP API, + * verifies the row directly in PostgreSQL, and removes all probe objects. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import express from 'express'; +import pg from 'pg'; +import { attachPageDataRoutes } from '../page-data-routes.mjs'; +import { createPageDataService } from '../page-data-service.mjs'; +import { createPageDataPublicService } from '../page-data-public-service.mjs'; +import { deletePageAccessPolicy, writePageAccessPolicy } from '../page-data-policy-store.mjs'; +import { createUserDataSpaceService } from '../user-data-space-service.mjs'; +import { deriveUserSpaceNames, quotePgIdentifier } from '../mindspace-userdata-postgres.mjs'; +import { loadH5Environment } from './load-env.mjs'; + +loadH5Environment(import.meta.dirname); + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const userId = process.argv[2] ?? '1c99b83b-0454-474f-a5d2-129d34506a32'; +const workspaceRoot = path.join(repoRoot, 'MindSpace', userId); +const suffix = `${Date.now()}_${process.pid}`; +const tableName = `survey_e2e_${suffix}`; +const datasetName = tableName; +const pageId = `page-survey-e2e-${suffix}`; +const names = deriveUserSpaceNames(userId); +const policyPath = path.join(workspaceRoot, '.mindspace', 'page-data-policies', `${pageId}.json`); +const logPath = path.join(workspaceRoot, '.mindspace', 'page-data-logs', `${pageId}.jsonl`); + +assert.equal(process.env.MINDSPACE_USERDATA_BACKEND, 'postgres', '本机未启用 PostgreSQL 用户数据后端'); +assert.ok(fs.existsSync(workspaceRoot), `用户工作区不存在: ${workspaceRoot}`); + +const pgPool = new pg.Pool({ + host: process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp', + port: Number(process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433), + database: process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev', + user: process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER, + max: 2, +}); + +function publicationPool() { + return { + async query(sql) { + if (sql.includes('FROM h5_publish_records')) { + return [[{ + id: `publication-${suffix}`, + user_id: userId, + page_id: pageId, + access_mode: 'public', + password_hash: null, + status: 'online', + expires_at: null, + }]]; + } + return [[]]; + }, + }; +} + +function buildApp() { + const fakePublicationPool = publicationPool(); + const publicService = createPageDataPublicService({ + getPool: () => fakePublicationPool, + resolveWorkspaceRootForOwner: () => workspaceRoot, + }); + const api = express.Router(); + api.use(express.json()); + attachPageDataRoutes(api, { + sendData: (res, _req, data, status = 200) => res.status(status).json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }), + getPageDataPublicService: () => publicService, + }); + const app = express(); + app.set('trust proxy', true); + app.use('/api', api); + return app; +} + +async function submitQuestionnaire(app, row) { + const server = app.listen(0, '127.0.0.1'); + await new Promise((resolve, reject) => { + server.once('listening', resolve); + server.once('error', reject); + }); + try { + const { port } = server.address(); + const apiPath = `/api/public/pages/${pageId}/data/${datasetName}/rows`; + const response = await fetch(`http://127.0.0.1:${port}${apiPath}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(row), + }); + return { apiPath, status: response.status, body: await response.json() }; + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +async function main() { + const ownerService = createUserDataSpaceService({ workspaceRoot, userId }); + let tableCreated = false; + try { + // Equivalent to the Agent's private_data_execute + register_dataset + bind policy flow. + await ownerService.executeSql(`CREATE TABLE ${tableName} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + satisfaction TEXT NOT NULL, + favorite_feature TEXT NOT NULL, + suggestion TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now', 'localtime')) + )`); + tableCreated = true; + const dataset = await ownerService.upsertDataset({ + name: datasetName, + table: tableName, + description: '本机 PG 问卷端到端验证', + actions: ['read', 'insert'], + columns: { + read: ['id', 'satisfaction', 'favorite_feature', 'suggestion', 'created_at'], + insert: ['satisfaction', 'favorite_feature', 'suggestion'], + }, + }); + writePageAccessPolicy(workspaceRoot, { + pageId, + ownerUserId: userId, + accessMode: 'public', + datasets: { + [datasetName]: { + insert: true, + read: false, + columns: { insert: ['satisfaction', 'favorite_feature', 'suggestion'] }, + }, + }, + }); + + const answer = { + satisfaction: '非常满意', + favorite_feature: '个人数据报表', + suggestion: '增加按月趋势分析', + }; + const submitted = await submitQuestionnaire(buildApp(), answer); + assert.equal(submitted.status, 201, JSON.stringify(submitted.body)); + assert.equal(submitted.body.data.dataset, datasetName); + assert.equal(submitted.body.data.row.satisfaction, answer.satisfaction); + + const direct = await pgPool.query( + `SELECT id,satisfaction,favorite_feature,suggestion,created_at + FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(tableName)} + ORDER BY id DESC LIMIT 1`, + ); + assert.equal(direct.rowCount, 1); + assert.equal(direct.rows[0].satisfaction, answer.satisfaction); + assert.equal(direct.rows[0].favorite_feature, answer.favorite_feature); + assert.equal(direct.rows[0].suggestion, answer.suggestion); + + const registered = await pgPool.query( + `SELECT name,table_name FROM ${quotePgIdentifier(names.schemaName)}.__page_data_datasets WHERE name=$1`, + [datasetName], + ); + assert.equal(registered.rowCount, 1); + + console.log(JSON.stringify({ + result: 'PASS', + localOnly: true, + userId, + schema: names.schemaName, + dataset: { name: dataset.name, table: dataset.table, actions: dataset.actions }, + pageDataApi: { method: 'POST', path: submitted.apiPath, status: submitted.status }, + postgresRow: direct.rows[0], + checks: [ + 'Agent-style table creation succeeded in the user schema', + 'Dataset registry was stored in PostgreSQL', + 'Questionnaire submission used the public Page Data HTTP API', + 'Submitted answers matched a direct PostgreSQL query', + ], + }, null, 2)); + } finally { + if (tableCreated) { + await ownerService.executeSql(`DELETE FROM __page_data_datasets WHERE name='${datasetName}'`).catch(() => {}); + await ownerService.executeSql(`DROP TABLE IF EXISTS ${tableName}`).catch(() => {}); + } + deletePageAccessPolicy(workspaceRoot, pageId); + fs.rmSync(logPath, { force: true }); + fs.rmSync(policyPath, { force: true }); + await pgPool.end(); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + process.exitCode = 1; +}); diff --git a/server.mjs b/server.mjs index 31f87ad..0a58eb8 100644 --- a/server.mjs +++ b/server.mjs @@ -14,6 +14,7 @@ import { } from './auth.mjs'; import { createDbPool, initSchema, isDatabaseConfigured } from './db.mjs'; import { createWorkspacePageDeliverService } from './mindspace-workspace-page-deliver.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; import { createToolGateway } from './tool-gateway.mjs'; import { createAgentRunGateway } from './agent-run-gateway.mjs'; import { @@ -164,6 +165,7 @@ import { registerChatDocxArtifactForConversation, } from './mindspace-chat-docx-package.mjs'; import { scanContent } from './mindspace-content-scan.mjs'; +import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; import { renderImageAssetViewerHtml, wantsInlineImageViewer } from './mindspace-asset-preview.mjs'; import { DEFAULT_IMAGE_UPLOAD_MAX_BYTES } from './user-image-normalize.mjs'; import { createRechargeService } from './billing-recharge.mjs'; @@ -535,6 +537,10 @@ async function bootstrapUserAuth() { h5Root: __dirname, defaultSignupBalanceCents: Number(process.env.H5_SIGNUP_BALANCE_CENTS ?? 500), subscriptionService, + provisionUserDataSpace: async ({ userId, workspaceRoot }) => { + const service = createUserDataSpaceService({ workspaceRoot, userId, query: pool }); + return service.ensureReady(); + }, }); sessionAccess = createSessionAccess({ userAuth, enabled: isSessionBrokerEnabled() }); if (sessionAccess.enabled) { @@ -622,6 +628,12 @@ async function bootstrapUserAuth() { expireStaleUploadsIntervalMs: 5 * 60 * 1000, }); await userAuth.ensureAdminUser(); + const userDataSpaceBackfill = await userAuth.ensureAllUserDataSpaces(); + if (userDataSpaceBackfill.errors.length > 0) { + console.warn( + `[PageData] PG space backfill incomplete: ${userDataSpaceBackfill.errors.length} user(s) failed`, + ); + } llmProviderService = createLlmProviderService(pool, { apiTarget: API_TARGET, apiTargets: API_TARGETS, @@ -717,8 +729,24 @@ async function bootstrapUserAuth() { messages: [userMessage], }); }, - syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId), + syncUserPagesOnSuccess: async ({ userId, sessionId, runStartedAtMs }) => + syncUserGeneratedPages(userId, { sessionId, sinceMs: runStartedAtMs }), isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0, + validateRunDeliverables: async ({ userId, deliverables }) => { + const publishDir = resolveMindSpaceUserPublishDir(__dirname, { id: userId }); + const violations = scanWorkspaceFilesForProhibitedBrowserStorage({ + publishDir, + relativePaths: (deliverables?.pages ?? []) + .map((page) => page.workspaceRelativePath) + .filter(Boolean), + }); + return { + errors: violations.map((violation) => ({ + code: 'browser_storage_forbidden', + message: `${violation.relativePath} 使用 ${violation.apis.join(', ')}`, + })), + }; + }, autoDispatch: ['1', 'true', 'yes', 'on'].includes( String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(), ), @@ -3231,10 +3259,36 @@ async function resolveExistingSavedPage(userId, { sessionId, messageId, relative const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']); // REGRESSION GUARD: mindspace-page-sync-thumbnail — remote 也经 pageSyncService RPC 同步 public HTML -async function syncUserGeneratedPages(userId) { +async function listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs = null } = {}) { + if (!authPool || !userId || !sessionId) return []; + const sinceClause = sinceMs == null ? '' : 'AND ca.created_at >= ?'; + const params = sinceMs == null ? [userId, sessionId] : [userId, sessionId, sinceMs]; + const [rows] = await authPool.query( + `SELECT ca.display_name + FROM h5_conversation_artifacts ca + JOIN h5_conversation_packages cp ON cp.id = ca.package_id + WHERE cp.user_id = ? + AND cp.session_id = ? + AND ca.artifact_kind = 'public_html' + ${sinceClause} + ORDER BY ca.sort_order ASC, ca.created_at ASC`, + params, + ); + return [...new Set((rows ?? []) + .map((row) => normalizeWorkspaceRelativePath(`public/${String(row.display_name ?? '').trim()}`)) + .filter((relativePath) => relativePath?.startsWith('public/') && relativePath.toLowerCase().endsWith('.html')))]; +} + +async function syncUserGeneratedPages(userId, { sessionId = null, sinceMs = null } = {}) { if (!userId) return; + // Agent-run/Finish delivery must stay scoped to the current conversation. + // A stale Page Data page elsewhere in the user's workspace must not turn a + // successfully completed current task into a failed run. + const pageDataRelativePaths = sessionId + ? await listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs }) + : null; if (workspacePageDeliver?.syncAndDeliver) { - return await workspacePageDeliver.syncAndDeliver(userId); + return await workspacePageDeliver.syncAndDeliver(userId, { pageDataRelativePaths }); } if (!mindSpacePageSync) return; return await mindSpacePageSync.syncUserGeneratedPages(userId); @@ -5099,7 +5153,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { .filter(Boolean) .join('\n') : ''; - await syncUserGeneratedPages(uid); + await syncUserGeneratedPages(uid, { sessionId: sid }); await maybeRepairPageDataAfterFinish({ sessionId: sid, userId: uid, diff --git a/skills/page-data-collect/SKILL.md b/skills/page-data-collect/SKILL.md index ba84841..701a02f 100644 --- a/skills/page-data-collect/SKILL.md +++ b/skills/page-data-collect/SKILL.md @@ -1,11 +1,11 @@ --- name: page-data-collect -description: 在 MindSpace 页面中收集、存储并管理结构化数据（问卷、报名、台账、后台查看），基于 Page Data API 与用户私有 SQLite +description: 在 MindSpace 页面中收集、存储并管理结构化数据（问卷、报名、台账、后台查看），基于 Page Data API 与用户隔离的 PostgreSQL 空间 --- # 页面数据收集（Page Data Collect） -在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互，并将提交持久化到当前用户的 `.mindspace/private-data.sqlite`，通过平台 **Page Data API** 受控读写。 +在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互，并将提交持久化到当前用户隔离的 PostgreSQL schema，通过平台 **Page Data API** 受控读写。运行时禁止创建、读取或回退到用户 SQLite。 详细 API 说明见工作区外文档 `docs/page-data-api-usage.md`（Memind 仓库）。 @@ -25,7 +25,7 @@ description: 在 MindSpace 页面中收集、存储并管理结构化数据（ ```text Agent 可以建模 SQL 并注册 dataset； HTML 页面只能调用 Page Data API（page-data-client.js）； -禁止自建 Express / 独立端口 / 直接暴露 SQLite 文件路径。 +禁止自建 Express / 独立端口 / 直接暴露数据库连接；禁止 SQLite / localStorage 数据回退。 ``` ## 方案选择：默认快车道 + 能力分支（必做） @@ -36,7 +36,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； 用户描述需求 → 匹配能力分支（下表 A/B/C/D，默认 A） → 仅冲突或无法匹配时，用 1 题让用户选分支或改口令 - → 输出「方案摘要」供确认（用户可只改口令/选项） + → 输出简短「方案摘要」；用户已明确要求直接创建/完成/发布时视为已确认并立即执行 → 再 load_skill / 建表 / write_file / bind ``` @@ -50,7 +50,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； | 提交后修改 | 不支持（一次性） | | 页面 | `public/*-survey.html` + `public/*-admin.html`（两文件各 bind 一次） | -用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A，先展示摘要再开工**。 +用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A**。若用户已说「直接做」「完整创建并发布」「不用询问」「持续推进」等终态指令，摘要后必须在同一轮立即调用工具开工，禁止停下来等待再次确认。 口令规则： @@ -83,6 +83,9 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； ```text - public：可匿名 insert；服务端禁止 update / softDelete - password：所有 API 须先 authenticate；口令 ≥8 位且须写入发布记录 +- `public` 策略中的 `insert: true` 等价于任何访客都能直接调用 API 写入；前端口令、隐藏按钮和 JavaScript 判断都不是权限控制 +- password 页面必须调用 `client.authenticate(password)` 获取服务端 token；禁止在 HTML 中硬编码口令或只做 `password === "..."` 比较 +- 同一公开页若要求「所有人可评价、只有所有者可写正文」：正文 dataset 在公开页必须 `insert: false`，评价 dataset 才能 `insert: true`；所有者写正文必须使用独立 `password` 作者页或 `login_required` 页面 - 同页不能同时「匿名 insert」+「口令 read 全量」→ 须拆前台 + 后台（方案 A/E） - password 无法区分访客身份；「只改自己的」须 login_required + own_rows（方案 B） - 匿名提交后「凭链接改自己的」无内置能力，须改 B 或接受一次性提交 @@ -101,7 +104,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； - 页面：public/xxx.html + public/xxx-admin.html - 内容：（由你的描述生成题目/字段/报表） -确认后开始建表与写页面。若需登录提交或团队共改，请说明，我换成 B/C 分支。 +若用户尚未授权创建，确认后开始建表与写页面；若用户已明确要求创建/完成/发布，此摘要仅用于告知，后面必须紧接工具调用，不得结束回复等待确认。 ``` ### 分支切换示例 @@ -121,8 +124,8 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； ```text load_skill → page-data-collect -→ 匹配分支（默认 A）→ 方案摘要 → 用户确认或仅改口令/选项 -→ 禁止未确认就 bind；禁止空转计划不调用工具 +→ 匹配分支（默认 A）→ 方案摘要 → 用户确认，或从明确的创建/完成/发布指令判定已确认 +→ 禁止未获创建授权就 bind；已获授权时禁止空转计划、重复确认或不调用工具 ``` ### 2. 数据层：建表 + 注册 dataset @@ -262,7 +265,7 @@ CREATE TABLE IF NOT EXISTS survey_responses ( ## 严格禁止 -1. **禁止**在方案摘要未确认时建表 / bind / 发布（默认 A 也须展示摘要；用户明确「按默认做」视为确认） +1. **禁止**在尚未获得创建授权时建表 / bind / 发布；用户明确要求「创建、完成、发布、直接做、不用询问、持续推进」均视为确认，禁止再次停下来询问 2. **禁止**让 LLM 自行发明 accessMode/拆页/口令；必须落在分支 A～E 与默认口令规则内 3. **禁止**连续两轮仅输出计划、不调用 `load_skill` / `list_dir` / `write_file` / `private_data_execute` / `bind` 4. **禁止**接受 <8 位发布口令；用户未指定口令时后台默认 **`88888888`** @@ -274,7 +277,7 @@ CREATE TABLE IF NOT EXISTS survey_responses ( 10. **禁止**只 `CREATE TABLE` 而不 `private_data_register_dataset` 11. **禁止**未配置 `private_data_set_page_policy` 就让页面调用公开 API 12. **禁止**先发布占位页（如 `

问卷页面

`）再让用户访问 `/u/.../pages/...` -13. **禁止**在 Page Data 页面中使用 `localStorage` / `sessionStorage` 存提交记录或做 API fallback +13. **禁止**在 Agent 生成的 HTML/JavaScript 中使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据或做 API fallback；所有需持久化的数据必须进入当前用户专属 PostgreSQL schema，禁止 SQLite、静态 JSON/JS 文件和内存 fallback ## 交付前自检 diff --git a/skills/static-page-publish/SKILL.md b/skills/static-page-publish/SKILL.md index 7a2802f..aebce39 100644 --- a/skills/static-page-publish/SKILL.md +++ b/skills/static-page-publish/SKILL.md @@ -27,6 +27,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报 10. **禁止**在 HTML 中用 CDN `https://...` 引用 `