From 8d0155346941459f2cc74e255e16a7f025e87fef Mon Sep 17 00:00:00 2001 From: john Date: Wed, 8 Jul 2026 12:44:23 +0800 Subject: [PATCH] feat(page-data): add private and public dataset API (phase 1-3) Extract UserDataSpaceService for shared SQLite access, wire logged-in Page Data routes, and add public insert plus password-token read/update/delete with policy storage, rate limits, and regression tests. Co-authored-by: Cursor --- mindspace-sandbox-mcp.mjs | 297 ++++++-------- mindspace-sandbox-mcp.test.mjs | 60 +++ package.json | 2 +- page-access-policy.mjs | 121 ++++++ page-access-policy.test.mjs | 67 ++++ page-data-integration.test.mjs | 254 ++++++++++++ page-data-policy-store.mjs | 52 +++ page-data-public-service.mjs | 373 ++++++++++++++++++ page-data-public-service.test.mjs | 176 +++++++++ page-data-routes.mjs | 245 ++++++++++++ page-data-routes.test.mjs | 140 +++++++ page-data-service.mjs | 109 ++++++ page-data-session-store.mjs | 82 ++++ scripts/run-memind-tests.mjs | 12 + server.mjs | 29 +- user-data-space-service.mjs | 618 ++++++++++++++++++++++++++++++ user-data-space-service.test.mjs | 139 +++++++ 17 files changed, 2587 insertions(+), 189 deletions(-) create mode 100644 page-access-policy.mjs create mode 100644 page-access-policy.test.mjs create mode 100644 page-data-integration.test.mjs create mode 100644 page-data-policy-store.mjs create mode 100644 page-data-public-service.mjs create mode 100644 page-data-public-service.test.mjs create mode 100644 page-data-routes.mjs create mode 100644 page-data-routes.test.mjs create mode 100644 page-data-service.mjs create mode 100644 page-data-session-store.mjs create mode 100644 user-data-space-service.mjs create mode 100644 user-data-space-service.test.mjs diff --git a/mindspace-sandbox-mcp.mjs b/mindspace-sandbox-mcp.mjs index 8152891..8ee3dd4 100644 --- a/mindspace-sandbox-mcp.mjs +++ b/mindspace-sandbox-mcp.mjs @@ -17,6 +17,9 @@ import mysql from 'mysql2/promise'; import { createScheduleService } from './schedule-service.mjs'; import { resolveScheduleTimestamp } from './schedule-time.mjs'; import { renderLongImage } from './mindspace-long-image.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; +import { writePageAccessPolicy } from './page-data-policy-store.mjs'; +import { normalizePageAccessPolicy } from './page-access-policy.mjs'; const SANDBOX_ROOT = process.argv[2]?.trim() || process.env.SANDBOX_ROOT?.trim(); if (!SANDBOX_ROOT) { @@ -27,7 +30,6 @@ if (!SANDBOX_ROOT) { const SANDBOX = path.resolve(SANDBOX_ROOT); const SANDBOX_MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); const PRIVATE_DATA_DIR = path.join(SANDBOX, '.mindspace'); -const PRIVATE_DATA_DB = path.join(PRIVATE_DATA_DIR, 'private-data.sqlite'); const SQLITE_BIN = process.env.SQLITE_BIN?.trim() || 'sqlite3'; const PRIVATE_DATA_MAX_BYTES = Number(process.env.PRIVATE_DATA_MAX_BYTES ?? 20 * 1024 * 1024); const PRIVATE_DATA_QUERY_TIMEOUT_MS = Number(process.env.PRIVATE_DATA_QUERY_TIMEOUT_MS ?? 5000); @@ -242,10 +244,60 @@ const ALL_TOOLS = [ required: ['sql'], }, }, + { + name: 'private_data_register_dataset', + description: + '注册或更新页面可访问的 dataset。Agent 建表后应注册 dataset，供 HTML 页面通过 Page Data API 受控读写。', + inputSchema: { + type: 'object', + properties: { + name: { type: 'string', description: 'dataset 名称，例如 registrations' }, + table: { type: 'string', description: '映射的 SQLite 表名' }, + description: { type: 'string', description: 'dataset 说明，可选' }, + actions: { + type: 'array', + items: { type: 'string' }, + description: '允许动作，例如 read、insert、update、soft_delete', + }, + columns: { + type: 'object', + description: '各动作允许的字段白名单，例如 { read: ["id","name"], insert: ["name"] }', + }, + limits: { + type: 'object', + description: '限制，例如 { maxRowsPerRead: 100, maxInsertBytes: 8192 }', + }, + }, + required: ['name', 'table', 'actions', 'columns'], + }, + }, + { + name: 'private_data_set_page_policy', + description: + '为已发布页面配置 Page Data 访问策略。策略保存在工作区 .mindspace/page-data-policies/ 下，供公开页面通过 Page Data API 受控读写。', + inputSchema: { + type: 'object', + properties: { + pageId: { type: 'string', description: '页面 ID（h5_page_records.id）' }, + ownerUserId: { type: 'string', description: '页面 owner 用户 ID，可选，默认当前会话用户' }, + accessMode: { + type: 'string', + description: '访问模式：public、password、login_required', + }, + datasets: { + type: 'object', + description: + 'dataset 授权，例如 { registrations: { insert: true, columns: { insert: ["name","phone"] } } }', + }, + }, + required: ['pageId', 'accessMode', 'datasets'], + }, + }, ]; let quotaPool = null; let scheduleService = null; +let userDataSpaceService = null; function isQuotaSyncConfigured() { return Boolean( @@ -352,169 +404,19 @@ if (isScheduleConfigured()) { const TOOLS = ALLOWED_TOOLS ? ALL_TOOLS.filter((t) => ALLOWED_TOOLS.has(t.name)) : ALL_TOOLS; -function ensurePrivateDataDb() { - fs.mkdirSync(PRIVATE_DATA_DIR, { recursive: true }); - if (!fs.existsSync(PRIVATE_DATA_DB)) { - runSqlite(['-batch', PRIVATE_DATA_DB, 'PRAGMA journal_mode=WAL; PRAGMA user_version = 1;']); +function getUserDataSpaceService() { + if (!userDataSpaceService) { + userDataSpaceService = createUserDataSpaceService({ + workspaceRoot: SANDBOX, + userId: PRIVATE_DATA_USER_ID || null, + query: getQuotaPool(), + sqliteBin: SQLITE_BIN, + maxBytes: PRIVATE_DATA_MAX_BYTES, + queryTimeoutMs: PRIVATE_DATA_QUERY_TIMEOUT_MS, + maxRows: PRIVATE_DATA_MAX_ROWS, + }); } - return PRIVATE_DATA_DB; -} - -function privateDataSize() { - let total = 0; - for (const file of fs.existsSync(PRIVATE_DATA_DIR) ? fs.readdirSync(PRIVATE_DATA_DIR) : []) { - if (file === 'private-data.sqlite' || file.startsWith('private-data.sqlite-')) { - total += fs.statSync(path.join(PRIVATE_DATA_DIR, file)).size; - } - } - return total; -} - -function runSqlite(args) { - return execFileSync(SQLITE_BIN, args, { - encoding: 'utf8', - timeout: PRIVATE_DATA_QUERY_TIMEOUT_MS, - maxBuffer: 1024 * 1024, - }); -} - -function sqliteScalar(sql) { - return Number(runSqlite(['-batch', '-noheader', PRIVATE_DATA_DB, sql]).trim() || 0); -} - -async function getQuotaState({ forUpdate = false, conn = null } = {}) { - const pool = getQuotaPool(); - if (!pool) return null; - const db = conn ?? pool; - const [rows] = await db.query( - `SELECT id, quota_bytes, used_bytes, reserved_bytes, status - FROM h5_user_spaces - WHERE user_id = ? - LIMIT 1 ${forUpdate ? 'FOR UPDATE' : ''}`, - [PRIVATE_DATA_USER_ID], - ); - const row = rows[0]; - if (!row) return null; - const quotaBytes = Number(row.quota_bytes ?? 0); - const usedBytes = Number(row.used_bytes ?? 0); - const reservedBytes = Number(row.reserved_bytes ?? 0); - return { - id: row.id, - status: row.status, - quotaBytes, - usedBytes, - reservedBytes, - availableBytes: Math.max(0, quotaBytes - usedBytes - reservedBytes), - }; -} - -async function sqliteMaxPagePragmaForQuota(currentBytes) { - const quota = await getQuotaState(); - if (!quota) return ''; - if (quota.status !== 'active') throw new Error('用户空间不可写'); - if (quota.availableBytes <= 0 && currentBytes >= PRIVATE_DATA_MAX_BYTES) { - throw new Error('用户私有数据空间配额不足'); - } - const allowedSize = Math.min(PRIVATE_DATA_MAX_BYTES, currentBytes + quota.availableBytes); - const pageSize = sqliteScalar('PRAGMA page_size;') || 4096; - const currentPages = sqliteScalar('PRAGMA page_count;') || 1; - const maxPages = Math.max(currentPages, Math.max(1, Math.floor(allowedSize / pageSize))); - return `PRAGMA max_page_count=${maxPages};`; -} - -async function syncPrivateDataQuota(deltaBytes) { - const pool = getQuotaPool(); - if (!pool || !deltaBytes) return null; - const conn = await pool.getConnection(); - try { - await conn.beginTransaction(); - const quota = await getQuotaState({ forUpdate: true, conn }); - if (!quota) { - await conn.rollback(); - return null; - } - if (quota.status !== 'active') throw new Error('用户空间不可写'); - if (deltaBytes > quota.availableBytes) { - throw new Error(`用户私有数据空间配额不足：需要 ${deltaBytes} 字节，可用 ${quota.availableBytes} 字节`); - } - await conn.query( - `UPDATE h5_user_spaces - SET used_bytes = GREATEST(0, used_bytes + ?), updated_at = ? - WHERE id = ? AND user_id = ?`, - [deltaBytes, Date.now(), quota.id, PRIVATE_DATA_USER_ID], - ); - await conn.commit(); - return { deltaBytes }; - } catch (err) { - await conn.rollback(); - throw err; - } finally { - conn.release(); - } -} - -function stripSqlComments(sql) { - return String(sql ?? '') - .replace(/--.*$/gm, '') - .replace(/\/\*[\s\S]*?\*\//g, '') - .trim(); -} - -function rejectDangerousSql(sql, { readonly = false } = {}) { - const cleaned = stripSqlComments(sql); - if (!cleaned) throw new Error('SQL 不能为空'); - if (cleaned.length > 20000) throw new Error('SQL 过长'); - if (/^\s*\./m.test(cleaned)) { - throw new Error('SQL 不允许使用 sqlite3 dot command'); - } - if (/\b(ATTACH|DETACH|LOAD_EXTENSION|VACUUM\s+INTO)\b/i.test(cleaned)) { - throw new Error('SQL 包含用户私有数据空间不允许的操作'); - } - if (/\bPRAGMA\s+writable_schema\b/i.test(cleaned)) { - throw new Error('SQL 包含不允许的 PRAGMA'); - } - if (readonly && !/^\s*(SELECT|WITH)\b/i.test(cleaned)) { - throw new Error('private_data_query 只允许 SELECT/WITH'); - } - return cleaned; -} - -async function queryPrivateData(sql) { - const db = ensurePrivateDataDb(); - const cleaned = rejectDangerousSql(sql, { readonly: true }); - const limited = `SELECT * FROM (${cleaned.replace(/;\s*$/, '')}) LIMIT ${PRIVATE_DATA_MAX_ROWS}`; - const output = runSqlite(['-json', db, limited]); - return output.trim() || '[]'; -} - -async function executePrivateData(sql) { - const db = ensurePrivateDataDb(); - const before = privateDataSize(); - if (before > PRIVATE_DATA_MAX_BYTES) throw new Error('用户私有数据空间已超过大小限制'); - const cleaned = rejectDangerousSql(sql); - const quotaPragma = await sqliteMaxPagePragmaForQuota(before); - runSqlite(['-batch', db, `${quotaPragma}\n${cleaned}`]); - const after = privateDataSize(); - if (after > PRIVATE_DATA_MAX_BYTES) { - throw new Error(`用户私有数据空间超过大小限制：${after}/${PRIVATE_DATA_MAX_BYTES} 字节`); - } - const delta = after - before; - const quotaSync = await syncPrivateDataQuota(delta); - return `已执行。当前数据空间大小 ${after} 字节${quotaSync ? `，已同步空间占用 ${delta} 字节` : ''}`; -} - -async function privateDataSchema() { - const db = ensurePrivateDataDb(); - const output = runSqlite([ - '-json', - db, - `SELECT m.name AS table_name, p.cid, p.name AS column_name, p.type, p."notnull" AS not_null, p.pk - FROM sqlite_master m - JOIN pragma_table_info(m.name) p - WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite_%' - ORDER BY m.name, p.cid`, - ]); - return output.trim() || '[]'; + return userDataSpaceService; } async function callTool(name, args) { @@ -595,32 +497,18 @@ async function callTool(name, args) { ]; } case 'private_data_info': { - ensurePrivateDataDb(); - const size = privateDataSize(); - const quota = await getQuotaState().catch(() => null); + const info = await getUserDataSpaceService().getInfo(); return [ { type: 'text', text: JSON.stringify( { - name: '用户私有数据空间', - database: '.mindspace/private-data.sqlite', - maxBytes: PRIVATE_DATA_MAX_BYTES, - sizeBytes: size, - quotaSyncEnabled: Boolean(getQuotaPool()), - quota: quota - ? { - status: quota.status, - quotaBytes: quota.quotaBytes, - usedBytes: quota.usedBytes, - reservedBytes: quota.reservedBytes, - availableBytes: quota.availableBytes, - } - : null, + ...info, rules: [ '每个用户只能使用这个唯一 SQLite 数据库', '适合问卷、表单、清单、调研数据和分析中间表', '不要存账号、计费、权限、审计、公开平台数据或跨用户数据', + 'HTML 页面通过 Page Data API 访问 dataset，不要直接暴露 SQL', ], }, null, @@ -629,12 +517,49 @@ async function callTool(name, args) { }, ]; } - case 'private_data_schema': - return [{ type: 'text', text: await privateDataSchema() }]; - case 'private_data_query': - return [{ type: 'text', text: await queryPrivateData(args.sql) }]; - case 'private_data_execute': - return [{ type: 'text', text: await executePrivateData(args.sql) }]; + case 'private_data_schema': { + const schema = await getUserDataSpaceService().getSchema(); + return [{ type: 'text', text: JSON.stringify(schema, null, 2) }]; + } + case 'private_data_query': { + const rows = await getUserDataSpaceService().querySql(args.sql); + return [{ type: 'text', text: JSON.stringify(rows, null, 2) }]; + } + case 'private_data_execute': { + const result = await getUserDataSpaceService().executeSql(args.sql); + return [ + { + type: 'text', + text: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `，已同步空间占用 ${result.deltaBytes} 字节` : ''}`, + }, + ]; + } + case 'private_data_register_dataset': { + const dataset = await getUserDataSpaceService().upsertDataset({ + name: args.name, + table: args.table, + description: args.description, + actions: args.actions, + columns: args.columns, + limits: args.limits, + }); + return [{ type: 'text', text: JSON.stringify(dataset, null, 2) }]; + } + case 'private_data_set_page_policy': { + const ownerUserId = String(args.ownerUserId ?? PRIVATE_DATA_USER_ID ?? '').trim(); + if (!ownerUserId) throw new Error('缺少 ownerUserId，无法保存页面数据策略'); + const policy = normalizePageAccessPolicy( + { + pageId: args.pageId, + ownerUserId, + accessMode: args.accessMode, + datasets: args.datasets, + }, + { fallbackPageId: args.pageId, fallbackOwnerUserId: ownerUserId }, + ); + const saved = writePageAccessPolicy(SANDBOX, policy); + return [{ type: 'text', text: JSON.stringify(saved, null, 2) }]; + } case 'schedule_create_item': { const timezone = args.timezone ?? process.env.H5_DEFAULT_TIMEZONE ?? 'Asia/Shanghai'; const startAt = resolveScheduleTimestamp({ diff --git a/mindspace-sandbox-mcp.test.mjs b/mindspace-sandbox-mcp.test.mjs index f1e43f2..4da241b 100644 --- a/mindspace-sandbox-mcp.test.mjs +++ b/mindspace-sandbox-mcp.test.mjs @@ -129,6 +129,66 @@ test('sandbox MCP exposes and protects the user private data space', async (t) = assert.match(dotCommand.result.content[0].text, /dot command/); }); +test('sandbox MCP registers datasets and page policies for Page Data API', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-page-data-')); + const server = startSandbox(root, { + ALLOWED_TOOLS: 'private_data_execute,private_data_register_dataset,private_data_set_page_policy', + PRIVATE_DATA_USER_ID: 'user-1', + }); + t.after(() => server.child.kill()); + + await server.request('initialize'); + + const create = await server.request('tools/call', { + name: 'private_data_execute', + arguments: { + sql: `CREATE TABLE signups ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + phone TEXT + );`, + }, + }); + assert.equal(create.result.isError, false); + + const register = await server.request('tools/call', { + name: 'private_data_register_dataset', + arguments: { + name: 'signups', + table: 'signups', + actions: ['read', 'insert'], + columns: { + read: ['id', 'name', 'phone'], + insert: ['name', 'phone'], + }, + }, + }); + assert.equal(register.result.isError, false); + assert.match(register.result.content[0].text, /"name": "signups"/); + + const policy = await server.request('tools/call', { + name: 'private_data_set_page_policy', + arguments: { + pageId: 'page-1', + accessMode: 'public', + datasets: { + signups: { + insert: true, + columns: { insert: ['name', 'phone'] }, + }, + }, + }, + }); + assert.equal(policy.result.isError, false); + assert.equal( + fs.existsSync(path.join(root, '.mindspace', 'page-data-policies', 'page-1.json')), + true, + ); + const saved = JSON.parse(fs.readFileSync(path.join(root, '.mindspace', 'page-data-policies', 'page-1.json'), 'utf8')); + assert.equal(saved.pageId, 'page-1'); + assert.equal(saved.datasets.signups.insert, true); +}); + test('sandbox MCP exposes schedule tools only when schedule env is configured', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-schedule-')); const server = startSandbox(root, { diff --git a/package.json b/package.json index c85a355..b85ce91 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "mock:memory-v2-services": "node scripts/mock-memory-v2-services.mjs", "test:memind": "node scripts/run-memind-tests.mjs", "test:scenario": "node scripts/run-scenario-test.mjs", - "test": "node --test auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", + "test": "node --test auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:mindspace-service": "node --test mindspace-service/mindspace-rpc-server.test.mjs", "verify:chat-finish-sync": "node scripts/verify-chat-finish-sync.mjs", "verify:public-finish-sync-runtime": "node scripts/verify-public-finish-sync-runtime.mjs", diff --git a/page-access-policy.mjs b/page-access-policy.mjs new file mode 100644 index 0000000..2896ea3 --- /dev/null +++ b/page-access-policy.mjs @@ -0,0 +1,121 @@ +import { assertSafeSqlIdentifier, normalizeDatasetConfig } from './user-data-space-service.mjs'; + +export const PAGE_DATA_ACCESS_MODES = new Set(['public', 'password', 'login_required']); + +export function normalizePolicyDataset(name, raw) { + const datasetName = assertSafeSqlIdentifier(name, 'dataset 名称'); + if (!raw || typeof raw !== 'object') { + throw Object.assign(new Error(`dataset 策略无效：${datasetName}`), { code: 'invalid_policy' }); + } + const columns = raw.columns && typeof raw.columns === 'object' ? raw.columns : {}; + const normalizedColumns = {}; + for (const [action, fields] of Object.entries(columns)) { + if (!Array.isArray(fields)) continue; + normalizedColumns[action] = fields.map((field) => assertSafeSqlIdentifier(field, `${action} 字段`)); + } + return { + name: datasetName, + read: Boolean(raw.read), + insert: Boolean(raw.insert), + update: Boolean(raw.update), + softDelete: Boolean(raw.softDelete ?? raw.soft_delete), + hardDelete: Boolean(raw.hardDelete ?? raw.hard_delete), + columns: normalizedColumns, + limits: + raw.limits && typeof raw.limits === 'object' + ? { + maxRowsPerRead: Number(raw.limits.maxRowsPerRead ?? 100), + maxInsertBytes: Number(raw.limits.maxInsertBytes ?? 8192), + } + : undefined, + }; +} + +export function normalizePageAccessPolicy(raw, { fallbackPageId = null, fallbackOwnerUserId = null } = {}) { + if (!raw || typeof raw !== 'object') { + throw Object.assign(new Error('Page Access Policy 无效'), { code: 'invalid_policy' }); + } + const pageId = String(raw.pageId ?? fallbackPageId ?? '').trim(); + const ownerUserId = String(raw.ownerUserId ?? fallbackOwnerUserId ?? '').trim(); + if (!pageId) throw Object.assign(new Error('缺少 pageId'), { code: 'invalid_policy' }); + if (!ownerUserId) throw Object.assign(new Error('缺少 ownerUserId'), { code: 'invalid_policy' }); + const accessMode = String(raw.accessMode ?? raw.access_mode ?? 'public').trim(); + if (!PAGE_DATA_ACCESS_MODES.has(accessMode)) { + throw Object.assign(new Error(`不支持的 accessMode：${accessMode}`), { code: 'invalid_policy' }); + } + const datasetsInput = raw.datasets && typeof raw.datasets === 'object' ? raw.datasets : {}; + const datasets = {}; + for (const [name, config] of Object.entries(datasetsInput)) { + datasets[assertSafeSqlIdentifier(name, 'dataset 名称')] = normalizePolicyDataset(name, config); + } + return { + pageId, + ownerUserId, + workspaceRef: String(raw.workspaceRef ?? raw.workspace_ref ?? '').trim() || null, + accessMode, + datasets, + updatedAt: raw.updatedAt ?? new Date().toISOString(), + }; +} + +export function buildEffectiveDataset(registryDataset, policyDataset) { + if (!registryDataset || !policyDataset) return null; + const actions = []; + if (policyDataset.read) actions.push('read'); + if (policyDataset.insert) actions.push('insert'); + if (policyDataset.update) actions.push('update'); + if (policyDataset.softDelete) actions.push('soft_delete'); + if (policyDataset.hardDelete) actions.push('hard_delete'); + const columns = { + read: policyDataset.columns?.read ?? registryDataset.columns?.read ?? [], + insert: policyDataset.columns?.insert ?? registryDataset.columns?.insert ?? [], + update: policyDataset.columns?.update ?? registryDataset.columns?.update ?? [], + soft_delete: policyDataset.columns?.soft_delete ?? registryDataset.columns?.soft_delete ?? ['id'], + }; + return normalizeDatasetConfig({ + name: registryDataset.name, + table: registryDataset.table, + description: registryDataset.description, + actions, + columns, + limits: policyDataset.limits ?? registryDataset.limits, + }); +} + +export function defaultDatasetsForAccessMode(accessMode, datasets = {}) { + const normalized = {}; + for (const [name, config] of Object.entries(datasets)) { + const base = normalizePolicyDataset(name, config); + if (accessMode === 'public') { + normalized[name] = { + ...base, + read: false, + update: false, + softDelete: false, + hardDelete: false, + }; + } else if (accessMode === 'password') { + normalized[name] = { + ...base, + hardDelete: false, + }; + } else { + normalized[name] = { + ...base, + hardDelete: false, + }; + } + } + return normalized; +} + +export function policyAllowsAction(policy, datasetName, action) { + const dataset = policy?.datasets?.[datasetName]; + if (!dataset) return false; + if (action === 'read') return dataset.read; + if (action === 'insert') return dataset.insert; + if (action === 'update') return dataset.update; + if (action === 'soft_delete') return dataset.softDelete; + if (action === 'hard_delete') return dataset.hardDelete; + return false; +} diff --git a/page-access-policy.test.mjs b/page-access-policy.test.mjs new file mode 100644 index 0000000..fb31488 --- /dev/null +++ b/page-access-policy.test.mjs @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildEffectiveDataset, + normalizePageAccessPolicy, + policyAllowsAction, +} from './page-access-policy.mjs'; + +test('normalizePageAccessPolicy validates access mode and datasets', () => { + const policy = normalizePageAccessPolicy( + { + pageId: 'page-1', + ownerUserId: 'user-1', + accessMode: 'public', + datasets: { + registrations: { + insert: true, + columns: { insert: ['name', 'phone'] }, + }, + }, + }, + { fallbackPageId: 'page-1', fallbackOwnerUserId: 'user-1' }, + ); + assert.equal(policy.accessMode, 'public'); + assert.equal(policy.datasets.registrations.insert, true); + assert.equal(policy.datasets.registrations.read, false); +}); + +test('buildEffectiveDataset intersects registry actions with policy permissions', () => { + const effective = buildEffectiveDataset( + { + name: 'registrations', + table: 'form_registrations', + description: '报名', + actions: ['read', 'insert', 'update', 'soft_delete'], + columns: { + read: ['id', 'name', 'phone', 'secret'], + insert: ['name', 'phone', 'secret'], + update: ['status'], + soft_delete: ['id'], + }, + }, + { + read: false, + insert: true, + update: false, + softDelete: false, + columns: { insert: ['name', 'phone'] }, + }, + ); + assert.deepEqual(effective.actions, ['insert']); + assert.deepEqual(effective.columns.insert, ['name', 'phone']); +}); + +test('policyAllowsAction respects per-dataset flags', () => { + const policy = normalizePageAccessPolicy({ + pageId: 'page-1', + ownerUserId: 'user-1', + accessMode: 'password', + datasets: { + ledger: { read: true, insert: true, update: true, softDelete: true, columns: {} }, + }, + }); + assert.equal(policyAllowsAction(policy, 'ledger', 'read'), true); + assert.equal(policyAllowsAction(policy, 'ledger', 'insert'), true); + assert.equal(policyAllowsAction(policy, 'missing', 'read'), false); +}); diff --git a/page-data-integration.test.mjs b/page-data-integration.test.mjs new file mode 100644 index 0000000..51cf4e0 --- /dev/null +++ b/page-data-integration.test.mjs @@ -0,0 +1,254 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import express from 'express'; +import { attachPageDataRoutes } from './page-data-routes.mjs'; +import { createPageDataService } from './page-data-service.mjs'; +import { createPageDataPublicService } from './page-data-public-service.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; +import { publicationInternals } from './mindspace-publications.mjs'; + +const PAGE_ID = 'page-integration-1'; +const OWNER_ID = 'user-integration-1'; + +function createPool(accessMode = 'public') { + return { + async query(sql) { + if (sql.includes('FROM h5_publish_records')) { + return [ + [ + { + id: 'pub-1', + user_id: OWNER_ID, + page_id: PAGE_ID, + access_mode: accessMode, + password_hash: null, + status: 'online', + expires_at: null, + }, + ], + ]; + } + return [[]]; + }, + }; +} + +async function setupWorkspace(workspaceRoot) { + const service = createUserDataSpaceService({ workspaceRoot }); + await service.executeSql(`CREATE TABLE entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + created_at TEXT + );`); + await service.upsertDataset({ + name: 'entries', + table: 'entries', + actions: ['read', 'insert'], + columns: { + read: ['id', 'title', 'created_at'], + insert: ['title'], + }, + }); + return service; +} + +function buildApp(workspaceRoot) { + const pageDataPublicService = createPageDataPublicService({ + getPool: () => createPool('public'), + resolveWorkspaceRootForOwner: () => workspaceRoot, + }); + const api = express.Router(); + api.use(express.json()); + api.use((req, _res, next) => { + if (req.headers['x-test-user']) { + req.currentUser = { id: OWNER_ID, workspaceRoot }; + } + next(); + }); + attachPageDataRoutes(api, { + sendData: (res, _req, data, status = 200) => res.status(status).json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => + createPageDataService({ + resolveWorkspaceRoot: async () => workspaceRoot, + }), + getPageDataPublicService: () => pageDataPublicService, + }); + const app = express(); + app.set('trust proxy', true); + app.use('/api', api); + return app; +} + +async function request(app, method, url, { body, headers } = {}) { + const server = app.listen(0); + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}${url}`, { + method, + headers: { + ...(body ? { 'content-type': 'application/json' } : {}), + ...headers, + }, + body: body ? JSON.stringify(body) : undefined, + }); + return { status: response.status, body: await response.json() }; + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +test('integration: owner private API and public insert coexist without breaking each other', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-integration-')); + const service = await setupWorkspace(workspaceRoot); + const app = buildApp(workspaceRoot); + + const ownerInsert = await request(app, 'POST', '/api/page-data/entries/rows', { + headers: { 'x-test-user': '1' }, + body: { title: 'owner 写入' }, + }); + assert.equal(ownerInsert.status, 201); + assert.equal(ownerInsert.body.data.row.title, 'owner 写入'); + + const ownerList = await request(app, 'GET', '/api/page-data/entries?limit=10', { + headers: { 'x-test-user': '1' }, + }); + assert.equal(ownerList.status, 200); + assert.equal(ownerList.body.data.rows.length, 1); + + const policy = { + pageId: PAGE_ID, + ownerUserId: OWNER_ID, + accessMode: 'public', + datasets: { + entries: { + insert: true, + columns: { insert: ['title'] }, + }, + }, + }; + const policyWrite = await request(app, 'PUT', `/api/page-data/policies/${PAGE_ID}`, { + headers: { 'x-test-user': '1' }, + body: policy, + }); + assert.equal(policyWrite.status, 200); + assert.equal(policyWrite.body.data.policy.pageId, PAGE_ID); + + const publicInsert = await request(app, 'POST', `/api/public/pages/${PAGE_ID}/data/entries/rows`, { + body: { title: 'public 写入' }, + }); + assert.equal(publicInsert.status, 201); + assert.equal(publicInsert.body.data.row.title, 'public 写入'); + + const publicReadDenied = await request(app, 'GET', `/api/public/pages/${PAGE_ID}/data/entries`); + assert.equal(publicReadDenied.status, 403); + assert.equal(publicReadDenied.body.error.code, 'action_not_allowed'); + + const ownerListAfterPublic = await request(app, 'GET', '/api/page-data/entries?limit=10', { + headers: { 'x-test-user': '1' }, + }); + assert.equal(ownerListAfterPublic.status, 200); + assert.equal(ownerListAfterPublic.body.data.rows.length, 2); + + const stats = service.getDatasetStats('entries'); + assert.equal(stats.total, 2); +}); + +test('integration: public insert rejects SQL injection style payload keys', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-integration-sql-')); + await setupWorkspace(workspaceRoot); + const app = buildApp(workspaceRoot); + await request(app, 'PUT', `/api/page-data/policies/${PAGE_ID}`, { + headers: { 'x-test-user': '1' }, + body: { + pageId: PAGE_ID, + ownerUserId: OWNER_ID, + accessMode: 'public', + datasets: { + entries: { insert: true, columns: { insert: ['title'] } }, + }, + }, + }); + + const denied = await request(app, 'POST', `/api/public/pages/${PAGE_ID}/data/entries/rows`, { + body: { title: 'ok', sql: 'DROP TABLE entries' }, + }); + assert.equal(denied.status, 403); + assert.equal(denied.body.error.code, 'columns_not_allowed'); +}); + +test('integration: password publication flow still works for read after data-auth', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-integration-password-')); + const service = await setupWorkspace(workspaceRoot); + await service.insertDatasetRow('entries', { title: '口令可见' }); + + const api = express.Router(); + api.use(express.json()); + api.use((req, _res, next) => { + if (req.headers['x-test-user']) { + req.currentUser = { id: OWNER_ID, workspaceRoot }; + } + next(); + }); + const pageDataPublicService = createPageDataPublicService({ + getPool: () => ({ + async query(sql) { + if (sql.includes('FROM h5_publish_records')) { + return [ + [ + { + id: 'pub-1', + user_id: OWNER_ID, + page_id: PAGE_ID, + access_mode: 'password', + password_hash: publicationInternals.hashPassword('secret-123'), + status: 'online', + expires_at: null, + }, + ], + ]; + } + return [[]]; + }, + }), + resolveWorkspaceRootForOwner: () => workspaceRoot, + }); + attachPageDataRoutes(api, { + sendData: (res, _req, data, status = 200) => res.status(status).json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }), + getPageDataPublicService: () => pageDataPublicService, + }); + const app = express(); + app.use('/api', api); + + await request(app, 'PUT', `/api/page-data/policies/${PAGE_ID}`, { + headers: { 'x-test-user': '1' }, + body: { + pageId: PAGE_ID, + ownerUserId: OWNER_ID, + accessMode: 'password', + datasets: { + entries: { + read: true, + columns: { read: ['id', 'title', 'created_at'] }, + }, + }, + }, + }); + + const auth = await request(app, 'POST', `/api/public/pages/${PAGE_ID}/data-auth`, { + body: { password: 'secret-123' }, + }); + assert.equal(auth.status, 200); + assert.ok(auth.body.data.token); + + const listed = await request(app, 'GET', `/api/public/pages/${PAGE_ID}/data/entries`, { + headers: { 'x-page-data-token': auth.body.data.token }, + }); + assert.equal(listed.status, 200); + assert.equal(listed.body.data.rows[0].title, '口令可见'); +}); diff --git a/page-data-policy-store.mjs b/page-data-policy-store.mjs new file mode 100644 index 0000000..b879bcb --- /dev/null +++ b/page-data-policy-store.mjs @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { normalizePageAccessPolicy } from './page-access-policy.mjs'; + +export function resolvePageDataPolicyDir(workspaceRoot) { + return path.join(workspaceRoot, '.mindspace', 'page-data-policies'); +} + +export function resolvePageDataPolicyPath(workspaceRoot, pageId) { + const safePageId = String(pageId ?? '').trim(); + if (!safePageId || /[\\/]/.test(safePageId)) { + throw Object.assign(new Error('pageId 无效'), { code: 'invalid_page_id' }); + } + return path.join(resolvePageDataPolicyDir(workspaceRoot), `${safePageId}.json`); +} + +export function readPageAccessPolicy(workspaceRoot, pageId) { + const policyPath = resolvePageDataPolicyPath(workspaceRoot, pageId); + if (!fs.existsSync(policyPath)) return null; + const raw = JSON.parse(fs.readFileSync(policyPath, 'utf8')); + return normalizePageAccessPolicy(raw, { fallbackPageId: pageId }); +} + +export function writePageAccessPolicy(workspaceRoot, policyInput) { + const policy = normalizePageAccessPolicy(policyInput); + const policyDir = resolvePageDataPolicyDir(workspaceRoot); + fs.mkdirSync(policyDir, { recursive: true }); + const policyPath = resolvePageDataPolicyPath(workspaceRoot, policy.pageId); + fs.writeFileSync( + policyPath, + `${JSON.stringify({ ...policy, updatedAt: new Date().toISOString() }, null, 2)}\n`, + 'utf8', + ); + return readPageAccessPolicy(workspaceRoot, policy.pageId); +} + +export function deletePageAccessPolicy(workspaceRoot, pageId) { + const policyPath = resolvePageDataPolicyPath(workspaceRoot, pageId); + if (!fs.existsSync(policyPath)) return false; + fs.unlinkSync(policyPath); + return true; +} + +export function listPageAccessPolicies(workspaceRoot) { + const policyDir = resolvePageDataPolicyDir(workspaceRoot); + if (!fs.existsSync(policyDir)) return []; + return fs + .readdirSync(policyDir) + .filter((name) => name.endsWith('.json')) + .map((name) => readPageAccessPolicy(workspaceRoot, name.replace(/\.json$/, ''))) + .filter(Boolean); +} diff --git a/page-data-public-service.mjs b/page-data-public-service.mjs new file mode 100644 index 0000000..00f37a1 --- /dev/null +++ b/page-data-public-service.mjs @@ -0,0 +1,373 @@ +import { resolveMindSpaceUserPublishDir } from './mindspace-runtime-config.mjs'; +import { publicationInternals } from './mindspace-publications.mjs'; +import { + buildEffectiveDataset, + normalizePageAccessPolicy, + policyAllowsAction, +} from './page-access-policy.mjs'; +import { readPageAccessPolicy, writePageAccessPolicy } from './page-data-policy-store.mjs'; +import { + createPageDataRateLimiter, + createPageDataSessionStore, + hashClientMeta, +} from './page-data-session-store.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; + +function mapPublicError(error) { + if (error?.status && error?.code && error?.message) { + return error; + } + const code = error?.code; + if (code === 'publication_not_found') { + return { status: 404, code: 'publication_not_found', message: '公开页面不存在或已下线' }; + } + if (code === 'policy_not_found') { + return { status: 404, code: 'policy_not_found', message: '页面数据策略未配置' }; + } + if (code === 'dataset_not_found') { + return { status: 404, code: 'dataset_not_found', message: '数据集不存在' }; + } + if (code === 'action_not_allowed') { + return { status: 403, code: 'action_not_allowed', message: error.message }; + } + if (code === 'columns_not_allowed') { + return { status: 403, code: 'columns_not_allowed', message: error.message }; + } + if (code === 'auth_required' || code === 'token_invalid' || code === 'password_invalid') { + return { status: 401, code, message: error.message }; + } + if (code === 'rate_limited') { + return { status: 429, code: 'rate_limited', message: error.message }; + } + if (code === 'invalid_identifier') { + return { status: 403, code: 'columns_not_allowed', message: error.message }; + } + if (code === 'forbidden') { + return { status: 403, code: 'forbidden', message: error.message }; + } + if (code === 'unauthorized') { + return { status: 401, code: 'unauthorized', message: error.message }; + } + if (code === 'invalid_payload' || code === 'payload_too_large' || code === 'invalid_policy') { + return { status: 400, code: code ?? 'invalid_request', message: error.message }; + } + return { + status: 400, + code: 'page_data_failed', + message: error instanceof Error ? error.message : '页面数据操作失败', + }; +} + +export function mapPageDataPublicError(error) { + return mapPublicError(error); +} + +function rethrowPublicError(error) { + throw mapPublicError(error); +} + +async function runPublicAction(fn) { + try { + return await fn(); + } catch (error) { + rethrowPublicError(error); + } +} + +function extractPageDataToken(req) { + const header = String(req.headers?.['x-page-data-token'] ?? req.headers?.authorization ?? '').trim(); + if (!header) return null; + if (header.toLowerCase().startsWith('bearer ')) return header.slice(7).trim(); + return header; +} + +export function isPageDataPublicPath(path, method) { + if (!path.startsWith('/public/pages/')) return false; + if (method === 'POST' && /^\/public\/pages\/[^/]+\/data-auth$/.test(path)) return true; + if (method === 'POST' && /^\/public\/pages\/[^/]+\/data\/[^/]+\/rows$/.test(path)) return true; + if (method === 'PATCH' && /^\/public\/pages\/[^/]+\/data\/[^/]+\/rows\/[^/]+$/.test(path)) return true; + if (method === 'DELETE' && /^\/public\/pages\/[^/]+\/data\/[^/]+\/rows\/[^/]+$/.test(path)) return true; + if (method !== 'GET') return false; + return ( + /^\/public\/pages\/[^/]+\/data\/[^/]+$/.test(path) || + /^\/public\/pages\/[^/]+\/data\/[^/]+\/schema$/.test(path) || + /^\/public\/pages\/[^/]+\/data\/[^/]+\/stats$/.test(path) + ); +} + +export function createPageDataPublicService(deps = {}) { + const getPool = deps.getPool ?? (() => null); + const resolveH5Root = deps.resolveH5Root ?? (() => process.cwd()); + const resolveWorkspaceRootForOwner = + deps.resolveWorkspaceRootForOwner ?? + ((ownerUserId) => resolveMindSpaceUserPublishDir(resolveH5Root(), { id: ownerUserId })); + const sessionStore = deps.sessionStore ?? createPageDataSessionStore(); + const rateLimiter = + deps.rateLimiter ?? + createPageDataRateLimiter({ + windowMs: Number(process.env.PAGE_DATA_RATE_WINDOW_MS ?? 60_000), + maxRequests: Number(process.env.PAGE_DATA_RATE_MAX_REQUESTS ?? 30), + }); + + async function queryPublication(pageId) { + const pool = getPool(); + if (!pool) { + throw Object.assign(new Error('数据库未配置'), { code: 'feature_disabled', status: 503 }); + } + const [rows] = await pool.query( + `SELECT pr.id, pr.user_id, pr.page_id, pr.access_mode, pr.password_hash, pr.status, pr.expires_at + FROM h5_publish_records pr + WHERE pr.page_id = ? AND pr.status = 'online' + ORDER BY pr.published_at DESC + LIMIT 1`, + [pageId], + ); + const row = rows[0]; + if (!row) { + throw Object.assign(new Error('公开页面不存在或已下线'), { code: 'publication_not_found' }); + } + if (row.expires_at && Number(row.expires_at) <= Date.now()) { + throw Object.assign(new Error('公开页面不存在或已下线'), { code: 'publication_not_found' }); + } + return row; + } + + function resolveWorkspaceRoot(ownerUserId) { + return resolveWorkspaceRootForOwner(ownerUserId); + } + + function createOwnerService(ownerUserId) { + const pool = getPool(); + return createUserDataSpaceService({ + workspaceRoot: resolveWorkspaceRoot(ownerUserId), + userId: ownerUserId, + query: pool ?? null, + }); + } + + function resolveEffectiveDataset(ownerUserId, policy, datasetName) { + const policyDataset = policy.datasets?.[datasetName]; + if (!policyDataset) { + throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' }); + } + const ownerService = createOwnerService(ownerUserId); + const registryDataset = ownerService.getDataset(datasetName); + if (!registryDataset) { + throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + } + const effective = buildEffectiveDataset(registryDataset, policyDataset); + if (!effective) { + throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' }); + } + return { ownerService, effective }; + } + + function enforceRateLimit(req, publication, action) { + const ip = req.ip ?? req.socket?.remoteAddress ?? 'unknown'; + rateLimiter.check(`${publication.page_id}:${action}:${ip}`); + } + + function resolveAccessContext({ publication, policy, req, action, datasetName }) { + const accessMode = publication.access_mode; + if (accessMode === 'public') { + if (action === 'read' || action === 'update' || action === 'soft_delete') { + throw Object.assign(new Error('当前页面未开放此数据操作'), { code: 'action_not_allowed' }); + } + if (!policyAllowsAction(policy, datasetName, action)) { + throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + return { accessMode, session: null }; + } + + if (accessMode === 'password') { + const token = extractPageDataToken(req); + const session = token ? sessionStore.verify(token) : null; + if (!session || session.pageId !== publication.page_id) { + throw Object.assign(new Error('需要有效的页面数据访问令牌'), { + code: 'token_invalid', + }); + } + if (!policyAllowsAction(policy, datasetName, action)) { + throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + return { accessMode, session }; + } + + if (accessMode === 'login_required') { + const viewerId = req.currentUser?.id ?? null; + if (!viewerId) { + throw Object.assign(new Error('需要登录后才能访问页面数据'), { code: 'auth_required' }); + } + if (!policyAllowsAction(policy, datasetName, action)) { + throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + return { accessMode, session: { sessionId: viewerId, ownerUserId: publication.user_id } }; + } + + throw Object.assign(new Error('当前页面访问模式不支持 Page Data API'), { code: 'action_not_allowed' }); + } + + async function loadPolicy(publication) { + const workspaceRoot = resolveWorkspaceRoot(publication.user_id); + const policy = readPageAccessPolicy(workspaceRoot, publication.page_id); + if (!policy) { + throw Object.assign(new Error('页面数据策略未配置'), { code: 'policy_not_found' }); + } + if (policy.ownerUserId !== publication.user_id) { + throw Object.assign(new Error('页面数据策略无效'), { code: 'invalid_policy' }); + } + return policy; + } + + async function authenticate(pageId, password, req) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + enforceRateLimit(req, publication, 'data-auth'); + if (publication.access_mode !== 'password') { + throw Object.assign(new Error('当前页面不需要口令认证'), { code: 'auth_not_required' }); + } + if (!publicationInternals.verifyPassword(password, publication.password_hash)) { + throw Object.assign(new Error('页面口令不正确'), { code: 'password_invalid' }); + } + return sessionStore.issue({ + pageId: publication.page_id, + ownerUserId: publication.user_id, + publicationId: publication.id, + accessMode: publication.access_mode, + }); + }); + } + + async function listRows(pageId, datasetName, req, query = {}) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); + enforceRateLimit(req, publication, 'read'); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.readRowsForDataset(effective, query); + }); + } + + async function getSchema(pageId, datasetName, req) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.getSchemaForDataset(effective); + }); + } + + async function getStats(pageId, datasetName, req) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.getStatsForDataset(effective); + }); + } + + async function insertRow(pageId, datasetName, req, payload) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + const { session } = resolveAccessContext({ + publication, + policy, + req, + action: 'insert', + datasetName, + }); + enforceRateLimit(req, publication, 'insert'); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.insertRowForDataset(effective, payload, { + createdIpHash: hashClientMeta(req.ip ?? req.socket?.remoteAddress), + userAgentHash: hashClientMeta(req.headers?.['user-agent']), + pageDataSessionId: session?.sessionId ?? null, + }); + }); + } + + async function updateRow(pageId, datasetName, rowId, req, payload) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + const { session } = resolveAccessContext({ + publication, + policy, + req, + action: 'update', + datasetName, + }); + enforceRateLimit(req, publication, 'update'); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.updateRowForDataset(effective, rowId, payload, { + updatedByLabel: req.body?.updated_by_label ?? session?.sessionId ?? 'public', + }); + }); + } + + async function softDeleteRow(pageId, datasetName, rowId, req) { + return runPublicAction(async () => { + const publication = await queryPublication(pageId); + const policy = await loadPolicy(publication); + const { session } = resolveAccessContext({ + publication, + policy, + req, + action: 'soft_delete', + datasetName, + }); + enforceRateLimit(req, publication, 'soft_delete'); + const { ownerService, effective } = resolveEffectiveDataset(publication.user_id, policy, datasetName); + return ownerService.softDeleteRowForDataset(effective, rowId, { + deletedBy: session?.sessionId ?? 'public', + }); + }); + } + + async function getOwnerPolicy(user, pageId) { + if (!user?.id) throw Object.assign(new Error('未授权'), { code: 'unauthorized', status: 401 }); + const workspaceRoot = user.workspaceRoot ?? resolveWorkspaceRoot(user.id); + const policy = readPageAccessPolicy(workspaceRoot, pageId); + if (!policy) return null; + if (policy.ownerUserId !== user.id) { + throw Object.assign(new Error('无权访问该页面策略'), { code: 'forbidden', status: 403 }); + } + return policy; + } + + async function saveOwnerPolicy(user, pageId, policyInput) { + if (!user?.id) throw Object.assign(new Error('未授权'), { code: 'unauthorized', status: 401 }); + const publication = await queryPublication(pageId).catch(() => null); + if (!publication || publication.user_id !== user.id) { + throw Object.assign(new Error('页面不存在或未发布'), { code: 'publication_not_found', status: 404 }); + } + const workspaceRoot = user.workspaceRoot ?? resolveWorkspaceRoot(user.id); + const policy = normalizePageAccessPolicy( + { + ...policyInput, + pageId, + ownerUserId: user.id, + accessMode: policyInput.accessMode ?? publication.access_mode, + }, + { fallbackPageId: pageId, fallbackOwnerUserId: user.id }, + ); + return writePageAccessPolicy(workspaceRoot, policy); + } + + return { + authenticate, + listRows, + getSchema, + getStats, + insertRow, + updateRow, + softDeleteRow, + getOwnerPolicy, + saveOwnerPolicy, + queryPublication, + }; +} diff --git a/page-data-public-service.test.mjs b/page-data-public-service.test.mjs new file mode 100644 index 0000000..27be3f8 --- /dev/null +++ b/page-data-public-service.test.mjs @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { createPageDataPublicService, isPageDataPublicPath } from './page-data-public-service.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; +import { writePageAccessPolicy } from './page-data-policy-store.mjs'; +import { publicationInternals } from './mindspace-publications.mjs'; + +const PAGE_ID = 'page-public-1'; +const OWNER_ID = 'user-owner-1'; + +function createPublicationPool({ accessMode = 'public', password = null } = {}) { + const passwordHash = password ? publicationInternals.hashPassword(password) : null; + return { + async query(sql) { + if (sql.includes('FROM h5_publish_records')) { + return [ + [ + { + id: 'pub-record-1', + user_id: OWNER_ID, + page_id: PAGE_ID, + access_mode: accessMode, + password_hash: passwordHash, + status: 'online', + expires_at: null, + }, + ], + ]; + } + return [[]]; + }, + }; +} + +async function setupPublicWorkspace(workspaceRoot, { accessMode = 'public', withRead = false } = {}) { + const service = createUserDataSpaceService({ workspaceRoot }); + await service.executeSql(`CREATE TABLE signups ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + phone TEXT, + status TEXT DEFAULT 'pending', + created_at TEXT, + deleted_at TEXT, + deleted_by TEXT, + updated_at TEXT, + updated_by_label TEXT + );`); + await service.upsertDataset({ + name: 'signups', + table: 'signups', + actions: ['read', 'insert', 'update', 'soft_delete'], + columns: { + read: ['id', 'name', 'phone', 'status', 'created_at'], + insert: ['name', 'phone', 'status'], + update: ['status'], + soft_delete: ['id'], + }, + }); + writePageAccessPolicy(workspaceRoot, { + pageId: PAGE_ID, + ownerUserId: OWNER_ID, + accessMode, + datasets: { + signups: { + read: withRead, + insert: true, + update: withRead, + softDelete: withRead, + columns: { + read: ['id', 'name', 'phone', 'status', 'created_at'], + insert: ['name', 'phone', 'status'], + update: ['status'], + }, + }, + }, + }); + return service; +} + +function createPublicService(workspaceRoot, poolOptions = {}) { + return createPageDataPublicService({ + getPool: () => createPublicationPool(poolOptions), + resolveWorkspaceRootForOwner: () => workspaceRoot, + }); +} + +test('public page insert works without login for public access mode', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-public-')); + await setupPublicWorkspace(workspaceRoot); + const service = createPublicService(workspaceRoot, { accessMode: 'public' }); + const result = await service.insertRow(PAGE_ID, 'signups', { + ip: '127.0.0.1', + headers: { 'user-agent': 'test' }, + }, { + name: '匿名用户', + phone: '13900000000', + }); + assert.equal(result.row.name, '匿名用户'); +}); + +test('public page read is denied in public access mode by default', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-public-read-')); + await setupPublicWorkspace(workspaceRoot); + const service = createPublicService(workspaceRoot, { accessMode: 'public' }); + await assert.rejects( + () => service.listRows(PAGE_ID, 'signups', { headers: {} }), + (error) => error.code === 'action_not_allowed', + ); +}); + +test('password page data-auth returns token and enables read', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-password-')); + const ownerService = await setupPublicWorkspace(workspaceRoot, { + accessMode: 'password', + withRead: true, + }); + await ownerService.insertRowForDataset(ownerService.getDataset('signups'), { + name: '协作项', + phone: '13800000001', + }); + + const service = createPublicService(workspaceRoot, { + accessMode: 'password', + password: 'team-2026', + }); + const auth = await service.authenticate(PAGE_ID, 'team-2026', { ip: '127.0.0.1' }); + assert.ok(auth.token); + + const rows = await service.listRows(PAGE_ID, 'signups', { + ip: '127.0.0.1', + headers: { 'x-page-data-token': auth.token }, + }); + assert.equal(rows.rows.length, 1); + assert.equal(rows.rows[0].name, '协作项'); +}); + +test('password page update and soft delete require token', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-password-mut-')); + const ownerService = await setupPublicWorkspace(workspaceRoot, { + accessMode: 'password', + withRead: true, + }); + const inserted = await ownerService.insertRowForDataset(ownerService.getDataset('signups'), { + name: '待更新', + status: 'pending', + }); + + const service = createPublicService(workspaceRoot, { + accessMode: 'password', + password: 'team-2026', + }); + const auth = await service.authenticate(PAGE_ID, 'team-2026', { ip: '127.0.0.1' }); + const req = { + ip: '127.0.0.1', + headers: { 'x-page-data-token': auth.token }, + body: { updated_by_label: '测试员' }, + }; + + const updated = await service.updateRow(PAGE_ID, 'signups', inserted.row.id, req, { + status: 'done', + }); + assert.equal(updated.row.status, 'done'); + + const deleted = await service.softDeleteRow(PAGE_ID, 'signups', inserted.row.id, req); + assert.equal(deleted.deleted, true); +}); + +test('isPageDataPublicPath allows public page data routes without auth', () => { + assert.equal(isPageDataPublicPath('/public/pages/page-1/data/signups/rows', 'POST'), true); + assert.equal(isPageDataPublicPath('/public/pages/page-1/data-auth', 'POST'), true); + assert.equal(isPageDataPublicPath('/public/pages/page-1/data/signups', 'GET'), true); + assert.equal(isPageDataPublicPath('/page-data/signups', 'GET'), false); +}); diff --git a/page-data-routes.mjs b/page-data-routes.mjs new file mode 100644 index 0000000..bd45bc5 --- /dev/null +++ b/page-data-routes.mjs @@ -0,0 +1,245 @@ +import { mapPageDataPublicError } from './page-data-public-service.mjs'; + +function requireUser(req, res, sendError) { + if (!req.currentUser?.id) { + sendError(res, req, 401, 'unauthorized', '未授权，请重新登录'); + return null; + } + return req.currentUser; +} + +function parsePositiveInt(value, fallback) { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) return fallback; + return Math.floor(parsed); +} + +function parseRowPayload(req) { + const payload = req.body?.row && typeof req.body.row === 'object' ? req.body.row : req.body; + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + return null; + } + return payload; +} + +function handlePublicError(res, req, sendError, error) { + const mapped = mapPageDataPublicError(error); + return sendError(res, req, mapped.status, mapped.code, mapped.message); +} + +export function attachPageDataRoutes(api, deps) { + const { sendData, sendError, getPageDataService, getPageDataPublicService } = deps; + + api.get('/page-data', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const service = getPageDataService?.(); + if (!service) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const datasets = await service.listDatasets(user); + return sendData(res, req, { datasets }); + } catch (error) { + const status = error?.status ?? 400; + return sendError(res, req, status, error?.code ?? 'page_data_failed', error?.message ?? '读取失败'); + } + }); + + api.get('/page-data/policies/:pageId', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const policy = await publicService.getOwnerPolicy(user, req.params.pageId); + if (!policy) return sendError(res, req, 404, 'policy_not_found', '页面数据策略未配置'); + return sendData(res, req, { policy }); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.put('/page-data/policies/:pageId', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const policy = await publicService.saveOwnerPolicy(user, req.params.pageId, req.body ?? {}); + return sendData(res, req, { policy }); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.get('/page-data/:dataset', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const service = getPageDataService?.(); + if (!service) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const result = await service.listRows(user, req.params.dataset, { + limit: parsePositiveInt(req.query.limit, undefined), + offset: parsePositiveInt(req.query.offset, 0), + orderBy: req.query.order_by, + orderDir: req.query.order_dir, + }); + return sendData(res, req, { + dataset: result.dataset.name, + rows: result.rows, + limit: result.limit, + offset: result.offset, + }); + } catch (error) { + const status = error?.status ?? 400; + return sendError(res, req, status, error?.code ?? 'page_data_failed', error?.message ?? '读取失败'); + } + }); + + api.get('/page-data/:dataset/schema', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const service = getPageDataService?.(); + if (!service) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const schema = await service.getSchema(user, req.params.dataset); + return sendData(res, req, schema); + } catch (error) { + const status = error?.status ?? 400; + return sendError(res, req, status, error?.code ?? 'page_data_failed', error?.message ?? '读取失败'); + } + }); + + api.get('/page-data/:dataset/stats', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const service = getPageDataService?.(); + if (!service) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const stats = await service.getStats(user, req.params.dataset); + return sendData(res, req, stats); + } catch (error) { + const status = error?.status ?? 400; + return sendError(res, req, status, error?.code ?? 'page_data_failed', error?.message ?? '读取失败'); + } + }); + + api.post('/page-data/:dataset/rows', async (req, res) => { + const user = requireUser(req, res, sendError); + if (!user) return; + const service = getPageDataService?.(); + if (!service) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + const payload = parseRowPayload(req); + if (!payload) return sendError(res, req, 400, 'invalid_request', '提交数据必须是 JSON 对象'); + try { + const result = await service.insertRow(user, req.params.dataset, payload); + return sendData(res, req, { dataset: result.dataset.name, row: result.row }, 201); + } catch (error) { + const status = error?.status ?? 400; + return sendError(res, req, status, error?.code ?? 'page_data_failed', error?.message ?? '写入失败'); + } + }); + + api.post('/public/pages/:pageId/data-auth', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + const password = String(req.body?.password ?? '').trim(); + if (!password) return sendError(res, req, 400, 'invalid_request', '缺少页面口令'); + try { + const auth = await publicService.authenticate(req.params.pageId, password, req); + return sendData(res, req, auth); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.get('/public/pages/:pageId/data/:dataset', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const result = await publicService.listRows(req.params.pageId, req.params.dataset, req, { + limit: parsePositiveInt(req.query.limit, undefined), + offset: parsePositiveInt(req.query.offset, 0), + orderBy: req.query.order_by, + orderDir: req.query.order_dir, + }); + return sendData(res, req, { + dataset: result.dataset.name, + rows: result.rows, + limit: result.limit, + offset: result.offset, + }); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.get('/public/pages/:pageId/data/:dataset/schema', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const schema = await publicService.getSchema(req.params.pageId, req.params.dataset, req); + return sendData(res, req, schema); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.get('/public/pages/:pageId/data/:dataset/stats', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const stats = await publicService.getStats(req.params.pageId, req.params.dataset, req); + return sendData(res, req, stats); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.post('/public/pages/:pageId/data/:dataset/rows', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + const payload = parseRowPayload(req); + if (!payload) return sendError(res, req, 400, 'invalid_request', '提交数据必须是 JSON 对象'); + try { + const result = await publicService.insertRow(req.params.pageId, req.params.dataset, req, payload); + return sendData(res, req, { dataset: result.dataset.name, row: result.row }, 201); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.patch('/public/pages/:pageId/data/:dataset/rows/:rowId', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + const payload = parseRowPayload(req); + if (!payload) return sendError(res, req, 400, 'invalid_request', '更新数据必须是 JSON 对象'); + try { + const result = await publicService.updateRow( + req.params.pageId, + req.params.dataset, + req.params.rowId, + req, + payload, + ); + return sendData(res, req, { dataset: result.dataset.name, row: result.row }); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); + + api.delete('/public/pages/:pageId/data/:dataset/rows/:rowId', async (req, res) => { + const publicService = getPageDataPublicService?.(); + if (!publicService) return sendError(res, req, 503, 'feature_disabled', 'Page Data API 未启用'); + try { + const result = await publicService.softDeleteRow( + req.params.pageId, + req.params.dataset, + req.params.rowId, + req, + ); + return sendData(res, req, result); + } catch (error) { + return handlePublicError(res, req, sendError, error); + } + }); +} diff --git a/page-data-routes.test.mjs b/page-data-routes.test.mjs new file mode 100644 index 0000000..09d4e98 --- /dev/null +++ b/page-data-routes.test.mjs @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import express from 'express'; +import { attachPageDataRoutes } from './page-data-routes.mjs'; +import { createPageDataService } from './page-data-service.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; + +function createResponseRecorder() { + return { + statusCode: 200, + body: null, + status(code) { + this.statusCode = code; + return this; + }, + json(payload) { + this.body = payload; + return this; + }, + }; +} + +function createApiApp({ workspaceRoot, user }) { + const api = express.Router(); + api.use(express.json()); + api.use((req, _res, next) => { + req.currentUser = user; + next(); + }); + const pageDataService = createPageDataService({ + resolveWorkspaceRoot: async () => workspaceRoot, + }); + attachPageDataRoutes(api, { + sendData: (res, _req, data, status = 200) => res.status(status).json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => pageDataService, + }); + const app = express(); + app.use('/api', api); + return app; +} + +async function requestJson(app, method, url, body) { + const server = app.listen(0); + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}${url}`, { + method, + headers: body ? { 'content-type': 'application/json' } : undefined, + body: body ? JSON.stringify(body) : undefined, + }); + const payload = await response.json(); + return { status: response.status, body: payload }; + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +test('page data routes allow logged-in owner to read and insert dataset rows', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-api-')); + const service = createUserDataSpaceService({ workspaceRoot }); + await service.executeSql(`CREATE TABLE feedback ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message TEXT NOT NULL, + created_at TEXT + );`); + await service.upsertDataset({ + name: 'feedback', + table: 'feedback', + actions: ['read', 'insert'], + columns: { + read: ['id', 'message', 'created_at'], + insert: ['message'], + }, + }); + + const app = createApiApp({ + workspaceRoot, + user: { id: 'user-1', workspaceRoot }, + }); + + const inserted = await requestJson(app, 'POST', '/api/page-data/feedback/rows', { + message: '很好用', + }); + assert.equal(inserted.status, 201); + assert.equal(inserted.body.data.row.message, '很好用'); + + const listed = await requestJson(app, 'GET', '/api/page-data/feedback?limit=10'); + assert.equal(listed.status, 200); + assert.equal(listed.body.data.rows.length, 1); + + const schema = await requestJson(app, 'GET', '/api/page-data/feedback/schema'); + assert.equal(schema.status, 200); + assert.equal(schema.body.data.dataset.name, 'feedback'); + + const stats = await requestJson(app, 'GET', '/api/page-data/feedback/stats'); + assert.equal(stats.status, 200); + assert.equal(stats.body.data.total, 1); +}); + +test('page data routes reject unauthorized dataset action', async () => { + const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-api-deny-')); + const service = createUserDataSpaceService({ workspaceRoot }); + await service.executeSql(`CREATE TABLE hidden ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + secret TEXT NOT NULL + );`); + await service.upsertDataset({ + name: 'hidden', + table: 'hidden', + actions: ['insert'], + columns: { insert: ['secret'] }, + }); + + const app = createApiApp({ + workspaceRoot, + user: { id: 'user-1', workspaceRoot }, + }); + + const denied = await requestJson(app, 'GET', '/api/page-data/hidden'); + assert.equal(denied.status, 403); + assert.equal(denied.body.error.code, 'action_not_allowed'); +}); + +test('page data routes require login', async () => { + const api = express.Router(); + attachPageDataRoutes(api, { + sendData: (res, _req, data) => res.json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => '/tmp' }), + }); + const handler = api.stack.find((layer) => layer.route?.path === '/page-data/:dataset')?.route.stack[0].handle; + const res = createResponseRecorder(); + await handler({ params: { dataset: 'demo' } }, res); + assert.equal(res.statusCode, 401); + assert.equal(res.body.error.code, 'unauthorized'); +}); diff --git a/page-data-service.mjs b/page-data-service.mjs new file mode 100644 index 0000000..53eb5ee --- /dev/null +++ b/page-data-service.mjs @@ -0,0 +1,109 @@ +import { createUserDataSpaceService } from './user-data-space-service.mjs'; + +function mapServiceError(error) { + const code = error?.code; + if (code === 'dataset_not_found') { + return { status: 404, code: 'dataset_not_found', message: '数据集不存在' }; + } + if (code === 'action_not_allowed') { + return { status: 403, code: 'action_not_allowed', message: error.message }; + } + if (code === 'columns_not_allowed') { + return { status: 403, code: 'columns_not_allowed', message: error.message }; + } + if (code === 'invalid_payload' || code === 'payload_too_large' || code === 'invalid_dataset_config') { + return { status: 400, code: code ?? 'invalid_request', message: error.message }; + } + if (code === 'table_not_found') { + return { status: 400, code: 'table_not_found', message: '数据集对应表不存在' }; + } + return { + status: 400, + code: 'page_data_failed', + message: error instanceof Error ? error.message : '页面数据操作失败', + }; +} + +export function createPageDataService(deps = {}) { + const getUserAuth = deps.getUserAuth ?? (() => null); + const getPool = deps.getPool ?? (() => null); + const resolveWorkspaceRoot = + deps.resolveWorkspaceRoot ?? + (async (user) => user?.workspaceRoot ?? null); + + async function createServiceForUser(user) { + if (!user?.id) { + throw Object.assign(new Error('未授权'), { code: 'unauthorized' }); + } + const workspaceRoot = await resolveWorkspaceRoot(user); + if (!workspaceRoot) { + throw Object.assign(new Error('用户工作区不存在'), { code: 'workspace_not_found' }); + } + const pool = getPool(); + return createUserDataSpaceService({ + workspaceRoot, + userId: user.id, + query: pool ?? null, + }); + } + + async function listRows(user, datasetName, query = {}) { + const service = await createServiceForUser(user); + try { + return service.readDatasetRows(datasetName, { + limit: query.limit, + offset: query.offset, + orderBy: query.orderBy ?? query.order_by, + orderDir: query.orderDir ?? query.order_dir, + }); + } catch (error) { + throw mapServiceError(error); + } + } + + async function getSchema(user, datasetName) { + const service = await createServiceForUser(user); + try { + return service.getDatasetSchema(datasetName); + } catch (error) { + throw mapServiceError(error); + } + } + + async function getStats(user, datasetName) { + const service = await createServiceForUser(user); + try { + return service.getDatasetStats(datasetName); + } catch (error) { + throw mapServiceError(error); + } + } + + async function insertRow(user, datasetName, payload) { + const service = await createServiceForUser(user); + try { + return service.insertDatasetRow(datasetName, payload); + } catch (error) { + throw mapServiceError(error); + } + } + + async function listDatasets(user) { + const service = await createServiceForUser(user); + return service.listDatasets().map((dataset) => ({ + name: dataset.name, + table: dataset.table, + description: dataset.description, + actions: dataset.actions, + })); + } + + return { + getUserAuth, + listRows, + getSchema, + getStats, + insertRow, + listDatasets, + }; +} diff --git a/page-data-session-store.mjs b/page-data-session-store.mjs new file mode 100644 index 0000000..977d415 --- /dev/null +++ b/page-data-session-store.mjs @@ -0,0 +1,82 @@ +import crypto from 'node:crypto'; + +export function createPageDataSessionStore(options = {}) { + const ttlMs = Number(options.ttlMs ?? 30 * 60 * 1000); + const sessions = new Map(); + + function hashToken(token) { + return crypto.createHash('sha256').update(String(token)).digest('hex'); + } + + function issue({ pageId, ownerUserId, publicationId, accessMode }) { + const token = crypto.randomBytes(32).toString('base64url'); + const tokenHash = hashToken(token); + const expiresAt = Date.now() + ttlMs; + sessions.set(tokenHash, { + pageId, + ownerUserId, + publicationId, + accessMode, + sessionId: crypto.randomUUID(), + expiresAt, + }); + return { + token, + expiresAt, + sessionId: sessions.get(tokenHash).sessionId, + }; + } + + function verify(token) { + const session = sessions.get(hashToken(token)); + if (!session) return null; + if (session.expiresAt <= Date.now()) { + sessions.delete(hashToken(token)); + return null; + } + return session; + } + + function revoke(token) { + return sessions.delete(hashToken(token)); + } + + function clearExpired() { + const now = Date.now(); + for (const [key, session] of sessions.entries()) { + if (session.expiresAt <= now) sessions.delete(key); + } + } + + return { issue, verify, revoke, clearExpired }; +} + +export function createPageDataRateLimiter(options = {}) { + const windowMs = Number(options.windowMs ?? 60_000); + const maxRequests = Number(options.maxRequests ?? 30); + const buckets = new Map(); + + function check(key) { + const now = Date.now(); + const bucket = buckets.get(key) ?? { count: 0, resetAt: now + windowMs }; + if (now >= bucket.resetAt) { + bucket.count = 0; + bucket.resetAt = now + windowMs; + } + bucket.count += 1; + buckets.set(key, bucket); + if (bucket.count > maxRequests) { + throw Object.assign(new Error('请求过于频繁，请稍后再试'), { + code: 'rate_limited', + status: 429, + }); + } + return { remaining: Math.max(0, maxRequests - bucket.count), resetAt: bucket.resetAt }; + } + + return { check }; +} + +export function hashClientMeta(value) { + return crypto.createHash('sha256').update(String(value ?? '')).digest('hex').slice(0, 16); +} diff --git a/scripts/run-memind-tests.mjs b/scripts/run-memind-tests.mjs index 28b3f7a..775d4c0 100644 --- a/scripts/run-memind-tests.mjs +++ b/scripts/run-memind-tests.mjs @@ -132,6 +132,18 @@ const SCOPE_RULES = [ 'plaza-ops.test.mjs', ], }, + { + id: 'page-data', + patterns: [/page-data/i, /page-access-policy/i, /user-data-space-service/i, /mindspace-sandbox-mcp/i], + tests: [ + 'user-data-space-service.test.mjs', + 'page-access-policy.test.mjs', + 'page-data-routes.test.mjs', + 'page-data-public-service.test.mjs', + 'page-data-integration.test.mjs', + 'mindspace-sandbox-mcp.test.mjs', + ], + }, { id: 'mindspace-core', patterns: [/mindspace/i, /server\.mjs$/, /src\/hooks\/useTKMind/i], diff --git a/server.mjs b/server.mjs index b01862e..6607433 100644 --- a/server.mjs +++ b/server.mjs @@ -185,6 +185,9 @@ import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs'; import { createWechatScheduleLlmConfigService } from './wechat-schedule-llm-config.mjs'; import { createExperienceService } from './experience-service.mjs'; import { attachAsrRoutes } from './asr-proxy.mjs'; +import { attachPageDataRoutes } from './page-data-routes.mjs'; +import { createPageDataService } from './page-data-service.mjs'; +import { createPageDataPublicService, isPageDataPublicPath } from './page-data-public-service.mjs'; import { isNativeH5ApiPath } from './policies.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -362,6 +365,8 @@ let scheduleReminderWorker = null; let llmProviderService = null; let wordFilterService = null; let authPool = null; +let pageDataService = null; +let pageDataPublicService = null; async function bootstrapUserAuth() { try { @@ -375,6 +380,19 @@ async function bootstrapUserAuth() { scheduleService = createScheduleService(pool, { defaultTimezone: process.env.H5_DEFAULT_TIMEZONE || 'Asia/Shanghai', }); + pageDataService = createPageDataService({ + getUserAuth: () => userAuth, + getPool: () => authPool, + resolveWorkspaceRoot: async (user) => { + if (user?.workspaceRoot) return user.workspaceRoot; + if (!userAuth || !user?.id) return null; + return userAuth.resolveWorkingDir(user.id); + }, + }); + pageDataPublicService = createPageDataPublicService({ + getPool: () => authPool, + resolveH5Root: () => __dirname, + }); feedbackService = createFeedbackService(pool); mindSpace = createMindSpaceService(pool, { maxFileBytes: mindSpaceServerRuntime.maxFileBytes, @@ -1833,10 +1851,11 @@ api.use(async (req, res, next) => { } const plazaPublic = isPlazaPublicRead(req.path, req.method); + const pageDataPublic = isPageDataPublicPath(req.path, req.method); if (userAuth && tkmindProxy) { if (req.userSessionError) { - if (plazaPublic) return next(); + if (plazaPublic || pageDataPublic) return next(); return res.status(503).json({ message: '用户认证服务不可用，请稍后重试' }); } try { @@ -1844,7 +1863,7 @@ api.use(async (req, res, next) => { const me = await userAuth.getMe(req.userToken); if (me) req.currentUser = me; } - if (plazaPublic) return next(); + if (plazaPublic || pageDataPublic) return next(); if (!req.userSession) { return res.status(401).json({ message: '未授权，请重新登录' }); } @@ -1863,6 +1882,12 @@ api.use(async (req, res, next) => { }); attachAsrRoutes(api, { sendError, sendData }); +attachPageDataRoutes(api, { + sendError, + sendData, + getPageDataService: () => pageDataService, + getPageDataPublicService: () => pageDataPublicService, +}); api.get('/config/blocked-words', async (_req, res) => { await userAuthReady; diff --git a/user-data-space-service.mjs b/user-data-space-service.mjs new file mode 100644 index 0000000..ec8ef8c --- /dev/null +++ b/user-data-space-service.mjs @@ -0,0 +1,618 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; + +export const DATASET_REGISTRY_TABLE = '__page_data_datasets'; + +const DEFAULT_MAX_BYTES = 20 * 1024 * 1024; +const DEFAULT_QUERY_TIMEOUT_MS = 5000; +const DEFAULT_MAX_ROWS = 200; + +function stripSqlComments(sql) { + return String(sql ?? '') + .replace(/--.*$/gm, '') + .replace(/\/\*[\s\S]*?\*\//g, '') + .trim(); +} + +export function assertSafeSqlIdentifier(name, label = '标识符') { + const value = String(name ?? '').trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { + throw Object.assign(new Error(`${label} 格式无效`), { code: 'invalid_identifier' }); + } + return value; +} + +export function rejectDangerousSql(sql, { readonly = false } = {}) { + const cleaned = stripSqlComments(sql); + if (!cleaned) throw new Error('SQL 不能为空'); + if (cleaned.length > 20000) throw new Error('SQL 过长'); + if (/^\s*\./m.test(cleaned)) { + throw new Error('SQL 不允许使用 sqlite3 dot command'); + } + if (/\b(ATTACH|DETACH|LOAD_EXTENSION|VACUUM\s+INTO)\b/i.test(cleaned)) { + throw new Error('SQL 包含用户私有数据空间不允许的操作'); + } + if (/\bPRAGMA\s+writable_schema\b/i.test(cleaned)) { + throw new Error('SQL 包含不允许的 PRAGMA'); + } + if (readonly && !/^\s*(SELECT|WITH)\b/i.test(cleaned)) { + throw new Error('private_data_query 只允许 SELECT/WITH'); + } + return cleaned; +} + +export function normalizeDatasetConfig(raw, { fallbackName = null } = {}) { + const config = typeof raw === 'string' ? JSON.parse(raw) : raw; + if (!config || typeof config !== 'object') { + throw Object.assign(new Error('dataset 配置无效'), { code: 'invalid_dataset_config' }); + } + const name = assertSafeSqlIdentifier(config.name ?? fallbackName, 'dataset 名称'); + const table = assertSafeSqlIdentifier(config.table ?? config.table_name, 'dataset 表名'); + const actions = Array.isArray(config.actions) + ? config.actions.map((action) => String(action).trim()).filter(Boolean) + : []; + const columns = config.columns && typeof config.columns === 'object' ? config.columns : {}; + for (const [action, fields] of Object.entries(columns)) { + if (!Array.isArray(fields)) { + throw Object.assign(new Error(`dataset 字段白名单无效：${action}`), { code: 'invalid_dataset_config' }); + } + columns[action] = fields.map((field) => assertSafeSqlIdentifier(field, `${action} 字段`)); + } + return { + name, + table, + description: String(config.description ?? '').trim(), + actions, + columns, + limits: + config.limits && typeof config.limits === 'object' + ? { + maxRowsPerRead: Number(config.limits.maxRowsPerRead ?? DEFAULT_MAX_ROWS), + maxInsertBytes: Number(config.limits.maxInsertBytes ?? 8192), + } + : { + maxRowsPerRead: DEFAULT_MAX_ROWS, + maxInsertBytes: 8192, + }, + }; +} + +function sqlLiteral(value) { + if (value === null || value === undefined) return 'NULL'; + if (typeof value === 'number' && Number.isFinite(value)) return String(value); + if (typeof value === 'boolean') return value ? '1' : '0'; + return `'${String(value).replace(/'/g, "''")}'`; +} + +export function createUserDataSpaceService(options = {}) { + const workspaceRoot = path.resolve(String(options.workspaceRoot ?? '').trim()); + if (!workspaceRoot) { + throw new Error('workspaceRoot 不能为空'); + } + + const userId = options.userId ? String(options.userId).trim() : null; + const query = typeof options.query === 'function' ? options.query : null; + const sqliteBin = options.sqliteBin?.trim() || process.env.SQLITE_BIN?.trim() || 'sqlite3'; + const maxBytes = Number(options.maxBytes ?? process.env.PRIVATE_DATA_MAX_BYTES ?? DEFAULT_MAX_BYTES); + const queryTimeoutMs = Number( + options.queryTimeoutMs ?? process.env.PRIVATE_DATA_QUERY_TIMEOUT_MS ?? DEFAULT_QUERY_TIMEOUT_MS, + ); + const maxRows = Number(options.maxRows ?? process.env.PRIVATE_DATA_MAX_ROWS ?? DEFAULT_MAX_ROWS); + + const privateDataDir = path.join(workspaceRoot, '.mindspace'); + const privateDataDb = path.join(privateDataDir, 'private-data.sqlite'); + + function ensurePrivateDataDb() { + fs.mkdirSync(privateDataDir, { recursive: true }); + if (!fs.existsSync(privateDataDb)) { + runSqlite(['-batch', privateDataDb, 'PRAGMA journal_mode=WAL; PRAGMA user_version = 1;']); + } + return privateDataDb; + } + + function privateDataSize() { + let total = 0; + for (const file of fs.existsSync(privateDataDir) ? fs.readdirSync(privateDataDir) : []) { + if (file === 'private-data.sqlite' || file.startsWith('private-data.sqlite-')) { + total += fs.statSync(path.join(privateDataDir, file)).size; + } + } + return total; + } + + function runSqlite(args) { + return execFileSync(sqliteBin, args, { + encoding: 'utf8', + timeout: queryTimeoutMs, + maxBuffer: 1024 * 1024, + }); + } + + function sqliteScalar(sql) { + return Number(runSqlite(['-batch', '-noheader', privateDataDb, sql]).trim() || 0); + } + + function runJsonQuery(sql) { + const output = runSqlite(['-json', privateDataDb, sql]); + return output.trim() ? JSON.parse(output) : []; + } + + async function getQuotaState({ forUpdate = false, conn = null } = {}) { + if (!query || !userId) return null; + const db = conn ?? { query }; + const [rows] = await db.query( + `SELECT id, quota_bytes, used_bytes, reserved_bytes, status + FROM h5_user_spaces + WHERE user_id = ? + LIMIT 1 ${forUpdate ? 'FOR UPDATE' : ''}`, + [userId], + ); + const row = rows[0]; + if (!row) return null; + const quotaBytes = Number(row.quota_bytes ?? 0); + const usedBytes = Number(row.used_bytes ?? 0); + const reservedBytes = Number(row.reserved_bytes ?? 0); + return { + id: row.id, + status: row.status, + quotaBytes, + usedBytes, + reservedBytes, + availableBytes: Math.max(0, quotaBytes - usedBytes - reservedBytes), + }; + } + + async function sqliteMaxPagePragmaForQuota(currentBytes) { + const quota = await getQuotaState(); + if (!quota) return ''; + if (quota.status !== 'active') throw new Error('用户空间不可写'); + if (quota.availableBytes <= 0 && currentBytes >= maxBytes) { + throw new Error('用户私有数据空间配额不足'); + } + const allowedSize = Math.min(maxBytes, currentBytes + quota.availableBytes); + const pageSize = sqliteScalar('PRAGMA page_size;') || 4096; + const currentPages = sqliteScalar('PRAGMA page_count;') || 1; + const maxPages = Math.max(currentPages, Math.max(1, Math.floor(allowedSize / pageSize))); + return `PRAGMA max_page_count=${maxPages};`; + } + + async function syncPrivateDataQuota(deltaBytes) { + if (!query || !userId || !deltaBytes) return null; + const conn = await query.getConnection?.(); + if (!conn?.beginTransaction) { + return null; + } + try { + await conn.beginTransaction(); + const quota = await getQuotaState({ forUpdate: true, conn }); + if (!quota) { + await conn.rollback(); + return null; + } + if (quota.status !== 'active') throw new Error('用户空间不可写'); + if (deltaBytes > quota.availableBytes) { + throw new Error(`用户私有数据空间配额不足：需要 ${deltaBytes} 字节，可用 ${quota.availableBytes} 字节`); + } + await conn.query( + `UPDATE h5_user_spaces + SET used_bytes = GREATEST(0, used_bytes + ?), updated_at = ? + WHERE id = ? AND user_id = ?`, + [deltaBytes, Date.now(), quota.id, userId], + ); + await conn.commit(); + return { deltaBytes }; + } catch (err) { + await conn.rollback(); + throw err; + } finally { + conn.release(); + } + } + + function ensureDatasetRegistryTable() { + ensurePrivateDataDb(); + runSqlite([ + '-batch', + privateDataDb, + `CREATE TABLE IF NOT EXISTS ${DATASET_REGISTRY_TABLE} ( + name TEXT PRIMARY KEY, + table_name TEXT NOT NULL, + config_json TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + );`, + ]); + } + + async function getInfo() { + ensurePrivateDataDb(); + const size = privateDataSize(); + const quota = await getQuotaState().catch(() => null); + return { + name: '用户私有数据空间', + database: '.mindspace/private-data.sqlite', + maxBytes, + sizeBytes: size, + quotaSyncEnabled: Boolean(query && userId), + quota: quota + ? { + status: quota.status, + quotaBytes: quota.quotaBytes, + usedBytes: quota.usedBytes, + reservedBytes: quota.reservedBytes, + availableBytes: quota.availableBytes, + } + : null, + }; + } + + async function getSchema() { + ensurePrivateDataDb(); + return runJsonQuery( + `SELECT m.name AS table_name, p.cid, p.name AS column_name, p.type, p."notnull" AS not_null, p.pk + FROM sqlite_master m + JOIN pragma_table_info(m.name) p + WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite_%' + ORDER BY m.name, p.cid`, + ); + } + + async function querySql(sql) { + ensurePrivateDataDb(); + const cleaned = rejectDangerousSql(sql, { readonly: true }); + const limited = `SELECT * FROM (${cleaned.replace(/;\s*$/, '')}) LIMIT ${maxRows}`; + return runJsonQuery(limited); + } + + async function executeSql(sql) { + ensurePrivateDataDb(); + const before = privateDataSize(); + if (before > maxBytes) throw new Error('用户私有数据空间已超过大小限制'); + const cleaned = rejectDangerousSql(sql); + const quotaPragma = await sqliteMaxPagePragmaForQuota(before); + runSqlite(['-batch', privateDataDb, `${quotaPragma}\n${cleaned}`]); + const after = privateDataSize(); + if (after > maxBytes) { + throw new Error(`用户私有数据空间超过大小限制：${after}/${maxBytes} 字节`); + } + const delta = after - before; + const quotaSync = await syncPrivateDataQuota(delta); + return { + sizeBytes: after, + deltaBytes: delta, + quotaSynced: Boolean(quotaSync), + }; + } + + function listTableColumns(tableName) { + const table = assertSafeSqlIdentifier(tableName, '表名'); + return runJsonQuery(`PRAGMA table_info(${table});`).map((row) => ({ + name: row.name, + type: row.type, + notNull: Boolean(row.notnull), + pk: Boolean(row.pk), + })); + } + + function tableHasColumn(tableName, columnName) { + return listTableColumns(tableName).some((column) => column.name === columnName); + } + + function getDatasetRow(name) { + ensureDatasetRegistryTable(); + const datasetName = assertSafeSqlIdentifier(name, 'dataset 名称'); + const rows = runJsonQuery( + `SELECT name, table_name, config_json, created_at, updated_at + FROM ${DATASET_REGISTRY_TABLE} + WHERE name = ${sqlLiteral(datasetName)} + LIMIT 1`, + ); + return rows[0] ?? null; + } + + function getDataset(name) { + const row = getDatasetRow(name); + if (!row) return null; + const config = normalizeDatasetConfig(row.config_json, { fallbackName: row.name }); + return { + ...config, + table: config.table || row.table_name, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; + } + + function listDatasets() { + ensureDatasetRegistryTable(); + return runJsonQuery( + `SELECT name, table_name, config_json, created_at, updated_at + FROM ${DATASET_REGISTRY_TABLE} + ORDER BY name`, + ).map((row) => { + const config = normalizeDatasetConfig(row.config_json, { fallbackName: row.name }); + return { + ...config, + table: config.table || row.table_name, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; + }); + } + + async function upsertDataset(configInput) { + const config = normalizeDatasetConfig(configInput); + ensureDatasetRegistryTable(); + const columns = listTableColumns(config.table); + if (!columns.length) { + throw Object.assign(new Error(`dataset 对应表不存在：${config.table}`), { code: 'table_not_found' }); + } + const payload = JSON.stringify(config); + const sql = `INSERT INTO ${DATASET_REGISTRY_TABLE} (name, table_name, config_json, updated_at) + VALUES (${sqlLiteral(config.name)}, ${sqlLiteral(config.table)}, ${sqlLiteral(payload)}, CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET + table_name = excluded.table_name, + config_json = excluded.config_json, + updated_at = CURRENT_TIMESTAMP;`; + await executeSql(sql); + return getDataset(config.name); + } + + function assertDatasetAction(dataset, action) { + if (!dataset.actions.includes(action)) { + throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + } + + function resolveReadColumns(dataset) { + const columns = dataset.columns?.read; + if (!Array.isArray(columns) || !columns.length) { + throw Object.assign(new Error('dataset 未配置可读字段'), { code: 'columns_not_allowed' }); + } + return columns.map((column) => assertSafeSqlIdentifier(column, '读字段')); + } + + function resolveInsertColumns(dataset, payload) { + const allowed = dataset.columns?.insert; + if (!Array.isArray(allowed) || !allowed.length) { + throw Object.assign(new Error('dataset 未配置可写字段'), { code: 'columns_not_allowed' }); + } + const allowedSet = new Set(allowed.map((column) => assertSafeSqlIdentifier(column, '写字段'))); + const keys = Object.keys(payload ?? {}); + if (!keys.length) { + throw Object.assign(new Error('提交数据不能为空'), { code: 'invalid_payload' }); + } + for (const key of keys) { + if (!allowedSet.has(key)) { + throw Object.assign(new Error(`字段未授权：${key}`), { code: 'columns_not_allowed' }); + } + } + return keys.map((key) => assertSafeSqlIdentifier(key, '写字段')); + } + + function resolveUpdateColumns(dataset, payload) { + const allowed = dataset.columns?.update; + if (!Array.isArray(allowed) || !allowed.length) { + throw Object.assign(new Error('dataset 未配置可更新字段'), { code: 'columns_not_allowed' }); + } + const allowedSet = new Set(allowed.map((column) => assertSafeSqlIdentifier(column, '更新字段'))); + const keys = Object.keys(payload ?? {}); + if (!keys.length) { + throw Object.assign(new Error('更新数据不能为空'), { code: 'invalid_payload' }); + } + for (const key of keys) { + if (!allowedSet.has(key)) { + throw Object.assign(new Error(`字段未授权：${key}`), { code: 'columns_not_allowed' }); + } + } + return keys.map((key) => assertSafeSqlIdentifier(key, '更新字段')); + } + + function readRowsForDataset(dataset, { limit, offset, orderBy, orderDir } = {}) { + assertDatasetAction(dataset, 'read'); + const columns = resolveReadColumns(dataset); + const table = assertSafeSqlIdentifier(dataset.table, '表名'); + const maxLimit = Math.min( + Math.max(1, Number(dataset.limits?.maxRowsPerRead ?? maxRows)), + maxRows, + ); + const safeLimit = Math.min(Math.max(1, Number(limit ?? maxLimit)), maxLimit); + const safeOffset = Math.max(0, Number(offset ?? 0)); + const where = tableHasColumn(table, 'deleted_at') ? 'WHERE deleted_at IS NULL' : ''; + let orderClause = ''; + if (orderBy) { + const column = assertSafeSqlIdentifier(orderBy, '排序字段'); + if (!columns.includes(column)) { + throw Object.assign(new Error(`排序字段未授权：${column}`), { code: 'columns_not_allowed' }); + } + const direction = String(orderDir ?? 'asc').toLowerCase() === 'desc' ? 'DESC' : 'ASC'; + orderClause = `ORDER BY "${column}" ${direction}`; + } else if (columns.includes('id')) { + orderClause = 'ORDER BY "id" DESC'; + } + const sql = `SELECT ${columns.map((column) => `"${column}"`).join(', ')} + FROM "${table}" + ${where} + ${orderClause} + LIMIT ${safeLimit} OFFSET ${safeOffset}`; + const rows = runJsonQuery(sql); + return { dataset, rows, limit: safeLimit, offset: safeOffset }; + } + + function readDatasetRows(datasetName, options = {}) { + const dataset = getDataset(datasetName); + if (!dataset) { + throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + } + return readRowsForDataset(dataset, options); + } + + function getStatsForDataset(dataset) { + assertDatasetAction(dataset, 'read'); + const table = assertSafeSqlIdentifier(dataset.table, '表名'); + const where = tableHasColumn(table, 'deleted_at') ? 'WHERE deleted_at IS NULL' : ''; + const total = sqliteScalar(`SELECT COUNT(*) FROM "${table}" ${where};`); + return { dataset: { name: dataset.name, table: dataset.table }, total }; + } + + function getDatasetStats(datasetName) { + const dataset = getDataset(datasetName); + if (!dataset) { + throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + } + return getStatsForDataset(dataset); + } + + async function insertRowForDataset(dataset, payload, meta = {}) { + assertDatasetAction(dataset, 'insert'); + const payloadBytes = Buffer.byteLength(JSON.stringify(payload ?? {}), 'utf8'); + const maxInsertBytes = Number(dataset.limits?.maxInsertBytes ?? 8192); + if (payloadBytes > maxInsertBytes) { + throw Object.assign(new Error('提交数据过大'), { code: 'payload_too_large' }); + } + const table = assertSafeSqlIdentifier(dataset.table, '表名'); + const columns = resolveInsertColumns(dataset, payload); + const values = { ...payload }; + if (tableHasColumn(table, 'created_at') && values.created_at == null) { + values.created_at = new Date().toISOString(); + } + if (tableHasColumn(table, 'created_ip_hash') && values.created_ip_hash == null && meta.createdIpHash) { + values.created_ip_hash = meta.createdIpHash; + } + if (tableHasColumn(table, 'user_agent_hash') && values.user_agent_hash == null && meta.userAgentHash) { + values.user_agent_hash = meta.userAgentHash; + } + if (tableHasColumn(table, 'page_data_session_id') && values.page_data_session_id == null && meta.pageDataSessionId) { + values.page_data_session_id = meta.pageDataSessionId; + } + const insertColumns = columns.filter((column) => values[column] !== undefined); + if (!insertColumns.length) { + throw Object.assign(new Error('提交数据不能为空'), { code: 'invalid_payload' }); + } + const sql = `INSERT INTO "${table}" (${insertColumns.map((column) => `"${column}"`).join(', ')}) + VALUES (${insertColumns.map((column) => sqlLiteral(values[column])).join(', ')});`; + await executeSql(sql); + const id = sqliteScalar('SELECT last_insert_rowid();'); + let row = { id }; + if (dataset.actions.includes('read')) { + const readBack = readRowsForDataset(dataset, { limit: 1, orderBy: 'id', orderDir: 'desc' }); + row = readBack.rows.find((item) => Number(item.id) === Number(id)) ?? readBack.rows[0] ?? row; + } else { + for (const column of insertColumns) { + row[column] = values[column]; + } + } + return { dataset, row }; + } + + async function insertDatasetRow(datasetName, payload, meta = {}) { + const dataset = getDataset(datasetName); + if (!dataset) { + throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + } + return insertRowForDataset(dataset, payload, meta); + } + + async function updateRowForDataset(dataset, rowId, payload, meta = {}) { + assertDatasetAction(dataset, 'update'); + const table = assertSafeSqlIdentifier(dataset.table, '表名'); + const id = Number(rowId); + if (!Number.isFinite(id) || id <= 0) { + throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' }); + } + const columns = resolveUpdateColumns(dataset, payload); + const values = { ...payload }; + if (tableHasColumn(table, 'updated_at')) { + values.updated_at = new Date().toISOString(); + } + if (tableHasColumn(table, 'updated_by_label') && meta.updatedByLabel) { + values.updated_by_label = meta.updatedByLabel; + } + const assignments = columns.map((column) => `"${column}" = ${sqlLiteral(values[column])}`); + if (tableHasColumn(table, 'updated_at') && !columns.includes('updated_at')) { + assignments.push(`"updated_at" = ${sqlLiteral(values.updated_at)}`); + } + if (tableHasColumn(table, 'updated_by_label') && meta.updatedByLabel && !columns.includes('updated_by_label')) { + assignments.push(`"updated_by_label" = ${sqlLiteral(meta.updatedByLabel)}`); + } + const deletedFilter = tableHasColumn(table, 'deleted_at') ? ' AND deleted_at IS NULL' : ''; + const sql = `UPDATE "${table}" + SET ${assignments.join(', ')} + WHERE id = ${id}${deletedFilter};`; + await executeSql(sql); + const readBack = readRowsForDataset(dataset, { limit: 1, orderBy: 'id', orderDir: 'desc' }); + const row = readBack.rows.find((item) => Number(item.id) === id) ?? { id }; + return { dataset, row }; + } + + async function softDeleteRowForDataset(dataset, rowId, meta = {}) { + assertDatasetAction(dataset, 'soft_delete'); + const table = assertSafeSqlIdentifier(dataset.table, '表名'); + const id = Number(rowId); + if (!Number.isFinite(id) || id <= 0) { + throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' }); + } + if (!tableHasColumn(table, 'deleted_at')) { + throw Object.assign(new Error('目标表不支持软删除'), { code: 'soft_delete_unsupported' }); + } + const assignments = ['deleted_at = CURRENT_TIMESTAMP']; + if (tableHasColumn(table, 'deleted_by')) { + assignments.push(`deleted_by = ${sqlLiteral(meta.deletedBy ?? meta.updatedByLabel ?? 'public')}`); + } + const sql = `UPDATE "${table}" + SET ${assignments.join(', ')} + WHERE id = ${id} AND deleted_at IS NULL;`; + await executeSql(sql); + return { dataset, id, deleted: true }; + } + + function getSchemaForDataset(dataset) { + const tableColumns = listTableColumns(dataset.table); + return { + dataset: { + name: dataset.name, + table: dataset.table, + description: dataset.description, + actions: dataset.actions, + columns: dataset.columns, + limits: dataset.limits, + }, + tableColumns, + }; + } + + function getDatasetSchema(datasetName) { + const dataset = getDataset(datasetName); + if (!dataset) { + throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + } + return getSchemaForDataset(dataset); + } + + return { + workspaceRoot, + privateDataDir, + privateDataDb, + ensurePrivateDataDb, + privateDataSize, + getInfo, + getSchema, + querySql, + executeSql, + listTableColumns, + ensureDatasetRegistryTable, + getDataset, + listDatasets, + upsertDataset, + readDatasetRows, + readRowsForDataset, + getDatasetStats, + getStatsForDataset, + insertDatasetRow, + insertRowForDataset, + updateRowForDataset, + softDeleteRowForDataset, + getDatasetSchema, + getSchemaForDataset, + }; +} diff --git a/user-data-space-service.test.mjs b/user-data-space-service.test.mjs new file mode 100644 index 0000000..b688eac --- /dev/null +++ b/user-data-space-service.test.mjs @@ -0,0 +1,139 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; + +function createTempWorkspace() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'user-data-space-')); +} + +test('createUserDataSpaceService supports agent SQL and dataset registry', async () => { + const workspaceRoot = createTempWorkspace(); + const service = createUserDataSpaceService({ workspaceRoot }); + + await service.executeSql(`CREATE TABLE form_registrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + phone TEXT, + status TEXT DEFAULT 'pending', + note TEXT, + created_at TEXT, + deleted_at TEXT + );`); + + const dataset = await service.upsertDataset({ + name: 'registrations', + table: 'form_registrations', + description: '活动报名数据', + actions: ['read', 'insert'], + columns: { + read: ['id', 'name', 'phone', 'status', 'note', 'created_at'], + insert: ['name', 'phone', 'status', 'note'], + }, + limits: { maxRowsPerRead: 50, maxInsertBytes: 4096 }, + }); + + assert.equal(dataset.name, 'registrations'); + assert.deepEqual(dataset.actions, ['read', 'insert']); + + const inserted = await service.insertDatasetRow('registrations', { + name: '张三', + phone: '13800000000', + status: 'pending', + note: '第一场', + }); + assert.equal(inserted.row.name, '张三'); + + const listed = service.readDatasetRows('registrations', { limit: 10 }); + assert.equal(listed.rows.length, 1); + assert.equal(listed.rows[0].phone, '13800000000'); + + const stats = service.getDatasetStats('registrations'); + assert.equal(stats.total, 1); + + const schema = service.getDatasetSchema('registrations'); + assert.equal(schema.dataset.name, 'registrations'); + assert.ok(schema.tableColumns.some((column) => column.name === 'name')); +}); + +test('dataset insert rejects unauthorized columns', async () => { + const workspaceRoot = createTempWorkspace(); + const service = createUserDataSpaceService({ workspaceRoot }); + + await service.executeSql(`CREATE TABLE secrets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + secret_code TEXT + );`); + + await service.upsertDataset({ + name: 'public_names', + table: 'secrets', + actions: ['insert'], + columns: { insert: ['name'] }, + }); + + await assert.rejects( + () => service.insertDatasetRow('public_names', { name: '公开项', secret_code: 'hidden' }), + (error) => error.code === 'columns_not_allowed', + ); +}); + +test('dataset read rejects missing dataset', () => { + const workspaceRoot = createTempWorkspace(); + const service = createUserDataSpaceService({ workspaceRoot }); + + assert.throws( + () => service.readDatasetRows('missing'), + (error) => error.code === 'dataset_not_found', + ); +}); + +test('rejectDangerousSql blocks attach and dot commands', async () => { + const workspaceRoot = createTempWorkspace(); + const service = createUserDataSpaceService({ workspaceRoot }); + + await assert.rejects( + () => service.querySql("ATTACH DATABASE '/tmp/other.sqlite' AS other; SELECT 1"), + /不允许/, + ); + await assert.rejects(() => service.executeSql('.open /tmp/other.sqlite'), /dot command/); +}); + +test('user-data-space-service supports update and soft delete', async () => { + const workspaceRoot = createTempWorkspace(); + const service = createUserDataSpaceService({ workspaceRoot }); + await service.executeSql(`CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + status TEXT DEFAULT 'open', + updated_at TEXT, + updated_by_label TEXT, + deleted_at TEXT, + deleted_by TEXT + );`); + await service.upsertDataset({ + name: 'tasks', + table: 'tasks', + actions: ['read', 'insert', 'update', 'soft_delete'], + columns: { + read: ['id', 'title', 'status', 'updated_at'], + insert: ['title', 'status'], + update: ['status'], + soft_delete: ['id'], + }, + }); + const inserted = await service.insertDatasetRow('tasks', { title: '整理资料', status: 'open' }); + const updated = await service.updateRowForDataset(service.getDataset('tasks'), inserted.row.id, { + status: 'done', + }, { updatedByLabel: 'owner' }); + assert.equal(updated.row.status, 'done'); + const deleted = await service.softDeleteRowForDataset(service.getDataset('tasks'), inserted.row.id, { + deletedBy: 'owner', + }); + assert.equal(deleted.deleted, true); + const listed = service.readDatasetRows('tasks', { limit: 10 }); + assert.equal(listed.rows.length, 0); +});