feat(mindspace): enforce PostgreSQL user data delivery
This commit is contained in:
+28
-1
@@ -242,4 +242,31 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind
|
||||
# MINDSPACE_AGENT_JOBS_ENABLED=true
|
||||
# MINDSPACE_INTERNAL_AGENT_SECRET=local-dev-secret
|
||||
# MINDSPACE_FREE_PUBLIC_PAGE_LIMIT=10
|
||||
# MINDSPACE_STORAGE_ROOT=/path/to/mindspace-storage
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Memind runtime profile(本地 vs 生产 必须区分)
|
||||
# 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。
|
||||
#
|
||||
# | profile | 场景 | MindSpace 数据路径 |
|
||||
# |----------------|-------------------|--------------------|
|
||||
# | local | 本机日常开发 | <repo>/data/mindspace |
|
||||
# | split-service | 本机验证 103 拆分 | /Users/john/MindSpace/data/mindspace |
|
||||
# | production | 103 生产服务器 | 103 .env 为准,勿提交 Git |
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# 【本地开发 — 复制到 .env 的默认值】
|
||||
MEMIND_RUNTIME_PROFILE=local
|
||||
# MINDSPACE_STORAGE_ROOT 留空即可,自动使用 <repo>/data/mindspace
|
||||
|
||||
# 【本机 split-service 联调 — 需要独立 MindSpace LaunchAgent 已启动】
|
||||
# MEMIND_RUNTIME_PROFILE=split-service
|
||||
# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
|
||||
# MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret
|
||||
# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace
|
||||
|
||||
# 【103 生产 — 仅在服务器维护,示例勿照抄密钥】
|
||||
# MEMIND_RUNTIME_PROFILE=production
|
||||
# MINDSPACE_SERVER_ADAPTER=remote
|
||||
# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
|
||||
# MINDSPACE_REMOTE_AUTH_TOKEN=<与 cn.tkmind.mindspace-service 一致>
|
||||
# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace
|
||||
|
||||
@@ -26,6 +26,7 @@ function mapDeliverableRows(rows) {
|
||||
publicationId: row.publication_id ? String(row.publication_id) : null,
|
||||
publicationStatus: row.publication_status ? String(row.publication_status) : null,
|
||||
publicUrl: row.public_url ? String(row.public_url) : null,
|
||||
workspaceRelativePath: row.workspace_relative_path ? String(row.workspace_relative_path) : null,
|
||||
})).filter((row) => row.pageId);
|
||||
const publicationCount = pages.filter((row) => row.publicationId).length;
|
||||
return {
|
||||
@@ -51,13 +52,16 @@ export function mergeDeliverableSummaries(...summaries) {
|
||||
};
|
||||
}
|
||||
|
||||
export async function detectSessionDeliverables(pool, userId, sessionId) {
|
||||
export async function detectSessionDeliverables(pool, userId, sessionId, { sinceMs = null } = {}) {
|
||||
if (!pool?.query || !userId || !sessionId) {
|
||||
return { pageCount: 0, publicationCount: 0, pages: [] };
|
||||
}
|
||||
const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : '';
|
||||
const params = sinceMs != null ? [userId, sessionId, sinceMs] : [userId, sessionId];
|
||||
const [rows] = await pool.query(
|
||||
`SELECT p.id AS page_id,
|
||||
p.title,
|
||||
p.workspace_relative_path,
|
||||
pr.id AS publication_id,
|
||||
pr.status AS publication_status,
|
||||
pr.public_url
|
||||
@@ -68,22 +72,39 @@ export async function detectSessionDeliverables(pool, userId, sessionId) {
|
||||
AND pr.status = 'online'
|
||||
WHERE p.user_id = ?
|
||||
AND p.source_session_id = ?
|
||||
${sinceClause}
|
||||
AND p.status <> 'deleted'
|
||||
ORDER BY p.created_at ASC`,
|
||||
[userId, sessionId],
|
||||
params,
|
||||
);
|
||||
return mapDeliverableRows(rows);
|
||||
}
|
||||
|
||||
export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs = null } = {}) {
|
||||
export async function detectWorkspaceSyncedDeliverables(
|
||||
pool,
|
||||
userId,
|
||||
{ sinceMs = null, onlyRelativePaths = null } = {},
|
||||
) {
|
||||
if (!pool?.query || !userId) {
|
||||
return { pageCount: 0, publicationCount: 0, pages: [] };
|
||||
}
|
||||
const normalizedPaths = onlyRelativePaths == null
|
||||
? null
|
||||
: [...new Set(onlyRelativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))];
|
||||
if (normalizedPaths?.length === 0) {
|
||||
return { pageCount: 0, publicationCount: 0, pages: [] };
|
||||
}
|
||||
const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : '';
|
||||
const params = sinceMs != null ? [userId, sinceMs] : [userId];
|
||||
const pathClause = normalizedPaths == null
|
||||
? ''
|
||||
: `AND p.workspace_relative_path IN (${normalizedPaths.map(() => '?').join(', ')})`;
|
||||
const params = [userId];
|
||||
if (sinceMs != null) params.push(sinceMs);
|
||||
if (normalizedPaths != null) params.push(...normalizedPaths);
|
||||
const [rows] = await pool.query(
|
||||
`SELECT p.id AS page_id,
|
||||
p.title,
|
||||
p.workspace_relative_path,
|
||||
pr.id AS publication_id,
|
||||
pr.status AS publication_status,
|
||||
pr.public_url
|
||||
@@ -96,6 +117,7 @@ export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs
|
||||
WHERE p.user_id = ?
|
||||
AND p.status <> 'deleted'
|
||||
${sinceClause}
|
||||
${pathClause}
|
||||
AND (
|
||||
p.workspace_relative_path LIKE 'public/%.html'
|
||||
OR JSON_UNQUOTE(JSON_EXTRACT(pv.source_snapshot_json, '$.relative_path')) LIKE 'public/%.html'
|
||||
@@ -116,15 +138,20 @@ export async function prepareAndDetectSessionDeliverables({
|
||||
sessionId,
|
||||
runStartedAtMs = null,
|
||||
prepareDeliverables = null,
|
||||
workspaceRelativePaths = null,
|
||||
} = {}) {
|
||||
let prepared = null;
|
||||
if (typeof prepareDeliverables === 'function') {
|
||||
await prepareDeliverables({ userId, sessionId }).catch(() => {});
|
||||
prepared = await prepareDeliverables({ userId, sessionId }).catch(() => null);
|
||||
}
|
||||
const sinceMs = runStartedAtMs == null
|
||||
? null
|
||||
: Math.max(0, Number(runStartedAtMs) - WORKSPACE_DELIVERABLE_LOOKBACK_MS);
|
||||
const bySession = await detectSessionDeliverables(pool, userId, sessionId);
|
||||
const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs });
|
||||
const bySession = await detectSessionDeliverables(pool, userId, sessionId, { sinceMs });
|
||||
const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, {
|
||||
sinceMs,
|
||||
onlyRelativePaths: workspaceRelativePaths ?? prepared?.pageDataRelativePaths ?? null,
|
||||
});
|
||||
return mergeDeliverableSummaries(bySession, byWorkspace);
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,40 @@ test('detectWorkspaceSyncedDeliverables filters auto-synced public html pages',
|
||||
assert.equal(summary.pageCount, 1);
|
||||
});
|
||||
|
||||
test('detectWorkspaceSyncedDeliverables stays inside current run artifact paths', async () => {
|
||||
const pool = {
|
||||
async query(sql, params) {
|
||||
assert.match(sql, /workspace_relative_path IN \(\?, \?\)/);
|
||||
assert.deepEqual(params, [
|
||||
'user-1',
|
||||
1000,
|
||||
'public/current.html',
|
||||
'public/current-admin.html',
|
||||
]);
|
||||
return [[{
|
||||
page_id: 'page-current',
|
||||
title: 'Current',
|
||||
workspace_relative_path: 'public/current.html',
|
||||
}]];
|
||||
},
|
||||
};
|
||||
const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', {
|
||||
sinceMs: 1000,
|
||||
onlyRelativePaths: ['public/current.html', 'public/current-admin.html'],
|
||||
});
|
||||
assert.equal(summary.pageCount, 1);
|
||||
assert.equal(summary.pages[0].workspaceRelativePath, 'public/current.html');
|
||||
});
|
||||
|
||||
test('detectWorkspaceSyncedDeliverables skips workspace sweep for an empty run scope', async () => {
|
||||
const pool = { async query() { throw new Error('must not query'); } };
|
||||
const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', {
|
||||
sinceMs: 1000,
|
||||
onlyRelativePaths: [],
|
||||
});
|
||||
assert.equal(summary.pageCount, 0);
|
||||
});
|
||||
|
||||
test('prepareAndDetectSessionDeliverables merges session and workspace pages after prepare hook', async () => {
|
||||
const prepareCalls = [];
|
||||
const pool = {
|
||||
|
||||
+47
-14
@@ -16,7 +16,7 @@ import {
|
||||
SESSION_FINISHED_STALE_GRACE_MS,
|
||||
tryRecoverRunFromDeliverables,
|
||||
} from './agent-run-deliverable-check.mjs';
|
||||
import { isPageDataIntent } from './chat-skills.mjs';
|
||||
import { isPageDataIntent, isPageGenerationIntent } from './chat-skills.mjs';
|
||||
|
||||
const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000];
|
||||
const TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
|
||||
@@ -262,6 +262,7 @@ export function createAgentRunGateway({
|
||||
syncUserPagesOnSuccess = null,
|
||||
observePersonalMemoryOnSuccess = null,
|
||||
isSessionExternallyBusy = null,
|
||||
validateRunDeliverables = null,
|
||||
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
|
||||
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
|
||||
maxConcurrentRuns = positiveInteger(
|
||||
@@ -420,7 +421,7 @@ export function createAgentRunGateway({
|
||||
|
||||
async function prepareRunDeliverables({ userId, sessionId, runId, runStartedAtMs = null }) {
|
||||
if (typeof syncUserPagesOnSuccess !== 'function') return;
|
||||
await syncUserPagesOnSuccess({
|
||||
return await syncUserPagesOnSuccess({
|
||||
userId,
|
||||
sessionId,
|
||||
runId,
|
||||
@@ -619,7 +620,7 @@ export function createAgentRunGateway({
|
||||
billed: Boolean(result.billing?.ok),
|
||||
});
|
||||
await appendRunSnapshot(runId);
|
||||
return { sessionId: result.sessionId };
|
||||
return { sessionId: result.sessionId, routing };
|
||||
} catch (err) {
|
||||
await appendEvent(runId, 'direct_chat_failed', {
|
||||
code: err?.code ?? null,
|
||||
@@ -673,7 +674,7 @@ export function createAgentRunGateway({
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
return { sessionId: row.agent_session_id ?? null };
|
||||
return { sessionId: row.agent_session_id ?? null, routing };
|
||||
}
|
||||
|
||||
let sessionId = resolveGatewayAgentSessionId({
|
||||
@@ -769,7 +770,7 @@ export function createAgentRunGateway({
|
||||
sessionId,
|
||||
err,
|
||||
})) {
|
||||
return { sessionId };
|
||||
return { sessionId, routing };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
@@ -785,16 +786,17 @@ export function createAgentRunGateway({
|
||||
},
|
||||
);
|
||||
}
|
||||
return { sessionId };
|
||||
return { sessionId, routing };
|
||||
}
|
||||
|
||||
async function finalizeSuccessfulRun(runId, row, sessionId) {
|
||||
async function finalizeSuccessfulRun(runId, row, sessionId, { routing = null } = {}) {
|
||||
let deliveryResult = null;
|
||||
if (typeof syncUserPagesOnSuccess === 'function') {
|
||||
deliveryResult = await syncUserPagesOnSuccess({
|
||||
userId: row.user_id,
|
||||
sessionId,
|
||||
runId,
|
||||
runStartedAtMs: row.started_at ?? null,
|
||||
});
|
||||
}
|
||||
const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors)
|
||||
@@ -806,17 +808,48 @@ export function createAgentRunGateway({
|
||||
error.retryable = false;
|
||||
throw error;
|
||||
}
|
||||
if (isPageDataIntent(extractRunMessageText(row))) {
|
||||
const runMessageText = extractRunMessageText(row);
|
||||
const pageDataIntent = isPageDataIntent(runMessageText)
|
||||
|| routing?.suggestedSkill === 'page-data-collect';
|
||||
const pageGenerationIntent = isPageGenerationIntent(runMessageText)
|
||||
|| routing?.suggestedSkill === 'static-page-publish';
|
||||
const requiresPageDeliverable = pageDataIntent || pageGenerationIntent;
|
||||
let deliverables = null;
|
||||
if (requiresPageDeliverable || typeof validateRunDeliverables === 'function') {
|
||||
const latest = await getRunById(runId);
|
||||
const deliverables = await prepareAndDetectSessionDeliverables({
|
||||
deliverables = await prepareAndDetectSessionDeliverables({
|
||||
pool,
|
||||
userId: row.user_id,
|
||||
sessionId,
|
||||
runStartedAtMs: latest?.started_at ?? row.started_at ?? null,
|
||||
workspaceRelativePaths: deliveryResult?.pageDataRelativePaths ?? [],
|
||||
});
|
||||
if (deliverables.pageCount < 1) {
|
||||
const error = new Error('Page Data 任务未生成可交付页面,不能标记成功');
|
||||
error.code = 'PAGE_DATA_DELIVERABLE_MISSING';
|
||||
if (requiresPageDeliverable && deliverables.pageCount < 1) {
|
||||
const error = new Error(
|
||||
pageDataIntent
|
||||
? 'Page Data 任务未生成可交付页面,不能标记成功'
|
||||
: '页面任务未生成 public HTML 交付物,不能标记成功',
|
||||
);
|
||||
error.code = pageDataIntent
|
||||
? 'PAGE_DATA_DELIVERABLE_MISSING'
|
||||
: 'PUBLIC_PAGE_DELIVERABLE_MISSING';
|
||||
error.retryable = false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (typeof validateRunDeliverables === 'function') {
|
||||
const validation = await validateRunDeliverables({
|
||||
userId: row.user_id,
|
||||
sessionId,
|
||||
runId,
|
||||
deliverables: deliverables ?? { pageCount: 0, publicationCount: 0, pages: [] },
|
||||
});
|
||||
const errors = Array.isArray(validation?.errors) ? validation.errors : [];
|
||||
if (errors.length > 0) {
|
||||
const error = new Error(
|
||||
`页面交付违反数据存储策略:${errors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`,
|
||||
);
|
||||
error.code = 'DELIVERABLE_DATA_STORAGE_FORBIDDEN';
|
||||
error.retryable = false;
|
||||
throw error;
|
||||
}
|
||||
@@ -856,8 +889,8 @@ export function createAgentRunGateway({
|
||||
const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt });
|
||||
|
||||
try {
|
||||
const { sessionId } = await runWithTimeout(runId, () => executeRun(row, runId));
|
||||
await finalizeSuccessfulRun(runId, row, sessionId);
|
||||
const { sessionId, routing } = await runWithTimeout(runId, () => executeRun(row, runId));
|
||||
await finalizeSuccessfulRun(runId, row, sessionId, { routing });
|
||||
} catch (err) {
|
||||
const latest = await getRunById(runId);
|
||||
const recoverySessionId = latest?.agent_session_id ?? row.agent_session_id ?? null;
|
||||
|
||||
+261
-5
@@ -387,6 +387,191 @@ test('agent run awaits session Finish before succeeding when proxy supports it',
|
||||
assert.equal(finishEvents.length, 1);
|
||||
});
|
||||
|
||||
test('Page Data run fails closed when Finish arrives without a generated page', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-page-data-missing' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
return { ok: true, finishEvent: { type: 'Finish' } };
|
||||
},
|
||||
},
|
||||
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-page-data-missing',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我创建调查问卷,保存提交记录并发布页面' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
|
||||
assert.match(pool.runs.get(run.id).error_message, /未生成可交付页面/);
|
||||
});
|
||||
|
||||
test('implicit sticky-note app run fails closed when Apps returns Finish without a public page', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
chatIntentRouter: {
|
||||
isEnabled() {
|
||||
return true;
|
||||
},
|
||||
async classify() {
|
||||
return {
|
||||
route: 'agent_orchestration',
|
||||
confidence: 0.96,
|
||||
reason: '页面需要数据交互与持久化',
|
||||
suggestedSkill: 'page-data-collect',
|
||||
source: 'rule',
|
||||
};
|
||||
},
|
||||
applyAgentOrchestration(message) {
|
||||
return message;
|
||||
},
|
||||
},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-sticky-note-missing' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
return { ok: true, finishEvent: { type: 'Finish' } };
|
||||
},
|
||||
},
|
||||
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-sticky-note-missing',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我设计一个便签提醒,可以写便签提交,时间轴来显示' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
|
||||
assert.match(pool.runs.get(run.id).error_message, /Page Data 任务未生成可交付页面/);
|
||||
assert.ok(pool.events.some((event) => event.eventType === 'intent_routed'));
|
||||
});
|
||||
|
||||
test('static page run fails closed when Finish arrives without public HTML', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-public-page-missing' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
return { ok: true, finishEvent: { type: 'Finish' } };
|
||||
},
|
||||
},
|
||||
syncUserPagesOnSuccess: async () => ({}),
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-public-page-missing',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我做一个秋夜诗的 H5 页面' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
|
||||
assert.match(pool.runs.get(run.id).error_message, /public HTML 交付物/);
|
||||
});
|
||||
|
||||
test('Page Data run succeeds only after a generated session page is detected', async () => {
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:session-page-data-ready': [{
|
||||
page_id: 'page-ready',
|
||||
title: '问卷',
|
||||
publication_id: 'pub-ready',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/u/john/pages/page-ready',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-page-data-ready' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
return { ok: true, finishEvent: { type: 'Finish' } };
|
||||
},
|
||||
},
|
||||
syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }),
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-page-data-ready',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '创建一个可以保存提交记录的问卷' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||
});
|
||||
|
||||
test('agent run fails closed when a generated page violates browser storage policy', async () => {
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:session-browser-storage': [{
|
||||
page_id: 'page-storage',
|
||||
title: '页面',
|
||||
workspace_relative_path: 'public/page.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-browser-storage' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
return { ok: true, finishEvent: { type: 'Finish' } };
|
||||
},
|
||||
},
|
||||
validateRunDeliverables: async ({ deliverables }) => ({
|
||||
errors: deliverables.pages.some((page) => page.workspaceRelativePath === 'public/page.html')
|
||||
? [{ code: 'browser_storage_forbidden', message: 'public/page.html 使用 localStorage' }]
|
||||
: [],
|
||||
}),
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-browser-storage',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我做一个展示页面' }],
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
|
||||
assert.match(pool.runs.get(run.id).error_message, /页面交付违反数据存储策略/);
|
||||
assert.match(pool.runs.get(run.id).error_message, /localStorage/);
|
||||
});
|
||||
|
||||
test('agent run succeeds when Finish is missing but session pages were already created', async () => {
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
@@ -563,7 +748,17 @@ test('agent run uses direct chat on regular agent sessions when llm routes direc
|
||||
});
|
||||
|
||||
test('agent run invalidates portal direct chat snapshot before submitting to goosed', async () => {
|
||||
const pool = createFakePool();
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:20260704_31': [{
|
||||
page_id: 'page-essay',
|
||||
title: '散文页面',
|
||||
publication_id: 'pub-essay',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/essay.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const submitted = [];
|
||||
const invalidated = [];
|
||||
const gateway = createAgentRunGateway({
|
||||
@@ -723,7 +918,17 @@ test('agent run falls back to backend session when router is disabled', async ()
|
||||
});
|
||||
|
||||
test('agent run uses chat intent router to enrich agent orchestration messages', async () => {
|
||||
const pool = createFakePool();
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:agent-session-1': [{
|
||||
page_id: 'page-router-agent',
|
||||
title: '路由页面',
|
||||
publication_id: 'pub-router-agent',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/router.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const submitted = [];
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
@@ -790,7 +995,17 @@ test('agent run uses chat intent router to enrich agent orchestration messages',
|
||||
});
|
||||
|
||||
test('agent run escalates direct sessions to a new backend session when forced', async () => {
|
||||
const pool = createFakePool();
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:deep-session-1': [{
|
||||
page_id: 'page-deep',
|
||||
title: '深度页面',
|
||||
publication_id: 'pub-deep',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const submitted = [];
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
@@ -820,7 +1035,17 @@ test('agent run escalates direct sessions to a new backend session when forced',
|
||||
});
|
||||
|
||||
test('agent run persists direct session transcript before escalating to goosed', async () => {
|
||||
const pool = createFakePool();
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:deep-session-1': [{
|
||||
page_id: 'page-deep-transcript',
|
||||
title: '深度页面',
|
||||
publication_id: 'pub-deep-transcript',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const submitted = [];
|
||||
const saved = [];
|
||||
const removed = [];
|
||||
@@ -914,7 +1139,17 @@ test('agent run rejects reused goosed session when broker ownership check fails'
|
||||
});
|
||||
|
||||
test('agent run persists portal direct snapshot before goosed submit on same session', async () => {
|
||||
const pool = createFakePool();
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:20260705_2': [{
|
||||
page_id: 'page-report',
|
||||
title: '报告页面',
|
||||
publication_id: 'pub-report',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/report.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const submitted = [];
|
||||
const saved = [];
|
||||
const gateway = createAgentRunGateway({
|
||||
@@ -1792,6 +2027,27 @@ test('createRun rejects with SESSION_RUN_CONFLICT when same session already has
|
||||
assert.equal(run3.requestId, 'req-conflict-3');
|
||||
});
|
||||
|
||||
test('createRun rejects while the same session is finishing page delivery', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {},
|
||||
autoDispatch: false,
|
||||
isSessionExternallyBusy: ({ sessionId }) => sessionId === 'sess-repairing',
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
gateway.createRun('user-1', {
|
||||
sessionId: 'sess-repairing',
|
||||
requestId: 'req-during-repair',
|
||||
userMessage: { role: 'user', content: [{ type: 'text', text: '继续' }] },
|
||||
}),
|
||||
(err) => err?.code === 'SESSION_RUN_CONFLICT' && err?.status === 409 && /自动修复/.test(err.message),
|
||||
);
|
||||
assert.equal(pool.runs.size, 0);
|
||||
});
|
||||
|
||||
test('createRun does not apply per-session conflict check for direct-chat sessions', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
|
||||
+1
-1
@@ -161,7 +161,7 @@ export const CAPABILITY_CATALOG = [
|
||||
];
|
||||
|
||||
/** Capabilities that must never be enabled for regular users, even via DB overrides. */
|
||||
export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin']);
|
||||
export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin', 'apps']);
|
||||
|
||||
export function clampUserCapabilities(capabilities) {
|
||||
const clamped = { ...capabilities };
|
||||
|
||||
@@ -164,6 +164,16 @@ test('clampUserCapabilities blocks extension_admin even when stored as true', ()
|
||||
assert.equal(clamped.shell, true);
|
||||
});
|
||||
|
||||
test('clampUserCapabilities blocks Apps for regular H5 users even when granted by role', () => {
|
||||
const clamped = clampUserCapabilities({
|
||||
...DEFAULT_USER_CAPABILITIES,
|
||||
apps: true,
|
||||
});
|
||||
assert.equal(clamped.apps, false);
|
||||
const policy = buildAgentExtensionPolicy(clamped);
|
||||
assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'apps'), false);
|
||||
});
|
||||
|
||||
test('catalog keys are unique', () => {
|
||||
const keys = CAPABILITY_CATALOG.map((item) => item.key);
|
||||
assert.equal(keys.length, new Set(keys).size);
|
||||
|
||||
@@ -888,7 +888,10 @@ export function createChatIntentRouter(options = {}) {
|
||||
|
||||
function getStatus() {
|
||||
return {
|
||||
enabled: Boolean(policy.enabled),
|
||||
enabled: true,
|
||||
ruleRoutingEnabled: true,
|
||||
llmRoutingEnabled: false,
|
||||
configuredLlmEnabled: Boolean(policy.enabled),
|
||||
modelProviderKeyId: policy.modelProviderKeyId ?? null,
|
||||
model: policy.model ?? null,
|
||||
modelApiType: policy.modelApiType ?? 'chat',
|
||||
@@ -903,8 +906,9 @@ export function createChatIntentRouter(options = {}) {
|
||||
}
|
||||
|
||||
function isEnabled() {
|
||||
// Skill + rule routing only; LLM intent router is intentionally disabled.
|
||||
return false;
|
||||
// Rule + skill routing is always active. The LLM classifier stays disabled
|
||||
// independently; returning false here would make the gateway skip rules too.
|
||||
return true;
|
||||
}
|
||||
|
||||
async function resolveRouterContext({ userId, sessionId, text, forceDeepReasoning = false }) {
|
||||
|
||||
@@ -67,6 +67,34 @@ test('classifyWithRules routes page data collection to page-data-collect', () =>
|
||||
assert.equal(result.suggestedSkill, 'page-data-collect');
|
||||
});
|
||||
|
||||
test('classifyWithRules routes ordinary household ledger language to page-data-collect', () => {
|
||||
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
|
||||
const result = classifyWithRules({
|
||||
text,
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text }],
|
||||
metadata: { displayText: text },
|
||||
},
|
||||
});
|
||||
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
|
||||
assert.equal(result.suggestedSkill, 'page-data-collect');
|
||||
});
|
||||
|
||||
test('classifyWithRules routes implicit sticky-note timeline apps to page-data-collect', () => {
|
||||
const text = '帮我设计一个便签提醒,可以写便签提交,时间轴来显示';
|
||||
const result = classifyWithRules({
|
||||
text,
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text }],
|
||||
metadata: { displayText: text },
|
||||
},
|
||||
});
|
||||
assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT);
|
||||
assert.equal(result.suggestedSkill, 'page-data-collect');
|
||||
});
|
||||
|
||||
test('coercePageDataSkill overrides static-page-publish when data intent is present', () => {
|
||||
const text = '帮我在这个页面增加调查问卷,密码 888 查看提交记录';
|
||||
const coerced = coercePageDataSkill(
|
||||
@@ -579,9 +607,9 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only'
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(await router.isEnabled(), false);
|
||||
assert.equal(await router.isEnabled(), true);
|
||||
states.shift();
|
||||
assert.equal(await router.isEnabled(), false);
|
||||
assert.equal(await router.isEnabled(), true);
|
||||
const result = await router.classify({
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
|
||||
+23
-2
@@ -43,10 +43,31 @@ const PAGE_DATA_INTENT_PATTERNS = [
|
||||
/(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u,
|
||||
];
|
||||
|
||||
const INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN = /(?:便签|便利贴|备忘录|待办|清单)/u;
|
||||
const INTERACTIVE_PAGE_DATA_SURFACE_PATTERN = /(?:应用|app|页面|网页|H5|h5|HTML|html|时间轴|看板)/u;
|
||||
const INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS = [
|
||||
/(?:写|填写|提交|保存)/u,
|
||||
/(?:添加|新增|创建)/u,
|
||||
/(?:编辑|修改)/u,
|
||||
/(?:删除|完成)/u,
|
||||
/(?:筛选|统计|汇总|管理)/u,
|
||||
/(?:提醒时间|优先级|分类标签|时间轴)/u,
|
||||
];
|
||||
|
||||
function isInteractivePageDataIntent(text) {
|
||||
if (!INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN.test(text)) return false;
|
||||
const featureCount = INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS.reduce(
|
||||
(count, pattern) => count + (pattern.test(text) ? 1 : 0),
|
||||
0,
|
||||
);
|
||||
return INTERACTIVE_PAGE_DATA_SURFACE_PATTERN.test(text) || featureCount >= 2;
|
||||
}
|
||||
|
||||
export function isPageDataIntent(text) {
|
||||
const normalized = String(text ?? '').trim();
|
||||
if (!normalized) return false;
|
||||
return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized))
|
||||
|| isInteractivePageDataIntent(normalized);
|
||||
}
|
||||
|
||||
export function isPageGenerationIntent(text) {
|
||||
@@ -161,7 +182,7 @@ export function buildChatSkillPrompt(promptKey, skillName) {
|
||||
`请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能:在 MindSpace 页面中实现可提交、可持久化的数据收集(问卷/报名/台账等),必须使用 Page Data API。` +
|
||||
'先匹配技能内能力分支(默认 A:匿名前台 public insert + 独立后台 password read,口令默认 88888888);“帮我做/创建”本身就是创建授权,展示简短方案摘要后必须同一轮继续执行,只有互斥需求或非法口令才追问。' +
|
||||
'流程:load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html(含 /assets/page-data-client.js)→ private_data_bind_workspace_page 发布并写策略。' +
|
||||
'禁止 localStorage / 浏览器本地存储 fallback;禁止自建 Express/独立端口(如 8899)、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl(/MindSpace/<用户ID>/public/...),禁止给用户 /u/用户名/pages/... 链接,并说明后台入口与口令。'
|
||||
'当前 session 的 private_data_* 已绑定当前用户专属 PostgreSQL schema;所有需保存或再次读取的数据必须走 PG + Page Data API。禁止 localStorage、sessionStorage、IndexedDB、SQLite、静态 JSON/JS 或内存持久化 fallback;禁止自建 Express/独立端口(如 8899)、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl(/MindSpace/<用户ID>/public/...),禁止给用户 /u/用户名/pages/... 链接,并说明后台入口与口令。'
|
||||
);
|
||||
case 'service-integration-smoke':
|
||||
return `请使用 ${skillName ?? 'service-integration-smoke'} 技能:按标准联调流程检查当前服务,覆盖身份、普通聊天、记忆读取,以及我本轮明确要求验证的技能/发布链路,并输出通过项、失败项、待确认项:`;
|
||||
|
||||
@@ -107,9 +107,36 @@ test('buildAutoChatSkillPrefix prefers page-data-collect for survey requests', (
|
||||
|
||||
test('isPageDataIntent matches survey and backend keywords', () => {
|
||||
assert.equal(isPageDataIntent('增加调查问卷和后台入口,密码 888'), true);
|
||||
assert.equal(
|
||||
isPageDataIntent('帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。'),
|
||||
true,
|
||||
);
|
||||
assert.equal(isPageDataIntent('帮我做一个苏州攻略页面'), false);
|
||||
});
|
||||
|
||||
test('isPageDataIntent matches implicit interactive sticky-note app requests', () => {
|
||||
const text = '帮我设计一个便签提醒,可以写便签提交,时间轴来显示';
|
||||
assert.equal(isPageDataIntent(text), true);
|
||||
assert.match(
|
||||
buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']),
|
||||
/page-data-collect/,
|
||||
);
|
||||
});
|
||||
|
||||
test('isPageDataIntent does not steal a simple schedule reminder', () => {
|
||||
assert.equal(isPageDataIntent('明天上午9点提醒我开会'), false);
|
||||
assert.equal(isPageDataIntent('创建一个待办'), false);
|
||||
});
|
||||
|
||||
test('buildAutoChatSkillPrefix routes ordinary household ledger language to Page Data', () => {
|
||||
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
|
||||
const prefix = buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']);
|
||||
assert.match(prefix, /page-data-collect/);
|
||||
assert.match(prefix, /同一轮继续执行/);
|
||||
assert.match(prefix, /禁止 localStorage/);
|
||||
assert.doesNotMatch(prefix, /static-page-publish 技能/);
|
||||
});
|
||||
|
||||
test('buildAutoChatSkillPrefix enables publish for implicit page requests', () => {
|
||||
const prefix = buildAutoChatSkillPrefix('苏州攻略页面', ['static-page-publish']);
|
||||
assert.match(prefix, /static-page-publish/);
|
||||
|
||||
@@ -63,6 +63,36 @@ pnpm dev:all
|
||||
| `PLAZA_APP_DIR` | ../memind_plaza | Plaza 专用脚本的源码路径(`pnpm dev:plaza` / `pnpm start:plaza` / `pnpm dev:all`) |
|
||||
| `MEMIND_SESSION_BROKER_ENABLED` | `0`(未设置) | Session Broker 灰度开关;Patch 2+ 本地验证时可设 `1`,见 [H5 Session 架构](./h5-session-architecture-20260706.md) |
|
||||
|
||||
### MindSpace:本地 vs 生产(必须区分)
|
||||
|
||||
`pnpm dev` / `server.mjs` 启动时会读取 `MEMIND_RUNTIME_PROFILE`(见 `scripts/memind-runtime-profile.mjs`):
|
||||
|
||||
| `MEMIND_RUNTIME_PROFILE` | 用途 | MindSpace 实现 |
|
||||
|--------------------------|------|----------------|
|
||||
| `local`(**默认**) | 本机日常开发 | Portal 内置 `MINDSPACE_SERVER_ADAPTER=local` |
|
||||
| `split-service` | 本机验证 103 拆分架构 | `remote` → `http://127.0.0.1:8082` 独立 MindSpace 服务 |
|
||||
| `production` | **仅 103 服务器** | 不覆盖 `.env`,以服务器配置为准 |
|
||||
|
||||
**本地 `.env` 推荐:**
|
||||
|
||||
```bash
|
||||
MEMIND_RUNTIME_PROFILE=local
|
||||
```
|
||||
|
||||
**本机要测 split-service 时:**
|
||||
|
||||
```bash
|
||||
MEMIND_RUNTIME_PROFILE=split-service
|
||||
MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082
|
||||
MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret # 必须与 MindSpace LaunchAgent 一致
|
||||
launchctl kickstart -k "gui/$(id -u)/cn.tkmind.mindspace-service"
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
启动后 Portal 日志应出现:`[Portal] Runtime profile: MEMIND_RUNTIME_PROFILE=local, MINDSPACE_SERVER_ADAPTER=local, ...`
|
||||
|
||||
**禁止:** 把 103 生产 `.env`、RDS 连接串、`MINDSPACE_REMOTE_AUTH_TOKEN` 生产值复制进本机 Git 仓库。
|
||||
|
||||
### H5 Session 架构本地全量联调(`0706-bug干净` 分支)
|
||||
|
||||
在 **本机 `.env`**(勿提交 Git)打开下列开关后重启 `pnpm dev`:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# MindSpace 浏览器存储迁移清单(仅本机)
|
||||
|
||||
更新时间:2026-07-13
|
||||
|
||||
## 当前策略
|
||||
|
||||
- 新建或修改的 Agent 页面禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 持久化数据。
|
||||
- 新 session 明确绑定当前用户的独立 PostgreSQL schema;持久化数据必须经 `private_data_*` 和 Page Data API。
|
||||
- 历史页面迁移期间继续可访问,不做全局 409 阻断。
|
||||
- 页面迁移并验证完成后,才考虑启用历史页面全局阻断。
|
||||
|
||||
## 待迁移页面
|
||||
|
||||
| 用户 ID | 页面 | 初步类型 | 状态 |
|
||||
|---|---|---|---|
|
||||
| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment.html` | 预约业务数据 | 待迁移 |
|
||||
| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment-admin.html` | 预约管理数据 | 待迁移 |
|
||||
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger.html` | 家庭记账数据 | 待迁移 |
|
||||
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger-manage.html` | 家庭记账管理 | 待迁移 |
|
||||
| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/index.html` | 工作区首页状态 | 待分类 |
|
||||
| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/oa-system.html` | OA 业务数据 | 待迁移 |
|
||||
| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/decision-comparison.html` | 决策工具状态 | 待分类 |
|
||||
| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/daily-admin.html` | 日报管理数据 | 待迁移 |
|
||||
| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/sales-register.html` | 销售登记数据 | 待迁移 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey.html` | 调查数据 | 待迁移 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey-admin.html` | 调查管理数据 | 待迁移 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/survey-youth.html` | 调查数据 | 待迁移 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/daily-report-system.html` | 日报业务数据 | 待迁移 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/weather-calendar.html` | 工具偏好或业务数据 | 待分类 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/runner-game.html` | 游戏存档 | 待决定 PG 或取消持久化 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/tetris-game.html` | 游戏存档 | 待决定 PG 或取消持久化 |
|
||||
| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/bird-key-fly.html` | 游戏存档 | 待决定 PG 或取消持久化 |
|
||||
|
||||
## 单页迁移完成条件
|
||||
|
||||
1. 建表位于页面所有者的独立 PG schema。
|
||||
2. dataset 注册完成,页面 policy 权限符合前台/管理页用途。
|
||||
3. HTML 只通过 Page Data API 访问数据,不包含浏览器持久化 fallback。
|
||||
4. 实际提交、刷新后读取、PG 直接查询三者结果一致。
|
||||
5. 原页面链接继续可用,管理页鉴权和跨用户隔离验证通过。
|
||||
@@ -0,0 +1,60 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const AGENT_PAGE_CODE_EXTENSIONS = new Set(['.html', '.htm', '.js', '.mjs', '.jsx', '.ts', '.tsx']);
|
||||
const PROHIBITED_BROWSER_STORAGE_PATTERNS = [
|
||||
{ api: 'localStorage', pattern: /\blocalStorage\b/i },
|
||||
{ api: 'sessionStorage', pattern: /\bsessionStorage\b/i },
|
||||
{ api: 'IndexedDB', pattern: /\b(?:indexedDB|IDBDatabase|IDBObjectStore|IDBTransaction)\b/i },
|
||||
];
|
||||
|
||||
export function isAgentPageCodePath(relativePath) {
|
||||
return AGENT_PAGE_CODE_EXTENSIONS.has(path.extname(String(relativePath ?? '')).toLowerCase());
|
||||
}
|
||||
|
||||
export function detectProhibitedBrowserStorage(content, { relativePath = '' } = {}) {
|
||||
if (!isAgentPageCodePath(relativePath)) return [];
|
||||
let source = String(content ?? '');
|
||||
// Fold adjacent string literals so computed-property obfuscation such as
|
||||
// window['local' + 'Storage'] cannot bypass the delivery policy.
|
||||
for (let pass = 0; pass < 8; pass += 1) {
|
||||
const folded = source.replace(
|
||||
/(['"])([^'"\\]*)\1\s*\+\s*(['"])([^'"\\]*)\3/g,
|
||||
(_match, quote, left, _rightQuote, right) => `${quote}${left}${right}${quote}`,
|
||||
);
|
||||
if (folded === source) break;
|
||||
source = folded;
|
||||
}
|
||||
return PROHIBITED_BROWSER_STORAGE_PATTERNS
|
||||
.filter(({ pattern }) => pattern.test(source))
|
||||
.map(({ api }) => api);
|
||||
}
|
||||
|
||||
export function assertNoProhibitedBrowserStorage(content, { relativePath = '' } = {}) {
|
||||
const APIs = detectProhibitedBrowserStorage(content, { relativePath });
|
||||
if (APIs.length === 0) return;
|
||||
const error = new Error(
|
||||
`${relativePath || '页面代码'} 禁止使用 ${APIs.join(', ')} 保存数据。`
|
||||
+ '用户业务数据必须通过 private_data_* 工具写入该用户专属 PostgreSQL schema,'
|
||||
+ 'HTML 必须通过 Page Data API 访问;请改写后再保存。',
|
||||
);
|
||||
error.code = 'BROWSER_STORAGE_FORBIDDEN';
|
||||
error.apis = APIs;
|
||||
error.relativePath = relativePath || null;
|
||||
throw error;
|
||||
}
|
||||
|
||||
export function scanWorkspaceFilesForProhibitedBrowserStorage({ publishDir, relativePaths = [] } = {}) {
|
||||
const root = path.resolve(String(publishDir ?? ''));
|
||||
if (!root) return [];
|
||||
const violations = [];
|
||||
for (const relativePath of new Set(relativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))) {
|
||||
if (!isAgentPageCodePath(relativePath)) continue;
|
||||
const absolutePath = path.resolve(root, relativePath);
|
||||
if (absolutePath !== root && !absolutePath.startsWith(`${root}${path.sep}`)) continue;
|
||||
if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) continue;
|
||||
const apis = detectProhibitedBrowserStorage(fs.readFileSync(absolutePath, 'utf8'), { relativePath });
|
||||
if (apis.length > 0) violations.push({ relativePath, apis });
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
isStubPublicHtmlContent,
|
||||
materializeMissingPublicHtmlWrites,
|
||||
} from './mindspace-public-finish-sync.mjs';
|
||||
import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
|
||||
|
||||
const PUBLIC_HTML_PATH_PATTERN_GLOBAL = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/gi;
|
||||
const PUBLIC_ASSET_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi;
|
||||
@@ -106,6 +107,26 @@ export function collectMissingPublicHtmlPaths(messages, currentUser, publishDir)
|
||||
);
|
||||
}
|
||||
|
||||
export function collectTouchedPublicCodePaths(messages, publishDir, syncResult = null) {
|
||||
const paths = new Set(
|
||||
(syncResult?.publicHtmlRelativePaths ?? [])
|
||||
.map((item) => normalizePublicRelativePath(item, { publishDir }))
|
||||
.filter(Boolean),
|
||||
);
|
||||
for (const message of Array.isArray(messages) ? messages : []) {
|
||||
for (const item of message?.content ?? []) {
|
||||
if (item?.type !== 'toolRequest') continue;
|
||||
const toolCall = item.toolCall?.value;
|
||||
const name = String(toolCall?.name ?? '').trim().split('__').at(-1);
|
||||
if (!['write_file', 'edit_file', 'write', 'edit'].includes(name)) continue;
|
||||
const args = toolCall?.arguments ?? {};
|
||||
const relativePath = normalizePublicRelativePath(args.path ?? args.file_path, { publishDir });
|
||||
if (relativePath) paths.add(relativePath);
|
||||
}
|
||||
}
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
export function collectMissingPublicAssetReferences(publishDir) {
|
||||
const root = path.resolve(String(publishDir ?? ''));
|
||||
const publicDir = path.join(root, 'public');
|
||||
@@ -135,6 +156,10 @@ export function evaluateH5HtmlFinishGuard({
|
||||
materializeMissingPublicHtmlWrites({ messages, publishDir });
|
||||
const missingHtml = collectMissingPublicHtmlPaths(messages, currentUser, publishDir);
|
||||
const missingAssets = collectMissingPublicAssetReferences(publishDir);
|
||||
const browserStorageViolations = scanWorkspaceFilesForProhibitedBrowserStorage({
|
||||
publishDir,
|
||||
relativePaths: collectTouchedPublicCodePaths(messages, publishDir, syncResult),
|
||||
});
|
||||
|
||||
const lastAssistant = [...(Array.isArray(messages) ? messages : [])]
|
||||
.reverse()
|
||||
@@ -159,18 +184,24 @@ export function evaluateH5HtmlFinishGuard({
|
||||
const needsRepair =
|
||||
missingHtml.length > 0 ||
|
||||
missingAssets.length > 0 ||
|
||||
browserStorageViolations.length > 0 ||
|
||||
htmlGenerationNeedsRetry;
|
||||
|
||||
return {
|
||||
needsRepair,
|
||||
missingHtml,
|
||||
missingAssets,
|
||||
browserStorageViolations,
|
||||
htmlGenerationNeedsRetry,
|
||||
linkExists,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } = {}) {
|
||||
export function buildH5HtmlRepairPrompt({
|
||||
missingHtml = [],
|
||||
missingAssets = [],
|
||||
browserStorageViolations = [],
|
||||
} = {}) {
|
||||
const lines = [
|
||||
'【系统补盘请求】检测到 public 页面交付不完整。请立即使用 write_file 写入缺失文件(完整内容),并在完成后确认磁盘存在。',
|
||||
'要求:先 load_skill static-page-publish,再逐个 write_file;不要用空回复或仅 edit_file 改 HTML 代替资源落盘。',
|
||||
@@ -185,6 +216,15 @@ export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] }
|
||||
lines.push(`- ${item.relativePath}(来自 ${item.htmlRelativePath})`);
|
||||
}
|
||||
}
|
||||
if (browserStorageViolations.length) {
|
||||
lines.push(
|
||||
'',
|
||||
'以下页面违反数据存储硬约束:',
|
||||
...browserStorageViolations.map((item) => `- ${item.relativePath}:禁止 ${item.apis.join(', ')}`),
|
||||
'',
|
||||
'必须删除浏览器存储代码,load_skill → page-data-collect,使用 private_data_* 在当前用户专属 PostgreSQL schema 建表和注册 dataset,HTML 只能通过 Page Data API 读写。禁止 SQLite 或任何本地 fallback。',
|
||||
);
|
||||
}
|
||||
lines.push('', '写完后请 list_dir public/assets 或对应目录,确认文件真实存在。');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
@@ -72,6 +72,61 @@ test('evaluateH5HtmlFinishGuard flags missing assets after html exists', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('evaluateH5HtmlFinishGuard blocks browser storage in any touched public page', () => {
|
||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-ban-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(publishDir, 'public', 'ledger.html'),
|
||||
'<html><script>sessionStorage.setItem("ledger", "[]"); indexedDB.open("ledger")</script></html>',
|
||||
'utf8',
|
||||
);
|
||||
const evaluation = evaluateH5HtmlFinishGuard({
|
||||
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
|
||||
name: 'write_file',
|
||||
arguments: { path: 'public/ledger.html' },
|
||||
} } }] }],
|
||||
currentUser: USER,
|
||||
publishDir,
|
||||
});
|
||||
assert.equal(evaluation.needsRepair, true);
|
||||
assert.deepEqual(evaluation.browserStorageViolations, [{
|
||||
relativePath: 'public/ledger.html',
|
||||
apis: ['sessionStorage', 'IndexedDB'],
|
||||
}]);
|
||||
assert.match(buildH5HtmlRepairPrompt(evaluation), /当前用户专属 PostgreSQL schema/);
|
||||
} finally {
|
||||
fs.rmSync(publishDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('evaluateH5HtmlFinishGuard blocks computed localStorage obfuscation', () => {
|
||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-obfuscation-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(publishDir, 'public', 'bypass.html'),
|
||||
`<script>window['local' + 'Sto' + 'rage'].setItem('data', '[]')</script>`,
|
||||
'utf8',
|
||||
);
|
||||
const evaluation = evaluateH5HtmlFinishGuard({
|
||||
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
|
||||
name: 'write_file',
|
||||
arguments: { path: 'public/bypass.html' },
|
||||
} } }] }],
|
||||
currentUser: USER,
|
||||
publishDir,
|
||||
});
|
||||
assert.equal(evaluation.needsRepair, true);
|
||||
assert.deepEqual(evaluation.browserStorageViolations, [{
|
||||
relativePath: 'public/bypass.html',
|
||||
apis: ['localStorage'],
|
||||
}]);
|
||||
} finally {
|
||||
fs.rmSync(publishDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('buildH5HtmlRepairPrompt lists html and asset targets', () => {
|
||||
const prompt = buildH5HtmlRepairPrompt({
|
||||
missingHtml: ['public/page.html'],
|
||||
|
||||
@@ -15,7 +15,9 @@ import {
|
||||
normalizePageDataApiBase,
|
||||
verifyPageDataDeliveryArtifacts,
|
||||
} from './page-data-delivery-assess.mjs';
|
||||
import { listPageAccessPolicies } from './page-data-policy-store.mjs';
|
||||
import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs';
|
||||
import { detectProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
|
||||
|
||||
export { inferPageDataBindAccessMode };
|
||||
|
||||
@@ -23,7 +25,7 @@ const PUBLICATION_ROUTE_LINK_PATTERN =
|
||||
/https?:\/\/[^\s<>"')\]]+\/u\/([0-9a-f-]{36}|[a-z0-9._-]+)\/pages\/([^\s<>"')\]]+)/gi;
|
||||
|
||||
const PAGE_DATA_CLIENT_SCRIPT_PATTERN = /\/assets\/page-data-client\.js/i;
|
||||
const LOCAL_STORAGE_DATA_PATTERN = /localStorage\.(?:getItem|setItem)\s*\(/i;
|
||||
const LEGACY_PAGE_DATA_OWNER_API_PATTERN = /['"`]\/api\/page-data(?:\/|['"`])/i;
|
||||
const LOCAL_STORAGE_FALLBACK_HINT_PATTERN = /fallback\s*到\s*localStorage|localStorage\s*fallback/i;
|
||||
|
||||
const repairAttemptsBySession = new Map();
|
||||
@@ -109,10 +111,14 @@ export function buildPageDataDeliveryArtifactsFromBindResult(autoBind, publishDi
|
||||
export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) {
|
||||
const content = String(html ?? '');
|
||||
const usage = detectPageDataDatasetUsageFromHtml(content);
|
||||
const prohibitedBrowserStorage = detectProhibitedBrowserStorage(content, { relativePath });
|
||||
const usesPageDataApi =
|
||||
usage.size > 0 ||
|
||||
PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content) ||
|
||||
/\bMindSpacePageData\b/.test(content);
|
||||
/\bMindSpacePageData\b/.test(content) ||
|
||||
LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content) ||
|
||||
prohibitedBrowserStorage.length > 0 ||
|
||||
LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content);
|
||||
|
||||
if (!usesPageDataApi) {
|
||||
return { usesPageDataApi: false, issues: [] };
|
||||
@@ -122,9 +128,15 @@ export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) {
|
||||
if (!PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content)) {
|
||||
issues.push('missing_page_data_client_script');
|
||||
}
|
||||
if (LOCAL_STORAGE_DATA_PATTERN.test(content)) {
|
||||
if (LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content)) {
|
||||
issues.push('forbidden_legacy_owner_api');
|
||||
}
|
||||
if (prohibitedBrowserStorage.includes('localStorage')) {
|
||||
issues.push('forbidden_local_storage');
|
||||
}
|
||||
if (prohibitedBrowserStorage.some((api) => api !== 'localStorage')) {
|
||||
issues.push('forbidden_browser_storage');
|
||||
}
|
||||
if (LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content)) {
|
||||
issues.push('forbidden_local_storage_fallback');
|
||||
}
|
||||
@@ -156,7 +168,22 @@ export function collectPageDataPublicHtmlFiles(publishDir) {
|
||||
}
|
||||
|
||||
function isPageDataHtmlWorkspaceReady({ publishDir, relativePath, html }) {
|
||||
return assessWorkspacePageDataReadiness({ publishDir, relativePath, html }).ready;
|
||||
const usage = detectPageDataDatasetUsageFromHtml(html);
|
||||
if (!usage.size) return true;
|
||||
return Boolean(findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }));
|
||||
}
|
||||
|
||||
function findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }) {
|
||||
const usage = detectPageDataDatasetUsageFromHtml(html);
|
||||
const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html);
|
||||
for (const policy of listPageAccessPolicies(publishDir)) {
|
||||
if (String(policy?.accessMode ?? '') !== expectedAccessMode) continue;
|
||||
if ([...usage.entries()].every(([name, perms]) => {
|
||||
const granted = policy?.datasets?.[name];
|
||||
return Boolean(granted && (!perms.read || granted.read) && (!perms.insert || granted.insert));
|
||||
})) return policy;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function collectUnboundPageDataFiles({
|
||||
@@ -186,11 +213,13 @@ async function collectUnboundPageDataFiles({
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!isPageDataHtmlWorkspaceReady({
|
||||
const workspace = await assessWorkspacePageDataReadiness({
|
||||
userId,
|
||||
publishDir,
|
||||
relativePath: file.relativePath,
|
||||
html: file.content,
|
||||
})) {
|
||||
});
|
||||
if (!workspace.ready) {
|
||||
unboundFiles.push(file);
|
||||
}
|
||||
}
|
||||
@@ -320,7 +349,7 @@ export function evaluatePageDataFinishGuard({
|
||||
);
|
||||
|
||||
const needsRepair =
|
||||
pageDataIntent &&
|
||||
(pageDataIntent || structuralFiles.length > 0) &&
|
||||
(htmlIssues.length > 0 ||
|
||||
unboundFiles.length > 0 ||
|
||||
(relevantFiles.length === 0 &&
|
||||
@@ -382,7 +411,7 @@ export function buildPageDataCollectFailureText() {
|
||||
'这次问卷/数据收集页面没有完成 Page Data API 绑定,所以我先不发链接。',
|
||||
'请直接重发一次完整需求(例如:调查问卷 + 后台查看),我会按 page-data-collect 技能:',
|
||||
'建表 → 注册 dataset → 写含 page-data-client.js 的 HTML → private_data_bind_workspace_page 发布。',
|
||||
'数据必须走平台 API 写入 SQLite,禁止 localStorage 或自建后端。',
|
||||
'数据必须走平台 API 写入用户私有数据空间,禁止 localStorage 或自建后端。',
|
||||
].join('');
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
buildPageDataCollectRepairPrompt,
|
||||
collectPageDataDeliveryArtifacts,
|
||||
evaluatePageDataFinishGuard,
|
||||
evaluatePageDataFinishGuardAsync,
|
||||
evaluatePageDataHtmlContent,
|
||||
extractRecentPageDataBindTargets,
|
||||
extractRecentPageDataHtmlWrites,
|
||||
@@ -42,6 +43,23 @@ test('evaluatePageDataHtmlContent flags missing client script and localStorage f
|
||||
const bad = evaluatePageDataHtmlContent(BAD_SURVEY_HTML, { relativePath: 'public/children-diet-survey.html' });
|
||||
assert.ok(bad.issues.includes('missing_page_data_client_script'));
|
||||
assert.ok(bad.issues.includes('forbidden_local_storage'));
|
||||
|
||||
const indexed = evaluatePageDataHtmlContent(
|
||||
'<script>sessionStorage.setItem("draft", "x"); indexedDB.open("drafts")</script>',
|
||||
{ relativePath: 'public/draft.html' },
|
||||
);
|
||||
assert.ok(indexed.issues.includes('forbidden_browser_storage'));
|
||||
|
||||
const legacyOwnerApi = evaluatePageDataHtmlContent(
|
||||
`<script>
|
||||
const PG_API = '/api/page-data';
|
||||
fetch(PG_API + '/query', { method: 'POST' });
|
||||
</script>`,
|
||||
{ relativePath: 'public/sticky-notes.html' },
|
||||
);
|
||||
assert.equal(legacyOwnerApi.usesPageDataApi, true);
|
||||
assert.ok(legacyOwnerApi.issues.includes('missing_page_data_client_script'));
|
||||
assert.ok(legacyOwnerApi.issues.includes('forbidden_legacy_owner_api'));
|
||||
});
|
||||
|
||||
test('inferPageDataBindAccessMode chooses password for admin html', () => {
|
||||
@@ -70,6 +88,35 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('finish guard blocks localStorage ledger generated from ordinary user language', () => {
|
||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-ledger-'));
|
||||
const text = '帮我做一个家庭记账页面,我每天可以记录日期、收入、支出、分类和备注,日期默认当天。再做一个管理页面,可以查看所有记录和收支汇总。';
|
||||
try {
|
||||
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
|
||||
fs.writeFileSync(path.join(publishDir, 'public', 'family-ledger.html'), `<!doctype html><script>
|
||||
localStorage.setItem('tk_family_ledger', JSON.stringify([]));
|
||||
</script>`, 'utf8');
|
||||
const evaluation = evaluatePageDataFinishGuard({
|
||||
publishDir,
|
||||
agentText: text,
|
||||
messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: {
|
||||
name: 'write_file',
|
||||
arguments: { path: 'public/family-ledger.html' },
|
||||
} } }] }],
|
||||
});
|
||||
assert.equal(evaluation.pageDataIntent, true);
|
||||
assert.equal(evaluation.needsRepair, true);
|
||||
assert.equal(
|
||||
evaluation.htmlIssues.some((item) => (
|
||||
item.issue === 'forbidden_local_storage' && item.relativePath === 'public/family-ledger.html'
|
||||
)),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(publishDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('finish guard ignores unrelated historical broken Page Data html', async () => {
|
||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-history-'));
|
||||
try {
|
||||
@@ -162,7 +209,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('evaluatePageDataFinishGuard flags html when dataset is not registered', () => {
|
||||
test('evaluatePageDataFinishGuard flags html when dataset is not registered', async () => {
|
||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-bound-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
|
||||
@@ -179,8 +226,9 @@ test('evaluatePageDataFinishGuard flags html when dataset is not registered', ()
|
||||
},
|
||||
},
|
||||
});
|
||||
const evaluation = evaluatePageDataFinishGuard({
|
||||
const evaluation = await evaluatePageDataFinishGuardAsync({
|
||||
publishDir,
|
||||
userId: 'user-1',
|
||||
agentText: '调查问卷和后台',
|
||||
messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }],
|
||||
});
|
||||
|
||||
@@ -11,12 +11,16 @@ export function createPageSyncService({
|
||||
} = {}) {
|
||||
const syncInFlight = new Map();
|
||||
|
||||
async function syncUserGeneratedPages(userId) {
|
||||
async function syncUserGeneratedPages(userId, { onlyRelativePaths = null } = {}) {
|
||||
if (!pool || !pageService || !userId) {
|
||||
return { created: 0, skipped: 0, updated: 0 };
|
||||
}
|
||||
|
||||
let inFlight = syncInFlight.get(userId);
|
||||
const scopeKey = onlyRelativePaths == null
|
||||
? '*'
|
||||
: [...new Set(onlyRelativePaths)].sort().join('|');
|
||||
const inFlightKey = `${userId}:${scopeKey}`;
|
||||
let inFlight = syncInFlight.get(inFlightKey);
|
||||
if (!inFlight) {
|
||||
inFlight = syncGeneratedPagesFromPublicAssets({
|
||||
pool,
|
||||
@@ -25,17 +29,18 @@ export function createPageSyncService({
|
||||
userId,
|
||||
publishDir: h5Root ? resolvePublishDir(h5Root, { id: userId }) : null,
|
||||
syncWorkspaceAssets,
|
||||
onlyRelativePaths,
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.warn?.('[MindSpace] page sync failed:', error?.message ?? error);
|
||||
return { created: 0, skipped: 0, updated: 0 };
|
||||
})
|
||||
.finally(() => {
|
||||
if (syncInFlight.get(userId) === inFlight) {
|
||||
syncInFlight.delete(userId);
|
||||
if (syncInFlight.get(inFlightKey) === inFlight) {
|
||||
syncInFlight.delete(inFlightKey);
|
||||
}
|
||||
});
|
||||
syncInFlight.set(userId, inFlight);
|
||||
syncInFlight.set(inFlightKey, inFlight);
|
||||
}
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
+10
-1
@@ -213,16 +213,23 @@ export async function syncGeneratedPagesFromPublicAssets({
|
||||
userId,
|
||||
publishDir = null,
|
||||
syncWorkspaceAssets = null,
|
||||
onlyRelativePaths = null,
|
||||
} = {}) {
|
||||
if (!pool || !pageService || !userId) {
|
||||
return { created: 0, skipped: 0, updated: 0 };
|
||||
}
|
||||
|
||||
if (typeof syncWorkspaceAssets === 'function') {
|
||||
await syncWorkspaceAssets(userId, { categoryCode: 'public' }).catch(() => {});
|
||||
await syncWorkspaceAssets(userId, {
|
||||
categoryCode: 'public',
|
||||
onlyRelativePaths,
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
const indexedPages = await loadIndexedWorkspacePages(pool, userId);
|
||||
const allowList = onlyRelativePaths == null
|
||||
? null
|
||||
: new Set(onlyRelativePaths.map((item) => normalizePublicHtmlPath(item)).filter(Boolean));
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
let skipped = 0;
|
||||
@@ -230,6 +237,7 @@ export async function syncGeneratedPagesFromPublicAssets({
|
||||
if (publishDir) {
|
||||
const htmlFiles = await listPublishPublicHtmlFiles(publishDir);
|
||||
for (const file of htmlFiles) {
|
||||
if (allowList && !allowList.has(file.relativePath)) continue;
|
||||
try {
|
||||
const content = await fs.readFile(file.absolutePath, 'utf8');
|
||||
if (!content.trim()) {
|
||||
@@ -285,6 +293,7 @@ export async function syncGeneratedPagesFromPublicAssets({
|
||||
? asset.original_filename
|
||||
: `public/${asset.original_filename}`,
|
||||
);
|
||||
if (allowList && !allowList.has(relativePath)) continue;
|
||||
if (indexedPages.has(relativePath)) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
|
||||
@@ -47,6 +47,40 @@ test('syncGeneratedPagesFromPublicAssets registers html files from publish publi
|
||||
assert.equal(created[0].source.snapshot.relative_path, 'public/demo.html');
|
||||
});
|
||||
|
||||
test('syncGeneratedPagesFromPublicAssets scopes agent-run sync to current session html', async () => {
|
||||
const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-scoped-'));
|
||||
const publicDir = path.join(publishDir, 'public');
|
||||
await fs.mkdir(publicDir, { recursive: true });
|
||||
await fs.writeFile(path.join(publicDir, 'current.html'), '<title>Current</title>', 'utf8');
|
||||
await fs.writeFile(path.join(publicDir, 'historical.html'), '<title>Historical</title>', 'utf8');
|
||||
|
||||
const created = [];
|
||||
const pool = {
|
||||
async query(sql) {
|
||||
if (sql.includes('FROM h5_page_records p')) return [[]];
|
||||
if (sql.includes('FROM h5_assets a')) return [[]];
|
||||
return [[]];
|
||||
},
|
||||
};
|
||||
const pageService = {
|
||||
async createFromChat(_userId, input, source) {
|
||||
created.push({ input, source });
|
||||
return { id: `page-${created.length}` };
|
||||
},
|
||||
};
|
||||
|
||||
await syncGeneratedPagesFromPublicAssets({
|
||||
pool,
|
||||
pageService,
|
||||
userId: 'user-1',
|
||||
publishDir,
|
||||
onlyRelativePaths: ['public/current.html'],
|
||||
});
|
||||
|
||||
assert.deepEqual(created.map((item) => item.source.snapshot.relative_path), ['public/current.html']);
|
||||
await fs.rm(publishDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('syncGeneratedPagesFromPublicAssets uses file mtime as createdAt', async () => {
|
||||
const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-mtime-'));
|
||||
const publicDir = path.join(publishDir, 'public');
|
||||
|
||||
@@ -516,6 +516,7 @@ export async function syncPublicHtmlAfterFinish({
|
||||
categoryCode: 'public',
|
||||
sourceSessionId: sessionId,
|
||||
sourceMessageId,
|
||||
onlyRelativePaths: publicHtmlRelativePaths,
|
||||
});
|
||||
synced = true;
|
||||
}
|
||||
|
||||
@@ -196,6 +196,7 @@ test('syncPublicHtmlAfterFinish forwards session source to workspace sync', asyn
|
||||
categoryCode: 'public',
|
||||
sourceSessionId: 'session-1',
|
||||
sourceMessageId: 'msg-public-1',
|
||||
onlyRelativePaths: ['public/session-page.html'],
|
||||
},
|
||||
]]);
|
||||
} finally {
|
||||
|
||||
@@ -121,6 +121,35 @@ test('resolveMindSpacePublicRequest coordinates canonical redirect, serve, and n
|
||||
});
|
||||
});
|
||||
|
||||
test('resolveMindSpacePublicRequest keeps legacy browser-storage pages available during migration', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-storage-ban-'));
|
||||
const ownerKey = '123e4567-e89b-12d3-a456-426614174000';
|
||||
const targetDir = path.join(root, 'MindSpace', ownerKey, 'public');
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(targetDir, 'ledger.html'),
|
||||
'<script>localStorage.setItem("ledger", "[]")</script>',
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(targetDir, 'ledger.js'),
|
||||
'sessionStorage.setItem("ledger", "[]")',
|
||||
);
|
||||
|
||||
const result = await resolveMindSpacePublicRequest({
|
||||
h5Root: root,
|
||||
requestPath: `/${ownerKey}/public/ledger.html`,
|
||||
});
|
||||
assert.equal(result.action, 'serve');
|
||||
assert.equal(result.filePath, path.join(targetDir, 'ledger.html'));
|
||||
|
||||
const scriptResult = await resolveMindSpacePublicRequest({
|
||||
h5Root: root,
|
||||
requestPath: `/${ownerKey}/public/ledger.js`,
|
||||
});
|
||||
assert.equal(scriptResult.action, 'serve');
|
||||
assert.equal(scriptResult.filePath, path.join(targetDir, 'ledger.js'));
|
||||
});
|
||||
|
||||
test('resolveMindSpacePublicRequest serves missing thumbnail png when svg sidecar exists', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-thumb-'));
|
||||
const ownerKey = '123e4567-e89b-12d3-a456-426614174000';
|
||||
|
||||
@@ -24,6 +24,7 @@ import { closePolicyDataset, normalizePageAccessPolicy } from './page-access-pol
|
||||
import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs';
|
||||
import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs';
|
||||
import { resolveMindSpaceStorageRoot } from './mindspace-runtime-config.mjs';
|
||||
import { assertNoProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
|
||||
|
||||
const SANDBOX_ROOT = process.argv[2]?.trim() || process.env.SANDBOX_ROOT?.trim();
|
||||
if (!SANDBOX_ROOT) {
|
||||
@@ -215,7 +216,7 @@ const ALL_TOOLS = [
|
||||
{
|
||||
name: 'private_data_info',
|
||||
description:
|
||||
'查看当前用户“用户私有数据空间”的状态。每个用户只有一个私有 SQLite 数据库,适合保存问卷、表单、清单、调研数据和分析中间表;不要创建额外 SQLite 文件。',
|
||||
'查看当前用户“用户私有数据空间”的状态。每个用户只有一个由平台分配和隔离的数据空间,适合保存问卷、表单、清单、调研数据和分析中间表;不要创建额外数据库文件。',
|
||||
inputSchema: { type: 'object', properties: {} },
|
||||
},
|
||||
{
|
||||
@@ -504,6 +505,7 @@ async function callTool(name, args) {
|
||||
}
|
||||
case 'write_file': {
|
||||
const abs = resolveSandboxed(args.path);
|
||||
assertNoProhibitedBrowserStorage(args.content ?? '', { relativePath: args.path });
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, args.content ?? '', 'utf8');
|
||||
return [{ type: 'text', text: `已写入 ${args.path}(${(args.content ?? '').length} 字节)` }];
|
||||
@@ -516,6 +518,7 @@ async function callTool(name, args) {
|
||||
throw new Error('edit_file: old_str 在文件中不存在,替换失败');
|
||||
}
|
||||
const updated = oldStr ? original.replace(oldStr, args.new_str ?? '') : (args.new_str ?? '');
|
||||
assertNoProhibitedBrowserStorage(updated, { relativePath: args.path });
|
||||
fs.writeFileSync(abs, updated, 'utf8');
|
||||
return [{ type: 'text', text: `已编辑 ${args.path}` }];
|
||||
}
|
||||
@@ -578,7 +581,9 @@ async function callTool(name, args) {
|
||||
{
|
||||
...info,
|
||||
rules: [
|
||||
'每个用户只能使用这个唯一 SQLite 数据库',
|
||||
info.backend === 'postgres'
|
||||
? '每个用户只能使用新 PG 中分配给自己的独立 schema'
|
||||
: '每个用户只能使用这个唯一 SQLite 数据库',
|
||||
'适合问卷、表单、清单、调研数据和分析中间表',
|
||||
'不要存账号、计费、权限、审计、公开平台数据或跨用户数据',
|
||||
'HTML 页面通过 Page Data API 访问 dataset,不要直接暴露 SQL',
|
||||
@@ -603,7 +608,9 @@ async function callTool(name, args) {
|
||||
return [
|
||||
{
|
||||
type: 'text',
|
||||
text: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `,已同步空间占用 ${result.deltaBytes} 字节` : ''}`,
|
||||
text: result.backend === 'postgres'
|
||||
? `已在用户专属 PG schema 中执行。命令 ${result.command ?? 'OK'},影响 ${result.affectedRows ?? 0} 行`
|
||||
: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `,已同步空间占用 ${result.deltaBytes} 字节` : ''}`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
@@ -85,6 +85,41 @@ function startSandbox(root, envOverrides = {}) {
|
||||
return { child, request };
|
||||
}
|
||||
|
||||
test('sandbox MCP rejects browser storage in generated page code', async (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-storage-ban-'));
|
||||
const server = startSandbox(root, { ALLOWED_TOOLS: 'write_file,edit_file' });
|
||||
t.after(() => server.child.kill());
|
||||
|
||||
await server.request('initialize');
|
||||
const rejectedWrite = await server.request('tools/call', {
|
||||
name: 'write_file',
|
||||
arguments: {
|
||||
path: 'public/ledger.html',
|
||||
content: '<script>localStorage.setItem("ledger", "[]")</script>',
|
||||
},
|
||||
});
|
||||
assert.equal(rejectedWrite.result.isError, true);
|
||||
assert.match(rejectedWrite.result.content[0].text, /BROWSER_STORAGE_FORBIDDEN|禁止使用 localStorage/);
|
||||
assert.equal(fs.existsSync(path.join(root, 'public', 'ledger.html')), false);
|
||||
|
||||
const acceptedWrite = await server.request('tools/call', {
|
||||
name: 'write_file',
|
||||
arguments: { path: 'public/ledger.html', content: '<script>client.insertRow("ledger", row)</script>' },
|
||||
});
|
||||
assert.equal(acceptedWrite.result.isError, false);
|
||||
|
||||
const rejectedEdit = await server.request('tools/call', {
|
||||
name: 'edit_file',
|
||||
arguments: {
|
||||
path: 'public/ledger.html',
|
||||
old_str: 'client.insertRow("ledger", row)',
|
||||
new_str: 'indexedDB.open("ledger")',
|
||||
},
|
||||
});
|
||||
assert.equal(rejectedEdit.result.isError, true);
|
||||
assert.doesNotMatch(fs.readFileSync(path.join(root, 'public', 'ledger.html'), 'utf8'), /indexedDB/);
|
||||
});
|
||||
|
||||
test('sandbox MCP exposes and protects the user private data space', async (t) => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-'));
|
||||
const server = startSandbox(root);
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
export const CONTROL_SCHEMA = 'mindspace_control';
|
||||
|
||||
export function assertUserId(value) {
|
||||
const userId = String(value ?? '').trim().toLowerCase();
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(userId)) {
|
||||
throw new Error('user id 必须是标准 UUID');
|
||||
}
|
||||
return userId;
|
||||
}
|
||||
|
||||
export function quotePgIdentifier(value) {
|
||||
return `"${String(value).replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
export function deriveUserSpaceNames(value) {
|
||||
const userId = assertUserId(value);
|
||||
const compact = userId.replaceAll('-', '');
|
||||
const short = compact.slice(0, 12);
|
||||
return {
|
||||
userId,
|
||||
schemaName: `u_${compact}`,
|
||||
ownerRole: `ms_u_${short}_owner`,
|
||||
agentRole: `ms_u_${short}_agent`,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildControlSchemaSql() {
|
||||
return `
|
||||
CREATE SCHEMA IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)};
|
||||
REVOKE ALL ON SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces (
|
||||
user_id uuid PRIMARY KEY,
|
||||
schema_name name NOT NULL UNIQUE,
|
||||
owner_role name NOT NULL UNIQUE,
|
||||
agent_role name NOT NULL UNIQUE,
|
||||
backend_type text NOT NULL DEFAULT 'postgres' CHECK (backend_type IN ('sqlite', 'postgres')),
|
||||
source_sqlite_path text,
|
||||
migration_state text NOT NULL DEFAULT 'pending' CHECK (
|
||||
migration_state IN ('pending', 'snapshot_created', 'schema_converted', 'data_imported', 'verified', 'shadow', 'cutover', 'rolled_back', 'failed')
|
||||
),
|
||||
schema_version integer NOT NULL DEFAULT 1,
|
||||
quota_bytes bigint NOT NULL DEFAULT 104857600 CHECK (quota_bytes > 0),
|
||||
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
cutover_at timestamptz,
|
||||
rollback_until timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs (
|
||||
id uuid PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE,
|
||||
source_path text NOT NULL,
|
||||
source_size_bytes bigint NOT NULL DEFAULT 0,
|
||||
source_sha256 text,
|
||||
status text NOT NULL,
|
||||
table_count integer NOT NULL DEFAULT 0,
|
||||
source_row_count bigint NOT NULL DEFAULT 0,
|
||||
target_row_count bigint NOT NULL DEFAULT 0,
|
||||
detail_json jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
started_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
finished_at timestamptz
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog (
|
||||
user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE,
|
||||
dataset_name text NOT NULL,
|
||||
physical_table name NOT NULL,
|
||||
config_json jsonb NOT NULL,
|
||||
sensitivity text NOT NULL DEFAULT 'normal' CHECK (sensitivity IN ('normal', 'personal', 'credential', 'health')),
|
||||
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (user_id, dataset_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.audit_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
user_id uuid,
|
||||
actor_type text NOT NULL,
|
||||
actor_id text,
|
||||
action text NOT NULL,
|
||||
object_type text,
|
||||
object_name text,
|
||||
statement_hash text,
|
||||
affected_rows bigint,
|
||||
duration_ms integer,
|
||||
result text NOT NULL,
|
||||
detail_json jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
REVOKE ALL ON ALL TABLES IN SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC;
|
||||
`.trim();
|
||||
}
|
||||
|
||||
export function buildProvisionUserSql(userId, { sourceSqlitePath = null, quotaBytes = 104857600 } = {}) {
|
||||
const names = deriveUserSpaceNames(userId);
|
||||
const schema = quotePgIdentifier(names.schemaName);
|
||||
const owner = quotePgIdentifier(names.ownerRole);
|
||||
const agent = quotePgIdentifier(names.agentRole);
|
||||
return {
|
||||
...names,
|
||||
statements: [
|
||||
`DO $$ BEGIN CREATE ROLE ${owner} NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN CREATE ROLE ${agent} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; EXCEPTION WHEN duplicate_object THEN NULL; END $$`,
|
||||
`DO $$ BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_auth_members am
|
||||
JOIN pg_roles r ON r.oid = am.roleid
|
||||
JOIN pg_roles m ON m.oid = am.member
|
||||
WHERE r.rolname = '${names.ownerRole}' AND m.rolname = CURRENT_USER AND am.set_option
|
||||
) THEN
|
||||
EXECUTE format('GRANT %I TO %I', '${names.ownerRole}', CURRENT_USER);
|
||||
END IF;
|
||||
END $$`,
|
||||
`CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION ${owner}`,
|
||||
`REVOKE ALL ON SCHEMA ${schema} FROM PUBLIC`,
|
||||
`GRANT USAGE, CREATE ON SCHEMA ${schema} TO ${agent}`,
|
||||
`ALTER ROLE ${agent} SET search_path = ${schema}, pg_catalog`,
|
||||
`ALTER ROLE ${agent} SET statement_timeout = '30s'`,
|
||||
`ALTER ROLE ${agent} SET lock_timeout = '5s'`,
|
||||
`ALTER ROLE ${agent} SET idle_in_transaction_session_timeout = '60s'`,
|
||||
`ALTER ROLE ${agent} SET work_mem = '4MB'`,
|
||||
`ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${agent}`,
|
||||
`ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO ${agent}`,
|
||||
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces
|
||||
(user_id, schema_name, owner_role, agent_role, source_sqlite_path, quota_bytes, updated_at)
|
||||
VALUES ($1::uuid, $2::name, $3::name, $4::name, $5, $6, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (user_id) DO UPDATE SET
|
||||
source_sqlite_path = EXCLUDED.source_sqlite_path,
|
||||
quota_bytes = EXCLUDED.quota_bytes,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
],
|
||||
params: [names.userId, names.schemaName, names.ownerRole, names.agentRole, sourceSqlitePath, quotaBytes],
|
||||
};
|
||||
}
|
||||
|
||||
export function mapSqliteTypeToPostgres(column) {
|
||||
const declared = String(column.type ?? '').trim().toUpperCase();
|
||||
if (Number(column.pk) === 1 && declared.includes('INT')) return 'bigint GENERATED BY DEFAULT AS IDENTITY';
|
||||
if (declared.includes('TEXT') && isSqliteTimestampDefault(column.dflt_value)) return 'timestamptz';
|
||||
if (declared.includes('INT')) return 'bigint';
|
||||
if (declared.includes('REAL') || declared.includes('FLOA') || declared.includes('DOUB')) return 'double precision';
|
||||
if (declared.includes('BLOB')) return 'bytea';
|
||||
if (declared.includes('NUM') || declared.includes('DEC')) return 'numeric';
|
||||
if (declared.includes('BOOL')) return 'boolean';
|
||||
return 'text';
|
||||
}
|
||||
|
||||
export function isSqliteTimestampDefault(value) {
|
||||
return /^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(String(value ?? '').trim());
|
||||
}
|
||||
|
||||
export function translateSqliteDefault(value) {
|
||||
if (value == null) return null;
|
||||
const input = String(value).trim();
|
||||
if (/^NULL$/i.test(input)) return 'NULL';
|
||||
if (/^[+-]?\d+(?:\.\d+)?$/.test(input)) return input;
|
||||
if (/^CURRENT_TIMESTAMP$/i.test(input)) return 'CURRENT_TIMESTAMP';
|
||||
if (/^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(input)) return 'CURRENT_TIMESTAMP';
|
||||
if (/^".*"$/.test(input)) return `'${input.slice(1, -1).replaceAll("'", "''")}'`;
|
||||
if (/^'.*'$/.test(input)) return input;
|
||||
return null;
|
||||
}
|
||||
|
||||
export function buildPostgresTableSql(schemaName, tableName, columns) {
|
||||
if (!Array.isArray(columns) || columns.length === 0) throw new Error(`表 ${tableName} 没有字段`);
|
||||
const primaryKeys = columns.filter((column) => Number(column.pk) > 0).sort((a, b) => Number(a.pk) - Number(b.pk));
|
||||
const definitions = columns.map((column) => {
|
||||
const parts = [quotePgIdentifier(column.name), mapSqliteTypeToPostgres(column)];
|
||||
if (Number(column.notnull) === 1) parts.push('NOT NULL');
|
||||
const translatedDefault = translateSqliteDefault(column.dflt_value);
|
||||
if (translatedDefault != null) parts.push(`DEFAULT ${translatedDefault}`);
|
||||
if (primaryKeys.length === 1 && primaryKeys[0].name === column.name) parts.push('PRIMARY KEY');
|
||||
return parts.join(' ');
|
||||
});
|
||||
if (primaryKeys.length > 1) {
|
||||
definitions.push(`PRIMARY KEY (${primaryKeys.map((column) => quotePgIdentifier(column.name)).join(', ')})`);
|
||||
}
|
||||
return `CREATE TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (\n ${definitions.join(',\n ')}\n)`;
|
||||
}
|
||||
|
||||
function stableObjectName(prefix, ...parts) {
|
||||
const readable = `${prefix}_${parts.join('_')}`.replace(/[^a-zA-Z0-9_]+/g, '_').toLowerCase();
|
||||
const hash = createHash('sha256').update(readable).digest('hex').slice(0, 8);
|
||||
return `${readable.slice(0, 50)}_${hash}`;
|
||||
}
|
||||
|
||||
export function extractSqliteChecks(createSql) {
|
||||
const sql = String(createSql ?? '');
|
||||
const checks = [];
|
||||
const pattern = /\bCHECK\s*\(/ig;
|
||||
let match;
|
||||
while ((match = pattern.exec(sql))) {
|
||||
const start = pattern.lastIndex;
|
||||
let depth = 1;
|
||||
let quote = '';
|
||||
let index = start;
|
||||
for (; index < sql.length && depth > 0; index += 1) {
|
||||
const char = sql[index];
|
||||
if (quote) {
|
||||
if (char === quote && sql[index + 1] === quote) index += 1;
|
||||
else if (char === quote) quote = '';
|
||||
continue;
|
||||
}
|
||||
if (char === "'" || char === '"') quote = char;
|
||||
else if (char === '(') depth += 1;
|
||||
else if (char === ')') depth -= 1;
|
||||
}
|
||||
if (depth === 0) checks.push(sql.slice(start, index - 1).trim());
|
||||
pattern.lastIndex = index;
|
||||
}
|
||||
return checks;
|
||||
}
|
||||
|
||||
export function buildPostgresIndexSql(schemaName, tableName, index) {
|
||||
if (!index.columns?.length) return null;
|
||||
if (index.partial) return null;
|
||||
const prefix = index.unique ? 'ux' : 'ix';
|
||||
const name = stableObjectName(prefix, tableName, ...index.columns);
|
||||
return `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX ${quotePgIdentifier(name)} ON ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (${index.columns.map(quotePgIdentifier).join(', ')})`;
|
||||
}
|
||||
|
||||
export function buildPostgresForeignKeySql(schemaName, tableName, foreignKey) {
|
||||
const name = stableObjectName('fk', tableName, foreignKey.id, foreignKey.table);
|
||||
const onUpdate = String(foreignKey.onUpdate ?? 'NO ACTION').toUpperCase();
|
||||
const onDelete = String(foreignKey.onDelete ?? 'NO ACTION').toUpperCase();
|
||||
const allowedActions = new Set(['NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT']);
|
||||
if (!allowedActions.has(onUpdate) || !allowedActions.has(onDelete)) throw new Error('SQLite 外键动作不受支持');
|
||||
return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} FOREIGN KEY (${foreignKey.from.map(quotePgIdentifier).join(', ')}) REFERENCES ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(foreignKey.table)} (${foreignKey.to.map(quotePgIdentifier).join(', ')}) ON UPDATE ${onUpdate} ON DELETE ${onDelete}`;
|
||||
}
|
||||
|
||||
export function buildPostgresCheckSql(schemaName, tableName, expression, index) {
|
||||
if (/;|--|\/\*/.test(expression)) throw new Error(`CHECK 表达式包含不安全内容:${tableName}`);
|
||||
const name = stableObjectName('ck', tableName, index);
|
||||
return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} CHECK (${expression})`;
|
||||
}
|
||||
|
||||
export function convertSqliteValueForPostgres(value, column) {
|
||||
if (value == null) return null;
|
||||
if (isSqliteTimestampDefault(column.dflt_value) && typeof value === 'string') {
|
||||
const trimmed = value.trim();
|
||||
if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(trimmed)) {
|
||||
return `${trimmed.replace(' ', 'T')}+08:00`;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function sqliteJson(sqliteBin, databasePath, sql) {
|
||||
const output = execFileSync(sqliteBin, ['-readonly', '-json', databasePath, sql], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
}).trim();
|
||||
return output ? JSON.parse(output) : [];
|
||||
}
|
||||
|
||||
function quoteSqliteIdentifier(value) {
|
||||
return `"${String(value).replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
export function inspectSqliteDatabase(databasePath, { sqliteBin = 'sqlite3' } = {}) {
|
||||
const resolved = path.resolve(databasePath);
|
||||
if (!fs.statSync(resolved).isFile()) throw new Error(`SQLite 文件不存在:${resolved}`);
|
||||
const integrity = sqliteJson(sqliteBin, resolved, 'PRAGMA integrity_check;');
|
||||
if (integrity[0]?.integrity_check !== 'ok') throw new Error(`SQLite integrity_check 未通过:${resolved}`);
|
||||
const tables = sqliteJson(
|
||||
sqliteBin,
|
||||
resolved,
|
||||
"SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;",
|
||||
).map((table) => {
|
||||
const columns = sqliteJson(sqliteBin, resolved, `PRAGMA table_info(${quoteSqliteIdentifier(table.name)});`);
|
||||
const rows = sqliteJson(sqliteBin, resolved, `SELECT * FROM ${quoteSqliteIdentifier(table.name)};`);
|
||||
const indexes = sqliteJson(sqliteBin, resolved, `PRAGMA index_list(${quoteSqliteIdentifier(table.name)});`).map((index) => ({
|
||||
name: index.name,
|
||||
unique: Boolean(index.unique),
|
||||
origin: index.origin,
|
||||
partial: Boolean(index.partial),
|
||||
columns: sqliteJson(sqliteBin, resolved, `PRAGMA index_info(${quoteSqliteIdentifier(index.name)});`).map((item) => item.name),
|
||||
})).filter((index) => index.origin !== 'pk');
|
||||
const foreignKeyRows = sqliteJson(sqliteBin, resolved, `PRAGMA foreign_key_list(${quoteSqliteIdentifier(table.name)});`);
|
||||
const foreignKeys = [...new Set(foreignKeyRows.map((item) => item.id))].map((id) => {
|
||||
const items = foreignKeyRows.filter((item) => item.id === id).sort((a, b) => a.seq - b.seq);
|
||||
return {
|
||||
id,
|
||||
table: items[0].table,
|
||||
from: items.map((item) => item.from),
|
||||
to: items.map((item) => item.to),
|
||||
onUpdate: items[0].on_update,
|
||||
onDelete: items[0].on_delete,
|
||||
};
|
||||
});
|
||||
return { ...table, columns, rows, indexes, foreignKeys, checks: extractSqliteChecks(table.sql) };
|
||||
});
|
||||
return { path: resolved, sizeBytes: fs.statSync(resolved).size, tables };
|
||||
}
|
||||
|
||||
export function createSqliteSnapshot(sourcePath, { sqliteBin = 'sqlite3' } = {}) {
|
||||
const target = path.join(os.tmpdir(), `mindspace-userdata-${Date.now()}-${process.pid}.sqlite`);
|
||||
if (/\s/.test(target)) throw new Error('SQLite snapshot 临时路径不能包含空格');
|
||||
execFileSync(sqliteBin, [path.resolve(sourcePath), `.backup ${target}`], { stdio: 'pipe' });
|
||||
// A standalone snapshot must not depend on the source WAL/SHM files.
|
||||
execFileSync(sqliteBin, [target, 'PRAGMA journal_mode=DELETE;'], { stdio: 'pipe' });
|
||||
return target;
|
||||
}
|
||||
|
||||
export async function provisionUserSpace(client, userId, options = {}) {
|
||||
const provision = buildProvisionUserSql(userId, options);
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
for (let index = 0; index < provision.statements.length; index += 1) {
|
||||
const params = index === provision.statements.length - 1 ? provision.params : [];
|
||||
await client.query(provision.statements[index], params);
|
||||
}
|
||||
await client.query('COMMIT');
|
||||
return provision;
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function migrateSqliteSnapshot(client, snapshot, userId, { replaceShadow = false, sourcePath = snapshot.path } = {}) {
|
||||
const names = deriveUserSpaceNames(userId);
|
||||
const owner = quotePgIdentifier(names.ownerRole);
|
||||
const agent = quotePgIdentifier(names.agentRole);
|
||||
const migrationId = randomUUID();
|
||||
let sourceRows = 0;
|
||||
let targetRows = 0;
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
const stateResult = await client.query(
|
||||
`SELECT migration_state FROM ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces WHERE user_id = $1::uuid FOR UPDATE`,
|
||||
[names.userId],
|
||||
);
|
||||
const migrationState = stateResult.rows[0]?.migration_state;
|
||||
if (migrationState === 'cutover') throw new Error('用户空间已经 cutover,禁止迁移器覆盖');
|
||||
const existingResult = await client.query(
|
||||
`SELECT count(*)::integer AS count FROM pg_catalog.pg_tables WHERE schemaname = $1`,
|
||||
[names.schemaName],
|
||||
);
|
||||
if (Number(existingResult.rows[0]?.count ?? 0) > 0 && !replaceShadow) {
|
||||
throw new Error('目标影子空间已有表;如确认重建,显式使用 --replace-shadow');
|
||||
}
|
||||
await client.query(
|
||||
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs
|
||||
(id, user_id, source_path, source_size_bytes, status, started_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, 'importing', CURRENT_TIMESTAMP)`,
|
||||
[migrationId, names.userId, sourcePath, snapshot.sizeBytes],
|
||||
);
|
||||
for (const table of snapshot.tables) {
|
||||
await client.query(`DROP TABLE IF EXISTS ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} CASCADE`);
|
||||
await client.query(buildPostgresTableSql(names.schemaName, table.name, table.columns));
|
||||
await client.query(`ALTER TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} OWNER TO ${owner}`);
|
||||
await client.query(
|
||||
`GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} TO ${agent}`,
|
||||
);
|
||||
for (const row of table.rows) {
|
||||
const columns = Object.keys(row);
|
||||
const params = columns.map((_, index) => `$${index + 1}`);
|
||||
await client.query(
|
||||
`INSERT INTO ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} (${columns.map(quotePgIdentifier).join(', ')}) VALUES (${params.join(', ')})`,
|
||||
columns.map((column) => convertSqliteValueForPostgres(
|
||||
row[column],
|
||||
table.columns.find((item) => item.name === column) ?? {},
|
||||
)),
|
||||
);
|
||||
}
|
||||
for (const column of table.columns.filter(
|
||||
(item) => Number(item.pk) === 1 && String(item.type ?? '').toUpperCase().includes('INT'),
|
||||
)) {
|
||||
const qualifiedTable = `${names.schemaName}.${table.name}`;
|
||||
await client.query(
|
||||
`SELECT setval(
|
||||
pg_get_serial_sequence($1, $2),
|
||||
GREATEST(COALESCE(MAX(${quotePgIdentifier(column.name)}), 0), 1),
|
||||
COALESCE(MAX(${quotePgIdentifier(column.name)}), 0) > 0
|
||||
)
|
||||
FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`,
|
||||
[qualifiedTable, column.name],
|
||||
);
|
||||
}
|
||||
sourceRows += table.rows.length;
|
||||
const count = await client.query(
|
||||
`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`,
|
||||
);
|
||||
targetRows += Number(count.rows[0].count);
|
||||
}
|
||||
for (const table of snapshot.tables) {
|
||||
for (const index of table.indexes ?? []) {
|
||||
const sql = buildPostgresIndexSql(names.schemaName, table.name, index);
|
||||
if (sql) await client.query(sql);
|
||||
}
|
||||
for (const foreignKey of table.foreignKeys ?? []) {
|
||||
await client.query(buildPostgresForeignKeySql(names.schemaName, table.name, foreignKey));
|
||||
}
|
||||
for (let index = 0; index < (table.checks ?? []).length; index += 1) {
|
||||
await client.query(buildPostgresCheckSql(names.schemaName, table.name, table.checks[index], index));
|
||||
}
|
||||
}
|
||||
await client.query(`GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA ${quotePgIdentifier(names.schemaName)} TO ${agent}`);
|
||||
const registry = snapshot.tables.find((table) => table.name === '__page_data_datasets');
|
||||
for (const row of registry?.rows ?? []) {
|
||||
await client.query(
|
||||
`INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog
|
||||
(user_id, dataset_name, physical_table, config_json, updated_at)
|
||||
VALUES ($1::uuid, $2, $3::name, $4::jsonb, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (user_id, dataset_name) DO UPDATE SET
|
||||
physical_table = EXCLUDED.physical_table,
|
||||
config_json = EXCLUDED.config_json,
|
||||
updated_at = CURRENT_TIMESTAMP`,
|
||||
[names.userId, row.name, row.table_name, row.config_json],
|
||||
);
|
||||
}
|
||||
await client.query(
|
||||
`UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces
|
||||
SET migration_state = 'shadow', updated_at = CURRENT_TIMESTAMP
|
||||
WHERE user_id = $1::uuid`,
|
||||
[names.userId],
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs
|
||||
SET status = 'verified', table_count = $2, source_row_count = $3,
|
||||
target_row_count = $4, finished_at = CURRENT_TIMESTAMP
|
||||
WHERE id = $1::uuid`,
|
||||
[migrationId, snapshot.tables.length, sourceRows, targetRows],
|
||||
);
|
||||
await client.query('COMMIT');
|
||||
return { migrationId, ...names, tableCount: snapshot.tables.length, sourceRows, targetRows };
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
buildControlSchemaSql,
|
||||
buildPostgresTableSql,
|
||||
buildPostgresCheckSql,
|
||||
buildPostgresForeignKeySql,
|
||||
buildPostgresIndexSql,
|
||||
buildProvisionUserSql,
|
||||
createSqliteSnapshot,
|
||||
convertSqliteValueForPostgres,
|
||||
deriveUserSpaceNames,
|
||||
extractSqliteChecks,
|
||||
mapSqliteTypeToPostgres,
|
||||
translateSqliteDefault,
|
||||
} from './mindspace-userdata-postgres.mjs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const USER_ID = 'ecc1c649-fff7-4361-a243-a69b460cc407';
|
||||
|
||||
test('deriveUserSpaceNames creates stable UUID based identifiers', () => {
|
||||
assert.deepEqual(deriveUserSpaceNames(USER_ID), {
|
||||
userId: USER_ID,
|
||||
schemaName: 'u_ecc1c649fff74361a243a69b460cc407',
|
||||
ownerRole: 'ms_u_ecc1c649fff7_owner',
|
||||
agentRole: 'ms_u_ecc1c649fff7_agent',
|
||||
});
|
||||
});
|
||||
|
||||
test('control schema is private and tracks migration lifecycle', () => {
|
||||
const sql = buildControlSchemaSql();
|
||||
assert.match(sql, /REVOKE ALL ON SCHEMA "mindspace_control" FROM PUBLIC/);
|
||||
assert.match(sql, /CREATE TABLE IF NOT EXISTS "mindspace_control"\.user_spaces/);
|
||||
assert.match(sql, /migration_runs/);
|
||||
assert.match(sql, /audit_events/);
|
||||
});
|
||||
|
||||
test('provision SQL creates isolated no-login roles and safety limits', () => {
|
||||
const provision = buildProvisionUserSql(USER_ID, { sourceSqlitePath: '/tmp/user.sqlite' });
|
||||
const sql = provision.statements.join('\n');
|
||||
assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_owner" NOLOGIN/);
|
||||
assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_agent" NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE/);
|
||||
assert.match(sql, /pg_auth_members/);
|
||||
assert.match(sql, /REVOKE ALL ON SCHEMA "u_ecc1c649fff74361a243a69b460cc407" FROM PUBLIC/);
|
||||
assert.match(sql, /statement_timeout = '30s'/);
|
||||
assert.match(sql, /NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION/);
|
||||
assert.doesNotMatch(sql, /temp_file_limit/);
|
||||
});
|
||||
|
||||
test('SQLite types and defaults map to conservative PostgreSQL equivalents', () => {
|
||||
assert.equal(mapSqliteTypeToPostgres({ type: 'INTEGER', pk: 1 }), 'bigint GENERATED BY DEFAULT AS IDENTITY');
|
||||
assert.equal(mapSqliteTypeToPostgres({ type: 'REAL', pk: 0 }), 'double precision');
|
||||
assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0 }), 'text');
|
||||
assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0, dflt_value: "datetime('now', 'localtime')" }), 'timestamptz');
|
||||
assert.equal(translateSqliteDefault("(datetime('now', '+8 hours'))"), 'CURRENT_TIMESTAMP');
|
||||
assert.equal(translateSqliteDefault('""'), "''");
|
||||
assert.equal(translateSqliteDefault('dangerous_function()'), null);
|
||||
});
|
||||
|
||||
test('SQLite checks, indexes and foreign keys become scoped PostgreSQL objects', () => {
|
||||
assert.deepEqual(extractSqliteChecks("CREATE TABLE x(status TEXT CHECK(status IN ('a','b')), score INT CHECK(score > 0))"), [
|
||||
"status IN ('a','b')",
|
||||
'score > 0',
|
||||
]);
|
||||
assert.match(buildPostgresIndexSql('u_test', 'users', { unique: true, columns: ['username'] }), /CREATE UNIQUE INDEX/);
|
||||
assert.match(buildPostgresForeignKeySql('u_test', 'child', {
|
||||
id: 0, table: 'parent', from: ['parent_id'], to: ['id'], onUpdate: 'NO ACTION', onDelete: 'CASCADE',
|
||||
}), /REFERENCES "u_test"\."parent" \("id"\).*ON DELETE CASCADE/);
|
||||
assert.match(buildPostgresCheckSql('u_test', 'users', "status IN ('a','b')", 0), /CHECK \(status IN \('a','b'\)\)/);
|
||||
});
|
||||
|
||||
test('SQLite local timestamps are made explicit for PostgreSQL', () => {
|
||||
assert.equal(
|
||||
convertSqliteValueForPostgres('2026-07-13 10:20:30', { dflt_value: "datetime('now', '+8 hours')" }),
|
||||
'2026-07-13T10:20:30+08:00',
|
||||
);
|
||||
assert.equal(convertSqliteValueForPostgres('plain', { dflt_value: null }), 'plain');
|
||||
});
|
||||
|
||||
test('table builder preserves primary key, not-null and safe defaults', () => {
|
||||
const sql = buildPostgresTableSql('u_example', 'survey', [
|
||||
{ name: 'id', type: 'INTEGER', pk: 1, notnull: 0, dflt_value: null },
|
||||
{ name: 'answer', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "''" },
|
||||
{ name: 'created_at', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "datetime('now', 'localtime')" },
|
||||
]);
|
||||
assert.match(sql, /"id" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY/);
|
||||
assert.match(sql, /"answer" text NOT NULL DEFAULT ''/);
|
||||
assert.match(sql, /"created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP/);
|
||||
});
|
||||
|
||||
test('SQLite snapshot includes committed rows without changing source', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-pg-test-'));
|
||||
const source = path.join(root, 'source.sqlite');
|
||||
execFileSync('sqlite3', [source, 'CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO sample VALUES (1, \'ok\');']);
|
||||
const snapshot = createSqliteSnapshot(source);
|
||||
try {
|
||||
const count = execFileSync('sqlite3', ['-readonly', snapshot, 'SELECT count(*) FROM sample;'], { encoding: 'utf8' }).trim();
|
||||
assert.equal(count, '1');
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
fs.rmSync(snapshot, { force: true });
|
||||
}
|
||||
});
|
||||
@@ -56,8 +56,8 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page
|
||||
},
|
||||
},
|
||||
pageDataEnsure: {
|
||||
async ensurePageDataHtmlPagesBound() {
|
||||
bound.push(true);
|
||||
async ensurePageDataHtmlPagesBound(options) {
|
||||
bound.push(options);
|
||||
return { bound: [{ relativePath: 'public/survey.html' }], skipped: [], errors: [] };
|
||||
},
|
||||
},
|
||||
@@ -65,8 +65,11 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page
|
||||
storageRoot: null,
|
||||
});
|
||||
|
||||
const result = await service.syncAndDeliver(userId);
|
||||
const result = await service.syncAndDeliver(userId, {
|
||||
pageDataRelativePaths: ['public/survey.html'],
|
||||
});
|
||||
assert.equal(bound.length, 1);
|
||||
assert.deepEqual(bound[0].onlyRelativePaths, ['public/survey.html']);
|
||||
assert.equal(result.publish.published, 0);
|
||||
assert.equal(result.publish.skipped, 1);
|
||||
assert.equal(published.length, 0);
|
||||
|
||||
@@ -95,7 +95,7 @@ export function createWorkspacePageDeliverService({
|
||||
return resolvePublishDir(h5Root, { id: userId });
|
||||
}
|
||||
|
||||
async function ensurePageDataBindings(userId) {
|
||||
async function ensurePageDataBindings(userId, { onlyRelativePaths = null } = {}) {
|
||||
if (!pageDataEnsure?.ensurePageDataHtmlPagesBound || !pool || !userId) {
|
||||
return { bound: [], skipped: [], errors: [] };
|
||||
}
|
||||
@@ -110,6 +110,7 @@ export function createWorkspacePageDeliverService({
|
||||
userId,
|
||||
workspaceRoot,
|
||||
findPageByRelativePath,
|
||||
onlyRelativePaths,
|
||||
logger,
|
||||
});
|
||||
}
|
||||
@@ -189,15 +190,20 @@ export function createWorkspacePageDeliverService({
|
||||
return { published, skipped, errors };
|
||||
}
|
||||
|
||||
async function syncAndDeliver(userId) {
|
||||
async function syncAndDeliver(userId, { pageDataRelativePaths = null } = {}) {
|
||||
let syncResult = { created: 0, updated: 0, skipped: 0 };
|
||||
if (pageSyncService?.syncUserGeneratedPages) {
|
||||
syncResult = await pageSyncService.syncUserGeneratedPages(userId);
|
||||
syncResult = await pageSyncService.syncUserGeneratedPages(userId, {
|
||||
onlyRelativePaths: pageDataRelativePaths,
|
||||
});
|
||||
}
|
||||
const pageDataBindResult = await ensurePageDataBindings(userId);
|
||||
const pageDataBindResult = await ensurePageDataBindings(userId, {
|
||||
onlyRelativePaths: pageDataRelativePaths,
|
||||
});
|
||||
const publishResult = await ensureWorkspaceHtmlPublications(userId);
|
||||
const refreshResult = await refreshOnlineWorkspacePublications(userId);
|
||||
return {
|
||||
pageDataRelativePaths: pageDataRelativePaths == null ? null : [...pageDataRelativePaths],
|
||||
sync: syncResult,
|
||||
pageDataBind: pageDataBindResult,
|
||||
publish: publishResult,
|
||||
|
||||
@@ -436,10 +436,20 @@ export function createWorkspaceAssetSync({
|
||||
}
|
||||
};
|
||||
|
||||
const syncCategory = async (userId, categoryCode, source = {}) => {
|
||||
const syncCategory = async (userId, categoryCode, source = {}, { onlyRelativePaths = null } = {}) => {
|
||||
if (!h5Root) return { imported: 0, updated: 0, skipped: 0 };
|
||||
const workspaceRoot = resolveUserWorkspaceRoot(h5Root, { id: userId });
|
||||
const files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode);
|
||||
let files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode);
|
||||
if (onlyRelativePaths != null) {
|
||||
const allowed = new Set(
|
||||
onlyRelativePaths
|
||||
.map((item) => normalizeWorkspaceRelativePath(item))
|
||||
.filter(Boolean),
|
||||
);
|
||||
files = files.filter((file) =>
|
||||
allowed.has(normalizeWorkspaceRelativePath(`${categoryCode}/${file.filename}`)),
|
||||
);
|
||||
}
|
||||
const [categories] = await pool.query(
|
||||
`SELECT c.id, c.space_id, c.category_code
|
||||
FROM h5_space_categories c
|
||||
@@ -487,14 +497,17 @@ export function createWorkspaceAssetSync({
|
||||
return { imported, updated, skipped };
|
||||
};
|
||||
|
||||
const syncUserWorkspace = async (userId, { categoryCode, sourceSessionId, sourceMessageId, title } = {}) => {
|
||||
const syncUserWorkspace = async (
|
||||
userId,
|
||||
{ categoryCode, sourceSessionId, sourceMessageId, title, onlyRelativePaths = null } = {},
|
||||
) => {
|
||||
const codes = categoryCode ? [categoryCode] : UPLOAD_ZONE_CODES;
|
||||
let imported = 0;
|
||||
let updated = 0;
|
||||
let skipped = 0;
|
||||
const source = { sessionId: sourceSessionId, messageId: sourceMessageId, title };
|
||||
for (const code of codes) {
|
||||
const result = await syncCategory(userId, code, source);
|
||||
const result = await syncCategory(userId, code, source, { onlyRelativePaths });
|
||||
imported += result.imported;
|
||||
updated += result.updated;
|
||||
skipped += result.skipped;
|
||||
|
||||
@@ -316,6 +316,7 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn
|
||||
const workspace = resolveUserWorkspaceRoot(h5Root, { id: 'user-1', username: 'john' });
|
||||
await fs.mkdir(path.join(workspace, 'oa', '暑假实习报告'), { recursive: true });
|
||||
await fs.writeFile(path.join(workspace, 'oa', '暑假实习报告', 'report.csv'), 'a,b\n1,2\n');
|
||||
await fs.writeFile(path.join(workspace, 'oa', 'historical.csv'), 'old,data\n3,4\n');
|
||||
|
||||
const state = {
|
||||
categories: [
|
||||
@@ -417,7 +418,10 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn
|
||||
idFactory: () => `id-${++nextId}`,
|
||||
});
|
||||
|
||||
const result = await sync.syncUserWorkspace('user-1', { categoryCode: 'oa' });
|
||||
const result = await sync.syncUserWorkspace('user-1', {
|
||||
categoryCode: 'oa',
|
||||
onlyRelativePaths: ['oa/暑假实习报告/report.csv'],
|
||||
});
|
||||
assert.equal(result.imported, 1);
|
||||
assert.equal(state.assets[0].original_filename, '暑假实习报告/report.csv');
|
||||
});
|
||||
|
||||
@@ -40,6 +40,33 @@ export function buildSmokeInsertRow(policy, datasetName) {
|
||||
return row;
|
||||
}
|
||||
|
||||
export function assessPageDataAuthenticationContract({ policy, html } = {}) {
|
||||
const accessMode = String(policy?.accessMode ?? '').trim();
|
||||
const content = String(html ?? '');
|
||||
const usesServerAuthentication = /\.\s*authenticate\s*\(/.test(content);
|
||||
const passwordValue = String.raw`\b(?:pwd|pass(?:word)?|passwordInput|adminPassword)\w*(?:\.value)?`;
|
||||
const passwordComparison = String.raw`(?:===?|!==?)`;
|
||||
const passwordOperand = String.raw`(?:[A-Za-z_$][\w$]*|['"\x60][^'"\x60]*['"\x60])`;
|
||||
const usesClientSidePasswordCheck = new RegExp(
|
||||
`(?:${passwordValue}\\s*${passwordComparison}\\s*${passwordOperand}|${passwordOperand}\\s*${passwordComparison}\\s*${passwordValue})`,
|
||||
'i',
|
||||
).test(content);
|
||||
const reasons = [];
|
||||
if (accessMode === 'password' && !usesServerAuthentication) {
|
||||
reasons.push('password_page_missing_server_authentication');
|
||||
}
|
||||
if (accessMode === 'public' && usesServerAuthentication) {
|
||||
reasons.push('public_page_cannot_use_password_authentication');
|
||||
}
|
||||
if (accessMode === 'public' && usesClientSidePasswordCheck) {
|
||||
reasons.push('public_page_uses_client_side_password_gate');
|
||||
}
|
||||
if (accessMode === 'password' && usesClientSidePasswordCheck) {
|
||||
reasons.push('password_page_uses_client_side_password_gate');
|
||||
}
|
||||
return reasons;
|
||||
}
|
||||
|
||||
export async function queryOnlinePublication(pool, pageId) {
|
||||
if (!pool || !pageId) return null;
|
||||
const [rows] = await pool.query(
|
||||
@@ -76,20 +103,13 @@ function findWorkspacePolicyForHtml({ publishDir, relativePath, html }) {
|
||||
return null;
|
||||
}
|
||||
|
||||
export function assessWorkspacePageDataReadiness({ publishDir, relativePath, html }) {
|
||||
function assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }) {
|
||||
const usage = detectPageDataDatasetUsageFromHtml(html);
|
||||
if (!usage.size) {
|
||||
return { ready: true, reasons: [] };
|
||||
}
|
||||
|
||||
const reasons = [];
|
||||
const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir });
|
||||
for (const datasetName of usage.keys()) {
|
||||
if (!dataSpace.getDataset(datasetName)) {
|
||||
reasons.push(`dataset_not_registered:${datasetName}`);
|
||||
}
|
||||
}
|
||||
|
||||
const policy = findWorkspacePolicyForHtml({ publishDir, relativePath, html });
|
||||
if (!policy) {
|
||||
reasons.push('missing_workspace_policy');
|
||||
@@ -98,6 +118,21 @@ export function assessWorkspacePageDataReadiness({ publishDir, relativePath, htm
|
||||
return { ready: reasons.length === 0, reasons, policy };
|
||||
}
|
||||
|
||||
export async function assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }) {
|
||||
const usage = detectPageDataDatasetUsageFromHtml(html);
|
||||
const policyAssessment = assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html });
|
||||
if (!usage.size) return policyAssessment;
|
||||
|
||||
const reasons = [...policyAssessment.reasons];
|
||||
const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir, userId });
|
||||
for (const datasetName of usage.keys()) {
|
||||
if (!(await dataSpace.getDataset(datasetName))) {
|
||||
reasons.push(`dataset_not_registered:${datasetName}`);
|
||||
}
|
||||
}
|
||||
return { ...policyAssessment, ready: reasons.length === 0, reasons };
|
||||
}
|
||||
|
||||
export async function assessPageDataHtmlBinding({
|
||||
pool,
|
||||
userId,
|
||||
@@ -111,7 +146,7 @@ export async function assessPageDataHtmlBinding({
|
||||
return { bound: true, reasons: [], pageId: null };
|
||||
}
|
||||
|
||||
const workspace = assessWorkspacePageDataReadiness({ publishDir, relativePath, html });
|
||||
const workspace = await assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html });
|
||||
const reasons = [...workspace.reasons];
|
||||
|
||||
let pageId = null;
|
||||
@@ -139,6 +174,7 @@ export async function assessPageDataHtmlBinding({
|
||||
if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) {
|
||||
reasons.push('policy_access_mode_mismatch');
|
||||
}
|
||||
reasons.push(...assessPageDataAuthenticationContract({ policy, html }));
|
||||
for (const [datasetName, perms] of usage) {
|
||||
const policyDataset = policy.datasets?.[datasetName];
|
||||
if (!policyDataset) continue;
|
||||
|
||||
@@ -1,11 +1,49 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
assessPageDataAuthenticationContract,
|
||||
buildSmokeInsertRow,
|
||||
extractInjectedPageIdFromHtml,
|
||||
normalizePageDataApiBase,
|
||||
} from './page-data-delivery-assess.mjs';
|
||||
|
||||
test('password pages must authenticate with the server before data reads', () => {
|
||||
assert.deepEqual(
|
||||
assessPageDataAuthenticationContract({
|
||||
policy: { accessMode: 'password' },
|
||||
html: '<script>if (pwd === ADMIN_PASSWORD) loadData()</script>',
|
||||
}),
|
||||
[
|
||||
'password_page_missing_server_authentication',
|
||||
'password_page_uses_client_side_password_gate',
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
assessPageDataAuthenticationContract({
|
||||
policy: { accessMode: 'password' },
|
||||
html: '<script>client.authenticate(password).then(loadData)</script>',
|
||||
}),
|
||||
[],
|
||||
);
|
||||
assert.deepEqual(
|
||||
assessPageDataAuthenticationContract({
|
||||
policy: { accessMode: 'password' },
|
||||
html: '<script>if (passwordInput.value !== "88888888") return; client.authenticate(passwordInput.value)</script>',
|
||||
}),
|
||||
['password_page_uses_client_side_password_gate'],
|
||||
);
|
||||
});
|
||||
|
||||
test('public pages cannot use a client-side password gate as authorization', () => {
|
||||
assert.deepEqual(
|
||||
assessPageDataAuthenticationContract({
|
||||
policy: { accessMode: 'public' },
|
||||
html: '<script>if (pwd === DIARY_PASSWORD) client.insertRow("diary_entries", row)</script>',
|
||||
}),
|
||||
['public_page_uses_client_side_password_gate'],
|
||||
);
|
||||
});
|
||||
|
||||
test('extractInjectedPageIdFromHtml reads injected pageId', () => {
|
||||
const html = `<html><head>
|
||||
<meta name="mindspace-page-data-page-id" content="page-abc">
|
||||
|
||||
@@ -171,13 +171,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) {
|
||||
async function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) {
|
||||
const policyDataset = policy.datasets?.[datasetName];
|
||||
if (!policyDataset) {
|
||||
throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' });
|
||||
}
|
||||
const ownerService = createOwnerService(ownerUserId);
|
||||
const registryDataset = ownerService.getDataset(datasetName);
|
||||
const registryDataset = await ownerService.getDataset(datasetName);
|
||||
if (!registryDataset) {
|
||||
throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
}
|
||||
@@ -190,9 +190,9 @@ export function createPageDataPublicService(deps = {}) {
|
||||
return { ownerService, effective, policyDataset };
|
||||
}
|
||||
|
||||
function resolveRowScope(ownerService, effective, policyDataset, visitorContext) {
|
||||
async function resolveRowScope(ownerService, effective, policyDataset, visitorContext) {
|
||||
if (!policyDataset?.rowPolicy || !visitorContext) return null;
|
||||
const tableColumns = ownerService.listTableColumns(effective.table);
|
||||
const tableColumns = await ownerService.listTableColumns(effective.table);
|
||||
return resolveRowAccessScope(policyDataset.rowPolicy, {
|
||||
visitorRole: visitorContext.visitorRole,
|
||||
visitorUserId: visitorContext.visitorUserId,
|
||||
@@ -303,13 +303,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
const policy = await loadPolicy(publication);
|
||||
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
|
||||
enforceRateLimit(req, publication, 'read');
|
||||
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
|
||||
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
access.rolePermissions ?? null,
|
||||
);
|
||||
const rowScope = resolveRowScope(
|
||||
const rowScope = await resolveRowScope(
|
||||
ownerService,
|
||||
effective,
|
||||
policyDataset,
|
||||
@@ -330,7 +330,7 @@ export function createPageDataPublicService(deps = {}) {
|
||||
const publication = await queryPublication(pageId);
|
||||
const policy = await loadPolicy(publication);
|
||||
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
|
||||
const { ownerService, effective } = resolveEffectiveDataset(
|
||||
const { ownerService, effective } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
@@ -345,13 +345,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
const publication = await queryPublication(pageId);
|
||||
const policy = await loadPolicy(publication);
|
||||
const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName });
|
||||
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
|
||||
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
access.rolePermissions ?? null,
|
||||
);
|
||||
const rowScope = resolveRowScope(
|
||||
const rowScope = await resolveRowScope(
|
||||
ownerService,
|
||||
effective,
|
||||
policyDataset,
|
||||
@@ -415,13 +415,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
}
|
||||
const payload = stripPageDataMetaFields(payloadInput);
|
||||
enforceRateLimit(req, publication, 'insert');
|
||||
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
|
||||
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
access.rolePermissions ?? null,
|
||||
);
|
||||
const rowScope = resolveRowScope(
|
||||
const rowScope = await resolveRowScope(
|
||||
ownerService,
|
||||
effective,
|
||||
policyDataset,
|
||||
@@ -465,13 +465,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
datasetName,
|
||||
});
|
||||
enforceRateLimit(req, publication, 'update');
|
||||
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
|
||||
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
access.rolePermissions ?? null,
|
||||
);
|
||||
const rowScope = resolveRowScope(
|
||||
const rowScope = await resolveRowScope(
|
||||
ownerService,
|
||||
effective,
|
||||
policyDataset,
|
||||
@@ -513,13 +513,13 @@ export function createPageDataPublicService(deps = {}) {
|
||||
datasetName,
|
||||
});
|
||||
enforceRateLimit(req, publication, 'soft_delete');
|
||||
const { ownerService, effective, policyDataset } = resolveEffectiveDataset(
|
||||
const { ownerService, effective, policyDataset } = await resolveEffectiveDataset(
|
||||
publication.user_id,
|
||||
policy,
|
||||
datasetName,
|
||||
access.rolePermissions ?? null,
|
||||
);
|
||||
const rowScope = resolveRowScope(
|
||||
const rowScope = await resolveRowScope(
|
||||
ownerService,
|
||||
effective,
|
||||
policyDataset,
|
||||
@@ -598,7 +598,7 @@ export function createPageDataPublicService(deps = {}) {
|
||||
throw Object.assign(new Error('缺少 dataset 名称'), { code: 'invalid_request', status: 400 });
|
||||
}
|
||||
const ownerService = createOwnerService(user.id);
|
||||
const registryDataset = ownerService.getDataset(normalizedDatasetName);
|
||||
const registryDataset = await ownerService.getDataset(normalizedDatasetName);
|
||||
if (!registryDataset) {
|
||||
throw Object.assign(new Error('dataset 未注册'), { code: 'dataset_not_found', status: 404 });
|
||||
}
|
||||
@@ -639,8 +639,8 @@ export function createPageDataPublicService(deps = {}) {
|
||||
for (const [name, config] of Object.entries(policy.datasets ?? {})) {
|
||||
let stats = null;
|
||||
try {
|
||||
if (ownerService.getDataset(name)) {
|
||||
stats = ownerService.getDatasetStats(name);
|
||||
if (await ownerService.getDataset(name)) {
|
||||
stats = await ownerService.getDatasetStats(name);
|
||||
}
|
||||
} catch {
|
||||
stats = null;
|
||||
@@ -660,7 +660,7 @@ export function createPageDataPublicService(deps = {}) {
|
||||
const recentSubmissions = logs.filter((entry) => entry.action === 'insert').slice(0, 10);
|
||||
const activeSessions = sessionStore.listByPageId(pageId);
|
||||
const index = await getPageDataPolicyIndex(getPool(), pageId);
|
||||
const registeredDatasets = ownerService.listDatasets().map((dataset) => dataset.name);
|
||||
const registeredDatasets = (await ownerService.listDatasets()).map((dataset) => dataset.name);
|
||||
const configuredNames = new Set(Object.keys(policy.datasets ?? {}));
|
||||
const unconfiguredDatasets = registeredDatasets.filter((name) => !configuredNames.has(name));
|
||||
return {
|
||||
|
||||
@@ -58,7 +58,7 @@ export function createPageDataService(deps = {}) {
|
||||
async function listRows(user, datasetName, query = {}) {
|
||||
const { service } = await createServiceForUser(user);
|
||||
try {
|
||||
return service.readDatasetRows(datasetName, {
|
||||
return await service.readDatasetRows(datasetName, {
|
||||
limit: query.limit,
|
||||
offset: query.offset,
|
||||
orderBy: query.orderBy ?? query.order_by,
|
||||
@@ -73,7 +73,7 @@ export function createPageDataService(deps = {}) {
|
||||
async function getSchema(user, datasetName) {
|
||||
const { service } = await createServiceForUser(user);
|
||||
try {
|
||||
return service.getDatasetSchema(datasetName);
|
||||
return await service.getDatasetSchema(datasetName);
|
||||
} catch (error) {
|
||||
throw mapServiceError(error);
|
||||
}
|
||||
@@ -82,7 +82,7 @@ export function createPageDataService(deps = {}) {
|
||||
async function getStats(user, datasetName) {
|
||||
const { service } = await createServiceForUser(user);
|
||||
try {
|
||||
return service.getDatasetStats(datasetName);
|
||||
return await service.getDatasetStats(datasetName);
|
||||
} catch (error) {
|
||||
throw mapServiceError(error);
|
||||
}
|
||||
@@ -139,7 +139,8 @@ export function createPageDataService(deps = {}) {
|
||||
|
||||
async function listDatasets(user) {
|
||||
const { service } = await createServiceForUser(user);
|
||||
return service.listDatasets().map((dataset) => ({
|
||||
const datasets = await service.listDatasets();
|
||||
return datasets.map((dataset) => ({
|
||||
name: dataset.name,
|
||||
table: dataset.table,
|
||||
description: dataset.description,
|
||||
|
||||
@@ -171,6 +171,16 @@ function assertRegisteredDatasetTables(userDataSpace, registryDatasets, usage) {
|
||||
}
|
||||
}
|
||||
|
||||
async function assertRegisteredDatasetTablesAsync(userDataSpace, registryDatasets, usage) {
|
||||
for (const dataset of registryDatasets) {
|
||||
const adapter = {
|
||||
listTableColumns: () => userDataSpace.listTableColumns(dataset.table),
|
||||
};
|
||||
const columns = await adapter.listTableColumns();
|
||||
assertRegisteredDatasetTables({ listTableColumns: () => columns }, [dataset], usage);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Page Data policy must be derived from a registered SQLite dataset, never
|
||||
* accepted solely from an Agent-supplied policy. This runs before page or
|
||||
@@ -213,6 +223,27 @@ export function resolveRegisteredPageDataPolicy({
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolveRegisteredPageDataPolicyAsync(options = {}) {
|
||||
const { html, userId, accessMode, pageDataPolicy = null, userDataSpace } = options;
|
||||
const usage = detectPageDataDatasetUsageFromHtml(html);
|
||||
if (!usage.size) return null;
|
||||
if (!userDataSpace) throw new Error('缺少 Page Data 数据空间');
|
||||
if (pageDataPolicy?.datasets) assertPolicyMatchesHtmlDatasets(html, pageDataPolicy.datasets);
|
||||
const registryDatasets = await userDataSpace.listDatasets();
|
||||
const datasets = buildPageDataPolicyDatasetsFromRegistry({ html, registryDatasets, usage });
|
||||
await assertRegisteredDatasetTablesAsync(
|
||||
userDataSpace,
|
||||
registryDatasets.filter((dataset) => datasets[dataset.name]),
|
||||
usage,
|
||||
);
|
||||
return {
|
||||
...pageDataPolicy,
|
||||
ownerUserId: String(pageDataPolicy?.ownerUserId ?? userId).trim(),
|
||||
accessMode: pageDataPolicy?.accessMode ?? accessMode,
|
||||
datasets,
|
||||
};
|
||||
}
|
||||
|
||||
export async function bindWorkspaceHtmlForPageData({
|
||||
pool,
|
||||
h5Root,
|
||||
@@ -236,8 +267,8 @@ export async function bindWorkspaceHtmlForPageData({
|
||||
const normalizedAccessMode = resolvePageDataBindAccess(accessMode);
|
||||
const resolvedPassword = resolvePageDataBindPassword(normalizedAccessMode, password);
|
||||
|
||||
const userDataSpace = createUserDataSpaceService({ workspaceRoot });
|
||||
const resolvedPageDataPolicy = resolveRegisteredPageDataPolicy({
|
||||
const userDataSpace = createUserDataSpaceService({ workspaceRoot, userId, query: pool });
|
||||
const resolvedPageDataPolicy = await resolveRegisteredPageDataPolicyAsync({
|
||||
html: content,
|
||||
userId,
|
||||
accessMode: normalizedAccessMode,
|
||||
|
||||
@@ -66,7 +66,7 @@ export async function ensureRegisteredDatasetFromHtml({
|
||||
datasetName,
|
||||
}) {
|
||||
const dataSpace = createUserDataSpaceService({ workspaceRoot, userId, query });
|
||||
const existing = dataSpace.getDataset(datasetName);
|
||||
const existing = await dataSpace.getDataset(datasetName);
|
||||
if (existing) return existing;
|
||||
|
||||
const insertColumns = inferInsertColumnsFromHtml(html, datasetName);
|
||||
|
||||
@@ -57,7 +57,13 @@ test('evaluatePageDataFinishGuard detects unbound homework html even when user o
|
||||
const evaluation = evaluatePageDataFinishGuard({
|
||||
publishDir,
|
||||
agentText: '确认发布',
|
||||
messages: [],
|
||||
messages: [{
|
||||
createdAt: Date.now(),
|
||||
content: [{
|
||||
type: 'toolRequest',
|
||||
toolCall: { value: { name: 'write_file', arguments: { path: 'public/homework.html' } } },
|
||||
}],
|
||||
}],
|
||||
});
|
||||
assert.equal(evaluation.pageDataIntent, false);
|
||||
assert.equal(evaluation.structuralPageData, true);
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
import pg from 'pg';
|
||||
import {
|
||||
buildControlSchemaSql,
|
||||
deriveUserSpaceNames,
|
||||
provisionUserSpace,
|
||||
quotePgIdentifier,
|
||||
} from './mindspace-userdata-postgres.mjs';
|
||||
|
||||
const pools = new Map();
|
||||
const provisionedUsers = new Set();
|
||||
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
|
||||
function ident(value, label = '标识符') {
|
||||
const text = String(value ?? '').trim();
|
||||
if (!IDENTIFIER.test(text)) throw Object.assign(new Error(`${label} 格式无效`), { code: 'invalid_identifier' });
|
||||
return text;
|
||||
}
|
||||
|
||||
function normalizeDataset(raw, fallbackName = null) {
|
||||
const value = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
||||
if (!value || typeof value !== 'object') throw Object.assign(new Error('dataset 配置无效'), { code: 'invalid_dataset_config' });
|
||||
const name = ident(value.name ?? fallbackName, 'dataset 名称');
|
||||
const table = ident(value.table ?? value.table_name, 'dataset 表名');
|
||||
const columns = value.columns && typeof value.columns === 'object' ? value.columns : {};
|
||||
for (const [action, fields] of Object.entries(columns)) {
|
||||
if (!Array.isArray(fields)) throw Object.assign(new Error(`dataset 字段白名单无效:${action}`), { code: 'invalid_dataset_config' });
|
||||
columns[action] = fields.map((field) => ident(field, `${action} 字段`));
|
||||
}
|
||||
return {
|
||||
name,
|
||||
table,
|
||||
description: String(value.description ?? ''),
|
||||
actions: Array.isArray(value.actions) ? value.actions.map(String) : [],
|
||||
columns,
|
||||
limits: {
|
||||
maxRowsPerRead: Number(value.limits?.maxRowsPerRead ?? 200),
|
||||
maxInsertBytes: Number(value.limits?.maxInsertBytes ?? 8192),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function poolConfig(options) {
|
||||
if (options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL) {
|
||||
return { connectionString: options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL, max: 5 };
|
||||
}
|
||||
return {
|
||||
host: options.pgHost ?? process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp',
|
||||
port: Number(options.pgPort ?? process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433),
|
||||
database: options.pgDatabase ?? process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev',
|
||||
user: options.pgUser ?? process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER,
|
||||
max: 5,
|
||||
};
|
||||
}
|
||||
|
||||
function sharedPool(options) {
|
||||
if (options.pgPool) return options.pgPool;
|
||||
const config = poolConfig(options);
|
||||
const key = JSON.stringify(config);
|
||||
if (!pools.has(key)) pools.set(key, new pg.Pool(config));
|
||||
return pools.get(key);
|
||||
}
|
||||
|
||||
function rejectSql(sql, { readonly = false } = {}) {
|
||||
const cleaned = String(sql ?? '').replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trim();
|
||||
if (!cleaned || cleaned.length > 20000) throw new Error('SQL 为空或过长');
|
||||
if (readonly && !/^(SELECT|WITH)\b/i.test(cleaned)) throw new Error('private_data_query 只允许 SELECT/WITH');
|
||||
if (/\b(CREATE|ALTER|DROP)\s+(ROLE|USER|DATABASE|TABLESPACE|EXTENSION)|ALTER\s+SYSTEM|COPY[\s\S]+PROGRAM|SECURITY\s+DEFINER|pg_(read|write)_file|lo_import|dblink/i.test(cleaned)) {
|
||||
throw new Error('SQL 包含用户空间不允许的操作');
|
||||
}
|
||||
if (/\bSET\s+(ROLE|SESSION_AUTHORIZATION|search_path)\b/i.test(cleaned)) throw new Error('SQL 不允许改变安全上下文');
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
function translateSqliteAgentDdl(sql) {
|
||||
return sql
|
||||
.replace(/INTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT/gi, 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY')
|
||||
.replace(/TEXT\s+NOT\s+NULL\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP')
|
||||
.replace(/TEXT\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP');
|
||||
}
|
||||
|
||||
export function createPostgresUserDataSpaceService(options = {}) {
|
||||
const names = deriveUserSpaceNames(options.userId);
|
||||
const pool = sharedPool(options);
|
||||
const schema = quotePgIdentifier(names.schemaName);
|
||||
const role = quotePgIdentifier(names.agentRole);
|
||||
const maxRows = Number(options.maxRows ?? 200);
|
||||
|
||||
async function ensureProvisioned({ force = false } = {}) {
|
||||
if (provisionedUsers.has(names.userId)) return;
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query(buildControlSchemaSql());
|
||||
const existing = await client.query('SELECT migration_state FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]);
|
||||
if (!existing.rows[0]) {
|
||||
if (!force && String(options.autoProvision ?? process.env.MINDSPACE_USERDATA_AUTO_PROVISION ?? '0') !== '1') {
|
||||
throw Object.assign(new Error('用户 PG 空间尚未分配'), { code: 'user_space_not_provisioned' });
|
||||
}
|
||||
await provisionUserSpace(client, names.userId, { sourceSqlitePath: options.workspaceRoot ? `${options.workspaceRoot}/.mindspace/private-data.sqlite` : null });
|
||||
await client.query('BEGIN');
|
||||
try {
|
||||
await client.query(`CREATE TABLE IF NOT EXISTS ${schema}.__page_data_datasets (name text PRIMARY KEY, table_name text NOT NULL, config_json text NOT NULL, created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)`);
|
||||
await client.query(`ALTER TABLE ${schema}.__page_data_datasets OWNER TO ${quotePgIdentifier(names.ownerRole)}`);
|
||||
await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON ${schema}.__page_data_datasets TO ${role}`);
|
||||
await client.query("UPDATE mindspace_control.user_spaces SET migration_state='cutover',cutover_at=CURRENT_TIMESTAMP,rollback_until=CURRENT_TIMESTAMP + INTERVAL '30 days',updated_at=CURRENT_TIMESTAMP WHERE user_id=$1::uuid", [names.userId]);
|
||||
await client.query('COMMIT');
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
provisionedUsers.add(names.userId);
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function tenant(fn, { write = false } = {}) {
|
||||
await ensureProvisioned();
|
||||
const client = await pool.connect();
|
||||
let quotaBytes = null;
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
if (write) {
|
||||
const quota = await client.query('SELECT quota_bytes FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]);
|
||||
quotaBytes = Number(quota.rows[0]?.quota_bytes ?? 0);
|
||||
}
|
||||
await client.query(`SET LOCAL ROLE ${role}`);
|
||||
await client.query(`SET LOCAL search_path = ${schema}, pg_catalog`);
|
||||
await client.query(`SET LOCAL statement_timeout = '${write ? 30 : 10}s'`);
|
||||
const result = await fn(client);
|
||||
if (write && quotaBytes > 0) {
|
||||
const usage = await client.query(
|
||||
`SELECT COALESCE(sum(pg_total_relation_size(c.oid)),0)::bigint AS bytes
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
WHERE n.nspname=$1 AND c.relkind IN ('r','p','m')`,
|
||||
[names.schemaName],
|
||||
);
|
||||
if (Number(usage.rows[0].bytes) > quotaBytes) {
|
||||
throw Object.assign(new Error(`用户 PG 空间超过配额:${usage.rows[0].bytes}/${quotaBytes}`), { code: 'quota_exceeded' });
|
||||
}
|
||||
}
|
||||
await client.query(write ? 'COMMIT' : 'ROLLBACK');
|
||||
if (write) {
|
||||
await client.query(
|
||||
`INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json)
|
||||
VALUES($1::uuid,'agent',$2,'postgres_write','success',$3::jsonb)`,
|
||||
[names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName })],
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK').catch(() => {});
|
||||
if (write) {
|
||||
await client.query(
|
||||
`INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json)
|
||||
VALUES($1::uuid,'agent',$2,'postgres_write','failed',$3::jsonb)`,
|
||||
[names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName, code: error?.code ?? null })],
|
||||
).catch(() => {});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function listTableColumns(tableName) {
|
||||
const table = ident(tableName, '表名');
|
||||
return tenant(async (client) => {
|
||||
const result = await client.query(
|
||||
`SELECT column_name AS name, data_type AS type,
|
||||
(is_nullable = 'NO') AS "notNull",
|
||||
(SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) WHERE i.indrelid=($1||'.'||$2)::regclass AND i.indisprimary AND a.attname=c.column_name)) AS pk
|
||||
FROM information_schema.columns c WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`,
|
||||
[names.schemaName, table],
|
||||
);
|
||||
return result.rows;
|
||||
});
|
||||
}
|
||||
|
||||
async function getSchema() {
|
||||
return tenant(async (client) => (await client.query(
|
||||
`SELECT table_name, column_name, data_type AS type, (is_nullable='NO') AS not_null
|
||||
FROM information_schema.columns WHERE table_schema=$1 ORDER BY table_name,ordinal_position`,
|
||||
[names.schemaName],
|
||||
)).rows);
|
||||
}
|
||||
|
||||
async function querySql(sql) {
|
||||
const cleaned = rejectSql(sql, { readonly: true }).replace(/;\s*$/, '');
|
||||
return tenant(async (client) => (await client.query(`SELECT * FROM (${cleaned}) AS q LIMIT ${maxRows}`)).rows);
|
||||
}
|
||||
|
||||
async function executeSql(sql) {
|
||||
const cleaned = translateSqliteAgentDdl(rejectSql(sql));
|
||||
return tenant(async (client) => {
|
||||
const result = await client.query(cleaned);
|
||||
return { backend: 'postgres', command: result.command, affectedRows: result.rowCount ?? 0, sizeBytes: null, deltaBytes: null, quotaSynced: false };
|
||||
}, { write: true });
|
||||
}
|
||||
|
||||
async function getDataset(name) {
|
||||
const datasetName = ident(name, 'dataset 名称');
|
||||
return tenant(async (client) => {
|
||||
const result = await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets WHERE name=$1 LIMIT 1', [datasetName]);
|
||||
if (!result.rows[0]) return null;
|
||||
return { ...normalizeDataset(result.rows[0].config_json, result.rows[0].name), createdAt: result.rows[0].created_at, updatedAt: result.rows[0].updated_at };
|
||||
});
|
||||
}
|
||||
|
||||
async function listDatasets() {
|
||||
return tenant(async (client) => (await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets ORDER BY name')).rows.map((row) => ({
|
||||
...normalizeDataset(row.config_json, row.name), createdAt: row.created_at, updatedAt: row.updated_at,
|
||||
})));
|
||||
}
|
||||
|
||||
async function upsertDataset(input) {
|
||||
const dataset = normalizeDataset(input);
|
||||
if (!(await listTableColumns(dataset.table)).length) throw Object.assign(new Error(`dataset 对应表不存在:${dataset.table}`), { code: 'table_not_found' });
|
||||
await tenant((client) => client.query(
|
||||
`INSERT INTO __page_data_datasets(name,table_name,config_json,updated_at) VALUES($1,$2,$3,CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(name) DO UPDATE SET table_name=EXCLUDED.table_name,config_json=EXCLUDED.config_json,updated_at=CURRENT_TIMESTAMP`,
|
||||
[dataset.name, dataset.table, JSON.stringify(dataset)],
|
||||
), { write: true });
|
||||
return getDataset(dataset.name);
|
||||
}
|
||||
|
||||
function assertAction(dataset, action) {
|
||||
if (!dataset.actions.includes(action)) throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' });
|
||||
}
|
||||
|
||||
async function readRowsForDataset(datasetInput, options = {}) {
|
||||
const dataset = await Promise.resolve(datasetInput);
|
||||
assertAction(dataset, 'read');
|
||||
const columns = dataset.columns.read?.map((item) => ident(item, '字段')) ?? [];
|
||||
if (!columns.length) throw Object.assign(new Error('dataset 未配置可读字段'), { code: 'columns_not_allowed' });
|
||||
const limit = Math.min(Math.max(1, Number(options.limit ?? dataset.limits.maxRowsPerRead)), maxRows, dataset.limits.maxRowsPerRead);
|
||||
const offset = Math.max(0, Number(options.offset ?? 0));
|
||||
const order = options.orderBy ? ident(options.orderBy, '排序字段') : columns.includes('id') ? 'id' : null;
|
||||
if (order && !columns.includes(order)) throw Object.assign(new Error('排序字段未授权'), { code: 'columns_not_allowed' });
|
||||
return tenant(async (client) => {
|
||||
const filters = [];
|
||||
if ((await listTableColumns(dataset.table)).some((column) => column.name === 'deleted_at') && !options.includeDeleted) filters.push('deleted_at IS NULL');
|
||||
if (options.rowScope?.whereClause) filters.push(`(${options.rowScope.whereClause})`);
|
||||
const where = filters.length ? ` WHERE ${filters.join(' AND ')}` : '';
|
||||
const result = await client.query(
|
||||
`SELECT ${columns.map(quotePgIdentifier).join(',')} FROM ${quotePgIdentifier(dataset.table)}${where}${order ? ` ORDER BY ${quotePgIdentifier(order)} ${String(options.orderDir).toLowerCase() === 'asc' ? 'ASC' : 'DESC'}` : ''} LIMIT $1 OFFSET $2`,
|
||||
[limit, offset],
|
||||
);
|
||||
return { dataset, rows: result.rows, limit, offset };
|
||||
});
|
||||
}
|
||||
|
||||
async function readDatasetRows(name, options) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return readRowsForDataset(dataset, options);
|
||||
}
|
||||
|
||||
async function getDatasetStats(name) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
assertAction(dataset, 'read');
|
||||
return tenant(async (client) => ({ dataset: { name: dataset.name, table: dataset.table }, total: Number((await client.query(`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(dataset.table)}`)).rows[0].count) }));
|
||||
}
|
||||
|
||||
async function insertWithDataset(dataset, payload, meta = {}) {
|
||||
assertAction(dataset, 'insert');
|
||||
const allowed = new Set(dataset.columns.insert ?? []);
|
||||
const keys = Object.keys(payload ?? {});
|
||||
if (keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('提交包含未授权字段'), { code: 'columns_not_allowed' });
|
||||
return tenant(async (client) => {
|
||||
const columns = await listTableColumns(dataset.table);
|
||||
const values = { ...payload };
|
||||
if (meta.rowScope?.ownerColumn && columns.some((c) => c.name === meta.rowScope.ownerColumn)) values[meta.rowScope.ownerColumn] ??= meta.rowScope.visitorUserId;
|
||||
const insertKeys = Object.keys(values).map((key) => ident(key, '字段'));
|
||||
const result = await client.query(
|
||||
`INSERT INTO ${quotePgIdentifier(dataset.table)} (${insertKeys.map(quotePgIdentifier).join(',')}) VALUES (${insertKeys.map((_, i) => `$${i + 1}`).join(',')}) RETURNING *`,
|
||||
insertKeys.map((key) => values[key]),
|
||||
);
|
||||
return { dataset, row: result.rows[0] };
|
||||
}, { write: true });
|
||||
}
|
||||
|
||||
async function insertDatasetRow(name, payload, meta = {}) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return insertWithDataset(dataset, payload, meta);
|
||||
}
|
||||
|
||||
async function insertRowForDataset(datasetInput, payload, meta = {}) {
|
||||
const dataset = await Promise.resolve(datasetInput);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return insertWithDataset(dataset, payload, meta);
|
||||
}
|
||||
|
||||
async function updateRowForDataset(datasetInput, rowId, payload, meta = {}) {
|
||||
const dataset = await Promise.resolve(datasetInput);
|
||||
assertAction(dataset, 'update');
|
||||
const id = Number(rowId);
|
||||
if (!Number.isFinite(id) || id <= 0) throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' });
|
||||
const allowed = new Set(dataset.columns.update ?? []);
|
||||
const keys = Object.keys(payload ?? {});
|
||||
if (!keys.length || keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('更新包含未授权字段'), { code: 'columns_not_allowed' });
|
||||
return tenant(async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE ${quotePgIdentifier(dataset.table)} SET ${keys.map((key, i) => `${quotePgIdentifier(ident(key, '字段'))}=$${i + 1}`).join(',')} WHERE id=$${keys.length + 1}${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING *`,
|
||||
[...keys.map((key) => payload[key]), id],
|
||||
);
|
||||
if (!result.rows[0]) throw Object.assign(new Error('无权更新该行或行不存在'), { code: 'row_not_allowed' });
|
||||
return { dataset, row: result.rows[0] };
|
||||
}, { write: true });
|
||||
}
|
||||
|
||||
async function updateDatasetRow(name, rowId, payload, meta) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return updateRowForDataset(dataset, rowId, payload, meta);
|
||||
}
|
||||
|
||||
async function softDeleteRowForDataset(datasetInput, rowId, meta = {}) {
|
||||
const dataset = await Promise.resolve(datasetInput);
|
||||
assertAction(dataset, 'soft_delete');
|
||||
const columns = await listTableColumns(dataset.table);
|
||||
if (!columns.some((item) => item.name === 'deleted_at')) throw Object.assign(new Error('目标表不支持软删除'), { code: 'soft_delete_unsupported' });
|
||||
const id = Number(rowId);
|
||||
return tenant(async (client) => {
|
||||
const hasDeletedBy = columns.some((item) => item.name === 'deleted_by');
|
||||
const result = await client.query(
|
||||
`UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=CURRENT_TIMESTAMP${hasDeletedBy ? ',deleted_by=$2' : ''} WHERE id=$1 AND deleted_at IS NULL${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING id,deleted_at`,
|
||||
hasDeletedBy ? [id, meta.deletedBy ?? meta.updatedByLabel ?? 'public'] : [id],
|
||||
);
|
||||
if (!result.rows[0]) throw Object.assign(new Error('无权删除该行或行不存在'), { code: 'row_not_allowed' });
|
||||
return { dataset, id, deleted: true };
|
||||
}, { write: true });
|
||||
}
|
||||
|
||||
async function softDeleteDatasetRow(name, rowId, meta) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return softDeleteRowForDataset(dataset, rowId, meta);
|
||||
}
|
||||
|
||||
async function restoreSoftDeletedRow(name, rowId) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
const columns = await listTableColumns(dataset.table);
|
||||
return tenant(async (client) => {
|
||||
const result = await client.query(
|
||||
`UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=NULL${columns.some((item) => item.name === 'deleted_by') ? ',deleted_by=NULL' : ''} WHERE id=$1 AND deleted_at IS NOT NULL RETURNING *`,
|
||||
[Number(rowId)],
|
||||
);
|
||||
if (!result.rows[0]) throw Object.assign(new Error('行不存在或未删除'), { code: 'row_not_allowed' });
|
||||
return { dataset, row: result.rows[0], restored: true };
|
||||
}, { write: true });
|
||||
}
|
||||
|
||||
async function exportDatasetRows(name, { format = 'json', includeDeleted = false, limit = 1000 } = {}) {
|
||||
const result = await readDatasetRows(name, { includeDeleted, limit: Math.min(Number(limit), 5000) });
|
||||
if (String(format).toLowerCase() !== 'csv') return { dataset: name, format: 'json', rows: result.rows, rowCount: result.rows.length };
|
||||
const columns = result.dataset.columns.read ?? [];
|
||||
const escape = (value) => {
|
||||
const text = value == null ? '' : String(value);
|
||||
return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
||||
};
|
||||
return { dataset: name, format: 'csv', content: `${columns.join(',')}\n${result.rows.map((row) => columns.map((column) => escape(row[column])).join(',')).join('\n')}\n`, rowCount: result.rows.length };
|
||||
}
|
||||
|
||||
async function getDatasetSchema(name) {
|
||||
const dataset = await getDataset(name);
|
||||
if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' });
|
||||
return { dataset, tableColumns: await listTableColumns(dataset.table) };
|
||||
}
|
||||
|
||||
async function getInfo() {
|
||||
return { name: '用户私有数据空间', backend: 'postgres', database: poolConfig(options).database, schema: names.schemaName, userId: names.userId };
|
||||
}
|
||||
|
||||
async function ensureReady() {
|
||||
await ensureProvisioned({ force: true });
|
||||
return getInfo();
|
||||
}
|
||||
|
||||
return {
|
||||
backend: 'postgres', workspaceRoot: options.workspaceRoot, privateDataDb: null,
|
||||
ensureReady, getInfo, getSchema, querySql, executeSql, listTableColumns, getDataset, listDatasets, upsertDataset,
|
||||
readRowsForDataset, readDatasetRows, getDatasetStats, getStatsForDataset: async (dataset) => getDatasetStats((await Promise.resolve(dataset)).name),
|
||||
getDatasetSchema, getSchemaForDataset: async (dataset) => ({ dataset: await Promise.resolve(dataset), tableColumns: await listTableColumns((await Promise.resolve(dataset)).table) }),
|
||||
insertDatasetRow, insertRowForDataset, updateDatasetRow, updateRowForDataset,
|
||||
softDeleteDatasetRow, softDeleteRowForDataset, restoreSoftDeletedRow, exportDatasetRows,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,11 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
applyMemindRuntimeProfile,
|
||||
describeMemindRuntimeProfile,
|
||||
loadMemindEnvFiles,
|
||||
} from './memind-runtime-profile.mjs';
|
||||
|
||||
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const opsDir = path.join(root, 'ops');
|
||||
@@ -23,22 +28,6 @@ const plazaPublicBase = (
|
||||
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
|
||||
).replace(/\/$/, '');
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq < 0) continue;
|
||||
const key = trimmed.slice(0, eq).trim();
|
||||
const value = trimmed.slice(eq + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '../../.env.local'));
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
|
||||
const logFile =
|
||||
process.env.MEMIND_DEV_LOG ?? path.join(os.homedir(), 'Library/Logs/memind-dev.log');
|
||||
|
||||
@@ -51,6 +40,10 @@ function log(message) {
|
||||
}
|
||||
}
|
||||
|
||||
loadMemindEnvFiles(root);
|
||||
applyMemindRuntimeProfile({ rootDir: root });
|
||||
log(`Runtime profile: ${describeMemindRuntimeProfile()}`);
|
||||
|
||||
function freePort(port) {
|
||||
try {
|
||||
execSync(`lsof -ti TCP:${port} -sTCP:LISTEN | xargs kill -9`, { stdio: 'ignore' });
|
||||
|
||||
+8
-15
@@ -3,6 +3,11 @@ import { spawn, execSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
applyMemindRuntimeProfile,
|
||||
describeMemindRuntimeProfile,
|
||||
loadMemindEnvFiles,
|
||||
} from './memind-runtime-profile.mjs';
|
||||
|
||||
const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const opsDir = path.join(root, 'ops');
|
||||
@@ -18,21 +23,9 @@ const plazaPublicBase = (
|
||||
process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}`
|
||||
).replace(/\/$/, '');
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq < 0) continue;
|
||||
const key = trimmed.slice(0, eq).trim();
|
||||
const value = trimmed.slice(eq + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '../../.env.local'));
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
loadMemindEnvFiles(root);
|
||||
applyMemindRuntimeProfile({ rootDir: root });
|
||||
console.log(`==> Runtime profile: ${describeMemindRuntimeProfile()}`);
|
||||
|
||||
function freePort(port) {
|
||||
try {
|
||||
|
||||
+7
-16
@@ -1,20 +1,11 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const separator = trimmed.indexOf('=');
|
||||
if (separator < 0) continue;
|
||||
const key = trimmed.slice(0, separator).trim();
|
||||
const value = trimmed.slice(separator + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
import {
|
||||
applyMemindRuntimeProfile,
|
||||
loadMemindEnvFiles,
|
||||
} from './memind-runtime-profile.mjs';
|
||||
|
||||
export function loadH5Environment(scriptDirectory) {
|
||||
loadEnvFile(path.join(scriptDirectory, '../../../.env.local'));
|
||||
loadEnvFile(path.join(scriptDirectory, '../.env'));
|
||||
const root = path.join(scriptDirectory, '..');
|
||||
loadMemindEnvFiles(root);
|
||||
applyMemindRuntimeProfile({ rootDir: root });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export const MEMIND_RUNTIME_PROFILES = new Set(['local', 'split-service', 'production']);
|
||||
|
||||
/**
|
||||
* Load Memind env files in standard order. Later files only fill unset keys.
|
||||
*/
|
||||
export function loadMemindEnvFiles(rootDir, env = process.env) {
|
||||
const root = path.resolve(rootDir);
|
||||
for (const relativePath of ['../../.env.local', '.env']) {
|
||||
loadEnvFile(path.join(root, relativePath), env);
|
||||
}
|
||||
}
|
||||
|
||||
function loadEnvFile(filePath, env = process.env) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const separator = trimmed.indexOf('=');
|
||||
if (separator < 0) continue;
|
||||
const key = trimmed.slice(0, separator).trim();
|
||||
const value = trimmed.slice(separator + 1).trim();
|
||||
if (!env[key]) env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function expandEnvReference(value, env = process.env) {
|
||||
const raw = String(value ?? '').trim();
|
||||
const match = raw.match(/^\$\{([A-Z0-9_]+)\}$/);
|
||||
if (!match) return raw;
|
||||
return String(env[match[1]] ?? '').trim() || raw;
|
||||
}
|
||||
|
||||
export function resolveMemindRuntimeProfile(env = process.env) {
|
||||
const explicit = String(env.MEMIND_RUNTIME_PROFILE ?? '').trim().toLowerCase();
|
||||
if (explicit) {
|
||||
if (!MEMIND_RUNTIME_PROFILES.has(explicit)) {
|
||||
throw new Error(
|
||||
`Unsupported MEMIND_RUNTIME_PROFILE "${explicit}". Expected: ${[...MEMIND_RUNTIME_PROFILES].join(', ')}`,
|
||||
);
|
||||
}
|
||||
return explicit;
|
||||
}
|
||||
if (String(env.NODE_ENV ?? '').trim().toLowerCase() === 'production') {
|
||||
return 'production';
|
||||
}
|
||||
return 'local';
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply environment-specific defaults after .env files are loaded.
|
||||
*
|
||||
* - local: monolith MindSpace adapter, repo-local storage (pnpm dev default)
|
||||
* - split-service: remote adapter against standalone MindSpace on 8082
|
||||
* - production: no overrides; 103 .env is source of truth
|
||||
*/
|
||||
export function applyMemindRuntimeProfile({
|
||||
rootDir = process.cwd(),
|
||||
env = process.env,
|
||||
profile,
|
||||
} = {}) {
|
||||
const resolvedProfile = profile ?? resolveMemindRuntimeProfile(env);
|
||||
env.MEMIND_RUNTIME_PROFILE = resolvedProfile;
|
||||
|
||||
if (resolvedProfile === 'local') {
|
||||
env.MINDSPACE_SERVER_ADAPTER = 'local';
|
||||
if (!env.MINDSPACE_STORAGE_ROOT) {
|
||||
env.MINDSPACE_STORAGE_ROOT = path.join(path.resolve(rootDir), 'data', 'mindspace');
|
||||
}
|
||||
return {
|
||||
profile: resolvedProfile,
|
||||
mindspaceAdapter: 'local',
|
||||
storageRoot: env.MINDSPACE_STORAGE_ROOT,
|
||||
};
|
||||
}
|
||||
|
||||
if (resolvedProfile === 'split-service') {
|
||||
env.MINDSPACE_SERVER_ADAPTER = 'remote';
|
||||
env.MINDSPACE_REMOTE_BASE_URL = String(
|
||||
env.MINDSPACE_REMOTE_BASE_URL ?? 'http://127.0.0.1:8082',
|
||||
).trim();
|
||||
const expandedToken = expandEnvReference(env.MINDSPACE_REMOTE_AUTH_TOKEN, env);
|
||||
if (expandedToken) {
|
||||
env.MINDSPACE_REMOTE_AUTH_TOKEN = expandedToken;
|
||||
} else if (env.TKMIND_SERVER__SECRET_KEY) {
|
||||
env.MINDSPACE_REMOTE_AUTH_TOKEN = String(env.TKMIND_SERVER__SECRET_KEY).trim();
|
||||
}
|
||||
if (!env.MINDSPACE_STORAGE_ROOT) {
|
||||
env.MINDSPACE_STORAGE_ROOT = '/Users/john/MindSpace/data/mindspace';
|
||||
}
|
||||
return {
|
||||
profile: resolvedProfile,
|
||||
mindspaceAdapter: 'remote',
|
||||
remoteBaseUrl: env.MINDSPACE_REMOTE_BASE_URL,
|
||||
storageRoot: env.MINDSPACE_STORAGE_ROOT,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
profile: resolvedProfile,
|
||||
mindspaceAdapter: String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local',
|
||||
storageRoot: env.MINDSPACE_STORAGE_ROOT ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function describeMemindRuntimeProfile(env = process.env) {
|
||||
const profile = resolveMemindRuntimeProfile(env);
|
||||
const adapter = String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local';
|
||||
const parts = [`MEMIND_RUNTIME_PROFILE=${profile}`, `MINDSPACE_SERVER_ADAPTER=${adapter}`];
|
||||
if (adapter === 'remote') {
|
||||
parts.push(`MINDSPACE_REMOTE_BASE_URL=${env.MINDSPACE_REMOTE_BASE_URL ?? '(unset)'}`);
|
||||
}
|
||||
if (env.MINDSPACE_STORAGE_ROOT) {
|
||||
parts.push(`MINDSPACE_STORAGE_ROOT=${env.MINDSPACE_STORAGE_ROOT}`);
|
||||
}
|
||||
return parts.join(', ');
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
applyMemindRuntimeProfile,
|
||||
resolveMemindRuntimeProfile,
|
||||
} from './memind-runtime-profile.mjs';
|
||||
|
||||
test('resolveMemindRuntimeProfile defaults to local for dev', () => {
|
||||
assert.equal(resolveMemindRuntimeProfile({}), 'local');
|
||||
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'development' }), 'local');
|
||||
});
|
||||
|
||||
test('resolveMemindRuntimeProfile honors explicit profile and production NODE_ENV', () => {
|
||||
assert.equal(resolveMemindRuntimeProfile({ MEMIND_RUNTIME_PROFILE: 'split-service' }), 'split-service');
|
||||
assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'production' }), 'production');
|
||||
});
|
||||
|
||||
test('applyMemindRuntimeProfile local forces monolith adapter and repo storage', () => {
|
||||
const env = {
|
||||
MINDSPACE_SERVER_ADAPTER: 'remote',
|
||||
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
|
||||
};
|
||||
const result = applyMemindRuntimeProfile({
|
||||
rootDir: '/tmp/memind',
|
||||
env,
|
||||
profile: 'local',
|
||||
});
|
||||
assert.equal(result.profile, 'local');
|
||||
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'local');
|
||||
assert.match(env.MINDSPACE_STORAGE_ROOT, /\/tmp\/memind\/data\/mindspace$/);
|
||||
});
|
||||
|
||||
test('applyMemindRuntimeProfile split-service expands auth token reference', () => {
|
||||
const env = {
|
||||
TKMIND_SERVER__SECRET_KEY: 'local-dev-secret',
|
||||
MINDSPACE_REMOTE_AUTH_TOKEN: '${TKMIND_SERVER__SECRET_KEY}',
|
||||
};
|
||||
applyMemindRuntimeProfile({ env, profile: 'split-service' });
|
||||
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
|
||||
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'local-dev-secret');
|
||||
});
|
||||
|
||||
test('applyMemindRuntimeProfile production does not override adapter', () => {
|
||||
const env = {
|
||||
MINDSPACE_SERVER_ADAPTER: 'remote',
|
||||
MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082',
|
||||
MINDSPACE_REMOTE_AUTH_TOKEN: 'prod-token',
|
||||
};
|
||||
const result = applyMemindRuntimeProfile({ env, profile: 'production' });
|
||||
assert.equal(result.profile, 'production');
|
||||
assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote');
|
||||
assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'prod-token');
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import {
|
||||
buildControlSchemaSql,
|
||||
createSqliteSnapshot,
|
||||
inspectSqliteDatabase,
|
||||
migrateSqliteSnapshot,
|
||||
provisionUserSpace,
|
||||
} from '../mindspace-userdata-postgres.mjs';
|
||||
|
||||
function usage() {
|
||||
return `
|
||||
Usage:
|
||||
node scripts/migrate-mindspace-sqlite-to-postgres.mjs --source <sqlite> --user-id <uuid> [--apply] [--replace-shadow]
|
||||
|
||||
Default is read-only dry-run. --apply requires MINDSPACE_USERDATA_PG_URL.
|
||||
The command creates a consistent SQLite snapshot and never modifies the source SQLite.
|
||||
`.trim();
|
||||
}
|
||||
|
||||
export function parseArgs(argv = []) {
|
||||
const options = { apply: false, replaceShadow: false, source: '', userId: '', help: false };
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--apply') options.apply = true;
|
||||
else if (arg === '--replace-shadow') options.replaceShadow = true;
|
||||
else if (arg === '--source') options.source = argv[++index] ?? '';
|
||||
else if (arg === '--user-id') options.userId = argv[++index] ?? '';
|
||||
else if (arg === '--help' || arg === '-h') options.help = true;
|
||||
else throw new Error(`未知参数:${arg}`);
|
||||
}
|
||||
if (!options.help && (!options.source || !options.userId)) throw new Error('--source 和 --user-id 必填');
|
||||
return options;
|
||||
}
|
||||
|
||||
export async function run(argv = process.argv.slice(2), env = process.env) {
|
||||
const options = parseArgs(argv);
|
||||
if (options.help) {
|
||||
console.log(usage());
|
||||
return { mode: 'help' };
|
||||
}
|
||||
const inspected = inspectSqliteDatabase(options.source);
|
||||
const sourceRows = inspected.tables.reduce((total, table) => total + table.rows.length, 0);
|
||||
if (!options.apply) {
|
||||
console.log(JSON.stringify({
|
||||
mode: 'dry-run',
|
||||
userId: options.userId,
|
||||
source: inspected.path,
|
||||
sizeBytes: inspected.sizeBytes,
|
||||
tables: inspected.tables.map((table) => ({ name: table.name, rows: table.rows.length })),
|
||||
sourceRows,
|
||||
note: '未修改 SQLite 或 PostgreSQL;使用 --apply 才会创建影子空间。',
|
||||
}, null, 2));
|
||||
return { mode: 'dry-run', tableCount: inspected.tables.length, sourceRows };
|
||||
}
|
||||
if (!env.MINDSPACE_USERDATA_PG_URL) throw new Error('--apply requires MINDSPACE_USERDATA_PG_URL');
|
||||
const snapshotPath = createSqliteSnapshot(options.source);
|
||||
const snapshot = inspectSqliteDatabase(snapshotPath);
|
||||
const { default: pg } = await import('pg');
|
||||
const client = new pg.Client({ connectionString: env.MINDSPACE_USERDATA_PG_URL });
|
||||
try {
|
||||
await client.connect();
|
||||
await client.query(buildControlSchemaSql());
|
||||
await provisionUserSpace(client, options.userId, {
|
||||
sourceSqlitePath: inspected.path,
|
||||
});
|
||||
const result = await migrateSqliteSnapshot(client, snapshot, options.userId, {
|
||||
replaceShadow: options.replaceShadow,
|
||||
sourcePath: inspected.path,
|
||||
});
|
||||
if (result.sourceRows !== result.targetRows) throw new Error('迁移行数校验失败');
|
||||
console.log(JSON.stringify({ mode: 'shadow', ...result }, null, 2));
|
||||
return { mode: 'shadow', ...result };
|
||||
} finally {
|
||||
await client.end().catch(() => {});
|
||||
fs.rmSync(snapshotPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
run().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { parseArgs } from './migrate-mindspace-sqlite-to-postgres.mjs';
|
||||
|
||||
test('migration CLI defaults to dry-run', () => {
|
||||
assert.deepEqual(parseArgs(['--source', '/tmp/a.sqlite', '--user-id', 'user-id']), {
|
||||
apply: false,
|
||||
replaceShadow: false,
|
||||
source: '/tmp/a.sqlite',
|
||||
userId: 'user-id',
|
||||
help: false,
|
||||
});
|
||||
});
|
||||
|
||||
test('migration CLI requires explicit source and user', () => {
|
||||
assert.throws(() => parseArgs([]), /--source 和 --user-id 必填/);
|
||||
});
|
||||
|
||||
test('migration CLI recognizes explicit apply', () => {
|
||||
assert.equal(parseArgs(['--apply', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).apply, true);
|
||||
});
|
||||
|
||||
test('migration CLI requires explicit replace-shadow for destructive reruns', () => {
|
||||
assert.equal(parseArgs(['--replace-shadow', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).replaceShadow, true);
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Local-only E2E proof for the questionnaire -> Page Data API -> PostgreSQL path.
|
||||
*
|
||||
* The probe uses an existing local user space, creates a uniquely named survey
|
||||
* table/dataset and page policy, submits one answer through the public HTTP API,
|
||||
* verifies the row directly in PostgreSQL, and removes all probe objects.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import express from 'express';
|
||||
import pg from 'pg';
|
||||
import { attachPageDataRoutes } from '../page-data-routes.mjs';
|
||||
import { createPageDataService } from '../page-data-service.mjs';
|
||||
import { createPageDataPublicService } from '../page-data-public-service.mjs';
|
||||
import { deletePageAccessPolicy, writePageAccessPolicy } from '../page-data-policy-store.mjs';
|
||||
import { createUserDataSpaceService } from '../user-data-space-service.mjs';
|
||||
import { deriveUserSpaceNames, quotePgIdentifier } from '../mindspace-userdata-postgres.mjs';
|
||||
import { loadH5Environment } from './load-env.mjs';
|
||||
|
||||
loadH5Environment(import.meta.dirname);
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const userId = process.argv[2] ?? '1c99b83b-0454-474f-a5d2-129d34506a32';
|
||||
const workspaceRoot = path.join(repoRoot, 'MindSpace', userId);
|
||||
const suffix = `${Date.now()}_${process.pid}`;
|
||||
const tableName = `survey_e2e_${suffix}`;
|
||||
const datasetName = tableName;
|
||||
const pageId = `page-survey-e2e-${suffix}`;
|
||||
const names = deriveUserSpaceNames(userId);
|
||||
const policyPath = path.join(workspaceRoot, '.mindspace', 'page-data-policies', `${pageId}.json`);
|
||||
const logPath = path.join(workspaceRoot, '.mindspace', 'page-data-logs', `${pageId}.jsonl`);
|
||||
|
||||
assert.equal(process.env.MINDSPACE_USERDATA_BACKEND, 'postgres', '本机未启用 PostgreSQL 用户数据后端');
|
||||
assert.ok(fs.existsSync(workspaceRoot), `用户工作区不存在: ${workspaceRoot}`);
|
||||
|
||||
const pgPool = new pg.Pool({
|
||||
host: process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp',
|
||||
port: Number(process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433),
|
||||
database: process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev',
|
||||
user: process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER,
|
||||
max: 2,
|
||||
});
|
||||
|
||||
function publicationPool() {
|
||||
return {
|
||||
async query(sql) {
|
||||
if (sql.includes('FROM h5_publish_records')) {
|
||||
return [[{
|
||||
id: `publication-${suffix}`,
|
||||
user_id: userId,
|
||||
page_id: pageId,
|
||||
access_mode: 'public',
|
||||
password_hash: null,
|
||||
status: 'online',
|
||||
expires_at: null,
|
||||
}]];
|
||||
}
|
||||
return [[]];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildApp() {
|
||||
const fakePublicationPool = publicationPool();
|
||||
const publicService = createPageDataPublicService({
|
||||
getPool: () => fakePublicationPool,
|
||||
resolveWorkspaceRootForOwner: () => workspaceRoot,
|
||||
});
|
||||
const api = express.Router();
|
||||
api.use(express.json());
|
||||
attachPageDataRoutes(api, {
|
||||
sendData: (res, _req, data, status = 200) => res.status(status).json({ data }),
|
||||
sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }),
|
||||
getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }),
|
||||
getPageDataPublicService: () => publicService,
|
||||
});
|
||||
const app = express();
|
||||
app.set('trust proxy', true);
|
||||
app.use('/api', api);
|
||||
return app;
|
||||
}
|
||||
|
||||
async function submitQuestionnaire(app, row) {
|
||||
const server = app.listen(0, '127.0.0.1');
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('listening', resolve);
|
||||
server.once('error', reject);
|
||||
});
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const apiPath = `/api/public/pages/${pageId}/data/${datasetName}/rows`;
|
||||
const response = await fetch(`http://127.0.0.1:${port}${apiPath}`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(row),
|
||||
});
|
||||
return { apiPath, status: response.status, body: await response.json() };
|
||||
} finally {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const ownerService = createUserDataSpaceService({ workspaceRoot, userId });
|
||||
let tableCreated = false;
|
||||
try {
|
||||
// Equivalent to the Agent's private_data_execute + register_dataset + bind policy flow.
|
||||
await ownerService.executeSql(`CREATE TABLE ${tableName} (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
satisfaction TEXT NOT NULL,
|
||||
favorite_feature TEXT NOT NULL,
|
||||
suggestion TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now', 'localtime'))
|
||||
)`);
|
||||
tableCreated = true;
|
||||
const dataset = await ownerService.upsertDataset({
|
||||
name: datasetName,
|
||||
table: tableName,
|
||||
description: '本机 PG 问卷端到端验证',
|
||||
actions: ['read', 'insert'],
|
||||
columns: {
|
||||
read: ['id', 'satisfaction', 'favorite_feature', 'suggestion', 'created_at'],
|
||||
insert: ['satisfaction', 'favorite_feature', 'suggestion'],
|
||||
},
|
||||
});
|
||||
writePageAccessPolicy(workspaceRoot, {
|
||||
pageId,
|
||||
ownerUserId: userId,
|
||||
accessMode: 'public',
|
||||
datasets: {
|
||||
[datasetName]: {
|
||||
insert: true,
|
||||
read: false,
|
||||
columns: { insert: ['satisfaction', 'favorite_feature', 'suggestion'] },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const answer = {
|
||||
satisfaction: '非常满意',
|
||||
favorite_feature: '个人数据报表',
|
||||
suggestion: '增加按月趋势分析',
|
||||
};
|
||||
const submitted = await submitQuestionnaire(buildApp(), answer);
|
||||
assert.equal(submitted.status, 201, JSON.stringify(submitted.body));
|
||||
assert.equal(submitted.body.data.dataset, datasetName);
|
||||
assert.equal(submitted.body.data.row.satisfaction, answer.satisfaction);
|
||||
|
||||
const direct = await pgPool.query(
|
||||
`SELECT id,satisfaction,favorite_feature,suggestion,created_at
|
||||
FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(tableName)}
|
||||
ORDER BY id DESC LIMIT 1`,
|
||||
);
|
||||
assert.equal(direct.rowCount, 1);
|
||||
assert.equal(direct.rows[0].satisfaction, answer.satisfaction);
|
||||
assert.equal(direct.rows[0].favorite_feature, answer.favorite_feature);
|
||||
assert.equal(direct.rows[0].suggestion, answer.suggestion);
|
||||
|
||||
const registered = await pgPool.query(
|
||||
`SELECT name,table_name FROM ${quotePgIdentifier(names.schemaName)}.__page_data_datasets WHERE name=$1`,
|
||||
[datasetName],
|
||||
);
|
||||
assert.equal(registered.rowCount, 1);
|
||||
|
||||
console.log(JSON.stringify({
|
||||
result: 'PASS',
|
||||
localOnly: true,
|
||||
userId,
|
||||
schema: names.schemaName,
|
||||
dataset: { name: dataset.name, table: dataset.table, actions: dataset.actions },
|
||||
pageDataApi: { method: 'POST', path: submitted.apiPath, status: submitted.status },
|
||||
postgresRow: direct.rows[0],
|
||||
checks: [
|
||||
'Agent-style table creation succeeded in the user schema',
|
||||
'Dataset registry was stored in PostgreSQL',
|
||||
'Questionnaire submission used the public Page Data HTTP API',
|
||||
'Submitted answers matched a direct PostgreSQL query',
|
||||
],
|
||||
}, null, 2));
|
||||
} finally {
|
||||
if (tableCreated) {
|
||||
await ownerService.executeSql(`DELETE FROM __page_data_datasets WHERE name='${datasetName}'`).catch(() => {});
|
||||
await ownerService.executeSql(`DROP TABLE IF EXISTS ${tableName}`).catch(() => {});
|
||||
}
|
||||
deletePageAccessPolicy(workspaceRoot, pageId);
|
||||
fs.rmSync(logPath, { force: true });
|
||||
fs.rmSync(policyPath, { force: true });
|
||||
await pgPool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
+58
-4
@@ -14,6 +14,7 @@ import {
|
||||
} from './auth.mjs';
|
||||
import { createDbPool, initSchema, isDatabaseConfigured } from './db.mjs';
|
||||
import { createWorkspacePageDeliverService } from './mindspace-workspace-page-deliver.mjs';
|
||||
import { createUserDataSpaceService } from './user-data-space-service.mjs';
|
||||
import { createToolGateway } from './tool-gateway.mjs';
|
||||
import { createAgentRunGateway } from './agent-run-gateway.mjs';
|
||||
import {
|
||||
@@ -164,6 +165,7 @@ import {
|
||||
registerChatDocxArtifactForConversation,
|
||||
} from './mindspace-chat-docx-package.mjs';
|
||||
import { scanContent } from './mindspace-content-scan.mjs';
|
||||
import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs';
|
||||
import { renderImageAssetViewerHtml, wantsInlineImageViewer } from './mindspace-asset-preview.mjs';
|
||||
import { DEFAULT_IMAGE_UPLOAD_MAX_BYTES } from './user-image-normalize.mjs';
|
||||
import { createRechargeService } from './billing-recharge.mjs';
|
||||
@@ -535,6 +537,10 @@ async function bootstrapUserAuth() {
|
||||
h5Root: __dirname,
|
||||
defaultSignupBalanceCents: Number(process.env.H5_SIGNUP_BALANCE_CENTS ?? 500),
|
||||
subscriptionService,
|
||||
provisionUserDataSpace: async ({ userId, workspaceRoot }) => {
|
||||
const service = createUserDataSpaceService({ workspaceRoot, userId, query: pool });
|
||||
return service.ensureReady();
|
||||
},
|
||||
});
|
||||
sessionAccess = createSessionAccess({ userAuth, enabled: isSessionBrokerEnabled() });
|
||||
if (sessionAccess.enabled) {
|
||||
@@ -622,6 +628,12 @@ async function bootstrapUserAuth() {
|
||||
expireStaleUploadsIntervalMs: 5 * 60 * 1000,
|
||||
});
|
||||
await userAuth.ensureAdminUser();
|
||||
const userDataSpaceBackfill = await userAuth.ensureAllUserDataSpaces();
|
||||
if (userDataSpaceBackfill.errors.length > 0) {
|
||||
console.warn(
|
||||
`[PageData] PG space backfill incomplete: ${userDataSpaceBackfill.errors.length} user(s) failed`,
|
||||
);
|
||||
}
|
||||
llmProviderService = createLlmProviderService(pool, {
|
||||
apiTarget: API_TARGET,
|
||||
apiTargets: API_TARGETS,
|
||||
@@ -717,8 +729,24 @@ async function bootstrapUserAuth() {
|
||||
messages: [userMessage],
|
||||
});
|
||||
},
|
||||
syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId),
|
||||
syncUserPagesOnSuccess: async ({ userId, sessionId, runStartedAtMs }) =>
|
||||
syncUserGeneratedPages(userId, { sessionId, sinceMs: runStartedAtMs }),
|
||||
isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0,
|
||||
validateRunDeliverables: async ({ userId, deliverables }) => {
|
||||
const publishDir = resolveMindSpaceUserPublishDir(__dirname, { id: userId });
|
||||
const violations = scanWorkspaceFilesForProhibitedBrowserStorage({
|
||||
publishDir,
|
||||
relativePaths: (deliverables?.pages ?? [])
|
||||
.map((page) => page.workspaceRelativePath)
|
||||
.filter(Boolean),
|
||||
});
|
||||
return {
|
||||
errors: violations.map((violation) => ({
|
||||
code: 'browser_storage_forbidden',
|
||||
message: `${violation.relativePath} 使用 ${violation.apis.join(', ')}`,
|
||||
})),
|
||||
};
|
||||
},
|
||||
autoDispatch: ['1', 'true', 'yes', 'on'].includes(
|
||||
String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(),
|
||||
),
|
||||
@@ -3231,10 +3259,36 @@ async function resolveExistingSavedPage(userId, { sessionId, messageId, relative
|
||||
const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']);
|
||||
|
||||
// REGRESSION GUARD: mindspace-page-sync-thumbnail — remote 也经 pageSyncService RPC 同步 public HTML
|
||||
async function syncUserGeneratedPages(userId) {
|
||||
async function listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs = null } = {}) {
|
||||
if (!authPool || !userId || !sessionId) return [];
|
||||
const sinceClause = sinceMs == null ? '' : 'AND ca.created_at >= ?';
|
||||
const params = sinceMs == null ? [userId, sessionId] : [userId, sessionId, sinceMs];
|
||||
const [rows] = await authPool.query(
|
||||
`SELECT ca.display_name
|
||||
FROM h5_conversation_artifacts ca
|
||||
JOIN h5_conversation_packages cp ON cp.id = ca.package_id
|
||||
WHERE cp.user_id = ?
|
||||
AND cp.session_id = ?
|
||||
AND ca.artifact_kind = 'public_html'
|
||||
${sinceClause}
|
||||
ORDER BY ca.sort_order ASC, ca.created_at ASC`,
|
||||
params,
|
||||
);
|
||||
return [...new Set((rows ?? [])
|
||||
.map((row) => normalizeWorkspaceRelativePath(`public/${String(row.display_name ?? '').trim()}`))
|
||||
.filter((relativePath) => relativePath?.startsWith('public/') && relativePath.toLowerCase().endsWith('.html')))];
|
||||
}
|
||||
|
||||
async function syncUserGeneratedPages(userId, { sessionId = null, sinceMs = null } = {}) {
|
||||
if (!userId) return;
|
||||
// Agent-run/Finish delivery must stay scoped to the current conversation.
|
||||
// A stale Page Data page elsewhere in the user's workspace must not turn a
|
||||
// successfully completed current task into a failed run.
|
||||
const pageDataRelativePaths = sessionId
|
||||
? await listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs })
|
||||
: null;
|
||||
if (workspacePageDeliver?.syncAndDeliver) {
|
||||
return await workspacePageDeliver.syncAndDeliver(userId);
|
||||
return await workspacePageDeliver.syncAndDeliver(userId, { pageDataRelativePaths });
|
||||
}
|
||||
if (!mindSpacePageSync) return;
|
||||
return await mindSpacePageSync.syncUserGeneratedPages(userId);
|
||||
@@ -5099,7 +5153,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: '';
|
||||
await syncUserGeneratedPages(uid);
|
||||
await syncUserGeneratedPages(uid, { sessionId: sid });
|
||||
await maybeRepairPageDataAfterFinish({
|
||||
sessionId: sid,
|
||||
userId: uid,
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
---
|
||||
name: page-data-collect
|
||||
description: 在 MindSpace 页面中收集、存储并管理结构化数据(问卷、报名、台账、后台查看),基于 Page Data API 与用户私有 SQLite
|
||||
description: 在 MindSpace 页面中收集、存储并管理结构化数据(问卷、报名、台账、后台查看),基于 Page Data API 与用户隔离的 PostgreSQL 空间
|
||||
---
|
||||
|
||||
# 页面数据收集(Page Data Collect)
|
||||
|
||||
在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互,并将提交持久化到当前用户的 `.mindspace/private-data.sqlite`,通过平台 **Page Data API** 受控读写。
|
||||
在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互,并将提交持久化到当前用户隔离的 PostgreSQL schema,通过平台 **Page Data API** 受控读写。运行时禁止创建、读取或回退到用户 SQLite。
|
||||
|
||||
详细 API 说明见工作区外文档 `docs/page-data-api-usage.md`(Memind 仓库)。
|
||||
|
||||
@@ -25,7 +25,7 @@ description: 在 MindSpace 页面中收集、存储并管理结构化数据(
|
||||
```text
|
||||
Agent 可以建模 SQL 并注册 dataset;
|
||||
HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
禁止自建 Express / 独立端口 / 直接暴露 SQLite 文件路径。
|
||||
禁止自建 Express / 独立端口 / 直接暴露数据库连接;禁止 SQLite / localStorage 数据回退。
|
||||
```
|
||||
|
||||
## 方案选择:默认快车道 + 能力分支(必做)
|
||||
@@ -36,7 +36,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
用户描述需求
|
||||
→ 匹配能力分支(下表 A/B/C/D,默认 A)
|
||||
→ 仅冲突或无法匹配时,用 1 题让用户选分支或改口令
|
||||
→ 输出「方案摘要」供确认(用户可只改口令/选项)
|
||||
→ 输出简短「方案摘要」;用户已明确要求直接创建/完成/发布时视为已确认并立即执行
|
||||
→ 再 load_skill / 建表 / write_file / bind
|
||||
```
|
||||
|
||||
@@ -50,7 +50,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
| 提交后修改 | 不支持(一次性) |
|
||||
| 页面 | `public/*-survey.html` + `public/*-admin.html`(两文件各 bind 一次) |
|
||||
|
||||
用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A,先展示摘要再开工**。
|
||||
用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A**。若用户已说「直接做」「完整创建并发布」「不用询问」「持续推进」等终态指令,摘要后必须在同一轮立即调用工具开工,禁止停下来等待再次确认。
|
||||
|
||||
口令规则:
|
||||
|
||||
@@ -83,6 +83,9 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
```text
|
||||
- public:可匿名 insert;服务端禁止 update / softDelete
|
||||
- password:所有 API 须先 authenticate;口令 ≥8 位且须写入发布记录
|
||||
- `public` 策略中的 `insert: true` 等价于任何访客都能直接调用 API 写入;前端口令、隐藏按钮和 JavaScript 判断都不是权限控制
|
||||
- password 页面必须调用 `client.authenticate(password)` 获取服务端 token;禁止在 HTML 中硬编码口令或只做 `password === "..."` 比较
|
||||
- 同一公开页若要求「所有人可评价、只有所有者可写正文」:正文 dataset 在公开页必须 `insert: false`,评价 dataset 才能 `insert: true`;所有者写正文必须使用独立 `password` 作者页或 `login_required` 页面
|
||||
- 同页不能同时「匿名 insert」+「口令 read 全量」→ 须拆前台 + 后台(方案 A/E)
|
||||
- password 无法区分访客身份;「只改自己的」须 login_required + own_rows(方案 B)
|
||||
- 匿名提交后「凭链接改自己的」无内置能力,须改 B 或接受一次性提交
|
||||
@@ -101,7 +104,7 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
- 页面:public/xxx.html + public/xxx-admin.html
|
||||
- 内容:(由你的描述生成题目/字段/报表)
|
||||
|
||||
确认后开始建表与写页面。若需登录提交或团队共改,请说明,我换成 B/C 分支。
|
||||
若用户尚未授权创建,确认后开始建表与写页面;若用户已明确要求创建/完成/发布,此摘要仅用于告知,后面必须紧接工具调用,不得结束回复等待确认。
|
||||
```
|
||||
|
||||
### 分支切换示例
|
||||
@@ -121,8 +124,8 @@ HTML 页面只能调用 Page Data API(page-data-client.js);
|
||||
|
||||
```text
|
||||
load_skill → page-data-collect
|
||||
→ 匹配分支(默认 A)→ 方案摘要 → 用户确认或仅改口令/选项
|
||||
→ 禁止未确认就 bind;禁止空转计划不调用工具
|
||||
→ 匹配分支(默认 A)→ 方案摘要 → 用户确认,或从明确的创建/完成/发布指令判定已确认
|
||||
→ 禁止未获创建授权就 bind;已获授权时禁止空转计划、重复确认或不调用工具
|
||||
```
|
||||
|
||||
### 2. 数据层:建表 + 注册 dataset
|
||||
@@ -262,7 +265,7 @@ CREATE TABLE IF NOT EXISTS survey_responses (
|
||||
|
||||
## 严格禁止
|
||||
|
||||
1. **禁止**在方案摘要未确认时建表 / bind / 发布(默认 A 也须展示摘要;用户明确「按默认做」视为确认)
|
||||
1. **禁止**在尚未获得创建授权时建表 / bind / 发布;用户明确要求「创建、完成、发布、直接做、不用询问、持续推进」均视为确认,禁止再次停下来询问
|
||||
2. **禁止**让 LLM 自行发明 accessMode/拆页/口令;必须落在分支 A~E 与默认口令规则内
|
||||
3. **禁止**连续两轮仅输出计划、不调用 `load_skill` / `list_dir` / `write_file` / `private_data_execute` / `bind`
|
||||
4. **禁止**接受 <8 位发布口令;用户未指定口令时后台默认 **`88888888`**
|
||||
@@ -274,7 +277,7 @@ CREATE TABLE IF NOT EXISTS survey_responses (
|
||||
10. **禁止**只 `CREATE TABLE` 而不 `private_data_register_dataset`
|
||||
11. **禁止**未配置 `private_data_set_page_policy` 就让页面调用公开 API
|
||||
12. **禁止**先发布占位页(如 `<p>问卷页面</p>`)再让用户访问 `/u/.../pages/...`
|
||||
13. **禁止**在 Page Data 页面中使用 `localStorage` / `sessionStorage` 存提交记录或做 API fallback
|
||||
13. **禁止**在 Agent 生成的 HTML/JavaScript 中使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据或做 API fallback;所有需持久化的数据必须进入当前用户专属 PostgreSQL schema,禁止 SQLite、静态 JSON/JS 文件和内存 fallback
|
||||
|
||||
## 交付前自检
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报
|
||||
10. **禁止**在 HTML 中用 CDN `https://...` 引用 `<script src>`;MindSpace 发布页 CSP 仅允许同源脚本。常用库优先用平台路径(如 Chart.js → `/assets/chart.umd.min.js`),或下载到 `public/assets/` 后相对引用
|
||||
11. **纯静态页(无 Page Data 问卷/后台)禁止调用 `private_data_bind_workspace_page`**;`write_file` / `edit_file` 落盘 `public/*.html` 后,直接按下方「回复格式」交付 **MindSpace 路径**(`/MindSpace/<用户ID>/public/...`),无需 bind、无需口令
|
||||
12. **微信分享必须使用 MindSpace 路径**:该路由会在服务端注入 `og:site_name=TKMind 智趣`、封面缩略图(`*.thumbnail.png`)与微信 JS-SDK 分享桥;禁止交付 `/u/用户名/pages/...` 作为用户可见链接
|
||||
13. **只要页面接收用户输入并要求以后查看、汇总、统计、修改或删除,就不是纯静态页**。禁止使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据;必须立即 `load_skill` → `page-data-collect`,通过 Page Data API 写入当前用户专属 PostgreSQL schema。禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化。记账、日记、打卡、清单、台账、问卷、报名及其管理页都适用。
|
||||
|
||||
详细约束以工作区内的 `.goosehints` 与 `.agents/skills/static-page-publish/SKILL.md` 为准。
|
||||
|
||||
|
||||
@@ -138,6 +138,9 @@ export function createUserAuth(pool, options = {}) {
|
||||
let rechargeNotifier =
|
||||
typeof options.onRechargeNotification === 'function' ? options.onRechargeNotification : null;
|
||||
const subscriptionService = options.subscriptionService ?? null;
|
||||
const provisionUserDataSpace = typeof options.provisionUserDataSpace === 'function'
|
||||
? options.provisionUserDataSpace
|
||||
: null;
|
||||
const sessions = new Map();
|
||||
const loginFailures = new Map();
|
||||
|
||||
@@ -277,6 +280,14 @@ export function createUserAuth(pool, options = {}) {
|
||||
};
|
||||
};
|
||||
|
||||
const ensureUserDataSpaceForUser = async (user, workspaceRoot = null) => {
|
||||
if (!provisionUserDataSpace || !user?.id || user.role === 'admin') return null;
|
||||
return provisionUserDataSpace({
|
||||
userId: user.id,
|
||||
workspaceRoot: workspaceRoot ?? user.workspaceRoot ?? user.workspace_root,
|
||||
});
|
||||
};
|
||||
|
||||
const listSkillGrants = async (subjectType, subjectId) => {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT skill_name, enabled
|
||||
@@ -463,6 +474,7 @@ export function createUserAuth(pool, options = {}) {
|
||||
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
|
||||
now,
|
||||
});
|
||||
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, workspaceRoot);
|
||||
await conn.commit();
|
||||
ensureWorkspace(workspaceRoot);
|
||||
ensureUserMemoryProfile(workspaceRoot, {
|
||||
@@ -697,6 +709,28 @@ export function createUserAuth(pool, options = {}) {
|
||||
}
|
||||
};
|
||||
|
||||
const ensureAllUserDataSpaces = async () => {
|
||||
if (!provisionUserDataSpace) return { provisioned: 0, errors: [] };
|
||||
const [rows] = await pool.query(
|
||||
`SELECT id, workspace_root FROM h5_users WHERE role = 'user' AND status = 'active'`,
|
||||
);
|
||||
let provisioned = 0;
|
||||
const errors = [];
|
||||
for (const row of rows) {
|
||||
try {
|
||||
await ensureUserDataSpaceForUser({ id: row.id, role: 'user' }, row.workspace_root);
|
||||
provisioned += 1;
|
||||
} catch (error) {
|
||||
errors.push({
|
||||
userId: row.id,
|
||||
code: error?.code ?? null,
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return { provisioned, errors };
|
||||
};
|
||||
|
||||
const seedRoleSkillDefaults = async () => {
|
||||
const now = Date.now();
|
||||
for (const [name, enabled] of Object.entries(DEFAULT_USER_SKILLS)) {
|
||||
@@ -941,6 +975,7 @@ export function createUserAuth(pool, options = {}) {
|
||||
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
|
||||
now,
|
||||
});
|
||||
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, root);
|
||||
}
|
||||
await conn.commit();
|
||||
ensureWorkspace(root);
|
||||
@@ -2634,6 +2669,7 @@ export function createUserAuth(pool, options = {}) {
|
||||
quotaBytes: Number(process.env.MINDSPACE_FREE_QUOTA_BYTES ?? 5 * 1024 * 1024),
|
||||
now,
|
||||
});
|
||||
await ensureUserDataSpaceForUser({ id: userId, role: 'user' }, workspaceRoot);
|
||||
await conn.commit();
|
||||
ensureWorkspace(workspaceRoot);
|
||||
ensureUserMemoryProfile(workspaceRoot, {
|
||||
@@ -2940,6 +2976,7 @@ export function createUserAuth(pool, options = {}) {
|
||||
getUserPublishLayout,
|
||||
isPathAllowed,
|
||||
repairAllUserPublishDirs,
|
||||
ensureAllUserDataSpaces,
|
||||
registerAgentSession,
|
||||
getSessionNode,
|
||||
getSessionTarget,
|
||||
|
||||
@@ -7,6 +7,32 @@ import test from 'node:test';
|
||||
import { Algorithm as Argon2Algorithm, hashRawSync as argon2HashRawSync } from '@node-rs/argon2';
|
||||
import { createUserAuth, clearUserLogoutCookies, userLoginCookies } from './user-auth.mjs';
|
||||
|
||||
test('ensureAllUserDataSpaces provisions every active non-admin user', async () => {
|
||||
const calls = [];
|
||||
const pool = {
|
||||
async query(sql) {
|
||||
if (sql.includes("WHERE role = 'user' AND status = 'active'")) {
|
||||
return [[
|
||||
{ id: 'user-1', workspace_root: '/tmp/user-1' },
|
||||
{ id: 'user-2', workspace_root: '/tmp/user-2' },
|
||||
]];
|
||||
}
|
||||
return [[]];
|
||||
},
|
||||
};
|
||||
const auth = createUserAuth(pool, {
|
||||
persistSessions: false,
|
||||
provisionUserDataSpace: async (input) => calls.push(input),
|
||||
});
|
||||
const result = await auth.ensureAllUserDataSpaces();
|
||||
assert.equal(result.provisioned, 2);
|
||||
assert.deepEqual(result.errors, []);
|
||||
assert.deepEqual(calls, [
|
||||
{ userId: 'user-1', workspaceRoot: '/tmp/user-1' },
|
||||
{ userId: 'user-2', workspaceRoot: '/tmp/user-2' },
|
||||
]);
|
||||
});
|
||||
|
||||
function hashPasswordPbkdf2(password, salt) {
|
||||
return crypto.pbkdf2Sync(password, salt, 100000, 64, 'sha512').toString('hex');
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createPostgresUserDataSpaceService } from './postgres-user-data-space-service.mjs';
|
||||
|
||||
export const DATASET_REGISTRY_TABLE = '__page_data_datasets';
|
||||
|
||||
@@ -92,6 +93,23 @@ export function createUserDataSpaceService(options = {}) {
|
||||
}
|
||||
|
||||
const userId = options.userId ? String(options.userId).trim() : null;
|
||||
const isNodeTest = Boolean(process.env.NODE_TEST_CONTEXT);
|
||||
const backend = String(
|
||||
options.backend ?? process.env.MINDSPACE_USERDATA_BACKEND ?? (isNodeTest ? 'sqlite' : 'postgres'),
|
||||
).trim().toLowerCase();
|
||||
if (backend === 'sqlite' && !isNodeTest) {
|
||||
throw Object.assign(
|
||||
new Error('用户私有数据运行时已禁用 SQLite,只允许使用 PostgreSQL'),
|
||||
{ code: 'sqlite_runtime_disabled' },
|
||||
);
|
||||
}
|
||||
if (backend !== 'postgres' && backend !== 'sqlite') {
|
||||
throw Object.assign(new Error(`不支持的用户数据后端:${backend}`), { code: 'invalid_userdata_backend' });
|
||||
}
|
||||
if (backend === 'postgres') {
|
||||
if (!userId) throw new Error('PostgreSQL 用户数据空间要求 userId');
|
||||
return createPostgresUserDataSpaceService({ ...options, userId });
|
||||
}
|
||||
const query = typeof options.query === 'function' ? options.query : null;
|
||||
const sqliteBin = options.sqliteBin?.trim() || process.env.SQLITE_BIN?.trim() || 'sqlite3';
|
||||
const maxBytes = Number(options.maxBytes ?? process.env.PRIVATE_DATA_MAX_BYTES ?? DEFAULT_MAX_BYTES);
|
||||
@@ -111,6 +129,11 @@ export function createUserDataSpaceService(options = {}) {
|
||||
return privateDataDb;
|
||||
}
|
||||
|
||||
function ensureReady() {
|
||||
ensurePrivateDataDb();
|
||||
return getInfo();
|
||||
}
|
||||
|
||||
function privateDataSize() {
|
||||
let total = 0;
|
||||
for (const file of fs.existsSync(privateDataDir) ? fs.readdirSync(privateDataDir) : []) {
|
||||
@@ -750,6 +773,7 @@ export function createUserDataSpaceService(options = {}) {
|
||||
workspaceRoot,
|
||||
privateDataDir,
|
||||
privateDataDb,
|
||||
ensureReady,
|
||||
ensurePrivateDataDb,
|
||||
privateDataSize,
|
||||
getInfo,
|
||||
|
||||
@@ -3,8 +3,29 @@ import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createUserDataSpaceService } from './user-data-space-service.mjs';
|
||||
|
||||
test('runtime rejects SQLite user data backend outside node tests', () => {
|
||||
const script = `
|
||||
import { createUserDataSpaceService } from './user-data-space-service.mjs';
|
||||
try {
|
||||
createUserDataSpaceService({ workspaceRoot: '/tmp/runtime-sqlite-denied', backend: 'sqlite' });
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error(error.code);
|
||||
process.exit(7);
|
||||
}
|
||||
`;
|
||||
const result = spawnSync(process.execPath, ['--input-type=module', '--eval', script], {
|
||||
cwd: process.cwd(),
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, NODE_TEST_CONTEXT: '' },
|
||||
});
|
||||
assert.equal(result.status, 7);
|
||||
assert.match(result.stderr, /sqlite_runtime_disabled/);
|
||||
});
|
||||
|
||||
function createTempWorkspace() {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'user-data-space-'));
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报
|
||||
6. 生成 HTML 后,向用户提供可访问链接,格式:
|
||||
\`${publicBaseUrl}/${PUBLISH_ROOT_DIR}/${slug}/public/<文件名>\`
|
||||
(写入 \`public/页面.html\` 时 URL **必须**含 \`public/\`;仅当 HTML 直接写在 workspace 根目录时才省略)
|
||||
7. **只要页面接收用户输入并要求以后查看、汇总、统计、修改或删除,就不是纯静态页**。禁止使用 \`localStorage\` / \`sessionStorage\` / \`IndexedDB\` 保存任何数据;必须立即 \`load_skill\` → \`page-data-collect\`,通过 Page Data API 写入当前用户专属 PostgreSQL schema。禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化。记账、日记、打卡、清单、台账、问卷、报名及其管理页都适用。
|
||||
|
||||
## 推荐工作流
|
||||
|
||||
@@ -298,6 +299,13 @@ ${renderBrandingBlock(addressName)}
|
||||
- 有 \`shell\` 时用 \`ls oa/\`、\`find .\`、\`cat oa/file.csv\`;有 \`tree\` 时用 \`tree oa/\`
|
||||
- 不要说「我没有 shell」——若系统提示已列出 shell,就必须用 shell 在工作区内操作
|
||||
|
||||
## 生成 / 发布 HTML 页面
|
||||
|
||||
- 用户要网页、交互应用或分享链接时,必须使用 对应 skill 并将最终 HTML 写入 \`public/*.html\`
|
||||
- 若页面接收用户输入并需要保存、查询、统计、修改或删除,必须使用 \`page-data-collect\` + Page Data API,禁止 localStorage / sessionStorage / IndexedDB
|
||||
- \`apps__create_app\` 只能用于设计预览,不是 H5 交付物;使用后仍必须写入 \`public/*.html\` 并返回真实可访问链接
|
||||
- 未验证文件已落盘且链接可访问前,禁止声称「已创建」「已打开」或「可在 Apps 标签找到」
|
||||
|
||||
## Word 下载页
|
||||
|
||||
- 若用户要求页面里可下载 Word / docx:必须先 \`load_skill\` → \`docx-generate\`
|
||||
@@ -322,6 +330,12 @@ export function buildSandboxSessionConstraints({ baseConstraints, developerTools
|
||||
const lines = [
|
||||
baseConstraints,
|
||||
'',
|
||||
'## 当前用户专属 PostgreSQL 数据空间(硬性)',
|
||||
'- 当前 session 已绑定当前用户身份;`private_data_*` 工具只连接该用户在新 PG 中的独立 schema,禁止跨用户访问',
|
||||
'- 任何需要保存、再次读取、统计、汇总、筛选、修改或删除的数据都必须使用 `private_data_*` 建表/注册 dataset,并由 HTML 通过 Page Data API 访问',
|
||||
'- Agent 生成的 HTML/JavaScript 禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 保存数据;write_file/edit_file 会直接拒绝此类代码',
|
||||
'- 不允许 SQLite,也不允许在浏览器存储、静态 JSON/JS 文件中伪装持久化业务数据;失败时必须修复 PG/Page Data 链路,不能降级',
|
||||
'',
|
||||
'## 当前会话可用文件工具',
|
||||
`- ${tools.join(', ')}`,
|
||||
hasSandboxMcp
|
||||
@@ -385,6 +399,8 @@ export function buildPublishConstraints({ slug, username, publicBaseUrl, publish
|
||||
'- **查找文件**:在工作区内对应分区搜索(如 `oa/2025-12-06T13-34_export.csv`),不要搜其它用户目录或公网 URL',
|
||||
'- **禁止**:访问 `assets/` 内部路径、其它用户目录、主机绝对路径;禁止用公网 URL 列目录或读 CSV',
|
||||
'- **路径规则**:只用相对路径;禁止 `../`;工作区外的路径会被系统拒绝(OS 层强制,非软约束)',
|
||||
'- **当前用户专属 PG**:本 session 的 `private_data_*` 工具已绑定当前用户,只能访问该用户在新 PostgreSQL 中的独立 schema;任何需持久化、再次读取、统计、汇总、修改或删除的数据都必须走 PG + Page Data API',
|
||||
'- **浏览器存储硬禁令**:Agent 生成的 HTML/JavaScript 禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 保存数据;禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化',
|
||||
'- **生成页面(必须亲自完成)**:先 `load_skill` → `static-page-publish`,再用 `write_file`/`edit_file` 写入 `public/页面.html`',
|
||||
'- **默认只生成 HTML**:不要在没有明确需求时强制生成 Word、PDF、长图等伴生文件',
|
||||
'- **Word 下载页(按需亲自完成)**:若用户明确要求 Word / docx 下载,必须先 `load_skill` → `docx-generate`,生成并确认 `public/*.docx` 已存在,再在 HTML 中用相对路径链接',
|
||||
|
||||
@@ -85,6 +85,10 @@ test('publish dir and public url use stable user id', () => {
|
||||
assert.match(skillText, /docx-generate/);
|
||||
assert.match(skillText, /long-image-download/);
|
||||
assert.match(skillText, /generate_long_image/);
|
||||
assert.match(skillText, /禁止使用 `localStorage`/);
|
||||
assert.match(skillText, /IndexedDB/);
|
||||
assert.match(skillText, /PostgreSQL schema/);
|
||||
assert.match(skillText, /page-data-collect/);
|
||||
assert.match(docxSkillText, /public\/文件名\.docx/);
|
||||
assert.match(longImageSkillText, /public\/report\.long\.png/);
|
||||
assert.match(skillText, /report\.docx|伴生下载/);
|
||||
@@ -137,6 +141,10 @@ test('buildSandboxSessionConstraints documents shell and forbids public url brow
|
||||
assert.match(text, /docx-generate/);
|
||||
assert.match(text, /public\/\*\.docx/);
|
||||
assert.match(text, /\/public\//);
|
||||
assert.match(text, /当前用户专属 PostgreSQL 数据空间/);
|
||||
assert.match(text, /localStorage/);
|
||||
assert.match(text, /IndexedDB/);
|
||||
assert.match(text, /write_file\/edit_file 会直接拒绝/);
|
||||
});
|
||||
|
||||
test('renderWorkspaceHints includes docx download guidance for published pages', () => {
|
||||
@@ -150,6 +158,9 @@ test('renderWorkspaceHints includes docx download guidance for published pages',
|
||||
assert.match(text, /public\/\*\.docx/);
|
||||
assert.match(text, /long-image-download/);
|
||||
assert.match(text, /generate_long_image/);
|
||||
assert.match(text, /apps__create_app/);
|
||||
assert.match(text, /Page Data API/);
|
||||
assert.match(text, /未验证文件已落盘/);
|
||||
});
|
||||
|
||||
test('isPathInsidePublishDir blocks escape', () => {
|
||||
|
||||
Reference in New Issue
Block a user