From e90a2da6cb36df103f431db9ec3fb34ae407aaee Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 13:59:51 +0800 Subject: [PATCH 1/5] feat(memory-v2): restore personal shadow pipeline and candidate store Recover WIP modules from stash so admin can persist capability config, observe chat writes in shadow mode, and review persisted candidates. Co-authored-by: Cursor --- memory-v2-admin-config.mjs | 61 +++++ memory-v2-admin-config.test.mjs | 49 +++++ memory-v2-personal-shadow.mjs | 232 ++++++++++++++++++++ memory-v2-personal-shadow.test.mjs | 193 ++++++++++++++++ memory-v2-personal-store.mjs | 127 +++++++++++ memory-v2-personal-store.test.mjs | 47 ++++ memory-v2-runtime.mjs | 15 ++ memory-v2.mjs | 35 ++- scripts/setup-memory-v2-personal-schema.mjs | 15 ++ 9 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 memory-v2-personal-shadow.mjs create mode 100644 memory-v2-personal-shadow.test.mjs create mode 100644 memory-v2-personal-store.mjs create mode 100644 memory-v2-personal-store.test.mjs create mode 100644 scripts/setup-memory-v2-personal-schema.mjs diff --git a/memory-v2-admin-config.mjs b/memory-v2-admin-config.mjs index e38c81a..c8c9068 100644 --- a/memory-v2-admin-config.mjs +++ b/memory-v2-admin-config.mjs @@ -22,6 +22,59 @@ const FIELD_SPECS = [ { env: 'MEMIND_CHAT_ROUTER_TIMEOUT_MS', group: 'chatIntentRouter', field: 'timeoutMs', type: 'number' }, { env: 'MEMIND_CHAT_ROUTER_FALLBACK_ROUTE', group: 'chatIntentRouter', field: 'fallbackRoute', type: 'string' }, + { env: 'MEMORY_CANDIDATE_ENABLED', group: 'candidateMemory', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_CANDIDATE_MODE', group: 'candidateMemory', field: 'mode', type: 'string' }, + { env: 'MEMORY_CANDIDATE_MIN_IMPORTANCE', group: 'candidateMemory', field: 'minImportance', type: 'number' }, + { env: 'MEMORY_CANDIDATE_MIN_CONFIDENCE', group: 'candidateMemory', field: 'minConfidence', type: 'number' }, + { env: 'MEMORY_CANDIDATE_MAX_PENDING', group: 'candidateMemory', field: 'maxPending', type: 'number' }, + { env: 'MEMORY_CANDIDATE_PERSISTENCE_ENABLED', group: 'candidateMemory', field: 'persistenceEnabled', type: 'boolean' }, + + { env: 'MEMORY_POLICY_ENABLED', group: 'policy', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_POLICY_SAVE_EXPLICIT', group: 'policy', field: 'saveExplicit', type: 'boolean' }, + { env: 'MEMORY_POLICY_REJECT_SENSITIVE', group: 'policy', field: 'rejectSensitive', type: 'boolean' }, + { env: 'MEMORY_POLICY_REQUIRE_EVIDENCE', group: 'policy', field: 'requireEvidence', type: 'boolean' }, + { env: 'MEMORY_POLICY_RETENTION_DAYS', group: 'policy', field: 'retentionDays', type: 'number' }, + + { env: 'MEMORY_RETRIEVER_ENABLED', group: 'retriever', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_RETRIEVER_EPISODIC_ENABLED', group: 'retriever', field: 'episodicEnabled', type: 'boolean' }, + { env: 'MEMORY_RETRIEVER_SEMANTIC_ENABLED', group: 'retriever', field: 'semanticEnabled', type: 'boolean' }, + { env: 'MEMORY_RETRIEVER_PREFERENCE_ENABLED', group: 'retriever', field: 'preferenceEnabled', type: 'boolean' }, + { env: 'MEMORY_RETRIEVER_GOAL_ENABLED', group: 'retriever', field: 'goalEnabled', type: 'boolean' }, + { env: 'MEMORY_RETRIEVER_LIMIT', group: 'retriever', field: 'limit', type: 'number' }, + { env: 'MEMORY_RETRIEVER_TOKEN_BUDGET', group: 'retriever', field: 'tokenBudget', type: 'number' }, + { env: 'MEMORY_RETRIEVER_TIMEOUT_MS', group: 'retriever', field: 'timeoutMs', type: 'number' }, + + { env: 'MEMORY_LIFECYCLE_ENABLED', group: 'lifecycle', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_LIFECYCLE_DEDUPE_ENABLED', group: 'lifecycle', field: 'dedupeEnabled', type: 'boolean' }, + { env: 'MEMORY_LIFECYCLE_CONFLICT_REVIEW', group: 'lifecycle', field: 'conflictReview', type: 'boolean' }, + { env: 'MEMORY_LIFECYCLE_DECAY_ENABLED', group: 'lifecycle', field: 'decayEnabled', type: 'boolean' }, + { env: 'MEMORY_LIFECYCLE_FORGETTING_ENABLED', group: 'lifecycle', field: 'forgettingEnabled', type: 'boolean' }, + { env: 'MEMORY_LIFECYCLE_COMPACT_INTERVAL_HOURS', group: 'lifecycle', field: 'compactIntervalHours', type: 'number' }, + + { env: 'MEMORY_PERSONA_ENABLED', group: 'persona', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_PERSONA_PROVIDER', group: 'persona', field: 'provider', type: 'string' }, + { env: 'MEMORY_PERSONA_SHADOW_MODE', group: 'persona', field: 'shadowMode', type: 'boolean' }, + { env: 'MEMORY_PERSONA_MAX_TOKENS', group: 'persona', field: 'maxTokens', type: 'number' }, + { env: 'MEMORY_PERSONA_CACHE_TTL_SECONDS', group: 'persona', field: 'cacheTtlSeconds', type: 'number' }, + + { env: 'MEMORY_GRAPH_ENABLED', group: 'graph', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_GRAPH_PROVIDER', group: 'graph', field: 'provider', type: 'string' }, + { env: 'MEMORY_GRAPH_MAX_DEPTH', group: 'graph', field: 'maxDepth', type: 'number' }, + { env: 'MEMORY_GRAPH_RELATION_LIMIT', group: 'graph', field: 'relationLimit', type: 'number' }, + + { env: 'MEMORY_USER_MANAGEMENT_ENABLED', group: 'userMemory', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_USER_REVIEW_ENABLED', group: 'userMemory', field: 'reviewEnabled', type: 'boolean' }, + { env: 'MEMORY_USER_CORRECTION_ENABLED', group: 'userMemory', field: 'correctionEnabled', type: 'boolean' }, + { env: 'MEMORY_USER_PIN_ENABLED', group: 'userMemory', field: 'pinEnabled', type: 'boolean' }, + { env: 'MEMORY_USER_FORGET_ENABLED', group: 'userMemory', field: 'forgetEnabled', type: 'boolean' }, + { env: 'MEMORY_USER_DELETE_PROPAGATION', group: 'userMemory', field: 'deletePropagation', type: 'boolean' }, + + { env: 'MEMORY_PLUGIN_HEALTH_ENABLED', group: 'pluginHealth', field: 'enabled', type: 'boolean' }, + { env: 'MEMORY_PLUGIN_HEALTH_INTERVAL_SECONDS', group: 'pluginHealth', field: 'intervalSeconds', type: 'number' }, + { env: 'MEMORY_PLUGIN_HEALTH_TIMEOUT_MS', group: 'pluginHealth', field: 'timeoutMs', type: 'number' }, + { env: 'MEMORY_PLUGIN_HEALTH_FAILURE_THRESHOLD', group: 'pluginHealth', field: 'failureThreshold', type: 'number' }, + { env: 'MEMORY_PLUGIN_HEALTH_AUTO_FALLBACK', group: 'pluginHealth', field: 'autoFallback', type: 'boolean' }, + { env: 'MEMORY_PGVECTOR_ENABLED', group: 'pgvector', field: 'enabled', type: 'boolean' }, { env: 'MEMORY_PGVECTOR_DATABASE_URL', group: 'pgvector', field: 'databaseUrl', type: 'secret' }, { env: 'MEMORY_PGVECTOR_TABLE', group: 'pgvector', field: 'table', type: 'string' }, @@ -95,6 +148,14 @@ const FIELD_SPECS = [ const GROUPS = [ 'global', 'chatIntentRouter', + 'candidateMemory', + 'policy', + 'retriever', + 'lifecycle', + 'persona', + 'graph', + 'userMemory', + 'pluginHealth', 'pgvector', 'qdrant', 'weaviate', diff --git a/memory-v2-admin-config.test.mjs b/memory-v2-admin-config.test.mjs index e2c3c19..5c639c1 100644 --- a/memory-v2-admin-config.test.mjs +++ b/memory-v2-admin-config.test.mjs @@ -81,6 +81,36 @@ test('memory v2 admin config service persists non-secret and secret patches', as url: 'http://127.0.0.1:6333', apiKey: 'secret-qdrant', }, + candidateMemory: { + enabled: true, + mode: 'shadow', + minImportance: '0.75', + minConfidence: '0.85', + maxPending: '500', + persistenceEnabled: true, + }, + policy: { + enabled: true, + saveExplicit: true, + rejectSensitive: true, + requireEvidence: true, + retentionDays: '365', + }, + retriever: { + enabled: true, + episodicEnabled: true, + semanticEnabled: true, + preferenceEnabled: true, + goalEnabled: true, + limit: '12', + tokenBudget: '1800', + timeoutMs: '1200', + }, + lifecycle: { enabled: true, dedupeEnabled: true, conflictReview: true }, + persona: { enabled: true, provider: 'ai-mind', shadowMode: true }, + graph: { enabled: true, provider: 'postgres', maxDepth: '2' }, + userMemory: { enabled: true, reviewEnabled: true, forgetEnabled: true, deletePropagation: true }, + pluginHealth: { enabled: true, intervalSeconds: '60', timeoutMs: '1500', failureThreshold: '3', autoFallback: true }, }, { updatedBy: 'admin-1' }); assert.equal(updated.config.global.enabled, true); @@ -91,6 +121,15 @@ test('memory v2 admin config service persists non-secret and secret patches', as assert.equal(updated.config.qdrant.enabled, true); assert.equal(updated.config.qdrant.url, 'http://127.0.0.1:6333'); assert.equal(updated.config.qdrant.apiKeyConfigured, true); + assert.equal(updated.config.candidateMemory.mode, 'shadow'); + assert.equal(updated.config.candidateMemory.persistenceEnabled, true); + assert.equal(updated.config.policy.requireEvidence, true); + assert.equal(updated.config.retriever.tokenBudget, '1800'); + assert.equal(updated.config.lifecycle.dedupeEnabled, true); + assert.equal(updated.config.persona.provider, 'ai-mind'); + assert.equal(updated.config.graph.provider, 'postgres'); + assert.equal(updated.config.userMemory.deletePropagation, true); + assert.equal(updated.config.pluginHealth.autoFallback, true); assert.equal(updated.updatedBy, 'admin-1'); const runtimeState = await service.getRuntimeState(); @@ -107,6 +146,16 @@ test('memory v2 admin config service persists non-secret and secret patches', as assert.equal(runtimeState.overrides.MEMORY_QDRANT_ENABLED, '1'); assert.equal(runtimeState.overrides.MEMORY_QDRANT_URL, 'http://127.0.0.1:6333'); assert.equal(runtimeState.overrides.MEMORY_QDRANT_API_KEY, 'secret-qdrant'); + assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_ENABLED, '1'); + assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_MODE, 'shadow'); + assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_PERSISTENCE_ENABLED, '1'); + assert.equal(runtimeState.overrides.MEMORY_POLICY_REQUIRE_EVIDENCE, '1'); + assert.equal(runtimeState.overrides.MEMORY_RETRIEVER_TOKEN_BUDGET, '1800'); + assert.equal(runtimeState.overrides.MEMORY_LIFECYCLE_DEDUPE_ENABLED, '1'); + assert.equal(runtimeState.overrides.MEMORY_PERSONA_PROVIDER, 'ai-mind'); + assert.equal(runtimeState.overrides.MEMORY_GRAPH_PROVIDER, 'postgres'); + assert.equal(runtimeState.overrides.MEMORY_USER_DELETE_PROPAGATION, '1'); + assert.equal(runtimeState.overrides.MEMORY_PLUGIN_HEALTH_AUTO_FALLBACK, '1'); }); test('memory v2 admin config internals flatten booleans and numbers consistently', () => { diff --git a/memory-v2-personal-shadow.mjs b/memory-v2-personal-shadow.mjs new file mode 100644 index 0000000..c389f2e --- /dev/null +++ b/memory-v2-personal-shadow.mjs @@ -0,0 +1,232 @@ +import crypto from 'node:crypto'; +import { deriveUserFacingText } from './conversation-display.mjs'; + +const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']); +const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']); +const SECRET_PATTERNS = [ + /\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i, + /\bsk-[a-z0-9_-]{12,}\b/i, + /-----BEGIN [A-Z ]+PRIVATE KEY-----/, + /(?:密码|口令|密钥|令牌)\s*[：:=]\s*\S+/i, +]; +const TRIVIAL_PATTERNS = [ + /^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[！!。.\s]*$/i, + /^(?:查一下|搜一下|看看|继续|下一步)[。.!！?？\s]*$/i, +]; + +function readFlag(env, key, fallback = false) { + const raw = env?.[key]; + if (raw == null || raw === '') return fallback; + const normalized = String(raw).trim().toLowerCase(); + if (TRUE_VALUES.has(normalized)) return true; + if (FALSE_VALUES.has(normalized)) return false; + return fallback; +} + +function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) { + const value = Number(env?.[key]); + if (!Number.isFinite(value)) return fallback; + return Math.min(max, Math.max(min, value)); +} + +function normalizeText(value, maxLength = 2000) { + const text = String(value ?? '').replace(/\s+/g, ' ').trim(); + if (!text) return ''; + return text.length > maxLength ? text.slice(0, maxLength) : text; +} + +function messageText(message) { + if (typeof message === 'string') return normalizeText(message); + if (!message || typeof message !== 'object') return ''; + const displayText = message.displayText + ?? message.display_text + ?? message.metadata?.displayText + ?? message.metadata?.display_text; + if (typeof displayText === 'string' && displayText.trim()) { + return normalizeText(displayText); + } + const direct = message.text ?? message.content ?? message.message ?? ''; + if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct)); + if (Array.isArray(direct)) { + return normalizeText(deriveUserFacingText( + direct.map((item) => item?.text ?? item?.content ?? '').join('\n'), + )); + } + return ''; +} + +function stableHash(value) { + return crypto.createHash('sha256').update(String(value)).digest('hex'); +} + +function classify(text) { + const rules = [ + { type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' }, + { type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' }, + { type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' }, + { type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' }, + { type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' }, + ]; + return rules.find((rule) => rule.pattern.test(text)) ?? null; +} + +export function resolvePersonalShadowConfig(env = process.env) { + const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false); + const requestedMode = normalizeText(env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'shadow' : 'off'), 20).toLowerCase(); + return { + enabled: candidateEnabled && requestedMode !== 'off', + requestedMode, + effectiveMode: candidateEnabled && requestedMode !== 'off' ? 'shadow' : 'off', + minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }), + minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }), + maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })), + policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true), + saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true), + rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true), + requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true), + healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true), + }; +} + +export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) { + const config = resolvePersonalShadowConfig(env); + const candidates = []; + const candidateHashes = new Set(); + const metrics = { + runs: 0, + observedMessages: 0, + accepted: 0, + rejected: 0, + deduped: 0, + errors: 0, + lastRunAt: null, + lastError: null, + }; + + function reject(reason) { + metrics.rejected += 1; + return { accepted: false, reason }; + } + + function evaluate({ userId, sessionId, message, index }) { + const text = messageText(message); + if (!text || text.length < 8) return reject('too_short'); + if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial'); + if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) { + return reject('sensitive_content'); + } + const classification = classify(text); + if (!classification) return reject('no_durable_signal'); + if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) { + return reject('explicit_memory_disabled'); + } + if (config.policyEnabled && classification.importance < config.minImportance) { + return reject('below_importance_threshold'); + } + if (config.policyEnabled && classification.confidence < config.minConfidence) { + return reject('below_confidence_threshold'); + } + const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`); + if (candidateHashes.has(contentHash)) { + metrics.deduped += 1; + return { accepted: false, reason: 'duplicate' }; + } + const capturedAt = now(); + const evidence = { + sourceType: 'message', + sourceId: `${sessionId || 'unknown'}:${index}`, + evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`), + capturedAt, + }; + if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required'); + const candidate = { + id: `pmc_${contentHash.slice(0, 24)}`, + userId: String(userId), + sessionId: sessionId ? String(sessionId) : null, + memoryType: classification.type, + content: text, + importance: classification.importance, + confidence: classification.confidence, + status: 'candidate', + policyReason: classification.reason, + evidence, + createdAt: capturedAt, + }; + candidateHashes.add(contentHash); + candidates.push(candidate); + while (candidates.length > config.maxPending) { + const removed = candidates.shift(); + if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`)); + } + metrics.accepted += 1; + return { accepted: true, candidate }; + } + + async function observeWrite({ userId, sessionId, messages = [] } = {}) { + if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' }; + metrics.runs += 1; + metrics.lastRunAt = now(); + if (!userId || !Array.isArray(messages)) { + return { enabled: true, skipped: true, reason: 'invalid_input' }; + } + try { + const results = []; + for (let index = 0; index < messages.length; index += 1) { + const message = messages[index]; + const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase(); + if (role && !['user', 'human'].includes(role)) continue; + metrics.observedMessages += 1; + const result = evaluate({ userId, sessionId, message, index }); + if (result.accepted && store?.saveCandidate) { + const persisted = await store.saveCandidate(result.candidate); + if (persisted?.inserted === false) { + metrics.deduped += 1; + } + } + results.push(result); + } + return { + enabled: true, + skipped: false, + mode: config.effectiveMode, + observed: results.length, + accepted: results.filter((result) => result.accepted).length, + rejected: results.filter((result) => !result.accepted).length, + }; + } catch (err) { + metrics.errors += 1; + metrics.lastError = err instanceof Error ? err.message : String(err); + return { enabled: true, skipped: true, reason: 'pipeline_failure' }; + } + } + + function getStatus() { + return { + enabled: config.enabled, + requestedMode: config.requestedMode, + effectiveMode: config.effectiveMode, + phase: 'shadow-candidate-v1', + injectionEnabled: false, + persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory', + pendingCandidates: candidates.length, + health: { + enabled: config.healthEnabled, + state: metrics.errors > 0 ? 'degraded' : 'healthy', + ...metrics, + }, + policy: { + enabled: config.policyEnabled, + minImportance: config.minImportance, + minConfidence: config.minConfidence, + rejectSensitive: config.rejectSensitive, + requireEvidence: config.requireEvidence, + }, + }; + } + + function listCandidates({ limit = 50 } = {}) { + return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse(); + } + + return { config, observeWrite, getStatus, listCandidates }; +} diff --git a/memory-v2-personal-shadow.test.mjs b/memory-v2-personal-shadow.test.mjs new file mode 100644 index 0000000..be272be --- /dev/null +++ b/memory-v2-personal-shadow.test.mjs @@ -0,0 +1,193 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createPersonalMemoryShadowPipeline, resolvePersonalShadowConfig } from './memory-v2-personal-shadow.mjs'; +import { createMemoryV2 } from './memory-v2.mjs'; + +test('personal memory shadow config never activates response injection', () => { + const config = resolvePersonalShadowConfig({ + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'active', + }); + assert.equal(config.enabled, true); + assert.equal(config.requestedMode, 'active'); + assert.equal(config.effectiveMode, 'shadow'); +}); + +test('shadow pipeline extracts durable candidates with evidence and deduplicates', async () => { + let clock = 1000; + const pipeline = createPersonalMemoryShadowPipeline({ + env: { + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'shadow', + MEMORY_POLICY_ENABLED: '1', + MEMORY_POLICY_REQUIRE_EVIDENCE: '1', + MEMORY_CANDIDATE_MIN_IMPORTANCE: '0.7', + MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8', + }, + now: () => clock += 1, + }); + const input = { + userId: 'u1', + sessionId: 's1', + messages: [ + { role: 'user', text: '我决定所有生产发布必须从完整 main 分支打包。' }, + { role: 'assistant', text: '收到。' }, + ], + }; + const first = await pipeline.observeWrite(input); + const second = await pipeline.observeWrite(input); + + assert.equal(first.accepted, 1); + assert.equal(second.accepted, 0); + assert.equal(pipeline.listCandidates().length, 1); + assert.equal(pipeline.listCandidates()[0].memoryType, 'episodic'); + assert.equal(pipeline.listCandidates()[0].evidence.sourceId, 's1:0'); + assert.equal(pipeline.getStatus().health.deduped, 1); + assert.equal(pipeline.getStatus().injectionEnabled, false); +}); + +test('shadow policy rejects trivial and sensitive messages', async () => { + const pipeline = createPersonalMemoryShadowPipeline({ + env: { + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'shadow', + MEMORY_POLICY_REJECT_SENSITIVE: '1', + }, + }); + const result = await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [ + { role: 'user', text: '谢谢' }, + { role: 'user', text: '请记住 API_KEY=sk-this-is-a-secret-token' }, + { role: 'user', text: '今天天气怎么样？' }, + ], + }); + assert.equal(result.accepted, 0); + assert.equal(result.rejected, 3); + assert.equal(pipeline.listCandidates().length, 0); +}); + +test('shadow pipeline prefers user-facing displayText over internal prompt content', async () => { + const pipeline = createPersonalMemoryShadowPipeline({ + env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' }, + }); + await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [{ + role: 'user', + content: [{ type: 'text', text: '[用户身份] 内部前缀\n我决定先测试再发布。' }], + metadata: { displayText: '我决定先测试再发布。' }, + }], + }); + assert.equal(pipeline.listCandidates().length, 1); + assert.equal(pipeline.listCandidates()[0].content, '我决定先测试再发布。'); + assert.doesNotMatch(pipeline.listCandidates()[0].content, /用户身份/); +}); + +test('shadow pipeline strips protected user identity prefix when displayText is absent', async () => { + const pipeline = createPersonalMemoryShadowPipeline({ + env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' }, + }); + await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [{ + role: 'user', + content: [{ + type: 'text', + text: '[用户身份]\n- 当前登录用户称呼：John\n- 内部提示\n\n我决定先测试再发布。', + }], + }], + }); + assert.equal(pipeline.listCandidates().length, 1); + assert.equal(pipeline.listCandidates()[0].content, '我决定先测试再发布。'); +}); + +test('shadow candidate pool remains bounded', async () => { + const pipeline = createPersonalMemoryShadowPipeline({ + env: { + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'shadow', + MEMORY_CANDIDATE_MAX_PENDING: '2', + }, + }); + await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [ + { role: 'user', text: '我决定项目一使用完整 main 发布。' }, + { role: 'user', text: '我决定项目二使用完整 main 发布。' }, + { role: 'user', text: '我决定项目三使用完整 main 发布。' }, + ], + }); + assert.equal(pipeline.listCandidates().length, 2); + assert.match(pipeline.listCandidates()[0].content, /项目三/); +}); + +test('shadow pipeline persists accepted candidates when a store is configured', async () => { + const saved = []; + const pipeline = createPersonalMemoryShadowPipeline({ + env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' }, + store: { async saveCandidate(candidate) { saved.push(candidate); return { inserted: true }; } }, + }); + await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [{ role: 'user', text: '我的长期目标是建设一个持续成长的 Personal Agent。' }], + }); + assert.equal(saved.length, 1); + assert.equal(saved[0].memoryType, 'goal'); + assert.equal(pipeline.getStatus().persistence, 'mysql'); +}); + +test('Memory V2 does not wait for the shadow pipeline before returning legacy write result', async () => { + let releaseShadow; + const shadowBlocked = new Promise((resolve) => { releaseShadow = resolve; }); + const memory = createMemoryV2({ + policy: { enabled: true, eventLogEnabled: true, failOpen: true, backend: 'legacy' }, + backends: [{ + name: 'legacy-conversation-memory', + isAvailable: () => true, + async write() { return { saved: 1, analyzed: 0, memories: 0 }; }, + }], + personalShadowPipeline: { + config: { enabled: true }, + observeWrite: () => shadowBlocked, + getStatus: () => ({ enabled: true }), + }, + }); + const result = await Promise.race([ + memory.write({ userId: 'u1', sessionId: 's1', messages: [] }), + new Promise((_, reject) => setTimeout(() => reject(new Error('write waited for shadow')), 50)), + ]); + assert.equal(result.saved, 1); + releaseShadow(); +}); + +test('Memory V2 exposes a shadow-only observation entry without calling legacy write', async () => { + let legacyWrites = 0; + const observed = []; + const memory = createMemoryV2({ + policy: { enabled: true, eventLogEnabled: true, failOpen: true, backend: 'legacy' }, + backends: [{ + name: 'legacy-conversation-memory', + isAvailable: () => true, + async write() { legacyWrites += 1; return { saved: 1 }; }, + }], + personalShadowPipeline: { + config: { enabled: true }, + async observeWrite(input) { observed.push(input); return { accepted: 1 }; }, + getStatus: () => ({ enabled: true }), + }, + }); + const result = await memory.observePersonalMemory({ + userId: 'u1', + sessionId: 's1', + messages: [{ role: 'user', text: '我决定先完成测试。' }], + }); + assert.equal(result.accepted, 1); + assert.equal(observed.length, 1); + assert.equal(legacyWrites, 0); +}); diff --git a/memory-v2-personal-store.mjs b/memory-v2-personal-store.mjs new file mode 100644 index 0000000..402b971 --- /dev/null +++ b/memory-v2-personal-store.mjs @@ -0,0 +1,127 @@ +const TABLE = 'h5_memory_v2_candidates'; +const ALLOWED_STATUSES = new Set(['candidate', 'accepted', 'rejected', 'forgotten']); + +export function buildPersonalMemoryCandidateSchemaSql({ table = TABLE } = {}) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error('Invalid candidate table name'); + return `CREATE TABLE IF NOT EXISTS \`${table}\` ( + id VARCHAR(64) PRIMARY KEY, + user_id CHAR(36) NOT NULL, + session_id VARCHAR(191) NULL, + memory_type VARCHAR(32) NOT NULL, + content TEXT NOT NULL, + importance DECIMAL(5,4) NOT NULL, + confidence DECIMAL(5,4) NOT NULL, + status VARCHAR(24) NOT NULL DEFAULT 'candidate', + policy_reason VARCHAR(64) NOT NULL, + evidence_json JSON NOT NULL, + reviewed_by CHAR(36) NULL, + reviewed_at BIGINT NULL, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + UNIQUE KEY uniq_user_type_content (user_id, memory_type, id), + KEY idx_candidate_status_created (status, created_at), + KEY idx_candidate_user_status (user_id, status, created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`; +} + +export async function ensurePersonalMemoryCandidateSchema(pool, options = {}) { + if (!pool?.query) throw new Error('Candidate schema requires a MySQL pool'); + await pool.query(buildPersonalMemoryCandidateSchemaSql(options)); +} + +function parseJson(value, fallback = null) { + if (value == null) return fallback; + if (typeof value === 'object') return value; + try { return JSON.parse(value); } catch { return fallback; } +} + +function normalizeRow(row) { + return { + id: String(row.id), + userId: String(row.user_id), + sessionId: row.session_id == null ? null : String(row.session_id), + memoryType: String(row.memory_type), + content: String(row.content), + importance: Number(row.importance), + confidence: Number(row.confidence), + status: String(row.status), + policyReason: String(row.policy_reason), + evidence: parseJson(row.evidence_json, {}), + reviewedBy: row.reviewed_by == null ? null : String(row.reviewed_by), + reviewedAt: row.reviewed_at == null ? null : Number(row.reviewed_at), + createdAt: Number(row.created_at), + updatedAt: Number(row.updated_at), + }; +} + +export function createPersonalMemoryCandidateStore(pool, { table = TABLE, now = () => Date.now() } = {}) { + if (!pool?.query) return null; + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error('Invalid candidate table name'); + const quoted = `\`${table}\``; + + return { + async saveCandidate(candidate) { + const updatedAt = now(); + const [result] = await pool.query( + `INSERT IGNORE INTO ${quoted} + (id, user_id, session_id, memory_type, content, importance, confidence, status, + policy_reason, evidence_json, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 'candidate', ?, ?, ?, ?)`, + [ + candidate.id, + candidate.userId, + candidate.sessionId, + candidate.memoryType, + candidate.content, + candidate.importance, + candidate.confidence, + candidate.policyReason, + JSON.stringify(candidate.evidence ?? {}), + candidate.createdAt, + updatedAt, + ], + ); + return { inserted: Number(result?.affectedRows ?? 0) > 0 }; + }, + + async listCandidates({ status = 'candidate', userId = null, limit = 50, offset = 0 } = {}) { + const normalizedStatus = String(status || 'candidate'); + if (!ALLOWED_STATUSES.has(normalizedStatus)) throw new Error('Invalid candidate status'); + const safeLimit = Math.max(1, Math.min(200, Number(limit) || 50)); + const safeOffset = Math.max(0, Number(offset) || 0); + const where = ['status = ?']; + const params = [normalizedStatus]; + if (userId) { + where.push('user_id = ?'); + params.push(String(userId)); + } + params.push(safeLimit, safeOffset); + const [rows] = await pool.query( + `SELECT * FROM ${quoted} WHERE ${where.join(' AND ')} + ORDER BY created_at DESC LIMIT ? OFFSET ?`, + params, + ); + return rows.map(normalizeRow); + }, + + async countByStatus() { + const [rows] = await pool.query( + `SELECT status, COUNT(*) AS count FROM ${quoted} GROUP BY status`, + ); + return Object.fromEntries(rows.map((row) => [String(row.status), Number(row.count)])); + }, + + async reviewCandidate(id, status, { reviewedBy = null } = {}) { + if (!['accepted', 'rejected'].includes(status)) throw new Error('Invalid review status'); + const reviewedAt = now(); + const [result] = await pool.query( + `UPDATE ${quoted} + SET status = ?, reviewed_by = ?, reviewed_at = ?, updated_at = ? + WHERE id = ? AND status = 'candidate'`, + [status, reviewedBy, reviewedAt, reviewedAt, String(id)], + ); + return { updated: Number(result?.affectedRows ?? 0) > 0, status, reviewedAt }; + }, + }; +} + diff --git a/memory-v2-personal-store.test.mjs b/memory-v2-personal-store.test.mjs new file mode 100644 index 0000000..60f1ae1 --- /dev/null +++ b/memory-v2-personal-store.test.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildPersonalMemoryCandidateSchemaSql, + createPersonalMemoryCandidateStore, + ensurePersonalMemoryCandidateSchema, +} from './memory-v2-personal-store.mjs'; + +test('candidate schema is explicit and idempotent', async () => { + const calls = []; + const pool = { async query(sql, params) { calls.push({ sql, params }); return [[], []]; } }; + await ensurePersonalMemoryCandidateSchema(pool); + assert.match(calls[0].sql, /CREATE TABLE IF NOT EXISTS/); + assert.match(calls[0].sql, /h5_memory_v2_candidates/); + assert.throws(() => buildPersonalMemoryCandidateSchemaSql({ table: 'bad-name' }), /Invalid/); +}); + +test('candidate store saves, lists, counts, and reviews with parameterized SQL', async () => { + const calls = []; + const pool = { + async query(sql, params = []) { + calls.push({ sql, params }); + if (sql.startsWith('INSERT')) return [{ affectedRows: 1 }, []]; + if (sql.startsWith('SELECT *')) return [[{ + id: 'pmc_1', user_id: 'u1', session_id: 's1', memory_type: 'episodic', + content: '决定使用 main 发布', importance: '0.9', confidence: '0.9', status: 'candidate', + policy_reason: 'decision_signal', evidence_json: '{"sourceId":"s1:0"}', + reviewed_by: null, reviewed_at: null, created_at: 1, updated_at: 1, + }], []]; + if (sql.startsWith('SELECT status')) return [[{ status: 'candidate', count: 1 }], []]; + if (sql.startsWith('UPDATE')) return [{ affectedRows: 1 }, []]; + throw new Error(`Unexpected SQL: ${sql}`); + }, + }; + const store = createPersonalMemoryCandidateStore(pool, { now: () => 10 }); + assert.deepEqual(await store.saveCandidate({ + id: 'pmc_1', userId: 'u1', sessionId: 's1', memoryType: 'episodic', + content: '决定使用 main 发布', importance: 0.9, confidence: 0.9, + policyReason: 'decision_signal', evidence: { sourceId: 's1:0' }, createdAt: 1, + }), { inserted: true }); + const rows = await store.listCandidates({ limit: 20 }); + assert.equal(rows[0].evidence.sourceId, 's1:0'); + assert.deepEqual(await store.countByStatus(), { candidate: 1 }); + assert.equal((await store.reviewCandidate('pmc_1', 'accepted', { reviewedBy: 'admin-1' })).updated, true); + assert.ok(calls.every((call) => !call.sql.includes('admin-1'))); +}); + diff --git a/memory-v2-runtime.mjs b/memory-v2-runtime.mjs index 3e877b2..a28e2f5 100644 --- a/memory-v2-runtime.mjs +++ b/memory-v2-runtime.mjs @@ -4,6 +4,8 @@ import { createLangGraphHttpClient, createLangGraphMemoryBackend } from './memor import { createLegacyMemoryBackend, createMemoryV2, resolveMemoryV2Policy } from './memory-v2.mjs'; import { createLettaHttpClient, createLettaMemoryBackend } from './memory-v2-letta.mjs'; import { createMemoryV2PluginBackends } from './memory-v2-plugin-backends.mjs'; +import { createPersonalMemoryShadowPipeline } from './memory-v2-personal-shadow.mjs'; +import { createPersonalMemoryCandidateStore } from './memory-v2-personal-store.mjs'; import { createMem0HttpClient, createMem0MemoryBackend } from './memory-v2-mem0.mjs'; import { createNeo4jHttpClient, createNeo4jMemoryBackend } from './memory-v2-neo4j.mjs'; import { createPgvectorMemoryBackend } from './memory-v2-pgvector.mjs'; @@ -400,11 +402,19 @@ export async function createMemoryV2Runtime({ ], })); + const personalCandidateStore = readFlag(env, 'MEMORY_CANDIDATE_PERSISTENCE_ENABLED', false) + ? createPersonalMemoryCandidateStore(mysqlPool) + : null; + const personalShadowPipeline = createPersonalMemoryShadowPipeline({ + env, + store: personalCandidateStore, + }); const memory = createMemoryV2({ legacyMemoryService, backends, env, logger, + personalShadowPipeline, }); const pgBackfillEnabled = readFlag(env, 'MEMORY_PGVECTOR_BACKFILL_ENABLED', pgvectorEnabled); @@ -593,6 +603,11 @@ export async function createManagedMemoryV2Runtime({ return runtime.compact(input); }, + async observePersonalMemory(input = {}) { + const runtime = await ensureRuntime(); + return runtime.observePersonalMemory(input); + }, + async close() { const runtime = activeRuntime; activeRuntime = null; diff --git a/memory-v2.mjs b/memory-v2.mjs index afc8b7c..f70d680 100644 --- a/memory-v2.mjs +++ b/memory-v2.mjs @@ -1,4 +1,5 @@ import { createMemoryV2PluginBackends } from './memory-v2-plugin-backends.mjs'; +import { createPersonalMemoryShadowPipeline } from './memory-v2-personal-shadow.mjs'; const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']); const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']); @@ -231,12 +232,14 @@ export function createMemoryV2({ policy = null, env = process.env, logger = console, + personalShadowPipeline = null, } = {}) { const resolvedPolicy = policy ?? resolveMemoryV2Policy({ env }); const resolvedBackends = backends ?? [ createLegacyMemoryBackend(legacyMemoryService), ...createMemoryV2PluginBackends(), ]; + const shadowPipeline = personalShadowPipeline ?? createPersonalMemoryShadowPipeline({ env }); async function failOpen(operation, err, fallback) { logger?.warn?.( @@ -278,6 +281,13 @@ export function createMemoryV2({ if (!backend?.write) return skippedWriteResult('no_backend'); try { const result = await backend.write(input); + if (shadowPipeline?.config?.enabled) { + void Promise.resolve(shadowPipeline.observeWrite(input)).catch((err) => { + logger?.warn?.( + `[memory-v2] personal shadow pipeline skipped: ${err instanceof Error ? err.message : err}`, + ); + }); + } return { ok: true, enabled: true, @@ -334,10 +344,28 @@ export function createMemoryV2({ } } + async function observePersonalMemory(input = {}) { + if (!shadowPipeline?.config?.enabled) { + return { enabled: false, skipped: true, reason: 'disabled' }; + } + try { + return await shadowPipeline.observeWrite(input); + } catch (err) { + logger?.warn?.( + `[memory-v2] personal shadow observation skipped: ${err instanceof Error ? err.message : err}`, + ); + return { + enabled: true, + skipped: true, + reason: 'pipeline_failure', + }; + } + } + function getStatus() { const backends = resolvedBackends.map((backend) => backendStatus(backend)); const selected = selectBackend(resolvedBackends, resolvedPolicy, 'resolve'); - return { + const status = { enabled: Boolean(resolvedPolicy.enabled), backend: resolvedPolicy.backend, selectedBackend: selected?.name ?? null, @@ -347,6 +375,10 @@ export function createMemoryV2({ failOpen: Boolean(resolvedPolicy.failOpen), backends, }; + if (shadowPipeline?.config?.enabled) { + status.personalMemory = shadowPipeline.getStatus(); + } + return status; } return { @@ -355,5 +387,6 @@ export function createMemoryV2({ resolve, write, compact, + observePersonalMemory, }; } diff --git a/scripts/setup-memory-v2-personal-schema.mjs b/scripts/setup-memory-v2-personal-schema.mjs new file mode 100644 index 0000000..6c09701 --- /dev/null +++ b/scripts/setup-memory-v2-personal-schema.mjs @@ -0,0 +1,15 @@ +import { createDbPool } from '../db.mjs'; +import { ensurePersonalMemoryCandidateSchema } from '../memory-v2-personal-store.mjs'; + +if (process.env.MEMORY_PERSONAL_SCHEMA_CONFIRM !== 'local-only') { + throw new Error('Refusing schema change: set MEMORY_PERSONAL_SCHEMA_CONFIRM=local-only explicitly'); +} + +const pool = createDbPool(); +try { + await ensurePersonalMemoryCandidateSchema(pool); + console.log('Memory V2 personal candidate schema ready.'); +} finally { + await pool.end(); +} + From a867592367536aeed89992584c77eb6b685ea670 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 13:59:59 +0800 Subject: [PATCH 2/5] feat(skill-runtime): restore admin config and manifest keyword routing Recover skill runtime policy/admin services and Skill Router v2 helpers so admins can toggle manifest routing and inspect the platform skill catalog. Co-authored-by: Cursor --- admin-bootstrap.mjs | 3 + admin-routes.mjs | 36 +++++ chat-skills.mjs | 96 ++++++++++++- skill-runtime-admin-config.mjs | 181 ++++++++++++++++++++++++ skill-runtime-admin-config.test.mjs | 74 ++++++++++ skill-runtime-policy.mjs | 30 ++++ skill-runtime-policy.test.mjs | 40 ++++++ skills/product-campaign-page/skill.yaml | 14 ++ 8 files changed, 470 insertions(+), 4 deletions(-) create mode 100644 skill-runtime-admin-config.mjs create mode 100644 skill-runtime-admin-config.test.mjs create mode 100644 skill-runtime-policy.mjs create mode 100644 skill-runtime-policy.test.mjs create mode 100644 skills/product-campaign-page/skill.yaml diff --git a/admin-bootstrap.mjs b/admin-bootstrap.mjs index 002cf92..feeb875 100644 --- a/admin-bootstrap.mjs +++ b/admin-bootstrap.mjs @@ -18,6 +18,7 @@ import { createUserAuth } from './user-auth.mjs'; import { createLlmProviderService } from './llm-providers.mjs'; import { createAssetGatewayConfigService } from './asset-gateway.mjs'; import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs'; +import { createSkillRuntimeAdminConfigService } from './skill-runtime-admin-config.mjs'; import { createWechatScheduleLlmConfigService } from './wechat-schedule-llm-config.mjs'; import { createPlazaPostService, formatPostRow } from './plaza-posts.mjs'; import { createPlazaInteractionService } from './plaza-interactions.mjs'; @@ -100,6 +101,7 @@ export async function createAdminServices(env = {}) { const llmProviderService = createLlmProviderService(pool, { apiTarget, apiSecret }); const assetGatewayConfigService = createAssetGatewayConfigService(pool, { llmProviderService }); const memoryV2ConfigService = createMemoryV2AdminConfigService(pool); + const skillRuntimeConfigService = createSkillRuntimeAdminConfigService(pool, { h5Root }); const wechatScheduleLlmConfigService = createWechatScheduleLlmConfigService(pool); const adminSystemTestService = createAdminSystemTestService({ pool, @@ -138,6 +140,7 @@ export async function createAdminServices(env = {}) { llmProviderService, assetGatewayConfigService, memoryV2ConfigService, + skillRuntimeConfigService, wechatScheduleLlmConfigService, adminSystemTestService, plazaPosts, diff --git a/admin-routes.mjs b/admin-routes.mjs index ae245a8..853ae64 100644 --- a/admin-routes.mjs +++ b/admin-routes.mjs @@ -31,6 +31,7 @@ function plazaRouteError(res, req, error) { * @param {object} deps.userAuth * @param {object|null} deps.llmProviderService * @param {object|null} deps.memoryV2ConfigService + * @param {object|null} deps.skillRuntimeConfigService * @param {object|null} deps.adminSystemTestService * @param {object|null} deps.plazaPosts * @param {object|null} deps.plazaOps @@ -44,6 +45,7 @@ export function createAdminApi({ llmProviderService, assetGatewayConfigService, memoryV2ConfigService, + skillRuntimeConfigService, adminSystemTestService, plazaPosts, plazaOps, @@ -148,6 +150,40 @@ export function createAdminApi({ return res.json(result); }); + adminApi.get('/skill-runtime/config', requireAdmin, async (_req, res) => { + if (!skillRuntimeConfigService?.getAdminConfig) { + return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' }); + } + return res.json(await skillRuntimeConfigService.getAdminConfig()); + }); + + const updateSkillRuntimeConfig = async (req, res) => { + if (!skillRuntimeConfigService?.updateAdminConfig) { + return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' }); + } + const result = await skillRuntimeConfigService.updateAdminConfig(req.body?.config ?? req.body ?? {}, { + updatedBy: req.currentUser.id, + }); + return res.json(result); + }; + + adminApi.put('/skill-runtime/config', requireAdmin, updateSkillRuntimeConfig); + adminApi.patch('/skill-runtime/config', requireAdmin, updateSkillRuntimeConfig); + + adminApi.get('/skill-runtime/catalog', requireAdmin, async (_req, res) => { + if (!skillRuntimeConfigService?.listCatalogSummary) { + return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' }); + } + return res.json({ catalog: await skillRuntimeConfigService.listCatalogSummary() }); + }); + + adminApi.get('/skill-runtime/runtime', requireAdmin, async (_req, res) => { + if (!skillRuntimeConfigService?.getPublicRuntimeConfig) { + return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' }); + } + return res.json(await skillRuntimeConfigService.getPublicRuntimeConfig()); + }); + adminApi.post('/system-tests/skill-validation', requireAdmin, async (req, res) => { if (!adminSystemTestService?.runSkillValidation) { return res.status(503).json({ message: '系统测试服务未启用' }); diff --git a/chat-skills.mjs b/chat-skills.mjs index 1538392..7d49846 100644 --- a/chat-skills.mjs +++ b/chat-skills.mjs @@ -1,6 +1,7 @@ // Keep browser-safe: do not import user-publish.mjs (uses node:fs/path/url). const PUBLISH_SKILL_NAME = 'static-page-publish'; export const PAGE_DATA_COLLECT_SKILL_NAME = 'page-data-collect'; +export const SKILL_ROUTER_V2_ENV = 'TKMIND_SKILL_ROUTER_V2'; const WEB_INTENT_PATTERNS = [ /(?:今天|今日|最新|热点|热搜|新闻|头条|头条新闻|最新消息|发生了什么|最近发生)/u, @@ -37,6 +38,9 @@ const PAGE_DATA_INTENT_PATTERNS = [ /(?:密码|口令).{0,12}(?:查看|后台|管理|进入)/u, /(?:sqlite|数据库|page[\s-]?data)/i, /(?:每个用户|各用户).{0,12}(?:提交|记录)/u, + /(?:记账|账本|收支|流水|日记|习惯打卡|每日打卡).{0,40}(?:页面|记录|管理|汇总|统计)/u, + /(?:管理页面|管理页).{0,30}(?:查看|记录|汇总|统计|明细|删除|修改)/u, + /(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u, ]; export function isPageDataIntent(text) { @@ -155,7 +159,7 @@ export function buildChatSkillPrompt(promptKey, skillName) { case 'page-data-collect': return ( `请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能：在 MindSpace 页面中实现可提交、可持久化的数据收集（问卷/报名/台账等），必须使用 Page Data API。` + - '先匹配技能内能力分支（默认 A：匿名前台 public insert + 独立后台 password read，口令默认 88888888）；展示方案摘要确认后再开工。' + + '先匹配技能内能力分支（默认 A：匿名前台 public insert + 独立后台 password read，口令默认 88888888）；“帮我做/创建”本身就是创建授权，展示简短方案摘要后必须同一轮继续执行，只有互斥需求或非法口令才追问。' + '流程：load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html（含 /assets/page-data-client.js）→ private_data_bind_workspace_page 发布并写策略。' + '禁止 localStorage / 浏览器本地存储 fallback；禁止自建 Express/独立端口（如 8899）、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl（/MindSpace/<用户ID>/public/...），禁止给用户 /u/用户名/pages/... 链接，并说明后台入口与口令。' ); @@ -206,9 +210,65 @@ export function filterChatSkills(options, ctx) { }); } -export function buildAutoChatSkillPrefix(text, grantedSkills = []) { - const trimmed = String(text ?? '').trim(); - if (!trimmed) return ''; +const PROMPT_KEY_BY_SKILL_NAME = new Map( + CHAT_SKILL_DEFINITIONS.filter((item) => item.skillName && item.promptKey).map((item) => [ + item.skillName, + item.promptKey, + ]), +); + +/** Opt-in Skill Router v2. Default off unless TKMIND_SKILL_ROUTER_V2=1 or options.skillRouterV2=true. */ +export function isSkillRouterV2Enabled(explicit) { + if (explicit === true) return true; + if (explicit === false) return false; + const raw = + typeof process !== 'undefined' && process?.env ? process.env[SKILL_ROUTER_V2_ENV] : undefined; + return /^(1|true|yes)$/i.test(String(raw ?? '')); +} + +/** Build manifest routes from platform catalog entries that declare trigger.keywords. */ +export function buildManifestRoutesFromCatalog(catalog = []) { + return catalog + .filter((item) => Array.isArray(item?.manifest?.trigger?.keywords) && item.manifest.trigger.keywords.length > 0) + .map((item) => ({ + skillName: item.name, + keywords: item.manifest.trigger.keywords, + priority: Number(item.manifest?.router?.priority ?? 0), + promptKey: + item.manifest?.router?.promptKey ?? + PROMPT_KEY_BY_SKILL_NAME.get(item.name) ?? + null, + promptVariant: item.manifest?.router?.promptVariant ?? null, + })) + .sort((a, b) => b.priority - a.priority || a.skillName.localeCompare(b.skillName)); +} + +export function matchManifestSkillRoute(text, routes = [], grantedSkills = []) { + const normalized = String(text ?? '').trim().toLowerCase(); + if (!normalized || !Array.isArray(routes) || routes.length === 0) return null; + const granted = new Set(grantedSkills); + for (const route of routes) { + if (!route?.skillName || !granted.has(route.skillName)) continue; + const keywords = Array.isArray(route.keywords) ? route.keywords : []; + if (keywords.some((keyword) => normalized.includes(String(keyword).trim().toLowerCase()))) { + return route; + } + } + return null; +} + +function buildManifestSkillPrompt(route) { + if (!route) return ''; + if (route.promptVariant === 'web-news') { + return buildWebNewsSkillPrompt(route.skillName); + } + if (route.promptKey) { + return buildChatSkillPrompt(route.promptKey, route.skillName); + } + return `请使用 ${route.skillName} 技能：`; +} + +function buildLegacyAutoChatSkillPrefix(trimmed, grantedSkills) { if ( grantedSkills.includes(PAGE_DATA_COLLECT_SKILL_NAME) && isPageDataIntent(trimmed) @@ -226,6 +286,34 @@ export function buildAutoChatSkillPrefix(text, grantedSkills = []) { return buildChatSkillPrompt('web', 'web'); } +export function buildAutoChatSkillPrefixOptions(publicConfig) { + if (!publicConfig?.routerV2Enabled || !publicConfig?.manifestRoutingEnabled) { + return {}; + } + return { + skillRouterV2: true, + manifestRoutes: publicConfig.manifestRoutes ?? [], + }; +} + +export function buildAutoChatSkillPrefix(text, grantedSkills = [], options = {}) { + const trimmed = String(text ?? '').trim(); + if (!trimmed) return ''; + + if ( + isSkillRouterV2Enabled(options.skillRouterV2) && + Array.isArray(options.manifestRoutes) && + options.manifestRoutes.length > 0 + ) { + const matched = matchManifestSkillRoute(trimmed, options.manifestRoutes, grantedSkills); + if (matched) { + return buildManifestSkillPrompt(matched); + } + } + + return buildLegacyAutoChatSkillPrefix(trimmed, grantedSkills); +} + export function stripKnownChatSkillPrompt(text) { let next = String(text ?? '').trimStart(); const candidates = []; diff --git a/skill-runtime-admin-config.mjs b/skill-runtime-admin-config.mjs new file mode 100644 index 0000000..6ee4ff5 --- /dev/null +++ b/skill-runtime-admin-config.mjs @@ -0,0 +1,181 @@ +import { SKILL_ROUTER_V2_ENV } from './chat-skills.mjs'; +import { + buildPublicSkillRuntimeConfig, + buildSkillRuntimeCatalogSummary, +} from './skill-runtime-policy.mjs'; + +const CONFIG_TABLE = 'h5_skill_runtime_config'; +const CONFIG_SCOPE = 'global'; + +function defaultConfigShape() { + return { + router: { + v2Enabled: false, + manifestRoutingEnabled: false, + }, + }; +} + +function normalizeBoolean(value, fallback = false) { + if (value == null || value === '') return fallback; + if (typeof value === 'boolean') return value; + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) return true; + if (['0', 'false', 'no', 'off'].includes(normalized)) return false; + return fallback; +} + +function cloneConfig(config = null) { + return structuredClone?.(config ?? defaultConfigShape()) + ?? JSON.parse(JSON.stringify(config ?? defaultConfigShape())); +} + +function parseJsonLike(value, fallback) { + if (value == null || value === '') return fallback; + if (typeof value === 'string') { + try { + return JSON.parse(value); + } catch { + return fallback; + } + } + if (typeof value === 'object') return value; + return fallback; +} + +function envRouterEnabled(env = process.env) { + return /^(1|true|yes)$/i.test(String(env?.[SKILL_ROUTER_V2_ENV] ?? '')); +} + +function applyEnv(config, env = process.env) { + const next = cloneConfig(config); + if (next.router.v2Enabled === false && envRouterEnabled(env)) { + next.router.v2Enabled = true; + } + if (next.router.manifestRoutingEnabled === false && next.router.v2Enabled) { + next.router.manifestRoutingEnabled = envRouterEnabled(env); + } + return next; +} + +function mergePatch(currentConfig, patch = {}) { + const next = cloneConfig(currentConfig); + if (patch?.router && 'v2Enabled' in patch.router) { + next.router.v2Enabled = normalizeBoolean(patch.router.v2Enabled, false); + } + if (patch?.router && 'manifestRoutingEnabled' in patch.router) { + next.router.manifestRoutingEnabled = normalizeBoolean(patch.router.manifestRoutingEnabled, false); + } + if (!next.router.v2Enabled) { + next.router.manifestRoutingEnabled = false; + } + return next; +} + +async function ensureConfigTable(pool) { + await pool.query(` + CREATE TABLE IF NOT EXISTS ${CONFIG_TABLE} ( + config_scope VARCHAR(32) PRIMARY KEY, + config_json JSON NOT NULL, + updated_by CHAR(36) NULL, + updated_at BIGINT NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + `); +} + +async function loadStoredState(pool) { + await ensureConfigTable(pool); + const [rows] = await pool.query( + `SELECT config_json, updated_by, updated_at + FROM ${CONFIG_TABLE} + WHERE config_scope = ? + LIMIT 1`, + [CONFIG_SCOPE], + ); + const row = rows[0]; + if (!row) return null; + return { + config: { ...defaultConfigShape(), ...parseJsonLike(row.config_json, {}) }, + updatedAt: Number(row.updated_at ?? 0) || null, + updatedBy: row.updated_by ?? null, + }; +} + +export function createSkillRuntimeAdminConfigService(pool, { env = process.env, h5Root = process.cwd() } = {}) { + async function loadEffectiveConfig() { + const stored = await loadStoredState(pool); + if (!stored) { + const config = applyEnv(defaultConfigShape(), env); + return { + config, + updatedAt: null, + updatedBy: null, + source: envRouterEnabled(env) ? 'env' : 'default', + }; + } + return { + config: cloneConfig(stored.config), + updatedAt: stored.updatedAt, + updatedBy: stored.updatedBy, + source: 'admin-db', + }; + } + + return { + async getAdminConfig() { + const state = await loadEffectiveConfig(); + return { + config: state.config, + updatedAt: state.updatedAt, + updatedBy: state.updatedBy, + source: state.source, + }; + }, + + async updateAdminConfig(patch = {}, { updatedBy = null } = {}) { + const stored = await loadStoredState(pool); + const base = stored?.config ?? defaultConfigShape(); + const nextConfig = mergePatch(base, patch.config ?? patch); + await ensureConfigTable(pool); + const now = Date.now(); + await pool.query( + `INSERT INTO ${CONFIG_TABLE} + (config_scope, config_json, updated_by, updated_at) + VALUES (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + config_json = VALUES(config_json), + updated_by = VALUES(updated_by), + updated_at = VALUES(updated_at)`, + [CONFIG_SCOPE, JSON.stringify(nextConfig), updatedBy, now], + ); + return this.getAdminConfig(); + }, + + async getRuntimeState() { + const state = await loadEffectiveConfig(); + return { + source: state.source, + updatedAt: state.updatedAt, + updatedBy: state.updatedBy, + config: state.config, + }; + }, + + async getPublicRuntimeConfig() { + const state = await loadEffectiveConfig(); + return buildPublicSkillRuntimeConfig(state.config, h5Root); + }, + + listCatalogSummary() { + return buildSkillRuntimeCatalogSummary(h5Root); + }, + }; +} + +export const skillRuntimeAdminConfigInternals = { + CONFIG_SCOPE, + CONFIG_TABLE, + defaultConfigShape, + applyEnv, + mergePatch, +}; diff --git a/skill-runtime-admin-config.test.mjs b/skill-runtime-admin-config.test.mjs new file mode 100644 index 0000000..9f4799b --- /dev/null +++ b/skill-runtime-admin-config.test.mjs @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + createSkillRuntimeAdminConfigService, + skillRuntimeAdminConfigInternals, +} from './skill-runtime-admin-config.mjs'; + +function createPool(seedRow = null) { + const state = { row: seedRow }; + return { + state, + async query(sql, params) { + if (sql.includes('CREATE TABLE')) return [[], []]; + if (sql.includes('SELECT config_json')) { + return [state.row ? [state.row] : [], []]; + } + if (sql.includes('INSERT INTO h5_skill_runtime_config')) { + state.row = { + config_json: params[1], + updated_by: params[2], + updated_at: params[3], + }; + return [[], []]; + } + throw new Error(`Unexpected query: ${sql}`); + }, + }; +} + +test('skill runtime admin config defaults to disabled router', async () => { + const service = createSkillRuntimeAdminConfigService(createPool(), { env: {} }); + const result = await service.getAdminConfig(); + assert.equal(result.config.router.v2Enabled, false); + assert.equal(result.config.router.manifestRoutingEnabled, false); +}); + +test('skill runtime admin config falls back to env when db row is absent', async () => { + const service = createSkillRuntimeAdminConfigService(createPool(), { + env: { TKMIND_SKILL_ROUTER_V2: '1' }, + }); + const result = await service.getAdminConfig(); + assert.equal(result.config.router.v2Enabled, true); + assert.equal(result.config.router.manifestRoutingEnabled, true); + assert.equal(result.source, 'env'); +}); + +test('skill runtime admin config persists router toggles', async () => { + const pool = createPool(); + const service = createSkillRuntimeAdminConfigService(pool, { env: {} }); + const updated = await service.updateAdminConfig( + { router: { v2Enabled: true, manifestRoutingEnabled: true } }, + { updatedBy: 'admin-1' }, + ); + assert.equal(updated.config.router.v2Enabled, true); + assert.equal(updated.config.router.manifestRoutingEnabled, true); + const runtime = await service.getPublicRuntimeConfig(); + assert.equal(runtime.routerV2Enabled, true); + assert.equal(runtime.manifestRoutingEnabled, true); + assert.ok(Array.isArray(runtime.manifestRoutes)); +}); + +test('skill runtime admin config keeps db authoritative over env', async () => { + const pool = createPool({ + config_json: JSON.stringify(skillRuntimeAdminConfigInternals.defaultConfigShape()), + updated_by: 'admin-1', + updated_at: 123, + }); + const service = createSkillRuntimeAdminConfigService(pool, { + env: { TKMIND_SKILL_ROUTER_V2: '1' }, + }); + const result = await service.getAdminConfig(); + assert.equal(result.config.router.v2Enabled, false); + assert.equal(result.source, 'admin-db'); +}); diff --git a/skill-runtime-policy.mjs b/skill-runtime-policy.mjs new file mode 100644 index 0000000..84bec4e --- /dev/null +++ b/skill-runtime-policy.mjs @@ -0,0 +1,30 @@ +import { buildManifestRoutesFromCatalog } from './chat-skills.mjs'; +import { listPlatformSkillCatalog } from './skills-registry.mjs'; + +export function buildPublicSkillRuntimeConfig(config, h5Root = process.cwd()) { + const routerV2Enabled = Boolean(config?.router?.v2Enabled); + const manifestRoutingEnabled = routerV2Enabled && Boolean(config?.router?.manifestRoutingEnabled); + const catalog = listPlatformSkillCatalog(h5Root); + const manifestRoutes = manifestRoutingEnabled ? buildManifestRoutesFromCatalog(catalog) : []; + return { + routerV2Enabled, + manifestRoutingEnabled, + manifestRoutes, + manifestSkillCount: catalog.filter((item) => item.manifest?.trigger?.keywords?.length).length, + }; +} + +export function buildSkillRuntimeCatalogSummary(h5Root = process.cwd()) { + const catalog = listPlatformSkillCatalog(h5Root); + return catalog.map((item) => ({ + name: item.name, + dirName: item.dirName, + description: item.description, + version: item.version ?? null, + executors: item.executors ?? ['goose'], + hasManifest: Boolean(item.manifest), + triggerKeywords: item.manifest?.trigger?.keywords ?? [], + routerPromptKey: item.manifest?.router?.promptKey ?? null, + routerPriority: item.manifest?.router?.priority ?? 0, + })); +} diff --git a/skill-runtime-policy.test.mjs b/skill-runtime-policy.test.mjs new file mode 100644 index 0000000..405febc --- /dev/null +++ b/skill-runtime-policy.test.mjs @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { + buildPublicSkillRuntimeConfig, + buildSkillRuntimeCatalogSummary, +} from './skill-runtime-policy.mjs'; +import { buildAutoChatSkillPrefixOptions } from './chat-skills.mjs'; + +const h5Root = path.dirname(fileURLToPath(import.meta.url)); + +test('buildPublicSkillRuntimeConfig omits manifest routes when disabled', () => { + const config = buildPublicSkillRuntimeConfig( + { router: { v2Enabled: false, manifestRoutingEnabled: false } }, + h5Root, + ); + assert.equal(config.routerV2Enabled, false); + assert.equal(config.manifestRoutingEnabled, false); + assert.deepEqual(config.manifestRoutes, []); +}); + +test('buildSkillRuntimeCatalogSummary includes product-campaign-page manifest', () => { + const catalog = buildSkillRuntimeCatalogSummary(h5Root); + const product = catalog.find((item) => item.name === 'product-campaign-page'); + assert.ok(product); + assert.equal(product.hasManifest, true); + assert.ok(product.triggerKeywords.includes('带货')); +}); + +test('buildAutoChatSkillPrefixOptions returns empty object when router is disabled', () => { + assert.deepEqual( + buildAutoChatSkillPrefixOptions({ + routerV2Enabled: false, + manifestRoutingEnabled: false, + manifestRoutes: [{ skillName: 'web', keywords: ['新闻'], priority: 1, promptKey: 'web', promptVariant: null }], + }), + {}, + ); +}); diff --git a/skills/product-campaign-page/skill.yaml b/skills/product-campaign-page/skill.yaml new file mode 100644 index 0000000..622c4c5 --- /dev/null +++ b/skills/product-campaign-page/skill.yaml @@ -0,0 +1,14 @@ +name: product-campaign-page +version: 1.0.0 +description: 商品宣传 / 活动页 +trigger: + keywords: + - 带货 + - 商品页 + - 种草 + - 电商页 +router: + promptKey: product-campaign-page + priority: 20 +executors: + - goose From 7f7aced7516bb1221d2f46c40f31521639284fc0 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 14:00:11 +0800 Subject: [PATCH 3/5] feat(runtime): wire personal memory observation and skill runtime config Hook shadow pipeline observation into session finish and agent runs, and expose skill runtime settings through auth and runtime status endpoints. Co-authored-by: Cursor --- agent-run-gateway.mjs | 63 +++++++++++++++++++++++++-- chat-agent-run-gate.mjs | 5 ++- chat-agent-run-gate.test.mjs | 11 +++++ server.mjs | 83 +++++++++++++++++++++++++++--------- 4 files changed, 136 insertions(+), 26 deletions(-) diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index 929c724..b7d6fe3 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -12,9 +12,11 @@ import { } from './conversation-transcript-persist.mjs'; import { ensureGooseUserMessageMetadata } from './goose-message.mjs'; import { + prepareAndDetectSessionDeliverables, SESSION_FINISHED_STALE_GRACE_MS, tryRecoverRunFromDeliverables, } from './agent-run-deliverable-check.mjs'; +import { isPageDataIntent } from './chat-skills.mjs'; const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000]; const TERMINAL_STATUSES = new Set(['succeeded', 'failed']); @@ -53,6 +55,17 @@ function serializeMessage(message) { return JSON.stringify(message ?? {}); } +function extractRunMessageText(row) { + const message = parseDbJsonColumn(row?.user_message_json, {}) ?? {}; + if (typeof message.content === 'string') return message.content; + if (!Array.isArray(message.content)) return ''; + return message.content + .filter((item) => item?.type === 'text') + .map((item) => String(item.text ?? '').trim()) + .filter(Boolean) + .join('\n'); +} + function positiveInteger(value, fallback) { const n = Number(value); if (!Number.isFinite(n) || n <= 0) return fallback; @@ -247,6 +260,8 @@ export function createAgentRunGateway({ sessionSnapshotService = null, conversationMemoryService = null, syncUserPagesOnSuccess = null, + observePersonalMemoryOnSuccess = null, + isSessionExternallyBusy = null, retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS, autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true), maxConcurrentRuns = positiveInteger( @@ -348,6 +363,15 @@ export function createAgentRunGateway({ // Prevent multiple concurrent agent runs on the same Goose session, which would // cause replies to arrive out of order and appear garbled in the chat UI. if (sessionId && !isDirectChatSessionId(sessionId)) { + if (typeof isSessionExternallyBusy === 'function' && await isSessionExternallyBusy({ + userId, + sessionId, + })) { + const conflict = new Error('该会话正在完成页面交付或自动修复，请稍候再发送'); + conflict.code = 'SESSION_RUN_CONFLICT'; + conflict.status = 409; + throw conflict; + } const [activeRows] = await pool.query( `SELECT id FROM h5_agent_runs WHERE agent_session_id = ? AND status NOT IN ('succeeded', 'failed') @@ -765,19 +789,52 @@ export function createAgentRunGateway({ } async function finalizeSuccessfulRun(runId, row, sessionId) { + let deliveryResult = null; + if (typeof syncUserPagesOnSuccess === 'function') { + deliveryResult = await syncUserPagesOnSuccess({ + userId: row.user_id, + sessionId, + runId, + }); + } + const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors) + ? deliveryResult.pageDataBind.errors + : []; + if (pageDataErrors.length > 0) { + const error = new Error(`Page Data 页面绑定失败：${pageDataErrors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`); + error.code = 'PAGE_DATA_DELIVERY_FAILED'; + error.retryable = false; + throw error; + } + if (isPageDataIntent(extractRunMessageText(row))) { + const latest = await getRunById(runId); + const deliverables = await prepareAndDetectSessionDeliverables({ + pool, + userId: row.user_id, + sessionId, + runStartedAtMs: latest?.started_at ?? row.started_at ?? null, + }); + if (deliverables.pageCount < 1) { + const error = new Error('Page Data 任务未生成可交付页面，不能标记成功'); + error.code = 'PAGE_DATA_DELIVERABLE_MISSING'; + error.retryable = false; + throw error; + } + } await markRun(runId, 'succeeded', { agent_session_id: sessionId, completed_at: nowMs(), error_message: null, }); - if (typeof syncUserPagesOnSuccess === 'function') { - await syncUserPagesOnSuccess({ + if (typeof observePersonalMemoryOnSuccess === 'function') { + await observePersonalMemoryOnSuccess({ userId: row.user_id, sessionId, runId, + userMessage: parseDbJsonColumn(row.user_message_json, {}), }).catch((err) => { console.warn( - '[AgentRun] workspace page deliver failed:', + '[AgentRun] personal memory shadow observation failed:', err instanceof Error ? err.message : err, ); }); diff --git a/chat-agent-run-gate.mjs b/chat-agent-run-gate.mjs index d254091..2445459 100644 --- a/chat-agent-run-gate.mjs +++ b/chat-agent-run-gate.mjs @@ -10,14 +10,15 @@ */ /** - * @param {{ chatState?: string; finishedViaPortalDirectChat?: boolean }} input + * @param {{ chatState?: string; finishedViaPortalDirectChat?: boolean; agentRunSucceeded?: boolean }} input * @returns {'idle' | 'streaming'} */ export function resolvePostAgentRunChatState({ chatState = 'waiting', finishedViaPortalDirectChat = false, + agentRunSucceeded = false, } = {}) { - if (finishedViaPortalDirectChat) return 'idle'; + if (finishedViaPortalDirectChat || agentRunSucceeded) return 'idle'; if (chatState === 'idle') return 'idle'; return 'streaming'; } diff --git a/chat-agent-run-gate.test.mjs b/chat-agent-run-gate.test.mjs index 97b170d..51a0e85 100644 --- a/chat-agent-run-gate.test.mjs +++ b/chat-agent-run-gate.test.mjs @@ -25,6 +25,17 @@ test('resolvePostAgentRunChatState prefers portal direct chat completion', () => ); }); +test('resolvePostAgentRunChatState idles after worker-side agent run success', () => { + assert.equal( + resolvePostAgentRunChatState({ chatState: 'waiting', agentRunSucceeded: true }), + 'idle', + ); + assert.equal( + resolvePostAgentRunChatState({ chatState: 'streaming', agentRunSucceeded: true }), + 'idle', + ); +}); + test('shouldPromoteSessionIdToStreaming skips re-streaming after Finish', () => { assert.equal(shouldPromoteSessionIdToStreaming('idle'), false); assert.equal(shouldPromoteSessionIdToStreaming('waiting'), true); diff --git a/server.mjs b/server.mjs index 01134d2..31f87ad 100644 --- a/server.mjs +++ b/server.mjs @@ -192,6 +192,7 @@ import { createSessionSnapshotService } from './session-snapshot.mjs'; import { createConversationMemoryService } from './conversation-memory.mjs'; import { createManagedMemoryV2Runtime } from './memory-v2-runtime.mjs'; import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs'; +import { createSkillRuntimeAdminConfigService } from './skill-runtime-admin-config.mjs'; import { createWechatScheduleLlmConfigService } from './wechat-schedule-llm-config.mjs'; import { createExperienceService } from './experience-service.mjs'; import { attachAsrRoutes } from './asr-proxy.mjs'; @@ -202,24 +203,16 @@ import { syncPageDataPolicyAccessMode } from './page-data-publish-sync.mjs'; import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs'; import { attachShenmeiOpinionFormRoutes } from './shenmei-opinion-form-routes.mjs'; import { isNativeH5ApiPath } from './policies.mjs'; +import { + applyMemindRuntimeProfile, + describeMemindRuntimeProfile, + loadMemindEnvFiles, +} from './scripts/memind-runtime-profile.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq < 0) continue; - const key = trimmed.slice(0, eq).trim(); - const value = trimmed.slice(eq + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} - -loadEnvFile(path.join(__dirname, '../../.env.local')); -loadEnvFile(path.join(__dirname, '.env')); +loadMemindEnvFiles(__dirname); +applyMemindRuntimeProfile({ rootDir: __dirname }); function parseApiTargets() { const csvTargets = (process.env.TKMIND_API_TARGETS ?? '') @@ -349,6 +342,15 @@ async function unregisterAgentSessionForUser(userId, sessionId) { await access.unregisterSession({ userId, sessionId }); } let agentRunGateway = null; +const sessionPageDeliveryLocks = new Map(); +function beginSessionPageDelivery(sessionId) { + sessionPageDeliveryLocks.set(sessionId, Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) + 1); +} +function endSessionPageDelivery(sessionId) { + const remaining = Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) - 1; + if (remaining > 0) sessionPageDeliveryLocks.set(sessionId, remaining); + else sessionPageDeliveryLocks.delete(sessionId); +} let chatIntentRouter = null; let toolGateway = null; let directChatService = null; @@ -356,6 +358,7 @@ let sessionSnapshotService = null; let conversationMemoryService = null; let memoryV2 = null; let memoryV2ConfigService = null; +let skillRuntimeConfigService = null; let wechatScheduleLlmConfigService = null; let mindSpace = null; let mindSpaceAssets = null; @@ -639,6 +642,7 @@ async function bootstrapUserAuth() { console.warn('LLM provider boot sync skipped:', err instanceof Error ? err.message : err); }); memoryV2ConfigService = createMemoryV2AdminConfigService(pool); + skillRuntimeConfigService = createSkillRuntimeAdminConfigService(pool, { h5Root: __dirname }); wechatScheduleLlmConfigService = createWechatScheduleLlmConfigService(pool); conversationMemoryService = createConversationMemoryService(pool, { llmProviderService, @@ -705,9 +709,16 @@ async function bootstrapUserAuth() { chatIntentRouter, sessionSnapshotService, conversationMemoryService, - syncUserPagesOnSuccess: async ({ userId }) => { - await syncUserGeneratedPages(userId); + observePersonalMemoryOnSuccess: async ({ userId, sessionId, userMessage }) => { + if (!memoryV2?.observePersonalMemory) return; + await memoryV2.observePersonalMemory({ + userId, + sessionId, + messages: [userMessage], + }); }, + syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId), + isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0, autoDispatch: ['1', 'true', 'yes', 'on'].includes( String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(), ), @@ -851,6 +862,16 @@ app.use(attachUserSession); // ============ Legacy password auth ============ +async function resolveSkillRuntimeForClient() { + if (!skillRuntimeConfigService?.getPublicRuntimeConfig) return null; + try { + return await skillRuntimeConfigService.getPublicRuntimeConfig(); + } catch (err) { + console.warn('[SkillRuntime] public config unavailable:', err instanceof Error ? err.message : err); + return null; + } +} + app.get('/auth/status', async (req, res) => { await userAuthReady; if (userAuth) { @@ -859,6 +880,7 @@ app.get('/auth/status', async (req, res) => { if (!me) return res.json({ authenticated: false, mode: 'user' }); const row = await userAuth.getUserById(me.id); const capabilityState = await userAuth.resolveUserCapabilities(row); + const skillRuntime = await resolveSkillRuntimeForClient(); return res.json({ authenticated: true, user: me, @@ -866,6 +888,7 @@ app.get('/auth/status', async (req, res) => { capabilities: capabilityState.capabilities, grantedSkills: capabilityState.grantedSkills ?? [], unrestricted: capabilityState.unrestricted, + skillRuntime, }); } catch (err) { console.error('[Auth] status failed:', err instanceof Error ? err.message : err); @@ -1309,10 +1332,11 @@ app.get('/auth/me', async (req, res) => { if (!userAuth) return res.status(503).json({ message: '未启用用户系统' }); const me = await userAuth.getMe(userToken(req)); if (!me) return res.status(401).json({ message: '未登录' }); - const [paths, capabilityState, subscription] = await Promise.all([ + const [paths, capabilityState, subscription, skillRuntime] = await Promise.all([ userAuth.listPathGrants(me.id), userAuth.resolveUserCapabilities(await userAuth.getUserById(me.id)), subscriptionService ? subscriptionService.getActiveSubscription(me.id) : null, + resolveSkillRuntimeForClient(), ]); return res.json({ user: { ...me, subscription }, @@ -1320,6 +1344,7 @@ app.get('/auth/me', async (req, res) => { capabilities: capabilityState.capabilities, grantedSkills: capabilityState.grantedSkills ?? [], unrestricted: capabilityState.unrestricted, + skillRuntime, }); }); @@ -3209,11 +3234,10 @@ const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']); async function syncUserGeneratedPages(userId) { if (!userId) return; if (workspacePageDeliver?.syncAndDeliver) { - await workspacePageDeliver.syncAndDeliver(userId); - return; + return await workspacePageDeliver.syncAndDeliver(userId); } if (!mindSpacePageSync) return; - await mindSpacePageSync.syncUserGeneratedPages(userId); + return await mindSpacePageSync.syncUserGeneratedPages(userId); } async function resolveChatSaveBundle(user, h5Root, input = {}) { @@ -5004,6 +5028,8 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { // workspace HTML into the asset store before a later restart rebuilds the // workspace from DB-backed assets only. const onAfterFinish = async (sid, uid) => { + beginSessionPageDelivery(sid); + try { const apiFetchFn = async (pathname, init) => { const target = await tkmindProxy.resolveTarget(sid); return tkmindProxy.apiFetchTo(target, pathname, init); @@ -5085,6 +5111,20 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { tkmindProxy, userText: lastUserText, }); + if (lastUserMessage && memoryV2?.observePersonalMemory) { + await memoryV2.observePersonalMemory({ + userId: uid, + sessionId: sid, + messages: [lastUserMessage], + }).catch((err) => { + console.warn( + `[memory-v2] finish shadow observation skipped for session ${sid}: ${err instanceof Error ? err.message : err}`, + ); + }); + } + } finally { + endSessionPageDelivery(sid); + } }; return tkmindProxy.proxySessionEvents(req, res, sessionId, { onAfterFinish, @@ -6233,6 +6273,7 @@ app.get('*', (_req, res) => { process.exit(1); } const server = app.listen(PORT, HOST, () => { + console.log(`[Portal] Runtime profile: ${describeMemindRuntimeProfile()}`); console.log(`TKMind H5 @ http://${HOST}:${PORT}`); console.log(`Proxy -> ${API_TARGETS.join(', ')}`); console.log(`Auth -> ${enabled ? 'multi-user (MySQL)' : legacyAuth ? 'legacy password' : 'disabled'}`); From b33c943b69086a1ba593a28a39b05ca0842bcaa5 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 14:16:57 +0800 Subject: [PATCH 4/5] feat(memory-v2): auto-accept candidate memories in active and canary modes Let Personal Memory candidates pass policy gates without per-user manual review, while keeping shadow mode for explicit human approval. Co-authored-by: Cursor --- memory-v2-personal-shadow.mjs | 48 +++++++++++++-- memory-v2-personal-shadow.test.mjs | 96 +++++++++++++++++++++++++++++- memory-v2-personal-store.mjs | 14 +++-- memory-v2-personal-store.test.mjs | 2 +- 4 files changed, 147 insertions(+), 13 deletions(-) diff --git a/memory-v2-personal-shadow.mjs b/memory-v2-personal-shadow.mjs index c389f2e..a215a79 100644 --- a/memory-v2-personal-shadow.mjs +++ b/memory-v2-personal-shadow.mjs @@ -70,13 +70,34 @@ function classify(text) { return rules.find((rule) => rule.pattern.test(text)) ?? null; } +const CANDIDATE_MODES = new Set(['off', 'shadow', 'canary', 'active']); + +function normalizeCandidateMode(value, fallback = 'shadow') { + const normalized = normalizeText(value, 20).toLowerCase(); + return CANDIDATE_MODES.has(normalized) ? normalized : fallback; +} + +export function shouldAutoAcceptCandidate(candidate, config) { + if (!candidate || !config?.enabled) return false; + const mode = normalizeCandidateMode(config.requestedMode, 'shadow'); + if (mode === 'shadow' || mode === 'off') return false; + if (mode === 'active') return true; + if (candidate.policyReason === 'explicit_memory_request') return true; + return Number(candidate.confidence) >= Math.max(Number(config.minConfidence) || 0, 0.9); +} + export function resolvePersonalShadowConfig(env = process.env) { const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false); - const requestedMode = normalizeText(env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'shadow' : 'off'), 20).toLowerCase(); + const requestedMode = normalizeCandidateMode( + env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'active' : 'off'), + candidateEnabled ? 'active' : 'off', + ); + const enabled = candidateEnabled && requestedMode !== 'off'; return { - enabled: candidateEnabled && requestedMode !== 'off', + enabled, requestedMode, - effectiveMode: candidateEnabled && requestedMode !== 'off' ? 'shadow' : 'off', + effectiveMode: enabled ? requestedMode : 'off', + autoReviewEnabled: enabled && requestedMode !== 'shadow', minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }), minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }), maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })), @@ -96,6 +117,8 @@ export function createPersonalMemoryShadowPipeline({ env = process.env, now = () runs: 0, observedMessages: 0, accepted: 0, + autoReviewed: 0, + pendingReview: 0, rejected: 0, deduped: 0, errors: 0, @@ -178,10 +201,21 @@ export function createPersonalMemoryShadowPipeline({ env = process.env, now = () metrics.observedMessages += 1; const result = evaluate({ userId, sessionId, message, index }); if (result.accepted && store?.saveCandidate) { - const persisted = await store.saveCandidate(result.candidate); + const autoAccept = shouldAutoAcceptCandidate(result.candidate, config); + const persisted = await store.saveCandidate(result.candidate, { autoAccept }); if (persisted?.inserted === false) { metrics.deduped += 1; + } else if (autoAccept) { + result.candidate.status = 'accepted'; + metrics.autoReviewed += 1; + } else { + metrics.pendingReview += 1; } + } else if (result.accepted) { + const autoAccept = shouldAutoAcceptCandidate(result.candidate, config); + result.candidate.status = autoAccept ? 'accepted' : 'candidate'; + if (autoAccept) metrics.autoReviewed += 1; + else metrics.pendingReview += 1; } results.push(result); } @@ -189,8 +223,11 @@ export function createPersonalMemoryShadowPipeline({ env = process.env, now = () enabled: true, skipped: false, mode: config.effectiveMode, + autoReviewEnabled: config.autoReviewEnabled, observed: results.length, accepted: results.filter((result) => result.accepted).length, + autoReviewed: results.filter((result) => result.accepted && result.candidate?.status === 'accepted').length, + pendingReview: results.filter((result) => result.accepted && result.candidate?.status === 'candidate').length, rejected: results.filter((result) => !result.accepted).length, }; } catch (err) { @@ -205,7 +242,8 @@ export function createPersonalMemoryShadowPipeline({ env = process.env, now = () enabled: config.enabled, requestedMode: config.requestedMode, effectiveMode: config.effectiveMode, - phase: 'shadow-candidate-v1', + phase: config.autoReviewEnabled ? 'auto-review-v1' : 'shadow-candidate-v1', + autoReviewEnabled: config.autoReviewEnabled, injectionEnabled: false, persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory', pendingCandidates: candidates.length, diff --git a/memory-v2-personal-shadow.test.mjs b/memory-v2-personal-shadow.test.mjs index be272be..bcbe5ac 100644 --- a/memory-v2-personal-shadow.test.mjs +++ b/memory-v2-personal-shadow.test.mjs @@ -1,16 +1,103 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { createPersonalMemoryShadowPipeline, resolvePersonalShadowConfig } from './memory-v2-personal-shadow.mjs'; +import { + createPersonalMemoryShadowPipeline, + resolvePersonalShadowConfig, + shouldAutoAcceptCandidate, +} from './memory-v2-personal-shadow.mjs'; import { createMemoryV2 } from './memory-v2.mjs'; -test('personal memory shadow config never activates response injection', () => { +test('personal memory shadow config maps active mode to auto review', () => { const config = resolvePersonalShadowConfig({ MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'active', }); assert.equal(config.enabled, true); assert.equal(config.requestedMode, 'active'); + assert.equal(config.effectiveMode, 'active'); + assert.equal(config.autoReviewEnabled, true); +}); + +test('personal memory shadow config keeps shadow mode for manual review', () => { + const config = resolvePersonalShadowConfig({ + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'shadow', + }); assert.equal(config.effectiveMode, 'shadow'); + assert.equal(config.autoReviewEnabled, false); +}); + +test('shouldAutoAcceptCandidate auto accepts explicit requests in canary mode', () => { + const config = resolvePersonalShadowConfig({ + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'canary', + MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8', + }); + assert.equal(shouldAutoAcceptCandidate({ + policyReason: 'explicit_memory_request', + confidence: 0.98, + }, config), true); + assert.equal(shouldAutoAcceptCandidate({ + policyReason: 'stable_fact_signal', + confidence: 0.82, + }, config), false); + assert.equal(shouldAutoAcceptCandidate({ + policyReason: 'decision_signal', + confidence: 0.9, + }, config), true); +}); + +test('shadow pipeline auto accepts durable candidates in active mode', async () => { + const saved = []; + const pipeline = createPersonalMemoryShadowPipeline({ + env: { + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'active', + MEMORY_POLICY_ENABLED: '1', + }, + store: { + async saveCandidate(candidate, options = {}) { + saved.push({ candidate, options }); + return { inserted: true, status: options.autoAccept ? 'accepted' : 'candidate' }; + }, + }, + }); + const result = await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [{ role: 'user', text: '我决定所有生产发布必须从完整 main 分支打包。' }], + }); + assert.equal(result.autoReviewed, 1); + assert.equal(result.pendingReview, 0); + assert.equal(saved.length, 1); + assert.equal(saved[0].options.autoAccept, true); + assert.equal(pipeline.getStatus().autoReviewEnabled, true); + assert.equal(pipeline.getStatus().phase, 'auto-review-v1'); +}); + +test('shadow pipeline keeps low-confidence canary candidates pending review', async () => { + const saved = []; + const pipeline = createPersonalMemoryShadowPipeline({ + env: { + MEMORY_CANDIDATE_ENABLED: '1', + MEMORY_CANDIDATE_MODE: 'canary', + MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8', + }, + store: { + async saveCandidate(candidate, options = {}) { + saved.push({ candidate, options }); + return { inserted: true, status: options.autoAccept ? 'accepted' : 'candidate' }; + }, + }, + }); + await pipeline.observeWrite({ + userId: 'u1', + sessionId: 's1', + messages: [{ role: 'user', text: '项目使用 pgvector 作为向量存储。' }], + }); + assert.equal(saved.length, 1); + assert.equal(saved[0].options.autoAccept, false); + assert.equal(pipeline.getStatus().health.pendingReview, 1); }); test('shadow pipeline extracts durable candidates with evidence and deduplicates', async () => { @@ -24,7 +111,10 @@ test('shadow pipeline extracts durable candidates with evidence and deduplicates MEMORY_CANDIDATE_MIN_IMPORTANCE: '0.7', MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8', }, - now: () => clock += 1, + now: () => { + clock += 1; + return clock; + }, }); const input = { userId: 'u1', diff --git a/memory-v2-personal-store.mjs b/memory-v2-personal-store.mjs index 402b971..a5c8d7c 100644 --- a/memory-v2-personal-store.mjs +++ b/memory-v2-personal-store.mjs @@ -60,13 +60,16 @@ export function createPersonalMemoryCandidateStore(pool, { table = TABLE, now = const quoted = `\`${table}\``; return { - async saveCandidate(candidate) { + async saveCandidate(candidate, { autoAccept = false, reviewedBy = 'system:auto-review' } = {}) { const updatedAt = now(); + const status = autoAccept ? 'accepted' : 'candidate'; + const reviewedAt = autoAccept ? updatedAt : null; + const reviewer = autoAccept ? String(reviewedBy || 'system:auto-review') : null; const [result] = await pool.query( `INSERT IGNORE INTO ${quoted} (id, user_id, session_id, memory_type, content, importance, confidence, status, - policy_reason, evidence_json, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, 'candidate', ?, ?, ?, ?)`, + policy_reason, evidence_json, reviewed_by, reviewed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ candidate.id, candidate.userId, @@ -75,13 +78,16 @@ export function createPersonalMemoryCandidateStore(pool, { table = TABLE, now = candidate.content, candidate.importance, candidate.confidence, + status, candidate.policyReason, JSON.stringify(candidate.evidence ?? {}), + reviewer, + reviewedAt, candidate.createdAt, updatedAt, ], ); - return { inserted: Number(result?.affectedRows ?? 0) > 0 }; + return { inserted: Number(result?.affectedRows ?? 0) > 0, status }; }, async listCandidates({ status = 'candidate', userId = null, limit = 50, offset = 0 } = {}) { diff --git a/memory-v2-personal-store.test.mjs b/memory-v2-personal-store.test.mjs index 60f1ae1..b4ff3f2 100644 --- a/memory-v2-personal-store.test.mjs +++ b/memory-v2-personal-store.test.mjs @@ -37,7 +37,7 @@ test('candidate store saves, lists, counts, and reviews with parameterized SQL', id: 'pmc_1', userId: 'u1', sessionId: 's1', memoryType: 'episodic', content: '决定使用 main 发布', importance: 0.9, confidence: 0.9, policyReason: 'decision_signal', evidence: { sourceId: 's1:0' }, createdAt: 1, - }), { inserted: true }); + }, { autoAccept: true }), { inserted: true, status: 'accepted' }); const rows = await store.listCandidates({ limit: 20 }); assert.equal(rows[0].evidence.sourceId, 's1:0'); assert.deepEqual(await store.countByStatus(), { candidate: 1 }); From a6620fb7193d818f748da83698abf471fbbda41e Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jul 2026 15:29:29 +0800 Subject: [PATCH 5/5] feat(mindspace): enforce PostgreSQL user data delivery --- .env.example | 29 +- agent-run-deliverable-check.mjs | 41 +- agent-run-deliverable-check.test.mjs | 34 ++ agent-run-gateway.mjs | 61 ++- agent-run-gateway.test.mjs | 266 ++++++++++- capabilities.mjs | 2 +- capabilities.test.mjs | 10 + chat-intent-router.mjs | 10 +- chat-intent-router.test.mjs | 32 +- chat-skills.mjs | 25 +- chat-skills.test.mjs | 27 ++ docs/local-dev.md | 30 ++ ...pace-browser-storage-pg-migration-local.md | 40 ++ mindspace-browser-storage-policy.mjs | 60 +++ mindspace-h5-html-finish-guard.mjs | 42 +- mindspace-h5-html-finish-guard.test.mjs | 55 +++ mindspace-page-data-finish-guard.mjs | 45 +- mindspace-page-data-finish-guard.test.mjs | 52 ++- mindspace-page-sync-service.mjs | 15 +- mindspace-page-sync.mjs | 11 +- mindspace-page-sync.test.mjs | 34 ++ mindspace-public-finish-sync.mjs | 1 + mindspace-public-finish-sync.test.mjs | 1 + mindspace-public-route.test.mjs | 29 ++ mindspace-sandbox-mcp.mjs | 13 +- mindspace-sandbox-mcp.test.mjs | 35 ++ mindspace-userdata-postgres.mjs | 439 ++++++++++++++++++ mindspace-userdata-postgres.test.mjs | 106 +++++ ...-workspace-page-deliver-page-data.test.mjs | 9 +- mindspace-workspace-page-deliver.mjs | 14 +- mindspace-workspace-sync.mjs | 21 +- mindspace-workspace-sync.test.mjs | 6 +- page-data-delivery-assess.mjs | 54 ++- page-data-delivery-assess.test.mjs | 38 ++ page-data-public-service.mjs | 38 +- page-data-service.mjs | 9 +- page-data-workspace-bind.mjs | 35 +- page-data-workspace-ensure.mjs | 2 +- page-data-workspace-ensure.test.mjs | 8 +- postgres-user-data-space-service.mjs | 391 ++++++++++++++++ scripts/dev-core-daemon.mjs | 25 +- scripts/dev-core.mjs | 23 +- scripts/load-env.mjs | 23 +- scripts/memind-runtime-profile.mjs | 119 +++++ scripts/memind-runtime-profile.test.mjs | 54 +++ .../migrate-mindspace-sqlite-to-postgres.mjs | 85 ++++ ...rate-mindspace-sqlite-to-postgres.test.mjs | 25 + scripts/verify-local-pg-questionnaire-e2e.mjs | 198 ++++++++ server.mjs | 62 ++- skills/page-data-collect/SKILL.md | 23 +- skills/static-page-publish/SKILL.md | 1 + user-auth.mjs | 37 ++ user-auth.test.mjs | 26 ++ user-data-space-service.mjs | 24 + user-data-space-service.test.mjs | 21 + user-publish.mjs | 16 + user-publish.test.mjs | 11 + 57 files changed, 2779 insertions(+), 164 deletions(-) create mode 100644 docs/mindspace-browser-storage-pg-migration-local.md create mode 100644 mindspace-browser-storage-policy.mjs create mode 100644 mindspace-userdata-postgres.mjs create mode 100644 mindspace-userdata-postgres.test.mjs create mode 100644 postgres-user-data-space-service.mjs create mode 100644 scripts/memind-runtime-profile.mjs create mode 100644 scripts/memind-runtime-profile.test.mjs create mode 100644 scripts/migrate-mindspace-sqlite-to-postgres.mjs create mode 100644 scripts/migrate-mindspace-sqlite-to-postgres.test.mjs create mode 100644 scripts/verify-local-pg-questionnaire-e2e.mjs diff --git a/.env.example b/.env.example index 27f31d1..6d9c654 100644 --- a/.env.example +++ b/.env.example @@ -242,4 +242,31 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind # MINDSPACE_AGENT_JOBS_ENABLED=true # MINDSPACE_INTERNAL_AGENT_SECRET=local-dev-secret # MINDSPACE_FREE_PUBLIC_PAGE_LIMIT=10 -# MINDSPACE_STORAGE_ROOT=/path/to/mindspace-storage + +# --------------------------------------------------------------------------- +# Memind runtime profile（本地 vs 生产 必须区分） +# 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。 +# +# | profile | 场景 | MindSpace 数据路径 | +# |----------------|-------------------|--------------------| +# | local | 本机日常开发 | /data/mindspace | +# | split-service | 本机验证 103 拆分 | /Users/john/MindSpace/data/mindspace | +# | production | 103 生产服务器 | 103 .env 为准，勿提交 Git | +# --------------------------------------------------------------------------- + +# 【本地开发 — 复制到 .env 的默认值】 +MEMIND_RUNTIME_PROFILE=local +# MINDSPACE_STORAGE_ROOT 留空即可，自动使用 /data/mindspace + +# 【本机 split-service 联调 — 需要独立 MindSpace LaunchAgent 已启动】 +# MEMIND_RUNTIME_PROFILE=split-service +# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +# MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret +# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace + +# 【103 生产 — 仅在服务器维护，示例勿照抄密钥】 +# MEMIND_RUNTIME_PROFILE=production +# MINDSPACE_SERVER_ADAPTER=remote +# MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +# MINDSPACE_REMOTE_AUTH_TOKEN=<与 cn.tkmind.mindspace-service 一致> +# MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace diff --git a/agent-run-deliverable-check.mjs b/agent-run-deliverable-check.mjs index dfe5685..eeba16c 100644 --- a/agent-run-deliverable-check.mjs +++ b/agent-run-deliverable-check.mjs @@ -26,6 +26,7 @@ function mapDeliverableRows(rows) { publicationId: row.publication_id ? String(row.publication_id) : null, publicationStatus: row.publication_status ? String(row.publication_status) : null, publicUrl: row.public_url ? String(row.public_url) : null, + workspaceRelativePath: row.workspace_relative_path ? String(row.workspace_relative_path) : null, })).filter((row) => row.pageId); const publicationCount = pages.filter((row) => row.publicationId).length; return { @@ -51,13 +52,16 @@ export function mergeDeliverableSummaries(...summaries) { }; } -export async function detectSessionDeliverables(pool, userId, sessionId) { +export async function detectSessionDeliverables(pool, userId, sessionId, { sinceMs = null } = {}) { if (!pool?.query || !userId || !sessionId) { return { pageCount: 0, publicationCount: 0, pages: [] }; } + const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : ''; + const params = sinceMs != null ? [userId, sessionId, sinceMs] : [userId, sessionId]; const [rows] = await pool.query( `SELECT p.id AS page_id, p.title, + p.workspace_relative_path, pr.id AS publication_id, pr.status AS publication_status, pr.public_url @@ -68,22 +72,39 @@ export async function detectSessionDeliverables(pool, userId, sessionId) { AND pr.status = 'online' WHERE p.user_id = ? AND p.source_session_id = ? + ${sinceClause} AND p.status <> 'deleted' ORDER BY p.created_at ASC`, - [userId, sessionId], + params, ); return mapDeliverableRows(rows); } -export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs = null } = {}) { +export async function detectWorkspaceSyncedDeliverables( + pool, + userId, + { sinceMs = null, onlyRelativePaths = null } = {}, +) { if (!pool?.query || !userId) { return { pageCount: 0, publicationCount: 0, pages: [] }; } + const normalizedPaths = onlyRelativePaths == null + ? null + : [...new Set(onlyRelativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))]; + if (normalizedPaths?.length === 0) { + return { pageCount: 0, publicationCount: 0, pages: [] }; + } const sinceClause = sinceMs != null ? 'AND p.updated_at >= ?' : ''; - const params = sinceMs != null ? [userId, sinceMs] : [userId]; + const pathClause = normalizedPaths == null + ? '' + : `AND p.workspace_relative_path IN (${normalizedPaths.map(() => '?').join(', ')})`; + const params = [userId]; + if (sinceMs != null) params.push(sinceMs); + if (normalizedPaths != null) params.push(...normalizedPaths); const [rows] = await pool.query( `SELECT p.id AS page_id, p.title, + p.workspace_relative_path, pr.id AS publication_id, pr.status AS publication_status, pr.public_url @@ -96,6 +117,7 @@ export async function detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs WHERE p.user_id = ? AND p.status <> 'deleted' ${sinceClause} + ${pathClause} AND ( p.workspace_relative_path LIKE 'public/%.html' OR JSON_UNQUOTE(JSON_EXTRACT(pv.source_snapshot_json, '$.relative_path')) LIKE 'public/%.html' @@ -116,15 +138,20 @@ export async function prepareAndDetectSessionDeliverables({ sessionId, runStartedAtMs = null, prepareDeliverables = null, + workspaceRelativePaths = null, } = {}) { + let prepared = null; if (typeof prepareDeliverables === 'function') { - await prepareDeliverables({ userId, sessionId }).catch(() => {}); + prepared = await prepareDeliverables({ userId, sessionId }).catch(() => null); } const sinceMs = runStartedAtMs == null ? null : Math.max(0, Number(runStartedAtMs) - WORKSPACE_DELIVERABLE_LOOKBACK_MS); - const bySession = await detectSessionDeliverables(pool, userId, sessionId); - const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { sinceMs }); + const bySession = await detectSessionDeliverables(pool, userId, sessionId, { sinceMs }); + const byWorkspace = await detectWorkspaceSyncedDeliverables(pool, userId, { + sinceMs, + onlyRelativePaths: workspaceRelativePaths ?? prepared?.pageDataRelativePaths ?? null, + }); return mergeDeliverableSummaries(bySession, byWorkspace); } diff --git a/agent-run-deliverable-check.test.mjs b/agent-run-deliverable-check.test.mjs index 8d3c03d..b335e63 100644 --- a/agent-run-deliverable-check.test.mjs +++ b/agent-run-deliverable-check.test.mjs @@ -98,6 +98,40 @@ test('detectWorkspaceSyncedDeliverables filters auto-synced public html pages', assert.equal(summary.pageCount, 1); }); +test('detectWorkspaceSyncedDeliverables stays inside current run artifact paths', async () => { + const pool = { + async query(sql, params) { + assert.match(sql, /workspace_relative_path IN \(\?, \?\)/); + assert.deepEqual(params, [ + 'user-1', + 1000, + 'public/current.html', + 'public/current-admin.html', + ]); + return [[{ + page_id: 'page-current', + title: 'Current', + workspace_relative_path: 'public/current.html', + }]]; + }, + }; + const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', { + sinceMs: 1000, + onlyRelativePaths: ['public/current.html', 'public/current-admin.html'], + }); + assert.equal(summary.pageCount, 1); + assert.equal(summary.pages[0].workspaceRelativePath, 'public/current.html'); +}); + +test('detectWorkspaceSyncedDeliverables skips workspace sweep for an empty run scope', async () => { + const pool = { async query() { throw new Error('must not query'); } }; + const summary = await detectWorkspaceSyncedDeliverables(pool, 'user-1', { + sinceMs: 1000, + onlyRelativePaths: [], + }); + assert.equal(summary.pageCount, 0); +}); + test('prepareAndDetectSessionDeliverables merges session and workspace pages after prepare hook', async () => { const prepareCalls = []; const pool = { diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index b7d6fe3..e3a01dd 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -16,7 +16,7 @@ import { SESSION_FINISHED_STALE_GRACE_MS, tryRecoverRunFromDeliverables, } from './agent-run-deliverable-check.mjs'; -import { isPageDataIntent } from './chat-skills.mjs'; +import { isPageDataIntent, isPageGenerationIntent } from './chat-skills.mjs'; const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000]; const TERMINAL_STATUSES = new Set(['succeeded', 'failed']); @@ -262,6 +262,7 @@ export function createAgentRunGateway({ syncUserPagesOnSuccess = null, observePersonalMemoryOnSuccess = null, isSessionExternallyBusy = null, + validateRunDeliverables = null, retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS, autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true), maxConcurrentRuns = positiveInteger( @@ -420,7 +421,7 @@ export function createAgentRunGateway({ async function prepareRunDeliverables({ userId, sessionId, runId, runStartedAtMs = null }) { if (typeof syncUserPagesOnSuccess !== 'function') return; - await syncUserPagesOnSuccess({ + return await syncUserPagesOnSuccess({ userId, sessionId, runId, @@ -619,7 +620,7 @@ export function createAgentRunGateway({ billed: Boolean(result.billing?.ok), }); await appendRunSnapshot(runId); - return { sessionId: result.sessionId }; + return { sessionId: result.sessionId, routing }; } catch (err) { await appendEvent(runId, 'direct_chat_failed', { code: err?.code ?? null, @@ -673,7 +674,7 @@ export function createAgentRunGateway({ }); throw err; } - return { sessionId: row.agent_session_id ?? null }; + return { sessionId: row.agent_session_id ?? null, routing }; } let sessionId = resolveGatewayAgentSessionId({ @@ -769,7 +770,7 @@ export function createAgentRunGateway({ sessionId, err, })) { - return { sessionId }; + return { sessionId, routing }; } throw err; } @@ -785,16 +786,17 @@ export function createAgentRunGateway({ }, ); } - return { sessionId }; + return { sessionId, routing }; } - async function finalizeSuccessfulRun(runId, row, sessionId) { + async function finalizeSuccessfulRun(runId, row, sessionId, { routing = null } = {}) { let deliveryResult = null; if (typeof syncUserPagesOnSuccess === 'function') { deliveryResult = await syncUserPagesOnSuccess({ userId: row.user_id, sessionId, runId, + runStartedAtMs: row.started_at ?? null, }); } const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors) @@ -806,17 +808,48 @@ export function createAgentRunGateway({ error.retryable = false; throw error; } - if (isPageDataIntent(extractRunMessageText(row))) { + const runMessageText = extractRunMessageText(row); + const pageDataIntent = isPageDataIntent(runMessageText) + || routing?.suggestedSkill === 'page-data-collect'; + const pageGenerationIntent = isPageGenerationIntent(runMessageText) + || routing?.suggestedSkill === 'static-page-publish'; + const requiresPageDeliverable = pageDataIntent || pageGenerationIntent; + let deliverables = null; + if (requiresPageDeliverable || typeof validateRunDeliverables === 'function') { const latest = await getRunById(runId); - const deliverables = await prepareAndDetectSessionDeliverables({ + deliverables = await prepareAndDetectSessionDeliverables({ pool, userId: row.user_id, sessionId, runStartedAtMs: latest?.started_at ?? row.started_at ?? null, + workspaceRelativePaths: deliveryResult?.pageDataRelativePaths ?? [], }); - if (deliverables.pageCount < 1) { - const error = new Error('Page Data 任务未生成可交付页面，不能标记成功'); - error.code = 'PAGE_DATA_DELIVERABLE_MISSING'; + if (requiresPageDeliverable && deliverables.pageCount < 1) { + const error = new Error( + pageDataIntent + ? 'Page Data 任务未生成可交付页面，不能标记成功' + : '页面任务未生成 public HTML 交付物，不能标记成功', + ); + error.code = pageDataIntent + ? 'PAGE_DATA_DELIVERABLE_MISSING' + : 'PUBLIC_PAGE_DELIVERABLE_MISSING'; + error.retryable = false; + throw error; + } + } + if (typeof validateRunDeliverables === 'function') { + const validation = await validateRunDeliverables({ + userId: row.user_id, + sessionId, + runId, + deliverables: deliverables ?? { pageCount: 0, publicationCount: 0, pages: [] }, + }); + const errors = Array.isArray(validation?.errors) ? validation.errors : []; + if (errors.length > 0) { + const error = new Error( + `页面交付违反数据存储策略：${errors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`, + ); + error.code = 'DELIVERABLE_DATA_STORAGE_FORBIDDEN'; error.retryable = false; throw error; } @@ -856,8 +889,8 @@ export function createAgentRunGateway({ const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt }); try { - const { sessionId } = await runWithTimeout(runId, () => executeRun(row, runId)); - await finalizeSuccessfulRun(runId, row, sessionId); + const { sessionId, routing } = await runWithTimeout(runId, () => executeRun(row, runId)); + await finalizeSuccessfulRun(runId, row, sessionId, { routing }); } catch (err) { const latest = await getRunById(runId); const recoverySessionId = latest?.agent_session_id ?? row.agent_session_id ?? null; diff --git a/agent-run-gateway.test.mjs b/agent-run-gateway.test.mjs index dc4872b..d426ee7 100644 --- a/agent-run-gateway.test.mjs +++ b/agent-run-gateway.test.mjs @@ -387,6 +387,191 @@ test('agent run awaits session Finish before succeeding when proxy supports it', assert.equal(finishEvents.length, 1); }); +test('Page Data run fails closed when Finish arrives without a generated page', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-page-data-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-page-data-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我创建调查问卷，保存提交记录并发布页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /未生成可交付页面/); +}); + +test('implicit sticky-note app run fails closed when Apps returns Finish without a public page', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + chatIntentRouter: { + isEnabled() { + return true; + }, + async classify() { + return { + route: 'agent_orchestration', + confidence: 0.96, + reason: '页面需要数据交互与持久化', + suggestedSkill: 'page-data-collect', + source: 'rule', + }; + }, + applyAgentOrchestration(message) { + return message; + }, + }, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-sticky-note-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-sticky-note-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我设计一个便签提醒，可以写便签提交，时间轴来显示' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /Page Data 任务未生成可交付页面/); + assert.ok(pool.events.some((event) => event.eventType === 'intent_routed')); +}); + +test('static page run fails closed when Finish arrives without public HTML', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-public-page-missing' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({}), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-public-page-missing', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我做一个秋夜诗的 H5 页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /public HTML 交付物/); +}); + +test('Page Data run succeeds only after a generated session page is detected', async () => { + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:session-page-data-ready': [{ + page_id: 'page-ready', + title: '问卷', + publication_id: 'pub-ready', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/u/john/pages/page-ready', + }], + }, + }); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-page-data-ready' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + syncUserPagesOnSuccess: async () => ({ pageDataBind: { errors: [] } }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-page-data-ready', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '创建一个可以保存提交记录的问卷' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded'); +}); + +test('agent run fails closed when a generated page violates browser storage policy', async () => { + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:session-browser-storage': [{ + page_id: 'page-storage', + title: '页面', + workspace_relative_path: 'public/page.html', + }], + }, + }); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: { + async startSessionForUser() { + return { id: 'session-browser-storage' }; + }, + async submitSessionReplyAndAwaitFinishForUser() { + return { ok: true, finishEvent: { type: 'Finish' } }; + }, + }, + validateRunDeliverables: async ({ deliverables }) => ({ + errors: deliverables.pages.some((page) => page.workspaceRelativePath === 'public/page.html') + ? [{ code: 'browser_storage_forbidden', message: 'public/page.html 使用 localStorage' }] + : [], + }), + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-browser-storage', + userMessage: { + role: 'user', + content: [{ type: 'text', text: '帮我做一个展示页面' }], + }, + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.match(pool.runs.get(run.id).error_message, /页面交付违反数据存储策略/); + assert.match(pool.runs.get(run.id).error_message, /localStorage/); +}); + test('agent run succeeds when Finish is missing but session pages were already created', async () => { const pool = createFakePool({ sessionDeliverables: { @@ -563,7 +748,17 @@ test('agent run uses direct chat on regular agent sessions when llm routes direc }); test('agent run invalidates portal direct chat snapshot before submitting to goosed', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:20260704_31': [{ + page_id: 'page-essay', + title: '散文页面', + publication_id: 'pub-essay', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/essay.html', + }], + }, + }); const submitted = []; const invalidated = []; const gateway = createAgentRunGateway({ @@ -723,7 +918,17 @@ test('agent run falls back to backend session when router is disabled', async () }); test('agent run uses chat intent router to enrich agent orchestration messages', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:agent-session-1': [{ + page_id: 'page-router-agent', + title: '路由页面', + publication_id: 'pub-router-agent', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/router.html', + }], + }, + }); const submitted = []; const gateway = createAgentRunGateway({ pool, @@ -790,7 +995,17 @@ test('agent run uses chat intent router to enrich agent orchestration messages', }); test('agent run escalates direct sessions to a new backend session when forced', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:deep-session-1': [{ + page_id: 'page-deep', + title: '深度页面', + publication_id: 'pub-deep', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html', + }], + }, + }); const submitted = []; const gateway = createAgentRunGateway({ pool, @@ -820,7 +1035,17 @@ test('agent run escalates direct sessions to a new backend session when forced', }); test('agent run persists direct session transcript before escalating to goosed', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:deep-session-1': [{ + page_id: 'page-deep-transcript', + title: '深度页面', + publication_id: 'pub-deep-transcript', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/a.html', + }], + }, + }); const submitted = []; const saved = []; const removed = []; @@ -914,7 +1139,17 @@ test('agent run rejects reused goosed session when broker ownership check fails' }); test('agent run persists portal direct snapshot before goosed submit on same session', async () => { - const pool = createFakePool(); + const pool = createFakePool({ + sessionDeliverables: { + 'user-1:20260705_2': [{ + page_id: 'page-report', + title: '报告页面', + publication_id: 'pub-report', + publication_status: 'online', + public_url: 'http://127.0.0.1:5173/MindSpace/user-1/public/report.html', + }], + }, + }); const submitted = []; const saved = []; const gateway = createAgentRunGateway({ @@ -1792,6 +2027,27 @@ test('createRun rejects with SESSION_RUN_CONFLICT when same session already has assert.equal(run3.requestId, 'req-conflict-3'); }); +test('createRun rejects while the same session is finishing page delivery', async () => { + const pool = createFakePool(); + const gateway = createAgentRunGateway({ + pool, + userAuth: {}, + tkmindProxy: {}, + autoDispatch: false, + isSessionExternallyBusy: ({ sessionId }) => sessionId === 'sess-repairing', + }); + + await assert.rejects( + gateway.createRun('user-1', { + sessionId: 'sess-repairing', + requestId: 'req-during-repair', + userMessage: { role: 'user', content: [{ type: 'text', text: '继续' }] }, + }), + (err) => err?.code === 'SESSION_RUN_CONFLICT' && err?.status === 409 && /自动修复/.test(err.message), + ); + assert.equal(pool.runs.size, 0); +}); + test('createRun does not apply per-session conflict check for direct-chat sessions', async () => { const pool = createFakePool(); const gateway = createAgentRunGateway({ diff --git a/capabilities.mjs b/capabilities.mjs index f3c722a..d400b2e 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -161,7 +161,7 @@ export const CAPABILITY_CATALOG = [ ]; /** Capabilities that must never be enabled for regular users, even via DB overrides. */ -export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin']); +export const USER_NON_GRANTABLE_CAPABILITIES = new Set(['extension_admin', 'apps']); export function clampUserCapabilities(capabilities) { const clamped = { ...capabilities }; diff --git a/capabilities.test.mjs b/capabilities.test.mjs index 2b1dd1d..1e4966a 100644 --- a/capabilities.test.mjs +++ b/capabilities.test.mjs @@ -164,6 +164,16 @@ test('clampUserCapabilities blocks extension_admin even when stored as true', () assert.equal(clamped.shell, true); }); +test('clampUserCapabilities blocks Apps for regular H5 users even when granted by role', () => { + const clamped = clampUserCapabilities({ + ...DEFAULT_USER_CAPABILITIES, + apps: true, + }); + assert.equal(clamped.apps, false); + const policy = buildAgentExtensionPolicy(clamped); + assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'apps'), false); +}); + test('catalog keys are unique', () => { const keys = CAPABILITY_CATALOG.map((item) => item.key); assert.equal(keys.length, new Set(keys).size); diff --git a/chat-intent-router.mjs b/chat-intent-router.mjs index d8e9a8b..e9acdff 100644 --- a/chat-intent-router.mjs +++ b/chat-intent-router.mjs @@ -888,7 +888,10 @@ export function createChatIntentRouter(options = {}) { function getStatus() { return { - enabled: Boolean(policy.enabled), + enabled: true, + ruleRoutingEnabled: true, + llmRoutingEnabled: false, + configuredLlmEnabled: Boolean(policy.enabled), modelProviderKeyId: policy.modelProviderKeyId ?? null, model: policy.model ?? null, modelApiType: policy.modelApiType ?? 'chat', @@ -903,8 +906,9 @@ export function createChatIntentRouter(options = {}) { } function isEnabled() { - // Skill + rule routing only; LLM intent router is intentionally disabled. - return false; + // Rule + skill routing is always active. The LLM classifier stays disabled + // independently; returning false here would make the gateway skip rules too. + return true; } async function resolveRouterContext({ userId, sessionId, text, forceDeepReasoning = false }) { diff --git a/chat-intent-router.test.mjs b/chat-intent-router.test.mjs index 374cea7..1cbdc30 100644 --- a/chat-intent-router.test.mjs +++ b/chat-intent-router.test.mjs @@ -67,6 +67,34 @@ test('classifyWithRules routes page data collection to page-data-collect', () => assert.equal(result.suggestedSkill, 'page-data-collect'); }); +test('classifyWithRules routes ordinary household ledger language to page-data-collect', () => { + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + const result = classifyWithRules({ + text, + userMessage: { + role: 'user', + content: [{ type: 'text', text }], + metadata: { displayText: text }, + }, + }); + assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT); + assert.equal(result.suggestedSkill, 'page-data-collect'); +}); + +test('classifyWithRules routes implicit sticky-note timeline apps to page-data-collect', () => { + const text = '帮我设计一个便签提醒，可以写便签提交，时间轴来显示'; + const result = classifyWithRules({ + text, + userMessage: { + role: 'user', + content: [{ type: 'text', text }], + metadata: { displayText: text }, + }, + }); + assert.equal(result.route, CHAT_INTENT_ROUTE.AGENT); + assert.equal(result.suggestedSkill, 'page-data-collect'); +}); + test('coercePageDataSkill overrides static-page-publish when data intent is present', () => { const text = '帮我在这个页面增加调查问卷，密码 888 查看提交记录'; const coerced = coercePageDataSkill( @@ -579,9 +607,9 @@ test('createManagedChatIntentRouter hot-loads admin config and stays skill-only' }, }); - assert.equal(await router.isEnabled(), false); + assert.equal(await router.isEnabled(), true); states.shift(); - assert.equal(await router.isEnabled(), false); + assert.equal(await router.isEnabled(), true); const result = await router.classify({ userMessage: { role: 'user', diff --git a/chat-skills.mjs b/chat-skills.mjs index 7d49846..4fa3bd3 100644 --- a/chat-skills.mjs +++ b/chat-skills.mjs @@ -43,10 +43,31 @@ const PAGE_DATA_INTENT_PATTERNS = [ /(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u, ]; +const INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN = /(?:便签|便利贴|备忘录|待办|清单)/u; +const INTERACTIVE_PAGE_DATA_SURFACE_PATTERN = /(?:应用|app|页面|网页|H5|h5|HTML|html|时间轴|看板)/u; +const INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS = [ + /(?:写|填写|提交|保存)/u, + /(?:添加|新增|创建)/u, + /(?:编辑|修改)/u, + /(?:删除|完成)/u, + /(?:筛选|统计|汇总|管理)/u, + /(?:提醒时间|优先级|分类标签|时间轴)/u, +]; + +function isInteractivePageDataIntent(text) { + if (!INTERACTIVE_PAGE_DATA_SUBJECT_PATTERN.test(text)) return false; + const featureCount = INTERACTIVE_PAGE_DATA_FEATURE_PATTERNS.reduce( + (count, pattern) => count + (pattern.test(text) ? 1 : 0), + 0, + ); + return INTERACTIVE_PAGE_DATA_SURFACE_PATTERN.test(text) || featureCount >= 2; +} + export function isPageDataIntent(text) { const normalized = String(text ?? '').trim(); if (!normalized) return false; - return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized)); + return PAGE_DATA_INTENT_PATTERNS.some((pattern) => pattern.test(normalized)) + || isInteractivePageDataIntent(normalized); } export function isPageGenerationIntent(text) { @@ -161,7 +182,7 @@ export function buildChatSkillPrompt(promptKey, skillName) { `请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能：在 MindSpace 页面中实现可提交、可持久化的数据收集（问卷/报名/台账等），必须使用 Page Data API。` + '先匹配技能内能力分支（默认 A：匿名前台 public insert + 独立后台 password read，口令默认 88888888）；“帮我做/创建”本身就是创建授权，展示简短方案摘要后必须同一轮继续执行，只有互斥需求或非法口令才追问。' + '流程：load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html（含 /assets/page-data-client.js）→ private_data_bind_workspace_page 发布并写策略。' + - '禁止 localStorage / 浏览器本地存储 fallback；禁止自建 Express/独立端口（如 8899）、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl（/MindSpace/<用户ID>/public/...），禁止给用户 /u/用户名/pages/... 链接，并说明后台入口与口令。' + '当前 session 的 private_data_* 已绑定当前用户专属 PostgreSQL schema；所有需保存或再次读取的数据必须走 PG + Page Data API。禁止 localStorage、sessionStorage、IndexedDB、SQLite、静态 JSON/JS 或内存持久化 fallback；禁止自建 Express/独立端口（如 8899）、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl（/MindSpace/<用户ID>/public/...），禁止给用户 /u/用户名/pages/... 链接，并说明后台入口与口令。' ); case 'service-integration-smoke': return `请使用 ${skillName ?? 'service-integration-smoke'} 技能：按标准联调流程检查当前服务，覆盖身份、普通聊天、记忆读取，以及我本轮明确要求验证的技能/发布链路，并输出通过项、失败项、待确认项：`; diff --git a/chat-skills.test.mjs b/chat-skills.test.mjs index 4ad38f6..8dfe9ca 100644 --- a/chat-skills.test.mjs +++ b/chat-skills.test.mjs @@ -107,9 +107,36 @@ test('buildAutoChatSkillPrefix prefers page-data-collect for survey requests', ( test('isPageDataIntent matches survey and backend keywords', () => { assert.equal(isPageDataIntent('增加调查问卷和后台入口，密码 888'), true); + assert.equal( + isPageDataIntent('帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'), + true, + ); assert.equal(isPageDataIntent('帮我做一个苏州攻略页面'), false); }); +test('isPageDataIntent matches implicit interactive sticky-note app requests', () => { + const text = '帮我设计一个便签提醒，可以写便签提交，时间轴来显示'; + assert.equal(isPageDataIntent(text), true); + assert.match( + buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']), + /page-data-collect/, + ); +}); + +test('isPageDataIntent does not steal a simple schedule reminder', () => { + assert.equal(isPageDataIntent('明天上午9点提醒我开会'), false); + assert.equal(isPageDataIntent('创建一个待办'), false); +}); + +test('buildAutoChatSkillPrefix routes ordinary household ledger language to Page Data', () => { + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + const prefix = buildAutoChatSkillPrefix(text, ['page-data-collect', 'static-page-publish']); + assert.match(prefix, /page-data-collect/); + assert.match(prefix, /同一轮继续执行/); + assert.match(prefix, /禁止 localStorage/); + assert.doesNotMatch(prefix, /static-page-publish 技能/); +}); + test('buildAutoChatSkillPrefix enables publish for implicit page requests', () => { const prefix = buildAutoChatSkillPrefix('苏州攻略页面', ['static-page-publish']); assert.match(prefix, /static-page-publish/); diff --git a/docs/local-dev.md b/docs/local-dev.md index 6016038..b712856 100644 --- a/docs/local-dev.md +++ b/docs/local-dev.md @@ -63,6 +63,36 @@ pnpm dev:all | `PLAZA_APP_DIR` | ../memind_plaza | Plaza 专用脚本的源码路径（`pnpm dev:plaza` / `pnpm start:plaza` / `pnpm dev:all`） | | `MEMIND_SESSION_BROKER_ENABLED` | `0`（未设置） | Session Broker 灰度开关；Patch 2+ 本地验证时可设 `1`，见 [H5 Session 架构](./h5-session-architecture-20260706.md) | +### MindSpace：本地 vs 生产（必须区分） + +`pnpm dev` / `server.mjs` 启动时会读取 `MEMIND_RUNTIME_PROFILE`（见 `scripts/memind-runtime-profile.mjs`）： + +| `MEMIND_RUNTIME_PROFILE` | 用途 | MindSpace 实现 | +|--------------------------|------|----------------| +| `local`（**默认**） | 本机日常开发 | Portal 内置 `MINDSPACE_SERVER_ADAPTER=local` | +| `split-service` | 本机验证 103 拆分架构 | `remote` → `http://127.0.0.1:8082` 独立 MindSpace 服务 | +| `production` | **仅 103 服务器** | 不覆盖 `.env`，以服务器配置为准 | + +**本地 `.env` 推荐：** + +```bash +MEMIND_RUNTIME_PROFILE=local +``` + +**本机要测 split-service 时：** + +```bash +MEMIND_RUNTIME_PROFILE=split-service +MINDSPACE_REMOTE_BASE_URL=http://127.0.0.1:8082 +MINDSPACE_REMOTE_AUTH_TOKEN=local-dev-secret # 必须与 MindSpace LaunchAgent 一致 +launchctl kickstart -k "gui/$(id -u)/cn.tkmind.mindspace-service" +pnpm dev +``` + +启动后 Portal 日志应出现：`[Portal] Runtime profile: MEMIND_RUNTIME_PROFILE=local, MINDSPACE_SERVER_ADAPTER=local, ...` + +**禁止：** 把 103 生产 `.env`、RDS 连接串、`MINDSPACE_REMOTE_AUTH_TOKEN` 生产值复制进本机 Git 仓库。 + ### H5 Session 架构本地全量联调（`0706-bug干净` 分支） 在 **本机 `.env`**（勿提交 Git）打开下列开关后重启 `pnpm dev`： diff --git a/docs/mindspace-browser-storage-pg-migration-local.md b/docs/mindspace-browser-storage-pg-migration-local.md new file mode 100644 index 0000000..7bb6853 --- /dev/null +++ b/docs/mindspace-browser-storage-pg-migration-local.md @@ -0,0 +1,40 @@ +# MindSpace 浏览器存储迁移清单（仅本机） + +更新时间：2026-07-13 + +## 当前策略 + +- 新建或修改的 Agent 页面禁止使用 `localStorage`、`sessionStorage`、`IndexedDB` 持久化数据。 +- 新 session 明确绑定当前用户的独立 PostgreSQL schema；持久化数据必须经 `private_data_*` 和 Page Data API。 +- 历史页面迁移期间继续可访问，不做全局 409 阻断。 +- 页面迁移并验证完成后，才考虑启用历史页面全局阻断。 + +## 待迁移页面 + +| 用户 ID | 页面 | 初步类型 | 状态 | +|---|---|---|---| +| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment.html` | 预约业务数据 | 待迁移 | +| `3f93530b-430d-45ce-a4d9-92be1cd08f26` | `public/appointment-admin.html` | 预约管理数据 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger.html` | 家庭记账数据 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/family-ledger-manage.html` | 家庭记账管理 | 待迁移 | +| `417afcc0-f3fc-40b7-b968-1f1cde74b9cf` | `public/index.html` | 工作区首页状态 | 待分类 | +| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/oa-system.html` | OA 业务数据 | 待迁移 | +| `a6fb1e97-2b0f-447b-b138-4561d8e5c53e` | `public/decision-comparison.html` | 决策工具状态 | 待分类 | +| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/daily-admin.html` | 日报管理数据 | 待迁移 | +| `1c99b83b-0454-474f-a5d2-129d34506a32` | `public/sales-register.html` | 销售登记数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey.html` | 调查数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/children-diet-survey-admin.html` | 调查管理数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/survey-youth.html` | 调查数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/daily-report-system.html` | 日报业务数据 | 待迁移 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/weather-calendar.html` | 工具偏好或业务数据 | 待分类 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/runner-game.html` | 游戏存档 | 待决定 PG 或取消持久化 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/tetris-game.html` | 游戏存档 | 待决定 PG 或取消持久化 | +| `a70ff537-8908-486e-9b6c-042e07cc25db` | `public/bird-key-fly.html` | 游戏存档 | 待决定 PG 或取消持久化 | + +## 单页迁移完成条件 + +1. 建表位于页面所有者的独立 PG schema。 +2. dataset 注册完成，页面 policy 权限符合前台/管理页用途。 +3. HTML 只通过 Page Data API 访问数据，不包含浏览器持久化 fallback。 +4. 实际提交、刷新后读取、PG 直接查询三者结果一致。 +5. 原页面链接继续可用，管理页鉴权和跨用户隔离验证通过。 diff --git a/mindspace-browser-storage-policy.mjs b/mindspace-browser-storage-policy.mjs new file mode 100644 index 0000000..0c528fb --- /dev/null +++ b/mindspace-browser-storage-policy.mjs @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const AGENT_PAGE_CODE_EXTENSIONS = new Set(['.html', '.htm', '.js', '.mjs', '.jsx', '.ts', '.tsx']); +const PROHIBITED_BROWSER_STORAGE_PATTERNS = [ + { api: 'localStorage', pattern: /\blocalStorage\b/i }, + { api: 'sessionStorage', pattern: /\bsessionStorage\b/i }, + { api: 'IndexedDB', pattern: /\b(?:indexedDB|IDBDatabase|IDBObjectStore|IDBTransaction)\b/i }, +]; + +export function isAgentPageCodePath(relativePath) { + return AGENT_PAGE_CODE_EXTENSIONS.has(path.extname(String(relativePath ?? '')).toLowerCase()); +} + +export function detectProhibitedBrowserStorage(content, { relativePath = '' } = {}) { + if (!isAgentPageCodePath(relativePath)) return []; + let source = String(content ?? ''); + // Fold adjacent string literals so computed-property obfuscation such as + // window['local' + 'Storage'] cannot bypass the delivery policy. + for (let pass = 0; pass < 8; pass += 1) { + const folded = source.replace( + /(['"])([^'"\\]*)\1\s*\+\s*(['"])([^'"\\]*)\3/g, + (_match, quote, left, _rightQuote, right) => `${quote}${left}${right}${quote}`, + ); + if (folded === source) break; + source = folded; + } + return PROHIBITED_BROWSER_STORAGE_PATTERNS + .filter(({ pattern }) => pattern.test(source)) + .map(({ api }) => api); +} + +export function assertNoProhibitedBrowserStorage(content, { relativePath = '' } = {}) { + const APIs = detectProhibitedBrowserStorage(content, { relativePath }); + if (APIs.length === 0) return; + const error = new Error( + `${relativePath || '页面代码'} 禁止使用 ${APIs.join(', ')} 保存数据。` + + '用户业务数据必须通过 private_data_* 工具写入该用户专属 PostgreSQL schema，' + + 'HTML 必须通过 Page Data API 访问；请改写后再保存。', + ); + error.code = 'BROWSER_STORAGE_FORBIDDEN'; + error.apis = APIs; + error.relativePath = relativePath || null; + throw error; +} + +export function scanWorkspaceFilesForProhibitedBrowserStorage({ publishDir, relativePaths = [] } = {}) { + const root = path.resolve(String(publishDir ?? '')); + if (!root) return []; + const violations = []; + for (const relativePath of new Set(relativePaths.map((item) => String(item ?? '').trim()).filter(Boolean))) { + if (!isAgentPageCodePath(relativePath)) continue; + const absolutePath = path.resolve(root, relativePath); + if (absolutePath !== root && !absolutePath.startsWith(`${root}${path.sep}`)) continue; + if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) continue; + const apis = detectProhibitedBrowserStorage(fs.readFileSync(absolutePath, 'utf8'), { relativePath }); + if (apis.length > 0) violations.push({ relativePath, apis }); + } + return violations; +} diff --git a/mindspace-h5-html-finish-guard.mjs b/mindspace-h5-html-finish-guard.mjs index c6a8e55..666872a 100644 --- a/mindspace-h5-html-finish-guard.mjs +++ b/mindspace-h5-html-finish-guard.mjs @@ -10,6 +10,7 @@ import { isStubPublicHtmlContent, materializeMissingPublicHtmlWrites, } from './mindspace-public-finish-sync.mjs'; +import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; const PUBLIC_HTML_PATH_PATTERN_GLOBAL = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/gi; const PUBLIC_ASSET_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi; @@ -106,6 +107,26 @@ export function collectMissingPublicHtmlPaths(messages, currentUser, publishDir) ); } +export function collectTouchedPublicCodePaths(messages, publishDir, syncResult = null) { + const paths = new Set( + (syncResult?.publicHtmlRelativePaths ?? []) + .map((item) => normalizePublicRelativePath(item, { publishDir })) + .filter(Boolean), + ); + for (const message of Array.isArray(messages) ? messages : []) { + for (const item of message?.content ?? []) { + if (item?.type !== 'toolRequest') continue; + const toolCall = item.toolCall?.value; + const name = String(toolCall?.name ?? '').trim().split('__').at(-1); + if (!['write_file', 'edit_file', 'write', 'edit'].includes(name)) continue; + const args = toolCall?.arguments ?? {}; + const relativePath = normalizePublicRelativePath(args.path ?? args.file_path, { publishDir }); + if (relativePath) paths.add(relativePath); + } + } + return [...paths]; +} + export function collectMissingPublicAssetReferences(publishDir) { const root = path.resolve(String(publishDir ?? '')); const publicDir = path.join(root, 'public'); @@ -135,6 +156,10 @@ export function evaluateH5HtmlFinishGuard({ materializeMissingPublicHtmlWrites({ messages, publishDir }); const missingHtml = collectMissingPublicHtmlPaths(messages, currentUser, publishDir); const missingAssets = collectMissingPublicAssetReferences(publishDir); + const browserStorageViolations = scanWorkspaceFilesForProhibitedBrowserStorage({ + publishDir, + relativePaths: collectTouchedPublicCodePaths(messages, publishDir, syncResult), + }); const lastAssistant = [...(Array.isArray(messages) ? messages : [])] .reverse() @@ -159,18 +184,24 @@ export function evaluateH5HtmlFinishGuard({ const needsRepair = missingHtml.length > 0 || missingAssets.length > 0 || + browserStorageViolations.length > 0 || htmlGenerationNeedsRetry; return { needsRepair, missingHtml, missingAssets, + browserStorageViolations, htmlGenerationNeedsRetry, linkExists, }; } -export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } = {}) { +export function buildH5HtmlRepairPrompt({ + missingHtml = [], + missingAssets = [], + browserStorageViolations = [], +} = {}) { const lines = [ '【系统补盘请求】检测到 public 页面交付不完整。请立即使用 write_file 写入缺失文件（完整内容），并在完成后确认磁盘存在。', '要求：先 load_skill static-page-publish，再逐个 write_file；不要用空回复或仅 edit_file 改 HTML 代替资源落盘。', @@ -185,6 +216,15 @@ export function buildH5HtmlRepairPrompt({ missingHtml = [], missingAssets = [] } lines.push(`- ${item.relativePath}（来自 ${item.htmlRelativePath}）`); } } + if (browserStorageViolations.length) { + lines.push( + '', + '以下页面违反数据存储硬约束：', + ...browserStorageViolations.map((item) => `- ${item.relativePath}：禁止 ${item.apis.join(', ')}`), + '', + '必须删除浏览器存储代码，load_skill → page-data-collect，使用 private_data_* 在当前用户专属 PostgreSQL schema 建表和注册 dataset，HTML 只能通过 Page Data API 读写。禁止 SQLite 或任何本地 fallback。', + ); + } lines.push('', '写完后请 list_dir public/assets 或对应目录，确认文件真实存在。'); return lines.join('\n'); } diff --git a/mindspace-h5-html-finish-guard.test.mjs b/mindspace-h5-html-finish-guard.test.mjs index 64dee35..681b8bb 100644 --- a/mindspace-h5-html-finish-guard.test.mjs +++ b/mindspace-h5-html-finish-guard.test.mjs @@ -72,6 +72,61 @@ test('evaluateH5HtmlFinishGuard flags missing assets after html exists', () => { } }); +test('evaluateH5HtmlFinishGuard blocks browser storage in any touched public page', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-ban-')); + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync( + path.join(publishDir, 'public', 'ledger.html'), + '', + 'utf8', + ); + const evaluation = evaluateH5HtmlFinishGuard({ + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/ledger.html' }, + } } }] }], + currentUser: USER, + publishDir, + }); + assert.equal(evaluation.needsRepair, true); + assert.deepEqual(evaluation.browserStorageViolations, [{ + relativePath: 'public/ledger.html', + apis: ['sessionStorage', 'IndexedDB'], + }]); + assert.match(buildH5HtmlRepairPrompt(evaluation), /当前用户专属 PostgreSQL schema/); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + +test('evaluateH5HtmlFinishGuard blocks computed localStorage obfuscation', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'h5-html-storage-obfuscation-')); + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync( + path.join(publishDir, 'public', 'bypass.html'), + ``, + 'utf8', + ); + const evaluation = evaluateH5HtmlFinishGuard({ + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/bypass.html' }, + } } }] }], + currentUser: USER, + publishDir, + }); + assert.equal(evaluation.needsRepair, true); + assert.deepEqual(evaluation.browserStorageViolations, [{ + relativePath: 'public/bypass.html', + apis: ['localStorage'], + }]); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + test('buildH5HtmlRepairPrompt lists html and asset targets', () => { const prompt = buildH5HtmlRepairPrompt({ missingHtml: ['public/page.html'], diff --git a/mindspace-page-data-finish-guard.mjs b/mindspace-page-data-finish-guard.mjs index edd78f3..f76a6f1 100644 --- a/mindspace-page-data-finish-guard.mjs +++ b/mindspace-page-data-finish-guard.mjs @@ -15,7 +15,9 @@ import { normalizePageDataApiBase, verifyPageDataDeliveryArtifacts, } from './page-data-delivery-assess.mjs'; +import { listPageAccessPolicies } from './page-data-policy-store.mjs'; import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs'; +import { detectProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; export { inferPageDataBindAccessMode }; @@ -23,7 +25,7 @@ const PUBLICATION_ROUTE_LINK_PATTERN = /https?:\/\/[^\s<>"')\]]+\/u\/([0-9a-f-]{36}|[a-z0-9._-]+)\/pages\/([^\s<>"')\]]+)/gi; const PAGE_DATA_CLIENT_SCRIPT_PATTERN = /\/assets\/page-data-client\.js/i; -const LOCAL_STORAGE_DATA_PATTERN = /localStorage\.(?:getItem|setItem)\s*\(/i; +const LEGACY_PAGE_DATA_OWNER_API_PATTERN = /['"`]\/api\/page-data(?:\/|['"`])/i; const LOCAL_STORAGE_FALLBACK_HINT_PATTERN = /fallback\s*到\s*localStorage|localStorage\s*fallback/i; const repairAttemptsBySession = new Map(); @@ -109,10 +111,14 @@ export function buildPageDataDeliveryArtifactsFromBindResult(autoBind, publishDi export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) { const content = String(html ?? ''); const usage = detectPageDataDatasetUsageFromHtml(content); + const prohibitedBrowserStorage = detectProhibitedBrowserStorage(content, { relativePath }); const usesPageDataApi = usage.size > 0 || PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content) || - /\bMindSpacePageData\b/.test(content); + /\bMindSpacePageData\b/.test(content) || + LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content) || + prohibitedBrowserStorage.length > 0 || + LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content); if (!usesPageDataApi) { return { usesPageDataApi: false, issues: [] }; @@ -122,9 +128,15 @@ export function evaluatePageDataHtmlContent(html, { relativePath = '' } = {}) { if (!PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content)) { issues.push('missing_page_data_client_script'); } - if (LOCAL_STORAGE_DATA_PATTERN.test(content)) { + if (LEGACY_PAGE_DATA_OWNER_API_PATTERN.test(content)) { + issues.push('forbidden_legacy_owner_api'); + } + if (prohibitedBrowserStorage.includes('localStorage')) { issues.push('forbidden_local_storage'); } + if (prohibitedBrowserStorage.some((api) => api !== 'localStorage')) { + issues.push('forbidden_browser_storage'); + } if (LOCAL_STORAGE_FALLBACK_HINT_PATTERN.test(content)) { issues.push('forbidden_local_storage_fallback'); } @@ -156,7 +168,22 @@ export function collectPageDataPublicHtmlFiles(publishDir) { } function isPageDataHtmlWorkspaceReady({ publishDir, relativePath, html }) { - return assessWorkspacePageDataReadiness({ publishDir, relativePath, html }).ready; + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) return true; + return Boolean(findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html })); +} + +function findWorkspacePolicyForPageDataHtml({ publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html); + for (const policy of listPageAccessPolicies(publishDir)) { + if (String(policy?.accessMode ?? '') !== expectedAccessMode) continue; + if ([...usage.entries()].every(([name, perms]) => { + const granted = policy?.datasets?.[name]; + return Boolean(granted && (!perms.read || granted.read) && (!perms.insert || granted.insert)); + })) return policy; + } + return null; } async function collectUnboundPageDataFiles({ @@ -186,11 +213,13 @@ async function collectUnboundPageDataFiles({ } continue; } - if (!isPageDataHtmlWorkspaceReady({ + const workspace = await assessWorkspacePageDataReadiness({ + userId, publishDir, relativePath: file.relativePath, html: file.content, - })) { + }); + if (!workspace.ready) { unboundFiles.push(file); } } @@ -320,7 +349,7 @@ export function evaluatePageDataFinishGuard({ ); const needsRepair = - pageDataIntent && + (pageDataIntent || structuralFiles.length > 0) && (htmlIssues.length > 0 || unboundFiles.length > 0 || (relevantFiles.length === 0 && @@ -382,7 +411,7 @@ export function buildPageDataCollectFailureText() { '这次问卷/数据收集页面没有完成 Page Data API 绑定，所以我先不发链接。', '请直接重发一次完整需求（例如：调查问卷 + 后台查看），我会按 page-data-collect 技能：', '建表 → 注册 dataset → 写含 page-data-client.js 的 HTML → private_data_bind_workspace_page 发布。', - '数据必须走平台 API 写入 SQLite，禁止 localStorage 或自建后端。', + '数据必须走平台 API 写入用户私有数据空间，禁止 localStorage 或自建后端。', ].join(''); } diff --git a/mindspace-page-data-finish-guard.test.mjs b/mindspace-page-data-finish-guard.test.mjs index 335b3ee..64bf312 100644 --- a/mindspace-page-data-finish-guard.test.mjs +++ b/mindspace-page-data-finish-guard.test.mjs @@ -8,6 +8,7 @@ import { buildPageDataCollectRepairPrompt, collectPageDataDeliveryArtifacts, evaluatePageDataFinishGuard, + evaluatePageDataFinishGuardAsync, evaluatePageDataHtmlContent, extractRecentPageDataBindTargets, extractRecentPageDataHtmlWrites, @@ -42,6 +43,23 @@ test('evaluatePageDataHtmlContent flags missing client script and localStorage f const bad = evaluatePageDataHtmlContent(BAD_SURVEY_HTML, { relativePath: 'public/children-diet-survey.html' }); assert.ok(bad.issues.includes('missing_page_data_client_script')); assert.ok(bad.issues.includes('forbidden_local_storage')); + + const indexed = evaluatePageDataHtmlContent( + '', + { relativePath: 'public/draft.html' }, + ); + assert.ok(indexed.issues.includes('forbidden_browser_storage')); + + const legacyOwnerApi = evaluatePageDataHtmlContent( + ``, + { relativePath: 'public/sticky-notes.html' }, + ); + assert.equal(legacyOwnerApi.usesPageDataApi, true); + assert.ok(legacyOwnerApi.issues.includes('missing_page_data_client_script')); + assert.ok(legacyOwnerApi.issues.includes('forbidden_legacy_owner_api')); }); test('inferPageDataBindAccessMode chooses password for admin html', () => { @@ -70,6 +88,35 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => { } }); +test('finish guard blocks localStorage ledger generated from ordinary user language', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-ledger-')); + const text = '帮我做一个家庭记账页面，我每天可以记录日期、收入、支出、分类和备注，日期默认当天。再做一个管理页面，可以查看所有记录和收支汇总。'; + try { + fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); + fs.writeFileSync(path.join(publishDir, 'public', 'family-ledger.html'), ``, 'utf8'); + const evaluation = evaluatePageDataFinishGuard({ + publishDir, + agentText: text, + messages: [{ role: 'assistant', content: [{ type: 'toolRequest', toolCall: { value: { + name: 'write_file', + arguments: { path: 'public/family-ledger.html' }, + } } }] }], + }); + assert.equal(evaluation.pageDataIntent, true); + assert.equal(evaluation.needsRepair, true); + assert.equal( + evaluation.htmlIssues.some((item) => ( + item.issue === 'forbidden_local_storage' && item.relativePath === 'public/family-ledger.html' + )), + true, + ); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + test('finish guard ignores unrelated historical broken Page Data html', async () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-history-')); try { @@ -162,7 +209,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => { } }); -test('evaluatePageDataFinishGuard flags html when dataset is not registered', () => { +test('evaluatePageDataFinishGuard flags html when dataset is not registered', async () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-bound-')); try { fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); @@ -179,8 +226,9 @@ test('evaluatePageDataFinishGuard flags html when dataset is not registered', () }, }, }); - const evaluation = evaluatePageDataFinishGuard({ + const evaluation = await evaluatePageDataFinishGuardAsync({ publishDir, + userId: 'user-1', agentText: '调查问卷和后台', messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }], }); diff --git a/mindspace-page-sync-service.mjs b/mindspace-page-sync-service.mjs index 3f3b6b3..c5a996f 100644 --- a/mindspace-page-sync-service.mjs +++ b/mindspace-page-sync-service.mjs @@ -11,12 +11,16 @@ export function createPageSyncService({ } = {}) { const syncInFlight = new Map(); - async function syncUserGeneratedPages(userId) { + async function syncUserGeneratedPages(userId, { onlyRelativePaths = null } = {}) { if (!pool || !pageService || !userId) { return { created: 0, skipped: 0, updated: 0 }; } - let inFlight = syncInFlight.get(userId); + const scopeKey = onlyRelativePaths == null + ? '*' + : [...new Set(onlyRelativePaths)].sort().join('|'); + const inFlightKey = `${userId}:${scopeKey}`; + let inFlight = syncInFlight.get(inFlightKey); if (!inFlight) { inFlight = syncGeneratedPagesFromPublicAssets({ pool, @@ -25,17 +29,18 @@ export function createPageSyncService({ userId, publishDir: h5Root ? resolvePublishDir(h5Root, { id: userId }) : null, syncWorkspaceAssets, + onlyRelativePaths, }) .catch((error) => { logger.warn?.('[MindSpace] page sync failed:', error?.message ?? error); return { created: 0, skipped: 0, updated: 0 }; }) .finally(() => { - if (syncInFlight.get(userId) === inFlight) { - syncInFlight.delete(userId); + if (syncInFlight.get(inFlightKey) === inFlight) { + syncInFlight.delete(inFlightKey); } }); - syncInFlight.set(userId, inFlight); + syncInFlight.set(inFlightKey, inFlight); } return inFlight; } diff --git a/mindspace-page-sync.mjs b/mindspace-page-sync.mjs index 5159b48..fcdfcbe 100644 --- a/mindspace-page-sync.mjs +++ b/mindspace-page-sync.mjs @@ -213,16 +213,23 @@ export async function syncGeneratedPagesFromPublicAssets({ userId, publishDir = null, syncWorkspaceAssets = null, + onlyRelativePaths = null, } = {}) { if (!pool || !pageService || !userId) { return { created: 0, skipped: 0, updated: 0 }; } if (typeof syncWorkspaceAssets === 'function') { - await syncWorkspaceAssets(userId, { categoryCode: 'public' }).catch(() => {}); + await syncWorkspaceAssets(userId, { + categoryCode: 'public', + onlyRelativePaths, + }).catch(() => {}); } const indexedPages = await loadIndexedWorkspacePages(pool, userId); + const allowList = onlyRelativePaths == null + ? null + : new Set(onlyRelativePaths.map((item) => normalizePublicHtmlPath(item)).filter(Boolean)); let created = 0; let updated = 0; let skipped = 0; @@ -230,6 +237,7 @@ export async function syncGeneratedPagesFromPublicAssets({ if (publishDir) { const htmlFiles = await listPublishPublicHtmlFiles(publishDir); for (const file of htmlFiles) { + if (allowList && !allowList.has(file.relativePath)) continue; try { const content = await fs.readFile(file.absolutePath, 'utf8'); if (!content.trim()) { @@ -285,6 +293,7 @@ export async function syncGeneratedPagesFromPublicAssets({ ? asset.original_filename : `public/${asset.original_filename}`, ); + if (allowList && !allowList.has(relativePath)) continue; if (indexedPages.has(relativePath)) { skipped += 1; continue; diff --git a/mindspace-page-sync.test.mjs b/mindspace-page-sync.test.mjs index 92593a5..8c97bcc 100644 --- a/mindspace-page-sync.test.mjs +++ b/mindspace-page-sync.test.mjs @@ -47,6 +47,40 @@ test('syncGeneratedPagesFromPublicAssets registers html files from publish publi assert.equal(created[0].source.snapshot.relative_path, 'public/demo.html'); }); +test('syncGeneratedPagesFromPublicAssets scopes agent-run sync to current session html', async () => { + const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-scoped-')); + const publicDir = path.join(publishDir, 'public'); + await fs.mkdir(publicDir, { recursive: true }); + await fs.writeFile(path.join(publicDir, 'current.html'), 'Current', 'utf8'); + await fs.writeFile(path.join(publicDir, 'historical.html'), 'Historical', 'utf8'); + + const created = []; + const pool = { + async query(sql) { + if (sql.includes('FROM h5_page_records p')) return [[]]; + if (sql.includes('FROM h5_assets a')) return [[]]; + return [[]]; + }, + }; + const pageService = { + async createFromChat(_userId, input, source) { + created.push({ input, source }); + return { id: `page-${created.length}` }; + }, + }; + + await syncGeneratedPagesFromPublicAssets({ + pool, + pageService, + userId: 'user-1', + publishDir, + onlyRelativePaths: ['public/current.html'], + }); + + assert.deepEqual(created.map((item) => item.source.snapshot.relative_path), ['public/current.html']); + await fs.rm(publishDir, { recursive: true, force: true }); +}); + test('syncGeneratedPagesFromPublicAssets uses file mtime as createdAt', async () => { const publishDir = await fs.mkdtemp(path.join(os.tmpdir(), 'page-sync-mtime-')); const publicDir = path.join(publishDir, 'public'); diff --git a/mindspace-public-finish-sync.mjs b/mindspace-public-finish-sync.mjs index 3d5c295..3e340e2 100644 --- a/mindspace-public-finish-sync.mjs +++ b/mindspace-public-finish-sync.mjs @@ -516,6 +516,7 @@ export async function syncPublicHtmlAfterFinish({ categoryCode: 'public', sourceSessionId: sessionId, sourceMessageId, + onlyRelativePaths: publicHtmlRelativePaths, }); synced = true; } diff --git a/mindspace-public-finish-sync.test.mjs b/mindspace-public-finish-sync.test.mjs index 90751c5..c5a4906 100644 --- a/mindspace-public-finish-sync.test.mjs +++ b/mindspace-public-finish-sync.test.mjs @@ -196,6 +196,7 @@ test('syncPublicHtmlAfterFinish forwards session source to workspace sync', asyn categoryCode: 'public', sourceSessionId: 'session-1', sourceMessageId: 'msg-public-1', + onlyRelativePaths: ['public/session-page.html'], }, ]]); } finally { diff --git a/mindspace-public-route.test.mjs b/mindspace-public-route.test.mjs index 05853aa..fcda274 100644 --- a/mindspace-public-route.test.mjs +++ b/mindspace-public-route.test.mjs @@ -121,6 +121,35 @@ test('resolveMindSpacePublicRequest coordinates canonical redirect, serve, and n }); }); +test('resolveMindSpacePublicRequest keeps legacy browser-storage pages available during migration', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-storage-ban-')); + const ownerKey = '123e4567-e89b-12d3-a456-426614174000'; + const targetDir = path.join(root, 'MindSpace', ownerKey, 'public'); + fs.mkdirSync(targetDir, { recursive: true }); + fs.writeFileSync( + path.join(targetDir, 'ledger.html'), + '', + ); + fs.writeFileSync( + path.join(targetDir, 'ledger.js'), + 'sessionStorage.setItem("ledger", "[]")', + ); + + const result = await resolveMindSpacePublicRequest({ + h5Root: root, + requestPath: `/${ownerKey}/public/ledger.html`, + }); + assert.equal(result.action, 'serve'); + assert.equal(result.filePath, path.join(targetDir, 'ledger.html')); + + const scriptResult = await resolveMindSpacePublicRequest({ + h5Root: root, + requestPath: `/${ownerKey}/public/ledger.js`, + }); + assert.equal(scriptResult.action, 'serve'); + assert.equal(scriptResult.filePath, path.join(targetDir, 'ledger.js')); +}); + test('resolveMindSpacePublicRequest serves missing thumbnail png when svg sidecar exists', async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ms-public-thumb-')); const ownerKey = '123e4567-e89b-12d3-a456-426614174000'; diff --git a/mindspace-sandbox-mcp.mjs b/mindspace-sandbox-mcp.mjs index 4cef617..f49753c 100644 --- a/mindspace-sandbox-mcp.mjs +++ b/mindspace-sandbox-mcp.mjs @@ -24,6 +24,7 @@ import { closePolicyDataset, normalizePageAccessPolicy } from './page-access-pol import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs'; import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs'; import { resolveMindSpaceStorageRoot } from './mindspace-runtime-config.mjs'; +import { assertNoProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; const SANDBOX_ROOT = process.argv[2]?.trim() || process.env.SANDBOX_ROOT?.trim(); if (!SANDBOX_ROOT) { @@ -215,7 +216,7 @@ const ALL_TOOLS = [ { name: 'private_data_info', description: - '查看当前用户“用户私有数据空间”的状态。每个用户只有一个私有 SQLite 数据库，适合保存问卷、表单、清单、调研数据和分析中间表；不要创建额外 SQLite 文件。', + '查看当前用户“用户私有数据空间”的状态。每个用户只有一个由平台分配和隔离的数据空间，适合保存问卷、表单、清单、调研数据和分析中间表；不要创建额外数据库文件。', inputSchema: { type: 'object', properties: {} }, }, { @@ -504,6 +505,7 @@ async function callTool(name, args) { } case 'write_file': { const abs = resolveSandboxed(args.path); + assertNoProhibitedBrowserStorage(args.content ?? '', { relativePath: args.path }); fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, args.content ?? '', 'utf8'); return [{ type: 'text', text: `已写入 ${args.path}（${(args.content ?? '').length} 字节）` }]; @@ -516,6 +518,7 @@ async function callTool(name, args) { throw new Error('edit_file: old_str 在文件中不存在，替换失败'); } const updated = oldStr ? original.replace(oldStr, args.new_str ?? '') : (args.new_str ?? ''); + assertNoProhibitedBrowserStorage(updated, { relativePath: args.path }); fs.writeFileSync(abs, updated, 'utf8'); return [{ type: 'text', text: `已编辑 ${args.path}` }]; } @@ -578,7 +581,9 @@ async function callTool(name, args) { { ...info, rules: [ - '每个用户只能使用这个唯一 SQLite 数据库', + info.backend === 'postgres' + ? '每个用户只能使用新 PG 中分配给自己的独立 schema' + : '每个用户只能使用这个唯一 SQLite 数据库', '适合问卷、表单、清单、调研数据和分析中间表', '不要存账号、计费、权限、审计、公开平台数据或跨用户数据', 'HTML 页面通过 Page Data API 访问 dataset，不要直接暴露 SQL', @@ -603,7 +608,9 @@ async function callTool(name, args) { return [ { type: 'text', - text: `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `，已同步空间占用 ${result.deltaBytes} 字节` : ''}`, + text: result.backend === 'postgres' + ? `已在用户专属 PG schema 中执行。命令 ${result.command ?? 'OK'}，影响 ${result.affectedRows ?? 0} 行` + : `已执行。当前数据空间大小 ${result.sizeBytes} 字节${result.quotaSynced ? `，已同步空间占用 ${result.deltaBytes} 字节` : ''}`, }, ]; } diff --git a/mindspace-sandbox-mcp.test.mjs b/mindspace-sandbox-mcp.test.mjs index 9833e7a..6bcb917 100644 --- a/mindspace-sandbox-mcp.test.mjs +++ b/mindspace-sandbox-mcp.test.mjs @@ -85,6 +85,41 @@ function startSandbox(root, envOverrides = {}) { return { child, request }; } +test('sandbox MCP rejects browser storage in generated page code', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-storage-ban-')); + const server = startSandbox(root, { ALLOWED_TOOLS: 'write_file,edit_file' }); + t.after(() => server.child.kill()); + + await server.request('initialize'); + const rejectedWrite = await server.request('tools/call', { + name: 'write_file', + arguments: { + path: 'public/ledger.html', + content: '', + }, + }); + assert.equal(rejectedWrite.result.isError, true); + assert.match(rejectedWrite.result.content[0].text, /BROWSER_STORAGE_FORBIDDEN|禁止使用 localStorage/); + assert.equal(fs.existsSync(path.join(root, 'public', 'ledger.html')), false); + + const acceptedWrite = await server.request('tools/call', { + name: 'write_file', + arguments: { path: 'public/ledger.html', content: '' }, + }); + assert.equal(acceptedWrite.result.isError, false); + + const rejectedEdit = await server.request('tools/call', { + name: 'edit_file', + arguments: { + path: 'public/ledger.html', + old_str: 'client.insertRow("ledger", row)', + new_str: 'indexedDB.open("ledger")', + }, + }); + assert.equal(rejectedEdit.result.isError, true); + assert.doesNotMatch(fs.readFileSync(path.join(root, 'public', 'ledger.html'), 'utf8'), /indexedDB/); +}); + test('sandbox MCP exposes and protects the user private data space', async (t) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-')); const server = startSandbox(root); diff --git a/mindspace-userdata-postgres.mjs b/mindspace-userdata-postgres.mjs new file mode 100644 index 0000000..39347a2 --- /dev/null +++ b/mindspace-userdata-postgres.mjs @@ -0,0 +1,439 @@ +import { execFileSync } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export const CONTROL_SCHEMA = 'mindspace_control'; + +export function assertUserId(value) { + const userId = String(value ?? '').trim().toLowerCase(); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(userId)) { + throw new Error('user id 必须是标准 UUID'); + } + return userId; +} + +export function quotePgIdentifier(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function deriveUserSpaceNames(value) { + const userId = assertUserId(value); + const compact = userId.replaceAll('-', ''); + const short = compact.slice(0, 12); + return { + userId, + schemaName: `u_${compact}`, + ownerRole: `ms_u_${short}_owner`, + agentRole: `ms_u_${short}_agent`, + }; +} + +export function buildControlSchemaSql() { + return ` +CREATE SCHEMA IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}; +REVOKE ALL ON SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC; + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces ( + user_id uuid PRIMARY KEY, + schema_name name NOT NULL UNIQUE, + owner_role name NOT NULL UNIQUE, + agent_role name NOT NULL UNIQUE, + backend_type text NOT NULL DEFAULT 'postgres' CHECK (backend_type IN ('sqlite', 'postgres')), + source_sqlite_path text, + migration_state text NOT NULL DEFAULT 'pending' CHECK ( + migration_state IN ('pending', 'snapshot_created', 'schema_converted', 'data_imported', 'verified', 'shadow', 'cutover', 'rolled_back', 'failed') + ), + schema_version integer NOT NULL DEFAULT 1, + quota_bytes bigint NOT NULL DEFAULT 104857600 CHECK (quota_bytes > 0), + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + cutover_at timestamptz, + rollback_until timestamptz +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE, + source_path text NOT NULL, + source_size_bytes bigint NOT NULL DEFAULT 0, + source_sha256 text, + status text NOT NULL, + table_count integer NOT NULL DEFAULT 0, + source_row_count bigint NOT NULL DEFAULT 0, + target_row_count bigint NOT NULL DEFAULT 0, + detail_json jsonb NOT NULL DEFAULT '{}'::jsonb, + started_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + finished_at timestamptz +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog ( + user_id uuid NOT NULL REFERENCES ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces(user_id) ON DELETE CASCADE, + dataset_name text NOT NULL, + physical_table name NOT NULL, + config_json jsonb NOT NULL, + sensitivity text NOT NULL DEFAULT 'normal' CHECK (sensitivity IN ('normal', 'personal', 'credential', 'health')), + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (user_id, dataset_name) +); + +CREATE TABLE IF NOT EXISTS ${quotePgIdentifier(CONTROL_SCHEMA)}.audit_events ( + id bigserial PRIMARY KEY, + user_id uuid, + actor_type text NOT NULL, + actor_id text, + action text NOT NULL, + object_type text, + object_name text, + statement_hash text, + affected_rows bigint, + duration_ms integer, + result text NOT NULL, + detail_json jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP +); +REVOKE ALL ON ALL TABLES IN SCHEMA ${quotePgIdentifier(CONTROL_SCHEMA)} FROM PUBLIC; +`.trim(); +} + +export function buildProvisionUserSql(userId, { sourceSqlitePath = null, quotaBytes = 104857600 } = {}) { + const names = deriveUserSpaceNames(userId); + const schema = quotePgIdentifier(names.schemaName); + const owner = quotePgIdentifier(names.ownerRole); + const agent = quotePgIdentifier(names.agentRole); + return { + ...names, + statements: [ + `DO $$ BEGIN CREATE ROLE ${owner} NOLOGIN; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN CREATE ROLE ${agent} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; EXCEPTION WHEN duplicate_object THEN NULL; END $$`, + `DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid + JOIN pg_roles m ON m.oid = am.member + WHERE r.rolname = '${names.ownerRole}' AND m.rolname = CURRENT_USER AND am.set_option + ) THEN + EXECUTE format('GRANT %I TO %I', '${names.ownerRole}', CURRENT_USER); + END IF; + END $$`, + `CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION ${owner}`, + `REVOKE ALL ON SCHEMA ${schema} FROM PUBLIC`, + `GRANT USAGE, CREATE ON SCHEMA ${schema} TO ${agent}`, + `ALTER ROLE ${agent} SET search_path = ${schema}, pg_catalog`, + `ALTER ROLE ${agent} SET statement_timeout = '30s'`, + `ALTER ROLE ${agent} SET lock_timeout = '5s'`, + `ALTER ROLE ${agent} SET idle_in_transaction_session_timeout = '60s'`, + `ALTER ROLE ${agent} SET work_mem = '4MB'`, + `ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${agent}`, + `ALTER DEFAULT PRIVILEGES FOR ROLE ${owner} IN SCHEMA ${schema} GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO ${agent}`, + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces + (user_id, schema_name, owner_role, agent_role, source_sqlite_path, quota_bytes, updated_at) + VALUES ($1::uuid, $2::name, $3::name, $4::name, $5, $6, CURRENT_TIMESTAMP) + ON CONFLICT (user_id) DO UPDATE SET + source_sqlite_path = EXCLUDED.source_sqlite_path, + quota_bytes = EXCLUDED.quota_bytes, + updated_at = CURRENT_TIMESTAMP`, + ], + params: [names.userId, names.schemaName, names.ownerRole, names.agentRole, sourceSqlitePath, quotaBytes], + }; +} + +export function mapSqliteTypeToPostgres(column) { + const declared = String(column.type ?? '').trim().toUpperCase(); + if (Number(column.pk) === 1 && declared.includes('INT')) return 'bigint GENERATED BY DEFAULT AS IDENTITY'; + if (declared.includes('TEXT') && isSqliteTimestampDefault(column.dflt_value)) return 'timestamptz'; + if (declared.includes('INT')) return 'bigint'; + if (declared.includes('REAL') || declared.includes('FLOA') || declared.includes('DOUB')) return 'double precision'; + if (declared.includes('BLOB')) return 'bytea'; + if (declared.includes('NUM') || declared.includes('DEC')) return 'numeric'; + if (declared.includes('BOOL')) return 'boolean'; + return 'text'; +} + +export function isSqliteTimestampDefault(value) { + return /^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(String(value ?? '').trim()); +} + +export function translateSqliteDefault(value) { + if (value == null) return null; + const input = String(value).trim(); + if (/^NULL$/i.test(input)) return 'NULL'; + if (/^[+-]?\d+(?:\.\d+)?$/.test(input)) return input; + if (/^CURRENT_TIMESTAMP$/i.test(input)) return 'CURRENT_TIMESTAMP'; + if (/^\(?datetime\(\s*'now'\s*(?:,\s*'(?:\+8 hours|localtime)'\s*)?\)\)?$/i.test(input)) return 'CURRENT_TIMESTAMP'; + if (/^".*"$/.test(input)) return `'${input.slice(1, -1).replaceAll("'", "''")}'`; + if (/^'.*'$/.test(input)) return input; + return null; +} + +export function buildPostgresTableSql(schemaName, tableName, columns) { + if (!Array.isArray(columns) || columns.length === 0) throw new Error(`表 ${tableName} 没有字段`); + const primaryKeys = columns.filter((column) => Number(column.pk) > 0).sort((a, b) => Number(a.pk) - Number(b.pk)); + const definitions = columns.map((column) => { + const parts = [quotePgIdentifier(column.name), mapSqliteTypeToPostgres(column)]; + if (Number(column.notnull) === 1) parts.push('NOT NULL'); + const translatedDefault = translateSqliteDefault(column.dflt_value); + if (translatedDefault != null) parts.push(`DEFAULT ${translatedDefault}`); + if (primaryKeys.length === 1 && primaryKeys[0].name === column.name) parts.push('PRIMARY KEY'); + return parts.join(' '); + }); + if (primaryKeys.length > 1) { + definitions.push(`PRIMARY KEY (${primaryKeys.map((column) => quotePgIdentifier(column.name)).join(', ')})`); + } + return `CREATE TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (\n ${definitions.join(',\n ')}\n)`; +} + +function stableObjectName(prefix, ...parts) { + const readable = `${prefix}_${parts.join('_')}`.replace(/[^a-zA-Z0-9_]+/g, '_').toLowerCase(); + const hash = createHash('sha256').update(readable).digest('hex').slice(0, 8); + return `${readable.slice(0, 50)}_${hash}`; +} + +export function extractSqliteChecks(createSql) { + const sql = String(createSql ?? ''); + const checks = []; + const pattern = /\bCHECK\s*\(/ig; + let match; + while ((match = pattern.exec(sql))) { + const start = pattern.lastIndex; + let depth = 1; + let quote = ''; + let index = start; + for (; index < sql.length && depth > 0; index += 1) { + const char = sql[index]; + if (quote) { + if (char === quote && sql[index + 1] === quote) index += 1; + else if (char === quote) quote = ''; + continue; + } + if (char === "'" || char === '"') quote = char; + else if (char === '(') depth += 1; + else if (char === ')') depth -= 1; + } + if (depth === 0) checks.push(sql.slice(start, index - 1).trim()); + pattern.lastIndex = index; + } + return checks; +} + +export function buildPostgresIndexSql(schemaName, tableName, index) { + if (!index.columns?.length) return null; + if (index.partial) return null; + const prefix = index.unique ? 'ux' : 'ix'; + const name = stableObjectName(prefix, tableName, ...index.columns); + return `CREATE ${index.unique ? 'UNIQUE ' : ''}INDEX ${quotePgIdentifier(name)} ON ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} (${index.columns.map(quotePgIdentifier).join(', ')})`; +} + +export function buildPostgresForeignKeySql(schemaName, tableName, foreignKey) { + const name = stableObjectName('fk', tableName, foreignKey.id, foreignKey.table); + const onUpdate = String(foreignKey.onUpdate ?? 'NO ACTION').toUpperCase(); + const onDelete = String(foreignKey.onDelete ?? 'NO ACTION').toUpperCase(); + const allowedActions = new Set(['NO ACTION', 'RESTRICT', 'CASCADE', 'SET NULL', 'SET DEFAULT']); + if (!allowedActions.has(onUpdate) || !allowedActions.has(onDelete)) throw new Error('SQLite 外键动作不受支持'); + return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} FOREIGN KEY (${foreignKey.from.map(quotePgIdentifier).join(', ')}) REFERENCES ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(foreignKey.table)} (${foreignKey.to.map(quotePgIdentifier).join(', ')}) ON UPDATE ${onUpdate} ON DELETE ${onDelete}`; +} + +export function buildPostgresCheckSql(schemaName, tableName, expression, index) { + if (/;|--|\/\*/.test(expression)) throw new Error(`CHECK 表达式包含不安全内容：${tableName}`); + const name = stableObjectName('ck', tableName, index); + return `ALTER TABLE ${quotePgIdentifier(schemaName)}.${quotePgIdentifier(tableName)} ADD CONSTRAINT ${quotePgIdentifier(name)} CHECK (${expression})`; +} + +export function convertSqliteValueForPostgres(value, column) { + if (value == null) return null; + if (isSqliteTimestampDefault(column.dflt_value) && typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(trimmed)) { + return `${trimmed.replace(' ', 'T')}+08:00`; + } + } + return value; +} + +function sqliteJson(sqliteBin, databasePath, sql) { + const output = execFileSync(sqliteBin, ['-readonly', '-json', databasePath, sql], { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + }).trim(); + return output ? JSON.parse(output) : []; +} + +function quoteSqliteIdentifier(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function inspectSqliteDatabase(databasePath, { sqliteBin = 'sqlite3' } = {}) { + const resolved = path.resolve(databasePath); + if (!fs.statSync(resolved).isFile()) throw new Error(`SQLite 文件不存在：${resolved}`); + const integrity = sqliteJson(sqliteBin, resolved, 'PRAGMA integrity_check;'); + if (integrity[0]?.integrity_check !== 'ok') throw new Error(`SQLite integrity_check 未通过：${resolved}`); + const tables = sqliteJson( + sqliteBin, + resolved, + "SELECT name, sql FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name;", + ).map((table) => { + const columns = sqliteJson(sqliteBin, resolved, `PRAGMA table_info(${quoteSqliteIdentifier(table.name)});`); + const rows = sqliteJson(sqliteBin, resolved, `SELECT * FROM ${quoteSqliteIdentifier(table.name)};`); + const indexes = sqliteJson(sqliteBin, resolved, `PRAGMA index_list(${quoteSqliteIdentifier(table.name)});`).map((index) => ({ + name: index.name, + unique: Boolean(index.unique), + origin: index.origin, + partial: Boolean(index.partial), + columns: sqliteJson(sqliteBin, resolved, `PRAGMA index_info(${quoteSqliteIdentifier(index.name)});`).map((item) => item.name), + })).filter((index) => index.origin !== 'pk'); + const foreignKeyRows = sqliteJson(sqliteBin, resolved, `PRAGMA foreign_key_list(${quoteSqliteIdentifier(table.name)});`); + const foreignKeys = [...new Set(foreignKeyRows.map((item) => item.id))].map((id) => { + const items = foreignKeyRows.filter((item) => item.id === id).sort((a, b) => a.seq - b.seq); + return { + id, + table: items[0].table, + from: items.map((item) => item.from), + to: items.map((item) => item.to), + onUpdate: items[0].on_update, + onDelete: items[0].on_delete, + }; + }); + return { ...table, columns, rows, indexes, foreignKeys, checks: extractSqliteChecks(table.sql) }; + }); + return { path: resolved, sizeBytes: fs.statSync(resolved).size, tables }; +} + +export function createSqliteSnapshot(sourcePath, { sqliteBin = 'sqlite3' } = {}) { + const target = path.join(os.tmpdir(), `mindspace-userdata-${Date.now()}-${process.pid}.sqlite`); + if (/\s/.test(target)) throw new Error('SQLite snapshot 临时路径不能包含空格'); + execFileSync(sqliteBin, [path.resolve(sourcePath), `.backup ${target}`], { stdio: 'pipe' }); + // A standalone snapshot must not depend on the source WAL/SHM files. + execFileSync(sqliteBin, [target, 'PRAGMA journal_mode=DELETE;'], { stdio: 'pipe' }); + return target; +} + +export async function provisionUserSpace(client, userId, options = {}) { + const provision = buildProvisionUserSql(userId, options); + await client.query('BEGIN'); + try { + for (let index = 0; index < provision.statements.length; index += 1) { + const params = index === provision.statements.length - 1 ? provision.params : []; + await client.query(provision.statements[index], params); + } + await client.query('COMMIT'); + return provision; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} + +export async function migrateSqliteSnapshot(client, snapshot, userId, { replaceShadow = false, sourcePath = snapshot.path } = {}) { + const names = deriveUserSpaceNames(userId); + const owner = quotePgIdentifier(names.ownerRole); + const agent = quotePgIdentifier(names.agentRole); + const migrationId = randomUUID(); + let sourceRows = 0; + let targetRows = 0; + await client.query('BEGIN'); + try { + const stateResult = await client.query( + `SELECT migration_state FROM ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces WHERE user_id = $1::uuid FOR UPDATE`, + [names.userId], + ); + const migrationState = stateResult.rows[0]?.migration_state; + if (migrationState === 'cutover') throw new Error('用户空间已经 cutover，禁止迁移器覆盖'); + const existingResult = await client.query( + `SELECT count(*)::integer AS count FROM pg_catalog.pg_tables WHERE schemaname = $1`, + [names.schemaName], + ); + if (Number(existingResult.rows[0]?.count ?? 0) > 0 && !replaceShadow) { + throw new Error('目标影子空间已有表；如确认重建，显式使用 --replace-shadow'); + } + await client.query( + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs + (id, user_id, source_path, source_size_bytes, status, started_at) + VALUES ($1::uuid, $2::uuid, $3, $4, 'importing', CURRENT_TIMESTAMP)`, + [migrationId, names.userId, sourcePath, snapshot.sizeBytes], + ); + for (const table of snapshot.tables) { + await client.query(`DROP TABLE IF EXISTS ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} CASCADE`); + await client.query(buildPostgresTableSql(names.schemaName, table.name, table.columns)); + await client.query(`ALTER TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} OWNER TO ${owner}`); + await client.query( + `GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} TO ${agent}`, + ); + for (const row of table.rows) { + const columns = Object.keys(row); + const params = columns.map((_, index) => `$${index + 1}`); + await client.query( + `INSERT INTO ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)} (${columns.map(quotePgIdentifier).join(', ')}) VALUES (${params.join(', ')})`, + columns.map((column) => convertSqliteValueForPostgres( + row[column], + table.columns.find((item) => item.name === column) ?? {}, + )), + ); + } + for (const column of table.columns.filter( + (item) => Number(item.pk) === 1 && String(item.type ?? '').toUpperCase().includes('INT'), + )) { + const qualifiedTable = `${names.schemaName}.${table.name}`; + await client.query( + `SELECT setval( + pg_get_serial_sequence($1, $2), + GREATEST(COALESCE(MAX(${quotePgIdentifier(column.name)}), 0), 1), + COALESCE(MAX(${quotePgIdentifier(column.name)}), 0) > 0 + ) + FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`, + [qualifiedTable, column.name], + ); + } + sourceRows += table.rows.length; + const count = await client.query( + `SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(table.name)}`, + ); + targetRows += Number(count.rows[0].count); + } + for (const table of snapshot.tables) { + for (const index of table.indexes ?? []) { + const sql = buildPostgresIndexSql(names.schemaName, table.name, index); + if (sql) await client.query(sql); + } + for (const foreignKey of table.foreignKeys ?? []) { + await client.query(buildPostgresForeignKeySql(names.schemaName, table.name, foreignKey)); + } + for (let index = 0; index < (table.checks ?? []).length; index += 1) { + await client.query(buildPostgresCheckSql(names.schemaName, table.name, table.checks[index], index)); + } + } + await client.query(`GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA ${quotePgIdentifier(names.schemaName)} TO ${agent}`); + const registry = snapshot.tables.find((table) => table.name === '__page_data_datasets'); + for (const row of registry?.rows ?? []) { + await client.query( + `INSERT INTO ${quotePgIdentifier(CONTROL_SCHEMA)}.dataset_catalog + (user_id, dataset_name, physical_table, config_json, updated_at) + VALUES ($1::uuid, $2, $3::name, $4::jsonb, CURRENT_TIMESTAMP) + ON CONFLICT (user_id, dataset_name) DO UPDATE SET + physical_table = EXCLUDED.physical_table, + config_json = EXCLUDED.config_json, + updated_at = CURRENT_TIMESTAMP`, + [names.userId, row.name, row.table_name, row.config_json], + ); + } + await client.query( + `UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.user_spaces + SET migration_state = 'shadow', updated_at = CURRENT_TIMESTAMP + WHERE user_id = $1::uuid`, + [names.userId], + ); + await client.query( + `UPDATE ${quotePgIdentifier(CONTROL_SCHEMA)}.migration_runs + SET status = 'verified', table_count = $2, source_row_count = $3, + target_row_count = $4, finished_at = CURRENT_TIMESTAMP + WHERE id = $1::uuid`, + [migrationId, snapshot.tables.length, sourceRows, targetRows], + ); + await client.query('COMMIT'); + return { migrationId, ...names, tableCount: snapshot.tables.length, sourceRows, targetRows }; + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } +} diff --git a/mindspace-userdata-postgres.test.mjs b/mindspace-userdata-postgres.test.mjs new file mode 100644 index 0000000..569fe38 --- /dev/null +++ b/mindspace-userdata-postgres.test.mjs @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { + buildControlSchemaSql, + buildPostgresTableSql, + buildPostgresCheckSql, + buildPostgresForeignKeySql, + buildPostgresIndexSql, + buildProvisionUserSql, + createSqliteSnapshot, + convertSqliteValueForPostgres, + deriveUserSpaceNames, + extractSqliteChecks, + mapSqliteTypeToPostgres, + translateSqliteDefault, +} from './mindspace-userdata-postgres.mjs'; +import { execFileSync } from 'node:child_process'; + +const USER_ID = 'ecc1c649-fff7-4361-a243-a69b460cc407'; + +test('deriveUserSpaceNames creates stable UUID based identifiers', () => { + assert.deepEqual(deriveUserSpaceNames(USER_ID), { + userId: USER_ID, + schemaName: 'u_ecc1c649fff74361a243a69b460cc407', + ownerRole: 'ms_u_ecc1c649fff7_owner', + agentRole: 'ms_u_ecc1c649fff7_agent', + }); +}); + +test('control schema is private and tracks migration lifecycle', () => { + const sql = buildControlSchemaSql(); + assert.match(sql, /REVOKE ALL ON SCHEMA "mindspace_control" FROM PUBLIC/); + assert.match(sql, /CREATE TABLE IF NOT EXISTS "mindspace_control"\.user_spaces/); + assert.match(sql, /migration_runs/); + assert.match(sql, /audit_events/); +}); + +test('provision SQL creates isolated no-login roles and safety limits', () => { + const provision = buildProvisionUserSql(USER_ID, { sourceSqlitePath: '/tmp/user.sqlite' }); + const sql = provision.statements.join('\n'); + assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_owner" NOLOGIN/); + assert.match(sql, /CREATE ROLE "ms_u_ecc1c649fff7_agent" NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE/); + assert.match(sql, /pg_auth_members/); + assert.match(sql, /REVOKE ALL ON SCHEMA "u_ecc1c649fff74361a243a69b460cc407" FROM PUBLIC/); + assert.match(sql, /statement_timeout = '30s'/); + assert.match(sql, /NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION/); + assert.doesNotMatch(sql, /temp_file_limit/); +}); + +test('SQLite types and defaults map to conservative PostgreSQL equivalents', () => { + assert.equal(mapSqliteTypeToPostgres({ type: 'INTEGER', pk: 1 }), 'bigint GENERATED BY DEFAULT AS IDENTITY'); + assert.equal(mapSqliteTypeToPostgres({ type: 'REAL', pk: 0 }), 'double precision'); + assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0 }), 'text'); + assert.equal(mapSqliteTypeToPostgres({ type: 'TEXT', pk: 0, dflt_value: "datetime('now', 'localtime')" }), 'timestamptz'); + assert.equal(translateSqliteDefault("(datetime('now', '+8 hours'))"), 'CURRENT_TIMESTAMP'); + assert.equal(translateSqliteDefault('""'), "''"); + assert.equal(translateSqliteDefault('dangerous_function()'), null); +}); + +test('SQLite checks, indexes and foreign keys become scoped PostgreSQL objects', () => { + assert.deepEqual(extractSqliteChecks("CREATE TABLE x(status TEXT CHECK(status IN ('a','b')), score INT CHECK(score > 0))"), [ + "status IN ('a','b')", + 'score > 0', + ]); + assert.match(buildPostgresIndexSql('u_test', 'users', { unique: true, columns: ['username'] }), /CREATE UNIQUE INDEX/); + assert.match(buildPostgresForeignKeySql('u_test', 'child', { + id: 0, table: 'parent', from: ['parent_id'], to: ['id'], onUpdate: 'NO ACTION', onDelete: 'CASCADE', + }), /REFERENCES "u_test"\."parent" \("id"\).*ON DELETE CASCADE/); + assert.match(buildPostgresCheckSql('u_test', 'users', "status IN ('a','b')", 0), /CHECK \(status IN \('a','b'\)\)/); +}); + +test('SQLite local timestamps are made explicit for PostgreSQL', () => { + assert.equal( + convertSqliteValueForPostgres('2026-07-13 10:20:30', { dflt_value: "datetime('now', '+8 hours')" }), + '2026-07-13T10:20:30+08:00', + ); + assert.equal(convertSqliteValueForPostgres('plain', { dflt_value: null }), 'plain'); +}); + +test('table builder preserves primary key, not-null and safe defaults', () => { + const sql = buildPostgresTableSql('u_example', 'survey', [ + { name: 'id', type: 'INTEGER', pk: 1, notnull: 0, dflt_value: null }, + { name: 'answer', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "''" }, + { name: 'created_at', type: 'TEXT', pk: 0, notnull: 1, dflt_value: "datetime('now', 'localtime')" }, + ]); + assert.match(sql, /"id" bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY/); + assert.match(sql, /"answer" text NOT NULL DEFAULT ''/); + assert.match(sql, /"created_at" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP/); +}); + +test('SQLite snapshot includes committed rows without changing source', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-pg-test-')); + const source = path.join(root, 'source.sqlite'); + execFileSync('sqlite3', [source, 'CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT); INSERT INTO sample VALUES (1, \'ok\');']); + const snapshot = createSqliteSnapshot(source); + try { + const count = execFileSync('sqlite3', ['-readonly', snapshot, 'SELECT count(*) FROM sample;'], { encoding: 'utf8' }).trim(); + assert.equal(count, '1'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(snapshot, { force: true }); + } +}); diff --git a/mindspace-workspace-page-deliver-page-data.test.mjs b/mindspace-workspace-page-deliver-page-data.test.mjs index 5cc7f5e..6614ae0 100644 --- a/mindspace-workspace-page-deliver-page-data.test.mjs +++ b/mindspace-workspace-page-deliver-page-data.test.mjs @@ -56,8 +56,8 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page }, }, pageDataEnsure: { - async ensurePageDataHtmlPagesBound() { - bound.push(true); + async ensurePageDataHtmlPagesBound(options) { + bound.push(options); return { bound: [{ relativePath: 'public/survey.html' }], skipped: [], errors: [] }; }, }, @@ -65,8 +65,11 @@ test('ensureWorkspaceHtmlPublications skips page-data html and delegates to page storageRoot: null, }); - const result = await service.syncAndDeliver(userId); + const result = await service.syncAndDeliver(userId, { + pageDataRelativePaths: ['public/survey.html'], + }); assert.equal(bound.length, 1); + assert.deepEqual(bound[0].onlyRelativePaths, ['public/survey.html']); assert.equal(result.publish.published, 0); assert.equal(result.publish.skipped, 1); assert.equal(published.length, 0); diff --git a/mindspace-workspace-page-deliver.mjs b/mindspace-workspace-page-deliver.mjs index b849ba2..36781bd 100644 --- a/mindspace-workspace-page-deliver.mjs +++ b/mindspace-workspace-page-deliver.mjs @@ -95,7 +95,7 @@ export function createWorkspacePageDeliverService({ return resolvePublishDir(h5Root, { id: userId }); } - async function ensurePageDataBindings(userId) { + async function ensurePageDataBindings(userId, { onlyRelativePaths = null } = {}) { if (!pageDataEnsure?.ensurePageDataHtmlPagesBound || !pool || !userId) { return { bound: [], skipped: [], errors: [] }; } @@ -110,6 +110,7 @@ export function createWorkspacePageDeliverService({ userId, workspaceRoot, findPageByRelativePath, + onlyRelativePaths, logger, }); } @@ -189,15 +190,20 @@ export function createWorkspacePageDeliverService({ return { published, skipped, errors }; } - async function syncAndDeliver(userId) { + async function syncAndDeliver(userId, { pageDataRelativePaths = null } = {}) { let syncResult = { created: 0, updated: 0, skipped: 0 }; if (pageSyncService?.syncUserGeneratedPages) { - syncResult = await pageSyncService.syncUserGeneratedPages(userId); + syncResult = await pageSyncService.syncUserGeneratedPages(userId, { + onlyRelativePaths: pageDataRelativePaths, + }); } - const pageDataBindResult = await ensurePageDataBindings(userId); + const pageDataBindResult = await ensurePageDataBindings(userId, { + onlyRelativePaths: pageDataRelativePaths, + }); const publishResult = await ensureWorkspaceHtmlPublications(userId); const refreshResult = await refreshOnlineWorkspacePublications(userId); return { + pageDataRelativePaths: pageDataRelativePaths == null ? null : [...pageDataRelativePaths], sync: syncResult, pageDataBind: pageDataBindResult, publish: publishResult, diff --git a/mindspace-workspace-sync.mjs b/mindspace-workspace-sync.mjs index 1a47a54..603708a 100644 --- a/mindspace-workspace-sync.mjs +++ b/mindspace-workspace-sync.mjs @@ -436,10 +436,20 @@ export function createWorkspaceAssetSync({ } }; - const syncCategory = async (userId, categoryCode, source = {}) => { + const syncCategory = async (userId, categoryCode, source = {}, { onlyRelativePaths = null } = {}) => { if (!h5Root) return { imported: 0, updated: 0, skipped: 0 }; const workspaceRoot = resolveUserWorkspaceRoot(h5Root, { id: userId }); - const files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode); + let files = await listWorkspaceZoneFiles(workspaceRoot, categoryCode); + if (onlyRelativePaths != null) { + const allowed = new Set( + onlyRelativePaths + .map((item) => normalizeWorkspaceRelativePath(item)) + .filter(Boolean), + ); + files = files.filter((file) => + allowed.has(normalizeWorkspaceRelativePath(`${categoryCode}/${file.filename}`)), + ); + } const [categories] = await pool.query( `SELECT c.id, c.space_id, c.category_code FROM h5_space_categories c @@ -487,14 +497,17 @@ export function createWorkspaceAssetSync({ return { imported, updated, skipped }; }; - const syncUserWorkspace = async (userId, { categoryCode, sourceSessionId, sourceMessageId, title } = {}) => { + const syncUserWorkspace = async ( + userId, + { categoryCode, sourceSessionId, sourceMessageId, title, onlyRelativePaths = null } = {}, + ) => { const codes = categoryCode ? [categoryCode] : UPLOAD_ZONE_CODES; let imported = 0; let updated = 0; let skipped = 0; const source = { sessionId: sourceSessionId, messageId: sourceMessageId, title }; for (const code of codes) { - const result = await syncCategory(userId, code, source); + const result = await syncCategory(userId, code, source, { onlyRelativePaths }); imported += result.imported; updated += result.updated; skipped += result.skipped; diff --git a/mindspace-workspace-sync.test.mjs b/mindspace-workspace-sync.test.mjs index 65f340f..b15da83 100644 --- a/mindspace-workspace-sync.test.mjs +++ b/mindspace-workspace-sync.test.mjs @@ -316,6 +316,7 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn const workspace = resolveUserWorkspaceRoot(h5Root, { id: 'user-1', username: 'john' }); await fs.mkdir(path.join(workspace, 'oa', '暑假实习报告'), { recursive: true }); await fs.writeFile(path.join(workspace, 'oa', '暑假实习报告', 'report.csv'), 'a,b\n1,2\n'); + await fs.writeFile(path.join(workspace, 'oa', 'historical.csv'), 'old,data\n3,4\n'); const state = { categories: [ @@ -417,7 +418,10 @@ test('syncUserWorkspace imports nested workspace files into asset library', asyn idFactory: () => `id-${++nextId}`, }); - const result = await sync.syncUserWorkspace('user-1', { categoryCode: 'oa' }); + const result = await sync.syncUserWorkspace('user-1', { + categoryCode: 'oa', + onlyRelativePaths: ['oa/暑假实习报告/report.csv'], + }); assert.equal(result.imported, 1); assert.equal(state.assets[0].original_filename, '暑假实习报告/report.csv'); }); diff --git a/page-data-delivery-assess.mjs b/page-data-delivery-assess.mjs index df0c39d..3bd3b3c 100644 --- a/page-data-delivery-assess.mjs +++ b/page-data-delivery-assess.mjs @@ -40,6 +40,33 @@ export function buildSmokeInsertRow(policy, datasetName) { return row; } +export function assessPageDataAuthenticationContract({ policy, html } = {}) { + const accessMode = String(policy?.accessMode ?? '').trim(); + const content = String(html ?? ''); + const usesServerAuthentication = /\.\s*authenticate\s*\(/.test(content); + const passwordValue = String.raw`\b(?:pwd|pass(?:word)?|passwordInput|adminPassword)\w*(?:\.value)?`; + const passwordComparison = String.raw`(?:===?|!==?)`; + const passwordOperand = String.raw`(?:[A-Za-z_$][\w$]*|['"\x60][^'"\x60]*['"\x60])`; + const usesClientSidePasswordCheck = new RegExp( + `(?:${passwordValue}\\s*${passwordComparison}\\s*${passwordOperand}|${passwordOperand}\\s*${passwordComparison}\\s*${passwordValue})`, + 'i', + ).test(content); + const reasons = []; + if (accessMode === 'password' && !usesServerAuthentication) { + reasons.push('password_page_missing_server_authentication'); + } + if (accessMode === 'public' && usesServerAuthentication) { + reasons.push('public_page_cannot_use_password_authentication'); + } + if (accessMode === 'public' && usesClientSidePasswordCheck) { + reasons.push('public_page_uses_client_side_password_gate'); + } + if (accessMode === 'password' && usesClientSidePasswordCheck) { + reasons.push('password_page_uses_client_side_password_gate'); + } + return reasons; +} + export async function queryOnlinePublication(pool, pageId) { if (!pool || !pageId) return null; const [rows] = await pool.query( @@ -76,20 +103,13 @@ function findWorkspacePolicyForHtml({ publishDir, relativePath, html }) { return null; } -export function assessWorkspacePageDataReadiness({ publishDir, relativePath, html }) { +function assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }) { const usage = detectPageDataDatasetUsageFromHtml(html); if (!usage.size) { return { ready: true, reasons: [] }; } const reasons = []; - const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir }); - for (const datasetName of usage.keys()) { - if (!dataSpace.getDataset(datasetName)) { - reasons.push(`dataset_not_registered:${datasetName}`); - } - } - const policy = findWorkspacePolicyForHtml({ publishDir, relativePath, html }); if (!policy) { reasons.push('missing_workspace_policy'); @@ -98,6 +118,21 @@ export function assessWorkspacePageDataReadiness({ publishDir, relativePath, htm return { ready: reasons.length === 0, reasons, policy }; } +export async function assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const policyAssessment = assessWorkspacePageDataPolicyReadiness({ publishDir, relativePath, html }); + if (!usage.size) return policyAssessment; + + const reasons = [...policyAssessment.reasons]; + const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir, userId }); + for (const datasetName of usage.keys()) { + if (!(await dataSpace.getDataset(datasetName))) { + reasons.push(`dataset_not_registered:${datasetName}`); + } + } + return { ...policyAssessment, ready: reasons.length === 0, reasons }; +} + export async function assessPageDataHtmlBinding({ pool, userId, @@ -111,7 +146,7 @@ export async function assessPageDataHtmlBinding({ return { bound: true, reasons: [], pageId: null }; } - const workspace = assessWorkspacePageDataReadiness({ publishDir, relativePath, html }); + const workspace = await assessWorkspacePageDataReadiness({ userId, publishDir, relativePath, html }); const reasons = [...workspace.reasons]; let pageId = null; @@ -139,6 +174,7 @@ export async function assessPageDataHtmlBinding({ if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) { reasons.push('policy_access_mode_mismatch'); } + reasons.push(...assessPageDataAuthenticationContract({ policy, html })); for (const [datasetName, perms] of usage) { const policyDataset = policy.datasets?.[datasetName]; if (!policyDataset) continue; diff --git a/page-data-delivery-assess.test.mjs b/page-data-delivery-assess.test.mjs index 0c0498e..b0db110 100644 --- a/page-data-delivery-assess.test.mjs +++ b/page-data-delivery-assess.test.mjs @@ -1,11 +1,49 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + assessPageDataAuthenticationContract, buildSmokeInsertRow, extractInjectedPageIdFromHtml, normalizePageDataApiBase, } from './page-data-delivery-assess.mjs'; +test('password pages must authenticate with the server before data reads', () => { + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + [ + 'password_page_missing_server_authentication', + 'password_page_uses_client_side_password_gate', + ], + ); + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + [], + ); + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'password' }, + html: '', + }), + ['password_page_uses_client_side_password_gate'], + ); +}); + +test('public pages cannot use a client-side password gate as authorization', () => { + assert.deepEqual( + assessPageDataAuthenticationContract({ + policy: { accessMode: 'public' }, + html: '', + }), + ['public_page_uses_client_side_password_gate'], + ); +}); + test('extractInjectedPageIdFromHtml reads injected pageId', () => { const html = ` diff --git a/page-data-public-service.mjs b/page-data-public-service.mjs index f474f59..3ef15c0 100644 --- a/page-data-public-service.mjs +++ b/page-data-public-service.mjs @@ -171,13 +171,13 @@ export function createPageDataPublicService(deps = {}) { }); } - function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) { + async function resolveEffectiveDataset(ownerUserId, policy, datasetName, rolePermissions = null) { const policyDataset = policy.datasets?.[datasetName]; if (!policyDataset) { throw Object.assign(new Error('dataset 未授权'), { code: 'action_not_allowed' }); } const ownerService = createOwnerService(ownerUserId); - const registryDataset = ownerService.getDataset(datasetName); + const registryDataset = await ownerService.getDataset(datasetName); if (!registryDataset) { throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); } @@ -190,9 +190,9 @@ export function createPageDataPublicService(deps = {}) { return { ownerService, effective, policyDataset }; } - function resolveRowScope(ownerService, effective, policyDataset, visitorContext) { + async function resolveRowScope(ownerService, effective, policyDataset, visitorContext) { if (!policyDataset?.rowPolicy || !visitorContext) return null; - const tableColumns = ownerService.listTableColumns(effective.table); + const tableColumns = await ownerService.listTableColumns(effective.table); return resolveRowAccessScope(policyDataset.rowPolicy, { visitorRole: visitorContext.visitorRole, visitorUserId: visitorContext.visitorUserId, @@ -303,13 +303,13 @@ export function createPageDataPublicService(deps = {}) { const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); enforceRateLimit(req, publication, 'read'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -330,7 +330,7 @@ export function createPageDataPublicService(deps = {}) { const publication = await queryPublication(pageId); const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); - const { ownerService, effective } = resolveEffectiveDataset( + const { ownerService, effective } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, @@ -345,13 +345,13 @@ export function createPageDataPublicService(deps = {}) { const publication = await queryPublication(pageId); const policy = await loadPolicy(publication); const access = resolveAccessContext({ publication, policy, req, action: 'read', datasetName }); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -415,13 +415,13 @@ export function createPageDataPublicService(deps = {}) { } const payload = stripPageDataMetaFields(payloadInput); enforceRateLimit(req, publication, 'insert'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -465,13 +465,13 @@ export function createPageDataPublicService(deps = {}) { datasetName, }); enforceRateLimit(req, publication, 'update'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -513,13 +513,13 @@ export function createPageDataPublicService(deps = {}) { datasetName, }); enforceRateLimit(req, publication, 'soft_delete'); - const { ownerService, effective, policyDataset } = resolveEffectiveDataset( + const { ownerService, effective, policyDataset } = await resolveEffectiveDataset( publication.user_id, policy, datasetName, access.rolePermissions ?? null, ); - const rowScope = resolveRowScope( + const rowScope = await resolveRowScope( ownerService, effective, policyDataset, @@ -598,7 +598,7 @@ export function createPageDataPublicService(deps = {}) { throw Object.assign(new Error('缺少 dataset 名称'), { code: 'invalid_request', status: 400 }); } const ownerService = createOwnerService(user.id); - const registryDataset = ownerService.getDataset(normalizedDatasetName); + const registryDataset = await ownerService.getDataset(normalizedDatasetName); if (!registryDataset) { throw Object.assign(new Error('dataset 未注册'), { code: 'dataset_not_found', status: 404 }); } @@ -639,8 +639,8 @@ export function createPageDataPublicService(deps = {}) { for (const [name, config] of Object.entries(policy.datasets ?? {})) { let stats = null; try { - if (ownerService.getDataset(name)) { - stats = ownerService.getDatasetStats(name); + if (await ownerService.getDataset(name)) { + stats = await ownerService.getDatasetStats(name); } } catch { stats = null; @@ -660,7 +660,7 @@ export function createPageDataPublicService(deps = {}) { const recentSubmissions = logs.filter((entry) => entry.action === 'insert').slice(0, 10); const activeSessions = sessionStore.listByPageId(pageId); const index = await getPageDataPolicyIndex(getPool(), pageId); - const registeredDatasets = ownerService.listDatasets().map((dataset) => dataset.name); + const registeredDatasets = (await ownerService.listDatasets()).map((dataset) => dataset.name); const configuredNames = new Set(Object.keys(policy.datasets ?? {})); const unconfiguredDatasets = registeredDatasets.filter((name) => !configuredNames.has(name)); return { diff --git a/page-data-service.mjs b/page-data-service.mjs index bd6a812..9124624 100644 --- a/page-data-service.mjs +++ b/page-data-service.mjs @@ -58,7 +58,7 @@ export function createPageDataService(deps = {}) { async function listRows(user, datasetName, query = {}) { const { service } = await createServiceForUser(user); try { - return service.readDatasetRows(datasetName, { + return await service.readDatasetRows(datasetName, { limit: query.limit, offset: query.offset, orderBy: query.orderBy ?? query.order_by, @@ -73,7 +73,7 @@ export function createPageDataService(deps = {}) { async function getSchema(user, datasetName) { const { service } = await createServiceForUser(user); try { - return service.getDatasetSchema(datasetName); + return await service.getDatasetSchema(datasetName); } catch (error) { throw mapServiceError(error); } @@ -82,7 +82,7 @@ export function createPageDataService(deps = {}) { async function getStats(user, datasetName) { const { service } = await createServiceForUser(user); try { - return service.getDatasetStats(datasetName); + return await service.getDatasetStats(datasetName); } catch (error) { throw mapServiceError(error); } @@ -139,7 +139,8 @@ export function createPageDataService(deps = {}) { async function listDatasets(user) { const { service } = await createServiceForUser(user); - return service.listDatasets().map((dataset) => ({ + const datasets = await service.listDatasets(); + return datasets.map((dataset) => ({ name: dataset.name, table: dataset.table, description: dataset.description, diff --git a/page-data-workspace-bind.mjs b/page-data-workspace-bind.mjs index c9f24b6..9d2ea45 100644 --- a/page-data-workspace-bind.mjs +++ b/page-data-workspace-bind.mjs @@ -171,6 +171,16 @@ function assertRegisteredDatasetTables(userDataSpace, registryDatasets, usage) { } } +async function assertRegisteredDatasetTablesAsync(userDataSpace, registryDatasets, usage) { + for (const dataset of registryDatasets) { + const adapter = { + listTableColumns: () => userDataSpace.listTableColumns(dataset.table), + }; + const columns = await adapter.listTableColumns(); + assertRegisteredDatasetTables({ listTableColumns: () => columns }, [dataset], usage); + } +} + /** * Page Data policy must be derived from a registered SQLite dataset, never * accepted solely from an Agent-supplied policy. This runs before page or @@ -213,6 +223,27 @@ export function resolveRegisteredPageDataPolicy({ }; } +export async function resolveRegisteredPageDataPolicyAsync(options = {}) { + const { html, userId, accessMode, pageDataPolicy = null, userDataSpace } = options; + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) return null; + if (!userDataSpace) throw new Error('缺少 Page Data 数据空间'); + if (pageDataPolicy?.datasets) assertPolicyMatchesHtmlDatasets(html, pageDataPolicy.datasets); + const registryDatasets = await userDataSpace.listDatasets(); + const datasets = buildPageDataPolicyDatasetsFromRegistry({ html, registryDatasets, usage }); + await assertRegisteredDatasetTablesAsync( + userDataSpace, + registryDatasets.filter((dataset) => datasets[dataset.name]), + usage, + ); + return { + ...pageDataPolicy, + ownerUserId: String(pageDataPolicy?.ownerUserId ?? userId).trim(), + accessMode: pageDataPolicy?.accessMode ?? accessMode, + datasets, + }; +} + export async function bindWorkspaceHtmlForPageData({ pool, h5Root, @@ -236,8 +267,8 @@ export async function bindWorkspaceHtmlForPageData({ const normalizedAccessMode = resolvePageDataBindAccess(accessMode); const resolvedPassword = resolvePageDataBindPassword(normalizedAccessMode, password); - const userDataSpace = createUserDataSpaceService({ workspaceRoot }); - const resolvedPageDataPolicy = resolveRegisteredPageDataPolicy({ + const userDataSpace = createUserDataSpaceService({ workspaceRoot, userId, query: pool }); + const resolvedPageDataPolicy = await resolveRegisteredPageDataPolicyAsync({ html: content, userId, accessMode: normalizedAccessMode, diff --git a/page-data-workspace-ensure.mjs b/page-data-workspace-ensure.mjs index 55e36c1..25643d6 100644 --- a/page-data-workspace-ensure.mjs +++ b/page-data-workspace-ensure.mjs @@ -66,7 +66,7 @@ export async function ensureRegisteredDatasetFromHtml({ datasetName, }) { const dataSpace = createUserDataSpaceService({ workspaceRoot, userId, query }); - const existing = dataSpace.getDataset(datasetName); + const existing = await dataSpace.getDataset(datasetName); if (existing) return existing; const insertColumns = inferInsertColumnsFromHtml(html, datasetName); diff --git a/page-data-workspace-ensure.test.mjs b/page-data-workspace-ensure.test.mjs index 8b58c76..bffd100 100644 --- a/page-data-workspace-ensure.test.mjs +++ b/page-data-workspace-ensure.test.mjs @@ -57,7 +57,13 @@ test('evaluatePageDataFinishGuard detects unbound homework html even when user o const evaluation = evaluatePageDataFinishGuard({ publishDir, agentText: '确认发布', - messages: [], + messages: [{ + createdAt: Date.now(), + content: [{ + type: 'toolRequest', + toolCall: { value: { name: 'write_file', arguments: { path: 'public/homework.html' } } }, + }], + }], }); assert.equal(evaluation.pageDataIntent, false); assert.equal(evaluation.structuralPageData, true); diff --git a/postgres-user-data-space-service.mjs b/postgres-user-data-space-service.mjs new file mode 100644 index 0000000..33ccdb4 --- /dev/null +++ b/postgres-user-data-space-service.mjs @@ -0,0 +1,391 @@ +import pg from 'pg'; +import { + buildControlSchemaSql, + deriveUserSpaceNames, + provisionUserSpace, + quotePgIdentifier, +} from './mindspace-userdata-postgres.mjs'; + +const pools = new Map(); +const provisionedUsers = new Set(); +const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function ident(value, label = '标识符') { + const text = String(value ?? '').trim(); + if (!IDENTIFIER.test(text)) throw Object.assign(new Error(`${label} 格式无效`), { code: 'invalid_identifier' }); + return text; +} + +function normalizeDataset(raw, fallbackName = null) { + const value = typeof raw === 'string' ? JSON.parse(raw) : raw; + if (!value || typeof value !== 'object') throw Object.assign(new Error('dataset 配置无效'), { code: 'invalid_dataset_config' }); + const name = ident(value.name ?? fallbackName, 'dataset 名称'); + const table = ident(value.table ?? value.table_name, 'dataset 表名'); + const columns = value.columns && typeof value.columns === 'object' ? value.columns : {}; + for (const [action, fields] of Object.entries(columns)) { + if (!Array.isArray(fields)) throw Object.assign(new Error(`dataset 字段白名单无效：${action}`), { code: 'invalid_dataset_config' }); + columns[action] = fields.map((field) => ident(field, `${action} 字段`)); + } + return { + name, + table, + description: String(value.description ?? ''), + actions: Array.isArray(value.actions) ? value.actions.map(String) : [], + columns, + limits: { + maxRowsPerRead: Number(value.limits?.maxRowsPerRead ?? 200), + maxInsertBytes: Number(value.limits?.maxInsertBytes ?? 8192), + }, + }; +} + +function poolConfig(options) { + if (options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL) { + return { connectionString: options.connectionString ?? process.env.MINDSPACE_USERDATA_PG_URL, max: 5 }; + } + return { + host: options.pgHost ?? process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp', + port: Number(options.pgPort ?? process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433), + database: options.pgDatabase ?? process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev', + user: options.pgUser ?? process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER, + max: 5, + }; +} + +function sharedPool(options) { + if (options.pgPool) return options.pgPool; + const config = poolConfig(options); + const key = JSON.stringify(config); + if (!pools.has(key)) pools.set(key, new pg.Pool(config)); + return pools.get(key); +} + +function rejectSql(sql, { readonly = false } = {}) { + const cleaned = String(sql ?? '').replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trim(); + if (!cleaned || cleaned.length > 20000) throw new Error('SQL 为空或过长'); + if (readonly && !/^(SELECT|WITH)\b/i.test(cleaned)) throw new Error('private_data_query 只允许 SELECT/WITH'); + if (/\b(CREATE|ALTER|DROP)\s+(ROLE|USER|DATABASE|TABLESPACE|EXTENSION)|ALTER\s+SYSTEM|COPY[\s\S]+PROGRAM|SECURITY\s+DEFINER|pg_(read|write)_file|lo_import|dblink/i.test(cleaned)) { + throw new Error('SQL 包含用户空间不允许的操作'); + } + if (/\bSET\s+(ROLE|SESSION_AUTHORIZATION|search_path)\b/i.test(cleaned)) throw new Error('SQL 不允许改变安全上下文'); + return cleaned; +} + +function translateSqliteAgentDdl(sql) { + return sql + .replace(/INTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT/gi, 'BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY') + .replace(/TEXT\s+NOT\s+NULL\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP') + .replace(/TEXT\s+DEFAULT\s*\(datetime\(\s*'now'\s*,\s*'(?:\+8 hours|localtime)'\s*\)\)/gi, 'TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP'); +} + +export function createPostgresUserDataSpaceService(options = {}) { + const names = deriveUserSpaceNames(options.userId); + const pool = sharedPool(options); + const schema = quotePgIdentifier(names.schemaName); + const role = quotePgIdentifier(names.agentRole); + const maxRows = Number(options.maxRows ?? 200); + + async function ensureProvisioned({ force = false } = {}) { + if (provisionedUsers.has(names.userId)) return; + const client = await pool.connect(); + try { + await client.query(buildControlSchemaSql()); + const existing = await client.query('SELECT migration_state FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]); + if (!existing.rows[0]) { + if (!force && String(options.autoProvision ?? process.env.MINDSPACE_USERDATA_AUTO_PROVISION ?? '0') !== '1') { + throw Object.assign(new Error('用户 PG 空间尚未分配'), { code: 'user_space_not_provisioned' }); + } + await provisionUserSpace(client, names.userId, { sourceSqlitePath: options.workspaceRoot ? `${options.workspaceRoot}/.mindspace/private-data.sqlite` : null }); + await client.query('BEGIN'); + try { + await client.query(`CREATE TABLE IF NOT EXISTS ${schema}.__page_data_datasets (name text PRIMARY KEY, table_name text NOT NULL, config_json text NOT NULL, created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP)`); + await client.query(`ALTER TABLE ${schema}.__page_data_datasets OWNER TO ${quotePgIdentifier(names.ownerRole)}`); + await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON ${schema}.__page_data_datasets TO ${role}`); + await client.query("UPDATE mindspace_control.user_spaces SET migration_state='cutover',cutover_at=CURRENT_TIMESTAMP,rollback_until=CURRENT_TIMESTAMP + INTERVAL '30 days',updated_at=CURRENT_TIMESTAMP WHERE user_id=$1::uuid", [names.userId]); + await client.query('COMMIT'); + } catch (error) { + await client.query('ROLLBACK'); + throw error; + } + } + provisionedUsers.add(names.userId); + } finally { + client.release(); + } + } + + async function tenant(fn, { write = false } = {}) { + await ensureProvisioned(); + const client = await pool.connect(); + let quotaBytes = null; + try { + await client.query('BEGIN'); + if (write) { + const quota = await client.query('SELECT quota_bytes FROM mindspace_control.user_spaces WHERE user_id=$1::uuid', [names.userId]); + quotaBytes = Number(quota.rows[0]?.quota_bytes ?? 0); + } + await client.query(`SET LOCAL ROLE ${role}`); + await client.query(`SET LOCAL search_path = ${schema}, pg_catalog`); + await client.query(`SET LOCAL statement_timeout = '${write ? 30 : 10}s'`); + const result = await fn(client); + if (write && quotaBytes > 0) { + const usage = await client.query( + `SELECT COALESCE(sum(pg_total_relation_size(c.oid)),0)::bigint AS bytes + FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=$1 AND c.relkind IN ('r','p','m')`, + [names.schemaName], + ); + if (Number(usage.rows[0].bytes) > quotaBytes) { + throw Object.assign(new Error(`用户 PG 空间超过配额：${usage.rows[0].bytes}/${quotaBytes}`), { code: 'quota_exceeded' }); + } + } + await client.query(write ? 'COMMIT' : 'ROLLBACK'); + if (write) { + await client.query( + `INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json) + VALUES($1::uuid,'agent',$2,'postgres_write','success',$3::jsonb)`, + [names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName })], + ); + } + return result; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + if (write) { + await client.query( + `INSERT INTO mindspace_control.audit_events(user_id,actor_type,actor_id,action,result,detail_json) + VALUES($1::uuid,'agent',$2,'postgres_write','failed',$3::jsonb)`, + [names.userId, names.agentRole, JSON.stringify({ schema: names.schemaName, code: error?.code ?? null })], + ).catch(() => {}); + } + throw error; + } finally { + client.release(); + } + } + + async function listTableColumns(tableName) { + const table = ident(tableName, '表名'); + return tenant(async (client) => { + const result = await client.query( + `SELECT column_name AS name, data_type AS type, + (is_nullable = 'NO') AS "notNull", + (SELECT EXISTS (SELECT 1 FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) WHERE i.indrelid=($1||'.'||$2)::regclass AND i.indisprimary AND a.attname=c.column_name)) AS pk + FROM information_schema.columns c WHERE table_schema=$1 AND table_name=$2 ORDER BY ordinal_position`, + [names.schemaName, table], + ); + return result.rows; + }); + } + + async function getSchema() { + return tenant(async (client) => (await client.query( + `SELECT table_name, column_name, data_type AS type, (is_nullable='NO') AS not_null + FROM information_schema.columns WHERE table_schema=$1 ORDER BY table_name,ordinal_position`, + [names.schemaName], + )).rows); + } + + async function querySql(sql) { + const cleaned = rejectSql(sql, { readonly: true }).replace(/;\s*$/, ''); + return tenant(async (client) => (await client.query(`SELECT * FROM (${cleaned}) AS q LIMIT ${maxRows}`)).rows); + } + + async function executeSql(sql) { + const cleaned = translateSqliteAgentDdl(rejectSql(sql)); + return tenant(async (client) => { + const result = await client.query(cleaned); + return { backend: 'postgres', command: result.command, affectedRows: result.rowCount ?? 0, sizeBytes: null, deltaBytes: null, quotaSynced: false }; + }, { write: true }); + } + + async function getDataset(name) { + const datasetName = ident(name, 'dataset 名称'); + return tenant(async (client) => { + const result = await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets WHERE name=$1 LIMIT 1', [datasetName]); + if (!result.rows[0]) return null; + return { ...normalizeDataset(result.rows[0].config_json, result.rows[0].name), createdAt: result.rows[0].created_at, updatedAt: result.rows[0].updated_at }; + }); + } + + async function listDatasets() { + return tenant(async (client) => (await client.query('SELECT name,table_name,config_json,created_at,updated_at FROM __page_data_datasets ORDER BY name')).rows.map((row) => ({ + ...normalizeDataset(row.config_json, row.name), createdAt: row.created_at, updatedAt: row.updated_at, + }))); + } + + async function upsertDataset(input) { + const dataset = normalizeDataset(input); + if (!(await listTableColumns(dataset.table)).length) throw Object.assign(new Error(`dataset 对应表不存在：${dataset.table}`), { code: 'table_not_found' }); + await tenant((client) => client.query( + `INSERT INTO __page_data_datasets(name,table_name,config_json,updated_at) VALUES($1,$2,$3,CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET table_name=EXCLUDED.table_name,config_json=EXCLUDED.config_json,updated_at=CURRENT_TIMESTAMP`, + [dataset.name, dataset.table, JSON.stringify(dataset)], + ), { write: true }); + return getDataset(dataset.name); + } + + function assertAction(dataset, action) { + if (!dataset.actions.includes(action)) throw Object.assign(new Error(`dataset 未授权 ${action}`), { code: 'action_not_allowed' }); + } + + async function readRowsForDataset(datasetInput, options = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'read'); + const columns = dataset.columns.read?.map((item) => ident(item, '字段')) ?? []; + if (!columns.length) throw Object.assign(new Error('dataset 未配置可读字段'), { code: 'columns_not_allowed' }); + const limit = Math.min(Math.max(1, Number(options.limit ?? dataset.limits.maxRowsPerRead)), maxRows, dataset.limits.maxRowsPerRead); + const offset = Math.max(0, Number(options.offset ?? 0)); + const order = options.orderBy ? ident(options.orderBy, '排序字段') : columns.includes('id') ? 'id' : null; + if (order && !columns.includes(order)) throw Object.assign(new Error('排序字段未授权'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const filters = []; + if ((await listTableColumns(dataset.table)).some((column) => column.name === 'deleted_at') && !options.includeDeleted) filters.push('deleted_at IS NULL'); + if (options.rowScope?.whereClause) filters.push(`(${options.rowScope.whereClause})`); + const where = filters.length ? ` WHERE ${filters.join(' AND ')}` : ''; + const result = await client.query( + `SELECT ${columns.map(quotePgIdentifier).join(',')} FROM ${quotePgIdentifier(dataset.table)}${where}${order ? ` ORDER BY ${quotePgIdentifier(order)} ${String(options.orderDir).toLowerCase() === 'asc' ? 'ASC' : 'DESC'}` : ''} LIMIT $1 OFFSET $2`, + [limit, offset], + ); + return { dataset, rows: result.rows, limit, offset }; + }); + } + + async function readDatasetRows(name, options) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return readRowsForDataset(dataset, options); + } + + async function getDatasetStats(name) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + assertAction(dataset, 'read'); + return tenant(async (client) => ({ dataset: { name: dataset.name, table: dataset.table }, total: Number((await client.query(`SELECT count(*)::bigint AS count FROM ${quotePgIdentifier(dataset.table)}`)).rows[0].count) })); + } + + async function insertWithDataset(dataset, payload, meta = {}) { + assertAction(dataset, 'insert'); + const allowed = new Set(dataset.columns.insert ?? []); + const keys = Object.keys(payload ?? {}); + if (keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('提交包含未授权字段'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const columns = await listTableColumns(dataset.table); + const values = { ...payload }; + if (meta.rowScope?.ownerColumn && columns.some((c) => c.name === meta.rowScope.ownerColumn)) values[meta.rowScope.ownerColumn] ??= meta.rowScope.visitorUserId; + const insertKeys = Object.keys(values).map((key) => ident(key, '字段')); + const result = await client.query( + `INSERT INTO ${quotePgIdentifier(dataset.table)} (${insertKeys.map(quotePgIdentifier).join(',')}) VALUES (${insertKeys.map((_, i) => `$${i + 1}`).join(',')}) RETURNING *`, + insertKeys.map((key) => values[key]), + ); + return { dataset, row: result.rows[0] }; + }, { write: true }); + } + + async function insertDatasetRow(name, payload, meta = {}) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return insertWithDataset(dataset, payload, meta); + } + + async function insertRowForDataset(datasetInput, payload, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return insertWithDataset(dataset, payload, meta); + } + + async function updateRowForDataset(datasetInput, rowId, payload, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'update'); + const id = Number(rowId); + if (!Number.isFinite(id) || id <= 0) throw Object.assign(new Error('行 id 无效'), { code: 'invalid_row_id' }); + const allowed = new Set(dataset.columns.update ?? []); + const keys = Object.keys(payload ?? {}); + if (!keys.length || keys.some((key) => !allowed.has(key))) throw Object.assign(new Error('更新包含未授权字段'), { code: 'columns_not_allowed' }); + return tenant(async (client) => { + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET ${keys.map((key, i) => `${quotePgIdentifier(ident(key, '字段'))}=$${i + 1}`).join(',')} WHERE id=$${keys.length + 1}${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING *`, + [...keys.map((key) => payload[key]), id], + ); + if (!result.rows[0]) throw Object.assign(new Error('无权更新该行或行不存在'), { code: 'row_not_allowed' }); + return { dataset, row: result.rows[0] }; + }, { write: true }); + } + + async function updateDatasetRow(name, rowId, payload, meta) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return updateRowForDataset(dataset, rowId, payload, meta); + } + + async function softDeleteRowForDataset(datasetInput, rowId, meta = {}) { + const dataset = await Promise.resolve(datasetInput); + assertAction(dataset, 'soft_delete'); + const columns = await listTableColumns(dataset.table); + if (!columns.some((item) => item.name === 'deleted_at')) throw Object.assign(new Error('目标表不支持软删除'), { code: 'soft_delete_unsupported' }); + const id = Number(rowId); + return tenant(async (client) => { + const hasDeletedBy = columns.some((item) => item.name === 'deleted_by'); + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=CURRENT_TIMESTAMP${hasDeletedBy ? ',deleted_by=$2' : ''} WHERE id=$1 AND deleted_at IS NULL${meta.rowScope?.whereClause ? ` AND ${meta.rowScope.whereClause}` : ''} RETURNING id,deleted_at`, + hasDeletedBy ? [id, meta.deletedBy ?? meta.updatedByLabel ?? 'public'] : [id], + ); + if (!result.rows[0]) throw Object.assign(new Error('无权删除该行或行不存在'), { code: 'row_not_allowed' }); + return { dataset, id, deleted: true }; + }, { write: true }); + } + + async function softDeleteDatasetRow(name, rowId, meta) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return softDeleteRowForDataset(dataset, rowId, meta); + } + + async function restoreSoftDeletedRow(name, rowId) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + const columns = await listTableColumns(dataset.table); + return tenant(async (client) => { + const result = await client.query( + `UPDATE ${quotePgIdentifier(dataset.table)} SET deleted_at=NULL${columns.some((item) => item.name === 'deleted_by') ? ',deleted_by=NULL' : ''} WHERE id=$1 AND deleted_at IS NOT NULL RETURNING *`, + [Number(rowId)], + ); + if (!result.rows[0]) throw Object.assign(new Error('行不存在或未删除'), { code: 'row_not_allowed' }); + return { dataset, row: result.rows[0], restored: true }; + }, { write: true }); + } + + async function exportDatasetRows(name, { format = 'json', includeDeleted = false, limit = 1000 } = {}) { + const result = await readDatasetRows(name, { includeDeleted, limit: Math.min(Number(limit), 5000) }); + if (String(format).toLowerCase() !== 'csv') return { dataset: name, format: 'json', rows: result.rows, rowCount: result.rows.length }; + const columns = result.dataset.columns.read ?? []; + const escape = (value) => { + const text = value == null ? '' : String(value); + return /[",\n\r]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; + }; + return { dataset: name, format: 'csv', content: `${columns.join(',')}\n${result.rows.map((row) => columns.map((column) => escape(row[column])).join(',')).join('\n')}\n`, rowCount: result.rows.length }; + } + + async function getDatasetSchema(name) { + const dataset = await getDataset(name); + if (!dataset) throw Object.assign(new Error('dataset 不存在'), { code: 'dataset_not_found' }); + return { dataset, tableColumns: await listTableColumns(dataset.table) }; + } + + async function getInfo() { + return { name: '用户私有数据空间', backend: 'postgres', database: poolConfig(options).database, schema: names.schemaName, userId: names.userId }; + } + + async function ensureReady() { + await ensureProvisioned({ force: true }); + return getInfo(); + } + + return { + backend: 'postgres', workspaceRoot: options.workspaceRoot, privateDataDb: null, + ensureReady, getInfo, getSchema, querySql, executeSql, listTableColumns, getDataset, listDatasets, upsertDataset, + readRowsForDataset, readDatasetRows, getDatasetStats, getStatsForDataset: async (dataset) => getDatasetStats((await Promise.resolve(dataset)).name), + getDatasetSchema, getSchemaForDataset: async (dataset) => ({ dataset: await Promise.resolve(dataset), tableColumns: await listTableColumns((await Promise.resolve(dataset)).table) }), + insertDatasetRow, insertRowForDataset, updateDatasetRow, updateRowForDataset, + softDeleteDatasetRow, softDeleteRowForDataset, restoreSoftDeletedRow, exportDatasetRows, + }; +} diff --git a/scripts/dev-core-daemon.mjs b/scripts/dev-core-daemon.mjs index 64231f3..9303c9c 100644 --- a/scripts/dev-core-daemon.mjs +++ b/scripts/dev-core-daemon.mjs @@ -8,6 +8,11 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + applyMemindRuntimeProfile, + describeMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const opsDir = path.join(root, 'ops'); @@ -23,22 +28,6 @@ const plazaPublicBase = ( process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}` ).replace(/\/$/, ''); -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq < 0) continue; - const key = trimmed.slice(0, eq).trim(); - const value = trimmed.slice(eq + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} - -loadEnvFile(path.join(root, '../../.env.local')); -loadEnvFile(path.join(root, '.env')); - const logFile = process.env.MEMIND_DEV_LOG ?? path.join(os.homedir(), 'Library/Logs/memind-dev.log'); @@ -51,6 +40,10 @@ function log(message) { } } +loadMemindEnvFiles(root); +applyMemindRuntimeProfile({ rootDir: root }); +log(`Runtime profile: ${describeMemindRuntimeProfile()}`); + function freePort(port) { try { execSync(`lsof -ti TCP:${port} -sTCP:LISTEN | xargs kill -9`, { stdio: 'ignore' }); diff --git a/scripts/dev-core.mjs b/scripts/dev-core.mjs index ff54454..e00455d 100644 --- a/scripts/dev-core.mjs +++ b/scripts/dev-core.mjs @@ -3,6 +3,11 @@ import { spawn, execSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { + applyMemindRuntimeProfile, + describeMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); const opsDir = path.join(root, 'ops'); @@ -18,21 +23,9 @@ const plazaPublicBase = ( process.env.PLAZA_PUBLIC_BASE ?? `http://127.0.0.1:${plazaPort}` ).replace(/\/$/, ''); -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const eq = trimmed.indexOf('='); - if (eq < 0) continue; - const key = trimmed.slice(0, eq).trim(); - const value = trimmed.slice(eq + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} - -loadEnvFile(path.join(root, '../../.env.local')); -loadEnvFile(path.join(root, '.env')); +loadMemindEnvFiles(root); +applyMemindRuntimeProfile({ rootDir: root }); +console.log(`==> Runtime profile: ${describeMemindRuntimeProfile()}`); function freePort(port) { try { diff --git a/scripts/load-env.mjs b/scripts/load-env.mjs index beeecce..d9eb1b6 100644 --- a/scripts/load-env.mjs +++ b/scripts/load-env.mjs @@ -1,20 +1,11 @@ -import fs from 'node:fs'; import path from 'node:path'; - -function loadEnvFile(filePath) { - if (!fs.existsSync(filePath)) return; - for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { - const trimmed = line.trim(); - if (!trimmed || trimmed.startsWith('#')) continue; - const separator = trimmed.indexOf('='); - if (separator < 0) continue; - const key = trimmed.slice(0, separator).trim(); - const value = trimmed.slice(separator + 1).trim(); - if (!process.env[key]) process.env[key] = value; - } -} +import { + applyMemindRuntimeProfile, + loadMemindEnvFiles, +} from './memind-runtime-profile.mjs'; export function loadH5Environment(scriptDirectory) { - loadEnvFile(path.join(scriptDirectory, '../../../.env.local')); - loadEnvFile(path.join(scriptDirectory, '../.env')); + const root = path.join(scriptDirectory, '..'); + loadMemindEnvFiles(root); + applyMemindRuntimeProfile({ rootDir: root }); } diff --git a/scripts/memind-runtime-profile.mjs b/scripts/memind-runtime-profile.mjs new file mode 100644 index 0000000..5d7e744 --- /dev/null +++ b/scripts/memind-runtime-profile.mjs @@ -0,0 +1,119 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const MEMIND_RUNTIME_PROFILES = new Set(['local', 'split-service', 'production']); + +/** + * Load Memind env files in standard order. Later files only fill unset keys. + */ +export function loadMemindEnvFiles(rootDir, env = process.env) { + const root = path.resolve(rootDir); + for (const relativePath of ['../../.env.local', '.env']) { + loadEnvFile(path.join(root, relativePath), env); + } +} + +function loadEnvFile(filePath, env = process.env) { + if (!fs.existsSync(filePath)) return; + for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const separator = trimmed.indexOf('='); + if (separator < 0) continue; + const key = trimmed.slice(0, separator).trim(); + const value = trimmed.slice(separator + 1).trim(); + if (!env[key]) env[key] = value; + } +} + +function expandEnvReference(value, env = process.env) { + const raw = String(value ?? '').trim(); + const match = raw.match(/^\$\{([A-Z0-9_]+)\}$/); + if (!match) return raw; + return String(env[match[1]] ?? '').trim() || raw; +} + +export function resolveMemindRuntimeProfile(env = process.env) { + const explicit = String(env.MEMIND_RUNTIME_PROFILE ?? '').trim().toLowerCase(); + if (explicit) { + if (!MEMIND_RUNTIME_PROFILES.has(explicit)) { + throw new Error( + `Unsupported MEMIND_RUNTIME_PROFILE "${explicit}". Expected: ${[...MEMIND_RUNTIME_PROFILES].join(', ')}`, + ); + } + return explicit; + } + if (String(env.NODE_ENV ?? '').trim().toLowerCase() === 'production') { + return 'production'; + } + return 'local'; +} + +/** + * Apply environment-specific defaults after .env files are loaded. + * + * - local: monolith MindSpace adapter, repo-local storage (pnpm dev default) + * - split-service: remote adapter against standalone MindSpace on 8082 + * - production: no overrides; 103 .env is source of truth + */ +export function applyMemindRuntimeProfile({ + rootDir = process.cwd(), + env = process.env, + profile, +} = {}) { + const resolvedProfile = profile ?? resolveMemindRuntimeProfile(env); + env.MEMIND_RUNTIME_PROFILE = resolvedProfile; + + if (resolvedProfile === 'local') { + env.MINDSPACE_SERVER_ADAPTER = 'local'; + if (!env.MINDSPACE_STORAGE_ROOT) { + env.MINDSPACE_STORAGE_ROOT = path.join(path.resolve(rootDir), 'data', 'mindspace'); + } + return { + profile: resolvedProfile, + mindspaceAdapter: 'local', + storageRoot: env.MINDSPACE_STORAGE_ROOT, + }; + } + + if (resolvedProfile === 'split-service') { + env.MINDSPACE_SERVER_ADAPTER = 'remote'; + env.MINDSPACE_REMOTE_BASE_URL = String( + env.MINDSPACE_REMOTE_BASE_URL ?? 'http://127.0.0.1:8082', + ).trim(); + const expandedToken = expandEnvReference(env.MINDSPACE_REMOTE_AUTH_TOKEN, env); + if (expandedToken) { + env.MINDSPACE_REMOTE_AUTH_TOKEN = expandedToken; + } else if (env.TKMIND_SERVER__SECRET_KEY) { + env.MINDSPACE_REMOTE_AUTH_TOKEN = String(env.TKMIND_SERVER__SECRET_KEY).trim(); + } + if (!env.MINDSPACE_STORAGE_ROOT) { + env.MINDSPACE_STORAGE_ROOT = '/Users/john/MindSpace/data/mindspace'; + } + return { + profile: resolvedProfile, + mindspaceAdapter: 'remote', + remoteBaseUrl: env.MINDSPACE_REMOTE_BASE_URL, + storageRoot: env.MINDSPACE_STORAGE_ROOT, + }; + } + + return { + profile: resolvedProfile, + mindspaceAdapter: String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local', + storageRoot: env.MINDSPACE_STORAGE_ROOT ?? null, + }; +} + +export function describeMemindRuntimeProfile(env = process.env) { + const profile = resolveMemindRuntimeProfile(env); + const adapter = String(env.MINDSPACE_SERVER_ADAPTER ?? 'local').trim().toLowerCase() || 'local'; + const parts = [`MEMIND_RUNTIME_PROFILE=${profile}`, `MINDSPACE_SERVER_ADAPTER=${adapter}`]; + if (adapter === 'remote') { + parts.push(`MINDSPACE_REMOTE_BASE_URL=${env.MINDSPACE_REMOTE_BASE_URL ?? '(unset)'}`); + } + if (env.MINDSPACE_STORAGE_ROOT) { + parts.push(`MINDSPACE_STORAGE_ROOT=${env.MINDSPACE_STORAGE_ROOT}`); + } + return parts.join(', '); +} diff --git a/scripts/memind-runtime-profile.test.mjs b/scripts/memind-runtime-profile.test.mjs new file mode 100644 index 0000000..6e7b73d --- /dev/null +++ b/scripts/memind-runtime-profile.test.mjs @@ -0,0 +1,54 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; +import { + applyMemindRuntimeProfile, + resolveMemindRuntimeProfile, +} from './memind-runtime-profile.mjs'; + +test('resolveMemindRuntimeProfile defaults to local for dev', () => { + assert.equal(resolveMemindRuntimeProfile({}), 'local'); + assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'development' }), 'local'); +}); + +test('resolveMemindRuntimeProfile honors explicit profile and production NODE_ENV', () => { + assert.equal(resolveMemindRuntimeProfile({ MEMIND_RUNTIME_PROFILE: 'split-service' }), 'split-service'); + assert.equal(resolveMemindRuntimeProfile({ NODE_ENV: 'production' }), 'production'); +}); + +test('applyMemindRuntimeProfile local forces monolith adapter and repo storage', () => { + const env = { + MINDSPACE_SERVER_ADAPTER: 'remote', + MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082', + }; + const result = applyMemindRuntimeProfile({ + rootDir: '/tmp/memind', + env, + profile: 'local', + }); + assert.equal(result.profile, 'local'); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'local'); + assert.match(env.MINDSPACE_STORAGE_ROOT, /\/tmp\/memind\/data\/mindspace$/); +}); + +test('applyMemindRuntimeProfile split-service expands auth token reference', () => { + const env = { + TKMIND_SERVER__SECRET_KEY: 'local-dev-secret', + MINDSPACE_REMOTE_AUTH_TOKEN: '${TKMIND_SERVER__SECRET_KEY}', + }; + applyMemindRuntimeProfile({ env, profile: 'split-service' }); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote'); + assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'local-dev-secret'); +}); + +test('applyMemindRuntimeProfile production does not override adapter', () => { + const env = { + MINDSPACE_SERVER_ADAPTER: 'remote', + MINDSPACE_REMOTE_BASE_URL: 'http://127.0.0.1:8082', + MINDSPACE_REMOTE_AUTH_TOKEN: 'prod-token', + }; + const result = applyMemindRuntimeProfile({ env, profile: 'production' }); + assert.equal(result.profile, 'production'); + assert.equal(env.MINDSPACE_SERVER_ADAPTER, 'remote'); + assert.equal(env.MINDSPACE_REMOTE_AUTH_TOKEN, 'prod-token'); +}); diff --git a/scripts/migrate-mindspace-sqlite-to-postgres.mjs b/scripts/migrate-mindspace-sqlite-to-postgres.mjs new file mode 100644 index 0000000..7fb698f --- /dev/null +++ b/scripts/migrate-mindspace-sqlite-to-postgres.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import { + buildControlSchemaSql, + createSqliteSnapshot, + inspectSqliteDatabase, + migrateSqliteSnapshot, + provisionUserSpace, +} from '../mindspace-userdata-postgres.mjs'; + +function usage() { + return ` +Usage: + node scripts/migrate-mindspace-sqlite-to-postgres.mjs --source --user-id [--apply] [--replace-shadow] + +Default is read-only dry-run. --apply requires MINDSPACE_USERDATA_PG_URL. +The command creates a consistent SQLite snapshot and never modifies the source SQLite. +`.trim(); +} + +export function parseArgs(argv = []) { + const options = { apply: false, replaceShadow: false, source: '', userId: '', help: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--apply') options.apply = true; + else if (arg === '--replace-shadow') options.replaceShadow = true; + else if (arg === '--source') options.source = argv[++index] ?? ''; + else if (arg === '--user-id') options.userId = argv[++index] ?? ''; + else if (arg === '--help' || arg === '-h') options.help = true; + else throw new Error(`未知参数：${arg}`); + } + if (!options.help && (!options.source || !options.userId)) throw new Error('--source 和 --user-id 必填'); + return options; +} + +export async function run(argv = process.argv.slice(2), env = process.env) { + const options = parseArgs(argv); + if (options.help) { + console.log(usage()); + return { mode: 'help' }; + } + const inspected = inspectSqliteDatabase(options.source); + const sourceRows = inspected.tables.reduce((total, table) => total + table.rows.length, 0); + if (!options.apply) { + console.log(JSON.stringify({ + mode: 'dry-run', + userId: options.userId, + source: inspected.path, + sizeBytes: inspected.sizeBytes, + tables: inspected.tables.map((table) => ({ name: table.name, rows: table.rows.length })), + sourceRows, + note: '未修改 SQLite 或 PostgreSQL；使用 --apply 才会创建影子空间。', + }, null, 2)); + return { mode: 'dry-run', tableCount: inspected.tables.length, sourceRows }; + } + if (!env.MINDSPACE_USERDATA_PG_URL) throw new Error('--apply requires MINDSPACE_USERDATA_PG_URL'); + const snapshotPath = createSqliteSnapshot(options.source); + const snapshot = inspectSqliteDatabase(snapshotPath); + const { default: pg } = await import('pg'); + const client = new pg.Client({ connectionString: env.MINDSPACE_USERDATA_PG_URL }); + try { + await client.connect(); + await client.query(buildControlSchemaSql()); + await provisionUserSpace(client, options.userId, { + sourceSqlitePath: inspected.path, + }); + const result = await migrateSqliteSnapshot(client, snapshot, options.userId, { + replaceShadow: options.replaceShadow, + sourcePath: inspected.path, + }); + if (result.sourceRows !== result.targetRows) throw new Error('迁移行数校验失败'); + console.log(JSON.stringify({ mode: 'shadow', ...result }, null, 2)); + return { mode: 'shadow', ...result }; + } finally { + await client.end().catch(() => {}); + fs.rmSync(snapshotPath, { force: true }); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + run().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs b/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs new file mode 100644 index 0000000..063561c --- /dev/null +++ b/scripts/migrate-mindspace-sqlite-to-postgres.test.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { parseArgs } from './migrate-mindspace-sqlite-to-postgres.mjs'; + +test('migration CLI defaults to dry-run', () => { + assert.deepEqual(parseArgs(['--source', '/tmp/a.sqlite', '--user-id', 'user-id']), { + apply: false, + replaceShadow: false, + source: '/tmp/a.sqlite', + userId: 'user-id', + help: false, + }); +}); + +test('migration CLI requires explicit source and user', () => { + assert.throws(() => parseArgs([]), /--source 和 --user-id 必填/); +}); + +test('migration CLI recognizes explicit apply', () => { + assert.equal(parseArgs(['--apply', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).apply, true); +}); + +test('migration CLI requires explicit replace-shadow for destructive reruns', () => { + assert.equal(parseArgs(['--replace-shadow', '--source', '/tmp/a.sqlite', '--user-id', 'user-id']).replaceShadow, true); +}); diff --git a/scripts/verify-local-pg-questionnaire-e2e.mjs b/scripts/verify-local-pg-questionnaire-e2e.mjs new file mode 100644 index 0000000..0582a3c --- /dev/null +++ b/scripts/verify-local-pg-questionnaire-e2e.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node +/** + * Local-only E2E proof for the questionnaire -> Page Data API -> PostgreSQL path. + * + * The probe uses an existing local user space, creates a uniquely named survey + * table/dataset and page policy, submits one answer through the public HTTP API, + * verifies the row directly in PostgreSQL, and removes all probe objects. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import express from 'express'; +import pg from 'pg'; +import { attachPageDataRoutes } from '../page-data-routes.mjs'; +import { createPageDataService } from '../page-data-service.mjs'; +import { createPageDataPublicService } from '../page-data-public-service.mjs'; +import { deletePageAccessPolicy, writePageAccessPolicy } from '../page-data-policy-store.mjs'; +import { createUserDataSpaceService } from '../user-data-space-service.mjs'; +import { deriveUserSpaceNames, quotePgIdentifier } from '../mindspace-userdata-postgres.mjs'; +import { loadH5Environment } from './load-env.mjs'; + +loadH5Environment(import.meta.dirname); + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const userId = process.argv[2] ?? '1c99b83b-0454-474f-a5d2-129d34506a32'; +const workspaceRoot = path.join(repoRoot, 'MindSpace', userId); +const suffix = `${Date.now()}_${process.pid}`; +const tableName = `survey_e2e_${suffix}`; +const datasetName = tableName; +const pageId = `page-survey-e2e-${suffix}`; +const names = deriveUserSpaceNames(userId); +const policyPath = path.join(workspaceRoot, '.mindspace', 'page-data-policies', `${pageId}.json`); +const logPath = path.join(workspaceRoot, '.mindspace', 'page-data-logs', `${pageId}.jsonl`); + +assert.equal(process.env.MINDSPACE_USERDATA_BACKEND, 'postgres', '本机未启用 PostgreSQL 用户数据后端'); +assert.ok(fs.existsSync(workspaceRoot), `用户工作区不存在: ${workspaceRoot}`); + +const pgPool = new pg.Pool({ + host: process.env.MINDSPACE_USERDATA_PG_HOST ?? '/tmp', + port: Number(process.env.MINDSPACE_USERDATA_PG_PORT ?? 5433), + database: process.env.MINDSPACE_USERDATA_PG_DATABASE ?? 'mindspace_userdata_dev', + user: process.env.MINDSPACE_USERDATA_PG_USER ?? process.env.USER, + max: 2, +}); + +function publicationPool() { + return { + async query(sql) { + if (sql.includes('FROM h5_publish_records')) { + return [[{ + id: `publication-${suffix}`, + user_id: userId, + page_id: pageId, + access_mode: 'public', + password_hash: null, + status: 'online', + expires_at: null, + }]]; + } + return [[]]; + }, + }; +} + +function buildApp() { + const fakePublicationPool = publicationPool(); + const publicService = createPageDataPublicService({ + getPool: () => fakePublicationPool, + resolveWorkspaceRootForOwner: () => workspaceRoot, + }); + const api = express.Router(); + api.use(express.json()); + attachPageDataRoutes(api, { + sendData: (res, _req, data, status = 200) => res.status(status).json({ data }), + sendError: (res, _req, status, code, message) => res.status(status).json({ error: { code, message } }), + getPageDataService: () => createPageDataService({ resolveWorkspaceRoot: async () => workspaceRoot }), + getPageDataPublicService: () => publicService, + }); + const app = express(); + app.set('trust proxy', true); + app.use('/api', api); + return app; +} + +async function submitQuestionnaire(app, row) { + const server = app.listen(0, '127.0.0.1'); + await new Promise((resolve, reject) => { + server.once('listening', resolve); + server.once('error', reject); + }); + try { + const { port } = server.address(); + const apiPath = `/api/public/pages/${pageId}/data/${datasetName}/rows`; + const response = await fetch(`http://127.0.0.1:${port}${apiPath}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(row), + }); + return { apiPath, status: response.status, body: await response.json() }; + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +async function main() { + const ownerService = createUserDataSpaceService({ workspaceRoot, userId }); + let tableCreated = false; + try { + // Equivalent to the Agent's private_data_execute + register_dataset + bind policy flow. + await ownerService.executeSql(`CREATE TABLE ${tableName} ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + satisfaction TEXT NOT NULL, + favorite_feature TEXT NOT NULL, + suggestion TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now', 'localtime')) + )`); + tableCreated = true; + const dataset = await ownerService.upsertDataset({ + name: datasetName, + table: tableName, + description: '本机 PG 问卷端到端验证', + actions: ['read', 'insert'], + columns: { + read: ['id', 'satisfaction', 'favorite_feature', 'suggestion', 'created_at'], + insert: ['satisfaction', 'favorite_feature', 'suggestion'], + }, + }); + writePageAccessPolicy(workspaceRoot, { + pageId, + ownerUserId: userId, + accessMode: 'public', + datasets: { + [datasetName]: { + insert: true, + read: false, + columns: { insert: ['satisfaction', 'favorite_feature', 'suggestion'] }, + }, + }, + }); + + const answer = { + satisfaction: '非常满意', + favorite_feature: '个人数据报表', + suggestion: '增加按月趋势分析', + }; + const submitted = await submitQuestionnaire(buildApp(), answer); + assert.equal(submitted.status, 201, JSON.stringify(submitted.body)); + assert.equal(submitted.body.data.dataset, datasetName); + assert.equal(submitted.body.data.row.satisfaction, answer.satisfaction); + + const direct = await pgPool.query( + `SELECT id,satisfaction,favorite_feature,suggestion,created_at + FROM ${quotePgIdentifier(names.schemaName)}.${quotePgIdentifier(tableName)} + ORDER BY id DESC LIMIT 1`, + ); + assert.equal(direct.rowCount, 1); + assert.equal(direct.rows[0].satisfaction, answer.satisfaction); + assert.equal(direct.rows[0].favorite_feature, answer.favorite_feature); + assert.equal(direct.rows[0].suggestion, answer.suggestion); + + const registered = await pgPool.query( + `SELECT name,table_name FROM ${quotePgIdentifier(names.schemaName)}.__page_data_datasets WHERE name=$1`, + [datasetName], + ); + assert.equal(registered.rowCount, 1); + + console.log(JSON.stringify({ + result: 'PASS', + localOnly: true, + userId, + schema: names.schemaName, + dataset: { name: dataset.name, table: dataset.table, actions: dataset.actions }, + pageDataApi: { method: 'POST', path: submitted.apiPath, status: submitted.status }, + postgresRow: direct.rows[0], + checks: [ + 'Agent-style table creation succeeded in the user schema', + 'Dataset registry was stored in PostgreSQL', + 'Questionnaire submission used the public Page Data HTTP API', + 'Submitted answers matched a direct PostgreSQL query', + ], + }, null, 2)); + } finally { + if (tableCreated) { + await ownerService.executeSql(`DELETE FROM __page_data_datasets WHERE name='${datasetName}'`).catch(() => {}); + await ownerService.executeSql(`DROP TABLE IF EXISTS ${tableName}`).catch(() => {}); + } + deletePageAccessPolicy(workspaceRoot, pageId); + fs.rmSync(logPath, { force: true }); + fs.rmSync(policyPath, { force: true }); + await pgPool.end(); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + process.exitCode = 1; +}); diff --git a/server.mjs b/server.mjs index 31f87ad..0a58eb8 100644 --- a/server.mjs +++ b/server.mjs @@ -14,6 +14,7 @@ import { } from './auth.mjs'; import { createDbPool, initSchema, isDatabaseConfigured } from './db.mjs'; import { createWorkspacePageDeliverService } from './mindspace-workspace-page-deliver.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; import { createToolGateway } from './tool-gateway.mjs'; import { createAgentRunGateway } from './agent-run-gateway.mjs'; import { @@ -164,6 +165,7 @@ import { registerChatDocxArtifactForConversation, } from './mindspace-chat-docx-package.mjs'; import { scanContent } from './mindspace-content-scan.mjs'; +import { scanWorkspaceFilesForProhibitedBrowserStorage } from './mindspace-browser-storage-policy.mjs'; import { renderImageAssetViewerHtml, wantsInlineImageViewer } from './mindspace-asset-preview.mjs'; import { DEFAULT_IMAGE_UPLOAD_MAX_BYTES } from './user-image-normalize.mjs'; import { createRechargeService } from './billing-recharge.mjs'; @@ -535,6 +537,10 @@ async function bootstrapUserAuth() { h5Root: __dirname, defaultSignupBalanceCents: Number(process.env.H5_SIGNUP_BALANCE_CENTS ?? 500), subscriptionService, + provisionUserDataSpace: async ({ userId, workspaceRoot }) => { + const service = createUserDataSpaceService({ workspaceRoot, userId, query: pool }); + return service.ensureReady(); + }, }); sessionAccess = createSessionAccess({ userAuth, enabled: isSessionBrokerEnabled() }); if (sessionAccess.enabled) { @@ -622,6 +628,12 @@ async function bootstrapUserAuth() { expireStaleUploadsIntervalMs: 5 * 60 * 1000, }); await userAuth.ensureAdminUser(); + const userDataSpaceBackfill = await userAuth.ensureAllUserDataSpaces(); + if (userDataSpaceBackfill.errors.length > 0) { + console.warn( + `[PageData] PG space backfill incomplete: ${userDataSpaceBackfill.errors.length} user(s) failed`, + ); + } llmProviderService = createLlmProviderService(pool, { apiTarget: API_TARGET, apiTargets: API_TARGETS, @@ -717,8 +729,24 @@ async function bootstrapUserAuth() { messages: [userMessage], }); }, - syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId), + syncUserPagesOnSuccess: async ({ userId, sessionId, runStartedAtMs }) => + syncUserGeneratedPages(userId, { sessionId, sinceMs: runStartedAtMs }), isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0, + validateRunDeliverables: async ({ userId, deliverables }) => { + const publishDir = resolveMindSpaceUserPublishDir(__dirname, { id: userId }); + const violations = scanWorkspaceFilesForProhibitedBrowserStorage({ + publishDir, + relativePaths: (deliverables?.pages ?? []) + .map((page) => page.workspaceRelativePath) + .filter(Boolean), + }); + return { + errors: violations.map((violation) => ({ + code: 'browser_storage_forbidden', + message: `${violation.relativePath} 使用 ${violation.apis.join(', ')}`, + })), + }; + }, autoDispatch: ['1', 'true', 'yes', 'on'].includes( String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(), ), @@ -3231,10 +3259,36 @@ async function resolveExistingSavedPage(userId, { sessionId, messageId, relative const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']); // REGRESSION GUARD: mindspace-page-sync-thumbnail — remote 也经 pageSyncService RPC 同步 public HTML -async function syncUserGeneratedPages(userId) { +async function listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs = null } = {}) { + if (!authPool || !userId || !sessionId) return []; + const sinceClause = sinceMs == null ? '' : 'AND ca.created_at >= ?'; + const params = sinceMs == null ? [userId, sessionId] : [userId, sessionId, sinceMs]; + const [rows] = await authPool.query( + `SELECT ca.display_name + FROM h5_conversation_artifacts ca + JOIN h5_conversation_packages cp ON cp.id = ca.package_id + WHERE cp.user_id = ? + AND cp.session_id = ? + AND ca.artifact_kind = 'public_html' + ${sinceClause} + ORDER BY ca.sort_order ASC, ca.created_at ASC`, + params, + ); + return [...new Set((rows ?? []) + .map((row) => normalizeWorkspaceRelativePath(`public/${String(row.display_name ?? '').trim()}`)) + .filter((relativePath) => relativePath?.startsWith('public/') && relativePath.toLowerCase().endsWith('.html')))]; +} + +async function syncUserGeneratedPages(userId, { sessionId = null, sinceMs = null } = {}) { if (!userId) return; + // Agent-run/Finish delivery must stay scoped to the current conversation. + // A stale Page Data page elsewhere in the user's workspace must not turn a + // successfully completed current task into a failed run. + const pageDataRelativePaths = sessionId + ? await listSessionPublicHtmlRelativePaths(userId, sessionId, { sinceMs }) + : null; if (workspacePageDeliver?.syncAndDeliver) { - return await workspacePageDeliver.syncAndDeliver(userId); + return await workspacePageDeliver.syncAndDeliver(userId, { pageDataRelativePaths }); } if (!mindSpacePageSync) return; return await mindSpacePageSync.syncUserGeneratedPages(userId); @@ -5099,7 +5153,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { .filter(Boolean) .join('\n') : ''; - await syncUserGeneratedPages(uid); + await syncUserGeneratedPages(uid, { sessionId: sid }); await maybeRepairPageDataAfterFinish({ sessionId: sid, userId: uid, diff --git a/skills/page-data-collect/SKILL.md b/skills/page-data-collect/SKILL.md index ba84841..701a02f 100644 --- a/skills/page-data-collect/SKILL.md +++ b/skills/page-data-collect/SKILL.md @@ -1,11 +1,11 @@ --- name: page-data-collect -description: 在 MindSpace 页面中收集、存储并管理结构化数据（问卷、报名、台账、后台查看），基于 Page Data API 与用户私有 SQLite +description: 在 MindSpace 页面中收集、存储并管理结构化数据（问卷、报名、台账、后台查看），基于 Page Data API 与用户隔离的 PostgreSQL 空间 --- # 页面数据收集（Page Data Collect） -在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互，并将提交持久化到当前用户的 `.mindspace/private-data.sqlite`，通过平台 **Page Data API** 受控读写。 +在 MindSpace **公开 HTML 页面**中嵌入表单、问卷、报名等交互，并将提交持久化到当前用户隔离的 PostgreSQL schema，通过平台 **Page Data API** 受控读写。运行时禁止创建、读取或回退到用户 SQLite。 详细 API 说明见工作区外文档 `docs/page-data-api-usage.md`（Memind 仓库）。 @@ -25,7 +25,7 @@ description: 在 MindSpace 页面中收集、存储并管理结构化数据（ ```text Agent 可以建模 SQL 并注册 dataset； HTML 页面只能调用 Page Data API（page-data-client.js）； -禁止自建 Express / 独立端口 / 直接暴露 SQLite 文件路径。 +禁止自建 Express / 独立端口 / 直接暴露数据库连接；禁止 SQLite / localStorage 数据回退。 ``` ## 方案选择：默认快车道 + 能力分支（必做） @@ -36,7 +36,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； 用户描述需求 → 匹配能力分支（下表 A/B/C/D，默认 A） → 仅冲突或无法匹配时，用 1 题让用户选分支或改口令 - → 输出「方案摘要」供确认（用户可只改口令/选项） + → 输出简短「方案摘要」；用户已明确要求直接创建/完成/发布时视为已确认并立即执行 → 再 load_skill / 建表 / write_file / bind ``` @@ -50,7 +50,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； | 提交后修改 | 不支持（一次性） | | 页面 | `public/*-survey.html` + `public/*-admin.html`（两文件各 bind 一次） | -用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A，先展示摘要再开工**。 +用户只说「问卷/报名/收集数据 + 后台查看」且未提登录、协作改单、同页后台时 → **用方案 A**。若用户已说「直接做」「完整创建并发布」「不用询问」「持续推进」等终态指令，摘要后必须在同一轮立即调用工具开工，禁止停下来等待再次确认。 口令规则： @@ -83,6 +83,9 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； ```text - public：可匿名 insert；服务端禁止 update / softDelete - password：所有 API 须先 authenticate；口令 ≥8 位且须写入发布记录 +- `public` 策略中的 `insert: true` 等价于任何访客都能直接调用 API 写入；前端口令、隐藏按钮和 JavaScript 判断都不是权限控制 +- password 页面必须调用 `client.authenticate(password)` 获取服务端 token；禁止在 HTML 中硬编码口令或只做 `password === "..."` 比较 +- 同一公开页若要求「所有人可评价、只有所有者可写正文」：正文 dataset 在公开页必须 `insert: false`，评价 dataset 才能 `insert: true`；所有者写正文必须使用独立 `password` 作者页或 `login_required` 页面 - 同页不能同时「匿名 insert」+「口令 read 全量」→ 须拆前台 + 后台（方案 A/E） - password 无法区分访客身份；「只改自己的」须 login_required + own_rows（方案 B） - 匿名提交后「凭链接改自己的」无内置能力，须改 B 或接受一次性提交 @@ -101,7 +104,7 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； - 页面：public/xxx.html + public/xxx-admin.html - 内容：（由你的描述生成题目/字段/报表） -确认后开始建表与写页面。若需登录提交或团队共改，请说明，我换成 B/C 分支。 +若用户尚未授权创建，确认后开始建表与写页面；若用户已明确要求创建/完成/发布，此摘要仅用于告知，后面必须紧接工具调用，不得结束回复等待确认。 ``` ### 分支切换示例 @@ -121,8 +124,8 @@ HTML 页面只能调用 Page Data API（page-data-client.js）； ```text load_skill → page-data-collect -→ 匹配分支（默认 A）→ 方案摘要 → 用户确认或仅改口令/选项 -→ 禁止未确认就 bind；禁止空转计划不调用工具 +→ 匹配分支（默认 A）→ 方案摘要 → 用户确认，或从明确的创建/完成/发布指令判定已确认 +→ 禁止未获创建授权就 bind；已获授权时禁止空转计划、重复确认或不调用工具 ``` ### 2. 数据层：建表 + 注册 dataset @@ -262,7 +265,7 @@ CREATE TABLE IF NOT EXISTS survey_responses ( ## 严格禁止 -1. **禁止**在方案摘要未确认时建表 / bind / 发布（默认 A 也须展示摘要；用户明确「按默认做」视为确认） +1. **禁止**在尚未获得创建授权时建表 / bind / 发布；用户明确要求「创建、完成、发布、直接做、不用询问、持续推进」均视为确认，禁止再次停下来询问 2. **禁止**让 LLM 自行发明 accessMode/拆页/口令；必须落在分支 A～E 与默认口令规则内 3. **禁止**连续两轮仅输出计划、不调用 `load_skill` / `list_dir` / `write_file` / `private_data_execute` / `bind` 4. **禁止**接受 <8 位发布口令；用户未指定口令时后台默认 **`88888888`** @@ -274,7 +277,7 @@ CREATE TABLE IF NOT EXISTS survey_responses ( 10. **禁止**只 `CREATE TABLE` 而不 `private_data_register_dataset` 11. **禁止**未配置 `private_data_set_page_policy` 就让页面调用公开 API 12. **禁止**先发布占位页（如 `

问卷页面

`）再让用户访问 `/u/.../pages/...` -13. **禁止**在 Page Data 页面中使用 `localStorage` / `sessionStorage` 存提交记录或做 API fallback +13. **禁止**在 Agent 生成的 HTML/JavaScript 中使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据或做 API fallback；所有需持久化的数据必须进入当前用户专属 PostgreSQL schema，禁止 SQLite、静态 JSON/JS 文件和内存 fallback ## 交付前自检 diff --git a/skills/static-page-publish/SKILL.md b/skills/static-page-publish/SKILL.md index 7a2802f..aebce39 100644 --- a/skills/static-page-publish/SKILL.md +++ b/skills/static-page-publish/SKILL.md @@ -27,6 +27,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报 10. **禁止**在 HTML 中用 CDN `https://...` 引用 `