Compare commits

...

14 Commits

Author SHA1 Message Date
john b33c943b69 feat(memory-v2): auto-accept candidate memories in active and canary modes
Let Personal Memory candidates pass policy gates without per-user manual review, while keeping shadow mode for explicit human approval.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 14:16:57 +08:00
john 7f7aced751 feat(runtime): wire personal memory observation and skill runtime config
Hook shadow pipeline observation into session finish and agent runs, and
expose skill runtime settings through auth and runtime status endpoints.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 14:00:11 +08:00
john a867592367 feat(skill-runtime): restore admin config and manifest keyword routing
Recover skill runtime policy/admin services and Skill Router v2 helpers so
admins can toggle manifest routing and inspect the platform skill catalog.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 13:59:59 +08:00
john e90a2da6cb feat(memory-v2): restore personal shadow pipeline and candidate store
Recover WIP modules from stash so admin can persist capability config,
observe chat writes in shadow mode, and review persisted candidates.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 13:59:51 +08:00
john 249b0697cb fix(page-data): scope finish guard to current delivery 2026-07-13 06:56:27 +08:00
john cdeb24ba92 fix(page-data): import ensurePageDataHtmlPagesBound in server bootstrap
The bundled portal runtime crashed on boot because workspace deliver referenced
ensurePageDataHtmlPagesBound without importing it from page-data-workspace-ensure.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 00:25:02 +08:00
john fb055c6721 fix(page-data): bind Page Data pages structurally on deliver, not by intent
Finish/sync now auto-registers datasets and binds any HTML using page-data-client.js
before publication, so user confirmation cannot publish unbound collect pages.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 00:18:49 +08:00
john da422a82e6 fix(wechat): restore session routing exports after merge
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-12 22:44:18 +08:00
john 7005b501af Merge branch 'hotfix/page-data-wechat-complete' 2026-07-12 22:43:46 +08:00
john 502334777e fix(page-data): trust live API smoke and WeChat survey submit compat
Stop blocking WeChat delivery when Page Data insert works but publish
quota blocks h5_publish_records; fix smoke URL missing /api; patch survey
radios for WeChat X5 and detect dataset constants in admin pages.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-12 22:43:42 +08:00
john 16fd99e8ba fix(page-data): trust live API smoke and WeChat survey submit compat
Stop blocking WeChat delivery when Page Data insert works but publish
quota blocks h5_publish_records; fix smoke URL missing /api; patch survey
radios for WeChat X5 and detect dataset constants in admin pages.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-12 22:42:47 +08:00
john 2f7bc3b6e4 fix(csp): allow page-data-client.js in WeChat WebView
WeChat UA uses a relaxed inline CSP without 'self', which blocked
/assets/page-data-client.js and broke public survey submissions.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-12 22:21:57 +08:00
john ca4805533b fix(wechat): isolate page data sessions 2026-07-12 21:13:45 +08:00
john cef3750ac6 fix(page-data): require registered datasets before binding 2026-07-12 18:39:13 +08:00
46 changed files with 2836 additions and 180 deletions
+1
View File
@@ -28,6 +28,7 @@ bash scripts/check-release-ready.sh
|------|------|
| MindSpace 公开页 `edit_file` 落盘 + 聊天 Finish 不丢消息 | [docs/regression-guards/mindspace-publish-and-chat-finish.md](docs/regression-guards/mindspace-publish-and-chat-finish.md) |
| MindSpace remote 页面 sync + storage 缺失缩略图 fallback | [docs/regression-guards/mindspace-remote-page-sync-and-thumbnail.md](docs/regression-guards/mindspace-remote-page-sync-and-thumbnail.md) |
| Page Data 数据集注册、绑定与交付验收 | [docs/regression-guards/page-data-delivery-contract.md](docs/regression-guards/page-data-delivery-contract.md) |
索引:[docs/regression-guards/README.md](docs/regression-guards/README.md)
+3
View File
@@ -18,6 +18,7 @@ import { createUserAuth } from './user-auth.mjs';
import { createLlmProviderService } from './llm-providers.mjs';
import { createAssetGatewayConfigService } from './asset-gateway.mjs';
import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs';
import { createSkillRuntimeAdminConfigService } from './skill-runtime-admin-config.mjs';
import { createWechatScheduleLlmConfigService } from './wechat-schedule-llm-config.mjs';
import { createPlazaPostService, formatPostRow } from './plaza-posts.mjs';
import { createPlazaInteractionService } from './plaza-interactions.mjs';
@@ -100,6 +101,7 @@ export async function createAdminServices(env = {}) {
const llmProviderService = createLlmProviderService(pool, { apiTarget, apiSecret });
const assetGatewayConfigService = createAssetGatewayConfigService(pool, { llmProviderService });
const memoryV2ConfigService = createMemoryV2AdminConfigService(pool);
const skillRuntimeConfigService = createSkillRuntimeAdminConfigService(pool, { h5Root });
const wechatScheduleLlmConfigService = createWechatScheduleLlmConfigService(pool);
const adminSystemTestService = createAdminSystemTestService({
pool,
@@ -138,6 +140,7 @@ export async function createAdminServices(env = {}) {
llmProviderService,
assetGatewayConfigService,
memoryV2ConfigService,
skillRuntimeConfigService,
wechatScheduleLlmConfigService,
adminSystemTestService,
plazaPosts,
+36
View File
@@ -31,6 +31,7 @@ function plazaRouteError(res, req, error) {
* @param {object} deps.userAuth
* @param {object|null} deps.llmProviderService
* @param {object|null} deps.memoryV2ConfigService
* @param {object|null} deps.skillRuntimeConfigService
* @param {object|null} deps.adminSystemTestService
* @param {object|null} deps.plazaPosts
* @param {object|null} deps.plazaOps
@@ -44,6 +45,7 @@ export function createAdminApi({
llmProviderService,
assetGatewayConfigService,
memoryV2ConfigService,
skillRuntimeConfigService,
adminSystemTestService,
plazaPosts,
plazaOps,
@@ -148,6 +150,40 @@ export function createAdminApi({
return res.json(result);
});
adminApi.get('/skill-runtime/config', requireAdmin, async (_req, res) => {
if (!skillRuntimeConfigService?.getAdminConfig) {
return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' });
}
return res.json(await skillRuntimeConfigService.getAdminConfig());
});
const updateSkillRuntimeConfig = async (req, res) => {
if (!skillRuntimeConfigService?.updateAdminConfig) {
return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' });
}
const result = await skillRuntimeConfigService.updateAdminConfig(req.body?.config ?? req.body ?? {}, {
updatedBy: req.currentUser.id,
});
return res.json(result);
};
adminApi.put('/skill-runtime/config', requireAdmin, updateSkillRuntimeConfig);
adminApi.patch('/skill-runtime/config', requireAdmin, updateSkillRuntimeConfig);
adminApi.get('/skill-runtime/catalog', requireAdmin, async (_req, res) => {
if (!skillRuntimeConfigService?.listCatalogSummary) {
return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' });
}
return res.json({ catalog: await skillRuntimeConfigService.listCatalogSummary() });
});
adminApi.get('/skill-runtime/runtime', requireAdmin, async (_req, res) => {
if (!skillRuntimeConfigService?.getPublicRuntimeConfig) {
return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' });
}
return res.json(await skillRuntimeConfigService.getPublicRuntimeConfig());
});
adminApi.post('/system-tests/skill-validation', requireAdmin, async (req, res) => {
if (!adminSystemTestService?.runSkillValidation) {
return res.status(503).json({ message: '系统测试服务未启用' });
+60 -3
View File
@@ -12,9 +12,11 @@ import {
} from './conversation-transcript-persist.mjs';
import { ensureGooseUserMessageMetadata } from './goose-message.mjs';
import {
prepareAndDetectSessionDeliverables,
SESSION_FINISHED_STALE_GRACE_MS,
tryRecoverRunFromDeliverables,
} from './agent-run-deliverable-check.mjs';
import { isPageDataIntent } from './chat-skills.mjs';
const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000];
const TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
@@ -53,6 +55,17 @@ function serializeMessage(message) {
return JSON.stringify(message ?? {});
}
function extractRunMessageText(row) {
const message = parseDbJsonColumn(row?.user_message_json, {}) ?? {};
if (typeof message.content === 'string') return message.content;
if (!Array.isArray(message.content)) return '';
return message.content
.filter((item) => item?.type === 'text')
.map((item) => String(item.text ?? '').trim())
.filter(Boolean)
.join('\n');
}
function positiveInteger(value, fallback) {
const n = Number(value);
if (!Number.isFinite(n) || n <= 0) return fallback;
@@ -247,6 +260,8 @@ export function createAgentRunGateway({
sessionSnapshotService = null,
conversationMemoryService = null,
syncUserPagesOnSuccess = null,
observePersonalMemoryOnSuccess = null,
isSessionExternallyBusy = null,
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
maxConcurrentRuns = positiveInteger(
@@ -348,6 +363,15 @@ export function createAgentRunGateway({
// Prevent multiple concurrent agent runs on the same Goose session, which would
// cause replies to arrive out of order and appear garbled in the chat UI.
if (sessionId && !isDirectChatSessionId(sessionId)) {
if (typeof isSessionExternallyBusy === 'function' && await isSessionExternallyBusy({
userId,
sessionId,
})) {
const conflict = new Error('该会话正在完成页面交付或自动修复,请稍候再发送');
conflict.code = 'SESSION_RUN_CONFLICT';
conflict.status = 409;
throw conflict;
}
const [activeRows] = await pool.query(
`SELECT id FROM h5_agent_runs
WHERE agent_session_id = ? AND status NOT IN ('succeeded', 'failed')
@@ -765,19 +789,52 @@ export function createAgentRunGateway({
}
async function finalizeSuccessfulRun(runId, row, sessionId) {
let deliveryResult = null;
if (typeof syncUserPagesOnSuccess === 'function') {
deliveryResult = await syncUserPagesOnSuccess({
userId: row.user_id,
sessionId,
runId,
});
}
const pageDataErrors = Array.isArray(deliveryResult?.pageDataBind?.errors)
? deliveryResult.pageDataBind.errors
: [];
if (pageDataErrors.length > 0) {
const error = new Error(`Page Data 页面绑定失败:${pageDataErrors.map((item) => item?.message ?? item?.code ?? 'unknown').join('; ')}`);
error.code = 'PAGE_DATA_DELIVERY_FAILED';
error.retryable = false;
throw error;
}
if (isPageDataIntent(extractRunMessageText(row))) {
const latest = await getRunById(runId);
const deliverables = await prepareAndDetectSessionDeliverables({
pool,
userId: row.user_id,
sessionId,
runStartedAtMs: latest?.started_at ?? row.started_at ?? null,
});
if (deliverables.pageCount < 1) {
const error = new Error('Page Data 任务未生成可交付页面,不能标记成功');
error.code = 'PAGE_DATA_DELIVERABLE_MISSING';
error.retryable = false;
throw error;
}
}
await markRun(runId, 'succeeded', {
agent_session_id: sessionId,
completed_at: nowMs(),
error_message: null,
});
if (typeof syncUserPagesOnSuccess === 'function') {
await syncUserPagesOnSuccess({
if (typeof observePersonalMemoryOnSuccess === 'function') {
await observePersonalMemoryOnSuccess({
userId: row.user_id,
sessionId,
runId,
userMessage: parseDbJsonColumn(row.user_message_json, {}),
}).catch((err) => {
console.warn(
'[AgentRun] workspace page deliver failed:',
'[AgentRun] personal memory shadow observation failed:',
err instanceof Error ? err.message : err,
);
});
+3 -2
View File
@@ -10,14 +10,15 @@
*/
/**
* @param {{ chatState?: string; finishedViaPortalDirectChat?: boolean }} input
* @param {{ chatState?: string; finishedViaPortalDirectChat?: boolean; agentRunSucceeded?: boolean }} input
* @returns {'idle' | 'streaming'}
*/
export function resolvePostAgentRunChatState({
chatState = 'waiting',
finishedViaPortalDirectChat = false,
agentRunSucceeded = false,
} = {}) {
if (finishedViaPortalDirectChat) return 'idle';
if (finishedViaPortalDirectChat || agentRunSucceeded) return 'idle';
if (chatState === 'idle') return 'idle';
return 'streaming';
}
+11
View File
@@ -25,6 +25,17 @@ test('resolvePostAgentRunChatState prefers portal direct chat completion', () =>
);
});
test('resolvePostAgentRunChatState idles after worker-side agent run success', () => {
assert.equal(
resolvePostAgentRunChatState({ chatState: 'waiting', agentRunSucceeded: true }),
'idle',
);
assert.equal(
resolvePostAgentRunChatState({ chatState: 'streaming', agentRunSucceeded: true }),
'idle',
);
});
test('shouldPromoteSessionIdToStreaming skips re-streaming after Finish', () => {
assert.equal(shouldPromoteSessionIdToStreaming('idle'), false);
assert.equal(shouldPromoteSessionIdToStreaming('waiting'), true);
+93 -4
View File
@@ -1,6 +1,7 @@
// Keep browser-safe: do not import user-publish.mjs (uses node:fs/path/url).
const PUBLISH_SKILL_NAME = 'static-page-publish';
export const PAGE_DATA_COLLECT_SKILL_NAME = 'page-data-collect';
export const SKILL_ROUTER_V2_ENV = 'TKMIND_SKILL_ROUTER_V2';
const WEB_INTENT_PATTERNS = [
/(?:今天|今日|最新|热点|热搜|新闻|头条|头条新闻|最新消息|发生了什么|最近发生)/u,
@@ -31,11 +32,15 @@ const PRODUCT_CAMPAIGN_INTENT_PATTERNS = [
const PAGE_DATA_INTENT_PATTERNS = [
/(?:问卷|调查|签到表|签到登记|签到收集|投票|意见反馈|数据上报)/u,
/(?:报名表|报名登记|在线报名|收集报名)/u,
/(?:作业登记|作业打卡|作业记录|每日作业|打卡登记|台账|记录表)/u,
/(?:表单|数据采集|数据交互|存数据|保存提交|提交记录)/u,
/(?:后台|管理入口|管理后台).{0,20}(?:查看|记录|数据|提交)/u,
/(?:密码|口令).{0,12}(?:查看|后台|管理|进入)/u,
/(?:sqlite|数据库|page[\s-]?data)/i,
/(?:每个用户|各用户).{0,12}(?:提交|记录)/u,
/(?:记账|账本|收支|流水|日记|习惯打卡|每日打卡).{0,40}(?:页面|记录|管理|汇总|统计)/u,
/(?:管理页面|管理页).{0,30}(?:查看|记录|汇总|统计|明细|删除|修改)/u,
/(?:页面|网页|H5|h5).{0,80}(?:每天|每日|新增|添加|填写|记录).{0,80}(?:所有记录|历史记录|管理|汇总|统计)/u,
];
export function isPageDataIntent(text) {
@@ -154,7 +159,7 @@ export function buildChatSkillPrompt(promptKey, skillName) {
case 'page-data-collect':
return (
`请使用 ${skillName ?? PAGE_DATA_COLLECT_SKILL_NAME} 技能:在 MindSpace 页面中实现可提交、可持久化的数据收集(问卷/报名/台账等),必须使用 Page Data API。` +
'先匹配技能内能力分支(默认 A:匿名前台 public insert + 独立后台 password read,口令默认 88888888);展示方案摘要确认后再开工。' +
'先匹配技能内能力分支(默认 A:匿名前台 public insert + 独立后台 password read,口令默认 88888888);“帮我做/创建”本身就是创建授权,展示简短方案摘要后必须同一轮继续执行,只有互斥需求或非法口令才追问。' +
'流程:load_skill → private_data_execute 建表 → private_data_register_dataset → write_file/edit_file 写 public/*.html(含 /assets/page-data-client.js)→ private_data_bind_workspace_page 发布并写策略。' +
'禁止 localStorage / 浏览器本地存储 fallback;禁止自建 Express/独立端口(如 8899)、禁止 HTML 硬编码 127.0.0.1 API、禁止连续空转不调用工具。HTML 视觉规范参照 static-page-publish。完成后只返回 workspaceUrl/MindSpace/<用户ID>/public/...),禁止给用户 /u/用户名/pages/... 链接,并说明后台入口与口令。'
);
@@ -205,9 +210,65 @@ export function filterChatSkills(options, ctx) {
});
}
export function buildAutoChatSkillPrefix(text, grantedSkills = []) {
const trimmed = String(text ?? '').trim();
if (!trimmed) return '';
const PROMPT_KEY_BY_SKILL_NAME = new Map(
CHAT_SKILL_DEFINITIONS.filter((item) => item.skillName && item.promptKey).map((item) => [
item.skillName,
item.promptKey,
]),
);
/** Opt-in Skill Router v2. Default off unless TKMIND_SKILL_ROUTER_V2=1 or options.skillRouterV2=true. */
export function isSkillRouterV2Enabled(explicit) {
if (explicit === true) return true;
if (explicit === false) return false;
const raw =
typeof process !== 'undefined' && process?.env ? process.env[SKILL_ROUTER_V2_ENV] : undefined;
return /^(1|true|yes)$/i.test(String(raw ?? ''));
}
/** Build manifest routes from platform catalog entries that declare trigger.keywords. */
export function buildManifestRoutesFromCatalog(catalog = []) {
return catalog
.filter((item) => Array.isArray(item?.manifest?.trigger?.keywords) && item.manifest.trigger.keywords.length > 0)
.map((item) => ({
skillName: item.name,
keywords: item.manifest.trigger.keywords,
priority: Number(item.manifest?.router?.priority ?? 0),
promptKey:
item.manifest?.router?.promptKey ??
PROMPT_KEY_BY_SKILL_NAME.get(item.name) ??
null,
promptVariant: item.manifest?.router?.promptVariant ?? null,
}))
.sort((a, b) => b.priority - a.priority || a.skillName.localeCompare(b.skillName));
}
export function matchManifestSkillRoute(text, routes = [], grantedSkills = []) {
const normalized = String(text ?? '').trim().toLowerCase();
if (!normalized || !Array.isArray(routes) || routes.length === 0) return null;
const granted = new Set(grantedSkills);
for (const route of routes) {
if (!route?.skillName || !granted.has(route.skillName)) continue;
const keywords = Array.isArray(route.keywords) ? route.keywords : [];
if (keywords.some((keyword) => normalized.includes(String(keyword).trim().toLowerCase()))) {
return route;
}
}
return null;
}
function buildManifestSkillPrompt(route) {
if (!route) return '';
if (route.promptVariant === 'web-news') {
return buildWebNewsSkillPrompt(route.skillName);
}
if (route.promptKey) {
return buildChatSkillPrompt(route.promptKey, route.skillName);
}
return `请使用 ${route.skillName} 技能:`;
}
function buildLegacyAutoChatSkillPrefix(trimmed, grantedSkills) {
if (
grantedSkills.includes(PAGE_DATA_COLLECT_SKILL_NAME) &&
isPageDataIntent(trimmed)
@@ -225,6 +286,34 @@ export function buildAutoChatSkillPrefix(text, grantedSkills = []) {
return buildChatSkillPrompt('web', 'web');
}
export function buildAutoChatSkillPrefixOptions(publicConfig) {
if (!publicConfig?.routerV2Enabled || !publicConfig?.manifestRoutingEnabled) {
return {};
}
return {
skillRouterV2: true,
manifestRoutes: publicConfig.manifestRoutes ?? [],
};
}
export function buildAutoChatSkillPrefix(text, grantedSkills = [], options = {}) {
const trimmed = String(text ?? '').trim();
if (!trimmed) return '';
if (
isSkillRouterV2Enabled(options.skillRouterV2) &&
Array.isArray(options.manifestRoutes) &&
options.manifestRoutes.length > 0
) {
const matched = matchManifestSkillRoute(trimmed, options.manifestRoutes, grantedSkills);
if (matched) {
return buildManifestSkillPrompt(matched);
}
}
return buildLegacyAutoChatSkillPrefix(trimmed, grantedSkills);
}
export function stripKnownChatSkillPrompt(text) {
let next = String(text ?? '').trimStart();
const candidates = [];
+1
View File
@@ -8,6 +8,7 @@
|------|----------|
| [mindspace-publish-and-chat-finish.md](./mindspace-publish-and-chat-finish.md) | ① `edit_file` 覆盖 `public/*.html` ② Finish 后聊天不清空、不暴露 agent 内部前缀 |
| [mindspace-remote-page-sync-and-thumbnail.md](./mindspace-remote-page-sync-and-thumbnail.md) | ① remote 模式 public HTML 入库 sync ② storage 缺失时缩略图/读页回退 workspace HTML |
| [page-data-delivery-contract.md](./page-data-delivery-contract.md) | 数据集注册、绑定、真实 page UUID 与交付验收 |
## 自动化
@@ -0,0 +1,28 @@
# Page Data 交付契约
Page Data 页面只有在下面所有条件满足后才可向用户交付链接:
1. HTML 实际调用的每个 dataset 已在工作区 `.mindspace/private-data.sqlite` 注册。
2. 每个注册 dataset 的真实 SQLite 表、声明字段和读写 action 都存在。
3. page record、online publication 与 policy 使用同一个真实 page UUID。
4. 公开页完成一次受权限约束的 insert smoke;后台页完成 password auth 后的 read smoke。
`private_data_bind_workspace_page` 是硬门:它在创建 page / publication / policy 前,必须从 SQLite registry 派生权限。Agent 传入的策略不能凭空创建 dataset 或字段权限。
运行时路径必须按语义区分:
- `workspaceRoot``MindSpace/<user>`,保存 HTML、policy 和 private-data.sqlite。
- `storageRoot`MindSpace service 的持久页面/资产存储。
- `usersRoot`:登录用户目录。
不要通过 `MINDSPACE_STORAGE_ROOT` 推断 Page Data 的 workspaceRoot。Portal、MindSpace service 与 sandbox MCP 必须显式使用同一 workspace contract。
回归命令:
```bash
npm run verify:page-data
npm run verify:mindspace-publish-guards:full
npm run verify:mindspace-page-sync-guards
```
涉及 H5 交付时,还必须验证:未注册 dataset 时不产生可用 Page Data policy,且最终链接交付被拒绝或进入明确 repair 状态。
+61
View File
@@ -22,6 +22,59 @@ const FIELD_SPECS = [
{ env: 'MEMIND_CHAT_ROUTER_TIMEOUT_MS', group: 'chatIntentRouter', field: 'timeoutMs', type: 'number' },
{ env: 'MEMIND_CHAT_ROUTER_FALLBACK_ROUTE', group: 'chatIntentRouter', field: 'fallbackRoute', type: 'string' },
{ env: 'MEMORY_CANDIDATE_ENABLED', group: 'candidateMemory', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_CANDIDATE_MODE', group: 'candidateMemory', field: 'mode', type: 'string' },
{ env: 'MEMORY_CANDIDATE_MIN_IMPORTANCE', group: 'candidateMemory', field: 'minImportance', type: 'number' },
{ env: 'MEMORY_CANDIDATE_MIN_CONFIDENCE', group: 'candidateMemory', field: 'minConfidence', type: 'number' },
{ env: 'MEMORY_CANDIDATE_MAX_PENDING', group: 'candidateMemory', field: 'maxPending', type: 'number' },
{ env: 'MEMORY_CANDIDATE_PERSISTENCE_ENABLED', group: 'candidateMemory', field: 'persistenceEnabled', type: 'boolean' },
{ env: 'MEMORY_POLICY_ENABLED', group: 'policy', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_POLICY_SAVE_EXPLICIT', group: 'policy', field: 'saveExplicit', type: 'boolean' },
{ env: 'MEMORY_POLICY_REJECT_SENSITIVE', group: 'policy', field: 'rejectSensitive', type: 'boolean' },
{ env: 'MEMORY_POLICY_REQUIRE_EVIDENCE', group: 'policy', field: 'requireEvidence', type: 'boolean' },
{ env: 'MEMORY_POLICY_RETENTION_DAYS', group: 'policy', field: 'retentionDays', type: 'number' },
{ env: 'MEMORY_RETRIEVER_ENABLED', group: 'retriever', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_RETRIEVER_EPISODIC_ENABLED', group: 'retriever', field: 'episodicEnabled', type: 'boolean' },
{ env: 'MEMORY_RETRIEVER_SEMANTIC_ENABLED', group: 'retriever', field: 'semanticEnabled', type: 'boolean' },
{ env: 'MEMORY_RETRIEVER_PREFERENCE_ENABLED', group: 'retriever', field: 'preferenceEnabled', type: 'boolean' },
{ env: 'MEMORY_RETRIEVER_GOAL_ENABLED', group: 'retriever', field: 'goalEnabled', type: 'boolean' },
{ env: 'MEMORY_RETRIEVER_LIMIT', group: 'retriever', field: 'limit', type: 'number' },
{ env: 'MEMORY_RETRIEVER_TOKEN_BUDGET', group: 'retriever', field: 'tokenBudget', type: 'number' },
{ env: 'MEMORY_RETRIEVER_TIMEOUT_MS', group: 'retriever', field: 'timeoutMs', type: 'number' },
{ env: 'MEMORY_LIFECYCLE_ENABLED', group: 'lifecycle', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_LIFECYCLE_DEDUPE_ENABLED', group: 'lifecycle', field: 'dedupeEnabled', type: 'boolean' },
{ env: 'MEMORY_LIFECYCLE_CONFLICT_REVIEW', group: 'lifecycle', field: 'conflictReview', type: 'boolean' },
{ env: 'MEMORY_LIFECYCLE_DECAY_ENABLED', group: 'lifecycle', field: 'decayEnabled', type: 'boolean' },
{ env: 'MEMORY_LIFECYCLE_FORGETTING_ENABLED', group: 'lifecycle', field: 'forgettingEnabled', type: 'boolean' },
{ env: 'MEMORY_LIFECYCLE_COMPACT_INTERVAL_HOURS', group: 'lifecycle', field: 'compactIntervalHours', type: 'number' },
{ env: 'MEMORY_PERSONA_ENABLED', group: 'persona', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_PERSONA_PROVIDER', group: 'persona', field: 'provider', type: 'string' },
{ env: 'MEMORY_PERSONA_SHADOW_MODE', group: 'persona', field: 'shadowMode', type: 'boolean' },
{ env: 'MEMORY_PERSONA_MAX_TOKENS', group: 'persona', field: 'maxTokens', type: 'number' },
{ env: 'MEMORY_PERSONA_CACHE_TTL_SECONDS', group: 'persona', field: 'cacheTtlSeconds', type: 'number' },
{ env: 'MEMORY_GRAPH_ENABLED', group: 'graph', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_GRAPH_PROVIDER', group: 'graph', field: 'provider', type: 'string' },
{ env: 'MEMORY_GRAPH_MAX_DEPTH', group: 'graph', field: 'maxDepth', type: 'number' },
{ env: 'MEMORY_GRAPH_RELATION_LIMIT', group: 'graph', field: 'relationLimit', type: 'number' },
{ env: 'MEMORY_USER_MANAGEMENT_ENABLED', group: 'userMemory', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_USER_REVIEW_ENABLED', group: 'userMemory', field: 'reviewEnabled', type: 'boolean' },
{ env: 'MEMORY_USER_CORRECTION_ENABLED', group: 'userMemory', field: 'correctionEnabled', type: 'boolean' },
{ env: 'MEMORY_USER_PIN_ENABLED', group: 'userMemory', field: 'pinEnabled', type: 'boolean' },
{ env: 'MEMORY_USER_FORGET_ENABLED', group: 'userMemory', field: 'forgetEnabled', type: 'boolean' },
{ env: 'MEMORY_USER_DELETE_PROPAGATION', group: 'userMemory', field: 'deletePropagation', type: 'boolean' },
{ env: 'MEMORY_PLUGIN_HEALTH_ENABLED', group: 'pluginHealth', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_PLUGIN_HEALTH_INTERVAL_SECONDS', group: 'pluginHealth', field: 'intervalSeconds', type: 'number' },
{ env: 'MEMORY_PLUGIN_HEALTH_TIMEOUT_MS', group: 'pluginHealth', field: 'timeoutMs', type: 'number' },
{ env: 'MEMORY_PLUGIN_HEALTH_FAILURE_THRESHOLD', group: 'pluginHealth', field: 'failureThreshold', type: 'number' },
{ env: 'MEMORY_PLUGIN_HEALTH_AUTO_FALLBACK', group: 'pluginHealth', field: 'autoFallback', type: 'boolean' },
{ env: 'MEMORY_PGVECTOR_ENABLED', group: 'pgvector', field: 'enabled', type: 'boolean' },
{ env: 'MEMORY_PGVECTOR_DATABASE_URL', group: 'pgvector', field: 'databaseUrl', type: 'secret' },
{ env: 'MEMORY_PGVECTOR_TABLE', group: 'pgvector', field: 'table', type: 'string' },
@@ -95,6 +148,14 @@ const FIELD_SPECS = [
const GROUPS = [
'global',
'chatIntentRouter',
'candidateMemory',
'policy',
'retriever',
'lifecycle',
'persona',
'graph',
'userMemory',
'pluginHealth',
'pgvector',
'qdrant',
'weaviate',
+49
View File
@@ -81,6 +81,36 @@ test('memory v2 admin config service persists non-secret and secret patches', as
url: 'http://127.0.0.1:6333',
apiKey: 'secret-qdrant',
},
candidateMemory: {
enabled: true,
mode: 'shadow',
minImportance: '0.75',
minConfidence: '0.85',
maxPending: '500',
persistenceEnabled: true,
},
policy: {
enabled: true,
saveExplicit: true,
rejectSensitive: true,
requireEvidence: true,
retentionDays: '365',
},
retriever: {
enabled: true,
episodicEnabled: true,
semanticEnabled: true,
preferenceEnabled: true,
goalEnabled: true,
limit: '12',
tokenBudget: '1800',
timeoutMs: '1200',
},
lifecycle: { enabled: true, dedupeEnabled: true, conflictReview: true },
persona: { enabled: true, provider: 'ai-mind', shadowMode: true },
graph: { enabled: true, provider: 'postgres', maxDepth: '2' },
userMemory: { enabled: true, reviewEnabled: true, forgetEnabled: true, deletePropagation: true },
pluginHealth: { enabled: true, intervalSeconds: '60', timeoutMs: '1500', failureThreshold: '3', autoFallback: true },
}, { updatedBy: 'admin-1' });
assert.equal(updated.config.global.enabled, true);
@@ -91,6 +121,15 @@ test('memory v2 admin config service persists non-secret and secret patches', as
assert.equal(updated.config.qdrant.enabled, true);
assert.equal(updated.config.qdrant.url, 'http://127.0.0.1:6333');
assert.equal(updated.config.qdrant.apiKeyConfigured, true);
assert.equal(updated.config.candidateMemory.mode, 'shadow');
assert.equal(updated.config.candidateMemory.persistenceEnabled, true);
assert.equal(updated.config.policy.requireEvidence, true);
assert.equal(updated.config.retriever.tokenBudget, '1800');
assert.equal(updated.config.lifecycle.dedupeEnabled, true);
assert.equal(updated.config.persona.provider, 'ai-mind');
assert.equal(updated.config.graph.provider, 'postgres');
assert.equal(updated.config.userMemory.deletePropagation, true);
assert.equal(updated.config.pluginHealth.autoFallback, true);
assert.equal(updated.updatedBy, 'admin-1');
const runtimeState = await service.getRuntimeState();
@@ -107,6 +146,16 @@ test('memory v2 admin config service persists non-secret and secret patches', as
assert.equal(runtimeState.overrides.MEMORY_QDRANT_ENABLED, '1');
assert.equal(runtimeState.overrides.MEMORY_QDRANT_URL, 'http://127.0.0.1:6333');
assert.equal(runtimeState.overrides.MEMORY_QDRANT_API_KEY, 'secret-qdrant');
assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_ENABLED, '1');
assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_MODE, 'shadow');
assert.equal(runtimeState.overrides.MEMORY_CANDIDATE_PERSISTENCE_ENABLED, '1');
assert.equal(runtimeState.overrides.MEMORY_POLICY_REQUIRE_EVIDENCE, '1');
assert.equal(runtimeState.overrides.MEMORY_RETRIEVER_TOKEN_BUDGET, '1800');
assert.equal(runtimeState.overrides.MEMORY_LIFECYCLE_DEDUPE_ENABLED, '1');
assert.equal(runtimeState.overrides.MEMORY_PERSONA_PROVIDER, 'ai-mind');
assert.equal(runtimeState.overrides.MEMORY_GRAPH_PROVIDER, 'postgres');
assert.equal(runtimeState.overrides.MEMORY_USER_DELETE_PROPAGATION, '1');
assert.equal(runtimeState.overrides.MEMORY_PLUGIN_HEALTH_AUTO_FALLBACK, '1');
});
test('memory v2 admin config internals flatten booleans and numbers consistently', () => {
+270
View File
@@ -0,0 +1,270 @@
import crypto from 'node:crypto';
import { deriveUserFacingText } from './conversation-display.mjs';
const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']);
const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
const SECRET_PATTERNS = [
/\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i,
/\bsk-[a-z0-9_-]{12,}\b/i,
/-----BEGIN [A-Z ]+PRIVATE KEY-----/,
/(?:密码|口令|密钥|令牌)\s*[:=]\s*\S+/i,
];
const TRIVIAL_PATTERNS = [
/^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[!。.\s]*$/i,
/^(?:查一下|搜一下|看看|继续|下一步)[。.!?\s]*$/i,
];
function readFlag(env, key, fallback = false) {
const raw = env?.[key];
if (raw == null || raw === '') return fallback;
const normalized = String(raw).trim().toLowerCase();
if (TRUE_VALUES.has(normalized)) return true;
if (FALSE_VALUES.has(normalized)) return false;
return fallback;
}
function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) {
const value = Number(env?.[key]);
if (!Number.isFinite(value)) return fallback;
return Math.min(max, Math.max(min, value));
}
function normalizeText(value, maxLength = 2000) {
const text = String(value ?? '').replace(/\s+/g, ' ').trim();
if (!text) return '';
return text.length > maxLength ? text.slice(0, maxLength) : text;
}
function messageText(message) {
if (typeof message === 'string') return normalizeText(message);
if (!message || typeof message !== 'object') return '';
const displayText = message.displayText
?? message.display_text
?? message.metadata?.displayText
?? message.metadata?.display_text;
if (typeof displayText === 'string' && displayText.trim()) {
return normalizeText(displayText);
}
const direct = message.text ?? message.content ?? message.message ?? '';
if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct));
if (Array.isArray(direct)) {
return normalizeText(deriveUserFacingText(
direct.map((item) => item?.text ?? item?.content ?? '').join('\n'),
));
}
return '';
}
function stableHash(value) {
return crypto.createHash('sha256').update(String(value)).digest('hex');
}
function classify(text) {
const rules = [
{ type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' },
{ type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' },
{ type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' },
{ type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' },
{ type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' },
];
return rules.find((rule) => rule.pattern.test(text)) ?? null;
}
const CANDIDATE_MODES = new Set(['off', 'shadow', 'canary', 'active']);
function normalizeCandidateMode(value, fallback = 'shadow') {
const normalized = normalizeText(value, 20).toLowerCase();
return CANDIDATE_MODES.has(normalized) ? normalized : fallback;
}
export function shouldAutoAcceptCandidate(candidate, config) {
if (!candidate || !config?.enabled) return false;
const mode = normalizeCandidateMode(config.requestedMode, 'shadow');
if (mode === 'shadow' || mode === 'off') return false;
if (mode === 'active') return true;
if (candidate.policyReason === 'explicit_memory_request') return true;
return Number(candidate.confidence) >= Math.max(Number(config.minConfidence) || 0, 0.9);
}
export function resolvePersonalShadowConfig(env = process.env) {
const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false);
const requestedMode = normalizeCandidateMode(
env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'active' : 'off'),
candidateEnabled ? 'active' : 'off',
);
const enabled = candidateEnabled && requestedMode !== 'off';
return {
enabled,
requestedMode,
effectiveMode: enabled ? requestedMode : 'off',
autoReviewEnabled: enabled && requestedMode !== 'shadow',
minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }),
minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }),
maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })),
policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true),
saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true),
rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true),
requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true),
healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true),
};
}
export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) {
const config = resolvePersonalShadowConfig(env);
const candidates = [];
const candidateHashes = new Set();
const metrics = {
runs: 0,
observedMessages: 0,
accepted: 0,
autoReviewed: 0,
pendingReview: 0,
rejected: 0,
deduped: 0,
errors: 0,
lastRunAt: null,
lastError: null,
};
function reject(reason) {
metrics.rejected += 1;
return { accepted: false, reason };
}
function evaluate({ userId, sessionId, message, index }) {
const text = messageText(message);
if (!text || text.length < 8) return reject('too_short');
if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial');
if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) {
return reject('sensitive_content');
}
const classification = classify(text);
if (!classification) return reject('no_durable_signal');
if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) {
return reject('explicit_memory_disabled');
}
if (config.policyEnabled && classification.importance < config.minImportance) {
return reject('below_importance_threshold');
}
if (config.policyEnabled && classification.confidence < config.minConfidence) {
return reject('below_confidence_threshold');
}
const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`);
if (candidateHashes.has(contentHash)) {
metrics.deduped += 1;
return { accepted: false, reason: 'duplicate' };
}
const capturedAt = now();
const evidence = {
sourceType: 'message',
sourceId: `${sessionId || 'unknown'}:${index}`,
evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`),
capturedAt,
};
if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required');
const candidate = {
id: `pmc_${contentHash.slice(0, 24)}`,
userId: String(userId),
sessionId: sessionId ? String(sessionId) : null,
memoryType: classification.type,
content: text,
importance: classification.importance,
confidence: classification.confidence,
status: 'candidate',
policyReason: classification.reason,
evidence,
createdAt: capturedAt,
};
candidateHashes.add(contentHash);
candidates.push(candidate);
while (candidates.length > config.maxPending) {
const removed = candidates.shift();
if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`));
}
metrics.accepted += 1;
return { accepted: true, candidate };
}
async function observeWrite({ userId, sessionId, messages = [] } = {}) {
if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' };
metrics.runs += 1;
metrics.lastRunAt = now();
if (!userId || !Array.isArray(messages)) {
return { enabled: true, skipped: true, reason: 'invalid_input' };
}
try {
const results = [];
for (let index = 0; index < messages.length; index += 1) {
const message = messages[index];
const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase();
if (role && !['user', 'human'].includes(role)) continue;
metrics.observedMessages += 1;
const result = evaluate({ userId, sessionId, message, index });
if (result.accepted && store?.saveCandidate) {
const autoAccept = shouldAutoAcceptCandidate(result.candidate, config);
const persisted = await store.saveCandidate(result.candidate, { autoAccept });
if (persisted?.inserted === false) {
metrics.deduped += 1;
} else if (autoAccept) {
result.candidate.status = 'accepted';
metrics.autoReviewed += 1;
} else {
metrics.pendingReview += 1;
}
} else if (result.accepted) {
const autoAccept = shouldAutoAcceptCandidate(result.candidate, config);
result.candidate.status = autoAccept ? 'accepted' : 'candidate';
if (autoAccept) metrics.autoReviewed += 1;
else metrics.pendingReview += 1;
}
results.push(result);
}
return {
enabled: true,
skipped: false,
mode: config.effectiveMode,
autoReviewEnabled: config.autoReviewEnabled,
observed: results.length,
accepted: results.filter((result) => result.accepted).length,
autoReviewed: results.filter((result) => result.accepted && result.candidate?.status === 'accepted').length,
pendingReview: results.filter((result) => result.accepted && result.candidate?.status === 'candidate').length,
rejected: results.filter((result) => !result.accepted).length,
};
} catch (err) {
metrics.errors += 1;
metrics.lastError = err instanceof Error ? err.message : String(err);
return { enabled: true, skipped: true, reason: 'pipeline_failure' };
}
}
function getStatus() {
return {
enabled: config.enabled,
requestedMode: config.requestedMode,
effectiveMode: config.effectiveMode,
phase: config.autoReviewEnabled ? 'auto-review-v1' : 'shadow-candidate-v1',
autoReviewEnabled: config.autoReviewEnabled,
injectionEnabled: false,
persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory',
pendingCandidates: candidates.length,
health: {
enabled: config.healthEnabled,
state: metrics.errors > 0 ? 'degraded' : 'healthy',
...metrics,
},
policy: {
enabled: config.policyEnabled,
minImportance: config.minImportance,
minConfidence: config.minConfidence,
rejectSensitive: config.rejectSensitive,
requireEvidence: config.requireEvidence,
},
};
}
function listCandidates({ limit = 50 } = {}) {
return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse();
}
return { config, observeWrite, getStatus, listCandidates };
}
+283
View File
@@ -0,0 +1,283 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
createPersonalMemoryShadowPipeline,
resolvePersonalShadowConfig,
shouldAutoAcceptCandidate,
} from './memory-v2-personal-shadow.mjs';
import { createMemoryV2 } from './memory-v2.mjs';
test('personal memory shadow config maps active mode to auto review', () => {
const config = resolvePersonalShadowConfig({
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'active',
});
assert.equal(config.enabled, true);
assert.equal(config.requestedMode, 'active');
assert.equal(config.effectiveMode, 'active');
assert.equal(config.autoReviewEnabled, true);
});
test('personal memory shadow config keeps shadow mode for manual review', () => {
const config = resolvePersonalShadowConfig({
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'shadow',
});
assert.equal(config.effectiveMode, 'shadow');
assert.equal(config.autoReviewEnabled, false);
});
test('shouldAutoAcceptCandidate auto accepts explicit requests in canary mode', () => {
const config = resolvePersonalShadowConfig({
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'canary',
MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8',
});
assert.equal(shouldAutoAcceptCandidate({
policyReason: 'explicit_memory_request',
confidence: 0.98,
}, config), true);
assert.equal(shouldAutoAcceptCandidate({
policyReason: 'stable_fact_signal',
confidence: 0.82,
}, config), false);
assert.equal(shouldAutoAcceptCandidate({
policyReason: 'decision_signal',
confidence: 0.9,
}, config), true);
});
test('shadow pipeline auto accepts durable candidates in active mode', async () => {
const saved = [];
const pipeline = createPersonalMemoryShadowPipeline({
env: {
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'active',
MEMORY_POLICY_ENABLED: '1',
},
store: {
async saveCandidate(candidate, options = {}) {
saved.push({ candidate, options });
return { inserted: true, status: options.autoAccept ? 'accepted' : 'candidate' };
},
},
});
const result = await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [{ role: 'user', text: '我决定所有生产发布必须从完整 main 分支打包。' }],
});
assert.equal(result.autoReviewed, 1);
assert.equal(result.pendingReview, 0);
assert.equal(saved.length, 1);
assert.equal(saved[0].options.autoAccept, true);
assert.equal(pipeline.getStatus().autoReviewEnabled, true);
assert.equal(pipeline.getStatus().phase, 'auto-review-v1');
});
test('shadow pipeline keeps low-confidence canary candidates pending review', async () => {
const saved = [];
const pipeline = createPersonalMemoryShadowPipeline({
env: {
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'canary',
MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8',
},
store: {
async saveCandidate(candidate, options = {}) {
saved.push({ candidate, options });
return { inserted: true, status: options.autoAccept ? 'accepted' : 'candidate' };
},
},
});
await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [{ role: 'user', text: '项目使用 pgvector 作为向量存储。' }],
});
assert.equal(saved.length, 1);
assert.equal(saved[0].options.autoAccept, false);
assert.equal(pipeline.getStatus().health.pendingReview, 1);
});
test('shadow pipeline extracts durable candidates with evidence and deduplicates', async () => {
let clock = 1000;
const pipeline = createPersonalMemoryShadowPipeline({
env: {
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'shadow',
MEMORY_POLICY_ENABLED: '1',
MEMORY_POLICY_REQUIRE_EVIDENCE: '1',
MEMORY_CANDIDATE_MIN_IMPORTANCE: '0.7',
MEMORY_CANDIDATE_MIN_CONFIDENCE: '0.8',
},
now: () => {
clock += 1;
return clock;
},
});
const input = {
userId: 'u1',
sessionId: 's1',
messages: [
{ role: 'user', text: '我决定所有生产发布必须从完整 main 分支打包。' },
{ role: 'assistant', text: '收到。' },
],
};
const first = await pipeline.observeWrite(input);
const second = await pipeline.observeWrite(input);
assert.equal(first.accepted, 1);
assert.equal(second.accepted, 0);
assert.equal(pipeline.listCandidates().length, 1);
assert.equal(pipeline.listCandidates()[0].memoryType, 'episodic');
assert.equal(pipeline.listCandidates()[0].evidence.sourceId, 's1:0');
assert.equal(pipeline.getStatus().health.deduped, 1);
assert.equal(pipeline.getStatus().injectionEnabled, false);
});
test('shadow policy rejects trivial and sensitive messages', async () => {
const pipeline = createPersonalMemoryShadowPipeline({
env: {
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'shadow',
MEMORY_POLICY_REJECT_SENSITIVE: '1',
},
});
const result = await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [
{ role: 'user', text: '谢谢' },
{ role: 'user', text: '请记住 API_KEY=sk-this-is-a-secret-token' },
{ role: 'user', text: '今天天气怎么样?' },
],
});
assert.equal(result.accepted, 0);
assert.equal(result.rejected, 3);
assert.equal(pipeline.listCandidates().length, 0);
});
test('shadow pipeline prefers user-facing displayText over internal prompt content', async () => {
const pipeline = createPersonalMemoryShadowPipeline({
env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' },
});
await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [{
role: 'user',
content: [{ type: 'text', text: '[用户身份] 内部前缀\n我决定先测试再发布。' }],
metadata: { displayText: '我决定先测试再发布。' },
}],
});
assert.equal(pipeline.listCandidates().length, 1);
assert.equal(pipeline.listCandidates()[0].content, '我决定先测试再发布。');
assert.doesNotMatch(pipeline.listCandidates()[0].content, /用户身份/);
});
test('shadow pipeline strips protected user identity prefix when displayText is absent', async () => {
const pipeline = createPersonalMemoryShadowPipeline({
env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' },
});
await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [{
role: 'user',
content: [{
type: 'text',
text: '[用户身份]\n- 当前登录用户称呼:John\n- 内部提示\n\n我决定先测试再发布。',
}],
}],
});
assert.equal(pipeline.listCandidates().length, 1);
assert.equal(pipeline.listCandidates()[0].content, '我决定先测试再发布。');
});
test('shadow candidate pool remains bounded', async () => {
const pipeline = createPersonalMemoryShadowPipeline({
env: {
MEMORY_CANDIDATE_ENABLED: '1',
MEMORY_CANDIDATE_MODE: 'shadow',
MEMORY_CANDIDATE_MAX_PENDING: '2',
},
});
await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [
{ role: 'user', text: '我决定项目一使用完整 main 发布。' },
{ role: 'user', text: '我决定项目二使用完整 main 发布。' },
{ role: 'user', text: '我决定项目三使用完整 main 发布。' },
],
});
assert.equal(pipeline.listCandidates().length, 2);
assert.match(pipeline.listCandidates()[0].content, /项目三/);
});
test('shadow pipeline persists accepted candidates when a store is configured', async () => {
const saved = [];
const pipeline = createPersonalMemoryShadowPipeline({
env: { MEMORY_CANDIDATE_ENABLED: '1', MEMORY_CANDIDATE_MODE: 'shadow' },
store: { async saveCandidate(candidate) { saved.push(candidate); return { inserted: true }; } },
});
await pipeline.observeWrite({
userId: 'u1',
sessionId: 's1',
messages: [{ role: 'user', text: '我的长期目标是建设一个持续成长的 Personal Agent。' }],
});
assert.equal(saved.length, 1);
assert.equal(saved[0].memoryType, 'goal');
assert.equal(pipeline.getStatus().persistence, 'mysql');
});
test('Memory V2 does not wait for the shadow pipeline before returning legacy write result', async () => {
let releaseShadow;
const shadowBlocked = new Promise((resolve) => { releaseShadow = resolve; });
const memory = createMemoryV2({
policy: { enabled: true, eventLogEnabled: true, failOpen: true, backend: 'legacy' },
backends: [{
name: 'legacy-conversation-memory',
isAvailable: () => true,
async write() { return { saved: 1, analyzed: 0, memories: 0 }; },
}],
personalShadowPipeline: {
config: { enabled: true },
observeWrite: () => shadowBlocked,
getStatus: () => ({ enabled: true }),
},
});
const result = await Promise.race([
memory.write({ userId: 'u1', sessionId: 's1', messages: [] }),
new Promise((_, reject) => setTimeout(() => reject(new Error('write waited for shadow')), 50)),
]);
assert.equal(result.saved, 1);
releaseShadow();
});
test('Memory V2 exposes a shadow-only observation entry without calling legacy write', async () => {
let legacyWrites = 0;
const observed = [];
const memory = createMemoryV2({
policy: { enabled: true, eventLogEnabled: true, failOpen: true, backend: 'legacy' },
backends: [{
name: 'legacy-conversation-memory',
isAvailable: () => true,
async write() { legacyWrites += 1; return { saved: 1 }; },
}],
personalShadowPipeline: {
config: { enabled: true },
async observeWrite(input) { observed.push(input); return { accepted: 1 }; },
getStatus: () => ({ enabled: true }),
},
});
const result = await memory.observePersonalMemory({
userId: 'u1',
sessionId: 's1',
messages: [{ role: 'user', text: '我决定先完成测试。' }],
});
assert.equal(result.accepted, 1);
assert.equal(observed.length, 1);
assert.equal(legacyWrites, 0);
});
+133
View File
@@ -0,0 +1,133 @@
const TABLE = 'h5_memory_v2_candidates';
const ALLOWED_STATUSES = new Set(['candidate', 'accepted', 'rejected', 'forgotten']);
export function buildPersonalMemoryCandidateSchemaSql({ table = TABLE } = {}) {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error('Invalid candidate table name');
return `CREATE TABLE IF NOT EXISTS \`${table}\` (
id VARCHAR(64) PRIMARY KEY,
user_id CHAR(36) NOT NULL,
session_id VARCHAR(191) NULL,
memory_type VARCHAR(32) NOT NULL,
content TEXT NOT NULL,
importance DECIMAL(5,4) NOT NULL,
confidence DECIMAL(5,4) NOT NULL,
status VARCHAR(24) NOT NULL DEFAULT 'candidate',
policy_reason VARCHAR(64) NOT NULL,
evidence_json JSON NOT NULL,
reviewed_by CHAR(36) NULL,
reviewed_at BIGINT NULL,
created_at BIGINT NOT NULL,
updated_at BIGINT NOT NULL,
UNIQUE KEY uniq_user_type_content (user_id, memory_type, id),
KEY idx_candidate_status_created (status, created_at),
KEY idx_candidate_user_status (user_id, status, created_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`;
}
export async function ensurePersonalMemoryCandidateSchema(pool, options = {}) {
if (!pool?.query) throw new Error('Candidate schema requires a MySQL pool');
await pool.query(buildPersonalMemoryCandidateSchemaSql(options));
}
function parseJson(value, fallback = null) {
if (value == null) return fallback;
if (typeof value === 'object') return value;
try { return JSON.parse(value); } catch { return fallback; }
}
function normalizeRow(row) {
return {
id: String(row.id),
userId: String(row.user_id),
sessionId: row.session_id == null ? null : String(row.session_id),
memoryType: String(row.memory_type),
content: String(row.content),
importance: Number(row.importance),
confidence: Number(row.confidence),
status: String(row.status),
policyReason: String(row.policy_reason),
evidence: parseJson(row.evidence_json, {}),
reviewedBy: row.reviewed_by == null ? null : String(row.reviewed_by),
reviewedAt: row.reviewed_at == null ? null : Number(row.reviewed_at),
createdAt: Number(row.created_at),
updatedAt: Number(row.updated_at),
};
}
export function createPersonalMemoryCandidateStore(pool, { table = TABLE, now = () => Date.now() } = {}) {
if (!pool?.query) return null;
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table)) throw new Error('Invalid candidate table name');
const quoted = `\`${table}\``;
return {
async saveCandidate(candidate, { autoAccept = false, reviewedBy = 'system:auto-review' } = {}) {
const updatedAt = now();
const status = autoAccept ? 'accepted' : 'candidate';
const reviewedAt = autoAccept ? updatedAt : null;
const reviewer = autoAccept ? String(reviewedBy || 'system:auto-review') : null;
const [result] = await pool.query(
`INSERT IGNORE INTO ${quoted}
(id, user_id, session_id, memory_type, content, importance, confidence, status,
policy_reason, evidence_json, reviewed_by, reviewed_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
candidate.id,
candidate.userId,
candidate.sessionId,
candidate.memoryType,
candidate.content,
candidate.importance,
candidate.confidence,
status,
candidate.policyReason,
JSON.stringify(candidate.evidence ?? {}),
reviewer,
reviewedAt,
candidate.createdAt,
updatedAt,
],
);
return { inserted: Number(result?.affectedRows ?? 0) > 0, status };
},
async listCandidates({ status = 'candidate', userId = null, limit = 50, offset = 0 } = {}) {
const normalizedStatus = String(status || 'candidate');
if (!ALLOWED_STATUSES.has(normalizedStatus)) throw new Error('Invalid candidate status');
const safeLimit = Math.max(1, Math.min(200, Number(limit) || 50));
const safeOffset = Math.max(0, Number(offset) || 0);
const where = ['status = ?'];
const params = [normalizedStatus];
if (userId) {
where.push('user_id = ?');
params.push(String(userId));
}
params.push(safeLimit, safeOffset);
const [rows] = await pool.query(
`SELECT * FROM ${quoted} WHERE ${where.join(' AND ')}
ORDER BY created_at DESC LIMIT ? OFFSET ?`,
params,
);
return rows.map(normalizeRow);
},
async countByStatus() {
const [rows] = await pool.query(
`SELECT status, COUNT(*) AS count FROM ${quoted} GROUP BY status`,
);
return Object.fromEntries(rows.map((row) => [String(row.status), Number(row.count)]));
},
async reviewCandidate(id, status, { reviewedBy = null } = {}) {
if (!['accepted', 'rejected'].includes(status)) throw new Error('Invalid review status');
const reviewedAt = now();
const [result] = await pool.query(
`UPDATE ${quoted}
SET status = ?, reviewed_by = ?, reviewed_at = ?, updated_at = ?
WHERE id = ? AND status = 'candidate'`,
[status, reviewedBy, reviewedAt, reviewedAt, String(id)],
);
return { updated: Number(result?.affectedRows ?? 0) > 0, status, reviewedAt };
},
};
}
+47
View File
@@ -0,0 +1,47 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
buildPersonalMemoryCandidateSchemaSql,
createPersonalMemoryCandidateStore,
ensurePersonalMemoryCandidateSchema,
} from './memory-v2-personal-store.mjs';
test('candidate schema is explicit and idempotent', async () => {
const calls = [];
const pool = { async query(sql, params) { calls.push({ sql, params }); return [[], []]; } };
await ensurePersonalMemoryCandidateSchema(pool);
assert.match(calls[0].sql, /CREATE TABLE IF NOT EXISTS/);
assert.match(calls[0].sql, /h5_memory_v2_candidates/);
assert.throws(() => buildPersonalMemoryCandidateSchemaSql({ table: 'bad-name' }), /Invalid/);
});
test('candidate store saves, lists, counts, and reviews with parameterized SQL', async () => {
const calls = [];
const pool = {
async query(sql, params = []) {
calls.push({ sql, params });
if (sql.startsWith('INSERT')) return [{ affectedRows: 1 }, []];
if (sql.startsWith('SELECT *')) return [[{
id: 'pmc_1', user_id: 'u1', session_id: 's1', memory_type: 'episodic',
content: '决定使用 main 发布', importance: '0.9', confidence: '0.9', status: 'candidate',
policy_reason: 'decision_signal', evidence_json: '{"sourceId":"s1:0"}',
reviewed_by: null, reviewed_at: null, created_at: 1, updated_at: 1,
}], []];
if (sql.startsWith('SELECT status')) return [[{ status: 'candidate', count: 1 }], []];
if (sql.startsWith('UPDATE')) return [{ affectedRows: 1 }, []];
throw new Error(`Unexpected SQL: ${sql}`);
},
};
const store = createPersonalMemoryCandidateStore(pool, { now: () => 10 });
assert.deepEqual(await store.saveCandidate({
id: 'pmc_1', userId: 'u1', sessionId: 's1', memoryType: 'episodic',
content: '决定使用 main 发布', importance: 0.9, confidence: 0.9,
policyReason: 'decision_signal', evidence: { sourceId: 's1:0' }, createdAt: 1,
}, { autoAccept: true }), { inserted: true, status: 'accepted' });
const rows = await store.listCandidates({ limit: 20 });
assert.equal(rows[0].evidence.sourceId, 's1:0');
assert.deepEqual(await store.countByStatus(), { candidate: 1 });
assert.equal((await store.reviewCandidate('pmc_1', 'accepted', { reviewedBy: 'admin-1' })).updated, true);
assert.ok(calls.every((call) => !call.sql.includes('admin-1')));
});
+15
View File
@@ -4,6 +4,8 @@ import { createLangGraphHttpClient, createLangGraphMemoryBackend } from './memor
import { createLegacyMemoryBackend, createMemoryV2, resolveMemoryV2Policy } from './memory-v2.mjs';
import { createLettaHttpClient, createLettaMemoryBackend } from './memory-v2-letta.mjs';
import { createMemoryV2PluginBackends } from './memory-v2-plugin-backends.mjs';
import { createPersonalMemoryShadowPipeline } from './memory-v2-personal-shadow.mjs';
import { createPersonalMemoryCandidateStore } from './memory-v2-personal-store.mjs';
import { createMem0HttpClient, createMem0MemoryBackend } from './memory-v2-mem0.mjs';
import { createNeo4jHttpClient, createNeo4jMemoryBackend } from './memory-v2-neo4j.mjs';
import { createPgvectorMemoryBackend } from './memory-v2-pgvector.mjs';
@@ -400,11 +402,19 @@ export async function createMemoryV2Runtime({
],
}));
const personalCandidateStore = readFlag(env, 'MEMORY_CANDIDATE_PERSISTENCE_ENABLED', false)
? createPersonalMemoryCandidateStore(mysqlPool)
: null;
const personalShadowPipeline = createPersonalMemoryShadowPipeline({
env,
store: personalCandidateStore,
});
const memory = createMemoryV2({
legacyMemoryService,
backends,
env,
logger,
personalShadowPipeline,
});
const pgBackfillEnabled = readFlag(env, 'MEMORY_PGVECTOR_BACKFILL_ENABLED', pgvectorEnabled);
@@ -593,6 +603,11 @@ export async function createManagedMemoryV2Runtime({
return runtime.compact(input);
},
async observePersonalMemory(input = {}) {
const runtime = await ensureRuntime();
return runtime.observePersonalMemory(input);
},
async close() {
const runtime = activeRuntime;
activeRuntime = null;
+34 -1
View File
@@ -1,4 +1,5 @@
import { createMemoryV2PluginBackends } from './memory-v2-plugin-backends.mjs';
import { createPersonalMemoryShadowPipeline } from './memory-v2-personal-shadow.mjs';
const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']);
const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
@@ -231,12 +232,14 @@ export function createMemoryV2({
policy = null,
env = process.env,
logger = console,
personalShadowPipeline = null,
} = {}) {
const resolvedPolicy = policy ?? resolveMemoryV2Policy({ env });
const resolvedBackends = backends ?? [
createLegacyMemoryBackend(legacyMemoryService),
...createMemoryV2PluginBackends(),
];
const shadowPipeline = personalShadowPipeline ?? createPersonalMemoryShadowPipeline({ env });
async function failOpen(operation, err, fallback) {
logger?.warn?.(
@@ -278,6 +281,13 @@ export function createMemoryV2({
if (!backend?.write) return skippedWriteResult('no_backend');
try {
const result = await backend.write(input);
if (shadowPipeline?.config?.enabled) {
void Promise.resolve(shadowPipeline.observeWrite(input)).catch((err) => {
logger?.warn?.(
`[memory-v2] personal shadow pipeline skipped: ${err instanceof Error ? err.message : err}`,
);
});
}
return {
ok: true,
enabled: true,
@@ -334,10 +344,28 @@ export function createMemoryV2({
}
}
async function observePersonalMemory(input = {}) {
if (!shadowPipeline?.config?.enabled) {
return { enabled: false, skipped: true, reason: 'disabled' };
}
try {
return await shadowPipeline.observeWrite(input);
} catch (err) {
logger?.warn?.(
`[memory-v2] personal shadow observation skipped: ${err instanceof Error ? err.message : err}`,
);
return {
enabled: true,
skipped: true,
reason: 'pipeline_failure',
};
}
}
function getStatus() {
const backends = resolvedBackends.map((backend) => backendStatus(backend));
const selected = selectBackend(resolvedBackends, resolvedPolicy, 'resolve');
return {
const status = {
enabled: Boolean(resolvedPolicy.enabled),
backend: resolvedPolicy.backend,
selectedBackend: selected?.name ?? null,
@@ -347,6 +375,10 @@ export function createMemoryV2({
failOpen: Boolean(resolvedPolicy.failOpen),
backends,
};
if (shadowPipeline?.config?.enabled) {
status.personalMemory = shadowPipeline.getStatus();
}
return status;
}
return {
@@ -355,5 +387,6 @@ export function createMemoryV2({
resolve,
write,
compact,
observePersonalMemory,
};
}
@@ -234,7 +234,13 @@ test('integration: finish guard auto-bind clears unbound state for valid survey
const before = evaluatePageDataFinishGuard({
publishDir: workspaceRoot,
agentText: SURVEY_USER_TEXT,
messages: [],
messages: [{
createdAt: Date.now(),
content: [{
type: 'toolRequest',
toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } },
}],
}],
});
assert.equal(before.unboundFiles.length, 1);
@@ -246,7 +252,7 @@ test('integration: finish guard auto-bind clears unbound state for valid survey
storageRoot: workspaceRoot,
});
assert.equal(autoBind.bound.length, 0);
assert.equal(autoBind.errors[0]?.code, 'database_unconfigured');
assert.equal(autoBind.errors[0]?.code, 'missing_context');
writePageAccessPolicy(workspaceRoot, {
pageId: 'page-diet-survey',
@@ -264,7 +270,13 @@ test('integration: finish guard auto-bind clears unbound state for valid survey
const after = evaluatePageDataFinishGuard({
publishDir: workspaceRoot,
agentText: SURVEY_USER_TEXT,
messages: [],
messages: [{
createdAt: Date.now(),
content: [{
type: 'toolRequest',
toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } },
}],
}],
});
assert.equal(after.unboundFiles.length, 0);
assert.equal(after.htmlIssues.length, 0);
@@ -285,7 +297,13 @@ test('integration: H5 finish guard triggers repair prompt for invalid survey htm
sessionId: 'session-h5-page-data',
userId: 'user-page-data',
publishDir: workspaceRoot,
messages: [],
messages: [{
createdAt: Date.now(),
content: [{
type: 'toolRequest',
toolCall: { value: { name: 'write_file', arguments: { path: 'public/children-diet-survey.html' } } },
}],
}],
pool: null,
h5Root: workspaceRoot,
storageRoot: workspaceRoot,
@@ -307,3 +325,28 @@ test('integration: H5 finish guard triggers repair prompt for invalid survey htm
fs.rmSync(workspaceRoot, { recursive: true, force: true });
}
});
test('integration: finish guard does not repair historical Page Data html without a current write', async () => {
const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-historical-skip-'));
try {
await setupSurveyWorkspace(workspaceRoot);
fs.writeFileSync(path.join(workspaceRoot, 'public', 'children-diet-survey.html'), VALID_SURVEY_HTML, 'utf8');
const result = await maybeRepairPageDataAfterFinish({
sessionId: 'session-with-history',
userId: 'user-page-data',
publishDir: workspaceRoot,
messages: [],
pool: null,
h5Root: workspaceRoot,
storageRoot: workspaceRoot,
userText: '确认发布',
tkmindProxy: { async submitSessionReplyForUser() { throw new Error('must not repair history'); } },
});
assert.equal(result.structuralPageData, false);
assert.equal(result.relevantFiles.length, 0);
assert.equal(result.needsRepair, false);
assert.equal(result.triggered, undefined);
} finally {
fs.rmSync(workspaceRoot, { recursive: true, force: true });
}
});
+119 -82
View File
@@ -7,11 +7,12 @@ import {
htmlUsesPageDataApi,
inferPageDataBindAccessMode,
} from './page-data-html-detect.mjs';
import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs';
import { createPageService } from './mindspace-pages.mjs';
import { ensurePageDataHtmlPagesBound } from './page-data-workspace-ensure.mjs';
import {
assessPageDataHtmlBinding,
assessWorkspacePageDataReadiness,
normalizePageDataApiBase,
verifyPageDataDeliveryArtifacts,
} from './page-data-delivery-assess.mjs';
import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs';
@@ -228,7 +229,8 @@ export function extractRecentPageDataHtmlWrites(messages = [], { sinceMs = 0 } =
if (item?.type !== 'toolRequest') continue;
const toolCall = item.toolCall?.value;
const name = String(toolCall?.name ?? '').trim();
if (!['write_file', 'edit_file', 'write', 'edit'].includes(name)) continue;
const normalizedName = name.split('__').at(-1);
if (!['write_file', 'edit_file', 'write', 'edit'].includes(normalizedName)) continue;
const args = toolCall?.arguments ?? {};
const candidate = String(args.path ?? args.file_path ?? '').trim().replace(/\\/g, '/');
if (!candidate.toLowerCase().endsWith('.html')) continue;
@@ -239,18 +241,66 @@ export function extractRecentPageDataHtmlWrites(messages = [], { sinceMs = 0 } =
return [...targets];
}
export function extractRecentPageDataBindTargets(messages = [], { sinceMs = 0 } = {}) {
const targets = new Set();
for (const message of Array.isArray(messages) ? messages : []) {
const createdAt = Number(message?.created ?? message?.createdAt ?? 0);
if (sinceMs > 0 && createdAt > 0 && createdAt < sinceMs) continue;
for (const item of message?.content ?? []) {
if (item?.type !== 'toolRequest') continue;
const toolCall = item.toolCall?.value;
const name = String(toolCall?.name ?? '').trim().split('__').at(-1);
if (name !== 'private_data_bind_workspace_page') continue;
const candidate = String(toolCall?.arguments?.relativePath ?? '')
.trim()
.replace(/\\/g, '/');
if (!candidate.toLowerCase().endsWith('.html')) continue;
targets.add(candidate.startsWith('public/') ? candidate : `public/${path.posix.basename(candidate)}`);
}
}
return [...targets];
}
function isStructuralPageDataHtmlFile(file) {
return Boolean(file?.evaluation?.usage?.size > 0 || file?.evaluation?.usesPageDataApi);
}
function resolvePageDataGuardAgentText({ agentText = '', messages = [] } = {}) {
const direct = String(agentText ?? '').trim();
if (isPageDataIntent(direct)) return direct;
for (let i = messages.length - 1; i >= 0; i -= 1) {
const message = messages[i];
if (message?.role !== 'user') continue;
const text = Array.isArray(message?.content)
? message.content
.filter((item) => item?.type === 'text')
.map((item) => String(item.text ?? '').trim())
.filter(Boolean)
.join('\n')
: String(message?.content ?? '').trim();
if (isPageDataIntent(text)) return text;
}
return direct;
}
export function evaluatePageDataFinishGuard({
publishDir,
agentText = '',
messages = [],
requestStartedAt = 0,
} = {}) {
const pageDataIntent = isPageDataIntent(agentText);
const resolvedAgentText = resolvePageDataGuardAgentText({ agentText, messages });
const pageDataIntent = isPageDataIntent(resolvedAgentText);
const pageDataFiles = collectPageDataPublicHtmlFiles(publishDir);
const recentWrites = extractRecentPageDataHtmlWrites(messages, { sinceMs: requestStartedAt });
const relevantFiles = pageDataFiles.filter((file) =>
pageDataIntent || recentWrites.includes(file.relativePath),
);
const recentBinds = extractRecentPageDataBindTargets(messages, { sinceMs: requestStartedAt });
const relevantPaths = new Set([...recentWrites, ...recentBinds]);
// A reused session/workspace can contain many historical Page Data pages.
// Only files touched or explicitly bound in this request belong to this
// delivery. Scanning the whole workspace lets one stale localStorage page
// incorrectly block every later, correctly bound survey.
const relevantFiles = pageDataFiles.filter((file) => relevantPaths.has(file.relativePath));
const structuralFiles = relevantFiles.filter(isStructuralPageDataHtmlFile);
const htmlIssues = relevantFiles.flatMap((file) =>
file.evaluation.issues.map((issue) => ({
@@ -273,10 +323,13 @@ export function evaluatePageDataFinishGuard({
pageDataIntent &&
(htmlIssues.length > 0 ||
unboundFiles.length > 0 ||
(relevantFiles.length === 0 && extractRecentPageDataHtmlWrites(messages, { sinceMs: requestStartedAt }).length === 0 && usedPageDataCollectSkill(messages)));
(relevantFiles.length === 0 &&
extractRecentPageDataHtmlWrites(messages, { sinceMs: requestStartedAt }).length === 0 &&
usedPageDataCollectSkill(messages)));
return {
pageDataIntent,
structuralPageData: structuralFiles.length > 0,
relevantFiles,
htmlIssues,
unboundFiles,
@@ -309,10 +362,11 @@ export async function evaluatePageDataFinishGuardAsync({
findPageByRelativePath,
});
const needsRepair =
base.pageDataIntent &&
base.structuralPageData &&
(base.htmlIssues.length > 0 ||
unboundFiles.length > 0 ||
(base.relevantFiles.length === 0 &&
(base.pageDataIntent &&
base.relevantFiles.length === 0 &&
extractRecentPageDataHtmlWrites(messages, { sinceMs: requestStartedAt }).length === 0 &&
usedPageDataCollectSkill(messages)));
@@ -402,16 +456,15 @@ export function resolvePageDataCollectOutcome({
requestStartedAt = 0,
}) {
const agentText = String(intent?.agentText ?? intent?.displayText ?? '').trim();
if (!isPageDataIntent(agentText)) {
return { action: 'skip' };
}
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText,
messages: reply?.messages ?? [],
requestStartedAt,
});
if (!evaluation.structuralPageData && !evaluation.pageDataIntent) {
return { action: 'skip' };
}
if (evaluation.htmlIssues.length > 0) {
return { action: 'fail', failureText: buildPageDataCollectFailureText(), reason: 'invalid_html', evaluation };
@@ -433,12 +486,10 @@ export async function resolvePageDataCollectOutcomeAsync({
pool = null,
userId = null,
findPageByRelativePath = null,
apiBase = null,
fetchImpl = fetch,
} = {}) {
const agentText = String(intent?.agentText ?? intent?.displayText ?? '').trim();
if (!isPageDataIntent(agentText)) {
return { action: 'skip' };
}
const evaluation = await evaluatePageDataFinishGuardAsync({
publishDir,
agentText,
@@ -448,10 +499,45 @@ export async function resolvePageDataCollectOutcomeAsync({
userId,
findPageByRelativePath,
});
if (!evaluation.structuralPageData && !evaluation.pageDataIntent) {
return { action: 'skip' };
}
if (evaluation.htmlIssues.length > 0) {
return { action: 'fail', failureText: buildPageDataCollectFailureText(), reason: 'invalid_html', evaluation };
}
if (evaluation.unboundFiles.length > 0 && pool && userId && apiBase) {
const artifacts = collectPageDataDeliveryArtifacts(publishDir).filter((artifact) =>
evaluation.relevantFiles.some((file) => file.relativePath === artifact.relativePath),
);
if (artifacts.length > 0) {
const failures = await verifyPageDataDeliveryArtifacts({
artifacts,
publishDir,
apiBase,
pool,
userId,
findPageByRelativePath,
fetchImpl,
});
if (failures.length === 0) {
const reassessment = await evaluatePageDataFinishGuardAsync({
publishDir,
agentText,
messages: reply?.messages ?? [],
requestStartedAt,
pool,
userId,
findPageByRelativePath,
});
if (reassessment.unboundFiles.length === 0 && reassessment.htmlIssues.length === 0) {
return { action: 'send', reason: 'verified_by_live_api', evaluation: reassessment };
}
}
}
}
if (evaluation.unboundFiles.length > 0) {
return { action: 'retry', reason: 'missing_bind', evaluation };
}
@@ -479,66 +565,15 @@ export async function maybeAutoBindPageDataHtmlPages({
onlyRelativePaths = null,
findPageByRelativePath = null,
} = {}) {
if (!pool) {
return { bound: [], skipped: [], errors: [{ code: 'database_unconfigured' }] };
}
const pageLookup =
typeof findPageByRelativePath === 'function'
? findPageByRelativePath
: createPageService(pool, { h5Root, storageRoot }).findPageByRelativePath;
const bound = [];
const skipped = [];
const errors = [];
const allowList = onlyRelativePaths ? new Set(onlyRelativePaths) : null;
for (const file of collectPageDataPublicHtmlFiles(publishDir)) {
if (allowList && !allowList.has(file.relativePath)) continue;
if (file.evaluation.issues.length > 0) {
skipped.push({ relativePath: file.relativePath, reason: 'invalid_html', issues: file.evaluation.issues });
continue;
}
if (file.evaluation.usage?.size === 0) {
skipped.push({ relativePath: file.relativePath, reason: 'no_dataset_usage' });
continue;
}
const assessment = await assessPageDataHtmlBinding({
pool,
userId,
publishDir,
relativePath: file.relativePath,
html: file.content,
findPageByRelativePath: pageLookup,
});
if (assessment.bound) {
skipped.push({ relativePath: file.relativePath, reason: 'already_bound' });
continue;
}
try {
const accessMode = inferPageDataBindAccessMode(file.relativePath, file.content);
const result = await bindWorkspaceHtmlForPageData({
pool,
h5Root,
storageRoot,
userId,
workspaceRoot: publishDir,
relativePath: file.relativePath,
accessMode,
password: accessMode === 'password' ? '88888888' : null,
});
bound.push({ relativePath: file.relativePath, pageId: result.pageId, workspaceUrl: result.workspaceUrl });
} catch (err) {
errors.push({
relativePath: file.relativePath,
message: err instanceof Error ? err.message : String(err),
code: err?.code ?? 'bind_failed',
});
}
}
return { bound, skipped, errors };
return ensurePageDataHtmlPagesBound({
pool,
h5Root,
storageRoot,
userId,
workspaceRoot: publishDir,
findPageByRelativePath,
onlyRelativePaths,
});
}
export async function ensurePageDataDeliveryReady({
@@ -558,7 +593,7 @@ export async function ensurePageDataDeliveryReady({
const failures = await verifyPageDataDeliveryArtifacts({
artifacts,
publishDir,
apiBase,
apiBase: normalizePageDataApiBase(apiBase),
pool,
userId,
findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService),
@@ -585,14 +620,14 @@ export async function maybeRepairPageDataAfterFinish({
const pageService = pool ? createPageService(pool, { h5Root, storageRoot }) : null;
const evaluation = await evaluatePageDataFinishGuardAsync({
publishDir,
agentText: recentUserText,
agentText: resolvePageDataGuardAgentText({ agentText: recentUserText, messages }),
messages,
pool,
userId,
findPageByRelativePath: pageService?.findPageByRelativePath?.bind(pageService) ?? null,
});
if (!evaluation.pageDataIntent && evaluation.relevantFiles.length === 0) {
if (!evaluation.structuralPageData && evaluation.relevantFiles.length === 0) {
resetPageDataFinishGuardAttempts(sessionId);
return { repaired: false, skipped: 'not_page_data', ...evaluation };
}
@@ -603,13 +638,15 @@ export async function maybeRepairPageDataAfterFinish({
publishDir,
h5Root,
storageRoot,
onlyRelativePaths: evaluation.relevantFiles.map((file) => file.relativePath),
onlyRelativePaths: evaluation.relevantFiles.length
? evaluation.relevantFiles.map((file) => file.relativePath)
: null,
findPageByRelativePath: pageService?.findPageByRelativePath?.bind(pageService) ?? null,
});
const afterBind = await evaluatePageDataFinishGuardAsync({
publishDir,
agentText: recentUserText,
agentText: resolvePageDataGuardAgentText({ agentText: recentUserText, messages }),
messages,
pool,
userId,
+52 -4
View File
@@ -9,12 +9,15 @@ import {
collectPageDataDeliveryArtifacts,
evaluatePageDataFinishGuard,
evaluatePageDataHtmlContent,
extractRecentPageDataBindTargets,
extractRecentPageDataHtmlWrites,
inferPageDataBindAccessMode,
maybeAutoBindPageDataHtmlPages,
rewritePageDataDeliveryLinks,
shouldRetryPageDataCollectReply,
} from './mindspace-page-data-finish-guard.mjs';
import { writePageAccessPolicy } from './page-data-policy-store.mjs';
import { createUserDataSpaceService } from './user-data-space-service.mjs';
const SURVEY_HTML = `<!doctype html><html><head><title>问卷</title></head><body>
<script src="/assets/page-data-client.js"></script>
@@ -57,7 +60,7 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => {
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: '帮我设计一个调查问卷,要加一个后台',
messages: [],
messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }],
});
assert.equal(evaluation.pageDataIntent, true);
assert.equal(evaluation.unboundFiles.length, 1);
@@ -67,6 +70,51 @@ test('evaluatePageDataFinishGuard detects unbound page data html', () => {
}
});
test('finish guard ignores unrelated historical broken Page Data html', async () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-history-'));
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(path.join(publishDir, 'public', 'historical-admin.html'), BAD_SURVEY_HTML, 'utf8');
fs.writeFileSync(path.join(publishDir, 'public', 'current-survey.html'), SURVEY_HTML, 'utf8');
const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir });
await dataSpace.executeSql('CREATE TABLE diet_survey (id INTEGER PRIMARY KEY AUTOINCREMENT);');
await dataSpace.upsertDataset({
name: 'diet_survey',
table: 'diet_survey',
actions: ['read', 'insert'],
columns: { read: ['id'], insert: [] },
});
writePageAccessPolicy(publishDir, {
pageId: 'current-page',
ownerUserId: 'user-1',
accessMode: 'public',
datasets: { diet_survey: { insert: true, read: false, columns: { insert: [] } } },
});
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: '调查问卷和后台',
messages: [{ content: [{ type: 'toolRequest', toolCall: { value: {
name: 'sandbox-fs__write_file',
arguments: { path: 'public/current-survey.html' },
} } }] }],
});
assert.deepEqual(evaluation.relevantFiles.map((file) => file.relativePath), ['public/current-survey.html']);
assert.deepEqual(evaluation.htmlIssues, []);
assert.deepEqual(evaluation.unboundFiles, []);
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
});
test('recent Page Data target extraction supports namespaced writes and bind-only delivery', () => {
const messages = [{ content: [
{ type: 'toolRequest', toolCall: { value: { name: 'sandbox-fs__write_file', arguments: { path: 'public/form.html' } } } },
{ type: 'toolRequest', toolCall: { value: { name: 'sandbox-fs__private_data_bind_workspace_page', arguments: { relativePath: 'public/form-admin.html' } } } },
] }];
assert.deepEqual(extractRecentPageDataHtmlWrites(messages), ['public/form.html']);
assert.deepEqual(extractRecentPageDataBindTargets(messages), ['public/form-admin.html']);
});
test('shouldRetryPageDataCollectReply retries when localStorage fallback exists', () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-retry-'));
try {
@@ -74,7 +122,7 @@ test('shouldRetryPageDataCollectReply retries when localStorage fallback exists'
fs.writeFileSync(path.join(publishDir, 'public', 'children-diet-survey.html'), BAD_SURVEY_HTML, 'utf8');
assert.equal(
shouldRetryPageDataCollectReply({
reply: { text: '问卷已发布', messages: [] },
reply: { text: '问卷已发布', messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/children-diet-survey.html' } } } }] }] },
intent: { agentText: '帮我设计一个调查问卷,要加一个后台' },
publishDir,
}),
@@ -108,7 +156,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => {
storageRoot: publishDir,
});
assert.equal(result.bound.length, 0);
assert.equal(result.errors[0]?.code, 'database_unconfigured');
assert.equal(result.errors[0]?.code, 'missing_context');
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
@@ -134,7 +182,7 @@ test('evaluatePageDataFinishGuard flags html when dataset is not registered', ()
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: '调查问卷和后台',
messages: [],
messages: [{ content: [{ type: 'toolRequest', toolCall: { value: { name: 'write_file', arguments: { path: 'public/diet-survey.html' } } } }] }],
});
assert.equal(evaluation.unboundFiles.length, 1);
assert.equal(evaluation.htmlIssues.length, 0);
@@ -0,0 +1,48 @@
/**
* WeChat WebView compatibility patches for Page Data survey HTML.
* X5 often fails to select radios hidden with pointer-events:none.
*/
export function htmlNeedsWechatSurveyCompat(html) {
const content = String(html ?? '');
if (!/\/assets\/page-data-client\.js/i.test(content)) return false;
if (!/\.insertRow\s*\(/.test(content)) return false;
return (
/pointer-events\s*:\s*none/i.test(content) ||
/\.genre-option\s+input/i.test(content) ||
/input\[type=["']radio["']\][^{]*pointer-events/i.test(content)
);
}
export function applyWechatSurveyCompat(html) {
const source = String(html ?? '');
if (!htmlNeedsWechatSurveyCompat(source)) {
return { html: source, patched: false };
}
let next = source;
next = next.replace(
/\.genre-option\s+input\s*\{[^}]*pointer-events\s*:\s*none\s*;?[^}]*\}/gi,
'.genre-option input { position: absolute; inset: 0; width: 100%; height: 100%; margin: 0; opacity: 0; cursor: pointer; z-index: 2; }',
);
if (!/data-mindspace-wechat-survey-compat/i.test(next)) {
const patch = `<style id="mindspace-wechat-survey-compat">
.genre-option { position: relative; }
.genre-option label { position: relative; z-index: 1; pointer-events: none; -webkit-tap-highlight-color: transparent; }
.genre-option input { position: absolute; inset: 0; width: 100%; height: 100%; margin: 0; opacity: 0; cursor: pointer; z-index: 2; }
.genre-option input:checked + label { pointer-events: none; }
button.submit-btn, .submit-btn { touch-action: manipulation; -webkit-tap-highlight-color: transparent; }
</style>`;
if (/<\/head>/i.test(next)) {
next = next.replace(/<\/head>/i, `${patch}\n</head>`);
} else if (/<body\b/i.test(next)) {
next = next.replace(/<body\b/i, `${patch}\n<body`);
} else {
next = `${patch}\n${next}`;
}
}
return { html: next, patched: true };
}
@@ -0,0 +1,26 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
applyWechatSurveyCompat,
htmlNeedsWechatSurveyCompat,
} from './mindspace-page-data-wechat-survey-compat.mjs';
const SURVEY_HTML = `<!doctype html><html><head><style>
.genre-option input { position: absolute; opacity: 0; pointer-events: none; }
</style>
<script src="/assets/page-data-client.js"></script></head><body>
<script>MindSpacePageData.createClient({ apiBase: '/api' }).insertRow('reading_survey', {});</script>
</body></html>`;
test('htmlNeedsWechatSurveyCompat detects pointer-events none on survey radios', () => {
assert.equal(htmlNeedsWechatSurveyCompat(SURVEY_HTML), true);
assert.equal(htmlNeedsWechatSurveyCompat('<html><body>plain</body></html>'), false);
});
test('applyWechatSurveyCompat removes pointer-events none from radio inputs', () => {
const result = applyWechatSurveyCompat(SURVEY_HTML);
assert.equal(result.patched, true);
assert.doesNotMatch(result.html, /\.genre-option\s+input\s*\{[^}]*pointer-events\s*:\s*none/i);
assert.match(result.html, /mindspace-wechat-survey-compat/);
assert.match(result.html, /z-index:\s*2/);
});
+4
View File
@@ -4,6 +4,7 @@ import path from 'node:path';
import { injectMindSpacePageDataContext } from './mindspace-public-page-context.mjs';
import { preparePublishedPlatformBrand } from './mindspace-page-tag.mjs';
import { injectPublicImageRetryScript } from './mindspace-public-image-retry.mjs';
import { applyWechatSurveyCompat } from './mindspace-page-data-wechat-survey-compat.mjs';
const INLINE_SCRIPT_PATTERN = /<script\b(?![^>]*\bsrc\b)[^>]*>([\s\S]*?)<\/script>/gi;
@@ -97,6 +98,9 @@ export function decorateMindSpacePublishedHtml({
if (pageDataContext?.pageId) {
nextHtml = injectMindSpacePageDataContext(nextHtml, pageDataContext);
}
if (!embed && isWechatUserAgent(userAgent || '')) {
nextHtml = applyWechatSurveyCompat(nextHtml).html;
}
const scriptHashes = collectInlineScriptHashes(nextHtml);
return {
html: nextHtml,
+8 -1
View File
@@ -21,7 +21,14 @@ export function publishedPageCsp(
return publishedPageCspForEmbed(true);
}
if (wechatShare && isFullHtml) {
return "default-src 'none'; style-src 'unsafe-inline' https:; img-src 'self' data: http: https:; font-src 'self' https: data:; connect-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'self'; script-src 'unsafe-inline' https://res.wx.qq.com";
const scriptSrc = scriptSrcDirective({
inline: true,
urls: [
...(htmlUsesExternalScriptSrc(html) ? ["'self'"] : []),
'https://res.wx.qq.com',
],
});
return `default-src 'none'; style-src 'unsafe-inline' https:; img-src 'self' data: http: https:; font-src 'self' https: data:; connect-src 'self'; base-uri 'none'; form-action 'self'; frame-ancestors 'self'; ${scriptSrc}`;
}
if (raw && isFullHtml) {
const scriptSrc = scriptSrcDirective({
+12
View File
@@ -23,3 +23,15 @@ test('publishedPageCsp non-raw full html allows self when external scripts are p
const csp = publishedPageCsp(PAGE_DATA_HTML, { raw: false });
assert.match(csp, /script-src 'self'/);
});
test('publishedPageCsp wechatShare mode allows self when page-data-client.js is referenced', () => {
const csp = publishedPageCsp(PAGE_DATA_HTML, { wechatShare: true });
assert.match(csp, /script-src 'unsafe-inline' 'self' https:\/\/res\.wx\.qq\.com/);
});
test('publishedPageCsp wechatShare mode keeps inline-only pages without self', () => {
const html = '<!doctype html><html><body><script>console.log(1)</script></body></html>';
const csp = publishedPageCsp(html, { wechatShare: true });
assert.match(csp, /script-src 'unsafe-inline' https:\/\/res\.wx\.qq\.com/);
assert.doesNotMatch(csp, /script-src 'unsafe-inline' 'self'/);
});
@@ -0,0 +1,74 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { createWorkspacePageDeliverService } from './mindspace-workspace-page-deliver.mjs';
const PAGE_DATA_HTML = `<!doctype html><html><body>
<script src="/assets/page-data-client.js"></script>
<script>MindSpacePageData.createClient({ apiBase: '/api' }).insertRow('survey', { q1: 'a' });</script>
</body></html>`;
test('ensureWorkspaceHtmlPublications skips page-data html and delegates to pageDataEnsure', async () => {
const h5Root = fs.mkdtempSync(path.join(os.tmpdir(), 'deliver-h5-'));
const userId = 'user-1';
const publishDir = path.join(h5Root, 'MindSpace', userId);
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(path.join(publishDir, 'public', 'survey.html'), PAGE_DATA_HTML, 'utf8');
const published = [];
const bound = [];
const service = createWorkspacePageDeliverService({
pool: {
async query() {
return [[{
page_id: 'page-1',
title: '问卷',
current_version_id: 'ver-1',
workspace_relative_path: 'public/survey.html',
source_snapshot_json: JSON.stringify({
auto_synced: true,
relative_path: 'public/survey.html',
content_mode: 'static_html',
}),
}]];
},
},
pageService: {
async getPage() {
return { id: 'page-1', title: '问卷', currentVersionId: 'ver-1' };
},
findPageByRelativePath: async () => null,
},
publicationService: {
async getCurrent() {
return null;
},
async publish(userIdArg, pageId, input) {
published.push({ userId: userIdArg, pageId, input });
return { id: 'pub-1' };
},
},
pageSyncService: {
async syncUserGeneratedPages() {
return { created: 0, updated: 0, skipped: 0 };
},
},
pageDataEnsure: {
async ensurePageDataHtmlPagesBound() {
bound.push(true);
return { bound: [{ relativePath: 'public/survey.html' }], skipped: [], errors: [] };
},
},
h5Root,
storageRoot: null,
});
const result = await service.syncAndDeliver(userId);
assert.equal(bound.length, 1);
assert.equal(result.publish.published, 0);
assert.equal(result.publish.skipped, 1);
assert.equal(published.length, 0);
fs.rmSync(h5Root, { recursive: true, force: true });
});
+64 -1
View File
@@ -1,5 +1,9 @@
import fs from 'node:fs';
import path from 'node:path';
import { slugFromPageTitle } from './mindspace-chat-plaza.mjs';
import { normalizeWorkspaceRelativePath } from './mindspace-pages.mjs';
import { htmlUsesPageDataApi } from './page-data-html-detect.mjs';
import { resolvePublishDir } from './user-publish.mjs';
function parseJsonColumn(value, fallback = {}) {
if (value == null || value === '') return { ...fallback };
@@ -16,11 +20,31 @@ function isPublicWorkspaceHtmlPath(relativePath) {
return Boolean(normalized?.startsWith('public/') && normalized.toLowerCase().endsWith('.html'));
}
function readWorkspaceHtmlContent(publishDir, relativePath) {
const normalized = normalizeWorkspaceRelativePath(relativePath);
if (!normalized || !publishDir) return '';
const absolutePath = path.join(publishDir, ...normalized.split('/'));
try {
return fs.readFileSync(absolutePath, 'utf8');
} catch {
return '';
}
}
function isPageDataWorkspaceHtml(publishDir, relativePath) {
const content = readWorkspaceHtmlContent(publishDir, relativePath);
return htmlUsesPageDataApi(content);
}
export function createWorkspacePageDeliverService({
pool,
pageService,
publicationService,
pageSyncService,
pageDataEnsure = null,
h5Root = null,
storageRoot = null,
findPageByRelativePath = null,
logger = console,
} = {}) {
async function listUnpublishedAutoSyncedWorkspacePages(userId) {
@@ -66,6 +90,30 @@ export function createWorkspacePageDeliverService({
});
}
function resolveWorkspaceRoot(userId) {
if (!h5Root || !userId) return null;
return resolvePublishDir(h5Root, { id: userId });
}
async function ensurePageDataBindings(userId) {
if (!pageDataEnsure?.ensurePageDataHtmlPagesBound || !pool || !userId) {
return { bound: [], skipped: [], errors: [] };
}
const workspaceRoot = resolveWorkspaceRoot(userId);
if (!workspaceRoot) {
return { bound: [], skipped: [], errors: [{ code: 'missing_workspace_root' }] };
}
return pageDataEnsure.ensurePageDataHtmlPagesBound({
pool,
h5Root,
storageRoot,
userId,
workspaceRoot,
findPageByRelativePath,
logger,
});
}
async function refreshOnlineWorkspacePublications(userId) {
if (!publicationService?.refreshOnlinePublicationHtml || !userId) {
return { refreshed: 0, skipped: 0, errors: [] };
@@ -101,11 +149,19 @@ export function createWorkspacePageDeliverService({
return { published: 0, skipped: 0, errors: [] };
}
const candidates = await listUnpublishedAutoSyncedWorkspacePages(userId);
const publishDir = resolveWorkspaceRoot(userId);
let published = 0;
let skipped = 0;
const errors = [];
for (const candidate of candidates) {
try {
const snapshot = parseJsonColumn(candidate.source_snapshot_json);
const relativePath =
candidate.workspace_relative_path ?? snapshot.relative_path ?? null;
if (isPageDataWorkspaceHtml(publishDir, relativePath)) {
skipped += 1;
continue;
}
const page = await pageService.getPage(userId, candidate.page_id);
const current = await publicationService.getCurrent?.(userId, candidate.page_id);
if (current?.status === 'online') {
@@ -138,13 +194,20 @@ export function createWorkspacePageDeliverService({
if (pageSyncService?.syncUserGeneratedPages) {
syncResult = await pageSyncService.syncUserGeneratedPages(userId);
}
const pageDataBindResult = await ensurePageDataBindings(userId);
const publishResult = await ensureWorkspaceHtmlPublications(userId);
const refreshResult = await refreshOnlineWorkspacePublications(userId);
return { sync: syncResult, publish: publishResult, refresh: refreshResult };
return {
sync: syncResult,
pageDataBind: pageDataBindResult,
publish: publishResult,
refresh: refreshResult,
};
}
return {
syncAndDeliver,
ensurePageDataBindings,
ensureWorkspaceHtmlPublications,
refreshOnlineWorkspacePublications,
};
+38 -29
View File
@@ -7,10 +7,17 @@ import { listPageAccessPolicies, readPageAccessPolicy } from './page-data-policy
import { createUserDataSpaceService } from './user-data-space-service.mjs';
const INJECTION_PAGE_ID_PATTERN =
/window\.__MINDSPACE_PAGE_DATA__\s*=\s*\{[^}]*"pageId"\s*:\s*"([^"]+)"/i;
/window\.__MINDSPACE_PAGE_DATA__\s*=\s*\{[^}]*(?:"pageId"|pageId)\s*:\s*"([^"]+)"/i;
const INJECTION_META_PAGE_ID_PATTERN =
/<meta[^>]+name=["']mindspace-page-data-page-id["'][^>]+content=["']([^"']+)["']/i;
export function normalizePageDataApiBase(apiBase) {
const raw = String(apiBase ?? '').trim().replace(/\/$/, '');
if (!raw) return '/api';
if (/\/api$/i.test(raw)) return raw;
return `${raw}/api`;
}
export function extractInjectedPageIdFromHtml(html) {
const content = String(html ?? '');
const fromScript = content.match(INJECTION_PAGE_ID_PATTERN)?.[1]?.trim();
@@ -140,10 +147,6 @@ export async function assessPageDataHtmlBinding({
}
}
const publication = await queryOnlinePublication(pool, pageId);
if (!publication) {
reasons.push('missing_online_publication');
}
}
return {
@@ -179,7 +182,7 @@ export async function smokeTestPageDataInsert({
policy,
fetchImpl = fetch,
}) {
const base = String(apiBase ?? '').replace(/\/$/, '') || '/api';
const base = normalizePageDataApiBase(apiBase);
const row = buildSmokeInsertRow(policy, dataset);
if (!pageId || !dataset || !Object.keys(row).length) {
return { ok: false, reason: 'smoke_payload_unavailable', status: 0, body: null };
@@ -229,6 +232,29 @@ export async function verifyPageDataDeliveryArtifacts({
continue;
}
const pageId = injection.pageId;
const policy =
readPageAccessPolicy(publishDir, pageId) ??
(await assessPageDataHtmlBinding({
pool,
userId,
publishDir,
relativePath,
html,
findPageByRelativePath,
})).policy;
const smoke = await smokeTestPageDataInsert({
apiBase,
pageId,
dataset: insertDataset,
policy,
fetchImpl,
});
if (smoke.ok) {
continue;
}
const assessment = await assessPageDataHtmlBinding({
pool,
userId,
@@ -237,30 +263,13 @@ export async function verifyPageDataDeliveryArtifacts({
html,
findPageByRelativePath,
});
if (!assessment.bound) {
failures.push({
relativePath,
stage: 'binding',
reason: assessment.reasons.join(','),
});
continue;
}
const smoke = await smokeTestPageDataInsert({
apiBase,
pageId: assessment.pageId,
dataset: insertDataset,
policy: assessment.policy,
fetchImpl,
failures.push({
relativePath,
stage: 'insert_smoke',
reason: smoke.reason,
status: smoke.status,
bindingReasons: assessment.reasons,
});
if (!smoke.ok) {
failures.push({
relativePath,
stage: 'insert_smoke',
reason: smoke.reason,
status: smoke.status,
});
}
}
return failures;
}
+13
View File
@@ -3,6 +3,7 @@ import test from 'node:test';
import {
buildSmokeInsertRow,
extractInjectedPageIdFromHtml,
normalizePageDataApiBase,
} from './page-data-delivery-assess.mjs';
test('extractInjectedPageIdFromHtml reads injected pageId', () => {
@@ -13,6 +14,18 @@ test('extractInjectedPageIdFromHtml reads injected pageId', () => {
assert.equal(extractInjectedPageIdFromHtml(html), 'page-abc');
});
test('extractInjectedPageIdFromHtml reads unquoted pageId keys', () => {
const html =
'<script>window.__MINDSPACE_PAGE_DATA__={pageId:"page-unquoted",accessMode:"public"};</script>';
assert.equal(extractInjectedPageIdFromHtml(html), 'page-unquoted');
});
test('normalizePageDataApiBase appends /api to public base url', () => {
assert.equal(normalizePageDataApiBase('https://m.tkmind.cn'), 'https://m.tkmind.cn/api');
assert.equal(normalizePageDataApiBase('https://m.tkmind.cn/api'), 'https://m.tkmind.cn/api');
assert.equal(normalizePageDataApiBase('/api'), '/api');
});
test('buildSmokeInsertRow fills insert columns', () => {
const row = buildSmokeInsertRow(
{
+19 -6
View File
@@ -17,16 +17,29 @@ export function htmlUsesPageDataApi(html) {
export function detectPageDataDatasetUsageFromHtml(html) {
const text = String(html ?? '');
const datasets = new Map();
const constants = new Map();
for (const match of text.matchAll(/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*['"]([^'"]+)['"]/g)) {
constants.set(match[1], match[2]);
}
function remember(name, patch) {
const key = String(name ?? '').trim();
if (!key) return;
datasets.set(key, { ...(datasets.get(key) ?? {}), ...patch });
}
for (const match of text.matchAll(/\.insertRow\(\s*['"]([^'"]+)['"]/g)) {
const name = match[1];
const prev = datasets.get(name) ?? {};
datasets.set(name, { ...prev, insert: true });
remember(match[1], { insert: true });
}
for (const match of text.matchAll(/\.listRows\(\s*['"]([^'"]+)['"]/g)) {
const name = match[1];
const prev = datasets.get(name) ?? {};
datasets.set(name, { ...prev, read: true });
remember(match[1], { read: true });
}
for (const match of text.matchAll(/\.insertRow\(\s*([A-Za-z_$][\w$]*)/g)) {
remember(constants.get(match[1]) ?? match[1], { insert: true });
}
for (const match of text.matchAll(/\.listRows\(\s*([A-Za-z_$][\w$]*)/g)) {
remember(constants.get(match[1]) ?? match[1], { read: true });
}
return datasets;
+9
View File
@@ -16,6 +16,15 @@ test('detectPageDataDatasetUsageFromHtml finds insert and read datasets', () =>
assert.deepEqual(usage.get('tkmind_exp_survey'), { insert: true, read: true });
});
test('detectPageDataDatasetUsageFromHtml resolves dataset constants', () => {
const html = `
const DATASET = 'reading_survey';
await client.listRows(DATASET, { limit: 500 });
`;
const usage = detectPageDataDatasetUsageFromHtml(html);
assert.deepEqual(usage.get('reading_survey'), { read: true });
});
test('assertPolicyMatchesHtmlDatasets rejects mismatched dataset names', () => {
const html = `await c.insertRow('tkmind_exp_survey', {});`;
assert.throws(
+87 -20
View File
@@ -135,6 +135,84 @@ function buildWorkspacePublicUrl(userId, relativePath) {
});
}
function assertRegisteredDatasetTables(userDataSpace, registryDatasets, usage) {
for (const dataset of registryDatasets) {
const requiredUsage = usage.get(dataset.name) ?? {};
if (requiredUsage.insert && !dataset.actions.includes('insert')) {
throw Object.assign(new Error(`dataset「${dataset.name}」未开放写入`), {
code: 'dataset_action_not_registered',
datasetName: dataset.name,
action: 'insert',
});
}
if (requiredUsage.read && !dataset.actions.includes('read')) {
throw Object.assign(new Error(`dataset「${dataset.name}」未开放读取`), {
code: 'dataset_action_not_registered',
datasetName: dataset.name,
action: 'read',
});
}
const actualColumns = userDataSpace.listTableColumns(dataset.table);
if (!actualColumns.length) {
throw Object.assign(new Error(`dataset 对应表不存在:${dataset.table}`), {
code: 'table_not_found',
datasetName: dataset.name,
});
}
const actualNames = new Set(actualColumns.map((column) => column.name));
const configuredColumns = Object.values(dataset.columns ?? {}).flat();
const missingColumns = [...new Set(configuredColumns)].filter((column) => !actualNames.has(column));
if (missingColumns.length) {
throw Object.assign(
new Error(`dataset「${dataset.name}」注册字段不存在:${missingColumns.join(', ')}`),
{ code: 'dataset_schema_mismatch', datasetName: dataset.name, missingColumns },
);
}
}
}
/**
* Page Data policy must be derived from a registered SQLite dataset, never
* accepted solely from an Agent-supplied policy. This runs before page or
* publication creation so an incomplete data setup cannot leave a live page.
*/
export function resolveRegisteredPageDataPolicy({
html,
userId,
accessMode,
pageDataPolicy = null,
userDataSpace,
} = {}) {
const usage = detectPageDataDatasetUsageFromHtml(html);
if (!usage.size) return null;
if (!userDataSpace) throw new Error('缺少 Page Data 数据空间');
if (pageDataPolicy?.datasets) {
assertPolicyMatchesHtmlDatasets(html, pageDataPolicy.datasets);
}
const registryDatasets = userDataSpace.listDatasets();
const datasets = buildPageDataPolicyDatasetsFromRegistry({
html,
registryDatasets,
usage,
});
assertRegisteredDatasetTables(
userDataSpace,
registryDatasets.filter((dataset) => datasets[dataset.name]),
usage,
);
return {
...pageDataPolicy,
ownerUserId: String(pageDataPolicy?.ownerUserId ?? userId).trim(),
accessMode: pageDataPolicy?.accessMode ?? accessMode,
// The registry is authoritative for actions and allowed columns. Agent
// input can describe the page, but cannot manufacture a dataset policy.
datasets,
};
}
export async function bindWorkspaceHtmlForPageData({
pool,
h5Root,
@@ -158,6 +236,15 @@ export async function bindWorkspaceHtmlForPageData({
const normalizedAccessMode = resolvePageDataBindAccess(accessMode);
const resolvedPassword = resolvePageDataBindPassword(normalizedAccessMode, password);
const userDataSpace = createUserDataSpaceService({ workspaceRoot });
const resolvedPageDataPolicy = resolveRegisteredPageDataPolicy({
html: content,
userId,
accessMode: normalizedAccessMode,
pageDataPolicy,
userDataSpace,
});
const mindSpacePages = createPageService(pool, { h5Root, storageRoot });
const mindSpacePublications = createPublicationService(pool, {
h5Root,
@@ -183,26 +270,6 @@ export async function bindWorkspaceHtmlForPageData({
});
let policy = null;
const htmlDatasetUsage = detectPageDataDatasetUsageFromHtml(content);
let resolvedPageDataPolicy = pageDataPolicy;
if (htmlDatasetUsage.size) {
if (pageDataPolicy?.datasets) {
assertPolicyMatchesHtmlDatasets(content, pageDataPolicy.datasets);
} else {
const userDataSpace = createUserDataSpaceService({ workspaceRoot });
const autoDatasets = buildPageDataPolicyDatasetsFromRegistry({
html: content,
registryDatasets: userDataSpace.listDatasets(),
usage: htmlDatasetUsage,
});
resolvedPageDataPolicy = {
ownerUserId: userId,
accessMode: normalizedAccessMode,
datasets: autoDatasets,
};
}
}
if (resolvedPageDataPolicy) {
const ownerUserId = String(resolvedPageDataPolicy.ownerUserId ?? userId).trim();
+89
View File
@@ -4,8 +4,18 @@ import {
DEFAULT_PAGE_DATA_ADMIN_PASSWORD,
resolvePageDataBindAccess,
resolvePageDataBindPassword,
resolveRegisteredPageDataPolicy,
} from './page-data-workspace-bind.mjs';
const HTML = '<script src="/assets/page-data-client.js"></script><script>MindSpacePageData.createClient().insertRow("activity_signups", { name: "test" })</script>';
function createDataSpace({ datasets = [], tables = {} } = {}) {
return {
listDatasets: () => datasets,
listTableColumns: (table) => tables[table] ?? [],
};
}
test('resolvePageDataBindPassword defaults for password mode', () => {
assert.equal(resolvePageDataBindPassword('password', null), DEFAULT_PAGE_DATA_ADMIN_PASSWORD);
assert.equal(resolvePageDataBindPassword('password', ''), DEFAULT_PAGE_DATA_ADMIN_PASSWORD);
@@ -32,3 +42,82 @@ test('resolvePageDataBindAccess normalizes access mode', () => {
assert.equal(resolvePageDataBindAccess('public'), 'public');
assert.equal(resolvePageDataBindAccess('password'), 'password');
});
test('binding rejects an Agent policy when its HTML dataset is not registered', () => {
assert.throws(
() =>
resolveRegisteredPageDataPolicy({
html: HTML,
userId: 'user-1',
accessMode: 'public',
pageDataPolicy: { datasets: { activity_signups: { insert: true } } },
userDataSpace: createDataSpace(),
}),
(error) => error.code === 'dataset_not_registered',
);
});
test('binding derives policy from the registered dataset and rejects a missing table', () => {
const dataset = {
name: 'activity_signups',
table: 'activity_signups',
actions: ['insert'],
columns: { insert: ['name'] },
limits: {},
};
assert.throws(
() =>
resolveRegisteredPageDataPolicy({
html: HTML,
userId: 'user-1',
accessMode: 'public',
pageDataPolicy: { datasets: { activity_signups: { insert: true, columns: { insert: ['forged'] } } } },
userDataSpace: createDataSpace({ datasets: [dataset] }),
}),
(error) => error.code === 'table_not_found',
);
});
test('binding uses registered columns instead of Agent-supplied policy columns', () => {
const dataset = {
name: 'activity_signups',
table: 'activity_signups',
actions: ['insert'],
columns: { insert: ['name'] },
limits: {},
};
const policy = resolveRegisteredPageDataPolicy({
html: HTML,
userId: 'user-1',
accessMode: 'public',
pageDataPolicy: { datasets: { activity_signups: { insert: true, columns: { insert: ['forged'] } } } },
userDataSpace: createDataSpace({
datasets: [dataset],
tables: { activity_signups: [{ name: 'id' }, { name: 'name' }] },
}),
});
assert.deepEqual(policy.datasets.activity_signups.columns.insert, ['name']);
});
test('binding rejects a registered dataset that does not allow the HTML action', () => {
const dataset = {
name: 'activity_signups',
table: 'activity_signups',
actions: ['read'],
columns: { insert: ['name'], read: ['id', 'name'] },
limits: {},
};
assert.throws(
() =>
resolveRegisteredPageDataPolicy({
html: HTML,
userId: 'user-1',
accessMode: 'public',
userDataSpace: createDataSpace({
datasets: [dataset],
tables: { activity_signups: [{ name: 'id' }, { name: 'name' }] },
}),
}),
(error) => error.code === 'dataset_action_not_registered' && error.action === 'insert',
);
});
+193
View File
@@ -0,0 +1,193 @@
import fs from 'node:fs';
import path from 'node:path';
import { assessPageDataHtmlBinding } from './page-data-delivery-assess.mjs';
import {
detectPageDataDatasetUsageFromHtml,
htmlUsesPageDataApi,
inferPageDataBindAccessMode,
} from './page-data-html-detect.mjs';
import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs';
import { createUserDataSpaceService } from './user-data-space-service.mjs';
import { assertSafeSqlIdentifier } from './user-data-space-service.mjs';
const PAGE_DATA_ADMIN_PASSWORD = '88888888';
function readPublicHtmlFiles(workspaceRoot) {
const publicDir = path.join(path.resolve(String(workspaceRoot ?? '')), 'public');
if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) return [];
return fs
.readdirSync(publicDir)
.filter((name) => name.toLowerCase().endsWith('.html'))
.map((name) => {
const relativePath = `public/${name}`;
const absolutePath = path.join(publicDir, name);
const content = fs.readFileSync(absolutePath, 'utf8');
return { relativePath, absolutePath, content };
});
}
export function inferInsertColumnsFromHtml(html, datasetName) {
const text = String(html ?? '');
const safeName = String(datasetName ?? '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const patterns = [
new RegExp(`\\.insertRow\\(\\s*['"]${safeName}['"]\\s*,\\s*\\{([\\s\\S]*?)\\}\\s*[,)]`, 'm'),
new RegExp(
`\\.insertRow\\(\\s*([A-Za-z_$][\\w$]*)\\s*,\\s*\\{([\\s\\S]*?)\\}\\s*[,)]`,
'm',
),
];
const constants = new Map();
for (const match of text.matchAll(/(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*['"]([^'"]+)['"]/g)) {
constants.set(match[1], match[2]);
}
for (const pattern of patterns) {
const match = pattern.exec(text);
if (!match) continue;
const resolvedDataset = match.length === 3 ? constants.get(match[1]) ?? match[1] : datasetName;
if (String(resolvedDataset).trim() !== String(datasetName).trim()) continue;
const body = match[match.length - 1] ?? '';
const columns = [];
for (const fieldMatch of body.matchAll(/([A-Za-z_][\w$]*)\s*:/g)) {
const column = assertSafeSqlIdentifier(fieldMatch[1], 'insert 字段');
if (!columns.includes(column)) columns.push(column);
}
if (columns.length) return columns;
}
return [];
}
export async function ensureRegisteredDatasetFromHtml({
workspaceRoot,
userId = null,
query = null,
html,
datasetName,
}) {
const dataSpace = createUserDataSpaceService({ workspaceRoot, userId, query });
const existing = dataSpace.getDataset(datasetName);
if (existing) return existing;
const insertColumns = inferInsertColumnsFromHtml(html, datasetName);
if (!insertColumns.length) {
throw Object.assign(
new Error(`无法从 HTML 推断 dataset「${datasetName}」的 insert 字段,请先 register_dataset`),
{ code: 'insert_columns_unknown', datasetName },
);
}
const tableName = assertSafeSqlIdentifier(datasetName, 'dataset 表名');
const columnSql = insertColumns
.map((column) => `${column} TEXT NOT NULL DEFAULT ''`)
.join(',\n ');
await dataSpace.executeSql(
`CREATE TABLE IF NOT EXISTS ${tableName} (
id INTEGER PRIMARY KEY AUTOINCREMENT,
${columnSql},
created_at TEXT NOT NULL DEFAULT (datetime('now', '+8 hours'))
);`,
);
const readColumns = ['id', ...insertColumns, 'created_at'];
return dataSpace.upsertDataset({
name: datasetName,
table: tableName,
description: `Auto-registered from workspace HTML (${datasetName})`,
actions: ['read', 'insert'],
columns: {
read: readColumns,
insert: insertColumns,
},
});
}
export function listPageDataHtmlFiles(workspaceRoot) {
return readPublicHtmlFiles(workspaceRoot).filter((file) => htmlUsesPageDataApi(file.content));
}
export async function ensurePageDataHtmlPagesBound({
pool,
h5Root,
storageRoot,
userId,
workspaceRoot,
findPageByRelativePath = null,
onlyRelativePaths = null,
logger = console,
} = {}) {
if (!pool || !userId || !workspaceRoot) {
return {
bound: [],
skipped: [],
errors: [{ code: 'missing_context', message: '缺少 pool/userId/workspaceRoot' }],
};
}
const allowList = onlyRelativePaths ? new Set(onlyRelativePaths) : null;
const bound = [];
const skipped = [];
const errors = [];
for (const file of listPageDataHtmlFiles(workspaceRoot)) {
if (allowList && !allowList.has(file.relativePath)) continue;
const usage = detectPageDataDatasetUsageFromHtml(file.content);
if (!usage.size) {
skipped.push({ relativePath: file.relativePath, reason: 'no_dataset_usage' });
continue;
}
const assessment = await assessPageDataHtmlBinding({
pool,
userId,
publishDir: workspaceRoot,
relativePath: file.relativePath,
html: file.content,
findPageByRelativePath,
});
if (assessment.bound) {
skipped.push({ relativePath: file.relativePath, reason: 'already_bound' });
continue;
}
try {
for (const datasetName of usage.keys()) {
await ensureRegisteredDatasetFromHtml({
workspaceRoot,
userId,
query: pool.query.bind(pool),
html: file.content,
datasetName,
});
}
const accessMode = inferPageDataBindAccessMode(file.relativePath, file.content);
const result = await bindWorkspaceHtmlForPageData({
pool,
h5Root,
storageRoot,
userId,
workspaceRoot,
relativePath: file.relativePath,
accessMode,
password: accessMode === 'password' ? PAGE_DATA_ADMIN_PASSWORD : null,
});
bound.push({
relativePath: file.relativePath,
pageId: result.pageId,
workspaceUrl: result.workspaceUrl,
});
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
errors.push({
relativePath: file.relativePath,
message,
code: err?.code ?? 'bind_failed',
});
logger.warn?.(
`[PageData] ensure bind failed for ${file.relativePath}: ${message}`,
);
}
}
return { bound, skipped, errors };
}
+69
View File
@@ -0,0 +1,69 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import {
ensureRegisteredDatasetFromHtml,
inferInsertColumnsFromHtml,
} from './page-data-workspace-ensure.mjs';
import { evaluatePageDataFinishGuard } from './mindspace-page-data-finish-guard.mjs';
const HOMEWORK_HTML = `<!doctype html><html><head><title>作业</title></head><body>
<script src="/assets/page-data-client.js"></script>
<script>
MindSpacePageData.createClient({ apiBase: '/api' }).insertRow('homework_records', {
student_name: 'a',
record_date: '2026-07-12',
chinese: '已完成',
math: '已完成',
english: '已完成',
note: ''
});
</script></body></html>`;
test('inferInsertColumnsFromHtml parses insertRow object keys', () => {
const columns = inferInsertColumnsFromHtml(HOMEWORK_HTML, 'homework_records');
assert.deepEqual(columns, [
'student_name',
'record_date',
'chinese',
'math',
'english',
'note',
]);
});
test('ensureRegisteredDatasetFromHtml creates sqlite registry from html', async () => {
const workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-ensure-'));
try {
await ensureRegisteredDatasetFromHtml({
workspaceRoot,
html: HOMEWORK_HTML,
datasetName: 'homework_records',
});
const dbPath = path.join(workspaceRoot, '.mindspace', 'private-data.sqlite');
assert.ok(fs.existsSync(dbPath));
} finally {
fs.rmSync(workspaceRoot, { recursive: true, force: true });
}
});
test('evaluatePageDataFinishGuard detects unbound homework html even when user only confirms', () => {
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-homework-'));
try {
fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true });
fs.writeFileSync(path.join(publishDir, 'public', 'homework.html'), HOMEWORK_HTML, 'utf8');
const evaluation = evaluatePageDataFinishGuard({
publishDir,
agentText: '确认发布',
messages: [],
});
assert.equal(evaluation.pageDataIntent, false);
assert.equal(evaluation.structuralPageData, true);
assert.equal(evaluation.unboundFiles.length, 1);
assert.equal(evaluation.needsRepair, true);
} finally {
fs.rmSync(publishDir, { recursive: true, force: true });
}
});
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env node
/**
* 生产 runtime 安全:补建作业登记 Page Datasqlite + workspace policy 文件 + policy index)。
* 不依赖完整 dev 模块树(db.mjs / page-data-workspace-bind 等)。
*/
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import mysql from 'mysql2/promise';
const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(repoRoot, '.env'));
const USER_ID = '0a763620-c0d6-4e88-9f0a-6cc68840cf7a';
const DATASET = 'homework_records';
const TABLE = 'homework_records';
const REGISTRY = '__page_data_datasets';
const SQLITE_BIN = process.env.SQLITE_BIN?.trim() || 'sqlite3';
const ADMIN_PASSWORD = '88888888';
const PAGES = [
{
pageId: '553c3b47-01eb-4294-9ce8-dfb6b2576990',
relativePath: 'public/homework.html',
accessMode: 'public',
policy: {
accessMode: 'public',
defaultVisitorRole: 'deny',
datasets: {
[DATASET]: {
read: false,
insert: true,
columns: {
insert: ['student_name', 'record_date', 'chinese', 'math', 'english', 'note'],
},
},
},
},
},
{
pageId: 'f8720a88-e04d-4c40-938f-11b7fcd41d50',
relativePath: 'public/homework-admin.html',
accessMode: 'password',
policy: {
accessMode: 'password',
defaultVisitorRole: 'deny',
datasets: {
[DATASET]: {
read: true,
insert: false,
columns: {
read: ['id', 'student_name', 'record_date', 'chinese', 'math', 'english', 'note', 'created_at'],
},
},
},
},
},
];
function runSqlite(dbPath, sql) {
execFileSync(SQLITE_BIN, ['-batch', dbPath, sql], { stdio: 'pipe' });
}
function sqlLiteral(value) {
return `'${String(value).replace(/'/g, "''")}'`;
}
function buildScopeHash(policy) {
const payload = {
accessMode: policy.accessMode,
datasets: Object.keys(policy.datasets).sort(),
};
return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex').slice(0, 16);
}
function writePolicy(workspaceRoot, pageId, policyInput) {
const policyDir = path.join(workspaceRoot, '.mindspace', 'page-data-policies');
fs.mkdirSync(policyDir, { recursive: true });
const policy = {
pageId,
ownerUserId: USER_ID,
workspaceRef: null,
...policyInput,
visitors: [],
roles: {},
updatedAt: new Date().toISOString(),
};
const policyPath = path.join(policyDir, `${pageId}.json`);
fs.writeFileSync(policyPath, `${JSON.stringify(policy, null, 2)}\n`, 'utf8');
return policy;
}
const workspaceRoot = path.join(repoRoot, 'MindSpace', USER_ID);
if (!fs.existsSync(workspaceRoot)) {
console.error(`工作区不存在: ${workspaceRoot}`);
process.exit(1);
}
const mindspaceDir = path.join(workspaceRoot, '.mindspace');
const dbPath = path.join(mindspaceDir, 'private-data.sqlite');
fs.mkdirSync(mindspaceDir, { recursive: true });
console.log('1/3 初始化 sqlite 表与 dataset 注册…');
runSqlite(
dbPath,
`PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS ${TABLE} (
id INTEGER PRIMARY KEY AUTOINCREMENT,
student_name TEXT NOT NULL,
record_date TEXT NOT NULL,
chinese TEXT NOT NULL,
math TEXT NOT NULL,
english TEXT NOT NULL,
note TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL DEFAULT (datetime('now', '+8 hours'))
);
CREATE TABLE IF NOT EXISTS ${REGISTRY} (
name TEXT PRIMARY KEY,
table_name TEXT NOT NULL,
config_json TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
);`,
);
const datasetConfig = {
name: DATASET,
table: TABLE,
description: '每日作业登记(语数外)',
actions: ['read', 'insert'],
columns: {
read: ['id', 'student_name', 'record_date', 'chinese', 'math', 'english', 'note', 'created_at'],
insert: ['student_name', 'record_date', 'chinese', 'math', 'english', 'note'],
},
limits: { maxRowsPerRead: 200, maxInsertBytes: 8192 },
};
runSqlite(
dbPath,
`INSERT INTO ${REGISTRY} (name, table_name, config_json, updated_at)
VALUES (${sqlLiteral(DATASET)}, ${sqlLiteral(TABLE)}, ${sqlLiteral(JSON.stringify(datasetConfig))}, CURRENT_TIMESTAMP)
ON CONFLICT(name) DO UPDATE SET
table_name = excluded.table_name,
config_json = excluded.config_json,
updated_at = CURRENT_TIMESTAMP;`,
);
console.log('2/3 写入 workspace policy 文件…');
const writtenPolicies = [];
for (const page of PAGES) {
const htmlPath = path.join(workspaceRoot, page.relativePath);
if (!fs.existsSync(htmlPath)) {
console.error(`缺少 HTML: ${page.relativePath}`);
process.exit(1);
}
const policy = writePolicy(workspaceRoot, page.pageId, page.policy);
writtenPolicies.push(policy);
console.log(`${page.relativePath}${page.pageId}.json`);
}
if (!process.env.DATABASE_URL && !(process.env.MYSQL_HOST && process.env.MYSQL_DATABASE)) {
console.warn('3/3 跳过 MySQL policy index(未配置数据库)');
process.exit(0);
}
console.log('3/3 同步 MySQL policy index…');
const pool = process.env.DATABASE_URL
? mysql.createPool({ uri: process.env.DATABASE_URL, connectionLimit: 2 })
: mysql.createPool({
host: process.env.MYSQL_HOST ?? 'localhost',
port: Number(process.env.MYSQL_PORT ?? 3306),
user: process.env.MYSQL_USER ?? 'boot',
password: process.env.MYSQL_PASSWORD ?? '',
database: process.env.MYSQL_DATABASE ?? 'tkmind',
connectionLimit: 2,
});
const now = Date.now();
for (const policy of writtenPolicies) {
const datasetCount = Object.keys(policy.datasets).length;
await pool.query(
`INSERT INTO h5_page_data_policy_index
(page_id, owner_user_id, access_mode, dataset_count, scope_hash, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
owner_user_id = VALUES(owner_user_id),
access_mode = VALUES(access_mode),
dataset_count = VALUES(dataset_count),
scope_hash = VALUES(scope_hash),
updated_at = VALUES(updated_at)`,
[policy.pageId, USER_ID, policy.accessMode, datasetCount, buildScopeHash(policy), now],
);
}
await pool.end();
console.log('\n完成。');
console.log(` 登记页:https://m.tkmind.cn/MindSpace/${USER_ID}/public/homework.html`);
console.log(` 后台页:https://m.tkmind.cn/MindSpace/${USER_ID}/public/homework-admin.html`);
console.log(` 后台口令:${ADMIN_PASSWORD}(平台要求至少 8 位;「888」无效)`);
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env node
/**
* Patch reading-survey HTML on disk for WeChat WebView radio/submit compatibility.
*
* Usage:
* node scripts/repair-reading-survey-wechat.mjs
* node scripts/repair-reading-survey-wechat.mjs --user a70ff537-8908-486e-9b6c-042e07cc25db
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { applyWechatSurveyCompat } from '../mindspace-page-data-wechat-survey-compat.mjs';
import { PUBLISH_ROOT_DIR } from '../user-publish.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');
function parseArgs(argv) {
const userIdx = argv.indexOf('--user');
return {
userId:
userIdx >= 0
? String(argv[userIdx + 1] ?? '').trim()
: 'a70ff537-8908-486e-9b6c-042e07cc25db',
};
}
function patchFile(absolutePath) {
const before = fs.readFileSync(absolutePath, 'utf8');
const { html, patched } = applyWechatSurveyCompat(before);
if (!patched) {
console.log(`SKIP ${absolutePath} (no compat patch needed)`);
return false;
}
fs.writeFileSync(absolutePath, html, 'utf8');
console.log(`OK ${absolutePath}`);
return true;
}
const { userId } = parseArgs(process.argv.slice(2));
const base = path.join(root, PUBLISH_ROOT_DIR, userId, 'public');
const targets = ['reading-survey.html'];
let changed = 0;
for (const name of targets) {
const filePath = path.join(base, name);
if (!fs.existsSync(filePath)) {
console.log(`MISS ${filePath}`);
continue;
}
if (patchFile(filePath)) changed += 1;
}
console.log(`patched ${changed} file(s)`);
process.exit(changed > 0 ? 0 : 1);
@@ -0,0 +1,15 @@
import { createDbPool } from '../db.mjs';
import { ensurePersonalMemoryCandidateSchema } from '../memory-v2-personal-store.mjs';
if (process.env.MEMORY_PERSONAL_SCHEMA_CONFIRM !== 'local-only') {
throw new Error('Refusing schema change: set MEMORY_PERSONAL_SCHEMA_CONFIRM=local-only explicitly');
}
const pool = createDbPool();
try {
await ensurePersonalMemoryCandidateSchema(pool);
console.log('Memory V2 personal candidate schema ready.');
} finally {
await pool.end();
}
+68 -22
View File
@@ -136,6 +136,7 @@ import {
} from './mindspace-public-finish-sync.mjs';
import { maybeRepairH5HtmlAfterFinish } from './mindspace-h5-html-finish-guard.mjs';
import { maybeRepairPageDataAfterFinish } from './mindspace-page-data-finish-guard.mjs';
import { ensurePageDataHtmlPagesBound } from './page-data-workspace-ensure.mjs';
import { quickPlazaFromChat, quickPlazaFromPublicHtml, getQuickPlazaFromPublicHtmlStatus } from './mindspace-chat-plaza.mjs';
import { injectPublicFileShareButton } from './mindspace-public-share-widget.mjs';
import { resolvePlazaPostPath, resolvePlazaPublicBase } from './src/utils/public-site-bases.mjs';
@@ -191,6 +192,7 @@ import { createSessionSnapshotService } from './session-snapshot.mjs';
import { createConversationMemoryService } from './conversation-memory.mjs';
import { createManagedMemoryV2Runtime } from './memory-v2-runtime.mjs';
import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs';
import { createSkillRuntimeAdminConfigService } from './skill-runtime-admin-config.mjs';
import { createWechatScheduleLlmConfigService } from './wechat-schedule-llm-config.mjs';
import { createExperienceService } from './experience-service.mjs';
import { attachAsrRoutes } from './asr-proxy.mjs';
@@ -201,24 +203,16 @@ import { syncPageDataPolicyAccessMode } from './page-data-publish-sync.mjs';
import { upsertPageDataPolicyIndex } from './page-data-policy-index.mjs';
import { attachShenmeiOpinionFormRoutes } from './shenmei-opinion-form-routes.mjs';
import { isNativeH5ApiPath } from './policies.mjs';
import {
applyMemindRuntimeProfile,
describeMemindRuntimeProfile,
loadMemindEnvFiles,
} from './scripts/memind-runtime-profile.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eq = trimmed.indexOf('=');
if (eq < 0) continue;
const key = trimmed.slice(0, eq).trim();
const value = trimmed.slice(eq + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(__dirname, '../../.env.local'));
loadEnvFile(path.join(__dirname, '.env'));
loadMemindEnvFiles(__dirname);
applyMemindRuntimeProfile({ rootDir: __dirname });
function parseApiTargets() {
const csvTargets = (process.env.TKMIND_API_TARGETS ?? '')
@@ -348,6 +342,15 @@ async function unregisterAgentSessionForUser(userId, sessionId) {
await access.unregisterSession({ userId, sessionId });
}
let agentRunGateway = null;
const sessionPageDeliveryLocks = new Map();
function beginSessionPageDelivery(sessionId) {
sessionPageDeliveryLocks.set(sessionId, Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) + 1);
}
function endSessionPageDelivery(sessionId) {
const remaining = Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) - 1;
if (remaining > 0) sessionPageDeliveryLocks.set(sessionId, remaining);
else sessionPageDeliveryLocks.delete(sessionId);
}
let chatIntentRouter = null;
let toolGateway = null;
let directChatService = null;
@@ -355,6 +358,7 @@ let sessionSnapshotService = null;
let conversationMemoryService = null;
let memoryV2 = null;
let memoryV2ConfigService = null;
let skillRuntimeConfigService = null;
let wechatScheduleLlmConfigService = null;
let mindSpace = null;
let mindSpaceAssets = null;
@@ -463,6 +467,10 @@ async function bootstrapUserAuth() {
pageService: mindSpacePages,
publicationService: mindSpacePublications,
pageSyncService: mindSpacePageSync,
pageDataEnsure: { ensurePageDataHtmlPagesBound },
h5Root: __dirname,
storageRoot: resolveMindSpaceRuntimeConfig(__dirname, process.env).storageRoot,
findPageByRelativePath: mindSpacePages.findPageByRelativePath.bind(mindSpacePages),
logger: console,
});
const resolveUserIdByDirKey = async (dirKey) => {
@@ -634,6 +642,7 @@ async function bootstrapUserAuth() {
console.warn('LLM provider boot sync skipped:', err instanceof Error ? err.message : err);
});
memoryV2ConfigService = createMemoryV2AdminConfigService(pool);
skillRuntimeConfigService = createSkillRuntimeAdminConfigService(pool, { h5Root: __dirname });
wechatScheduleLlmConfigService = createWechatScheduleLlmConfigService(pool);
conversationMemoryService = createConversationMemoryService(pool, {
llmProviderService,
@@ -700,9 +709,16 @@ async function bootstrapUserAuth() {
chatIntentRouter,
sessionSnapshotService,
conversationMemoryService,
syncUserPagesOnSuccess: async ({ userId }) => {
await syncUserGeneratedPages(userId);
observePersonalMemoryOnSuccess: async ({ userId, sessionId, userMessage }) => {
if (!memoryV2?.observePersonalMemory) return;
await memoryV2.observePersonalMemory({
userId,
sessionId,
messages: [userMessage],
});
},
syncUserPagesOnSuccess: async ({ userId }) => syncUserGeneratedPages(userId),
isSessionExternallyBusy: ({ sessionId }) => Number(sessionPageDeliveryLocks.get(sessionId) ?? 0) > 0,
autoDispatch: ['1', 'true', 'yes', 'on'].includes(
String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(),
),
@@ -846,6 +862,16 @@ app.use(attachUserSession);
// ============ Legacy password auth ============
async function resolveSkillRuntimeForClient() {
if (!skillRuntimeConfigService?.getPublicRuntimeConfig) return null;
try {
return await skillRuntimeConfigService.getPublicRuntimeConfig();
} catch (err) {
console.warn('[SkillRuntime] public config unavailable:', err instanceof Error ? err.message : err);
return null;
}
}
app.get('/auth/status', async (req, res) => {
await userAuthReady;
if (userAuth) {
@@ -854,6 +880,7 @@ app.get('/auth/status', async (req, res) => {
if (!me) return res.json({ authenticated: false, mode: 'user' });
const row = await userAuth.getUserById(me.id);
const capabilityState = await userAuth.resolveUserCapabilities(row);
const skillRuntime = await resolveSkillRuntimeForClient();
return res.json({
authenticated: true,
user: me,
@@ -861,6 +888,7 @@ app.get('/auth/status', async (req, res) => {
capabilities: capabilityState.capabilities,
grantedSkills: capabilityState.grantedSkills ?? [],
unrestricted: capabilityState.unrestricted,
skillRuntime,
});
} catch (err) {
console.error('[Auth] status failed:', err instanceof Error ? err.message : err);
@@ -1304,10 +1332,11 @@ app.get('/auth/me', async (req, res) => {
if (!userAuth) return res.status(503).json({ message: '未启用用户系统' });
const me = await userAuth.getMe(userToken(req));
if (!me) return res.status(401).json({ message: '未登录' });
const [paths, capabilityState, subscription] = await Promise.all([
const [paths, capabilityState, subscription, skillRuntime] = await Promise.all([
userAuth.listPathGrants(me.id),
userAuth.resolveUserCapabilities(await userAuth.getUserById(me.id)),
subscriptionService ? subscriptionService.getActiveSubscription(me.id) : null,
resolveSkillRuntimeForClient(),
]);
return res.json({
user: { ...me, subscription },
@@ -1315,6 +1344,7 @@ app.get('/auth/me', async (req, res) => {
capabilities: capabilityState.capabilities,
grantedSkills: capabilityState.grantedSkills ?? [],
unrestricted: capabilityState.unrestricted,
skillRuntime,
});
});
@@ -3204,11 +3234,10 @@ const SAVE_TARGET_CATEGORIES = new Set(['draft', 'oa', 'public']);
async function syncUserGeneratedPages(userId) {
if (!userId) return;
if (workspacePageDeliver?.syncAndDeliver) {
await workspacePageDeliver.syncAndDeliver(userId);
return;
return await workspacePageDeliver.syncAndDeliver(userId);
}
if (!mindSpacePageSync) return;
await mindSpacePageSync.syncUserGeneratedPages(userId);
return await mindSpacePageSync.syncUserGeneratedPages(userId);
}
async function resolveChatSaveBundle(user, h5Root, input = {}) {
@@ -4999,6 +5028,8 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
// workspace HTML into the asset store before a later restart rebuilds the
// workspace from DB-backed assets only.
const onAfterFinish = async (sid, uid) => {
beginSessionPageDelivery(sid);
try {
const apiFetchFn = async (pathname, init) => {
const target = await tkmindProxy.resolveTarget(sid);
return tkmindProxy.apiFetchTo(target, pathname, init);
@@ -5068,6 +5099,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
.filter(Boolean)
.join('\n')
: '';
await syncUserGeneratedPages(uid);
await maybeRepairPageDataAfterFinish({
sessionId: sid,
userId: uid,
@@ -5079,7 +5111,20 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
tkmindProxy,
userText: lastUserText,
});
await syncUserGeneratedPages(uid);
if (lastUserMessage && memoryV2?.observePersonalMemory) {
await memoryV2.observePersonalMemory({
userId: uid,
sessionId: sid,
messages: [lastUserMessage],
}).catch((err) => {
console.warn(
`[memory-v2] finish shadow observation skipped for session ${sid}: ${err instanceof Error ? err.message : err}`,
);
});
}
} finally {
endSessionPageDelivery(sid);
}
};
return tkmindProxy.proxySessionEvents(req, res, sessionId, {
onAfterFinish,
@@ -6228,6 +6273,7 @@ app.get('*', (_req, res) => {
process.exit(1);
}
const server = app.listen(PORT, HOST, () => {
console.log(`[Portal] Runtime profile: ${describeMemindRuntimeProfile()}`);
console.log(`TKMind H5 @ http://${HOST}:${PORT}`);
console.log(`Proxy -> ${API_TARGETS.join(', ')}`);
console.log(`Auth -> ${enabled ? 'multi-user (MySQL)' : legacyAuth ? 'legacy password' : 'disabled'}`);
+181
View File
@@ -0,0 +1,181 @@
import { SKILL_ROUTER_V2_ENV } from './chat-skills.mjs';
import {
buildPublicSkillRuntimeConfig,
buildSkillRuntimeCatalogSummary,
} from './skill-runtime-policy.mjs';
const CONFIG_TABLE = 'h5_skill_runtime_config';
const CONFIG_SCOPE = 'global';
function defaultConfigShape() {
return {
router: {
v2Enabled: false,
manifestRoutingEnabled: false,
},
};
}
function normalizeBoolean(value, fallback = false) {
if (value == null || value === '') return fallback;
if (typeof value === 'boolean') return value;
const normalized = String(value).trim().toLowerCase();
if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
return fallback;
}
function cloneConfig(config = null) {
return structuredClone?.(config ?? defaultConfigShape())
?? JSON.parse(JSON.stringify(config ?? defaultConfigShape()));
}
function parseJsonLike(value, fallback) {
if (value == null || value === '') return fallback;
if (typeof value === 'string') {
try {
return JSON.parse(value);
} catch {
return fallback;
}
}
if (typeof value === 'object') return value;
return fallback;
}
function envRouterEnabled(env = process.env) {
return /^(1|true|yes)$/i.test(String(env?.[SKILL_ROUTER_V2_ENV] ?? ''));
}
function applyEnv(config, env = process.env) {
const next = cloneConfig(config);
if (next.router.v2Enabled === false && envRouterEnabled(env)) {
next.router.v2Enabled = true;
}
if (next.router.manifestRoutingEnabled === false && next.router.v2Enabled) {
next.router.manifestRoutingEnabled = envRouterEnabled(env);
}
return next;
}
function mergePatch(currentConfig, patch = {}) {
const next = cloneConfig(currentConfig);
if (patch?.router && 'v2Enabled' in patch.router) {
next.router.v2Enabled = normalizeBoolean(patch.router.v2Enabled, false);
}
if (patch?.router && 'manifestRoutingEnabled' in patch.router) {
next.router.manifestRoutingEnabled = normalizeBoolean(patch.router.manifestRoutingEnabled, false);
}
if (!next.router.v2Enabled) {
next.router.manifestRoutingEnabled = false;
}
return next;
}
async function ensureConfigTable(pool) {
await pool.query(`
CREATE TABLE IF NOT EXISTS ${CONFIG_TABLE} (
config_scope VARCHAR(32) PRIMARY KEY,
config_json JSON NOT NULL,
updated_by CHAR(36) NULL,
updated_at BIGINT NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
`);
}
async function loadStoredState(pool) {
await ensureConfigTable(pool);
const [rows] = await pool.query(
`SELECT config_json, updated_by, updated_at
FROM ${CONFIG_TABLE}
WHERE config_scope = ?
LIMIT 1`,
[CONFIG_SCOPE],
);
const row = rows[0];
if (!row) return null;
return {
config: { ...defaultConfigShape(), ...parseJsonLike(row.config_json, {}) },
updatedAt: Number(row.updated_at ?? 0) || null,
updatedBy: row.updated_by ?? null,
};
}
export function createSkillRuntimeAdminConfigService(pool, { env = process.env, h5Root = process.cwd() } = {}) {
async function loadEffectiveConfig() {
const stored = await loadStoredState(pool);
if (!stored) {
const config = applyEnv(defaultConfigShape(), env);
return {
config,
updatedAt: null,
updatedBy: null,
source: envRouterEnabled(env) ? 'env' : 'default',
};
}
return {
config: cloneConfig(stored.config),
updatedAt: stored.updatedAt,
updatedBy: stored.updatedBy,
source: 'admin-db',
};
}
return {
async getAdminConfig() {
const state = await loadEffectiveConfig();
return {
config: state.config,
updatedAt: state.updatedAt,
updatedBy: state.updatedBy,
source: state.source,
};
},
async updateAdminConfig(patch = {}, { updatedBy = null } = {}) {
const stored = await loadStoredState(pool);
const base = stored?.config ?? defaultConfigShape();
const nextConfig = mergePatch(base, patch.config ?? patch);
await ensureConfigTable(pool);
const now = Date.now();
await pool.query(
`INSERT INTO ${CONFIG_TABLE}
(config_scope, config_json, updated_by, updated_at)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
config_json = VALUES(config_json),
updated_by = VALUES(updated_by),
updated_at = VALUES(updated_at)`,
[CONFIG_SCOPE, JSON.stringify(nextConfig), updatedBy, now],
);
return this.getAdminConfig();
},
async getRuntimeState() {
const state = await loadEffectiveConfig();
return {
source: state.source,
updatedAt: state.updatedAt,
updatedBy: state.updatedBy,
config: state.config,
};
},
async getPublicRuntimeConfig() {
const state = await loadEffectiveConfig();
return buildPublicSkillRuntimeConfig(state.config, h5Root);
},
listCatalogSummary() {
return buildSkillRuntimeCatalogSummary(h5Root);
},
};
}
export const skillRuntimeAdminConfigInternals = {
CONFIG_SCOPE,
CONFIG_TABLE,
defaultConfigShape,
applyEnv,
mergePatch,
};
+74
View File
@@ -0,0 +1,74 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
createSkillRuntimeAdminConfigService,
skillRuntimeAdminConfigInternals,
} from './skill-runtime-admin-config.mjs';
function createPool(seedRow = null) {
const state = { row: seedRow };
return {
state,
async query(sql, params) {
if (sql.includes('CREATE TABLE')) return [[], []];
if (sql.includes('SELECT config_json')) {
return [state.row ? [state.row] : [], []];
}
if (sql.includes('INSERT INTO h5_skill_runtime_config')) {
state.row = {
config_json: params[1],
updated_by: params[2],
updated_at: params[3],
};
return [[], []];
}
throw new Error(`Unexpected query: ${sql}`);
},
};
}
test('skill runtime admin config defaults to disabled router', async () => {
const service = createSkillRuntimeAdminConfigService(createPool(), { env: {} });
const result = await service.getAdminConfig();
assert.equal(result.config.router.v2Enabled, false);
assert.equal(result.config.router.manifestRoutingEnabled, false);
});
test('skill runtime admin config falls back to env when db row is absent', async () => {
const service = createSkillRuntimeAdminConfigService(createPool(), {
env: { TKMIND_SKILL_ROUTER_V2: '1' },
});
const result = await service.getAdminConfig();
assert.equal(result.config.router.v2Enabled, true);
assert.equal(result.config.router.manifestRoutingEnabled, true);
assert.equal(result.source, 'env');
});
test('skill runtime admin config persists router toggles', async () => {
const pool = createPool();
const service = createSkillRuntimeAdminConfigService(pool, { env: {} });
const updated = await service.updateAdminConfig(
{ router: { v2Enabled: true, manifestRoutingEnabled: true } },
{ updatedBy: 'admin-1' },
);
assert.equal(updated.config.router.v2Enabled, true);
assert.equal(updated.config.router.manifestRoutingEnabled, true);
const runtime = await service.getPublicRuntimeConfig();
assert.equal(runtime.routerV2Enabled, true);
assert.equal(runtime.manifestRoutingEnabled, true);
assert.ok(Array.isArray(runtime.manifestRoutes));
});
test('skill runtime admin config keeps db authoritative over env', async () => {
const pool = createPool({
config_json: JSON.stringify(skillRuntimeAdminConfigInternals.defaultConfigShape()),
updated_by: 'admin-1',
updated_at: 123,
});
const service = createSkillRuntimeAdminConfigService(pool, {
env: { TKMIND_SKILL_ROUTER_V2: '1' },
});
const result = await service.getAdminConfig();
assert.equal(result.config.router.v2Enabled, false);
assert.equal(result.source, 'admin-db');
});
+30
View File
@@ -0,0 +1,30 @@
import { buildManifestRoutesFromCatalog } from './chat-skills.mjs';
import { listPlatformSkillCatalog } from './skills-registry.mjs';
export function buildPublicSkillRuntimeConfig(config, h5Root = process.cwd()) {
const routerV2Enabled = Boolean(config?.router?.v2Enabled);
const manifestRoutingEnabled = routerV2Enabled && Boolean(config?.router?.manifestRoutingEnabled);
const catalog = listPlatformSkillCatalog(h5Root);
const manifestRoutes = manifestRoutingEnabled ? buildManifestRoutesFromCatalog(catalog) : [];
return {
routerV2Enabled,
manifestRoutingEnabled,
manifestRoutes,
manifestSkillCount: catalog.filter((item) => item.manifest?.trigger?.keywords?.length).length,
};
}
export function buildSkillRuntimeCatalogSummary(h5Root = process.cwd()) {
const catalog = listPlatformSkillCatalog(h5Root);
return catalog.map((item) => ({
name: item.name,
dirName: item.dirName,
description: item.description,
version: item.version ?? null,
executors: item.executors ?? ['goose'],
hasManifest: Boolean(item.manifest),
triggerKeywords: item.manifest?.trigger?.keywords ?? [],
routerPromptKey: item.manifest?.router?.promptKey ?? null,
routerPriority: item.manifest?.router?.priority ?? 0,
}));
}
+40
View File
@@ -0,0 +1,40 @@
import assert from 'node:assert/strict';
import path from 'node:path';
import test from 'node:test';
import { fileURLToPath } from 'node:url';
import {
buildPublicSkillRuntimeConfig,
buildSkillRuntimeCatalogSummary,
} from './skill-runtime-policy.mjs';
import { buildAutoChatSkillPrefixOptions } from './chat-skills.mjs';
const h5Root = path.dirname(fileURLToPath(import.meta.url));
test('buildPublicSkillRuntimeConfig omits manifest routes when disabled', () => {
const config = buildPublicSkillRuntimeConfig(
{ router: { v2Enabled: false, manifestRoutingEnabled: false } },
h5Root,
);
assert.equal(config.routerV2Enabled, false);
assert.equal(config.manifestRoutingEnabled, false);
assert.deepEqual(config.manifestRoutes, []);
});
test('buildSkillRuntimeCatalogSummary includes product-campaign-page manifest', () => {
const catalog = buildSkillRuntimeCatalogSummary(h5Root);
const product = catalog.find((item) => item.name === 'product-campaign-page');
assert.ok(product);
assert.equal(product.hasManifest, true);
assert.ok(product.triggerKeywords.includes('带货'));
});
test('buildAutoChatSkillPrefixOptions returns empty object when router is disabled', () => {
assert.deepEqual(
buildAutoChatSkillPrefixOptions({
routerV2Enabled: false,
manifestRoutingEnabled: false,
manifestRoutes: [{ skillName: 'web', keywords: ['新闻'], priority: 1, promptKey: 'web', promptVariant: null }],
}),
{},
);
});
+14
View File
@@ -0,0 +1,14 @@
name: product-campaign-page
version: 1.0.0
description: 商品宣传 / 活动页
trigger:
keywords:
- 带货
- 商品页
- 种草
- 电商页
router:
promptKey: product-campaign-page
priority: 20
executors:
- goose
+34 -1
View File
@@ -21,6 +21,7 @@ import {
} from './wechat/handlers/sync-replies.mjs';
import { classifyWechatIntent } from './wechat/intent/classifier.mjs';
import { isPageGenerateText, isTopicResetText } from './wechat/intent/patterns.mjs';
import { isPageDataIntent } from './chat-skills.mjs';
import { resolvePageGenerateOutcome } from './wechat/handlers/page-generate.mjs';
import { buildWechatAgentPrompt } from './wechat/prompts/chat-general.mjs';
import {
@@ -973,6 +974,14 @@ function isTopicResetIntent(text) {
return isTopicResetText(text);
}
export function shouldForceNewWechatAgentSession(wechatIntent, resetCandidate) {
return (
wechatIntent?.kind === 'session.reset' ||
isTopicResetIntent(resetCandidate) ||
isPageDataIntent(resetCandidate)
);
}
export function isRecoverableWechatAgentSessionError(message) {
const normalized = String(message ?? '').trim();
if (!normalized) return false;
@@ -1065,6 +1074,7 @@ async function enforcePageDataCollectDelivery({
pool: pageDataFinishGuard?.pool ?? null,
userId,
findPageByRelativePath: null,
apiBase: publicBaseUrl,
});
let autoBind = null;
if (outcome.action === 'skip') return outcome;
@@ -1091,6 +1101,7 @@ async function enforcePageDataCollectDelivery({
pool: pageDataFinishGuard.pool,
userId,
findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService),
apiBase: publicBaseUrl,
});
}
@@ -1905,7 +1916,11 @@ export function createWechatMpService({
const wechatIntent = classifyWechatIntent(intent);
const resetCandidate =
intent.msgType === 'text' || intent.msgType === 'voice' ? intent.agentText : '';
const forceNew = wechatIntent.kind === 'session.reset' || isTopicResetIntent(resetCandidate);
// Page Data delivery owns persistent files, datasets and two publication
// policies. Reusing a conversational route here can make a new request
// inspect/retry unrelated historical pages from that session.
const isPageDataRequest = isPageDataIntent(resetCandidate);
const forceNew = shouldForceNewWechatAgentSession(wechatIntent, resetCandidate);
let route = await ensureWechatAgentSession({
userId: user.userId,
openid: inbound.fromUserName,
@@ -2064,6 +2079,24 @@ export function createWechatMpService({
return { sessionId };
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
// A Page Data request must fail closed. Retrying a poisoned completion in
// another session while the finish guard is also active can turn one
// request into repairs against historical pages. Drop only this user's
// route so the next explicit request starts cleanly.
if (isPageDataRequest) {
await userAuth.clearWechatAgentRoute(config.appId, inbound.fromUserName).catch((clearErr) => {
logger.warn?.('WeChat MP page data route clear failed:', clearErr);
});
if (!err?.wechatUserNotified) {
const text = buildPageDataCollectFailureText();
try {
await sendCustomerServiceText(inbound.fromUserName, text, user);
} catch (sendErr) {
logger.error?.('WeChat MP page data failure notice failed:', sendErr);
}
}
throw markWechatUserNotified(err instanceof Error ? err : new Error(message));
}
const mayBeStaleSession = sessionId && isRecoverableWechatAgentSessionError(message);
if (mayBeStaleSession) {
route = await ensureWechatAgentSession({
+10
View File
@@ -16,6 +16,7 @@ import {
loadWechatMpConfig,
maybeAttachPublishedHtmlLink,
shouldRetryHtmlGenerationReply,
shouldForceNewWechatAgentSession,
splitWechatText,
verifyWechatMpSignature,
verifyWechatMpUrlChallenge,
@@ -144,6 +145,15 @@ test('splitWechatText respects WeChat 2048-byte customer service limit', () => {
assert.equal(chunks.join(''), longChinese);
});
test('Page Data requests always rotate away from an existing WeChat route', () => {
assert.equal(
shouldForceNewWechatAgentSession({ kind: 'chat.general' }, '做一个问卷,后台可以查看提交数据'),
true,
);
assert.equal(shouldForceNewWechatAgentSession({ kind: 'chat.general' }, '继续聊德川家康'), false);
assert.equal(shouldForceNewWechatAgentSession({ kind: 'session.reset' }, '继续聊德川家康'), true);
});
test('buildWechatAgentPrompt requires docx generation before html when Word download is requested', () => {
const prompt = buildWechatAgentPrompt({
msgType: 'text',