From 73b308af0a4cb795114f743770a405f8c33f999b Mon Sep 17 00:00:00 2001 From: john Date: Sun, 12 Jul 2026 15:30:52 +0800 Subject: [PATCH] fix(page-data): enforce full workspace bind delivery and repair tooling Prevent false-positive Finish Guard passes and sync-created fake bindings by assessing publication, policy, dataset, injection, and insert smoke before H5 delivery. Co-authored-by: Cursor --- mindspace-page-data-finish-guard.mjs | 203 +++++++++++-- mindspace-page-data-finish-guard.test.mjs | 4 +- mindspace-page-sync.mjs | 5 + package.json | 5 +- page-data-delivery-assess.mjs | 266 ++++++++++++++++++ page-data-delivery-assess.test.mjs | 30 ++ page-data-html-detect.mjs | 22 ++ scripts/ensure-page-data-datasets.mjs | 112 ++++++++ scripts/release-portal-runtime-prod.sh | 1 + .../repair-page-data-workspace-bindings.mjs | 154 ++++++++++ scripts/scenario-test-lib.mjs | 8 +- wechat-mp.mjs | 42 ++- 12 files changed, 819 insertions(+), 33 deletions(-) create mode 100644 page-data-delivery-assess.mjs create mode 100644 page-data-delivery-assess.test.mjs create mode 100644 scripts/ensure-page-data-datasets.mjs create mode 100644 scripts/repair-page-data-workspace-bindings.mjs diff --git a/mindspace-page-data-finish-guard.mjs b/mindspace-page-data-finish-guard.mjs index 4fb93cf..e96e864 100644 --- a/mindspace-page-data-finish-guard.mjs +++ b/mindspace-page-data-finish-guard.mjs @@ -2,11 +2,22 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { isPageDataIntent } from './chat-skills.mjs'; -import { detectPageDataDatasetUsageFromHtml } from './page-data-html-detect.mjs'; +import { + detectPageDataDatasetUsageFromHtml, + htmlUsesPageDataApi, + inferPageDataBindAccessMode, +} from './page-data-html-detect.mjs'; import { bindWorkspaceHtmlForPageData } from './page-data-workspace-bind.mjs'; -import { listPageAccessPolicies } from './page-data-policy-store.mjs'; +import { createPageService } from './mindspace-pages.mjs'; +import { + assessPageDataHtmlBinding, + assessWorkspacePageDataReadiness, + verifyPageDataDeliveryArtifacts, +} from './page-data-delivery-assess.mjs'; import { buildPublicUrl, resolvePublicBaseUrl } from './user-publish.mjs'; +export { inferPageDataBindAccessMode }; + const PUBLICATION_ROUTE_LINK_PATTERN = /https?:\/\/[^\s<>"')\]]+\/u\/([0-9a-f-]{36}|[a-z0-9._-]+)\/pages\/([^\s<>"')\]]+)/gi; @@ -143,26 +154,46 @@ export function collectPageDataPublicHtmlFiles(publishDir) { .filter((file) => file.evaluation.usesPageDataApi); } -export function inferPageDataBindAccessMode(relativePath, html) { - const usage = detectPageDataDatasetUsageFromHtml(html); - const hasRead = [...usage.values()].some((item) => item.read); - const hasInsert = [...usage.values()].some((item) => item.insert); - if (/-admin\.html$/i.test(String(relativePath ?? '')) || (hasRead && !hasInsert)) { - return 'password'; - } - return 'public'; +function isPageDataHtmlWorkspaceReady({ publishDir, relativePath, html }) { + return assessWorkspacePageDataReadiness({ publishDir, relativePath, html }).ready; } -function pageHasBoundPolicy({ publishDir, relativePath, html }) { - const usage = detectPageDataDatasetUsageFromHtml(html); - if (!usage.size) return true; - const policies = listPageAccessPolicies(publishDir); - if (!policies.length) return false; - const htmlDatasetNames = [...usage.keys()].sort().join(','); - return policies.some((policy) => { - const policyNames = Object.keys(policy?.datasets ?? {}).sort().join(','); - return policyNames === htmlDatasetNames; - }); +async function collectUnboundPageDataFiles({ + relevantFiles, + publishDir, + pool = null, + userId = null, + findPageByRelativePath = null, +}) { + const unboundFiles = []; + for (const file of relevantFiles) { + if (file.evaluation.usage?.size === 0) continue; + if (pool && userId && typeof findPageByRelativePath === 'function') { + const assessment = await assessPageDataHtmlBinding({ + pool, + userId, + publishDir, + relativePath: file.relativePath, + html: file.content, + findPageByRelativePath, + }); + if (!assessment.bound) { + unboundFiles.push({ + ...file, + bindReasons: assessment.reasons, + }); + } + continue; + } + if (!isPageDataHtmlWorkspaceReady({ + publishDir, + relativePath: file.relativePath, + html: file.content, + })) { + unboundFiles.push(file); + } + } + return unboundFiles; } export function usedPageDataCollectSkill(messages = []) { @@ -231,7 +262,7 @@ export function evaluatePageDataFinishGuard({ const unboundFiles = relevantFiles.filter( (file) => file.evaluation.usage?.size > 0 && - !pageHasBoundPolicy({ + !isPageDataHtmlWorkspaceReady({ publishDir, relativePath: file.relativePath, html: file.content, @@ -255,6 +286,43 @@ export function evaluatePageDataFinishGuard({ }; } +export async function evaluatePageDataFinishGuardAsync({ + publishDir, + agentText = '', + messages = [], + requestStartedAt = 0, + pool = null, + userId = null, + findPageByRelativePath = null, +} = {}) { + const base = evaluatePageDataFinishGuard({ + publishDir, + agentText, + messages, + requestStartedAt, + }); + const unboundFiles = await collectUnboundPageDataFiles({ + relevantFiles: base.relevantFiles, + publishDir, + pool, + userId, + findPageByRelativePath, + }); + const needsRepair = + base.pageDataIntent && + (base.htmlIssues.length > 0 || + unboundFiles.length > 0 || + (base.relevantFiles.length === 0 && + extractRecentPageDataHtmlWrites(messages, { sinceMs: requestStartedAt }).length === 0 && + usedPageDataCollectSkill(messages))); + + return { + ...base, + unboundFiles, + needsRepair, + }; +} + export function buildPageDataCollectFailureText() { return [ '这次问卷/数据收集页面没有完成 Page Data API 绑定，所以我先不发链接。', @@ -284,7 +352,10 @@ export function buildPageDataCollectRepairPrompt({ if (unboundFiles.length) { lines.push('', '尚未 bind 的 Page Data 页面：'); for (const file of unboundFiles) { - lines.push(`- ${file.relativePath}`); + const reasons = Array.isArray(file.bindReasons) && file.bindReasons.length + ? ` (${file.bindReasons.join(', ')})` + : ''; + lines.push(`- ${file.relativePath}${reasons}`); } } lines.push('', '修复完成前不要告诉用户“已发布/已可提交”。'); @@ -354,6 +425,42 @@ export function resolvePageDataCollectOutcome({ return { action: 'send', evaluation }; } +export async function resolvePageDataCollectOutcomeAsync({ + reply, + intent, + publishDir, + requestStartedAt = 0, + pool = null, + userId = null, + findPageByRelativePath = null, +} = {}) { + const agentText = String(intent?.agentText ?? intent?.displayText ?? '').trim(); + if (!isPageDataIntent(agentText)) { + return { action: 'skip' }; + } + + const evaluation = await evaluatePageDataFinishGuardAsync({ + publishDir, + agentText, + messages: reply?.messages ?? [], + requestStartedAt, + pool, + userId, + findPageByRelativePath, + }); + + if (evaluation.htmlIssues.length > 0) { + return { action: 'fail', failureText: buildPageDataCollectFailureText(), reason: 'invalid_html', evaluation }; + } + if (evaluation.unboundFiles.length > 0) { + return { action: 'retry', reason: 'missing_bind', evaluation }; + } + if (shouldRetryPageDataCollectReply({ reply, intent, publishDir, requestStartedAt })) { + return { action: 'retry', reason: 'incomplete_delivery', evaluation }; + } + return { action: 'send', evaluation }; +} + export function isPageDataFinishGuardEnabled(env = process.env) { return envFlag(env.MEMIND_PAGE_DATA_FINISH_GUARD, true); } @@ -370,11 +477,17 @@ export async function maybeAutoBindPageDataHtmlPages({ h5Root, storageRoot, onlyRelativePaths = null, + findPageByRelativePath = null, } = {}) { if (!pool) { return { bound: [], skipped: [], errors: [{ code: 'database_unconfigured' }] }; } + const pageLookup = + typeof findPageByRelativePath === 'function' + ? findPageByRelativePath + : createPageService(pool, { h5Root, storageRoot }).findPageByRelativePath; + const bound = []; const skipped = []; const errors = []; @@ -390,11 +503,15 @@ export async function maybeAutoBindPageDataHtmlPages({ skipped.push({ relativePath: file.relativePath, reason: 'no_dataset_usage' }); continue; } - if (pageHasBoundPolicy({ + const assessment = await assessPageDataHtmlBinding({ + pool, + userId, publishDir, relativePath: file.relativePath, html: file.content, - })) { + findPageByRelativePath: pageLookup, + }); + if (assessment.bound) { skipped.push({ relativePath: file.relativePath, reason: 'already_bound' }); continue; } @@ -424,6 +541,32 @@ export async function maybeAutoBindPageDataHtmlPages({ return { bound, skipped, errors }; } +export async function ensurePageDataDeliveryReady({ + publishDir, + userId, + pool, + h5Root, + storageRoot, + apiBase, + artifacts = [], + fetchImpl = fetch, +}) { + if (!pool || !userId) { + return { ok: false, failures: [{ stage: 'binding', reason: 'database_unconfigured' }] }; + } + const pageService = createPageService(pool, { h5Root, storageRoot }); + const failures = await verifyPageDataDeliveryArtifacts({ + artifacts, + publishDir, + apiBase, + pool, + userId, + findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService), + fetchImpl, + }); + return { ok: failures.length === 0, failures }; +} + export async function maybeRepairPageDataAfterFinish({ sessionId, userId, @@ -439,10 +582,14 @@ export async function maybeRepairPageDataAfterFinish({ userText = '', } = {}) { const recentUserText = String(userText ?? '').trim(); - const evaluation = evaluatePageDataFinishGuard({ + const pageService = pool ? createPageService(pool, { h5Root, storageRoot }) : null; + const evaluation = await evaluatePageDataFinishGuardAsync({ publishDir, agentText: recentUserText, messages, + pool, + userId, + findPageByRelativePath: pageService?.findPageByRelativePath?.bind(pageService) ?? null, }); if (!evaluation.pageDataIntent && evaluation.relevantFiles.length === 0) { @@ -457,12 +604,16 @@ export async function maybeRepairPageDataAfterFinish({ h5Root, storageRoot, onlyRelativePaths: evaluation.relevantFiles.map((file) => file.relativePath), + findPageByRelativePath: pageService?.findPageByRelativePath?.bind(pageService) ?? null, }); - const afterBind = evaluatePageDataFinishGuard({ + const afterBind = await evaluatePageDataFinishGuardAsync({ publishDir, agentText: recentUserText, messages, + pool, + userId, + findPageByRelativePath: pageService?.findPageByRelativePath?.bind(pageService) ?? null, }); if (!afterBind.needsRepair) { diff --git a/mindspace-page-data-finish-guard.test.mjs b/mindspace-page-data-finish-guard.test.mjs index 037680f..5205b92 100644 --- a/mindspace-page-data-finish-guard.test.mjs +++ b/mindspace-page-data-finish-guard.test.mjs @@ -114,7 +114,7 @@ test('maybeAutoBindPageDataHtmlPages skips invalid html', async () => { } }); -test('evaluatePageDataFinishGuard passes when policy already exists', () => { +test('evaluatePageDataFinishGuard flags html when dataset is not registered', () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'page-data-guard-bound-')); try { fs.mkdirSync(path.join(publishDir, 'public'), { recursive: true }); @@ -136,7 +136,7 @@ test('evaluatePageDataFinishGuard passes when policy already exists', () => { agentText: '调查问卷和后台', messages: [], }); - assert.equal(evaluation.unboundFiles.length, 0); + assert.equal(evaluation.unboundFiles.length, 1); assert.equal(evaluation.htmlIssues.length, 0); } finally { fs.rmSync(publishDir, { recursive: true, force: true }); diff --git a/mindspace-page-sync.mjs b/mindspace-page-sync.mjs index e5063b4..5159b48 100644 --- a/mindspace-page-sync.mjs +++ b/mindspace-page-sync.mjs @@ -1,5 +1,6 @@ import fs from 'node:fs/promises'; import path from 'node:path'; +import { htmlUsesPageDataApi } from './page-data-html-detect.mjs'; import { normalizeWorkspaceRelativePath } from './mindspace-pages.mjs'; const PUBLIC_HTML_SKIP_DIRS = new Set([ @@ -235,6 +236,10 @@ export async function syncGeneratedPagesFromPublicAssets({ skipped += 1; continue; } + if (htmlUsesPageDataApi(content)) { + skipped += 1; + continue; + } const result = await upsertWorkspaceHtmlPage({ pool, pageService, diff --git a/package.json b/package.json index 4b86ab7..f8ed31c 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,10 @@ "verify:mindspace-publish-guards": "node scripts/verify-mindspace-publish-guards.mjs", "verify:mindspace-publish-guards:full": "node scripts/verify-mindspace-publish-guards.mjs --with-runtime", "verify:mindspace-page-sync-guards": "node scripts/verify-mindspace-page-sync-guards.mjs", - "verify:page-data": "node --test page-data-acceptance.test.mjs page-data-integration.test.mjs page-data-public-service.test.mjs page-data-ops.test.mjs mindspace-page-data-finish-guard.test.mjs mindspace-page-data-finish-guard.integration.test.mjs", + "verify:page-data": "node --test page-data-acceptance.test.mjs page-data-integration.test.mjs page-data-public-service.test.mjs page-data-ops.test.mjs page-data-delivery-assess.test.mjs mindspace-page-data-finish-guard.test.mjs mindspace-page-data-finish-guard.integration.test.mjs", + "verify:page-data-delivery": "node scripts/repair-page-data-workspace-bindings.mjs --dry-run", + "repair:page-data-bindings": "node scripts/repair-page-data-workspace-bindings.mjs", + "repair:page-data:103": "node scripts/ensure-page-data-datasets.mjs && node scripts/repair-page-data-workspace-bindings.mjs", "verify:wechat-channel-isolation": "node scripts/check-wechat-channel-isolation.mjs", "verify:goosed-proxy-boundary": "node scripts/check-goosed-proxy-boundary.mjs", "verify:h5-session-patches": "node scripts/verify-h5-session-patches.mjs", diff --git a/page-data-delivery-assess.mjs b/page-data-delivery-assess.mjs new file mode 100644 index 0000000..11296de --- /dev/null +++ b/page-data-delivery-assess.mjs @@ -0,0 +1,266 @@ +import fs from 'node:fs/promises'; +import { + detectPageDataDatasetUsageFromHtml, + inferPageDataBindAccessMode, +} from './page-data-html-detect.mjs'; +import { listPageAccessPolicies, readPageAccessPolicy } from './page-data-policy-store.mjs'; +import { createUserDataSpaceService } from './user-data-space-service.mjs'; + +const INJECTION_PAGE_ID_PATTERN = + /window\.__MINDSPACE_PAGE_DATA__\s*=\s*\{[^}]*"pageId"\s*:\s*"([^"]+)"/i; +const INJECTION_META_PAGE_ID_PATTERN = + /]+name=["']mindspace-page-data-page-id["'][^>]+content=["']([^"']+)["']/i; + +export function extractInjectedPageIdFromHtml(html) { + const content = String(html ?? ''); + const fromScript = content.match(INJECTION_PAGE_ID_PATTERN)?.[1]?.trim(); + if (fromScript) return fromScript; + return content.match(INJECTION_META_PAGE_ID_PATTERN)?.[1]?.trim() ?? null; +} + +export function buildSmokeInsertRow(policy, datasetName) { + const columns = policy?.datasets?.[datasetName]?.columns?.insert ?? []; + const row = {}; + for (const column of columns) { + if (/rating|score|星级|评分/i.test(column)) { + row[column] = 3; + } else if (/phone|手机|电话/i.test(column)) { + row[column] = '13800000000'; + } else { + row[column] = 'smoke-test'; + } + } + return row; +} + +export async function queryOnlinePublication(pool, pageId) { + if (!pool || !pageId) return null; + const [rows] = await pool.query( + `SELECT id, page_id, status, access_mode + FROM h5_publish_records + WHERE page_id = ? AND status = 'online' + ORDER BY published_at DESC + LIMIT 1`, + [pageId], + ); + return rows[0] ?? null; +} + +function findWorkspacePolicyForHtml({ publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const htmlDatasetNames = [...usage.keys()].sort().join(','); + const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html); + for (const policy of listPageAccessPolicies(publishDir)) { + const policyNames = Object.keys(policy?.datasets ?? {}).sort().join(','); + if (policyNames !== htmlDatasetNames) continue; + if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) continue; + let matches = true; + for (const [datasetName, perms] of usage) { + const policyDataset = policy.datasets?.[datasetName]; + if (!policyDataset) { + matches = false; + break; + } + if (perms.insert && !policyDataset.insert) matches = false; + if (perms.read && !policyDataset.read) matches = false; + } + if (matches) return policy; + } + return null; +} + +export function assessWorkspacePageDataReadiness({ publishDir, relativePath, html }) { + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) { + return { ready: true, reasons: [] }; + } + + const reasons = []; + const dataSpace = createUserDataSpaceService({ workspaceRoot: publishDir }); + for (const datasetName of usage.keys()) { + if (!dataSpace.getDataset(datasetName)) { + reasons.push(`dataset_not_registered:${datasetName}`); + } + } + + const policy = findWorkspacePolicyForHtml({ publishDir, relativePath, html }); + if (!policy) { + reasons.push('missing_workspace_policy'); + } + + return { ready: reasons.length === 0, reasons, policy }; +} + +export async function assessPageDataHtmlBinding({ + pool, + userId, + publishDir, + relativePath, + html, + findPageByRelativePath, +}) { + const usage = detectPageDataDatasetUsageFromHtml(html); + if (!usage.size) { + return { bound: true, reasons: [], pageId: null }; + } + + const workspace = assessWorkspacePageDataReadiness({ publishDir, relativePath, html }); + const reasons = [...workspace.reasons]; + + let pageId = null; + if (typeof findPageByRelativePath === 'function') { + const page = await findPageByRelativePath(userId, relativePath).catch(() => null); + pageId = page?.id ? String(page.id).trim() : null; + if (!pageId) { + reasons.push('missing_page_record'); + } + } else { + reasons.push('missing_page_lookup'); + } + + if (pageId) { + const policy = readPageAccessPolicy(publishDir, pageId); + if (!policy) { + reasons.push('missing_policy_for_page'); + } else { + const htmlDatasetNames = [...usage.keys()].sort().join(','); + const policyNames = Object.keys(policy.datasets ?? {}).sort().join(','); + if (htmlDatasetNames !== policyNames) { + reasons.push('policy_dataset_mismatch'); + } + const expectedAccessMode = inferPageDataBindAccessMode(relativePath, html); + if (String(policy.accessMode ?? '').trim() !== expectedAccessMode) { + reasons.push('policy_access_mode_mismatch'); + } + for (const [datasetName, perms] of usage) { + const policyDataset = policy.datasets?.[datasetName]; + if (!policyDataset) continue; + if (perms.insert && !policyDataset.insert) reasons.push('missing_insert_permission'); + if (perms.read && !policyDataset.read) reasons.push('missing_read_permission'); + } + } + + const publication = await queryOnlinePublication(pool, pageId); + if (!publication) { + reasons.push('missing_online_publication'); + } + } + + return { + bound: reasons.length === 0, + reasons: [...new Set(reasons)], + pageId, + policy: pageId ? readPageAccessPolicy(publishDir, pageId) : workspace.policy ?? null, + }; +} + +export async function fetchWorkspaceHtmlInjection({ url, fetchImpl = fetch }) { + const response = await fetchImpl(String(url)); + if (!response.ok) { + return { + ok: false, + reason: `http_${response.status}`, + pageId: null, + html: '', + }; + } + const html = await response.text(); + const pageId = extractInjectedPageIdFromHtml(html); + if (!pageId) { + return { ok: false, reason: 'missing_page_data_injection', pageId: null, html }; + } + return { ok: true, reason: null, pageId, html }; +} + +export async function smokeTestPageDataInsert({ + apiBase, + pageId, + dataset, + policy, + fetchImpl = fetch, +}) { + const base = String(apiBase ?? '').replace(/\/$/, '') || '/api'; + const row = buildSmokeInsertRow(policy, dataset); + if (!pageId || !dataset || !Object.keys(row).length) { + return { ok: false, reason: 'smoke_payload_unavailable', status: 0, body: null }; + } + + const response = await fetchImpl( + `${base}/public/pages/${encodeURIComponent(pageId)}/data/${encodeURIComponent(dataset)}/rows`, + { + method: 'POST', + headers: { 'content-type': 'application/json', accept: 'application/json' }, + body: JSON.stringify(row), + }, + ); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + return { + ok: false, + reason: body?.error?.code ?? 'insert_failed', + status: response.status, + body, + }; + } + return { ok: true, reason: null, status: response.status, body }; +} + +export async function verifyPageDataDeliveryArtifacts({ + artifacts = [], + publishDir, + apiBase, + pool, + userId, + findPageByRelativePath, + fetchImpl = fetch, +}) { + const failures = []; + for (const artifact of artifacts) { + if (artifact.isAdmin) continue; + const relativePath = artifact.relativePath; + const html = await fs.readFile(artifact.localPath, 'utf8').catch(() => ''); + const usage = detectPageDataDatasetUsageFromHtml(html); + const insertDataset = [...usage.entries()].find(([, perms]) => perms.insert)?.[0] ?? null; + if (!insertDataset) continue; + + const injection = await fetchWorkspaceHtmlInjection({ url: artifact.url, fetchImpl }); + if (!injection.ok) { + failures.push({ relativePath, stage: 'injection', reason: injection.reason }); + continue; + } + + const assessment = await assessPageDataHtmlBinding({ + pool, + userId, + publishDir, + relativePath, + html, + findPageByRelativePath, + }); + if (!assessment.bound) { + failures.push({ + relativePath, + stage: 'binding', + reason: assessment.reasons.join(','), + }); + continue; + } + + const smoke = await smokeTestPageDataInsert({ + apiBase, + pageId: assessment.pageId, + dataset: insertDataset, + policy: assessment.policy, + fetchImpl, + }); + if (!smoke.ok) { + failures.push({ + relativePath, + stage: 'insert_smoke', + reason: smoke.reason, + status: smoke.status, + }); + } + } + return failures; +} diff --git a/page-data-delivery-assess.test.mjs b/page-data-delivery-assess.test.mjs new file mode 100644 index 0000000..e7c2feb --- /dev/null +++ b/page-data-delivery-assess.test.mjs @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + buildSmokeInsertRow, + extractInjectedPageIdFromHtml, +} from './page-data-delivery-assess.mjs'; + +test('extractInjectedPageIdFromHtml reads injected pageId', () => { + const html = ` + + +`; + assert.equal(extractInjectedPageIdFromHtml(html), 'page-abc'); +}); + +test('buildSmokeInsertRow fills insert columns', () => { + const row = buildSmokeInsertRow( + { + datasets: { + dining_survey: { + columns: { insert: ['q1_frequency', 'overall_rating', 'phone'] }, + }, + }, + }, + 'dining_survey', + ); + assert.equal(row.q1_frequency, 'smoke-test'); + assert.equal(row.overall_rating, 3); + assert.equal(row.phone, '13800000000'); +}); diff --git a/page-data-html-detect.mjs b/page-data-html-detect.mjs index ae3454b..944372e 100644 --- a/page-data-html-detect.mjs +++ b/page-data-html-detect.mjs @@ -2,6 +2,18 @@ * 从 MindSpace 公开 HTML 中解析 Page Data API 引用的 dataset 与读写意图。 */ +const PAGE_DATA_CLIENT_SCRIPT_PATTERN = /\/assets\/page-data-client\.js/i; + +export function htmlUsesPageDataApi(html) { + const content = String(html ?? ''); + const usage = detectPageDataDatasetUsageFromHtml(content); + return ( + usage.size > 0 || + PAGE_DATA_CLIENT_SCRIPT_PATTERN.test(content) || + /\bMindSpacePageData\b/.test(content) + ); +} + export function detectPageDataDatasetUsageFromHtml(html) { const text = String(html ?? ''); const datasets = new Map(); @@ -20,6 +32,16 @@ export function detectPageDataDatasetUsageFromHtml(html) { return datasets; } +export function inferPageDataBindAccessMode(relativePath, html) { + const usage = detectPageDataDatasetUsageFromHtml(html); + const hasRead = [...usage.values()].some((item) => item.read); + const hasInsert = [...usage.values()].some((item) => item.insert); + if (/-admin\.html$/i.test(String(relativePath ?? '')) || (hasRead && !hasInsert)) { + return 'password'; + } + return 'public'; +} + export function assertPolicyMatchesHtmlDatasets(html, policyDatasets) { const usage = detectPageDataDatasetUsageFromHtml(html); const htmlNames = [...usage.keys()].sort(); diff --git a/scripts/ensure-page-data-datasets.mjs b/scripts/ensure-page-data-datasets.mjs new file mode 100644 index 0000000..8645633 --- /dev/null +++ b/scripts/ensure-page-data-datasets.mjs @@ -0,0 +1,112 @@ +#!/usr/bin/env node +/** + * 为已知破损问卷补注册 dataset（表已存在但 __page_data_datasets 未登记的场景）。 + */ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { loadH5Environment } from './load-env.mjs'; +import { createUserDataSpaceService } from '../user-data-space-service.mjs'; + +const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +loadH5Environment(import.meta.dirname); + +const PRESETS = { + 'a70ff537-8908-486e-9b6c-042e07cc25db': [ + { + name: 'company_suggestions', + table: 'company_suggestions', + description: '公司建议问卷调查', + actions: ['read', 'insert'], + columns: { + read: [ + 'id', + 'department', + 'q_work_env', + 'q_management', + 'q_teamwork', + 'q_salary_welfare', + 'q_dev_opportunity', + 'q_other_suggestion', + 'overall_rating', + 'created_at', + ], + insert: [ + 'department', + 'q_work_env', + 'q_management', + 'q_teamwork', + 'q_salary_welfare', + 'q_dev_opportunity', + 'q_other_suggestion', + 'overall_rating', + ], + }, + }, + { + name: 'diet_survey', + table: 'diet_survey', + description: '儿童饮食偏好调查', + actions: ['read', 'insert'], + columns: { + read: ['id', 'child_age', 'veggie_habit', 'snack_type', 'created_at'], + insert: ['child_age', 'veggie_habit', 'snack_type'], + }, + }, + ], + 'a6fb1e97-2b0f-447b-b138-4561d8e5c53e': [ + { + name: 'dining_survey', + table: 'dining_survey', + description: '餐饮偏好调查问卷', + sql: `CREATE TABLE IF NOT EXISTS dining_survey ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + q1_frequency TEXT NOT NULL DEFAULT '', + q2_priority TEXT NOT NULL DEFAULT '', + q3_cuisine TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (datetime('now', '+8 hours')) + );`, + actions: ['read', 'insert'], + columns: { + read: ['id', 'q1_frequency', 'q2_priority', 'q3_cuisine', 'created_at'], + insert: ['q1_frequency', 'q2_priority', 'q3_cuisine'], + }, + }, + ], +}; + +async function main() { + const userIds = process.argv.slice(2); + const targets = userIds.length ? userIds : Object.keys(PRESETS); + for (const userId of targets) { + const presets = PRESETS[userId]; + if (!presets) { + console.warn(`跳过 ${userId}：无 preset`); + continue; + } + const workspaceRoot = path.join(repoRoot, 'MindSpace', userId); + const dataSpace = createUserDataSpaceService({ workspaceRoot }); + for (const preset of presets) { + if (preset.sql) { + await dataSpace.executeSql(preset.sql); + console.log(`[${userId}] 已确保表 ${preset.table}`); + } + if (!dataSpace.getDataset(preset.name)) { + await dataSpace.upsertDataset({ + name: preset.name, + table: preset.table, + description: preset.description, + actions: preset.actions, + columns: preset.columns, + }); + console.log(`[${userId}] 已注册 dataset ${preset.name}`); + } else { + console.log(`[${userId}] dataset ${preset.name} 已存在`); + } + } + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + process.exit(1); +}); diff --git a/scripts/release-portal-runtime-prod.sh b/scripts/release-portal-runtime-prod.sh index 1f7d51c..55ba606 100755 --- a/scripts/release-portal-runtime-prod.sh +++ b/scripts/release-portal-runtime-prod.sh @@ -133,6 +133,7 @@ if [[ "${SKIP_TESTS}" -ne 1 ]]; then cd "${ROOT}" npm test -- --test-name-pattern='publish|space|billing' >/dev/null npm run verify:mindspace-publish-guards >/dev/null + npm run verify:page-data >/dev/null ) fi diff --git a/scripts/repair-page-data-workspace-bindings.mjs b/scripts/repair-page-data-workspace-bindings.mjs new file mode 100644 index 0000000..dc1e116 --- /dev/null +++ b/scripts/repair-page-data-workspace-bindings.mjs @@ -0,0 +1,154 @@ +#!/usr/bin/env node +/** + * 扫描并补绑工作区中未完整交付的 Page Data 问卷页（publication + policy + dataset）。 + * + * Usage: + * node scripts/repair-page-data-workspace-bindings.mjs + * node scripts/repair-page-data-workspace-bindings.mjs --user-id + * node scripts/repair-page-data-workspace-bindings.mjs --dry-run + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { loadH5Environment } from './load-env.mjs'; +import { createDbPool } from '../db.mjs'; +import { buildPublicUrl, PUBLISH_ROOT_DIR, resolvePublicBaseUrl } from '../user-publish.mjs'; +import { resolveMindSpaceStorageRoot } from '../mindspace-runtime-config.mjs'; +import { createPageService } from '../mindspace-pages.mjs'; +import { assessPageDataHtmlBinding } from '../page-data-delivery-assess.mjs'; +import { + collectPageDataPublicHtmlFiles, + maybeAutoBindPageDataHtmlPages, +} from '../mindspace-page-data-finish-guard.mjs'; + +const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); +loadH5Environment(import.meta.dirname); + +function parseArgs(argv) { + const options = { userId: null, dryRun: false }; + for (let i = 2; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--dry-run') options.dryRun = true; + else if (arg === '--user-id' && argv[i + 1]) options.userId = argv[++i]; + else if (arg === '-h' || arg === '--help') { + console.log('Usage: node scripts/repair-page-data-workspace-bindings.mjs [--user-id ] [--dry-run]'); + process.exit(0); + } else { + throw new Error(`未知参数: ${arg}`); + } + } + return options; +} + +function listWorkspaceUserIds(mindspaceRoot) { + if (!fs.existsSync(mindspaceRoot)) return []; + return fs + .readdirSync(mindspaceRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && /^[0-9a-f-]{36}$/i.test(entry.name)) + .map((entry) => entry.name); +} + +async function assessUserWorkspace({ userId, pool, h5Root, storageRoot, publicBaseUrl }) { + const publishDir = path.join(h5Root, 'MindSpace', userId); + if (!fs.existsSync(publishDir)) return []; + const pageService = createPageService(pool, { h5Root, storageRoot }); + const broken = []; + for (const file of collectPageDataPublicHtmlFiles(publishDir)) { + if (file.evaluation.usage?.size === 0) continue; + const assessment = await assessPageDataHtmlBinding({ + pool, + userId, + publishDir, + relativePath: file.relativePath, + html: file.content, + findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService), + }); + if (!assessment.bound) { + broken.push({ + userId, + relativePath: file.relativePath, + url: buildPublicUrl(publicBaseUrl, userId, file.relativePath), + reasons: assessment.reasons, + pageId: assessment.pageId, + }); + } + } + return broken; +} + +async function main() { + const options = parseArgs(process.argv); + const pool = createDbPool(); + const h5Root = repoRoot; + const storageRoot = resolveMindSpaceStorageRoot(h5Root); + const publicBaseUrl = resolvePublicBaseUrl(); + const userIds = options.userId ? [options.userId] : listWorkspaceUserIds(path.join(h5Root, 'MindSpace')); + + const allBroken = []; + for (const userId of userIds) { + const broken = await assessUserWorkspace({ + userId, + pool, + h5Root, + storageRoot, + publicBaseUrl, + }); + allBroken.push(...broken); + } + + if (!allBroken.length) { + console.log('未发现需要补绑的 Page Data 页面。'); + await pool.end(); + return; + } + + console.log(`发现 ${allBroken.length} 个未完整绑定的 Page Data 页面：`); + for (const item of allBroken) { + console.log(`- [${item.userId}] ${item.relativePath}`); + console.log(` url: ${item.url}`); + console.log(` reasons: ${item.reasons.join(', ')}`); + } + + if (options.dryRun) { + console.log('\n(dry-run) 未执行补绑。'); + await pool.end(); + return; + } + + const byUser = new Map(); + for (const item of allBroken) { + const list = byUser.get(item.userId) ?? []; + list.push(item.relativePath); + byUser.set(item.userId, list); + } + + for (const [userId, relativePaths] of byUser.entries()) { + const publishDir = path.join(h5Root, 'MindSpace', userId); + const pageService = createPageService(pool, { h5Root, storageRoot }); + const result = await maybeAutoBindPageDataHtmlPages({ + pool, + userId, + publishDir, + h5Root, + storageRoot, + onlyRelativePaths: relativePaths, + findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService), + }); + console.log( + `\n[user ${userId}] bound=${result.bound.length} skipped=${result.skipped.length} errors=${result.errors.length}`, + ); + for (const item of result.bound) { + console.log(` ✓ ${item.relativePath} -> ${item.pageId}`); + } + for (const item of result.errors) { + console.log(` ✗ ${item.relativePath}: ${item.message}`); + } + } + + await pool.end(); +} + +main().catch(async (error) => { + console.error(error instanceof Error ? error.stack ?? error.message : error); + process.exit(1); +}); diff --git a/scripts/scenario-test-lib.mjs b/scripts/scenario-test-lib.mjs index e0db538..ad33a67 100644 --- a/scripts/scenario-test-lib.mjs +++ b/scripts/scenario-test-lib.mjs @@ -579,8 +579,14 @@ export async function verifyChildrenHobbyDietSurvey({ if (response.status !== 200) { reporter.fail(label, `${response.status} ${url}`); ok = false; + continue; + } + const html = await response.text(); + if (!html.includes('__MINDSPACE_PAGE_DATA__')) { + reporter.fail(`${label} Page Data 注入`, '响应缺少 __MINDSPACE_PAGE_DATA__'); + ok = false; } else { - reporter.pass(label, url); + reporter.pass(`${label} Page Data 注入`, url); } } diff --git a/wechat-mp.mjs b/wechat-mp.mjs index 7d91a63..fe066ce 100644 --- a/wechat-mp.mjs +++ b/wechat-mp.mjs @@ -32,8 +32,9 @@ import { resolveBillingTokenState } from './billing-token-state.mjs'; import { buildPageDataCollectFailureText, buildPageDataDeliveryArtifactsFromBindResult, + ensurePageDataDeliveryReady, maybeAutoBindPageDataHtmlPages, - resolvePageDataCollectOutcome, + resolvePageDataCollectOutcomeAsync, rewritePageDataDeliveryLinks, } from './mindspace-page-data-finish-guard.mjs'; @@ -1056,28 +1057,40 @@ async function enforcePageDataCollectDelivery({ requestStartedAt = 0, notifyFailure, }) { - let outcome = resolvePageDataCollectOutcome({ + let outcome = await resolvePageDataCollectOutcomeAsync({ reply, intent, publishDir: workingDir, requestStartedAt, + pool: pageDataFinishGuard?.pool ?? null, + userId, + findPageByRelativePath: null, }); let autoBind = null; if (outcome.action === 'skip') return outcome; if (pageDataFinishGuard?.pool) { + const { createPageService } = await import('./mindspace-pages.mjs'); + const pageService = createPageService(pageDataFinishGuard.pool, { + h5Root: pageDataFinishGuard.h5Root, + storageRoot: pageDataFinishGuard.storageRoot, + }); autoBind = await maybeAutoBindPageDataHtmlPages({ pool: pageDataFinishGuard.pool, userId, publishDir: workingDir, h5Root: pageDataFinishGuard.h5Root, storageRoot: pageDataFinishGuard.storageRoot, + findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService), }); - outcome = resolvePageDataCollectOutcome({ + outcome = await resolvePageDataCollectOutcomeAsync({ reply, intent, publishDir: workingDir, requestStartedAt, + pool: pageDataFinishGuard.pool, + userId, + findPageByRelativePath: pageService.findPageByRelativePath.bind(pageService), }); } @@ -1095,6 +1108,29 @@ async function enforcePageDataCollectDelivery({ const deliveryArtifacts = buildPageDataDeliveryArtifactsFromBindResult(autoBind, workingDir, { publicBaseUrl, }); + if (deliveryArtifacts.length > 0 && pageDataFinishGuard?.pool) { + const { createPageService } = await import('./mindspace-pages.mjs'); + const pageService = createPageService(pageDataFinishGuard.pool, { + h5Root: pageDataFinishGuard.h5Root, + storageRoot: pageDataFinishGuard.storageRoot, + }); + const deliveryCheck = await ensurePageDataDeliveryReady({ + publishDir: workingDir, + userId, + pool: pageDataFinishGuard.pool, + h5Root: pageDataFinishGuard.h5Root, + storageRoot: pageDataFinishGuard.storageRoot, + apiBase: publicBaseUrl, + artifacts: deliveryArtifacts, + }); + if (!deliveryCheck.ok) { + const text = buildPageDataCollectFailureText(); + if (typeof notifyFailure === 'function') { + await notifyFailure(text); + } + throw markWechatUserNotified(new Error(text)); + } + } if (deliveryArtifacts.length > 0 && reply && typeof reply.text === 'string') { reply.text = rewritePageDataDeliveryLinks(reply.text, deliveryArtifacts); }