diff --git a/docs/regression-guards/mindspace-seo-geo.md b/docs/regression-guards/mindspace-seo-geo.md index aeaeafa..b0439d7 100644 --- a/docs/regression-guards/mindspace-seo-geo.md +++ b/docs/regression-guards/mindspace-seo-geo.md @@ -4,8 +4,8 @@ ## 保护内容 -1. **收录策略硬规则**:仅 `access_mode=public` 且 `status=online` 且未过期的发布页允许 SEO/GEO 注入与 sitemap/llms 收录。不再要求 `user_confirmed_at`。 -2. **私有页强制 noindex**:密码、登录可见、owner_only、已过期、工作区预览直链等必须注入 `noindex, nofollow` 并设置 `X-Robots-Tag`。 +1. **收录策略硬规则**:`h5_publish_records` 中 `access_mode=public` 且 `status=online` 且未过期的发布页允许 SEO/GEO 注入与 sitemap/llms 收录;**此外**,工作区 `public/*.html` 直链在无在线非 public 发布记录阻挡时,默认视为可收录(implicit public)。不再要求 `user_confirmed_at`。 +2. **私有页强制 noindex**:在线 `password` / `private_link` / `login_required` / `owner_only` 发布、已过期、embed 模式等必须注入 `noindex, nofollow` 并设置 `X-Robots-Tag`。 3. **总开关默认开启**:`mindspace_config.seo_geo_config` 缺省为全开;库内已保存的旧值仍以数据库为准,需在 memind_adm MindSpace 配置页保存后才会改写生产。 4. **配置来源**:memind_adm MindSpace 配置页 → `PATCH /admin-api/mindspace/config` → Portal `loadMindSpaceConfigCached()`。 5. **百度推送**:仅在 admin 开启 `seo.baiduPush` 且页面可索引时触发;公开页发布与 Plaza 发帖共用该开关。 diff --git a/mindspace-index-policy.mjs b/mindspace-index-policy.mjs index 4d5f397..6503561 100644 --- a/mindspace-index-policy.mjs +++ b/mindspace-index-policy.mjs @@ -1,5 +1,36 @@ const INDEXABLE_ACCESS_MODE = 'public'; +export function isWorkspacePublicHtmlRelativePath(value) { + const normalized = String(value ?? '') + .replace(/\\/g, '/') + .replace(/^\/+/, ''); + if (!/^public\/[^/]+\.html$/i.test(normalized)) return false; + if (/\/_archived[^/]*\.html$/i.test(normalized)) return false; + return true; +} + +export function buildImplicitWorkspacePublicPublication({ + publicUrl = '', + pageId = null, + updatedAt = null, + title = null, + summary = null, +} = {}) { + const url = String(publicUrl ?? '').trim(); + if (!url) return null; + return normalizePublicationSnapshot({ + accessMode: INDEXABLE_ACCESS_MODE, + status: 'online', + publicUrl: url, + pageId, + userConfirmedAt: null, + expiresAt: null, + updatedAt: updatedAt ?? Date.now(), + title, + summary, + }); +} + export function normalizePublicationSnapshot(input = {}) { if (!input || typeof input !== 'object') return null; const accessMode = String( @@ -83,4 +114,5 @@ export function resolveIndexPolicy({ export const indexPolicyInternals = { INDEXABLE_ACCESS_MODE, + isWorkspacePublicHtmlRelativePath, }; diff --git a/mindspace-index-policy.test.mjs b/mindspace-index-policy.test.mjs index e1dd30f..2bbedb1 100644 --- a/mindspace-index-policy.test.mjs +++ b/mindspace-index-policy.test.mjs @@ -1,7 +1,9 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + buildImplicitWorkspacePublicPublication, isPublicationIndexable, + isWorkspacePublicHtmlRelativePath, resolveIndexPolicy, } from './mindspace-index-policy.mjs'; @@ -73,3 +75,25 @@ test('resolveIndexPolicy skips embed delivery', () => { }); assert.equal(policy.mode, 'off'); }); + +test('isWorkspacePublicHtmlRelativePath accepts public html and rejects archived', () => { + assert.equal(isWorkspacePublicHtmlRelativePath('public/demo.html'), true); + assert.equal(isWorkspacePublicHtmlRelativePath('public/_archived-demo.html'), false); + assert.equal(isWorkspacePublicHtmlRelativePath('private/demo.html'), false); +}); + +test('buildImplicitWorkspacePublicPublication marks workspace public html indexable', () => { + const snapshot = buildImplicitWorkspacePublicPublication({ + publicUrl: '/MindSpace/user/public/demo.html', + }); + assert.equal(isPublicationIndexable(snapshot), true); + const policy = resolveIndexPolicy({ + seoGeoConfig: { + enabled: true, + seo: { enabled: true, canonical: true }, + geo: { enabled: true, jsonLd: true }, + }, + publication: snapshot, + }); + assert.equal(policy.mode, 'indexable'); +}); diff --git a/mindspace-seo-discovery-service.mjs b/mindspace-seo-discovery-service.mjs index eb12202..c17d70b 100644 --- a/mindspace-seo-discovery-service.mjs +++ b/mindspace-seo-discovery-service.mjs @@ -1,5 +1,30 @@ -import { isPublicationIndexable, normalizePublicationSnapshot } from './mindspace-index-policy.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import { + buildImplicitWorkspacePublicPublication, + isPublicationIndexable, + isWorkspacePublicHtmlRelativePath, + normalizePublicationSnapshot, +} from './mindspace-index-policy.mjs'; +import { buildMindSpacePublicRoutePath } from './mindspace-runtime-config.mjs'; import { PLATFORM_STATIC_DISCOVERY_ENTRIES } from './platform-seo-html.mjs'; +import { PUBLISH_ROOT_DIR } from './user-publish.mjs'; + +function mapPublicationRow(row) { + if (!row) return null; + return normalizePublicationSnapshot({ + id: row.id, + pageId: row.page_id, + publicUrl: row.public_url, + accessMode: row.access_mode, + status: row.status, + userConfirmedAt: row.user_confirmed_at, + expiresAt: row.expires_at, + updatedAt: row.updated_at ?? row.published_at, + title: row.title, + summary: row.summary, + }); +} function escapeXml(value) { return String(value ?? '') @@ -18,9 +43,88 @@ function toAbsoluteUrl(origin, value) { return raw.split('#')[0]; } +async function loadOnlineNonPublicWorkspacePaths(pool) { + if (!pool) return new Set(); + const [rows] = await pool.query( + `SELECT LOWER(pr.user_id) AS user_id, p.workspace_relative_path + FROM h5_publish_records pr + JOIN h5_page_records p ON p.id = pr.page_id + WHERE pr.status = 'online' + AND pr.access_mode <> 'public' + AND p.workspace_relative_path LIKE 'public/%.html'`, + ); + return new Set( + rows.map((row) => `${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`), + ); +} + +async function loadWorkspacePageMetaByPath(pool) { + if (!pool) return new Map(); + const [rows] = await pool.query( + `SELECT id, LOWER(user_id) AS user_id, workspace_relative_path, title, summary, updated_at + FROM h5_page_records + WHERE status <> 'deleted' + AND workspace_relative_path LIKE 'public/%.html'`, + ); + const map = new Map(); + for (const row of rows) { + map.set(`${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`, row); + } + return map; +} + +export async function listWorkspacePublicHtmlIndexEntries( + pool, + { h5Root = null, now = Date.now() } = {}, +) { + if (!h5Root) return []; + const root = path.join(path.resolve(String(h5Root)), PUBLISH_ROOT_DIR); + if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) return []; + + const blocked = await loadOnlineNonPublicWorkspacePaths(pool); + const pageMeta = await loadWorkspacePageMetaByPath(pool); + const entries = []; + + for (const userId of fs.readdirSync(root)) { + if (!userId || userId.startsWith('.')) continue; + const publicDir = path.join(root, userId, 'public'); + if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) continue; + + for (const name of fs.readdirSync(publicDir)) { + if (!name.toLowerCase().endsWith('.html')) continue; + const relativePath = `public/${name}`; + if (!isWorkspacePublicHtmlRelativePath(relativePath)) continue; + const blockKey = `${userId.toLowerCase()}\0${relativePath}`; + if (blocked.has(blockKey)) continue; + + const meta = pageMeta.get(blockKey); + const filePath = path.join(publicDir, name); + let updatedAt = now; + try { + updatedAt = fs.statSync(filePath).mtimeMs; + } catch { + updatedAt = now; + } + + const snapshot = buildImplicitWorkspacePublicPublication({ + publicUrl: buildMindSpacePublicRoutePath(userId, relativePath.split('/')), + pageId: meta?.id ?? null, + updatedAt: meta?.updated_at ?? updatedAt, + title: meta?.title ?? null, + summary: meta?.summary ?? null, + }); + if (snapshot && isPublicationIndexable(snapshot, { now })) { + entries.push(snapshot); + } + } + } + + return entries; +} + export async function listIndexablePublications( pool, - { limit = 5000, offset = 0 } = {}, + { limit = 5000, offset = 0, h5Root = null } = {}, ) { if (!pool) return []; const safeLimit = Math.min(Math.max(Number(limit) || 5000, 1), 10000); @@ -38,22 +142,14 @@ export async function listIndexablePublications( LIMIT ? OFFSET ?`, [Date.now(), safeLimit, safeOffset], ); - return rows - .map((row) => - normalizePublicationSnapshot({ - id: row.id, - pageId: row.page_id, - publicUrl: row.public_url, - accessMode: row.access_mode, - status: row.status, - userConfirmedAt: row.user_confirmed_at, - expiresAt: row.expires_at, - updatedAt: row.updated_at ?? row.published_at, - title: row.title, - summary: row.summary, - }), - ) + const fromDb = rows + .map((row) => mapPublicationRow(row)) .filter((entry) => isPublicationIndexable(entry)); + if (!h5Root || safeOffset > 0) { + return dedupeDiscoveryEntriesByPage(fromDb); + } + const fromWorkspace = await listWorkspacePublicHtmlIndexEntries(pool, { h5Root }); + return dedupeDiscoveryEntriesByPage([...fromDb, ...fromWorkspace]); } export async function resolvePublicationIndexSnapshot( @@ -90,19 +186,61 @@ export async function resolvePublicationIndexSnapshot( } sql += ' ORDER BY pr.published_at DESC LIMIT 1'; const [rows] = await pool.query(sql, params); - const row = rows[0]; - if (!row) return null; - return normalizePublicationSnapshot({ - id: row.id, - pageId: row.page_id, - publicUrl: row.public_url, - accessMode: row.access_mode, - status: row.status, - userConfirmedAt: row.user_confirmed_at, - expiresAt: row.expires_at, - updatedAt: row.updated_at ?? row.published_at, - title: row.title, - summary: row.summary, + return mapPublicationRow(rows[0]); +} + +export async function resolveWorkspacePublicationIndexSnapshot( + pool, + { + publicationId = null, + pageId = null, + userId = null, + workspaceRelativePath = '', + publicUrl = '', + } = {}, +) { + const online = await resolvePublicationIndexSnapshot(pool, { + publicationId, + pageId, + userId, + }); + if (online) { + return online; + } + + const relativePath = String(workspaceRelativePath ?? '').trim(); + const owner = String(userId ?? '').trim(); + if (!isWorkspacePublicHtmlRelativePath(relativePath) || !owner) { + return null; + } + + if (pool) { + const [rows] = await pool.query( + `SELECT pr.id, pr.page_id, pr.public_url, pr.access_mode, pr.status, + pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at, + p.title, p.summary + FROM h5_publish_records pr + JOIN h5_page_records p ON p.id = pr.page_id + WHERE pr.status = 'online' + AND pr.user_id = ? + AND p.workspace_relative_path = ? + AND pr.access_mode <> 'public' + ORDER BY pr.published_at DESC + LIMIT 1`, + [owner, relativePath], + ); + const blocking = mapPublicationRow(rows[0]); + if (blocking) { + return blocking; + } + } + + const resolvedPublicUrl = + String(publicUrl ?? '').trim() || + buildMindSpacePublicRoutePath(owner, relativePath.split('/')); + return buildImplicitWorkspacePublicPublication({ + publicUrl: resolvedPublicUrl, + pageId: pageId ?? null, }); } @@ -210,11 +348,16 @@ export function renderLlmsTxt(entries, { origin = '' } = {}) { return `${header.concat(body).join('\n')}\n`; } -export function createMindspaceSeoDiscoveryService(pool) { +export function createMindspaceSeoDiscoveryService(pool, { h5Root = null } = {}) { return { - listIndexablePublications: (options) => listIndexablePublications(pool, options), + listIndexablePublications: (options) => + listIndexablePublications(pool, { h5Root, ...options }), resolvePublicationIndexSnapshot: (options) => resolvePublicationIndexSnapshot(pool, options), + resolveWorkspacePublicationIndexSnapshot: (options) => + resolveWorkspacePublicationIndexSnapshot(pool, options), + listWorkspacePublicHtmlIndexEntries: (options) => + listWorkspacePublicHtmlIndexEntries(pool, { h5Root, ...options }), dedupeDiscoveryEntriesByPage, mergeDiscoveryEntries, renderSitemapXml, diff --git a/mindspace-seo-discovery-service.test.mjs b/mindspace-seo-discovery-service.test.mjs index 7883f81..6c7a9c3 100644 --- a/mindspace-seo-discovery-service.test.mjs +++ b/mindspace-seo-discovery-service.test.mjs @@ -1,11 +1,16 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import test from 'node:test'; import { dedupeDiscoveryEntriesByPage, + listWorkspacePublicHtmlIndexEntries, mergeDiscoveryEntries, renderLlmsTxt, renderRobotsTxt, renderSitemapXml, + resolveWorkspacePublicationIndexSnapshot, } from './mindspace-seo-discovery-service.mjs'; test('renderSitemapXml emits only provided urls', () => { @@ -78,3 +83,59 @@ test('renderLlmsTxt lists markdown links', () => { ); assert.match(body, /\[仙居玩水\]\(https:\/\/m\.tkmind\.cn\/u\/john\/pages\/demo\)/); }); + +test('listWorkspacePublicHtmlIndexEntries includes unpublished public html on disk', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-seo-disk-')); + const userId = 'user-implicit-seo'; + const publicDir = path.join(root, 'MindSpace', userId, 'public'); + fs.mkdirSync(publicDir, { recursive: true }); + fs.writeFileSync( + path.join(publicDir, 'weather-broadcast-20260916.html'), + '