john
745e9b691d
Memind CI / Test, build, and release guards (pull_request) Successful in 3m35s
未点公开发布但已落盘到 public/ 的页面，在无在线非 public 发布阻挡时 合成 implicit public snapshot，注入 canonical/JSON-LD 并扫描进 sitemap。 Co-authored-by: Cursor <cursoragent@cursor.com>
368 lines
11 KiB
JavaScript
368 lines
11 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import {
|
|
buildImplicitWorkspacePublicPublication,
|
|
isPublicationIndexable,
|
|
isWorkspacePublicHtmlRelativePath,
|
|
normalizePublicationSnapshot,
|
|
} from './mindspace-index-policy.mjs';
|
|
import { buildMindSpacePublicRoutePath } from './mindspace-runtime-config.mjs';
|
|
import { PLATFORM_STATIC_DISCOVERY_ENTRIES } from './platform-seo-html.mjs';
|
|
import { PUBLISH_ROOT_DIR } from './user-publish.mjs';
|
|
|
|
function mapPublicationRow(row) {
|
|
if (!row) return null;
|
|
return normalizePublicationSnapshot({
|
|
id: row.id,
|
|
pageId: row.page_id,
|
|
publicUrl: row.public_url,
|
|
accessMode: row.access_mode,
|
|
status: row.status,
|
|
userConfirmedAt: row.user_confirmed_at,
|
|
expiresAt: row.expires_at,
|
|
updatedAt: row.updated_at ?? row.published_at,
|
|
title: row.title,
|
|
summary: row.summary,
|
|
});
|
|
}
|
|
|
|
function escapeXml(value) {
|
|
return String(value ?? '')
|
|
.replaceAll('&', '&')
|
|
.replaceAll('<', '<')
|
|
.replaceAll('>', '>')
|
|
.replaceAll('"', '"')
|
|
.replaceAll("'", ''');
|
|
}
|
|
|
|
function toAbsoluteUrl(origin, value) {
|
|
const raw = String(value ?? '').trim();
|
|
if (!raw) return '';
|
|
if (/^https?:\/\//i.test(raw)) return raw.split('#')[0];
|
|
if (raw.startsWith('/') && origin) return `${origin}${raw}`.split('#')[0];
|
|
return raw.split('#')[0];
|
|
}
|
|
|
|
async function loadOnlineNonPublicWorkspacePaths(pool) {
|
|
if (!pool) return new Set();
|
|
const [rows] = await pool.query(
|
|
`SELECT LOWER(pr.user_id) AS user_id, p.workspace_relative_path
|
|
FROM h5_publish_records pr
|
|
JOIN h5_page_records p ON p.id = pr.page_id
|
|
WHERE pr.status = 'online'
|
|
AND pr.access_mode <> 'public'
|
|
AND p.workspace_relative_path LIKE 'public/%.html'`,
|
|
);
|
|
return new Set(
|
|
rows.map((row) => `${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`),
|
|
);
|
|
}
|
|
|
|
async function loadWorkspacePageMetaByPath(pool) {
|
|
if (!pool) return new Map();
|
|
const [rows] = await pool.query(
|
|
`SELECT id, LOWER(user_id) AS user_id, workspace_relative_path, title, summary, updated_at
|
|
FROM h5_page_records
|
|
WHERE status <> 'deleted'
|
|
AND workspace_relative_path LIKE 'public/%.html'`,
|
|
);
|
|
const map = new Map();
|
|
for (const row of rows) {
|
|
map.set(`${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`, row);
|
|
}
|
|
return map;
|
|
}
|
|
|
|
export async function listWorkspacePublicHtmlIndexEntries(
|
|
pool,
|
|
{ h5Root = null, now = Date.now() } = {},
|
|
) {
|
|
if (!h5Root) return [];
|
|
const root = path.join(path.resolve(String(h5Root)), PUBLISH_ROOT_DIR);
|
|
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) return [];
|
|
|
|
const blocked = await loadOnlineNonPublicWorkspacePaths(pool);
|
|
const pageMeta = await loadWorkspacePageMetaByPath(pool);
|
|
const entries = [];
|
|
|
|
for (const userId of fs.readdirSync(root)) {
|
|
if (!userId || userId.startsWith('.')) continue;
|
|
const publicDir = path.join(root, userId, 'public');
|
|
if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) continue;
|
|
|
|
for (const name of fs.readdirSync(publicDir)) {
|
|
if (!name.toLowerCase().endsWith('.html')) continue;
|
|
const relativePath = `public/${name}`;
|
|
if (!isWorkspacePublicHtmlRelativePath(relativePath)) continue;
|
|
const blockKey = `${userId.toLowerCase()}\0${relativePath}`;
|
|
if (blocked.has(blockKey)) continue;
|
|
|
|
const meta = pageMeta.get(blockKey);
|
|
const filePath = path.join(publicDir, name);
|
|
let updatedAt = now;
|
|
try {
|
|
updatedAt = fs.statSync(filePath).mtimeMs;
|
|
} catch {
|
|
updatedAt = now;
|
|
}
|
|
|
|
const snapshot = buildImplicitWorkspacePublicPublication({
|
|
publicUrl: buildMindSpacePublicRoutePath(userId, relativePath.split('/')),
|
|
pageId: meta?.id ?? null,
|
|
updatedAt: meta?.updated_at ?? updatedAt,
|
|
title: meta?.title ?? null,
|
|
summary: meta?.summary ?? null,
|
|
});
|
|
if (snapshot && isPublicationIndexable(snapshot, { now })) {
|
|
entries.push(snapshot);
|
|
}
|
|
}
|
|
}
|
|
|
|
return entries;
|
|
}
|
|
|
|
export async function listIndexablePublications(
|
|
pool,
|
|
{ limit = 5000, offset = 0, h5Root = null } = {},
|
|
) {
|
|
if (!pool) return [];
|
|
const safeLimit = Math.min(Math.max(Number(limit) || 5000, 1), 10000);
|
|
const safeOffset = Math.max(Number(offset) || 0, 0);
|
|
const [rows] = await pool.query(
|
|
`SELECT pr.id, pr.page_id, pr.public_url, pr.access_mode, pr.status,
|
|
pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at,
|
|
p.title, p.summary
|
|
FROM h5_publish_records pr
|
|
LEFT JOIN h5_page_records p ON p.id = pr.page_id
|
|
WHERE pr.status = 'online'
|
|
AND pr.access_mode = 'public'
|
|
AND (pr.expires_at IS NULL OR pr.expires_at > ?)
|
|
ORDER BY pr.published_at DESC
|
|
LIMIT ? OFFSET ?`,
|
|
[Date.now(), safeLimit, safeOffset],
|
|
);
|
|
const fromDb = rows
|
|
.map((row) => mapPublicationRow(row))
|
|
.filter((entry) => isPublicationIndexable(entry));
|
|
if (!h5Root || safeOffset > 0) {
|
|
return dedupeDiscoveryEntriesByPage(fromDb);
|
|
}
|
|
const fromWorkspace = await listWorkspacePublicHtmlIndexEntries(pool, { h5Root });
|
|
return dedupeDiscoveryEntriesByPage([...fromDb, ...fromWorkspace]);
|
|
}
|
|
|
|
export async function resolvePublicationIndexSnapshot(
|
|
pool,
|
|
{
|
|
publicationId = null,
|
|
pageId = null,
|
|
userId = null,
|
|
} = {},
|
|
) {
|
|
if (!pool) return null;
|
|
const id = String(publicationId ?? '').trim();
|
|
const page = String(pageId ?? '').trim();
|
|
const owner = String(userId ?? '').trim();
|
|
if (!id && !page) return null;
|
|
|
|
const params = [];
|
|
let sql = `SELECT pr.id, pr.page_id, pr.user_id, pr.public_url, pr.access_mode, pr.status,
|
|
pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at,
|
|
p.title, p.summary
|
|
FROM h5_publish_records pr
|
|
LEFT JOIN h5_page_records p ON p.id = pr.page_id
|
|
WHERE pr.status = 'online'`;
|
|
if (id) {
|
|
sql += ' AND pr.id = ?';
|
|
params.push(id);
|
|
} else {
|
|
sql += ' AND pr.page_id = ?';
|
|
params.push(page);
|
|
if (owner) {
|
|
sql += ' AND pr.user_id = ?';
|
|
params.push(owner);
|
|
}
|
|
}
|
|
sql += ' ORDER BY pr.published_at DESC LIMIT 1';
|
|
const [rows] = await pool.query(sql, params);
|
|
return mapPublicationRow(rows[0]);
|
|
}
|
|
|
|
export async function resolveWorkspacePublicationIndexSnapshot(
|
|
pool,
|
|
{
|
|
publicationId = null,
|
|
pageId = null,
|
|
userId = null,
|
|
workspaceRelativePath = '',
|
|
publicUrl = '',
|
|
} = {},
|
|
) {
|
|
const online = await resolvePublicationIndexSnapshot(pool, {
|
|
publicationId,
|
|
pageId,
|
|
userId,
|
|
});
|
|
if (online) {
|
|
return online;
|
|
}
|
|
|
|
const relativePath = String(workspaceRelativePath ?? '').trim();
|
|
const owner = String(userId ?? '').trim();
|
|
if (!isWorkspacePublicHtmlRelativePath(relativePath) || !owner) {
|
|
return null;
|
|
}
|
|
|
|
if (pool) {
|
|
const [rows] = await pool.query(
|
|
`SELECT pr.id, pr.page_id, pr.public_url, pr.access_mode, pr.status,
|
|
pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at,
|
|
p.title, p.summary
|
|
FROM h5_publish_records pr
|
|
JOIN h5_page_records p ON p.id = pr.page_id
|
|
WHERE pr.status = 'online'
|
|
AND pr.user_id = ?
|
|
AND p.workspace_relative_path = ?
|
|
AND pr.access_mode <> 'public'
|
|
ORDER BY pr.published_at DESC
|
|
LIMIT 1`,
|
|
[owner, relativePath],
|
|
);
|
|
const blocking = mapPublicationRow(rows[0]);
|
|
if (blocking) {
|
|
return blocking;
|
|
}
|
|
}
|
|
|
|
const resolvedPublicUrl =
|
|
String(publicUrl ?? '').trim() ||
|
|
buildMindSpacePublicRoutePath(owner, relativePath.split('/'));
|
|
return buildImplicitWorkspacePublicPublication({
|
|
publicUrl: resolvedPublicUrl,
|
|
pageId: pageId ?? null,
|
|
});
|
|
}
|
|
|
|
function scoreDiscoveryEntry(entry) {
|
|
const url = String(entry?.publicUrl ?? entry?.public_url ?? '').trim();
|
|
if (!url) return -100;
|
|
if (/\/_archived/i.test(url)) return -50;
|
|
if (url.includes('/u/')) return 100;
|
|
if (url.includes('/MindSpace/')) return 10;
|
|
return 20;
|
|
}
|
|
|
|
export function dedupeDiscoveryEntriesByPage(publications = []) {
|
|
const byPage = new Map();
|
|
for (const entry of publications) {
|
|
const snapshot = normalizePublicationSnapshot(entry);
|
|
if (!snapshot) continue;
|
|
const pageKey =
|
|
String(snapshot.pageId ?? '').trim() ||
|
|
String(snapshot.publicUrl ?? '').trim();
|
|
if (!pageKey) continue;
|
|
const existing = byPage.get(pageKey);
|
|
if (
|
|
!existing ||
|
|
scoreDiscoveryEntry(snapshot) > scoreDiscoveryEntry(existing)
|
|
) {
|
|
byPage.set(pageKey, snapshot);
|
|
}
|
|
}
|
|
return [...byPage.values()];
|
|
}
|
|
|
|
export function mergeDiscoveryEntries(publications = [], staticEntries = PLATFORM_STATIC_DISCOVERY_ENTRIES) {
|
|
const merged = [];
|
|
const seen = new Set();
|
|
const dedupedPublications = dedupeDiscoveryEntriesByPage(publications);
|
|
for (const entry of [...dedupedPublications, ...staticEntries]) {
|
|
const key = String(entry.publicUrl ?? entry.public_url ?? '').trim();
|
|
if (!key || seen.has(key)) continue;
|
|
seen.add(key);
|
|
merged.push(normalizePublicationSnapshot(entry) ?? entry);
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
export function renderSitemapXml(entries, { origin = '' } = {}) {
|
|
const urls = entries
|
|
.map((entry) => {
|
|
const loc = toAbsoluteUrl(origin, entry.publicUrl);
|
|
if (!loc) return '';
|
|
const lastmod = entry.updatedAt
|
|
? new Date(Number(entry.updatedAt)).toISOString()
|
|
: new Date().toISOString();
|
|
return ` <url>
|
|
<loc>${escapeXml(loc)}</loc>
|
|
<lastmod>${escapeXml(lastmod)}</lastmod>
|
|
</url>`;
|
|
})
|
|
.filter(Boolean)
|
|
.join('\n');
|
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
|
${urls}
|
|
</urlset>
|
|
`;
|
|
}
|
|
|
|
export function renderRobotsTxt({
|
|
origin = '',
|
|
sitemapEnabled = false,
|
|
llmsTxtEnabled = false,
|
|
} = {}) {
|
|
const lines = [
|
|
'User-agent: *',
|
|
'Allow: /u/',
|
|
'Allow: /learn/',
|
|
'Disallow: /api/',
|
|
'Disallow: /admin-api/',
|
|
'Disallow: /mindspace/',
|
|
];
|
|
if (sitemapEnabled && origin) {
|
|
lines.push(`Sitemap: ${origin}/sitemap.xml`);
|
|
}
|
|
if (llmsTxtEnabled && origin) {
|
|
lines.push(`# AI discovery: ${origin}/llms.txt`);
|
|
}
|
|
lines.push('');
|
|
return `${lines.join('\n')}`;
|
|
}
|
|
|
|
export function renderLlmsTxt(entries, { origin = '' } = {}) {
|
|
const header = [
|
|
'# TKMind MindSpace public pages',
|
|
'# Public online publications are listed here.',
|
|
'',
|
|
];
|
|
const body = entries
|
|
.map((entry) => {
|
|
const url = toAbsoluteUrl(origin, entry.publicUrl);
|
|
if (!url) return '';
|
|
const summary = String(entry.summary ?? entry.title ?? '').trim();
|
|
return summary ? `- [${summary}](${url})` : `- ${url}`;
|
|
})
|
|
.filter(Boolean);
|
|
return `${header.concat(body).join('\n')}\n`;
|
|
}
|
|
|
|
export function createMindspaceSeoDiscoveryService(pool, { h5Root = null } = {}) {
|
|
return {
|
|
listIndexablePublications: (options) =>
|
|
listIndexablePublications(pool, { h5Root, ...options }),
|
|
resolvePublicationIndexSnapshot: (options) =>
|
|
resolvePublicationIndexSnapshot(pool, options),
|
|
resolveWorkspacePublicationIndexSnapshot: (options) =>
|
|
resolveWorkspacePublicationIndexSnapshot(pool, options),
|
|
listWorkspacePublicHtmlIndexEntries: (options) =>
|
|
listWorkspacePublicHtmlIndexEntries(pool, { h5Root, ...options }),
|
|
dedupeDiscoveryEntriesByPage,
|
|
mergeDiscoveryEntries,
|
|
renderSitemapXml,
|
|
renderRobotsTxt,
|
|
renderLlmsTxt,
|
|
};
|
|
}
|