import fs from 'node:fs'; import path from 'node:path'; import { buildImplicitWorkspacePublicPublication, isPublicationIndexable, isWorkspacePublicHtmlRelativePath, normalizePublicationSnapshot, } from './mindspace-index-policy.mjs'; import { buildMindSpacePublicRoutePath } from './mindspace-runtime-config.mjs'; import { PLATFORM_STATIC_DISCOVERY_ENTRIES } from './platform-seo-html.mjs'; import { PUBLISH_ROOT_DIR } from './user-publish.mjs'; function mapPublicationRow(row) { if (!row) return null; return normalizePublicationSnapshot({ id: row.id, pageId: row.page_id, publicUrl: row.public_url, accessMode: row.access_mode, status: row.status, userConfirmedAt: row.user_confirmed_at, expiresAt: row.expires_at, updatedAt: row.updated_at ?? row.published_at, title: row.title, summary: row.summary, }); } function escapeXml(value) { return String(value ?? '') .replaceAll('&', '&') .replaceAll('<', '<') .replaceAll('>', '>') .replaceAll('"', '"') .replaceAll("'", '''); } function toAbsoluteUrl(origin, value) { const raw = String(value ?? '').trim(); if (!raw) return ''; if (/^https?:\/\//i.test(raw)) return raw.split('#')[0]; if (raw.startsWith('/') && origin) return `${origin}${raw}`.split('#')[0]; return raw.split('#')[0]; } async function loadOnlineNonPublicWorkspacePaths(pool) { if (!pool) return new Set(); const [rows] = await pool.query( `SELECT LOWER(pr.user_id) AS user_id, p.workspace_relative_path FROM h5_publish_records pr JOIN h5_page_records p ON p.id = pr.page_id WHERE pr.status = 'online' AND pr.access_mode <> 'public' AND p.workspace_relative_path LIKE 'public/%.html'`, ); return new Set( rows.map((row) => `${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`), ); } async function loadWorkspacePageMetaByPath(pool) { if (!pool) return new Map(); const [rows] = await pool.query( `SELECT id, LOWER(user_id) AS user_id, workspace_relative_path, title, summary, updated_at FROM h5_page_records WHERE status <> 'deleted' AND workspace_relative_path LIKE 'public/%.html'`, ); const map = new Map(); for (const row of rows) { map.set(`${String(row.user_id ?? '').toLowerCase()}\0${row.workspace_relative_path}`, row); } return map; } export async function listWorkspacePublicHtmlIndexEntries( pool, { h5Root = null, now = Date.now() } = {}, ) { if (!h5Root) return []; const root = path.join(path.resolve(String(h5Root)), PUBLISH_ROOT_DIR); if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) return []; const blocked = await loadOnlineNonPublicWorkspacePaths(pool); const pageMeta = await loadWorkspacePageMetaByPath(pool); const entries = []; for (const userId of fs.readdirSync(root)) { if (!userId || userId.startsWith('.')) continue; const publicDir = path.join(root, userId, 'public'); if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) continue; for (const name of fs.readdirSync(publicDir)) { if (!name.toLowerCase().endsWith('.html')) continue; const relativePath = `public/${name}`; if (!isWorkspacePublicHtmlRelativePath(relativePath)) continue; const blockKey = `${userId.toLowerCase()}\0${relativePath}`; if (blocked.has(blockKey)) continue; const meta = pageMeta.get(blockKey); const filePath = path.join(publicDir, name); let updatedAt = now; try { updatedAt = fs.statSync(filePath).mtimeMs; } catch { updatedAt = now; } const snapshot = buildImplicitWorkspacePublicPublication({ publicUrl: buildMindSpacePublicRoutePath(userId, relativePath.split('/')), pageId: meta?.id ?? null, updatedAt: meta?.updated_at ?? updatedAt, title: meta?.title ?? null, summary: meta?.summary ?? null, }); if (snapshot && isPublicationIndexable(snapshot, { now })) { entries.push(snapshot); } } } return entries; } export async function listIndexablePublications( pool, { limit = 5000, offset = 0, h5Root = null } = {}, ) { if (!pool) return []; const safeLimit = Math.min(Math.max(Number(limit) || 5000, 1), 10000); const safeOffset = Math.max(Number(offset) || 0, 0); const [rows] = await pool.query( `SELECT pr.id, pr.page_id, pr.public_url, pr.access_mode, pr.status, pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at, p.title, p.summary FROM h5_publish_records pr LEFT JOIN h5_page_records p ON p.id = pr.page_id WHERE pr.status = 'online' AND pr.access_mode = 'public' AND (pr.expires_at IS NULL OR pr.expires_at > ?) ORDER BY pr.published_at DESC LIMIT ? OFFSET ?`, [Date.now(), safeLimit, safeOffset], ); const fromDb = rows .map((row) => mapPublicationRow(row)) .filter((entry) => isPublicationIndexable(entry)); if (!h5Root || safeOffset > 0) { return dedupeDiscoveryEntriesByPage(fromDb); } const fromWorkspace = await listWorkspacePublicHtmlIndexEntries(pool, { h5Root }); return dedupeDiscoveryEntriesByPage([...fromDb, ...fromWorkspace]); } export async function resolvePublicationIndexSnapshot( pool, { publicationId = null, pageId = null, userId = null, } = {}, ) { if (!pool) return null; const id = String(publicationId ?? '').trim(); const page = String(pageId ?? '').trim(); const owner = String(userId ?? '').trim(); if (!id && !page) return null; const params = []; let sql = `SELECT pr.id, pr.page_id, pr.user_id, pr.public_url, pr.access_mode, pr.status, pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at, p.title, p.summary FROM h5_publish_records pr LEFT JOIN h5_page_records p ON p.id = pr.page_id WHERE pr.status = 'online'`; if (id) { sql += ' AND pr.id = ?'; params.push(id); } else { sql += ' AND pr.page_id = ?'; params.push(page); if (owner) { sql += ' AND pr.user_id = ?'; params.push(owner); } } sql += ' ORDER BY pr.published_at DESC LIMIT 1'; const [rows] = await pool.query(sql, params); return mapPublicationRow(rows[0]); } export async function resolveWorkspacePublicationIndexSnapshot( pool, { publicationId = null, pageId = null, userId = null, workspaceRelativePath = '', publicUrl = '', } = {}, ) { const online = await resolvePublicationIndexSnapshot(pool, { publicationId, pageId, userId, }); if (online) { return online; } const relativePath = String(workspaceRelativePath ?? '').trim(); const owner = String(userId ?? '').trim(); if (!isWorkspacePublicHtmlRelativePath(relativePath) || !owner) { return null; } if (pool) { const [rows] = await pool.query( `SELECT pr.id, pr.page_id, pr.public_url, pr.access_mode, pr.status, pr.user_confirmed_at, pr.expires_at, pr.updated_at, pr.published_at, p.title, p.summary FROM h5_publish_records pr JOIN h5_page_records p ON p.id = pr.page_id WHERE pr.status = 'online' AND pr.user_id = ? AND p.workspace_relative_path = ? AND pr.access_mode <> 'public' ORDER BY pr.published_at DESC LIMIT 1`, [owner, relativePath], ); const blocking = mapPublicationRow(rows[0]); if (blocking) { return blocking; } } const resolvedPublicUrl = String(publicUrl ?? '').trim() || buildMindSpacePublicRoutePath(owner, relativePath.split('/')); return buildImplicitWorkspacePublicPublication({ publicUrl: resolvedPublicUrl, pageId: pageId ?? null, }); } function scoreDiscoveryEntry(entry) { const url = String(entry?.publicUrl ?? entry?.public_url ?? '').trim(); if (!url) return -100; if (/\/_archived/i.test(url)) return -50; if (url.includes('/u/')) return 100; if (url.includes('/MindSpace/')) return 10; return 20; } export function dedupeDiscoveryEntriesByPage(publications = []) { const byPage = new Map(); for (const entry of publications) { const snapshot = normalizePublicationSnapshot(entry); if (!snapshot) continue; const pageKey = String(snapshot.pageId ?? '').trim() || String(snapshot.publicUrl ?? '').trim(); if (!pageKey) continue; const existing = byPage.get(pageKey); if ( !existing || scoreDiscoveryEntry(snapshot) > scoreDiscoveryEntry(existing) ) { byPage.set(pageKey, snapshot); } } return [...byPage.values()]; } export function mergeDiscoveryEntries(publications = [], staticEntries = PLATFORM_STATIC_DISCOVERY_ENTRIES) { const merged = []; const seen = new Set(); const dedupedPublications = dedupeDiscoveryEntriesByPage(publications); for (const entry of [...dedupedPublications, ...staticEntries]) { const key = String(entry.publicUrl ?? entry.public_url ?? '').trim(); if (!key || seen.has(key)) continue; seen.add(key); merged.push(normalizePublicationSnapshot(entry) ?? entry); } return merged; } export function renderSitemapXml(entries, { origin = '' } = {}) { const urls = entries .map((entry) => { const loc = toAbsoluteUrl(origin, entry.publicUrl); if (!loc) return ''; const lastmod = entry.updatedAt ? new Date(Number(entry.updatedAt)).toISOString() : new Date().toISOString(); return ` ${escapeXml(loc)} ${escapeXml(lastmod)} `; }) .filter(Boolean) .join('\n'); return ` ${urls} `; } export function renderRobotsTxt({ origin = '', sitemapEnabled = false, llmsTxtEnabled = false, } = {}) { const lines = [ 'User-agent: *', 'Allow: /u/', 'Allow: /learn/', 'Disallow: /api/', 'Disallow: /admin-api/', 'Disallow: /mindspace/', ]; if (sitemapEnabled && origin) { lines.push(`Sitemap: ${origin}/sitemap.xml`); } if (llmsTxtEnabled && origin) { lines.push(`# AI discovery: ${origin}/llms.txt`); } lines.push(''); return `${lines.join('\n')}`; } export function renderLlmsTxt(entries, { origin = '' } = {}) { const header = [ '# TKMind MindSpace public pages', '# Public online publications are listed here.', '', ]; const body = entries .map((entry) => { const url = toAbsoluteUrl(origin, entry.publicUrl); if (!url) return ''; const summary = String(entry.summary ?? entry.title ?? '').trim(); return summary ? `- [${summary}](${url})` : `- ${url}`; }) .filter(Boolean); return `${header.concat(body).join('\n')}\n`; } export function createMindspaceSeoDiscoveryService(pool, { h5Root = null } = {}) { return { listIndexablePublications: (options) => listIndexablePublications(pool, { h5Root, ...options }), resolvePublicationIndexSnapshot: (options) => resolvePublicationIndexSnapshot(pool, options), resolveWorkspacePublicationIndexSnapshot: (options) => resolveWorkspacePublicationIndexSnapshot(pool, options), listWorkspacePublicHtmlIndexEntries: (options) => listWorkspacePublicHtmlIndexEntries(pool, { h5Root, ...options }), dedupeDiscoveryEntriesByPage, mergeDiscoveryEntries, renderSitemapXml, renderRobotsTxt, renderLlmsTxt, }; }