feat(seo): 固化 m.tkmind.cn 百度验证、推送与 sitemap 去重

2026-09-16 08:45:14 +00:00
parent 0f692ab68f f281f15788
commit 1e4f6433ff
11 changed files with 459 additions and 10 deletions
@@ -1,6 +1,7 @@
 <!doctype html>
 <html lang="zh-CN" data-theme="light">
  <head>
+    <meta name="baidu-site-verification" content="codeva-32NHQfJ8qL" />
    <meta charset="UTF-8" />
    <meta name="theme-color" content="#f4f1ea" />
    <script>
@@ -0,0 +1,19 @@
+export const BAIDU_SITE_VERIFICATION_CODE = 'codeva-32NHQfJ8qL';
+
+export const BAIDU_SITE_VERIFICATION_META =
+  `<meta name="baidu-site-verification" content="${BAIDU_SITE_VERIFICATION_CODE}" />`;
+
+export function injectBaiduSiteVerification(html) {
+  const source = String(html ?? '');
+  const tag = BAIDU_SITE_VERIFICATION_META;
+  if (/baidu-site-verification/i.test(source)) {
+    return source.replace(
+      /<meta[^>]+name=["']baidu-site-verification["'][^>]*>/i,
+      tag,
+    );
+  }
+  if (/<head[^>]*>/i.test(source)) {
+    return source.replace(/<head([^>]*)>/i, `<head$1>\n  ${tag}`);
+  }
+  return `${tag}\n${source}`;
+}
@@ -0,0 +1,24 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  BAIDU_SITE_VERIFICATION_CODE,
+  injectBaiduSiteVerification,
+} from './mindspace-baidu-site-verification.mjs';
+
+test('injectBaiduSiteVerification inserts meta as first head child', () => {
+  const html = '<!doctype html><html><head><title>TKMind</title></head><body></body></html>';
+  const next = injectBaiduSiteVerification(html);
+  assert.match(next, /<head>\s*<meta name="baidu-site-verification" content="codeva-32NHQfJ8qL" \/>/);
+});
+
+test('injectBaiduSiteVerification normalizes existing meta', () => {
+  const html =
+    '<html><head><meta name="baidu-site-verification" content="old" /><title>x</title></head></html>';
+  const next = injectBaiduSiteVerification(html);
+  assert.match(next, /content="codeva-32NHQfJ8qL"/);
+  assert.doesNotMatch(next, /content="old"/);
+});
+
+test('verification code stays stable', () => {
+  assert.equal(BAIDU_SITE_VERIFICATION_CODE, 'codeva-32NHQfJ8qL');
+});
@@ -106,10 +106,40 @@ export async function resolvePublicationIndexSnapshot(
  });
 }

+function scoreDiscoveryEntry(entry) {
+  const url = String(entry?.publicUrl ?? entry?.public_url ?? '').trim();
+  if (!url) return -100;
+  if (/\/_archived/i.test(url)) return -50;
+  if (url.includes('/u/')) return 100;
+  if (url.includes('/MindSpace/')) return 10;
+  return 20;
+}
+
+export function dedupeDiscoveryEntriesByPage(publications = []) {
+  const byPage = new Map();
+  for (const entry of publications) {
+    const snapshot = normalizePublicationSnapshot(entry);
+    if (!snapshot) continue;
+    const pageKey =
+      String(snapshot.pageId ?? '').trim() ||
+      String(snapshot.publicUrl ?? '').trim();
+    if (!pageKey) continue;
+    const existing = byPage.get(pageKey);
+    if (
+      !existing ||
+      scoreDiscoveryEntry(snapshot) > scoreDiscoveryEntry(existing)
+    ) {
+      byPage.set(pageKey, snapshot);
+    }
+  }
+  return [...byPage.values()];
+}
+
 export function mergeDiscoveryEntries(publications = [], staticEntries = PLATFORM_STATIC_DISCOVERY_ENTRIES) {
  const merged = [];
  const seen = new Set();
-  for (const entry of [...publications, ...staticEntries]) {
+  const dedupedPublications = dedupeDiscoveryEntriesByPage(publications);
+  for (const entry of [...dedupedPublications, ...staticEntries]) {
    const key = String(entry.publicUrl ?? entry.public_url ?? '').trim();
    if (!key || seen.has(key)) continue;
    seen.add(key);
@@ -185,6 +215,7 @@ export function createMindspaceSeoDiscoveryService(pool) {
    listIndexablePublications: (options) => listIndexablePublications(pool, options),
    resolvePublicationIndexSnapshot: (options) =>
      resolvePublicationIndexSnapshot(pool, options),
+    dedupeDiscoveryEntriesByPage,
    mergeDiscoveryEntries,
    renderSitemapXml,
    renderRobotsTxt,
@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
 import {
+  dedupeDiscoveryEntriesByPage,
  mergeDiscoveryEntries,
  renderLlmsTxt,
  renderRobotsTxt,
@@ -25,6 +26,44 @@ test('renderRobotsTxt references sitemap when enabled', () => {
  assert.match(body, /llms\.txt/);
 });

+test('dedupeDiscoveryEntriesByPage prefers /u/ urls for the same page', () => {
+  const merged = dedupeDiscoveryEntriesByPage([
+    {
+      pageId: 'page-1',
+      publicUrl: '/MindSpace/user/public/demo.html',
+      accessMode: 'public',
+      status: 'online',
+    },
+    {
+      pageId: 'page-1',
+      publicUrl: '/u/john/pages/demo',
+      accessMode: 'public',
+      status: 'online',
+    },
+  ]);
+  assert.equal(merged.length, 1);
+  assert.equal(merged[0].publicUrl, '/u/john/pages/demo');
+});
+
+test('dedupeDiscoveryEntriesByPage drops archived duplicates', () => {
+  const merged = dedupeDiscoveryEntriesByPage([
+    {
+      pageId: 'page-2',
+      publicUrl: '/MindSpace/user/public/_archived-demo.html',
+      accessMode: 'public',
+      status: 'online',
+    },
+    {
+      pageId: 'page-2',
+      publicUrl: '/u/john/pages/demo',
+      accessMode: 'public',
+      status: 'online',
+    },
+  ]);
+  assert.equal(merged.length, 1);
+  assert.equal(merged[0].publicUrl, '/u/john/pages/demo');
+});
+
 test('mergeDiscoveryEntries appends platform static pages', () => {
  const merged = mergeDiscoveryEntries([
    { publicUrl: '/u/john/pages/demo', accessMode: 'public', status: 'online' },
@@ -0,0 +1 @@
+codeva-32NHQfJ8qL
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+WRAPPER="${MEMIND_BAIDU_PUSH_WRAPPER:-$ROOT/scripts/run-baidu-sitemap-push-prod.sh}"
+LABEL="${MEMIND_BAIDU_PUSH_LABEL:-cn.tkmind.memind-baidu-sitemap-push}"
+PLIST="$HOME/Library/LaunchAgents/${LABEL}.plist"
+LOG="$HOME/Library/Logs/memind-baidu-sitemap-push.log"
+GUI="gui/$(id -u)"
+HOUR="${MEMIND_BAIDU_PUSH_HOUR:-1}"
+MINUTE="${MEMIND_BAIDU_PUSH_MINUTE:-0}"
+STATE_FILE="${BAIDU_PUSH_STATE_FILE:-$ROOT/data/baidu-push-state.json}"
+
+mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs" "$(dirname "$STATE_FILE")"
+
+if [[ ! -x "$WRAPPER" ]]; then
+  chmod +x "$WRAPPER"
+fi
+if [[ ! -f "$WRAPPER" ]]; then
+  echo "baidu push wrapper not found: $WRAPPER" >&2
+  exit 1
+fi
+
+cat > "$PLIST" <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>$LABEL</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/bash</string>
+    <string>$WRAPPER</string>
+  </array>
+  <key>WorkingDirectory</key>
+  <string>$ROOT</string>
+  <key>StartCalendarInterval</key>
+  <dict>
+    <key>Hour</key>
+    <integer>$HOUR</integer>
+    <key>Minute</key>
+    <integer>$MINUTE</integer>
+  </dict>
+  <key>StandardOutPath</key>
+  <string>$LOG</string>
+  <key>StandardErrorPath</key>
+  <string>$LOG</string>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
+    <key>BAIDU_PUSH_STATE_FILE</key>
+    <string>$STATE_FILE</string>
+  </dict>
+</dict>
+</plist>
+EOF
+
+plutil -lint "$PLIST"
+launchctl bootout "$GUI/$LABEL" 2>/dev/null || true
+launchctl bootstrap "$GUI" "$PLIST"
+launchctl enable "$GUI/$LABEL"
+
+echo "installed $PLIST"
+echo "wrapper: $WRAPPER"
+echo "schedule: daily ${HOUR}:$(printf '%02d' "$MINUTE") (local time)"
+echo "state_file: $STATE_FILE"
+echo "log: $LOG"
@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+/**
+ * 从 sitemap 批量推送 URL 到百度（受每日配额限制）。
+ *
+ * 用法:
+ *   node scripts/push-baidu-sitemap.mjs
+ *   node scripts/push-baidu-sitemap.mjs --dry-run
+ *   node scripts/push-baidu-sitemap.mjs --sitemap https://m.tkmind.cn/sitemap.xml
+ *
+ * 环境变量:
+ *   MINDSPACE_BAIDU_SITE          默认 m.tkmind.cn
+ *   MINDSPACE_BAIDU_PUSH_TOKEN    必填
+ *   BAIDU_PUSH_STATE_FILE         默认 data/baidu-push-state.json
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+function parseArgs(argv) {
+  const args = {
+    dryRun: false,
+    sitemap: process.env.BAIDU_PUSH_SITEMAP ?? 'https://m.tkmind.cn/sitemap.xml',
+    batchSize: Number(process.env.BAIDU_PUSH_BATCH_SIZE ?? 10),
+    stateFile:
+      process.env.BAIDU_PUSH_STATE_FILE ??
+      path.join(ROOT, 'data', 'baidu-push-state.json'),
+  };
+  for (let i = 2; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--dry-run') args.dryRun = true;
+    else if (arg === '--sitemap') args.sitemap = argv[++i] ?? args.sitemap;
+    else if (arg === '--batch-size') args.batchSize = Number(argv[++i] ?? args.batchSize);
+    else if (arg === '--state-file') args.stateFile = argv[++i] ?? args.stateFile;
+  }
+  return args;
+}
+
+function loadState(stateFile) {
+  try {
+    return JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+  } catch {
+    return { pushed: {}, lastRunAt: null, history: [] };
+  }
+}
+
+function saveState(stateFile, state) {
+  fs.mkdirSync(path.dirname(stateFile), { recursive: true });
+  fs.writeFileSync(stateFile, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
+}
+
+function extractSitemapUrls(xml) {
+  return [...String(xml ?? '').matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim());
+}
+
+function isArchivedDiscoveryUrl(url) {
+  return /\/_archived/i.test(String(url ?? ''));
+}
+
+function prioritizeUrls(urls) {
+  const filtered = urls.filter((url) => url && !isArchivedDiscoveryUrl(url));
+  const uUrls = filtered.filter((url) => url.includes('/u/'));
+  const mindspaceUrls = filtered.filter((url) => url.includes('/MindSpace/'));
+  const otherUrls = filtered.filter(
+    (url) => !url.includes('/u/') && !url.includes('/MindSpace/'),
+  );
+  const seen = new Set();
+  const ordered = [];
+  for (const url of [...uUrls, ...otherUrls, ...mindspaceUrls]) {
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    ordered.push(url);
+  }
+  return ordered;
+}
+
+async function fetchSitemap(url) {
+  const response = await fetch(url, { signal: AbortSignal.timeout(30_000) });
+  if (!response.ok) {
+    throw new Error(`sitemap fetch failed: ${response.status}`);
+  }
+  return response.text();
+}
+
+async function pingBaidu({ site, token, urls }) {
+  const endpoint = `http://data.zz.baidu.com/urls?site=${encodeURIComponent(site)}&token=${encodeURIComponent(token)}`;
+  const response = await fetch(endpoint, {
+    method: 'POST',
+    headers: { 'Content-Type': 'text/plain' },
+    body: urls.join('\n'),
+    signal: AbortSignal.timeout(60_000),
+  });
+  const payload = await response.json().catch(() => ({}));
+  if (!response.ok) {
+    const error = new Error(payload?.message || 'baidu_push_failed');
+    error.code = payload?.error ?? response.status;
+    error.details = payload;
+    throw error;
+  }
+  return payload;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  const site = String(process.env.MINDSPACE_BAIDU_SITE ?? 'm.tkmind.cn').trim();
+  const token = String(
+    process.env.MINDSPACE_BAIDU_PUSH_TOKEN ??
+      process.env.PLAZA_BAIDU_PUSH_TOKEN ??
+      '',
+  ).trim();
+  if (!token) {
+    console.error('缺少 MINDSPACE_BAIDU_PUSH_TOKEN');
+    process.exit(1);
+  }
+
+  const xml = await fetchSitemap(args.sitemap);
+  const allUrls = prioritizeUrls(extractSitemapUrls(xml));
+  const state = loadState(args.stateFile);
+  const pending = allUrls.filter((url) => !state.pushed[url]);
+
+  console.log(
+    JSON.stringify(
+      {
+        site,
+        sitemap: args.sitemap,
+        totalInSitemap: allUrls.length,
+        alreadyPushed: allUrls.length - pending.length,
+        pending: pending.length,
+        dryRun: args.dryRun,
+      },
+      null,
+      2,
+    ),
+  );
+
+  if (args.dryRun || pending.length === 0) {
+    return;
+  }
+
+  let pushedThisRun = 0;
+  const runHistory = {
+    at: new Date().toISOString(),
+    attempted: 0,
+    success: 0,
+    batches: [],
+  };
+
+  for (let i = 0; i < pending.length; i += args.batchSize) {
+    const batch = pending.slice(i, i + args.batchSize);
+    runHistory.attempted += batch.length;
+    try {
+      const result = await pingBaidu({ site, token, urls: batch });
+      const successCount = Number(result.success ?? 0);
+      runHistory.success += successCount;
+      runHistory.batches.push({ size: batch.length, result });
+      for (const url of batch.slice(0, successCount)) {
+        state.pushed[url] = runHistory.at;
+        pushedThisRun += 1;
+      }
+      console.log(
+        JSON.stringify(
+          {
+            batch: batch.length,
+            success: successCount,
+            remain: result.remain ?? null,
+            notSameSite: result.not_same_site ?? null,
+            notValid: result.not_valid ?? null,
+          },
+          null,
+          2,
+        ),
+      );
+      if (successCount < batch.length) break;
+      if (Number(result.remain ?? 0) <= 0) break;
+    } catch (error) {
+      runHistory.batches.push({
+        size: batch.length,
+        error: error.message,
+        details: error.details ?? null,
+      });
+      console.error(JSON.stringify({ batch: batch.length, error: error.message, details: error.details ?? null }, null, 2));
+      break;
+    }
+  }
+
+  state.lastRunAt = runHistory.at;
+  state.history = [...(state.history ?? []), runHistory].slice(-30);
+  saveState(args.stateFile, state);
+
+  console.log(
+    JSON.stringify(
+      {
+        pushedThisRun,
+        totalTracked: Object.keys(state.pushed).length,
+        pendingRemaining: allUrls.filter((url) => !state.pushed[url]).length,
+      },
+      null,
+      2,
+    ),
+  );
+}
+
+main().catch((error) => {
+  console.error(error?.stack || error?.message || String(error));
+  process.exit(1);
+});
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+if [[ -f "${ROOT}/.env" ]]; then
+  set -a
+  # shellcheck disable=SC1091
+  source "${ROOT}/.env"
+  set +a
+fi
+
+NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
+if [[ ! -x "${NODE_BIN}" ]]; then
+  NODE_BIN="$(command -v node)"
+fi
+
+exec "${NODE_BIN}" "${ROOT}/scripts/push-baidu-sitemap.mjs" "$@"
@@ -6,6 +6,7 @@ import {
  injectOgTags,
  renderWechatSharePreviewHtml,
 } from '../mindspace-og-tags.mjs';
+import { injectBaiduSiteVerification } from '../mindspace-baidu-site-verification.mjs';

 export function attachPortalStaticDeliveryRoutes({
  app,
@@ -285,6 +286,37 @@ export function attachPortalStaticDeliveryRoutes({
    },
  );

+  const sendPortalIndexHtml = (res) => {
+    const indexPath = pathApi.join(h5Root, 'dist', 'index.html');
+    const html = fsApi.readFileSync(indexPath, 'utf8');
+    res
+      .type('text/html; charset=utf-8')
+      .set('Cache-Control', 'public, max-age=0')
+      .send(injectBaiduSiteVerification(html));
+  };
+
+  app.get(
+    /^\/baidu_verify_[A-Za-z0-9-]+\.html$/,
+    (req, res) => {
+      const fileName = pathApi.basename(req.path);
+      const filePath = pathApi.join(
+        h5Root,
+        'public',
+        fileName,
+      );
+      if (!fsApi.existsSync(filePath)) {
+        return res.status(404).end();
+      }
+      return res
+        .type('text/html; charset=utf-8')
+        .sendFile(filePath);
+    },
+  );
+
+  app.get('/', (_req, res) => {
+    sendPortalIndexHtml(res);
+  });
+
  app.use('/auth', (_req, res) => {
    res.status(404).json({
      message:
@@ -305,12 +337,6 @@ export function attachPortalStaticDeliveryRoutes({
    ),
  );
  app.get('*', (_req, res) => {
-    res.sendFile(
-      pathApi.join(
-        h5Root,
-        'dist',
-        'index.html',
-      ),
-    );
+    sendPortalIndexHtml(res);
  });
 }
@@ -188,6 +188,11 @@ test('registers static delivery routes and middleware in legacy order', () => {
        'get',
        '/^\\/MP_verify_[A-Za-z0-9]+\\.txt$/',
      ],
+      [
+        'get',
+        '/^\\/baidu_verify_[A-Za-z0-9-]+\\.html$/',
+      ],
+      ['get', '/'],
      ['use', '/auth'],
      ['use', '/admin-api'],
      ['use', null],
@@ -394,6 +399,9 @@ test('preserves verification, API fallbacks, and SPA fallback', async () => {
      existsSync() {
        return false;
      },
+      readFileSync() {
+        return '<!doctype html><html><head><title>TKMind</title></head><body></body></html>';
+      },
    },
  });
  const verification = missingSetup.routes.find(
@@ -432,8 +440,12 @@ test('preserves verification, API fallbacks, and SPA fallback', async () => {
    'get',
    '*',
  );
+  assert.match(
+    spa.response.body,
+    /baidu-site-verification/,
+  );
  assert.equal(
-    spa.response.sentFile,
-    '/project/dist/index.html',
+    spa.response.contentType,
+    'text/html; charset=utf-8',
  );
 });