memind/scripts/push-baidu-sitemap.mjs

#!/usr/bin/env node
/**
 * 从 sitemap 批量推送 URL 到百度（受每日配额限制）。
 *
 * 用法:
 *   node scripts/push-baidu-sitemap.mjs
 *   node scripts/push-baidu-sitemap.mjs --dry-run
 *   node scripts/push-baidu-sitemap.mjs --sitemap https://m.tkmind.cn/sitemap.xml
 *
 * 环境变量:
 *   MINDSPACE_BAIDU_SITE          默认 m.tkmind.cn
 *   MINDSPACE_BAIDU_PUSH_TOKEN    必填
 *   BAIDU_PUSH_STATE_FILE         默认 data/baidu-push-state.json
 */

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');

function parseArgs(argv) {
  const args = {
    dryRun: false,
    sitemap: process.env.BAIDU_PUSH_SITEMAP ?? 'https://m.tkmind.cn/sitemap.xml',
    batchSize: Number(process.env.BAIDU_PUSH_BATCH_SIZE ?? 10),
    stateFile:
      process.env.BAIDU_PUSH_STATE_FILE ??
      path.join(ROOT, 'data', 'baidu-push-state.json'),
  };
  for (let i = 2; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--dry-run') args.dryRun = true;
    else if (arg === '--sitemap') args.sitemap = argv[++i] ?? args.sitemap;
    else if (arg === '--batch-size') args.batchSize = Number(argv[++i] ?? args.batchSize);
    else if (arg === '--state-file') args.stateFile = argv[++i] ?? args.stateFile;
  }
  return args;
}

function loadState(stateFile) {
  try {
    return JSON.parse(fs.readFileSync(stateFile, 'utf8'));
  } catch {
    return { pushed: {}, lastRunAt: null, history: [] };
  }
}

function saveState(stateFile, state) {
  fs.mkdirSync(path.dirname(stateFile), { recursive: true });
  fs.writeFileSync(stateFile, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
}

function extractSitemapUrls(xml) {
  return [...String(xml ?? '').matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim());
}

function isArchivedDiscoveryUrl(url) {
  return /\/_archived/i.test(String(url ?? ''));
}

function prioritizeUrls(urls) {
  const filtered = urls.filter((url) => url && !isArchivedDiscoveryUrl(url));
  const uUrls = filtered.filter((url) => url.includes('/u/'));
  const mindspaceUrls = filtered.filter((url) => url.includes('/MindSpace/'));
  const otherUrls = filtered.filter(
    (url) => !url.includes('/u/') && !url.includes('/MindSpace/'),
  );
  const seen = new Set();
  const ordered = [];
  for (const url of [...uUrls, ...otherUrls, ...mindspaceUrls]) {
    if (!url || seen.has(url)) continue;
    seen.add(url);
    ordered.push(url);
  }
  return ordered;
}

async function fetchSitemap(url) {
  const response = await fetch(url, { signal: AbortSignal.timeout(30_000) });
  if (!response.ok) {
    throw new Error(`sitemap fetch failed: ${response.status}`);
  }
  return response.text();
}

async function pingBaidu({ site, token, urls }) {
  const endpoint = `http://data.zz.baidu.com/urls?site=${encodeURIComponent(site)}&token=${encodeURIComponent(token)}`;
  const response = await fetch(endpoint, {
    method: 'POST',
    headers: { 'Content-Type': 'text/plain' },
    body: urls.join('\n'),
    signal: AbortSignal.timeout(60_000),
  });
  const payload = await response.json().catch(() => ({}));
  if (!response.ok) {
    const error = new Error(payload?.message || 'baidu_push_failed');
    error.code = payload?.error ?? response.status;
    error.details = payload;
    throw error;
  }
  return payload;
}

async function main() {
  const args = parseArgs(process.argv);
  const site = String(process.env.MINDSPACE_BAIDU_SITE ?? 'm.tkmind.cn').trim();
  const token = String(
    process.env.MINDSPACE_BAIDU_PUSH_TOKEN ??
      process.env.PLAZA_BAIDU_PUSH_TOKEN ??
      '',
  ).trim();
  if (!token) {
    console.error('缺少 MINDSPACE_BAIDU_PUSH_TOKEN');
    process.exit(1);
  }

  const xml = await fetchSitemap(args.sitemap);
  const allUrls = prioritizeUrls(extractSitemapUrls(xml));
  const state = loadState(args.stateFile);
  const pending = allUrls.filter((url) => !state.pushed[url]);

  console.log(
    JSON.stringify(
      {
        site,
        sitemap: args.sitemap,
        totalInSitemap: allUrls.length,
        alreadyPushed: allUrls.length - pending.length,
        pending: pending.length,
        dryRun: args.dryRun,
      },
      null,
      2,
    ),
  );

  if (args.dryRun || pending.length === 0) {
    return;
  }

  let pushedThisRun = 0;
  const runHistory = {
    at: new Date().toISOString(),
    attempted: 0,
    success: 0,
    batches: [],
  };

  for (let i = 0; i < pending.length; i += args.batchSize) {
    const batch = pending.slice(i, i + args.batchSize);
    runHistory.attempted += batch.length;
    try {
      const result = await pingBaidu({ site, token, urls: batch });
      const successCount = Number(result.success ?? 0);
      runHistory.success += successCount;
      runHistory.batches.push({ size: batch.length, result });
      for (const url of batch.slice(0, successCount)) {
        state.pushed[url] = runHistory.at;
        pushedThisRun += 1;
      }
      console.log(
        JSON.stringify(
          {
            batch: batch.length,
            success: successCount,
            remain: result.remain ?? null,
            notSameSite: result.not_same_site ?? null,
            notValid: result.not_valid ?? null,
          },
          null,
          2,
        ),
      );
      if (successCount < batch.length) break;
      if (Number(result.remain ?? 0) <= 0) break;
    } catch (error) {
      runHistory.batches.push({
        size: batch.length,
        error: error.message,
        details: error.details ?? null,
      });
      console.error(JSON.stringify({ batch: batch.length, error: error.message, details: error.details ?? null }, null, 2));
      break;
    }
  }

  state.lastRunAt = runHistory.at;
  state.history = [...(state.history ?? []), runHistory].slice(-30);
  saveState(args.stateFile, state);

  console.log(
    JSON.stringify(
      {
        pushedThisRun,
        totalTracked: Object.keys(state.pushed).length,
        pendingRemaining: allUrls.filter((url) => !state.pushed[url]).length,
      },
      null,
      2,
    ),
  );
}

main().catch((error) => {
  console.error(error?.stack || error?.message || String(error));
  process.exit(1);
});