memind/wechat-news-morning-dedup.test.mjs

import assert from 'node:assert/strict';
import test from 'node:test';
import {
  auditNewsMorningDuplicates,
  dedupeNewsMorningHtml,
  extractNewsMorningCards,
  extractNewsMorningSections,
  formatNewsMorningDedupeSummary,
  isArticlePermalink,
  isSameNewsEvent,
} from './wechat-news-morning-dedup.mjs';

function card({ id = '', title, url = '', cls = 'card' }) {
  return [
    `<div class="${cls}"${id ? ` data-event-key="${id}"` : ''}>`,
    '<div class="body">',
    `<h3>${title}</h3>`,
    '<p>正文摘要。</p>',
    `<div class="source"><a href="${url}">来源</a></div>`,
    '</div>',
    '</div>',
  ].join('\n');
}

function page(sections) {
  const body = sections
    .map(([id, cards]) => `<div class="section" id="${id}">\n<div class="section-title"><h2>${id}</h2></div>\n${cards.join('\n')}\n</div>`)
    .join('\n');
  const nav = sections.map(([id]) => `<a class="nav-link" href="#${id}">${id}</a>`).join('');
  return `<body>\n<nav class="nav">${nav}</nav>\n<div class="container">\n${body}\n</div>\n</body>`;
}

test('extractNewsMorningCards matches class tokens, not substrings', () => {
  const html = page([
    ['headlines', [card({ title: '甲事件发生了', cls: 'lead-card' })]],
    ['weather', ['<div class="weather-card"><div class="wc">北京</div></div>']],
    ['domestic', [card({ title: '乙事件发生了' })]],
  ]);
  const cards = extractNewsMorningCards(html);
  assert.equal(cards.length, 2, 'weather-card 不应被当作新闻卡片');
  assert.equal(cards[0].isLead, true);
  assert.deepEqual(cards.map((item) => item.title), ['甲事件发生了', '乙事件发生了']);
});

test('extractNewsMorningSections returns balanced section ranges', () => {
  const html = page([['headlines', [card({ title: '甲' })]], ['domestic', [card({ title: '乙' })]]]);
  const sections = extractNewsMorningSections(html);
  assert.deepEqual(sections.map((item) => item.id), ['headlines', 'domestic']);
  assert.ok(sections[0].end <= sections[1].start);
});

test('isArticlePermalink rejects section landing pages', () => {
  // 0918 生产页的 7 个「来源」全是这类栏目首页，不能当事件身份。
  assert.equal(isArticlePermalink('https://www.zzd.pub/'), false);
  assert.equal(isArticlePermalink('https://finance.sina.com.cn/stock/cpbd/'), false);
  assert.equal(isArticlePermalink('https://www.news.cn/politics'), false);
  assert.equal(isArticlePermalink('https://worldcup.cctv.com/index.shtml'), false);
  assert.equal(isArticlePermalink('not-a-url'), false);

  assert.equal(isArticlePermalink('https://www.news.cn/politics/20260918/abc/c.html'), true);
  assert.equal(isArticlePermalink('https://www.reuters.com/world/china/call-2026-09-18/'), true);
  assert.equal(isArticlePermalink('https://example.com/a/b/c-some-slug'), true);
});

test('isSameNewsEvent catches Chinese paraphrases that word overlap misses', () => {
  assert.equal(
    isSameNewsEvent('王毅同美国国务卿鲁比奥通电话，双方就中美关系交换意见', '王毅同鲁比奥通电话，就中美关系交换意见'),
    true,
  );
  assert.equal(
    isSameNewsEvent('第23届中国—东盟博览会南宁开幕，东帝汶首次以共办方身份参展', '东博会南宁开幕，东帝汶首次以共办方身份参展'),
    true,
  );
  assert.equal(
    isSameNewsEvent('国际油价高位震荡，地缘因素扰动原油市场', 'A股市场关注政策与业绩信号，AI、半导体受关注'),
    false,
  );
});

test('auditNewsMorningDuplicates prefers data-event-key over text similarity', () => {
  const html = page([
    ['headlines', [card({ id: 'wangyi-rubio-call', title: '王毅同鲁比奥通电话' })]],
    ['world', [card({ id: 'wangyi-rubio-call', title: '中美高层通话为紧张局势提供沟通渠道' })]],
  ]);
  const report = auditNewsMorningDuplicates(html);
  assert.equal(report.total, 2);
  assert.equal(report.uniqueCount, 1);
  assert.equal(report.duplicates[0].reason, 'event-key');
});

test('auditNewsMorningDuplicates skips history and knowledge sections', () => {
  const html = page([
    ['headlines', [card({ title: '甲事件今天发生了重要变化' })]],
    ['history', [card({ title: '甲事件今天发生了重要变化' })]],
  ]);
  const report = auditNewsMorningDuplicates(html);
  assert.equal(report.total, 1);
  assert.equal(report.duplicateCount, 0);
});

test('dedupeNewsMorningHtml removes duplicate cards and keeps div balance', () => {
  const html = page([
    ['headlines', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
    ['world', [
      card({ id: 'a', title: '甲事件的另一个说法' }),
      card({ id: 'b', title: '乙事件在海外引发广泛关注' }),
    ]],
  ]);
  const result = dedupeNewsMorningHtml(html);

  assert.equal(result.changed, true);
  assert.equal(result.report.duplicateCount, 1);
  assert.ok(!result.html.includes('甲事件的另一个说法'));
  assert.ok(result.html.includes('乙事件在海外引发广泛关注'));
  assert.deepEqual(result.removedSectionIds, [], 'world 还剩一条，不应整节删除');

  const opens = (result.html.match(/<div\b/gi) ?? []).length;
  const closes = (result.html.match(/<\/div>/gi) ?? []).length;
  assert.equal(opens, closes, 'div 必须配平');
});

test('dedupeNewsMorningHtml drops emptied flex sections and their nav anchor', () => {
  const html = page([
    ['headlines', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
    ['living', [card({ id: 'a', title: '甲事件换个角度再写一遍' })]],
  ]);
  const result = dedupeNewsMorningHtml(html);

  assert.deepEqual(result.removedSectionIds, ['living']);
  assert.ok(!result.html.includes('id="living"'));
  assert.ok(!result.html.includes('href="#living"'), '导航锚点必须一起移除');
  assert.ok(result.html.includes('id="headlines"'));
});

test('dedupeNewsMorningHtml never empties protected sections', () => {
  const html = page([
    ['headlines', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
    ['weather', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
  ]);
  const result = dedupeNewsMorningHtml(html);
  assert.deepEqual(result.removedSectionIds, []);
  assert.ok(result.html.includes('id="weather"'));
});

test('dedupeNewsMorningHtml is a no-op on a clean page', () => {
  const html = page([
    ['headlines', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
    ['world', [card({ id: 'b', title: '乙事件在海外引发广泛关注' })]],
  ]);
  const result = dedupeNewsMorningHtml(html);
  assert.equal(result.changed, false);
  assert.equal(result.html, html);
  assert.match(formatNewsMorningDedupeSummary(result), /卡片 2 条，去重后 2 条/);
});

test('formatNewsMorningDedupeSummary reports reasons and removed sections', () => {
  const html = page([
    ['headlines', [card({ id: 'a', title: '甲事件今天发生了重要变化' })]],
    ['living', [card({ id: 'a', title: '甲事件换个角度再写一遍' })]],
  ]);
  const summary = formatNewsMorningDedupeSummary(dedupeNewsMorningHtml(html));
  assert.match(summary, /剔除 1 条重复（event-key:1）/);
  assert.match(summary, /移除空栏目 living/);
});

test('countNewsMorningStoryCards counts news001 and news002 card shapes', async () => {
  const { countNewsMorningStoryCards } = await import('./wechat-news-morning-dedup.mjs');
  const html = page([
    ['headlines', [
      card({ title: '头条一条', cls: 'lead-card' }),
      card({ title: '头条两条', cls: 'card highlight-box' }),
    ]],
    ['domestic', [card({ title: '国内一条' })]],
    ['weather', ['<div class="weather-card"><div class="wc">北京</div></div>']],
  ]);
  assert.equal(countNewsMorningStoryCards(html), 3, 'lead-card/highlight-box 要计入，weather-card 不计入');
});