feat: consume independent news engine

docs/news-engine-mvp.md

@@ -0,0 +1,58 @@
+# MeMind News Engine MVP
+
+## Boundary
+
+- `memind-news-engine` owns collection, event clustering, ranking, and briefing candidates.
+- `Memind` consumes ranked candidates and owns page generation and delivery.
+- `memind_adm` owns source configuration, ranking policy editing, review, run monitoring, and publish controls.
+- The legacy `ops/` UI is not extended.
+
+## Current pipeline
+
+```text
+Memind SearXNG query groups
+  -> memind-news-engine HTTP API
+  -> event clustering by canonical URL and title similarity
+  -> deterministic rule score
+  -> optional structured LLM assessment
+  -> global score plus optional user-interest score
+  -> breaking-news override
+  -> category quotas and source diversity
+  -> ranked candidates in the existing briefing prompt
+  -> existing page generation, dedupe, image, and WeChat delivery guards
+```
+
+The first slice is deliberately deterministic. The model may provide the six semantic dimensions, but application code calculates the weighted score and owns final selection.
+
+## Score contract
+
+LLM assessment values are 0-10:
+
+```json
+{
+  "importance": 8,
+  "novelty": 9,
+  "impact": 8,
+  "credibility": 9,
+  "relevance": 8,
+  "actionability": 7,
+  "category": "ai_tech",
+  "reason": "A flagship model release changes capability and API economics."
+}
+```
+
+Semantic score weights are 25% importance, 20% impact, 15% novelty, 15% credibility, 15% relevance, and 10% actionability. When the LLM assessment is present, global score is 35% rule score plus 65% semantic score. When a user-interest score is present, final score is 70% global plus 30% personal.
+
+Breaking override is active when importance is at least 9, there are at least two distinct sources, and credibility is at least 8.
+
+## Runtime controls
+
+- `MEMIND_NEWS_ENGINE_DAILY_LIMIT`: selected event count, default `15`.
+- `MEMIND_NEWS_ENGINE_MINIMUM_SCORE`: minimum final score, default `0` during the observation phase.
+- `MEMIND_NEWS_ENGINE_URL`: independent service base URL; ranking is skipped when unset.
+- `MEMIND_NEWS_ENGINE_API_TOKEN`: service-to-service bearer token.
+- `MEMIND_NEWS_ENGINE_TIMEOUT_MS`: request timeout, default `5000`.
+
+## Next slice
+
+The engine repository now owns persistence migrations and the ranking API. Next work is the bounded LLM scoring worker, collection scheduler, and `memind_adm` management UI.

news-engine-client.mjs

@@ -0,0 +1,87 @@
+const EMPTY_RANKING = Object.freeze({
+  candidateCount: 0,
+  eventCount: 0,
+  events: [],
+  selected: [],
+});
+
+export function emptyNewsEngineRanking() {
+  return {
+    candidateCount: 0,
+    eventCount: 0,
+    events: [],
+    selected: [],
+  };
+}
+
+export function resolveNewsEngineEndpoint(env = process.env) {
+  return String(env.MEMIND_NEWS_ENGINE_URL ?? '').trim().replace(/\/+$/, '');
+}
+
+export async function rankNewsWithEngine(groups, {
+  env = process.env,
+  now = Date.now(),
+  fetchImpl = fetch,
+  logger = console,
+} = {}) {
+  const endpoint = resolveNewsEngineEndpoint(env);
+  if (!endpoint) return emptyNewsEngineRanking();
+
+  const timeoutMs = Math.max(
+    500,
+    Number(env.MEMIND_NEWS_ENGINE_TIMEOUT_MS ?? 5000) || 5000,
+  );
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  timer.unref?.();
+  try {
+    const token = String(env.MEMIND_NEWS_ENGINE_API_TOKEN ?? '').trim();
+    const response = await fetchImpl(`${endpoint}/v1/rank`, {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        ...(token ? { authorization: `Bearer ${token}` } : {}),
+      },
+      body: JSON.stringify({
+        groups: Array.isArray(groups) ? groups : [],
+        options: {
+          now,
+          limit: Number(env.MEMIND_NEWS_ENGINE_DAILY_LIMIT ?? 15) || 15,
+          minimumScore: Number(env.MEMIND_NEWS_ENGINE_MINIMUM_SCORE ?? 0) || 0,
+        },
+      }),
+      signal: controller.signal,
+    });
+    if (!response.ok) throw new Error(`HTTP ${response.status}`);
+    const ranking = await response.json();
+    if (!ranking || !Array.isArray(ranking.events) || !Array.isArray(ranking.selected)) {
+      throw new Error('invalid ranking response');
+    }
+    return ranking;
+  } catch (error) {
+    logger.warn?.(
+      '[NewsEngine] ranking unavailable:',
+      error instanceof Error ? error.message : error,
+    );
+    return { ...EMPTY_RANKING, events: [], selected: [] };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export function formatRankedNewsCandidates(ranking) {
+  const selected = Array.isArray(ranking?.selected) ? ranking.selected : [];
+  if (!selected.length) return '';
+  const lines = [
+    '【News Engine 编辑候选】以下事件已完成聚类、规则评分和栏目配额选择。生成早报时优先采用；must_include=true 的事件必须进入正文。',
+  ];
+  for (const event of selected) {
+    const source = event.primaryArticle?.sourceDomain || 'unknown';
+    lines.push(
+      `${event.rank}. [${event.category}] ${event.canonicalTitle}`,
+      `   score=${event.finalScore} sources=${event.sourceCount} must_include=${event.mustInclude} primary=${source}`,
+    );
+    if (event.primaryArticle?.url) lines.push(`   ${event.primaryArticle.url}`);
+  }
+  return lines.join('\n');
+}

news-engine-client.test.mjs

@@ -0,0 +1,66 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  formatRankedNewsCandidates,
+  rankNewsWithEngine,
+  resolveNewsEngineEndpoint,
+} from './news-engine-client.mjs';
+
+test('news engine client is disabled without an endpoint', async () => {
+  assert.equal(resolveNewsEngineEndpoint({}), '');
+  const ranking = await rankNewsWithEngine([], {
+    env: {},
+    fetchImpl: async () => { throw new Error('must not fetch'); },
+  });
+  assert.deepEqual(ranking.selected, []);
+});
+
+test('news engine client sends groups and authentication to the independent service', async () => {
+  const calls = [];
+  const ranking = await rankNewsWithEngine([{ id: 'tech', results: [] }], {
+    now: 123,
+    env: {
+      MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092/',
+      MEMIND_NEWS_ENGINE_API_TOKEN: 'service-token',
+      MEMIND_NEWS_ENGINE_DAILY_LIMIT: '12',
+    },
+    fetchImpl: async (url, init) => {
+      calls.push({ url, init, body: JSON.parse(init.body) });
+      return {
+        ok: true,
+        async json() {
+          return {
+            candidateCount: 1,
+            eventCount: 1,
+            events: [],
+            selected: [{
+              rank: 1,
+              category: 'ai_tech',
+              canonicalTitle: '模型发布',
+              finalScore: 8.8,
+              sourceCount: 2,
+              mustInclude: true,
+              primaryArticle: { sourceDomain: 'example.com', url: 'https://example.com/a' },
+            }],
+          };
+        },
+      };
+    },
+  });
+  assert.equal(calls[0].url, 'http://127.0.0.1:8092/v1/rank');
+  assert.equal(calls[0].init.headers.authorization, 'Bearer service-token');
+  assert.equal(calls[0].body.options.limit, 12);
+  assert.equal(ranking.selected[0].mustInclude, true);
+  assert.match(formatRankedNewsCandidates(ranking), /模型发布/);
+});
+
+test('news engine client fails open when the service is unavailable', async () => {
+  const warnings = [];
+  const ranking = await rankNewsWithEngine([], {
+    env: { MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092' },
+    fetchImpl: async () => { throw new Error('offline'); },
+    logger: { warn: (...args) => warnings.push(args) },
+  });
+  assert.deepEqual(ranking.selected, []);
+  assert.match(String(warnings[0]), /offline/);
+});

wechat-news-morning-search.mjs

@@ -2,6 +2,10 @@ import { searchSearxng } from './mindsearch-providers.mjs';
 import { resolvePortalSearxngEndpoint } from './mindsearch-prefetch.mjs';
 import { getLocalParts } from './schedule-time.mjs';
 import { enrichNewsMorningSearchGroupsWithImages } from './wechat-news-morning-images.mjs';
+import {
+  formatRankedNewsCandidates,
+  rankNewsWithEngine,
+} from './news-engine-client.mjs';
 
 function envFlag(value, fallback = false) {
   const raw = String(value ?? '').trim().toLowerCase();
@@ -191,7 +195,11 @@ function truncateSnippet(value, max = 90) {
   return text.length > max ? `${text.slice(0, max - 1)}…` : text;
 }
 
-export function formatNewsMorningSearchBrief(groups = [], { dateLabel = null, dedupeRemovedCount = 0 } = {}) {
+export function formatNewsMorningSearchBrief(groups = [], {
+  dateLabel = null,
+  dedupeRemovedCount = 0,
+  ranking = null,
+} = {}) {
   const lines = [
     '【专用联网搜索预取】以下结果来自专用新闻检索（与 web_search 并列，不是替代）。',
     '写页面时必须：1）优先从下列结果覆盖各栏目；2）同一轮再调用 tkmind_search(type=news) 与 web_search 补漏并核对来源；3）禁止只使用 web_search；4）全页去重：同一 URL/同一事件只允许出现一次，优先保留要闻/国内/国际/财经/科技，扩展栏目用不同角度或跳过；5）英文栏目标题可英文，正文用中文摘要并附原文链接。',
@@ -201,6 +209,8 @@ export function formatNewsMorningSearchBrief(groups = [], { dateLabel = null, de
   if (dedupeRemovedCount > 0) {
     lines.push(`预取跨栏目去重：已剔除 ${dedupeRemovedCount} 条重复 URL/标题，写页时仍须对双路搜索结果再次去重。`);
   }
+  const rankedCandidates = formatRankedNewsCandidates(ranking);
+  if (rankedCandidates) lines.push('', rankedCandidates);
   lines.push('');
 
   let hitCount = 0;
@@ -262,18 +272,33 @@ export async function prefetchNewsMorningSearxngBundle({
   env = process.env,
   searchImpl = searchSearxng,
   fetchImpl = fetch,
+  newsEngineFetchImpl = fetch,
+  logger = console,
   limit = 5,
 } = {}) {
   const dateLabel = formatLocalDateParts(now, timezone);
   const queries = buildNewsMorningSearchQueries(dateLabel);
   const emptyGroups = queries.map((item) => ({ ...item, results: [] }));
   const emptyBrief = formatNewsMorningSearchBrief(emptyGroups, { dateLabel });
+  const emptyRanking = { candidateCount: 0, eventCount: 0, events: [], selected: [] };
   if (!shouldPrefetchNewsMorningSearxng(env)) {
-    return { brief: emptyBrief, groups: emptyGroups, dateLabel, dedupeRemovedCount: 0 };
+    return {
+      brief: emptyBrief,
+      groups: emptyGroups,
+      ranking: emptyRanking,
+      dateLabel,
+      dedupeRemovedCount: 0,
+    };
   }
   const endpoint = resolvePortalSearxngEndpoint(env);
   if (!endpoint) {
-    return { brief: emptyBrief, groups: emptyGroups, dateLabel, dedupeRemovedCount: 0 };
+    return {
+      brief: emptyBrief,
+      groups: emptyGroups,
+      ranking: emptyRanking,
+      dateLabel,
+      dedupeRemovedCount: 0,
+    };
   }
   const timeoutMs = Number(env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000) || 8000;
   const useNewsCategory = shouldUseNewsMorningNewsCategory(env);
@@ -294,6 +319,12 @@ export async function prefetchNewsMorningSearxngBundle({
     ).slice(0, limit);
     return { ...item, results };
   });
+  const ranking = await rankNewsWithEngine(groups, {
+    env,
+    now,
+    fetchImpl: newsEngineFetchImpl,
+    logger,
+  });
   const { groups: dedupedGroups, removedCount } = dedupeNewsMorningSearchGroups(groups);
   const imageEnrichment = await enrichNewsMorningSearchGroupsWithImages(dedupedGroups, {
     fetchImpl,
@@ -305,8 +336,10 @@ export async function prefetchNewsMorningSearxngBundle({
     brief: formatNewsMorningSearchBrief(imageEnrichment.groups, {
       dateLabel,
       dedupeRemovedCount: removedCount,
+      ranking,
     }),
     groups: imageEnrichment.groups,
+    ranking,
     dateLabel,
     dedupeRemovedCount: removedCount,
   };

wechat-news-morning-search.test.mjs

@@ -139,7 +139,10 @@ test('prefetchNewsMorningSearxngBrief queries news category then falls back', as
 test('prefetchNewsMorningSearxngBundle returns structured groups for fallback', async () => {
   const bundle = await prefetchNewsMorningSearxngBundle({
     now: Date.parse('2026-09-15T05:00:00+08:00'),
-    env: { TKMIND_SEARCH_SEARXNG_URL: 'http://127.0.0.1:20080/search' },
+    env: {
+      TKMIND_SEARCH_SEARXNG_URL: 'http://127.0.0.1:20080/search',
+      MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092',
+    },
     searchImpl: async () => ([
       { title: '样例新闻', url: 'https://news.example/story', snippet: '摘要' },
     ]),
@@ -148,10 +151,35 @@ test('prefetchNewsMorningSearxngBundle returns structured groups for fallback',
       headers: { get: () => 'text/html' },
       text: async () => '<meta property="og:image" content="https://cdn.example/story.jpg">',
     }),
+    newsEngineFetchImpl: async () => ({
+      ok: true,
+      async json() {
+        return {
+          candidateCount: 1,
+          eventCount: 1,
+          events: [],
+          selected: [{
+            rank: 1,
+            category: 'international',
+            canonicalTitle: '样例新闻',
+            finalScore: 8.1,
+            sourceCount: 2,
+            mustInclude: false,
+            primaryArticle: {
+              sourceDomain: 'news.example',
+              url: 'https://news.example/story',
+            },
+          }],
+        };
+      },
+    }),
   });
   assert.ok(Array.isArray(bundle.groups));
   assert.ok(bundle.groups.length > 10);
   assert.ok(bundle.groups.some((group) => group.results.length > 0));
+  assert.ok(bundle.ranking.eventCount > 0);
+  assert.ok(bundle.ranking.selected.length > 0);
+  assert.match(bundle.brief, /News Engine 编辑候选/);
   assert.match(bundle.brief, /专用联网搜索预取/);
 });
 
@@ -177,13 +205,19 @@ test('prefetchNewsMorningSearxngBrief skips news category by default', async ()
 });
 
 test('prefetchNewsMorningSearxngBrief skips live calls without endpoint', async () => {
-  const brief = await prefetchNewsMorningSearxngBrief({
+  const bundle = await prefetchNewsMorningSearxngBundle({
     now: Date.parse('2026-09-15T05:00:00+08:00'),
     env: {},
     searchImpl: async () => {
       throw new Error('should not run');
     },
   });
-  assert.match(brief, /预取没有返回条目|本路预取为空/);
-  assert.match(brief, /tkmind_search/);
+  assert.deepEqual(bundle.ranking, {
+    candidateCount: 0,
+    eventCount: 0,
+    events: [],
+    selected: [],
+  });
+  assert.match(bundle.brief, /预取没有返回条目|本路预取为空/);
+  assert.match(bundle.brief, /tkmind_search/);
 });