diff --git a/docs/news-engine-mvp.md b/docs/news-engine-mvp.md new file mode 100644 index 0000000..db8ae24 --- /dev/null +++ b/docs/news-engine-mvp.md @@ -0,0 +1,58 @@ +# MeMind News Engine MVP + +## Boundary + +- `memind-news-engine` owns collection, event clustering, ranking, and briefing candidates. +- `Memind` consumes ranked candidates and owns page generation and delivery. +- `memind_adm` owns source configuration, ranking policy editing, review, run monitoring, and publish controls. +- The legacy `ops/` UI is not extended. + +## Current pipeline + +```text +Memind SearXNG query groups + -> memind-news-engine HTTP API + -> event clustering by canonical URL and title similarity + -> deterministic rule score + -> optional structured LLM assessment + -> global score plus optional user-interest score + -> breaking-news override + -> category quotas and source diversity + -> ranked candidates in the existing briefing prompt + -> existing page generation, dedupe, image, and WeChat delivery guards +``` + +The first slice is deliberately deterministic. The model may provide the six semantic dimensions, but application code calculates the weighted score and owns final selection. + +## Score contract + +LLM assessment values are 0-10: + +```json +{ + "importance": 8, + "novelty": 9, + "impact": 8, + "credibility": 9, + "relevance": 8, + "actionability": 7, + "category": "ai_tech", + "reason": "A flagship model release changes capability and API economics." +} +``` + +Semantic score weights are 25% importance, 20% impact, 15% novelty, 15% credibility, 15% relevance, and 10% actionability. When the LLM assessment is present, global score is 35% rule score plus 65% semantic score. When a user-interest score is present, final score is 70% global plus 30% personal. + +Breaking override is active when importance is at least 9, there are at least two distinct sources, and credibility is at least 8. + +## Runtime controls + +- `MEMIND_NEWS_ENGINE_DAILY_LIMIT`: selected event count, default `15`. +- `MEMIND_NEWS_ENGINE_MINIMUM_SCORE`: minimum final score, default `0` during the observation phase. +- `MEMIND_NEWS_ENGINE_URL`: independent service base URL; ranking is skipped when unset. +- `MEMIND_NEWS_ENGINE_API_TOKEN`: service-to-service bearer token. +- `MEMIND_NEWS_ENGINE_TIMEOUT_MS`: request timeout, default `5000`. + +## Next slice + +The engine repository now owns persistence migrations and the ranking API. Next work is the bounded LLM scoring worker, collection scheduler, and `memind_adm` management UI. diff --git a/news-engine-client.mjs b/news-engine-client.mjs new file mode 100644 index 0000000..ce069ea --- /dev/null +++ b/news-engine-client.mjs @@ -0,0 +1,87 @@ +const EMPTY_RANKING = Object.freeze({ + candidateCount: 0, + eventCount: 0, + events: [], + selected: [], +}); + +export function emptyNewsEngineRanking() { + return { + candidateCount: 0, + eventCount: 0, + events: [], + selected: [], + }; +} + +export function resolveNewsEngineEndpoint(env = process.env) { + return String(env.MEMIND_NEWS_ENGINE_URL ?? '').trim().replace(/\/+$/, ''); +} + +export async function rankNewsWithEngine(groups, { + env = process.env, + now = Date.now(), + fetchImpl = fetch, + logger = console, +} = {}) { + const endpoint = resolveNewsEngineEndpoint(env); + if (!endpoint) return emptyNewsEngineRanking(); + + const timeoutMs = Math.max( + 500, + Number(env.MEMIND_NEWS_ENGINE_TIMEOUT_MS ?? 5000) || 5000, + ); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + timer.unref?.(); + try { + const token = String(env.MEMIND_NEWS_ENGINE_API_TOKEN ?? '').trim(); + const response = await fetchImpl(`${endpoint}/v1/rank`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + body: JSON.stringify({ + groups: Array.isArray(groups) ? groups : [], + options: { + now, + limit: Number(env.MEMIND_NEWS_ENGINE_DAILY_LIMIT ?? 15) || 15, + minimumScore: Number(env.MEMIND_NEWS_ENGINE_MINIMUM_SCORE ?? 0) || 0, + }, + }), + signal: controller.signal, + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const ranking = await response.json(); + if (!ranking || !Array.isArray(ranking.events) || !Array.isArray(ranking.selected)) { + throw new Error('invalid ranking response'); + } + return ranking; + } catch (error) { + logger.warn?.( + '[NewsEngine] ranking unavailable:', + error instanceof Error ? error.message : error, + ); + return { ...EMPTY_RANKING, events: [], selected: [] }; + } finally { + clearTimeout(timer); + } +} + +export function formatRankedNewsCandidates(ranking) { + const selected = Array.isArray(ranking?.selected) ? ranking.selected : []; + if (!selected.length) return ''; + const lines = [ + '【News Engine 编辑候选】以下事件已完成聚类、规则评分和栏目配额选择。生成早报时优先采用;must_include=true 的事件必须进入正文。', + ]; + for (const event of selected) { + const source = event.primaryArticle?.sourceDomain || 'unknown'; + lines.push( + `${event.rank}. [${event.category}] ${event.canonicalTitle}`, + ` score=${event.finalScore} sources=${event.sourceCount} must_include=${event.mustInclude} primary=${source}`, + ); + if (event.primaryArticle?.url) lines.push(` ${event.primaryArticle.url}`); + } + return lines.join('\n'); +} diff --git a/news-engine-client.test.mjs b/news-engine-client.test.mjs new file mode 100644 index 0000000..bd464cd --- /dev/null +++ b/news-engine-client.test.mjs @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + formatRankedNewsCandidates, + rankNewsWithEngine, + resolveNewsEngineEndpoint, +} from './news-engine-client.mjs'; + +test('news engine client is disabled without an endpoint', async () => { + assert.equal(resolveNewsEngineEndpoint({}), ''); + const ranking = await rankNewsWithEngine([], { + env: {}, + fetchImpl: async () => { throw new Error('must not fetch'); }, + }); + assert.deepEqual(ranking.selected, []); +}); + +test('news engine client sends groups and authentication to the independent service', async () => { + const calls = []; + const ranking = await rankNewsWithEngine([{ id: 'tech', results: [] }], { + now: 123, + env: { + MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092/', + MEMIND_NEWS_ENGINE_API_TOKEN: 'service-token', + MEMIND_NEWS_ENGINE_DAILY_LIMIT: '12', + }, + fetchImpl: async (url, init) => { + calls.push({ url, init, body: JSON.parse(init.body) }); + return { + ok: true, + async json() { + return { + candidateCount: 1, + eventCount: 1, + events: [], + selected: [{ + rank: 1, + category: 'ai_tech', + canonicalTitle: '模型发布', + finalScore: 8.8, + sourceCount: 2, + mustInclude: true, + primaryArticle: { sourceDomain: 'example.com', url: 'https://example.com/a' }, + }], + }; + }, + }; + }, + }); + assert.equal(calls[0].url, 'http://127.0.0.1:8092/v1/rank'); + assert.equal(calls[0].init.headers.authorization, 'Bearer service-token'); + assert.equal(calls[0].body.options.limit, 12); + assert.equal(ranking.selected[0].mustInclude, true); + assert.match(formatRankedNewsCandidates(ranking), /模型发布/); +}); + +test('news engine client fails open when the service is unavailable', async () => { + const warnings = []; + const ranking = await rankNewsWithEngine([], { + env: { MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092' }, + fetchImpl: async () => { throw new Error('offline'); }, + logger: { warn: (...args) => warnings.push(args) }, + }); + assert.deepEqual(ranking.selected, []); + assert.match(String(warnings[0]), /offline/); +}); diff --git a/package.json b/package.json index 47f1937..753415c 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-voice-reco.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs scheduled-task-intent.test.mjs scheduled-task-service.test.mjs scheduled-task-executor.test.mjs scheduled-task-worker.test.mjs scheduled-task-worker-config.test.mjs wechat-news-morning-draft.test.mjs wechat-news-morning-draft-worker.test.mjs wechat-news-morning-draft-worker-config.test.mjs wechat/handlers/scheduled-task.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs health-channel-state.test.mjs health-intent-rules.test.mjs health-extraction.test.mjs health-observation-validate.test.mjs health-baseline-maturity.test.mjs health-publish-guard.test.mjs health-p0-experiment.test.mjs health-wechat-channel.test.mjs server/portal-health-routes.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-long-image.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-save-service.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-wechat-html-delivery.test.mjs mindspace-wechat-mp-config.test.mjs mindspace-wechat-page-draft.test.mjs server/portal-mindspace-wechat-routes.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-artifact-service.test.mjs mindspace-conversation-package-audit.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-workspace-publication-delivery-service.test.mjs mindspace-workspace-tool-service.test.mjs mindspace-mcp-scoped-token.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", + "test:news-engine": "node --test news-engine-client.test.mjs wechat-news-morning-search.test.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", "test:mindspace-service": "node --test mindspace-service/mindspace-rpc-server.test.mjs", "verify:chat-finish-sync": "node scripts/verify-chat-finish-sync.mjs", diff --git a/wechat-news-morning-search.mjs b/wechat-news-morning-search.mjs index 18502b5..4886357 100644 --- a/wechat-news-morning-search.mjs +++ b/wechat-news-morning-search.mjs @@ -2,6 +2,10 @@ import { searchSearxng } from './mindsearch-providers.mjs'; import { resolvePortalSearxngEndpoint } from './mindsearch-prefetch.mjs'; import { getLocalParts } from './schedule-time.mjs'; import { enrichNewsMorningSearchGroupsWithImages } from './wechat-news-morning-images.mjs'; +import { + formatRankedNewsCandidates, + rankNewsWithEngine, +} from './news-engine-client.mjs'; function envFlag(value, fallback = false) { const raw = String(value ?? '').trim().toLowerCase(); @@ -191,7 +195,11 @@ function truncateSnippet(value, max = 90) { return text.length > max ? `${text.slice(0, max - 1)}…` : text; } -export function formatNewsMorningSearchBrief(groups = [], { dateLabel = null, dedupeRemovedCount = 0 } = {}) { +export function formatNewsMorningSearchBrief(groups = [], { + dateLabel = null, + dedupeRemovedCount = 0, + ranking = null, +} = {}) { const lines = [ '【专用联网搜索预取】以下结果来自专用新闻检索(与 web_search 并列,不是替代)。', '写页面时必须:1)优先从下列结果覆盖各栏目;2)同一轮再调用 tkmind_search(type=news) 与 web_search 补漏并核对来源;3)禁止只使用 web_search;4)全页去重:同一 URL/同一事件只允许出现一次,优先保留要闻/国内/国际/财经/科技,扩展栏目用不同角度或跳过;5)英文栏目标题可英文,正文用中文摘要并附原文链接。', @@ -201,6 +209,8 @@ export function formatNewsMorningSearchBrief(groups = [], { dateLabel = null, de if (dedupeRemovedCount > 0) { lines.push(`预取跨栏目去重:已剔除 ${dedupeRemovedCount} 条重复 URL/标题,写页时仍须对双路搜索结果再次去重。`); } + const rankedCandidates = formatRankedNewsCandidates(ranking); + if (rankedCandidates) lines.push('', rankedCandidates); lines.push(''); let hitCount = 0; @@ -262,18 +272,33 @@ export async function prefetchNewsMorningSearxngBundle({ env = process.env, searchImpl = searchSearxng, fetchImpl = fetch, + newsEngineFetchImpl = fetch, + logger = console, limit = 5, } = {}) { const dateLabel = formatLocalDateParts(now, timezone); const queries = buildNewsMorningSearchQueries(dateLabel); const emptyGroups = queries.map((item) => ({ ...item, results: [] })); const emptyBrief = formatNewsMorningSearchBrief(emptyGroups, { dateLabel }); + const emptyRanking = { candidateCount: 0, eventCount: 0, events: [], selected: [] }; if (!shouldPrefetchNewsMorningSearxng(env)) { - return { brief: emptyBrief, groups: emptyGroups, dateLabel, dedupeRemovedCount: 0 }; + return { + brief: emptyBrief, + groups: emptyGroups, + ranking: emptyRanking, + dateLabel, + dedupeRemovedCount: 0, + }; } const endpoint = resolvePortalSearxngEndpoint(env); if (!endpoint) { - return { brief: emptyBrief, groups: emptyGroups, dateLabel, dedupeRemovedCount: 0 }; + return { + brief: emptyBrief, + groups: emptyGroups, + ranking: emptyRanking, + dateLabel, + dedupeRemovedCount: 0, + }; } const timeoutMs = Number(env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000) || 8000; const useNewsCategory = shouldUseNewsMorningNewsCategory(env); @@ -294,6 +319,12 @@ export async function prefetchNewsMorningSearxngBundle({ ).slice(0, limit); return { ...item, results }; }); + const ranking = await rankNewsWithEngine(groups, { + env, + now, + fetchImpl: newsEngineFetchImpl, + logger, + }); const { groups: dedupedGroups, removedCount } = dedupeNewsMorningSearchGroups(groups); const imageEnrichment = await enrichNewsMorningSearchGroupsWithImages(dedupedGroups, { fetchImpl, @@ -305,8 +336,10 @@ export async function prefetchNewsMorningSearxngBundle({ brief: formatNewsMorningSearchBrief(imageEnrichment.groups, { dateLabel, dedupeRemovedCount: removedCount, + ranking, }), groups: imageEnrichment.groups, + ranking, dateLabel, dedupeRemovedCount: removedCount, }; diff --git a/wechat-news-morning-search.test.mjs b/wechat-news-morning-search.test.mjs index caa8e25..2d92667 100644 --- a/wechat-news-morning-search.test.mjs +++ b/wechat-news-morning-search.test.mjs @@ -139,7 +139,10 @@ test('prefetchNewsMorningSearxngBrief queries news category then falls back', as test('prefetchNewsMorningSearxngBundle returns structured groups for fallback', async () => { const bundle = await prefetchNewsMorningSearxngBundle({ now: Date.parse('2026-09-15T05:00:00+08:00'), - env: { TKMIND_SEARCH_SEARXNG_URL: 'http://127.0.0.1:20080/search' }, + env: { + TKMIND_SEARCH_SEARXNG_URL: 'http://127.0.0.1:20080/search', + MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092', + }, searchImpl: async () => ([ { title: '样例新闻', url: 'https://news.example/story', snippet: '摘要' }, ]), @@ -148,10 +151,35 @@ test('prefetchNewsMorningSearxngBundle returns structured groups for fallback', headers: { get: () => 'text/html' }, text: async () => '', }), + newsEngineFetchImpl: async () => ({ + ok: true, + async json() { + return { + candidateCount: 1, + eventCount: 1, + events: [], + selected: [{ + rank: 1, + category: 'international', + canonicalTitle: '样例新闻', + finalScore: 8.1, + sourceCount: 2, + mustInclude: false, + primaryArticle: { + sourceDomain: 'news.example', + url: 'https://news.example/story', + }, + }], + }; + }, + }), }); assert.ok(Array.isArray(bundle.groups)); assert.ok(bundle.groups.length > 10); assert.ok(bundle.groups.some((group) => group.results.length > 0)); + assert.ok(bundle.ranking.eventCount > 0); + assert.ok(bundle.ranking.selected.length > 0); + assert.match(bundle.brief, /News Engine 编辑候选/); assert.match(bundle.brief, /专用联网搜索预取/); }); @@ -177,13 +205,19 @@ test('prefetchNewsMorningSearxngBrief skips news category by default', async () }); test('prefetchNewsMorningSearxngBrief skips live calls without endpoint', async () => { - const brief = await prefetchNewsMorningSearxngBrief({ + const bundle = await prefetchNewsMorningSearxngBundle({ now: Date.parse('2026-09-15T05:00:00+08:00'), env: {}, searchImpl: async () => { throw new Error('should not run'); }, }); - assert.match(brief, /预取没有返回条目|本路预取为空/); - assert.match(brief, /tkmind_search/); + assert.deepEqual(bundle.ranking, { + candidateCount: 0, + eventCount: 0, + events: [], + selected: [], + }); + assert.match(bundle.brief, /预取没有返回条目|本路预取为空/); + assert.match(bundle.brief, /tkmind_search/); });