From 4a26db5e84c51fba7962b3dd854a1af5c3fcbc52 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 22 Sep 2026 14:05:40 +0800 Subject: [PATCH] feat(news): ship news002 fill gate and full WeChat draft push Lock scheduled morning news to news002 with section-fill gates, headline lead-card normalization, SearXNG top-up to max, and push that keeps imageless cards as text while requiring uploaded images where present. Co-authored-by: Cursor --- .env.example | 12 + docs/news-morning-templates.md | 3 +- news-engine-client.mjs | 122 +++++++ news-engine-client.test.mjs | 99 ++++++ news-morning-templates.mjs | 203 +++++++++++- news-morning-templates.test.mjs | 22 +- package.json | 1 + .../run-news-morning-generate-push-now.mjs | 305 ++++++++++++++++++ scripts/trigger-news002-generate-draft.mjs | 14 +- scripts/verify-news-engine-integration.mjs | 60 ++++ wechat-news-morning-draft-worker.mjs | 91 ++++-- wechat-news-morning-draft.mjs | 14 +- wechat-news-morning-draft.test.mjs | 27 ++ wechat-news-morning-images.mjs | 178 +++++++--- wechat-news-morning-images.test.mjs | 28 ++ wechat-news-morning-search.mjs | 145 +++++++-- wechat-news-morning-search.test.mjs | 131 +++++++- wechat-news-morning-searxng-fallback.mjs | 52 ++- 18 files changed, 1384 insertions(+), 123 deletions(-) create mode 100644 scripts/run-news-morning-generate-push-now.mjs create mode 100644 scripts/verify-news-engine-integration.mjs diff --git a/.env.example b/.env.example index 9ac35e4..418c0bb 100644 --- a/.env.example +++ b/.env.example @@ -239,6 +239,11 @@ H5_PUBLIC_BASE_URL=http://127.0.0.1:5173 # H5_WECHAT_NEWS_MORNING_DRAFT_MINUTE=0 # H5_WECHAT_NEWS_MORNING_DRAFT_AUTO_GENERATE=1 # H5_WECHAT_NEWS_MORNING_DRAFT_GENERATE_LEAD_MINUTES=60 +# 早报模板:news002(订阅版,写满各栏 + 有图必留、无图保留文字卡) +# H5_WECHAT_NEWS_MORNING_TEMPLATE_ID=news002 +# MEMIND_NEWS_MORNING_NEWS002_REQUIRE_FILL=1 +# MEMIND_NEWS_MORNING_SEARXNG_FALLBACK=1 +# MEMIND_NEWS_MORNING_LOCALIZE_IMAGES=1 # Intent Transaction Layer(Phase A:Draft → Confirm → Commit;默认关闭) # H5_INTENT_TRANSACTION_ENABLED=1 @@ -536,6 +541,13 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind # MEMIND_NEWS_MORNING_SEARXNG_NEWS_CATEGORY=0 # MEMIND_NEWS_MORNING_SEARXNG_CONCURRENCY=3 # MEMIND_NEWS_MORNING_SEARXNG_ENGINES=yandex,bing,360search +# 独立 news-engine 服务:早报预取完成后 POST groups 到 /v1/rank,brief 注入「News Engine 编辑候选」 +# MEMIND_NEWS_ENGINE_URL=http://127.0.0.1:8092 +# MEMIND_NEWS_ENGINE_API_TOKEN= +# MEMIND_NEWS_ENGINE_DAILY_LIMIT=15 +# MEMIND_NEWS_ENGINE_MINIMUM_SCORE=0 +# MEMIND_NEWS_ENGINE_TIMEOUT_MS=120000 +# MEMIND_NEWS_ENGINE_USE_LATEST_COLLECTION=true # Portal 在 web / search-enhanced 技能执行前预取 SearXNG;设为 0 可关闭。 # MEMIND_LIVE_SEARCH_PREFETCH=1 # TKMIND_SEARCH_MCP_HOST_GATEWAY=host.docker.internal diff --git a/docs/news-morning-templates.md b/docs/news-morning-templates.md index b665bb1..271a2e8 100644 --- a/docs/news-morning-templates.md +++ b/docs/news-morning-templates.md @@ -102,7 +102,8 @@ news002 的卡片图会先上传到微信正文素材接口,再由 inline 渲 静态页与微信草稿均使用 **左上角 72~92px 小图**(`.card` / `.lead-card` 左侧缩略图),不再用全宽大图。 能拿到真实配图的卡片必须显示并上传;没有真实图就只保留文字卡,不放默认小图。 远程配图会先下载到当日 MindSpace `public/`,改写成本站相对 URL,避免海外图源在国内打不开。 -已有配图上传失败、头条不是 3~4 条、事件卡超过 28 条,或页面仍混有 news001 DOM, +推送前不再删除无图故事卡(微信草稿保留纯文字卡);有图卡必须成功上传微信 CDN。 +生成闸门会拦截:配图上传失败、头条不是 3~4 条、事件卡超过 30 条,或页面仍混有 news001 DOM, 才会由生成/推送闸门阻断,不允许进入草稿箱。 真实图片来源按优先级取 SearXNG 结果里的 `img_src`/`thumbnail`,以及文章来源的 diff --git a/news-engine-client.mjs b/news-engine-client.mjs index ce069ea..0c04c2b 100644 --- a/news-engine-client.mjs +++ b/news-engine-client.mjs @@ -18,6 +18,127 @@ export function resolveNewsEngineEndpoint(env = process.env) { return String(env.MEMIND_NEWS_ENGINE_URL ?? '').trim().replace(/\/+$/, ''); } +export function isNewsEngineMediaUrl(value, env = process.env) { + const raw = String(value ?? '').trim(); + if (!raw) return false; + if (raw.startsWith('/media/')) return true; + const base = resolveNewsEngineEndpoint(env); + if (!base) return false; + try { + const parsed = new URL(raw); + const origin = new URL(`${base}/`).origin; + return parsed.origin === origin && parsed.pathname.startsWith('/media/'); + } catch { + return false; + } +} + +export function applyRankingImagesToGroups(groups = [], ranking = null, { + normalizeUrl = (value) => String(value ?? '').trim(), +} = {}) { + const imageByUrl = new Map(); + for (const event of [...(ranking?.selected ?? []), ...(ranking?.events ?? [])]) { + const url = normalizeUrl(event?.primaryArticle?.url); + const image = String(event?.primaryArticle?.image ?? '').trim(); + if (url && image) imageByUrl.set(url, image); + } + return (Array.isArray(groups) ? groups : []).map((group) => ({ + ...group, + results: (Array.isArray(group?.results) ? group.results : []).map((item) => { + const url = normalizeUrl(item?.url); + const image = imageByUrl.get(url); + if (!image) return item; + return { ...item, image, imageSource: 'news-engine' }; + }), + })); +} + +function envFlag(value, fallback = false) { + const raw = String(value ?? '').trim().toLowerCase(); + if (!raw) return fallback; + return ['1', 'true', 'yes', 'on'].includes(raw); +} + +export function shouldUseNewsEngineLatestCollection(env = process.env) { + if (!resolveNewsEngineEndpoint(env)) return false; + return envFlag(env.MEMIND_NEWS_ENGINE_USE_LATEST_COLLECTION, true); +} + +export async function fetchLatestCollectionFromEngine({ + env = process.env, + fetchImpl = fetch, + logger = console, +} = {}) { + const endpoint = resolveNewsEngineEndpoint(env); + if (!endpoint) return null; + const timeoutMs = Math.max( + 500, + Number(env.MEMIND_NEWS_ENGINE_TIMEOUT_MS ?? 5000) || 5000, + ); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + timer.unref?.(); + try { + const token = String(env.MEMIND_NEWS_ENGINE_API_TOKEN ?? '').trim(); + const response = await fetchImpl(`${endpoint}/v1/collections/latest`, { + headers: { + accept: 'application/json', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + signal: controller.signal, + }); + if (response.status === 404) return null; + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const collection = await response.json(); + if (!collection || typeof collection !== 'object') return null; + return collection; + } catch (error) { + logger.warn?.( + '[NewsEngine] latest collection unavailable:', + error instanceof Error ? error.message : error, + ); + return null; + } finally { + clearTimeout(timer); + } +} + +export function resolveNewsEngineMediaFetchUrl(imageUrl, env = process.env) { + const raw = String(imageUrl ?? '').trim(); + if (!raw) return ''; + if (raw.startsWith('/media/')) { + const endpoint = resolveNewsEngineEndpoint(env); + return endpoint ? `${endpoint}${raw}` : ''; + } + return isNewsEngineMediaUrl(raw, env) ? raw : ''; +} + +export async function downloadNewsEngineMediaBytes(imageUrl, { + env = process.env, + fetchImpl = fetch, + timeoutMs = 8000, +} = {}) { + const fetchUrl = resolveNewsEngineMediaFetchUrl(imageUrl, env); + if (!fetchUrl) return null; + try { + const token = String(env.MEMIND_NEWS_ENGINE_API_TOKEN ?? '').trim(); + const response = await fetchImpl(fetchUrl, { + redirect: 'follow', + signal: AbortSignal.timeout(timeoutMs), + headers: { + Accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + }); + if (!response.ok) return null; + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.length > 2_500_000 || buffer.length < 12) return null; + return buffer; + } catch { + return null; + } +} + export async function rankNewsWithEngine(groups, { env = process.env, now = Date.now(), @@ -82,6 +203,7 @@ export function formatRankedNewsCandidates(ranking) { ` score=${event.finalScore} sources=${event.sourceCount} must_include=${event.mustInclude} primary=${source}`, ); if (event.primaryArticle?.url) lines.push(` ${event.primaryArticle.url}`); + if (event.primaryArticle?.image) lines.push(` 图:${event.primaryArticle.image}`); } return lines.join('\n'); } diff --git a/news-engine-client.test.mjs b/news-engine-client.test.mjs index bd464cd..0934b37 100644 --- a/news-engine-client.test.mjs +++ b/news-engine-client.test.mjs @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + applyRankingImagesToGroups, + downloadNewsEngineMediaBytes, + fetchLatestCollectionFromEngine, formatRankedNewsCandidates, + isNewsEngineMediaUrl, rankNewsWithEngine, resolveNewsEngineEndpoint, + shouldUseNewsEngineLatestCollection, } from './news-engine-client.mjs'; test('news engine client is disabled without an endpoint', async () => { @@ -54,6 +59,100 @@ test('news engine client sends groups and authentication to the independent serv assert.match(formatRankedNewsCandidates(ranking), /模型发布/); }); +test('isNewsEngineMediaUrl detects local news-engine media paths', () => { + assert.equal(isNewsEngineMediaUrl('/media/abc.jpg'), true); + assert.equal( + isNewsEngineMediaUrl('http://127.0.0.1:8092/media/abc.jpg', { MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092' }), + true, + ); + assert.equal(isNewsEngineMediaUrl('https://cdn.example.com/a.jpg'), false); +}); + +test('applyRankingImagesToGroups merges primaryArticle images by url', () => { + const groups = applyRankingImagesToGroups([{ + id: 'tech', + results: [{ title: 'A', url: 'https://example.com/a' }], + }], { + selected: [{ + primaryArticle: { + url: 'https://example.com/a', + image: 'http://127.0.0.1:8092/media/abc.jpg', + }, + }], + }); + assert.equal(groups[0].results[0].image, 'http://127.0.0.1:8092/media/abc.jpg'); + assert.equal(groups[0].results[0].imageSource, 'news-engine'); +}); + +test('shouldUseNewsEngineLatestCollection requires endpoint', () => { + assert.equal(shouldUseNewsEngineLatestCollection({}), false); + assert.equal( + shouldUseNewsEngineLatestCollection({ MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092' }), + true, + ); + assert.equal( + shouldUseNewsEngineLatestCollection({ + MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092', + MEMIND_NEWS_ENGINE_USE_LATEST_COLLECTION: 'false', + }), + false, + ); +}); + +test('fetchLatestCollectionFromEngine returns latest snapshot', async () => { + const collection = await fetchLatestCollectionFromEngine({ + env: { + MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092', + MEMIND_NEWS_ENGINE_API_TOKEN: 'token', + }, + fetchImpl: async (url, init) => { + assert.equal(url, 'http://127.0.0.1:8092/v1/collections/latest'); + assert.equal(init.headers.authorization, 'Bearer token'); + return { + ok: true, + status: 200, + async json() { + return { id: 'c1', status: 'ok', dateLabel: '2026-09-22', groups: [{ id: 'tech', results: [{}] }] }; + }, + }; + }, + }); + assert.equal(collection.id, 'c1'); +}); + +test('downloadNewsEngineMediaBytes fetches /media assets from news-engine', async () => { + const bytes = await downloadNewsEngineMediaBytes('http://127.0.0.1:8092/media/abc.jpg', { + env: { MEMIND_NEWS_ENGINE_URL: 'http://127.0.0.1:8092' }, + fetchImpl: async (url) => { + assert.equal(url, 'http://127.0.0.1:8092/media/abc.jpg'); + return { + ok: true, + arrayBuffer: async () => Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0, 0, 0, 0, 0]), + }; + }, + }); + assert.ok(Buffer.isBuffer(bytes)); +}); + +test('formatRankedNewsCandidates includes local image urls', () => { + const block = formatRankedNewsCandidates({ + selected: [{ + rank: 1, + category: 'tech', + canonicalTitle: '测试', + finalScore: 8, + sourceCount: 1, + mustInclude: false, + primaryArticle: { + url: 'https://example.com/a', + image: 'http://127.0.0.1:8092/media/abc.jpg', + sourceDomain: 'example.com', + }, + }], + }); + assert.match(block, /图:http:\/\/127\.0\.0\.1:8092\/media\/abc\.jpg/); +}); + test('news engine client fails open when the service is unavailable', async () => { const warnings = []; const ranking = await rankNewsWithEngine([], { diff --git a/news-morning-templates.mjs b/news-morning-templates.mjs index 01910bd..b30dbcc 100644 --- a/news-morning-templates.mjs +++ b/news-morning-templates.mjs @@ -115,7 +115,185 @@ export const NEWS002_FLEX_SECTION_IDS = NEWS002_SECTIONS .map((item) => item.id); /** 全页独立事件卡上限;超过这个数就是在灌水,不是在做早报。 */ -export const NEWS002_MAX_STORY_CARDS = 28; +/** core(20) + 3×flex(9) 写满时最多 29 条故事卡,留 1 条余量给 SearXNG 补栏。 */ +export const NEWS002_MAX_STORY_CARDS = 30; + +/** news002 flex 栏目至少应出现的节数(写满策略下仍要求多栏目覆盖)。 */ +export const NEWS002_MIN_FLEX_SECTIONS = 2; + +const NEWS002_SECTION_ALIASES = { + business: ['business', 'finance'], + deepdive: ['deepdive', 'deep'], +}; + +function sliceNews002BalancedBlock(html, tagName, openStart, openEnd) { + const open = new RegExp(`<${tagName}\\b`, 'gi'); + const close = new RegExp(``, 'gi'); + let depth = 1; + let cursor = openEnd; + while (depth > 0 && cursor < html.length) { + open.lastIndex = cursor; + close.lastIndex = cursor; + const nextOpen = open.exec(html); + const nextClose = close.exec(html); + if (!nextClose) return { start: openStart, end: html.length }; + if (nextOpen && nextOpen.index < nextClose.index) { + depth += 1; + cursor = nextOpen.index + nextOpen[0].length; + } else { + depth -= 1; + cursor = nextClose.index + nextClose[0].length; + } + } + return { start: openStart, end: cursor }; +} + +function extractNews002SectionBlock(html, sectionId) { + const source = String(html ?? ''); + const aliases = NEWS002_SECTION_ALIASES[sectionId] ?? [sectionId]; + for (const alias of aliases) { + if (alias === 'brief') { + const match = source.match( + /]*\bsection\b[^>]*\bid=["']brief["'][^>]*>/i, + ); + if (!match) continue; + const start = match.index; + const end = sliceNews002BalancedBlock(source, 'div', start, start + match[0].length).end; + return source.slice(start, end); + } + const match = source.match( + new RegExp( + `]*class=["'][^"']*\\bsection\\b[^"']*["'][^>]*id=["']${alias}["'][^>]*>`, + 'i', + ), + ); + if (!match) continue; + const start = match.index; + const end = sliceNews002BalancedBlock(source, 'div', start, start + match[0].length).end; + return source.slice(start, end); + } + return ''; +} + +function countNews002StoryCardsInBlock(block) { + const source = String(block ?? ''); + const re = /<(div|a)\b[^>]*\bclass="([^"]*)"[^>]*>/gi; + let count = 0; + let match; + while ((match = re.exec(source)) != null) { + const tokens = String(match[2]).split(/\s+/); + if (tokens.includes('card') || tokens.includes('lead-card')) count += 1; + re.lastIndex = match.index + match[0].length; + } + return count; +} + +/** 统计 news002 各栏目当前条数。 */ +export function findNews002SectionIdForOffset(html, offset) { + const before = String(html ?? '').slice(0, offset); + const matches = [...before.matchAll( + /]*class=["'][^"']*\bsection\b[^"']*["'][^>]*id=["']([^"']+)["']/gi, + )]; + return matches.length ? matches[matches.length - 1][1] : ''; +} + +export function countNews002HeadlineLeadCards(html) { + const block = extractNews002SectionBlock(html, 'headlines'); + if (!block) return 0; + return (block.match(/\blead-card\b/gi) ?? []).length; +} + +export function countNews002SectionItems(html, sectionId) { + const block = extractNews002SectionBlock(html, sectionId); + if (!block) return 0; + if (sectionId === 'brief') return (block.match(/]*\bclass="[^"]*\bdeep\b/i.test(block) ? 1 : 0; + } + if (sectionId === 'weather') { + return (block.match(/class="[^"]*\bweather-card\b/gi) ?? []).length; + } + if (sectionId === 'knowledge') { + return (block.match(/class="[^"]*\bk-item\b/gi) ?? []).length; + } + if (sectionId === 'history') { + return (block.match(/class="[^"]*\bh-item\b/gi) ?? []).length; + } + return countNews002StoryCardsInBlock(block); +} + +/** + * news002 栏目写满审计:core 栏目必须达到 max;flex 至少 minFlexSections 节且每节达到 max。 + */ +export function auditNews002SectionFill( + html, + { + minFlexSections = NEWS002_MIN_FLEX_SECTIONS, + requireCoreMax = true, + requireFlexMax = true, + } = {}, +) { + const counts = Object.fromEntries( + NEWS002_SECTIONS.map((section) => [section.id, countNews002SectionItems(html, section.id)]), + ); + const violations = []; + + for (const section of NEWS002_SECTIONS.filter((item) => item.tier === 'core')) { + const count = counts[section.id] ?? 0; + if (section.id === 'deepdive') { + if (count !== section.max) { + violations.push(`underfilled:${section.id}:${count}/${section.max}`); + } + continue; + } + if (requireCoreMax && count < section.max) { + violations.push(`underfilled:${section.id}:${count}/${section.max}`); + } + } + + const flexPresent = NEWS002_FLEX_SECTION_IDS.filter((id) => extractNews002SectionBlock(html, id)); + if (flexPresent.length < minFlexSections) { + violations.push(`insufficient-flex-sections:${flexPresent.length}/${minFlexSections}`); + } + if (requireFlexMax) { + for (const id of flexPresent) { + const section = NEWS002_SECTIONS.find((item) => item.id === id); + const count = counts[id] ?? 0; + if (section && count < section.max) { + violations.push(`underfilled:${id}:${count}/${section.max}`); + } + } + } + + return { + ok: violations.length === 0, + violations, + counts, + flexPresent, + }; +} + +/** 交给 Agent 的「必须写满」附加约束(可写入 taskSpecAppend)。 */ +export const NEWS002_FILL_REQUIREMENT_SPEC = [ + '【news002 写满硬约束 · 未达标即作废重写】', + '1. core 栏目(brief/headlines/deepdive/domestic/world/business/tech/weather/knowledge/history)每一条都必须写到该栏目的上限条数,不能只写 min。', + `2. 各栏上限:${NEWS002_SECTIONS.map((s) => `${s.id}=${s.max}`).join(',')}。`, + `3. flex 栏目(${NEWS002_FLEX_SECTION_IDS.join('/')})至少出现 ${NEWS002_MIN_FLEX_SECTIONS} 个,且每个出现的 flex 栏目也必须写到上限。`, + '4. 写页前必须先合并预取 + tkmind_search(type=news) + web_search,事件表不足时继续搜索补事件,禁止用同一事件拆条凑数。', + `5. 全页独立事件仍遵守去重与 ${NEWS002_MAX_STORY_CARDS} 条上限;为写满各栏,必须覆盖更多不同事件,而不是重复。`, + '6. `.meta` 里的「今日 M 条」必须等于速读条数 + 各栏故事卡 + 知识卡 + 天气城数 + 历史条目的合计,且与实际上屏一致。', +].join('\n'); + +/** 定时任务默认附加约束:禁止回退 news001 + 必须写满各栏。 */ +export const NEWS002_DEFAULT_TASK_SPEC_APPEND = [ + '【强制 news002 版式 · 覆盖一切旧稿参考】', + '1. 禁止输出 news001 结构:不得出现 class="hero"、class="date-badge"、class="quick-links"、class="highlight-box"、背景 #f7fafc / 主色 #b0191e。', + '2. 必须使用 news002 结构:div.masthead + div.lede + a.lead-card + ol.brief + div.card[data-event-key],并内嵌 taskSpec 里的 NEWS002_STYLE_BLOCK(暖纸 #FAF7F2、主色 #C8362F)。', + '3. 写页前不要 read_file 任何旧 daily-news-*.html 学版式;旧稿正文禁止复用。只允许按本 taskSpec 生成。', + `4. 全页每张故事卡必须带 data-event-key;独立事件 ≤${NEWS002_MAX_STORY_CARDS};flex 栏目无独立事件则整节删除。`, + '5. 导航仅保留 news002 8 个主锚点(速读/头条/深读/国内/国际/财经/科技/天气/知识),禁止 25 项 quick-links。', + NEWS002_FILL_REQUIREMENT_SPEC, +].join('\n'); /** news002 标准