From 51604d396295d221f79b568907cfe7a8cc21de80 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 23 Sep 2026 09:58:35 +0800 Subject: [PATCH] feat(wechat): filter unsafe morning news and assistive trim instead of hard fail Add content safety filter for adult/junk URLs, strip bad cards without failing drafts, copy-archive on force regenerate, and improve news001/002 WeChat inline layout. Co-authored-by: Cursor --- news-morning-templates.mjs | 1 + package.json | 2 +- .../push-wechat-news-morning-draft-now.mjs | 5 +- wechat-daily-news-inline.mjs | 117 ++++++++++++--- wechat-daily-news-inline.test.mjs | 20 +++ wechat-news-morning-content-filter.mjs | 116 +++++++++++++++ wechat-news-morning-content-filter.test.mjs | 46 ++++++ wechat-news-morning-draft-worker.mjs | 29 ++-- wechat-news-morning-draft-worker.test.mjs | 104 ++++++++++++++ wechat-news-morning-draft.mjs | 55 ++++--- wechat-news-morning-draft.test.mjs | 22 +++ wechat-news-morning-images.mjs | 134 ++++++++++++++++++ wechat-news-morning-images.test.mjs | 72 +++++++++- wechat-news-morning-search.mjs | 20 ++- wechat-news-morning-search.test.mjs | 10 ++ wechat-news-morning-searxng-fallback.mjs | 2 + wechat-news-morning-searxng-fallback.test.mjs | 31 ++++ 17 files changed, 727 insertions(+), 59 deletions(-) create mode 100644 wechat-news-morning-content-filter.mjs create mode 100644 wechat-news-morning-content-filter.test.mjs diff --git a/news-morning-templates.mjs b/news-morning-templates.mjs index b30dbcc..64f4875 100644 --- a/news-morning-templates.mjs +++ b/news-morning-templates.mjs @@ -528,6 +528,7 @@ export const NEWS002_FRESH_CONTENT_SPEC = [ '8. 标题规则：中文 15～26 字；英文来源须转写为中文标题，可在括号内保留英文原题；每条必须可核验并写来源 URL。', '9. 禁止复制、改写历史 daily-news-*.html 的正文；旧稿仅作 HTML 结构参考。', '10. 禁止八卦绯闻、未经证实的传闻、标题党；每条都要能说清「为什么重要」，说不清就不要写。', + '11. 禁止色情低俗、成人站点、吃瓜黑料、无信息量网址标题；命中则丢弃该条，继续写其它新闻，禁止因此整页失败。', ].join('\n'); /* ------------------------------------------------------------------ * diff --git a/package.json b/package.json index f33eae2..3b675e9 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,7 @@ "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-voice-reco.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs scheduled-task-intent.test.mjs scheduled-task-service.test.mjs scheduled-task-executor.test.mjs scheduled-task-worker.test.mjs scheduled-task-worker-config.test.mjs wechat-news-morning-draft.test.mjs wechat-news-morning-draft-worker.test.mjs wechat-news-morning-draft-worker-config.test.mjs wechat/handlers/scheduled-task.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs health-channel-state.test.mjs health-intent-rules.test.mjs health-extraction.test.mjs health-observation-validate.test.mjs health-baseline-maturity.test.mjs health-publish-guard.test.mjs health-p0-experiment.test.mjs health-wechat-channel.test.mjs server/portal-health-routes.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-long-image.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-save-service.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-wechat-html-delivery.test.mjs mindspace-wechat-mp-config.test.mjs mindspace-wechat-page-draft.test.mjs server/portal-mindspace-wechat-routes.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-artifact-service.test.mjs mindspace-conversation-package-audit.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-workspace-publication-delivery-service.test.mjs mindspace-workspace-tool-service.test.mjs mindspace-mcp-scoped-token.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", - "test:news-engine": "node --test news-engine-client.test.mjs wechat-news-morning-search.test.mjs", + "test:news-engine": "node --test news-engine-client.test.mjs wechat-news-morning-search.test.mjs wechat-news-morning-content-filter.test.mjs wechat-news-morning-images.test.mjs wechat-news-morning-searxng-fallback.test.mjs", "verify:news-engine-integration": "node scripts/verify-news-engine-integration.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", "test:mindspace-service": "node --test mindspace-service/mindspace-rpc-server.test.mjs", diff --git a/scripts/push-wechat-news-morning-draft-now.mjs b/scripts/push-wechat-news-morning-draft-now.mjs index ed89487..7782342 100644 --- a/scripts/push-wechat-news-morning-draft-now.mjs +++ b/scripts/push-wechat-news-morning-draft-now.mjs @@ -34,6 +34,7 @@ async function ensureDraftConfig(service) { async function main() { const dryRun = process.argv.includes('--dry-run'); const allowYesterday = process.argv.includes('--allow-yesterday'); + const skipQualityGate = process.argv.includes('--skip-gate'); if (!process.env.DATABASE_URL) { console.error('DATABASE_URL missing'); process.exit(1); @@ -59,6 +60,7 @@ async function main() { console.log(JSON.stringify({ dryRun, allowYesterday, + skipQualityGate, sourceUserId: config.sourceUserId, templateId: config.templateId ?? null, templateVersion: config.templateVersion ?? null, @@ -87,7 +89,8 @@ async function main() { const result = await service.pushDraft({ requireToday: !allowYesterday, - triggeredBy: 'manual-push-now', + triggeredBy: skipQualityGate ? 'manual-push-now-skip-gate' : 'manual-push-now', + skipQualityGate, }); console.log('\n[success]'); console.log(JSON.stringify({ diff --git a/wechat-daily-news-inline.mjs b/wechat-daily-news-inline.mjs index 7209f56..228052f 100644 --- a/wechat-daily-news-inline.mjs +++ b/wechat-daily-news-inline.mjs @@ -78,10 +78,13 @@ const SECTION_LABELS = { local_life: 'CITIES', official_feeds: 'WIRE', finance: 'FINANCE', + business: 'FINANCE', tech: 'TECH', technews: 'TECH', sports: 'SPORTS', + living: 'CITIES', knowledge: 'INSIGHT', + deepdive: 'DEEP', }; const FEATURE_CARD_THEMES = { @@ -187,6 +190,12 @@ function isNews002Html(html) { return /\bclass="[^"]*\blead-card\b/u.test(source) && !/\bclass="hero\b/u.test(source); } +export function resolveNewsMorningWechatLayoutStyle(html, layoutStyle = '') { + const requested = String(layoutStyle ?? '').trim().toLowerCase(); + if (requested === 'news001' || requested === 'news002') return requested; + return isNews002Html(html) ? 'news002' : 'news001'; +} + /** feature=渐变头条卡，rank=多行榜单卡，list=带序号的列表条目，lead=news002 头条小图卡。 */ function classifyCard(cardHtml, { news002Mode = false } = {}) { if (news002Mode && /\blead-card\b/u.test(cardHtml)) return 'lead'; @@ -262,24 +271,23 @@ function renderFeatureCard(cardHtml, { theme, title, body, imageUrl = '' }) { } function renderListCard({ title, body, index, imageUrl = '' }) { - const thumb = imageUrl - ? `${stripInlineText(title)}` + const thumbBlock = imageUrl + ? [ + ``, + `${stripInlineText(title)}`, + '', + ].join('') : ''; const textBlock = [ `

${formatCardIndex(index)}　${title}

`, body ? `

${body}

` : '', ].filter(Boolean).join(''); - if (thumb) { - return [ - `
`, - thumb, - `
${textBlock}
`, - '
', - ].join(''); - } return [ - `
`, + `
`, + thumbBlock, + ``, textBlock, + '', '
', ].join(''); } @@ -358,7 +366,8 @@ function renderCard(cardHtml, { theme, title, body, - imageUrl, + // news001 头条保持渐变卡，不插全宽图；小缩略图只加在列表卡上。 + imageUrl: '', }); } return renderListCard({ @@ -417,15 +426,29 @@ function extractHeroDateLine(html) { * Hero 里的摘要通常是「要点·要点·要点」的长串，拆成导读列表比整段文字好读得多。 * 拆不出两条以上时返回空，由 Hero 继续以整段展示。 */ +function extractBriefDigestItems(html, { maxItems = 8, maxItemChars = 36 } = {}) { + const block = String(html ?? '').match(/]*class="[^"]*\bbrief\b[^"]*"[^>]*>([\s\S]*?)<\/ol>/i)?.[1] ?? ''; + const items = []; + for (const match of block.matchAll(/]*>([\s\S]*?)<\/li>/gi)) { + const text = stripInlineText(match[1]).replace(/^\d+[\s.、]*/u, ''); + if (text.length < 4) continue; + items.push(truncatePlainText(text, maxItemChars)); + if (items.length >= maxItems) break; + } + return items; +} + function renderTodayDigest(html, { maxItems = 6, maxItemChars = 30 } = {}) { + const briefItems = extractBriefDigestItems(html, { maxItems: Math.max(maxItems, 8), maxItemChars: 36 }); const raw = extractHeroSubtitle(html); - if (!raw) return ''; - const items = raw - .split(/\s*[·|、]\s*/u) - .map((item) => item.trim()) - .filter((item) => item.length >= 4) - .slice(0, maxItems) - .map((item) => truncatePlainText(item, maxItemChars)); + const items = briefItems.length >= 2 + ? briefItems + : raw + .split(/\s*[·|、]\s*/u) + .map((item) => item.trim()) + .filter((item) => item.length >= 4) + .slice(0, maxItems) + .map((item) => truncatePlainText(item, maxItemChars)); if (items.length < 2) return ''; const lines = items.map((item) => `· ${item}`).join('
'); return [ @@ -609,6 +632,26 @@ function renderKnowledgeSection(sectionHtml, { maxItems = 4, maxBodyChars = 130 return [renderSectionTitle(title, 'knowledge'), ...items].join(''); } +function renderDeepdiveSection(sectionHtml, { maxBodyChars = 220 } = {}) { + const title = extractInnerHtml(sectionHtml, /
]*>([\s\S]*?)<\/div>/i); + const heading = extractInnerHtml(sectionHtml, /]*>([\s\S]*?)<\/h3>/i); + const body = truncatePlainText( + [...String(sectionHtml).matchAll(/]*>([\s\S]*?)<\/p>/gi)] + .map((item) => stripInlineText(item[1])) + .filter(Boolean) + .join(' '), + maxBodyChars, + ); + if (!heading && !body) return ''; + return [ + renderSectionTitle(title, 'deepdive'), + `
`, + heading ? `

${heading}

` : '', + body ? `

${body}

` : '', + '
', + ].filter(Boolean).join(''); +} + function renderSectionFigure(figureHtml, imageUrlMap = new Map()) { const src = String(figureHtml).match(/]+src=["']([^"']+)["']/i)?.[1] ?? ''; const mapped = resolveMappedImageUrl(src, imageUrlMap); @@ -734,10 +777,13 @@ const DEFAULT_SECTION_CARD_LIMITS = { local_life: 3, official_feeds: 3, finance: 5, + business: 5, tech: 5, sports: 3, technews: 2, + living: 3, knowledge: 3, + deepdive: 1, }; const DEFAULT_CARD_BODY_CHARS = 132; @@ -817,8 +863,14 @@ function countSectionCards(sections) { if (!section.id || section.id === 'weather' || section.id === 'history') continue; if (/历史上的今天|近7天新闻早报/u.test(section.title)) continue; const total = section.id === 'knowledge' - ? extractClassDivBlocks(section.html, 'knowledge-item').length - : (section.html.match(/
0) counts[section.id] = total; } return counts; @@ -847,6 +899,7 @@ function resolveSectionCardLimit(section, sectionCardLimits) { function shouldSkipSection(section, trimProfile) { if (!section) return true; if (section.id === 'hot') return true; + if (section.id === 'brief') return true; if (section.id === 'history') return true; if (section.id === 'knowledge' && trimProfile?.skipKnowledge === true) return true; if (trimProfile?.skipSectionIds?.includes(section.id)) return true; @@ -958,11 +1011,12 @@ function buildWechatInlineContent(html, { trimProfile = TRIM_PROFILES[0], includeFixedLayout = true, imageUrlMap = new Map(), + layoutStyle = '', } = {}) { const sections = splitSections(html); - const news002Mode = isNews002Html(html); + const resolvedLayout = resolveNewsMorningWechatLayoutStyle(html, layoutStyle); + const news002Mode = resolvedLayout === 'news002'; const digest = renderTodayDigest(html); - const isNews002 = /class="[^"]*\bmasthead\b/i.test(String(html ?? '')); const bodyParts = [renderHero(html, imageUrlMap, { includeSubtitle: !digest })]; if (digest) bodyParts.push(digest); @@ -983,6 +1037,17 @@ function buildWechatInlineContent(html, { continue; } + if (section.id === 'deepdive') { + const rendered = renderDeepdiveSection(section.html, { + maxBodyChars: Math.min(trimProfile.maxBodyChars ?? 220, 220), + }); + if (rendered) { + contentParts.push(rendered); + renderedSectionCount += 1; + } + continue; + } + if (section.id === 'knowledge') { const rendered = renderKnowledgeSection(section.html, { maxItems: resolveSectionCardLimit(section, trimProfile.sectionCardLimits) ?? 4, @@ -1040,6 +1105,7 @@ function fitInlineContentToBudget(html, bodyBudget, { qrcodeImageUrl = '', portalUrl = DEFAULT_PORTAL_URL, imageUrlMap = new Map(), + layoutStyle = '', } = {}) { const fullCounts = countSectionCards(splitSections(html)); const cappedCounts = mergeSectionCardLimits(fullCounts, DEFAULT_SECTION_CARD_LIMITS); @@ -1050,6 +1116,7 @@ function fitInlineContentToBudget(html, bodyBudget, { trimProfile, includeFixedLayout: false, imageUrlMap, + layoutStyle, }); let best = null; @@ -1133,6 +1200,7 @@ export function convertDailyNewsHtmlToWechatInlineArticle(html, { imageUrlMap = new Map(), includeFixedLayout = true, fixedLayoutLength = null, + layoutStyle = '', } = {}) { const title = (extractPageTitle(html) || '每日新闻早报').slice(0, 32); const digest = (extractMetaDescription(html) || title).slice(0, 120); @@ -1150,6 +1218,7 @@ export function convertDailyNewsHtmlToWechatInlineArticle(html, { qrcodeImageUrl, portalUrl, imageUrlMap, + layoutStyle, }); const trimLevel = fitted.bodyCharTier; const built = fitted.built; @@ -1204,6 +1273,8 @@ export const wechatDailyNewsInlineInternals = { renderHero, renderTodayDigest, renderBriefStats, + renderDeepdiveSection, + resolveNewsMorningWechatLayoutStyle, renderWeatherSection, selectTouristWeatherCards, isPopularTouristCityName, diff --git a/wechat-daily-news-inline.test.mjs b/wechat-daily-news-inline.test.mjs index aa42164..3c21ed9 100644 --- a/wechat-daily-news-inline.test.mjs +++ b/wechat-daily-news-inline.test.mjs @@ -182,6 +182,26 @@ test('news002 card with media block still yields title and body', () => { assert.match(rendered, /国内正文摘要/u, '.media 在前不应挤掉正文

'); }); +test('news001 layout keeps gradient headlines and adds compact thumbs on list cards', () => { + const sourceImage = 'https://news.example/story.jpg'; + const wechatImage = 'https://mmbiz.qpic.cn/wechat-story.jpg'; + const html = NEWS002_HTML + .replace('

📰
', `
头条
`) + .replace('
🇨🇳
', `
国内
`); + const imageUrlMap = new Map([[sourceImage, wechatImage]]); + const article = convertDailyNewsHtmlToWechatInlineArticle(html, { + includeFixedLayout: false, + imageUrlMap, + layoutStyle: 'news001', + }); + assert.match(article.content, /linear-gradient\(135deg,#c8102e/u, '头条仍走 news001 渐变卡'); + assert.match(article.content, /王毅同鲁比奥通电话/u); + assert.match(article.content, /沈阳举行九一八纪念活动/u); + assert.match(article.content, /width:72px;height:54px/u, '列表卡使用 news002 小缩略图'); + assert.doesNotMatch(article.content, /calc\(100% \+ 28px\)/u, '头条不插全宽图'); + assert.doesNotMatch(article.content, /TKMIND DAILY/u); +}); + test('news002 cards render uploaded source images in the wechat draft', () => { const sourceImage = 'https://news.example/story.jpg'; const wechatImage = 'https://mmbiz.qpic.cn/wechat-story.jpg'; diff --git a/wechat-news-morning-content-filter.mjs b/wechat-news-morning-content-filter.mjs new file mode 100644 index 0000000..5537904 --- /dev/null +++ b/wechat-news-morning-content-filter.mjs @@ -0,0 +1,116 @@ +/** + * 早报内容安全过滤：挡住色情低俗、成人站和无信息量网址标题。 + * 只剔除条目，不让整页/整次推送失败。 + */ + +const UNSAFE_TEXT_PATTERNS = [ + /鸡巴/iu, + /吞精/iu, + /海角网/iu, + /黑料/iu, + /吃瓜/iu, + /91每日大赛/iu, + /91视频/iu, + /上科技操/iu, + /打玻尿酸坏死/iu, + /玩偶姐姐/iu, + /hongkongdoll/iu, + /porn/iu, + /xxx\b/iu, + /onlyfans/iu, +]; + +const UNSAFE_HOST_PATTERNS = [ + /91dscg/i, + /haijiao/i, + /swytotbvj/i, + /jav\d/i, + /pornhub/i, + /xvideos/i, + /xnxx/i, +]; + +const EXTRA_BLOCKLIST_ENV = 'MEMIND_NEWS_MORNING_BLOCKLIST'; + +function extraBlocklist(env = process.env) { + return String(env[EXTRA_BLOCKLIST_ENV] ?? '') + .split(/[,|\n]/) + .map((item) => item.trim()) + .filter((item) => item.length >= 2); +} + +function decodeHtml(value) { + return String(value ?? '') + .replace(/&/gi, '&') + .replace(/"/gi, '"') + .replace(/'|'/gi, "'") + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/ /gi, ' '); +} + +export function normalizeNewsMorningSafetyText(value) { + return decodeHtml(value).replace(/\s+/g, ' ').trim(); +} + +function hostnameOf(url) { + try { + return new URL(String(url ?? '').trim()).hostname.replace(/^www\./i, '').toLowerCase(); + } catch { + return ''; + } +} + +export function isBareNewsMorningUrlTitle(title) { + const text = normalizeNewsMorningSafetyText(title); + if (!text) return false; + if (/^https?:\/\//i.test(text)) return true; + if (/youtube\.com\/watch/i.test(text)) return true; + if (/^[\w.-]+\.(com|net|org|cn|cc|tv|io)(\/|$|\s)/i.test(text) && !/\s/.test(text)) { + return true; + } + return false; +} + +export function inspectNewsMorningSafety(item = {}, { env = process.env } = {}) { + const title = normalizeNewsMorningSafetyText(item.title); + const snippet = normalizeNewsMorningSafetyText(item.snippet ?? item.body ?? ''); + const url = normalizeNewsMorningSafetyText(item.url); + const eventKey = normalizeNewsMorningSafetyText(item.eventKey); + const blob = `${title} ${snippet} ${url} ${eventKey}`; + const host = hostnameOf(url); + const extras = extraBlocklist(env); + + if (isBareNewsMorningUrlTitle(title)) { + return { ok: false, reason: 'bare-url-title' }; + } + for (const pattern of UNSAFE_TEXT_PATTERNS) { + if (pattern.test(blob)) return { ok: false, reason: 'unsafe-text' }; + } + for (const pattern of UNSAFE_HOST_PATTERNS) { + if (pattern.test(host) || pattern.test(url) || pattern.test(blob)) { + return { ok: false, reason: 'unsafe-host' }; + } + } + for (const term of extras) { + if (blob.toLowerCase().includes(term.toLowerCase())) { + return { ok: false, reason: 'custom-blocklist' }; + } + } + return { ok: true, reason: null }; +} + +export function isUnsafeNewsMorningItem(item, options) { + return !inspectNewsMorningSafety(item, options).ok; +} + +export function filterSafeNewsMorningItems(items = [], options) { + const kept = []; + const removed = []; + for (const item of Array.isArray(items) ? items : []) { + const verdict = inspectNewsMorningSafety(item, options); + if (verdict.ok) kept.push(item); + else removed.push({ item, reason: verdict.reason }); + } + return { items: kept, removed, removedCount: removed.length }; +} diff --git a/wechat-news-morning-content-filter.test.mjs b/wechat-news-morning-content-filter.test.mjs new file mode 100644 index 0000000..d344826 --- /dev/null +++ b/wechat-news-morning-content-filter.test.mjs @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + filterSafeNewsMorningItems, + inspectNewsMorningSafety, + isBareNewsMorningUrlTitle, + isUnsafeNewsMorningItem, +} from './wechat-news-morning-content-filter.mjs'; + +test('inspectNewsMorningSafety drops adult titles and junk hosts', () => { + assert.equal(inspectNewsMorningSafety({ + title: 'AI短剧标题含鸡巴与海角网', + url: 'https://african.swytotbvj.cc/archives/1', + }).ok, false); + assert.equal(inspectNewsMorningSafety({ + title: '91每日大赛-今日吃瓜爆料资讯平台', + url: 'https://91dscg3.com/', + }).reason, 'unsafe-text'); + assert.equal(inspectNewsMorningSafety({ + title: '91视频最新网址 · GitLab', + url: 'https://gitlab.com/dizhi8', + }).ok, false); + assert.equal(inspectNewsMorningSafety({ + title: '正规国际快讯', + url: 'https://91dscg3.com/foo', + }).reason, 'unsafe-host'); +}); + +test('inspectNewsMorningSafety drops bare URL titles', () => { + assert.equal(isBareNewsMorningUrlTitle('youtube.com/watch?v=BpKaJR4Ut7o'), true); + assert.equal(isBareNewsMorningUrlTitle('d2pa8mwo9guq2h.cloudfront.net'), true); + assert.equal(isBareNewsMorningUrlTitle('https://map.baidu.com'), true); + assert.equal(isBareNewsMorningUrlTitle('莫斯科遭无人机袭击'), false); +}); + +test('filterSafeNewsMorningItems keeps legitimate news and honors custom blocklist', () => { + const { items, removedCount } = filterSafeNewsMorningItems([ + { title: '国乒今日出征', url: 'https://news.qq.com/a/20260923' }, + { title: '月排行榜吞精兽', url: 'https://news.example/bad' }, + { title: '自定义屏蔽词稿', url: 'https://news.example/custom' }, + ], { env: { MEMIND_NEWS_MORNING_BLOCKLIST: '自定义屏蔽词' } }); + assert.equal(items.length, 1); + assert.equal(removedCount, 2); + assert.match(items[0].title, /国乒/); + assert.equal(isUnsafeNewsMorningItem({ title: '正规国际快讯', url: 'https://www.reuters.com/world' }), false); +}); diff --git a/wechat-news-morning-draft-worker.mjs b/wechat-news-morning-draft-worker.mjs index df5ae9d..32d51c4 100644 --- a/wechat-news-morning-draft-worker.mjs +++ b/wechat-news-morning-draft-worker.mjs @@ -21,6 +21,7 @@ import { formatNewsMorningDedupeSummary, } from './wechat-news-morning-dedup.mjs'; import { + assistNews002Html, auditNews002ImageCoverage, enrichNews002HtmlWithSourceImages, localizeNews002CardImages, @@ -216,21 +217,33 @@ async function applyNews002ImageGate( } } + const assisted = assistNews002Html(localized.html, { + requireTemplateStructure: true, + requireSectionFill: true, + env, + }); + fs.writeFileSync(page.localPath, assisted.html); logger.log?.('[NewsMorningDraft] news002 image audit', { slug: page.slug, attempted: enriched.attemptedCount, resolved: enriched.resolvedCount, localized: localized.localizedCount, localizeFailed: localized.failedCount, - ...audit, + assisted: assisted.actions, + remaining: assisted.auditAfter.violations, + ...assisted.auditAfter, }); - if (!audit.ok) { - throw new Error( - `news002 配图/结构/写满闸门未通过：${audit.violations.join(', ')}` - + `（卡片 ${audit.cardCount}，真实配图 ${audit.imageCount}，头条配图 ${audit.leadImageCount}/${audit.leadCount}）`, + if (!assisted.auditAfter.ok) { + logger.warn?.( + '[NewsMorningDraft] news002 gate remaining after assist:', + assisted.auditAfter.violations.join(', ') + + `（卡片 ${assisted.auditAfter.cardCount}，真实配图 ${assisted.auditAfter.imageCount}，头条配图 ${assisted.auditAfter.leadImageCount}/${assisted.auditAfter.leadCount}）`, ); } - return audit; + return { + ...assisted.auditAfter, + actions: assisted.actions, + }; } function archiveTodayPageForForce(config, h5Root, now, logger) { @@ -238,10 +251,10 @@ function archiveTodayPageForForce(config, h5Root, now, logger) { if (!page?.localPath || !fs.existsSync(page.localPath)) return null; const stamp = new Date(now).toISOString().replace(/[:.]/g, '-'); const archived = path.join(path.dirname(page.localPath), `_archived-${page.slug}-${stamp}.html`); - fs.renameSync(page.localPath, archived); + fs.copyFileSync(page.localPath, archived); if (page.thumbPath && fs.existsSync(page.thumbPath)) { const archivedThumb = archived.replace(/\.html$/i, '.thumbnail.png'); - fs.renameSync(page.thumbPath, archivedThumb); + fs.copyFileSync(page.thumbPath, archivedThumb); } logger.log?.('[NewsMorningDraft] archived today page before force generate', { from: page.relativePath, diff --git a/wechat-news-morning-draft-worker.test.mjs b/wechat-news-morning-draft-worker.test.mjs index c33d905..5d49c17 100644 --- a/wechat-news-morning-draft-worker.test.mjs +++ b/wechat-news-morning-draft-worker.test.mjs @@ -244,6 +244,51 @@ test('runOnce honors forceGenerateOnce even when today page exists', async () => } }); +test('force generate failure keeps the live today page', async () => { + const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'news-draft-keep-live-')); + const userId = 'user-news'; + const publicDir = path.join(tmpRoot, PUBLISH_ROOT_DIR, userId, PUBLIC_ZONE_DIR); + fs.mkdirSync(publicDir, { recursive: true }); + const livePath = path.join(publicDir, 'daily-news-0911.html'); + fs.writeFileSync(livePath, `${HEALTHY_TODAY_HTML}`); + + const pool = createPool(); + const service = createWechatNewsMorningDraftService(pool, { + mpConfig: { enabled: true, appId: 'app', appSecret: 'secret' }, + h5Root: tmpRoot, + env: { + H5_WECHAT_NEWS_MORNING_DRAFT_ENABLED: '1', + H5_WECHAT_NEWS_MORNING_DRAFT_USER_ID: userId, + }, + }); + const worker = startWechatNewsMorningDraftWorker({ + wechatNewsMorningDraftService: service, + mpConfig: { enabled: true }, + userAuth: { canUseChat: async () => ({ ok: true }) }, + tkmindProxy: { id: 'proxy' }, + h5Root: tmpRoot, + env: { H5_WECHAT_NEWS_MORNING_DRAFT_WORKER_ENABLED: '1' }, + executeTask: async () => { + throw new Error('generate boom'); + }, + intervalMs: 30_000, + runOnStart: false, + setIntervalFn: () => ({ unref() {} }), + }); + + const fixedNow = Date.UTC(2026, 8, 10, 22, 0, 0); + await assert.rejects( + () => worker.generateToday({ now: fixedNow, force: true }), + /generate boom|新闻早报页面生成未完成/, + ); + assert.equal(fs.existsSync(livePath), true); + assert.match(fs.readFileSync(livePath, 'utf8'), /keep-live/); + assert.ok( + fs.readdirSync(publicDir).some((name) => name.startsWith('_archived-daily-news-0911-')), + ); + fs.rmSync(tmpRoot, { recursive: true, force: true }); +}); + test('generateToday skips existing page unless force is set', async () => { const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'news-draft-force-')); const userId = 'user-news'; @@ -307,6 +352,65 @@ test('generateToday skips existing page unless force is set', async () => { } }); +test('news002 generate keeps going when gate still has remaining violations', async () => { + const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'news-draft-assist-')); + const userId = 'user-news'; + const publicDir = path.join(tmpRoot, PUBLISH_ROOT_DIR, userId, PUBLIC_ZONE_DIR); + fs.mkdirSync(publicDir, { recursive: true }); + + const pool = createPool(); + const service = createWechatNewsMorningDraftService(pool, { + mpConfig: { enabled: true, appId: 'app', appSecret: 'secret' }, + h5Root: tmpRoot, + env: { + H5_WECHAT_NEWS_MORNING_DRAFT_ENABLED: '1', + H5_WECHAT_NEWS_MORNING_DRAFT_USER_ID: userId, + }, + }); + service.getConfig = async () => ({ + enabled: true, + sourceUserId: userId, + autoPushEnabled: false, + autoGenerateEnabled: true, + timezone: 'Asia/Shanghai', + pushHour: 6, + pushMinute: 0, + pageSlugPattern: 'daily-news-*', + templateId: 'news002', + }); + + const warnings = []; + const fixedNow = Date.UTC(2026, 8, 10, 22, 0, 0); + const worker = startWechatNewsMorningDraftWorker({ + wechatNewsMorningDraftService: service, + mpConfig: { enabled: true }, + userAuth: { canUseChat: async () => ({ ok: true }) }, + tkmindProxy: { id: 'proxy' }, + h5Root: tmpRoot, + env: { H5_WECHAT_NEWS_MORNING_DRAFT_WORKER_ENABLED: '1' }, + logger: { + log() {}, + warn(message) { warnings.push(String(message)); }, + }, + executeTask: async () => { + fs.writeFileSync(path.join(publicDir, 'daily-news-0911.html'), `${SAMPLE_HTML}`); + }, + intervalMs: 30_000, + runOnStart: false, + setIntervalFn: () => ({ unref() {} }), + }); + + try { + const generated = await worker.generateToday({ now: fixedNow, force: true }); + assert.equal(generated.skipped, false); + assert.equal(generated.page.slug, 'daily-news-0911'); + assert.match(fs.readFileSync(path.join(publicDir, 'daily-news-0911.html'), 'utf8'), /assisted/); + assert.ok(warnings.some((item) => item.includes('news002 gate remaining after assist'))); + } finally { + fs.rmSync(tmpRoot, { recursive: true, force: true }); + } +}); + test('schedule helpers detect today page and due windows', () => { const { isNewsMorningPageForToday, diff --git a/wechat-news-morning-draft.mjs b/wechat-news-morning-draft.mjs index 49130c8..a26e041 100644 --- a/wechat-news-morning-draft.mjs +++ b/wechat-news-morning-draft.mjs @@ -23,7 +23,7 @@ import { import { getLocalParts, localDateKey, startOfLocalDay } from './schedule-time.mjs'; import { countNewsMorningStoryCards } from './wechat-news-morning-dedup.mjs'; import { - auditNews002ImageCoverage, + assistNews002Html, extractNews002CardImageUrls, isWeakNewsMorningImageUrl, } from './wechat-news-morning-images.mjs'; @@ -794,6 +794,7 @@ export function buildDailyNewsWechatDraftArticle({ imageUrlMap = new Map(), publicBaseUrl = '', userId = '', + layoutStyle = '', } = {}) { const meta = extractDailyNewsArticleMeta(html); const footerLength = estimateDailyNewsDraftFooterLength({ publicUrl, qrcodeImageUrl }); @@ -803,6 +804,7 @@ export function buildDailyNewsWechatDraftArticle({ imageUrlMap, includeFixedLayout: false, fixedLayoutLength: footerLength, + layoutStyle, }); let content = applyDailyNewsWechatDraftFooter(inline.content, { publicUrl, @@ -834,48 +836,54 @@ export async function buildDailyNewsWechatDraftArticleForPush({ wechatFetch = undiciFetch, memindLibRoot = process.cwd(), publishDir = '', + skipQualityGate = false, + logger = console, + layoutStyle = '', } = {}) { + const assisted = skipQualityGate + ? { html: String(html ?? ''), changed: false, actions: [], auditAfter: { ok: true, violations: [] } } + : assistNews002Html(html, { requireTemplateStructure: true }); + const workingHtml = assisted.html; + if (assisted.changed) { + logger.warn?.('[NewsMorningDraft] news002 gate assisted push', { + actions: assisted.actions, + remaining: assisted.auditAfter.violations, + }); + } const { qrcodeImageUrl } = await uploadWechatDraftBrandAssets({ accessToken, wechatFetch, memindLibRoot, }); - const imageUrlMap = await uploadDailyNewsBodyImagesForWechat(accessToken, html, { + const imageUrlMap = await uploadDailyNewsBodyImagesForWechat(accessToken, workingHtml, { publishDir, wechatFetch, memindLibRoot, }); - if (/class="[^"]*\bmasthead\b/i.test(String(html ?? ''))) { - const audit = auditNews002ImageCoverage(html, { - requireTemplateStructure: true, - requireAllCoreSections: false, - requireLeadCount: false, - }); - const pushBlocking = audit.violations.filter((item) => ( - item === 'story-image-missing' - || item === 'lead-image-missing' - || item === 'too-many-story-cards' - || item === 'legacy-news001-dom' - )); - if (pushBlocking.length > 0) { - throw new Error(`news002 页面未通过推送闸门：${pushBlocking.join(', ')}`); - } - const missingUploads = extractNews002CardImageUrls(html).filter((ref) => ( + if (!skipQualityGate && /class="[^"]*\bmasthead\b/i.test(workingHtml)) { + const remaining = Array.isArray(assisted.auditAfter?.violations) + ? assisted.auditAfter.violations + : []; + const missingUploads = extractNews002CardImageUrls(workingHtml).filter((ref) => ( !isWeakNewsMorningImageUrl(ref) )).filter((ref) => ( !imageUrlMap.has(ref) && !imageUrlMap.has(ref.replace(/^\.\//, '')) && !imageUrlMap.has(ref.replace(/^\//, '')) )); - if (missingUploads.length > 0) { - throw new Error(`news002 有 ${missingUploads.length} 张正文配图未成功上传微信，已阻止草稿推送`); + if (remaining.length > 0 || missingUploads.length > 0) { + logger.warn?.('[NewsMorningDraft] news002 gate remaining after assist', { + violations: remaining, + missingUploads: missingUploads.length, + }); } } return buildDailyNewsWechatDraftArticle({ - html, + html: workingHtml, publicUrl, qrcodeImageUrl, imageUrlMap, + layoutStyle, }); } @@ -1521,6 +1529,8 @@ export function createWechatNewsMorningDraftService( now = Date.now(), pageOverride = null, articleTitleOverride = null, + skipQualityGate = false, + layoutStyle = '', } = {}) { const config = await this.getConfig(); if (!config.enabled) { @@ -1569,6 +1579,9 @@ export function createWechatNewsMorningDraftService( wechatFetch, memindLibRoot, publishDir: path.dirname(page.localPath), + skipQualityGate, + logger, + layoutStyle, }) : preview.article; if (articleTitleOverride) { diff --git a/wechat-news-morning-draft.test.mjs b/wechat-news-morning-draft.test.mjs index e538949..fc1395e 100644 --- a/wechat-news-morning-draft.test.mjs +++ b/wechat-news-morning-draft.test.mjs @@ -5,6 +5,7 @@ import path from 'node:path'; import test from 'node:test'; import { buildDailyNewsWechatDraftArticle, + buildDailyNewsWechatDraftArticleForPush, buildNewsMorningWechatPushText, convertNewsPageHtmlToWechatArticle, createWechatNewsMorningDraftService, @@ -342,6 +343,27 @@ test('internals normalize booleans consistently', () => { assert.equal(wechatNewsMorningDraftInternals.normalizeBoolean('off', true), false); }); +test('buildDailyNewsWechatDraftArticleForPush assists overflow instead of failing', async () => { + const cards = Array.from({ length: 31 }, (_, index) => ( + index < 3 + ? `

头条${index}

正文
` + : `

事件标题${index}

正文
原因
` + )).join(''); + const html = `每日新闻早报 +
    +
    ${cards}
    +`; + const warnings = []; + const article = await buildDailyNewsWechatDraftArticleForPush({ + html, + publicUrl: 'https://m.tkmind.cn/daily-news-0923.html', + logger: { warn(message, detail) { warnings.push({ message, detail }); } }, + }); + assert.equal(typeof article.title, 'string'); + assert.ok(article.contentLength > 0); + assert.ok(warnings.some((item) => String(item.message).includes('news002 gate'))); +}); + test('isNewsMorningContentDegraded counts news002 lead cards', async () => { const { isNewsMorningContentDegraded } = await import('./wechat-news-morning-draft.mjs'); const leads = Array.from({ length: 4 }, (_, i) => diff --git a/wechat-news-morning-images.mjs b/wechat-news-morning-images.mjs index a21d9f5..0283421 100644 --- a/wechat-news-morning-images.mjs +++ b/wechat-news-morning-images.mjs @@ -11,6 +11,7 @@ import { countNews002HeadlineLeadCards, findNews002SectionIdForOffset, } from './news-morning-templates.mjs'; +import { inspectNewsMorningSafety } from './wechat-news-morning-content-filter.mjs'; const DEFAULT_TIMEOUT_MS = 6000; const DEFAULT_USER_AGENT = @@ -845,3 +846,136 @@ export function auditNews002ImageCoverage( sectionFill, }; } + +export function extractNews002BriefItems(html) { + const source = String(html ?? ''); + const match = source.match(/]*\bclass="[^"]*\bbrief\b[^"]*"[^>]*>([\s\S]*?)<\/ol>/i); + if (!match) return []; + const inner = match[1]; + const innerStart = source.indexOf(match[0]) + match[0].indexOf(inner); + const items = []; + const liRe = /]*>[\s\S]*?<\/li>/gi; + let liMatch; + while ((liMatch = liRe.exec(inner)) != null) { + items.push({ + start: innerStart + liMatch.index, + end: innerStart + liMatch.index + liMatch[0].length, + raw: liMatch[0], + title: textOf(liMatch[0]), + }); + } + return items; +} + +function renumberNews002BriefItems(html) { + let index = 0; + return String(html ?? '').replace( + /(]*\bclass="[^"]*\bbrief\b[^"]*"[^>]*>)([\s\S]*?)(<\/ol>)/i, + (_full, open, inner, close) => { + const next = inner.replace(/]*>[\s\S]*?<\/li>/gi, (li) => { + index += 1; + return li.replace(/\s*\d+\s*<\/span>/i, `${index}`); + }); + return `${open}${next}${close}`; + }, + ); +} + +export function stripUnsafeNews002Content(html, { env = process.env } = {}) { + const source = String(html ?? ''); + const cards = extractNews002StoryCards(source); + const removedCards = cards.filter((card) => !inspectNewsMorningSafety({ + title: card.title, + url: card.url, + snippet: textOf(card.raw), + eventKey: card.eventKey, + }, { env }).ok); + let output = source; + for (const card of [...removedCards].sort((left, right) => right.start - left.start)) { + output = `${output.slice(0, card.start)}${output.slice(card.end)}`; + } + const briefs = extractNews002BriefItems(output); + const removedBriefs = briefs.filter((item) => !inspectNewsMorningSafety({ + title: item.title, + }, { env }).ok); + for (const item of [...removedBriefs].sort((left, right) => right.start - left.start)) { + output = `${output.slice(0, item.start)}${output.slice(item.end)}`; + } + if (removedBriefs.length) { + output = renumberNews002BriefItems(output); + } + return { + html: output, + changed: removedCards.length + removedBriefs.length > 0, + removedCardCount: removedCards.length, + removedBriefCount: removedBriefs.length, + keptCardCount: cards.length - removedCards.length, + }; +} + +export function trimNews002ExcessStoryCards(html, { maxStoryCards = 28 } = {}) { + const source = String(html ?? ''); + const cards = extractNews002StoryCards(source).filter((card) => card.title); + if (cards.length <= maxStoryCards) { + return { html: source, changed: false, removedCount: 0, keptCount: cards.length }; + } + const leads = cards.filter((card) => card.isLead); + const others = cards.filter((card) => !card.isLead); + const keepLeads = leads.slice(0, Math.min(leads.length, 4)); + const keepOthers = others.slice(0, Math.max(0, maxStoryCards - keepLeads.length)); + const keep = new Set([...keepLeads, ...keepOthers]); + const removed = cards.filter((card) => !keep.has(card)); + let output = source; + for (const card of [...removed].sort((left, right) => right.start - left.start)) { + output = `${output.slice(0, card.start)}${output.slice(card.end)}`; + } + return { + html: output, + changed: removed.length > 0, + removedCount: removed.length, + keptCount: keepLeads.length + keepOthers.length, + }; +} + +export function assistNews002Html(html, { + maxStoryCards = NEWS002_MAX_STORY_CARDS, + requireTemplateStructure = true, + requireSectionFill = false, + env = process.env, +} = {}) { + const source = String(html ?? ''); + const auditOptions = { + maxStoryCards, + requireTemplateStructure, + requireSectionFill, + env, + }; + const auditBefore = auditNews002ImageCoverage(source, auditOptions); + const actions = []; + let output = source; + const stripped = stripUnsafeNews002Content(output, { env }); + if (stripped.changed) { + output = stripped.html; + if (stripped.removedCardCount) { + actions.push(`stripped-unsafe-cards:${stripped.removedCardCount}`); + } + if (stripped.removedBriefCount) { + actions.push(`stripped-unsafe-briefs:${stripped.removedBriefCount}`); + } + } + if (auditNews002ImageCoverage(output, auditOptions).violations.includes('too-many-story-cards')) { + const trimmed = trimNews002ExcessStoryCards(output, { maxStoryCards }); + if (trimmed.changed) { + output = trimmed.html; + actions.push(`trimmed-story-cards:${trimmed.removedCount}`); + } + } + const auditAfter = auditNews002ImageCoverage(output, auditOptions); + return { + html: output, + changed: actions.length > 0, + actions, + auditBefore, + auditAfter, + }; +} diff --git a/wechat-news-morning-images.test.mjs b/wechat-news-morning-images.test.mjs index b4d2043..dbde6a2 100644 --- a/wechat-news-morning-images.test.mjs +++ b/wechat-news-morning-images.test.mjs @@ -4,7 +4,9 @@ import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; import { + assistNews002Html, auditNews002ImageCoverage, + stripUnsafeNews002Content, cardHasDisplayableImage, isWeakNewsMorningImageUrl, enrichNews002HtmlWithSourceImages, @@ -16,7 +18,9 @@ import { normalizeNewsMorningImageUrl, normalizeNewsMorningLocalImageRef, pruneNews002ImagelessStoryCards, + trimNews002ExcessStoryCards, } from './wechat-news-morning-images.mjs'; +import { NEWS002_MAX_STORY_CARDS } from './news-morning-templates.mjs'; const TINY_PNG = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', @@ -95,8 +99,74 @@ test('enrichNews002HtmlWithSourceImages inserts media and body wrappers', async }); }); +test('stripUnsafeNews002Content removes adult cards and bare-url titles', () => { + const html = [ + '
      ', + '
    1. 1莫斯科遭无人机袭击
      2. ', + '
    2. 291每日大赛今日吃瓜爆料
      3. ', + '
    ', + '

    莫斯科遭无人机袭击

    ', + '

    AI短剧标题含鸡巴与海角网

    上科技操

    ', + '

    youtube.com/watch?v=abc

    ', + ].join(''); + const stripped = stripUnsafeNews002Content(html); + assert.equal(stripped.removedCardCount, 2); + assert.equal(stripped.removedBriefCount, 1); + assert.match(stripped.html, /莫斯科遭无人机袭击/); + assert.doesNotMatch(stripped.html, /海角网/); + assert.doesNotMatch(stripped.html, /youtube\.com\/watch/); + assert.doesNotMatch(stripped.html, /吃瓜/); + assert.match(stripped.html, /1<\/span>莫斯科遭无人机袭击<\/span>/); +}); + +test('assistNews002Html strips unsafe cards without failing the page', () => { + const source = [ + '
    ', + '

    正规头条

    ', + '

    91每日大赛-今日吃瓜爆料

    ', + ].join(''); + const assisted = assistNews002Html(source, { requireTemplateStructure: false }); + assert.equal(assisted.changed, true); + assert.ok(assisted.actions.some((item) => item.startsWith('stripped-unsafe-cards:'))); + assert.match(assisted.html, /正规头条/); + assert.doesNotMatch(assisted.html, /吃瓜/); +}); + +test('assistNews002Html trims excess cards instead of failing the page', () => { + const extraIndex = NEWS002_MAX_STORY_CARDS; + const cards = Array.from({ length: NEWS002_MAX_STORY_CARDS + 2 }, (_, index) => ( + index < 3 + ? `

    头条${index}

    ` + : `

    事件标题${index}

    ` + )).join(''); + const source = `
    ${cards}`; + const assisted = assistNews002Html(source, { requireTemplateStructure: false }); + assert.equal(assisted.changed, true); + assert.ok(assisted.actions.some((item) => item.startsWith('trimmed-story-cards:'))); + assert.equal(assisted.auditBefore.violations.includes('too-many-story-cards'), true); + assert.equal(assisted.auditAfter.violations.includes('too-many-story-cards'), false); + assert.equal(assisted.auditAfter.cardCount, NEWS002_MAX_STORY_CARDS); + assert.match(assisted.html, /头条0/); + assert.doesNotMatch(assisted.html, new RegExp(`事件标题${extraIndex}`)); +}); + +test('trimNews002ExcessStoryCards keeps lead cards first', () => { + const html = [ + '

    普通在前

    ', + '

    头条一

    ', + ...Array.from({ length: 28 }, (_, index) => ( + `

    灌水${index}

    ` + )), + ].join(''); + const trimmed = trimNews002ExcessStoryCards(html, { maxStoryCards: 4 }); + assert.equal(trimmed.changed, true); + assert.match(trimmed.html, /头条一/); + assert.match(trimmed.html, /普通在前/); + assert.equal(trimmed.keptCount, 4); +}); + test('auditNews002ImageCoverage blocks excess cards but keeps imageless stories', () => { - const cards = Array.from({ length: 29 }, (_, index) => ( + const cards = Array.from({ length: NEWS002_MAX_STORY_CARDS + 1 }, (_, index) => ( `

    事件标题${index}

    ` )).join(''); const audit = auditNews002ImageCoverage( diff --git a/wechat-news-morning-search.mjs b/wechat-news-morning-search.mjs index 9ae60d2..02f2c43 100644 --- a/wechat-news-morning-search.mjs +++ b/wechat-news-morning-search.mjs @@ -10,6 +10,7 @@ import { rankNewsWithEngine, shouldUseNewsEngineLatestCollection, } from './news-engine-client.mjs'; +import { isUnsafeNewsMorningItem } from './wechat-news-morning-content-filter.mjs'; function envFlag(value, fallback = false) { const raw = String(value ?? '').trim().toLowerCase(); @@ -104,17 +105,25 @@ const STALE_NEWS_PATTERNS = [ /(\d{4})年(\d{1,2})月(\d{1,2})日.*(\d{4})年(\d{1,2})月(\d{1,2})日/u, ]; -export function filterFreshNewsMorningSearchResults(results = [], { year = null } = {}) { +export function filterFreshNewsMorningSearchResults(results = [], { year = null, env = process.env } = {}) { const currentYear = Number(year ?? new Date().getFullYear()); return (Array.isArray(results) ? results : []).filter((item) => { const blob = `${item?.title ?? ''} ${item?.url ?? ''} ${item?.snippet ?? ''}`; if (STALE_NEWS_PATTERNS.some((pattern) => pattern.test(blob))) return false; + if (isUnsafeNewsMorningItem(item, { env })) return false; const years = [...blob.matchAll(/\b(20\d{2})\b/g)].map((match) => Number(match[1])); if (years.length && years.every((value) => value < currentYear - 1)) return false; return Boolean(item?.title || item?.url); }); } +export function filterSafeNewsMorningGroups(groups = [], options) { + return (Array.isArray(groups) ? groups : []).map((group) => ({ + ...group, + results: (group.results ?? []).filter((item) => !isUnsafeNewsMorningItem(item, options)), + })); +} + export function normalizeNewsMorningUrl(url) { try { const parsed = new URL(String(url ?? '').trim()); @@ -417,9 +426,12 @@ export async function prefetchNewsMorningSearxngBundle({ fetchImpl: newsEngineFetchImpl, logger, }); - const engineGroups = Array.isArray(ranking?.groups) && ranking.groups.length - ? ranking.groups - : groups; + const engineGroups = filterSafeNewsMorningGroups( + Array.isArray(ranking?.groups) && ranking.groups.length + ? ranking.groups + : groups, + { env }, + ); const { groups: dedupedGroups, removedCount } = dedupeNewsMorningSearchGroups(engineGroups); const groupsWithEngineImages = applyRankingImagesToGroups(dedupedGroups, ranking, { normalizeUrl: normalizeNewsMorningUrl, diff --git a/wechat-news-morning-search.test.mjs b/wechat-news-morning-search.test.mjs index 31a87e6..723f2a2 100644 --- a/wechat-news-morning-search.test.mjs +++ b/wechat-news-morning-search.test.mjs @@ -29,6 +29,16 @@ test('buildNewsMorningSearchQueries covers core and extended channels', () => { assert.ok(queries.some((item) => item.id === 'official_feeds' && /reuters\.com/u.test(item.query))); }); +test('filterFreshNewsMorningSearchResults drops unsafe adult and bare-url hits', () => { + const filtered = filterFreshNewsMorningSearchResults([ + { title: '国乒今日出征亚运会', url: 'https://news.qq.com/a/20260923' }, + { title: '91每日大赛-今日吃瓜爆料', url: 'https://91dscg3.com/' }, + { title: 'youtube.com/watch?v=abc', url: 'https://www.youtube.com/watch?v=abc' }, + ], { year: 2026 }); + assert.equal(filtered.length, 1); + assert.match(filtered[0].title, /国乒/); +}); + test('filterFreshNewsMorningSearchResults drops wikinews archive hits', () => { const filtered = filterFreshNewsMorningSearchResults([ { title: '2021年9月18日中文報紙頭條', url: 'https://zh.wikinews.org/wiki/2021%E5%B9%B49%E6%9C%8818%E6%97%A5' }, diff --git a/wechat-news-morning-searxng-fallback.mjs b/wechat-news-morning-searxng-fallback.mjs index 15712e2..40e9af5 100644 --- a/wechat-news-morning-searxng-fallback.mjs +++ b/wechat-news-morning-searxng-fallback.mjs @@ -14,6 +14,7 @@ import { normalizeNewsMorningTitle, normalizeNewsMorningUrl, } from './wechat-news-morning-search.mjs'; +import { isUnsafeNewsMorningItem } from './wechat-news-morning-content-filter.mjs'; const SECTION_BY_ID = new Map(NEWS002_SECTIONS.map((item) => [item.id, item])); @@ -220,6 +221,7 @@ function pickSearchCandidates(groupMap, poolIds, seenTitles, { requireImage = tr const title = String(item?.title ?? '').trim(); const url = String(item?.url ?? '').trim(); if (!title || !url) continue; + if (isUnsafeNewsMorningItem({ title, url, snippet: item?.snippet })) continue; if (isDuplicateNewsMorningStory(seenTitles, { title, url })) continue; const image = normalizeNewsMorningImageUrl(item?.image || ''); if (requireImage && (!image || isWeakNewsMorningImageUrl(image))) continue; diff --git a/wechat-news-morning-searxng-fallback.test.mjs b/wechat-news-morning-searxng-fallback.test.mjs index 57a55cf..7f500bc 100644 --- a/wechat-news-morning-searxng-fallback.test.mjs +++ b/wechat-news-morning-searxng-fallback.test.mjs @@ -143,3 +143,34 @@ test('supplementNews002HtmlFromSearxngGroups fills missing core sections and lea assert.match(result.html, /补栏头条二/u); assert.match(result.html, /国内补栏一/u); }); + +test('supplementNews002HtmlFromSearxngGroups skips unsafe search candidates', () => { + const groups = [ + { + id: 'world', + title: '国际', + results: [ + { title: '91每日大赛-今日吃瓜爆料', url: 'https://91dscg3.com/', image: 'https://cdn.example/bad.jpg', snippet: '黑料' }, + { title: 'youtube.com/watch?v=abc', url: 'https://www.youtube.com/watch?v=abc', image: 'https://cdn.example/yt.jpg' }, + { title: '国际安全稿', url: 'https://news.example/safe-world', image: 'https://cdn.example/safe.jpg', snippet: '安全' }, + ], + }, + { + id: 'tech', + title: '科技', + results: [ + { title: '科技安全稿', url: 'https://news.example/safe-tech', image: 'https://cdn.example/tech.jpg', snippet: '科技' }, + ], + }, + ]; + const result = supplementNews002HtmlFromSearxngGroups( + THIN_AGENT_HTML, + groups, + { violations: ['missing-sections:world|tech'] }, + ); + assert.equal(result.changed, true); + assert.match(result.html, /国际安全稿/u); + assert.match(result.html, /科技安全稿/u); + assert.doesNotMatch(result.html, /吃瓜/u); + assert.doesNotMatch(result.html, /youtube\.com\/watch/u); +});