Files
memind/wechat-news-morning-draft-worker.mjs
T
john 57a9aadf53
Memind CI / Test, build, and release guards (push) Successful in 4m43s
fix(news): repair instead of block in daily news generation and push
- news002 image/structure audit no longer throws: prune empty sections and
  publish a best-effort page with a warning
- WeChat push strips images that failed to upload instead of refusing the draft
- forceGenerateOnce is cleared before running, so a failed run cannot loop;
  the archived page is restored when generation fails
- degraded-page regeneration runs at most once per day
- content safety rules cover vulgar sexual terms and gossip/adult sites

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-23 11:29:35 +08:00

697 lines
24 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import { executeScheduledTask } from './scheduled-task-executor.mjs';
import { localDateKey } from './schedule-time.mjs';
import {
buildNewsMorningAutoGenerationTask,
findLatestNewsMorningPage,
isLocalScheduleDue,
isNewsMorningPageDegraded,
isNewsMorningPageForToday,
isWithinNewsMorningGenerateLeadWindow,
} from './wechat-news-morning-draft.mjs';
import { prefetchNewsMorningSearxngBundle } from './wechat-news-morning-search.mjs';
import {
repairNews002ImagelessCardsFromSearxng,
shouldUseNewsMorningSearxngFallback,
supplementNews002HtmlFromSearxngGroups,
} from './wechat-news-morning-searxng-fallback.mjs';
import {
dedupeNewsMorningHtml,
formatNewsMorningDedupeSummary,
} from './wechat-news-morning-dedup.mjs';
import {
formatNewsMorningContentSafetySummary,
sanitizeNewsMorningHtml,
sanitizeNewsMorningHtmlSync,
} from './wechat-news-morning-content-safety.mjs';
import {
auditNews002ImageCoverage,
enrichNews002HtmlWithSourceImages,
localizeNews002CardImages,
promoteNews002LeadCards,
pruneEmptyNews002Sections,
} from './wechat-news-morning-images.mjs';
import {
isWechatNewsMorningDraftWorkerEnabled,
wechatNewsMorningDraftExecutionTimeoutMs,
wechatNewsMorningDraftWorkerIntervalMs,
} from './wechat-news-morning-draft-worker-config.mjs';
import { isPassiveCanaryRuntime } from './server/portal-runtime-role.mjs';
function resolveTodayPage(config, h5Root, now = Date.now()) {
const page = findLatestNewsMorningPage({
h5Root,
userId: config.sourceUserId,
slugPattern: config.pageSlugPattern,
date: new Date(now),
timezone: config.timezone,
});
if (!page || !isNewsMorningPageForToday(page, { date: new Date(now), timezone: config.timezone })) {
return null;
}
return page;
}
/**
* 落盘后的事件去重闸门:prompt 层的「禁止重复」无法验证,这里做一次程序化清理。
* 去重失败不阻断生成流程,只告警。
*/
function applyNewsMorningDedupe(page, logger) {
if (!page?.localPath || !fs.existsSync(page.localPath)) return null;
try {
const result = dedupeNewsMorningHtml(fs.readFileSync(page.localPath, 'utf8'));
const summary = formatNewsMorningDedupeSummary(result);
if (result.changed) {
fs.writeFileSync(page.localPath, result.html);
logger.log?.('[NewsMorningDraft] deduped page', { slug: page.slug, summary });
} else {
logger.log?.('[NewsMorningDraft] dedupe clean', { slug: page.slug, summary });
}
return result;
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] dedupe failed:',
error instanceof Error ? error.message : error,
);
return null;
}
}
/**
* 内容安全闸门(底线):成人 / 未成年人 / 法律合规 / 人文伦理。
* 只过滤不阻断:清洗异常时退回同步规则版,再失败也只告警,绝不抛出。
*/
async function applyNewsMorningContentSafety(page, { reviewer, reviewCache, logger, stage }) {
if (!page?.localPath || !fs.existsSync(page.localPath)) return null;
const source = fs.readFileSync(page.localPath, 'utf8');
let result = null;
try {
result = await sanitizeNewsMorningHtml(source, { reviewer, reviewCache, logger });
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] content safety failed, falling back to rules-only:',
error instanceof Error ? error.message : error,
);
try {
result = sanitizeNewsMorningHtmlSync(source);
} catch (fallbackError) {
logger.warn?.(
'[NewsMorningDraft] content safety rules-only fallback failed:',
fallbackError instanceof Error ? fallbackError.message : fallbackError,
);
return null;
}
}
if (result.changed) fs.writeFileSync(page.localPath, result.html);
logger.log?.('[NewsMorningDraft] content safety', {
slug: page.slug,
stage,
summary: formatNewsMorningContentSafetySummary(result),
});
return result;
}
async function applyNews002ImageGate(
page,
config,
logger,
env = process.env,
{ searchGroups = [], safetyExemptSectionIds = [] } = {},
) {
if (config?.templateId !== 'news002') return null;
if (!page?.localPath || !fs.existsSync(page.localPath)) return null;
let source = fs.readFileSync(page.localPath, 'utf8');
const normalizedBefore = promoteNews002LeadCards(source);
if (normalizedBefore.promotedCount > 0 || normalizedBefore.demotedCount > 0) {
logger.log?.('[NewsMorningDraft] news002 normalized headline cards', {
slug: page.slug,
promotedCount: normalizedBefore.promotedCount,
demotedCount: normalizedBefore.demotedCount,
});
source = normalizedBefore.html;
}
let enriched = await enrichNews002HtmlWithSourceImages(source, {
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_TIMEOUT_MS ?? 6000) || 6000,
concurrency: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_CONCURRENCY ?? 8) || 8,
});
let audit = auditNews002ImageCoverage(enriched.html, {
requireTemplateStructure: true,
exemptSectionIds: safetyExemptSectionIds,
});
if (
!audit.ok
&& searchGroups.length > 0
&& shouldUseNewsMorningSearxngFallback(env)
) {
const supplemented = supplementNews002HtmlFromSearxngGroups(
enriched.html,
searchGroups,
audit,
);
if (supplemented.changed) {
logger.log?.('[NewsMorningDraft] searxng fallback supplement', {
slug: page.slug,
...supplemented.stats,
});
enriched = await enrichNews002HtmlWithSourceImages(supplemented.html, {
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_TIMEOUT_MS ?? 6000) || 6000,
concurrency: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_CONCURRENCY ?? 8) || 8,
});
const normalizedAfterSupplement = promoteNews002LeadCards(enriched.html);
if (normalizedAfterSupplement.promotedCount > 0 || normalizedAfterSupplement.demotedCount > 0) {
enriched = { ...enriched, html: normalizedAfterSupplement.html };
logger.log?.('[NewsMorningDraft] news002 normalized headline cards after searxng', {
slug: page.slug,
promotedCount: normalizedAfterSupplement.promotedCount,
demotedCount: normalizedAfterSupplement.demotedCount,
});
}
audit = auditNews002ImageCoverage(enriched.html, {
requireTemplateStructure: true,
exemptSectionIds: safetyExemptSectionIds,
});
}
if (!audit.ok && audit.missingTitles?.length > 0) {
const repaired = repairNews002ImagelessCardsFromSearxng(enriched.html, searchGroups);
if (repaired.changed) {
logger.log?.('[NewsMorningDraft] searxng fallback image repair', {
slug: page.slug,
...repaired.stats,
});
enriched = await enrichNews002HtmlWithSourceImages(repaired.html, {
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_TIMEOUT_MS ?? 6000) || 6000,
concurrency: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_CONCURRENCY ?? 8) || 8,
});
const normalizedAfterRepair = promoteNews002LeadCards(enriched.html);
if (normalizedAfterRepair.promotedCount > 0 || normalizedAfterRepair.demotedCount > 0) {
enriched = { ...enriched, html: normalizedAfterRepair.html };
}
audit = auditNews002ImageCoverage(enriched.html, {
requireTemplateStructure: true,
exemptSectionIds: safetyExemptSectionIds,
});
}
}
}
if (!audit.ok && audit.violations.includes('invalid-lead-count')) {
const normalizedFinal = promoteNews002LeadCards(enriched.html);
if (normalizedFinal.promotedCount > 0 || normalizedFinal.demotedCount > 0) {
enriched = { ...enriched, html: normalizedFinal.html };
audit = auditNews002ImageCoverage(enriched.html, {
requireTemplateStructure: true,
exemptSectionIds: safetyExemptSectionIds,
});
logger.log?.('[NewsMorningDraft] news002 normalized headline cards before localize', {
slug: page.slug,
promotedCount: normalizedFinal.promotedCount,
demotedCount: normalizedFinal.demotedCount,
});
}
}
let localized = await localizeNews002CardImages(enriched.html, {
env,
publishDir: path.dirname(page.localPath),
slug: page.slug,
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_DOWNLOAD_TIMEOUT_MS ?? 8000) || 8000,
});
fs.writeFileSync(page.localPath, localized.html);
audit = auditNews002ImageCoverage(localized.html, {
requireTemplateStructure: true,
requireSectionFill: true,
exemptSectionIds: safetyExemptSectionIds,
env,
});
if (
!audit.ok
&& searchGroups.length > 0
&& shouldUseNewsMorningSearxngFallback(env)
&& audit.violations.some((item) => item.startsWith('underfilled:'))
) {
const fillSupplement = supplementNews002HtmlFromSearxngGroups(
localized.html,
searchGroups,
audit,
);
logger.log?.('[NewsMorningDraft] searxng fill supplement after section audit', {
slug: page.slug,
changed: fillSupplement.changed,
...fillSupplement.stats,
});
if (fillSupplement.changed) {
enriched = await enrichNews002HtmlWithSourceImages(fillSupplement.html, {
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_TIMEOUT_MS ?? 6000) || 6000,
concurrency: Number(env.MEMIND_NEWS_MORNING_IMAGE_META_CONCURRENCY ?? 8) || 8,
});
const normalizedAfterFill = promoteNews002LeadCards(enriched.html);
if (normalizedAfterFill.promotedCount > 0 || normalizedAfterFill.demotedCount > 0) {
enriched = { ...enriched, html: normalizedAfterFill.html };
}
localized = await localizeNews002CardImages(enriched.html, {
env,
publishDir: path.dirname(page.localPath),
slug: page.slug,
timeoutMs: Number(env.MEMIND_NEWS_MORNING_IMAGE_DOWNLOAD_TIMEOUT_MS ?? 8000) || 8000,
});
fs.writeFileSync(page.localPath, localized.html);
audit = auditNews002ImageCoverage(localized.html, {
requireTemplateStructure: true,
requireSectionFill: true,
exemptSectionIds: safetyExemptSectionIds,
env,
});
}
}
if (!audit.ok) {
const pruned = pruneEmptyNews002Sections(localized.html, { requireImage: false });
if (pruned.removedSections > 0) {
localized = { ...localized, html: pruned.html };
fs.writeFileSync(page.localPath, localized.html);
audit = auditNews002ImageCoverage(localized.html, {
requireTemplateStructure: true,
requireSectionFill: true,
exemptSectionIds: safetyExemptSectionIds,
env,
});
}
}
logger.log?.('[NewsMorningDraft] news002 image audit', {
slug: page.slug,
attempted: enriched.attemptedCount,
resolved: enriched.resolvedCount,
localized: localized.localizedCount,
localizeFailed: localized.failedCount,
...audit,
});
if (!audit.ok) {
logger.warn?.('[NewsMorningDraft] news002 audit incomplete, publishing best-effort page', {
slug: page.slug,
violations: audit.violations,
cardCount: audit.cardCount,
imageCount: audit.imageCount,
leadImages: `${audit.leadImageCount}/${audit.leadCount}`,
});
}
return { ...audit, degraded: !audit.ok };
}
function archiveTodayPageForForce(config, h5Root, now, logger) {
const page = resolveTodayPage(config, h5Root, now);
if (!page?.localPath || !fs.existsSync(page.localPath)) return null;
const stamp = new Date(now).toISOString().replace(/[:.]/g, '-');
const archived = path.join(path.dirname(page.localPath), `_archived-${page.slug}-${stamp}.html`);
fs.renameSync(page.localPath, archived);
let archivedThumb = null;
if (page.thumbPath && fs.existsSync(page.thumbPath)) {
archivedThumb = archived.replace(/\.html$/i, '.thumbnail.png');
fs.renameSync(page.thumbPath, archivedThumb);
}
logger.log?.('[NewsMorningDraft] archived today page before force generate', {
from: page.relativePath,
to: path.basename(archived),
});
return {
originalPath: page.localPath,
archivedPath: archived,
originalThumbPath: archivedThumb ? page.thumbPath : null,
archivedThumbPath: archivedThumb,
};
}
/** 强制重生成失败时把原稿放回原位;新稿已落盘则保留新稿。 */
function restoreArchivedTodayPage(archived, logger) {
if (!archived?.archivedPath || !fs.existsSync(archived.archivedPath)) return false;
if (fs.existsSync(archived.originalPath)) return false;
try {
fs.renameSync(archived.archivedPath, archived.originalPath);
if (
archived.archivedThumbPath
&& fs.existsSync(archived.archivedThumbPath)
&& !fs.existsSync(archived.originalThumbPath)
) {
fs.renameSync(archived.archivedThumbPath, archived.originalThumbPath);
}
logger.log?.('[NewsMorningDraft] restored archived today page after failed generate', {
to: path.basename(archived.originalPath),
});
return true;
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] restore archived today page failed:',
error instanceof Error ? error.message : error,
);
return false;
}
}
export function startWechatNewsMorningDraftWorker({
wechatNewsMorningDraftService = null,
mpConfig = null,
userAuth = null,
tkmindProxy = null,
agentRunGateway = null,
cursorExecutorPolicyService = null,
sessionSnapshotService = null,
pool = null,
h5Root = null,
env = process.env,
executeTask = executeScheduledTask,
prefetchNewsSearch = prefetchNewsMorningSearxngBundle,
contentReviewer = null,
logger = console,
intervalMs = wechatNewsMorningDraftWorkerIntervalMs(env),
executionTimeoutMs = wechatNewsMorningDraftExecutionTimeoutMs(env),
runOnStart = true,
setIntervalFn = setInterval,
} = {}) {
if (
isPassiveCanaryRuntime(env)
|| !isWechatNewsMorningDraftWorkerEnabled(env)
|| !wechatNewsMorningDraftService
|| !mpConfig?.enabled
|| !userAuth
|| !tkmindProxy
|| !h5Root
) {
return { stop() {} };
}
let stopped = false;
let running = false;
let generationInFlightDateKey = null;
let degradedRegenerateDateKey = null;
const ensureTodayPage = async (config, dateKey, { now = Date.now(), force = false } = {}) => {
if (generationInFlightDateKey === dateKey) return null;
generationInFlightDateKey = dateKey;
let archived = null;
try {
if (force) {
archived = archiveTodayPageForForce(config, h5Root, now, logger);
}
let searchBrief = '';
let searchGroups = [];
try {
const searchBundle = await prefetchNewsSearch({
now,
timezone: config.timezone,
env,
});
searchBrief = String(searchBundle?.brief ?? searchBundle ?? '').trim();
searchGroups = Array.isArray(searchBundle?.groups) ? searchBundle.groups : [];
} catch (prefetchError) {
logger.warn?.(
'[NewsMorningDraft] searxng prefetch failed:',
prefetchError instanceof Error ? prefetchError.message : prefetchError,
);
}
const task = buildNewsMorningAutoGenerationTask(config, { now, searchBrief });
await executeTask(task, {
userAuth,
tkmindProxy,
agentRunGateway,
cursorExecutorPolicyService,
sessionSnapshotService,
pool,
h5Root,
timeoutMs: executionTimeoutMs,
logger,
});
const page = resolveTodayPage(config, h5Root, now);
if (!page) {
throw new Error('新闻早报页面生成未完成');
}
applyNewsMorningDedupe(page, logger);
const reviewCache = new Map();
const safety = await applyNewsMorningContentSafety(page, {
reviewer: contentReviewer,
reviewCache,
logger,
stage: 'post-generate',
});
const safetyExemptSectionIds = safety?.exemptSectionIds ?? [];
try {
await applyNews002ImageGate(page, config, logger, env, { searchGroups, safetyExemptSectionIds });
} catch (imageError) {
logger.warn?.(
'[NewsMorningDraft] news002 image repair failed, keeping generated page:',
imageError instanceof Error ? imageError.message : imageError,
);
}
await applyNewsMorningContentSafety(page, {
reviewer: contentReviewer,
reviewCache,
logger,
stage: 'post-image-gate',
});
await wechatNewsMorningDraftService.recordAutoGenerationRun({
status: 'success',
pageSlug: page.slug,
});
logger.log?.('[NewsMorningDraft] auto-generated today page', {
slug: page.slug,
userId: config.sourceUserId,
force,
});
return page;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
await wechatNewsMorningDraftService.recordAutoGenerationRun({
status: 'failed',
errorMessage: message,
}).catch(() => {});
logger.warn?.('[NewsMorningDraft] auto-generate failed:', message);
restoreArchivedTodayPage(archived, logger);
throw error;
} finally {
if (generationInFlightDateKey === dateKey) {
generationInFlightDateKey = null;
}
}
};
const runOnce = async () => {
if (running || stopped) return;
running = true;
try {
const config = await wechatNewsMorningDraftService.getConfig();
if (!config.enabled || !config.sourceUserId) return;
const now = Date.now();
if (config.forcePushOnce) {
try {
const result = await wechatNewsMorningDraftService.pushDraft({
triggeredBy: 'manual-push',
requireToday: config.forcePushRequireToday !== false,
now,
});
await wechatNewsMorningDraftService.updateConfig(
{ forcePushOnce: false },
{ updatedBy: 'manual-push' },
);
logger.log?.('[NewsMorningDraft] force push succeeded', {
draftMediaId: result.draftMediaId,
coverSource: result.cover?.source ?? null,
pageSlug: result.preview?.page?.slug ?? null,
});
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] force push failed:',
error instanceof Error ? error.message : error,
);
}
return;
}
const timezone = config.timezone;
const dateKey = localDateKey(now, timezone);
const toleranceMinutes = Math.max(1, Math.ceil(Number(intervalMs) / 60_000));
let todayPage = resolveTodayPage(config, h5Root, now);
const todayPageDegraded = todayPage && isNewsMorningPageDegraded(todayPage);
if (
todayPageDegraded
&& !config.forceGenerateOnce
&& generationInFlightDateKey !== dateKey
&& degradedRegenerateDateKey !== dateKey
&& (
isWithinNewsMorningGenerateLeadWindow(config, now)
|| isLocalScheduleDue(
{ hour: config.pushHour, minute: config.pushMinute, timezone },
now,
toleranceMinutes + 180,
)
)
) {
logger.warn?.('[NewsMorningDraft] today page degraded, regenerating', {
slug: todayPage.slug,
userId: config.sourceUserId,
dateKey,
});
degradedRegenerateDateKey = dateKey;
try {
todayPage = await ensureTodayPage(config, dateKey, { now, force: true });
if (
todayPage
&& config.autoPushEnabled
&& !isNewsMorningPageDegraded(todayPage)
&& isLocalScheduleDue(
{ hour: config.pushHour, minute: config.pushMinute, timezone },
now,
toleranceMinutes + 180,
)
) {
const result = await wechatNewsMorningDraftService.pushDraft({
triggeredBy: 'degraded-regenerate',
now,
});
logger.log?.('[NewsMorningDraft] degraded regenerate push succeeded', {
draftMediaId: result.draftMediaId,
pageSlug: result.preview?.page?.slug ?? todayPage.slug,
});
}
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] degraded page regenerate failed:',
error instanceof Error ? error.message : error,
);
}
}
if (config.forceGenerateOnce && generationInFlightDateKey !== dateKey) {
try {
await wechatNewsMorningDraftService.updateConfig(
{ forceGenerateOnce: false },
{ updatedBy: 'auto-generate' },
);
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] clear forceGenerateOnce failed, skip force generate:',
error instanceof Error ? error.message : error,
);
return;
}
try {
todayPage = await ensureTodayPage(config, dateKey, { now, force: true });
if (config.autoPushEnabled && todayPage) {
const result = await wechatNewsMorningDraftService.pushDraft({
triggeredBy: 'manual-generate',
now,
});
logger.log?.('[NewsMorningDraft] force generate push succeeded', {
draftMediaId: result.draftMediaId,
pageSlug: result.preview?.page?.slug ?? todayPage.slug,
});
}
} catch (error) {
logger.warn?.(
'[NewsMorningDraft] force generate failed:',
error instanceof Error ? error.message : error,
);
}
return;
}
const shouldGenerate =
config.autoGenerateEnabled !== false
&& !todayPage
&& (
isWithinNewsMorningGenerateLeadWindow(config, now)
|| isLocalScheduleDue(
{ hour: config.pushHour, minute: config.pushMinute, timezone },
now,
toleranceMinutes + 120,
)
);
if (shouldGenerate && generationInFlightDateKey !== dateKey) {
try {
todayPage = await ensureTodayPage(config, dateKey, { now });
} catch {
// keep trying during generate window on later scans
}
}
if (!config.autoPushEnabled) return;
if (
!isLocalScheduleDue(
{ hour: config.pushHour, minute: config.pushMinute, timezone },
now,
toleranceMinutes,
)
) {
return;
}
if (await wechatNewsMorningDraftService.hasSuccessfulAutoPushToday({ now })) {
return;
}
todayPage = todayPage ?? resolveTodayPage(config, h5Root, now);
if (!todayPage) {
logger.warn?.('[NewsMorningDraft] skip auto push: today page missing', {
userId: config.sourceUserId,
dateKey,
});
return;
}
const result = await wechatNewsMorningDraftService.pushDraft({
triggeredBy: 'auto-push',
now,
});
logger.log?.('[NewsMorningDraft] auto push succeeded', {
draftMediaId: result.draftMediaId,
pageSlug: result.preview?.page?.slug ?? todayPage.slug,
});
} catch (error) {
logger.warn?.('[NewsMorningDraft] auto worker failed:', error);
} finally {
running = false;
}
};
const timer = setIntervalFn(runOnce, Math.max(5_000, Number(intervalMs) || 30_000));
timer.unref?.();
if (runOnStart) void runOnce();
return {
runOnce,
async generateToday({ force = false, now = Date.now(), pushDraft = false } = {}) {
const config = await wechatNewsMorningDraftService.getConfig();
if (!config.enabled || !config.sourceUserId) {
throw new Error('新闻早报未配置来源用户或未启用');
}
const dateKey = localDateKey(now, config.timezone);
const existing = resolveTodayPage(config, h5Root, now);
if (existing && !force) {
return { page: existing, skipped: true, reason: 'today-page-already-exists' };
}
const page = await ensureTodayPage(config, dateKey, { now, force: true });
if (!page) {
throw new Error('新闻早报页面正在生成中');
}
let draft = null;
if (pushDraft) {
draft = await wechatNewsMorningDraftService.pushDraft({
triggeredBy: 'manual-generate',
now,
});
}
return { page, skipped: false, draft };
},
stop() {
stopped = true;
clearInterval(timer);
},
};
}