From 105a72c1f1490f728c2b9448470bd9673322f2af Mon Sep 17 00:00:00 2001 From: john Date: Mon, 20 Jul 2026 07:37:43 +0800 Subject: [PATCH] feat: add guarded image generation controls --- agent-run-gateway.mjs | 47 +++++++++++++-- agent-run-gateway.test.mjs | 28 ++++++++- asset-gateway.mjs | 21 +++++-- asset-gateway.test.mjs | 22 +++++++ chat-intent-router.mjs | 92 +++++++++++++++++++++++++++-- chat-intent-router.test.mjs | 48 +++++++++++++++ image-make-client.mjs | 4 +- mindspace-sandbox-mcp.mjs | 4 +- session-reply-wait.mjs | 79 +++++++++++++++++++++++++ session-reply-wait.test.mjs | 42 +++++++++++++ skills/static-page-publish/SKILL.md | 11 ++++ src/components/ChatPanel.tsx | 32 +++++++++- src/components/ChatView.tsx | 1 + src/components/SpaceChatPanel.tsx | 2 + src/hooks/usePageEditSubChat.ts | 12 +++- src/hooks/useTKMindChat.ts | 3 + src/index.css | 31 ++++++++++ src/utils/imageGeneration.ts | 13 ++++ user-publish.mjs | 13 +++- 19 files changed, 482 insertions(+), 23 deletions(-) create mode 100644 src/utils/imageGeneration.ts diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index 7ceea2c..0c59c47 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -66,6 +66,33 @@ function extractRunMessageText(row) { .join('\n'); } +function resolveRequiredImageGeneration(row, routing) { + const message = parseDbJsonColumn(row?.user_message_json, {}) ?? {}; + const metadata = message.metadata ?? {}; + const runMetadata = metadata[RUN_METADATA_KEY] ?? metadata.agentRun ?? {}; + const requestedMode = String( + runMetadata.imageGenerationMode ?? runMetadata.image_generation_mode ?? metadata.imageGenerationMode ?? '', + ).trim().toLowerCase(); + if (requestedMode === 'required') return true; + if (requestedMode === 'disabled') return false; + return routing?.imageGeneration?.mode === 'required'; +} + +export function assertRequiredImageGenerationCompleted(row, routing, toolEvidence) { + if (!resolveRequiredImageGeneration(row, routing)) return; + if (toolEvidence?.generateImage?.succeeded === true) return; + if (toolEvidence?.generateImage?.called !== true) { + const error = new Error('本轮明确要求生成图片，但 Agent 没有实际调用 generate_image；历史回复不能作为本轮工具证据'); + error.code = 'IMAGE_GENERATION_REQUIRED_NOT_CALLED'; + error.retryable = false; + throw error; + } + const error = new Error('本轮明确要求生成图片，但未获得 image_make 的有效 PNG/JPEG/WebP 产物，不能标记成功'); + error.code = 'IMAGE_GENERATION_REQUIRED_MISSING'; + error.retryable = false; + throw error; +} + function positiveInteger(value, fallback) { const n = Number(value); if (!Number.isFinite(n) || n <= 0) return fallback; @@ -780,6 +807,7 @@ export function createAgentRunGateway({ }); } await invalidatePortalDirectChatSnapshot(sessionId); + let toolEvidence = null; const awaitSessionFinish = envFlag(process.env.MEMIND_AGENT_RUN_AWAIT_SESSION_FINISH, true) && runOptions.toolMode === 'chat' && typeof tkmindProxy.submitSessionReplyAndAwaitFinishForUser === 'function'; @@ -799,7 +827,10 @@ export function createAgentRunGateway({ await appendEvent(runId, 'session_finished', { sessionId, tokenState: finish.tokenState ?? null, + toolCalls: finish.toolEvidence?.calls ?? [], + generateImage: finish.toolEvidence?.generateImage ?? null, }); + toolEvidence = finish.toolEvidence ?? null; } catch (err) { if (await recoverRunFromDeliverables({ runId, @@ -823,10 +854,16 @@ export function createAgentRunGateway({ }, ); } - return { sessionId, routing }; + return { sessionId, routing, toolEvidence }; } - async function finalizeSuccessfulRun(runId, row, sessionId, { routing = null } = {}) { + async function finalizeSuccessfulRun( + runId, + row, + sessionId, + { routing = null, toolEvidence = null } = {}, + ) { + assertRequiredImageGenerationCompleted(row, routing, toolEvidence); let deliveryResult = null; if (typeof syncUserPagesOnSuccess === 'function') { deliveryResult = await syncUserPagesOnSuccess({ @@ -933,12 +970,12 @@ export function createAgentRunGateway({ const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt }); try { - const { sessionId, routing } = await runWithTimeout(runId, () => executeRun(row, runId)); - await finalizeSuccessfulRun(runId, row, sessionId, { routing }); + const execution = await runWithTimeout(runId, () => executeRun(row, runId)); + await finalizeSuccessfulRun(runId, row, execution.sessionId, execution); } catch (err) { const latest = await getRunById(runId); const recoverySessionId = latest?.agent_session_id ?? row.agent_session_id ?? null; - if (await recoverRunFromDeliverables({ + if (err?.code !== 'IMAGE_GENERATION_REQUIRED_MISSING' && await recoverRunFromDeliverables({ runId, userId: row.user_id, sessionId: recoverySessionId, diff --git a/agent-run-gateway.test.mjs b/agent-run-gateway.test.mjs index 2df0068..3213fca 100644 --- a/agent-run-gateway.test.mjs +++ b/agent-run-gateway.test.mjs @@ -4,7 +4,33 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; -import { createAgentRunGateway } from './agent-run-gateway.mjs'; +import { + assertRequiredImageGenerationCompleted, + createAgentRunGateway, +} from './agent-run-gateway.mjs'; + +test('required image generation cannot succeed without a verified raster image_make result', () => { + const row = { + user_message_json: JSON.stringify({ + metadata: { memindRun: { imageGenerationMode: 'required' } }, + }), + }; + assert.throws( + () => assertRequiredImageGenerationCompleted(row, null, { + generateImage: { called: false, succeeded: false }, + }), + (error) => error?.code === 'IMAGE_GENERATION_REQUIRED_NOT_CALLED', + ); + assert.throws( + () => assertRequiredImageGenerationCompleted(row, null, { + generateImage: { called: true, succeeded: false }, + }), + (error) => error?.code === 'IMAGE_GENERATION_REQUIRED_MISSING', + ); + assert.doesNotThrow(() => assertRequiredImageGenerationCompleted(row, null, { + generateImage: { called: true, succeeded: true, jobId: 'job-1', mimeType: 'image/webp' }, + })); +}); function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} } = {}) { const runs = new Map(); diff --git a/asset-gateway.mjs b/asset-gateway.mjs index ef96a0b..f469bba 100644 --- a/asset-gateway.mjs +++ b/asset-gateway.mjs @@ -32,10 +32,10 @@ export const ASSET_PLUGIN_CATALOG = [ ]; export const IMAGE_MAKE_PURPOSE_CATALOG = [ - { id: 'inline_image', label: '页面正文图片', presetId: 'memind_square_illustration' }, - { id: 'hero', label: '页面头图', presetId: 'memind_dark_hero' }, - { id: 'card_cover', label: '卡片封面', presetId: 'memind_card_cover' }, - { id: 'feed_cover', label: '信息流与缩略图源图', presetId: 'memind_feed_cover_source' }, + { id: 'inline_image', label: '页面正文图片', presetId: 'memind_square_illustration', strategy: 'generate', sourcePurpose: null }, + { id: 'hero', label: '页面头图', presetId: 'memind_dark_hero', strategy: 'generate', sourcePurpose: null }, + { id: 'card_cover', label: '卡片封面图', presetId: 'memind_card_cover', strategy: 'derive', sourcePurpose: 'hero' }, + { id: 'feed_cover', label: '信息流缩略图', presetId: 'memind_feed_cover_source', strategy: 'derive', sourcePurpose: 'hero' }, ]; const imageMakePurposeIds = new Set(IMAGE_MAKE_PURPOSE_CATALOG.map((purpose) => purpose.id)); @@ -281,10 +281,21 @@ export function createAssetGatewayConfigService(pool, { llmProviderService = nul if (!resolved.plugin.purposes?.[purpose]) { return { ok: false, code: 'purpose_disabled', message: '该图片用途未启用，可安全降级到原有流程' }; } + const purposeConfig = IMAGE_MAKE_PURPOSE_CATALOG.find((item) => item.id === purpose); + if (purposeConfig?.strategy === 'derive' && resolved.plugin.purposes?.hero) { + return { + ok: false, + code: 'reuse_hero_required', + message: '页面头图已启用：请只生成一次 hero，并从该主图派生卡片封面和信息流缩略图', + sourcePurpose: 'hero', + }; + } return { ok: true, purpose, - presetId: IMAGE_MAKE_PURPOSE_CATALOG.find((item) => item.id === purpose)?.presetId, + presetId: purposeConfig?.presetId, + strategy: purposeConfig?.strategy ?? 'generate', + sourcePurpose: purposeConfig?.sourcePurpose ?? null, plugin: resolved.plugin, }; }, diff --git a/asset-gateway.test.mjs b/asset-gateway.test.mjs index 1cc6866..bbc2fad 100644 --- a/asset-gateway.test.mjs +++ b/asset-gateway.test.mjs @@ -120,6 +120,12 @@ test('image_make purposes are disabled by default and resolved independently', a feed_cover: true, }); assert.equal((await service.resolveImageGenerationPurpose('hero')).ok, true); + assert.deepEqual(await service.resolveImageGenerationPurpose('feed_cover'), { + ok: false, + code: 'reuse_hero_required', + message: '页面头图已启用：请只生成一次 hero，并从该主图派生卡片封面和信息流缩略图', + sourcePurpose: 'hero', + }); assert.equal(plugin.llmProviderKeyId, null); assert.equal(plugin.llmModel, null); assert.deepEqual(await service.resolveImageGenerationPurpose('card_cover'), { @@ -128,3 +134,19 @@ test('image_make purposes are disabled by default and resolved independently', a message: '该图片用途未启用，可安全降级到原有流程', }); }); + +test('derived cover purpose can generate one source when hero output is disabled', async () => { + const service = createAssetGatewayConfigService(createPool(), { llmProviderService }); + await service.updateGlobalConfig({ enabled: true }, { updatedBy: 'admin-1' }); + await service.updatePluginConfig('asset-generate', { + enabled: true, + provider: 'image_make', + purposes: { hero: false, card_cover: true }, + }, { updatedBy: 'admin-1' }); + + const resolved = await service.resolveImageGenerationPurpose('card_cover'); + assert.equal(resolved.ok, true); + assert.equal(resolved.strategy, 'derive'); + assert.equal(resolved.sourcePurpose, 'hero'); + assert.equal(resolved.presetId, 'memind_card_cover'); +}); diff --git a/chat-intent-router.mjs b/chat-intent-router.mjs index 0332b8a..093d349 100644 --- a/chat-intent-router.mjs +++ b/chat-intent-router.mjs @@ -84,6 +84,83 @@ const OBVIOUS_DIRECT_CHAT_PATTERNS = [ /(?:继续|再).{0,8}(?:写|讲|说|编).{0,10}(?:诗|故事|笑话|散文)/u, ]; +export const IMAGE_GENERATION_MODE = { + AUTO: 'auto', + REQUIRED: 'required', + DISABLED: 'disabled', +}; + +const IMAGE_GENERATION_REQUIRED_PATTERNS = [ + /(?:生成|制作|创建|画|绘制|配|添加|使用).{0,16}(?:AI\s*)?(?:图片|图像|背景图|封面图|插画|照片|主图|配图)/iu, + /(?:图片|图像|背景图|封面图|插画|照片|主图|配图).{0,16}(?:生成|制作|创建|画|绘制|作为|用作)/iu, + /(?:全屏|页面|网页|卡片|信息流).{0,12}(?:背景|封面|主图).{0,12}(?:图片|图像|照片)/iu, + /(?:AI配图|AI图片|真图|真实图片|生图)/iu, +]; + +const IMAGE_GENERATION_DISABLED_PATTERNS = [ + /(?:不要|无需|不需要|禁止|关闭|取消).{0,12}(?:AI\s*)?(?:生图|图片生成|生成图片|配图)/iu, + /(?:纯文字|只要文字|不要图片|不要配图|不用图片|不用配图)/iu, +]; + +const VISUAL_PAGE_IMAGE_HINT_PATTERNS = [ + /(?:精美|沉浸式|视觉化|视觉效果|全屏背景|大唐盛景|海报风|摄影风)/u, + /(?:旅行|美食|活动|品牌|产品|促销|诗词|唐诗).{0,16}(?:页面|网页|H5|h5|封面|背景)/u, +]; + +export function normalizeImageGenerationMode(value) { + const normalized = String(value ?? '').trim().toLowerCase(); + if (normalized === IMAGE_GENERATION_MODE.REQUIRED) return IMAGE_GENERATION_MODE.REQUIRED; + if (normalized === IMAGE_GENERATION_MODE.DISABLED) return IMAGE_GENERATION_MODE.DISABLED; + return IMAGE_GENERATION_MODE.AUTO; +} + +export function resolveImageGenerationDecision({ text, requestedMode = IMAGE_GENERATION_MODE.AUTO } = {}) { + const normalizedText = String(text ?? '').trim(); + const normalizedMode = normalizeImageGenerationMode(requestedMode); + if (normalizedMode === IMAGE_GENERATION_MODE.REQUIRED) { + return { mode: IMAGE_GENERATION_MODE.REQUIRED, source: 'user_override', reason: '用户在输入区强制开启 AI 配图' }; + } + if (normalizedMode === IMAGE_GENERATION_MODE.DISABLED) { + return { mode: IMAGE_GENERATION_MODE.DISABLED, source: 'user_override', reason: '用户在输入区关闭 AI 配图' }; + } + if (IMAGE_GENERATION_DISABLED_PATTERNS.some((pattern) => pattern.test(normalizedText))) { + return { mode: IMAGE_GENERATION_MODE.DISABLED, source: 'intent', reason: '用户明确要求不生成图片' }; + } + if (IMAGE_GENERATION_REQUIRED_PATTERNS.some((pattern) => pattern.test(normalizedText))) { + return { mode: IMAGE_GENERATION_MODE.REQUIRED, source: 'intent', reason: '检测到明确的图片生成需求' }; + } + if ( + isPageGenerationIntent(normalizedText) + && VISUAL_PAGE_IMAGE_HINT_PATTERNS.some((pattern) => pattern.test(normalizedText)) + ) { + return { mode: IMAGE_GENERATION_MODE.REQUIRED, source: 'intent', reason: '视觉类页面需要真实主图和缩略图源图' }; + } + return { mode: IMAGE_GENERATION_MODE.AUTO, source: 'default', reason: '未检测到必须生成或禁止生成图片的要求' }; +} + +function requestedImageGenerationMode(userMessage) { + const metadata = userMessage?.metadata ?? {}; + const runMetadata = metadata.memindRun ?? metadata.agentRun ?? {}; + return normalizeImageGenerationMode( + runMetadata.imageGenerationMode ?? runMetadata.image_generation_mode ?? metadata.imageGenerationMode, + ); +} + +function buildImageGenerationInstruction(decision) { + if (decision?.mode === IMAGE_GENERATION_MODE.REQUIRED) { + return [ + '图片策略：强制生成。必须先调用 sandbox-fs__generate_image，并使用返回的 workspaceRelativePath/publicUrl 作为页面真实主图或背景图，再生成 HTML 与缩略图。', + '性能规则：页面头图、卡片封面和信息流缩略图只能共享一张主图。优先只调用一次 purpose=hero；hero 成功后必须复用该资产并裁剪派生，禁止再调用 card_cover 或 feed_cover。只有 hero 用途被后台关闭、且确实只需要对应封面时，才允许改用一个 card_cover 或 feed_cover 请求。正文插图 inline_image 仅在后台开启且正文确有需要时单独生成。', + '本轮证据要求：必须在当前这一次执行中发起新的 generate_image 工具请求；历史对话中的成功或失败都不能作为本轮结果。禁止在没有新 toolRequest/toolResponse 的情况下复述“已尝试”“Provider 未启用”或其他旧结论。', + '完成条件：返回结果必须 ok=true，包含 jobId，且产物为 PNG/JPEG/WebP；不得用 SVG、CSS 绘图、旧素材或网页搜索图片冒充本次生图。生图失败时必须明确报告失败，不得静默降级后宣称完成。', + ].join('\n'); + } + if (decision?.mode === IMAGE_GENERATION_MODE.DISABLED) { + return '图片策略：关闭。本轮禁止调用 generate_image；如需页面视觉，只能使用现有合法资源或纯 CSS，并如实说明未生成新图片。'; + } + return '图片策略：自动。仅在用户需求确实需要新主图、背景图、正文配图或封面时调用 generate_image；不要为了装饰无条件生图。'; +} + /** Realtime / web-search prompts — fast-path to agent even when LLM router is enabled. */ const REALTIME_INFO_AGENT_PATTERNS = [ /(?:世界杯|欧冠|NBA|英超|西甲|意甲|德甲|法甲|欧洲杯|亚洲杯|奥运会).{0,16}(?:赛况|赛程|比分|战况|结果|积分|排名|进展|情况)/u, @@ -719,6 +796,7 @@ export function buildAgentOrchestrationAgentText({ `路由判定：${classification.reason}`, classification.agentBrief ? `执行要点：${classification.agentBrief}` : '', classification.suggestedSkill ? `建议 skill：${classification.suggestedSkill}` : '', + buildImageGenerationInstruction(classification.imageGeneration), skillPrompt, memoryLines.length ? [ @@ -1114,8 +1192,7 @@ export function createChatIntentRouter(options = {}) { if (!classification) return classification; const base = { ...classification }; delete base.decision; - return finalizeRouterClassification( - coercePageGenerationSkill( + const coerced = coercePageGenerationSkill( coercePageDataSkill( coerceRealtimeWebSkill(base, text, { grantedSkills }), text, @@ -1123,9 +1200,14 @@ export function createChatIntentRouter(options = {}) { ), text, { grantedSkills }, - ), - decisionContext, - ); + ); + return finalizeRouterClassification({ + ...coerced, + imageGeneration: resolveImageGenerationDecision({ + text, + requestedMode: requestedImageGenerationMode(userMessage), + }), + }, decisionContext); }; const ruleResult = classifyWithRules({ text, diff --git a/chat-intent-router.test.mjs b/chat-intent-router.test.mjs index a98a87f..2cb1068 100644 --- a/chat-intent-router.test.mjs +++ b/chat-intent-router.test.mjs @@ -23,10 +23,58 @@ import { isRealtimeInfoQuestion, isRichChannelPageIntent, isRichPublishableContentIntent, + IMAGE_GENERATION_MODE, + resolveImageGenerationDecision, coerceRealtimeWebSkill, resolveChatIntentRouterPolicy, } from './chat-intent-router.mjs'; +test('image generation intent requires a real image for explicit background requests', () => { + const decision = resolveImageGenerationDecision({ + text: '帮我生成一首唐诗页面，背景是大唐盛景图片，页面要精美', + }); + assert.equal(decision.mode, IMAGE_GENERATION_MODE.REQUIRED); + assert.equal(decision.source, 'intent'); +}); + +test('image generation user override can force or disable image generation', () => { + assert.equal(resolveImageGenerationDecision({ + text: '做一个纯文字页面', + requestedMode: 'required', + }).mode, IMAGE_GENERATION_MODE.REQUIRED); + assert.equal(resolveImageGenerationDecision({ + text: '生成一个带背景图的页面', + requestedMode: 'disabled', + }).mode, IMAGE_GENERATION_MODE.DISABLED); +}); + +test('router injects a fail-closed generate_image contract for required visual pages', async () => { + const router = createChatIntentRouter(); + const text = '帮我生成一个精美唐诗页面，背景是大唐盛景图片'; + const userMessage = { + role: 'user', + content: [{ type: 'text', text }], + metadata: { + displayText: text, + memindRun: { imageGenerationMode: 'auto' }, + }, + }; + const classification = await router.classify({ + userMessage, + grantedSkills: ['static-page-publish'], + }); + assert.equal(classification.imageGeneration.mode, IMAGE_GENERATION_MODE.REQUIRED); + const enriched = router.applyAgentOrchestration(userMessage, classification, { + grantedSkills: ['static-page-publish'], + }); + assert.match(enriched.content[0].text, /sandbox-fs__generate_image/); + assert.match(enriched.content[0].text, /当前这一次执行中发起新的 generate_image/); + assert.match(enriched.content[0].text, /历史对话中的成功或失败都不能作为本轮结果/); + assert.match(enriched.content[0].text, /hero 成功后必须复用该资产/); + assert.match(enriched.content[0].text, /禁止再调用 card_cover 或 feed_cover/); + assert.match(enriched.content[0].text, /不得用 SVG/); +}); + test('classifyWithRules routes greetings to direct chat', () => { const result = classifyWithRules({ text: '你好', diff --git a/image-make-client.mjs b/image-make-client.mjs index 318c659..bf5af30 100644 --- a/image-make-client.mjs +++ b/image-make-client.mjs @@ -52,7 +52,7 @@ export function createImageMakeClient({ token, fetchImpl = globalThis.fetch, requestTimeoutMs = 15_000, - generationTimeoutMs = 10 * 60_000, + generationTimeoutMs = 15 * 60_000, pollIntervalMs = 1_000, maxResultBytes = DEFAULT_MAX_RESULT_BYTES, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), @@ -62,7 +62,7 @@ export function createImageMakeClient({ const bearerToken = String(token ?? '').trim(); if (!bearerToken) throw new Error('image_make token is required'); const perRequestTimeout = positiveInteger(requestTimeoutMs, 15_000); - const overallTimeout = positiveInteger(generationTimeoutMs, 10 * 60_000); + const overallTimeout = positiveInteger(generationTimeoutMs, 15 * 60_000); const pollingDelay = positiveInteger(pollIntervalMs, 1_000); const resultLimit = positiveInteger(maxResultBytes, DEFAULT_MAX_RESULT_BYTES); diff --git a/mindspace-sandbox-mcp.mjs b/mindspace-sandbox-mcp.mjs index 9a02ea5..7ce7dc9 100644 --- a/mindspace-sandbox-mcp.mjs +++ b/mindspace-sandbox-mcp.mjs @@ -142,7 +142,7 @@ async function generateMindSpaceImage({ purpose, prompt, negativePrompt, idempot prompt, negative_prompt: negativePrompt, }), - signal: AbortSignal.timeout(Number(process.env.IMAGE_MAKE_GENERATION_TIMEOUT_MS ?? 600_000)), + signal: AbortSignal.timeout(Number(process.env.IMAGE_MAKE_GENERATION_TIMEOUT_MS ?? 900_000)), }); let body = null; try { @@ -233,7 +233,7 @@ const ALL_TOOLS = [ { name: 'generate_image', description: - '通过平台 image_make 服务生成页面图片并直接写入当前用户 MindSpace。仅在后台开关开启时生效；失败时继续原页面流程。返回 canonical publicUrl 与 workspaceRelativePath。', + '通过平台 image_make 服务生成页面图片并直接写入当前用户 MindSpace。页面头图、卡片封面和信息流缩略图必须共享一次生成结果：优先只调用一次 hero，再从该图派生其他尺寸；禁止在 hero 成功后继续调用 card_cover/feed_cover。正文插图 inline_image 仅按需单独生成。仅在后台对应开关开启时生效，返回 canonical publicUrl 与 workspaceRelativePath。', inputSchema: { type: 'object', properties: { diff --git a/session-reply-wait.mjs b/session-reply-wait.mjs index 25622f0..c9b007f 100644 --- a/session-reply-wait.mjs +++ b/session-reply-wait.mjs @@ -24,6 +24,82 @@ export function eventMatchesRequest(event, requestId) { return !routingId || routingId === requestId; } +function messageContentItems(event) { + const candidates = [ + event?.message?.content, + event?.data?.message?.content, + event?.content, + ]; + return candidates.find((content) => Array.isArray(content)) ?? []; +} + +function parseGenerateImageResult(toolResult) { + const content = toolResult?.value?.content; + if (!Array.isArray(content)) return null; + for (const item of content) { + if (item?.type !== 'text' || !String(item.text ?? '').trim()) continue; + try { + const result = JSON.parse(String(item.text)); + const mimeType = String(result?.source?.mimeType ?? result?.asset?.mimeType ?? '').toLowerCase(); + if ( + result?.ok === true + && String(result?.jobId ?? '').trim() + && /^image\/(?:png|jpeg|webp)$/.test(mimeType) + ) { + return { + jobId: String(result.jobId), + mimeType, + assetId: String(result?.asset?.id ?? '').trim() || null, + publicUrl: String(result?.asset?.publicUrl ?? '').trim() || null, + workspaceRelativePath: String(result?.asset?.workspaceRelativePath ?? '').trim() || null, + }; + } + } catch { + // Ignore non-JSON tool text; it cannot prove a successful image_make delivery. + } + } + return null; +} + +function createToolEvidenceCollector() { + const requestNames = new Map(); + const calls = new Set(); + const successfulCalls = new Set(); + let generateImage = { called: false, succeeded: false }; + + return { + observe(event) { + for (const item of messageContentItems(event)) { + if (item?.type === 'toolRequest') { + const name = String(item?.toolCall?.value?.name ?? '').trim(); + if (!name) continue; + calls.add(name); + if (item.id) requestNames.set(String(item.id), name); + if (name.endsWith('generate_image')) generateImage = { called: true, succeeded: false }; + continue; + } + if (item?.type !== 'toolResponse') continue; + const name = requestNames.get(String(item.id ?? '')) ?? ''; + const result = item.toolResult ?? {}; + const succeeded = result.status === 'success' && result?.value?.isError !== true; + if (name && succeeded) successfulCalls.add(name); + if (!name.endsWith('generate_image')) continue; + const generated = succeeded ? parseGenerateImageResult(result) : null; + generateImage = generated + ? { called: true, succeeded: true, ...generated } + : { called: true, succeeded: false }; + } + }, + snapshot() { + return { + calls: [...calls], + successfulCalls: [...successfulCalls], + generateImage, + }; + }, + }; +} + export async function consumeSessionEventsUntilFinish( body, { @@ -40,6 +116,7 @@ export async function consumeSessionEventsUntilFinish( const reader = Readable.fromWeb(body); const decoder = new TextDecoder(); + const toolEvidence = createToolEvidenceCollector(); let buffer = ''; const deadline = Date.now() + Math.max(1, Number(timeoutMs) || 1); @@ -60,6 +137,7 @@ export async function consumeSessionEventsUntilFinish( const event = parseSessionStreamEvent(trimmed); if (!event) continue; if (!eventMatchesRequest(event, requestId)) continue; + toolEvidence.observe(event); onEvent?.(event); if (event.type === 'Error') { const err = new Error(String(event.error ?? 'session reply failed')); @@ -71,6 +149,7 @@ export async function consumeSessionEventsUntilFinish( return { finishEvent: event, tokenState: event.token_state ?? null, + toolEvidence: toolEvidence.snapshot(), }; } } diff --git a/session-reply-wait.test.mjs b/session-reply-wait.test.mjs index c40e4d1..9b0d7e6 100644 --- a/session-reply-wait.test.mjs +++ b/session-reply-wait.test.mjs @@ -33,3 +33,45 @@ test('consumeSessionEventsUntilFinish resolves on Finish', async () => { assert.equal(result.finishEvent.type, 'Finish'); assert.equal(result.tokenState.totalTokens, 12); }); + +test('consumeSessionEventsUntilFinish records a successful raster generate_image result', async () => { + const toolResult = JSON.stringify({ + ok: true, + jobId: 'job-1', + source: { mimeType: 'image/webp' }, + asset: { + id: 'asset-1', + publicUrl: '/MindSpace/user/public/images/hero.webp', + workspaceRelativePath: 'public/images/hero.webp', + }, + }); + const frames = [ + `data: ${JSON.stringify({ + type: 'Message', request_id: 'req-1', message: { + content: [{ + type: 'toolRequest', id: 'call-1', + toolCall: { value: { name: 'sandbox-fs__generate_image', arguments: { purpose: 'hero' } } }, + }], + }, + })}\n\n`, + `data: ${JSON.stringify({ + type: 'Message', request_id: 'req-1', message: { + content: [{ + type: 'toolResponse', id: 'call-1', + toolResult: { status: 'success', value: { content: [{ type: 'text', text: toolResult }], isError: false } }, + }], + }, + })}\n\n`, + 'data: {"type":"Finish","request_id":"req-1"}\n\n', + ]; + const stream = new ReadableStream({ + start(controller) { + for (const frame of frames) controller.enqueue(new TextEncoder().encode(frame)); + controller.close(); + }, + }); + const result = await consumeSessionEventsUntilFinish(stream, { requestId: 'req-1', timeoutMs: 5000 }); + assert.equal(result.toolEvidence.generateImage.called, true); + assert.equal(result.toolEvidence.generateImage.succeeded, true); + assert.equal(result.toolEvidence.generateImage.jobId, 'job-1'); +}); diff --git a/skills/static-page-publish/SKILL.md b/skills/static-page-publish/SKILL.md index aebce39..b75d20e 100644 --- a/skills/static-page-publish/SKILL.md +++ b/skills/static-page-publish/SKILL.md @@ -28,6 +28,7 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报 11. **纯静态页（无 Page Data 问卷/后台）禁止调用 `private_data_bind_workspace_page`**；`write_file` / `edit_file` 落盘 `public/*.html` 后，直接按下方「回复格式」交付 **MindSpace 路径**（`/MindSpace/<用户ID>/public/...`），无需 bind、无需口令 12. **微信分享必须使用 MindSpace 路径**：该路由会在服务端注入 `og:site_name=TKMind 智趣`、封面缩略图（`*.thumbnail.png`）与微信 JS-SDK 分享桥；禁止交付 `/u/用户名/pages/...` 作为用户可见链接 13. **只要页面接收用户输入并要求以后查看、汇总、统计、修改或删除，就不是纯静态页**。禁止使用 `localStorage` / `sessionStorage` / `IndexedDB` 保存任何数据；必须立即 `load_skill` → `page-data-collect`，通过 Page Data API 写入当前用户专属 PostgreSQL schema。禁止 SQLite、静态 JSON/JS 文件或内存 fallback 冒充持久化。记账、日记、打卡、清单、台账、问卷、报名及其管理页都适用。 +14. 当编排提示为“图片策略：强制生成”或用户明确要求新图片、背景图、主图、封面、插画、照片时，必须先调用 `sandbox-fs__generate_image`；不得用 SVG、CSS 绘图、旧素材或网页搜索图片冒充本次生成结果。 详细约束以工作区内的 `.goosehints` 与 `.agents/skills/static-page-publish/SKILL.md` 为准。 @@ -41,6 +42,16 @@ description: 在专属 MindSpace 目录生成可公开访问的静态 HTML 报 6. 若用户明确要求 Word/docx 下载，必须用 `generate_docx`（sandbox-fs 工具）生成 `public/<同名>.docx`，再确认链接目标已落盘 7. 若用户明确要求长图下载，必须用 `long-image-download` 生成 `public/<同名>.long.png`，并确认文件存在 +## AI 图片生成（按意图与用户选择） + +- **强制生成**：页面主图优先只调用一次 `generate_image(purpose=hero)`；卡片封面和信息流缩略图必须复用这张主图裁剪派生，hero 成功后禁止再调用 `card_cover` / `feed_cover` +- 只有后台关闭 `hero`、且任务确实只需要某一种封面时，才允许改用一次 `card_cover` 或 `feed_cover`；正文配图 `inline_image` 仅在后台开启且正文确有需要时单独生成 +- 成功结果必须满足 `ok=true`、包含 `jobId`，且 `source.mimeType` 为 `image/png`、`image/jpeg` 或 `image/webp` +- HTML 的 ``、背景图及 `mindspace-cover.cover` 必须引用本次返回的 `asset.publicUrl` 或对应工作区相对路径；页面主图默认同时作为缩略图源图 +- **自动**：只在用户需求确实需要新主图、背景图、正文配图或封面时调用 +- **关闭**：禁止调用 `generate_image` +- 强制生成失败时必须如实报告，禁止静默改用 SVG/旧图后宣称图片生成完成 + ## 按需伴生下载文件 - 默认不生成伴生文件；只有用户明确要求下载附件时才生成 diff --git a/src/components/ChatPanel.tsx b/src/components/ChatPanel.tsx index ac805c2..30b9389 100644 --- a/src/components/ChatPanel.tsx +++ b/src/components/ChatPanel.tsx @@ -1,10 +1,15 @@ import { ChangeEvent, useCallback, useEffect, useLayoutEffect, useRef, useState, type ClipboardEvent } from 'react'; -import { BrainCircuit, Database } from 'lucide-react'; +import { BrainCircuit, Database, Image, ImageOff, ImagePlus } from 'lucide-react'; import { useNetworkStatus } from '../hooks/useNetworkStatus'; import { openAvatarPicker } from '../utils/userAvatar'; import { CHAT_SKILL_OPTIONS, filterChatSkills } from '../utils/chatSkills'; import { getMessageSaveActions } from '../utils/messageSave'; import { getDisplayText } from '../utils/message'; +import { + IMAGE_GENERATION_MODE_LABELS, + nextImageGenerationMode, + type ImageGenerationMode, +} from '../utils/imageGeneration'; import { downloadChatMessageDocx, } from '../api/client'; @@ -186,6 +191,7 @@ export function ChatPanel({ messageId?: string; forceDeepReasoning?: boolean; pgRequired?: boolean; + imageGenerationMode?: ImageGenerationMode; selectedChatSkill?: string; fileAttachments?: ChatFileAttachment[]; }, @@ -219,6 +225,7 @@ export function ChatPanel({ const [voiceNotice, setVoiceNotice] = useState(null); const [forceDeepReasoning, setForceDeepReasoning] = useState(false); const [pgRequired, setPgRequired] = useState(false); + const [imageGenerationMode, setImageGenerationMode] = useState('auto'); const [chatControlOnboardingStep, setChatControlOnboardingStep] = useState<0 | 1 | 2 | null>(null); const [pendingImages, setPendingImages] = useState([]); const [pendingFiles, setPendingFiles] = useState([]); @@ -745,6 +752,7 @@ export function ChatPanel({ messageId: outgoingMessageId, forceDeepReasoning, pgRequired, + imageGenerationMode, selectedChatSkill, fileAttachments: fileAttachmentsToSend, }); @@ -774,6 +782,7 @@ export function ChatPanel({ }, [ forceDeepReasoning, pgRequired, + imageGenerationMode, input, onSubmit, onUploadFile, @@ -1229,6 +1238,27 @@ export function ChatPanel({ 需要长期使用的数据，放进专属空间 )} +