tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: expose image generation to page agents

Browse Source
This commit is contained in:
john
2026-07-19 18:39:46 +08:00
parent 5b88f8ece5
commit 9cf12e3786
10 changed files with 220 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+2
View File
@@ -259,6 +259,8 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind
# IMAGE_MAKE_GENERATION_TIMEOUT_MS=600000
# IMAGE_MAKE_POLL_INTERVAL_MS=1000
# IMAGE_MAKE_MAX_RESULT_BYTES=20971520
# Sandbox MCP 调用 Portal 内部生图入口；容器内通常使用 host.docker.internal。
# MINDSPACE_AGENT_API_BASE_URL=http://127.0.0.1:8081/api
# ---------------------------------------------------------------------------
# Memind runtime profile（本地 vs 生产 必须区分）
capabilities.mjs
+17 -1
View File
@@ -288,7 +288,15 @@ export function sandboxDeveloperTools(capabilities) {
export function sandboxMcpTools(capabilities) {
const tools = [];
if (capabilities.static_publish) {
tools.push('read_file', 'write_file', 'edit_file', 'create_dir', 'generate_docx', 'generate_long_image');
tools.push(
'read_file',
'write_file',
'edit_file',
'create_dir',
'generate_image',
'generate_docx',
'generate_long_image',
);
if (capabilities.shell || capabilities.code_browse) tools.push('list_dir');
}
if (capabilities.private_data_space) {
@@ -344,6 +352,14 @@ function sandboxMcpEnvs(sandboxMcp, mcpTools) {
if (sandboxMcp.workspaceRoot || localRoot) envs.MINDSPACE_WORKSPACE_ROOT = sandboxMcp.workspaceRoot || localRoot;
if (sandboxMcp.workspaceRef) envs.MINDSPACE_WORKSPACE_REF = sandboxMcp.workspaceRef;
if (sandboxMcp.userId) envs.PRIVATE_DATA_USER_ID = sandboxMcp.userId;
if (mcpTools.includes('generate_image')) {
if (sandboxMcp.agentApiBaseUrl) {
envs.MINDSPACE_AGENT_API_BASE_URL = sandboxMcp.agentApiBaseUrl;
}
if (sandboxMcp.internalAgentSecret) {
envs.MINDSPACE_INTERNAL_AGENT_SECRET = sandboxMcp.internalAgentSecret;
}
}
if (mcpTools.includes('private_data_info')) {
const userDataPgUrl = resolveSandboxMcpUserDataPgUrl({
portalUrl: sandboxMcp.userDataPgUrl ?? process.env.MINDSPACE_USERDATA_PG_URL,
capabilities.test.mjs
+10 -1
View File
@@ -256,6 +256,7 @@ test('sandboxMcpTools returns correct tool list based on capabilities', () => {
'write_file',
'edit_file',
'create_dir',
'generate_image',
'generate_docx',
'generate_long_image',
'private_data_info',
@@ -328,7 +329,12 @@ test('private_data_space alone exposes private data tools through sandbox MCP',
test('static_publish with sandboxMcp uses stdio sandbox-fs extension instead of developer', () => {
const caps = { ...DEFAULT_USER_CAPABILITIES, static_publish: true };
const sandboxMcp = { serverPath: '/opt/h5/mindspace-sandbox-mcp.mjs', sandboxRoot: '/opt/h5/MindSpace/abc123' };
const sandboxMcp = {
serverPath: '/opt/h5/mindspace-sandbox-mcp.mjs',
sandboxRoot: '/opt/h5/MindSpace/abc123',
agentApiBaseUrl: 'http://host.docker.internal:8081/api',
internalAgentSecret: 'internal-secret',
};
const policy = buildAgentExtensionPolicy(caps, { sandboxMcp });
const sandboxExt = policy.extensionOverrides.find((ext) => ext.name === 'sandbox-fs');
@@ -337,10 +343,13 @@ test('static_publish with sandboxMcp uses stdio sandbox-fs extension instead of
assert.equal(sandboxExt.cmd, process.execPath);
assert.equal(sandboxExt.envs.SANDBOX_ROOT, '/opt/h5/MindSpace/abc123');
assert.equal(sandboxExt.args[1], '/opt/h5/MindSpace/abc123'); // also passed as argv[2]
assert.equal(sandboxExt.envs.MINDSPACE_AGENT_API_BASE_URL, 'http://host.docker.internal:8081/api');
assert.equal(sandboxExt.envs.MINDSPACE_INTERNAL_AGENT_SECRET, 'internal-secret');
assert.ok(sandboxExt.available_tools.includes('write_file'));
assert.ok(sandboxExt.available_tools.includes('read_file'));
assert.ok(sandboxExt.available_tools.includes('generate_docx'));
assert.ok(sandboxExt.available_tools.includes('generate_long_image'));
assert.ok(sandboxExt.available_tools.includes('generate_image'));
// built-in developer extension should only remain for read_image (image_read: true by default)
const developer = policy.extensionOverrides.find((ext) => ext.name === 'developer');
docs/image-make-integration.md
+6
View File
@@ -58,6 +58,12 @@ IMAGE_MAKE_MAX_RESULT_BYTES=20971520
Token 只进入 Memind 运行环境，不在 `memind_adm` 页面或数据库中保存、展示。
页面 Agent 通过 sandbox MCP 的 `generate_image` 工具调用 Portal 内部入口。需要额外配置
`MINDSPACE_AGENT_API_BASE_URL`；MCP 使用现有 `MINDSPACE_INTERNAL_AGENT_SECRET` 鉴权，成功结果
直接返回 MindSpace `publicUrl` 与 `workspaceRelativePath`。工具始终受同一后台总开关、Provider
开关和用途开关约束。容器部署时 base URL 应使用容器可访问的 Portal 地址，不能使用容器自身的
`127.0.0.1`。
## 回滚
在 `memind_adm` 关闭图片生成插件或资产能力总开关即可立即停止新调用。该回滚不删除已经入库
mindspace-image-generation-routes.mjs
+30 -6
View File
@@ -1,8 +1,6 @@
const PURPOSES = new Set(['inline_image', 'hero', 'card_cover', 'feed_cover']);
export function attachMindSpaceImageGenerationRoutes(router, { getService } = {}) {
router.post('/mindspace/v1/images/generate', async (req, res) => {
const service = getService?.();
async function handleGenerate(req, res, { service, userId, consumerPrefix }) {
if (!service) {
return res.status(503).json({
ok: false,
@@ -26,11 +24,11 @@ export function attachMindSpaceImageGenerationRoutes(router, { getService } = {}
return res.status(400).json({ message: '幂等键不能超过 128 字符' });
}
const result = await service.generate({
userId: req.currentUser.id,
userId,
purpose,
prompt,
negativePrompt: req.body?.negative_prompt,
consumerRef: `mindspace:${req.requestId ?? 'request'}:${purpose}`,
consumerRef: `${consumerPrefix}:${req.requestId ?? 'request'}:${purpose}`,
idempotencyKey,
});
if (!result.ok) {
@@ -43,5 +41,31 @@ export function attachMindSpaceImageGenerationRoutes(router, { getService } = {}
return res.status(unavailable ? 503 : 409).json(result);
}
return res.status(201).json({ data: result });
});
}
export function attachMindSpaceImageGenerationRoutes(
router,
{ getService, requireInternal = null } = {},
) {
router.post('/mindspace/v1/images/generate', async (req, res) => {
const service = getService?.();
return handleGenerate(req, res, {
service,
userId: req.currentUser.id,
consumerPrefix: 'mindspace',
});
});
if (requireInternal) {
router.post('/agent/mindspace_image_generate', async (req, res) => {
if (!requireInternal(req, res)) return;
const userId = String(req.body?.user_id ?? req.body?.userId ?? '').trim();
if (!userId) return res.status(400).json({ message: '缺少 user_id' });
return handleGenerate(req, res, {
service: getService?.(),
userId,
consumerPrefix: 'mindspace-agent',
});
});
}
}
mindspace-image-generation-routes.test.mjs
+28 -2
View File
@@ -4,7 +4,7 @@ import test from 'node:test';
import express from 'express';
import { attachMindSpaceImageGenerationRoutes } from './mindspace-image-generation-routes.mjs';
async function start(service) {
async function start(service, requireInternal = null) {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
@@ -12,7 +12,7 @@ async function start(service) {
req.requestId = 'req-1';
next();
});
attachMindSpaceImageGenerationRoutes(app, { getService: () => service });
attachMindSpaceImageGenerationRoutes(app, { getService: () => service, requireInternal });
const server = app.listen(0, '127.0.0.1');
await once(server, 'listening');
return {
@@ -66,3 +66,29 @@ test('image generation route exposes safe fallback without changing existing flo
});
} finally { await fixture.close(); }
});
test('internal image generation route requires its injected secret guard and explicit user', async () => {
const calls = [];
const fixture = await start({
async generate(input) { calls.push(input); return { ok: true, asset: { id: 'asset-1' } }; },
}, (req, res) => {
if (req.get('authorization') === 'Bearer internal-secret') return true;
res.status(401).json({ message: 'invalid secret' });
return false;
});
try {
const denied = await fetch(`${fixture.url}/agent/mindspace_image_generate`, {
method: 'POST', headers: { 'content-type': 'application/json' },
body: JSON.stringify({ user_id: 'user-2', purpose: 'hero', prompt: 'x' }),
});
assert.equal(denied.status, 401);
const allowed = await fetch(`${fixture.url}/agent/mindspace_image_generate`, {
method: 'POST',
headers: { 'content-type': 'application/json', authorization: 'Bearer internal-secret' },
body: JSON.stringify({ user_id: 'user-2', purpose: 'hero', prompt: 'x' }),
});
assert.equal(allowed.status, 201);
assert.equal(calls[0].userId, 'user-2');
assert.equal(calls[0].consumerRef, 'mindspace-agent:req-1:hero');
} finally { await fixture.close(); }
});
mindspace-sandbox-mcp.mjs
+66
View File
@@ -40,6 +40,8 @@ const PRIVATE_DATA_MAX_BYTES = Number(process.env.PRIVATE_DATA_MAX_BYTES ?? 20 *
const PRIVATE_DATA_QUERY_TIMEOUT_MS = Number(process.env.PRIVATE_DATA_QUERY_TIMEOUT_MS ?? 5000);
const PRIVATE_DATA_MAX_ROWS = Number(process.env.PRIVATE_DATA_MAX_ROWS ?? 200);
const PRIVATE_DATA_USER_ID = process.env.PRIVATE_DATA_USER_ID?.trim();
const AGENT_API_BASE_URL = process.env.MINDSPACE_AGENT_API_BASE_URL?.trim();
const INTERNAL_AGENT_SECRET = process.env.MINDSPACE_INTERNAL_AGENT_SECRET?.trim();
const allowedToolsEnv = process.env.ALLOWED_TOOLS?.trim();
const ALLOWED_TOOLS = allowedToolsEnv ? new Set(allowedToolsEnv.split(',').map((s) => s.trim())) : null;
@@ -121,6 +123,39 @@ function runGenerateDocxScript({ outputPath, title, sections }) {
return { bytes: size, stdout: String(stdout ?? '').trim() };
}
async function generateMindSpaceImage({ purpose, prompt, negativePrompt, idempotencyKey }) {
if (!AGENT_API_BASE_URL || !INTERNAL_AGENT_SECRET || !PRIVATE_DATA_USER_ID) {
throw new Error('generate_image: Memind 内部生图入口未配置');
}
const endpoint = new URL('agent/mindspace_image_generate', `${AGENT_API_BASE_URL.replace(/\/+$/, '')}/`);
const response = await fetch(endpoint, {
method: 'POST',
headers: {
accept: 'application/json',
authorization: `Bearer ${INTERNAL_AGENT_SECRET}`,
'content-type': 'application/json',
...(idempotencyKey ? { 'idempotency-key': idempotencyKey } : {}),
},
body: JSON.stringify({
user_id: PRIVATE_DATA_USER_ID,
purpose,
prompt,
negative_prompt: negativePrompt,
}),
signal: AbortSignal.timeout(Number(process.env.IMAGE_MAKE_GENERATION_TIMEOUT_MS ?? 600_000)),
});
let body = null;
try {
body = await response.json();
} catch {
// Safe error below intentionally excludes the response body.
}
if (!response.ok && !body?.fallback) {
throw new Error(`generate_image: Memind 返回 ${response.status}`);
}
return body?.data ?? body ?? { ok: false, fallback: true, code: 'empty_response' };
}
const ALL_TOOLS = [
{
name: 'read_file',
@@ -195,6 +230,25 @@ const ALL_TOOLS = [
required: ['html_path'],
},
},
{
name: 'generate_image',
description:
'通过平台 image_make 服务生成页面图片并直接写入当前用户 MindSpace。仅在后台开关开启时生效；失败时继续原页面流程。返回 canonical publicUrl 与 workspaceRelativePath。',
inputSchema: {
type: 'object',
properties: {
purpose: {
type: 'string',
enum: ['inline_image', 'hero', 'card_cover', 'feed_cover'],
description: '图片用途',
},
prompt: { type: 'string', description: '图片描述，画面内不要包含文字、水印或二维码' },
negative_prompt: { type: 'string', description: '可选负面描述' },
idempotency_key: { type: 'string', description: '可选幂等键；同一页面同一用途应复用' },
},
required: ['purpose', 'prompt'],
},
},
{
name: 'generate_docx',
description:
@@ -534,6 +588,18 @@ async function callTool(name, args) {
fs.mkdirSync(abs, { recursive: true });
return [{ type: 'text', text: `已创建目录 ${args.path}` }];
}
case 'generate_image': {
const purpose = String(args.purpose ?? '').trim();
const prompt = String(args.prompt ?? '').trim();
if (!prompt) throw new Error('generate_image: prompt 不能为空');
const result = await generateMindSpaceImage({
purpose,
prompt,
negativePrompt: String(args.negative_prompt ?? '').trim(),
idempotencyKey: String(args.idempotency_key ?? '').trim(),
});
return [{ type: 'text', text: JSON.stringify(result, null, 2) }];
}
case 'generate_long_image': {
const htmlPath = String(args.html_path ?? args.path ?? '').trim();
if (!htmlPath.toLowerCase().endsWith('.html')) {
mindspace-sandbox-mcp.test.mjs
+56
View File
@@ -4,6 +4,8 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import http from 'node:http';
import { syncPublicDocxDownloads } from './mindspace-public-finish-sync.mjs';
function copyDocxGenerateSkill(root) {
@@ -85,6 +87,60 @@ function startSandbox(root, envOverrides = {}) {
return { child, request };
}
test('generate_image calls the authenticated Memind internal route and returns MindSpace asset data', async (t) => {
const requests = [];
const api = http.createServer((req, res) => {
let body = '';
req.setEncoding('utf8');
req.on('data', (chunk) => { body += chunk; });
req.on('end', () => {
requests.push({ url: req.url, authorization: req.headers.authorization, body: JSON.parse(body) });
res.setHeader('content-type', 'application/json');
res.end(JSON.stringify({
data: {
ok: true,
purpose: 'hero',
asset: {
id: 'asset-1',
publicUrl: 'https://m.example/image.webp',
workspaceRelativePath: 'public/images/2026-07-19/image.webp',
},
},
}));
});
});
api.listen(0, '127.0.0.1');
await once(api, 'listening');
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-generate-image-'));
const sandbox = startSandbox(root, {
ALLOWED_TOOLS: 'generate_image',
PRIVATE_DATA_USER_ID: 'user-1',
MINDSPACE_AGENT_API_BASE_URL: `http://127.0.0.1:${api.address().port}/api`,
MINDSPACE_INTERNAL_AGENT_SECRET: 'internal-secret',
});
t.after(() => sandbox.child.kill());
t.after(() => api.close());
await sandbox.request('initialize');
const called = await sandbox.request('tools/call', {
name: 'generate_image',
arguments: { purpose: 'hero', prompt: 'calm reading room', idempotency_key: 'imgreq-1' },
});
assert.equal(called.result.isError, false);
const result = JSON.parse(called.result.content[0].text);
assert.equal(result.asset.id, 'asset-1');
assert.deepEqual(requests, [{
url: '/api/agent/mindspace_image_generate',
authorization: 'Bearer internal-secret',
body: {
user_id: 'user-1',
purpose: 'hero',
prompt: 'calm reading room',
negative_prompt: '',
},
}]);
});
test('sandbox MCP rejects browser storage in generated page code', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mindspace-sandbox-storage-ban-'));
const server = startSandbox(root, { ALLOWED_TOOLS: 'write_file,edit_file' });
server.mjs
+2
View File
@@ -2084,6 +2084,7 @@ api.use(async (req, res, next) => {
if (req.path === '/agent/mindspace_page_patch') return next();
if (req.path === '/agent/mindspace_asset_delete') return next();
if (req.path === '/agent/mindspace_asset_download') return next();
if (req.path === '/agent/mindspace_image_generate') return next();
if (req.path === '/config/blocked-words') return next();
if (req.method === 'GET' && /^\/mindspace\/v1\/assets\/[^/]+\/download$/.test(req.path)) {
return next();
@@ -2127,6 +2128,7 @@ api.use(async (req, res, next) => {
attachAsrRoutes(api, { sendError, sendData });
attachMindSpaceImageGenerationRoutes(api, {
getService: () => mindSpaceImageGeneration,
requireInternal: requireInternalAgentSecret,
});
attachPageDataRoutes(api, {
sendError,
user-auth.mjs
+3
View File
@@ -1908,6 +1908,9 @@ export function createUserAuth(pool, options = {}) {
userDataMcpPgUrl: env.MINDSPACE_USERDATA_MCP_PG_URL,
userDataPgHostGateway: env.MINDSPACE_USERDATA_MCP_PG_HOST,
userDataAutoProvision: env.MINDSPACE_USERDATA_AUTO_PROVISION,
agentApiBaseUrl: env.MINDSPACE_AGENT_API_BASE_URL,
internalAgentSecret:
env.MINDSPACE_INTERNAL_AGENT_SECRET ?? env.TKMIND_SERVER__SECRET_KEY,
};
} catch (err) {
console.warn('[getAgentSessionPolicy] sandbox MCP setup failed, falling back:', err?.message);