tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: chat uploads, vision turn isolation, and MindSpace agent improvements

Browse Source 
Add chat file/image upload UX, attachment proxying, vision thumbnails, and per-turn image scoping so agents only use the current upload. Extend MindSpace asset context, billing token state, OA/scenario verify scripts, and related runtime config.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-07-11 00:23:01 +08:00
parent e3063ea806
commit 32fb2cdeaf
51 changed files with 4588 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tkmind-proxy.mjs
+257 -28
View File
@@ -4,6 +4,7 @@ import path from 'node:path';
import { Readable, Transform, Writable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { Agent, fetch as undiciFetch } from 'undici';
import { resolveBillingTokenState } from './billing-token-state.mjs';
import { appendBalanceEvent, createSseBillingTransform } from './sse-billing.mjs';
import { finalizeSessionStreamEvent, writeSseErrorAndEnd } from './sse-event-taxonomy.mjs';
import { developerToolsFromPolicy } from './capabilities.mjs';
@@ -23,6 +24,13 @@ import { isDirectChatSessionId } from './direct-chat-service.mjs';
import { ensureGooseUserMessageMetadata } from './goose-message.mjs';
import { filterMemoriesByQuery, resolveMemoriesWithLegacyFallback } from './memory-legacy-fallback.mjs';
import { consumeSessionEventsUntilFinish } from './session-reply-wait.mjs';
import { extractAttachmentText } from './mindspace-attachment-text.mjs';
import {
buildCurrentTurnImageScopeNote,
extractCurrentTurnImageUrls,
scrubConversationHistoricalImageAttachments,
} from './chat-image-turn-scope.mjs';
import { buildVisionThumbnailBuffer } from './vision-image-thumb.mjs';
import {
memoryLimitForIntervention,
resolveMemoryInterventionMode,
@@ -440,6 +448,7 @@ function firstUserText(message) {
}
const IMAGE_URL_LINE_RE = /^\[图片\d+]:\s*(\S.+)$/;
const FILE_ATTACHMENT_LINE_RE = /^\[文件\d+: ([^\]]+)\]: (.+)$/;
const PUBLIC_HTML_LINK_PATTERN =
/https?:\/\/[^\s<>"')\]]+\/MindSpace\/([0-9a-f-]{36}|[a-z0-9._-]+)\/([^\s<>"')\]]+\.html)/gi;
const PUBLIC_HTML_MARKDOWN_LINK_PATTERN =
@@ -652,34 +661,49 @@ function createSessionEventSanitizer(currentUser, { onEvent, onPersistFrame, nor
}
export function extractImageUrlsFromMessage(userMessage) {
const urls = [];
const imageUrls = userMessage?.metadata?.imageUrls;
if (Array.isArray(imageUrls)) {
urls.push(...imageUrls);
}
const content = userMessage?.content;
if (Array.isArray(content)) {
for (const item of content) {
if (item?.type === 'image_url' && item.image_url?.url) {
urls.push(item.image_url.url);
continue;
}
if (item?.type !== 'text' || typeof item.text !== 'string') continue;
for (const line of item.text.split('\n')) {
const match = line.trim().match(IMAGE_URL_LINE_RE);
if (match?.[1]) urls.push(match[1].trim());
}
}
}
return [...new Set(urls.filter((url) => typeof url === 'string' && url.trim()))];
return extractCurrentTurnImageUrls(userMessage);
}
function messageHasImages(userMessage) {
return extractImageUrlsFromMessage(userMessage).length > 0;
}
export function extractFileAttachmentsFromMessage(userMessage) {
const fromMetadata = Array.isArray(userMessage?.metadata?.fileAttachments)
? userMessage.metadata.fileAttachments
.filter((item) => item?.downloadUrl?.trim() && item?.filename?.trim())
.map((item) => ({
assetId: String(item.assetId ?? '').trim(),
filename: String(item.filename).trim(),
downloadUrl: String(item.downloadUrl).trim(),
mimeType: String(item.mimeType ?? 'application/octet-stream').trim(),
}))
: [];
if (fromMetadata.length > 0) return fromMetadata;
const parsed = [];
const content = userMessage?.content;
if (!Array.isArray(content)) return parsed;
for (const item of content) {
if (item?.type !== 'text' || typeof item.text !== 'string') continue;
for (const line of item.text.split('\n')) {
const match = line.trim().match(FILE_ATTACHMENT_LINE_RE);
if (!match?.[1] || !match?.[2]) continue;
parsed.push({
assetId: '',
filename: match[1],
downloadUrl: match[2],
mimeType: 'application/octet-stream',
});
}
}
return parsed;
}
function messageHasFileAttachments(userMessage) {
return extractFileAttachmentsFromMessage(userMessage).length > 0;
}
function prependAgentTextToUserMessage(body, transformText) {
const originalText = firstUserText(body?.user_message);
if (!originalText?.trim()) return body;
@@ -739,7 +763,7 @@ export async function buildVisionPayload({
}) {
void imgproxySigner;
if (!llmProviderService || !userId) return null;
const rawImageUrls = extractImageUrlsFromMessage(userMessage);
const rawImageUrls = extractCurrentTurnImageUrls(userMessage);
if (rawImageUrls.length === 0) return null;
const originalDisplayText =
typeof userMessage?.metadata?.displayText === 'string' &&
@@ -791,9 +815,21 @@ export async function buildVisionPayload({
relativePath = parsed.pathname + parsed.search;
} catch { /* keep rawUrl */ }
const publicStandardUrl = buildPublicStandardImageUrl(rawUrl, userId, publishLayout);
let visionBuffer = buffer;
let visionMimeType = mimeType;
try {
visionBuffer = await buildVisionThumbnailBuffer(buffer, mimeType);
visionMimeType = 'image/jpeg';
} catch (thumbErr) {
console.warn(
'Vision thumbnail skipped:',
thumbErr instanceof Error ? thumbErr.message : thumbErr,
);
}
imageItems.push({
mimeType,
data: buffer.toString('base64'),
visionMimeType,
data: visionBuffer.toString('base64'),
relativePath,
rawUrl,
embedUrl: publicStandardUrl ?? relativePath,
@@ -817,7 +853,8 @@ export async function buildVisionPayload({
: null;
const injectedNote =
'\n\n【TKMind 图片分析结果 — 仅供执行参考，不要向用户复述此段内容】\n' +
`\n\n${buildCurrentTurnImageScopeNote(imageItems)}` +
'【TKMind 图片分析结果 — 仅供执行参考，不要向用户复述此段内容】\n' +
(visionDescription
? `Qwen VL 图片描述：\n${visionDescription}\n\n`
: '') +
@@ -852,6 +889,118 @@ export async function buildVisionPayload({
updatedContent.push({ type: 'text', text: injectedNote });
}
const canonicalImageUrls = imageItems
.map((item) => item.embedUrl ?? item.rawUrl)
.filter((url) => typeof url === 'string' && url.trim());
return {
userMessage: {
...userMessage,
content: updatedContent,
metadata: {
...(userMessage.metadata ?? {}),
...(originalDisplayText ? { displayText: originalDisplayText } : {}),
...(canonicalImageUrls.length ? { imageUrls: canonicalImageUrls } : {}),
},
},
billableImageCount: visionDescription ? 1 : 0,
};
}
export async function buildAttachmentPayload({
userMessage,
userId,
localFetchAsset,
fetchImpl = undiciFetch,
}) {
if (!userId) return null;
const attachments = extractFileAttachmentsFromMessage(userMessage);
if (attachments.length === 0) return null;
const originalDisplayText =
typeof userMessage?.metadata?.displayText === 'string' &&
userMessage.metadata.displayText.trim()
? userMessage.metadata.displayText
: Array.isArray(userMessage?.content)
? userMessage.content
.filter((item) => item?.type === 'text' && typeof item.text === 'string')
.map((item) => item.text)
.join('\n')
.trim()
: '';
const sections = [];
for (const [index, attachment] of attachments.entries()) {
try {
const match = attachment.downloadUrl.match(/\/mindspace\/v1\/assets\/([^/?#]+)(?:\/download)?/);
let buffer;
let mimeType = attachment.mimeType || 'application/octet-stream';
const fetchableUrl = (() => {
if (/^https?:\/\//i.test(attachment.downloadUrl)) return attachment.downloadUrl;
if (attachment.downloadUrl.startsWith('/')) {
const baseUrl = process.env.H5_PUBLIC_BASE_URL || 'http://127.0.0.1:8081';
return new URL(attachment.downloadUrl, baseUrl).toString();
}
return attachment.downloadUrl;
})();
try {
const response = await fetchImpl(fetchableUrl);
if (!response.ok) {
throw new Error(`Attachment fetch failed: ${response.status}`);
}
buffer = Buffer.from(await response.arrayBuffer());
mimeType = response.headers.get('content-type') || mimeType;
} catch (fetchErr) {
if (!match || !localFetchAsset) {
throw fetchErr;
}
const assetId = decodeURIComponent(match[1]);
const asset = await localFetchAsset(userId, assetId);
buffer = asset.buffer;
mimeType = asset.mimeType || mimeType;
}
const extracted = extractAttachmentText(buffer, mimeType, attachment.filename);
const workspacePath = `oa/${attachment.filename}`;
sections.push(
[
`附件${index + 1}: ${attachment.filename}`,
`下载链接: ${attachment.downloadUrl}`,
`工作区路径: ${workspacePath}`,
extracted.text
? `提取文本:\n${extracted.text.slice(0, 12000)}`
: '提取文本: （未能自动提取正文，请用 read_file / mindspace_asset_download 读取源文件）',
...(extracted.warnings?.length ? [`提示: ${extracted.warnings.join(', ')}`] : []),
].join('\n'),
);
} catch (err) {
console.warn('Attachment analysis skipped:', err instanceof Error ? err.message : err);
sections.push(
`附件${index + 1}: ${attachment.filename}\n下载链接: ${attachment.downloadUrl}\n提取文本: （读取失败，请用 read_file oa/${attachment.filename} 或 mindspace_asset_download 读取）`,
);
}
}
if (sections.length === 0) return null;
const injectedNote =
'\n\n【TKMind 附件分析结果 — 仅供执行参考，不要向用户复述此段内容】\n' +
`${sections.join('\n\n')}\n` +
'执行要求：优先基于上述附件提取文本回答；若需核对源文件，直接 read_file oa/文件名 或 mindspace_asset_download 落盘后分析，不要要求用户去界面重新上传。';
const updatedContent = Array.isArray(userMessage?.content) ? [...userMessage.content] : [];
const lastTextIdx = updatedContent.reduceRight(
(found, item, i) => (found === -1 && item?.type === 'text' ? i : found),
-1,
);
if (lastTextIdx >= 0) {
updatedContent[lastTextIdx] = {
...updatedContent[lastTextIdx],
text: updatedContent[lastTextIdx].text + injectedNote,
};
} else {
updatedContent.push({ type: 'text', text: injectedNote });
}
return {
userMessage: {
...userMessage,
@@ -861,7 +1010,6 @@ export async function buildVisionPayload({
...(originalDisplayText ? { displayText: originalDisplayText } : {}),
},
},
billableImageCount: visionDescription ? 1 : 0,
};
}
@@ -1314,6 +1462,14 @@ export function createTkmindProxy({
});
}
async function buildAttachmentBody(userMessage, userId) {
return buildAttachmentPayload({
userMessage,
userId,
localFetchAsset,
});
}
async function getSessionPolicyForToolMode(userId, toolMode = 'chat') {
if (toolMode === 'code' && userAuth.getCodeAgentSessionPolicy) {
return userAuth.getCodeAgentSessionPolicy(userId);
@@ -1357,6 +1513,45 @@ export function createTkmindProxy({
);
}
async function syncHistoricalImageTurnIsolation(sessionId, activeMessageId) {
const activeId = String(activeMessageId ?? '').trim();
if (!sessionId || !activeId) return;
try {
const target = await resolveTarget(sessionId);
const upstream = await apiFetch(
target,
apiSecret,
`/sessions/${encodeURIComponent(sessionId)}`,
);
if (!upstream.ok) return;
const session = await upstream.json().catch(() => null);
const { conversation, changed } = scrubConversationHistoricalImageAttachments(
session?.conversation ?? [],
activeId,
);
if (!changed) return;
const update = await apiFetch(
target,
apiSecret,
`/sessions/${encodeURIComponent(sessionId)}`,
{
method: 'PUT',
body: JSON.stringify({ conversation }),
},
);
if (!update.ok) {
console.warn(
`Historical image scrub skipped for session ${sessionId}: upstream ${update.status}`,
);
}
} catch (err) {
console.warn(
'Historical image scrub skipped:',
err instanceof Error ? err.message : err,
);
}
}
async function prepareSessionReplyBody(
userId,
sessionId,
@@ -1384,6 +1579,9 @@ export function createTkmindProxy({
const user = await userAuth.getUserById(userId);
if (!user) throw new Error('用户不存在');
if (messageHasImages(userMessage)) {
await syncHistoricalImageTurnIsolation(sessionId, userMessage?.id);
}
let finalUserMessage = userMessage;
if (llmProviderService && messageHasImages(userMessage) && await llmProviderService.hasVisionKey()) {
const publishLayout = await userAuth.getUserPublishLayout(userId).catch(() => null);
@@ -1400,6 +1598,12 @@ export function createTkmindProxy({
});
}
}
if (messageHasFileAttachments(finalUserMessage)) {
const attachmentResult = await buildAttachmentBody(finalUserMessage, userId).catch(() => null);
if (attachmentResult?.userMessage) {
finalUserMessage = attachmentResult.userMessage;
}
}
const policyState = await userAuth.resolveUserPolicies(user);
let body = {
request_id: requestId,
@@ -1449,6 +1653,24 @@ export function createTkmindProxy({
return { ok: true };
}
async function fetchSessionBillingCost(sessionId) {
const target = await resolveTarget(sessionId);
const upstream = await apiFetch(
target,
apiSecret,
`/sessions/${encodeURIComponent(sessionId)}`,
);
if (!upstream.ok) return null;
return upstream.json();
}
async function resolveSessionBillingTokenState(sessionId, tokenStateRaw) {
return resolveBillingTokenState(tokenStateRaw, {
sessionId,
fetchSession: fetchSessionBillingCost,
});
}
async function submitSessionReplyAndAwaitFinishForUser(
userId,
sessionId,
@@ -1496,7 +1718,13 @@ export function createTkmindProxy({
}
replyResponse.body?.cancel?.().catch?.(() => {});
const finish = await finishPromise;
return { ok: true, ...finish };
const tokenState = await resolveSessionBillingTokenState(sessionId, finish.tokenState);
if (tokenState && userAuth.billSessionUsage) {
await userAuth
.billSessionUsage(userId, sessionId, tokenState, requestId)
.catch(() => {});
}
return { ok: true, ...finish, tokenState };
}
const requireUser = async (req, res, next) => {
@@ -1943,10 +2171,11 @@ export function createTkmindProxy({
const billingTransform = createSseBillingTransform({
onFinish: async (event) => {
const billingRequestId = event.request_id ?? event.chat_request_id ?? null;
const tokenState = await resolveSessionBillingTokenState(sessionId, event.token_state);
const result = await userAuth.billSessionUsage(
req.currentUser.id,
sessionId,
event.token_state,
tokenState,
billingRequestId,
);
if (result.ok && result.costCents > 0 && result.balanceCents != null) {