feat: chat uploads, vision turn isolation, and MindSpace agent improvements

Add chat file/image upload UX, attachment proxying, vision thumbnails, and per-turn image scoping so agents only use the current upload. Extend MindSpace asset context, billing token state, OA/scenario verify scripts, and related runtime config.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-07-11 00:23:01 +08:00
parent e3063ea806
commit 32fb2cdeaf
51 changed files with 4588 additions and 186 deletions
+62 -6
View File
@@ -1,4 +1,4 @@
import type { Message, MessageContent } from '../types';
import type { ChatFileAttachment, Message, MessageContent } from '../types';
import { mergeMessageContent } from '../../message-stream.mjs';
import { mergeConversationSnapshot as mergeConversationSnapshotCore } from '../../chat-finish-sync.mjs';
import { deriveUserFacingText, deriveAssistantFacingText } from '../../conversation-display.mjs';
@@ -6,8 +6,11 @@ import { stripUserAddressPrefix } from './userAddress';
const IMAGE_URL_LINE_RE = /^\[图片\d+]: (.+)$/;
const IMAGE_URL_LINES_RE = /\n*\[图片\d+]: [^\n]+/g;
const FILE_ATTACHMENT_LINE_RE = /^\[文件\d+: ([^\]]+)\]: (.+)$/;
const FILE_ATTACHMENT_LINES_RE = /\n*\[文件\d+: [^\]]+\]: [^\n]+/g;
const USER_IDENTITY_BLOCK_RE = /^\[用户身份\][\s\S]*?(?:\n{2,}|$)/;
const TKMIND_VISION_NOTE_RE = /\n*【TKMind 图片分析结果[\s\S]*$/;
const TKMIND_ATTACHMENT_NOTE_RE = /\n*【TKMind 附件分析结果[\s\S]*$/;
function normalizeImageUrl(url: string): string {
const value = String(url ?? '').trim();
@@ -41,10 +44,19 @@ function stripImageUrlLines(text: string): string {
return String(text ?? '')
.replace(USER_IDENTITY_BLOCK_RE, '')
.replace(TKMIND_VISION_NOTE_RE, '')
.replace(TKMIND_ATTACHMENT_NOTE_RE, '')
.replace(IMAGE_URL_LINES_RE, '')
.replace(FILE_ATTACHMENT_LINES_RE, '')
.trim();
}
function formatFileAttachmentsForAgentText(attachments: ChatFileAttachment[]): string {
return attachments
.filter((item) => item?.downloadUrl?.trim())
.map((item, index) => `[文件${index + 1}: ${item.filename}]: ${item.downloadUrl.trim()}`)
.join('\n');
}
function parseImageUrlsFromText(text: string): string[] {
const urls: string[] = [];
for (const line of text.split('\n')) {
@@ -82,6 +94,7 @@ export function normalizeUserMessageForApi(message: Message): Message {
if (message.role !== 'user') return message;
const imageUrls = getImageUrls(message);
const fileAttachments = getFileAttachments(message);
const displayText =
message.metadata.displayText ??
stripImageUrlLines(
@@ -97,12 +110,17 @@ export function normalizeUserMessageForApi(message: Message): Message {
.join('\n')
.trim();
if (imageUrls.length === 0 && message.content.every((item) => item.type === 'text')) {
if (
imageUrls.length === 0 &&
fileAttachments.length === 0 &&
message.content.every((item) => item.type === 'text')
) {
return message;
}
const imageText = formatImageUrlsForAgentText(imageUrls);
const agentText = [stripImageUrlLines(textOnly), imageText]
const fileText = formatFileAttachmentsForAgentText(fileAttachments);
const agentText = [stripImageUrlLines(textOnly), imageText, fileText]
.filter(Boolean)
.join('\n\n');
@@ -113,6 +131,7 @@ export function normalizeUserMessageForApi(message: Message): Message {
...message.metadata,
displayText,
...(imageUrls.length ? { imageUrls } : {}),
...(fileAttachments.length ? { fileAttachments } : {}),
},
};
}
@@ -125,6 +144,7 @@ export function buildUserMessage(
displayText?: string;
imageUrls?: string[];
previewImageUrls?: string[];
fileAttachments?: ChatFileAttachment[];
},
): Message {
const displayText = stripImageUrlLines(options?.displayText ?? text);
@@ -132,9 +152,13 @@ export function buildUserMessage(
const previewImageUrls = (options?.previewImageUrls ?? []).filter(
(value) => typeof value === 'string' && value.trim(),
);
const fileAttachments = (options?.fileAttachments ?? []).filter(
(item) => item?.downloadUrl?.trim() && item?.filename?.trim(),
);
const baseAgentText = options?.agentText ?? text;
const imageText = formatImageUrlsForAgentText(imageUrls);
const agentText = [baseAgentText.trim(), imageText].filter(Boolean).join('\n\n');
const fileText = formatFileAttachmentsForAgentText(fileAttachments);
const agentText = [baseAgentText.trim(), imageText, fileText].filter(Boolean).join('\n\n');
return {
id: options?.id ?? crypto.randomUUID(),
@@ -147,6 +171,7 @@ export function buildUserMessage(
displayText,
...(imageUrls.length ? { imageUrls } : {}),
...(previewImageUrls.length ? { previewImageUrls } : {}),
...(fileAttachments.length ? { fileAttachments } : {}),
},
};
}
@@ -199,6 +224,7 @@ export function getDisplayText(message: Message): string {
export function shouldShowChatMessage(message: Message): boolean {
if (getDisplayText(message).trim()) return true;
if (message.role === 'user' && getImageUrls(message).length > 0) return true;
if (message.role === 'user' && getFileAttachments(message).length > 0) return true;
if (getThinking(message)) return true;
if (getSystemNotificationText(message)) return true;
if (message.role !== 'assistant') return false;
@@ -230,7 +256,10 @@ export function getVisibleText(message: Message): string {
}
export function getImageUrls(message: Message): string[] {
const fromMetadata = message.metadata.imageUrls?.filter(
const fromMetadata = (
message.metadata.imageUrls
?? message.metadata.archivedImageUrls
)?.filter(
(url) => typeof url === 'string' && url.trim(),
);
if (fromMetadata?.length) return fromMetadata.map(normalizeImageUrl);
@@ -245,9 +274,36 @@ export function getImageUrls(message: Message): string[] {
return parseImageUrlsFromText(text).map(normalizeImageUrl);
}
export function getFileAttachments(message: Message): ChatFileAttachment[] {
const fromMetadata = message.metadata.fileAttachments?.filter(
(item) => item?.downloadUrl?.trim() && item?.filename?.trim(),
);
if (fromMetadata?.length) return fromMetadata;
const text = message.content
.filter((item): item is Extract<MessageContent, { type: 'text' }> => item.type === 'text')
.map((item) => item.text)
.join('\n');
const parsed: ChatFileAttachment[] = [];
for (const line of text.split('\n')) {
const match = line.trim().match(FILE_ATTACHMENT_LINE_RE);
if (!match?.[1] || !match?.[2]) continue;
parsed.push({
assetId: '',
filename: match[1],
downloadUrl: match[2],
mimeType: 'application/octet-stream',
});
}
return parsed;
}
export function getRenderableImageUrls(message: Message): Array<{ src: string; href: string }> {
const imageUrls = getImageUrls(message);
const previewImageUrls = message.metadata.previewImageUrls?.filter(
const previewImageUrls = (
message.metadata.previewImageUrls
?? message.metadata.archivedPreviewImageUrls
)?.filter(
(url) => typeof url === 'string' && url.trim(),
) ?? [];
return imageUrls.map((href, index) => ({