feat: chat uploads, vision turn isolation, and MindSpace agent improvements
Add chat file/image upload UX, attachment proxying, vision thumbnails, and per-turn image scoping so agents only use the current upload. Extend MindSpace asset context, billing token state, OA/scenario verify scripts, and related runtime config. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+62
-6
@@ -1,4 +1,4 @@
|
||||
import type { Message, MessageContent } from '../types';
|
||||
import type { ChatFileAttachment, Message, MessageContent } from '../types';
|
||||
import { mergeMessageContent } from '../../message-stream.mjs';
|
||||
import { mergeConversationSnapshot as mergeConversationSnapshotCore } from '../../chat-finish-sync.mjs';
|
||||
import { deriveUserFacingText, deriveAssistantFacingText } from '../../conversation-display.mjs';
|
||||
@@ -6,8 +6,11 @@ import { stripUserAddressPrefix } from './userAddress';
|
||||
|
||||
const IMAGE_URL_LINE_RE = /^\[图片\d+]: (.+)$/;
|
||||
const IMAGE_URL_LINES_RE = /\n*\[图片\d+]: [^\n]+/g;
|
||||
const FILE_ATTACHMENT_LINE_RE = /^\[文件\d+: ([^\]]+)\]: (.+)$/;
|
||||
const FILE_ATTACHMENT_LINES_RE = /\n*\[文件\d+: [^\]]+\]: [^\n]+/g;
|
||||
const USER_IDENTITY_BLOCK_RE = /^\[用户身份\][\s\S]*?(?:\n{2,}|$)/;
|
||||
const TKMIND_VISION_NOTE_RE = /\n*【TKMind 图片分析结果[\s\S]*$/;
|
||||
const TKMIND_ATTACHMENT_NOTE_RE = /\n*【TKMind 附件分析结果[\s\S]*$/;
|
||||
|
||||
function normalizeImageUrl(url: string): string {
|
||||
const value = String(url ?? '').trim();
|
||||
@@ -41,10 +44,19 @@ function stripImageUrlLines(text: string): string {
|
||||
return String(text ?? '')
|
||||
.replace(USER_IDENTITY_BLOCK_RE, '')
|
||||
.replace(TKMIND_VISION_NOTE_RE, '')
|
||||
.replace(TKMIND_ATTACHMENT_NOTE_RE, '')
|
||||
.replace(IMAGE_URL_LINES_RE, '')
|
||||
.replace(FILE_ATTACHMENT_LINES_RE, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function formatFileAttachmentsForAgentText(attachments: ChatFileAttachment[]): string {
|
||||
return attachments
|
||||
.filter((item) => item?.downloadUrl?.trim())
|
||||
.map((item, index) => `[文件${index + 1}: ${item.filename}]: ${item.downloadUrl.trim()}`)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function parseImageUrlsFromText(text: string): string[] {
|
||||
const urls: string[] = [];
|
||||
for (const line of text.split('\n')) {
|
||||
@@ -82,6 +94,7 @@ export function normalizeUserMessageForApi(message: Message): Message {
|
||||
if (message.role !== 'user') return message;
|
||||
|
||||
const imageUrls = getImageUrls(message);
|
||||
const fileAttachments = getFileAttachments(message);
|
||||
const displayText =
|
||||
message.metadata.displayText ??
|
||||
stripImageUrlLines(
|
||||
@@ -97,12 +110,17 @@ export function normalizeUserMessageForApi(message: Message): Message {
|
||||
.join('\n')
|
||||
.trim();
|
||||
|
||||
if (imageUrls.length === 0 && message.content.every((item) => item.type === 'text')) {
|
||||
if (
|
||||
imageUrls.length === 0 &&
|
||||
fileAttachments.length === 0 &&
|
||||
message.content.every((item) => item.type === 'text')
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
|
||||
const imageText = formatImageUrlsForAgentText(imageUrls);
|
||||
const agentText = [stripImageUrlLines(textOnly), imageText]
|
||||
const fileText = formatFileAttachmentsForAgentText(fileAttachments);
|
||||
const agentText = [stripImageUrlLines(textOnly), imageText, fileText]
|
||||
.filter(Boolean)
|
||||
.join('\n\n');
|
||||
|
||||
@@ -113,6 +131,7 @@ export function normalizeUserMessageForApi(message: Message): Message {
|
||||
...message.metadata,
|
||||
displayText,
|
||||
...(imageUrls.length ? { imageUrls } : {}),
|
||||
...(fileAttachments.length ? { fileAttachments } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -125,6 +144,7 @@ export function buildUserMessage(
|
||||
displayText?: string;
|
||||
imageUrls?: string[];
|
||||
previewImageUrls?: string[];
|
||||
fileAttachments?: ChatFileAttachment[];
|
||||
},
|
||||
): Message {
|
||||
const displayText = stripImageUrlLines(options?.displayText ?? text);
|
||||
@@ -132,9 +152,13 @@ export function buildUserMessage(
|
||||
const previewImageUrls = (options?.previewImageUrls ?? []).filter(
|
||||
(value) => typeof value === 'string' && value.trim(),
|
||||
);
|
||||
const fileAttachments = (options?.fileAttachments ?? []).filter(
|
||||
(item) => item?.downloadUrl?.trim() && item?.filename?.trim(),
|
||||
);
|
||||
const baseAgentText = options?.agentText ?? text;
|
||||
const imageText = formatImageUrlsForAgentText(imageUrls);
|
||||
const agentText = [baseAgentText.trim(), imageText].filter(Boolean).join('\n\n');
|
||||
const fileText = formatFileAttachmentsForAgentText(fileAttachments);
|
||||
const agentText = [baseAgentText.trim(), imageText, fileText].filter(Boolean).join('\n\n');
|
||||
|
||||
return {
|
||||
id: options?.id ?? crypto.randomUUID(),
|
||||
@@ -147,6 +171,7 @@ export function buildUserMessage(
|
||||
displayText,
|
||||
...(imageUrls.length ? { imageUrls } : {}),
|
||||
...(previewImageUrls.length ? { previewImageUrls } : {}),
|
||||
...(fileAttachments.length ? { fileAttachments } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -199,6 +224,7 @@ export function getDisplayText(message: Message): string {
|
||||
export function shouldShowChatMessage(message: Message): boolean {
|
||||
if (getDisplayText(message).trim()) return true;
|
||||
if (message.role === 'user' && getImageUrls(message).length > 0) return true;
|
||||
if (message.role === 'user' && getFileAttachments(message).length > 0) return true;
|
||||
if (getThinking(message)) return true;
|
||||
if (getSystemNotificationText(message)) return true;
|
||||
if (message.role !== 'assistant') return false;
|
||||
@@ -230,7 +256,10 @@ export function getVisibleText(message: Message): string {
|
||||
}
|
||||
|
||||
export function getImageUrls(message: Message): string[] {
|
||||
const fromMetadata = message.metadata.imageUrls?.filter(
|
||||
const fromMetadata = (
|
||||
message.metadata.imageUrls
|
||||
?? message.metadata.archivedImageUrls
|
||||
)?.filter(
|
||||
(url) => typeof url === 'string' && url.trim(),
|
||||
);
|
||||
if (fromMetadata?.length) return fromMetadata.map(normalizeImageUrl);
|
||||
@@ -245,9 +274,36 @@ export function getImageUrls(message: Message): string[] {
|
||||
return parseImageUrlsFromText(text).map(normalizeImageUrl);
|
||||
}
|
||||
|
||||
export function getFileAttachments(message: Message): ChatFileAttachment[] {
|
||||
const fromMetadata = message.metadata.fileAttachments?.filter(
|
||||
(item) => item?.downloadUrl?.trim() && item?.filename?.trim(),
|
||||
);
|
||||
if (fromMetadata?.length) return fromMetadata;
|
||||
|
||||
const text = message.content
|
||||
.filter((item): item is Extract<MessageContent, { type: 'text' }> => item.type === 'text')
|
||||
.map((item) => item.text)
|
||||
.join('\n');
|
||||
const parsed: ChatFileAttachment[] = [];
|
||||
for (const line of text.split('\n')) {
|
||||
const match = line.trim().match(FILE_ATTACHMENT_LINE_RE);
|
||||
if (!match?.[1] || !match?.[2]) continue;
|
||||
parsed.push({
|
||||
assetId: '',
|
||||
filename: match[1],
|
||||
downloadUrl: match[2],
|
||||
mimeType: 'application/octet-stream',
|
||||
});
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function getRenderableImageUrls(message: Message): Array<{ src: string; href: string }> {
|
||||
const imageUrls = getImageUrls(message);
|
||||
const previewImageUrls = message.metadata.previewImageUrls?.filter(
|
||||
const previewImageUrls = (
|
||||
message.metadata.previewImageUrls
|
||||
?? message.metadata.archivedPreviewImageUrls
|
||||
)?.filter(
|
||||
(url) => typeof url === 'string' && url.trim(),
|
||||
) ?? [];
|
||||
return imageUrls.map((href, index) => ({
|
||||
|
||||
Reference in New Issue
Block a user