diff --git a/chat-image-turn-scope.mjs b/chat-image-turn-scope.mjs index 1d5aff7..fd207cb 100644 --- a/chat-image-turn-scope.mjs +++ b/chat-image-turn-scope.mjs @@ -130,6 +130,25 @@ export function scrubUserMessageImageAttachments(message) { }; } +export function messageContentHasImageUrl(content) { + if (!Array.isArray(content)) return false; + return content.some((item) => item?.type === 'image_url' && item?.image_url?.url); +} + +/** + * Any persisted image_url part will break DeepSeek / other text-only providers. + * User metadata.imageUrls alone is not enough — Goose may have expanded them into + * content parts on assistant or user turns. + */ +export function conversationHasImageUrlContent(conversation, { excludeMessageId = null } = {}) { + if (!Array.isArray(conversation)) return false; + const excluded = String(excludeMessageId ?? '').trim(); + return conversation.some((message) => { + if (excluded && String(message?.id ?? '').trim() === excluded) return false; + return messageContentHasImageUrl(message?.content); + }); +} + export function scrubConversationHistoricalImageAttachments(conversation, activeMessageId) { const activeId = String(activeMessageId ?? '').trim(); if (!Array.isArray(conversation) || !activeId) { diff --git a/chat-image-turn-scope.test.mjs b/chat-image-turn-scope.test.mjs index 9d50704..8211812 100644 --- a/chat-image-turn-scope.test.mjs +++ b/chat-image-turn-scope.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { buildCurrentTurnImageScopeNote, + conversationHasImageUrlContent, dedupeImageUrlsByAssetKey, extractCurrentTurnImageUrls, scrubConversationHistoricalImageAttachments, @@ -113,3 +114,46 @@ test('buildCurrentTurnImageScopeNote states one independent topic per upload', ( assert.match(note, /不得与历史轮次混用/); assert.match(note, /asset=asset-9/); }); + +test('conversationHasImageUrlContent detects historical poison and ignores active turn', () => { + const conversation = [ + { + id: 'assistant-old', + role: 'assistant', + content: [ + { type: 'text', text: '看图' }, + { type: 'image_url', image_url: { url: 'https://example.com/old.png' } }, + ], + }, + { + id: 'user-new', + role: 'user', + content: [ + { type: 'text', text: '这是什么' }, + { type: 'image_url', image_url: { url: 'https://example.com/new.png' } }, + ], + }, + ]; + + assert.equal(conversationHasImageUrlContent(conversation), true); + assert.equal( + conversationHasImageUrlContent(conversation, { excludeMessageId: 'user-new' }), + true, + ); + assert.equal( + conversationHasImageUrlContent( + [ + { + id: 'user-new', + role: 'user', + content: [ + { type: 'text', text: '这是什么' }, + { type: 'image_url', image_url: { url: 'https://example.com/new.png' } }, + ], + }, + ], + { excludeMessageId: 'user-new' }, + ), + false, + ); +}); diff --git a/mindspace-scan.mjs b/mindspace-scan.mjs index ec069c1..2b46ae4 100644 --- a/mindspace-scan.mjs +++ b/mindspace-scan.mjs @@ -7,6 +7,7 @@ const BLOCKED_ACTIVE_PATTERNS = [ const SCRIPT_TAG_PATTERN = /]*)>([\s\S]*?)<\/script>/gi; const SCRIPT_SRC_PATTERN = /\bsrc\s*=\s*(['"])([^'"]+)\1/i; const TRUSTED_SCRIPT_SRC_PATTERNS = [ + /^\/assets\/page-data-client\.js(?:[?#].*)?$/i, /^\/assets\/chart\.umd\.min\.js(?:[?#].*)?$/i, /^https:\/\/cdn\.jsdelivr\.net\/npm\/chart\.js(?:@[^/]+)?\/dist\/chart\.(?:umd\.)?min\.js(?:[?#].*)?$/i, /^https:\/\/cdnjs\.cloudflare\.com\/ajax\/libs\/Chart\.js\/[^/]+\/chart\.(?:umd\.)?min\.js(?:[?#].*)?$/i, diff --git a/mindspace-scan.test.mjs b/mindspace-scan.test.mjs index a7f4d96..1bccbbf 100644 --- a/mindspace-scan.test.mjs +++ b/mindspace-scan.test.mjs @@ -35,6 +35,21 @@ test('runBasicFileScan warns for sandboxed html with inline script and trusted C assert.deepEqual(result.findings, ['trusted_html_active_content']); }); +test('runBasicFileScan warns for sandboxed html with page-data-client and inline script', () => { + const result = runBasicFileScan( + Buffer.from(` + + `), + { + filename: 'survey.html', + mimeType: 'text/html', + htmlActiveContentPolicy: 'sandbox_warn', + }, + ); + assert.equal(result.scanStatus, 'warned'); + assert.deepEqual(result.findings, ['trusted_html_active_content']); +}); + test('runBasicFileScan still blocks unsafe html active content in sandbox mode', () => { const javascriptUrl = runBasicFileScan(Buffer.from('go'), { filename: 'dashboard.html', diff --git a/page-data-browser-client.mjs b/page-data-browser-client.mjs index 7e7118d..ea19644 100644 --- a/page-data-browser-client.mjs +++ b/page-data-browser-client.mjs @@ -88,6 +88,9 @@ export function createPageDataBrowserClient({ async listRows(dataset, query = {}) { return request('GET', buildPageDataPublicPath(apiBase, pageId, dataset), { query }); }, + async readRows(dataset, query = {}) { + return request('GET', buildPageDataPublicPath(apiBase, pageId, dataset), { query }); + }, async getSchema(dataset) { return request('GET', buildPageDataPublicPath(apiBase, pageId, dataset, 'schema')); }, diff --git a/public/assets/page-data-client.js b/public/assets/page-data-client.js index d3ef5c6..2bc8fd4 100644 --- a/public/assets/page-data-client.js +++ b/public/assets/page-data-client.js @@ -106,6 +106,9 @@ listRows: function (dataset, query) { return request('GET', buildDataPath(apiBase, pageId, dataset), { query: query || {} }); }, + readRows: function (dataset, query) { + return request('GET', buildDataPath(apiBase, pageId, dataset), { query: query || {} }); + }, getSchema: function (dataset) { return request('GET', buildDataPath(apiBase, pageId, dataset, 'schema')); }, diff --git a/tkmind-proxy-vision.test.mjs b/tkmind-proxy-vision.test.mjs index 068d665..c4fca8e 100644 --- a/tkmind-proxy-vision.test.mjs +++ b/tkmind-proxy-vision.test.mjs @@ -140,3 +140,35 @@ test('buildVisionPayload does not mark billable usage when vision analysis fails assert.equal(result?.billableImageCount, 0); assert.doesNotMatch(result?.userMessage?.content?.[0]?.text ?? '', /Qwen VL 图片描述/); }); + +test('buildVisionPayload strips image_url content parts for text-only Goose providers', async () => { + const result = await buildVisionPayload({ + userId: 'user-1', + publishLayout: { publicUrl: 'https://m.tkmind.cn/MindSpace/user-1' }, + userMessage: { + content: [ + { type: 'text', text: '这是什么' }, + { + type: 'image_url', + image_url: { url: '/api/mindspace/v1/assets/asset-7/download?inline=1' }, + }, + ], + metadata: { + imageUrls: ['/api/mindspace/v1/assets/asset-7/download?inline=1'], + }, + }, + localFetchAsset: async () => ({ + buffer: Buffer.from('fake-image'), + mimeType: 'image/png', + }), + llmProviderService: { + analyzeImagesWithVision: async () => '蓝色方块', + }, + }); + + assert.equal( + (result?.userMessage?.content ?? []).some((item) => item?.type === 'image_url'), + false, + ); + assert.match(result?.userMessage?.content?.[0]?.text ?? '', /蓝色方块/); +}); diff --git a/tkmind-proxy.mjs b/tkmind-proxy.mjs index c8aff0c..43004c8 100644 --- a/tkmind-proxy.mjs +++ b/tkmind-proxy.mjs @@ -34,6 +34,7 @@ import { import { extractAttachmentText } from './mindspace-attachment-text.mjs'; import { buildCurrentTurnImageScopeNote, + conversationHasImageUrlContent, extractCurrentTurnImageUrls, scrubConversationHistoricalImageAttachments, } from './chat-image-turn-scope.mjs'; @@ -974,6 +975,9 @@ export async function buildVisionPayload({ '不要向用户展示 HTML 代码块。'; let updatedContent = Array.isArray(userMessage?.content) ? [...userMessage.content] : []; + // Text-only Goose providers cannot accept image_url parts. After VL analysis, + // keep only text (with the injected vision note) for the agent turn. + updatedContent = updatedContent.filter((item) => item?.type !== 'image_url'); for (const item of imageItems) { updatedContent = updatedContent.map((c) => { if (c?.type !== 'text' || typeof c.text !== 'string') return c; @@ -1698,27 +1702,59 @@ export function createTkmindProxy({ return { changed: false, updated: false, status: upstream.status }; } const session = await upstream.json().catch(() => null); - const { conversation, changed } = scrubConversationHistoricalImageAttachments( - session?.conversation ?? [], + const conversation = Array.isArray(session?.conversation) ? session.conversation : []; + const hasImageUrlContent = conversationHasImageUrlContent(conversation, { + excludeMessageId: activeId, + }); + const { conversation: scrubbedConversation, changed } = scrubConversationHistoricalImageAttachments( + conversation, activeId, ); - if (!changed) return { changed: false, updated: false }; + // Text-only providers (DeepSeek) reject any lingering image_url parts. If Goose + // cannot persist a scrub (405/404), callers must rotate to a fresh session. + if (!changed && !hasImageUrlContent) { + return { changed: false, updated: false, hasImageUrlContent: false }; + } + if (!changed && hasImageUrlContent) { + return { + changed: true, + updated: false, + status: 'image_url_content_present', + hasImageUrlContent: true, + }; + } const update = await apiFetch( target, apiSecret, `/sessions/${encodeURIComponent(sessionId)}`, { method: 'PUT', - body: JSON.stringify({ conversation }), + body: JSON.stringify({ conversation: scrubbedConversation }), }, ); if (!update.ok) { console.warn( `Historical image scrub skipped for session ${sessionId}: upstream ${update.status}`, ); - return { changed: true, updated: false, status: update.status }; + return { + changed: true, + updated: false, + status: update.status, + hasImageUrlContent: + hasImageUrlContent + || conversationHasImageUrlContent(scrubbedConversation, { + excludeMessageId: activeId, + }), + }; } - return { changed: true, updated: true, status: update.status }; + return { + changed: true, + updated: true, + status: update.status, + hasImageUrlContent: conversationHasImageUrlContent(scrubbedConversation, { + excludeMessageId: activeId, + }), + }; } catch (err) { console.warn( 'Historical image scrub skipped:', @@ -1938,7 +1974,17 @@ export function createTkmindProxy({ if (!user) throw new Error('用户不存在'); if (requireHistoricalImageIsolation || messageHasImages(userMessage)) { const imageIsolation = await syncHistoricalImageTurnIsolation(sessionId, userMessage?.id); - if (requireHistoricalImageIsolation && imageIsolation.changed && !imageIsolation.updated) { + const scrubUnsupported = + imageIsolation.changed + && !imageIsolation.updated + && ( + requireHistoricalImageIsolation + || imageIsolation.hasImageUrlContent + || Number(imageIsolation.status) === 404 + || Number(imageIsolation.status) === 405 + || imageIsolation.status === 'image_url_content_present' + ); + if (scrubUnsupported) { const error = new Error( `historical_image_session_update_unsupported:${imageIsolation.status ?? 'unknown'}`, ); diff --git a/tkmind-proxy.test.mjs b/tkmind-proxy.test.mjs index 1458e64..04b12fa 100644 --- a/tkmind-proxy.test.mjs +++ b/tkmind-proxy.test.mjs @@ -1534,6 +1534,58 @@ test('submitSessionReplyForUser fails closed when historical image scrub is unsu }); }); +test('submitSessionReplyForUser rotates when assistant history still has image_url', async () => { + await withFakeGoosedSession(async ({ apiTarget, workingDir, replyBodies }) => { + const proxy = createTkmindProxy({ + apiTarget, + apiSecret: 'test-secret', + userAuth: { + ...createMemoryTestUserAuth(workingDir), + async ownsSession() { + return true; + }, + async canUseChat() { + return { ok: true }; + }, + async getUserById() { + return { id: 'user-1' }; + }, + async resolveUserPolicies() { + return { unrestricted: true, policies: {} }; + }, + }, + }); + + await assert.rejects( + proxy.submitSessionReplyForUser( + 'user-1', + 'session-1', + 'request-after-assistant-image', + { + id: 'message-current', + role: 'user', + content: [{ type: 'text', text: '这张图是什么' }], + metadata: { imageUrls: ['https://example.com/new.png'] }, + }, + { requireHistoricalImageIsolation: true }, + ), + /historical_image_session_update_unsupported:image_url_content_present/, + ); + assert.equal(replyBodies.length, 0); + }, { + conversation: [ + { + id: 'assistant-old-image', + role: 'assistant', + content: [ + { type: 'text', text: '我看到了图片' }, + { type: 'image_url', image_url: { url: 'https://example.com/old.png' } }, + ], + }, + ], + }); +}); + test('visual fallback session removes read_image while preserving the text task path', async () => { await withFakeGoosedSession(async ({ apiTarget,