diff --git a/crates/goose-provider-types/src/conversation/message.rs b/crates/goose-provider-types/src/conversation/message.rs index 996ae6e63..9ad830336 100644 --- a/crates/goose-provider-types/src/conversation/message.rs +++ b/crates/goose-provider-types/src/conversation/message.rs @@ -706,7 +706,10 @@ impl From for Message { // Convert and add the content let content = match prompt_message.content { - ContentBlock::Text(text) => MessageContentBlock::Text(text), + ContentBlock::Text(mut text) => { + text.text = sanitize_unicode_tags(&text.text); + MessageContentBlock::Text(text) + } ContentBlock::Image(image) => MessageContentBlock::Image(image), ContentBlock::ResourceLink(_) => MessageContentBlock::text("[Resource link]"), ContentBlock::Resource(resource) => { @@ -1901,6 +1904,27 @@ mod tests { } } + #[test] + fn test_from_prompt_message_text_preserves_visible_unicode() { + let prompt_message = PromptMessage::new(Role::User, ContentBlock::text("Grüße 你好 🪿")); + + let message = Message::from(prompt_message); + + assert_eq!(message.as_concat_text(), "Grüße 你好 🪿"); + } + + #[test] + fn test_from_prompt_message_text_removes_unicode_tags() { + let prompt_message = PromptMessage::new( + Role::User, + ContentBlock::text("visible\u{E0000}\u{E0041}\u{E007F} text"), + ); + + let message = Message::from(prompt_message); + + assert_eq!(message.as_concat_text(), "visible text"); + } + #[test] fn test_from_prompt_message_image() { let prompt_content = ContentBlock::image("base64data", "image/jpeg");