tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

fix: use safe_truncate to truncate charactor (#3263) (#3264)

Browse Source 
Signed-off-by: toyamagu2021@gmail.com <toyamagu2021@gmail.com>
This commit is contained in:
toyamagu-2021
2025-07-15 19:34:40 +09:00
committed by GitHub
parent c6225e38d0
commit d3beeace95
13 changed files with 46 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/goose/src/agents/large_response_handler.rs
+2 -2
View File
@@ -17,14 +17,14 @@ pub fn process_tool_response(
match content {
Content::Text(text_content) => {
// Check if text exceeds threshold
if text_content.text.len() > LARGE_TEXT_THRESHOLD {
if text_content.text.chars().count() > LARGE_TEXT_THRESHOLD {
// Write to temp file
match write_large_text_to_file(&text_content.text) {
Ok(file_path) => {
// Create a new text content with reference to the file
let message = format!(
"The response returned from the tool call was larger ({} characters) and is stored in the file which you can use other tools to examine or search in: {}",
text_content.text.len(),
text_content.text.chars().count(),
file_path
);
processed_contents.push(Content::text(message));
crates/goose/src/context_mgmt/truncate.rs
+10 -9
View File
@@ -1,4 +1,5 @@
use crate::message::{Message, MessageContent};
use crate::utils::safe_truncate;
use anyhow::{anyhow, Result};
use mcp_core::{Content, ResourceContents, Role};
use std::collections::HashSet;
@@ -75,11 +76,11 @@ fn truncate_message_content(message: &Message, max_content_size: usize) -> Resul
for content in &mut new_message.content {
match content {
MessageContent::Text(text_content) => {
if text_content.text.len() > max_content_size {
if text_content.text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... content truncated from {} to {} characters ...]",
&text_content.text[..max_content_size.min(text_content.text.len())],
text_content.text.len(),
safe_truncate(&text_content.text, max_content_size),
text_content.text.chars().count(),
max_content_size
);
text_content.text = truncated;
@@ -89,11 +90,11 @@ fn truncate_message_content(message: &Message, max_content_size: usize) -> Resul
if let Ok(ref mut result) = tool_response.tool_result {
for content_item in result {
if let Content::Text(ref mut text_content) = content_item {
if text_content.text.len() > max_content_size {
if text_content.text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... tool response truncated from {} to {} characters ...]",
&text_content.text[..max_content_size.min(text_content.text.len())],
text_content.text.len(),
safe_truncate(&text_content.text, max_content_size),
text_content.text.chars().count(),
max_content_size
);
text_content.text = truncated;
@@ -104,11 +105,11 @@ fn truncate_message_content(message: &Message, max_content_size: usize) -> Resul
if let ResourceContents::TextResourceContents { text, .. } =
&mut resource_content.resource
{
if text.len() > max_content_size {
if text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... resource content truncated from {} to {} characters ...]",
&text[..max_content_size.min(text.len())],
text.len(),
safe_truncate(text, max_content_size),
text.chars().count(),
max_content_size
);
*text = truncated;
crates/goose/src/lib.rs
+1
View File
@@ -15,6 +15,7 @@ pub mod temporal_scheduler;
pub mod token_counter;
pub mod tool_monitor;
pub mod tracing;
pub mod utils;
#[cfg(test)]
mod cron_test;
crates/goose/src/session/storage.rs
+18 -21
View File
@@ -7,6 +7,7 @@
use crate::message::Message;
use crate::providers::base::Provider;
use crate::utils::safe_truncate;
use anyhow::Result;
use chrono::Local;
use etcetera::{choose_app_strategy, AppStrategy, AppStrategyArgs};
@@ -605,7 +606,7 @@ pub fn read_messages_with_truncation(
// Log details about corrupted lines (with limited detail for security)
for (num, line) in &corrupted_lines {
let preview = if line.len() > 50 {
format!("{}... (truncated)", &line[..50])
format!("{}... (truncated)", safe_truncate(line, 50))
} else {
line.clone()
};
@@ -678,11 +679,11 @@ fn truncate_message_content_in_place(message: &mut Message, max_content_size: us
for content in &mut message.content {
match content {
MessageContent::Text(text_content) => {
if text_content.text.len() > max_content_size {
if text_content.text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... content truncated during session loading from {} to {} characters ...]",
&text_content.text[..max_content_size.min(text_content.text.len())],
text_content.text.len(),
safe_truncate(&text_content.text, max_content_size),
text_content.text.chars().count(),
max_content_size
);
text_content.text = truncated;
@@ -693,11 +694,11 @@ fn truncate_message_content_in_place(message: &mut Message, max_content_size: us
for content_item in result {
match content_item {
Content::Text(ref mut text_content) => {
if text_content.text.len() > max_content_size {
if text_content.text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... tool response truncated during session loading from {} to {} characters ...]",
&text_content.text[..max_content_size.min(text_content.text.len())],
text_content.text.len(),
safe_truncate(&text_content.text, max_content_size),
text_content.text.chars().count(),
max_content_size
);
text_content.text = truncated;
@@ -707,11 +708,11 @@ fn truncate_message_content_in_place(message: &mut Message, max_content_size: us
if let ResourceContents::TextResourceContents { text, .. } =
&mut resource_content.resource
{
if text.len() > max_content_size {
if text.chars().count() > max_content_size {
let truncated = format!(
"{}\n\n[... resource content truncated during session loading from {} to {} characters ...]",
&text[..max_content_size.min(text.len())],
text.len(),
safe_truncate(text, max_content_size),
text.chars().count(),
max_content_size
);
*text = truncated;
@@ -751,7 +752,7 @@ fn attempt_corruption_recovery(json_str: &str, max_content_size: Option<usize>)
// Strategy 4: Create a placeholder message with the raw content
println!("[SESSION] All recovery strategies failed, creating placeholder message");
let preview = if json_str.len() > 200 {
format!("{}...", &json_str[..200])
format!("{}...", safe_truncate(json_str, 200))
} else {
json_str.to_string()
};
@@ -968,7 +969,7 @@ fn truncate_json_string(json_str: &str, max_content_size: usize) -> String {
if text_content.len() > max_content_size {
let truncated_text = format!(
"{}\n\n[... content truncated during JSON parsing from {} to {} characters ...]",
&text_content[..max_content_size.min(text_content.len())],
safe_truncate(text_content, max_content_size),
text_content.len(),
max_content_size
);
@@ -1269,11 +1270,7 @@ pub async fn generate_description_with_schedule_id(
.take(3) // Use up to first 3 user messages for context
.map(|m| {
let text = m.as_concat_text();
if text.len() > 300 {
format!("{}...", &text[..300])
} else {
text
}
safe_truncate(&text, 300)
})
.collect();
@@ -1302,9 +1299,9 @@ pub async fn generate_description_with_schedule_id(
let description = result.0.as_concat_text();
// Validate description length for security
let sanitized_description = if description.len() > 100 {
let sanitized_description = if description.chars().count() > 100 {
tracing::warn!("Generated description too long, truncating");
format!("{}...", &description[..97])
safe_truncate(&description, 100)
} else {
description
};
@@ -1379,9 +1376,9 @@ mod tests {
println!(
"[TEST] Input: {}",
if corrupt_json.len() > 100 {
&corrupt_json[..100]
safe_truncate(corrupt_json, 100)
} else {
corrupt_json
corrupt_json.to_string()
}
);
crates/goose/src/utils.rs
+49
View File
@@ -0,0 +1,49 @@
/// Safely truncate a string at character boundaries, not byte boundaries
///
/// This function ensures that multi-byte UTF-8 characters (like Japanese, emoji, etc.)
/// are not split in the middle, which would cause a panic.
///
/// # Arguments
/// * `s` - The string to truncate
/// * `max_chars` - Maximum number of characters to keep
///
/// # Returns
/// A truncated string with "..." appended if truncation occurred
pub fn safe_truncate(s: &str, max_chars: usize) -> String {
if s.chars().count() <= max_chars {
s.to_string()
} else {
let truncated: String = s.chars().take(max_chars.saturating_sub(3)).collect();
format!("{}...", truncated)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_safe_truncate_ascii() {
assert_eq!(safe_truncate("hello world", 20), "hello world");
assert_eq!(safe_truncate("hello world", 8), "hello...");
assert_eq!(safe_truncate("hello", 5), "hello");
assert_eq!(safe_truncate("hello", 3), "...");
}
#[test]
fn test_safe_truncate_japanese() {
// Japanese characters: "こんにちは世界" (Hello World)
let japanese = "こんにちは世界";
assert_eq!(safe_truncate(japanese, 10), japanese);
assert_eq!(safe_truncate(japanese, 5), "こん...");
assert_eq!(safe_truncate(japanese, 7), japanese);
}
#[test]
fn test_safe_truncate_mixed() {
// Mixed ASCII and Japanese
let mixed = "Hello こんにちは";
assert_eq!(safe_truncate(mixed, 20), mixed);
assert_eq!(safe_truncate(mixed, 8), "Hello...");
}
}