diff --git a/crates/goose-test-support/src/otel.rs b/crates/goose-test-support/src/otel.rs index 1b7459f37..210aac659 100644 --- a/crates/goose-test-support/src/otel.rs +++ b/crates/goose-test-support/src/otel.rs @@ -26,9 +26,6 @@ impl Drop for OtelTestGuard { } pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGuard { - let prev_tracer = global::tracer_provider(); - let prev_meter = global::meter_provider(); - let mut keys: Vec<&'static str> = vec![ "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", @@ -40,6 +37,7 @@ pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGua "OTEL_EXPORTER_OTLP_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "OTEL_LOG_LEVEL", "OTEL_LOGS_EXPORTER", "OTEL_METRICS_EXPORTER", @@ -55,6 +53,8 @@ pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGua } let guard = env_lock::lock_env(keys.into_iter().map(|k| (k, None::<&str>))); + let prev_tracer = global::tracer_provider(); + let prev_meter = global::meter_provider(); for &(k, v) in overrides { env::set_var(k, v); } diff --git a/crates/goose/src/agents/agent.rs b/crates/goose/src/agents/agent.rs index d8abdf0fe..973cb4bc0 100644 --- a/crates/goose/src/agents/agent.rs +++ b/crates/goose/src/agents/agent.rs @@ -1067,11 +1067,13 @@ impl Agent { cancellation_token: Option, session: &Session, ) -> (String, Result) { - let input_summary = serde_json::json!({ - "tool": tool_call.name, - "arguments": tool_call.arguments, - }); - tracing::Span::current().record("input", tracing::field::display(&input_summary)); + if gen_ai_telemetry::capture_message_content() { + let input_summary = serde_json::json!({ + "tool": tool_call.name, + "arguments": tool_call.arguments, + }); + tracing::Span::current().record("input", tracing::field::display(&input_summary)); + } gen_ai_telemetry::record_tool_arguments(&tracing::Span::current(), &tool_call); self.prompt_manager @@ -2045,9 +2047,9 @@ impl Agent { let session_manager = self.config.session_manager.clone(); let message_text_for_trace = agent_visible_message_text(&user_message); - tracing::Span::current().record("user_message", message_text_for_trace.as_str()); - tracing::Span::current().record("trace_input", message_text_for_trace.as_str()); if gen_ai_telemetry::capture_message_content() { + tracing::Span::current().record("user_message", message_text_for_trace.as_str()); + tracing::Span::current().record("trace_input", message_text_for_trace.as_str()); tracing::Span::current().record( "gen_ai.input.messages", gen_ai_telemetry::simple_input_json(&message_text_for_trace).as_str(), @@ -3596,17 +3598,13 @@ impl Agent { tokio::task::yield_now().await; } - if !last_assistant_text.is_empty() { + if !last_assistant_text.is_empty() + && gen_ai_telemetry::capture_message_content() + { tracing::Span::current().record("trace_output", last_assistant_text.as_str()); - if gen_ai_telemetry::capture_message_content() { - let output_json = - gen_ai_telemetry::simple_output_json(&last_assistant_text); - tracing::Span::current().record( - "gen_ai.output.messages", - output_json.as_str(), - ); - reply_span.record("gen_ai.output.messages", output_json.as_str()); - } + let output_json = gen_ai_telemetry::simple_output_json(&last_assistant_text); + tracing::Span::current().record("gen_ai.output.messages", output_json.as_str()); + reply_span.record("gen_ai.output.messages", output_json.as_str()); } gen_ai_telemetry::record_usage(&tracing::Span::current(), &turn_total_usage); gen_ai_telemetry::record_usage(&reply_span, &turn_total_usage); @@ -4352,6 +4350,50 @@ mod tests { (agent, session, data_dir) } + async fn capture_tool_dispatch_fields( + capture_setting: Option<&'static str>, + ) -> serde_json::Map { + use goose_test_support::otel::clear_otel_env; + use rmcp::object; + + let _env = match capture_setting { + Some(value) => { + clear_otel_env(&[(gen_ai_telemetry::CAPTURE_MESSAGE_CONTENT_ENV, value)]) + } + None => clear_otel_env(&[]), + }; + let capture = SpanFieldCapture::new("dispatch_tool_call"); + let _subscriber = capture.clone().set_default(); + let (agent, session, _data_dir) = tracing_test_agent_and_session().await; + let tool_call = CallToolRequestParams::new( + crate::agents::platform_extensions::scheduler::MANAGE_SCHEDULE_TOOL_NAME_COMPLETE, + ) + .with_arguments(object!({ + "action": "list", + "api_key": "tool-input-super-secret-token", + })); + + let (_, result) = agent + .dispatch_tool_call(tool_call, "call-secret".to_string(), None, &session) + .await; + let _ = result.unwrap().result.await; + capture.fields() + } + + #[tokio::test] + async fn tool_arguments_are_not_traced_without_content_capture() { + for capture_setting in [None, Some("false")] { + let fields = capture_tool_dispatch_fields(capture_setting).await; + let recorded = serde_json::to_string(&fields).unwrap(); + + assert!(!recorded.contains("tool-input-super-secret-token")); + assert!(!fields.contains_key("input")); + assert!(!fields.contains_key("gen_ai.tool.call.arguments")); + assert_eq!(fields["gen_ai.operation.name"], "execute_tool"); + assert_eq!(fields["gen_ai.tool.call.id"], "call-secret"); + } + } + #[tokio::test] async fn tool_dispatch_records_gen_ai_span_attributes() { use goose_test_support::otel::clear_otel_env; @@ -4383,6 +4425,8 @@ mod tests { assert_eq!(fields["gen_ai.tool.name"], tool_name); assert_eq!(fields["gen_ai.tool.call.id"], "call-42"); assert_eq!(fields["gen_ai.conversation.id"], session.id); + let input: Value = serde_json::from_str(fields["input"].as_str().unwrap()).unwrap(); + assert_eq!(input["arguments"]["action"], "list"); let arguments: Value = serde_json::from_str(fields["gen_ai.tool.call.arguments"].as_str().unwrap()).unwrap(); assert_eq!(arguments["action"], "list"); @@ -5259,6 +5303,118 @@ echo start >> "$PLUGIN_ROOT/hook.log" Ok((agent, session.id)) } + struct TraceContentProvider; + + #[async_trait::async_trait] + impl crate::providers::base::Provider for TraceContentProvider { + async fn stream( + &self, + _model_config: &goose_providers::model::ModelConfig, + _system_prompt: &str, + _messages: &[Message], + _tools: &[Tool], + ) -> Result { + let message = Message::assistant().with_text("output-super-secret-token"); + let usage = ProviderUsage::new("mock-model".to_string(), Usage::default()); + Ok(stream_from_single_message(message, usage)) + } + + fn get_name(&self) -> &str { + "trace-content" + } + } + + async fn capture_legacy_reply_fields( + span_name: &'static str, + capture_setting: Option<&'static str>, + ) -> Result> { + use goose_test_support::otel::clear_otel_env; + + let mut overrides = vec![("GOOSE_STATE_MACHINE", "0")]; + if let Some(value) = capture_setting { + overrides.push((gen_ai_telemetry::CAPTURE_MESSAGE_CONTENT_ENV, value)); + } + let _env = clear_otel_env(&overrides); + let capture = SpanFieldCapture::new(span_name); + let _subscriber = capture.clone().set_default(); + let temp_dir = tempfile::tempdir()?; + let hook_manager = crate::hooks::HookManager::from_plugins_for_test(vec![]); + let (agent, session_id) = create_test_agent( + temp_dir.path().join("data"), + hook_manager, + Arc::new(TraceContentProvider), + ) + .await?; + let session_config = SessionConfig { + id: session_id, + schedule_id: None, + max_turns: Some(1), + retry_config: None, + }; + let reply_stream = agent + .reply( + Message::user().with_text("input-super-secret-token"), + session_config, + None, + ) + .await?; + tokio::pin!(reply_stream); + while let Some(event) = reply_stream.next().await { + event?; + } + + Ok(capture.fields()) + } + + #[tokio::test] + async fn legacy_reply_trace_omits_content_without_capture() -> Result<()> { + for capture_setting in [None, Some("false")] { + let reply_fields = capture_legacy_reply_fields("reply", capture_setting).await?; + let reply_json = serde_json::to_string(&reply_fields)?; + assert!(!reply_json.contains("super-secret-token")); + assert!(!reply_fields.contains_key("user_message")); + assert!(!reply_fields.contains_key("trace_input")); + assert!(!reply_fields.contains_key("gen_ai.input.messages")); + assert!(!reply_fields.contains_key("gen_ai.output.messages")); + + let stream_fields = + capture_legacy_reply_fields("reply_stream", capture_setting).await?; + let stream_json = serde_json::to_string(&stream_fields)?; + assert!(!stream_json.contains("super-secret-token")); + assert!(!stream_fields.contains_key("trace_output")); + assert!(!stream_fields.contains_key("gen_ai.input.messages")); + assert!(!stream_fields.contains_key("gen_ai.output.messages")); + } + Ok(()) + } + + #[tokio::test] + async fn legacy_reply_trace_retains_content_with_capture() -> Result<()> { + let reply_fields = capture_legacy_reply_fields("reply", Some("true")).await?; + assert_eq!(reply_fields["user_message"], "input-super-secret-token"); + assert_eq!(reply_fields["trace_input"], "input-super-secret-token"); + assert!(reply_fields["gen_ai.input.messages"] + .as_str() + .unwrap() + .contains("input-super-secret-token")); + assert!(reply_fields["gen_ai.output.messages"] + .as_str() + .unwrap() + .contains("output-super-secret-token")); + + let stream_fields = capture_legacy_reply_fields("reply_stream", Some("true")).await?; + assert_eq!(stream_fields["trace_output"], "output-super-secret-token"); + assert!(stream_fields["gen_ai.input.messages"] + .as_str() + .unwrap() + .contains("input-super-secret-token")); + assert!(stream_fields["gen_ai.output.messages"] + .as_str() + .unwrap() + .contains("output-super-secret-token")); + Ok(()) + } + async fn create_stop_hook_test_agent( env: &StopHookTestEnv, stop_hook_block_cap: u32, diff --git a/crates/goose/src/agents/state_machine/session.rs b/crates/goose/src/agents/state_machine/session.rs index 8cbddefad..2e9fa5f95 100644 --- a/crates/goose/src/agents/state_machine/session.rs +++ b/crates/goose/src/agents/state_machine/session.rs @@ -177,16 +177,20 @@ pub(crate) async fn run( "gen_ai.agent.name", crate::agents::gen_ai_telemetry::agent_name(&entry_session), ); - if let Some(input) = entry_session - .conversation() - .and_then(|conversation| { - crate::agents::state_machine::messages_since_kickoff(conversation).ok() - }) - .and_then(|messages| messages.first()) - .map(crate::conversation::message::Message::user_visible_content) - .map(|message| message.as_concat_text()) - .filter(|text| !text.is_empty()) - { + let trace_input = if crate::agents::gen_ai_telemetry::capture_message_content() { + entry_session + .conversation() + .and_then(|conversation| { + crate::agents::state_machine::messages_since_kickoff(conversation).ok() + }) + .and_then(|messages| messages.first()) + .map(crate::conversation::message::Message::user_visible_content) + .map(|message| message.as_concat_text()) + .filter(|text| !text.is_empty()) + } else { + None + }; + if let Some(input) = trace_input { tracing::Span::current().record("trace_input", input.as_str()); } @@ -224,8 +228,8 @@ pub(crate) async fn run( .unwrap_or_default(); if !last_assistant_text.is_empty() { let span = tracing::Span::current(); - span.record("trace_output", last_assistant_text.as_str()); if crate::agents::gen_ai_telemetry::capture_message_content() { + span.record("trace_output", last_assistant_text.as_str()); let output = crate::agents::gen_ai_telemetry::simple_output_json(&last_assistant_text); span.record("gen_ai.output.messages", output.as_str()); } diff --git a/crates/goose/src/agents/state_machine/tests/mod.rs b/crates/goose/src/agents/state_machine/tests/mod.rs index f5f2ad7ee..6f00f2c2f 100644 --- a/crates/goose/src/agents/state_machine/tests/mod.rs +++ b/crates/goose/src/agents/state_machine/tests/mod.rs @@ -5,6 +5,7 @@ use self::pipeline::MessageKind::{Agent, ToolCall}; use self::pipeline::{test_pipeline, MAX_TURNS}; use crate::agents::state_machine; use crate::agents::state_machine::ops_retry::NUDGED; +use crate::agents::state_machine::Emitter; mod agent_reply; mod calculator_extension; @@ -19,6 +20,85 @@ mod reconstruction_isolation_lifecycle; mod steering_lifecycle; mod tool_lifecycle; +async fn capture_state_machine_trace_fields( + capture_setting: Option<&'static str>, +) -> Result> { + use goose_test_support::otel::clear_otel_env; + use tokio::sync::mpsc; + use tokio_util::sync::CancellationToken; + use tracing_futures::Instrument; + + use crate::agents::gen_ai_telemetry::{ + test_support::SpanFieldCapture, CAPTURE_MESSAGE_CONTENT_ENV, + }; + use crate::conversation::message::Message; + + let _env = match capture_setting { + Some(value) => clear_otel_env(&[(CAPTURE_MESSAGE_CONTENT_ENV, value)]), + None => clear_otel_env(&[]), + }; + let capture = SpanFieldCapture::new("state_machine_security_trace"); + let _subscriber = capture.clone().set_default(); + let (pipeline, api) = test_pipeline().await?; + api.on("input-super-secret-token") + .reply("output-super-secret-token"); + pipeline + .session_manager + .add_message( + &pipeline.session_id, + &Message::user().with_text("input-super-secret-token"), + ) + .await?; + + let cancel = CancellationToken::new(); + let machine = pipeline.machine(cancel.clone()); + let (tx, _rx) = mpsc::channel(1024); + let emit = Emitter::new(tx, cancel); + let span = tracing::info_span!( + "state_machine_security_trace", + trace_input = tracing::field::Empty, + trace_output = tracing::field::Empty, + gen_ai.agent.name = tracing::field::Empty, + gen_ai.output.messages = tracing::field::Empty, + ); + super::session::run( + &machine, + pipeline.session_manager.as_ref(), + &pipeline.session_id, + &emit, + ) + .instrument(span) + .await?; + + Ok(capture.fields()) +} + +#[tokio::test] +async fn state_machine_trace_omits_content_without_capture() -> Result<()> { + for capture_setting in [None, Some("false")] { + let fields = capture_state_machine_trace_fields(capture_setting).await?; + let recorded = serde_json::to_string(&fields)?; + assert!(!recorded.contains("super-secret-token")); + assert!(!fields.contains_key("trace_input")); + assert!(!fields.contains_key("trace_output")); + assert!(!fields.contains_key("gen_ai.output.messages")); + assert_eq!(fields["gen_ai.agent.name"], "goose"); + } + Ok(()) +} + +#[tokio::test] +async fn state_machine_trace_retains_content_with_capture() -> Result<()> { + let fields = capture_state_machine_trace_fields(Some("true")).await?; + assert_eq!(fields["trace_input"], "input-super-secret-token"); + assert_eq!(fields["trace_output"], "output-super-secret-token"); + assert!(fields["gen_ai.output.messages"] + .as_str() + .unwrap() + .contains("output-super-secret-token")); + Ok(()) +} + #[tokio::test] async fn bang_shell_requests_the_shell_tool() -> Result<()> { let (pipeline, api) = test_pipeline().await?; diff --git a/documentation/docs/guides/environment-variables.md b/documentation/docs/guides/environment-variables.md index 60bdbba77..3ef0f8e5e 100644 --- a/documentation/docs/guides/environment-variables.md +++ b/documentation/docs/guides/environment-variables.md @@ -428,6 +428,7 @@ You can control each signal (traces, metrics, logs) independently with `OTEL_{SI | `OTEL_EXPORTER_OTLP_{SIGNAL}_ENDPOINT` | Override endpoint for a specific signal | URL | | `OTEL_{SIGNAL}_EXPORTER` | Exporter type per signal | `otlp`, `console`, `none` | | `OTEL_SDK_DISABLED` | Disable all OTel export | `true` | +| `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` | Include model messages and tool arguments/results in exported traces | `true`, `false` (default) | Additional variables like `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`, and `OTEL_EXPORTER_OTLP_TIMEOUT` are also supported. diff --git a/documentation/docs/tutorials/langfuse.md b/documentation/docs/tutorials/langfuse.md index 588aef23d..6e4726baf 100644 --- a/documentation/docs/tutorials/langfuse.md +++ b/documentation/docs/tutorials/langfuse.md @@ -28,6 +28,14 @@ export LANGFUSE_URL=https://cloud.langfuse.com # EU data region 🇪🇺 # https://localhost:3000 if you're self-hosting ``` +By default, traces exclude model messages and tool arguments and results. To include that content, explicitly enable content capture: + +```bash +export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true +``` + +Only enable content capture when your telemetry storage is appropriate for potentially sensitive content. + ## Run goose with Langfuse Integration Now, you can run goose and monitor your AI requests and actions through Langfuse. @@ -37,4 +45,3 @@ With goose running and the environment variables set, Langfuse will start captur _[Example trace (public) in Langfuse](https://cloud.langfuse.com/project/cloramnkj0002jz088vzn1ja4/traces/cea4ed38-0c44-4b0a-8c20-4b0b6b9e8d73?timestamp=2025-01-31T15%3A52%3A30.362Z&observation=7c8e5807-3c29-4c28-9c6f-7d7427be401f)_ ![goose trace in Langfuse](https://langfuse.com//images/docs/goose-integration/goose-example-trace.png) -