fix(tracing): honor content capture opt-out (#11782)

This commit is contained in:
Jasper
2026-09-03 10:36:12 +02:00
committed by GitHub
parent 35524facf0
commit 0338189c66
6 changed files with 280 additions and 32 deletions
+3 -3
View File
@@ -26,9 +26,6 @@ impl Drop for OtelTestGuard {
}
pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGuard {
let prev_tracer = global::tracer_provider();
let prev_meter = global::meter_provider();
let mut keys: Vec<&'static str> = vec![
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
@@ -40,6 +37,7 @@ pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGua
"OTEL_EXPORTER_OTLP_TIMEOUT",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL",
"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT",
"OTEL_LOG_LEVEL",
"OTEL_LOGS_EXPORTER",
"OTEL_METRICS_EXPORTER",
@@ -55,6 +53,8 @@ pub fn clear_otel_env(overrides: &[(&'static str, &'static str)]) -> OtelTestGua
}
let guard = env_lock::lock_env(keys.into_iter().map(|k| (k, None::<&str>)));
let prev_tracer = global::tracer_provider();
let prev_meter = global::meter_provider();
for &(k, v) in overrides {
env::set_var(k, v);
}
+173 -17
View File
@@ -1067,11 +1067,13 @@ impl Agent {
cancellation_token: Option<CancellationToken>,
session: &Session,
) -> (String, Result<ToolCallResult, ErrorData>) {
let input_summary = serde_json::json!({
"tool": tool_call.name,
"arguments": tool_call.arguments,
});
tracing::Span::current().record("input", tracing::field::display(&input_summary));
if gen_ai_telemetry::capture_message_content() {
let input_summary = serde_json::json!({
"tool": tool_call.name,
"arguments": tool_call.arguments,
});
tracing::Span::current().record("input", tracing::field::display(&input_summary));
}
gen_ai_telemetry::record_tool_arguments(&tracing::Span::current(), &tool_call);
self.prompt_manager
@@ -2045,9 +2047,9 @@ impl Agent {
let session_manager = self.config.session_manager.clone();
let message_text_for_trace = agent_visible_message_text(&user_message);
tracing::Span::current().record("user_message", message_text_for_trace.as_str());
tracing::Span::current().record("trace_input", message_text_for_trace.as_str());
if gen_ai_telemetry::capture_message_content() {
tracing::Span::current().record("user_message", message_text_for_trace.as_str());
tracing::Span::current().record("trace_input", message_text_for_trace.as_str());
tracing::Span::current().record(
"gen_ai.input.messages",
gen_ai_telemetry::simple_input_json(&message_text_for_trace).as_str(),
@@ -3596,17 +3598,13 @@ impl Agent {
tokio::task::yield_now().await;
}
if !last_assistant_text.is_empty() {
if !last_assistant_text.is_empty()
&& gen_ai_telemetry::capture_message_content()
{
tracing::Span::current().record("trace_output", last_assistant_text.as_str());
if gen_ai_telemetry::capture_message_content() {
let output_json =
gen_ai_telemetry::simple_output_json(&last_assistant_text);
tracing::Span::current().record(
"gen_ai.output.messages",
output_json.as_str(),
);
reply_span.record("gen_ai.output.messages", output_json.as_str());
}
let output_json = gen_ai_telemetry::simple_output_json(&last_assistant_text);
tracing::Span::current().record("gen_ai.output.messages", output_json.as_str());
reply_span.record("gen_ai.output.messages", output_json.as_str());
}
gen_ai_telemetry::record_usage(&tracing::Span::current(), &turn_total_usage);
gen_ai_telemetry::record_usage(&reply_span, &turn_total_usage);
@@ -4352,6 +4350,50 @@ mod tests {
(agent, session, data_dir)
}
async fn capture_tool_dispatch_fields(
capture_setting: Option<&'static str>,
) -> serde_json::Map<String, Value> {
use goose_test_support::otel::clear_otel_env;
use rmcp::object;
let _env = match capture_setting {
Some(value) => {
clear_otel_env(&[(gen_ai_telemetry::CAPTURE_MESSAGE_CONTENT_ENV, value)])
}
None => clear_otel_env(&[]),
};
let capture = SpanFieldCapture::new("dispatch_tool_call");
let _subscriber = capture.clone().set_default();
let (agent, session, _data_dir) = tracing_test_agent_and_session().await;
let tool_call = CallToolRequestParams::new(
crate::agents::platform_extensions::scheduler::MANAGE_SCHEDULE_TOOL_NAME_COMPLETE,
)
.with_arguments(object!({
"action": "list",
"api_key": "tool-input-super-secret-token",
}));
let (_, result) = agent
.dispatch_tool_call(tool_call, "call-secret".to_string(), None, &session)
.await;
let _ = result.unwrap().result.await;
capture.fields()
}
#[tokio::test]
async fn tool_arguments_are_not_traced_without_content_capture() {
for capture_setting in [None, Some("false")] {
let fields = capture_tool_dispatch_fields(capture_setting).await;
let recorded = serde_json::to_string(&fields).unwrap();
assert!(!recorded.contains("tool-input-super-secret-token"));
assert!(!fields.contains_key("input"));
assert!(!fields.contains_key("gen_ai.tool.call.arguments"));
assert_eq!(fields["gen_ai.operation.name"], "execute_tool");
assert_eq!(fields["gen_ai.tool.call.id"], "call-secret");
}
}
#[tokio::test]
async fn tool_dispatch_records_gen_ai_span_attributes() {
use goose_test_support::otel::clear_otel_env;
@@ -4383,6 +4425,8 @@ mod tests {
assert_eq!(fields["gen_ai.tool.name"], tool_name);
assert_eq!(fields["gen_ai.tool.call.id"], "call-42");
assert_eq!(fields["gen_ai.conversation.id"], session.id);
let input: Value = serde_json::from_str(fields["input"].as_str().unwrap()).unwrap();
assert_eq!(input["arguments"]["action"], "list");
let arguments: Value =
serde_json::from_str(fields["gen_ai.tool.call.arguments"].as_str().unwrap()).unwrap();
assert_eq!(arguments["action"], "list");
@@ -5259,6 +5303,118 @@ echo start >> "$PLUGIN_ROOT/hook.log"
Ok((agent, session.id))
}
struct TraceContentProvider;
#[async_trait::async_trait]
impl crate::providers::base::Provider for TraceContentProvider {
async fn stream(
&self,
_model_config: &goose_providers::model::ModelConfig,
_system_prompt: &str,
_messages: &[Message],
_tools: &[Tool],
) -> Result<MessageStream, ProviderError> {
let message = Message::assistant().with_text("output-super-secret-token");
let usage = ProviderUsage::new("mock-model".to_string(), Usage::default());
Ok(stream_from_single_message(message, usage))
}
fn get_name(&self) -> &str {
"trace-content"
}
}
async fn capture_legacy_reply_fields(
span_name: &'static str,
capture_setting: Option<&'static str>,
) -> Result<serde_json::Map<String, Value>> {
use goose_test_support::otel::clear_otel_env;
let mut overrides = vec![("GOOSE_STATE_MACHINE", "0")];
if let Some(value) = capture_setting {
overrides.push((gen_ai_telemetry::CAPTURE_MESSAGE_CONTENT_ENV, value));
}
let _env = clear_otel_env(&overrides);
let capture = SpanFieldCapture::new(span_name);
let _subscriber = capture.clone().set_default();
let temp_dir = tempfile::tempdir()?;
let hook_manager = crate::hooks::HookManager::from_plugins_for_test(vec![]);
let (agent, session_id) = create_test_agent(
temp_dir.path().join("data"),
hook_manager,
Arc::new(TraceContentProvider),
)
.await?;
let session_config = SessionConfig {
id: session_id,
schedule_id: None,
max_turns: Some(1),
retry_config: None,
};
let reply_stream = agent
.reply(
Message::user().with_text("input-super-secret-token"),
session_config,
None,
)
.await?;
tokio::pin!(reply_stream);
while let Some(event) = reply_stream.next().await {
event?;
}
Ok(capture.fields())
}
#[tokio::test]
async fn legacy_reply_trace_omits_content_without_capture() -> Result<()> {
for capture_setting in [None, Some("false")] {
let reply_fields = capture_legacy_reply_fields("reply", capture_setting).await?;
let reply_json = serde_json::to_string(&reply_fields)?;
assert!(!reply_json.contains("super-secret-token"));
assert!(!reply_fields.contains_key("user_message"));
assert!(!reply_fields.contains_key("trace_input"));
assert!(!reply_fields.contains_key("gen_ai.input.messages"));
assert!(!reply_fields.contains_key("gen_ai.output.messages"));
let stream_fields =
capture_legacy_reply_fields("reply_stream", capture_setting).await?;
let stream_json = serde_json::to_string(&stream_fields)?;
assert!(!stream_json.contains("super-secret-token"));
assert!(!stream_fields.contains_key("trace_output"));
assert!(!stream_fields.contains_key("gen_ai.input.messages"));
assert!(!stream_fields.contains_key("gen_ai.output.messages"));
}
Ok(())
}
#[tokio::test]
async fn legacy_reply_trace_retains_content_with_capture() -> Result<()> {
let reply_fields = capture_legacy_reply_fields("reply", Some("true")).await?;
assert_eq!(reply_fields["user_message"], "input-super-secret-token");
assert_eq!(reply_fields["trace_input"], "input-super-secret-token");
assert!(reply_fields["gen_ai.input.messages"]
.as_str()
.unwrap()
.contains("input-super-secret-token"));
assert!(reply_fields["gen_ai.output.messages"]
.as_str()
.unwrap()
.contains("output-super-secret-token"));
let stream_fields = capture_legacy_reply_fields("reply_stream", Some("true")).await?;
assert_eq!(stream_fields["trace_output"], "output-super-secret-token");
assert!(stream_fields["gen_ai.input.messages"]
.as_str()
.unwrap()
.contains("input-super-secret-token"));
assert!(stream_fields["gen_ai.output.messages"]
.as_str()
.unwrap()
.contains("output-super-secret-token"));
Ok(())
}
async fn create_stop_hook_test_agent(
env: &StopHookTestEnv,
stop_hook_block_cap: u32,
@@ -177,16 +177,20 @@ pub(crate) async fn run(
"gen_ai.agent.name",
crate::agents::gen_ai_telemetry::agent_name(&entry_session),
);
if let Some(input) = entry_session
.conversation()
.and_then(|conversation| {
crate::agents::state_machine::messages_since_kickoff(conversation).ok()
})
.and_then(|messages| messages.first())
.map(crate::conversation::message::Message::user_visible_content)
.map(|message| message.as_concat_text())
.filter(|text| !text.is_empty())
{
let trace_input = if crate::agents::gen_ai_telemetry::capture_message_content() {
entry_session
.conversation()
.and_then(|conversation| {
crate::agents::state_machine::messages_since_kickoff(conversation).ok()
})
.and_then(|messages| messages.first())
.map(crate::conversation::message::Message::user_visible_content)
.map(|message| message.as_concat_text())
.filter(|text| !text.is_empty())
} else {
None
};
if let Some(input) = trace_input {
tracing::Span::current().record("trace_input", input.as_str());
}
@@ -224,8 +228,8 @@ pub(crate) async fn run(
.unwrap_or_default();
if !last_assistant_text.is_empty() {
let span = tracing::Span::current();
span.record("trace_output", last_assistant_text.as_str());
if crate::agents::gen_ai_telemetry::capture_message_content() {
span.record("trace_output", last_assistant_text.as_str());
let output = crate::agents::gen_ai_telemetry::simple_output_json(&last_assistant_text);
span.record("gen_ai.output.messages", output.as_str());
}
@@ -5,6 +5,7 @@ use self::pipeline::MessageKind::{Agent, ToolCall};
use self::pipeline::{test_pipeline, MAX_TURNS};
use crate::agents::state_machine;
use crate::agents::state_machine::ops_retry::NUDGED;
use crate::agents::state_machine::Emitter;
mod agent_reply;
mod calculator_extension;
@@ -19,6 +20,85 @@ mod reconstruction_isolation_lifecycle;
mod steering_lifecycle;
mod tool_lifecycle;
async fn capture_state_machine_trace_fields(
capture_setting: Option<&'static str>,
) -> Result<serde_json::Map<String, serde_json::Value>> {
use goose_test_support::otel::clear_otel_env;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing_futures::Instrument;
use crate::agents::gen_ai_telemetry::{
test_support::SpanFieldCapture, CAPTURE_MESSAGE_CONTENT_ENV,
};
use crate::conversation::message::Message;
let _env = match capture_setting {
Some(value) => clear_otel_env(&[(CAPTURE_MESSAGE_CONTENT_ENV, value)]),
None => clear_otel_env(&[]),
};
let capture = SpanFieldCapture::new("state_machine_security_trace");
let _subscriber = capture.clone().set_default();
let (pipeline, api) = test_pipeline().await?;
api.on("input-super-secret-token")
.reply("output-super-secret-token");
pipeline
.session_manager
.add_message(
&pipeline.session_id,
&Message::user().with_text("input-super-secret-token"),
)
.await?;
let cancel = CancellationToken::new();
let machine = pipeline.machine(cancel.clone());
let (tx, _rx) = mpsc::channel(1024);
let emit = Emitter::new(tx, cancel);
let span = tracing::info_span!(
"state_machine_security_trace",
trace_input = tracing::field::Empty,
trace_output = tracing::field::Empty,
gen_ai.agent.name = tracing::field::Empty,
gen_ai.output.messages = tracing::field::Empty,
);
super::session::run(
&machine,
pipeline.session_manager.as_ref(),
&pipeline.session_id,
&emit,
)
.instrument(span)
.await?;
Ok(capture.fields())
}
#[tokio::test]
async fn state_machine_trace_omits_content_without_capture() -> Result<()> {
for capture_setting in [None, Some("false")] {
let fields = capture_state_machine_trace_fields(capture_setting).await?;
let recorded = serde_json::to_string(&fields)?;
assert!(!recorded.contains("super-secret-token"));
assert!(!fields.contains_key("trace_input"));
assert!(!fields.contains_key("trace_output"));
assert!(!fields.contains_key("gen_ai.output.messages"));
assert_eq!(fields["gen_ai.agent.name"], "goose");
}
Ok(())
}
#[tokio::test]
async fn state_machine_trace_retains_content_with_capture() -> Result<()> {
let fields = capture_state_machine_trace_fields(Some("true")).await?;
assert_eq!(fields["trace_input"], "input-super-secret-token");
assert_eq!(fields["trace_output"], "output-super-secret-token");
assert!(fields["gen_ai.output.messages"]
.as_str()
.unwrap()
.contains("output-super-secret-token"));
Ok(())
}
#[tokio::test]
async fn bang_shell_requests_the_shell_tool() -> Result<()> {
let (pipeline, api) = test_pipeline().await?;
@@ -428,6 +428,7 @@ You can control each signal (traces, metrics, logs) independently with `OTEL_{SI
| `OTEL_EXPORTER_OTLP_{SIGNAL}_ENDPOINT` | Override endpoint for a specific signal | URL |
| `OTEL_{SIGNAL}_EXPORTER` | Exporter type per signal | `otlp`, `console`, `none` |
| `OTEL_SDK_DISABLED` | Disable all OTel export | `true` |
| `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` | Include model messages and tool arguments/results in exported traces | `true`, `false` (default) |
Additional variables like `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`,
and `OTEL_EXPORTER_OTLP_TIMEOUT` are also supported.
+8 -1
View File
@@ -28,6 +28,14 @@ export LANGFUSE_URL=https://cloud.langfuse.com # EU data region 🇪🇺
# https://localhost:3000 if you're self-hosting
```
By default, traces exclude model messages and tool arguments and results. To include that content, explicitly enable content capture:
```bash
export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
```
Only enable content capture when your telemetry storage is appropriate for potentially sensitive content.
## Run goose with Langfuse Integration
Now, you can run goose and monitor your AI requests and actions through Langfuse.
@@ -37,4 +45,3 @@ With goose running and the environment variables set, Langfuse will start captur
_[Example trace (public) in Langfuse](https://cloud.langfuse.com/project/cloramnkj0002jz088vzn1ja4/traces/cea4ed38-0c44-4b0a-8c20-4b0b6b9e8d73?timestamp=2025-01-31T15%3A52%3A30.362Z&observation=7c8e5807-3c29-4c28-9c6f-7d7427be401f)_
![goose trace in Langfuse](https://langfuse.com//images/docs/goose-integration/goose-example-trace.png)