add MCP conformance tests to goose CI (combines #10800 + #10801) (#10940)

Signed-off-by: Jack Amadeo <jackamadeo@squareup.com> Co-authored-by: Jack Amadeo <jackamadeo@squareup.com>
2026-08-20 21:51:53 +00:00
parent 810bb68fff
commit 2f5169af35
18 changed files with 1380 additions and 126 deletions
@@ -19,6 +19,10 @@ path = "src/main.rs"
 name = "generate_manpages"
 path = "src/bin/generate_manpages.rs"

+[[bin]]
+name = "mcp_conformance_driver"
+path = "src/bin/mcp_conformance_driver.rs"
+
 [dependencies]
 clap_mangen = { version = "0.3", default-features = false }
 goose = { path = "../goose", default-features = false }
@@ -0,0 +1,132 @@
+use std::io::Write;
+use std::process::{Command, Stdio};
+
+use serde_json::{json, Map, Value};
+
+fn script_for_scenario(scenario: Option<&str>) -> Value {
+    let context: Map<String, Value> = std::env::var("MCP_CONFORMANCE_CONTEXT")
+        .ok()
+        .and_then(|raw| serde_json::from_str(&raw).ok())
+        .unwrap_or_default();
+
+    let mut script = match scenario {
+        Some("tools_call") => json!({
+            "steps": [{ "action": "callTool", "name": "add_numbers", "arguments": { "a": 2, "b": 3 } }],
+        }),
+        Some("elicitation-sep1034-client-defaults") => json!({
+            "steps": [{ "action": "callTool", "name": "test_client_elicitation_defaults", "arguments": {} }],
+            "elicitation": { "action": "acceptSchemaDefaults" },
+        }),
+        Some("auth/scope-step-up") => json!({
+            "steps": [{ "action": "callTool", "name": "test-tool", "arguments": {} }],
+        }),
+        Some("sse-retry") => json!({
+            "steps": [{ "action": "callTool", "name": "test_reconnection", "arguments": {} }],
+        }),
+        Some("auth/basic-cimd") => json!({
+            "steps": [{ "action": "listTools" }],
+            "oauth": { "clientMetadataUrl": "https://conformance-test.local/client-metadata.json" },
+        }),
+        Some("auth/pre-registration") => json!({
+            "steps": [{ "action": "listTools" }],
+            "oauth": { "clientId": context.get("client_id"), "clientSecret": context.get("client_secret") },
+        }),
+        Some("sep-2322-client-request-state") => json!({
+            "steps": [
+                { "action": "callTool", "name": "test_mrtr_echo_state", "arguments": {} },
+                { "action": "callTool", "name": "test_mrtr_no_state", "arguments": {} },
+                { "action": "callTool", "name": "test_mrtr_unrelated", "arguments": {} },
+                { "action": "callTool", "name": "test_mrtr_no_result_type", "arguments": {} },
+            ],
+            "elicitation": { "action": "accept", "content": { "confirmed": true } },
+        }),
+        Some("http-custom-headers") => {
+            let steps: Vec<Value> = context
+                .get("toolCalls")
+                .and_then(Value::as_array)
+                .cloned()
+                .unwrap_or_default()
+                .into_iter()
+                .map(|call| {
+                    let mut step = Map::new();
+                    step.insert("action".to_string(), json!("callTool"));
+                    if let Value::Object(call) = call {
+                        step.extend(call);
+                    }
+                    Value::Object(step)
+                })
+                .collect();
+            json!({ "steps": steps })
+        }
+        Some("http-invalid-tool-headers") => json!({
+            "steps": [{ "action": "callTool", "name": "valid_tool", "arguments": {} }],
+        }),
+        Some("http-standard-headers") => json!({
+            "steps": [
+                { "action": "listTools" },
+                { "action": "callTool", "name": "test_headers", "arguments": {} },
+                { "action": "listPrompts" },
+                { "action": "getPrompt", "name": "test_prompt", "arguments": {} },
+                { "action": "listResources" },
+                { "action": "readResource", "uri": "file:///path/to/file%20name.txt" },
+            ],
+        }),
+        _ => json!({
+            "steps": [
+                { "action": "listTools" },
+                { "action": "listPrompts" },
+                { "action": "listResources" },
+            ],
+        }),
+    };
+
+    // Runner 0.1.16 does not set MCP_CONFORMANCE_PROTOCOL_VERSION; default to
+    // the 2025-11-25 spec version those scenarios expect.
+    let protocol_version = std::env::var("MCP_CONFORMANCE_PROTOCOL_VERSION")
+        .unwrap_or_else(|_| "2025-11-25".to_string());
+    {
+        script["protocolVersion"] = json!(protocol_version);
+    }
+    script
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let [target] = args.as_slice() else {
+        eprintln!("usage: mcp_conformance_driver <server-url-or-stdio-command>");
+        std::process::exit(2);
+    };
+
+    let scenario = std::env::var("MCP_CONFORMANCE_SCENARIO").ok();
+    let script = script_for_scenario(scenario.as_deref());
+
+    let goose = std::env::var("GOOSE_BIN").unwrap_or_else(|_| "target/debug/goose".to_string());
+    let path_root = tempfile::Builder::new()
+        .prefix("goose-mcp-conformance-")
+        .tempdir()
+        .unwrap_or_else(|err| {
+            eprintln!("failed to create temporary GOOSE_PATH_ROOT: {err}");
+            std::process::exit(1);
+        });
+    let mut child = Command::new(&goose)
+        .args(["mcp-probe", target, "--script", "-"])
+        .env("GOOSE_OAUTH_AUTOMATIC_CALLBACK", "1")
+        .env("GOOSE_DISABLE_KEYRING", "1")
+        .env("GOOSE_PATH_ROOT", path_root.path())
+        .stdin(Stdio::piped())
+        .spawn()
+        .unwrap_or_else(|err| {
+            eprintln!("failed to spawn {goose}: {err}");
+            std::process::exit(1);
+        });
+
+    child
+        .stdin
+        .take()
+        .expect("stdin was piped")
+        .write_all(script.to_string().as_bytes())
+        .expect("write probe script to goose stdin");
+
+    let status = child.wait().expect("wait for goose");
+    std::process::exit(status.code().unwrap_or(1));
+}
@@ -1194,6 +1194,23 @@ enum Command {
        #[arg(help = "Path to the bundled-extensions.json file")]
        file: PathBuf,
    },
+
+    #[command(
+        name = "mcp-probe",
+        about = "Start a Goose MCP session without an LLM and inspect a stdio MCP server",
+        hide = true
+    )]
+    McpProbe {
+        #[arg(help = "Stdio MCP server command to inspect")]
+        extension: String,
+
+        #[arg(
+            long,
+            value_name = "PATH|-",
+            help = "JSON probe script; use - for stdin"
+        )]
+        script: Option<String>,
+    },
 }

 #[cfg(feature = "local-inference")]
@@ -1358,10 +1375,216 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
        Some(Command::Completion { .. }) => "completion",
        Some(Command::Review { .. }) => "review",
        Some(Command::ValidateExtensions { .. }) => "validate-extensions",
+        Some(Command::McpProbe { .. }) => "mcp-probe",
        None => "default_session",
    }
 }

+#[derive(serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct McpProbeScript {
+    #[serde(default)]
+    steps: Vec<McpProbeStep>,
+    elicitation: Option<McpProbeElicitation>,
+    #[serde(default)]
+    oauth: goose::oauth::OAuthFlowConfig,
+    protocol_version: Option<String>,
+}
+
+#[derive(serde::Deserialize)]
+#[serde(tag = "action", rename_all = "camelCase")]
+enum McpProbeStep {
+    ListTools,
+    ListPrompts,
+    ListResources,
+    CallTool {
+        name: String,
+        #[serde(default)]
+        arguments: serde_json::Map<String, serde_json::Value>,
+    },
+}
+
+#[derive(Clone, serde::Deserialize)]
+#[serde(tag = "action", rename_all = "camelCase")]
+enum McpProbeElicitation {
+    Accept { content: serde_json::Value },
+    AcceptSchemaDefaults,
+    Decline,
+    Cancel,
+}
+
+async fn handle_mcp_probe(extension_command: String, script_path: Option<String>) -> Result<()> {
+    use goose::agents::{Agent, AgentConfig, ToolCallContext};
+    use goose::config::ExtensionConfig;
+    use rmcp::model::{ElicitRequestParams, ElicitResult, ElicitationAction};
+    use tokio_util::sync::CancellationToken;
+
+    let script = if let Some(path) = script_path {
+        let json = if path == "-" {
+            let mut json = String::new();
+            std::io::stdin().read_to_string(&mut json)?;
+            json
+        } else {
+            std::fs::read_to_string(path)?
+        };
+        serde_json::from_str::<McpProbeScript>(&json)?
+    } else {
+        McpProbeScript {
+            steps: vec![
+                McpProbeStep::ListTools,
+                McpProbeStep::ListPrompts,
+                McpProbeStep::ListResources,
+            ],
+            elicitation: None,
+            oauth: goose::oauth::OAuthFlowConfig::default(),
+            protocol_version: None,
+        }
+    };
+
+    let mut extension = if url::Url::parse(&extension_command)
+        .is_ok_and(|url| matches!(url.scheme(), "http" | "https"))
+    {
+        crate::session::CliSession::parse_streamable_http_extension(
+            &extension_command,
+            goose::config::DEFAULT_EXTENSION_TIMEOUT,
+        )
+    } else {
+        crate::session::CliSession::parse_stdio_extension(&extension_command)?
+    };
+    match &mut extension {
+        ExtensionConfig::Stdio { name, .. } | ExtensionConfig::StreamableHttp { name, .. } => {
+            *name = "probe".to_string();
+        }
+        _ => unreachable!("MCP probe only creates stdio or streamable HTTP extensions"),
+    }
+
+    if let Some(client_id) = &script.oauth.client_id {
+        std::env::set_var("GOOSE_MCP_OAUTH_CLIENT_ID", client_id);
+    }
+    if let Some(client_secret) = &script.oauth.client_secret {
+        std::env::set_var("GOOSE_MCP_OAUTH_CLIENT_SECRET", client_secret);
+    }
+    if let Some(client_metadata_url) = &script.oauth.client_metadata_url {
+        std::env::set_var("GOOSE_MCP_OAUTH_CLIENT_METADATA_URL", client_metadata_url);
+    }
+
+    let config = goose::config::Config::global();
+    let mut agent_config = AgentConfig::new(
+        std::sync::Arc::new(SessionManager::instance()),
+        goose::config::permission::PermissionManager::instance(),
+        None,
+        config.get_goose_mode().unwrap_or_default(),
+        true,
+        GoosePlatform::GooseCli,
+    );
+    if let Some(protocol_version) = script.protocol_version.as_deref() {
+        agent_config.mcp_protocol_version = Some(serde_json::from_value(
+            serde_json::Value::String(protocol_version.to_string()),
+        )?);
+    }
+    if let Some(action) = script.elicitation.clone() {
+        agent_config.elicitation_handler =
+            Some(std::sync::Arc::new(move |request| match &action {
+                McpProbeElicitation::Accept { content } => {
+                    ElicitResult::new(ElicitationAction::Accept).with_content(content.clone())
+                }
+                McpProbeElicitation::AcceptSchemaDefaults => {
+                    let content = match request {
+                        ElicitRequestParams::FormElicitationParams {
+                            requested_schema, ..
+                        } => serde_json::to_value(requested_schema)
+                            .ok()
+                            .and_then(|schema| schema.get("properties").cloned())
+                            .and_then(|properties| properties.as_object().cloned())
+                            .map(|properties| {
+                                properties
+                                    .into_iter()
+                                    .filter_map(|(name, schema)| {
+                                        schema.get("default").cloned().map(|value| (name, value))
+                                    })
+                                    .collect()
+                            })
+                            .unwrap_or_default(),
+                        _ => serde_json::Map::new(),
+                    };
+                    ElicitResult::new(ElicitationAction::Accept)
+                        .with_content(serde_json::Value::Object(content))
+                }
+                McpProbeElicitation::Decline => ElicitResult::new(ElicitationAction::Decline),
+                McpProbeElicitation::Cancel => ElicitResult::new(ElicitationAction::Cancel),
+            }));
+    }
+    let agent = Agent::with_config(agent_config);
+    let session = agent
+        .config
+        .session_manager
+        .create_session(
+            std::env::current_dir()?,
+            "MCP Probe".to_string(),
+            goose::session::session_manager::SessionType::Hidden,
+            agent.config.goose_mode,
+        )
+        .await?;
+    let session_id = session.id.as_str();
+    agent.add_extension(extension, session_id).await?;
+
+    let mut results = Vec::new();
+    for step in script.steps {
+        let result = match step {
+            McpProbeStep::ListTools => serde_json::json!({
+                "action": "listTools",
+                "result": agent.extension_manager.list_tools_from_extension(
+                    session_id,
+                    "probe",
+                    CancellationToken::new(),
+                ).await?,
+            }),
+            McpProbeStep::ListPrompts => serde_json::json!({
+                "action": "listPrompts",
+                "result": agent.extension_manager.list_prompts_from_extension(
+                    session_id,
+                    "probe",
+                    CancellationToken::new(),
+                ).await?,
+            }),
+            McpProbeStep::ListResources => serde_json::json!({
+                "action": "listResources",
+                "result": agent.extension_manager.list_resources_result_from_extension(
+                    session_id,
+                    "probe",
+                    CancellationToken::new(),
+                ).await?,
+            }),
+            McpProbeStep::CallTool { name, arguments } => {
+                let scoped_name = format!("probe__{name}");
+                let ctx = ToolCallContext::new(
+                    session_id.to_string(),
+                    Some(std::env::current_dir()?),
+                    Some("mcp-probe-tool-call".to_string()),
+                );
+                let result = agent
+                    .extension_manager
+                    .dispatch_tool_call(
+                        &ctx,
+                        rmcp::model::CallToolRequestParams::new(scoped_name)
+                            .with_arguments(arguments),
+                        CancellationToken::new(),
+                    )
+                    .await?
+                    .result
+                    .await?;
+                serde_json::json!({ "action": "callTool", "name": name, "result": result })
+            }
+        };
+        results.push(result);
+    }
+    println!(
+        "{}",
+        serde_json::to_string_pretty(&serde_json::json!({ "results": results }))?
+    );
+    Ok(())
+}
+
 async fn handle_mcp_command(server: McpCommand) -> Result<()> {
    let name = server.name();
    let _ = crate::logging::setup_logging(Some(&format!("mcp-{name}")));
@@ -2385,6 +2608,7 @@ pub async fn cli() -> anyhow::Result<()> {
                }
            }
        }
+        Some(Command::McpProbe { extension, script }) => handle_mcp_probe(extension, script).await,
        None => handle_default_session().await,
    }
 }
@@ -118,12 +118,7 @@ impl McpClientTrait for MockClient {
        _next_cursor: Option<String>,
        _cancel_token: CancellationToken,
    ) -> Result<ListPromptsResult, Error> {
-        Ok(ListPromptsResult {
-            prompts: vec![],
-            next_cursor: None,
-            meta: None,
-            ..Default::default()
-        })
+        Ok(ListPromptsResult::with_all_items(vec![]))
    }

    async fn get_prompt(
@@ -0,0 +1,13 @@
+# MCP conformance baseline: known-failing client scenarios.
+#
+# Spec version:        2025-11-25
+# Conformance version: 0.1.16
+#
+# Scenarios listed here are allowed to fail (or emit warnings) without failing
+# CI. The conformance runner will error if any of these scenarios start passing
+# (stale entry) or if a scenario not listed here fails (unexpected failure), so
+# keep this list in sync as behavior changes.
+#
+# All scenarios currently pass; keep this file so CI flags any regression as an
+# unexpected failure.
+client: []
@@ -0,0 +1,13 @@
+# MCP conformance baseline: known-failing client scenarios.
+#
+# Spec version:        2025-11-25
+# Conformance version: 0.2.0-alpha.10
+#
+# Scenarios listed here are allowed to fail (or emit warnings) without failing
+# CI. The conformance runner will error if any of these scenarios start passing
+# (stale entry) or if a scenario not listed here fails (unexpected failure), so
+# keep this list in sync as behavior changes.
+#
+# All scenarios currently pass; keep this file so CI flags any regression as an
+# unexpected failure.
+client: []
@@ -0,0 +1,13 @@
+# MCP conformance baseline: known-failing client scenarios.
+#
+# Spec version:        2026-07-28
+# Conformance version: 0.2.0-alpha.10
+#
+# Scenarios listed here are allowed to fail (or emit warnings) without failing
+# CI. The conformance runner will error if any of these scenarios start passing
+# (stale entry) or if a scenario not listed here fails (unexpected failure), so
+# keep this list in sync as behavior changes.
+#
+# All scenarios currently pass; keep this file so CI flags any regression as an
+# unexpected failure.
+client: []