add MCP conformance tests to goose CI (combines #10800 + #10801) (#10940)

Signed-off-by: Jack Amadeo <jackamadeo@squareup.com> Co-authored-by: Jack Amadeo <jackamadeo@squareup.com>
2026-08-20 21:51:53 +00:00
parent 810bb68fff
commit 2f5169af35
18 changed files with 1380 additions and 126 deletions
@@ -0,0 +1,91 @@
+name: MCP Conformance
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  merge_group:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build Conformance Binaries
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+
+      - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
+
+      - name: Install Dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y libdbus-1-dev libxcb1-dev
+
+      - name: Cache Cargo artifacts
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4  # v2.9.1
+        with:
+          key: mcp-conformance
+
+      - name: Build
+        run: |
+          source ./bin/activate-hermit
+          export CARGO_INCREMENTAL=0
+          just mcp-conformance-build
+
+      - name: Upload Conformance Binaries
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: mcp-conformance-binaries
+          path: |
+            target/debug/goose
+            target/debug/mcp_conformance_driver
+          retention-days: 1
+          if-no-files-found: error
+
+  conformance:
+    name: Conformance ${{ matrix.spec-version }} / ${{ matrix.conformance-version }}
+    runs-on: ubuntu-latest
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - spec-version: "2025-11-25"
+            conformance-version: "0.1.16"
+            baseline: "crates/goose-cli/tests/mcp-conformance/expected-failures-2025-11-25-0.1.16.yaml"
+          - spec-version: "2025-11-25"
+            conformance-version: "0.2.0-alpha.10"
+            baseline: "crates/goose-cli/tests/mcp-conformance/expected-failures-2025-11-25-0.2.0-alpha.10.yaml"
+          - spec-version: "2026-07-28"
+            conformance-version: "0.2.0-alpha.10"
+            baseline: "crates/goose-cli/tests/mcp-conformance/expected-failures-2026-07-28-0.2.0-alpha.10.yaml"
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+
+      - name: Install Dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install -y libdbus-1-dev libxcb1-dev
+
+      - name: Download Conformance Binaries
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: mcp-conformance-binaries
+          path: target/debug
+
+      - name: Restore Executable Permissions
+        run: chmod +x target/debug/goose target/debug/mcp_conformance_driver
+
+      - name: Run Conformance Suite
+        env:
+          GOOSE_DISABLE_KEYRING: "1"
+        run: |
+          source ./bin/activate-hermit
+          just mcp-conformance "${{ matrix.spec-version }}" all "${{ matrix.conformance-version }}" false "${{ matrix.baseline }}"