Add out of context compaction test via error proxy (#5805)

.github/workflows/pr-smoke-test.yml

@@ -143,6 +143,14 @@ jobs:
         run: |
           bash scripts/test_subrecipes.sh
 
+      - name: Set up Python (for error proxy)
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv (for error proxy)
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # pin@v6
+
       - name: Run Compaction Tests
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

crates/goose/src/providers/retry.rs

@@ -109,8 +109,17 @@ pub trait ProviderRetry {
                             _ => config.delay_for_attempt(attempts),
                         };
 
-                        tracing::info!("Backing off for {:?} before retry", delay);
+                        let skip_backoff = std::env::var("GOOSE_PROVIDER_SKIP_BACKOFF")
-                        sleep(delay).await;
+                            .unwrap_or_default()
+                            .parse::<bool>()
+                            .unwrap_or(false);
+
+                        if skip_backoff {
+                            tracing::info!("Skipping backoff due to GOOSE_PROVIDER_SKIP_BACKOFF");
+                        } else {
+                            tracing::info!("Backing off for {:?} before retry", delay);
+                            sleep(delay).await;
+                        }
                         continue;
                     }
 

scripts/provider-error-proxy/README.md

@@ -64,6 +64,38 @@ Use a custom port:
 uv run proxy.py --port 9000
 ```
 
+Start the proxy with an initial error mode (for automated testing):
+
+```bash
+# Start with context length error (3 times)
+uv run proxy.py --mode "c 3"
+
+# Start with rate limit error (30% of requests)
+uv run proxy.py --mode "r 30%"
+
+# Start with server error (all requests)
+uv run proxy.py --mode "u *"
+```
+
+Command-line options:
+- `--port PORT` - Port to listen on (default: 8888)
+- `--mode COMMAND` - Initial error mode command (e.g., "c 3", "r 30%", "u *", "n")
+  - Same syntax as interactive commands
+- `--no-stdin` - Disable stdin reader (for background/automated mode)
+
+For automated tests or background usage, combine `--no-stdin` with `--mode`:
+
+```bash
+# Run in background for automated testing
+uv run proxy.py --mode "c 3" --no-stdin &
+PROXY_PID=$!
+
+# ... run your tests ...
+
+# Stop the proxy
+kill $PROXY_PID
+```
+
 ### Interactive Commands
 
 Once the proxy is running, you can control error injection interactively:

scripts/provider-error-proxy/proxy.py

@@ -553,6 +553,73 @@ class ErrorProxy:
             )
 
 
+def parse_command(command: str) -> tuple[Optional[ErrorMode], int, float, Optional[str]]:
+    """
+    Parse a command string and return the error mode, count, and percentage.
+
+    Args:
+        command: Command string (e.g., "c", "c 3", "r 30%", "u *")
+
+    Returns:
+        Tuple of (mode, count, percentage, error_message)
+        If error_message is not None, parsing failed
+    """
+    # Parse command - remove all whitespace and parse
+    command_no_space = command.strip().replace(" ", "")
+    if not command_no_space:
+        return (None, 0, 0.0, "Empty command")
+
+    # Get the first character (error type letter)
+    error_letter = command_no_space[0].lower()
+
+    # Map letter to ErrorMode
+    mode_map = {
+        'n': ErrorMode.NO_ERROR,
+        'c': ErrorMode.CONTEXT_LENGTH,
+        'r': ErrorMode.RATE_LIMIT,
+        'u': ErrorMode.SERVER_ERROR
+    }
+
+    if error_letter not in mode_map:
+        return (None, 0, 0.0, f"Invalid command: '{error_letter}'. Use n, c, r, or u")
+
+    mode = mode_map[error_letter]
+
+    # Parse the rest as count or percentage
+    count = 1
+    percentage = 0.0
+
+    if len(command_no_space) > 1:
+        value_str = command_no_space[1:]
+
+        try:
+            # Check for * (100%)
+            if value_str == '*':
+                percentage = 1.0
+                count = 0  # Percentage mode
+            # Check for percentage with % sign (e.g., "30%")
+            elif value_str.endswith('%'):
+                percentage = float(value_str[:-1]) / 100.0
+                if percentage < 0.0 or percentage > 1.0:
+                    return (None, 0, 0.0, f"Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%")
+                count = 0  # Percentage mode
+            # Check if it's a decimal (percentage as 0.0-1.0)
+            elif '.' in value_str:
+                percentage = float(value_str)
+                if percentage < 0.0 or percentage > 1.0:
+                    return (None, 0, 0.0, f"Invalid percentage: {percentage}. Must be between 0.0 and 1.0")
+                count = 0  # Percentage mode
+            else:
+                # It's an integer count
+                count = int(value_str)
+                if count < 0:
+                    return (None, 0, 0.0, f"Invalid count: {count}. Must be >= 0")
+        except ValueError:
+            return (None, 0, 0.0, f"Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)")
+
+    return (mode, count, percentage, None)
+
+
 def print_status(proxy: ErrorProxy):
     """Print the current proxy status."""
     mode, count, percentage = proxy.get_error_config()
@@ -562,7 +629,7 @@ def print_status(proxy: ErrorProxy):
         ErrorMode.RATE_LIMIT: "⏱️  Rate limit exceeded",
         ErrorMode.SERVER_ERROR: "💥 Server error (500)"
     }
-    
+
     print("\n" + "=" * 60)
     mode_str = mode_names.get(mode, 'Unknown')
     if mode != ErrorMode.NO_ERROR:
@@ -589,79 +656,28 @@ def print_status(proxy: ErrorProxy):
 def stdin_reader(proxy: ErrorProxy, loop):
     """Read commands from stdin in a separate thread."""
     print_status(proxy)
-    
+
     while True:
         try:
             command = input("Enter command: ").strip()
-            
+
             if command.lower() == 'q':
                 print("\n🛑 Shutting down proxy...")
                 # Schedule the shutdown in the event loop
                 asyncio.run_coroutine_threadsafe(shutdown_server(loop), loop)
                 break
-            
+
-            # Parse command - remove all whitespace and parse
+            # Parse the command using the shared parser
-            command_no_space = command.replace(" ", "")
+            mode, count, percentage, error_msg = parse_command(command)
-            if not command_no_space:
+
+            if error_msg:
+                print(f"❌ {error_msg}")
                 continue
-            
+
-            # Get the first character (error type letter)
-            error_letter = command_no_space[0].lower()
-            
-            # Map letter to ErrorMode
-            mode_map = {
-                'n': ErrorMode.NO_ERROR,
-                'c': ErrorMode.CONTEXT_LENGTH,
-                'r': ErrorMode.RATE_LIMIT,
-                'u': ErrorMode.SERVER_ERROR
-            }
-            
-            if error_letter not in mode_map:
-                print(f"❌ Invalid command: '{error_letter}'. Use n, c, r, u, or q")
-                continue
-            
-            mode = mode_map[error_letter]
-            
-            # Parse the rest as count or percentage
-            count = 1
-            percentage = 0.0
-            
-            if len(command_no_space) > 1:
-                value_str = command_no_space[1:]
-                
-                try:
-                    # Check for * (100%)
-                    if value_str == '*':
-                        percentage = 1.0
-                        count = 0  # Percentage mode
-                    # Check for percentage with % sign (e.g., "30%")
-                    elif value_str.endswith('%'):
-                        percentage = float(value_str[:-1]) / 100.0
-                        if percentage < 0.0 or percentage > 1.0:
-                            print(f"❌ Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%")
-                            continue
-                        count = 0  # Percentage mode
-                    # Check if it's a decimal (percentage as 0.0-1.0)
-                    elif '.' in value_str:
-                        percentage = float(value_str)
-                        if percentage < 0.0 or percentage > 1.0:
-                            print(f"❌ Invalid percentage: {percentage}. Must be between 0.0 and 1.0")
-                            continue
-                        count = 0  # Percentage mode
-                    else:
-                        # It's an integer count
-                        count = int(value_str)
-                        if count < 0:
-                            print(f"❌ Invalid count: {count}. Must be >= 0")
-                            continue
-                except ValueError:
-                    print(f"❌ Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)")
-                    continue
-            
             # Set the error mode
             proxy.set_error_mode(mode, count, percentage)
             print_status(proxy)
-                
+
         except EOFError:
             # Handle Ctrl+D
             print("\n🛑 Shutting down proxy...")
@@ -716,7 +732,17 @@ def main():
         default=8888,
         help='Port to listen on (default: 8888)'
     )
-    
+    parser.add_argument(
+        '--mode',
+        type=str,
+        help='Error mode command (e.g., "c 3", "r 30%%", "u *", "n")'
+    )
+    parser.add_argument(
+        '--no-stdin',
+        action='store_true',
+        help='Disable stdin reader (for background/automated mode)'
+    )
+
     args = parser.parse_args()
     
     print("=" * 60)
@@ -735,14 +761,38 @@ def main():
     
     # Create proxy instance
     proxy = ErrorProxy()
-    
+
+    # Set initial error mode from command-line arguments
+    if args.mode:
+        mode, count, percentage, error_msg = parse_command(args.mode)
+
+        if error_msg:
+            print(f"❌ Error parsing --mode argument: {error_msg}")
+            print(f"   Example usage: --mode \"c 3\" or --mode \"r 30%\"")
+            return
+
+        proxy.set_error_mode(mode, count, percentage)
+        print()
+        print(f"Initial mode set from command-line arguments:")
+        print(f"  Mode: {mode.name}")
+        if percentage > 0.0:
+            print(f"  Percentage: {percentage*100:.0f}%")
+        elif count > 0:
+            print(f"  Count: {count}")
+        print()
+
     # Create event loop
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    
+
-    # Start stdin reader thread
+    # Start stdin reader thread only if not disabled
-    stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True)
+    if not args.no_stdin:
-    stdin_thread.start()
+        stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True)
+        stdin_thread.start()
+    else:
+        print("Running in no-stdin mode (background/automated)")
+        print("Use SIGINT (Ctrl+C) or SIGTERM to stop the proxy")
+        print()
     
     # Create and run the app
     app = loop.run_until_complete(create_app(proxy))

scripts/test_compaction.sh

@@ -85,13 +85,6 @@ echo "COMPACTION SMOKE TESTS"
 echo "=================================================="
 echo ""
 
-# Check if jq is available
-if ! command -v jq &> /dev/null; then
-  echo "⚠ WARNING: jq is not installed. Compaction structure validation will be limited."
-  echo "   Install jq to enable full validation: brew install jq (macOS) or apt-get install jq (Linux)"
-  echo ""
-fi
-
 RESULTS=()
 
 # ==================================================
@@ -219,6 +212,126 @@ rm -rf "$TESTDIR"
 echo ""
 echo ""
 
+# ==================================================
+# TEST 3: Out-of-Context Error Compaction
+# ==================================================
+echo "---------------------------------------------------"
+echo "TEST 3: Compaction via out-of-context error (proxy)"
+echo "---------------------------------------------------"
+
+TESTDIR=$(mktemp -d)
+echo "test content" > "$TESTDIR/test.txt"
+echo "Test directory: $TESTDIR"
+echo ""
+
+# Use a random port to avoid conflicts
+PROXY_PORT=$((9000 + RANDOM % 1000))
+PROXY_DIR="$SCRIPT_DIR/scripts/provider-error-proxy"
+
+OUTPUT=$(mktemp)
+PROXY_LOG=$(mktemp)
+PROXY_SETUP_LOG=$(mktemp)
+
+# Pre-install proxy dependencies (so first run doesn't take forever)
+echo "Installing proxy dependencies..."
+export UV_INDEX_URL="https://pypi.org/simple"
+if ! (cd "$PROXY_DIR" && uv sync 2>&1 | tee "$PROXY_SETUP_LOG"); then
+  echo "✗ FAILED: Could not install proxy dependencies"
+  echo "Setup log:"
+  cat "$PROXY_SETUP_LOG"
+  RESULTS+=("✗ Out-of-Context Error (dependency install failed)")
+else
+  echo "✓ Dependencies installed"
+
+  # Start the error proxy in context-length error mode (3 errors)
+  echo "Starting error proxy on port $PROXY_PORT with context-length error mode..."
+  (cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 3" --no-stdin > "$PROXY_LOG" 2>&1) &
+  PROXY_PID=$!
+
+  # Wait for proxy to be ready (check if port is listening)
+  echo "Waiting for proxy to be ready..."
+  PROXY_READY=false
+  for i in {1..60}; do
+    if kill -0 $PROXY_PID 2>/dev/null; then
+      # Check if port is listening using /dev/tcp
+      if timeout 1 bash -c "echo -n > /dev/tcp/localhost/$PROXY_PORT" 2>/dev/null; then
+        PROXY_READY=true
+        echo "✓ Proxy is ready on port $PROXY_PORT"
+        break
+      fi
+    else
+      echo "✗ FAILED: Error proxy process died"
+      break
+    fi
+    sleep 0.5
+  done
+
+  # Check if proxy is running and ready
+  if [ "$PROXY_READY" != "true" ]; then
+    echo "✗ FAILED: Error proxy failed to become ready"
+    echo "Proxy log:"
+    cat "$PROXY_LOG"
+    kill $PROXY_PID 2>/dev/null || true
+    RESULTS+=("✗ Out-of-Context Test Error (proxy failed)")
+  else
+    # Configure provider to use proxy and skip backoff
+    export ANTHROPIC_HOST="http://localhost:$PROXY_PORT"
+    export GOOSE_PROVIDER_SKIP_BACKOFF=true
+    export GOOSE_PROVIDER=anthropic
+    export GOOSE_MODEL=claude-haiku-4-5
+
+    echo "Step 1: Creating session (should trigger context-length error and compaction)..."
+    (cd "$TESTDIR" && "$GOOSE_BIN" run --text "hello world" 2>&1) | tee "$OUTPUT"
+
+    SESSION_ID=$("$GOOSE_BIN" session list --format json 2>/dev/null | jq -r '.[0].id' 2>/dev/null)
+
+    if [ -z "$SESSION_ID" ] || [ "$SESSION_ID" = "null" ]; then
+      echo "✗ FAILED: Could not create session"
+      RESULTS+=("✗ Out-of-Context Test Error (no session)")
+    else
+      echo ""
+      echo "Session created: $SESSION_ID"
+      echo "Checking for compaction evidence..."
+
+      # Check for compaction in the output
+      if grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then
+        echo "✓ SUCCESS: Out-of-context Test error triggered compaction"
+
+        if validate_compaction "$SESSION_ID" "out-of-context error compaction"; then
+          RESULTS+=("✓ Out-of-Context Test Error")
+        else
+          RESULTS+=("✗ Out-of-Context Test Error (structure validation failed)")
+        fi
+      else
+        echo "✗ FAILED: No evidence of compaction after context-length error"
+        echo "   Output:"
+        cat "$OUTPUT"
+        RESULTS+=("✗ Out-of-Context Test Error")
+      fi
+    fi
+
+    # Clean up
+    echo ""
+    echo "Stopping error proxy..."
+    # Kill the entire process group to ensure UV and Python processes are terminated
+    kill -- -$PROXY_PID 2>/dev/null || true
+    # Also explicitly kill any remaining UV processes on this port
+    pkill -f "uv run.*--port $PROXY_PORT" 2>/dev/null || true
+    wait $PROXY_PID 2>/dev/null || true
+    unset ANTHROPIC_HOST
+    unset GOOSE_PROVIDER_SKIP_BACKOFF
+    unset GOOSE_PROVIDER
+    unset GOOSE_MODEL
+    unset UV_INDEX_URL
+  fi
+fi
+
+rm -f "$OUTPUT" "$PROXY_LOG" "$PROXY_SETUP_LOG"
+rm -rf "$TESTDIR"
+
+echo ""
+echo ""
+
 # ==================================================
 # Summary
 # ==================================================