diff --git a/.github/workflows/pr-smoke-test.yml b/.github/workflows/pr-smoke-test.yml index 60346711..096459dc 100644 --- a/.github/workflows/pr-smoke-test.yml +++ b/.github/workflows/pr-smoke-test.yml @@ -143,6 +143,14 @@ jobs: run: | bash scripts/test_subrecipes.sh + - name: Set up Python (for error proxy) + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv (for error proxy) + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # pin@v6 + - name: Run Compaction Tests env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/crates/goose/src/providers/retry.rs b/crates/goose/src/providers/retry.rs index f1097839..4c8c4975 100644 --- a/crates/goose/src/providers/retry.rs +++ b/crates/goose/src/providers/retry.rs @@ -109,8 +109,17 @@ pub trait ProviderRetry { _ => config.delay_for_attempt(attempts), }; - tracing::info!("Backing off for {:?} before retry", delay); - sleep(delay).await; + let skip_backoff = std::env::var("GOOSE_PROVIDER_SKIP_BACKOFF") + .unwrap_or_default() + .parse::() + .unwrap_or(false); + + if skip_backoff { + tracing::info!("Skipping backoff due to GOOSE_PROVIDER_SKIP_BACKOFF"); + } else { + tracing::info!("Backing off for {:?} before retry", delay); + sleep(delay).await; + } continue; } diff --git a/scripts/provider-error-proxy/README.md b/scripts/provider-error-proxy/README.md index 70b31b94..ead861fb 100644 --- a/scripts/provider-error-proxy/README.md +++ b/scripts/provider-error-proxy/README.md @@ -64,6 +64,38 @@ Use a custom port: uv run proxy.py --port 9000 ``` +Start the proxy with an initial error mode (for automated testing): + +```bash +# Start with context length error (3 times) +uv run proxy.py --mode "c 3" + +# Start with rate limit error (30% of requests) +uv run proxy.py --mode "r 30%" + +# Start with server error (all requests) +uv run proxy.py --mode "u *" +``` + +Command-line options: +- `--port PORT` - Port to listen on (default: 8888) +- `--mode COMMAND` - Initial error mode command (e.g., "c 3", "r 30%", "u *", "n") + - Same syntax as interactive commands +- `--no-stdin` - Disable stdin reader (for background/automated mode) + +For automated tests or background usage, combine `--no-stdin` with `--mode`: + +```bash +# Run in background for automated testing +uv run proxy.py --mode "c 3" --no-stdin & +PROXY_PID=$! + +# ... run your tests ... + +# Stop the proxy +kill $PROXY_PID +``` + ### Interactive Commands Once the proxy is running, you can control error injection interactively: diff --git a/scripts/provider-error-proxy/proxy.py b/scripts/provider-error-proxy/proxy.py index 3d2dda71..7e764cc6 100755 --- a/scripts/provider-error-proxy/proxy.py +++ b/scripts/provider-error-proxy/proxy.py @@ -553,6 +553,73 @@ class ErrorProxy: ) +def parse_command(command: str) -> tuple[Optional[ErrorMode], int, float, Optional[str]]: + """ + Parse a command string and return the error mode, count, and percentage. + + Args: + command: Command string (e.g., "c", "c 3", "r 30%", "u *") + + Returns: + Tuple of (mode, count, percentage, error_message) + If error_message is not None, parsing failed + """ + # Parse command - remove all whitespace and parse + command_no_space = command.strip().replace(" ", "") + if not command_no_space: + return (None, 0, 0.0, "Empty command") + + # Get the first character (error type letter) + error_letter = command_no_space[0].lower() + + # Map letter to ErrorMode + mode_map = { + 'n': ErrorMode.NO_ERROR, + 'c': ErrorMode.CONTEXT_LENGTH, + 'r': ErrorMode.RATE_LIMIT, + 'u': ErrorMode.SERVER_ERROR + } + + if error_letter not in mode_map: + return (None, 0, 0.0, f"Invalid command: '{error_letter}'. Use n, c, r, or u") + + mode = mode_map[error_letter] + + # Parse the rest as count or percentage + count = 1 + percentage = 0.0 + + if len(command_no_space) > 1: + value_str = command_no_space[1:] + + try: + # Check for * (100%) + if value_str == '*': + percentage = 1.0 + count = 0 # Percentage mode + # Check for percentage with % sign (e.g., "30%") + elif value_str.endswith('%'): + percentage = float(value_str[:-1]) / 100.0 + if percentage < 0.0 or percentage > 1.0: + return (None, 0, 0.0, f"Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%") + count = 0 # Percentage mode + # Check if it's a decimal (percentage as 0.0-1.0) + elif '.' in value_str: + percentage = float(value_str) + if percentage < 0.0 or percentage > 1.0: + return (None, 0, 0.0, f"Invalid percentage: {percentage}. Must be between 0.0 and 1.0") + count = 0 # Percentage mode + else: + # It's an integer count + count = int(value_str) + if count < 0: + return (None, 0, 0.0, f"Invalid count: {count}. Must be >= 0") + except ValueError: + return (None, 0, 0.0, f"Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)") + + return (mode, count, percentage, None) + + def print_status(proxy: ErrorProxy): """Print the current proxy status.""" mode, count, percentage = proxy.get_error_config() @@ -562,7 +629,7 @@ def print_status(proxy: ErrorProxy): ErrorMode.RATE_LIMIT: "ā±ļø Rate limit exceeded", ErrorMode.SERVER_ERROR: "šŸ’„ Server error (500)" } - + print("\n" + "=" * 60) mode_str = mode_names.get(mode, 'Unknown') if mode != ErrorMode.NO_ERROR: @@ -589,79 +656,28 @@ def print_status(proxy: ErrorProxy): def stdin_reader(proxy: ErrorProxy, loop): """Read commands from stdin in a separate thread.""" print_status(proxy) - + while True: try: command = input("Enter command: ").strip() - + if command.lower() == 'q': print("\nšŸ›‘ Shutting down proxy...") # Schedule the shutdown in the event loop asyncio.run_coroutine_threadsafe(shutdown_server(loop), loop) break - - # Parse command - remove all whitespace and parse - command_no_space = command.replace(" ", "") - if not command_no_space: + + # Parse the command using the shared parser + mode, count, percentage, error_msg = parse_command(command) + + if error_msg: + print(f"āŒ {error_msg}") continue - - # Get the first character (error type letter) - error_letter = command_no_space[0].lower() - - # Map letter to ErrorMode - mode_map = { - 'n': ErrorMode.NO_ERROR, - 'c': ErrorMode.CONTEXT_LENGTH, - 'r': ErrorMode.RATE_LIMIT, - 'u': ErrorMode.SERVER_ERROR - } - - if error_letter not in mode_map: - print(f"āŒ Invalid command: '{error_letter}'. Use n, c, r, u, or q") - continue - - mode = mode_map[error_letter] - - # Parse the rest as count or percentage - count = 1 - percentage = 0.0 - - if len(command_no_space) > 1: - value_str = command_no_space[1:] - - try: - # Check for * (100%) - if value_str == '*': - percentage = 1.0 - count = 0 # Percentage mode - # Check for percentage with % sign (e.g., "30%") - elif value_str.endswith('%'): - percentage = float(value_str[:-1]) / 100.0 - if percentage < 0.0 or percentage > 1.0: - print(f"āŒ Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%") - continue - count = 0 # Percentage mode - # Check if it's a decimal (percentage as 0.0-1.0) - elif '.' in value_str: - percentage = float(value_str) - if percentage < 0.0 or percentage > 1.0: - print(f"āŒ Invalid percentage: {percentage}. Must be between 0.0 and 1.0") - continue - count = 0 # Percentage mode - else: - # It's an integer count - count = int(value_str) - if count < 0: - print(f"āŒ Invalid count: {count}. Must be >= 0") - continue - except ValueError: - print(f"āŒ Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)") - continue - + # Set the error mode proxy.set_error_mode(mode, count, percentage) print_status(proxy) - + except EOFError: # Handle Ctrl+D print("\nšŸ›‘ Shutting down proxy...") @@ -716,7 +732,17 @@ def main(): default=8888, help='Port to listen on (default: 8888)' ) - + parser.add_argument( + '--mode', + type=str, + help='Error mode command (e.g., "c 3", "r 30%%", "u *", "n")' + ) + parser.add_argument( + '--no-stdin', + action='store_true', + help='Disable stdin reader (for background/automated mode)' + ) + args = parser.parse_args() print("=" * 60) @@ -735,14 +761,38 @@ def main(): # Create proxy instance proxy = ErrorProxy() - + + # Set initial error mode from command-line arguments + if args.mode: + mode, count, percentage, error_msg = parse_command(args.mode) + + if error_msg: + print(f"āŒ Error parsing --mode argument: {error_msg}") + print(f" Example usage: --mode \"c 3\" or --mode \"r 30%\"") + return + + proxy.set_error_mode(mode, count, percentage) + print() + print(f"Initial mode set from command-line arguments:") + print(f" Mode: {mode.name}") + if percentage > 0.0: + print(f" Percentage: {percentage*100:.0f}%") + elif count > 0: + print(f" Count: {count}") + print() + # Create event loop loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - - # Start stdin reader thread - stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True) - stdin_thread.start() + + # Start stdin reader thread only if not disabled + if not args.no_stdin: + stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True) + stdin_thread.start() + else: + print("Running in no-stdin mode (background/automated)") + print("Use SIGINT (Ctrl+C) or SIGTERM to stop the proxy") + print() # Create and run the app app = loop.run_until_complete(create_app(proxy)) diff --git a/scripts/test_compaction.sh b/scripts/test_compaction.sh index f3bf6d43..5e51f57d 100755 --- a/scripts/test_compaction.sh +++ b/scripts/test_compaction.sh @@ -85,13 +85,6 @@ echo "COMPACTION SMOKE TESTS" echo "==================================================" echo "" -# Check if jq is available -if ! command -v jq &> /dev/null; then - echo "āš  WARNING: jq is not installed. Compaction structure validation will be limited." - echo " Install jq to enable full validation: brew install jq (macOS) or apt-get install jq (Linux)" - echo "" -fi - RESULTS=() # ================================================== @@ -219,6 +212,126 @@ rm -rf "$TESTDIR" echo "" echo "" +# ================================================== +# TEST 3: Out-of-Context Error Compaction +# ================================================== +echo "---------------------------------------------------" +echo "TEST 3: Compaction via out-of-context error (proxy)" +echo "---------------------------------------------------" + +TESTDIR=$(mktemp -d) +echo "test content" > "$TESTDIR/test.txt" +echo "Test directory: $TESTDIR" +echo "" + +# Use a random port to avoid conflicts +PROXY_PORT=$((9000 + RANDOM % 1000)) +PROXY_DIR="$SCRIPT_DIR/scripts/provider-error-proxy" + +OUTPUT=$(mktemp) +PROXY_LOG=$(mktemp) +PROXY_SETUP_LOG=$(mktemp) + +# Pre-install proxy dependencies (so first run doesn't take forever) +echo "Installing proxy dependencies..." +export UV_INDEX_URL="https://pypi.org/simple" +if ! (cd "$PROXY_DIR" && uv sync 2>&1 | tee "$PROXY_SETUP_LOG"); then + echo "āœ— FAILED: Could not install proxy dependencies" + echo "Setup log:" + cat "$PROXY_SETUP_LOG" + RESULTS+=("āœ— Out-of-Context Error (dependency install failed)") +else + echo "āœ“ Dependencies installed" + + # Start the error proxy in context-length error mode (3 errors) + echo "Starting error proxy on port $PROXY_PORT with context-length error mode..." + (cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 3" --no-stdin > "$PROXY_LOG" 2>&1) & + PROXY_PID=$! + + # Wait for proxy to be ready (check if port is listening) + echo "Waiting for proxy to be ready..." + PROXY_READY=false + for i in {1..60}; do + if kill -0 $PROXY_PID 2>/dev/null; then + # Check if port is listening using /dev/tcp + if timeout 1 bash -c "echo -n > /dev/tcp/localhost/$PROXY_PORT" 2>/dev/null; then + PROXY_READY=true + echo "āœ“ Proxy is ready on port $PROXY_PORT" + break + fi + else + echo "āœ— FAILED: Error proxy process died" + break + fi + sleep 0.5 + done + + # Check if proxy is running and ready + if [ "$PROXY_READY" != "true" ]; then + echo "āœ— FAILED: Error proxy failed to become ready" + echo "Proxy log:" + cat "$PROXY_LOG" + kill $PROXY_PID 2>/dev/null || true + RESULTS+=("āœ— Out-of-Context Test Error (proxy failed)") + else + # Configure provider to use proxy and skip backoff + export ANTHROPIC_HOST="http://localhost:$PROXY_PORT" + export GOOSE_PROVIDER_SKIP_BACKOFF=true + export GOOSE_PROVIDER=anthropic + export GOOSE_MODEL=claude-haiku-4-5 + + echo "Step 1: Creating session (should trigger context-length error and compaction)..." + (cd "$TESTDIR" && "$GOOSE_BIN" run --text "hello world" 2>&1) | tee "$OUTPUT" + + SESSION_ID=$("$GOOSE_BIN" session list --format json 2>/dev/null | jq -r '.[0].id' 2>/dev/null) + + if [ -z "$SESSION_ID" ] || [ "$SESSION_ID" = "null" ]; then + echo "āœ— FAILED: Could not create session" + RESULTS+=("āœ— Out-of-Context Test Error (no session)") + else + echo "" + echo "Session created: $SESSION_ID" + echo "Checking for compaction evidence..." + + # Check for compaction in the output + if grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then + echo "āœ“ SUCCESS: Out-of-context Test error triggered compaction" + + if validate_compaction "$SESSION_ID" "out-of-context error compaction"; then + RESULTS+=("āœ“ Out-of-Context Test Error") + else + RESULTS+=("āœ— Out-of-Context Test Error (structure validation failed)") + fi + else + echo "āœ— FAILED: No evidence of compaction after context-length error" + echo " Output:" + cat "$OUTPUT" + RESULTS+=("āœ— Out-of-Context Test Error") + fi + fi + + # Clean up + echo "" + echo "Stopping error proxy..." + # Kill the entire process group to ensure UV and Python processes are terminated + kill -- -$PROXY_PID 2>/dev/null || true + # Also explicitly kill any remaining UV processes on this port + pkill -f "uv run.*--port $PROXY_PORT" 2>/dev/null || true + wait $PROXY_PID 2>/dev/null || true + unset ANTHROPIC_HOST + unset GOOSE_PROVIDER_SKIP_BACKOFF + unset GOOSE_PROVIDER + unset GOOSE_MODEL + unset UV_INDEX_URL + fi +fi + +rm -f "$OUTPUT" "$PROXY_LOG" "$PROXY_SETUP_LOG" +rm -rf "$TESTDIR" + +echo "" +echo "" + # ================================================== # Summary # ==================================================