diff --git a/scripts/provider-error-proxy/proxy.py b/scripts/provider-error-proxy/proxy.py index 7e764cc69..8025930bf 100755 --- a/scripts/provider-error-proxy/proxy.py +++ b/scripts/provider-error-proxy/proxy.py @@ -62,6 +62,42 @@ ALWAYS_FORWARD_PATHS = [ '/api/2.0/', # Databricks management API ] +# Path fragments identifying an inference/completion request. +# +# Error injection only applies to these. Every error mode the proxy offers +# describes a completion failure (context length exceeded, rate limited, +# upstream 500), so injecting them into metadata traffic such as model listing +# is meaningless. It is also actively harmful in count mode: a pre-flight +# request would silently consume the error budget intended for a completion, +# making "inject N errors" depend on goose's incidental call pattern rather +# than on the completions the test is actually exercising. +COMPLETION_PATHS = [ + '/chat/completions', # OpenAI and compatible + '/completions', # OpenAI legacy + '/messages', # Anthropic + '/responses', # OpenAI Responses API + ':generatecontent', # Google Gemini + ':streamgeneratecontent', + '/converse', # Bedrock + '/invoke', # Bedrock / Databricks serving + '/invocations', + '/serving-endpoints', # Databricks +] + +# Path suffixes identifying known non-inference (metadata) traffic. Used as +# the exclusion list when classifying requests that do not match +# COMPLETION_PATHS: a POST to an unrecognized path (e.g. a provider configured +# with a custom inference base path) is still treated as a completion, but +# known metadata endpoints are not. Matched as suffixes, not substrings, so a +# custom inference path that merely contains one of these segments (e.g. +# /v1/models/my-model/predict) is still classified as a completion. +METADATA_PATH_SUFFIXES = [ + '/models', # model listing (OpenAI-compatible) + '/api/tags', # Ollama model listing + '/api/show', # Ollama model info (POSTed before completions for context limits) + '/embeddings', # embeddings are not turn completions +] + class ErrorMode(Enum): """Error injection modes.""" @@ -383,7 +419,33 @@ class ErrorProxy: for forward_path in ALWAYS_FORWARD_PATHS: if forward_path in path: return True - return False + return not self.is_completion_request(request) + + def is_completion_request(self, request: Request) -> bool: + """ + Check whether this request is an inference/completion call. + + Only completion requests are eligible for error injection; see + COMPLETION_PATHS. + + Args: + request: The incoming HTTP request + + Returns: + True if this is a completion request + """ + path = request.path.lower() + if any(fragment in path for fragment in COMPLETION_PATHS): + return True + # Providers configured with a custom inference base path (e.g. an + # OpenAI-compatible endpoint at "/v1") post completions to + # routes that match none of the known fragments. Treat any other POST + # that is not known metadata traffic as a completion so error + # injection still applies to custom inference routes. + if request.method.upper() != 'POST': + return False + path = path.rstrip('/') + return not any(path.endswith(suffix) for suffix in METADATA_PATH_SUFFIXES) def get_target_url(self, request: Request, provider: str) -> str: """ @@ -462,7 +524,13 @@ class ErrorProxy: # Check if this request should always be forwarded if self.should_always_forward(request): - logger.info(f"🔄 Always forwarding: {request.path}") + if self.is_completion_request(request): + logger.info(f"🔄 Always forwarding: {request.path}") + else: + logger.info( + f"🔄 Forwarding non-completion request (not eligible for " + f"error injection): {request.path}" + ) else: # Capture the error mode BEFORE checking if we should inject (since that modifies state) mode_before_check = self.get_error_mode() diff --git a/scripts/test_compaction.sh b/scripts/test_compaction.sh index 0743cc10a..69fd5140a 100755 --- a/scripts/test_compaction.sh +++ b/scripts/test_compaction.sh @@ -37,6 +37,16 @@ if [ -n "$COMPACTION_PROVIDER" ] || [ -n "$COMPACTION_MODEL" ]; then echo "" fi +# Print the error proxy's request ledger. Without this, a TEST 3 failure gives +# no way to tell which upstream requests were made or which one received the +# injected error, leaving the request ordering to guesswork. +dump_proxy_log() { + if [ -n "$PROXY_LOG" ] && [ -s "$PROXY_LOG" ]; then + echo " Proxy request log:" + cat "$PROXY_LOG" + fi +} + # Validation function to check compaction structure in session JSON validate_compaction() { local session_id=$1 @@ -263,9 +273,19 @@ if ! (cd "$PROXY_DIR" && uv sync 2>&1 | tee "$PROXY_SETUP_LOG"); then else echo "✓ Dependencies installed" - # Start the error proxy in context-length error mode (3 errors) + # Start the error proxy in context-length error mode. + # + # Inject exactly ONE error. The proxy only injects into completion requests + # (see COMPLETION_PATHS in provider-error-proxy/proxy.py), so this error lands + # on the turn completion and the summarizer call that follows succeeds, + # producing the summary message this test validates. + # + # Do not raise this number to "give compaction more retries". A context-length + # error is deterministic for a fixed prompt, so retrying an identical request + # is not a scenario worth asserting; a higher count just consumes the + # summarizer's own calls and prevents the summary this test exists to check. echo "Starting error proxy on port $PROXY_PORT with context-length error mode..." - (cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 3" --no-stdin > "$PROXY_LOG" 2>&1) & + (cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 1" --no-stdin > "$PROXY_LOG" 2>&1) & PROXY_PID=$! # Wait for proxy to be ready (check if port is listening) @@ -300,6 +320,14 @@ else export GOOSE_PROVIDER=anthropic export GOOSE_MODEL=claude-haiku-4-5 + # Session naming runs as a background completion request spawned at the + # start of the turn, so it races the turn completion for the proxy's single + # injected error. When it wins, the naming call absorbs the error (it only + # logs a warning), the turn succeeds, and no compaction ever happens. + # Disable it so the turn completion is deterministically the first + # completion request the proxy sees. + export GOOSE_DISABLE_SESSION_NAMING=true + echo "Step 1: Creating session (should trigger context-length error and compaction)..." (cd "$TESTDIR" && "$GOOSE_BIN" run --text "hello world" 2>&1) | tee "$OUTPUT" @@ -313,19 +341,29 @@ else echo "Session created: $SESSION_ID" echo "Checking for compaction evidence..." - # Check for compaction in the output - if grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then + # The compaction failure message itself contains the word "compact", so a + # bare keyword grep matches it and reports success on a failed run. Fail + # explicitly on the error text before checking for evidence of compaction. + if grep -qi "error trying to compact" "$OUTPUT"; then + echo "✗ FAILED: Compaction was attempted but returned an error" + echo " Output:" + cat "$OUTPUT" + dump_proxy_log + RESULTS+=("✗ Out-of-Context Test Error (compaction errored)") + elif grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then echo "✓ SUCCESS: Out-of-context Test error triggered compaction" if validate_compaction "$SESSION_ID" "out-of-context error compaction"; then RESULTS+=("✓ Out-of-Context Test Error") else + dump_proxy_log RESULTS+=("✗ Out-of-Context Test Error (structure validation failed)") fi else echo "✗ FAILED: No evidence of compaction after context-length error" echo " Output:" cat "$OUTPUT" + dump_proxy_log RESULTS+=("✗ Out-of-Context Test Error") fi fi