tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

fix(ci): hotfix Compaction Tests smoke test broken on main by fast-fail compaction (#11547)

Browse Source 
Signed-off-by: Michael Neale <michael.neale@gmail.com>

Merging as these tests are run on main and only with PRs from maintainers, so it can slip in.
This commit is contained in:
Michael Neale
2026-08-25 13:44:11 +10:00
committed by GitHub
parent 98dd5305b9
commit 65838e9fb7
2 changed files with 112 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/provider-error-proxy/proxy.py
+70 -2
View File
@@ -62,6 +62,42 @@ ALWAYS_FORWARD_PATHS = [
'/api/2.0/', # Databricks management API
]
# Path fragments identifying an inference/completion request.
#
# Error injection only applies to these. Every error mode the proxy offers
# describes a completion failure (context length exceeded, rate limited,
# upstream 500), so injecting them into metadata traffic such as model listing
# is meaningless. It is also actively harmful in count mode: a pre-flight
# request would silently consume the error budget intended for a completion,
# making "inject N errors" depend on goose's incidental call pattern rather
# than on the completions the test is actually exercising.
COMPLETION_PATHS = [
'/chat/completions', # OpenAI and compatible
'/completions', # OpenAI legacy
'/messages', # Anthropic
'/responses', # OpenAI Responses API
':generatecontent', # Google Gemini
':streamgeneratecontent',
'/converse', # Bedrock
'/invoke', # Bedrock / Databricks serving
'/invocations',
'/serving-endpoints', # Databricks
]
# Path suffixes identifying known non-inference (metadata) traffic. Used as
# the exclusion list when classifying requests that do not match
# COMPLETION_PATHS: a POST to an unrecognized path (e.g. a provider configured
# with a custom inference base path) is still treated as a completion, but
# known metadata endpoints are not. Matched as suffixes, not substrings, so a
# custom inference path that merely contains one of these segments (e.g.
# /v1/models/my-model/predict) is still classified as a completion.
METADATA_PATH_SUFFIXES = [
'/models', # model listing (OpenAI-compatible)
'/api/tags', # Ollama model listing
'/api/show', # Ollama model info (POSTed before completions for context limits)
'/embeddings', # embeddings are not turn completions
]
class ErrorMode(Enum):
"""Error injection modes."""
@@ -383,7 +419,33 @@ class ErrorProxy:
for forward_path in ALWAYS_FORWARD_PATHS:
if forward_path in path:
return True
return False
return not self.is_completion_request(request)
def is_completion_request(self, request: Request) -> bool:
"""
Check whether this request is an inference/completion call.
Only completion requests are eligible for error injection; see
COMPLETION_PATHS.
Args:
request: The incoming HTTP request
Returns:
True if this is a completion request
"""
path = request.path.lower()
if any(fragment in path for fragment in COMPLETION_PATHS):
return True
# Providers configured with a custom inference base path (e.g. an
# OpenAI-compatible endpoint at "<project_id>/v1") post completions to
# routes that match none of the known fragments. Treat any other POST
# that is not known metadata traffic as a completion so error
# injection still applies to custom inference routes.
if request.method.upper() != 'POST':
return False
path = path.rstrip('/')
return not any(path.endswith(suffix) for suffix in METADATA_PATH_SUFFIXES)
def get_target_url(self, request: Request, provider: str) -> str:
"""
@@ -462,7 +524,13 @@ class ErrorProxy:
# Check if this request should always be forwarded
if self.should_always_forward(request):
logger.info(f"🔄 Always forwarding: {request.path}")
if self.is_completion_request(request):
logger.info(f"🔄 Always forwarding: {request.path}")
else:
logger.info(
f"🔄 Forwarding non-completion request (not eligible for "
f"error injection): {request.path}"
)
else:
# Capture the error mode BEFORE checking if we should inject (since that modifies state)
mode_before_check = self.get_error_mode()
scripts/test_compaction.sh
+42 -4
View File
@@ -37,6 +37,16 @@ if [ -n "$COMPACTION_PROVIDER" ] || [ -n "$COMPACTION_MODEL" ]; then
echo ""
fi
# Print the error proxy's request ledger. Without this, a TEST 3 failure gives
# no way to tell which upstream requests were made or which one received the
# injected error, leaving the request ordering to guesswork.
dump_proxy_log() {
if [ -n "$PROXY_LOG" ] && [ -s "$PROXY_LOG" ]; then
echo " Proxy request log:"
cat "$PROXY_LOG"
fi
}
# Validation function to check compaction structure in session JSON
validate_compaction() {
local session_id=$1
@@ -263,9 +273,19 @@ if ! (cd "$PROXY_DIR" && uv sync 2>&1 | tee "$PROXY_SETUP_LOG"); then
else
echo "✓ Dependencies installed"
# Start the error proxy in context-length error mode (3 errors)
# Start the error proxy in context-length error mode.
#
# Inject exactly ONE error. The proxy only injects into completion requests
# (see COMPLETION_PATHS in provider-error-proxy/proxy.py), so this error lands
# on the turn completion and the summarizer call that follows succeeds,
# producing the summary message this test validates.
#
# Do not raise this number to "give compaction more retries". A context-length
# error is deterministic for a fixed prompt, so retrying an identical request
# is not a scenario worth asserting; a higher count just consumes the
# summarizer's own calls and prevents the summary this test exists to check.
echo "Starting error proxy on port $PROXY_PORT with context-length error mode..."
(cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 3" --no-stdin > "$PROXY_LOG" 2>&1) &
(cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 1" --no-stdin > "$PROXY_LOG" 2>&1) &
PROXY_PID=$!
# Wait for proxy to be ready (check if port is listening)
@@ -300,6 +320,14 @@ else
export GOOSE_PROVIDER=anthropic
export GOOSE_MODEL=claude-haiku-4-5
# Session naming runs as a background completion request spawned at the
# start of the turn, so it races the turn completion for the proxy's single
# injected error. When it wins, the naming call absorbs the error (it only
# logs a warning), the turn succeeds, and no compaction ever happens.
# Disable it so the turn completion is deterministically the first
# completion request the proxy sees.
export GOOSE_DISABLE_SESSION_NAMING=true
echo "Step 1: Creating session (should trigger context-length error and compaction)..."
(cd "$TESTDIR" && "$GOOSE_BIN" run --text "hello world" 2>&1) | tee "$OUTPUT"
@@ -313,19 +341,29 @@ else
echo "Session created: $SESSION_ID"
echo "Checking for compaction evidence..."
# Check for compaction in the output
if grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then
# The compaction failure message itself contains the word "compact", so a
# bare keyword grep matches it and reports success on a failed run. Fail
# explicitly on the error text before checking for evidence of compaction.
if grep -qi "error trying to compact" "$OUTPUT"; then
echo "✗ FAILED: Compaction was attempted but returned an error"
echo " Output:"
cat "$OUTPUT"
dump_proxy_log
RESULTS+=("✗ Out-of-Context Test Error (compaction errored)")
elif grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then
echo "✓ SUCCESS: Out-of-context Test error triggered compaction"
if validate_compaction "$SESSION_ID" "out-of-context error compaction"; then
RESULTS+=("✓ Out-of-Context Test Error")
else
dump_proxy_log
RESULTS+=("✗ Out-of-Context Test Error (structure validation failed)")
fi
else
echo "✗ FAILED: No evidence of compaction after context-length error"
echo " Output:"
cat "$OUTPUT"
dump_proxy_log
RESULTS+=("✗ Out-of-Context Test Error")
fi
fi