tkmind/tkmind_go
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Add out of context compaction test via error proxy (#5805)

Browse Source
This commit is contained in:
David Katz
2025-11-21 14:51:01 -05:00
committed by GitHub
parent 0354d9ede4
commit c1c772b267
5 changed files with 289 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-smoke-test.yml
+8
View File
@@ -143,6 +143,14 @@ jobs:
run: |
bash scripts/test_subrecipes.sh
- name: Set up Python (for error proxy)
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install uv (for error proxy)
uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # pin@v6
- name: Run Compaction Tests
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
crates/goose/src/providers/retry.rs
+11 -2
View File
@@ -109,8 +109,17 @@ pub trait ProviderRetry {
_ => config.delay_for_attempt(attempts),
};
tracing::info!("Backing off for {:?} before retry", delay);
sleep(delay).await;
let skip_backoff = std::env::var("GOOSE_PROVIDER_SKIP_BACKOFF")
.unwrap_or_default()
.parse::<bool>()
.unwrap_or(false);
if skip_backoff {
tracing::info!("Skipping backoff due to GOOSE_PROVIDER_SKIP_BACKOFF");
} else {
tracing::info!("Backing off for {:?} before retry", delay);
sleep(delay).await;
}
continue;
}
scripts/provider-error-proxy/README.md
+32
View File
@@ -64,6 +64,38 @@ Use a custom port:
uv run proxy.py --port 9000
```
Start the proxy with an initial error mode (for automated testing):
```bash
# Start with context length error (3 times)
uv run proxy.py --mode "c 3"
# Start with rate limit error (30% of requests)
uv run proxy.py --mode "r 30%"
# Start with server error (all requests)
uv run proxy.py --mode "u *"
```
Command-line options:
- `--port PORT` - Port to listen on (default: 8888)
- `--mode COMMAND` - Initial error mode command (e.g., "c 3", "r 30%", "u *", "n")
- Same syntax as interactive commands
- `--no-stdin` - Disable stdin reader (for background/automated mode)
For automated tests or background usage, combine `--no-stdin` with `--mode`:
```bash
# Run in background for automated testing
uv run proxy.py --mode "c 3" --no-stdin &
PROXY_PID=$!
# ... run your tests ...
# Stop the proxy
kill $PROXY_PID
```
### Interactive Commands
Once the proxy is running, you can control error injection interactively:
scripts/provider-error-proxy/proxy.py
+118 -68
View File
@@ -553,6 +553,73 @@ class ErrorProxy:
)
def parse_command(command: str) -> tuple[Optional[ErrorMode], int, float, Optional[str]]:
"""
Parse a command string and return the error mode, count, and percentage.
Args:
command: Command string (e.g., "c", "c 3", "r 30%", "u *")
Returns:
Tuple of (mode, count, percentage, error_message)
If error_message is not None, parsing failed
"""
# Parse command - remove all whitespace and parse
command_no_space = command.strip().replace(" ", "")
if not command_no_space:
return (None, 0, 0.0, "Empty command")
# Get the first character (error type letter)
error_letter = command_no_space[0].lower()
# Map letter to ErrorMode
mode_map = {
'n': ErrorMode.NO_ERROR,
'c': ErrorMode.CONTEXT_LENGTH,
'r': ErrorMode.RATE_LIMIT,
'u': ErrorMode.SERVER_ERROR
}
if error_letter not in mode_map:
return (None, 0, 0.0, f"Invalid command: '{error_letter}'. Use n, c, r, or u")
mode = mode_map[error_letter]
# Parse the rest as count or percentage
count = 1
percentage = 0.0
if len(command_no_space) > 1:
value_str = command_no_space[1:]
try:
# Check for * (100%)
if value_str == '*':
percentage = 1.0
count = 0 # Percentage mode
# Check for percentage with % sign (e.g., "30%")
elif value_str.endswith('%'):
percentage = float(value_str[:-1]) / 100.0
if percentage < 0.0 or percentage > 1.0:
return (None, 0, 0.0, f"Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%")
count = 0 # Percentage mode
# Check if it's a decimal (percentage as 0.0-1.0)
elif '.' in value_str:
percentage = float(value_str)
if percentage < 0.0 or percentage > 1.0:
return (None, 0, 0.0, f"Invalid percentage: {percentage}. Must be between 0.0 and 1.0")
count = 0 # Percentage mode
else:
# It's an integer count
count = int(value_str)
if count < 0:
return (None, 0, 0.0, f"Invalid count: {count}. Must be >= 0")
except ValueError:
return (None, 0, 0.0, f"Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)")
return (mode, count, percentage, None)
def print_status(proxy: ErrorProxy):
"""Print the current proxy status."""
mode, count, percentage = proxy.get_error_config()
@@ -562,7 +629,7 @@ def print_status(proxy: ErrorProxy):
ErrorMode.RATE_LIMIT: "⏱️ Rate limit exceeded",
ErrorMode.SERVER_ERROR: "💥 Server error (500)"
}
print("\n" + "=" * 60)
mode_str = mode_names.get(mode, 'Unknown')
if mode != ErrorMode.NO_ERROR:
@@ -589,79 +656,28 @@ def print_status(proxy: ErrorProxy):
def stdin_reader(proxy: ErrorProxy, loop):
"""Read commands from stdin in a separate thread."""
print_status(proxy)
while True:
try:
command = input("Enter command: ").strip()
if command.lower() == 'q':
print("\n🛑 Shutting down proxy...")
# Schedule the shutdown in the event loop
asyncio.run_coroutine_threadsafe(shutdown_server(loop), loop)
break
# Parse command - remove all whitespace and parse
command_no_space = command.replace(" ", "")
if not command_no_space:
# Parse the command using the shared parser
mode, count, percentage, error_msg = parse_command(command)
if error_msg:
print(f"❌ {error_msg}")
continue
# Get the first character (error type letter)
error_letter = command_no_space[0].lower()
# Map letter to ErrorMode
mode_map = {
'n': ErrorMode.NO_ERROR,
'c': ErrorMode.CONTEXT_LENGTH,
'r': ErrorMode.RATE_LIMIT,
'u': ErrorMode.SERVER_ERROR
}
if error_letter not in mode_map:
print(f"❌ Invalid command: '{error_letter}'. Use n, c, r, u, or q")
continue
mode = mode_map[error_letter]
# Parse the rest as count or percentage
count = 1
percentage = 0.0
if len(command_no_space) > 1:
value_str = command_no_space[1:]
try:
# Check for * (100%)
if value_str == '*':
percentage = 1.0
count = 0 # Percentage mode
# Check for percentage with % sign (e.g., "30%")
elif value_str.endswith('%'):
percentage = float(value_str[:-1]) / 100.0
if percentage < 0.0 or percentage > 1.0:
print(f"❌ Invalid percentage: {percentage*100:.0f}%. Must be between 0% and 100%")
continue
count = 0 # Percentage mode
# Check if it's a decimal (percentage as 0.0-1.0)
elif '.' in value_str:
percentage = float(value_str)
if percentage < 0.0 or percentage > 1.0:
print(f"❌ Invalid percentage: {percentage}. Must be between 0.0 and 1.0")
continue
count = 0 # Percentage mode
else:
# It's an integer count
count = int(value_str)
if count < 0:
print(f"❌ Invalid count: {count}. Must be >= 0")
continue
except ValueError:
print(f"❌ Invalid value: '{value_str}'. Must be an integer, decimal, percentage (30%), or * (100%)")
continue
# Set the error mode
proxy.set_error_mode(mode, count, percentage)
print_status(proxy)
except EOFError:
# Handle Ctrl+D
print("\n🛑 Shutting down proxy...")
@@ -716,7 +732,17 @@ def main():
default=8888,
help='Port to listen on (default: 8888)'
)
parser.add_argument(
'--mode',
type=str,
help='Error mode command (e.g., "c 3", "r 30%%", "u *", "n")'
)
parser.add_argument(
'--no-stdin',
action='store_true',
help='Disable stdin reader (for background/automated mode)'
)
args = parser.parse_args()
print("=" * 60)
@@ -735,14 +761,38 @@ def main():
# Create proxy instance
proxy = ErrorProxy()
# Set initial error mode from command-line arguments
if args.mode:
mode, count, percentage, error_msg = parse_command(args.mode)
if error_msg:
print(f"❌ Error parsing --mode argument: {error_msg}")
print(f" Example usage: --mode \"c 3\" or --mode \"r 30%\"")
return
proxy.set_error_mode(mode, count, percentage)
print()
print(f"Initial mode set from command-line arguments:")
print(f" Mode: {mode.name}")
if percentage > 0.0:
print(f" Percentage: {percentage*100:.0f}%")
elif count > 0:
print(f" Count: {count}")
print()
# Create event loop
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# Start stdin reader thread
stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True)
stdin_thread.start()
# Start stdin reader thread only if not disabled
if not args.no_stdin:
stdin_thread = threading.Thread(target=stdin_reader, args=(proxy, loop), daemon=True)
stdin_thread.start()
else:
print("Running in no-stdin mode (background/automated)")
print("Use SIGINT (Ctrl+C) or SIGTERM to stop the proxy")
print()
# Create and run the app
app = loop.run_until_complete(create_app(proxy))
scripts/test_compaction.sh
+120 -7
View File
@@ -85,13 +85,6 @@ echo "COMPACTION SMOKE TESTS"
echo "=================================================="
echo ""
# Check if jq is available
if ! command -v jq &> /dev/null; then
echo "⚠ WARNING: jq is not installed. Compaction structure validation will be limited."
echo " Install jq to enable full validation: brew install jq (macOS) or apt-get install jq (Linux)"
echo ""
fi
RESULTS=()
# ==================================================
@@ -219,6 +212,126 @@ rm -rf "$TESTDIR"
echo ""
echo ""
# ==================================================
# TEST 3: Out-of-Context Error Compaction
# ==================================================
echo "---------------------------------------------------"
echo "TEST 3: Compaction via out-of-context error (proxy)"
echo "---------------------------------------------------"
TESTDIR=$(mktemp -d)
echo "test content" > "$TESTDIR/test.txt"
echo "Test directory: $TESTDIR"
echo ""
# Use a random port to avoid conflicts
PROXY_PORT=$((9000 + RANDOM % 1000))
PROXY_DIR="$SCRIPT_DIR/scripts/provider-error-proxy"
OUTPUT=$(mktemp)
PROXY_LOG=$(mktemp)
PROXY_SETUP_LOG=$(mktemp)
# Pre-install proxy dependencies (so first run doesn't take forever)
echo "Installing proxy dependencies..."
export UV_INDEX_URL="https://pypi.org/simple"
if ! (cd "$PROXY_DIR" && uv sync 2>&1 | tee "$PROXY_SETUP_LOG"); then
echo "✗ FAILED: Could not install proxy dependencies"
echo "Setup log:"
cat "$PROXY_SETUP_LOG"
RESULTS+=("✗ Out-of-Context Error (dependency install failed)")
else
echo "✓ Dependencies installed"
# Start the error proxy in context-length error mode (3 errors)
echo "Starting error proxy on port $PROXY_PORT with context-length error mode..."
(cd "$PROXY_DIR" && UV_INDEX_URL="https://pypi.org/simple" uv run proxy.py --port "$PROXY_PORT" --mode "c 3" --no-stdin > "$PROXY_LOG" 2>&1) &
PROXY_PID=$!
# Wait for proxy to be ready (check if port is listening)
echo "Waiting for proxy to be ready..."
PROXY_READY=false
for i in {1..60}; do
if kill -0 $PROXY_PID 2>/dev/null; then
# Check if port is listening using /dev/tcp
if timeout 1 bash -c "echo -n > /dev/tcp/localhost/$PROXY_PORT" 2>/dev/null; then
PROXY_READY=true
echo "✓ Proxy is ready on port $PROXY_PORT"
break
fi
else
echo "✗ FAILED: Error proxy process died"
break
fi
sleep 0.5
done
# Check if proxy is running and ready
if [ "$PROXY_READY" != "true" ]; then
echo "✗ FAILED: Error proxy failed to become ready"
echo "Proxy log:"
cat "$PROXY_LOG"
kill $PROXY_PID 2>/dev/null || true
RESULTS+=("✗ Out-of-Context Test Error (proxy failed)")
else
# Configure provider to use proxy and skip backoff
export ANTHROPIC_HOST="http://localhost:$PROXY_PORT"
export GOOSE_PROVIDER_SKIP_BACKOFF=true
export GOOSE_PROVIDER=anthropic
export GOOSE_MODEL=claude-haiku-4-5
echo "Step 1: Creating session (should trigger context-length error and compaction)..."
(cd "$TESTDIR" && "$GOOSE_BIN" run --text "hello world" 2>&1) | tee "$OUTPUT"
SESSION_ID=$("$GOOSE_BIN" session list --format json 2>/dev/null | jq -r '.[0].id' 2>/dev/null)
if [ -z "$SESSION_ID" ] || [ "$SESSION_ID" = "null" ]; then
echo "✗ FAILED: Could not create session"
RESULTS+=("✗ Out-of-Context Test Error (no session)")
else
echo ""
echo "Session created: $SESSION_ID"
echo "Checking for compaction evidence..."
# Check for compaction in the output
if grep -qi "context.*length\|compacting\|compacted\|compaction" "$OUTPUT"; then
echo "✓ SUCCESS: Out-of-context Test error triggered compaction"
if validate_compaction "$SESSION_ID" "out-of-context error compaction"; then
RESULTS+=("✓ Out-of-Context Test Error")
else
RESULTS+=("✗ Out-of-Context Test Error (structure validation failed)")
fi
else
echo "✗ FAILED: No evidence of compaction after context-length error"
echo " Output:"
cat "$OUTPUT"
RESULTS+=("✗ Out-of-Context Test Error")
fi
fi
# Clean up
echo ""
echo "Stopping error proxy..."
# Kill the entire process group to ensure UV and Python processes are terminated
kill -- -$PROXY_PID 2>/dev/null || true
# Also explicitly kill any remaining UV processes on this port
pkill -f "uv run.*--port $PROXY_PORT" 2>/dev/null || true
wait $PROXY_PID 2>/dev/null || true
unset ANTHROPIC_HOST
unset GOOSE_PROVIDER_SKIP_BACKOFF
unset GOOSE_PROVIDER
unset GOOSE_MODEL
unset UV_INDEX_URL
fi
fi
rm -f "$OUTPUT" "$PROXY_LOG" "$PROXY_SETUP_LOG"
rm -rf "$TESTDIR"
echo ""
echo ""
# ==================================================
# Summary
# ==================================================