fix(circuit-breaker): eliminate JSON field false positives in error detection
Fixes circuit breaker opening prematurely due to naive error pattern matching that treated JSON field names like "is_error": false as actual errors. Changes: - ralph_loop.sh: Implement two-stage error detection with JSON filtering - lib/response_analyzer.sh: Apply same filtering to error counting - tests/test_error_detection.sh: Add comprehensive test suite (12 scenarios) Error detection now: - Filters out JSON field patterns before searching for errors - Uses context-specific patterns (^Error:, ]: error, Exception, Fatal) - Avoids type annotations (error: Error) and code identifiers - Includes debug logging when VERBOSE_PROGRESS=true Test coverage validates: ✓ JSON fields don't trigger false positives ✓ Real error messages are correctly detected ✓ Mixed content handled properly ✓ Code diffs and documentation excluded This prevents the consecutive_same_error counter from incrementing on false positives, eliminating unnecessary circuit breaker trips.
This commit is contained in:
frankbria
parent
e370c1ec2e
commit
8fc53755bf
3 changed files with 200 additions and 2 deletions
|
|
@ -89,7 +89,12 @@ analyze_response() {
|
|||
fi
|
||||
|
||||
# 4. Detect stuck/error loops
|
||||
error_count=$(grep -c -i "error\|failed\|cannot\|unable" "$output_file" 2>/dev/null | head -1 || echo "0")
|
||||
# Use two-stage filtering to avoid counting JSON field names as errors
|
||||
# Stage 1: Filter out JSON field patterns
|
||||
# Stage 2: Count actual error messages (avoid type annotations like "error: Error")
|
||||
error_count=$(grep -v '"[^"]*\(error\|failed\)"[^"]*":' "$output_file" 2>/dev/null | \
|
||||
grep -cE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL|cannot|unable to)' \
|
||||
2>/dev/null || echo "0")
|
||||
error_count=$(echo "$error_count" | tr -d '[:space:]')
|
||||
error_count=${error_count:-0}
|
||||
error_count=$((error_count + 0))
|
||||
|
|
|
|||
|
|
@ -386,8 +386,25 @@ EOF
|
|||
# Get file change count for circuit breaker
|
||||
local files_changed=$(git diff --name-only 2>/dev/null | wc -l || echo 0)
|
||||
local has_errors="false"
|
||||
if grep -q "error\|Error\|ERROR" "$output_file"; then
|
||||
|
||||
# Two-stage error detection to avoid JSON field false positives
|
||||
# Stage 1: Filter out JSON field patterns like "is_error": false
|
||||
# Stage 2: Look for actual error messages in specific contexts
|
||||
# Avoid type annotations like "error: Error" by requiring lowercase after ": error"
|
||||
if grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
|
||||
grep -qE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then
|
||||
has_errors="true"
|
||||
|
||||
# Debug logging: show what triggered error detection
|
||||
if [[ "$VERBOSE_PROGRESS" == "true" ]]; then
|
||||
log_status "DEBUG" "Error patterns found:"
|
||||
grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
|
||||
grep -nE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' | \
|
||||
head -3 | while IFS= read -r line; do
|
||||
log_status "DEBUG" " $line"
|
||||
done
|
||||
fi
|
||||
|
||||
log_status "WARN" "Errors detected in output, check: $output_file"
|
||||
fi
|
||||
local output_length=$(wc -c < "$output_file" 2>/dev/null || echo 0)
|
||||
|
|
|
|||
176
tests/test_error_detection.sh
Executable file
176
tests/test_error_detection.sh
Executable file
|
|
@ -0,0 +1,176 @@
|
|||
#!/bin/bash
|
||||
# Test script for error detection fix
|
||||
# Validates that JSON field names don't trigger false positives
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Test counter
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
|
||||
# Create temporary directory for test files
|
||||
TEST_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$TEST_DIR"' EXIT
|
||||
|
||||
# Helper function to run tests
|
||||
run_test() {
|
||||
local test_name="$1"
|
||||
local test_file="$2"
|
||||
local expected_result="$3" # "true" or "false"
|
||||
|
||||
echo -e "\n${YELLOW}Running test: $test_name${NC}"
|
||||
|
||||
# Apply the error detection logic (same as in ralph_loop.sh)
|
||||
local has_errors="false"
|
||||
if grep -v '"[^"]*error[^"]*":' "$test_file" 2>/dev/null | \
|
||||
grep -qE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then
|
||||
has_errors="true"
|
||||
fi
|
||||
|
||||
# Check result
|
||||
if [[ "$has_errors" == "$expected_result" ]]; then
|
||||
echo -e "${GREEN}✓ PASS${NC} - Expected: $expected_result, Got: $has_errors"
|
||||
TESTS_PASSED=$((TESTS_PASSED + 1))
|
||||
else
|
||||
echo -e "${RED}✗ FAIL${NC} - Expected: $expected_result, Got: $has_errors"
|
||||
echo "File contents:"
|
||||
cat "$test_file"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
echo "========================================"
|
||||
echo "Error Detection Test Suite"
|
||||
echo "========================================"
|
||||
|
||||
# Test 1: JSON with "is_error": false should NOT trigger
|
||||
cat > "$TEST_DIR/test1.txt" << 'EOF'
|
||||
{
|
||||
"status": "success",
|
||||
"is_error": false,
|
||||
"message": "Operation completed successfully"
|
||||
}
|
||||
EOF
|
||||
run_test "JSON with is_error: false" "$TEST_DIR/test1.txt" "false"
|
||||
|
||||
# Test 2: JSON with "error": null should NOT trigger
|
||||
cat > "$TEST_DIR/test2.txt" << 'EOF'
|
||||
{
|
||||
"status": "ok",
|
||||
"error": null,
|
||||
"error_count": 0,
|
||||
"has_error": false
|
||||
}
|
||||
EOF
|
||||
run_test "JSON with error: null" "$TEST_DIR/test2.txt" "false"
|
||||
|
||||
# Test 3: Actual error message SHOULD trigger
|
||||
cat > "$TEST_DIR/test3.txt" << 'EOF'
|
||||
Running build process...
|
||||
Error: Failed to compile src/main.ts
|
||||
Type error on line 42
|
||||
EOF
|
||||
run_test "Actual error message" "$TEST_DIR/test3.txt" "true"
|
||||
|
||||
# Test 4: Exception in log SHOULD trigger
|
||||
cat > "$TEST_DIR/test4.txt" << 'EOF'
|
||||
[2025-12-31 10:30:00] INFO: Starting process
|
||||
[2025-12-31 10:30:05] ERROR: Unhandled exception in handler
|
||||
Exception: NullPointerException at line 123
|
||||
EOF
|
||||
run_test "Exception in log" "$TEST_DIR/test4.txt" "true"
|
||||
|
||||
# Test 5: Mixed content - JSON + real error SHOULD trigger
|
||||
cat > "$TEST_DIR/test5.txt" << 'EOF'
|
||||
{
|
||||
"is_error": false,
|
||||
"status": "running"
|
||||
}
|
||||
Processing files...
|
||||
Fatal: Segmentation fault in module X
|
||||
EOF
|
||||
run_test "Mixed JSON and real error" "$TEST_DIR/test5.txt" "true"
|
||||
|
||||
# Test 6: Normal output without errors should NOT trigger
|
||||
cat > "$TEST_DIR/test6.txt" << 'EOF'
|
||||
Build successful
|
||||
All tests passed
|
||||
Deployment complete
|
||||
EOF
|
||||
run_test "Normal output no errors" "$TEST_DIR/test6.txt" "false"
|
||||
|
||||
# Test 7: Error in context (colon after) SHOULD trigger
|
||||
cat > "$TEST_DIR/test7.txt" << 'EOF'
|
||||
[BUILD] Compiling...
|
||||
[BUILD] Link: error: undefined reference to 'main'
|
||||
[BUILD] Failed
|
||||
EOF
|
||||
run_test "Error with context (colon)" "$TEST_DIR/test7.txt" "true"
|
||||
|
||||
# Test 8: Multiple JSON fields with "error" should NOT trigger
|
||||
cat > "$TEST_DIR/test8.txt" << 'EOF'
|
||||
{
|
||||
"error_message": "",
|
||||
"is_error": false,
|
||||
"error_code": 0,
|
||||
"has_errors": false,
|
||||
"error_list": []
|
||||
}
|
||||
EOF
|
||||
run_test "Multiple JSON error fields" "$TEST_DIR/test8.txt" "false"
|
||||
|
||||
# Test 9: Case sensitivity - ERROR SHOULD trigger
|
||||
cat > "$TEST_DIR/test9.txt" << 'EOF'
|
||||
SYSTEM LOG:
|
||||
ERROR: Database connection failed
|
||||
Retrying...
|
||||
EOF
|
||||
run_test "Uppercase ERROR message" "$TEST_DIR/test9.txt" "true"
|
||||
|
||||
# Test 10: Words "cannot" and "unable" in actual error context SHOULD trigger
|
||||
cat > "$TEST_DIR/test10.txt" << 'EOF'
|
||||
Build process started
|
||||
Error: unable to access file system
|
||||
cannot proceed with deployment
|
||||
EOF
|
||||
run_test "Cannot/unable in error context" "$TEST_DIR/test10.txt" "true"
|
||||
|
||||
# Test 11: Documentation mentioning "error" should NOT trigger
|
||||
cat > "$TEST_DIR/test11.txt" << 'EOF'
|
||||
# Error Handling Guide
|
||||
|
||||
This document describes how to handle errors in the application.
|
||||
When an error occurs, the system will log it and continue.
|
||||
EOF
|
||||
run_test "Documentation about errors" "$TEST_DIR/test11.txt" "false"
|
||||
|
||||
# Test 12: Git diff with error keywords should NOT trigger
|
||||
cat > "$TEST_DIR/test12.txt" << 'EOF'
|
||||
diff --git a/src/error.ts b/src/error.ts
|
||||
+export class ErrorHandler {
|
||||
+ handleError(error: Error) {
|
||||
+ console.log(error);
|
||||
EOF
|
||||
run_test "Git diff with error class" "$TEST_DIR/test12.txt" "false"
|
||||
|
||||
# Print summary
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "Test Summary"
|
||||
echo "========================================"
|
||||
echo -e "${GREEN}Passed: $TESTS_PASSED${NC}"
|
||||
echo -e "${RED}Failed: $TESTS_FAILED${NC}"
|
||||
echo "========================================"
|
||||
|
||||
if [[ $TESTS_FAILED -gt 0 ]]; then
|
||||
exit 1
|
||||
else
|
||||
echo -e "${GREEN}All tests passed!${NC}"
|
||||
exit 0
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue