From 8fc53755bf836ed4e041edb4106467e9f42df647 Mon Sep 17 00:00:00 2001 From: frankbria Date: Wed, 31 Dec 2025 13:25:09 -0700 Subject: [PATCH] fix(circuit-breaker): eliminate JSON field false positives in error detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes circuit breaker opening prematurely due to naive error pattern matching that treated JSON field names like "is_error": false as actual errors. Changes: - ralph_loop.sh: Implement two-stage error detection with JSON filtering - lib/response_analyzer.sh: Apply same filtering to error counting - tests/test_error_detection.sh: Add comprehensive test suite (12 scenarios) Error detection now: - Filters out JSON field patterns before searching for errors - Uses context-specific patterns (^Error:, ]: error, Exception, Fatal) - Avoids type annotations (error: Error) and code identifiers - Includes debug logging when VERBOSE_PROGRESS=true Test coverage validates: ✓ JSON fields don't trigger false positives ✓ Real error messages are correctly detected ✓ Mixed content handled properly ✓ Code diffs and documentation excluded This prevents the consecutive_same_error counter from incrementing on false positives, eliminating unnecessary circuit breaker trips. --- lib/response_analyzer.sh | 7 +- ralph_loop.sh | 19 +++- tests/test_error_detection.sh | 176 ++++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100755 tests/test_error_detection.sh diff --git a/lib/response_analyzer.sh b/lib/response_analyzer.sh index 1664491..12baec6 100644 --- a/lib/response_analyzer.sh +++ b/lib/response_analyzer.sh @@ -89,7 +89,12 @@ analyze_response() { fi # 4. Detect stuck/error loops - error_count=$(grep -c -i "error\|failed\|cannot\|unable" "$output_file" 2>/dev/null | head -1 || echo "0") + # Use two-stage filtering to avoid counting JSON field names as errors + # Stage 1: Filter out JSON field patterns + # Stage 2: Count actual error messages (avoid type annotations like "error: Error") + error_count=$(grep -v '"[^"]*\(error\|failed\)"[^"]*":' "$output_file" 2>/dev/null | \ + grep -cE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL|cannot|unable to)' \ + 2>/dev/null || echo "0") error_count=$(echo "$error_count" | tr -d '[:space:]') error_count=${error_count:-0} error_count=$((error_count + 0)) diff --git a/ralph_loop.sh b/ralph_loop.sh index 1131bc1..6881366 100755 --- a/ralph_loop.sh +++ b/ralph_loop.sh @@ -386,8 +386,25 @@ EOF # Get file change count for circuit breaker local files_changed=$(git diff --name-only 2>/dev/null | wc -l || echo 0) local has_errors="false" - if grep -q "error\|Error\|ERROR" "$output_file"; then + + # Two-stage error detection to avoid JSON field false positives + # Stage 1: Filter out JSON field patterns like "is_error": false + # Stage 2: Look for actual error messages in specific contexts + # Avoid type annotations like "error: Error" by requiring lowercase after ": error" + if grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \ + grep -qE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then has_errors="true" + + # Debug logging: show what triggered error detection + if [[ "$VERBOSE_PROGRESS" == "true" ]]; then + log_status "DEBUG" "Error patterns found:" + grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \ + grep -nE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' | \ + head -3 | while IFS= read -r line; do + log_status "DEBUG" " $line" + done + fi + log_status "WARN" "Errors detected in output, check: $output_file" fi local output_length=$(wc -c < "$output_file" 2>/dev/null || echo 0) diff --git a/tests/test_error_detection.sh b/tests/test_error_detection.sh new file mode 100755 index 0000000..31935dd --- /dev/null +++ b/tests/test_error_detection.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# Test script for error detection fix +# Validates that JSON field names don't trigger false positives + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Test counter +TESTS_PASSED=0 +TESTS_FAILED=0 + +# Create temporary directory for test files +TEST_DIR=$(mktemp -d) +trap 'rm -rf "$TEST_DIR"' EXIT + +# Helper function to run tests +run_test() { + local test_name="$1" + local test_file="$2" + local expected_result="$3" # "true" or "false" + + echo -e "\n${YELLOW}Running test: $test_name${NC}" + + # Apply the error detection logic (same as in ralph_loop.sh) + local has_errors="false" + if grep -v '"[^"]*error[^"]*":' "$test_file" 2>/dev/null | \ + grep -qE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)'; then + has_errors="true" + fi + + # Check result + if [[ "$has_errors" == "$expected_result" ]]; then + echo -e "${GREEN}✓ PASS${NC} - Expected: $expected_result, Got: $has_errors" + TESTS_PASSED=$((TESTS_PASSED + 1)) + else + echo -e "${RED}✗ FAIL${NC} - Expected: $expected_result, Got: $has_errors" + echo "File contents:" + cat "$test_file" + TESTS_FAILED=$((TESTS_FAILED + 1)) + fi +} + +echo "========================================" +echo "Error Detection Test Suite" +echo "========================================" + +# Test 1: JSON with "is_error": false should NOT trigger +cat > "$TEST_DIR/test1.txt" << 'EOF' +{ + "status": "success", + "is_error": false, + "message": "Operation completed successfully" +} +EOF +run_test "JSON with is_error: false" "$TEST_DIR/test1.txt" "false" + +# Test 2: JSON with "error": null should NOT trigger +cat > "$TEST_DIR/test2.txt" << 'EOF' +{ + "status": "ok", + "error": null, + "error_count": 0, + "has_error": false +} +EOF +run_test "JSON with error: null" "$TEST_DIR/test2.txt" "false" + +# Test 3: Actual error message SHOULD trigger +cat > "$TEST_DIR/test3.txt" << 'EOF' +Running build process... +Error: Failed to compile src/main.ts + Type error on line 42 +EOF +run_test "Actual error message" "$TEST_DIR/test3.txt" "true" + +# Test 4: Exception in log SHOULD trigger +cat > "$TEST_DIR/test4.txt" << 'EOF' +[2025-12-31 10:30:00] INFO: Starting process +[2025-12-31 10:30:05] ERROR: Unhandled exception in handler +Exception: NullPointerException at line 123 +EOF +run_test "Exception in log" "$TEST_DIR/test4.txt" "true" + +# Test 5: Mixed content - JSON + real error SHOULD trigger +cat > "$TEST_DIR/test5.txt" << 'EOF' +{ + "is_error": false, + "status": "running" +} +Processing files... +Fatal: Segmentation fault in module X +EOF +run_test "Mixed JSON and real error" "$TEST_DIR/test5.txt" "true" + +# Test 6: Normal output without errors should NOT trigger +cat > "$TEST_DIR/test6.txt" << 'EOF' +Build successful +All tests passed +Deployment complete +EOF +run_test "Normal output no errors" "$TEST_DIR/test6.txt" "false" + +# Test 7: Error in context (colon after) SHOULD trigger +cat > "$TEST_DIR/test7.txt" << 'EOF' +[BUILD] Compiling... +[BUILD] Link: error: undefined reference to 'main' +[BUILD] Failed +EOF +run_test "Error with context (colon)" "$TEST_DIR/test7.txt" "true" + +# Test 8: Multiple JSON fields with "error" should NOT trigger +cat > "$TEST_DIR/test8.txt" << 'EOF' +{ + "error_message": "", + "is_error": false, + "error_code": 0, + "has_errors": false, + "error_list": [] +} +EOF +run_test "Multiple JSON error fields" "$TEST_DIR/test8.txt" "false" + +# Test 9: Case sensitivity - ERROR SHOULD trigger +cat > "$TEST_DIR/test9.txt" << 'EOF' +SYSTEM LOG: +ERROR: Database connection failed +Retrying... +EOF +run_test "Uppercase ERROR message" "$TEST_DIR/test9.txt" "true" + +# Test 10: Words "cannot" and "unable" in actual error context SHOULD trigger +cat > "$TEST_DIR/test10.txt" << 'EOF' +Build process started +Error: unable to access file system +cannot proceed with deployment +EOF +run_test "Cannot/unable in error context" "$TEST_DIR/test10.txt" "true" + +# Test 11: Documentation mentioning "error" should NOT trigger +cat > "$TEST_DIR/test11.txt" << 'EOF' +# Error Handling Guide + +This document describes how to handle errors in the application. +When an error occurs, the system will log it and continue. +EOF +run_test "Documentation about errors" "$TEST_DIR/test11.txt" "false" + +# Test 12: Git diff with error keywords should NOT trigger +cat > "$TEST_DIR/test12.txt" << 'EOF' +diff --git a/src/error.ts b/src/error.ts ++export class ErrorHandler { ++ handleError(error: Error) { ++ console.log(error); +EOF +run_test "Git diff with error class" "$TEST_DIR/test12.txt" "false" + +# Print summary +echo "" +echo "========================================" +echo "Test Summary" +echo "========================================" +echo -e "${GREEN}Passed: $TESTS_PASSED${NC}" +echo -e "${RED}Failed: $TESTS_FAILED${NC}" +echo "========================================" + +if [[ $TESTS_FAILED -gt 0 ]]; then + exit 1 +else + echo -e "${GREEN}All tests passed!${NC}" + exit 0 +fi