diff --git a/CLAUDE.md b/CLAUDE.md index 67a23b0..24dbfd6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co This is the Ralph for Claude Code repository - an autonomous AI development loop system that enables continuous development cycles with intelligent exit detection and rate limiting. -**Version**: v0.11.0 | **Tests**: 420 passing (100% pass rate) | **CI/CD**: GitHub Actions +**Version**: v0.11.1 | **Tests**: 424 passing (100% pass rate) | **CI/CD**: GitHub Actions ## Core Architecture @@ -410,7 +410,7 @@ Ralph uses advanced error detection with two-stage filtering to eliminate false | `test_cli_modern.bats` | 29 | Modern CLI commands (Phase 1.1) + build_claude_command fix | | `test_json_parsing.bats` | 45 | JSON output format parsing + Claude CLI format + session management + array format | | `test_session_continuity.bats` | 28 | Session lifecycle management + circuit breaker integration + issue #91 fix | -| `test_exit_detection.bats` | 20 | Exit signal detection | +| `test_exit_detection.bats` | 35 | Exit signal detection + EXIT_SIGNAL-based completion indicators | | `test_rate_limiting.bats` | 15 | Rate limiting behavior | | `test_loop_execution.bats` | 20 | Integration tests | | `test_edge_cases.bats` | 20 | Edge case handling | @@ -436,6 +436,20 @@ bats tests/unit/test_cli_parsing.bats ## Recent Improvements +### Completion Indicators Fix (v0.11.1) +- Fixed premature exit after exactly 5 loops in JSON output mode +- Root cause: `update_exit_signals()` used confidence threshold (≥60) to populate `completion_indicators` + - JSON mode always has confidence ≥70 due to deterministic scoring (+50 for JSON format, +20 for result field) + - This caused every successful JSON response to increment `completion_indicators` + - After 5 loops, safety circuit breaker triggered even when Claude set `EXIT_SIGNAL: false` +- Fix: Replaced confidence-based heuristic with explicit EXIT_SIGNAL checking + - `completion_indicators` now only accumulates when `exit_signal == "true"` + - Aligns with documented behavior in CLAUDE.md and README.md + - Confidence scoring retained for analysis/logging purposes +- Updated safety circuit breaker documentation in `ralph_loop.sh` to reflect new behavior +- Added 4 new TDD tests (Tests 32-35) for `update_exit_signals()` behavior +- Test count: 424 (up from 420) + ### Ralph Enable Command (v0.11.0) - Added `ralph-enable` interactive wizard for enabling Ralph in existing projects - 5-phase wizard: Environment Detection → Task Source Selection → Configuration → File Generation → Verification diff --git a/lib/response_analyzer.sh b/lib/response_analyzer.sh index d3d3cf3..eda242d 100644 --- a/lib/response_analyzer.sh +++ b/lib/response_analyzer.sh @@ -560,9 +560,12 @@ update_exit_signals() { signals=$(echo "$signals" | jq ".done_signals += [$loop_number]") fi - # Update completion_indicators array (strong signals) - local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file") - if [[ $confidence -ge 60 ]]; then + # Update completion_indicators array (only when Claude explicitly signals exit) + # Note: Previously used confidence >= 60, but JSON mode always has confidence >= 70 + # due to deterministic scoring (+50 for JSON format, +20 for result field). + # This caused premature exits after 5 loops. Now we respect Claude's explicit intent. + local exit_signal=$(jq -r '.analysis.exit_signal // false' "$analysis_file") + if [[ "$exit_signal" == "true" ]]; then signals=$(echo "$signals" | jq ".completion_indicators += [$loop_number]") fi diff --git a/ralph_loop.sh b/ralph_loop.sh index 0b4442f..451b124 100755 --- a/ralph_loop.sh +++ b/ralph_loop.sh @@ -384,12 +384,14 @@ should_exit_gracefully() { return 0 fi - # 3. Safety circuit breaker - force exit after 5 consecutive completion indicators - # Bug #2 Fix: Prevents infinite loops when EXIT_SIGNAL is not explicitly set - # but completion patterns clearly indicate work is done. Threshold of 5 is higher - # than normal threshold (2) to avoid false positives while preventing API waste. + # 3. Safety circuit breaker - force exit after 5 consecutive EXIT_SIGNAL=true responses + # Note: completion_indicators only accumulates when Claude explicitly sets EXIT_SIGNAL=true + # (not based on confidence score). This safety breaker catches cases where Claude signals + # completion 5+ times but the normal exit path (completion_indicators >= 2 + EXIT_SIGNAL=true) + # didn't trigger for some reason. Threshold of 5 prevents API waste while being higher than + # the normal threshold (2) to avoid false positives. if [[ $recent_completion_indicators -ge 5 ]]; then - log_status "WARN" "🚨 SAFETY CIRCUIT BREAKER: Force exit after 5 consecutive completion indicators ($recent_completion_indicators)" >&2 + log_status "WARN" "🚨 SAFETY CIRCUIT BREAKER: Force exit after 5 consecutive EXIT_SIGNAL=true responses ($recent_completion_indicators)" >&2 echo "safety_circuit_breaker" return 0 fi diff --git a/tests/unit/test_exit_detection.bats b/tests/unit/test_exit_detection.bats index f377034..a162b03 100644 --- a/tests/unit/test_exit_detection.bats +++ b/tests/unit/test_exit_detection.bats @@ -519,3 +519,183 @@ EOF # EXIT_SIGNAL=false should take precedence, continue working assert_equal "$result" "" } + +# ============================================================================= +# UPDATE_EXIT_SIGNALS TESTS (Issue: Confidence-based completion indicators) +# ============================================================================= +# These tests verify that update_exit_signals() only adds to completion_indicators +# when EXIT_SIGNAL is true, not based on confidence score alone. +# This is critical for JSON mode where confidence is always >= 70. + +# Source the response_analyzer library for direct testing +# Note: These tests source the library to test update_exit_signals() directly + +# Test 32: update_exit_signals should NOT add to completion_indicators when exit_signal=false +@test "update_exit_signals does NOT add to completion_indicators when exit_signal=false" { + # Source the response analyzer library + source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh" + + # Initialize exit signals file + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" + + # Create analysis file with HIGH confidence (70) but exit_signal=false + # This simulates JSON mode where confidence is always >= 70 + cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' +{ + "loop_number": 1, + "timestamp": "2026-01-12T10:00:00Z", + "output_format": "json", + "analysis": { + "has_completion_signal": false, + "is_test_only": false, + "is_stuck": false, + "has_progress": true, + "files_modified": 5, + "confidence_score": 70, + "exit_signal": false, + "work_summary": "Implementing feature, still in progress" + } +} +EOF + + # Call update_exit_signals + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + + # Verify completion_indicators was NOT incremented + local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE") + assert_equal "$indicator_count" "0" +} + +# Test 33: update_exit_signals SHOULD add to completion_indicators when exit_signal=true +@test "update_exit_signals adds to completion_indicators when exit_signal=true" { + # Source the response analyzer library + source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh" + + # Initialize exit signals file + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" + + # Create analysis file with exit_signal=true + cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' +{ + "loop_number": 1, + "timestamp": "2026-01-12T10:00:00Z", + "output_format": "json", + "analysis": { + "has_completion_signal": true, + "is_test_only": false, + "is_stuck": false, + "has_progress": false, + "files_modified": 0, + "confidence_score": 100, + "exit_signal": true, + "work_summary": "All tasks complete" + } +} +EOF + + # Call update_exit_signals + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + + # Verify completion_indicators WAS incremented + local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE") + assert_equal "$indicator_count" "1" + + # Verify the loop number was recorded + local loop_recorded=$(jq '.completion_indicators[0]' "$EXIT_SIGNALS_FILE") + assert_equal "$loop_recorded" "1" +} + +# Test 34: update_exit_signals accumulates completion_indicators only on exit_signal=true +@test "update_exit_signals accumulates completion_indicators only when exit_signal=true" { + # Source the response analyzer library + source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh" + + # Initialize exit signals file + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" + + # Loop 1: exit_signal=false (should NOT add) + cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' +{ + "loop_number": 1, + "analysis": { + "has_completion_signal": false, + "is_test_only": false, + "has_progress": true, + "confidence_score": 80, + "exit_signal": false + } +} +EOF + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + + # Loop 2: exit_signal=false (should NOT add) + cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' +{ + "loop_number": 2, + "analysis": { + "has_completion_signal": true, + "is_test_only": false, + "has_progress": true, + "confidence_score": 90, + "exit_signal": false + } +} +EOF + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + + # Loop 3: exit_signal=true (SHOULD add) + cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF' +{ + "loop_number": 3, + "analysis": { + "has_completion_signal": true, + "is_test_only": false, + "has_progress": false, + "confidence_score": 100, + "exit_signal": true + } +} +EOF + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + + # Verify only 1 completion indicator (from loop 3) + local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE") + assert_equal "$indicator_count" "1" + + local loop_recorded=$(jq '.completion_indicators[0]' "$EXIT_SIGNALS_FILE") + assert_equal "$loop_recorded" "3" +} + +# Test 35: JSON mode simulation - 5 loops with exit_signal=false should NOT trigger safety breaker +@test "update_exit_signals JSON mode - 5 loops with exit_signal=false does not fill completion_indicators" { + # Source the response analyzer library + source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh" + + # Initialize exit signals file + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE" + + # Simulate 5 JSON mode loops with high confidence but exit_signal=false + # This is the exact scenario that caused the bug + for i in 1 2 3 4 5; do + cat > "$RESPONSE_ANALYSIS_FILE" << EOF +{ + "loop_number": $i, + "output_format": "json", + "analysis": { + "has_completion_signal": false, + "is_test_only": false, + "has_progress": true, + "files_modified": 3, + "confidence_score": 70, + "exit_signal": false, + "work_summary": "Working on feature $i" + } +} +EOF + update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE" + done + + # Verify completion_indicators is EMPTY (not filled with 5 indicators) + local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE") + assert_equal "$indicator_count" "0" +}