fix(exit-detection): use explicit EXIT_SIGNAL instead of confidence threshold (#132)
Replace confidence-based heuristic in update_exit_signals() with explicit EXIT_SIGNAL checking. JSON mode always has confidence >= 70 due to deterministic scoring, causing completion_indicators to fill after 5 loops and triggering premature exits even when Claude sets EXIT_SIGNAL: false. Changes: - lib/response_analyzer.sh: Check exit_signal == "true" instead of confidence >= 60 when updating completion_indicators array - ralph_loop.sh: Update safety circuit breaker comment to reflect that completion_indicators now only accumulates on EXIT_SIGNAL=true - tests/unit/test_exit_detection.bats: Add 4 TDD tests (32-35) validating the fix for update_exit_signals() behavior - CLAUDE.md: Document fix as v0.11.1, update test counts (420 → 424) Test count: 424 passing (100% pass rate) Co-authored-by: Test User <test@example.com>
This commit is contained in:
Frank Bria
GitHub
parent
c7e7a1c6a3
commit
f6fde6780b
4 changed files with 209 additions and 10 deletions
18
CLAUDE.md
18
CLAUDE.md
|
|
@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
|||
|
||||
This is the Ralph for Claude Code repository - an autonomous AI development loop system that enables continuous development cycles with intelligent exit detection and rate limiting.
|
||||
|
||||
**Version**: v0.11.0 | **Tests**: 420 passing (100% pass rate) | **CI/CD**: GitHub Actions
|
||||
**Version**: v0.11.1 | **Tests**: 424 passing (100% pass rate) | **CI/CD**: GitHub Actions
|
||||
|
||||
## Core Architecture
|
||||
|
||||
|
|
@ -410,7 +410,7 @@ Ralph uses advanced error detection with two-stage filtering to eliminate false
|
|||
| `test_cli_modern.bats` | 29 | Modern CLI commands (Phase 1.1) + build_claude_command fix |
|
||||
| `test_json_parsing.bats` | 45 | JSON output format parsing + Claude CLI format + session management + array format |
|
||||
| `test_session_continuity.bats` | 28 | Session lifecycle management + circuit breaker integration + issue #91 fix |
|
||||
| `test_exit_detection.bats` | 20 | Exit signal detection |
|
||||
| `test_exit_detection.bats` | 35 | Exit signal detection + EXIT_SIGNAL-based completion indicators |
|
||||
| `test_rate_limiting.bats` | 15 | Rate limiting behavior |
|
||||
| `test_loop_execution.bats` | 20 | Integration tests |
|
||||
| `test_edge_cases.bats` | 20 | Edge case handling |
|
||||
|
|
@ -436,6 +436,20 @@ bats tests/unit/test_cli_parsing.bats
|
|||
|
||||
## Recent Improvements
|
||||
|
||||
### Completion Indicators Fix (v0.11.1)
|
||||
- Fixed premature exit after exactly 5 loops in JSON output mode
|
||||
- Root cause: `update_exit_signals()` used confidence threshold (≥60) to populate `completion_indicators`
|
||||
- JSON mode always has confidence ≥70 due to deterministic scoring (+50 for JSON format, +20 for result field)
|
||||
- This caused every successful JSON response to increment `completion_indicators`
|
||||
- After 5 loops, safety circuit breaker triggered even when Claude set `EXIT_SIGNAL: false`
|
||||
- Fix: Replaced confidence-based heuristic with explicit EXIT_SIGNAL checking
|
||||
- `completion_indicators` now only accumulates when `exit_signal == "true"`
|
||||
- Aligns with documented behavior in CLAUDE.md and README.md
|
||||
- Confidence scoring retained for analysis/logging purposes
|
||||
- Updated safety circuit breaker documentation in `ralph_loop.sh` to reflect new behavior
|
||||
- Added 4 new TDD tests (Tests 32-35) for `update_exit_signals()` behavior
|
||||
- Test count: 424 (up from 420)
|
||||
|
||||
### Ralph Enable Command (v0.11.0)
|
||||
- Added `ralph-enable` interactive wizard for enabling Ralph in existing projects
|
||||
- 5-phase wizard: Environment Detection → Task Source Selection → Configuration → File Generation → Verification
|
||||
|
|
|
|||
|
|
@ -560,9 +560,12 @@ update_exit_signals() {
|
|||
signals=$(echo "$signals" | jq ".done_signals += [$loop_number]")
|
||||
fi
|
||||
|
||||
# Update completion_indicators array (strong signals)
|
||||
local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
|
||||
if [[ $confidence -ge 60 ]]; then
|
||||
# Update completion_indicators array (only when Claude explicitly signals exit)
|
||||
# Note: Previously used confidence >= 60, but JSON mode always has confidence >= 70
|
||||
# due to deterministic scoring (+50 for JSON format, +20 for result field).
|
||||
# This caused premature exits after 5 loops. Now we respect Claude's explicit intent.
|
||||
local exit_signal=$(jq -r '.analysis.exit_signal // false' "$analysis_file")
|
||||
if [[ "$exit_signal" == "true" ]]; then
|
||||
signals=$(echo "$signals" | jq ".completion_indicators += [$loop_number]")
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -384,12 +384,14 @@ should_exit_gracefully() {
|
|||
return 0
|
||||
fi
|
||||
|
||||
# 3. Safety circuit breaker - force exit after 5 consecutive completion indicators
|
||||
# Bug #2 Fix: Prevents infinite loops when EXIT_SIGNAL is not explicitly set
|
||||
# but completion patterns clearly indicate work is done. Threshold of 5 is higher
|
||||
# than normal threshold (2) to avoid false positives while preventing API waste.
|
||||
# 3. Safety circuit breaker - force exit after 5 consecutive EXIT_SIGNAL=true responses
|
||||
# Note: completion_indicators only accumulates when Claude explicitly sets EXIT_SIGNAL=true
|
||||
# (not based on confidence score). This safety breaker catches cases where Claude signals
|
||||
# completion 5+ times but the normal exit path (completion_indicators >= 2 + EXIT_SIGNAL=true)
|
||||
# didn't trigger for some reason. Threshold of 5 prevents API waste while being higher than
|
||||
# the normal threshold (2) to avoid false positives.
|
||||
if [[ $recent_completion_indicators -ge 5 ]]; then
|
||||
log_status "WARN" "🚨 SAFETY CIRCUIT BREAKER: Force exit after 5 consecutive completion indicators ($recent_completion_indicators)" >&2
|
||||
log_status "WARN" "🚨 SAFETY CIRCUIT BREAKER: Force exit after 5 consecutive EXIT_SIGNAL=true responses ($recent_completion_indicators)" >&2
|
||||
echo "safety_circuit_breaker"
|
||||
return 0
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -519,3 +519,183 @@ EOF
|
|||
# EXIT_SIGNAL=false should take precedence, continue working
|
||||
assert_equal "$result" ""
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# UPDATE_EXIT_SIGNALS TESTS (Issue: Confidence-based completion indicators)
|
||||
# =============================================================================
|
||||
# These tests verify that update_exit_signals() only adds to completion_indicators
|
||||
# when EXIT_SIGNAL is true, not based on confidence score alone.
|
||||
# This is critical for JSON mode where confidence is always >= 70.
|
||||
|
||||
# Source the response_analyzer library for direct testing
|
||||
# Note: These tests source the library to test update_exit_signals() directly
|
||||
|
||||
# Test 32: update_exit_signals should NOT add to completion_indicators when exit_signal=false
|
||||
@test "update_exit_signals does NOT add to completion_indicators when exit_signal=false" {
|
||||
# Source the response analyzer library
|
||||
source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh"
|
||||
|
||||
# Initialize exit signals file
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Create analysis file with HIGH confidence (70) but exit_signal=false
|
||||
# This simulates JSON mode where confidence is always >= 70
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"timestamp": "2026-01-12T10:00:00Z",
|
||||
"output_format": "json",
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"is_stuck": false,
|
||||
"has_progress": true,
|
||||
"files_modified": 5,
|
||||
"confidence_score": 70,
|
||||
"exit_signal": false,
|
||||
"work_summary": "Implementing feature, still in progress"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Call update_exit_signals
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Verify completion_indicators was NOT incremented
|
||||
local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$indicator_count" "0"
|
||||
}
|
||||
|
||||
# Test 33: update_exit_signals SHOULD add to completion_indicators when exit_signal=true
|
||||
@test "update_exit_signals adds to completion_indicators when exit_signal=true" {
|
||||
# Source the response analyzer library
|
||||
source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh"
|
||||
|
||||
# Initialize exit signals file
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Create analysis file with exit_signal=true
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"timestamp": "2026-01-12T10:00:00Z",
|
||||
"output_format": "json",
|
||||
"analysis": {
|
||||
"has_completion_signal": true,
|
||||
"is_test_only": false,
|
||||
"is_stuck": false,
|
||||
"has_progress": false,
|
||||
"files_modified": 0,
|
||||
"confidence_score": 100,
|
||||
"exit_signal": true,
|
||||
"work_summary": "All tasks complete"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Call update_exit_signals
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Verify completion_indicators WAS incremented
|
||||
local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$indicator_count" "1"
|
||||
|
||||
# Verify the loop number was recorded
|
||||
local loop_recorded=$(jq '.completion_indicators[0]' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$loop_recorded" "1"
|
||||
}
|
||||
|
||||
# Test 34: update_exit_signals accumulates completion_indicators only on exit_signal=true
|
||||
@test "update_exit_signals accumulates completion_indicators only when exit_signal=true" {
|
||||
# Source the response analyzer library
|
||||
source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh"
|
||||
|
||||
# Initialize exit signals file
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Loop 1: exit_signal=false (should NOT add)
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"has_progress": true,
|
||||
"confidence_score": 80,
|
||||
"exit_signal": false
|
||||
}
|
||||
}
|
||||
EOF
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Loop 2: exit_signal=false (should NOT add)
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 2,
|
||||
"analysis": {
|
||||
"has_completion_signal": true,
|
||||
"is_test_only": false,
|
||||
"has_progress": true,
|
||||
"confidence_score": 90,
|
||||
"exit_signal": false
|
||||
}
|
||||
}
|
||||
EOF
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Loop 3: exit_signal=true (SHOULD add)
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 3,
|
||||
"analysis": {
|
||||
"has_completion_signal": true,
|
||||
"is_test_only": false,
|
||||
"has_progress": false,
|
||||
"confidence_score": 100,
|
||||
"exit_signal": true
|
||||
}
|
||||
}
|
||||
EOF
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Verify only 1 completion indicator (from loop 3)
|
||||
local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$indicator_count" "1"
|
||||
|
||||
local loop_recorded=$(jq '.completion_indicators[0]' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$loop_recorded" "3"
|
||||
}
|
||||
|
||||
# Test 35: JSON mode simulation - 5 loops with exit_signal=false should NOT trigger safety breaker
|
||||
@test "update_exit_signals JSON mode - 5 loops with exit_signal=false does not fill completion_indicators" {
|
||||
# Source the response analyzer library
|
||||
source "${BATS_TEST_DIRNAME}/../../lib/response_analyzer.sh"
|
||||
|
||||
# Initialize exit signals file
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Simulate 5 JSON mode loops with high confidence but exit_signal=false
|
||||
# This is the exact scenario that caused the bug
|
||||
for i in 1 2 3 4 5; do
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << EOF
|
||||
{
|
||||
"loop_number": $i,
|
||||
"output_format": "json",
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"has_progress": true,
|
||||
"files_modified": 3,
|
||||
"confidence_score": 70,
|
||||
"exit_signal": false,
|
||||
"work_summary": "Working on feature $i"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
update_exit_signals "$RESPONSE_ANALYSIS_FILE" "$EXIT_SIGNALS_FILE"
|
||||
done
|
||||
|
||||
# Verify completion_indicators is EMPTY (not filled with 5 indicators)
|
||||
local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$indicator_count" "0"
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue