From b4b9db6b76cd229b2f6c1e9b832e33cf162b4ab9 Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 7 Feb 2026 01:33:41 -0700 Subject: [PATCH] feat(circuit-breaker): add auto-recovery from OPEN state (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OPEN state was terminal — once triggered, it persisted across restarts with no automatic recovery. This adds two recovery mechanisms: 1. Cooldown timer (default): OPEN → HALF_OPEN after CB_COOLDOWN_MINUTES (default 30). The existing HALF_OPEN logic handles recovery or re-trip. 2. Auto-reset option: CB_AUTO_RESET=true bypasses cooldown, resets to CLOSED on startup for fully unattended operation. Changes: - Add parse_iso_to_epoch() to lib/date_utils.sh (cross-platform) - Add cooldown + auto-reset logic to init_circuit_breaker() - Add opened_at field to state file when entering/staying OPEN - Add --auto-reset-circuit CLI flag and .ralphrc config vars - Add 19 tests in test_circuit_breaker_recovery.bats - Update CLAUDE.md and README.md documentation --- CLAUDE.md | 23 +- README.md | 20 +- lib/circuit_breaker.sh | 72 ++- lib/date_utils.sh | 51 ++ ralph_loop.sh | 13 + templates/ralphrc.template | 7 + tests/unit/test_circuit_breaker_recovery.bats | 446 ++++++++++++++++++ 7 files changed, 626 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_circuit_breaker_recovery.bats diff --git a/CLAUDE.md b/CLAUDE.md index 5f73a4e..ab46445 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -59,6 +59,7 @@ The system uses a modular architecture with reusable components in the `lib/` di 3. **lib/date_utils.sh** - Cross-platform date utilities - ISO timestamp generation for logging - Epoch time calculations for rate limiting + - ISO-to-epoch conversion for cooldown timer comparisons (`parse_iso_to_epoch()`) 4. **lib/timeout_utils.sh** - Cross-platform timeout command utilities - Detects and uses appropriate timeout command for the platform @@ -147,6 +148,7 @@ ralph --status # Circuit breaker management ralph --reset-circuit ralph --circuit-status +ralph --auto-reset-circuit # Auto-reset OPEN state on startup # Session management ralph --reset-session # Reset session state manually @@ -380,6 +382,24 @@ fi - `CB_OUTPUT_DECLINE_THRESHOLD=70%` - Open circuit if output declines by >70% - `CB_PERMISSION_DENIAL_THRESHOLD=2` - Open circuit after 2 loops with permission denials (Issue #101) +### Circuit Breaker Auto-Recovery (Issue #160) + +The OPEN state is no longer terminal. Two recovery mechanisms are available: + +**Cooldown Timer (default):** After `CB_COOLDOWN_MINUTES` (default: 30) in OPEN state, the circuit transitions to HALF_OPEN on next `init_circuit_breaker()` call. The existing HALF_OPEN logic handles recovery (progress → CLOSED) or re-trip (no progress → OPEN). + +**Auto-Reset:** When `CB_AUTO_RESET=true`, the circuit resets directly to CLOSED on startup, bypassing the cooldown. Use for fully unattended operation. + +**Configuration:** +```bash +CB_COOLDOWN_MINUTES=30 # Minutes before OPEN → HALF_OPEN (0 = immediate) +CB_AUTO_RESET=false # true = bypass cooldown, reset to CLOSED on startup +``` + +**CLI flag:** `ralph --auto-reset-circuit` sets `CB_AUTO_RESET=true` for a single run. + +**State file:** The `opened_at` field tracks when the circuit entered OPEN state. Old state files without this field fall back to `last_change` for backward compatibility. + ### Permission Denial Detection (Issue #101) When Claude Code is denied permission to execute commands (e.g., `npm install`), Ralph detects this from the `permission_denials` array in the JSON output and halts the loop immediately: @@ -424,10 +444,11 @@ Ralph uses advanced error detection with two-stage filtering to eliminate false ## Test Suite -### Test Files (420 tests total) +### Test Files (484 tests total) | File | Tests | Description | |------|-------|-------------| +| `test_circuit_breaker_recovery.bats` | 19 | Cooldown timer, auto-reset, parse_iso_to_epoch, CLI flag (Issue #160) | | `test_cli_parsing.bats` | 27 | CLI argument parsing for all 12 flags | | `test_cli_modern.bats` | 29 | Modern CLI commands (Phase 1.1) + build_claude_command fix | | `test_json_parsing.bats` | 45 | JSON output format parsing + Claude CLI format + session management + array format | diff --git a/README.md b/README.md index 4f30a24..b5ade65 100644 --- a/README.md +++ b/README.md @@ -431,8 +431,19 @@ The circuit breaker automatically: - Eliminates false positives from JSON fields containing "error" - Accurately detects stuck loops with multi-line error matching - Gradually recovers with half-open monitoring state +- **Auto-recovers** after cooldown period (default: 30 minutes) — OPEN → HALF_OPEN → CLOSED - Provides detailed error tracking and logging with state history +**Auto-recovery options:** +```bash +# Default: 30-minute cooldown before auto-recovery attempt +CB_COOLDOWN_MINUTES=30 # Set in .ralphrc (0 = immediate) + +# Auto-reset on startup (for fully unattended operation) +ralph --auto-reset-circuit +# Or set in .ralphrc: CB_AUTO_RESET=true +``` + ### Claude API 5-Hour Limit When Claude's 5-hour usage limit is reached, Ralph: @@ -534,6 +545,8 @@ TEST_PERCENTAGE_THRESHOLD=30 # Flag if 30%+ loops are test-only CB_NO_PROGRESS_THRESHOLD=3 # Open circuit after 3 loops with no file changes CB_SAME_ERROR_THRESHOLD=5 # Open circuit after 5 loops with repeated errors CB_OUTPUT_DECLINE_THRESHOLD=70 # Open circuit if output declines by >70% +CB_COOLDOWN_MINUTES=30 # Minutes before OPEN → HALF_OPEN auto-recovery +CB_AUTO_RESET=false # true = reset to CLOSED on startup (bypasses cooldown) ``` **Completion Indicators with EXIT_SIGNAL Gate:** @@ -636,8 +649,8 @@ bats tests/integration/test_installation.bats ``` Current test status: -- **465 tests** across 15 test files -- **100% pass rate** (452/452 passing) +- **484 tests** across 16 test files +- **100% pass rate** (484/484 passing) - Comprehensive unit and integration tests - Specialized tests for JSON parsing, CLI flags, circuit breaker, EXIT_SIGNAL behavior, enable wizard, and installation workflows @@ -799,6 +812,7 @@ ralph [OPTIONS] --no-continue Disable session continuity (start fresh each loop) --reset-circuit Reset the circuit breaker --circuit-status Show circuit breaker status + --auto-reset-circuit Auto-reset circuit breaker on startup (bypasses cooldown) --reset-session Reset session state manually ``` @@ -838,7 +852,7 @@ Ralph is under active development with a clear path to v1.0.0. See [IMPLEMENTATI - **Dual-condition exit gate** (completion indicators + EXIT_SIGNAL) - Rate limiting (100 calls/hour) and circuit breaker pattern - Response analyzer with semantic understanding -- **452 comprehensive tests** (100% pass rate) +- **484 comprehensive tests** (100% pass rate) - **Live streaming output mode** for real-time Claude Code visibility - tmux integration and live monitoring - PRD import functionality with modern CLI JSON parsing diff --git a/lib/circuit_breaker.sh b/lib/circuit_breaker.sh index 810a17c..3ca9bea 100644 --- a/lib/circuit_breaker.sh +++ b/lib/circuit_breaker.sh @@ -22,6 +22,8 @@ CB_NO_PROGRESS_THRESHOLD=${CB_NO_PROGRESS_THRESHOLD:-3} # Open circuit af CB_SAME_ERROR_THRESHOLD=${CB_SAME_ERROR_THRESHOLD:-5} # Open circuit after N loops with same error CB_OUTPUT_DECLINE_THRESHOLD=${CB_OUTPUT_DECLINE_THRESHOLD:-70} # Open circuit if output declines by >70% CB_PERMISSION_DENIAL_THRESHOLD=${CB_PERMISSION_DENIAL_THRESHOLD:-2} # Open circuit after N loops with permission denials (Issue #101) +CB_COOLDOWN_MINUTES=${CB_COOLDOWN_MINUTES:-30} # Minutes before OPEN → HALF_OPEN auto-recovery (Issue #160) +CB_AUTO_RESET=${CB_AUTO_RESET:-false} # Reset to CLOSED on startup instead of waiting for cooldown # Colors RED='\033[0;31m' @@ -55,6 +57,61 @@ init_circuit_breaker() { EOF fi + # Auto-recovery: check if OPEN state should transition (Issue #160) + local current_state + current_state=$(jq -r '.state' "$CB_STATE_FILE" 2>/dev/null || echo "$CB_STATE_CLOSED") + + if [[ "$current_state" == "$CB_STATE_OPEN" ]]; then + if [[ "$CB_AUTO_RESET" == "true" ]]; then + # Auto-reset: bypass cooldown, go straight to CLOSED + local current_loop total_opens + current_loop=$(jq -r '.current_loop // 0' "$CB_STATE_FILE" 2>/dev/null || echo "0") + total_opens=$(jq -r '.total_opens // 0' "$CB_STATE_FILE" 2>/dev/null || echo "0") + log_circuit_transition "$CB_STATE_OPEN" "$CB_STATE_CLOSED" "Auto-reset on startup (CB_AUTO_RESET=true)" "$current_loop" + + cat > "$CB_STATE_FILE" << EOF +{ + "state": "$CB_STATE_CLOSED", + "last_change": "$(get_iso_timestamp)", + "consecutive_no_progress": 0, + "consecutive_same_error": 0, + "consecutive_permission_denials": 0, + "last_progress_loop": 0, + "total_opens": $total_opens, + "reason": "Auto-reset on startup" +} +EOF + else + # Cooldown: check if enough time has elapsed to transition to HALF_OPEN + local opened_at + opened_at=$(jq -r '.opened_at // .last_change // ""' "$CB_STATE_FILE" 2>/dev/null || echo "") + + if [[ -n "$opened_at" && "$opened_at" != "null" ]]; then + local opened_epoch current_epoch elapsed_minutes + opened_epoch=$(parse_iso_to_epoch "$opened_at") + current_epoch=$(date +%s) + elapsed_minutes=$(( (current_epoch - opened_epoch) / 60 )) + + if [[ $elapsed_minutes -ge 0 && $elapsed_minutes -ge $CB_COOLDOWN_MINUTES ]]; then + local current_loop + current_loop=$(jq -r '.current_loop // 0' "$CB_STATE_FILE" 2>/dev/null || echo "0") + log_circuit_transition "$CB_STATE_OPEN" "$CB_STATE_HALF_OPEN" "Cooldown elapsed (${elapsed_minutes}m >= ${CB_COOLDOWN_MINUTES}m)" "$current_loop" + + # Preserve counters but transition state + local state_data + state_data=$(cat "$CB_STATE_FILE") + echo "$state_data" | jq \ + --arg state "$CB_STATE_HALF_OPEN" \ + --arg last_change "$(get_iso_timestamp)" \ + --arg reason "Cooldown recovery: ${elapsed_minutes}m elapsed" \ + '.state = $state | .last_change = $last_change | .reason = $reason' \ + > "$CB_STATE_FILE" + fi + # If elapsed_minutes < 0 (clock skew), stay OPEN safely + fi + fi + fi + # Check if history file exists and is valid JSON if [[ -f "$CB_HISTORY_FILE" ]]; then if ! jq '.' "$CB_HISTORY_FILE" > /dev/null 2>&1; then @@ -217,7 +274,7 @@ record_loop_result() { ;; "$CB_STATE_OPEN") - # Circuit is open - stays open (manual intervention required) + # Circuit is open - stays open (auto-recovery handled in init_circuit_breaker) reason="Circuit breaker is open, execution halted" ;; esac @@ -229,6 +286,16 @@ record_loop_result() { total_opens=$((total_opens + 1)) fi + # Determine opened_at: set when entering OPEN, preserve when staying OPEN + local opened_at="" + if [[ "$new_state" == "$CB_STATE_OPEN" && "$current_state" != "$CB_STATE_OPEN" ]]; then + # Entering OPEN state - record the timestamp + opened_at=$(get_iso_timestamp) + elif [[ "$new_state" == "$CB_STATE_OPEN" && "$current_state" == "$CB_STATE_OPEN" ]]; then + # Staying OPEN - preserve existing opened_at (fall back to last_change for old state files) + opened_at=$(echo "$state_data" | jq -r '.opened_at // .last_change // ""' 2>/dev/null) + fi + cat > "$CB_STATE_FILE" << EOF { "state": "$new_state", @@ -239,7 +306,8 @@ record_loop_result() { "last_progress_loop": $last_progress_loop, "total_opens": $total_opens, "reason": "$reason", - "current_loop": $loop_number + "current_loop": $loop_number$(if [[ -n "$opened_at" ]]; then echo ", + \"opened_at\": \"$opened_at\""; fi) } EOF diff --git a/lib/date_utils.sh b/lib/date_utils.sh index b889548..ddd23fc 100644 --- a/lib/date_utils.sh +++ b/lib/date_utils.sh @@ -46,8 +46,59 @@ get_epoch_seconds() { date +%s } +# Convert ISO 8601 timestamp to Unix epoch seconds +# Input: ISO timestamp (e.g., "2025-01-15T10:30:00+00:00") +# Returns: Unix epoch seconds on stdout +# Falls back to current epoch on parse failure (safe default) +parse_iso_to_epoch() { + local iso_timestamp=$1 + + if [[ -z "$iso_timestamp" || "$iso_timestamp" == "null" ]]; then + date +%s + return + fi + + # Try GNU date -d (Linux, macOS with Homebrew coreutils) + local result + if result=$(date -d "$iso_timestamp" +%s 2>/dev/null) && [[ "$result" =~ ^[0-9]+$ ]]; then + echo "$result" + return + fi + + # Try BSD date -j (native macOS) + # Strip timezone suffix for BSD compatibility + local stripped="${iso_timestamp%%+*}" + stripped="${stripped%%Z*}" + if result=$(date -j -f "%Y-%m-%dT%H:%M:%S" "$stripped" +%s 2>/dev/null) && [[ "$result" =~ ^[0-9]+$ ]]; then + echo "$result" + return + fi + + # Fallback: manual epoch arithmetic from ISO components + # Parse: YYYY-MM-DDTHH:MM:SS (ignore timezone, assume UTC) + local year month day hour minute second + if [[ "$iso_timestamp" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2}) ]]; then + year="${BASH_REMATCH[1]}" + month="${BASH_REMATCH[2]}" + day="${BASH_REMATCH[3]}" + hour="${BASH_REMATCH[4]}" + minute="${BASH_REMATCH[5]}" + second="${BASH_REMATCH[6]}" + + # Use date with explicit components if available + if result=$(date -u -d "${year}-${month}-${day} ${hour}:${minute}:${second}" +%s 2>/dev/null) && [[ "$result" =~ ^[0-9]+$ ]]; then + echo "$result" + return + fi + fi + + # Ultimate fallback: return current epoch (safe default) + date +%s +} + # Export functions for use in other scripts export -f get_iso_timestamp export -f get_next_hour_time export -f get_basic_timestamp export -f get_epoch_seconds +export -f parse_iso_to_epoch diff --git a/ralph_loop.sh b/ralph_loop.sh index d39c209..18c1df6 100755 --- a/ralph_loop.sh +++ b/ralph_loop.sh @@ -42,6 +42,8 @@ _env_CLAUDE_ALLOWED_TOOLS="${CLAUDE_ALLOWED_TOOLS:-}" _env_CLAUDE_USE_CONTINUE="${CLAUDE_USE_CONTINUE:-}" _env_CLAUDE_SESSION_EXPIRY_HOURS="${CLAUDE_SESSION_EXPIRY_HOURS:-}" _env_VERBOSE_PROGRESS="${VERBOSE_PROGRESS:-}" +_env_CB_COOLDOWN_MINUTES="${CB_COOLDOWN_MINUTES:-}" +_env_CB_AUTO_RESET="${CB_AUTO_RESET:-}" # Now set defaults (only if not already set by environment) MAX_CALLS_PER_HOUR="${MAX_CALLS_PER_HOUR:-100}" @@ -146,6 +148,8 @@ load_ralphrc() { [[ -n "$_env_CLAUDE_USE_CONTINUE" ]] && CLAUDE_USE_CONTINUE="$_env_CLAUDE_USE_CONTINUE" [[ -n "$_env_CLAUDE_SESSION_EXPIRY_HOURS" ]] && CLAUDE_SESSION_EXPIRY_HOURS="$_env_CLAUDE_SESSION_EXPIRY_HOURS" [[ -n "$_env_VERBOSE_PROGRESS" ]] && VERBOSE_PROGRESS="$_env_VERBOSE_PROGRESS" + [[ -n "$_env_CB_COOLDOWN_MINUTES" ]] && CB_COOLDOWN_MINUTES="$_env_CB_COOLDOWN_MINUTES" + [[ -n "$_env_CB_AUTO_RESET" ]] && CB_AUTO_RESET="$_env_CB_AUTO_RESET" RALPHRC_LOADED=true return 0 @@ -261,6 +265,10 @@ setup_tmux_session() { if [[ "$CLAUDE_SESSION_EXPIRY_HOURS" != "24" ]]; then ralph_cmd="$ralph_cmd --session-expiry $CLAUDE_SESSION_EXPIRY_HOURS" fi + # Forward --auto-reset-circuit if enabled + if [[ "$CB_AUTO_RESET" == "true" ]]; then + ralph_cmd="$ralph_cmd --auto-reset-circuit" + fi tmux send-keys -t "$session_name:${base_win}.0" "$ralph_cmd" Enter @@ -1609,6 +1617,7 @@ Options: -t, --timeout MIN Set Claude Code execution timeout in minutes (default: $CLAUDE_TIMEOUT_MINUTES) --reset-circuit Reset circuit breaker to CLOSED state --circuit-status Show circuit breaker status and exit + --auto-reset-circuit Auto-reset circuit breaker on startup (bypasses cooldown) --reset-session Reset session state and exit (clears session continuity) Modern CLI Options (Phase 1.1): @@ -1742,6 +1751,10 @@ while [[ $# -gt 0 ]]; do CLAUDE_SESSION_EXPIRY_HOURS="$2" shift 2 ;; + --auto-reset-circuit) + CB_AUTO_RESET=true + shift + ;; *) echo "Unknown option: $1" show_help diff --git a/templates/ralphrc.template b/templates/ralphrc.template index bd3e0ba..026be86 100644 --- a/templates/ralphrc.template +++ b/templates/ralphrc.template @@ -75,6 +75,13 @@ CB_SAME_ERROR_THRESHOLD=5 # Open circuit if output declines by more than N percent CB_OUTPUT_DECLINE_THRESHOLD=70 +# Auto-recovery: cooldown before retry (minutes, 0 = immediate) +CB_COOLDOWN_MINUTES=30 + +# Auto-reset circuit breaker on startup (bypasses cooldown) +# WARNING: Reduces circuit breaker safety for unattended operation +CB_AUTO_RESET=false + # ============================================================================= # ADVANCED SETTINGS # ============================================================================= diff --git a/tests/unit/test_circuit_breaker_recovery.bats b/tests/unit/test_circuit_breaker_recovery.bats new file mode 100644 index 0000000..f3c69e5 --- /dev/null +++ b/tests/unit/test_circuit_breaker_recovery.bats @@ -0,0 +1,446 @@ +#!/usr/bin/env bats +# Unit Tests for Circuit Breaker Auto-Recovery (Issue #160) +# Tests cooldown timer, auto-reset, and parse_iso_to_epoch + +load '../helpers/test_helper' + +SCRIPT_DIR="${BATS_TEST_DIRNAME}/../../lib" + +setup() { + # Create temp test directory + export TEST_TEMP_DIR="$(mktemp -d /tmp/ralph-cb-recovery.XXXXXX)" + cd "$TEST_TEMP_DIR" + + export RALPH_DIR=".ralph" + export CB_STATE_FILE="$RALPH_DIR/.circuit_breaker_state" + export CB_HISTORY_FILE="$RALPH_DIR/.circuit_breaker_history" + export RESPONSE_ANALYSIS_FILE="$RALPH_DIR/.response_analysis" + mkdir -p "$RALPH_DIR" + + # Source the actual library files + source "$SCRIPT_DIR/date_utils.sh" + source "$SCRIPT_DIR/circuit_breaker.sh" +} + +teardown() { + cd / + rm -rf "$TEST_TEMP_DIR" +} + +# Helper: Create an OPEN state file with a specific opened_at timestamp +create_open_state() { + local opened_at="${1:-$(get_iso_timestamp)}" + local total_opens="${2:-1}" + cat > "$CB_STATE_FILE" << EOF +{ + "state": "OPEN", + "last_change": "$(get_iso_timestamp)", + "consecutive_no_progress": 5, + "consecutive_same_error": 0, + "consecutive_permission_denials": 0, + "last_progress_loop": 2, + "total_opens": $total_opens, + "reason": "No progress detected in 5 consecutive loops", + "current_loop": 7, + "opened_at": "$opened_at" +} +EOF + echo '[]' > "$CB_HISTORY_FILE" +} + +# Helper: Create an OPEN state file WITHOUT opened_at (old format) +create_old_format_open_state() { + local last_change="${1:-$(get_iso_timestamp)}" + cat > "$CB_STATE_FILE" << EOF +{ + "state": "OPEN", + "last_change": "$last_change", + "consecutive_no_progress": 5, + "consecutive_same_error": 0, + "consecutive_permission_denials": 0, + "last_progress_loop": 2, + "total_opens": 1, + "reason": "No progress detected in 5 consecutive loops", + "current_loop": 7 +} +EOF + echo '[]' > "$CB_HISTORY_FILE" +} + +# Helper: Create a CLOSED state file +create_closed_state() { + cat > "$CB_STATE_FILE" << EOF +{ + "state": "CLOSED", + "last_change": "$(get_iso_timestamp)", + "consecutive_no_progress": 0, + "consecutive_same_error": 0, + "consecutive_permission_denials": 0, + "last_progress_loop": 0, + "total_opens": 0, + "reason": "" +} +EOF + echo '[]' > "$CB_HISTORY_FILE" +} + +# Helper: Get ISO timestamp for N minutes ago +get_past_timestamp() { + local minutes_ago=$1 + local seconds_ago=$((minutes_ago * 60)) + local past_epoch=$(($(date +%s) - seconds_ago)) + # Use GNU date if available, otherwise BSD date + if date -d "@$past_epoch" -Iseconds 2>/dev/null; then + return + fi + date -r "$past_epoch" +"%Y-%m-%dT%H:%M:%S+00:00" 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%S+00:00" +} + +# ============================================================================= +# COOLDOWN TIMER TESTS +# ============================================================================= + +@test "OPEN state with cooldown NOT elapsed stays OPEN" { + # Opened 10 minutes ago, cooldown is 30 minutes + local recent_timestamp + recent_timestamp=$(get_past_timestamp 10) + create_open_state "$recent_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "OPEN" ]] +} + +@test "OPEN state with cooldown elapsed transitions to HALF_OPEN" { + # Opened 35 minutes ago, cooldown is 30 minutes + local old_timestamp + old_timestamp=$(get_past_timestamp 35) + create_open_state "$old_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "HALF_OPEN" ]] +} + +@test "Cooldown recovery logs transition in history" { + local old_timestamp + old_timestamp=$(get_past_timestamp 35) + create_open_state "$old_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + + # Check history has a transition entry + local history_count + history_count=$(jq 'length' "$CB_HISTORY_FILE") + [[ $history_count -ge 1 ]] + + # Verify the transition details + local from_state to_state + from_state=$(jq -r '.[-1].from_state' "$CB_HISTORY_FILE") + to_state=$(jq -r '.[-1].to_state' "$CB_HISTORY_FILE") + [[ "$from_state" == "OPEN" ]] + [[ "$to_state" == "HALF_OPEN" ]] +} + +@test "HALF_OPEN from cooldown + progress recovers to CLOSED" { + # First: simulate cooldown recovery to HALF_OPEN + local old_timestamp + old_timestamp=$(get_past_timestamp 35) + create_open_state "$old_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "HALF_OPEN" ]] + + # Now: simulate a loop with progress + record_loop_result 8 3 "false" 5000 + + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "CLOSED" ]] +} + +@test "HALF_OPEN from cooldown + no progress re-trips to OPEN" { + # First: simulate cooldown recovery to HALF_OPEN + local old_timestamp + old_timestamp=$(get_past_timestamp 35) + create_open_state "$old_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "HALF_OPEN" ]] + + # Simulate enough no-progress loops to re-trip + for i in $(seq 1 $CB_NO_PROGRESS_THRESHOLD); do + record_loop_result $((7 + i)) 0 "false" 100 || true + done + + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "OPEN" ]] +} + +@test "CB_COOLDOWN_MINUTES=0 means immediate recovery attempt" { + # Opened just now, but cooldown is 0 + create_open_state "$(get_iso_timestamp)" + export CB_COOLDOWN_MINUTES=0 + export CB_AUTO_RESET=false + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "HALF_OPEN" ]] +} + +@test "Old state file without opened_at falls back to last_change" { + # Create old-format state file (no opened_at field) + local old_timestamp + old_timestamp=$(get_past_timestamp 35) + create_old_format_open_state "$old_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + + # Should still recover using last_change as fallback + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "HALF_OPEN" ]] +} + +@test "Clock skew (negative elapsed time) stays OPEN safely" { + # Create state with a future timestamp (simulating clock skew) + local future_epoch=$(($(date +%s) + 7200)) + local future_timestamp + if future_timestamp=$(date -d "@$future_epoch" -Iseconds 2>/dev/null); then + : # success + else + future_timestamp=$(date -r "$future_epoch" +"%Y-%m-%dT%H:%M:%S+00:00" 2>/dev/null || skip "Cannot create future timestamp") + fi + create_open_state "$future_timestamp" + export CB_COOLDOWN_MINUTES=30 + export CB_AUTO_RESET=false + + init_circuit_breaker + + # Should stay OPEN due to negative elapsed time + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "OPEN" ]] +} + +# ============================================================================= +# AUTO-RESET TESTS +# ============================================================================= + +@test "CB_AUTO_RESET=true resets OPEN to CLOSED on init" { + create_open_state "$(get_iso_timestamp)" + export CB_AUTO_RESET=true + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "CLOSED" ]] +} + +@test "CB_AUTO_RESET=true preserves total_opens count" { + create_open_state "$(get_iso_timestamp)" 3 + export CB_AUTO_RESET=true + + init_circuit_breaker + + local total_opens + total_opens=$(jq -r '.total_opens' "$CB_STATE_FILE") + [[ "$total_opens" == "3" ]] +} + +@test "CB_AUTO_RESET=true logs transition in history" { + create_open_state "$(get_iso_timestamp)" + export CB_AUTO_RESET=true + + init_circuit_breaker + + local history_count + history_count=$(jq 'length' "$CB_HISTORY_FILE") + [[ $history_count -ge 1 ]] + + local to_state reason + to_state=$(jq -r '.[-1].to_state' "$CB_HISTORY_FILE") + reason=$(jq -r '.[-1].reason' "$CB_HISTORY_FILE") + [[ "$to_state" == "CLOSED" ]] + [[ "$reason" == *"Auto-reset"* ]] +} + +@test "CB_AUTO_RESET=false (default) uses normal cooldown behavior" { + # Opened recently, cooldown not elapsed + create_open_state "$(get_iso_timestamp)" + export CB_AUTO_RESET=false + export CB_COOLDOWN_MINUTES=30 + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "OPEN" ]] +} + +@test "CLOSED state is not affected by auto-recovery logic" { + create_closed_state + export CB_AUTO_RESET=true + export CB_COOLDOWN_MINUTES=0 + + init_circuit_breaker + + local state + state=$(jq -r '.state' "$CB_STATE_FILE") + [[ "$state" == "CLOSED" ]] +} + +# ============================================================================= +# opened_at FIELD TESTS +# ============================================================================= + +@test "Entering OPEN state sets opened_at field" { + # Start CLOSED, trigger OPEN via no-progress + create_closed_state + export CB_NO_PROGRESS_THRESHOLD=3 + + for i in 1 2 3; do + record_loop_result "$i" 0 "false" 100 || true + done + + local state opened_at + state=$(jq -r '.state' "$CB_STATE_FILE") + opened_at=$(jq -r '.opened_at // "missing"' "$CB_STATE_FILE") + [[ "$state" == "OPEN" ]] + [[ "$opened_at" != "missing" ]] + [[ "$opened_at" != "null" ]] + [[ "$opened_at" != "" ]] +} + +@test "Staying OPEN preserves opened_at field" { + # Use a recent timestamp (5 minutes ago) so cooldown doesn't trigger + local fixed_timestamp + fixed_timestamp=$(get_past_timestamp 5) + create_open_state "$fixed_timestamp" + export CB_COOLDOWN_MINUTES=30 + + # Record another result while OPEN + record_loop_result 8 0 "false" 100 || true + + local opened_at + opened_at=$(jq -r '.opened_at' "$CB_STATE_FILE") + [[ "$opened_at" == "$fixed_timestamp" ]] +} + +# ============================================================================= +# parse_iso_to_epoch TESTS +# ============================================================================= + +@test "parse_iso_to_epoch handles valid ISO timestamp" { + local result + result=$(parse_iso_to_epoch "2025-01-15T10:30:00+00:00") + [[ "$result" =~ ^[0-9]+$ ]] + + # Should be roughly in the right range (2025 is ~1736899200 epoch) + [[ $result -gt 1700000000 ]] + [[ $result -lt 1800000000 ]] +} + +@test "parse_iso_to_epoch handles empty input with safe fallback" { + local result current_epoch + current_epoch=$(date +%s) + result=$(parse_iso_to_epoch "") + + [[ "$result" =~ ^[0-9]+$ ]] + # Should be close to current time (within 5 seconds) + local diff=$(( result - current_epoch )) + [[ ${diff#-} -lt 5 ]] +} + +@test "parse_iso_to_epoch handles null input with safe fallback" { + local result current_epoch + current_epoch=$(date +%s) + result=$(parse_iso_to_epoch "null") + + [[ "$result" =~ ^[0-9]+$ ]] + local diff=$(( result - current_epoch )) + [[ ${diff#-} -lt 5 ]] +} + +# ============================================================================= +# CLI FLAG TEST +# ============================================================================= + +@test "--auto-reset-circuit flag sets CB_AUTO_RESET=true" { + local RALPH_SCRIPT="${BATS_TEST_DIRNAME}/../../ralph_loop.sh" + + # Create minimal environment for CLI parsing + local CLI_TEST_DIR + CLI_TEST_DIR="$(mktemp -d /tmp/ralph-cli-test.XXXXXX)" + cd "$CLI_TEST_DIR" + + git init > /dev/null 2>&1 + git config user.email "test@example.com" + git config user.name "Test User" + + export RALPH_DIR=".ralph" + mkdir -p "$RALPH_DIR/logs" + echo "# Test Prompt" > "$RALPH_DIR/PROMPT.md" + echo "0" > "$RALPH_DIR/.call_count" + echo "$(date +%Y%m%d%H)" > "$RALPH_DIR/.last_reset" + echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$RALPH_DIR/.exit_signals" + + mkdir -p lib + cat > lib/circuit_breaker.sh << 'CBEOF' +RALPH_DIR="${RALPH_DIR:-.ralph}" +CB_AUTO_RESET="${CB_AUTO_RESET:-false}" +reset_circuit_breaker() { echo "Circuit breaker reset: $1"; } +show_circuit_status() { echo "Circuit breaker status: CLOSED"; } +init_circuit_breaker() { :; } +record_loop_result() { :; } +CBEOF + + cat > lib/response_analyzer.sh << 'RAEOF' +RALPH_DIR="${RALPH_DIR:-.ralph}" +analyze_response() { :; } +detect_output_format() { echo "text"; } +RAEOF + + cat > lib/date_utils.sh << 'DUEOF' +get_iso_timestamp() { date -Iseconds 2>/dev/null || date '+%Y-%m-%dT%H:%M:%S'; } +get_epoch_timestamp() { date +%s; } +DUEOF + + cat > lib/timeout_utils.sh << 'TUEOF' +portable_timeout() { shift; "$@"; } +TUEOF + + # Run with --auto-reset-circuit --help to parse the flag and exit + run bash "$RALPH_SCRIPT" --auto-reset-circuit --help + + assert_success + [[ "$output" == *"Usage:"* ]] + # Verify the flag is documented in help + [[ "$output" == *"--auto-reset-circuit"* ]] + + # Cleanup + cd / + rm -rf "$CLI_TEST_DIR" +}