ralph-claude-code/lib/response_analyzer.sh

#!/bin/bash
# Response Analyzer Component for Ralph
# Analyzes Claude Code output to detect completion signals, test-only loops, and progress

# Response Analysis Functions
# Based on expert recommendations from Martin Fowler, Michael Nygard, Sam Newman

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Analysis configuration
COMPLETION_KEYWORDS=("done" "complete" "finished" "all tasks complete" "project complete" "ready for review")
TEST_ONLY_PATTERNS=("npm test" "bats" "pytest" "jest" "cargo test" "go test" "running tests")
NO_WORK_PATTERNS=("nothing to do" "no changes" "already implemented" "up to date")

# Analyze Claude Code response and extract signals
analyze_response() {
    local output_file=$1
    local loop_number=$2
    local analysis_result_file=${3:-".response_analysis"}

    # Initialize analysis result
    local has_completion_signal=false
    local is_test_only=false
    local is_stuck=false
    local has_progress=false
    local confidence_score=0
    local exit_signal=false
    local work_summary=""
    local files_modified=0

    # Read output file
    if [[ ! -f "$output_file" ]]; then
        echo "ERROR: Output file not found: $output_file"
        return 1
    fi

    local output_content=$(cat "$output_file")
    local output_length=${#output_content}

    # 1. Check for explicit structured output (if Claude follows schema)
    if grep -q -- "---RALPH_STATUS---" "$output_file"; then
        # Parse structured output
        local status=$(grep "STATUS:" "$output_file" | cut -d: -f2 | xargs)
        local exit_sig=$(grep "EXIT_SIGNAL:" "$output_file" | cut -d: -f2 | xargs)

        if [[ "$exit_sig" == "true" || "$status" == "COMPLETE" ]]; then
            has_completion_signal=true
            exit_signal=true
            confidence_score=100
        fi
    fi

    # 2. Detect completion keywords in natural language output
    for keyword in "${COMPLETION_KEYWORDS[@]}"; do
        if grep -qi "$keyword" "$output_file"; then
            has_completion_signal=true
            ((confidence_score+=10))
            break
        fi
    done

    # 3. Detect test-only loops
    local test_command_count=0
    local implementation_count=0
    local error_count=0

    test_command_count=$(grep -c -i "running tests\|npm test\|bats\|pytest\|jest" "$output_file" 2>/dev/null | head -1 || echo "0")
    implementation_count=$(grep -c -i "implementing\|creating\|writing\|adding\|function\|class" "$output_file" 2>/dev/null | head -1 || echo "0")

    # Strip whitespace and ensure it's a number
    test_command_count=$(echo "$test_command_count" | tr -d '[:space:]')
    implementation_count=$(echo "$implementation_count" | tr -d '[:space:]')

    # Convert to integers with default fallback
    test_command_count=${test_command_count:-0}
    implementation_count=${implementation_count:-0}
    test_command_count=$((test_command_count + 0))
    implementation_count=$((implementation_count + 0))

    if [[ $test_command_count -gt 0 ]] && [[ $implementation_count -eq 0 ]]; then
        is_test_only=true
        work_summary="Test execution only, no implementation"
    fi

    # 4. Detect stuck/error loops
    # Use two-stage filtering to avoid counting JSON field names as errors
    # Stage 1: Filter out JSON field patterns like "is_error": false
    # Stage 2: Count actual error messages in specific contexts
    # Pattern aligned with ralph_loop.sh to ensure consistent behavior
    error_count=$(grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
                  grep -cE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' \
                  2>/dev/null || echo "0")
    error_count=$(echo "$error_count" | tr -d '[:space:]')
    error_count=${error_count:-0}
    error_count=$((error_count + 0))

    if [[ $error_count -gt 5 ]]; then
        is_stuck=true
    fi

    # 5. Detect "nothing to do" patterns
    for pattern in "${NO_WORK_PATTERNS[@]}"; do
        if grep -qi "$pattern" "$output_file"; then
            has_completion_signal=true
            ((confidence_score+=15))
            work_summary="No work remaining"
            break
        fi
    done

    # 6. Check for file changes (git integration)
    if command -v git &>/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then
        files_modified=$(git diff --name-only 2>/dev/null | wc -l)
        if [[ $files_modified -gt 0 ]]; then
            has_progress=true
            ((confidence_score+=20))
        fi
    fi

    # 7. Analyze output length trends (detect declining engagement)
    if [[ -f ".last_output_length" ]]; then
        local last_length=$(cat ".last_output_length")
        local length_ratio=$((output_length * 100 / last_length))

        if [[ $length_ratio -lt 50 ]]; then
            # Output is less than 50% of previous - possible completion
            ((confidence_score+=10))
        fi
    fi
    echo "$output_length" > ".last_output_length"

    # 8. Extract work summary from output
    if [[ -z "$work_summary" ]]; then
        # Try to find summary in output
        work_summary=$(grep -i "summary\|completed\|implemented" "$output_file" | head -1 | cut -c 1-100)
        if [[ -z "$work_summary" ]]; then
            work_summary="Output analyzed, no explicit summary found"
        fi
    fi

    # 9. Determine exit signal based on confidence
    if [[ $confidence_score -ge 40 || "$has_completion_signal" == "true" ]]; then
        exit_signal=true
    fi

    # Write analysis results to file
    cat > "$analysis_result_file" << EOF
{
    "loop_number": $loop_number,
    "timestamp": "$(date -Iseconds)",
    "output_file": "$output_file",
    "analysis": {
        "has_completion_signal": $has_completion_signal,
        "is_test_only": $is_test_only,
        "is_stuck": $is_stuck,
        "has_progress": $has_progress,
        "files_modified": $files_modified,
        "confidence_score": $confidence_score,
        "exit_signal": $exit_signal,
        "work_summary": "$work_summary",
        "output_length": $output_length
    }
}
EOF

    # Always return 0 (success) - callers should check the JSON result file
    # Returning non-zero would cause issues with set -e and test frameworks
    return 0
}

# Update exit signals file based on analysis
update_exit_signals() {
    local analysis_file=${1:-".response_analysis"}
    local exit_signals_file=${2:-".exit_signals"}

    if [[ ! -f "$analysis_file" ]]; then
        echo "ERROR: Analysis file not found: $analysis_file"
        return 1
    fi

    # Read analysis results
    local is_test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
    local has_completion_signal=$(jq -r '.analysis.has_completion_signal' "$analysis_file")
    local loop_number=$(jq -r '.loop_number' "$analysis_file")
    local has_progress=$(jq -r '.analysis.has_progress' "$analysis_file")

    # Read current exit signals
    local signals=$(cat "$exit_signals_file" 2>/dev/null || echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}')

    # Update test_only_loops array
    if [[ "$is_test_only" == "true" ]]; then
        signals=$(echo "$signals" | jq ".test_only_loops += [$loop_number]")
    else
        # Clear test_only_loops if we had implementation
        if [[ "$has_progress" == "true" ]]; then
            signals=$(echo "$signals" | jq '.test_only_loops = []')
        fi
    fi

    # Update done_signals array
    if [[ "$has_completion_signal" == "true" ]]; then
        signals=$(echo "$signals" | jq ".done_signals += [$loop_number]")
    fi

    # Update completion_indicators array (strong signals)
    local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
    if [[ $confidence -ge 60 ]]; then
        signals=$(echo "$signals" | jq ".completion_indicators += [$loop_number]")
    fi

    # Keep only last 5 signals (rolling window)
    signals=$(echo "$signals" | jq '.test_only_loops = .test_only_loops[-5:]')
    signals=$(echo "$signals" | jq '.done_signals = .done_signals[-5:]')
    signals=$(echo "$signals" | jq '.completion_indicators = .completion_indicators[-5:]')

    # Write updated signals
    echo "$signals" > "$exit_signals_file"

    return 0
}

# Log analysis results in human-readable format
log_analysis_summary() {
    local analysis_file=${1:-".response_analysis"}

    if [[ ! -f "$analysis_file" ]]; then
        return 1
    fi

    local loop=$(jq -r '.loop_number' "$analysis_file")
    local exit_sig=$(jq -r '.analysis.exit_signal' "$analysis_file")
    local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
    local test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
    local files_changed=$(jq -r '.analysis.files_modified' "$analysis_file")
    local summary=$(jq -r '.analysis.work_summary' "$analysis_file")

    echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
    echo -e "${BLUE}║           Response Analysis - Loop #$loop                 ║${NC}"
    echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
    echo -e "${YELLOW}Exit Signal:${NC}      $exit_sig"
    echo -e "${YELLOW}Confidence:${NC}       $confidence%"
    echo -e "${YELLOW}Test Only:${NC}        $test_only"
    echo -e "${YELLOW}Files Changed:${NC}    $files_changed"
    echo -e "${YELLOW}Summary:${NC}          $summary"
    echo ""
}

# Detect if Claude is stuck (repeating same errors)
detect_stuck_loop() {
    local current_output=$1
    local history_dir=${2:-"logs"}

    # Get last 3 output files
    local recent_outputs=$(ls -t "$history_dir"/claude_output_*.log 2>/dev/null | head -3)

    if [[ -z "$recent_outputs" ]]; then
        return 1  # Not enough history
    fi

    # Extract key errors from current output using two-stage filtering
    # Stage 1: Filter out JSON field patterns to avoid false positives
    # Stage 2: Extract actual error messages
    local current_errors=$(grep -v '"[^"]*error[^"]*":' "$current_output" 2>/dev/null | \
                          grep -E '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' 2>/dev/null | \
                          sort | uniq)

    if [[ -z "$current_errors" ]]; then
        return 1  # No errors
    fi

    # Check if same errors appear in all recent outputs
    local stuck_count=0
    while IFS= read -r output_file; do
        if grep -q "$current_errors" "$output_file" 2>/dev/null; then
            ((stuck_count++))
        fi
    done <<< "$recent_outputs"

    if [[ $stuck_count -ge 3 ]]; then
        return 0  # Stuck on same error
    else
        return 1  # Making progress or different errors
    fi
}

# Export functions for use in ralph_loop.sh
export -f analyze_response
export -f update_exit_signals
export -f log_analysis_summary
export -f detect_stuck_loop