3d7db2c3ae
Add cross-platform date utility library to handle differences between GNU date (Linux) and BSD date (macOS). Fixes issues with: - ISO 8601 timestamp formatting (-Iseconds flag) - Date arithmetic operations (-d vs -v flags) Changes: - Created lib/date_utils.sh with get_iso_timestamp() and get_next_hour_time() - Updated ralph_loop.sh to use date utilities (2 instances) - Updated lib/circuit_breaker.sh to use date utilities (4 instances) - Updated lib/response_analyzer.sh to use date utilities (1 instance) All date operations now work consistently across both platforms without modification. The utility automatically detects the OS and uses the appropriate date command syntax. Tested on Linux with GNU date - all syntax checks and integration tests pass.
309 lines
12 KiB
Bash
309 lines
12 KiB
Bash
#!/bin/bash
|
|
# Response Analyzer Component for Ralph
|
|
# Analyzes Claude Code output to detect completion signals, test-only loops, and progress
|
|
|
|
# Source date utilities for cross-platform compatibility
|
|
source "$(dirname "${BASH_SOURCE[0]}")/date_utils.sh"
|
|
|
|
# Response Analysis Functions
|
|
# Based on expert recommendations from Martin Fowler, Michael Nygard, Sam Newman
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
# Analysis configuration
|
|
COMPLETION_KEYWORDS=("done" "complete" "finished" "all tasks complete" "project complete" "ready for review")
|
|
TEST_ONLY_PATTERNS=("npm test" "bats" "pytest" "jest" "cargo test" "go test" "running tests")
|
|
NO_WORK_PATTERNS=("nothing to do" "no changes" "already implemented" "up to date")
|
|
|
|
# Analyze Claude Code response and extract signals
|
|
analyze_response() {
|
|
local output_file=$1
|
|
local loop_number=$2
|
|
local analysis_result_file=${3:-".response_analysis"}
|
|
|
|
# Initialize analysis result
|
|
local has_completion_signal=false
|
|
local is_test_only=false
|
|
local is_stuck=false
|
|
local has_progress=false
|
|
local confidence_score=0
|
|
local exit_signal=false
|
|
local work_summary=""
|
|
local files_modified=0
|
|
|
|
# Read output file
|
|
if [[ ! -f "$output_file" ]]; then
|
|
echo "ERROR: Output file not found: $output_file"
|
|
return 1
|
|
fi
|
|
|
|
local output_content=$(cat "$output_file")
|
|
local output_length=${#output_content}
|
|
|
|
# 1. Check for explicit structured output (if Claude follows schema)
|
|
if grep -q -- "---RALPH_STATUS---" "$output_file"; then
|
|
# Parse structured output
|
|
local status=$(grep "STATUS:" "$output_file" | cut -d: -f2 | xargs)
|
|
local exit_sig=$(grep "EXIT_SIGNAL:" "$output_file" | cut -d: -f2 | xargs)
|
|
|
|
if [[ "$exit_sig" == "true" || "$status" == "COMPLETE" ]]; then
|
|
has_completion_signal=true
|
|
exit_signal=true
|
|
confidence_score=100
|
|
fi
|
|
fi
|
|
|
|
# 2. Detect completion keywords in natural language output
|
|
for keyword in "${COMPLETION_KEYWORDS[@]}"; do
|
|
if grep -qi "$keyword" "$output_file"; then
|
|
has_completion_signal=true
|
|
((confidence_score+=10))
|
|
break
|
|
fi
|
|
done
|
|
|
|
# 3. Detect test-only loops
|
|
local test_command_count=0
|
|
local implementation_count=0
|
|
local error_count=0
|
|
|
|
test_command_count=$(grep -c -i "running tests\|npm test\|bats\|pytest\|jest" "$output_file" 2>/dev/null | head -1 || echo "0")
|
|
implementation_count=$(grep -c -i "implementing\|creating\|writing\|adding\|function\|class" "$output_file" 2>/dev/null | head -1 || echo "0")
|
|
|
|
# Strip whitespace and ensure it's a number
|
|
test_command_count=$(echo "$test_command_count" | tr -d '[:space:]')
|
|
implementation_count=$(echo "$implementation_count" | tr -d '[:space:]')
|
|
|
|
# Convert to integers with default fallback
|
|
test_command_count=${test_command_count:-0}
|
|
implementation_count=${implementation_count:-0}
|
|
test_command_count=$((test_command_count + 0))
|
|
implementation_count=$((implementation_count + 0))
|
|
|
|
if [[ $test_command_count -gt 0 ]] && [[ $implementation_count -eq 0 ]]; then
|
|
is_test_only=true
|
|
work_summary="Test execution only, no implementation"
|
|
fi
|
|
|
|
# 4. Detect stuck/error loops
|
|
# Use two-stage filtering to avoid counting JSON field names as errors
|
|
# Stage 1: Filter out JSON field patterns like "is_error": false
|
|
# Stage 2: Count actual error messages in specific contexts
|
|
# Pattern aligned with ralph_loop.sh to ensure consistent behavior
|
|
error_count=$(grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
|
|
grep -cE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' \
|
|
2>/dev/null || echo "0")
|
|
error_count=$(echo "$error_count" | tr -d '[:space:]')
|
|
error_count=${error_count:-0}
|
|
error_count=$((error_count + 0))
|
|
|
|
if [[ $error_count -gt 5 ]]; then
|
|
is_stuck=true
|
|
fi
|
|
|
|
# 5. Detect "nothing to do" patterns
|
|
for pattern in "${NO_WORK_PATTERNS[@]}"; do
|
|
if grep -qi "$pattern" "$output_file"; then
|
|
has_completion_signal=true
|
|
((confidence_score+=15))
|
|
work_summary="No work remaining"
|
|
break
|
|
fi
|
|
done
|
|
|
|
# 6. Check for file changes (git integration)
|
|
if command -v git &>/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then
|
|
files_modified=$(git diff --name-only 2>/dev/null | wc -l)
|
|
if [[ $files_modified -gt 0 ]]; then
|
|
has_progress=true
|
|
((confidence_score+=20))
|
|
fi
|
|
fi
|
|
|
|
# 7. Analyze output length trends (detect declining engagement)
|
|
if [[ -f ".last_output_length" ]]; then
|
|
local last_length=$(cat ".last_output_length")
|
|
local length_ratio=$((output_length * 100 / last_length))
|
|
|
|
if [[ $length_ratio -lt 50 ]]; then
|
|
# Output is less than 50% of previous - possible completion
|
|
((confidence_score+=10))
|
|
fi
|
|
fi
|
|
echo "$output_length" > ".last_output_length"
|
|
|
|
# 8. Extract work summary from output
|
|
if [[ -z "$work_summary" ]]; then
|
|
# Try to find summary in output
|
|
work_summary=$(grep -i "summary\|completed\|implemented" "$output_file" | head -1 | cut -c 1-100)
|
|
if [[ -z "$work_summary" ]]; then
|
|
work_summary="Output analyzed, no explicit summary found"
|
|
fi
|
|
fi
|
|
|
|
# 9. Determine exit signal based on confidence
|
|
if [[ $confidence_score -ge 40 || "$has_completion_signal" == "true" ]]; then
|
|
exit_signal=true
|
|
fi
|
|
|
|
# Write analysis results to file
|
|
cat > "$analysis_result_file" << EOF
|
|
{
|
|
"loop_number": $loop_number,
|
|
"timestamp": "$(get_iso_timestamp)",
|
|
"output_file": "$output_file",
|
|
"analysis": {
|
|
"has_completion_signal": $has_completion_signal,
|
|
"is_test_only": $is_test_only,
|
|
"is_stuck": $is_stuck,
|
|
"has_progress": $has_progress,
|
|
"files_modified": $files_modified,
|
|
"confidence_score": $confidence_score,
|
|
"exit_signal": $exit_signal,
|
|
"work_summary": "$work_summary",
|
|
"output_length": $output_length
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Always return 0 (success) - callers should check the JSON result file
|
|
# Returning non-zero would cause issues with set -e and test frameworks
|
|
return 0
|
|
}
|
|
|
|
# Update exit signals file based on analysis
|
|
update_exit_signals() {
|
|
local analysis_file=${1:-".response_analysis"}
|
|
local exit_signals_file=${2:-".exit_signals"}
|
|
|
|
if [[ ! -f "$analysis_file" ]]; then
|
|
echo "ERROR: Analysis file not found: $analysis_file"
|
|
return 1
|
|
fi
|
|
|
|
# Read analysis results
|
|
local is_test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
|
|
local has_completion_signal=$(jq -r '.analysis.has_completion_signal' "$analysis_file")
|
|
local loop_number=$(jq -r '.loop_number' "$analysis_file")
|
|
local has_progress=$(jq -r '.analysis.has_progress' "$analysis_file")
|
|
|
|
# Read current exit signals
|
|
local signals=$(cat "$exit_signals_file" 2>/dev/null || echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}')
|
|
|
|
# Update test_only_loops array
|
|
if [[ "$is_test_only" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq ".test_only_loops += [$loop_number]")
|
|
else
|
|
# Clear test_only_loops if we had implementation
|
|
if [[ "$has_progress" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq '.test_only_loops = []')
|
|
fi
|
|
fi
|
|
|
|
# Update done_signals array
|
|
if [[ "$has_completion_signal" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq ".done_signals += [$loop_number]")
|
|
fi
|
|
|
|
# Update completion_indicators array (strong signals)
|
|
local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
|
|
if [[ $confidence -ge 60 ]]; then
|
|
signals=$(echo "$signals" | jq ".completion_indicators += [$loop_number]")
|
|
fi
|
|
|
|
# Keep only last 5 signals (rolling window)
|
|
signals=$(echo "$signals" | jq '.test_only_loops = .test_only_loops[-5:]')
|
|
signals=$(echo "$signals" | jq '.done_signals = .done_signals[-5:]')
|
|
signals=$(echo "$signals" | jq '.completion_indicators = .completion_indicators[-5:]')
|
|
|
|
# Write updated signals
|
|
echo "$signals" > "$exit_signals_file"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Log analysis results in human-readable format
|
|
log_analysis_summary() {
|
|
local analysis_file=${1:-".response_analysis"}
|
|
|
|
if [[ ! -f "$analysis_file" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local loop=$(jq -r '.loop_number' "$analysis_file")
|
|
local exit_sig=$(jq -r '.analysis.exit_signal' "$analysis_file")
|
|
local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
|
|
local test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
|
|
local files_changed=$(jq -r '.analysis.files_modified' "$analysis_file")
|
|
local summary=$(jq -r '.analysis.work_summary' "$analysis_file")
|
|
|
|
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${BLUE}║ Response Analysis - Loop #$loop ║${NC}"
|
|
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
|
echo -e "${YELLOW}Exit Signal:${NC} $exit_sig"
|
|
echo -e "${YELLOW}Confidence:${NC} $confidence%"
|
|
echo -e "${YELLOW}Test Only:${NC} $test_only"
|
|
echo -e "${YELLOW}Files Changed:${NC} $files_changed"
|
|
echo -e "${YELLOW}Summary:${NC} $summary"
|
|
echo ""
|
|
}
|
|
|
|
# Detect if Claude is stuck (repeating same errors)
|
|
detect_stuck_loop() {
|
|
local current_output=$1
|
|
local history_dir=${2:-"logs"}
|
|
|
|
# Get last 3 output files
|
|
local recent_outputs=$(ls -t "$history_dir"/claude_output_*.log 2>/dev/null | head -3)
|
|
|
|
if [[ -z "$recent_outputs" ]]; then
|
|
return 1 # Not enough history
|
|
fi
|
|
|
|
# Extract key errors from current output using two-stage filtering
|
|
# Stage 1: Filter out JSON field patterns to avoid false positives
|
|
# Stage 2: Extract actual error messages
|
|
local current_errors=$(grep -v '"[^"]*error[^"]*":' "$current_output" 2>/dev/null | \
|
|
grep -E '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' 2>/dev/null | \
|
|
sort | uniq)
|
|
|
|
if [[ -z "$current_errors" ]]; then
|
|
return 1 # No errors
|
|
fi
|
|
|
|
# Check if same errors appear in all recent outputs
|
|
# For multi-line errors, verify ALL error lines appear in ALL history files
|
|
local all_files_match=true
|
|
while IFS= read -r output_file; do
|
|
local file_matches_all=true
|
|
while IFS= read -r error_line; do
|
|
# Use -F for literal fixed-string matching (not regex)
|
|
if ! grep -qF "$error_line" "$output_file" 2>/dev/null; then
|
|
file_matches_all=false
|
|
break
|
|
fi
|
|
done <<< "$current_errors"
|
|
|
|
if [[ "$file_matches_all" != "true" ]]; then
|
|
all_files_match=false
|
|
break
|
|
fi
|
|
done <<< "$recent_outputs"
|
|
|
|
if [[ "$all_files_match" == "true" ]]; then
|
|
return 0 # Stuck on same error(s)
|
|
else
|
|
return 1 # Making progress or different errors
|
|
fi
|
|
}
|
|
|
|
# Export functions for use in ralph_loop.sh
|
|
export -f analyze_response
|
|
export -f update_exit_signals
|
|
export -f log_analysis_summary
|
|
export -f detect_stuck_loop
|