c7c6c9389d
Replace two heredocs that write analysis results with jq construction to prevent JSON injection via work_summary or other string fields. Fixed locations: - Line 205-235: JSON parsing path analysis result - Line 351-381: Text parsing path analysis result Both now use jq with --arg for strings and --argjson for numeric/boolean fields, ensuring proper escaping of special characters.
520 lines
20 KiB
Bash
520 lines
20 KiB
Bash
#!/bin/bash
|
|
# Response Analyzer Component for Ralph
|
|
# Analyzes Claude Code output to detect completion signals, test-only loops, and progress
|
|
|
|
# Source date utilities for cross-platform compatibility
|
|
source "$(dirname "${BASH_SOURCE[0]}")/date_utils.sh"
|
|
|
|
# Response Analysis Functions
|
|
# Based on expert recommendations from Martin Fowler, Michael Nygard, Sam Newman
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
# Analysis configuration
|
|
COMPLETION_KEYWORDS=("done" "complete" "finished" "all tasks complete" "project complete" "ready for review")
|
|
TEST_ONLY_PATTERNS=("npm test" "bats" "pytest" "jest" "cargo test" "go test" "running tests")
|
|
NO_WORK_PATTERNS=("nothing to do" "no changes" "already implemented" "up to date")
|
|
|
|
# =============================================================================
|
|
# JSON OUTPUT FORMAT DETECTION AND PARSING
|
|
# =============================================================================
|
|
|
|
# Detect output format (json or text)
|
|
# Returns: "json" if valid JSON, "text" otherwise
|
|
detect_output_format() {
|
|
local output_file=$1
|
|
|
|
if [[ ! -f "$output_file" ]] || [[ ! -s "$output_file" ]]; then
|
|
echo "text"
|
|
return
|
|
fi
|
|
|
|
# Check if file starts with { or [ (JSON indicators)
|
|
local first_char=$(head -c 1 "$output_file" 2>/dev/null | tr -d '[:space:]')
|
|
|
|
if [[ "$first_char" != "{" && "$first_char" != "[" ]]; then
|
|
echo "text"
|
|
return
|
|
fi
|
|
|
|
# Validate as JSON using jq
|
|
if jq empty "$output_file" 2>/dev/null; then
|
|
echo "json"
|
|
else
|
|
echo "text"
|
|
fi
|
|
}
|
|
|
|
# Parse JSON response and extract structured fields
|
|
# Creates .json_parse_result with normalized analysis data
|
|
parse_json_response() {
|
|
local output_file=$1
|
|
local result_file="${2:-.json_parse_result}"
|
|
|
|
if [[ ! -f "$output_file" ]]; then
|
|
echo "ERROR: Output file not found: $output_file" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Validate JSON first
|
|
if ! jq empty "$output_file" 2>/dev/null; then
|
|
echo "ERROR: Invalid JSON in output file" >&2
|
|
return 1
|
|
fi
|
|
|
|
# Extract fields with defaults
|
|
local status=$(jq -r '.status // "UNKNOWN"' "$output_file" 2>/dev/null)
|
|
local exit_signal=$(jq -r '.exit_signal // false' "$output_file" 2>/dev/null)
|
|
local work_type=$(jq -r '.work_type // "UNKNOWN"' "$output_file" 2>/dev/null)
|
|
local files_modified=$(jq -r '.files_modified // 0' "$output_file" 2>/dev/null)
|
|
local error_count=$(jq -r '.error_count // 0' "$output_file" 2>/dev/null)
|
|
local summary=$(jq -r '.summary // ""' "$output_file" 2>/dev/null)
|
|
|
|
# Extract nested metadata if present
|
|
local loop_number=$(jq -r '.metadata.loop_number // .loop_number // 0' "$output_file" 2>/dev/null)
|
|
local session_id=$(jq -r '.metadata.session_id // ""' "$output_file" 2>/dev/null)
|
|
local confidence=$(jq -r '.confidence // 0' "$output_file" 2>/dev/null)
|
|
|
|
# Normalize values
|
|
# Convert exit_signal to boolean string
|
|
if [[ "$exit_signal" == "true" || "$status" == "COMPLETE" ]]; then
|
|
exit_signal="true"
|
|
else
|
|
exit_signal="false"
|
|
fi
|
|
|
|
# Determine is_test_only from work_type
|
|
local is_test_only="false"
|
|
if [[ "$work_type" == "TEST_ONLY" ]]; then
|
|
is_test_only="true"
|
|
fi
|
|
|
|
# Determine is_stuck from error_count
|
|
local is_stuck="false"
|
|
error_count=$((error_count + 0)) # Ensure integer
|
|
if [[ $error_count -gt 5 ]]; then
|
|
is_stuck="true"
|
|
fi
|
|
|
|
# Ensure files_modified is integer
|
|
files_modified=$((files_modified + 0))
|
|
|
|
# Calculate has_completion_signal
|
|
local has_completion_signal="false"
|
|
if [[ "$status" == "COMPLETE" || "$exit_signal" == "true" ]]; then
|
|
has_completion_signal="true"
|
|
fi
|
|
|
|
# Write normalized result using jq for safe JSON construction
|
|
# String fields use --arg (auto-escapes), numeric/boolean use --argjson
|
|
jq -n \
|
|
--arg status "$status" \
|
|
--argjson exit_signal "$exit_signal" \
|
|
--argjson is_test_only "$is_test_only" \
|
|
--argjson is_stuck "$is_stuck" \
|
|
--argjson has_completion_signal "$has_completion_signal" \
|
|
--argjson files_modified "$files_modified" \
|
|
--argjson error_count "$error_count" \
|
|
--arg summary "$summary" \
|
|
--argjson loop_number "$loop_number" \
|
|
--arg session_id "$session_id" \
|
|
--argjson confidence "$confidence" \
|
|
'{
|
|
status: $status,
|
|
exit_signal: $exit_signal,
|
|
is_test_only: $is_test_only,
|
|
is_stuck: $is_stuck,
|
|
has_completion_signal: $has_completion_signal,
|
|
files_modified: $files_modified,
|
|
error_count: $error_count,
|
|
summary: $summary,
|
|
loop_number: $loop_number,
|
|
session_id: $session_id,
|
|
confidence: $confidence,
|
|
metadata: {
|
|
loop_number: $loop_number,
|
|
session_id: $session_id
|
|
}
|
|
}' > "$result_file"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Analyze Claude Code response and extract signals
|
|
analyze_response() {
|
|
local output_file=$1
|
|
local loop_number=$2
|
|
local analysis_result_file=${3:-".response_analysis"}
|
|
|
|
# Initialize analysis result
|
|
local has_completion_signal=false
|
|
local is_test_only=false
|
|
local is_stuck=false
|
|
local has_progress=false
|
|
local confidence_score=0
|
|
local exit_signal=false
|
|
local work_summary=""
|
|
local files_modified=0
|
|
|
|
# Read output file
|
|
if [[ ! -f "$output_file" ]]; then
|
|
echo "ERROR: Output file not found: $output_file"
|
|
return 1
|
|
fi
|
|
|
|
local output_content=$(cat "$output_file")
|
|
local output_length=${#output_content}
|
|
|
|
# Detect output format and try JSON parsing first
|
|
local output_format=$(detect_output_format "$output_file")
|
|
|
|
if [[ "$output_format" == "json" ]]; then
|
|
# Try JSON parsing
|
|
if parse_json_response "$output_file" ".json_parse_result" 2>/dev/null; then
|
|
# Extract values from JSON parse result
|
|
has_completion_signal=$(jq -r '.has_completion_signal' .json_parse_result 2>/dev/null || echo "false")
|
|
exit_signal=$(jq -r '.exit_signal' .json_parse_result 2>/dev/null || echo "false")
|
|
is_test_only=$(jq -r '.is_test_only' .json_parse_result 2>/dev/null || echo "false")
|
|
is_stuck=$(jq -r '.is_stuck' .json_parse_result 2>/dev/null || echo "false")
|
|
work_summary=$(jq -r '.summary' .json_parse_result 2>/dev/null || echo "")
|
|
files_modified=$(jq -r '.files_modified' .json_parse_result 2>/dev/null || echo "0")
|
|
local json_confidence=$(jq -r '.confidence' .json_parse_result 2>/dev/null || echo "0")
|
|
|
|
# JSON parsing provides high confidence
|
|
if [[ "$exit_signal" == "true" ]]; then
|
|
confidence_score=100
|
|
else
|
|
confidence_score=$((json_confidence + 50))
|
|
fi
|
|
|
|
# Check for file changes via git (supplements JSON data)
|
|
if command -v git &>/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then
|
|
local git_files=$(git diff --name-only 2>/dev/null | wc -l)
|
|
if [[ $git_files -gt 0 ]]; then
|
|
has_progress=true
|
|
files_modified=$git_files
|
|
fi
|
|
fi
|
|
|
|
# Write analysis results for JSON path using jq for safe construction
|
|
jq -n \
|
|
--argjson loop_number "$loop_number" \
|
|
--arg timestamp "$(get_iso_timestamp)" \
|
|
--arg output_file "$output_file" \
|
|
--arg output_format "json" \
|
|
--argjson has_completion_signal "$has_completion_signal" \
|
|
--argjson is_test_only "$is_test_only" \
|
|
--argjson is_stuck "$is_stuck" \
|
|
--argjson has_progress "$has_progress" \
|
|
--argjson files_modified "$files_modified" \
|
|
--argjson confidence_score "$confidence_score" \
|
|
--argjson exit_signal "$exit_signal" \
|
|
--arg work_summary "$work_summary" \
|
|
--argjson output_length "$output_length" \
|
|
'{
|
|
loop_number: $loop_number,
|
|
timestamp: $timestamp,
|
|
output_file: $output_file,
|
|
output_format: $output_format,
|
|
analysis: {
|
|
has_completion_signal: $has_completion_signal,
|
|
is_test_only: $is_test_only,
|
|
is_stuck: $is_stuck,
|
|
has_progress: $has_progress,
|
|
files_modified: $files_modified,
|
|
confidence_score: $confidence_score,
|
|
exit_signal: $exit_signal,
|
|
work_summary: $work_summary,
|
|
output_length: $output_length
|
|
}
|
|
}' > "$analysis_result_file"
|
|
rm -f ".json_parse_result"
|
|
return 0
|
|
fi
|
|
# If JSON parsing failed, fall through to text parsing
|
|
fi
|
|
|
|
# Text parsing fallback (original logic)
|
|
|
|
# 1. Check for explicit structured output (if Claude follows schema)
|
|
if grep -q -- "---RALPH_STATUS---" "$output_file"; then
|
|
# Parse structured output
|
|
local status=$(grep "STATUS:" "$output_file" | cut -d: -f2 | xargs)
|
|
local exit_sig=$(grep "EXIT_SIGNAL:" "$output_file" | cut -d: -f2 | xargs)
|
|
|
|
if [[ "$exit_sig" == "true" || "$status" == "COMPLETE" ]]; then
|
|
has_completion_signal=true
|
|
exit_signal=true
|
|
confidence_score=100
|
|
fi
|
|
fi
|
|
|
|
# 2. Detect completion keywords in natural language output
|
|
for keyword in "${COMPLETION_KEYWORDS[@]}"; do
|
|
if grep -qi "$keyword" "$output_file"; then
|
|
has_completion_signal=true
|
|
((confidence_score+=10))
|
|
break
|
|
fi
|
|
done
|
|
|
|
# 3. Detect test-only loops
|
|
local test_command_count=0
|
|
local implementation_count=0
|
|
local error_count=0
|
|
|
|
test_command_count=$(grep -c -i "running tests\|npm test\|bats\|pytest\|jest" "$output_file" 2>/dev/null | head -1 || echo "0")
|
|
implementation_count=$(grep -c -i "implementing\|creating\|writing\|adding\|function\|class" "$output_file" 2>/dev/null | head -1 || echo "0")
|
|
|
|
# Strip whitespace and ensure it's a number
|
|
test_command_count=$(echo "$test_command_count" | tr -d '[:space:]')
|
|
implementation_count=$(echo "$implementation_count" | tr -d '[:space:]')
|
|
|
|
# Convert to integers with default fallback
|
|
test_command_count=${test_command_count:-0}
|
|
implementation_count=${implementation_count:-0}
|
|
test_command_count=$((test_command_count + 0))
|
|
implementation_count=$((implementation_count + 0))
|
|
|
|
if [[ $test_command_count -gt 0 ]] && [[ $implementation_count -eq 0 ]]; then
|
|
is_test_only=true
|
|
work_summary="Test execution only, no implementation"
|
|
fi
|
|
|
|
# 4. Detect stuck/error loops
|
|
# Use two-stage filtering to avoid counting JSON field names as errors
|
|
# Stage 1: Filter out JSON field patterns like "is_error": false
|
|
# Stage 2: Count actual error messages in specific contexts
|
|
# Pattern aligned with ralph_loop.sh to ensure consistent behavior
|
|
error_count=$(grep -v '"[^"]*error[^"]*":' "$output_file" 2>/dev/null | \
|
|
grep -cE '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' \
|
|
2>/dev/null || echo "0")
|
|
error_count=$(echo "$error_count" | tr -d '[:space:]')
|
|
error_count=${error_count:-0}
|
|
error_count=$((error_count + 0))
|
|
|
|
if [[ $error_count -gt 5 ]]; then
|
|
is_stuck=true
|
|
fi
|
|
|
|
# 5. Detect "nothing to do" patterns
|
|
for pattern in "${NO_WORK_PATTERNS[@]}"; do
|
|
if grep -qi "$pattern" "$output_file"; then
|
|
has_completion_signal=true
|
|
((confidence_score+=15))
|
|
work_summary="No work remaining"
|
|
break
|
|
fi
|
|
done
|
|
|
|
# 6. Check for file changes (git integration)
|
|
if command -v git &>/dev/null && git rev-parse --git-dir >/dev/null 2>&1; then
|
|
files_modified=$(git diff --name-only 2>/dev/null | wc -l)
|
|
if [[ $files_modified -gt 0 ]]; then
|
|
has_progress=true
|
|
((confidence_score+=20))
|
|
fi
|
|
fi
|
|
|
|
# 7. Analyze output length trends (detect declining engagement)
|
|
if [[ -f ".last_output_length" ]]; then
|
|
local last_length=$(cat ".last_output_length")
|
|
local length_ratio=$((output_length * 100 / last_length))
|
|
|
|
if [[ $length_ratio -lt 50 ]]; then
|
|
# Output is less than 50% of previous - possible completion
|
|
((confidence_score+=10))
|
|
fi
|
|
fi
|
|
echo "$output_length" > ".last_output_length"
|
|
|
|
# 8. Extract work summary from output
|
|
if [[ -z "$work_summary" ]]; then
|
|
# Try to find summary in output
|
|
work_summary=$(grep -i "summary\|completed\|implemented" "$output_file" | head -1 | cut -c 1-100)
|
|
if [[ -z "$work_summary" ]]; then
|
|
work_summary="Output analyzed, no explicit summary found"
|
|
fi
|
|
fi
|
|
|
|
# 9. Determine exit signal based on confidence
|
|
if [[ $confidence_score -ge 40 || "$has_completion_signal" == "true" ]]; then
|
|
exit_signal=true
|
|
fi
|
|
|
|
# Write analysis results to file (text parsing path) using jq for safe construction
|
|
jq -n \
|
|
--argjson loop_number "$loop_number" \
|
|
--arg timestamp "$(get_iso_timestamp)" \
|
|
--arg output_file "$output_file" \
|
|
--arg output_format "text" \
|
|
--argjson has_completion_signal "$has_completion_signal" \
|
|
--argjson is_test_only "$is_test_only" \
|
|
--argjson is_stuck "$is_stuck" \
|
|
--argjson has_progress "$has_progress" \
|
|
--argjson files_modified "$files_modified" \
|
|
--argjson confidence_score "$confidence_score" \
|
|
--argjson exit_signal "$exit_signal" \
|
|
--arg work_summary "$work_summary" \
|
|
--argjson output_length "$output_length" \
|
|
'{
|
|
loop_number: $loop_number,
|
|
timestamp: $timestamp,
|
|
output_file: $output_file,
|
|
output_format: $output_format,
|
|
analysis: {
|
|
has_completion_signal: $has_completion_signal,
|
|
is_test_only: $is_test_only,
|
|
is_stuck: $is_stuck,
|
|
has_progress: $has_progress,
|
|
files_modified: $files_modified,
|
|
confidence_score: $confidence_score,
|
|
exit_signal: $exit_signal,
|
|
work_summary: $work_summary,
|
|
output_length: $output_length
|
|
}
|
|
}' > "$analysis_result_file"
|
|
|
|
# Always return 0 (success) - callers should check the JSON result file
|
|
# Returning non-zero would cause issues with set -e and test frameworks
|
|
return 0
|
|
}
|
|
|
|
# Update exit signals file based on analysis
|
|
update_exit_signals() {
|
|
local analysis_file=${1:-".response_analysis"}
|
|
local exit_signals_file=${2:-".exit_signals"}
|
|
|
|
if [[ ! -f "$analysis_file" ]]; then
|
|
echo "ERROR: Analysis file not found: $analysis_file"
|
|
return 1
|
|
fi
|
|
|
|
# Read analysis results
|
|
local is_test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
|
|
local has_completion_signal=$(jq -r '.analysis.has_completion_signal' "$analysis_file")
|
|
local loop_number=$(jq -r '.loop_number' "$analysis_file")
|
|
local has_progress=$(jq -r '.analysis.has_progress' "$analysis_file")
|
|
|
|
# Read current exit signals
|
|
local signals=$(cat "$exit_signals_file" 2>/dev/null || echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}')
|
|
|
|
# Update test_only_loops array
|
|
if [[ "$is_test_only" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq ".test_only_loops += [$loop_number]")
|
|
else
|
|
# Clear test_only_loops if we had implementation
|
|
if [[ "$has_progress" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq '.test_only_loops = []')
|
|
fi
|
|
fi
|
|
|
|
# Update done_signals array
|
|
if [[ "$has_completion_signal" == "true" ]]; then
|
|
signals=$(echo "$signals" | jq ".done_signals += [$loop_number]")
|
|
fi
|
|
|
|
# Update completion_indicators array (strong signals)
|
|
local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
|
|
if [[ $confidence -ge 60 ]]; then
|
|
signals=$(echo "$signals" | jq ".completion_indicators += [$loop_number]")
|
|
fi
|
|
|
|
# Keep only last 5 signals (rolling window)
|
|
signals=$(echo "$signals" | jq '.test_only_loops = .test_only_loops[-5:]')
|
|
signals=$(echo "$signals" | jq '.done_signals = .done_signals[-5:]')
|
|
signals=$(echo "$signals" | jq '.completion_indicators = .completion_indicators[-5:]')
|
|
|
|
# Write updated signals
|
|
echo "$signals" > "$exit_signals_file"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Log analysis results in human-readable format
|
|
log_analysis_summary() {
|
|
local analysis_file=${1:-".response_analysis"}
|
|
|
|
if [[ ! -f "$analysis_file" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local loop=$(jq -r '.loop_number' "$analysis_file")
|
|
local exit_sig=$(jq -r '.analysis.exit_signal' "$analysis_file")
|
|
local confidence=$(jq -r '.analysis.confidence_score' "$analysis_file")
|
|
local test_only=$(jq -r '.analysis.is_test_only' "$analysis_file")
|
|
local files_changed=$(jq -r '.analysis.files_modified' "$analysis_file")
|
|
local summary=$(jq -r '.analysis.work_summary' "$analysis_file")
|
|
|
|
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${BLUE}║ Response Analysis - Loop #$loop ║${NC}"
|
|
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
|
echo -e "${YELLOW}Exit Signal:${NC} $exit_sig"
|
|
echo -e "${YELLOW}Confidence:${NC} $confidence%"
|
|
echo -e "${YELLOW}Test Only:${NC} $test_only"
|
|
echo -e "${YELLOW}Files Changed:${NC} $files_changed"
|
|
echo -e "${YELLOW}Summary:${NC} $summary"
|
|
echo ""
|
|
}
|
|
|
|
# Detect if Claude is stuck (repeating same errors)
|
|
detect_stuck_loop() {
|
|
local current_output=$1
|
|
local history_dir=${2:-"logs"}
|
|
|
|
# Get last 3 output files
|
|
local recent_outputs=$(ls -t "$history_dir"/claude_output_*.log 2>/dev/null | head -3)
|
|
|
|
if [[ -z "$recent_outputs" ]]; then
|
|
return 1 # Not enough history
|
|
fi
|
|
|
|
# Extract key errors from current output using two-stage filtering
|
|
# Stage 1: Filter out JSON field patterns to avoid false positives
|
|
# Stage 2: Extract actual error messages
|
|
local current_errors=$(grep -v '"[^"]*error[^"]*":' "$current_output" 2>/dev/null | \
|
|
grep -E '(^Error:|^ERROR:|^error:|\]: error|Link: error|Error occurred|failed with error|[Ee]xception|Fatal|FATAL)' 2>/dev/null | \
|
|
sort | uniq)
|
|
|
|
if [[ -z "$current_errors" ]]; then
|
|
return 1 # No errors
|
|
fi
|
|
|
|
# Check if same errors appear in all recent outputs
|
|
# For multi-line errors, verify ALL error lines appear in ALL history files
|
|
local all_files_match=true
|
|
while IFS= read -r output_file; do
|
|
local file_matches_all=true
|
|
while IFS= read -r error_line; do
|
|
# Use -F for literal fixed-string matching (not regex)
|
|
if ! grep -qF "$error_line" "$output_file" 2>/dev/null; then
|
|
file_matches_all=false
|
|
break
|
|
fi
|
|
done <<< "$current_errors"
|
|
|
|
if [[ "$file_matches_all" != "true" ]]; then
|
|
all_files_match=false
|
|
break
|
|
fi
|
|
done <<< "$recent_outputs"
|
|
|
|
if [[ "$all_files_match" == "true" ]]; then
|
|
return 0 # Stuck on same error(s)
|
|
else
|
|
return 1 # Making progress or different errors
|
|
fi
|
|
}
|
|
|
|
# Export functions for use in ralph_loop.sh
|
|
export -f detect_output_format
|
|
export -f parse_json_response
|
|
export -f analyze_response
|
|
export -f update_exit_signals
|
|
export -f log_analysis_summary
|
|
export -f detect_stuck_loop
|