When Claude Code is denied permission to execute commands (e.g., npm install), Ralph now detects this from the permission_denials array in the JSON output and halts the loop immediately with clear guidance for the user. Changes: - Add permission denial detection to parse_json_response() in response_analyzer.sh - Extract permission_denials array from Claude Code JSON output - Track has_permission_denials, permission_denial_count, denied_commands - Add analyze_response() support for permission denial fields - Add permission denial exit condition to should_exit_gracefully() in ralph_loop.sh - Permission denial takes highest priority among exit conditions - Display helpful guidance for updating ALLOWED_TOOLS in .ralphrc - Update circuit breaker with CB_PERMISSION_DENIAL_THRESHOLD=2 - Track consecutive_permission_denials in state file - Open circuit after 2 consecutive loops with permission denials - Add 11 new TDD tests (6 in test_json_parsing.bats, 5 in test_exit_detection.bats) - Update documentation in CLAUDE.md and README.md Test count: 452 (up from 452 - added 11 new tests) Fixes #101 Co-authored-by: Test User <test@example.com>
This commit is contained in:
Frank Bria
GitHub
parent
5cad271eac
commit
328294847d
7 changed files with 519 additions and 6 deletions
|
|
@ -699,3 +699,182 @@ EOF
|
|||
local indicator_count=$(jq '.completion_indicators | length' "$EXIT_SIGNALS_FILE")
|
||||
assert_equal "$indicator_count" "0"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# PERMISSION DENIAL EXIT TESTS (Issue #101)
|
||||
# =============================================================================
|
||||
# When Claude Code is denied permission to run commands, Ralph should detect
|
||||
# this from the permission_denials field and halt the loop to allow user intervention.
|
||||
|
||||
# Helper function with permission denial support
|
||||
should_exit_gracefully_with_denials() {
|
||||
if [[ ! -f "$EXIT_SIGNALS_FILE" ]]; then
|
||||
echo ""
|
||||
return 1
|
||||
fi
|
||||
|
||||
local signals=$(cat "$EXIT_SIGNALS_FILE")
|
||||
|
||||
local recent_test_loops
|
||||
local recent_done_signals
|
||||
local recent_completion_indicators
|
||||
|
||||
recent_test_loops=$(echo "$signals" | jq '.test_only_loops | length' 2>/dev/null || echo "0")
|
||||
recent_done_signals=$(echo "$signals" | jq '.done_signals | length' 2>/dev/null || echo "0")
|
||||
recent_completion_indicators=$(echo "$signals" | jq '.completion_indicators | length' 2>/dev/null || echo "0")
|
||||
|
||||
# Check for permission denials first (highest priority - Issue #101)
|
||||
if [[ -f "$RESPONSE_ANALYSIS_FILE" ]]; then
|
||||
local has_permission_denials=$(jq -r '.analysis.has_permission_denials // false' "$RESPONSE_ANALYSIS_FILE" 2>/dev/null || echo "false")
|
||||
if [[ "$has_permission_denials" == "true" ]]; then
|
||||
echo "permission_denied"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# 1. Too many consecutive test-only loops
|
||||
if [[ $recent_test_loops -ge $MAX_CONSECUTIVE_TEST_LOOPS ]]; then
|
||||
echo "test_saturation"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 2. Multiple "done" signals
|
||||
if [[ $recent_done_signals -ge $MAX_CONSECUTIVE_DONE_SIGNALS ]]; then
|
||||
echo "completion_signals"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 3. Strong completion indicators (only if Claude's EXIT_SIGNAL is true)
|
||||
local claude_exit_signal="false"
|
||||
if [[ -f "$RESPONSE_ANALYSIS_FILE" ]]; then
|
||||
claude_exit_signal=$(jq -r '.analysis.exit_signal // false' "$RESPONSE_ANALYSIS_FILE" 2>/dev/null || echo "false")
|
||||
fi
|
||||
|
||||
if [[ $recent_completion_indicators -ge 2 ]] && [[ "$claude_exit_signal" == "true" ]]; then
|
||||
echo "project_complete"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
return 1
|
||||
}
|
||||
|
||||
# Test 36: Exit on permission denial detected
|
||||
@test "should_exit_gracefully exits on permission_denied" {
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Create response analysis with permission denials
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"output_format": "json",
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"is_stuck": false,
|
||||
"has_progress": false,
|
||||
"files_modified": 0,
|
||||
"confidence_score": 70,
|
||||
"exit_signal": false,
|
||||
"work_summary": "Tried to run npm install but permission denied",
|
||||
"has_permission_denials": true,
|
||||
"permission_denial_count": 1,
|
||||
"denied_commands": ["npm install"]
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
result=$(should_exit_gracefully_with_denials)
|
||||
assert_equal "$result" "permission_denied"
|
||||
}
|
||||
|
||||
# Test 37: No exit when no permission denials
|
||||
@test "should_exit_gracefully continues when no permission denials" {
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Create response analysis without permission denials
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"output_format": "json",
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"is_stuck": false,
|
||||
"has_progress": true,
|
||||
"files_modified": 3,
|
||||
"confidence_score": 70,
|
||||
"exit_signal": false,
|
||||
"work_summary": "Implementing feature",
|
||||
"has_permission_denials": false,
|
||||
"permission_denial_count": 0,
|
||||
"denied_commands": []
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
result=$(should_exit_gracefully_with_denials || true)
|
||||
assert_equal "$result" ""
|
||||
}
|
||||
|
||||
# Test 38: Permission denial takes priority over other signals
|
||||
@test "permission_denied takes priority over test_saturation" {
|
||||
# Set up test saturation condition
|
||||
echo '{"test_only_loops": [1,2,3], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Create response analysis with permission denials
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 3,
|
||||
"analysis": {
|
||||
"is_test_only": true,
|
||||
"has_permission_denials": true,
|
||||
"permission_denial_count": 1,
|
||||
"denied_commands": ["npm install"]
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Permission denied should take priority
|
||||
result=$(should_exit_gracefully_with_denials)
|
||||
assert_equal "$result" "permission_denied"
|
||||
}
|
||||
|
||||
# Test 39: Multiple permission denials detected
|
||||
@test "should_exit_gracefully detects multiple permission denials" {
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"analysis": {
|
||||
"has_permission_denials": true,
|
||||
"permission_denial_count": 3,
|
||||
"denied_commands": ["npm install", "pnpm install", "yarn add lodash"]
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
result=$(should_exit_gracefully_with_denials)
|
||||
assert_equal "$result" "permission_denied"
|
||||
}
|
||||
|
||||
# Test 40: Missing has_permission_denials field defaults to false (backward compat)
|
||||
@test "should_exit_gracefully handles missing permission denial fields" {
|
||||
echo '{"test_only_loops": [], "done_signals": [], "completion_indicators": []}' > "$EXIT_SIGNALS_FILE"
|
||||
|
||||
# Old format response analysis without permission denial fields
|
||||
cat > "$RESPONSE_ANALYSIS_FILE" << 'EOF'
|
||||
{
|
||||
"loop_number": 1,
|
||||
"analysis": {
|
||||
"has_completion_signal": false,
|
||||
"is_test_only": false,
|
||||
"exit_signal": false
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
result=$(should_exit_gracefully_with_denials || true)
|
||||
assert_equal "$result" ""
|
||||
}
|
||||
|
|
|
|||
|
|
@ -919,3 +919,193 @@ EOF
|
|||
local session_id=$(jq -r '.session_id' "$result_file")
|
||||
assert_equal "$session_id" "session-in-result-only"
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# PERMISSION DENIAL DETECTION TESTS (Issue #101)
|
||||
# =============================================================================
|
||||
# Tests for detecting permission_denials from Claude Code JSON output.
|
||||
# When Claude Code is denied permission to execute commands (e.g., npm install),
|
||||
# the JSON output contains a permission_denials array that Ralph should detect.
|
||||
|
||||
@test "parse_json_response detects permission_denials array" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
# Create JSON output with permission denials (as Claude Code outputs)
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"result": "I tried to run npm install but was denied permission.",
|
||||
"sessionId": "session-denied-123",
|
||||
"is_error": false,
|
||||
"permission_denials": [
|
||||
{"tool": "Bash", "command": "npm install", "reason": "Tool not in allowed list"}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
# Should extract has_permission_denials flag
|
||||
local has_denials=$(jq -r '.has_permission_denials' "$result_file")
|
||||
assert_equal "$has_denials" "true"
|
||||
}
|
||||
|
||||
@test "parse_json_response extracts permission_denial_count" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"result": "Multiple commands were denied.",
|
||||
"sessionId": "session-multi-deny",
|
||||
"permission_denials": [
|
||||
{"tool": "Bash", "command": "npm install", "reason": "Not allowed"},
|
||||
{"tool": "Bash", "command": "pnpm install", "reason": "Not allowed"},
|
||||
{"tool": "Bash", "command": "yarn add lodash", "reason": "Not allowed"}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
# Should count denials correctly
|
||||
local denial_count=$(jq -r '.permission_denial_count' "$result_file")
|
||||
assert_equal "$denial_count" "3"
|
||||
}
|
||||
|
||||
@test "parse_json_response extracts denied_commands list" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"result": "Permission denied for npm install",
|
||||
"sessionId": "session-extract-cmds",
|
||||
"permission_denials": [
|
||||
{"tool": "Bash", "command": "npm install express", "reason": "Not allowed"}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
# Should extract the denied commands
|
||||
local denied_cmds=$(jq -r '.denied_commands[0]' "$result_file")
|
||||
[[ "$denied_cmds" == *"npm install"* ]]
|
||||
}
|
||||
|
||||
@test "parse_json_response handles empty permission_denials array" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"result": "All commands executed successfully.",
|
||||
"sessionId": "session-no-denials",
|
||||
"permission_denials": []
|
||||
}
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
# Should set has_permission_denials to false
|
||||
local has_denials=$(jq -r '.has_permission_denials' "$result_file")
|
||||
assert_equal "$has_denials" "false"
|
||||
|
||||
local denial_count=$(jq -r '.permission_denial_count' "$result_file")
|
||||
assert_equal "$denial_count" "0"
|
||||
}
|
||||
|
||||
@test "parse_json_response handles missing permission_denials field (backward compat)" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
# Old format without permission_denials field
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"status": "COMPLETE",
|
||||
"exit_signal": true,
|
||||
"work_type": "IMPLEMENTATION",
|
||||
"files_modified": 5
|
||||
}
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
# Should default to no denials
|
||||
local has_denials=$(jq -r '.has_permission_denials' "$result_file")
|
||||
assert_equal "$has_denials" "false"
|
||||
|
||||
local denial_count=$(jq -r '.permission_denial_count' "$result_file")
|
||||
assert_equal "$denial_count" "0"
|
||||
}
|
||||
|
||||
@test "analyze_response includes permission denial info in analysis result" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
cat > "$output_file" << 'EOF'
|
||||
{
|
||||
"result": "Tried npm install but permission was denied.",
|
||||
"sessionId": "session-analyze-denial",
|
||||
"permission_denials": [
|
||||
{"tool": "Bash", "command": "npm install", "reason": "Tool not allowed"}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
analyze_response "$output_file" 1
|
||||
|
||||
assert_file_exists "$RALPH_DIR/.response_analysis"
|
||||
|
||||
# Should include permission denial in analysis
|
||||
local has_denials=$(jq -r '.analysis.has_permission_denials' "$RALPH_DIR/.response_analysis")
|
||||
assert_equal "$has_denials" "true"
|
||||
|
||||
local denial_count=$(jq -r '.analysis.permission_denial_count' "$RALPH_DIR/.response_analysis")
|
||||
assert_equal "$denial_count" "1"
|
||||
}
|
||||
|
||||
@test "parse_json_response handles Claude CLI array format with permission denials" {
|
||||
local output_file="$LOG_DIR/test_output.log"
|
||||
|
||||
# Claude CLI array format with permission denials in result
|
||||
cat > "$output_file" << 'EOF'
|
||||
[
|
||||
{"type": "system", "subtype": "init", "session_id": "session-array-deny"},
|
||||
{"type": "assistant", "message": {"content": [{"type": "text", "text": "Trying to install..."}]}},
|
||||
{
|
||||
"type": "result",
|
||||
"subtype": "success",
|
||||
"result": "Could not run npm install - permission denied",
|
||||
"session_id": "session-array-deny",
|
||||
"permission_denials": [
|
||||
{"tool": "Bash", "command": "npm install", "reason": "Not in allowed tools"}
|
||||
]
|
||||
}
|
||||
]
|
||||
EOF
|
||||
|
||||
run parse_json_response "$output_file"
|
||||
assert_equal "$status" "0"
|
||||
|
||||
local result_file="$RALPH_DIR/.json_parse_result"
|
||||
[[ -f "$result_file" ]]
|
||||
|
||||
local has_denials=$(jq -r '.has_permission_denials' "$result_file")
|
||||
assert_equal "$has_denials" "true"
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue