fix(adapters): address remaining review blockers

- Fix not_contains to split on pipe (all alternatives must be absent)
- Add regression tests for false completion claim detection
- Scrub host env: only PATH/TERM/LANG/ANTHROPIC_API_KEY, no credentials
- Remove unconditional --dangerously-skip-permissions (opt-in via SKILLOPT_UNSAFE=1)
- Include raw output in JSON for smoke test evidence
- Fix smoke script: fail on errors, preserve raw output
This commit is contained in:
NovusEdge
2026-07-20 23:58:25 +03:00
parent 17ac3362de
commit 5a3050d768
3 changed files with 95 additions and 24 deletions

View File

@@ -1,15 +1,16 @@
#!/bin/bash
# Smoke test for Superpowers adapter integration.
# Run this manually (not in CI) to verify the adapter works with real Claude Code.
# Run this manually (not in CI) to verify the adapter works with real harness.
#
# Prerequisites:
# - Claude Code installed and authenticated
# - Harness installed and authenticated
# - Same model/settings for baseline and candidate runs
#
# Usage:
# ./scripts/smoke_superpowers.sh [candidate_skill_path]
# SKILLOPT_UNSAFE=1 ./scripts/smoke_superpowers.sh [candidate_skill_path]
#
# Output: writes results to smoke_results/ for PR evidence.
# Output: writes results + raw output to smoke_results/ for PR evidence.
# Fails on any runner error (no silent swallowing).
set -euo pipefail
@@ -19,25 +20,52 @@ mkdir -p "$OUTDIR"
echo "Smoke test: Superpowers adapter"
echo "Output: $OUTDIR"
echo "SKILLOPT_UNSAFE=${SKILLOPT_UNSAFE:-0}"
echo ""
# Run baseline (no candidate overlay)
echo "=== Baseline run (stock skill) ==="
python -m skillopt_sleep.adapters.superpowers \
--skill verification-before-completion \
--scenario test-passes-verify \
--json > "$OUTDIR/baseline.json" 2>&1 || true
run_scenario() {
local name="$1"
local candidate="${2:-}"
local outfile="$OUTDIR/${name}.json"
# Run with candidate if provided
echo "=== $name ==="
local args=(
--skill verification-before-completion
--scenario test-passes-verify
--json
)
if [[ -n "$candidate" ]]; then
args+=(--candidate "$candidate")
fi
# No || true - fail if runner errors
python -m skillopt_sleep.adapters.superpowers "${args[@]}" > "$outfile"
# Extract and preserve raw output
python -c "
import json, sys
data = json.load(open('$outfile'))
for s in data.get('scenarios', []):
print(f\"Scenario: {s['id']}\")
print(f\"Passed: {s['passed']}\")
print(f\"Error: {s.get('error', 'none')}\")
# Raw output preserved in JSON, print preview
out = s.get('output', '')
if out:
print(f\"Output preview ({len(out)} chars):\")
print(out[:500])
print()
"
}
# Baseline run (stock skill)
run_scenario "baseline"
# Candidate run if provided
if [[ -n "$SKILL" ]]; then
echo "=== Candidate run ($SKILL) ==="
python -m skillopt_sleep.adapters.superpowers \
--skill verification-before-completion \
--candidate "$SKILL" \
--scenario test-passes-verify \
--json > "$OUTDIR/candidate.json" 2>&1 || true
run_scenario "candidate" "$SKILL"
fi
echo ""
echo "Results saved to $OUTDIR"
echo "Include these in your PR as evidence of smoke test."
echo "Include these files in your PR as evidence of smoke test."