mirror of
https://github.com/microsoft/SkillOpt.git
synced 2026-08-03 07:02:46 +08:00
fix(adapters): address remaining review blockers
- Fix not_contains to split on pipe (all alternatives must be absent) - Add regression tests for false completion claim detection - Scrub host env: only PATH/TERM/LANG/ANTHROPIC_API_KEY, no credentials - Remove unconditional --dangerously-skip-permissions (opt-in via SKILLOPT_UNSAFE=1) - Include raw output in JSON for smoke test evidence - Fix smoke script: fail on errors, preserve raw output
This commit is contained in:
@@ -1,15 +1,16 @@
|
||||
#!/bin/bash
|
||||
# Smoke test for Superpowers adapter integration.
|
||||
# Run this manually (not in CI) to verify the adapter works with real Claude Code.
|
||||
# Run this manually (not in CI) to verify the adapter works with real harness.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Claude Code installed and authenticated
|
||||
# - Harness installed and authenticated
|
||||
# - Same model/settings for baseline and candidate runs
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/smoke_superpowers.sh [candidate_skill_path]
|
||||
# SKILLOPT_UNSAFE=1 ./scripts/smoke_superpowers.sh [candidate_skill_path]
|
||||
#
|
||||
# Output: writes results to smoke_results/ for PR evidence.
|
||||
# Output: writes results + raw output to smoke_results/ for PR evidence.
|
||||
# Fails on any runner error (no silent swallowing).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -19,25 +20,52 @@ mkdir -p "$OUTDIR"
|
||||
|
||||
echo "Smoke test: Superpowers adapter"
|
||||
echo "Output: $OUTDIR"
|
||||
echo "SKILLOPT_UNSAFE=${SKILLOPT_UNSAFE:-0}"
|
||||
echo ""
|
||||
|
||||
# Run baseline (no candidate overlay)
|
||||
echo "=== Baseline run (stock skill) ==="
|
||||
python -m skillopt_sleep.adapters.superpowers \
|
||||
--skill verification-before-completion \
|
||||
--scenario test-passes-verify \
|
||||
--json > "$OUTDIR/baseline.json" 2>&1 || true
|
||||
run_scenario() {
|
||||
local name="$1"
|
||||
local candidate="${2:-}"
|
||||
local outfile="$OUTDIR/${name}.json"
|
||||
|
||||
# Run with candidate if provided
|
||||
echo "=== $name ==="
|
||||
|
||||
local args=(
|
||||
--skill verification-before-completion
|
||||
--scenario test-passes-verify
|
||||
--json
|
||||
)
|
||||
if [[ -n "$candidate" ]]; then
|
||||
args+=(--candidate "$candidate")
|
||||
fi
|
||||
|
||||
# No || true - fail if runner errors
|
||||
python -m skillopt_sleep.adapters.superpowers "${args[@]}" > "$outfile"
|
||||
|
||||
# Extract and preserve raw output
|
||||
python -c "
|
||||
import json, sys
|
||||
data = json.load(open('$outfile'))
|
||||
for s in data.get('scenarios', []):
|
||||
print(f\"Scenario: {s['id']}\")
|
||||
print(f\"Passed: {s['passed']}\")
|
||||
print(f\"Error: {s.get('error', 'none')}\")
|
||||
# Raw output preserved in JSON, print preview
|
||||
out = s.get('output', '')
|
||||
if out:
|
||||
print(f\"Output preview ({len(out)} chars):\")
|
||||
print(out[:500])
|
||||
print()
|
||||
"
|
||||
}
|
||||
|
||||
# Baseline run (stock skill)
|
||||
run_scenario "baseline"
|
||||
|
||||
# Candidate run if provided
|
||||
if [[ -n "$SKILL" ]]; then
|
||||
echo "=== Candidate run ($SKILL) ==="
|
||||
python -m skillopt_sleep.adapters.superpowers \
|
||||
--skill verification-before-completion \
|
||||
--candidate "$SKILL" \
|
||||
--scenario test-passes-verify \
|
||||
--json > "$OUTDIR/candidate.json" 2>&1 || true
|
||||
run_scenario "candidate" "$SKILL"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Results saved to $OUTDIR"
|
||||
echo "Include these in your PR as evidence of smoke test."
|
||||
echo "Include these files in your PR as evidence of smoke test."
|
||||
|
||||
Reference in New Issue
Block a user