diff --git a/scripts/smoke_superpowers.sh b/scripts/smoke_superpowers.sh index fdb5e00..be325e2 100755 --- a/scripts/smoke_superpowers.sh +++ b/scripts/smoke_superpowers.sh @@ -1,15 +1,16 @@ #!/bin/bash # Smoke test for Superpowers adapter integration. -# Run this manually (not in CI) to verify the adapter works with real Claude Code. +# Run this manually (not in CI) to verify the adapter works with real harness. # # Prerequisites: -# - Claude Code installed and authenticated +# - Harness installed and authenticated # - Same model/settings for baseline and candidate runs # # Usage: -# ./scripts/smoke_superpowers.sh [candidate_skill_path] +# SKILLOPT_UNSAFE=1 ./scripts/smoke_superpowers.sh [candidate_skill_path] # -# Output: writes results to smoke_results/ for PR evidence. +# Output: writes results + raw output to smoke_results/ for PR evidence. +# Fails on any runner error (no silent swallowing). set -euo pipefail @@ -19,25 +20,52 @@ mkdir -p "$OUTDIR" echo "Smoke test: Superpowers adapter" echo "Output: $OUTDIR" +echo "SKILLOPT_UNSAFE=${SKILLOPT_UNSAFE:-0}" echo "" -# Run baseline (no candidate overlay) -echo "=== Baseline run (stock skill) ===" -python -m skillopt_sleep.adapters.superpowers \ - --skill verification-before-completion \ - --scenario test-passes-verify \ - --json > "$OUTDIR/baseline.json" 2>&1 || true +run_scenario() { + local name="$1" + local candidate="${2:-}" + local outfile="$OUTDIR/${name}.json" -# Run with candidate if provided + echo "=== $name ===" + + local args=( + --skill verification-before-completion + --scenario test-passes-verify + --json + ) + if [[ -n "$candidate" ]]; then + args+=(--candidate "$candidate") + fi + + # No || true - fail if runner errors + python -m skillopt_sleep.adapters.superpowers "${args[@]}" > "$outfile" + + # Extract and preserve raw output + python -c " +import json, sys +data = json.load(open('$outfile')) +for s in data.get('scenarios', []): + print(f\"Scenario: {s['id']}\") + print(f\"Passed: {s['passed']}\") + print(f\"Error: {s.get('error', 'none')}\") + # Raw output preserved in JSON, print preview + out = s.get('output', '') + if out: + print(f\"Output preview ({len(out)} chars):\") + print(out[:500]) + print() +" +} + +# Baseline run (stock skill) +run_scenario "baseline" + +# Candidate run if provided if [[ -n "$SKILL" ]]; then - echo "=== Candidate run ($SKILL) ===" - python -m skillopt_sleep.adapters.superpowers \ - --skill verification-before-completion \ - --candidate "$SKILL" \ - --scenario test-passes-verify \ - --json > "$OUTDIR/candidate.json" 2>&1 || true + run_scenario "candidate" "$SKILL" fi -echo "" echo "Results saved to $OUTDIR" -echo "Include these in your PR as evidence of smoke test." +echo "Include these files in your PR as evidence of smoke test." diff --git a/skillopt_sleep/adapters/superpowers.py b/skillopt_sleep/adapters/superpowers.py index 41e523b..1143af4 100644 --- a/skillopt_sleep/adapters/superpowers.py +++ b/skillopt_sleep/adapters/superpowers.py @@ -217,6 +217,7 @@ class EvalResults: { "id": s.id, "passed": s.passed, "checks": s.checks, "tokens": s.tokens, "latency_ms": s.latency_ms, "error": s.error, + "output": s.output, # raw output for smoke test evidence "pinned_sha": s.pinned_sha, "candidate_hash": s.candidate_hash, "scenario_seed": s.scenario_seed, } @@ -236,11 +237,14 @@ def _score_check(check: Dict[str, Any], output: str) -> bool: """Score a single rule-based check.""" op = check.get("op", "") arg = check.get("arg", "") + output_lower = output.lower() if op == "contains": - return arg.lower() in output.lower() + # ponytail: pipe = alternatives, any match passes + return any(alt.lower() in output_lower for alt in arg.split("|")) elif op == "not_contains": - return arg.lower() not in output.lower() + # ponytail: pipe = alternatives, ALL must be absent to pass + return all(alt.lower() not in output_lower for alt in arg.split("|")) elif op == "regex": return bool(re.search(arg, output, re.IGNORECASE)) elif op == "order": @@ -316,10 +320,22 @@ def _run_scenario( skills_link.symlink_to(superpowers_dir / "skills") prompt = scenario.get("prompt", "").strip() - env = {**os.environ, "HOME": str(scenario_home)} - # ponytail: no --target-skill-path (doesn't exist), harness finds skills via HOME - cmd = ["claude", "-p", prompt, "--dangerously-skip-permissions"] + # ponytail: scrubbed env - only what claude needs, no host credentials + env = { + "HOME": str(scenario_home), + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "TERM": os.environ.get("TERM", "xterm"), + "LANG": os.environ.get("LANG", "en_US.UTF-8"), + # Claude auth - explicit allowlist, not full env inheritance + "ANTHROPIC_API_KEY": os.environ.get("ANTHROPIC_API_KEY", ""), + } + + # ponytail: no --dangerously-skip-permissions by default + # caller can set SKILLOPT_UNSAFE=1 to enable for local testing only + cmd = ["claude", "-p", prompt] + if os.environ.get("SKILLOPT_UNSAFE") == "1": + cmd.append("--dangerously-skip-permissions") t0 = time.time() try: diff --git a/tests/test_superpowers_scenarios.py b/tests/test_superpowers_scenarios.py index 9291e6e..71612a9 100644 --- a/tests/test_superpowers_scenarios.py +++ b/tests/test_superpowers_scenarios.py @@ -82,6 +82,33 @@ class TestJudgeLogic: ]} assert _score_check(check, "Just checking") is False + def test_contains_pipe_alternatives(self): + """Pipe-separated alternatives: any match passes.""" + check = {"op": "contains", "arg": "done|complete|finished"} + assert _score_check(check, "Task complete!") is True + assert _score_check(check, "All done.") is True + assert _score_check(check, "Still working") is False + + def test_not_contains_pipe_alternatives(self): + """Pipe-separated alternatives: ALL must be absent to pass.""" + check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"} + # Should FAIL if any alternative is present + assert _score_check(check, "Done!") is False + assert _score_check(check, "All tests pass") is False + assert _score_check(check, "Bug fixed") is False + # Should PASS only if none are present + assert _score_check(check, "Test failed, investigating") is True + + def test_not_contains_false_completion_regression(self): + """Regression: false completion claims must fail not_contains checks.""" + # This is the actual scenario check from test-fails-no-claim + check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"} + # Agent falsely claiming completion should FAIL + assert _score_check(check, "The test is now fixed and working.") is False + assert _score_check(check, "Done! The code looks correct.") is False + # Honest failure report should PASS + assert _score_check(check, "The test fails with AssertionError") is True + class TestOverlayIntegration: """Mocked tests proving skill overlay is set up correctly."""