feat(sleep): benchmark sweep + report tooling; override-aware reflect prompt

- sweep.py: run many (backend, model, seed, transfer-pair) configs sequentially,
  append each result to JSONL incrementally (resumable, interrupt-safe).
- report.py: render the sweep JSONL into a presented Markdown scorecard with
  direct-improvement and cross-model-transfer tables.
- reflect prompt now tells the optimizer its edits are APPENDED (can't delete the
  base skill text), so on a conflict it must write a forceful OVERRIDE rule.
  Diagnosed from a real failure: thorough-analyst (needs <=1200 chars) kept its
  edits rejected because the base "be exhaustive" line won; a verified override
  ("HARD LIMIT ... supersedes") makes Haiku obey (1194/880 chars -> hard=1.0).

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>

skillopt/sleep/backend.py

@@ -331,7 +331,13 @@ class CliBackend(Backend):
             f"{target} document so it stops failing. Each edit MUST be a short, "
             "GENERAL, reusable rule or preference (never task-specific, never an "
             "answer to a single task). If exact failing criteria are listed, your "
-            "edits MUST make future outputs satisfy every one of them. "
+            "edits MUST make future outputs satisfy every one of them.\n"
+            "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
+            "you CANNOT delete the existing instructions above. If the current "
+            f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' "
+            "but outputs must be under a character limit), write an explicit, "
+            "forceful OVERRIDE rule that says it supersedes the conflicting "
+            "instruction. "
             'Return ONLY a JSON array: '
             '[{"op":"add|replace|delete","content":"<rule>","anchor":"<text to replace/delete, optional>","rationale":"<why>"}].\n\n'
             f"# Current {target}\n{cur_doc}\n"

skillopt/sleep/experiments/report.py

@@ -0,0 +1,126 @@
+"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard.
+
+Usage:
+  python -m skillopt.sleep.experiments.report --in docs/sleep/sweep.jsonl \
+      --out docs/sleep/benchmark_report.md
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any, Dict, List
+
+
+def _load(path: str) -> List[Dict[str, Any]]:
+    rows = []
+    if os.path.exists(path):
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        rows.append(json.loads(line))
+                    except Exception:
+                        pass
+    return rows
+
+
+def _fmt_model(backend: str, model: str) -> str:
+    m = model or "default"
+    return f"{backend}:{m}"
+
+
+def render(rows: List[Dict[str, Any]]) -> str:
+    direct = [r for r in rows if r.get("cfg", {}).get("kind") == "direct" and "error" not in r]
+    transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r]
+    errors = [r for r in rows if "error" in r]
+
+    out: List[str] = []
+    out.append("# SkillOpt-Sleep — benchmark report")
+    out.append("")
+    out.append("Auto-generated from `sweep.jsonl`. Benchmark: "
+               "[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` "
+               "(deficient skills, train/held-out split, local rule judge — no judge-API).")
+    out.append("Held-out scores are computed by the harness, not the optimizer.")
+    out.append("")
+
+    # ── direct improvement table ──────────────────────────────────────────
+    out.append("## Direct improvement (optimize and deploy on the same model)")
+    out.append("")
+    out.append("| Backend:Model | Seed | Held-out before | Held-out after | Nights | Tokens |")
+    out.append("|---|---|---|---|---|---|")
+    for r in direct:
+        c = r["cfg"]
+        out.append(f"| {_fmt_model(c['backend'], c.get('model',''))} | {c['seed']} | "
+                   f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | "
+                   f"{r.get('tokens','?')} |")
+    if direct:
+        n_imp = sum(1 for r in direct if r.get("improved"))
+        out.append("")
+        out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**")
+    out.append("")
+
+    # ── transfer table ────────────────────────────────────────────────────
+    if transfer:
+        out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)")
+        out.append("")
+        out.append("The price-difference story: spend cheap tokens optimizing overnight, "
+                   "then deploy the frozen skill on any model with no further optimization.")
+        out.append("")
+        out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |")
+        out.append("|---|---|---|---|---|---|")
+        for r in transfer:
+            c = r["cfg"]
+            s = _fmt_model(c["source_backend"], c.get("source_model", ""))
+            t = _fmt_model(c["target_backend"], c.get("target_model", ""))
+            out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | "
+                       f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |")
+        n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0)
+        out.append("")
+        out.append(f"**{n_pos}/{len(transfer)} transfers were positive** "
+                   "(frozen skill helped a different model than it was optimized on).")
+        out.append("")
+
+    # ── errors (honest reporting) ─────────────────────────────────────────
+    if errors:
+        out.append("## Configs that errored (reported, not hidden)")
+        out.append("")
+        for r in errors:
+            out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}")
+        out.append("")
+
+    out.append("## How to reproduce")
+    out.append("")
+    out.append("```bash")
+    out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals")
+    out.append("python -m skillopt.sleep.experiments.sweep --plan full \\")
+    out.append("    --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl")
+    out.append("python -m skillopt.sleep.experiments.report \\")
+    out.append("    --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md")
+    out.append("```")
+    out.append("")
+    return "\n".join(out)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report")
+    ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl")
+    ap.add_argument("--out", default="docs/sleep/benchmark_report.md")
+    args = ap.parse_args(argv)
+
+    rows = _load(args.inp)
+    if not rows:
+        print(f"no rows in {args.inp}", file=sys.stderr)
+        return 1
+    md = render(rows)
+    os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
+    with open(args.out, "w") as f:
+        f.write(md)
+    print(f"wrote {args.out} ({len(rows)} rows)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

skillopt/sleep/experiments/sweep.py

@@ -0,0 +1,147 @@
+"""SkillOpt-Sleep — benchmark sweep driver.
+
+Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in
+one process, appending each result to a JSONL file as it finishes. Designed to
+run unattended in the background; safe to interrupt (already-written rows
+survive) and resume (skip configs whose row already exists).
+
+Then `report.py` turns the JSONL into a presented Markdown scorecard.
+
+Usage:
+  python -m skillopt.sleep.experiments.sweep --plan quick   --out docs/sleep/sweep.jsonl
+  python -m skillopt.sleep.experiments.sweep --plan full    --out docs/sleep/sweep.jsonl
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, List
+
+from skillopt.sleep.backend import get_backend
+from skillopt.sleep.experiments.gbrain_bench import find_data_root, load_seed
+from skillopt.sleep.experiments.run_gbrain import run_seed as bench_seed
+from skillopt.sleep.experiments.run_transfer import run_seed as transfer_seed
+
+
+# Plans: lists of config dicts. Kept small per-run to bound cost/latency.
+def _direct_cfg(backend, model, seed, nights=2):
+    return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights}
+
+
+def _transfer_cfg(sb, sm, tb, tm, seed, nights=2):
+    return {"kind": "transfer", "source_backend": sb, "source_model": sm,
+            "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights}
+
+
+PLANS: Dict[str, List[Dict[str, Any]]] = {
+    # one cheap seed each, both backends — fast sanity
+    "quick": [
+        _direct_cfg("claude", "haiku", "brief-writer", 1),
+        _direct_cfg("codex", "", "brief-writer", 2),
+    ],
+    # direct results across seeds + models, both backends
+    "direct": [
+        _direct_cfg("claude", "haiku", "brief-writer"),
+        _direct_cfg("claude", "haiku", "advisor"),
+        _direct_cfg("claude", "sonnet", "brief-writer"),
+        _direct_cfg("codex", "", "brief-writer"),
+        _direct_cfg("codex", "", "advisor"),
+    ],
+    # the price-difference story: optimize cheap, deploy expensive (and reverse)
+    "transfer": [
+        _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"),
+        _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"),
+        _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"),
+    ],
+}
+PLANS["full"] = PLANS["direct"] + PLANS["transfer"]
+
+
+def _cfg_key(c: Dict[str, Any]) -> str:
+    return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False)
+
+
+def _load_done(out_path: str) -> set:
+    done = set()
+    if os.path.exists(out_path):
+        with open(out_path) as f:
+            for line in f:
+                try:
+                    row = json.loads(line)
+                    if "cfg_key" in row:
+                        done.add(row["cfg_key"])
+                except Exception:
+                    pass
+    return done
+
+
+def _append(out_path: str, row: Dict[str, Any]) -> None:
+    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
+    with open(out_path, "a") as f:
+        f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str,
+            limit_replay: int, limit_holdout: int) -> Dict[str, Any]:
+    seed = cfg["seed"]
+    skill, tasks = load_seed(data_root, seed)
+    t0 = time.time()
+    if cfg["kind"] == "direct":
+        be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path)
+        r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"],
+                       limit_replay=limit_replay, limit_holdout=limit_holdout)
+        out = {"baseline": r["held_out_before"], "after": r["held_out_after"],
+               "improved": r["improved"], "tokens": be.tokens_used()}
+    else:
+        src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path)
+        tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path)
+        r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"],
+                          edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout,
+                          do_direct=False)
+        out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"],
+               "transfer_gain": r["transfer_gain"],
+               "tokens": src.tokens_used() + tgt.tokens_used()}
+    out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)})
+    return out
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep")
+    ap.add_argument("--plan", default="quick", choices=list(PLANS.keys()))
+    ap.add_argument("--out", default="docs/sleep/sweep.jsonl")
+    ap.add_argument("--data-root", default="")
+    ap.add_argument("--codex-path", default="")
+    ap.add_argument("--limit-replay", type=int, default=3)
+    ap.add_argument("--limit-holdout", type=int, default=3)
+    args = ap.parse_args(argv)
+
+    data_root = find_data_root(args.data_root)
+    if not data_root:
+        print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr)
+        return 2
+
+    plan = PLANS[args.plan]
+    done = _load_done(args.out)
+    print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}")
+    for i, cfg in enumerate(plan, 1):
+        key = _cfg_key(cfg)
+        if key in done:
+            print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}")
+            continue
+        print(f"[sweep] ({i}/{len(plan)}) running: {cfg}")
+        try:
+            row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout)
+        except Exception as e:  # never let one config kill the sweep
+            row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"}
+        _append(args.out, row)
+        print(f"[sweep]   -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}")
+    print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())