From a0419bfdbbc1130f599055e10324ca4736d36f11 Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Mon, 8 Jun 2026 14:31:51 +0000 Subject: [PATCH] feat(sleep): benchmark sweep + report tooling; override-aware reflect prompt - sweep.py: run many (backend, model, seed, transfer-pair) configs sequentially, append each result to JSONL incrementally (resumable, interrupt-safe). - report.py: render the sweep JSONL into a presented Markdown scorecard with direct-improvement and cross-model-transfer tables. - reflect prompt now tells the optimizer its edits are APPENDED (can't delete the base skill text), so on a conflict it must write a forceful OVERRIDE rule. Diagnosed from a real failure: thorough-analyst (needs <=1200 chars) kept its edits rejected because the base "be exhaustive" line won; a verified override ("HARD LIMIT ... supersedes") makes Haiku obey (1194/880 chars -> hard=1.0). Co-Authored-By: Claude Opus 4 --- skillopt/sleep/backend.py | 8 +- skillopt/sleep/experiments/report.py | 126 +++++++++++++++++++++++ skillopt/sleep/experiments/sweep.py | 147 +++++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 skillopt/sleep/experiments/report.py create mode 100644 skillopt/sleep/experiments/sweep.py diff --git a/skillopt/sleep/backend.py b/skillopt/sleep/backend.py index bdc8e57..078a9d9 100644 --- a/skillopt/sleep/backend.py +++ b/skillopt/sleep/backend.py @@ -331,7 +331,13 @@ class CliBackend(Backend): f"{target} document so it stops failing. Each edit MUST be a short, " "GENERAL, reusable rule or preference (never task-specific, never an " "answer to a single task). If exact failing criteria are listed, your " - "edits MUST make future outputs satisfy every one of them. " + "edits MUST make future outputs satisfy every one of them.\n" + "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; " + "you CANNOT delete the existing instructions above. If the current " + f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' " + "but outputs must be under a character limit), write an explicit, " + "forceful OVERRIDE rule that says it supersedes the conflicting " + "instruction. " 'Return ONLY a JSON array: ' '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n' f"# Current {target}\n{cur_doc}\n" diff --git a/skillopt/sleep/experiments/report.py b/skillopt/sleep/experiments/report.py new file mode 100644 index 0000000..2bde5ad --- /dev/null +++ b/skillopt/sleep/experiments/report.py @@ -0,0 +1,126 @@ +"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard. + +Usage: + python -m skillopt.sleep.experiments.report --in docs/sleep/sweep.jsonl \ + --out docs/sleep/benchmark_report.md +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List + + +def _load(path: str) -> List[Dict[str, Any]]: + rows = [] + if os.path.exists(path): + with open(path) as f: + for line in f: + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except Exception: + pass + return rows + + +def _fmt_model(backend: str, model: str) -> str: + m = model or "default" + return f"{backend}:{m}" + + +def render(rows: List[Dict[str, Any]]) -> str: + direct = [r for r in rows if r.get("cfg", {}).get("kind") == "direct" and "error" not in r] + transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r] + errors = [r for r in rows if "error" in r] + + out: List[str] = [] + out.append("# SkillOpt-Sleep — benchmark report") + out.append("") + out.append("Auto-generated from `sweep.jsonl`. Benchmark: " + "[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` " + "(deficient skills, train/held-out split, local rule judge — no judge-API).") + out.append("Held-out scores are computed by the harness, not the optimizer.") + out.append("") + + # ── direct improvement table ────────────────────────────────────────── + out.append("## Direct improvement (optimize and deploy on the same model)") + out.append("") + out.append("| Backend:Model | Seed | Held-out before | Held-out after | Nights | Tokens |") + out.append("|---|---|---|---|---|---|") + for r in direct: + c = r["cfg"] + out.append(f"| {_fmt_model(c['backend'], c.get('model',''))} | {c['seed']} | " + f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | " + f"{r.get('tokens','?')} |") + if direct: + n_imp = sum(1 for r in direct if r.get("improved")) + out.append("") + out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**") + out.append("") + + # ── transfer table ──────────────────────────────────────────────────── + if transfer: + out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)") + out.append("") + out.append("The price-difference story: spend cheap tokens optimizing overnight, " + "then deploy the frozen skill on any model with no further optimization.") + out.append("") + out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |") + out.append("|---|---|---|---|---|---|") + for r in transfer: + c = r["cfg"] + s = _fmt_model(c["source_backend"], c.get("source_model", "")) + t = _fmt_model(c["target_backend"], c.get("target_model", "")) + out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | " + f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |") + n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0) + out.append("") + out.append(f"**{n_pos}/{len(transfer)} transfers were positive** " + "(frozen skill helped a different model than it was optimized on).") + out.append("") + + # ── errors (honest reporting) ───────────────────────────────────────── + if errors: + out.append("## Configs that errored (reported, not hidden)") + out.append("") + for r in errors: + out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}") + out.append("") + + out.append("## How to reproduce") + out.append("") + out.append("```bash") + out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals") + out.append("python -m skillopt.sleep.experiments.sweep --plan full \\") + out.append(" --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl") + out.append("python -m skillopt.sleep.experiments.report \\") + out.append(" --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md") + out.append("```") + out.append("") + return "\n".join(out) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report") + ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl") + ap.add_argument("--out", default="docs/sleep/benchmark_report.md") + args = ap.parse_args(argv) + + rows = _load(args.inp) + if not rows: + print(f"no rows in {args.inp}", file=sys.stderr) + return 1 + md = render(rows) + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + with open(args.out, "w") as f: + f.write(md) + print(f"wrote {args.out} ({len(rows)} rows)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt/sleep/experiments/sweep.py b/skillopt/sleep/experiments/sweep.py new file mode 100644 index 0000000..b0d7db1 --- /dev/null +++ b/skillopt/sleep/experiments/sweep.py @@ -0,0 +1,147 @@ +"""SkillOpt-Sleep — benchmark sweep driver. + +Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in +one process, appending each result to a JSONL file as it finishes. Designed to +run unattended in the background; safe to interrupt (already-written rows +survive) and resume (skip configs whose row already exists). + +Then `report.py` turns the JSONL into a presented Markdown scorecard. + +Usage: + python -m skillopt.sleep.experiments.sweep --plan quick --out docs/sleep/sweep.jsonl + python -m skillopt.sleep.experiments.sweep --plan full --out docs/sleep/sweep.jsonl +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from typing import Any, Dict, List + +from skillopt.sleep.backend import get_backend +from skillopt.sleep.experiments.gbrain_bench import find_data_root, load_seed +from skillopt.sleep.experiments.run_gbrain import run_seed as bench_seed +from skillopt.sleep.experiments.run_transfer import run_seed as transfer_seed + + +# Plans: lists of config dicts. Kept small per-run to bound cost/latency. +def _direct_cfg(backend, model, seed, nights=2): + return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights} + + +def _transfer_cfg(sb, sm, tb, tm, seed, nights=2): + return {"kind": "transfer", "source_backend": sb, "source_model": sm, + "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights} + + +PLANS: Dict[str, List[Dict[str, Any]]] = { + # one cheap seed each, both backends — fast sanity + "quick": [ + _direct_cfg("claude", "haiku", "brief-writer", 1), + _direct_cfg("codex", "", "brief-writer", 2), + ], + # direct results across seeds + models, both backends + "direct": [ + _direct_cfg("claude", "haiku", "brief-writer"), + _direct_cfg("claude", "haiku", "advisor"), + _direct_cfg("claude", "sonnet", "brief-writer"), + _direct_cfg("codex", "", "brief-writer"), + _direct_cfg("codex", "", "advisor"), + ], + # the price-difference story: optimize cheap, deploy expensive (and reverse) + "transfer": [ + _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"), + _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"), + _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"), + _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"), + ], +} +PLANS["full"] = PLANS["direct"] + PLANS["transfer"] + + +def _cfg_key(c: Dict[str, Any]) -> str: + return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False) + + +def _load_done(out_path: str) -> set: + done = set() + if os.path.exists(out_path): + with open(out_path) as f: + for line in f: + try: + row = json.loads(line) + if "cfg_key" in row: + done.add(row["cfg_key"]) + except Exception: + pass + return done + + +def _append(out_path: str, row: Dict[str, Any]) -> None: + os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) + with open(out_path, "a") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str, + limit_replay: int, limit_holdout: int) -> Dict[str, Any]: + seed = cfg["seed"] + skill, tasks = load_seed(data_root, seed) + t0 = time.time() + if cfg["kind"] == "direct": + be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path) + r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"], + limit_replay=limit_replay, limit_holdout=limit_holdout) + out = {"baseline": r["held_out_before"], "after": r["held_out_after"], + "improved": r["improved"], "tokens": be.tokens_used()} + else: + src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path) + tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path) + r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"], + edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout, + do_direct=False) + out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"], + "transfer_gain": r["transfer_gain"], + "tokens": src.tokens_used() + tgt.tokens_used()} + out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)}) + return out + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep") + ap.add_argument("--plan", default="quick", choices=list(PLANS.keys())) + ap.add_argument("--out", default="docs/sleep/sweep.jsonl") + ap.add_argument("--data-root", default="") + ap.add_argument("--codex-path", default="") + ap.add_argument("--limit-replay", type=int, default=3) + ap.add_argument("--limit-holdout", type=int, default=3) + args = ap.parse_args(argv) + + data_root = find_data_root(args.data_root) + if not data_root: + print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr) + return 2 + + plan = PLANS[args.plan] + done = _load_done(args.out) + print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}") + for i, cfg in enumerate(plan, 1): + key = _cfg_key(cfg) + if key in done: + print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}") + continue + print(f"[sweep] ({i}/{len(plan)}) running: {cfg}") + try: + row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout) + except Exception as e: # never let one config kill the sweep + row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"} + _append(args.out, row) + print(f"[sweep] -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}") + print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())