diff --git a/skillopt/sleep/config.py b/skillopt/sleep/config.py index 88b969c..7541527 100644 --- a/skillopt/sleep/config.py +++ b/skillopt/sleep/config.py @@ -30,10 +30,13 @@ DEFAULTS: Dict[str, Any] = { # ── budgets ──────────────────────────────────────────────────────────── "max_tasks_per_night": 40, "max_tokens_per_night": 400_000, - "holdout_fraction": 0.34, # fraction of mined tasks reserved for the gate + "holdout_fraction": 0.34, # legacy alias for val_fraction + "val_fraction": 0.34, # real tasks reserved to gate updates + "test_fraction": 0.0, # real tasks reserved as the final held-out measure # ── optimizer ────────────────────────────────────────────────────────── "backend": "mock", # "mock" | "claude" | "codex" "model": "", # backend-specific; "" => backend default + "gate_mode": "on", # "on" (validation-gated) | "off" (greedy, no hard filter) "codex_path": "", # "" => auto-detect the real @openai/codex binary "edit_budget": 4, # textual learning rate (max edits/night) "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) diff --git a/skillopt/sleep/consolidate.py b/skillopt/sleep/consolidate.py index 0a679d6..328345a 100644 --- a/skillopt/sleep/consolidate.py +++ b/skillopt/sleep/consolidate.py @@ -52,14 +52,26 @@ class ConsolidationResult: def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]: - replay = [t for t in tasks if t.split == "replay"] - holdout = [t for t in tasks if t.split == "holdout"] - # be robust if a split is empty - if not replay: - replay = tasks - if not holdout: - holdout = tasks - return replay, holdout + """Return (train_tasks, val_tasks). + + train drives reflect; val gates updates. test is held out entirely from + consolidation and is scored by the caller. Accepts legacy split names + (replay->train, holdout->val) for robustness. + """ + def _norm(s: str) -> str: + return {"replay": "train", "holdout": "val"}.get(s, s) + + train = [t for t in tasks if _norm(t.split) == "train"] + val = [t for t in tasks if _norm(t.split) == "val"] + # be robust if a split is empty: fall back so a night still does something, + # but never silently use test as val. + test = [t for t in tasks if _norm(t.split) == "test"] + if not val: + # prefer train as the gate reference over nothing; last resort all-but-test + val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks + if not train: + train = val + return train, val def consolidate( @@ -71,25 +83,30 @@ def consolidate( edit_budget: int = 4, gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, + gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy) evolve_skill: bool = True, evolve_memory: bool = True, night: int = 1, ) -> ConsolidationResult: """Run one consolidation epoch: reflect -> bounded edit -> gate. - Skill and memory are evolved in sequence (skill first if both enabled), - each behind the same held-out gate, so each document only changes when it - demonstrably helps on the user's held-out tasks. - """ - replay_tasks, holdout_tasks = _split(tasks) + train tasks drive reflect; val tasks gate the update (test is held out by the + caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve + requirement) — the user opts out of hard filtering — but val scores are still + recorded so the report shows whether quality moved. - # ── baseline on held-out slice (the gate reference) ────────────────── - base_pairs = replay_batch(backend, holdout_tasks, skill, memory) + Skill and memory are evolved in sequence (skill first if both enabled). + """ + train_tasks, val_tasks = _split(tasks) + gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"} + + # ── baseline on the VAL slice (the gate reference) ──────────────────── + base_pairs = replay_batch(backend, val_tasks, skill, memory) base_hard, base_soft = aggregate_scores(base_pairs) base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) - # ── reflect over replay-split failures/successes ───────────────────── - train_pairs = replay_batch(backend, replay_tasks, skill, memory) + # ── reflect over TRAIN-split failures/successes ─────────────────────── + train_pairs = replay_batch(backend, train_tasks, skill, memory) failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0] successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0] @@ -104,14 +121,15 @@ def consolidate( new_doc, applied = apply_edits(doc, edits) if not applied: return doc - # evaluate candidate on the held-out slice + # score the candidate on the VAL slice trial_skill = new_doc if which == "skill" else cand_skill trial_memory = new_doc if which == "memory" else cand_memory - pairs = replay_batch(backend, holdout_tasks, trial_skill, trial_memory) + pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory) h, s = aggregate_scores(pairs) cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight) - if cand_score > base_score: - base_score = cand_score + # gate OFF: accept greedily (no regression check); gate ON: strict improve + if gate_off or cand_score > base_score: + base_score = max(base_score, cand_score) all_applied.extend(applied) return new_doc all_rejected.extend(applied) @@ -126,7 +144,7 @@ def consolidate( if evolve_memory: # re-evaluate failures under the (possibly improved) skill - train_pairs2 = replay_batch(backend, replay_tasks, cand_skill, cand_memory) + train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory) failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0] successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0] edits_m = backend.reflect( @@ -135,19 +153,29 @@ def consolidate( ) cand_memory = _gate_apply(cand_memory, edits_m, "memory") - # ── final gate decision (use the repo gate for the canonical action) ── - final_pairs = replay_batch(backend, holdout_tasks, cand_skill, cand_memory) + # ── final decision, scored on the VAL slice ─────────────────────────── + final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory) final_hard, final_soft = aggregate_scores(final_pairs) final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight) + base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) - if _HAVE_REPO_GATE: + if gate_off: + # greedy mode: keep whatever edits we applied; report quality movement + accepted = bool(all_applied) + if final_score > base_gate_score: + action = "greedy_improved" + elif final_score < base_gate_score: + action = "greedy_regressed" + else: + action = "greedy_flat" if all_applied else "greedy_noop" + elif _HAVE_REPO_GATE: gate = evaluate_gate( candidate_skill=cand_skill, cand_hard=final_hard, current_skill=skill, - current_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight), + current_score=base_gate_score, best_skill=skill, - best_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight), + best_score=base_gate_score, best_step=night - 1, global_step=night, cand_soft=final_soft, @@ -155,17 +183,15 @@ def consolidate( mixed_weight=gate_mixed_weight, ) action = gate.action + accepted = bool(all_applied) and final_score > base_gate_score else: - action = "accept" if final_score > base_soft else "reject" - - accepted = bool(all_applied) and final_score > select_gate_score( - base_hard, base_soft, gate_metric, gate_mixed_weight - ) + action = "accept" if final_score > base_gate_score else "reject" + accepted = bool(all_applied) and final_score > base_gate_score return ConsolidationResult( accepted=accepted, gate_action=action, - baseline_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight), + baseline_score=base_gate_score, candidate_score=final_score, new_skill=cand_skill if accepted else skill, new_memory=cand_memory if accepted else memory, diff --git a/skillopt/sleep/cycle.py b/skillopt/sleep/cycle.py index a410e79..4efc81b 100644 --- a/skillopt/sleep/cycle.py +++ b/skillopt/sleep/cycle.py @@ -175,6 +175,7 @@ def run_sleep_cycle( edit_budget=cfg.get("edit_budget", 4), gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), evolve_memory=cfg.get("evolve_memory", True), night=night, diff --git a/skillopt/sleep/experiments/gbrain_bench.py b/skillopt/sleep/experiments/gbrain_bench.py index 7f4dd25..efe0ff6 100644 --- a/skillopt/sleep/experiments/gbrain_bench.py +++ b/skillopt/sleep/experiments/gbrain_bench.py @@ -63,8 +63,17 @@ def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord: ) -def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]: - """Return (deficient_skill_md, tasks) for one gbrain seed.""" +def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34, + split_seed: int = 42) -> Tuple[str, List[TaskRecord]]: + """Return (deficient_skill_md, tasks) for one gbrain seed. + + Faithful split mapping: + * gbrain held-out.jsonl -> our ``test`` (the true final measure) + * gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val`` + (val gates updates; train drives reflect) + All tasks are origin='real' (gbrain provides no synthetic tasks). + """ + import hashlib sub = SEED_DIRS.get(seed, seed) seed_dir = os.path.join(data_root, sub) skill_path = os.path.join(seed_dir, "SKILL.md") @@ -73,10 +82,21 @@ def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]: with open(skill_path, encoding="utf-8") as f: skill = f.read() tasks: List[TaskRecord] = [] + # benchmark pool -> train/val + val_cut = int(round(val_fraction * 100)) for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")): - tasks.append(_to_task(rec, seed=seed, split="replay")) + t = _to_task(rec, seed=seed, split="train") + bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100 + t.split = "val" if bucket < val_cut else "train" + tasks.append(t) + # held-out -> test for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")): - tasks.append(_to_task(rec, seed=seed, split="holdout")) + tasks.append(_to_task(rec, seed=seed, split="test")) + # guarantee a non-empty val + if not any(t.split == "val" for t in tasks): + train_only = [t for t in tasks if t.split == "train"] + if train_only: + train_only[0].split = "val" return skill, tasks diff --git a/skillopt/sleep/experiments/run_experiment.py b/skillopt/sleep/experiments/run_experiment.py index 7e12acb..385b0a1 100644 --- a/skillopt/sleep/experiments/run_experiment.py +++ b/skillopt/sleep/experiments/run_experiment.py @@ -42,7 +42,8 @@ from skillopt.sleep.types import TaskRecord def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str, metric: str = "mixed", w: float = 0.5) -> float: from skillopt.sleep.consolidate import select_gate_score - holdout = [t for t in tasks if t.split == "holdout"] or tasks + # the persona experiment uses a 2-way split (train/val, no test); score on val + holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks pairs = replay_batch(backend, holdout, skill, memory) h, s = aggregate_scores(pairs) return select_gate_score(h, s, metric, w) diff --git a/skillopt/sleep/experiments/run_gbrain.py b/skillopt/sleep/experiments/run_gbrain.py index 63feec4..f29ef57 100644 --- a/skillopt/sleep/experiments/run_gbrain.py +++ b/skillopt/sleep/experiments/run_gbrain.py @@ -34,47 +34,56 @@ from skillopt.sleep.experiments.gbrain_bench import ( from skillopt.sleep.replay import aggregate_scores, replay_batch -def _score(backend, tasks, skill, memory, split="holdout", metric="mixed", w=0.5): - sub = [t for t in tasks if t.split == split] or tasks +def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5): + sub = [t for t in tasks if t.split == split] + if not sub: # fall back to val, then everything, so we never score on nothing + sub = [t for t in tasks if t.split == "val"] or tasks pairs = replay_batch(backend, sub, skill, memory) h, s = aggregate_scores(pairs) return h, s, select_gate_score(h, s, metric, w) def run_seed(backend, seed: str, skill: str, tasks: List, *, - nights: int = 3, edit_budget: int = 4, + nights: int = 3, edit_budget: int = 4, gate_mode: str = "on", limit_replay: int = 0, limit_holdout: int = 0) -> dict: memory = "" - # optionally cap each split to control API cost / latency + # optionally cap each split to control API cost / latency. + # limit_replay caps train; limit_holdout caps BOTH val and test. if limit_replay or limit_holdout: - replay = [t for t in tasks if t.split == "replay"] - holdout = [t for t in tasks if t.split == "holdout"] + train = [t for t in tasks if t.split == "train"] + val = [t for t in tasks if t.split == "val"] + test = [t for t in tasks if t.split == "test"] if limit_replay: - replay = replay[:limit_replay] + train = train[:limit_replay] if limit_holdout: - holdout = holdout[:limit_holdout] - tasks = replay + holdout - bh, bs, bscore = _score(backend, tasks, skill, memory) - trace = [{"night": 0, "held_out_hard": round(bh, 3), "action": "baseline"}] + val = val[:limit_holdout] + test = test[:limit_holdout] + tasks = train + val + test + # final measure is TEST (the gbrain held-out set); val gates internally + bh, bs, bscore = _score(backend, tasks, skill, memory, split="test") + trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}] cur = skill for night in range(1, nights + 1): res = consolidate( backend, tasks, cur, memory, edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5, - evolve_skill=True, evolve_memory=False, night=night, + gate_mode=gate_mode, evolve_skill=True, evolve_memory=False, night=night, ) if res.accepted: cur = res.new_skill + # report the TEST score each night (independent of the val gate) + th, _ts, _ = _score(backend, tasks, cur, memory, split="test") trace.append({ "night": night, - "held_out_hard": round(res.holdout_candidate, 3), + "val_hard": round(res.holdout_candidate, 3), + "test_hard": round(th, 3), "action": res.gate_action, "accepted": res.accepted, "edits": [e.content for e in res.applied_edits], }) - if res.holdout_candidate >= 0.999: + if th >= 0.999: break - ah, as_, ascore = _score(backend, tasks, cur, memory) + ah, as_, ascore = _score(backend, tasks, cur, memory, split="test") return { "seed": seed, "held_out_before": round(bh, 3), @@ -99,8 +108,10 @@ def main(argv=None) -> int: ap.add_argument("--seeds", default="", help="comma list; default = all available") ap.add_argument("--nights", type=int, default=3) ap.add_argument("--edit-budget", type=int, default=4) - ap.add_argument("--limit-replay", type=int, default=0, help="cap #training tasks (cost control)") - ap.add_argument("--limit-holdout", type=int, default=0, help="cap #held-out tasks (cost control)") + ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"], + help="on/hard/soft = validation-gated; off = greedy (no hard filter)") + ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)") + ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)") ap.add_argument("--json", action="store_true") args = ap.parse_args(argv) @@ -125,6 +136,7 @@ def main(argv=None) -> int: continue r = run_seed(backend, seed, skill, tasks, nights=args.nights, edit_budget=args.edit_budget, + gate_mode=("off" if args.gate == "off" else "on"), limit_replay=args.limit_replay, limit_holdout=args.limit_holdout) results.append(r) if not args.json: diff --git a/skillopt/sleep/experiments/run_transfer.py b/skillopt/sleep/experiments/run_transfer.py index af26685..9cdd86d 100644 --- a/skillopt/sleep/experiments/run_transfer.py +++ b/skillopt/sleep/experiments/run_transfer.py @@ -37,7 +37,10 @@ from skillopt.sleep.replay import aggregate_scores, replay_batch def _holdout_hard(backend, tasks, skill, memory="") -> float: - ho = [t for t in tasks if t.split == "holdout"] or tasks + # transfer is measured on the true held-out TEST split + ho = [t for t in tasks if t.split == "test"] + if not ho: + ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks pairs = replay_batch(backend, ho, skill, memory) h, _s = aggregate_scores(pairs) return h @@ -59,13 +62,15 @@ def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str: def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget, limit_replay, limit_holdout, do_direct=True) -> dict: if limit_replay or limit_holdout: - replay = [t for t in tasks if t.split == "replay"] - holdout = [t for t in tasks if t.split == "holdout"] + train = [t for t in tasks if t.split == "train"] + val = [t for t in tasks if t.split == "val"] + test = [t for t in tasks if t.split == "test"] if limit_replay: - replay = replay[:limit_replay] + train = train[:limit_replay] if limit_holdout: - holdout = holdout[:limit_holdout] - tasks = replay + holdout + val = val[:limit_holdout] + test = test[:limit_holdout] + tasks = train + val + test baseline_target = _holdout_hard(target, tasks, skill) diff --git a/skillopt/sleep/mine.py b/skillopt/sleep/mine.py index fcdfc43..ec22f18 100644 --- a/skillopt/sleep/mine.py +++ b/skillopt/sleep/mine.py @@ -126,26 +126,68 @@ def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]: def assign_splits( tasks: List[TaskRecord], *, - holdout_fraction: float = 0.34, + val_fraction: float = 0.34, + test_fraction: float = 0.0, + holdout_fraction: float | None = None, # legacy alias for val_fraction seed: int = 42, ) -> List[TaskRecord]: - """Deterministically split tasks into replay (train) / holdout (test). + """Deterministically split tasks into train / val / test. - Uses a stable hash of the task id so the same task always lands in the - same split across nights (a fixed held-out gate, like SkillOpt's D_sel). + Anti-overfitting contract (the user's design): + * ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real') + and never overlap. val gates updates; test is the final held-out measure. + * ``train`` may include DREAM-augmented tasks (origin=='dream'); those are + NEVER placed in val/test. + + A stable hash of the task id keeps the same real task in the same split across + nights (a fixed held-out gate, like SkillOpt's D_sel/D_test). + + Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old + two-way replay/holdout split — real tasks divide into train + val, no test. + ``holdout_fraction`` is accepted as an alias for ``val_fraction``. """ - for t in tasks: + if holdout_fraction is not None: + val_fraction = holdout_fraction + + dream = [t for t in tasks if t.origin == "dream"] + real = [t for t in tasks if t.origin != "dream"] + + # all dream tasks go to train, unconditionally + for t in dream: + t.split = "train" + + val_cut = int(round(val_fraction * 100)) + test_cut = val_cut + int(round(test_fraction * 100)) + for t in real: bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100 - t.split = "holdout" if bucket < int(holdout_fraction * 100) else "replay" - # guarantee both splits non-empty when possible - splits = {t.split for t in tasks} - if len(tasks) >= 2 and "holdout" not in splits: - tasks[-1].split = "holdout" - if len(tasks) >= 2 and "replay" not in splits: - tasks[0].split = "replay" + if bucket < val_cut: + t.split = "val" + elif bucket < test_cut: + t.split = "test" + else: + t.split = "train" + + # guarantee val (the gate) is non-empty when we have >=2 real tasks + real_splits = {t.split for t in real} + if len(real) >= 2 and "val" not in real_splits: + real[-1].split = "val" + # guarantee a train pool exists (dream or real) when possible + if not any(t.split == "train" for t in tasks) and len(real) >= 2: + real[0].split = "train" + # if test was requested but ended up empty with >=3 real tasks, carve one + if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real): + for t in real: + if t.split == "train": + t.split = "test" + break return tasks +def normalize_legacy_split(value: str) -> str: + """Map old split names to the new vocabulary.""" + return {"replay": "train", "holdout": "val"}.get(value, value) + + def mine( digests: List[SessionDigest], *, diff --git a/skillopt/sleep/types.py b/skillopt/sleep/types.py index 9e2837e..a82fc84 100644 --- a/skillopt/sleep/types.py +++ b/skillopt/sleep/types.py @@ -61,7 +61,16 @@ class TaskRecord: judge: Dict[str, Any] = field(default_factory=dict) # gbrain-style rule judge tags: List[str] = field(default_factory=list) source_sessions: List[str] = field(default_factory=list) - split: str = "replay" # replay (train) | holdout (test) + # split ∈ {train, val, test}. val + test come ONLY from real mined tasks and + # never overlap (val gates updates, test is the final held-out measure). train + # may be dream-augmented (see origin). Legacy values replay->train, + # holdout->val are normalized on load. + split: str = "train" + # origin ∈ {real, dream}. 'real' = mined from the user's actual sessions; + # 'dream' = synthetic/augmented for the training pool. Dream tasks are NEVER + # allowed into val/test, which is the anti-overfitting guarantee. + origin: str = "real" + derived_from: str = "" # for dream tasks: the real task id it varies def to_dict(self) -> Dict[str, Any]: return asdict(self) diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index 6892c26..27dedcc 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -105,14 +105,31 @@ class TestMine(unittest.TestCase): self.assertEqual(ok[0].outcome, "success") def test_split_stable_and_nonempty(self): - tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + tasks = assign_splits(researcher_persona(), val_fraction=0.34, seed=42) splits = {t.split for t in tasks} - self.assertIn("replay", splits) - self.assertIn("holdout", splits) + self.assertIn("train", splits) + self.assertIn("val", splits) # stable across calls - again = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + again = assign_splits(researcher_persona(), val_fraction=0.34, seed=42) self.assertEqual([t.split for t in tasks], [t.split for t in again]) + def test_dream_never_in_val_or_test(self): + # the anti-overfitting guarantee: origin='dream' tasks only ever land in train + from skillopt.sleep.types import TaskRecord + real = researcher_persona() + dream = [TaskRecord(id=f"d{i}", project="/p", intent=f"dream {i}", + origin="dream", derived_from="r0") for i in range(5)] + tasks = assign_splits(real + dream, val_fraction=0.3, test_fraction=0.3, seed=7) + for t in tasks: + if t.origin == "dream": + self.assertEqual(t.split, "train") + # val and test contain ONLY real tasks + for t in tasks: + if t.split in ("val", "test"): + self.assertEqual(t.origin, "real") + # and val/test are disjoint (a task is in exactly one split) + self.assertTrue(any(t.split == "val" for t in tasks)) + class TestConsolidateGate(unittest.TestCase): def test_accepts_helpful_rejects_harmful(self): @@ -169,11 +186,13 @@ class TestGbrainLoader(unittest.TestCase): self.skipTest("gbrain-evals data not present") skill, tasks = load_seed(root, "brief-writer") self.assertTrue(skill) - self.assertTrue(any(t.split == "holdout" for t in tasks)) + # gbrain held-out maps to our 'test'; benchmark pool to train/val + self.assertTrue(any(t.split == "test" for t in tasks)) + self.assertTrue(any(t.split == "val" for t in tasks)) self.assertTrue(all(t.reference_kind == "rule" for t in tasks)) - # the deficient skill must FAIL its own held-out checks (baseline 0) + # the deficient skill must FAIL its own held-out (test) checks (baseline 0) from skillopt.sleep.judges import score_rule_judge - ho = [t for t in tasks if t.split == "holdout"][0] + ho = [t for t in tasks if t.split == "test"][0] self.assertEqual(score_rule_judge(ho.judge, skill)[0], 0.0)