feat(sleep): 3-way train/val/test split + gate_mode on|off

Data-split refactor (the anti-overfitting foundation the user asked for):
  - TaskRecord gains split∈{train,val,test} and origin∈{real,dream}.
  - assign_splits: real tasks deterministically split into val/test (disjoint);
    DREAM-augmented tasks (origin='dream') NEVER enter val/test — they only go to
    train. val gates updates; test is the final held-out measure.
  - gbrain loader maps its held-out.jsonl -> test, benchmark.jsonl -> train/val,
    so the gbrain held-out stays the true final score.
  - consolidate(): train drives reflect, val gates; adds gate_mode='off' (greedy,
    no hard filter) reporting val movement (greedy_improved/regressed/flat).
  - run_gbrain/transfer/experiment score on test (val fallback); run_gbrain gains
    --gate on|off. Legacy replay/holdout names normalized.

New test proves dream tasks never land in val/test. 21 tests pass; mock
experiment + gate=off both green.

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>

skillopt/sleep/config.py

@@ -30,10 +30,13 @@ DEFAULTS: Dict[str, Any] = {
     # ── budgets ────────────────────────────────────────────────────────────
     "max_tasks_per_night": 40,
     "max_tokens_per_night": 400_000,
-    "holdout_fraction": 0.34,     # fraction of mined tasks reserved for the gate
+    "holdout_fraction": 0.34,     # legacy alias for val_fraction
+    "val_fraction": 0.34,         # real tasks reserved to gate updates
+    "test_fraction": 0.0,         # real tasks reserved as the final held-out measure
     # ── optimizer ──────────────────────────────────────────────────────────
     "backend": "mock",            # "mock" | "claude" | "codex"
     "model": "",                  # backend-specific; "" => backend default
+    "gate_mode": "on",            # "on" (validation-gated) | "off" (greedy, no hard filter)
     "codex_path": "",             # "" => auto-detect the real @openai/codex binary
     "edit_budget": 4,             # textual learning rate (max edits/night)
     "gate_metric": "mixed",       # hard | soft | mixed (mixed best for tiny holdouts)

skillopt/sleep/consolidate.py

@@ -52,14 +52,26 @@ class ConsolidationResult:
 
 
 def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]:
-    replay = [t for t in tasks if t.split == "replay"]
-    holdout = [t for t in tasks if t.split == "holdout"]
-    # be robust if a split is empty
-    if not replay:
-        replay = tasks
-    if not holdout:
-        holdout = tasks
-    return replay, holdout
+    """Return (train_tasks, val_tasks).
+
+    train drives reflect; val gates updates. test is held out entirely from
+    consolidation and is scored by the caller. Accepts legacy split names
+    (replay->train, holdout->val) for robustness.
+    """
+    def _norm(s: str) -> str:
+        return {"replay": "train", "holdout": "val"}.get(s, s)
+
+    train = [t for t in tasks if _norm(t.split) == "train"]
+    val = [t for t in tasks if _norm(t.split) == "val"]
+    # be robust if a split is empty: fall back so a night still does something,
+    # but never silently use test as val.
+    test = [t for t in tasks if _norm(t.split) == "test"]
+    if not val:
+        # prefer train as the gate reference over nothing; last resort all-but-test
+        val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks
+    if not train:
+        train = val
+    return train, val
 
 
 def consolidate(
@@ -71,25 +83,30 @@ def consolidate(
     edit_budget: int = 4,
     gate_metric: str = "mixed",
     gate_mixed_weight: float = 0.5,
+    gate_mode: str = "on",       # "on" (hard/soft per gate_metric) | "off" (greedy)
     evolve_skill: bool = True,
     evolve_memory: bool = True,
     night: int = 1,
 ) -> ConsolidationResult:
     """Run one consolidation epoch: reflect -> bounded edit -> gate.
 
-    Skill and memory are evolved in sequence (skill first if both enabled),
-    each behind the same held-out gate, so each document only changes when it
-    demonstrably helps on the user's held-out tasks.
-    """
-    replay_tasks, holdout_tasks = _split(tasks)
+    train tasks drive reflect; val tasks gate the update (test is held out by the
+    caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve
+    requirement) — the user opts out of hard filtering — but val scores are still
+    recorded so the report shows whether quality moved.
 
-    # ── baseline on held-out slice (the gate reference) ──────────────────
-    base_pairs = replay_batch(backend, holdout_tasks, skill, memory)
+    Skill and memory are evolved in sequence (skill first if both enabled).
+    """
+    train_tasks, val_tasks = _split(tasks)
+    gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
+
+    # ── baseline on the VAL slice (the gate reference) ────────────────────
+    base_pairs = replay_batch(backend, val_tasks, skill, memory)
     base_hard, base_soft = aggregate_scores(base_pairs)
     base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
 
-    # ── reflect over replay-split failures/successes ─────────────────────
-    train_pairs = replay_batch(backend, replay_tasks, skill, memory)
+    # ── reflect over TRAIN-split failures/successes ───────────────────────
+    train_pairs = replay_batch(backend, train_tasks, skill, memory)
     failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0]
     successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0]
 
@@ -104,14 +121,15 @@ def consolidate(
         new_doc, applied = apply_edits(doc, edits)
         if not applied:
             return doc
-        # evaluate candidate on the held-out slice
+        # score the candidate on the VAL slice
         trial_skill = new_doc if which == "skill" else cand_skill
         trial_memory = new_doc if which == "memory" else cand_memory
-        pairs = replay_batch(backend, holdout_tasks, trial_skill, trial_memory)
+        pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
         h, s = aggregate_scores(pairs)
         cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
-        if cand_score > base_score:
-            base_score = cand_score
+        # gate OFF: accept greedily (no regression check); gate ON: strict improve
+        if gate_off or cand_score > base_score:
+            base_score = max(base_score, cand_score)
             all_applied.extend(applied)
             return new_doc
         all_rejected.extend(applied)
@@ -126,7 +144,7 @@ def consolidate(
 
     if evolve_memory:
         # re-evaluate failures under the (possibly improved) skill
-        train_pairs2 = replay_batch(backend, replay_tasks, cand_skill, cand_memory)
+        train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory)
         failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0]
         successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0]
         edits_m = backend.reflect(
@@ -135,19 +153,29 @@ def consolidate(
         )
         cand_memory = _gate_apply(cand_memory, edits_m, "memory")
 
-    # ── final gate decision (use the repo gate for the canonical action) ──
-    final_pairs = replay_batch(backend, holdout_tasks, cand_skill, cand_memory)
+    # ── final decision, scored on the VAL slice ───────────────────────────
+    final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
     final_hard, final_soft = aggregate_scores(final_pairs)
     final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
+    base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
 
-    if _HAVE_REPO_GATE:
+    if gate_off:
+        # greedy mode: keep whatever edits we applied; report quality movement
+        accepted = bool(all_applied)
+        if final_score > base_gate_score:
+            action = "greedy_improved"
+        elif final_score < base_gate_score:
+            action = "greedy_regressed"
+        else:
+            action = "greedy_flat" if all_applied else "greedy_noop"
+    elif _HAVE_REPO_GATE:
         gate = evaluate_gate(
             candidate_skill=cand_skill,
             cand_hard=final_hard,
             current_skill=skill,
-            current_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
+            current_score=base_gate_score,
             best_skill=skill,
-            best_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
+            best_score=base_gate_score,
             best_step=night - 1,
             global_step=night,
             cand_soft=final_soft,
@@ -155,17 +183,15 @@ def consolidate(
             mixed_weight=gate_mixed_weight,
         )
         action = gate.action
+        accepted = bool(all_applied) and final_score > base_gate_score
     else:
-        action = "accept" if final_score > base_soft else "reject"
-
-    accepted = bool(all_applied) and final_score > select_gate_score(
-        base_hard, base_soft, gate_metric, gate_mixed_weight
-    )
+        action = "accept" if final_score > base_gate_score else "reject"
+        accepted = bool(all_applied) and final_score > base_gate_score
 
     return ConsolidationResult(
         accepted=accepted,
         gate_action=action,
-        baseline_score=select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight),
+        baseline_score=base_gate_score,
         candidate_score=final_score,
         new_skill=cand_skill if accepted else skill,
         new_memory=cand_memory if accepted else memory,

skillopt/sleep/cycle.py

@@ -175,6 +175,7 @@ def run_sleep_cycle(
         edit_budget=cfg.get("edit_budget", 4),
         gate_metric=cfg.get("gate_metric", "mixed"),
         gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5),
+        gate_mode=cfg.get("gate_mode", "on"),
         evolve_skill=cfg.get("evolve_skill", True),
         evolve_memory=cfg.get("evolve_memory", True),
         night=night,

skillopt/sleep/experiments/gbrain_bench.py

@@ -63,8 +63,17 @@ def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord:
     )
 
 
-def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
-    """Return (deficient_skill_md, tasks) for one gbrain seed."""
+def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34,
+              split_seed: int = 42) -> Tuple[str, List[TaskRecord]]:
+    """Return (deficient_skill_md, tasks) for one gbrain seed.
+
+    Faithful split mapping:
+      * gbrain held-out.jsonl  -> our ``test`` (the true final measure)
+      * gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val``
+        (val gates updates; train drives reflect)
+    All tasks are origin='real' (gbrain provides no synthetic tasks).
+    """
+    import hashlib
     sub = SEED_DIRS.get(seed, seed)
     seed_dir = os.path.join(data_root, sub)
     skill_path = os.path.join(seed_dir, "SKILL.md")
@@ -73,10 +82,21 @@ def load_seed(data_root: str, seed: str) -> Tuple[str, List[TaskRecord]]:
         with open(skill_path, encoding="utf-8") as f:
             skill = f.read()
     tasks: List[TaskRecord] = []
+    # benchmark pool -> train/val
+    val_cut = int(round(val_fraction * 100))
     for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")):
-        tasks.append(_to_task(rec, seed=seed, split="replay"))
+        t = _to_task(rec, seed=seed, split="train")
+        bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100
+        t.split = "val" if bucket < val_cut else "train"
+        tasks.append(t)
+    # held-out -> test
     for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")):
-        tasks.append(_to_task(rec, seed=seed, split="holdout"))
+        tasks.append(_to_task(rec, seed=seed, split="test"))
+    # guarantee a non-empty val
+    if not any(t.split == "val" for t in tasks):
+        train_only = [t for t in tasks if t.split == "train"]
+        if train_only:
+            train_only[0].split = "val"
     return skill, tasks
 
 

skillopt/sleep/experiments/run_experiment.py

@@ -42,7 +42,8 @@ from skillopt.sleep.types import TaskRecord
 def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str,
                    metric: str = "mixed", w: float = 0.5) -> float:
     from skillopt.sleep.consolidate import select_gate_score
-    holdout = [t for t in tasks if t.split == "holdout"] or tasks
+    # the persona experiment uses a 2-way split (train/val, no test); score on val
+    holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks
     pairs = replay_batch(backend, holdout, skill, memory)
     h, s = aggregate_scores(pairs)
     return select_gate_score(h, s, metric, w)

skillopt/sleep/experiments/run_gbrain.py

@@ -34,47 +34,56 @@ from skillopt.sleep.experiments.gbrain_bench import (
 from skillopt.sleep.replay import aggregate_scores, replay_batch
 
 
-def _score(backend, tasks, skill, memory, split="holdout", metric="mixed", w=0.5):
-    sub = [t for t in tasks if t.split == split] or tasks
+def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5):
+    sub = [t for t in tasks if t.split == split]
+    if not sub:  # fall back to val, then everything, so we never score on nothing
+        sub = [t for t in tasks if t.split == "val"] or tasks
     pairs = replay_batch(backend, sub, skill, memory)
     h, s = aggregate_scores(pairs)
     return h, s, select_gate_score(h, s, metric, w)
 
 
 def run_seed(backend, seed: str, skill: str, tasks: List, *,
-             nights: int = 3, edit_budget: int = 4,
+             nights: int = 3, edit_budget: int = 4, gate_mode: str = "on",
              limit_replay: int = 0, limit_holdout: int = 0) -> dict:
     memory = ""
-    # optionally cap each split to control API cost / latency
+    # optionally cap each split to control API cost / latency.
+    # limit_replay caps train; limit_holdout caps BOTH val and test.
     if limit_replay or limit_holdout:
-        replay = [t for t in tasks if t.split == "replay"]
-        holdout = [t for t in tasks if t.split == "holdout"]
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
         if limit_replay:
-            replay = replay[:limit_replay]
+            train = train[:limit_replay]
         if limit_holdout:
-            holdout = holdout[:limit_holdout]
-        tasks = replay + holdout
-    bh, bs, bscore = _score(backend, tasks, skill, memory)
-    trace = [{"night": 0, "held_out_hard": round(bh, 3), "action": "baseline"}]
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
+    # final measure is TEST (the gbrain held-out set); val gates internally
+    bh, bs, bscore = _score(backend, tasks, skill, memory, split="test")
+    trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}]
     cur = skill
     for night in range(1, nights + 1):
         res = consolidate(
             backend, tasks, cur, memory,
             edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5,
-            evolve_skill=True, evolve_memory=False, night=night,
+            gate_mode=gate_mode, evolve_skill=True, evolve_memory=False, night=night,
         )
         if res.accepted:
             cur = res.new_skill
+        # report the TEST score each night (independent of the val gate)
+        th, _ts, _ = _score(backend, tasks, cur, memory, split="test")
         trace.append({
             "night": night,
-            "held_out_hard": round(res.holdout_candidate, 3),
+            "val_hard": round(res.holdout_candidate, 3),
+            "test_hard": round(th, 3),
             "action": res.gate_action,
             "accepted": res.accepted,
             "edits": [e.content for e in res.applied_edits],
         })
-        if res.holdout_candidate >= 0.999:
+        if th >= 0.999:
             break
-    ah, as_, ascore = _score(backend, tasks, cur, memory)
+    ah, as_, ascore = _score(backend, tasks, cur, memory, split="test")
     return {
         "seed": seed,
         "held_out_before": round(bh, 3),
@@ -99,8 +108,10 @@ def main(argv=None) -> int:
     ap.add_argument("--seeds", default="", help="comma list; default = all available")
     ap.add_argument("--nights", type=int, default=3)
     ap.add_argument("--edit-budget", type=int, default=4)
-    ap.add_argument("--limit-replay", type=int, default=0, help="cap #training tasks (cost control)")
-    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #held-out tasks (cost control)")
+    ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"],
+                    help="on/hard/soft = validation-gated; off = greedy (no hard filter)")
+    ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)")
+    ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)")
     ap.add_argument("--json", action="store_true")
     args = ap.parse_args(argv)
 
@@ -125,6 +136,7 @@ def main(argv=None) -> int:
             continue
         r = run_seed(backend, seed, skill, tasks, nights=args.nights,
                      edit_budget=args.edit_budget,
+                     gate_mode=("off" if args.gate == "off" else "on"),
                      limit_replay=args.limit_replay, limit_holdout=args.limit_holdout)
         results.append(r)
         if not args.json:

skillopt/sleep/experiments/run_transfer.py

@@ -37,7 +37,10 @@ from skillopt.sleep.replay import aggregate_scores, replay_batch
 
 
 def _holdout_hard(backend, tasks, skill, memory="") -> float:
-    ho = [t for t in tasks if t.split == "holdout"] or tasks
+    # transfer is measured on the true held-out TEST split
+    ho = [t for t in tasks if t.split == "test"]
+    if not ho:
+        ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks
     pairs = replay_batch(backend, ho, skill, memory)
     h, _s = aggregate_scores(pairs)
     return h
@@ -59,13 +62,15 @@ def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str:
 def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget,
              limit_replay, limit_holdout, do_direct=True) -> dict:
     if limit_replay or limit_holdout:
-        replay = [t for t in tasks if t.split == "replay"]
-        holdout = [t for t in tasks if t.split == "holdout"]
+        train = [t for t in tasks if t.split == "train"]
+        val = [t for t in tasks if t.split == "val"]
+        test = [t for t in tasks if t.split == "test"]
         if limit_replay:
-            replay = replay[:limit_replay]
+            train = train[:limit_replay]
         if limit_holdout:
-            holdout = holdout[:limit_holdout]
-        tasks = replay + holdout
+            val = val[:limit_holdout]
+            test = test[:limit_holdout]
+        tasks = train + val + test
 
     baseline_target = _holdout_hard(target, tasks, skill)
 

skillopt/sleep/mine.py

@@ -126,26 +126,68 @@ def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]:
 def assign_splits(
     tasks: List[TaskRecord],
     *,
-    holdout_fraction: float = 0.34,
+    val_fraction: float = 0.34,
+    test_fraction: float = 0.0,
+    holdout_fraction: float | None = None,  # legacy alias for val_fraction
     seed: int = 42,
 ) -> List[TaskRecord]:
-    """Deterministically split tasks into replay (train) / holdout (test).
+    """Deterministically split tasks into train / val / test.
 
-    Uses a stable hash of the task id so the same task always lands in the
-    same split across nights (a fixed held-out gate, like SkillOpt's D_sel).
+    Anti-overfitting contract (the user's design):
+      * ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real')
+        and never overlap. val gates updates; test is the final held-out measure.
+      * ``train`` may include DREAM-augmented tasks (origin=='dream'); those are
+        NEVER placed in val/test.
+
+    A stable hash of the task id keeps the same real task in the same split across
+    nights (a fixed held-out gate, like SkillOpt's D_sel/D_test).
+
+    Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old
+    two-way replay/holdout split — real tasks divide into train + val, no test.
+    ``holdout_fraction`` is accepted as an alias for ``val_fraction``.
     """
-    for t in tasks:
+    if holdout_fraction is not None:
+        val_fraction = holdout_fraction
+
+    dream = [t for t in tasks if t.origin == "dream"]
+    real = [t for t in tasks if t.origin != "dream"]
+
+    # all dream tasks go to train, unconditionally
+    for t in dream:
+        t.split = "train"
+
+    val_cut = int(round(val_fraction * 100))
+    test_cut = val_cut + int(round(test_fraction * 100))
+    for t in real:
         bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100
-        t.split = "holdout" if bucket < int(holdout_fraction * 100) else "replay"
-    # guarantee both splits non-empty when possible
-    splits = {t.split for t in tasks}
-    if len(tasks) >= 2 and "holdout" not in splits:
-        tasks[-1].split = "holdout"
-    if len(tasks) >= 2 and "replay" not in splits:
-        tasks[0].split = "replay"
+        if bucket < val_cut:
+            t.split = "val"
+        elif bucket < test_cut:
+            t.split = "test"
+        else:
+            t.split = "train"
+
+    # guarantee val (the gate) is non-empty when we have >=2 real tasks
+    real_splits = {t.split for t in real}
+    if len(real) >= 2 and "val" not in real_splits:
+        real[-1].split = "val"
+    # guarantee a train pool exists (dream or real) when possible
+    if not any(t.split == "train" for t in tasks) and len(real) >= 2:
+        real[0].split = "train"
+    # if test was requested but ended up empty with >=3 real tasks, carve one
+    if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real):
+        for t in real:
+            if t.split == "train":
+                t.split = "test"
+                break
     return tasks
 
 
+def normalize_legacy_split(value: str) -> str:
+    """Map old split names to the new vocabulary."""
+    return {"replay": "train", "holdout": "val"}.get(value, value)
+
+
 def mine(
     digests: List[SessionDigest],
     *,

skillopt/sleep/types.py

@@ -61,7 +61,16 @@ class TaskRecord:
     judge: Dict[str, Any] = field(default_factory=dict)  # gbrain-style rule judge
     tags: List[str] = field(default_factory=list)
     source_sessions: List[str] = field(default_factory=list)
-    split: str = "replay"             # replay (train) | holdout (test)
+    # split ∈ {train, val, test}.  val + test come ONLY from real mined tasks and
+    # never overlap (val gates updates, test is the final held-out measure). train
+    # may be dream-augmented (see origin).  Legacy values replay->train,
+    # holdout->val are normalized on load.
+    split: str = "train"
+    # origin ∈ {real, dream}.  'real' = mined from the user's actual sessions;
+    # 'dream' = synthetic/augmented for the training pool. Dream tasks are NEVER
+    # allowed into val/test, which is the anti-overfitting guarantee.
+    origin: str = "real"
+    derived_from: str = ""            # for dream tasks: the real task id it varies
 
     def to_dict(self) -> Dict[str, Any]:
         return asdict(self)

tests/test_sleep_engine.py

@@ -105,14 +105,31 @@ class TestMine(unittest.TestCase):
         self.assertEqual(ok[0].outcome, "success")
 
     def test_split_stable_and_nonempty(self):
-        tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
+        tasks = assign_splits(researcher_persona(), val_fraction=0.34, seed=42)
         splits = {t.split for t in tasks}
-        self.assertIn("replay", splits)
-        self.assertIn("holdout", splits)
+        self.assertIn("train", splits)
+        self.assertIn("val", splits)
         # stable across calls
-        again = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
+        again = assign_splits(researcher_persona(), val_fraction=0.34, seed=42)
         self.assertEqual([t.split for t in tasks], [t.split for t in again])
 
+    def test_dream_never_in_val_or_test(self):
+        # the anti-overfitting guarantee: origin='dream' tasks only ever land in train
+        from skillopt.sleep.types import TaskRecord
+        real = researcher_persona()
+        dream = [TaskRecord(id=f"d{i}", project="/p", intent=f"dream {i}",
+                            origin="dream", derived_from="r0") for i in range(5)]
+        tasks = assign_splits(real + dream, val_fraction=0.3, test_fraction=0.3, seed=7)
+        for t in tasks:
+            if t.origin == "dream":
+                self.assertEqual(t.split, "train")
+        # val and test contain ONLY real tasks
+        for t in tasks:
+            if t.split in ("val", "test"):
+                self.assertEqual(t.origin, "real")
+        # and val/test are disjoint (a task is in exactly one split)
+        self.assertTrue(any(t.split == "val" for t in tasks))
+
 
 class TestConsolidateGate(unittest.TestCase):
     def test_accepts_helpful_rejects_harmful(self):
@@ -169,11 +186,13 @@ class TestGbrainLoader(unittest.TestCase):
             self.skipTest("gbrain-evals data not present")
         skill, tasks = load_seed(root, "brief-writer")
         self.assertTrue(skill)
-        self.assertTrue(any(t.split == "holdout" for t in tasks))
+        # gbrain held-out maps to our 'test'; benchmark pool to train/val
+        self.assertTrue(any(t.split == "test" for t in tasks))
+        self.assertTrue(any(t.split == "val" for t in tasks))
         self.assertTrue(all(t.reference_kind == "rule" for t in tasks))
-        # the deficient skill must FAIL its own held-out checks (baseline 0)
+        # the deficient skill must FAIL its own held-out (test) checks (baseline 0)
         from skillopt.sleep.judges import score_rule_judge
-        ho = [t for t in tasks if t.split == "holdout"][0]
+        ho = [t for t in tasks if t.split == "test"][0]
         self.assertEqual(score_rule_judge(ho.judge, skill)[0], 0.0)