mirror of
https://github.com/microsoft/SkillOpt.git
synced 2026-07-03 14:02:58 +08:00
4eb4c64b2a
The shipped env_template.py and loader_template.py described the same fictional async execute / evaluate / build_prompt API documented in docs/reference/api.md. As a result TemplateBenchmarkEnv(cfg) raised 'TypeError: Can't instantiate abstract class' for every copy-and-paste user who followed the in-tree scaffold. Rewrite the template so it's a working starting point: - env_template.py: TemplateBenchmarkEnv(EnvAdapter) now implements all five real abstract methods (build_train_env, build_eval_env, rollout, reflect, get_task_types) with no-op defaults documented as TODO. Instantiable today; pytest 60/60 still passes. - loader_template.py: TemplateBenchmarkLoader(SplitDataLoader) implements load_split_items for .json / .jsonl input and explains the optional load_raw_items override for split_mode="ratio". - README.md: usage steps now point at scripts/train.py's _ENV_REGISTRY (the real registry) instead of a non-existent BENCHMARK_REGISTRY in skillopt/envs/__init__.py, and link to the rewritten new-benchmark guide. - config_template.yaml: _base_ is a string path (not a list, which the loader rejects); skill_init is commented out with a note so the template config doesn't reference a file the user hasn't created. Verified locally: 'from skillopt.envs._template.env_template import TemplateBenchmarkEnv; TemplateBenchmarkEnv()' succeeds. Refs microsoft/SkillOpt#30. Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
"""
|
|
Benchmark Data Loader Template
|
|
================================
|
|
Copy this file and implement ``load_split_items`` to load your benchmark
|
|
data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader`
|
|
subclass — the base class handles both ``split_mode="split_dir"`` (read
|
|
an existing train/val/test layout) and ``split_mode="ratio"`` (build the
|
|
splits from a single raw file deterministically).
|
|
|
|
For a fully worked example see
|
|
``skillopt/envs/officeqa/dataloader.py``.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from skillopt.datasets.base import SplitDataLoader
|
|
|
|
|
|
def _normalize_item(raw: dict) -> dict:
|
|
"""
|
|
Normalise one raw entry into the dict shape SkillOpt expects.
|
|
|
|
The only **hard** requirement is ``"id"`` (str). Add whatever extra
|
|
fields your :class:`TemplateBenchmarkEnv.rollout` needs.
|
|
"""
|
|
return {
|
|
"id": str(raw.get("uid") or raw.get("id") or ""),
|
|
"question": str(raw.get("question") or raw.get("prompt") or ""),
|
|
"ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""),
|
|
"task_type": str(raw.get("category") or raw.get("task_type") or "template"),
|
|
# ── add benchmark-specific keys here ──
|
|
}
|
|
|
|
|
|
class TemplateBenchmarkLoader(SplitDataLoader):
|
|
"""
|
|
Data loader for <Your Benchmark Name>.
|
|
|
|
Subclass note: you usually only need to implement
|
|
:meth:`load_split_items`. The base class drives ``setup(cfg)``,
|
|
materialises ratio-mode splits, exposes ``train_items``,
|
|
``val_items``, ``test_items``, and builds ``BatchSpec`` objects on
|
|
demand.
|
|
|
|
If you want to support ``split_mode="ratio"`` (auto-split a single
|
|
file into train/val/test), also implement
|
|
:meth:`load_raw_items(data_path)` returning the full list of items.
|
|
"""
|
|
|
|
def load_split_items(self, split_path: str) -> list[dict]:
|
|
"""Load all items for one split directory.
|
|
|
|
``split_path`` is e.g. ``data/your_benchmark/train/``. Return a
|
|
list of dicts, each shaped like :func:`_normalize_item`'s output.
|
|
"""
|
|
path = Path(split_path)
|
|
|
|
json_files = sorted(path.glob("*.json"))
|
|
if json_files:
|
|
with json_files[0].open(encoding="utf-8") as f:
|
|
payload = json.load(f)
|
|
if not isinstance(payload, list):
|
|
raise ValueError(
|
|
f"Expected JSON array at top level of {json_files[0]}"
|
|
)
|
|
return [_normalize_item(row) for row in payload]
|
|
|
|
jsonl_files = sorted(path.glob("*.jsonl"))
|
|
if jsonl_files:
|
|
items: list[dict] = []
|
|
with jsonl_files[0].open(encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
items.append(_normalize_item(json.loads(line)))
|
|
return items
|
|
|
|
raise FileNotFoundError(
|
|
f"No .json or .jsonl file found in {split_path}"
|
|
)
|
|
|
|
# Optional — only needed if you intend to use ``split_mode='ratio'``.
|
|
# def load_raw_items(self, data_path: str) -> list[dict]:
|
|
# ...
|