Initial commit

scripts/codex_azure_mi.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="/home/azureuser/workspace-gzy/SkillReflection"
+PY="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+REAL_CODEX="/home/azureuser/.nvm/versions/node/v18.20.8/bin/codex"
+CLIENT_ID="8cafa2b1-a2a7-4ad9-814a-ffe4aed7e800"
+SCOPE="https://cognitiveservices.azure.com/.default"
+
+token="$("$PY" - <<PY
+from azure.identity import ManagedIdentityCredential, get_bearer_token_provider
+cred = ManagedIdentityCredential(client_id="$CLIENT_ID")
+print(get_bearer_token_provider(cred, "$SCOPE")())
+PY
+)"
+
+export CODEX_HOME="${CODEX_HOME:-$ROOT/.codex_azure}"
+export AZURE_OPENAI_AUTH_HEADER="Bearer $token"
+
+exec "$REAL_CODEX" "$@"

scripts/download_babyvision.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Download BabyVision from Hugging Face and convert it to local meta_data.jsonl + images/ format."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--out_dir", type=str, required=True)
+    p.add_argument("--dataset", type=str, default="UnipatAI/BabyVision")
+    p.add_argument("--split", type=str, default="train")
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    try:
+        from datasets import load_dataset
+    except ImportError as exc:  # pragma: no cover
+        raise SystemExit("Please install `datasets` first: pip install datasets pillow") from exc
+
+    out_dir = Path(args.out_dir).resolve()
+    images_dir = out_dir / "images"
+    meta_path = out_dir / "meta_data.jsonl"
+    images_dir.mkdir(parents=True, exist_ok=True)
+
+    dataset = load_dataset(args.dataset, split=args.split)
+    with open(meta_path, "w", encoding="utf-8") as outf:
+        for idx, row in enumerate(dataset):
+            image = row.get("image")
+            if image is None:
+                continue
+            task_id = str(row.get("taskId") or row.get("id") or idx + 1)
+            image_name = f"{task_id}.png"
+            image_path = images_dir / image_name
+            image.save(image_path)
+
+            record = dict(row)
+            record["image"] = image_name
+            outf.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+    print(f"Saved BabyVision to {out_dir}")
+    print(f"Metadata: {meta_path}")
+    print(f"Images:   {images_dir}")
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_livemathematicianbench_baseline.py

@@ -0,0 +1,333 @@
+#!/usr/bin/env python3
+"""Evaluate LiveMathematicianBench under current or official-style prompts."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import random
+import re
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+from reflact.envs.livemathematicianbench.dataloader import load_items
+from reflact.envs.livemathematicianbench.evaluator import evaluate as current_evaluate
+from reflact.envs.livemathematicianbench.rollout import _build_system, _build_user
+from reflact.model import (
+    chat_with_deployment,
+    configure_azure_openai,
+    set_backend,
+    set_reasoning_effort,
+)
+
+_LABELS = ["A", "B", "C", "D", "E"]
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--data_path", type=str, required=True)
+    p.add_argument("--model", type=str, default="gpt-5.4")
+    p.add_argument("--backend", type=str, choices=["azure_openai", "codex", "claude"], default="azure_openai")
+    p.add_argument("--mode", type=str, choices=["current", "official"], required=True)
+    p.add_argument("--reasoning_effort", type=str, default=None)
+    p.add_argument("--azure_endpoint", type=str, default="")
+    p.add_argument("--azure_api_version", type=str, default="")
+    p.add_argument("--azure_api_key", type=str, default="")
+    p.add_argument("--max_completion_tokens", type=int, default=0)
+    p.add_argument("--workers", type=int, default=8)
+    p.add_argument("--seed", type=int, default=20260227)
+    p.add_argument("--skill_path", type=str, default="reflact/envs/livemathematicianbench/skills/initial.md")
+    p.add_argument("--limit", type=int, default=0)
+    p.add_argument("--resume", action="store_true")
+    p.add_argument("--output_json", type=str, required=True)
+    return p.parse_args()
+
+def read_skill(skill_path: str) -> str:
+    with open(skill_path, encoding="utf-8") as f:
+        return f.read()
+
+
+def official_extract_answer(response_text: str) -> str | None:
+    if not response_text:
+        return None
+    boxed_match = re.search(r"\\boxed\{([A-Ea-e])\}", response_text)
+    if boxed_match:
+        return boxed_match.group(1).upper()
+    boxed_match = re.search(r"boxed\{([A-Ea-e])\}", response_text)
+    if boxed_match:
+        return boxed_match.group(1).upper()
+    answer_match = re.search(r"answer is[:\s]*([A-Ea-e])", response_text, re.IGNORECASE)
+    if answer_match:
+        return answer_match.group(1).upper()
+    answer_match = re.search(r"Answer[:\s]*\(?([A-Ea-e])\)?", response_text)
+    if answer_match:
+        return answer_match.group(1).upper()
+    final_match = re.search(r"\b([A-Ea-e])\b\s*[.)]?\s*$", response_text.strip())
+    if final_match:
+        return final_match.group(1).upper()
+    return None
+
+
+def official_format_mcq_prompt(question: str, choices: list[dict]) -> str:
+    lines = [
+        "Answer the following multiple-choice question.",
+        "Think carefully, then provide your final answer in the format: \\boxed{X} where X is A, B, C, D, or E.",
+        "",
+        "Question:",
+        question,
+        "",
+        "Choices:",
+    ]
+    for choice in choices:
+        lines.append(f"{choice['label']}. {choice['text']}")
+    lines.append("")
+    lines.append("Your answer:")
+    return "\n".join(lines)
+
+
+def shuffle_choices(item: dict, seed: int) -> tuple[list[dict], dict]:
+    correct_choice = dict(item["correct_choice"])
+    all_choices = [dict(choice) for choice in item["choices"]]
+    rng = random.Random(f"{seed}:{item['id']}")
+    rng.shuffle(all_choices)
+
+    shuffled: list[dict] = []
+    new_correct = dict(correct_choice)
+    correct_text = correct_choice["text"]
+
+    for idx, choice in enumerate(all_choices[: len(_LABELS)]):
+        relabeled = {"label": _LABELS[idx], "text": choice["text"]}
+        shuffled.append(relabeled)
+        if choice["text"] == correct_text:
+            new_correct = dict(relabeled)
+
+    return shuffled, new_correct
+
+
+def load_existing(output_path: Path) -> dict[str, dict]:
+    if not output_path.exists():
+        return {}
+    with open(output_path, encoding="utf-8") as f:
+        payload = json.load(f)
+    existing = {}
+    for row in payload.get("results", []):
+        existing[str(row["id"])] = row
+    return existing
+
+
+def save_results(output_path: Path, meta: dict, results: list[dict]) -> None:
+    correct = sum(1 for row in results if row.get("is_correct"))
+    total = len(results)
+    payload = {
+        **meta,
+        "summary": {
+            "correct": correct,
+            "total": total,
+            "accuracy": (correct / total) if total else 0.0,
+        },
+        "results": sorted(results, key=lambda row: str(row["id"])),
+    }
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(payload, f, ensure_ascii=False, indent=2)
+
+
+def call_model(
+    *,
+    model: str,
+    system: str,
+    user: str,
+    max_completion_tokens: int | None,
+    reasoning_effort: str | None,
+) -> str:
+    last_error: Exception | None = None
+    for attempt in range(5):
+        try:
+            set_reasoning_effort(reasoning_effort)
+            raw, _ = chat_with_deployment(
+                deployment=model,
+                system=system,
+                user=user,
+                max_completion_tokens=max_completion_tokens if max_completion_tokens and max_completion_tokens > 0 else 4096,
+                retries=1,
+                stage="rollout",
+            )
+            return str(raw or "")
+        except Exception as exc:  # noqa: BLE001
+            last_error = exc
+            if attempt == 4:
+                break
+            time.sleep(min(2 ** attempt, 10))
+    raise RuntimeError(f"LLM call failed after retries: {last_error}")
+
+
+def evaluate_one(
+    item: dict,
+    *,
+    mode: str,
+    model: str,
+    skill_content: str,
+    max_completion_tokens: int,
+    reasoning_effort: str | None,
+    seed: int,
+) -> dict:
+    shuffled_choices, correct_choice = shuffle_choices(item, seed)
+
+    if mode == "official":
+        system = "You are an expert mathematician. Answer accurately."
+        user = official_format_mcq_prompt(item["question"], shuffled_choices)
+        effective_max_completion_tokens = max_completion_tokens if max_completion_tokens > 0 else None
+    else:
+        materialized = dict(item)
+        materialized["choices"] = shuffled_choices
+        materialized["correct_choice"] = correct_choice
+        system = _build_system(skill_content)
+        user = _build_user(materialized, use_theorem=False, use_sketch=False)
+        effective_max_completion_tokens = max_completion_tokens if max_completion_tokens > 0 else 768
+
+    t0 = time.time()
+    response = call_model(
+        model=model,
+        system=system,
+        user=user,
+        max_completion_tokens=effective_max_completion_tokens,
+        reasoning_effort=reasoning_effort,
+    )
+    elapsed = time.time() - t0
+
+    if mode == "official":
+        predicted = official_extract_answer(response)
+        predicted_text = ""
+        for choice in shuffled_choices:
+            if choice["label"] == predicted:
+                predicted_text = choice["text"]
+                break
+        is_correct = predicted == correct_choice["label"]
+        return {
+            "id": item["id"],
+            "question": item["question"],
+            "correct_label": correct_choice["label"],
+            "correct_text": correct_choice["text"],
+            "predicted_label": predicted,
+            "predicted_text": predicted_text,
+            "is_correct": is_correct,
+            "elapsed_seconds": elapsed,
+            "response": response,
+        }
+
+    eval_result = current_evaluate(response, correct_choice, shuffled_choices)
+    return {
+        "id": item["id"],
+        "question": item["question"],
+        "correct_label": correct_choice["label"],
+        "correct_text": correct_choice["text"],
+        "predicted_label": eval_result["predicted_label"],
+        "predicted_text": eval_result["predicted_text"],
+        "is_correct": bool(eval_result["em"]),
+        "elapsed_seconds": elapsed,
+        "response": response,
+    }
+
+
+def main() -> None:
+    args = parse_args()
+    set_backend(args.backend)
+    configure_azure_openai(
+        endpoint=args.azure_endpoint or None,
+        api_version=args.azure_api_version or None,
+        api_key=args.azure_api_key or None,
+    )
+    set_reasoning_effort(args.reasoning_effort)
+    output_path = Path(args.output_json).resolve()
+    skill_content = read_skill(args.skill_path) if args.mode == "current" else ""
+
+    items = load_items(args.data_path)
+    if args.limit:
+        items = items[:args.limit]
+
+    existing = load_existing(output_path) if args.resume else {}
+    pending = [item for item in items if str(item["id"]) not in existing]
+    results = list(existing.values())
+
+    print("=" * 72, flush=True)
+    print("LiveMathematicianBench baseline eval", flush=True)
+    print("=" * 72, flush=True)
+    print(f"Mode: {args.mode}", flush=True)
+    print(f"Model: {args.model}", flush=True)
+    print(f"Reasoning effort: {args.reasoning_effort}", flush=True)
+    print(f"Items: {len(items)} total, {len(pending)} pending, {len(existing)} resumed", flush=True)
+    print(f"Output: {output_path}", flush=True)
+    print("=" * 72, flush=True)
+
+    meta = {
+        "mode": args.mode,
+        "model": args.model,
+        "reasoning_effort": args.reasoning_effort,
+        "seed": args.seed,
+        "max_completion_tokens": args.max_completion_tokens,
+    }
+
+    if not pending:
+        save_results(output_path, meta, results)
+        summary = json.loads(output_path.read_text(encoding="utf-8"))["summary"]
+        print(f"Accuracy: {summary['correct']}/{summary['total']} = {summary['accuracy']:.4f}", flush=True)
+        return
+
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        futs = {
+            ex.submit(
+                evaluate_one,
+                item,
+                mode=args.mode,
+                model=args.model,
+                skill_content=skill_content,
+                max_completion_tokens=args.max_completion_tokens,
+                reasoning_effort=args.reasoning_effort,
+                seed=args.seed,
+            ): item
+            for item in pending
+        }
+        completed = 0
+        for fut in as_completed(futs):
+            item = futs[fut]
+            try:
+                row = fut.result()
+            except Exception as exc:  # noqa: BLE001
+                row = {
+                    "id": item["id"],
+                    "question": item["question"],
+                    "correct_label": None,
+                    "correct_text": item["correct_choice"]["text"],
+                    "predicted_label": None,
+                    "predicted_text": "",
+                    "is_correct": False,
+                    "elapsed_seconds": 0.0,
+                    "response": "",
+                    "error": str(exc),
+                }
+            results.append(row)
+            completed += 1
+            correct = sum(1 for result in results if result.get("is_correct"))
+            total = len(results)
+            print(
+                f"[{completed}/{len(pending)}] id={row['id']} "
+                f"pred={row['predicted_label']} gold={row['correct_label']} "
+                f"acc={correct}/{total}={correct/total:.4f}",
+                flush=True,
+            )
+            save_results(output_path, meta, results)
+
+    summary = json.loads(output_path.read_text(encoding="utf-8"))["summary"]
+    print("=" * 72, flush=True)
+    print(f"Accuracy: {summary['correct']}/{summary['total']} = {summary['accuracy']:.4f}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_only.py

@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+"""ReflACT eval-only: run a single skill on a dataset without training.
+
+Usage
+-----
+    python scripts/eval_only.py \
+        --config configs/spreadsheetbench/default.yaml \
+        --skill reflact/envs/spreadsheetbench/skills/initial.md \
+        --split_dir /path/to/split \
+        --out_root outputs/eval_skill0
+
+All YAML keys can be overridden from the CLI, same as train.py.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import json
+import os
+import sys
+
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+from reflact.model import (
+    configure_azure_openai,
+    configure_claude_code_exec,
+    configure_codex_exec,
+    set_reasoning_effort,
+    set_student_backend,
+    set_student_deployment,
+    set_teacher_backend,
+    set_teacher_deployment,
+)
+from reflact.model.common import default_model_for_backend, normalize_backend_name
+
+_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"}
+from reflact.utils import compute_score
+
+
+# ── Reuse registry from train.py ───────────────────────────────────────────
+
+_ENV_REGISTRY: dict[str, type] = {}
+
+
+def _register_builtins() -> None:
+    try:
+        from reflact.envs.alfworld.adapter import ALFWorldAdapter
+        _ENV_REGISTRY["alfworld"] = ALFWorldAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.searchqa.adapter import SearchQAAdapter
+        _ENV_REGISTRY["searchqa"] = SearchQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter
+        _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.babyvision.adapter import BabyVisionAdapter
+        _ENV_REGISTRY["babyvision"] = BabyVisionAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter
+        _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.mmrb.adapter import MMRBAdapter
+        _ENV_REGISTRY["mmrb"] = MMRBAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.docvqa.adapter import DocVQAAdapter
+        _ENV_REGISTRY["docvqa"] = DocVQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.mathverse.adapter import MathVerseAdapter
+        _ENV_REGISTRY["mathverse"] = MathVerseAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.officeqa.adapter import OfficeQAAdapter
+        _ENV_REGISTRY["officeqa"] = OfficeQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.sealqa.adapter import SealQAAdapter
+        _ENV_REGISTRY["sealqa"] = SealQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.swebench.adapter import SWEBenchAdapter
+        _ENV_REGISTRY["swebench"] = SWEBenchAdapter
+    except ImportError:
+        pass
+
+
+def get_adapter(cfg: dict):
+    _register_builtins()
+    env_name = cfg.get("env", "alfworld")
+    if env_name not in _ENV_REGISTRY:
+        raise ValueError(
+            f"Unknown environment '{env_name}'. "
+            f"Available: {list(_ENV_REGISTRY.keys())}"
+        )
+    adapter_cls = _ENV_REGISTRY[env_name]
+
+    import inspect
+    sig = inspect.signature(adapter_cls.__init__)
+    accepted = set(sig.parameters.keys()) - {"self"}
+    adapter_kwargs = {k: cfg[k] for k in accepted if k in cfg}
+    return adapter_cls(**adapter_kwargs)
+
+
+# ── CLI ────────────────────────────────────────────────────────────────────
+
+_BOOL = lambda x: str(x).lower() in ("true", "1", "yes")  # noqa: E731
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="ReflACT eval-only")
+    p.add_argument("--config", type=str, required=True)
+    p.add_argument("--skill", type=str, required=True,
+                   help="Path to skill .md file to evaluate")
+    p.add_argument("--split", type=str, default="all",
+                   help="Which split to eval: train/valid_seen/valid_unseen/all (default: all)")
+    p.add_argument("--cfg-options", nargs="+", default=[],
+                   help="Override config: section.key=value")
+    # Legacy flat overrides
+    p.add_argument("--env", type=str)
+    p.add_argument("--backend", type=str,
+                   choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec"])
+    p.add_argument("--teacher_model", type=str)
+    p.add_argument("--student_model", type=str)
+    p.add_argument("--teacher_backend", type=str)
+    p.add_argument("--student_backend", type=str)
+    p.add_argument("--reasoning_effort", type=str,
+                   choices=["", "low", "medium", "high", "xhigh", "max"])
+    p.add_argument("--azure_endpoint", type=str)
+    p.add_argument("--azure_api_version", type=str)
+    p.add_argument("--azure_api_key", type=str)
+    p.add_argument("--azure_openai_endpoint", type=str)
+    p.add_argument("--azure_openai_api_version", type=str)
+    p.add_argument("--azure_openai_api_key", type=str)
+    p.add_argument("--azure_openai_auth_mode", type=str)
+    p.add_argument("--azure_openai_ad_scope", type=str)
+    p.add_argument("--azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--teacher_azure_openai_endpoint", type=str)
+    p.add_argument("--teacher_azure_openai_api_version", type=str)
+    p.add_argument("--teacher_azure_openai_api_key", type=str)
+    p.add_argument("--teacher_azure_openai_auth_mode", type=str)
+    p.add_argument("--teacher_azure_openai_ad_scope", type=str)
+    p.add_argument("--teacher_azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--student_azure_openai_endpoint", type=str)
+    p.add_argument("--student_azure_openai_api_version", type=str)
+    p.add_argument("--student_azure_openai_api_key", type=str)
+    p.add_argument("--student_azure_openai_auth_mode", type=str)
+    p.add_argument("--student_azure_openai_ad_scope", type=str)
+    p.add_argument("--student_azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--codex_exec_path", type=str)
+    p.add_argument("--codex_exec_sandbox", type=str)
+    p.add_argument("--codex_exec_profile", type=str)
+    p.add_argument("--codex_exec_full_auto", type=_BOOL)
+    p.add_argument("--codex_exec_reasoning_effort", type=str)
+    p.add_argument("--codex_exec_use_sdk", type=str)
+    p.add_argument("--codex_exec_network_access", type=_BOOL)
+    p.add_argument("--codex_exec_web_search", type=_BOOL)
+    p.add_argument("--codex_exec_approval_policy", type=str)
+    p.add_argument("--claude_code_exec_path", type=str)
+    p.add_argument("--claude_code_exec_profile", type=str)
+    p.add_argument("--claude_code_exec_use_sdk", type=str)
+    p.add_argument("--claude_code_exec_effort", type=str)
+    p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
+    p.add_argument("--out_root", type=str)
+    p.add_argument("--data_path", type=str)
+    p.add_argument("--split_mode", type=str,
+                   choices=["ratio", "split_dir"])
+    p.add_argument("--split_ratio", type=str)
+    p.add_argument("--split_seed", type=int)
+    p.add_argument("--split_dir", type=str)
+    p.add_argument("--split_output_dir", type=str)
+    p.add_argument("--data_root", type=str)
+    p.add_argument("--max_turns", type=int)
+    p.add_argument("--workers", type=int)
+    p.add_argument("--max_api_workers", type=int)
+    p.add_argument("--seed", type=int)
+    p.add_argument("--test_env_num", type=int)
+    p.add_argument("--mode", type=str,
+                   help="SpreadsheetBench: single/multi/react (default comes from config)")
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+
+    from reflact.config import load_config as _load, flatten_config, is_structured
+
+    cfg = _load(args.config, overrides=args.cfg_options)
+    structured = is_structured(cfg)
+
+    # Apply legacy --key value overrides
+    cli = {k: v for k, v in vars(args).items()
+           if v is not None and k not in ("config", "skill", "split", "cfg_options")}
+    if cli:
+        if structured:
+            from reflact.config import apply_overrides
+            _MAP = {
+                "backend": "model.backend",
+                "teacher_model": "model.teacher",
+                "student_model": "model.student",
+                "teacher_backend": "model.teacher_backend",
+                "student_backend": "model.student_backend",
+                "reasoning_effort": "model.reasoning_effort",
+                "azure_endpoint": "model.azure_endpoint",
+                "azure_api_version": "model.azure_api_version",
+                "azure_api_key": "model.azure_api_key",
+                "azure_openai_endpoint": "model.azure_openai_endpoint",
+                "azure_openai_api_version": "model.azure_openai_api_version",
+                "azure_openai_api_key": "model.azure_openai_api_key",
+                "azure_openai_auth_mode": "model.azure_openai_auth_mode",
+                "azure_openai_ad_scope": "model.azure_openai_ad_scope",
+                "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id",
+                "teacher_azure_openai_endpoint": "model.teacher_azure_openai_endpoint",
+                "teacher_azure_openai_api_version": "model.teacher_azure_openai_api_version",
+                "teacher_azure_openai_api_key": "model.teacher_azure_openai_api_key",
+                "teacher_azure_openai_auth_mode": "model.teacher_azure_openai_auth_mode",
+                "teacher_azure_openai_ad_scope": "model.teacher_azure_openai_ad_scope",
+                "teacher_azure_openai_managed_identity_client_id": "model.teacher_azure_openai_managed_identity_client_id",
+                "student_azure_openai_endpoint": "model.student_azure_openai_endpoint",
+                "student_azure_openai_api_version": "model.student_azure_openai_api_version",
+                "student_azure_openai_api_key": "model.student_azure_openai_api_key",
+                "student_azure_openai_auth_mode": "model.student_azure_openai_auth_mode",
+                "student_azure_openai_ad_scope": "model.student_azure_openai_ad_scope",
+                "student_azure_openai_managed_identity_client_id": "model.student_azure_openai_managed_identity_client_id",
+                "codex_exec_path": "model.codex_exec_path",
+                "codex_exec_sandbox": "model.codex_exec_sandbox",
+                "codex_exec_profile": "model.codex_exec_profile",
+                "codex_exec_full_auto": "model.codex_exec_full_auto",
+                "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort",
+                "codex_exec_use_sdk": "model.codex_exec_use_sdk",
+                "codex_exec_network_access": "model.codex_exec_network_access",
+                "codex_exec_web_search": "model.codex_exec_web_search",
+                "codex_exec_approval_policy": "model.codex_exec_approval_policy",
+                "claude_code_exec_path": "model.claude_code_exec_path",
+                "claude_code_exec_profile": "model.claude_code_exec_profile",
+                "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
+                "claude_code_exec_effort": "model.claude_code_exec_effort",
+                "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
+                "seed": "train.seed",
+                "test_env_num": "evaluation.test_env_num",
+                "env": "env.name",
+                "out_root": "env.out_root",
+            }
+            mapped = []
+            for k, v in cli.items():
+                dotted = _MAP.get(k)
+                if dotted:
+                    mapped.append(f"{dotted}={v}")
+                else:
+                    mapped.append(f"env.{k}={v}")
+            apply_overrides(cfg, mapped)
+        else:
+            cfg.update(cli)
+
+    cfg = flatten_config(cfg) if structured else cfg
+
+    for new_key, old_key in (
+        ("azure_openai_endpoint", "azure_endpoint"),
+        ("azure_openai_api_version", "azure_api_version"),
+        ("azure_openai_api_key", "azure_api_key"),
+    ):
+        if cfg.get(new_key) in (None, "") and cfg.get(old_key) not in (None, ""):
+            cfg[new_key] = cfg[old_key]
+
+    explicit_backend = getattr(args, "backend", None)
+    if explicit_backend is None:
+        for option in args.cfg_options or []:
+            key = str(option).split("=", 1)[0].strip()
+            if key == "model.backend":
+                explicit_backend = str(option).split("=", 1)[1].strip()
+                break
+
+    backend = normalize_backend_name(cfg.get("model_backend") or cfg.get("student_backend") or "azure_openai")
+
+    def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
+        if getattr(args, legacy_key, None) is not None:
+            return True
+        for option in args.cfg_options or []:
+            key = str(option).split("=", 1)[0].strip()
+            if key == dotted_key:
+                return True
+        return False
+
+    if explicit_backend is not None:
+        backend = normalize_backend_name(explicit_backend)
+        cfg["model_backend"] = backend
+        if backend in {"claude", "claude_chat"}:
+            cfg.setdefault("teacher_backend", "claude_chat")
+            cfg.setdefault("student_backend", "claude_chat")
+        elif backend in {"codex", "codex_exec"}:
+            cfg.setdefault("teacher_backend", "openai_chat")
+            cfg.setdefault("student_backend", "codex_exec")
+        elif backend == "claude_code_exec":
+            cfg.setdefault("teacher_backend", "openai_chat")
+            cfg.setdefault("student_backend", "claude_code_exec")
+        else:
+            cfg.setdefault("teacher_backend", "openai_chat")
+            cfg.setdefault("student_backend", "openai_chat")
+    else:
+        cfg.setdefault("teacher_backend", "openai_chat")
+        cfg.setdefault("student_backend", "openai_chat")
+
+    if cfg.get("teacher_backend") == "claude_chat":
+        if (
+            str(cfg.get("teacher_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.teacher", "teacher_model")
+        ):
+            cfg["teacher_model"] = default_model_for_backend("claude_chat")
+    if cfg.get("student_backend") == "claude_chat":
+        if (
+            str(cfg.get("student_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.student", "student_model")
+        ):
+            cfg["student_model"] = default_model_for_backend("claude_chat")
+    if cfg.get("student_backend") == "claude_code_exec":
+        if (
+            str(cfg.get("student_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.student", "student_model")
+        ):
+            cfg["student_model"] = default_model_for_backend("claude_chat")
+
+    if not cfg.get("out_root"):
+        env = cfg.get("env", "unknown")
+        model = cfg.get("student_model", "unknown").replace("/", "-")
+        ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}")
+
+    cfg["out_root"] = os.path.abspath(cfg["out_root"])
+
+    out_root = cfg["out_root"]
+    os.makedirs(out_root, exist_ok=True)
+
+    # Load skill
+    skill_path = os.path.abspath(args.skill)
+    with open(skill_path) as f:
+        skill_content = f.read()
+    print(f"  [skill] {skill_path} ({len(skill_content)} chars)")
+
+    # Configure models
+    configure_azure_openai(
+        endpoint=(cfg.get("azure_openai_endpoint") or cfg.get("azure_endpoint") or None),
+        api_version=(cfg.get("azure_openai_api_version") or cfg.get("azure_api_version") or None),
+        api_key=(cfg.get("azure_openai_api_key") or cfg.get("azure_api_key") or None),
+        auth_mode=cfg.get("azure_openai_auth_mode") or None,
+        ad_scope=cfg.get("azure_openai_ad_scope") or None,
+        managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None,
+        teacher_endpoint=cfg.get("teacher_azure_openai_endpoint") or None,
+        teacher_api_version=cfg.get("teacher_azure_openai_api_version") or None,
+        teacher_api_key=cfg.get("teacher_azure_openai_api_key") or None,
+        teacher_auth_mode=cfg.get("teacher_azure_openai_auth_mode") or None,
+        teacher_ad_scope=cfg.get("teacher_azure_openai_ad_scope") or None,
+        teacher_managed_identity_client_id=(
+            cfg.get("teacher_azure_openai_managed_identity_client_id") or None
+        ),
+        student_endpoint=cfg.get("student_azure_openai_endpoint") or None,
+        student_api_version=cfg.get("student_azure_openai_api_version") or None,
+        student_api_key=cfg.get("student_azure_openai_api_key") or None,
+        student_auth_mode=cfg.get("student_azure_openai_auth_mode") or None,
+        student_ad_scope=cfg.get("student_azure_openai_ad_scope") or None,
+        student_managed_identity_client_id=(
+            cfg.get("student_azure_openai_managed_identity_client_id") or None
+        ),
+    )
+    set_teacher_backend(cfg.get("teacher_backend", "openai_chat"))
+    set_student_backend(cfg.get("student_backend", "openai_chat"))
+    set_teacher_deployment(cfg.get("teacher_model", default_model_for_backend(backend)))
+    set_student_deployment(cfg.get("student_model", default_model_for_backend(backend)))
+    configure_codex_exec(
+        path=cfg.get("codex_exec_path", "codex"),
+        sandbox=cfg.get("codex_exec_sandbox", "workspace-write"),
+        profile=cfg.get("codex_exec_profile", ""),
+        full_auto=cfg.get("codex_exec_full_auto", False),
+        reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"),
+        use_sdk=cfg.get("codex_exec_use_sdk", None),
+        network_access=cfg.get("codex_exec_network_access", False),
+        web_search=cfg.get("codex_exec_web_search", False),
+        approval_policy=cfg.get("codex_exec_approval_policy", "never"),
+    )
+    configure_claude_code_exec(
+        path=cfg.get("claude_code_exec_path", "claude"),
+        profile=cfg.get("claude_code_exec_profile", ""),
+        use_sdk=cfg.get("claude_code_exec_use_sdk", None),
+        effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
+        max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
+    )
+    set_reasoning_effort(cfg.get("reasoning_effort", "") or None)
+
+    # Build adapter
+    adapter = get_adapter(cfg)
+    adapter.setup(cfg)
+
+    seed = cfg.get("seed", 42)
+    split = args.split or "all"
+
+    if split == "all":
+        items = (
+            adapter.build_eval_env(0, "train", seed)
+            + adapter.build_eval_env(0, "valid_seen", seed)
+            + adapter.build_eval_env(0, "valid_unseen", seed)
+        )
+    else:
+        env_num = cfg.get("test_env_num", 0)
+        items = adapter.build_eval_env(env_num, split, seed)
+
+    print(f"\n  [eval] split={split}  items={len(items)}")
+    print(f"  [eval] out_root={out_root}")
+    print(f"{'='*60}")
+
+    # Run rollout
+    results = adapter.rollout(items, skill_content, out_root)
+
+    # Score
+    hard, soft = compute_score(results)
+    print(f"\n{'='*60}")
+    print(f"  Results: hard={hard:.4f}  soft={soft:.4f}  (n={len(results)})")
+    print(f"{'='*60}")
+
+    # Save summary
+    summary = {
+        "skill": skill_path,
+        "split": split,
+        "n_items": len(results),
+        "hard": hard,
+        "soft": soft,
+    }
+    with open(os.path.join(out_root, "eval_summary.json"), "w") as f:
+        json.dump(summary, f, indent=2, ensure_ascii=False)
+
+    print(f"  Saved to: {out_root}")
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_prompt_custom.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""Standalone eval: CUSTOM prompt (with Critical Rules) on verified-400.
+
+Usage:
+    python scripts/eval_prompt_custom.py --workers 8
+    python scripts/eval_prompt_custom.py --workers 32 --limit 20
+"""
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import random
+import re
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
+
+import openpyxl
+
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+from reflact.model import (
+    chat_messages_with_deployment,
+    configure_azure_openai,
+    set_backend,
+    set_student_deployment,
+)
+from reflact.envs.spreadsheetbench.evaluator import evaluate
+
+
+# ── Config ──────────────────────────────────────────────────────────────────
+
+DATA_ROOT = "/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH = os.path.join(DATA_ROOT, "dataset.json")
+MODEL = "gpt-5-mini"
+
+# ── Custom Prompt (with Critical Rules) ─────────────────────────────────────
+
+_SYSTEM_TEMPLATE = """\
+You are an expert Python programmer specializing in spreadsheet manipulation.
+You will be given a user instruction together with a preview of an input .xlsx file.
+Your job is to write a single self-contained Python script that reads the input file
+at the path stored in the variable INPUT_PATH, performs the requested manipulation,
+and saves the result to OUTPUT_PATH.
+
+## Critical Rules
+1. NEVER write Excel formulas to cells. openpyxl does NOT compute formulas —
+   the evaluator will see None. Compute results in Python and write literal values.
+2. Use only: standard library, openpyxl, pandas.
+3. Do NOT hardcode cell values from the preview — iterate over actual rows.
+4. The script must define INPUT_PATH and OUTPUT_PATH at the top.
+
+{skill_section}\
+Return ONLY the Python code inside a single ```python ... ``` fenced block.
+"""
+
+
+def build_system(skill_content: str = "") -> str:
+    if skill_content.strip():
+        skill_section = f"## Skill\n{skill_content.strip()}\n\n"
+    else:
+        skill_section = ""
+    return _SYSTEM_TEMPLATE.format(skill_section=skill_section)
+
+
+def build_user(instruction, input_xlsx, instruction_type="", answer_position=""):
+    try:
+        preview = _preview_workbook(input_xlsx)
+    except Exception as e:
+        preview = f"(failed to preview: {e})"
+    extra = ""
+    if instruction_type:
+        extra += f"\nInstruction type: {instruction_type}"
+    if answer_position:
+        extra += f"\nExpected answer position: {answer_position}"
+    return (
+        f"# Instruction\n{instruction}\n{extra}\n\n"
+        f"# Input spreadsheet preview\n{preview}\n\n"
+        "# Task\n"
+        "Write a Python script that reads the workbook from the variable `INPUT_PATH`, "
+        "applies the instruction, and writes the modified workbook to `OUTPUT_PATH`. "
+        "Preserve all other cells unchanged. "
+        "The preview may be truncated — do not hardcode row counts; "
+        "iterate over all actual rows in the workbook instead.\n"
+        "Return only a ```python``` code block."
+    )
+
+
+# ── Shared utilities ────────────────────────────────────────────────────────
+
+def _preview_workbook(path, max_rows=5, max_cols=20):
+    wb = openpyxl.load_workbook(path, data_only=False)
+    chunks = []
+    for sn in wb.sheetnames:
+        ws = wb[sn]
+        chunks.append(f"## Sheet: {sn}  (dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})")
+        for row in ws.iter_rows(min_row=1, max_row=min(ws.max_row, max_rows),
+                                max_col=min(ws.max_column, max_cols), values_only=False):
+            cells = []
+            for c in row:
+                v = c.value
+                s = "" if v is None else str(v)
+                if len(s) > 40: s = s[:37] + "..."
+                cells.append(f"{c.coordinate}={s}")
+            chunks.append(" | ".join(cells))
+        if ws.max_row > max_rows:
+            chunks.append(f"... ({ws.max_row - max_rows} more rows)")
+        chunks.append("")
+    wb.close()
+    return "\n".join(chunks)
+
+
+def extract_code(text):
+    if "```" not in text:
+        return text.strip()
+    start = text.find("```")
+    nl = text.find("\n", start)
+    end = text.find("```", nl + 1)
+    if nl == -1 or end == -1:
+        return text.strip()
+    return text[nl + 1:end].strip()
+
+
+_PATH_RE = re.compile(r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE)
+
+def strip_paths(code):
+    return _PATH_RE.sub("", code)
+
+
+RUNNER_TEMPLATE = textwrap.dedent("""
+    import os, sys, traceback
+    INPUT_PATH = {input_path!r}
+    OUTPUT_PATH = {output_path!r}
+    try:
+    {code_indented}
+    except Exception:
+        traceback.print_exc()
+        sys.exit(2)
+""")
+
+
+def run_code(code, input_path, output_path, timeout=120):
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    cleaned = strip_paths(code)
+    indented = textwrap.indent(cleaned, "    ")
+    script = RUNNER_TEMPLATE.format(input_path=input_path, output_path=output_path, code_indented=indented)
+    with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
+        f.write(script)
+        tmp = f.name
+    try:
+        proc = subprocess.run([sys.executable, tmp], capture_output=True, text=True, timeout=timeout)
+        if proc.returncode != 0:
+            return False, (proc.stdout + "\n" + proc.stderr).strip()
+        if not os.path.exists(output_path):
+            return False, "output file was not created"
+        return True, ""
+    except subprocess.TimeoutExpired:
+        return False, f"timeout after {timeout}s"
+    finally:
+        try: os.unlink(tmp)
+        except OSError: pass
+
+
+def find_test_cases(task_dir):
+    cases = []
+    for ip in sorted(glob.glob(os.path.join(task_dir, "*_input.xlsx"))):
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_input.xlsx", "_answer.xlsx")
+        if os.path.exists(ap): cases.append((no, ip, ap))
+    for ip in sorted(glob.glob(os.path.join(task_dir, "*_init.xlsx"))):
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_init.xlsx", "_golden.xlsx")
+        if os.path.exists(ap): cases.append((no, ip, ap))
+    if not cases:
+        bare_init = os.path.join(task_dir, "initial.xlsx")
+        bare_gold = os.path.join(task_dir, "golden.xlsx")
+        if os.path.exists(bare_init) and os.path.exists(bare_gold):
+            cases.append(("1", bare_init, bare_gold))
+    return cases
+
+
+def load_items(path):
+    if path.endswith(".json"):
+        with open(path) as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            data = data.get("data") or list(data.values())
+        return list(data)
+    items = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line: items.append(json.loads(line))
+    return items
+
+
+# ── LLM call ────────────────────────────────────────────────────────────────
+
+def llm_call(messages, deployment, max_tokens=16384, retries=5, llm_timeout=120):
+    raw, _ = chat_messages_with_deployment(
+        deployment=deployment,
+        messages=messages,
+        max_completion_tokens=max_tokens,
+        retries=retries,
+        stage="rollout",
+        timeout=llm_timeout,
+    )
+    return str(raw or "")
+
+
+# ── Process one task ────────────────────────────────────────────────────────
+
+def process_one(item, data_root, out_root, model):
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position = f"{answer_sheet}!{answer_position}"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {"id": task_id, "ok": False, "hard": 0, "soft": 0.0,
+              "n_cases": 0, "n_pass": 0, "fail_reason": "", "error": ""}
+    try:
+        cases = find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out, exist_ok=True)
+
+        # LLM call
+        system = build_system("")
+        user = build_user(instruction, cases[0][1], instruction_type, answer_position)
+        messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+        raw = llm_call(messages, model)
+        time.sleep(3)
+        code = extract_code(raw)
+
+        with open(os.path.join(task_out, "code.py"), "w") as f: f.write(code)
+        with open(os.path.join(task_out, "raw.txt"), "w") as f: f.write(raw)
+
+        if not code.strip():
+            result["fail_reason"] = "empty-code"
+            return result
+
+        # Execute + evaluate each test case
+        for no, ip, ap in cases:
+            pred = os.path.join(task_out, f"{no}_pred.xlsx")
+            ok_exec, err = run_code(code, ip, pred)
+            if not ok_exec:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"exec: {err[:200]}"
+                continue
+            try:
+                ev = evaluate(pred, ap, instruction_type, answer_position)
+            except Exception as e:
+                ev = {"ok": False, "reason": str(e)}
+            if ev["ok"]:
+                result["n_pass"] += 1
+
+        nc, np = result["n_cases"], result["n_pass"]
+        result["soft"] = np / nc if nc else 0.0
+        result["hard"] = 1 if nc > 0 and np == nc else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]: result["fail_reason"] = ""
+        return result
+    except Exception as e:
+        result["fail_reason"] = f"unexpected: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Main ────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="Eval CUSTOM prompt on verified-400")
+    ap.add_argument("--model", default=MODEL)
+    ap.add_argument("--backend", choices=["azure_openai", "codex", "claude"], default="azure_openai")
+    ap.add_argument("--azure_endpoint", default="")
+    ap.add_argument("--azure_api_version", default="")
+    ap.add_argument("--azure_api_key", default="")
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--limit", type=int, default=0)
+    ap.add_argument("--out_root", default="")
+    args = ap.parse_args()
+
+    set_backend(args.backend)
+    configure_azure_openai(
+        endpoint=args.azure_endpoint or None,
+        api_version=args.azure_api_version or None,
+        api_key=args.azure_api_key or None,
+    )
+    set_student_deployment(args.model)
+    ts = time.strftime("%Y%m%d_%H%M%S")
+    out_root = args.out_root or os.path.join(_PROJECT_ROOT, "outputs", f"prompt_custom_{args.model}_{ts}")
+    out_root = os.path.abspath(out_root)
+    os.makedirs(out_root, exist_ok=True)
+
+    items = load_items(JSONL_PATH)
+    if args.limit: items = items[:args.limit]
+
+    print(f"{'='*60}")
+    print(f"  Prompt: CUSTOM (Critical Rules)")
+    print(f"  Model:  {args.model}")
+    print(f"  Items:  {len(items)}")
+    print(f"  Output: {out_root}")
+    print(f"{'='*60}")
+
+    t0 = time.time()
+    results = []
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        futs = {ex.submit(process_one, it, DATA_ROOT, out_root, args.model): it for it in items}
+        for i, fut in enumerate(as_completed(futs), 1):
+            item = futs[fut]
+            try:
+                res = fut.result(timeout=300)
+            except FuturesTimeoutError:
+                res = {"id": str(item["id"]), "ok": False, "hard": 0, "soft": 0.0,
+                       "n_cases": 0, "n_pass": 0, "fail_reason": "timeout"}
+            except Exception as e:
+                res = {"id": str(item["id"]), "ok": False, "hard": 0, "soft": 0.0,
+                       "n_cases": 0, "n_pass": 0, "fail_reason": str(e)}
+            results.append(res)
+            status = "PASS" if res.get("hard") else "FAIL"
+            dt = time.time() - t0
+            print(f"  {i}/{len(items)} id={res['id']:<10} {status}  cases={res.get('n_pass',0)}/{res.get('n_cases',0)}  dt={dt:.0f}s")
+
+    # Summary
+    hard_sum = sum(r.get("hard", 0) for r in results)
+    soft_sum = sum(r.get("soft", 0.0) for r in results)
+    n = len(results)
+    print(f"\n{'='*60}")
+    print(f"  CUSTOM prompt: hard={hard_sum}/{n}={hard_sum/n:.4f}  soft={soft_sum/n:.4f}")
+    print(f"{'='*60}")
+
+    with open(os.path.join(out_root, "results.jsonl"), "w") as f:
+        for r in results:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    with open(os.path.join(out_root, "summary.json"), "w") as f:
+        json.dump({"prompt": "custom", "model": args.model, "n": n,
+                   "hard": hard_sum/n, "soft": soft_sum/n}, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_prompt_official.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""Standalone eval: OFFICIAL prompt (SpreadsheetBench original) on verified-400.
+
+Usage:
+    python scripts/eval_prompt_official.py --workers 8
+    python scripts/eval_prompt_official.py --workers 32 --limit 20
+"""
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import random
+import re
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError
+
+import openpyxl
+
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+from reflact.model import (
+    chat_messages_with_deployment,
+    configure_azure_openai,
+    set_backend,
+    set_student_deployment,
+)
+from reflact.envs.spreadsheetbench.evaluator import evaluate
+
+
+# ── Config ──────────────────────────────────────────────────────────────────
+
+DATA_ROOT = "/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH = os.path.join(DATA_ROOT, "dataset.json")
+MODEL = "gpt-5-mini"
+
+# ── Official Prompt (from SpreadsheetBench src/prompt.py) ───────────────────
+
+_SYSTEM_PROMPT = (
+    "You are an expert Python programmer specializing in spreadsheet manipulation. "
+    "You will be given a user instruction together with a preview of an input .xlsx file. "
+    "Your job is to write a single self-contained Python script that reads the input file "
+    "at the path stored in the variable INPUT_PATH, performs the requested manipulation, "
+    "and saves the result to OUTPUT_PATH. Use only the standard library, openpyxl, and pandas. "
+    "Do not print anything. Do not use input(). Do not hardcode file paths. "
+    "Return ONLY the Python code inside a single ```python ... ``` fenced block."
+)
+
+
+def build_system(skill_content: str = "") -> str:
+    base = _SYSTEM_PROMPT
+    if skill_content.strip():
+        base += f"\n\n## Skill\n{skill_content.strip()}"
+    return base
+
+
+def build_user(instruction, input_xlsx, instruction_type="", answer_position=""):
+    try:
+        preview = _preview_workbook(input_xlsx)
+    except Exception as e:
+        preview = f"(failed to preview: {e})"
+    extra = ""
+    if instruction_type:
+        extra += f"\nInstruction type: {instruction_type}"
+    if answer_position:
+        extra += f"\nExpected answer position: {answer_position}"
+    return (
+        f"# Instruction\n{instruction}\n{extra}\n\n"
+        f"# Input spreadsheet preview\n{preview}\n\n"
+        "# Task\n"
+        "Write a Python script that reads the workbook from the variable `INPUT_PATH`, "
+        "applies the instruction, and writes the modified workbook to `OUTPUT_PATH`. "
+        "Preserve all other cells unchanged. "
+        "The preview may be truncated — do not hardcode row counts or assume the data ends at the last previewed row; "
+        "iterate over all actual rows in the workbook instead. "
+        "Return only a ```python``` code block."
+    )
+
+
+# ── Shared utilities (identical to custom version) ──────────────────────────
+
+def _preview_workbook(path, max_rows=5, max_cols=20):
+    wb = openpyxl.load_workbook(path, data_only=False)
+    chunks = []
+    for sn in wb.sheetnames:
+        ws = wb[sn]
+        chunks.append(f"## Sheet: {sn}  (dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})")
+        for row in ws.iter_rows(min_row=1, max_row=min(ws.max_row, max_rows),
+                                max_col=min(ws.max_column, max_cols), values_only=False):
+            cells = []
+            for c in row:
+                v = c.value
+                s = "" if v is None else str(v)
+                if len(s) > 40: s = s[:37] + "..."
+                cells.append(f"{c.coordinate}={s}")
+            chunks.append(" | ".join(cells))
+        if ws.max_row > max_rows:
+            chunks.append(f"... ({ws.max_row - max_rows} more rows)")
+        chunks.append("")
+    wb.close()
+    return "\n".join(chunks)
+
+
+def extract_code(text):
+    if "```" not in text:
+        return text.strip()
+    start = text.find("```")
+    nl = text.find("\n", start)
+    end = text.find("```", nl + 1)
+    if nl == -1 or end == -1:
+        return text.strip()
+    return text[nl + 1:end].strip()
+
+
+_PATH_RE = re.compile(r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE)
+
+def strip_paths(code):
+    return _PATH_RE.sub("", code)
+
+
+RUNNER_TEMPLATE = textwrap.dedent("""
+    import os, sys, traceback
+    INPUT_PATH = {input_path!r}
+    OUTPUT_PATH = {output_path!r}
+    try:
+    {code_indented}
+    except Exception:
+        traceback.print_exc()
+        sys.exit(2)
+""")
+
+
+def run_code(code, input_path, output_path, timeout=120):
+    os.makedirs(os.path.dirname(output_path), exist_ok=True)
+    cleaned = strip_paths(code)
+    indented = textwrap.indent(cleaned, "    ")
+    script = RUNNER_TEMPLATE.format(input_path=input_path, output_path=output_path, code_indented=indented)
+    with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
+        f.write(script)
+        tmp = f.name
+    try:
+        proc = subprocess.run([sys.executable, tmp], capture_output=True, text=True, timeout=timeout)
+        if proc.returncode != 0:
+            return False, (proc.stdout + "\n" + proc.stderr).strip()
+        if not os.path.exists(output_path):
+            return False, "output file was not created"
+        return True, ""
+    except subprocess.TimeoutExpired:
+        return False, f"timeout after {timeout}s"
+    finally:
+        try: os.unlink(tmp)
+        except OSError: pass
+
+
+def find_test_cases(task_dir):
+    cases = []
+    for ip in sorted(glob.glob(os.path.join(task_dir, "*_input.xlsx"))):
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_input.xlsx", "_answer.xlsx")
+        if os.path.exists(ap): cases.append((no, ip, ap))
+    for ip in sorted(glob.glob(os.path.join(task_dir, "*_init.xlsx"))):
+        no = os.path.basename(ip).split("_", 1)[0]
+        ap = ip.replace("_init.xlsx", "_golden.xlsx")
+        if os.path.exists(ap): cases.append((no, ip, ap))
+    if not cases:
+        bare_init = os.path.join(task_dir, "initial.xlsx")
+        bare_gold = os.path.join(task_dir, "golden.xlsx")
+        if os.path.exists(bare_init) and os.path.exists(bare_gold):
+            cases.append(("1", bare_init, bare_gold))
+    return cases
+
+
+def load_items(path):
+    if path.endswith(".json"):
+        with open(path) as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            data = data.get("data") or list(data.values())
+        return list(data)
+    items = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if line: items.append(json.loads(line))
+    return items
+
+
+# ── LLM call ────────────────────────────────────────────────────────────────
+
+def llm_call(messages, deployment, max_tokens=16384, retries=5, llm_timeout=120):
+    raw, _ = chat_messages_with_deployment(
+        deployment=deployment,
+        messages=messages,
+        max_completion_tokens=max_tokens,
+        retries=retries,
+        stage="rollout",
+        timeout=llm_timeout,
+    )
+    return str(raw or "")
+
+
+# ── Process one task ────────────────────────────────────────────────────────
+
+def process_one(item, data_root, out_root, model):
+    task_id = str(item["id"])
+    instruction = item["instruction"]
+    instruction_type = item.get("instruction_type", "")
+    answer_position = item.get("answer_position", "")
+    answer_sheet = item.get("answer_sheet", "")
+    if answer_position and answer_sheet and "!" not in answer_position:
+        answer_position = f"{answer_sheet}!{answer_position}"
+
+    sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}")
+    task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp)
+
+    result = {"id": task_id, "ok": False, "hard": 0, "soft": 0.0,
+              "n_cases": 0, "n_pass": 0, "fail_reason": "", "error": ""}
+    try:
+        cases = find_test_cases(task_dir)
+        result["n_cases"] = len(cases)
+        if not cases:
+            result["fail_reason"] = "no-test-cases"
+            return result
+
+        task_out = os.path.join(out_root, "predictions", task_id)
+        os.makedirs(task_out, exist_ok=True)
+
+        # LLM call
+        system = build_system("")
+        user = build_user(instruction, cases[0][1], instruction_type, answer_position)
+        messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+        raw = llm_call(messages, model)
+        time.sleep(3)
+        code = extract_code(raw)
+
+        with open(os.path.join(task_out, "code.py"), "w") as f: f.write(code)
+        with open(os.path.join(task_out, "raw.txt"), "w") as f: f.write(raw)
+
+        if not code.strip():
+            result["fail_reason"] = "empty-code"
+            return result
+
+        # Execute + evaluate each test case
+        for no, ip, ap in cases:
+            pred = os.path.join(task_out, f"{no}_pred.xlsx")
+            ok_exec, err = run_code(code, ip, pred)
+            if not ok_exec:
+                if not result["fail_reason"]:
+                    result["fail_reason"] = f"exec: {err[:200]}"
+                continue
+            try:
+                ev = evaluate(pred, ap, instruction_type, answer_position)
+            except Exception as e:
+                ev = {"ok": False, "reason": str(e)}
+            if ev["ok"]:
+                result["n_pass"] += 1
+
+        nc, np = result["n_cases"], result["n_pass"]
+        result["soft"] = np / nc if nc else 0.0
+        result["hard"] = 1 if nc > 0 and np == nc else 0
+        result["ok"] = bool(result["hard"])
+        if result["ok"]: result["fail_reason"] = ""
+        return result
+    except Exception as e:
+        result["fail_reason"] = f"unexpected: {e}"
+        result["error"] = traceback.format_exc()
+        return result
+
+
+# ── Main ────────────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description="Eval OFFICIAL prompt on verified-400")
+    ap.add_argument("--model", default=MODEL)
+    ap.add_argument("--backend", choices=["azure_openai", "codex", "claude"], default="azure_openai")
+    ap.add_argument("--azure_endpoint", default="")
+    ap.add_argument("--azure_api_version", default="")
+    ap.add_argument("--azure_api_key", default="")
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--limit", type=int, default=0)
+    ap.add_argument("--out_root", default="")
+    args = ap.parse_args()
+
+    set_backend(args.backend)
+    configure_azure_openai(
+        endpoint=args.azure_endpoint or None,
+        api_version=args.azure_api_version or None,
+        api_key=args.azure_api_key or None,
+    )
+    set_student_deployment(args.model)
+    ts = time.strftime("%Y%m%d_%H%M%S")
+    out_root = args.out_root or os.path.join(_PROJECT_ROOT, "outputs", f"prompt_official_{args.model}_{ts}")
+    out_root = os.path.abspath(out_root)
+    os.makedirs(out_root, exist_ok=True)
+
+    items = load_items(JSONL_PATH)
+    if args.limit: items = items[:args.limit]
+
+    print(f"{'='*60}")
+    print(f"  Prompt: OFFICIAL (SpreadsheetBench original)")
+    print(f"  Model:  {args.model}")
+    print(f"  Items:  {len(items)}")
+    print(f"  Output: {out_root}")
+    print(f"{'='*60}")
+
+    t0 = time.time()
+    results = []
+    with ThreadPoolExecutor(max_workers=args.workers) as ex:
+        futs = {ex.submit(process_one, it, DATA_ROOT, out_root, args.model): it for it in items}
+        for i, fut in enumerate(as_completed(futs), 1):
+            item = futs[fut]
+            try:
+                res = fut.result(timeout=300)
+            except FuturesTimeoutError:
+                res = {"id": str(item["id"]), "ok": False, "hard": 0, "soft": 0.0,
+                       "n_cases": 0, "n_pass": 0, "fail_reason": "timeout"}
+            except Exception as e:
+                res = {"id": str(item["id"]), "ok": False, "hard": 0, "soft": 0.0,
+                       "n_cases": 0, "n_pass": 0, "fail_reason": str(e)}
+            results.append(res)
+            status = "PASS" if res.get("hard") else "FAIL"
+            dt = time.time() - t0
+            print(f"  {i}/{len(items)} id={res['id']:<10} {status}  cases={res.get('n_pass',0)}/{res.get('n_cases',0)}  dt={dt:.0f}s")
+
+    # Summary
+    hard_sum = sum(r.get("hard", 0) for r in results)
+    soft_sum = sum(r.get("soft", 0.0) for r in results)
+    n = len(results)
+    print(f"\n{'='*60}")
+    print(f"  OFFICIAL prompt: hard={hard_sum}/{n}={hard_sum/n:.4f}  soft={soft_sum/n:.4f}")
+    print(f"{'='*60}")
+
+    with open(os.path.join(out_root, "results.jsonl"), "w") as f:
+        for r in results:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    with open(os.path.join(out_root, "summary.json"), "w") as f:
+        json.dump({"prompt": "official", "model": args.model, "n": n,
+                   "hard": hard_sum/n, "soft": soft_sum/n}, f, indent=2)
+
+
+if __name__ == "__main__":
+    main()

scripts/eval_searchqa_val500.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT3 — SearchQA Eval-Only (验证集 500)
+#
+# Usage:
+#   bash scripts/eval_searchqa_val500.sh --skill_path outputs/xxx/best_skill.md
+#   bash scripts/eval_searchqa_val500.sh --skill_path outputs/xxx/best_skill.md --workers 32
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5-mini}"
+
+VAL_PATH="/home/azureuser/workspace-yqh/refleAct/search-qa/data/searchqa_val_500.json"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/searchqa_eval_val500_${STUDENT_DEPLOYMENT}_${TIMESTAMP}"
+
+echo "============================================================"
+echo "  ReflACT3 — SearchQA Eval-Only (val-500)"
+echo "  Student:  ${STUDENT_DEPLOYMENT}"
+echo "  Data:     ${VAL_PATH}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/eval_only.py \
+    --config configs/searchqa_default.yaml \
+    --data_path "${VAL_PATH}" \
+    --out_root "${DEFAULT_OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"

scripts/eval_verified400.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# Eval skill0 on full SpreadsheetBench verified-400
+#
+# Usage:
+#   bash scripts/eval_verified400.sh
+#   bash scripts/eval_verified400.sh --workers 64
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+DATA_ROOT="/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH="${DATA_ROOT}/dataset.json"
+SKILL_PATH="${PROJECT_ROOT}/reflact/envs/spreadsheetbench/skills/initial.md"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUT_ROOT="${PROJECT_ROOT}/outputs/eval_verified400_${TIMESTAMP}"
+
+echo "============================================================"
+echo "  Eval skill0 on verified-400 (full)"
+echo "============================================================"
+echo "  data_root:  ${DATA_ROOT}"
+echo "  skill:      ${SKILL_PATH}"
+echo "  out_root:   ${OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/eval_only.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --skill "${SKILL_PATH}" \
+    --split all \
+    --data_root "${DATA_ROOT}" \
+    --jsonl_path "${JSONL_PATH}" \
+    --out_root "${OUT_ROOT}" \
+    "$@"

scripts/eval_verified400_multi.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# Eval skill0 on full SpreadsheetBench verified-400 (MULTI-ROUND codegen)
+#
+# Usage:
+#   bash scripts/eval_verified400_multi.sh
+#   bash scripts/eval_verified400_multi.sh --workers 64 --max_turns 5
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+DATA_ROOT="/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH="${DATA_ROOT}/dataset.json"
+SKILL_PATH="${PROJECT_ROOT}/reflact/envs/spreadsheetbench/skills/initial.md"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUT_ROOT="${PROJECT_ROOT}/outputs/eval_multi_verified400_${TIMESTAMP}"
+
+echo "============================================================"
+echo "  Eval skill0 — MULTI-ROUND codegen — verified-400"
+echo "============================================================"
+echo "  data_root:  ${DATA_ROOT}"
+echo "  skill:      ${SKILL_PATH}"
+echo "  mode:       multi"
+echo "  out_root:   ${OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/eval_only.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --skill "${SKILL_PATH}" \
+    --split all \
+    --mode multi \
+    --data_root "${DATA_ROOT}" \
+    --jsonl_path "${JSONL_PATH}" \
+    --out_root "${OUT_ROOT}" \
+    "$@"

scripts/eval_verified400_single.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# Eval skill0 on full SpreadsheetBench verified-400 (SINGLE-ROUND codegen)
+#
+# Usage:
+#   bash scripts/eval_verified400_single.sh
+#   bash scripts/eval_verified400_single.sh --workers 64
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+DATA_ROOT="/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH="${DATA_ROOT}/dataset.json"
+SKILL_PATH="${PROJECT_ROOT}/reflact/envs/spreadsheetbench/skills/initial.md"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUT_ROOT="${PROJECT_ROOT}/outputs/eval_single_verified400_${TIMESTAMP}"
+
+echo "============================================================"
+echo "  Eval skill0 — SINGLE-ROUND codegen — verified-400"
+echo "============================================================"
+echo "  data_root:  ${DATA_ROOT}"
+echo "  skill:      ${SKILL_PATH}"
+echo "  mode:       single"
+echo "  out_root:   ${OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/eval_only.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --skill "${SKILL_PATH}" \
+    --split all \
+    --mode single \
+    --data_root "${DATA_ROOT}" \
+    --jsonl_path "${JSONL_PATH}" \
+    --out_root "${OUT_ROOT}" \
+    "$@"

scripts/launch_harness_bestsetting_from_scratch.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PY="${PY:-python}"
+RUN_ROOT="${RUN_ROOT:-$ROOT/outputs/harness_bestsetting_fromscratch_$(date -u +%Y%m%d_%H%M%S)_run}"
+MAX_PARALLEL="${MAX_PARALLEL:-2}"
+
+mkdir -p "$RUN_ROOT/logs"
+cd "$ROOT"
+export PYTHONPATH="$ROOT:${PYTHONPATH:-}"
+
+COMMON=(
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint=https://t2vgoaigpt4o3.openai.azure.com/
+  model.teacher_azure_openai_api_version=2024-12-01-preview
+  model.teacher_azure_openai_auth_mode=azure_cli
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.min_learning_rate=2
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+)
+
+CODEX=(
+  model.student_backend=codex_exec
+  model.student=gpt-5.5
+  model.codex_exec_use_sdk=auto
+  model.codex_exec_sandbox=workspace-write
+  model.codex_exec_approval_policy=never
+  model.codex_trace_to_teacher=true
+)
+
+CLAUDE=(
+  model.student_backend=claude_code_exec
+  model.student=claude-sonnet-4-6
+  model.claude_code_exec_use_sdk=auto
+  model.codex_trace_to_teacher=false
+)
+
+active=0
+launch() {
+  local run_id="$1"; shift
+  local config="$1"; shift
+  local out="$RUN_ROOT/$run_id"
+  local log="$RUN_ROOT/logs/$run_id.log"
+  echo "START $run_id"
+  setsid "$PY" -u scripts/train.py \
+    --config "$config" \
+    --cfg-options "${COMMON[@]}" "$@" "env.out_root=$out" \
+    > "$log" 2>&1 < /dev/null &
+  active=$((active + 1))
+  if (( active >= MAX_PARALLEL )); then
+    wait -n
+    active=$((active - 1))
+  fi
+}
+
+# SearchQA best openai-chat setting: optimizer.lr_scheduler=constant.
+launch HARNESS-BESTSETTING-searchqa-codex configs/searchqa/default.yaml \
+  "${CODEX[@]}" \
+  train.batch_size=40 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant \
+  env.split_dir=data/searchqa/splits
+
+launch HARNESS-BESTSETTING-searchqa-claude configs/searchqa/default.yaml \
+  "${CLAUDE[@]}" \
+  train.batch_size=40 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant \
+  env.split_dir=data/searchqa/splits
+
+# SpreadsheetBench best openai-chat setting: optimizer.lr_scheduler=constant.
+# Must stay env.mode=multi; exec-backend multi support is fixed on this branch.
+launch HARNESS-BESTSETTING-spreadsheetbench-codex configs/spreadsheetbench/default.yaml \
+  "${CODEX[@]}" \
+  train.batch_size=40 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi env.workers=4
+
+launch HARNESS-BESTSETTING-spreadsheetbench-claude configs/spreadsheetbench/default.yaml \
+  "${CLAUDE[@]}" \
+  train.batch_size=40 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi env.workers=4
+
+# LiveMathBench best openai-chat setting: optimizer.learning_rate=8.
+launch HARNESS-BESTSETTING-livemathematicianbench-codex configs/livemathematicianbench/default.yaml \
+  "${CODEX[@]}" \
+  train.batch_size=40 optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant \
+  env.split_dir=data/livemathbench/splits
+
+launch HARNESS-BESTSETTING-livemathematicianbench-claude configs/livemathematicianbench/default.yaml \
+  "${CLAUDE[@]}" \
+  train.batch_size=40 optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant \
+  env.split_dir=data/livemathbench/splits
+
+# DocVQA best openai-chat setting was full batch. On 10% harness split, train=107.
+launch HARNESS-BESTSETTING-docvqa10pct-codex configs/docvqa/default.yaml \
+  "${CODEX[@]}" \
+  train.batch_size=107 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct
+
+launch HARNESS-BESTSETTING-docvqa10pct-claude configs/docvqa/default.yaml \
+  "${CLAUDE[@]}" \
+  train.batch_size=107 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct
+
+wait
+echo "All launched runs finished or exited. RUN_ROOT=$RUN_ROOT"

scripts/launch_harness_canonical_claude18.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+# Claude Code on this machine must use the local copilot-api proxy.
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/harness_canonical_claude18_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-harness_canon_claude18_${stamp}}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  model.student_backend=claude_code_exec
+  model.student=claude-sonnet-4-6
+  model.claude_code_exec_use_sdk=sdk
+  model.claude_code_exec_effort=medium
+  model.claude_code_exec_max_thinking_tokens=16384
+  model.codex_trace_to_teacher=false
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local config="$2"
+  local skill="$3"
+  shift 3
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.skill_init="$skill"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+    printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+    printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+    printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+    printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+    printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+SEARCHQA_SKILL="docs/harness_source_skills/searchqa_best_skill.md"
+LIVEMATH_SKILL="docs/harness_source_skills/livemathematicianbench_best_skill.md"
+DOCVQA_SKILL="docs/harness_source_skills/docvqa_best_skill.md"
+SPREADSHEET_SKILL="docs/harness_source_skills/spreadsheetbench_best_skill.md"
+
+launch_run "HARNESS-Claude-SearchQA-sched-constant" "configs/searchqa/default.yaml" "$SEARCHQA_SKILL" \
+  env.split_dir=data/searchqa/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-SearchQA-sched-linear" "configs/searchqa/default.yaml" "$SEARCHQA_SKILL" \
+  env.split_dir=data/searchqa/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=linear
+launch_run "HARNESS-Claude-SearchQA-batch-full" "configs/searchqa/default.yaml" "$SEARCHQA_SKILL" \
+  env.split_dir=data/searchqa/splits \
+  train.batch_size=400 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+launch_run "HARNESS-Claude-SearchQA-lr8" "configs/searchqa/default.yaml" "$SEARCHQA_SKILL" \
+  env.split_dir=data/searchqa/splits \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-Claude-LiveMath-lr8" "configs/livemathematicianbench/default.yaml" "$LIVEMATH_SKILL" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-LiveMath-lr16" "configs/livemathematicianbench/default.yaml" "$LIVEMATH_SKILL" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=16 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-LiveMath-slow10" "configs/livemathematicianbench/default.yaml" "$LIVEMATH_SKILL" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine optimizer.slow_update_samples=10
+launch_run "HARNESS-Claude-LiveMath-sched-linear" "configs/livemathematicianbench/default.yaml" "$LIVEMATH_SKILL" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=linear
+launch_run "HARNESS-Claude-LiveMath-minibatch4" "configs/livemathematicianbench/default.yaml" "$LIVEMATH_SKILL" \
+  env.split_dir=data/livemathbench/splits \
+  gradient.minibatch_size=4 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+
+launch_run "HARNESS-Claude-DocVQA10-batch-full" "configs/docvqa/default.yaml" "$DOCVQA_SKILL" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  train.batch_size=107 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+launch_run "HARNESS-Claude-DocVQA10-lr16" "configs/docvqa/default.yaml" "$DOCVQA_SKILL" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  optimizer.learning_rate=16 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-DocVQA10-lr8" "configs/docvqa/default.yaml" "$DOCVQA_SKILL" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-DocVQA10-minibatch32" "configs/docvqa/default.yaml" "$DOCVQA_SKILL" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  gradient.minibatch_size=32 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+launch_run "HARNESS-Claude-DocVQA10-batch24" "configs/docvqa/default.yaml" "$DOCVQA_SKILL" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  train.batch_size=24 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+
+launch_run "HARNESS-Claude-Spreadsheet-sched-constant-multi" "configs/spreadsheetbench/default.yaml" "$SPREADSHEET_SKILL" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-Spreadsheet-lr4-multi" "configs/spreadsheetbench/default.yaml" "$SPREADSHEET_SKILL" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-Spreadsheet-lr16-multi" "configs/spreadsheetbench/default.yaml" "$SPREADSHEET_SKILL" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  optimizer.learning_rate=16 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+launch_run "HARNESS-Claude-Spreadsheet-minibatch16-multi" "configs/spreadsheetbench/default.yaml" "$SPREADSHEET_SKILL" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  gradient.minibatch_size=16 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"

scripts/launch_harness_canonical_claude4_smoke.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/harness_canonical_claude4_smoke_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-harness_canon_claude4_${stamp}}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  model.student_backend=claude_code_exec
+  model.student=claude-sonnet-4-6
+  model.claude_code_exec_use_sdk=sdk
+  model.claude_code_exec_effort=medium
+  model.claude_code_exec_max_thinking_tokens=16384
+  model.codex_trace_to_teacher=false
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local config="$2"
+  local skill="$3"
+  shift 3
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.skill_init="$skill"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+    printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+    printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+    printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+    printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+    printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+launch_run "HARNESS-Claude-SearchQA-sched-constant" "configs/searchqa/default.yaml" "docs/harness_source_skills/searchqa_best_skill.md" \
+  env.split_dir=data/searchqa/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-Claude-LiveMath-lr8" "configs/livemathematicianbench/default.yaml" "docs/harness_source_skills/livemathematicianbench_best_skill.md" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-Claude-DocVQA10-lr8" "configs/docvqa/default.yaml" "docs/harness_source_skills/docvqa_best_skill.md" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-Claude-Spreadsheet-lr4-multi" "configs/spreadsheetbench/default.yaml" "docs/harness_source_skills/spreadsheetbench_best_skill.md" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"

scripts/launch_harness_canonical_wave1.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+CODEX_WRAPPER="$REPO/scripts/codex_azure_mi.sh"
+
+cd "$REPO"
+
+# Claude Code is routed through the local copilot-api proxy on this machine.
+# Do not rely on interactive Claude login state inside tmux/train workers.
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/harness_canonical_step12_wave1_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-harness_canon_wave1_${stamp}}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local backend="$2"
+  local config="$3"
+  local skill="$4"
+  local split_dir="$5"
+  shift 5
+
+  local -a backend_cfg=()
+  if [[ "$backend" == "codex" ]]; then
+    backend_cfg=(
+      model.student_backend=codex_exec
+      model.student=gpt-5.5
+      model.codex_exec_path="$CODEX_WRAPPER"
+      model.codex_exec_use_sdk=auto
+      model.codex_exec_sandbox=workspace-write
+      model.codex_exec_approval_policy=never
+      model.codex_exec_reasoning_effort=medium
+      model.codex_trace_to_teacher=true
+    )
+  elif [[ "$backend" == "claude" ]]; then
+    backend_cfg=(
+      model.student_backend=claude_code_exec
+      model.student=claude-sonnet-4-6
+      model.claude_code_exec_use_sdk=sdk
+      model.claude_code_exec_effort=medium
+      model.claude_code_exec_max_thinking_tokens=16384
+      model.codex_trace_to_teacher=false
+    )
+  else
+    echo "unknown backend: $backend" >&2
+    exit 1
+  fi
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    "${backend_cfg[@]}"
+    env.split_dir="$split_dir"
+    env.skill_init="$skill"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+    printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+    printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+    printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+    printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+    printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+SEARCHQA_SKILL="docs/harness_source_skills/searchqa_best_skill.md"
+LIVEMATH_SKILL="docs/harness_source_skills/livemathematicianbench_best_skill.md"
+
+SEARCHQA_CFG="configs/searchqa/default.yaml"
+LIVEMATH_CFG="configs/livemathematicianbench/default.yaml"
+
+SEARCHQA_SPLIT="data/searchqa/splits"
+LIVEMATH_SPLIT="data/livemathbench/splits"
+
+for backend in codex claude; do
+  prefix="HARNESS-Codex"
+  [[ "$backend" == "claude" ]] && prefix="HARNESS-Claude"
+
+  launch_run "${prefix}-SearchQA-sched-constant" "$backend" "$SEARCHQA_CFG" "$SEARCHQA_SKILL" "$SEARCHQA_SPLIT" \
+    optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant
+  launch_run "${prefix}-SearchQA-sched-linear" "$backend" "$SEARCHQA_CFG" "$SEARCHQA_SKILL" "$SEARCHQA_SPLIT" \
+    optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=linear
+  launch_run "${prefix}-SearchQA-batch-full" "$backend" "$SEARCHQA_CFG" "$SEARCHQA_SKILL" "$SEARCHQA_SPLIT" \
+    train.batch_size=400 optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=cosine
+  launch_run "${prefix}-SearchQA-lr8" "$backend" "$SEARCHQA_CFG" "$SEARCHQA_SKILL" "$SEARCHQA_SPLIT" \
+    optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+  launch_run "${prefix}-LiveMath-lr8" "$backend" "$LIVEMATH_CFG" "$LIVEMATH_SKILL" "$LIVEMATH_SPLIT" \
+    optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+  launch_run "${prefix}-LiveMath-lr16" "$backend" "$LIVEMATH_CFG" "$LIVEMATH_SKILL" "$LIVEMATH_SPLIT" \
+    optimizer.learning_rate=16 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+done
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"

scripts/launch_harness_initial_claude4.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/harness_initial_claude4_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-harness_initial_claude4_${stamp}}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  model.student_backend=claude_code_exec
+  model.student=claude-sonnet-4-6
+  model.claude_code_exec_use_sdk=sdk
+  model.claude_code_exec_effort=medium
+  model.claude_code_exec_max_thinking_tokens=16384
+  model.codex_trace_to_teacher=false
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local config="$2"
+  shift 2
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+    printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+    printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+    printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+    printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+    printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+launch_run "HARNESS-ClaudeInit-SearchQA-sched-constant" "configs/searchqa/default.yaml" \
+  env.split_dir=data/searchqa/splits \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=2 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-ClaudeInit-LiveMath-lr8" "configs/livemathematicianbench/default.yaml" \
+  env.split_dir=data/livemathbench/splits \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-ClaudeInit-DocVQA10-lr8" "configs/docvqa/default.yaml" \
+  env.split_dir=data/harness_splits/docvqa_zisu_first10pct \
+  optimizer.learning_rate=8 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+launch_run "HARNESS-ClaudeInit-Spreadsheet-lr4-multi" "configs/spreadsheetbench/default.yaml" \
+  env.split_dir=data/spreadsheetbench env.data_root=data/spreadsheetbench/files env.mode=multi \
+  optimizer.learning_rate=4 optimizer.min_learning_rate=1 optimizer.lr_scheduler=constant
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"

scripts/launch_harness_initial_spreadsheet_clean.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/harness_initial_spreadsheet_clean_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-harness_initial_spreadsheet_clean_${stamp}}"
+RUN_ID="HARNESS-ClaudeInit-Spreadsheet-lr4-multi-clean"
+SPLIT_DIR="${SPREADSHEET_SPLIT_DIR:-data/harness_splits/spreadsheetbench_full}"
+DATA_ROOT="${SPREADSHEET_DATA_ROOT:-data/spreadsheetbench/files}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+cmd_file="$RUN_ROOT/commands/${RUN_ID}.sh"
+log_file="$RUN_ROOT/logs/${RUN_ID}.log"
+out_root="$RUN_ROOT/$RUN_ID"
+
+cmd=(
+  "$PYTHON" -u scripts/train.py
+  --config configs/spreadsheetbench/default.yaml
+  --cfg-options
+  model.teacher_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.lr_control_mode=fixed
+  optimizer.skill_update_mode=patch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  model.student_backend=claude_code_exec
+  model.student=claude-sonnet-4-6
+  model.claude_code_exec_use_sdk=sdk
+  model.claude_code_exec_effort=medium
+  model.claude_code_exec_max_thinking_tokens=16384
+  model.codex_trace_to_teacher=false
+  env.out_root="$out_root"
+  env.split_dir="$SPLIT_DIR"
+  env.data_root="$DATA_ROOT"
+  env.mode=multi
+  optimizer.learning_rate=4
+  optimizer.min_learning_rate=1
+  optimizer.lr_scheduler=constant
+)
+
+{
+  echo "#!/usr/bin/env bash"
+  echo "set -euo pipefail"
+  echo "cd '$REPO'"
+  printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+  printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+  printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+  printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+  printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+  printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+  printf '%q ' "${cmd[@]}"
+  printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+} > "$cmd_file"
+chmod +x "$cmd_file"
+
+tmux new-session -d -s "$SESSION" -n "$RUN_ID" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"
+echo "RUN_ID=$RUN_ID"

scripts/launch_lrctrl_fullrewrite_neutral3.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/lrctrl_fullrewrite_neutral3_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-lrctrl_fullrewrite_neutral3_${stamp}}"
+SEED="${3:-42}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.student_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.student=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.student_azure_openai_endpoint="${STUDENT_AZURE_OPENAI_ENDPOINT:-$TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.student_azure_openai_api_version="${STUDENT_AZURE_OPENAI_API_VERSION:-$TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.student_azure_openai_auth_mode="${STUDENT_AZURE_OPENAI_AUTH_MODE:-$TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.student_azure_openai_managed_identity_client_id="${STUDENT_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID:-$TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.student_azure_openai_ad_scope="${STUDENT_AZURE_OPENAI_AD_SCOPE:-$TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed="${SEED}"
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.learning_rate=4
+  optimizer.min_learning_rate=2
+  optimizer.lr_scheduler=cosine
+  optimizer.lr_control_mode=none
+  optimizer.skill_update_mode=full_rewrite_minibatch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local config="$2"
+  shift 2
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+launch_run "LRCTRL-searchqa-full-rewrite-neutral3-seed${SEED}" "configs/searchqa/default.yaml" \
+  env.split_dir=data/ablation_splits/searchqa/2-1-7_seed42
+
+launch_run "LRCTRL-spreadsheetbench-full-rewrite-neutral3-seed${SEED}" "configs/spreadsheetbench/default.yaml" \
+  env.split_dir=data/ablation_splits/spreadsheetbench/2-1-7_seed42 \
+  env.data_root=data/spreadsheetbench_verified_400 \
+  env.mode=multi
+
+launch_run "LRCTRL-livemathematicianbench-full-rewrite-neutral3-seed${SEED}" "configs/livemathematicianbench/default.yaml" \
+  env.split_dir=data/ablation_splits/livemathematicianbench/2-1-7_seed42
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"
+echo "SEED=$SEED"

scripts/launch_lrctrl_fullrewrite_neutral3_spreadsheet_repeats.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/lrctrl_fullrewrite_neutral3_spreadsheet_promptweak_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-lrctrl_fr_spreadsheet_promptweak_${stamp}}"
+START_INDEX="${3:-4}"
+N_REPEATS="${4:-3}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.student_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.student=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.student_azure_openai_endpoint="${STUDENT_AZURE_OPENAI_ENDPOINT:-$TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.student_azure_openai_api_version="${STUDENT_AZURE_OPENAI_API_VERSION:-$TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.student_azure_openai_auth_mode="${STUDENT_AZURE_OPENAI_AUTH_MODE:-$TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.student_azure_openai_managed_identity_client_id="${STUDENT_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID:-$TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.student_azure_openai_ad_scope="${STUDENT_AZURE_OPENAI_AD_SCOPE:-$TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.learning_rate=4
+  optimizer.min_learning_rate=2
+  optimizer.lr_scheduler=cosine
+  optimizer.lr_control_mode=none
+  optimizer.skill_update_mode=full_rewrite_minibatch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  env.split_dir=data/ablation_splits/spreadsheetbench/2-1-7_seed42
+  env.data_root=data/spreadsheetbench_verified_400
+  env.mode=multi
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config configs/spreadsheetbench/default.yaml
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.out_root="$out_root"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+for ((i=START_INDEX; i<START_INDEX+N_REPEATS; i++)); do
+  launch_run "LRCTRL-spreadsheetbench-full-rewrite-neutral3-promptweak-r${i}"
+done
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"
+echo "START_INDEX=$START_INDEX"
+echo "N_REPEATS=$N_REPEATS"

scripts/launch_lrctrl_fullrewrite_neutral3_sq_lm_repeats.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/lrctrl_fullrewrite_neutral3_sq_lm_extra_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-lrctrl_fr_sq_lm_extra_${stamp}}"
+START_INDEX="${3:-4}"
+N_REPEATS="${4:-3}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+COMMON_CFG=(
+  model.teacher_backend=openai_chat
+  model.student_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.student=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.student_azure_openai_endpoint="${STUDENT_AZURE_OPENAI_ENDPOINT:-$TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.student_azure_openai_api_version="${STUDENT_AZURE_OPENAI_API_VERSION:-$TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.student_azure_openai_auth_mode="${STUDENT_AZURE_OPENAI_AUTH_MODE:-$TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.student_azure_openai_managed_identity_client_id="${STUDENT_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID:-$TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.student_azure_openai_ad_scope="${STUDENT_AZURE_OPENAI_AD_SCOPE:-$TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.learning_rate=4
+  optimizer.min_learning_rate=2
+  optimizer.lr_scheduler=cosine
+  optimizer.lr_control_mode=none
+  optimizer.skill_update_mode=full_rewrite_minibatch
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_skill=true
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+)
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  local config="$2"
+  shift 2
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+  local out_root="$RUN_ROOT/$run_id"
+
+  local -a cmd=(
+    "$PYTHON" -u scripts/train.py
+    --config "$config"
+    --cfg-options
+    "${COMMON_CFG[@]}"
+    env.out_root="$out_root"
+    "$@"
+  )
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf '%q ' "${cmd[@]}"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+for ((i=START_INDEX; i<START_INDEX+N_REPEATS; i++)); do
+  launch_run "LRCTRL-searchqa-full-rewrite-neutral3-extra-r${i}" "configs/searchqa/default.yaml" \
+    env.split_dir=data/ablation_splits/searchqa/2-1-7_seed42
+
+  launch_run "LRCTRL-livemathematicianbench-full-rewrite-neutral3-extra-r${i}" "configs/livemathematicianbench/default.yaml" \
+    env.split_dir=data/ablation_splits/livemathematicianbench/2-1-7_seed42
+done
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"
+echo "START_INDEX=$START_INDEX"
+echo "N_REPEATS=$N_REPEATS"

scripts/launch_spreadsheet_full_replacements.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="/home/azureuser/workspace-gzy/SkillReflection"
+PYTHON="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+
+cd "$REPO"
+
+export ANTHROPIC_BASE_URL="${ANTHROPIC_BASE_URL:-http://127.0.0.1:4343}"
+export ANTHROPIC_AUTH_TOKEN="${ANTHROPIC_AUTH_TOKEN:-dummy}"
+export ANTHROPIC_MODEL="${ANTHROPIC_MODEL:-claude-sonnet-4-6}"
+export ANTHROPIC_SMALL_FAST_MODEL="${ANTHROPIC_SMALL_FAST_MODEL:-claude-sonnet-4-6}"
+export DISABLE_NON_ESSENTIAL_MODEL_CALLS="${DISABLE_NON_ESSENTIAL_MODEL_CALLS:-1}"
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="${CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC:-1}"
+
+if [[ -f ".secrets/teacher_oaidr9.env" ]]; then
+  # shellcheck disable=SC1091
+  source ".secrets/teacher_oaidr9.env"
+else
+  echo "missing .secrets/teacher_oaidr9.env" >&2
+  exit 1
+fi
+
+stamp="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${1:-outputs/spreadsheet_full_replacements_workers2_timeout1020_${stamp}_run}"
+SESSION="${2:-spreadsheet_full_replacements_${stamp}}"
+
+mkdir -p "$RUN_ROOT/logs" "$RUN_ROOT/commands"
+
+tmux_started=0
+
+launch_run() {
+  local run_id="$1"
+  shift
+
+  local cmd_file="$RUN_ROOT/commands/${run_id}.sh"
+  local log_file="$RUN_ROOT/logs/${run_id}.log"
+
+  {
+    echo "#!/usr/bin/env bash"
+    echo "set -euo pipefail"
+    echo "cd '$REPO'"
+    printf 'export ANTHROPIC_BASE_URL=%q\n' "$ANTHROPIC_BASE_URL"
+    printf 'export ANTHROPIC_AUTH_TOKEN=%q\n' "$ANTHROPIC_AUTH_TOKEN"
+    printf 'export ANTHROPIC_MODEL=%q\n' "$ANTHROPIC_MODEL"
+    printf 'export ANTHROPIC_SMALL_FAST_MODEL=%q\n' "$ANTHROPIC_SMALL_FAST_MODEL"
+    printf 'export DISABLE_NON_ESSENTIAL_MODEL_CALLS=%q\n' "$DISABLE_NON_ESSENTIAL_MODEL_CALLS"
+    printf 'export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=%q\n' "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"
+    printf '%q ' "$@"
+    printf ' >%q 2>&1 < /dev/null\n' "$log_file"
+  } > "$cmd_file"
+  chmod +x "$cmd_file"
+
+  if [[ "$tmux_started" -eq 0 ]]; then
+    tmux new-session -d -s "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+    tmux_started=1
+  else
+    tmux new-window -t "$SESSION" -n "$run_id" "bash '$cmd_file'; code=\$?; echo EXIT:\$code; sleep 3600"
+  fi
+  echo "$run_id"
+}
+
+OPENAI_COMMON=(
+  "$PYTHON" -u scripts/train.py
+  --config configs/spreadsheetbench/default.yaml
+  --cfg-options
+  model.teacher_backend=openai_chat
+  model.student_backend=openai_chat
+  model.teacher=gpt-5.5
+  model.student=gpt-5.5
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.student_azure_openai_endpoint="${STUDENT_AZURE_OPENAI_ENDPOINT:-$TEACHER_AZURE_OPENAI_ENDPOINT}"
+  model.student_azure_openai_api_version="${STUDENT_AZURE_OPENAI_API_VERSION:-$TEACHER_AZURE_OPENAI_API_VERSION}"
+  model.student_azure_openai_auth_mode="${STUDENT_AZURE_OPENAI_AUTH_MODE:-$TEACHER_AZURE_OPENAI_AUTH_MODE}"
+  model.student_azure_openai_managed_identity_client_id="${STUDENT_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID:-$TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}"
+  model.student_azure_openai_ad_scope="${STUDENT_AZURE_OPENAI_AD_SCOPE:-$TEACHER_AZURE_OPENAI_AD_SCOPE}"
+  model.reasoning_effort=medium
+  train.num_epochs=4
+  train.train_size=0
+  train.batch_size=40
+  train.accumulation=1
+  train.seed=42
+  gradient.minibatch_size=8
+  gradient.merge_batch_size=8
+  gradient.analyst_workers=16
+  gradient.use_deep_reflect=false
+  optimizer.learning_rate=4
+  optimizer.min_learning_rate=2
+  optimizer.lr_scheduler=cosine
+  optimizer.use_slow_update=true
+  optimizer.slow_update_samples=20
+  optimizer.use_meta_reflect=false
+  evaluation.use_gate=true
+  evaluation.eval_test=true
+  env.split_mode=split_dir
+  env.workers=2
+  env.exec_timeout=1020
+  env.split_dir=data/ablation_splits/spreadsheetbench/2-1-7_seed42
+  env.data_root=data/spreadsheetbench_verified_400
+  env.mode=multi
+)
+
+HARNESS_RUN="HARNESS-ClaudeInit-Spreadsheet-lr4-multi-full"
+launch_run "$HARNESS_RUN" \
+  "$PYTHON" -u scripts/train.py \
+  --config configs/spreadsheetbench/default.yaml \
+  --cfg-options \
+  model.teacher_backend=openai_chat \
+  model.teacher=gpt-5.5 \
+  model.teacher_azure_openai_endpoint="${TEACHER_AZURE_OPENAI_ENDPOINT}" \
+  model.teacher_azure_openai_api_version="${TEACHER_AZURE_OPENAI_API_VERSION}" \
+  model.teacher_azure_openai_auth_mode="${TEACHER_AZURE_OPENAI_AUTH_MODE}" \
+  model.teacher_azure_openai_managed_identity_client_id="${TEACHER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID}" \
+  model.teacher_azure_openai_ad_scope="${TEACHER_AZURE_OPENAI_AD_SCOPE}" \
+  model.reasoning_effort=medium \
+  train.num_epochs=4 \
+  train.train_size=0 \
+  train.batch_size=40 \
+  train.accumulation=1 \
+  train.seed=42 \
+  gradient.minibatch_size=8 \
+  gradient.merge_batch_size=8 \
+  gradient.analyst_workers=16 \
+  gradient.use_deep_reflect=false \
+  optimizer.lr_control_mode=fixed \
+  optimizer.skill_update_mode=patch \
+  optimizer.use_slow_update=true \
+  optimizer.slow_update_samples=20 \
+  optimizer.use_meta_skill=true \
+  optimizer.use_meta_reflect=false \
+  evaluation.use_gate=true \
+  evaluation.eval_test=true \
+  env.split_mode=split_dir \
+  env.workers=2 \
+  env.exec_timeout=1020 \
+  model.student_backend=claude_code_exec \
+  model.student=claude-sonnet-4-6 \
+  model.claude_code_exec_use_sdk=sdk \
+  model.claude_code_exec_effort=medium \
+  model.claude_code_exec_max_thinking_tokens=16384 \
+  model.codex_trace_to_teacher=false \
+  env.out_root="$RUN_ROOT/$HARNESS_RUN" \
+  env.split_dir=data/spreadsheetbench/splits \
+  env.data_root=data/spreadsheetbench/files \
+  env.mode=multi \
+  optimizer.learning_rate=4 \
+  optimizer.min_learning_rate=1 \
+  optimizer.lr_scheduler=constant
+
+for repeat in r1 r2 r3; do
+  run_id="LRCTRL-spreadsheetbench-full-rewrite-neutral3-full-${repeat}"
+  launch_run "$run_id" \
+    "${OPENAI_COMMON[@]}" \
+    optimizer.lr_control_mode=none \
+    optimizer.skill_update_mode=full_rewrite_minibatch \
+    optimizer.use_meta_skill=true \
+    env.out_root="$RUN_ROOT/$run_id"
+done
+
+for repeat in r1 r2 r3; do
+  run_id="SLOWMETA-spreadsheetbench-true-false-full-${repeat}"
+  launch_run "$run_id" \
+    "${OPENAI_COMMON[@]}" \
+    optimizer.lr_control_mode=fixed \
+    optimizer.skill_update_mode=patch \
+    optimizer.use_meta_skill=false \
+    env.out_root="$RUN_ROOT/$run_id"
+done
+
+echo "RUN_ROOT=$RUN_ROOT"
+echo "SESSION=$SESSION"

scripts/monitor_harness_claude18.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="${1:?usage: monitor_harness_claude18.sh RUN_ROOT}"
+
+while true; do
+  ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  echo "===== $ts CLAUDE18 ====="
+  uptime | sed 's/^/uptime=/'
+  active="$(pgrep -af "scripts/train.py.*${ROOT}" | grep -v pgrep | wc -l || true)"
+  claude_child="$(pgrep -af 'claude.*--output-format stream-json' | grep -v pgrep | wc -l || true)"
+  echo "active_train_total=$active"
+  echo "active_codex_train=0"
+  echo "active_claude_train=$active"
+  echo "claude_child=$claude_child"
+
+  for d in "$ROOT"/HARNESS-Claude-*; do
+    [[ -d "$d" ]] || continue
+    rid="$(basename "$d")"
+    read -r base best < <(python3 - "$d" <<'PY'
+import json, sys
+from pathlib import Path
+d = Path(sys.argv[1])
+s = d / "summary.json"
+if not s.exists():
+    print("pending pending")
+    raise SystemExit
+try:
+    obj = json.loads(s.read_text())
+except Exception:
+    print("pending pending")
+    raise SystemExit
+base = obj.get("baseline_test_hard", obj.get("base_test", "pending"))
+best = obj.get("test_hard", obj.get("best_test", "pending"))
+def fmt(x):
+    if isinstance(x, (int, float)):
+        return f"{x:.4f}"
+    return str(x)
+print(fmt(base), fmt(best))
+PY
+)
+    scan_files="$(mktemp)"
+    find "$d" \
+      \( -path '*/codex_exec' -o -path '*/codex_multi' \) -prune -o \
+      -maxdepth 6 -type f \
+      \( -name 'claude_trace_summary.txt' -o -name 'codex_trace_summary.txt' -o -name '*.log' -o -name 'summary.json' \) \
+      -print > "$scan_files" 2>/dev/null || true
+    auth="$({ xargs -r rg -l 'Not logged in|authentication_failed' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    e429="$({ xargs -r rg -l 'Too Many Requests|RateLimitError|Error code: 429|api_error_status.: 429|rate_limit|too_many_requests' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    e401="$({ xargs -r rg -l '401 Unauthorized|Error code: 401|HTTP 401|AuthenticationTypeDisabled|PermissionDeniedError' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    timeout="$({ xargs -r rg -l 'TimeoutError|Task timed out|timed out after|subprocess.TimeoutExpired|timeout_exceeded' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    teacher="$({ xargs -r rg -l 'APITimeoutError|APIConnectionError|AuthenticationError|Azure OpenAI Responses API is enabled only|teacher.*error' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    results="$(find "$d" -maxdepth 5 -path '*/results.jsonl' -type f -print0 2>/dev/null | xargs -0 -r wc -l | awk 'END{print $1+0}')"
+    empty="$({ xargs -r rg -l 'final response chars: 0|\"final_response\"\\s*:\\s*\"\"|\"result\"\\s*:\\s*\"\"' < "$scan_files" 2>/dev/null || true; } | wc -l | tr -d ' ')"
+    rm -f "$scan_files"
+    errors=$((auth + e429 + e401 + timeout + teacher))
+    echo "$rid Base=$base Best=$best Errors=$errors auth=$auth 429=$e429 401=$e401 timeout=$timeout teacher=$teacher Results=$results Empty=$empty"
+  done | sort
+  echo
+  sleep 60
+done

scripts/prepare_ablation_splits.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Prepare fixed data splits for ablation experiments."""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+
+DATASETS = {
+    "searchqa": {
+        "raw": PROJECT_ROOT / "data/searchqa_train_2000.json",
+        "out": PROJECT_ROOT / "data/ablation_splits/searchqa",
+        "filenames": {"train": "train.json", "val": "selection.json", "test": "test.json"},
+    },
+    "spreadsheetbench": {
+        "raw": PROJECT_ROOT / "data/spreadsheetbench_verified_400/dataset.json",
+        "out": PROJECT_ROOT / "data/ablation_splits/spreadsheetbench",
+        "filenames": {"train": "train.json", "val": "sel.json", "test": "test.json"},
+    },
+}
+
+SPLITS = ("1shot", "1:1:8", "2:1:7", "4:1:5")
+
+
+def load_items(path: Path) -> list[dict]:
+    with path.open(encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise TypeError(f"Expected JSON array in {path}, got {type(data).__name__}")
+    return data
+
+
+def split_counts(total: int, split: str) -> tuple[int, int, int]:
+    if split == "1shot":
+        if total < 3:
+            raise ValueError(f"Need at least 3 items for 1shot split, got {total}")
+        return 1, 1, total - 2
+
+    ratio = split
+    weights = [int(part) for part in ratio.split(":")]
+    if len(weights) != 3 or min(weights) <= 0:
+        raise ValueError(f"Invalid ratio: {ratio}")
+    denom = sum(weights)
+    raw = [total * weight / denom for weight in weights]
+    counts = [int(value) for value in raw]
+    remaining = total - sum(counts)
+    order = sorted(
+        range(3),
+        key=lambda idx: (raw[idx] - counts[idx], weights[idx]),
+        reverse=True,
+    )
+    for idx in order[:remaining]:
+        counts[idx] += 1
+    return counts[0], counts[1], counts[2]
+
+
+def split_tag(split: str) -> str:
+    return "1shot" if split == "1shot" else split.replace(":", "-")
+
+
+def write_json(path: Path, items: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(items, f, ensure_ascii=False, indent=2)
+
+
+def prepare_dataset(name: str, *, seed: int, force: bool) -> None:
+    spec = DATASETS[name]
+    raw_path = spec["raw"]
+    out_root = spec["out"]
+    filenames = spec["filenames"]
+
+    items = load_items(raw_path)
+    for split in SPLITS:
+        ratio_tag = split_tag(split)
+        split_dir = out_root / f"{ratio_tag}_seed{seed}"
+        manifest_path = split_dir / "split_manifest.json"
+        if manifest_path.exists() and not force:
+            print(f"skip {name} {split}: {split_dir} exists")
+            continue
+
+        shuffled = list(items)
+        random.Random(seed).shuffle(shuffled)
+        train_n, val_n, test_n = split_counts(len(shuffled), split)
+        train_items = shuffled[:train_n]
+        val_items = shuffled[train_n: train_n + val_n]
+        test_items = shuffled[train_n + val_n: train_n + val_n + test_n]
+
+        write_json(split_dir / "train" / filenames["train"], train_items)
+        write_json(split_dir / "val" / filenames["val"], val_items)
+        write_json(split_dir / "test" / filenames["test"], test_items)
+        write_json(
+            manifest_path,
+            {
+                "dataset": name,
+                "source": str(raw_path),
+                "split_mode": "precomputed_ratio",
+                "split_name": split,
+                "split_ratio": split if split != "1shot" else "1 train / 1 val / rest test",
+                "split_seed": seed,
+                "counts": {
+                    "train": len(train_items),
+                    "val": len(val_items),
+                    "test": len(test_items),
+                },
+            },
+        )
+        print(
+            f"wrote {name} {split} -> {split_dir} "
+            f"(train={len(train_items)}, val={len(val_items)}, test={len(test_items)})"
+        )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--force", action="store_true")
+    parser.add_argument("--dataset", choices=sorted(DATASETS), action="append")
+    args = parser.parse_args()
+
+    for name in args.dataset or sorted(DATASETS):
+        prepare_dataset(name, seed=args.seed, force=args.force)
+
+
+if __name__ == "__main__":
+    main()

scripts/run_ablation_matrix.py

@@ -0,0 +1,680 @@
+#!/usr/bin/env python3
+"""Launch the SearchQA / SpreadsheetBench ablation matrix.
+
+By default this script prints commands only. Pass --execute to actually start
+runs. Every run writes to a unique out_root under the run root and logs stdout
+/ stderr to logs/<run_id>.log.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+PYTHON_BIN = Path("/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python")
+
+T2_ENDPOINT = "https://t2vgoaigpt4o3.openai.azure.com/"
+SEARCHAGENT5_ENDPOINT = "https://searchagent5.cognitiveservices.azure.com/"
+
+
+@dataclass(frozen=True)
+class Experiment:
+    run_id: str
+    benchmark: str
+    config: str
+    overrides: tuple[str, ...]
+
+
+BENCH_CONFIG = {
+    "searchqa": "configs/searchqa/default.yaml",
+    "spreadsheetbench": "configs/spreadsheetbench/default.yaml",
+    "livemathematicianbench": "configs/livemathematicianbench/default.yaml",
+    "alfworld": "configs/alfworld/default.yaml",
+    "docvqa": "configs/docvqa/default.yaml",
+}
+
+DEFAULT_SPLIT = {
+    "searchqa": "data/ablation_splits/searchqa/2-1-7_seed42",
+    "spreadsheetbench": "data/ablation_splits/spreadsheetbench/2-1-7_seed42",
+    "livemathematicianbench": "data/ablation_splits/livemathematicianbench/2-1-7_seed42",
+    "alfworld": "data/ablation_splits/alfworld/2-1-7_seed42",
+    "docvqa": "/home/azureuser/zisu/SkillReflection/data/docvqa/splits",
+}
+
+DEFAULT_TRAIN_SIZE = {
+    "searchqa": 400,
+    "spreadsheetbench": 80,
+    "livemathematicianbench": 35,
+    "alfworld": 39,
+    "docvqa": 1070,
+}
+
+BATCH_SIZE_VALUES: tuple[int | str, ...] = (8, 24, 40, 56, "full")
+
+SPLITS = {
+    "searchqa": {
+        "1shot": ("data/ablation_splits/searchqa/1shot_seed42", ("optimizer.slow_update_samples=1",)),
+        "1-1-8": ("data/ablation_splits/searchqa/1-1-8_seed42", ()),
+        "2-1-7": ("data/ablation_splits/searchqa/2-1-7_seed42", ()),
+        "4-1-5": ("data/ablation_splits/searchqa/4-1-5_seed42", ()),
+    },
+    "spreadsheetbench": {
+        "1shot": ("data/ablation_splits/spreadsheetbench/1shot_seed42", ("optimizer.slow_update_samples=1",)),
+        "1-1-8": ("data/ablation_splits/spreadsheetbench/1-1-8_seed42", ()),
+        "2-1-7": ("data/ablation_splits/spreadsheetbench/2-1-7_seed42", ()),
+        "4-1-5": ("data/ablation_splits/spreadsheetbench/4-1-5_seed42", ()),
+    },
+    "livemathematicianbench": {
+        "1shot": ("data/ablation_splits/livemathematicianbench/1shot_seed42", ("optimizer.slow_update_samples=1",)),
+        "1-1-8": ("data/ablation_splits/livemathematicianbench/1-1-8_seed42", ()),
+        "2-1-7": ("data/ablation_splits/livemathematicianbench/2-1-7_seed42", ()),
+        "4-1-5": ("data/ablation_splits/livemathematicianbench/4-1-5_seed42", ()),
+    },
+    "alfworld": {
+        "1shot": ("data/ablation_splits/alfworld/1shot_seed42", ("optimizer.slow_update_samples=1",)),
+        "1-1-8": ("data/ablation_splits/alfworld/1-1-8_seed42", ()),
+        "2-1-7": ("data/ablation_splits/alfworld/2-1-7_seed42", ()),
+        "4-1-5": ("data/ablation_splits/alfworld/4-1-5_seed42", ()),
+    },
+    "docvqa": {
+        "1shot": ("data/ablation_splits/docvqa/1shot_seed42", ("optimizer.slow_update_samples=1",)),
+        "1-1-8": ("data/ablation_splits/docvqa/1-1-8_seed42", ()),
+        "2-1-7": ("/home/azureuser/zisu/SkillReflection/data/docvqa/splits", ()),
+        "4-1-5": ("data/ablation_splits/docvqa/4-1-5_seed42", ()),
+    },
+}
+
+
+def common_overrides(benchmark: str, out_root: Path) -> list[str]:
+    return [
+        "model.teacher_backend=openai_chat",
+        "model.student_backend=openai_chat",
+        "model.teacher=gpt-5.5",
+        "model.student=gpt-5.5",
+        f"model.teacher_azure_openai_endpoint={T2_ENDPOINT}",
+        "model.teacher_azure_openai_api_version=2024-12-01-preview",
+        "model.teacher_azure_openai_auth_mode=azure_cli",
+        f"model.student_azure_openai_endpoint={T2_ENDPOINT}",
+        "model.student_azure_openai_api_version=2024-12-01-preview",
+        "model.student_azure_openai_auth_mode=azure_cli",
+        "model.reasoning_effort=medium",
+        "train.num_epochs=4",
+        "train.train_size=0",
+        "train.batch_size=40",
+        "train.accumulation=1",
+        "train.seed=42",
+        "gradient.minibatch_size=8",
+        "gradient.merge_batch_size=8",
+        "gradient.analyst_workers=16",
+        "gradient.use_deep_reflect=false",
+        "optimizer.learning_rate=4",
+        "optimizer.min_learning_rate=2",
+        "optimizer.lr_scheduler=cosine",
+        "optimizer.skill_update_mode=patch",
+        "optimizer.use_slow_update=true",
+        "optimizer.slow_update_samples=20",
+        "optimizer.use_meta_skill=true",
+        "optimizer.use_meta_reflect=false",
+        "evaluation.use_gate=true",
+        "evaluation.eval_test=true",
+        "env.split_mode=split_dir",
+        f"env.split_dir={DEFAULT_SPLIT[benchmark]}",
+        f"env.out_root={out_root}",
+    ]
+
+
+def make_experiment(
+    group: str,
+    benchmark: str,
+    suffix: str,
+    run_root: Path,
+    overrides: list[str],
+) -> Experiment:
+    run_id = f"{group}-{benchmark}-{suffix}"
+    out_root = run_root / run_id
+    all_overrides = common_overrides(benchmark, out_root)
+    all_overrides.extend(overrides)
+    return Experiment(
+        run_id=run_id,
+        benchmark=benchmark,
+        config=BENCH_CONFIG[benchmark],
+        overrides=tuple(all_overrides),
+    )
+
+
+def build_matrix(
+    groups: set[str],
+    benchmarks: list[str],
+    run_root: Path,
+    *,
+    include_duplicate_defaults: bool = False,
+) -> list[Experiment]:
+    exps: list[Experiment] = []
+    group_order = [
+        "default",
+        "split",
+        "batch",
+        "mbs",
+        "lr",
+        "sched",
+        "slown",
+        "mod",
+        "smodel",
+        "longpair",
+        "lrctrl",
+    ]
+
+    for group in group_order:
+        if group not in groups:
+            continue
+        for benchmark in benchmarks:
+            if group == "default":
+                exps.append(make_experiment("DEFAULT", benchmark, "5.5", run_root, []))
+                continue
+
+            if group == "split":
+                for tag, (split_dir, extra) in SPLITS[benchmark].items():
+                    if not include_duplicate_defaults and tag == "2-1-7":
+                        continue
+                    exps.append(make_experiment(
+                        "SPLIT",
+                        benchmark,
+                        tag,
+                        run_root,
+                        [f"env.split_dir={split_dir}", *extra],
+                    ))
+                continue
+
+            if group == "mbs":
+                for value in (1, 2, 4, 8, 16, 32):
+                    if not include_duplicate_defaults and value == 8:
+                        continue
+                    exps.append(make_experiment(
+                        "MBS",
+                        benchmark,
+                        str(value),
+                        run_root,
+                        [f"gradient.minibatch_size={value}"],
+                    ))
+                continue
+
+            if group == "batch":
+                for value in BATCH_SIZE_VALUES:
+                    if not include_duplicate_defaults and value == 40:
+                        continue
+                    batch_size = DEFAULT_TRAIN_SIZE[benchmark] if value == "full" else int(value)
+                    exps.append(make_experiment(
+                        "BATCH",
+                        benchmark,
+                        str(value),
+                        run_root,
+                        [
+                            f"train.batch_size={batch_size}",
+                            "gradient.minibatch_size=8",
+                        ],
+                    ))
+                continue
+
+            if group == "lr":
+                for value in (1, 2, 4, 8, 16):
+                    exps.append(make_experiment(
+                        "LR",
+                        benchmark,
+                        str(value),
+                        run_root,
+                        [
+                            "optimizer.lr_scheduler=constant",
+                            "optimizer.min_learning_rate=1",
+                            f"optimizer.learning_rate={value}",
+                        ],
+                    ))
+                continue
+
+            if group == "sched":
+                for value in ("constant", "cosine", "linear"):
+                    if not include_duplicate_defaults and value == "cosine":
+                        continue
+                    exps.append(make_experiment(
+                        "SCHED",
+                        benchmark,
+                        value,
+                        run_root,
+                        [f"optimizer.lr_scheduler={value}"],
+                    ))
+                continue
+
+            if group == "slown":
+                for value in (5, 10, 20, 40):
+                    if not include_duplicate_defaults and value == 20:
+                        continue
+                    exps.append(make_experiment(
+                        "SLOWN",
+                        benchmark,
+                        str(value),
+                        run_root,
+                        [f"optimizer.slow_update_samples={value}"],
+                    ))
+                continue
+
+            if group == "mod":
+                settings = {
+                    "slow-meta": ("true", "true"),
+                    "slow-only": ("true", "false"),
+                    "meta-only": ("false", "true"),
+                    "none": ("false", "false"),
+                }
+                for tag, (slow, meta) in settings.items():
+                    if not include_duplicate_defaults and tag == "slow-meta":
+                        continue
+                    exps.append(make_experiment(
+                        "MOD",
+                        benchmark,
+                        tag,
+                        run_root,
+                        [
+                            f"optimizer.use_slow_update={slow}",
+                            f"optimizer.use_meta_skill={meta}",
+                        ],
+                    ))
+                continue
+
+            if group == "smodel":
+                student_settings = {
+                    "5.4": [
+                        "model.student=gpt-5.4-pro",
+                        f"model.student_azure_openai_endpoint={T2_ENDPOINT}",
+                        "model.student_azure_openai_api_version=2025-03-01-preview",
+                        "model.student_azure_openai_auth_mode=azure_cli",
+                    ],
+                    "5.4-mini": [
+                        "model.student=gpt-5.4-mini",
+                        f"model.student_azure_openai_endpoint={SEARCHAGENT5_ENDPOINT}",
+                        "model.student_azure_openai_api_version=2024-12-01-preview",
+                        "model.student_azure_openai_auth_mode=azure_cli",
+                    ],
+                    "5.5": [],
+                }
+                for tag, overrides in student_settings.items():
+                    if not include_duplicate_defaults and tag == "5.5":
+                        continue
+                    exps.append(make_experiment("SMODEL", benchmark, tag, run_root, overrides))
+                continue
+
+            if group == "longpair":
+                for value in ("changed", "unchanged"):
+                    exps.append(make_experiment(
+                        "LONGPAIR",
+                        benchmark,
+                        value,
+                        run_root,
+                        [f"optimizer.longitudinal_pair_policy={value}"],
+                    ))
+                continue
+
+            if group == "lrctrl":
+                settings = {
+                    "autonomous": ["optimizer.lr_control_mode=autonomous"],
+                    "full-rewrite": [
+                        "optimizer.lr_control_mode=none",
+                        "optimizer.skill_update_mode=full_rewrite_minibatch",
+                    ],
+                }
+                for tag, overrides in settings.items():
+                    exps.append(make_experiment("LRCTRL", benchmark, tag, run_root, overrides))
+                continue
+
+    return exps
+
+
+def _build_matrix_legacy(
+    groups: set[str],
+    benchmarks: list[str],
+    run_root: Path,
+    *,
+    include_duplicate_defaults: bool = False,
+) -> list[Experiment]:
+    exps: list[Experiment] = []
+    for benchmark in benchmarks:
+        if "default" in groups:
+            exps.append(make_experiment("DEFAULT", benchmark, "5.5", run_root, []))
+
+        if "split" in groups:
+            for tag, (split_dir, extra) in SPLITS[benchmark].items():
+                if not include_duplicate_defaults and tag == "2-1-7":
+                    continue
+                exps.append(make_experiment(
+                    "SPLIT",
+                    benchmark,
+                    tag,
+                    run_root,
+                    [f"env.split_dir={split_dir}", *extra],
+                ))
+
+        if "mbs" in groups:
+            for value in (1, 2, 4, 8, 16, 32):
+                if not include_duplicate_defaults and value == 8:
+                    continue
+                exps.append(make_experiment(
+                    "MBS",
+                    benchmark,
+                    str(value),
+                    run_root,
+                    [f"gradient.minibatch_size={value}"],
+                ))
+
+        if "batch" in groups:
+            for value in BATCH_SIZE_VALUES:
+                if not include_duplicate_defaults and value == 40:
+                    continue
+                batch_size = DEFAULT_TRAIN_SIZE[benchmark] if value == "full" else int(value)
+                exps.append(make_experiment(
+                    "BATCH",
+                    benchmark,
+                    str(value),
+                    run_root,
+                    [
+                        f"train.batch_size={batch_size}",
+                        "gradient.minibatch_size=8",
+                    ],
+                ))
+
+        if "lr" in groups:
+            for value in (1, 2, 4, 8, 16):
+                exps.append(make_experiment(
+                    "LR",
+                    benchmark,
+                    str(value),
+                    run_root,
+                    [
+                        "optimizer.lr_scheduler=constant",
+                        "optimizer.min_learning_rate=1",
+                        f"optimizer.learning_rate={value}",
+                    ],
+                ))
+
+        if "sched" in groups:
+            for value in ("constant", "cosine", "linear"):
+                if not include_duplicate_defaults and value == "cosine":
+                    continue
+                exps.append(make_experiment(
+                    "SCHED",
+                    benchmark,
+                    value,
+                    run_root,
+                    [f"optimizer.lr_scheduler={value}"],
+                ))
+
+        if "slown" in groups:
+            for value in (5, 10, 20, 40):
+                if not include_duplicate_defaults and value == 20:
+                    continue
+                exps.append(make_experiment(
+                    "SLOWN",
+                    benchmark,
+                    str(value),
+                    run_root,
+                    [f"optimizer.slow_update_samples={value}"],
+                ))
+
+        if "mod" in groups:
+            settings = {
+                "slow-meta": ("true", "true"),
+                "slow-only": ("true", "false"),
+                "meta-only": ("false", "true"),
+                "none": ("false", "false"),
+            }
+            for tag, (slow, meta) in settings.items():
+                if not include_duplicate_defaults and tag == "slow-meta":
+                    continue
+                exps.append(make_experiment(
+                    "MOD",
+                    benchmark,
+                    tag,
+                    run_root,
+                    [
+                        f"optimizer.use_slow_update={slow}",
+                        f"optimizer.use_meta_skill={meta}",
+                    ],
+                ))
+
+        if "smodel" in groups:
+            student_settings = {
+                "5.4": [
+                    "model.student=gpt-5.4-pro",
+                    f"model.student_azure_openai_endpoint={T2_ENDPOINT}",
+                    "model.student_azure_openai_api_version=2025-03-01-preview",
+                    "model.student_azure_openai_auth_mode=azure_cli",
+                ],
+                "5.4-mini": [
+                    "model.student=gpt-5.4-mini",
+                    f"model.student_azure_openai_endpoint={SEARCHAGENT5_ENDPOINT}",
+                    "model.student_azure_openai_api_version=2024-12-01-preview",
+                    "model.student_azure_openai_auth_mode=azure_cli",
+                ],
+                "5.5": [],
+            }
+            for tag, overrides in student_settings.items():
+                if not include_duplicate_defaults and tag == "5.5":
+                    continue
+                exps.append(make_experiment("SMODEL", benchmark, tag, run_root, overrides))
+
+        if "longpair" in groups:
+            for value in ("changed", "unchanged"):
+                exps.append(make_experiment(
+                    "LONGPAIR",
+                    benchmark,
+                    value,
+                    run_root,
+                    [f"optimizer.longitudinal_pair_policy={value}"],
+                ))
+
+        if "lrctrl" in groups:
+            settings = {
+                "autonomous": ["optimizer.lr_control_mode=autonomous"],
+                "full-rewrite": [
+                    "optimizer.lr_control_mode=none",
+                    "optimizer.skill_update_mode=full_rewrite_minibatch",
+                ],
+            }
+            for tag, overrides in settings.items():
+                exps.append(make_experiment("LRCTRL", benchmark, tag, run_root, overrides))
+
+    return exps
+
+
+def command_for(exp: Experiment) -> list[str]:
+    return [
+        str(PYTHON_BIN),
+        "scripts/train.py",
+        "--config",
+        exp.config,
+        "--cfg-options",
+        *exp.overrides,
+    ]
+
+
+def active_run_ids(run_root: Path, valid_run_ids: set[str] | None = None) -> set[str]:
+    try:
+        raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
+    except subprocess.CalledProcessError:
+        return set()
+    pattern = re.compile(re.escape(str(run_root)) + r"/([^\s]+)")
+    active: set[str] = set()
+    for line in raw.splitlines():
+        for match in pattern.finditer(line):
+            run_id = match.group(1).strip("'\"")
+            if run_id.endswith(".log") or "/" in run_id:
+                continue
+            if valid_run_ids is not None and run_id not in valid_run_ids:
+                continue
+            active.add(run_id)
+    return active
+
+
+def completed_run_ids(run_root: Path) -> set[str]:
+    return {
+        path.parent.name
+        for path in run_root.glob("*/summary.json")
+        if path.is_file()
+    }
+
+
+def print_commands(exps: list[Experiment]) -> None:
+    for exp in exps:
+        cmd = command_for(exp)
+        print(f"\n# {exp.run_id}")
+        print(" ".join(subprocess.list2cmdline([part]) for part in cmd))
+
+
+def run_commands(
+    exps: list[Experiment],
+    run_root: Path,
+    max_parallel: int,
+    run_retries: int,
+) -> int:
+    logs_dir = run_root / "logs"
+    logs_dir.mkdir(parents=True, exist_ok=True)
+    active: list[tuple[Experiment, subprocess.Popen, object]] = []
+    valid_run_ids = {exp.run_id for exp in exps}
+    skipped_completed = completed_run_ids(run_root)
+    skipped_active = active_run_ids(run_root, valid_run_ids)
+    pending: list[tuple[Experiment, int]] = [
+        (exp, 0)
+        for exp in exps
+        if exp.run_id not in skipped_completed and exp.run_id not in skipped_active
+    ]
+    for run_id in sorted(skipped_completed):
+        print(f"[SKIP_COMPLETED] {run_id}", flush=True)
+    for run_id in sorted(skipped_active):
+        print(f"[SKIP_ACTIVE] {run_id}", flush=True)
+    failures = 0
+
+    while pending or active:
+        external_active = active_run_ids(run_root, valid_run_ids) - {exp.run_id for exp, _, _ in active}
+        while pending and len(active) + len(external_active) < max_parallel:
+            exp, attempt = pending.pop(0)
+            log_path = logs_dir / f"{exp.run_id}.log"
+            if attempt:
+                log_path = logs_dir / f"{exp.run_id}.retry{attempt}.log"
+            log_f = open(log_path, "w", encoding="utf-8")
+            print(f"[START] {exp.run_id} attempt={attempt + 1} log={log_path}", flush=True)
+            proc = subprocess.Popen(
+                command_for(exp),
+                cwd=PROJECT_ROOT,
+                stdout=log_f,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+            setattr(proc, "_attempt", attempt)
+            active.append((exp, proc, log_f))
+
+        time.sleep(5)
+        still_active: list[tuple[Experiment, subprocess.Popen, object]] = []
+        for exp, proc, log_f in active:
+            rc = proc.poll()
+            if rc is None:
+                still_active.append((exp, proc, log_f))
+                continue
+            log_f.close()
+            if rc == 0:
+                print(f"[DONE]  {exp.run_id}", flush=True)
+            else:
+                if getattr(proc, "_attempt", 0) < run_retries:
+                    next_attempt = getattr(proc, "_attempt", 0) + 1
+                    pending.append((exp, next_attempt))
+                    print(f"[RETRY] {exp.run_id} rc={rc} next_attempt={next_attempt + 1}", flush=True)
+                else:
+                    failures += 1
+                    print(f"[FAIL]  {exp.run_id} rc={rc}", flush=True)
+        active = still_active
+
+    return failures
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--groups",
+        nargs="+",
+        default=["default"],
+        choices=[
+            "default",
+            "split",
+            "batch",
+            "mbs",
+            "lr",
+            "sched",
+            "slown",
+            "mod",
+            "smodel",
+            "longpair",
+            "lrctrl",
+            "all",
+        ],
+        help="Experiment groups to include. Default: default.",
+    )
+    parser.add_argument(
+        "--bench",
+        nargs="+",
+        default=["searchqa", "spreadsheetbench"],
+        choices=["searchqa", "spreadsheetbench", "livemathematicianbench", "alfworld", "docvqa"],
+    )
+    parser.add_argument("--run-root", default="", help="Output root. Default: outputs/ablation_<UTC timestamp>.")
+    parser.add_argument("--max-parallel", type=int, default=1)
+    parser.add_argument("--run-retries", type=int, default=1, help="Retry failed runs this many times. Default: 1.")
+    parser.add_argument(
+        "--include-duplicate-defaults",
+        action="store_true",
+        help="Also run ablation points that are exactly the default setting.",
+    )
+    parser.add_argument("--execute", action="store_true", help="Actually start runs. Without this, print commands only.")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    groups = set(args.groups)
+    if "all" in groups:
+        groups = {"default", "split", "batch", "mbs", "lr", "sched", "slown", "mod", "smodel"}
+
+    ts = time.strftime("%Y%m%d_%H%M%S", time.gmtime())
+    run_root = Path(args.run_root) if args.run_root else PROJECT_ROOT / "outputs" / f"ablation_{ts}"
+    if not run_root.is_absolute():
+        run_root = PROJECT_ROOT / run_root
+    run_root.mkdir(parents=True, exist_ok=True)
+
+    exps = build_matrix(
+        groups,
+        args.bench,
+        run_root,
+        include_duplicate_defaults=args.include_duplicate_defaults,
+    )
+    print(f"run_root={run_root}")
+    print(f"num_experiments={len(exps)}")
+    print(f"groups={','.join(sorted(groups))}")
+    print(f"bench={','.join(args.bench)}")
+
+    if not args.execute:
+        print_commands(exps)
+        return
+
+    max_parallel = max(1, int(args.max_parallel))
+    failures = run_commands(
+        exps,
+        run_root,
+        max_parallel=max_parallel,
+        run_retries=max(0, int(args.run_retries)),
+    )
+    if failures:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/run_alfworld.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT — ALFWorld training launch script
+#
+# Usage:
+#   bash scripts/run_alfworld.sh
+#   bash scripts/run_alfworld.sh --num_epochs 2 --edit_budget 6
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+WORKSPACE="/home/azureuser/workspace-gzy"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+# Activate conda environment
+export PATH="${WORKSPACE}/miniconda3/envs/reflact/bin:${WORKSPACE}/miniconda3/bin:${PATH}"
+
+# ALFWorld data — uses ~/.cache/alfworld by default (standard alfworld location)
+export ALFWORLD_DATA="${ALFWORLD_DATA:-${HOME}/.cache/alfworld}"
+
+# Ensure ReflACT is importable
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+# ── Verify ALFWorld data exists ──────────────────────────────────────────────
+if [ ! -d "${ALFWORLD_DATA}/json_2.1.1" ]; then
+    echo "ERROR: ALFWorld data not found at ${ALFWORLD_DATA}/json_2.1.1"
+    echo ""
+    echo "To download ALFWorld data, run:"
+    echo "  pip install alfworld[full]"
+    echo "  alfworld-download"
+    echo ""
+    echo "Or set ALFWORLD_DATA to the directory containing json_2.1.1/"
+    exit 1
+fi
+
+# ── Azure OpenAI credentials ────────────────────────────────────────────────
+export AZURE_OPENAI_ENDPOINT="${AZURE_OPENAI_ENDPOINT:-https://agl-dev.cognitiveservices.azure.com/}"
+export AZURE_OPENAI_API_KEY="${AZURE_OPENAI_API_KEY:-<your-azure-openai-api-key>}"
+export AZURE_OPENAI_API_VERSION="${AZURE_OPENAI_API_VERSION:-2025-04-01-preview}"
+
+# ── Model configuration ─────────────────────────────────────────────────────
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+# ── Output directory ─────────────────────────────────────────────────────────
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/reflact_alfworld_${STUDENT_DEPLOYMENT}_${TIMESTAMP}"
+
+# ── Run ──────────────────────────────────────────────────────────────────────
+echo "============================================================"
+echo "  ReflACT — Reflective Agent Tuning (ALFWorld)"
+echo "============================================================"
+echo "  Teacher:       ${TEACHER_DEPLOYMENT}"
+echo "  Student:       ${STUDENT_DEPLOYMENT}"
+echo "  ALFWORLD_DATA: ${ALFWORLD_DATA}"
+echo "  Output:        ${DEFAULT_OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/alfworld_default.yaml \
+    --out_root "${DEFAULT_OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"

scripts/run_meta_skill_ablation.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PROJECT_ROOT="/home/azureuser/workspace-gzy/SkillReflection_dev"
+PYTHON_BIN="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+TS="$(date -u +%Y%m%d_%H%M%S)"
+RUN_ROOT="${PROJECT_ROOT}/outputs/meta_skill_ablation_${TS}"
+
+mkdir -p "${RUN_ROOT}"
+
+run_train() {
+  local benchmark="$1"
+  local reasoning="$2"
+  local condition="$3"
+  local config_path=""
+  local reasoning_override=""
+  local meta_skill_flag=""
+
+  case "${benchmark}" in
+    searchqa)
+      config_path="configs/searchqa/default.yaml"
+      ;;
+    spreadsheetbench)
+      config_path="configs/spreadsheetbench/default.yaml"
+      ;;
+    *)
+      echo "Unknown benchmark: ${benchmark}" >&2
+      exit 1
+      ;;
+  esac
+
+  case "${reasoning}" in
+    medium)
+      reasoning_override="model.reasoning_effort=medium"
+      ;;
+    none)
+      reasoning_override="model.reasoning_effort="
+      ;;
+    *)
+      echo "Unknown reasoning setting: ${reasoning}" >&2
+      exit 1
+      ;;
+  esac
+
+  case "${condition}" in
+    slow)
+      meta_skill_flag="optimizer.use_meta_skill=false"
+      ;;
+    slow_meta)
+      meta_skill_flag="optimizer.use_meta_skill=true"
+      ;;
+    *)
+      echo "Unknown condition: ${condition}" >&2
+      exit 1
+      ;;
+  esac
+
+  local out_root="${RUN_ROOT}/${benchmark}_${reasoning}_${condition}"
+
+  echo
+  echo "============================================================"
+  echo "START ${benchmark} ${reasoning} ${condition}"
+  echo "out_root=${out_root}"
+  echo "============================================================"
+
+  (
+    cd "${PROJECT_ROOT}"
+    "${PYTHON_BIN}" scripts/train.py \
+      --config "${config_path}" \
+      --cfg-options \
+        "${reasoning_override}" \
+        "optimizer.use_slow_update=true" \
+        "${meta_skill_flag}" \
+        "optimizer.use_meta_reflect=false" \
+        "gradient.use_deep_reflect=false" \
+        "env.out_root=${out_root}"
+  )
+
+  echo
+  echo "============================================================"
+  echo "DONE ${benchmark} ${reasoning} ${condition}"
+  echo "============================================================"
+}
+
+for benchmark in searchqa spreadsheetbench; do
+  for reasoning in medium none; do
+    run_train "${benchmark}" "${reasoning}" "slow"
+    run_train "${benchmark}" "${reasoning}" "slow_meta"
+  done
+done
+
+echo
+echo "All runs completed."
+echo "Run root: ${RUN_ROOT}"

scripts/run_missing_meta_parallel.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PROJECT_ROOT="/home/azureuser/workspace-gzy/SkillReflection_dev"
+PYTHON_BIN="/home/azureuser/workspace-gzy/miniconda3/envs/reflact/bin/python"
+RUN_ROOT="${PROJECT_ROOT}/outputs/meta_skill_parallel_20260430_072356"
+LOG_DIR="${PROJECT_ROOT}/logs/meta_skill_parallel_20260430_072356"
+
+mkdir -p "${RUN_ROOT}" "${LOG_DIR}"
+
+start_run() {
+  local name="$1"
+  local config_path="$2"
+  local meta_skill="$3"
+  local out_root="${RUN_ROOT}/${name}"
+  local log_path="${LOG_DIR}/${name}.log"
+
+  echo "[START] ${name}"
+  echo "        out_root=${out_root}"
+  echo "        log=${log_path}"
+
+  (
+    cd "${PROJECT_ROOT}"
+    PYTHONUNBUFFERED=1 "${PYTHON_BIN}" scripts/train.py \
+      --config "${config_path}" \
+      --cfg-options \
+        "model.reasoning_effort=medium" \
+        "optimizer.use_slow_update=true" \
+        "optimizer.use_meta_skill=${meta_skill}" \
+        "optimizer.use_meta_reflect=false" \
+        "gradient.use_deep_reflect=false" \
+        "env.out_root=${out_root}"
+  ) > "${log_path}" 2>&1 &
+
+  echo "$!" > "${LOG_DIR}/${name}.pid"
+}
+
+start_run "searchqa_medium_slow_meta" "configs/searchqa/default.yaml" "true"
+start_run "spreadsheetbench_medium_slow" "configs/spreadsheetbench/default.yaml" "false"
+start_run "spreadsheetbench_medium_slow_meta" "configs/spreadsheetbench/default.yaml" "true"
+
+echo "[WAIT] missing comparison runs are active"
+wait

scripts/run_searchqa.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT — SearchQA training launch script
+#
+# Usage:
+#   bash scripts/run_searchqa.sh
+#   bash scripts/run_searchqa.sh --data_path data/searchqa_train_2000.json
+#   bash scripts/run_searchqa.sh --num_epochs 2 --edit_budget 6
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+# Ensure ReflACT is importable
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+# ── Model configuration ─────────────────────────────────────────────────────
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+# ── Output directory ─────────────────────────────────────────────────────────
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/reflact_searchqa_${STUDENT_DEPLOYMENT}_${TIMESTAMP}"
+
+# ── Run ──────────────────────────────────────────────────────────────────────
+echo "============================================================"
+echo "  ReflACT — Reflective Agent Tuning (SearchQA)"
+echo "============================================================"
+echo "  Teacher:  ${TEACHER_DEPLOYMENT}"
+echo "  Student:  ${STUDENT_DEPLOYMENT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/searchqa_default.yaml \
+    --out_root "${DEFAULT_OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"

scripts/run_spreadsheetbench.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT — SpreadsheetBench training launch script
+#
+# Usage:
+#   bash scripts/run_spreadsheetbench.sh \
+#       --data_root /path/to/data \
+#       --jsonl_path /path/to/benchmark.jsonl
+#
+#   bash scripts/run_spreadsheetbench.sh \
+#       --data_root /path/to/data \
+#       --jsonl_path /path/to/benchmark.jsonl \
+#       --num_epochs 2 --edit_budget 6
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+# ── Paths ────────────────────────────────────────────────────────────────────
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+# Ensure ReflACT is importable
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+# ── Model configuration ─────────────────────────────────────────────────────
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+# ── Output directory ─────────────────────────────────────────────────────────
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/reflact_spreadsheetbench_${STUDENT_DEPLOYMENT}_${TIMESTAMP}"
+
+# ── Run ──────────────────────────────────────────────────────────────────────
+echo "============================================================"
+echo "  ReflACT — Reflective Agent Tuning (SpreadsheetBench)"
+echo "============================================================"
+echo "  Teacher:  ${TEACHER_DEPLOYMENT}"
+echo "  Student:  ${STUDENT_DEPLOYMENT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --out_root "${DEFAULT_OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"

scripts/train.py

@@ -0,0 +1,483 @@
+#!/usr/bin/env python3
+"""ReflACT unified training entry point.
+
+Usage
+-----
+    python scripts/train.py --config configs/alfworld/default.yaml
+
+Any YAML key can be overridden from the command line::
+
+    python scripts/train.py --config configs/alfworld/default.yaml \\
+        --batch_size 40 --num_epochs 2 --seed 123
+
+Run ``python scripts/train.py --help`` for a full list of options.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import os
+import sys
+
+# Ensure the project root is on sys.path so ``import reflact`` works
+# regardless of where the script is invoked from.
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
+if _PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, _PROJECT_ROOT)
+
+from reflact.model.common import default_model_for_backend, normalize_backend_name
+
+_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"}
+
+
+# ── Environment registry ────────────────────────────────────────────────────
+
+_ENV_REGISTRY: dict[str, type] = {}
+
+
+def _register_builtins() -> None:
+    """Lazy-import built-in adapters so we don't pull heavy deps at CLI parse time."""
+    try:
+        from reflact.envs.alfworld.adapter import ALFWorldAdapter
+        _ENV_REGISTRY["alfworld"] = ALFWorldAdapter
+    except ImportError:
+        pass  # ALFWorld deps not installed — skip
+    try:
+        from reflact.envs.searchqa.adapter import SearchQAAdapter
+        _ENV_REGISTRY["searchqa"] = SearchQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter
+        _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.babyvision.adapter import BabyVisionAdapter
+        _ENV_REGISTRY["babyvision"] = BabyVisionAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter
+        _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.mmrb.adapter import MMRBAdapter
+        _ENV_REGISTRY["mmrb"] = MMRBAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.docvqa.adapter import DocVQAAdapter
+        _ENV_REGISTRY["docvqa"] = DocVQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.mathverse.adapter import MathVerseAdapter
+        _ENV_REGISTRY["mathverse"] = MathVerseAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.officeqa.adapter import OfficeQAAdapter
+        _ENV_REGISTRY["officeqa"] = OfficeQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.sealqa.adapter import SealQAAdapter
+        _ENV_REGISTRY["sealqa"] = SealQAAdapter
+    except ImportError:
+        pass
+    try:
+        from reflact.envs.swebench.adapter import SWEBenchAdapter
+        _ENV_REGISTRY["swebench"] = SWEBenchAdapter
+    except ImportError:
+        pass
+
+
+def get_adapter(cfg: dict):
+    """Instantiate the environment adapter specified in ``cfg["env"]``."""
+    _register_builtins()
+    env_name = cfg.get("env", "alfworld")
+    if env_name not in _ENV_REGISTRY:
+        raise ValueError(
+            f"Unknown environment '{env_name}'. "
+            f"Available: {list(_ENV_REGISTRY.keys())}"
+        )
+    adapter_cls = _ENV_REGISTRY[env_name]
+
+    # Inspect adapter __init__ signature and only pass accepted kwargs
+    import inspect
+    sig = inspect.signature(adapter_cls.__init__)
+    accepted = set(sig.parameters.keys()) - {"self"}
+    adapter_kwargs: dict = {}
+    for key in accepted:
+        if key in cfg:
+            adapter_kwargs[key] = cfg[key]
+
+    return adapter_cls(**adapter_kwargs)
+
+
+# ── CLI ──────────────────────────────────────────────────────────────────────
+
+_BOOL = lambda x: x.lower() in ("true", "1", "yes")  # noqa: E731
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        description="ReflACT: Reflective Agent Tuning",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    p.add_argument("--config", type=str, required=True,
+                   help="Path to YAML config file")
+    p.add_argument("--cfg-options", nargs="+", default=[],
+                   help="Override config: section.key=value (e.g. train.batch_size=40)")
+
+    # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage)
+    p.add_argument("--env", type=str)
+    p.add_argument("--backend", type=str,
+                   choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec"])
+    p.add_argument("--teacher_model", type=str)
+    p.add_argument("--student_model", type=str)
+    p.add_argument("--teacher_backend", type=str)
+    p.add_argument("--student_backend", type=str)
+    p.add_argument("--reasoning_effort", type=str,
+                   choices=["", "low", "medium", "high", "xhigh", "max"])
+    p.add_argument("--rewrite_reasoning_effort", type=str)
+    p.add_argument("--rewrite_max_completion_tokens", type=int)
+    p.add_argument("--azure_endpoint", type=str)
+    p.add_argument("--azure_api_version", type=str)
+    p.add_argument("--azure_api_key", type=str)
+    p.add_argument("--azure_openai_endpoint", type=str)
+    p.add_argument("--azure_openai_api_version", type=str)
+    p.add_argument("--azure_openai_api_key", type=str)
+    p.add_argument("--azure_openai_auth_mode", type=str)
+    p.add_argument("--azure_openai_ad_scope", type=str)
+    p.add_argument("--azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--teacher_azure_openai_endpoint", type=str)
+    p.add_argument("--teacher_azure_openai_api_version", type=str)
+    p.add_argument("--teacher_azure_openai_api_key", type=str)
+    p.add_argument("--teacher_azure_openai_auth_mode", type=str)
+    p.add_argument("--teacher_azure_openai_ad_scope", type=str)
+    p.add_argument("--teacher_azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--student_azure_openai_endpoint", type=str)
+    p.add_argument("--student_azure_openai_api_version", type=str)
+    p.add_argument("--student_azure_openai_api_key", type=str)
+    p.add_argument("--student_azure_openai_auth_mode", type=str)
+    p.add_argument("--student_azure_openai_ad_scope", type=str)
+    p.add_argument("--student_azure_openai_managed_identity_client_id", type=str)
+    p.add_argument("--codex_exec_path", type=str)
+    p.add_argument("--codex_exec_sandbox", type=str)
+    p.add_argument("--codex_exec_profile", type=str)
+    p.add_argument("--codex_exec_full_auto", type=_BOOL)
+    p.add_argument("--codex_exec_reasoning_effort", type=str)
+    p.add_argument("--codex_exec_use_sdk", type=str)
+    p.add_argument("--codex_exec_network_access", type=_BOOL)
+    p.add_argument("--codex_exec_web_search", type=_BOOL)
+    p.add_argument("--codex_exec_approval_policy", type=str)
+    p.add_argument("--claude_code_exec_path", type=str)
+    p.add_argument("--claude_code_exec_profile", type=str)
+    p.add_argument("--claude_code_exec_use_sdk", type=str)
+    p.add_argument("--claude_code_exec_effort", type=str)
+    p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
+    p.add_argument("--codex_trace_to_teacher", type=_BOOL)
+    p.add_argument("--skill_init", type=str)
+    p.add_argument("--num_epochs", type=int)
+    p.add_argument("--train_size", type=int)
+    p.add_argument("--steps_per_epoch", type=int)
+    p.add_argument("--batch_size", type=int)
+    p.add_argument("--accumulation", type=int)
+    p.add_argument("--seed", type=int)
+    p.add_argument("--edit_budget", type=int)
+    p.add_argument("--min_edit_budget", type=int)
+    p.add_argument("--lr_scheduler", type=str,
+                   choices=["constant", "linear", "cosine", "autonomous"])
+    p.add_argument("--lr_control_mode", type=str,
+                   choices=["fixed", "autonomous", "none"])
+    p.add_argument("--merge_batch_size", type=int)
+    p.add_argument("--max_analyst_rounds", type=int)
+    p.add_argument("--sel_env_num", type=int)
+    p.add_argument("--test_env_num", type=int)
+    p.add_argument("--eval_test", type=_BOOL)
+    p.add_argument("--use_gate", type=_BOOL)
+    p.add_argument("--max_steps", type=int)
+    p.add_argument("--max_api_workers", type=int)
+    p.add_argument("--analyst_workers", type=int)
+    p.add_argument("--failure_only", type=_BOOL)
+    p.add_argument("--minibatch_size", type=int)
+    p.add_argument("--use_meta_reflect", type=_BOOL)
+    p.add_argument("--meta_edit_budget", type=int)
+    p.add_argument("--skill_update_mode", type=str,
+                   choices=[
+                       "patch",
+                       "rewrite_from_suggestions",
+                       "rewrite",
+                       "suggestions",
+                       "full_rewrite",
+                       "full_rewrite_minibatch",
+                       "minibatch_full_rewrite",
+                   ])
+    p.add_argument("--use_deep_reflect", type=_BOOL)
+    p.add_argument("--deep_reflect_failures", type=int)
+    p.add_argument("--deep_reflect_successes", type=int)
+    p.add_argument("--use_slow_update", type=_BOOL)
+    p.add_argument("--slow_update_samples", type=int)
+    p.add_argument("--longitudinal_pair_policy", type=str,
+                   choices=["mixed", "changed", "unchanged"])
+    p.add_argument("--use_meta_skill", type=_BOOL)
+    p.add_argument("--data_path", type=str)
+    p.add_argument("--split_mode", type=str,
+                   choices=["ratio", "split_dir"])
+    p.add_argument("--split_ratio", type=str)
+    p.add_argument("--split_seed", type=int)
+    p.add_argument("--split_dir", type=str)
+    p.add_argument("--split_output_dir", type=str)
+    p.add_argument("--data_root", type=str)
+    p.add_argument("--max_turns", type=int)
+    p.add_argument("--workers", type=int)
+    p.add_argument("--limit", type=int)
+    p.add_argument("--shuffle_choices", type=_BOOL)
+    p.add_argument("--use_theorem", type=_BOOL)
+    p.add_argument("--use_sketch", type=_BOOL)
+    p.add_argument("--image_detail", type=str)
+    p.add_argument("--judge_model", type=str)
+    p.add_argument("--judge_max_completion_tokens", type=int)
+    p.add_argument("--judge_retries", type=int)
+    p.add_argument("--out_root", type=str)
+    p.add_argument("--mode", type=str)
+
+    return p.parse_args()
+
+
+# ── Flat key → structured path mapping (for legacy CLI → structured config) ──
+
+_LEGACY_TO_STRUCTURED: dict[str, str] = {
+    "backend": "model.backend",
+    "teacher_model": "model.teacher",
+    "student_model": "model.student",
+    "teacher_backend": "model.teacher_backend",
+    "student_backend": "model.student_backend",
+    "reasoning_effort": "model.reasoning_effort",
+    "rewrite_reasoning_effort": "model.rewrite_reasoning_effort",
+    "rewrite_max_completion_tokens": "model.rewrite_max_completion_tokens",
+    "azure_endpoint": "model.azure_endpoint",
+    "azure_api_version": "model.azure_api_version",
+    "azure_api_key": "model.azure_api_key",
+    "azure_openai_endpoint": "model.azure_openai_endpoint",
+    "azure_openai_api_version": "model.azure_openai_api_version",
+    "azure_openai_api_key": "model.azure_openai_api_key",
+    "azure_openai_auth_mode": "model.azure_openai_auth_mode",
+    "azure_openai_ad_scope": "model.azure_openai_ad_scope",
+    "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id",
+    "teacher_azure_openai_endpoint": "model.teacher_azure_openai_endpoint",
+    "teacher_azure_openai_api_version": "model.teacher_azure_openai_api_version",
+    "teacher_azure_openai_api_key": "model.teacher_azure_openai_api_key",
+    "teacher_azure_openai_auth_mode": "model.teacher_azure_openai_auth_mode",
+    "teacher_azure_openai_ad_scope": "model.teacher_azure_openai_ad_scope",
+    "teacher_azure_openai_managed_identity_client_id": "model.teacher_azure_openai_managed_identity_client_id",
+    "student_azure_openai_endpoint": "model.student_azure_openai_endpoint",
+    "student_azure_openai_api_version": "model.student_azure_openai_api_version",
+    "student_azure_openai_api_key": "model.student_azure_openai_api_key",
+    "student_azure_openai_auth_mode": "model.student_azure_openai_auth_mode",
+    "student_azure_openai_ad_scope": "model.student_azure_openai_ad_scope",
+    "student_azure_openai_managed_identity_client_id": "model.student_azure_openai_managed_identity_client_id",
+    "codex_exec_path": "model.codex_exec_path",
+    "codex_exec_sandbox": "model.codex_exec_sandbox",
+    "codex_exec_profile": "model.codex_exec_profile",
+    "codex_exec_full_auto": "model.codex_exec_full_auto",
+    "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort",
+    "codex_exec_use_sdk": "model.codex_exec_use_sdk",
+    "codex_exec_network_access": "model.codex_exec_network_access",
+    "codex_exec_web_search": "model.codex_exec_web_search",
+    "codex_exec_approval_policy": "model.codex_exec_approval_policy",
+    "claude_code_exec_path": "model.claude_code_exec_path",
+    "claude_code_exec_profile": "model.claude_code_exec_profile",
+    "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
+    "claude_code_exec_effort": "model.claude_code_exec_effort",
+    "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
+    "codex_trace_to_teacher": "model.codex_trace_to_teacher",
+    "num_epochs": "train.num_epochs",
+    "train_size": "train.train_size",
+    "steps_per_epoch": "train.steps_per_epoch",
+    "batch_size": "train.batch_size",
+    "accumulation": "train.accumulation",
+    "seed": "train.seed",
+    "minibatch_size": "gradient.minibatch_size",
+    "merge_batch_size": "gradient.merge_batch_size",
+    "analyst_workers": "gradient.analyst_workers",
+    "max_analyst_rounds": "gradient.max_analyst_rounds",
+    "failure_only": "gradient.failure_only",
+    "use_deep_reflect": "gradient.use_deep_reflect",
+    "deep_reflect_failures": "gradient.deep_reflect_failures",
+    "deep_reflect_successes": "gradient.deep_reflect_successes",
+    "edit_budget": "optimizer.learning_rate",
+    "min_edit_budget": "optimizer.min_learning_rate",
+    "lr_scheduler": "optimizer.lr_scheduler",
+    "lr_control_mode": "optimizer.lr_control_mode",
+    "skill_update_mode": "optimizer.skill_update_mode",
+    "use_meta_reflect": "optimizer.use_meta_reflect",
+    "meta_edit_budget": "optimizer.meta_learning_rate",
+    "use_slow_update": "optimizer.use_slow_update",
+    "slow_update_samples": "optimizer.slow_update_samples",
+    "longitudinal_pair_policy": "optimizer.longitudinal_pair_policy",
+    "use_meta_skill": "optimizer.use_meta_skill",
+    "use_gate": "evaluation.use_gate",
+    "sel_env_num": "evaluation.sel_env_num",
+    "test_env_num": "evaluation.test_env_num",
+    "eval_test": "evaluation.eval_test",
+    "env": "env.name",
+    "skill_init": "env.skill_init",
+    "out_root": "env.out_root",
+}
+
+
+def load_config(args: argparse.Namespace) -> dict:
+    """Load config with _base_ inheritance, then apply CLI overrides."""
+    from reflact.config import load_config as _load, flatten_config, is_structured
+
+    cfg = _load(args.config, overrides=args.cfg_options)
+    structured = is_structured(cfg)
+
+    # Apply legacy --key value overrides
+    cli = {k: v for k, v in vars(args).items()
+           if v is not None and k not in ("config", "cfg_options")}
+    if cli:
+        if structured:
+            from reflact.config import apply_overrides
+            mapped = []
+            for k, v in cli.items():
+                dotted = _LEGACY_TO_STRUCTURED.get(k)
+                if dotted:
+                    mapped.append(f"{dotted}={v}")
+                else:
+                    mapped.append(f"env.{k}={v}")
+            apply_overrides(cfg, mapped)
+        else:
+            cfg.update(cli)
+
+    # Flatten structured config → flat dict for trainer/adapter
+    flat = flatten_config(cfg) if structured else cfg
+
+    for new_key, old_key in (
+        ("azure_openai_endpoint", "azure_endpoint"),
+        ("azure_openai_api_version", "azure_api_version"),
+        ("azure_openai_api_key", "azure_api_key"),
+    ):
+        if flat.get(new_key) in (None, "") and flat.get(old_key) not in (None, ""):
+            flat[new_key] = flat[old_key]
+
+    explicit_backend = getattr(args, "backend", None)
+    if explicit_backend is None:
+        for option in args.cfg_options or []:
+            key = str(option).split("=", 1)[0].strip()
+            if key == "model.backend":
+                explicit_backend = str(option).split("=", 1)[1].strip()
+                break
+
+    backend = normalize_backend_name(flat.get("model_backend") or flat.get("student_backend") or "azure_openai")
+
+    def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
+        if getattr(args, legacy_key, None) is not None:
+            return True
+        for option in args.cfg_options or []:
+            key = str(option).split("=", 1)[0].strip()
+            if key == dotted_key:
+                return True
+        return False
+
+    if explicit_backend is not None:
+        backend = normalize_backend_name(explicit_backend)
+        flat["model_backend"] = backend
+        if backend in {"claude", "claude_chat"}:
+            flat.setdefault("teacher_backend", "claude_chat")
+            flat.setdefault("student_backend", "claude_chat")
+        elif backend in {"codex", "codex_exec"}:
+            flat.setdefault("teacher_backend", "openai_chat")
+            flat.setdefault("student_backend", "codex_exec")
+        elif backend == "claude_code_exec":
+            flat.setdefault("teacher_backend", "openai_chat")
+            flat.setdefault("student_backend", "claude_code_exec")
+        else:
+            flat.setdefault("teacher_backend", "openai_chat")
+            flat.setdefault("student_backend", "openai_chat")
+    else:
+        flat.setdefault("teacher_backend", "openai_chat")
+        flat.setdefault("student_backend", "openai_chat")
+
+    if flat.get("teacher_backend") == "claude_chat":
+        if (
+            str(flat.get("teacher_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.teacher", "teacher_model")
+        ):
+            flat["teacher_model"] = default_model_for_backend("claude_chat")
+    if flat.get("student_backend") == "claude_chat":
+        if (
+            str(flat.get("student_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.student", "student_model")
+        ):
+            flat["student_model"] = default_model_for_backend("claude_chat")
+    if flat.get("student_backend") == "claude_code_exec":
+        if (
+            str(flat.get("student_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+            and not _has_model_override("model.student", "student_model")
+        ):
+            flat["student_model"] = default_model_for_backend("claude_chat")
+
+    # Auto-generate output root
+    if not flat.get("out_root"):
+        env = flat.get("env", "unknown")
+        model = flat.get("teacher_model", "unknown").replace("/", "-")
+        ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        flat["out_root"] = os.path.join("outputs", f"reflact_{env}_{model}_{ts}")
+
+    flat["out_root"] = os.path.abspath(flat["out_root"])
+    return flat
+
+
+# ── Main ─────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    args = parse_args()
+    cfg = load_config(args)
+
+    print(f"\n{'='*60}")
+    print(f"  ReflACT — Reflective Agent Tuning")
+    print(f"{'='*60}")
+    print(f"  env:            {cfg.get('env')}")
+    print(f"  teacher_model:  {cfg.get('teacher_model')}")
+    print(f"  student_model:  {cfg.get('student_model')}")
+    print(f"  teacher_backend:{cfg.get('teacher_backend', 'openai_chat')}")
+    print(f"  student_backend:{cfg.get('student_backend', 'openai_chat')}")
+    print(f"  reasoning:      {cfg.get('reasoning_effort') or 'off'}")
+    print(f"  rewrite_effort: {cfg.get('rewrite_reasoning_effort') or 'off'}")
+    print(f"  epochs:         {cfg.get('num_epochs')}")
+    print(f"  train_size:     {cfg.get('train_size') or 'from dataset'}")
+    print(f"  steps/epoch:    auto")
+    print(f"  batch_size:     {cfg.get('batch_size')}")
+    print(f"  edit_budget:    {cfg.get('edit_budget')}")
+    print(f"  lr_scheduler:   {cfg.get('lr_scheduler', 'constant')}")
+    print(f"  update_mode:    {cfg.get('skill_update_mode', 'patch')}")
+    print(f"  min_edit_budget:{cfg.get('min_edit_budget', 2)}")
+    print(f"  minibatch_size: {cfg.get('minibatch_size')}")
+    print(f"  seed:           {cfg.get('seed')}")
+    print(f"  meta_reflect:   {cfg.get('use_meta_reflect', False)}")
+    print(f"  meta_skill:     {cfg.get('use_meta_skill', False)}")
+    print(f"  out_root:       {cfg.get('out_root')}")
+    print(f"{'='*60}\n")
+
+    # Build adapter
+    adapter = get_adapter(cfg)
+
+    # Build trainer and run
+    from reflact.engine.trainer import ReflACTTrainer
+    trainer = ReflACTTrainer(cfg, adapter)
+    summary = trainer.train()
+
+    print(f"\n  Output saved to: {cfg['out_root']}")
+    if summary.get("test_hard") is not None:
+        print(f"  Final test: {summary['test_hard']:.4f}")
+
+
+if __name__ == "__main__":
+    main()

scripts/train_searchqa.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT3 — SearchQA Training
+#
+# Usage:
+#   bash scripts/train_searchqa.sh
+#   bash scripts/train_searchqa.sh --limit 50
+#   bash scripts/train_searchqa.sh --num_epochs 2 --workers 32
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+
+# ── Models ───────────────────────────────────────────────────────────────────
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+# ── Data ─────────────────────────────────────────────────────────────────────
+DATA_PATH="/home/azureuser/workspace-yqh/refleAct/search-qa/data/searchqa_train_2000.json"
+
+# ── Output ───────────────────────────────────────────────────────────────────
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/searchqa-metaskill/searchqa_${STUDENT_DEPLOYMENT}"
+
+echo "============================================================"
+echo "  ReflACT3 — SearchQA Training"
+echo "  Teacher:  ${TEACHER_DEPLOYMENT}"
+echo "  Student:  ${STUDENT_DEPLOYMENT}"
+echo "  Data:     ${DATA_PATH}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/searchqa_default.yaml \
+    --data_path "${DATA_PATH}" \
+    --out_root "${DEFAULT_OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"

scripts/train_spreadsheet_multi.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT — SpreadsheetBench training (MULTI-ROUND codegen, no tool-call)
+#
+# Usage:
+#   bash scripts/train_spreadsheet_multi.sh
+#   bash scripts/train_spreadsheet_multi.sh --num_epochs 2 --max_turns 5
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+DATA_ROOT="/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH="${DATA_ROOT}/dataset.json"
+SPLIT_DIR="/home/azureuser/workspace-yqh/refleACT3/data/spreadsheetbench_split_2_1_7"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUT_ROOT="${PROJECT_ROOT}/outputs/spreadsheet-metaskill-new/train_multi_${STUDENT_DEPLOYMENT}"
+
+echo "============================================================"
+echo "  ReflACT — SpreadsheetBench Training (MULTI-ROUND)"
+echo "============================================================"
+echo "  Teacher:    ${TEACHER_DEPLOYMENT}"
+echo "  Student:    ${STUDENT_DEPLOYMENT}"
+echo "  Mode:       multi"
+echo "  Data:       ${DATA_ROOT}"
+echo "  Split:      ${SPLIT_DIR}"
+echo "  Output:     ${OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --mode multi \
+    --data_root "${DATA_ROOT}" \
+    --jsonl_path "${JSONL_PATH}" \
+    --split_dir "${SPLIT_DIR}" \
+    --out_root "${OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${OUT_ROOT}"

scripts/train_spreadsheet_single.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+# ReflACT — SpreadsheetBench training (SINGLE-ROUND codegen, no tool-call)
+#
+# Usage:
+#   bash scripts/train_spreadsheet_single.sh
+#   bash scripts/train_spreadsheet_single.sh --num_epochs 2 --edit_budget 6
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
+
+export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
+export TEACHER_DEPLOYMENT="${TEACHER_DEPLOYMENT:-gpt-5.5}"
+export STUDENT_DEPLOYMENT="${STUDENT_DEPLOYMENT:-gpt-5.5}"
+
+DATA_ROOT="/home/azureuser/workspace-yqh/sr/spreadsheetbench/data/spreadsheetbench_verified_400"
+JSONL_PATH="${DATA_ROOT}/dataset.json"
+SPLIT_DIR="/home/azureuser/workspace-yqh/refleACT3/data/spreadsheetbench_split_2_1_7"
+
+TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+OUT_ROOT="${PROJECT_ROOT}/outputs/spreadsheet-metaskill-new/train_single_${STUDENT_DEPLOYMENT}"
+
+echo "============================================================"
+echo "  ReflACT — SpreadsheetBench Training (SINGLE-ROUND)"
+echo "============================================================"
+echo "  Teacher:    ${TEACHER_DEPLOYMENT}"
+echo "  Student:    ${STUDENT_DEPLOYMENT}"
+echo "  Mode:       single"
+echo "  Data:       ${DATA_ROOT}"
+echo "  Split:      ${SPLIT_DIR}"
+echo "  Output:     ${OUT_ROOT}"
+echo "============================================================"
+
+cd "${PROJECT_ROOT}"
+
+python scripts/train.py \
+    --config configs/spreadsheetbench_default.yaml \
+    --mode single \
+    --data_root "${DATA_ROOT}" \
+    --jsonl_path "${JSONL_PATH}" \
+    --split_dir "${SPLIT_DIR}" \
+    --out_root "${OUT_ROOT}" \
+    "$@"
+
+echo ""
+echo "Done! Results saved to: ${OUT_ROOT}"

scripts/watch_ablation.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Watch an ablation run root and rerun final failures.
+
+This watcher is intended to run in tmux next to scripts/run_ablation_matrix.py.
+It writes STATUS.md on every poll and starts a direct rerun for any run that
+the launcher marks as final [FAIL].
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+
+from run_ablation_matrix import PROJECT_ROOT, build_matrix, command_for
+
+
+RUN_RE = re.compile(r"\[(START|DONE|FAIL|RETRY)\]\s+([^\s]+)")
+ERROR_RE = re.compile(
+    r"Traceback|RuntimeError|AuthenticationError|PermissionDenied|"
+    r"DeploymentNotFound|LLM call failed|LLM message call failed|"
+    r"BadRequestError|RateLimitError",
+    re.IGNORECASE,
+)
+
+
+def read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8", errors="replace")
+    except FileNotFoundError:
+        return ""
+
+
+def parse_launcher(path: Path) -> dict[str, list[str]]:
+    events = {"START": [], "DONE": [], "FAIL": [], "RETRY": []}
+    for line in read_text(path).splitlines():
+        match = RUN_RE.search(line)
+        if match:
+            events[match.group(1)].append(match.group(2))
+    return events
+
+
+def active_run_ids(run_root: Path) -> list[str]:
+    try:
+        raw = subprocess.check_output(["pgrep", "-af", "scripts/train.py"], text=True)
+    except subprocess.CalledProcessError:
+        return []
+    active: list[str] = []
+    pattern = re.compile(re.escape(str(run_root)) + r"/([^\s]+)")
+    for line in raw.splitlines():
+        for match in pattern.finditer(line):
+            active.append(match.group(1))
+    return sorted(set(active))
+
+
+def scan_errors(logs_dir: Path) -> dict[str, str]:
+    errors: dict[str, str] = {}
+    for log_path in sorted(logs_dir.glob("*.log")):
+        text = read_text(log_path)
+        match = ERROR_RE.search(text)
+        if match:
+            run_id = log_path.name.split(".watchrerun", 1)[0].removesuffix(".log")
+            errors[run_id] = match.group(0)
+    return errors
+
+
+def load_state(path: Path) -> dict:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {"reruns": {}}
+
+
+def save_state(path: Path, state: dict) -> None:
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(state, indent=2, ensure_ascii=False), encoding="utf-8")
+    tmp.replace(path)
+
+
+def write_status(
+    run_root: Path,
+    total: int,
+    events: dict[str, list[str]],
+    active: list[str],
+    completed: list[str],
+    pending: list[str],
+    errors: dict[str, str],
+    reruns: dict[str, int],
+) -> None:
+    now = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime())
+    failed = sorted(set(events["FAIL"]))
+    retrying = sorted(set(events["RETRY"]))
+    lines = [
+        "# Ablation Status",
+        "",
+        f"Updated: {now}",
+        f"Run root: `{run_root}`",
+        "",
+        "| Metric | Count |",
+        "| --- | ---: |",
+        f"| Total planned | {total} |",
+        f"| Completed summaries | {len(completed)} |",
+        f"| Active train processes | {len(active)} |",
+        f"| Pending/not summarized | {len(pending)} |",
+        f"| Launcher final fails | {len(failed)} |",
+        f"| Launcher retries | {len(retrying)} |",
+        f"| Logs with error patterns | {len(errors)} |",
+        "",
+        "## Active",
+        "",
+        *(f"- `{run_id}`" for run_id in active),
+        "",
+        "## Final Fails",
+        "",
+        *(f"- `{run_id}` watcher_reruns={reruns.get(run_id, 0)}" for run_id in failed),
+        "",
+        "## Error Patterns",
+        "",
+        *(f"- `{run_id}`: `{err}`" for run_id, err in sorted(errors.items())),
+        "",
+        "## Recent Launcher",
+        "",
+        "```text",
+        "\n".join(read_text(run_root / "launcher.log").splitlines()[-30:]),
+        "```",
+    ]
+    (run_root / "STATUS.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--run-root", required=True)
+    parser.add_argument("--interval", type=int, default=60)
+    parser.add_argument("--watcher-retries", type=int, default=1)
+    parser.add_argument("--groups", nargs="+", default=["all"])
+    parser.add_argument("--bench", nargs="+", default=["searchqa", "spreadsheetbench"])
+    args = parser.parse_args()
+
+    run_root = Path(args.run_root).resolve()
+    logs_dir = run_root / "logs"
+    logs_dir.mkdir(parents=True, exist_ok=True)
+    state_path = run_root / "watcher_state.json"
+
+    groups = set(args.groups)
+    if "all" in groups:
+        groups = {"default", "split", "mbs", "lr", "sched", "slown", "mod", "smodel"}
+    experiments = {
+        exp.run_id: exp
+        for exp in build_matrix(groups, args.bench, run_root, include_duplicate_defaults=False)
+    }
+
+    active_reruns: dict[str, subprocess.Popen] = {}
+    while True:
+        state = load_state(state_path)
+        reruns = state.setdefault("reruns", {})
+        events = parse_launcher(run_root / "launcher.log")
+        active = active_run_ids(run_root)
+        completed = sorted(
+            run_id for run_id in experiments
+            if (run_root / run_id / "summary.json").exists()
+        )
+        pending = sorted(set(experiments) - set(completed))
+        errors = scan_errors(logs_dir)
+
+        # Reap watcher-started reruns.
+        for run_id, proc in list(active_reruns.items()):
+            rc = proc.poll()
+            if rc is None:
+                continue
+            active_reruns.pop(run_id, None)
+            with open(logs_dir / f"{run_id}.watcher.log", "a", encoding="utf-8") as f:
+                f.write(f"\n[WATCHER_DONE] rc={rc} time={time.time()}\n")
+
+        for run_id in sorted(set(events["FAIL"])):
+            if run_id not in experiments:
+                continue
+            if (run_root / run_id / "summary.json").exists():
+                continue
+            if run_id in active or run_id in active_reruns:
+                continue
+            count = int(reruns.get(run_id, 0))
+            if count >= args.watcher_retries:
+                continue
+            reruns[run_id] = count + 1
+            save_state(state_path, state)
+            log_path = logs_dir / f"{run_id}.watchrerun{count + 1}.log"
+            with open(log_path, "w", encoding="utf-8") as log_f:
+                log_f.write(f"[WATCHER_START] run_id={run_id} attempt={count + 1}\n")
+                log_f.flush()
+                proc = subprocess.Popen(
+                    command_for(experiments[run_id]),
+                    cwd=PROJECT_ROOT,
+                    stdout=log_f,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    close_fds=True,
+                )
+            active_reruns[run_id] = proc
+
+        save_state(state_path, state)
+        write_status(
+            run_root=run_root,
+            total=len(experiments),
+            events=events,
+            active=active,
+            completed=completed,
+            pending=pending,
+            errors=errors,
+            reruns={k: int(v) for k, v in reruns.items()},
+        )
+        time.sleep(max(5, int(args.interval)))
+
+
+if __name__ == "__main__":
+    main()