diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json
new file mode 100644
index 0000000..55a8659
--- /dev/null
+++ b/.cursor-plugin/marketplace.json
@@ -0,0 +1,17 @@
+{
+ "name": "skillopt",
+ "owner": {
+ "name": "Yifan Yang",
+ "email": "yifanyang@microsoft.com"
+ },
+ "metadata": {
+ "description": "Official SkillOpt plugins for usage-driven, validation-gated agent improvement."
+ },
+ "plugins": [
+ {
+ "name": "skillopt-sleep",
+ "source": "plugins/cursor",
+ "description": "Review recent Cursor sessions, replay recurring work, and stage validation-gated improvements to a Cursor skill for explicit adoption."
+ }
+ ]
+}
diff --git a/.gitignore b/.gitignore
index 4a5142a..7c5950d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,3 +64,6 @@ docs/莊*
tests/run_*.sh
tests/launch_*.py
*.launch.log
+uv.lock
+# Superpowers smoke runs: raw agent output + local paths, share sanitized excerpts instead
+smoke_results/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6a9dadc..63b1a0b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,19 @@ All notable changes to SkillOpt are documented here. This project adheres to
## [Unreleased]
### Added
+- A non-destructive Devin installer and SessionEnd activity marker, preserving
+ existing project hooks across repeated installation.
+- Per-night SkillOpt-Sleep `evidence.jsonl` chains for reconstructing harvest,
+ mining, replay, reflection, and gate decisions, plus a live prompt-template
+ registry with user overrides.
+- Native SkillOpt-Sleep support for Cursor, including a local plugin command
+ and skill, Cursor transcript harvesting, and an optional Cursor Agent CLI
+ backend. Cursor tool-aware replay remains disabled pending live permission-
+ boundary validation.
+- **Cursor Agent research target harness** (`cursor_exec`) for running
+ supported benchmark rollouts through an installed, authenticated
+ `cursor-agent`, with sandboxed workspaces, structured trace capture, and
+ target-only optimizer separation.
- **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep â runs the
sleep cycle with no model subprocess or API key: the engine writes each
pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the
diff --git a/README.md b/README.md
index f4ccbac..8836325 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,7 @@
---
## News đĽđĽđĽ
+- **[2026-07-24]** đ° **SkillOpt in the news.** Read the official [Microsoft Research feature](https://www.microsoft.com/en-us/research/blog/skillopt-agent-skills-as-trainable-parameters/), along with recent coverage from [VentureBeat](https://venturebeat.com/orchestration/microsofts-open-source-skillopt-automatically-upgrades-ai-agent-skills-without-touching-model-weights), [Synced (ćşĺ¨äšĺż)](https://mp.weixin.qq.com/s/pMlyj3a3KOh8L7cIHClRXA), [Flowtivity](https://flowtivity.ai/blog/microsoft-skillopt-train-ai-agent-skills/), and [The Decoder](https://the-decoder.com/microsofts-skillopt-boosts-gpt-5-5-by-using-nothing-but-a-trained-markdown-file/).
- **[2026-07-02]** đ **SkillOpt [v0.2.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) is out on [PyPI](https://pypi.org/project/skillopt/)!** Headline feature: **SkillOpt-Sleep**, a nightly offline self-evolution engine (harvest â mine â replay â consolidate behind a held-out validation gate), now shipped as the `skillopt-sleep` CLI. It also includes experimental multi-objective, replay, and dream-rollout controls; the main CLI keeps conservative defaults and does not expose every experiment-harness control as a flag. The release source adds integration shells for **Claude Code, Codex, Copilot, and Devin**, plus an **OpenClaw reference adaptation**; these plugin/MCP files live in the repository rather than the PyPI wheel. It also adds SearchQA split materialization, Windows robustness, and hardened JSON parsing. See the [release notes](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) for full release details and contributor acknowledgements.
- **[2026-06-15]** đ´ **SkillOpt-Sleep (preview)** â a nightly offline self-evolution companion for local coding agents (Claude Code / Codex / Copilot): review past sessions, replay recurring tasks, and consolidate validated skills behind a held-out gate. See **[`docs/sleep/README.md`](docs/sleep/README.md)** for what it is, how to use it, and results.
- **[2026-06-03]** đ **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.**
@@ -65,13 +66,13 @@ https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7
A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`,
`qwen_chat`, `minimax_chat`, `openai_compatible`, `codex_exec`,
-`claude_code_exec`). If a provider implements the OpenAI Chat Completions
+`claude_code_exec`, `cursor_exec`). If a provider implements the OpenAI Chat Completions
protocol, try the built-in `openai_compatible` backend before adding code. See
[`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full
-contract; in short you add a `skillopt/model/_backend.py` module,
-register it in `skillopt/model/common.py` + `backend_config.py`, and wire
-it through the router in `skillopt/model/__init__.py`. `qwen_backend.py`
-and `minimax_backend.py` are good templates.
+contract. Chat backends add a `skillopt/model/_backend.py` module;
+target-only exec backends use the shared harness in `codex_harness.py`.
+Both register through `common.py`, `backend_config.py`, and
+`skillopt/model/__init__.py`.
### Adding a new benchmark
diff --git a/blog/gating-reflection-safe-updates/index.html b/blog/gating-reflection-safe-updates/index.html
index 96005f5..3a752d8 100644
--- a/blog/gating-reflection-safe-updates/index.html
+++ b/blog/gating-reflection-safe-updates/index.html
@@ -1697,27 +1697,13 @@
Citation
-
Cite this blog post
-
-
-
- If you find this report useful, please cite it as:
-
-
@misc{zhou2026skillopttechnicalreport,
- title = {Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep},
- author = {Zhou, Ziwei and Gong, Ziyang and Yang, Yifan},
- year = {2026},
- url = {https://microsoft.github.io/SkillOpt/blog/gating-reflection-safe-updates/},
- note = {SkillOpt Technical Blog post}
-}
-
-
-
SkillOpt paper
+
Cite SkillOpt
- For the underlying SkillOpt method, please also cite the
- SkillOpt paper:
+ If you find this report useful, please cite the main SkillOpt paper using the
+ citation maintained in the
+ SkillOpt repository:
@article{yang2026skillopt,
title={Skillopt: Executive strategy for self-evolving agent skills},
diff --git a/configs/_base_/default.yaml b/configs/_base_/default.yaml
index fcfdd71..c8c068d 100644
--- a/configs/_base_/default.yaml
+++ b/configs/_base_/default.yaml
@@ -24,6 +24,8 @@ model:
claude_code_exec_use_sdk: auto
claude_code_exec_effort: medium
claude_code_exec_max_thinking_tokens: 16384
+ cursor_exec_path: "" # blank uses CURSOR_EXEC_PATH or cursor-agent
+ cursor_exec_sandbox: "" # blank uses CURSOR_EXEC_SANDBOX or enabled
codex_trace_to_optimizer: true
azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/"
azure_openai_api_version: "2024-12-01-preview"
diff --git a/data/README.md b/data/README.md
index a31c337..73e5744 100644
--- a/data/README.md
+++ b/data/README.md
@@ -29,7 +29,7 @@ Each `items.json` contains only stable IDs or source-path hints.
| Manifest directory | Benchmark | Counts | Coverage | Raw data source | `split_dir` |
|---|---|---:|---|---|---|
| `searchqa_id_split/` | SearchQA | 400 / 200 / 1400 | Official HF dataset IDs | [lucadiliello/searchqa](https://huggingface.co/datasets/lucadiliello/searchqa) | `data/searchqa_split` |
-| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 18 / 124 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` |
+| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 17 / 125 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` |
| `docvqa_id_split/` | DocVQA | 107 / 53 / 374 | 10% subset of validation | [lmms-lab/DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA) | `data/docvqa/splits` |
| `officeqa_id_split/` | OfficeQA | 50 / 24 / 172 | OfficeQA Full | [databricks/officeqa](https://huggingface.co/datasets/databricks/officeqa) | `data/officeqa_split` |
| `spreadsheetbench_id_split/` | SpreadsheetBench | 80 / 40 / 280 | SpreadsheetBench Verified 400 | [KAKA22/SpreadsheetBench](https://huggingface.co/datasets/KAKA22/SpreadsheetBench) | `data/spreadsheetbench_split` |
diff --git a/data/livemathematicianbench_id_split/split_manifest.json b/data/livemathematicianbench_id_split/split_manifest.json
index 9af68a0..75c6bea 100644
--- a/data/livemathematicianbench_id_split/split_manifest.json
+++ b/data/livemathematicianbench_id_split/split_manifest.json
@@ -16,8 +16,8 @@
"split_seed": 42,
"counts": {
"train": 35,
- "val": 18,
- "test": 124
+ "val": 17,
+ "test": 125
},
"item_fields": [
"id",
diff --git a/data/livemathematicianbench_id_split/test/items.json b/data/livemathematicianbench_id_split/test/items.json
index 83006f3..575080d 100644
--- a/data/livemathematicianbench_id_split/test/items.json
+++ b/data/livemathematicianbench_id_split/test/items.json
@@ -866,5 +866,12 @@
"no": 37,
"paper_link": "http://arxiv.org/abs/2602.08644v1",
"source_file": "data/202602/qa_202602_final.json"
+ },
+ {
+ "id": "202512:46",
+ "month": "202512",
+ "no": 46,
+ "paper_link": "http://arxiv.org/abs/2512.05945v1",
+ "source_file": "data/202512/qa_202512_final.json"
}
-]
+]
\ No newline at end of file
diff --git a/data/livemathematicianbench_id_split/val/items.json b/data/livemathematicianbench_id_split/val/items.json
index 4298592..4e63fea 100644
--- a/data/livemathematicianbench_id_split/val/items.json
+++ b/data/livemathematicianbench_id_split/val/items.json
@@ -117,12 +117,5 @@
"no": 6,
"paper_link": "http://arxiv.org/abs/2602.01571v1",
"source_file": "data/202602/qa_202602_final.json"
- },
- {
- "id": "202512:46",
- "month": "202512",
- "no": 46,
- "paper_link": "http://arxiv.org/abs/2512.05945v1",
- "source_file": "data/202512/qa_202512_final.json"
}
-]
+]
\ No newline at end of file
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index 8600c44..2a72a64 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -45,6 +45,7 @@ model:
| `minimax_chat` | â | â | MiniMax API |
| `codex_exec` | â | â | Codex CLI execution harness |
| `claude_code_exec` | â | â | Claude Code CLI execution harness |
+| `cursor_exec` | â | â | Cursor Agent CLI execution harness |
The current MiniMax adapter has one shared deployment. Set
`model.minimax_model` when MiniMax is the target; a mixed-backend run cannot
@@ -183,6 +184,9 @@ Model credentials are loaded from environment variables:
| `OPENAI_COMPATIBLE_MODEL` | `openai_compatible` | Shared provider model ID for direct library use; train/eval YAML role models take precedence |
| `CLAUDE_CLI_BIN` | `claude_chat` | Optional path to the `claude` executable; defaults to `claude` |
| `ANTHROPIC_API_KEY` | `claude_chat` | Optional authentication method understood by the Claude CLI, not a direct SkillOpt API client |
+| `CURSOR_EXEC_PATH` | `cursor_exec` | Optional path to `cursor-agent`; defaults to `cursor-agent` |
+| `CURSOR_EXEC_SANDBOX` | `cursor_exec` | Cursor sandbox mode: `enabled` (default) or `disabled` |
+| `CURSOR_API_KEY` | `cursor_exec` | Optional authentication method understood directly by Cursor Agent |
| `QWEN_CHAT_BASE_URL` | `qwen_chat` | Local Qwen/vLLM endpoint |
| `QWEN_CHAT_MODEL` | `qwen_chat` | Served model name for direct library use; train/eval YAML role models take precedence |
| `MINIMAX_BASE_URL` | `minimax_chat` | MiniMax-compatible base URL |
@@ -197,6 +201,14 @@ and authenticate that CLI before use. Setting `ANTHROPIC_API_KEY` is one way
the CLI may authenticate, but SkillOpt does not call the Anthropic API
directly through this backend.
+`cursor_exec` is a target-only benchmark harness. Install and authenticate
+Cursor Agent first, then select it with `model.target_backend=cursor_exec`.
+Read-only rollouts use Ask mode; artifact-producing rollouts add Cursor's
+headless `--force` flag inside the benchmark workspace. SkillOpt enables the
+Cursor sandbox by default and rejects file-edit rollouts if it is disabled;
+read-only Ask-mode rollouts may explicitly disable it. SkillOpt does not approve
+MCP servers automatically.
+
### Three OpenAI-compatible paths
- Research, generic provider: select `openai_compatible` and use
diff --git a/docs/guide/installation.md b/docs/guide/installation.md
index 1b24a2d..e35d555 100644
--- a/docs/guide/installation.md
+++ b/docs/guide/installation.md
@@ -26,9 +26,10 @@ checkout for those files.
!!! important "PyPI versus `main`"
These docs track the latest `main`. The current PyPI release is `0.2.0`.
The generic research `openai_compatible` backend, SkillOpt-Sleep handoff,
- Sleep support for non-Azure OpenAI-compatible endpoints, and the Sleep
- `--preferences` flag landed after that release and require a source install
- from `main` until the next release.
+ Sleep support for non-Azure OpenAI-compatible endpoints, the Sleep
+ `--preferences` flag, and Cursor source/backend/plugin support landed after
+ that release and require a source install from `main` until the next
+ release.
### Source checkout
@@ -128,6 +129,13 @@ Anthropic API client. Install and authenticate `claude`, and set
`CLAUDE_CLI_BIN` only if the executable is not available as `claude` on
`PATH`. `ANTHROPIC_API_KEY` is one authentication option the CLI may consume.
+The SkillOpt-Sleep `cursor` backend similarly requires a separately installed
+and authenticated `cursor-agent`; harvesting with `--source cursor` alone does
+not. Set `SKILLOPT_SLEEP_CURSOR_PATH` when the executable is not on `PATH`, and
+`SKILLOPT_SLEEP_CURSOR_MODEL` to override its model. Cursor plugin installation
+and the explicit project skill target are documented in the
+[Cursor integration guide](https://github.com/microsoft/SkillOpt/blob/main/plugins/cursor/README.md).
+
OpenAI-compatible servers have three distinct entry points:
1. The research engine's generic `openai_compatible` backend uses
diff --git a/docs/guide/new-backend.md b/docs/guide/new-backend.md
index 6785527..98d598f 100644
--- a/docs/guide/new-backend.md
+++ b/docs/guide/new-backend.md
@@ -146,9 +146,9 @@ Provider-specific configuration helpers and `count_tokens()` are optional, but
their state must be safe to update while calls may run concurrently. Keep
credentials out of logs and persisted artifacts.
-Exec-style targets do not implement this chat contract. They are target-only
-and are integrated through `codex_harness.py` plus environment-specific rollout
-code.
+Exec-style targets such as `claude_code_exec` and `cursor_exec` do not
+implement this chat contract. They are target-only and are integrated through
+`codex_harness.py` plus environment-specific rollout code.
## Step 2: register and route the backend
diff --git a/docs/guideline.html b/docs/guideline.html
index 3bd625d..801ccaf 100644
--- a/docs/guideline.html
+++ b/docs/guideline.html
@@ -295,9 +295,10 @@ python -m pip install -e ".[dev]" # tests and linting
Release boundary
This guide tracks main. PyPI currently serves 0.2.0; the
generic research openai_compatible backend, Sleep handoff,
- SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, and the
- Sleep --preferences flag require a source install from
- main until the next release.
+ SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, the
+ Sleep --preferences flag, and Cursor source/backend/plugin
+ support require a source install from main until the next
+ release.
See the installation guide
for platform notes and dependency boundaries.
--project scopes collection but does not automatically
choose a project's skill file. Use --target-skill-path when
you intend to evolve a particular SKILL.md. Transcript source
- (claude, codex, or auto) and replay
- backend are independent settings.
+ (claude, codex, cursor, or
+ auto) and replay backend are independent settings. The existing
+ auto precedence remains Codex then Claude; select Cursor
+ explicitly with --source cursor.
+
Cursor transcripts default to
+ ~/.cursor/projects/<workspace>/agent-transcripts; override
+ that home with --cursor-home. Model-driven replay requires an
+ installed, authenticated cursor-agent and
+ --backend cursor; use --cursor-path when the CLI is
+ not on PATH. Target
+ .cursor/skills/skillopt-sleep-learned/SKILL.md explicitly so
+ adoption updates a project skill rather than the plugin's workflow skill.
+
The Cursor backend inserts that skill text into prompts; it does not
+ invoke the file as a native skill. Ordinary calls run in read-only Ask mode
+ in an empty temporary workspace and cannot inspect files under
+ --project. Cursor tasks containing a tool_called
+ check fail before Agent mode starts; use another backend for those tasks.
+ This validates textual guidance, not end-to-end repository, browser,
+ service, or filesystem workflows.
For subscription-based workflows that should not launch an API or model
subprocess, use --backend handoff and follow the generated
prompt/answer loop. Read the
@@ -467,6 +486,7 @@ skillopt-sleep adopt --project "$PWD"
Single rollout by default; values above 1 enable experimental contrastive replay.
dream_factor
0
Synthetic task variants are off by default.
recall_k
0
Historical associative recall is off by default.
+
replay_mode
mock
Reporting label for prompt replay; fresh-worktree replay is not implemented.
@@ -498,6 +519,13 @@ skillopt-sleep adopt --project "$PWD"
reward/budget controls as advanced features that require task-specific
validation. The reported experiments and their exact settings are in
RESULTS.md.
+
The managed scheduler persists only project, backend, time, and optional
+ auto-adopt. For Cursor schedules, put transcript_source,
+ cursor_home, cursor_path, model, and
+ target_skill_path in
+ ~/.skillopt-sleep/config.json. Use an absolute Cursor CLI path
+ and verify authentication for the scheduled account because schedulers may
+ run with a minimal environment.
@@ -510,6 +538,13 @@ skillopt-sleep adopt --project "$PWD"
is not a guarantee that every outbound model prompt is free of sensitive
content. In particular, do not treat raw coding-agent transcripts as
pre-sanitized.
+
The Cursor source excludes tool arguments and outputs, retaining only
+ user/assistant text, explicit turn errors, and tool names. The Cursor
+ backend still sends transcript-derived prompts through
+ cursor-agent to Cursor's selected model provider.
+
A real-backend dry-run still performs provider calls; it
+ suppresses staging rather than spend. Session and task limits are not hard
+ provider-call, token, time, or monetary budgets.
Updates are staged for review by default. Use
--auto-adopt only when you have an independent rollback and
validation process.
diff --git a/docs/reference/api.md b/docs/reference/api.md
index 287a717..34176d7 100644
--- a/docs/reference/api.md
+++ b/docs/reference/api.md
@@ -192,6 +192,7 @@ not via a base class subclass. Supported values (as of this writing):
| `openai_compatible` | â | â |
| `codex_exec` | â | â |
| `claude_code_exec` | â | â |
+| `cursor_exec` | â | â |
See `skillopt/model/backend_config.py` for the live whitelist and
[`docs/reference/config.md`](./config.md) for the per-backend
diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index e4d277a..69fa943 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -2,9 +2,10 @@
> **Version note.** This reference tracks `main`. PyPI 0.2.0 does not yet
> include the generic research `openai_compatible` backend, Sleep handoff,
-> Sleep support for non-Azure OpenAI-compatible endpoints, or the Sleep
-> `--preferences` flag; use a source install from `main` for those features
-> until the next release.
+> Sleep support for non-Azure OpenAI-compatible endpoints, the Sleep
+> `--preferences` flag, the research `cursor_exec` target harness, or Cursor
+> source/backend/plugin support; use a source install from `main` for those
+> features until the next release.
## Training
@@ -91,6 +92,27 @@ python scripts/train.py \
model.target=deepseek-chat
```
+To benchmark an installed, authenticated Cursor Agent through an environment
+that supports exec targets:
+
+```bash
+python scripts/eval_only.py \
+ --config configs/searchqa/default.yaml \
+ --skill skills/my_skill.md \
+ --cfg-options \
+ model.optimizer_backend=openai_chat \
+ model.target_backend=cursor_exec \
+ model.target=composer-2.5
+```
+
+`cursor_exec` runs the target only; the optimizer remains separately
+configured. Read-only rollouts use Cursor Ask mode. Rollouts that request file
+edits use `--force` inside the benchmark workspace, with Cursor sandboxing
+enabled. The harness refuses file-edit rollouts when the Cursor sandbox is
+disabled. Read-only Ask-mode rollouts may explicitly disable it. Override the
+executable or sandbox through `model.cursor_exec_path` and
+`model.cursor_exec_sandbox`.
+
## SkillOpt-Sleep
```bash
@@ -104,11 +126,13 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and
| Argument | Description |
|---|---|
-| `--project PATH` | Project to evolve (default: current directory) |
+| `--project PATH` | Project used for transcript scope, targets, state, and staging (default: current directory) |
| `--scope invoked\|all` | Harvest this project or all projects |
-| `--source claude\|codex\|auto` | Transcript source |
-| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | Replay/optimizer backend |
+| `--source claude\|codex\|cursor\|auto` | Transcript source; `auto` keeps Codex-then-Claude precedence and does not select Cursor |
+| `--backend mock\|claude\|codex\|copilot\|cursor\|handoff\|azure_openai` | Replay/optimizer backend |
| `--model NAME` | Backend-specific model override |
+| `--cursor-home PATH` | Override `~/.cursor` for Cursor transcript harvesting |
+| `--cursor-path PATH` | Path to the installed Cursor Agent CLI |
| `--preferences TEXT` | House rules supplied to reflection |
| `--lookback-hours N` | Initial transcript lookback; `0` scans all history |
| `--max-sessions N` / `--max-tasks N` | Bound the harvested workload |
@@ -118,6 +142,82 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and
| `--progress` / `--json` | Progress or machine-readable output |
| `--auto-adopt` | Apply an accepted staged proposal automatically |
+### Cursor source and backend
+
+`--source cursor` reads local Cursor JSONL transcripts from
+`~/.cursor/projects//agent-transcripts/*/*.jsonl`. Invoked scope uses
+Cursor's recorded workspace path, including when `--project` is a nested
+directory, and falls back to the sanitized storage name when metadata is not
+available. `--scope all` scans every workspace below `cursor_home`. The
+harvester retains user/assistant text, explicit turn errors, and tool names,
+while excluding tool arguments, tool outputs, and non-message records. It
+redacts known secret patterns and filters SkillOpt-generated replay sessions,
+but redaction is not a guarantee that outbound prompts contain no sensitive
+data.
+
+`--backend cursor` launches an installed, authenticated `cursor-agent`, sends
+prompts over stdin, and parses its JSON result. SkillOpt reads the target skill
+and includes its text in replay prompts; it does not invoke that file as a native
+Cursor skill. Ordinary mining, replay, judging, and reflection calls use
+read-only Ask mode in a new empty temporary workspace. Project file reads, file
+writes, and MCP tools are denied. `--project` does not change that execution
+workspace.
+
+Cursor tool-aware replay is temporarily disabled pending live Cursor
+permission-boundary validation. A task with a `tool_called` check fails nonzero
+before Agent mode starts and does not stage, adopt, cache, persist state, or
+advance the harvest checkpoint. Use another backend for such tasks. The current
+Cursor backend therefore does not provide end-to-end validation for skills that
+need repository inspection, real CLIs, browsers, running services, or file
+changes.
+
+There is no implemented fresh-worktree Cursor replay. If a report says
+`replay: mock`, that is the prompt-replay label and does not mean the mock model
+backend was selected. Both `run` and `dry-run` perform real-backend provider
+calls; `dry-run` suppresses staging, adoption, and persisted state changes, not
+spend. Session and task limits do not impose hard provider-call, token, time, or
+monetary budgets.
+Cursor and its selected model provider can receive the prompt content.
+
+Cursor-specific settings are available through the CLI, config, and environment:
+
+| Purpose | CLI | `~/.skillopt-sleep/config.json` | Environment |
+|---|---|---|---|
+| Transcript home | `--cursor-home PATH` | `"cursor_home": "/path/to/.cursor"` | none |
+| Agent executable | `--cursor-path PATH` | `"cursor_path": "/path/to/cursor-agent"` | `SKILLOPT_SLEEP_CURSOR_PATH` |
+| Model | `--model NAME` | `"model": "NAME"` | `SKILLOPT_SLEEP_CURSOR_MODEL` |
+
+Use `cursor-agent --list-models` to inspect model identifiers available to the
+authenticated account. When cost depends on a model variant, confirm the billed
+variant in Cursor's usage reporting rather than relying only on its display
+name.
+
+Target the learned project skill explicitly so accepted updates are visible to
+Cursor without modifying the plugin's own `skillopt-sleep` workflow skill:
+
+```bash
+skillopt-sleep run --project "$(pwd)" \
+ --source cursor --backend cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --progress
+```
+
+The first harvest uses a 72-hour lookback unless `--lookback-hours` is set. A
+value of `0` considers all available history while still respecting
+`--max-sessions`. A stateful `run`, including a run that mines no tasks, records
+a new harvest checkpoint; subsequent runs use that checkpoint rather than the
+initial lookback. Use `harvest` or `dry-run` to verify counts before the first
+stateful run.
+
+The managed `schedule` command persists the project, backend, time, and optional
+auto-adopt setting only. It does not copy source, Cursor paths, model, or target
+skill flags into the scheduled command. Put `transcript_source`, `cursor_home`,
+`cursor_path`, `model`, and `target_skill_path` in the user config before
+scheduling Cursor. Keep `target_skill_path` project-relative as
+`.cursor/skills/skillopt-sleep-learned/SKILL.md`, prefer an absolute
+`cursor_path`, and verify authentication for the scheduled account because cron
+and Task Scheduler may have a minimal environment.
+
Backend-specific setup for compatible endpoints is documented in
[OpenAI-compatible endpoints for SkillOpt-Sleep](../sleep/openai-compatible-endpoints.md).
diff --git a/docs/reference/config.md b/docs/reference/config.md
index b65c204..b8243c7 100644
--- a/docs/reference/config.md
+++ b/docs/reference/config.md
@@ -18,6 +18,7 @@ selecting the generic OpenAI-compatible backend.
| `minimax_chat` | â | â |
| `codex_exec` | â | â |
| `claude_code_exec` | â | â |
+| `cursor_exec` | â | â |
MiniMax currently has one shared deployment. `model.minimax_model` is applied
when MiniMax is the target; mixed-backend runs cannot independently choose a
@@ -64,6 +65,8 @@ defaults to `claude` and can be overridden with `CLAUDE_CLI_BIN`.
| `model.minimax_*` | MiniMax `base_url`, `api_key`, shared `minimax_model`, `temperature`, `max_tokens`, and `enable_thinking`; `minimax_model` applies when MiniMax is the target |
| `model.codex_exec_*` | Codex path, sandbox, profile, SDK mode, reasoning, network/search, and approval policy |
| `model.claude_code_exec_*` | Claude path, profile, SDK mode, effort, and thinking-token cap |
+| `model.cursor_exec_path` | Cursor Agent executable path; default `cursor-agent` |
+| `model.cursor_exec_sandbox` | Cursor sandbox mode: `enabled` (default) or `disabled`; file-edit rollouts require `enabled` |
## Training (`train`)
diff --git a/docs/sleep/README.md b/docs/sleep/README.md
index 2e36eb1..2b740ac 100644
--- a/docs/sleep/README.md
+++ b/docs/sleep/README.md
@@ -17,7 +17,7 @@ normal agent requests.
One "night":
```
-harvest Claude Code / Codex transcripts â mine recurring tasks â replay offline
+harvest Claude Code / Codex / Cursor transcripts â mine recurring tasks â replay via the configured backend (isolation varies by backend; mock/handoff make no network calls)
â consolidate (reflect â bounded edit â GATE on real held-out tasks)
â stage proposal â (you) adopt
```
@@ -33,6 +33,24 @@ experience â long-term competence).
> review your transcript source and provider policy before running on sensitive
> projects. For a reviewable workflow, harvest to a task file, inspect/redact it, mark
> it `"reviewed": true`, and then replay that file with the real backend.
+>
+> The Cursor source reads local user/assistant message text, explicit turn errors,
+> and tool names, but excludes tool arguments, tool outputs, and non-message records.
+> Known secret-shaped strings are redacted as defense in depth. The Cursor backend
+> sends prompts through `cursor-agent`; ordinary calls use read-only Ask mode in an
+> empty temporary workspace with project files denied. Cursor tool-aware replay is
+> temporarily disabled pending live permission-boundary validation.
+> Cursor and the model provider selected by Cursor may therefore receive
+> transcript-derived content.
+>
+> By default, each stateful night also writes a local `evidence.jsonl` under
+> the project staging tree (beside the report when one is staged); dry-runs
+> write evidence under the configured Sleep state directory. The log contains
+> best-effort-redacted, per-field-truncated copies
+> of miner, replay, judge, and reflection prompts and replies. Treat it as
+> sensitive local data and apply an appropriate retention policy. Set
+> `"evidence_log": false` to disable it; setting `"redact_secrets": false`
+> deliberately disables this defense-in-depth redaction.
## How to use it
@@ -48,13 +66,14 @@ skillopt-sleep schedule # install a nightly cron entry for this project
```
> **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base
-> commands above. Sleep handoff, non-Azure OpenAI-compatible endpoints, and
-> `--preferences` landed later and require a source install from `main` until
-> the next release.
+> commands above. Cursor source/backend/plugin support, Sleep handoff, non-Azure
+> OpenAI-compatible endpoints, and `--preferences` landed later and require a
+> source install from `main` until the next release.
The per-agent integrations below still come from the repo; the CLI above is the
-standalone, pip-only way to run a cycle. Claude Code, Codex, Copilot, and Devin wrap
-the shared engine. OpenClaw is a separate reference adaptation and has its own setup.
+standalone, pip-only way to run a cycle. Claude Code, Codex, Cursor, Copilot, and
+Devin wrap the shared engine. OpenClaw is a separate reference adaptation and has
+its own setup.
One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins)):
@@ -62,10 +81,62 @@ One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft
|---|---|---|
| **Claude Code** | [`plugins/claude-code`](https://github.com/microsoft/SkillOpt/tree/main/plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` â `/skillopt-sleep` |
| **Codex** | [`plugins/codex`](https://github.com/microsoft/SkillOpt/tree/main/plugins/codex) | `bash plugins/codex/install.sh` â `skillopt-sleep` skill |
+| **Cursor** | [`plugins/cursor`](https://github.com/microsoft/SkillOpt/tree/main/plugins/cursor) | `bash plugins/cursor/install.sh` â `/skillopt-sleep` |
| **Copilot** | [`plugins/copilot`](https://github.com/microsoft/SkillOpt/tree/main/plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server |
| **Devin** | [`plugins/devin`](https://github.com/microsoft/SkillOpt/tree/main/plugins/devin) | register `plugins/devin/mcp_server.py` as an MCP server |
| **OpenClaw** | [`plugins/openclaw`](https://github.com/microsoft/SkillOpt/tree/main/plugins/openclaw) | adapt the reference wrapper and paths for your installation |
+### Cursor
+
+Cursor transcript harvesting and model execution are independent. Use
+`--source cursor` to read
+`~/.cursor/projects//agent-transcripts/*/*.jsonl`; `--scope invoked`
+uses Cursor's recorded workspace path, with the sanitized storage directory as
+a fallback, while `--scope all` scans every Cursor workspace. Use
+`--cursor-home` for a different Cursor home. `--source auto` keeps its existing
+Codex-then-Claude precedence and does not select Cursor.
+
+`--backend cursor` requires an installed, authenticated `cursor-agent`. If it is
+not on `PATH`, select it with `--cursor-path`, `SKILLOPT_SLEEP_CURSOR_PATH`, or
+the `cursor_path` config key. Select a model with `--model` or
+`SKILLOPT_SLEEP_CURSOR_MODEL`. Point adoption at a project Cursor skill rather
+than at the plugin's workflow skill:
+
+```bash
+skillopt-sleep run --project "$(pwd)" \
+ --source cursor --backend cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --progress
+```
+
+The target skill is supplied to Cursor as prompt text; it is not invoked as a
+native skill. `--project` selects transcript scope, target files, state, and
+staging, but ordinary Cursor calls cannot inspect that project's files. The
+current backend therefore evaluates textual guidance rather than end-to-end
+repository, CLI, browser, or service workflows.
+
+Cursor tool-aware replay is temporarily disabled pending live Cursor
+permission-boundary validation. If a task contains a `tool_called` check, the
+Cursor backend exits nonzero before starting Agent mode and does not stage,
+adopt, or advance state. Use another backend for those tasks.
+
+The initial harvest window is 72 hours. Set `--lookback-hours N` explicitly when
+older sessions should be considered; `0` scans all history subject to the
+session limit. A stateful `run`, even with no mined tasks, advances the harvest
+checkpoint. Use `harvest` or `dry-run` to inspect counts first. A real-backend
+`dry-run` still incurs provider calls and spend, and session/task limits are not
+hard call, token, time, or monetary budgets.
+
+The managed scheduler records only the project, backend, time, and optional
+auto-adopt setting. It does not preserve Cursor source, home, CLI path, model, or
+target-skill flags. Before `skillopt-sleep schedule --backend cursor`, put
+`transcript_source`, `cursor_home`, `cursor_path`, `model`, and
+`target_skill_path` in `~/.skillopt-sleep/config.json`. The target may remain
+project-relative as `.cursor/skills/skillopt-sleep-learned/SKILL.md`. Use an
+absolute `cursor_path` and verify that the scheduled account is already
+authenticated, because cron and Task Scheduler may run with a minimal
+environment.
+
To use DeepSeek, vLLM, Ollama, or another Chat Completions server, see
**[OpenAI-compatible endpoints](openai-compatible-endpoints.md)**. That guide also
documents the separate HTTPS-only boundary for Azure managed-identity credentials.
diff --git a/docs/superpowers/SECURITY.md b/docs/superpowers/SECURITY.md
new file mode 100644
index 0000000..a946d72
--- /dev/null
+++ b/docs/superpowers/SECURITY.md
@@ -0,0 +1,82 @@
+# Security Considerations for Superpowers Adapter
+
+## Scope: trusted candidates only
+
+This adapter evaluates **trusted, locally-authored** candidate skills. It is
+explicitly **not** hardened against a hostile candidate, and must not be pointed
+at model-generated or third-party skills.
+
+There is no OS-level boundary. The evaluated agent gets `Bash`, `Read`, `Write`
+and `Edit`, and runs as the same OS user as the harness, so it can:
+
+- Execute arbitrary shell commands
+- Read/write any file that user can reach, including the harness's own evidence
+- Read the environment passed to the process
+- Make unrestricted network requests
+
+`--allowedTools` scopes which tools the agent may call. It is **not** an
+isolation boundary.
+
+## What the adapter does do
+
+1. **No host credential reuse by default.** The scenario `HOME` is empty; host
+ `~/.claude/credentials.json` and `settings.json` are never copied or
+ symlinked. Reuse is opt-in via `SKILLOPT_HOST_AUTH=1`, which warns.
+2. **Fail closed.** With neither `ANTHROPIC_API_KEY` nor `SKILLOPT_HOST_AUTH=1`,
+ the scenario errors (`NO_AUTH`) instead of running unauthenticated.
+3. **Scrubbed environment.** Only `HOME`, `PATH`, `TERM`, `LANG` and (if set)
+ `ANTHROPIC_API_KEY` are passed; the host environment is not inherited. `PATH`
+ is minimal by default (shim dir + `/usr/bin:/bin`); opt in to the host `PATH`
+ with `SKILLOPT_INHERIT_PATH=1`. Hygiene, not a boundary â a `Bash`-holding
+ agent can still call absolute paths.
+4. **Isolated project and HOME** per scenario, inside a temp workspace.
+5. **Execution evidence.** All of it is tamper-**evident**, not tamper-proof â
+ the agent can reach the shim, the nonce and the audit log, and can modify the
+ project tree the harness re-runs from. It is meaningful because the candidate
+ is trusted; it is not an adversarial oracle.
+ - `harness_test_passes` â the harness re-runs the protected test paths after
+ the agent exits, ignoring project pytest config and `conftest.py`, so agent
+ *output* alone cannot fake a pass. A pass requires at least one executed
+ passing test and no failures, errors or skips.
+ - `pytest_runs` â count of nonce-tagged invocations of the `pytest`/`python`
+ shims.
+ - `pytest_successes` / `pytest_failures` â completed shim invocations are
+ classified from JUnit results; exit code 0 without an executed passing test
+ (for example `pytest --help` or an all-skipped run) is inconclusive and is
+ counted as neither a success nor a failure.
+ - `pytest_after_edit` â the last shim invocation is newer than the newest
+ project `*.py`, so "fix, then claim done without re-running" fails.
+ - Protected scenario fixtures are hashed before the agent runs. A modified,
+ deleted or symlink-replaced fixture fails the scenario and is not executed
+ by the harness re-run.
+
+ â ď¸ The verification re-run **executes agent-modified project code on the
+ host**. `ANTHROPIC_API_KEY` is dropped from that re-run's environment, but
+ nothing else confines it.
+
+## Known Limitations
+
+- **API key exposure**: `ANTHROPIC_API_KEY`, if set, is visible to the agent
+ process. Use a scoped/disposable key.
+- **`SKILLOPT_HOST_AUTH=1` exposes host credentials** to the candidate.
+- **`SKILLOPT_UNSAFE=1`** disables permission checks entirely.
+- **No network isolation**, for either the agent or the verification re-run.
+
+## Usage
+
+```bash
+ANTHROPIC_API_KEY=... python -m skillopt_sleep.adapters.superpowers --candidate my_skill.md
+```
+
+## Follow-up Work
+
+Supporting untrusted candidates is deliberately out of scope for this adapter as
+shipped. It would need, at minimum:
+
+- [ ] Verification oracle and evidence (nonce, shim, audit log) held outside
+ every agent-writable mount
+- [ ] Harness re-run from an immutable copy of the test inputs
+- [ ] A validated OS-level sandbox: published image with Claude Code + pytest,
+ exercised in CI, fail-closed on an unrecognised mode
+- [ ] Network egress allowlist (api.anthropic.com only)
+- [ ] Per-run scoped API keys
diff --git a/plugins/README.md b/plugins/README.md
index 5123022..61d222c 100644
--- a/plugins/README.md
+++ b/plugins/README.md
@@ -10,13 +10,14 @@ runtime dependency on the paper's `skillopt/` experiment package.
## Available integrations
-Four integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate
+Five integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate
reference adaptation with its own backend and setup assumptions.
| Platform | Folder | Mechanism | Status |
|---|---|---|---|
| **Claude Code** | [`claude-code/`](claude-code) | marketplace plugin, commands, skill, and hooks | installable shared-engine integration |
| **Codex** | [`codex/`](codex) | user-level skill and shared runner | installable shared-engine integration |
+| **Cursor** | [`cursor/`](cursor) | native command and skill, project skill target, and shared runner | installable shared-engine integration |
| **GitHub Copilot** | [`copilot/`](copilot) | MCP server exposing seven `sleep_*` tools | shared-engine MCP integration |
| **Devin** | [`devin/`](devin) | MCP server plus Devin transcript conversion | shared-engine MCP integration |
| **OpenClaw** | [`openclaw/`](openclaw) | custom DeepSeek/Ollama wrapper | independent reference adaptation; review and adapt before use |
@@ -30,6 +31,7 @@ for your workflow.
|---|---|---|
| **Claude Code** | from the repository root, `/plugin marketplace add ./plugins/claude-code`, then `/plugin install skillopt-sleep@skillopt-sleep` | `/skillopt-sleep status` |
| **Codex** | `bash plugins/codex/install.sh` | ask Codex to use the `skillopt-sleep` skill |
+| **Cursor** | `bash plugins/cursor/install.sh` (macOS/Linux) or `powershell -File plugins/cursor/install.ps1` (Windows) | `/skillopt-sleep status` |
| **Copilot** | register `plugins/copilot/mcp_server.py` using its example MCP config | ask Copilot to run `sleep_status` |
| **Devin** | register `plugins/devin/mcp_server.py` using its example MCP config | ask Devin to run `sleep_status` |
| **OpenClaw** | follow and adapt [`openclaw/README.md`](openclaw/README.md) | validate paths, credentials, and tasks locally |
@@ -44,9 +46,9 @@ an importable `skillopt_sleep` module. Install with `uv tool install skillopt` o
`pip install skillopt` when using that fallback.
> **Version note.** This integration reference tracks `main`. PyPI 0.2.0
-> supports the base Sleep CLI, while handoff, Sleep support for non-Azure
-> OpenAI-compatible endpoints, and `--preferences` require a source checkout
-> from `main` until the next release.
+> supports the base Sleep CLI, while Cursor source/backend/plugin support,
+> handoff, Sleep support for non-Azure OpenAI-compatible endpoints, and
+> `--preferences` require a source checkout from `main` until the next release.
## One sleep cycle
@@ -66,6 +68,17 @@ optimization.
data path and no API spend.
- A real backend sends truncated transcript excerpts and derived task content to
the provider selected for mining, replay, judging, and reflection.
+- The Cursor source reads local user/assistant message text, explicit turn
+ errors, and tool names from `~/.cursor/projects/*/agent-transcripts`; it does
+ not retain tool arguments, tool outputs, or other record types. Known
+ secret-shaped strings are redacted, but this is defense in depth rather than
+ a guarantee that outbound prompts are secret-free.
+- The Cursor backend sends prompts through the installed, authenticated
+ `cursor-agent` CLI. Ordinary calls use read-only Ask mode in a new empty
+ temporary workspace with project file access denied. Cursor tasks containing
+ `tool_called` validation fail before Agent mode starts; use another backend for
+ those tasks. Cursor and the model provider selected by Cursor can receive the
+ resulting prompt content.
- Outbound prompts are not currently guaranteed to be free of secrets. Do not
use a third-party provider on sensitive transcripts without reviewing the data
source and the provider's retention policy.
@@ -101,9 +114,11 @@ Common implemented flags include:
| Flag | Default | Purpose |
|---|---|---|
-| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls |
+| `--backend mock\|claude\|codex\|cursor\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls |
| `--model NAME` | backend default | select a backend-specific model |
-| `--source claude\|codex\|auto` | `claude` | select the transcript source |
+| `--source claude\|codex\|cursor\|auto` | `claude` | select the transcript source; `auto` retains Codex-then-Claude precedence and does not select Cursor |
+| `--cursor-home PATH` | `~/.cursor` | override the Cursor transcript home |
+| `--cursor-path PATH` | auto-detect `cursor-agent` | select the Cursor Agent CLI executable |
| `--project PATH` | current directory | select the project and invoked harvest scope |
| `--scope invoked\|all` | `invoked` | limit transcript harvesting |
| `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt |
@@ -119,6 +134,14 @@ The nightly CLI does **not** currently expose `--gate`, `--rollouts-k`,
`--optimizer-model`, `--target-model`, `--budget-tokens`, or `--budget-minutes`.
Do not pass experiment-harness flags to the main CLI.
+For the Cursor backend, `--project` also selects target files, state, and the
+staging location, but it does not make that directory the Cursor Agent execution
+workspace. The target skill is inserted as prompt text rather than invoked as a
+native skill. Real-backend `dry-run` performs the same mining and replay model
+calls while suppressing staging, adoption, and persisted state changes. The
+current Sleep cycle does not implement fresh-worktree replay; a `replay: mock`
+report label describes prompt replay and is independent of `--backend mock`.
+
### Preferences
`--preferences` is the main user-facing steering knob:
@@ -130,6 +153,26 @@ python -m skillopt_sleep run --backend codex --project "$(pwd)" \
Preferences guide reflection but remain subject to the validation gate.
+### Cursor source and backend
+
+Cursor transcript harvesting is explicit: use `--source cursor` rather than
+`--source auto`. Invoked-project scope uses Cursor's recorded workspace path,
+with the sanitized storage directory as a fallback; `--scope all` scans every
+Cursor workspace under `~/.cursor/projects`. The model-driven backend requires
+an installed, authenticated `cursor-agent`; use `--cursor-path`,
+`SKILLOPT_SLEEP_CURSOR_PATH`, or the `cursor_path` config key when it is not on
+`PATH`, and use `--model` or `SKILLOPT_SLEEP_CURSOR_MODEL` to choose a model.
+
+Target the project skill explicitly so accepted learning becomes visible to
+Cursor without changing the plugin's own workflow skill:
+
+```bash
+python -m skillopt_sleep run --project "$(pwd)" \
+ --source cursor --backend cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --progress
+```
+
### Advanced config
The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys,
@@ -138,6 +181,15 @@ including `gate_mode`, `gate_metric`, `dream_rollouts`, `dream_factor`, `recall_
unsupported CLI flags listed above. Shipping defaults are conservative:
`gate_mode="on"`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`.
+The managed `schedule` command stores only the project, backend, time, and
+optional auto-adopt setting. It does not copy `--source`, `--cursor-home`,
+`--cursor-path`, `--model`, or `--target-skill-path` into the scheduled command.
+For a Cursor schedule, set `transcript_source`, `cursor_home`, `cursor_path`,
+`model`, and `target_skill_path` in `~/.skillopt-sleep/config.json` first. Keep
+the target project-relative, use an absolute CLI path because cron and Task
+Scheduler may have a minimal `PATH`, and confirm that `cursor-agent` is
+authenticated for the account that runs the job.
+
### Handoff backend
`--backend handoff` keeps model subprocesses out of the engine. It writes pending
diff --git a/plugins/cursor/.cursor-plugin/plugin.json b/plugins/cursor/.cursor-plugin/plugin.json
new file mode 100644
index 0000000..97252bb
--- /dev/null
+++ b/plugins/cursor/.cursor-plugin/plugin.json
@@ -0,0 +1,30 @@
+{
+ "name": "skillopt-sleep",
+ "displayName": "SkillOpt-Sleep",
+ "version": "0.1.0",
+ "description": "Review recent Cursor sessions, replay recurring work, and stage validation-gated improvements to a Cursor skill for explicit adoption.",
+ "author": {
+ "name": "Yifan Yang",
+ "email": "yifanyang@microsoft.com"
+ },
+ "homepage": "https://github.com/microsoft/SkillOpt",
+ "repository": "https://github.com/microsoft/SkillOpt",
+ "license": "MIT",
+ "keywords": [
+ "skillopt",
+ "cursor",
+ "self-improvement",
+ "memory-consolidation",
+ "sleep",
+ "skills",
+ "offline-optimization"
+ ],
+ "category": "developer-tools",
+ "tags": [
+ "automation",
+ "memory",
+ "transcripts"
+ ],
+ "commands": "./commands/",
+ "skills": "./skills/"
+}
diff --git a/plugins/cursor/LICENSE b/plugins/cursor/LICENSE
new file mode 100644
index 0000000..cf7bcb2
--- /dev/null
+++ b/plugins/cursor/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/plugins/cursor/README.md b/plugins/cursor/README.md
new file mode 100644
index 0000000..3310a63
--- /dev/null
+++ b/plugins/cursor/README.md
@@ -0,0 +1,249 @@
+# SkillOpt-Sleep - Cursor integration
+
+Give Cursor an on-demand or explicitly scheduled sleep cycle: review recent
+local Cursor sessions, replay recurring tasks through a selected backend, and
+stage validation-gated improvements to a project Cursor skill. Nothing runs at
+session end, and nothing live changes until the user adopts an accepted staged
+proposal (unless they explicitly request `--auto-adopt`).
+
+This package is a native Cursor plugin containing a command and an agent skill.
+It does not install hooks or an MCP server.
+
+## Requirements
+
+- Cursor with plugin and agent-skill support.
+- Python 3.10 or newer.
+- Either a SkillOpt source checkout or an installed `skillopt-sleep` command.
+- For `--backend cursor`, an installed and authenticated Cursor Agent CLI
+ (`cursor-agent`). The default `mock` backend needs no provider login or spend.
+
+The plugin and transcript harvester work on native Windows. Cursor documents
+the Agent CLI for Windows through WSL; run provider-backed `--backend cursor`
+inside WSL unless a native `cursor-agent` is available in your environment.
+
+## Install the local plugin
+
+Clone the repository, then run the installer for your platform.
+
+macOS or Linux:
+
+```bash
+git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+bash plugins/cursor/install.sh
+export SKILLOPT_SLEEP_REPO="$(pwd)"
+```
+
+Windows PowerShell:
+
+```powershell
+git clone https://github.com/microsoft/SkillOpt.git
+Set-Location SkillOpt
+powershell -File plugins/cursor/install.ps1
+[System.Environment]::SetEnvironmentVariable("SKILLOPT_SLEEP_REPO", "$(pwd)", "User")
+```
+
+The installer copies the plugin to
+`~/.cursor/plugins/local/skillopt-sleep` (or
+`%USERPROFILE%\.cursor\plugins\local\skillopt-sleep`). Quit and reopen Cursor
+after changing user environment variables, then confirm that **SkillOpt-Sleep**
+appears in Settings > Plugins under Installed.
+
+The plugin and engine have separate installation boundaries. The copied plugin
+teaches Cursor how to operate SkillOpt-Sleep; the engine still runs from the
+source checkout through `plugins/run-sleep.sh` / `plugins/run-sleep.ps1`, or
+from an installed command:
+
+```bash
+uv tool install skillopt
+# or: python -m pip install skillopt
+```
+
+Use a release that includes Cursor source/backend support when choosing the
+installed-command route. The source-checkout route uses the implementation in
+the checkout directly.
+
+## Use from Cursor
+
+Run the native command, for example:
+
+```text
+/skillopt-sleep status
+/skillopt-sleep dry-run --backend mock --max-sessions 5 --max-tasks 3
+/skillopt-sleep run --backend cursor --max-sessions 5 --max-tasks 3 --progress
+/skillopt-sleep adopt
+```
+
+The `skillopt-sleep` agent skill remains independently available if a Cursor
+version does not surface plugin commands.
+
+The native command's default Cursor-visible target is:
+
+```text
+.cursor/skills/skillopt-sleep-learned/SKILL.md
+```
+
+Use `--target-skill-path` with that value on harvest, dry-run, and run commands.
+Without an explicit target, the shared engine defaults to a Claude-managed
+skill, which Cursor does not load as a project skill.
+
+### Source-checkout commands
+
+macOS or Linux:
+
+```bash
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)"
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run \
+ --project "$(pwd)" --source cursor --backend mock \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md
+bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run \
+ --project "$(pwd)" --source cursor --backend cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --progress
+```
+
+Windows PowerShell:
+
+```powershell
+powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" status --project "$(pwd)"
+powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" dry-run `
+ --project "$(pwd)" --source cursor --backend mock `
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md
+powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" run `
+ --project "$(pwd)" --source cursor --backend cursor `
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md `
+ --max-sessions 5 --max-tasks 3 --progress
+```
+
+### Installed-command equivalents
+
+```bash
+skillopt-sleep status --project "$(pwd)"
+skillopt-sleep dry-run --project "$(pwd)" --source cursor --backend mock \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md
+skillopt-sleep run --project "$(pwd)" --source cursor --backend cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --progress
+skillopt-sleep adopt --project "$(pwd)"
+```
+
+`--source cursor` reads local JSONL transcripts below
+`~/.cursor/projects//agent-transcripts/`. Use
+`--cursor-home /path/to/.cursor` for a different Cursor home. Invoked scope
+selects the current workspace; `--scope all` includes every Cursor workspace.
+The source converter retains user/assistant text, tool names, and explicit turn
+errors, but excludes raw tool arguments and outputs.
+
+The first harvest uses a 72-hour lookback by default. Use
+`--lookback-hours N` to choose a wider initial window, or
+`--lookback-hours 0` to consider all available history while still respecting
+`--max-sessions`. A successful `run`, including one that mines no tasks,
+records a new harvest checkpoint. Later runs use that checkpoint instead of
+the initial lookback. Use `harvest` or `dry-run` to inspect session and task
+counts before the first stateful run; neither action advances the checkpoint.
+
+`--backend cursor` invokes the authenticated Cursor Agent CLI. Use
+`--cursor-path /path/to/cursor-agent` or `SKILLOPT_SLEEP_CURSOR_PATH` if it is
+not on PATH, and `--model` or `SKILLOPT_SLEEP_CURSOR_MODEL` to override its
+model. Check available identifiers with `cursor-agent --list-models` and verify
+the billed variant in Cursor's usage reporting when cost matters. The child
+process receives only an explicit runtime/authentication/locale/proxy/CA
+environment allowlist; unrelated cloud and model-provider credentials are not
+forwarded.
+
+## What Cursor replay evaluates
+
+SkillOpt reads the target skill and inserts its text into mined task prompts. It
+does not invoke that file as a native Cursor skill or execute commands described
+by the skill. All ordinary Cursor model calls, including mining, replay,
+judging, and reflection, run in a new empty temporary workspace in read-only Ask
+mode. File reads, file writes, and MCP tools are denied. `--project` selects the
+transcript scope, target files, state, and staging location; it does not make the
+project the Cursor Agent execution workspace.
+
+Cursor tool-aware replay is temporarily disabled pending live Cursor
+permission-boundary validation. Tasks containing a `tool_called` check fail
+nonzero before Agent mode starts. The failed replay does not add a cache entry,
+stage, adopt, persist state, or advance the harvest checkpoint. Use another
+backend for those tasks.
+
+This replay is useful for textual procedures, response conventions, and output
+formats. It is not an end-to-end evaluation of skills that depend on repository
+inspection, real CLIs, browsers, running services, or filesystem changes. There
+is currently no Cursor option that enables a fresh project worktree or real
+project tools. The `replay: mock` report label refers to the prompt-replay mode,
+not to the selected model backend.
+
+The shared engine also supports `mock`, `claude`, `codex`, `copilot`,
+`handoff`, and `azure_openai` backends. Cursor is the native model-driven
+choice for this integration; `mock` remains the no-provider default.
+
+## Review sensitive data before provider calls
+
+Harvesting is local and read-only, and `--backend mock` makes no provider calls.
+Known secret-shaped strings are redacted from harvested Cursor content, and raw
+tool payloads are excluded, but pattern-based redaction is not a guarantee.
+A real backend sends truncated transcript excerpts and derived tasks to that
+backend's provider for mining, replay, judging, and reflection.
+
+Both `run` and `dry-run` perform those real-backend calls; `dry-run` prevents
+staging but does not prevent provider spend. `--max-sessions` and `--max-tasks`
+bound harvested work, not provider calls, tokens, elapsed time, or money. One
+task can require several attempt, judge, and reflection calls. Start with small
+limits and review the provider's usage reporting before increasing them.
+
+For sensitive work, split the flow at the review boundary:
+
+```bash
+skillopt-sleep harvest --project "$(pwd)" --source cursor \
+ --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \
+ --max-sessions 5 --max-tasks 3 --output reviewed-tasks.json
+
+skillopt-sleep dry-run --project "$(pwd)" --backend cursor \
+ --tasks-file reviewed-tasks.json --progress --json
+```
+
+Inspect and redact the JSON, then set its top-level `"reviewed"` field to
+`true`. Real backends reject task files that remain unreviewed. Keep raw
+transcripts, credentials, and task files out of commits.
+
+## Scheduling
+
+Runs remain user-triggered unless the user explicitly schedules them. Before
+scheduling, put the Cursor source and target in
+`~/.skillopt-sleep/config.json`, because the scheduler persists the project,
+backend, time, and optional auto-adopt flag, but not command-line source or
+target overrides:
+
+```json
+{
+ "transcript_source": "cursor",
+ "target_skill_path": ".cursor/skills/skillopt-sleep-learned/SKILL.md",
+ "cursor_home": "/absolute/path/to/.cursor",
+ "backend": "cursor"
+}
+```
+
+Then schedule or remove the managed entry:
+
+```bash
+skillopt-sleep schedule --project "$(pwd)" --backend cursor --hour 3 --minute 17
+skillopt-sleep unschedule --project "$(pwd)"
+```
+
+On Unix this uses cron; on Windows it uses Task Scheduler. Scheduled runs stage
+proposals for later review by default. Do not add `--auto-adopt` unless the user
+has explicitly chosen unattended adoption.
+
+## Adoption and memory
+
+`run` stages accepted proposals under
+`/.skillopt-sleep/staging//`. Read the staged `report.md`
+and show the held-out baseline-to-candidate score plus exact edits before
+running `adopt`. Adoption backs up an existing target before replacing it.
+
+The shared engine may also propose project `CLAUDE.md` memory updates; existing
+memory behavior is unchanged. To restrict a Cursor setup to the explicit Cursor
+skill, set `"evolve_memory": false` in `~/.skillopt-sleep/config.json`.
+
+There is deliberately no session-end hook or automatic plugin execution.
diff --git a/plugins/cursor/commands/skillopt-sleep.md b/plugins/cursor/commands/skillopt-sleep.md
new file mode 100644
index 0000000..a62020f
--- /dev/null
+++ b/plugins/cursor/commands/skillopt-sleep.md
@@ -0,0 +1,19 @@
+# SkillOpt-Sleep
+
+Use the bundled `skillopt-sleep` skill to run or manage SkillOpt-Sleep for the
+current Cursor workspace.
+
+Requested action: `$ARGUMENTS`
+
+If no action was supplied, use `status`. Preserve all options supplied after
+the action. For `harvest`, `dry-run`, and `run`, ensure the engine receives:
+
+```text
+--project --scope invoked --source cursor --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md
+```
+
+Do not add `--backend cursor` unless the user requested provider-backed replay;
+the repository default remains the no-provider `mock` backend. Follow the
+skill's runner selection, review, data-boundary, scheduling, and adoption rules.
+Never edit the learned skill or `CLAUDE.md` directly as a substitute for the
+engine's staged adoption flow.
diff --git a/plugins/cursor/install.ps1 b/plugins/cursor/install.ps1
new file mode 100644
index 0000000..dd7b232
--- /dev/null
+++ b/plugins/cursor/install.ps1
@@ -0,0 +1,39 @@
+# Install the SkillOpt-Sleep Cursor integration as a local Cursor plugin on Windows.
+# Idempotent; prints what it does.
+
+$ErrorActionPreference = "Stop"
+
+$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
+$CursorHome = if ($env:CURSOR_HOME) { $env:CURSOR_HOME } else { Join-Path $env:USERPROFILE ".cursor" }
+$PluginDir = Join-Path $CursorHome "plugins\local\skillopt-sleep"
+$SourceDir = Join-Path $RepoRoot "plugins\cursor"
+$ManifestDir = Join-Path $PluginDir ".cursor-plugin"
+$CommandDir = Join-Path $PluginDir "commands"
+$SkillDir = Join-Path $PluginDir "skills\skillopt-sleep"
+
+Write-Output "[install] repo: $RepoRoot"
+
+New-Item -ItemType Directory -Path $ManifestDir -Force | Out-Null
+New-Item -ItemType Directory -Path $CommandDir -Force | Out-Null
+New-Item -ItemType Directory -Path $SkillDir -Force | Out-Null
+Copy-Item (Join-Path $SourceDir ".cursor-plugin\plugin.json") (Join-Path $ManifestDir "plugin.json") -Force
+Copy-Item (Join-Path $SourceDir "commands\skillopt-sleep.md") (Join-Path $CommandDir "skillopt-sleep.md") -Force
+Copy-Item (Join-Path $SourceDir "skills\skillopt-sleep\SKILL.md") (Join-Path $SkillDir "SKILL.md") -Force
+Copy-Item (Join-Path $SourceDir "README.md") (Join-Path $PluginDir "README.md") -Force
+Copy-Item (Join-Path $SourceDir "LICENSE") (Join-Path $PluginDir "LICENSE") -Force
+
+Write-Output "[install] plugin manifest -> $(Join-Path $ManifestDir 'plugin.json')"
+Write-Output "[install] command -> $(Join-Path $CommandDir 'skillopt-sleep.md')"
+Write-Output "[install] skill -> $(Join-Path $SkillDir 'SKILL.md')"
+Write-Output ""
+Write-Output "[install] Quit and reopen Cursor. The plugin should appear in Settings >"
+Write-Output "Plugins under Installed."
+Write-Output ""
+Write-Output "For source-checkout runs, add this user environment variable:"
+Write-Output " [System.Environment]::SetEnvironmentVariable('SKILLOPT_SLEEP_REPO', '$RepoRoot', 'User')"
+Write-Output ""
+Write-Output "Alternatively, install a SkillOpt release that includes Cursor support so the"
+Write-Output "skillopt-sleep command is on PATH."
+Write-Output ""
+Write-Output "Done. Try in Cursor:"
+Write-Output " /skillopt-sleep status"
diff --git a/plugins/cursor/install.sh b/plugins/cursor/install.sh
new file mode 100755
index 0000000..9b75d70
--- /dev/null
+++ b/plugins/cursor/install.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Install the SkillOpt-Sleep Cursor integration as a local Cursor plugin.
+# Idempotent; prints what it does.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+CURSOR_HOME="${CURSOR_HOME:-$HOME/.cursor}"
+PLUGIN_DIR="$CURSOR_HOME/plugins/local/skillopt-sleep"
+SOURCE_DIR="$REPO_ROOT/plugins/cursor"
+
+echo "[install] repo: $REPO_ROOT"
+
+mkdir -p "$PLUGIN_DIR/.cursor-plugin" "$PLUGIN_DIR/commands" "$PLUGIN_DIR/skills/skillopt-sleep"
+cp "$SOURCE_DIR/.cursor-plugin/plugin.json" "$PLUGIN_DIR/.cursor-plugin/plugin.json"
+cp "$SOURCE_DIR/commands/skillopt-sleep.md" "$PLUGIN_DIR/commands/skillopt-sleep.md"
+cp "$SOURCE_DIR/skills/skillopt-sleep/SKILL.md" "$PLUGIN_DIR/skills/skillopt-sleep/SKILL.md"
+cp "$SOURCE_DIR/README.md" "$PLUGIN_DIR/README.md"
+cp "$SOURCE_DIR/LICENSE" "$PLUGIN_DIR/LICENSE"
+
+echo "[install] plugin manifest -> $PLUGIN_DIR/.cursor-plugin/plugin.json"
+echo "[install] command -> $PLUGIN_DIR/commands/skillopt-sleep.md"
+echo "[install] skill -> $PLUGIN_DIR/skills/skillopt-sleep/SKILL.md"
+
+cat <
+Plugins under Installed.
+
+For source-checkout runs, add this to your shell profile:
+ export SKILLOPT_SLEEP_REPO="$REPO_ROOT"
+
+Alternatively, install a SkillOpt release that includes Cursor support so the
+\`skillopt-sleep\` command is on PATH.
+
+Done. Try in Cursor:
+ /skillopt-sleep status
+EOF
diff --git a/plugins/cursor/skills/skillopt-sleep/SKILL.md b/plugins/cursor/skills/skillopt-sleep/SKILL.md
new file mode 100644
index 0000000..7caa8fc
--- /dev/null
+++ b/plugins/cursor/skills/skillopt-sleep/SKILL.md
@@ -0,0 +1,218 @@
+---
+name: skillopt-sleep
+description: "Use when the user wants Cursor to learn from recent local sessions, asks for an offline sleep or dream cycle, wants to consolidate recurring work into a Cursor skill, or requests SkillOpt-Sleep status, harvest, dry-run, run, scheduling, review, or adoption. Drives the validation-gated skillopt_sleep engine with Cursor transcripts and the optional Cursor Agent CLI backend."
+---
+
+# SkillOpt-Sleep for Cursor
+
+SkillOpt-Sleep reviews recent local Cursor sessions, mines recurring tasks,
+replays those tasks, and proposes bounded improvements to a project Cursor
+skill. With the default gate enabled, a proposal is accepted only when it
+improves the held-out score. A normal run stages the proposal for review;
+nothing live changes until explicit adoption. There is no model-weight training.
+
+This plugin has no session-end hook and no MCP server. Run the cycle only when
+the user asks, or install a schedule only when the user explicitly requests one.
+
+## Cursor target
+
+Always use this project-relative target for Cursor-visible learning:
+
+```text
+.cursor/skills/skillopt-sleep-learned/SKILL.md
+```
+
+Pass it through `--target-skill-path` on `harvest`, `dry-run`, and `run`.
+Without an explicit target, the shared engine uses a Claude-managed skill under
+`~/.claude/skills`, which is not the intended Cursor project skill.
+
+The shared engine can also evolve project `CLAUDE.md`. If that secondary memory
+target is unwanted, set `"evolve_memory": false` in
+`~/.skillopt-sleep/config.json` before running.
+
+## Choose the runner
+
+Use one of these supported command paths consistently:
+
+1. Source checkout on macOS/Linux:
+ `bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" ...`
+2. Source checkout on Windows:
+ `powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" ...`
+3. Installed engine on any platform:
+ `skillopt-sleep ...`
+
+If `SKILLOPT_SLEEP_REPO` is not set and `skillopt-sleep` is unavailable, stop
+and explain that the engine must be installed or a SkillOpt checkout must be
+selected. Do not substitute a hand-written edit for the engine workflow.
+
+## Core workflow
+
+1. **Harvest** local Cursor JSONL transcripts read-only.
+2. **Mine** recurring, checkable task records from session digests.
+3. **Replay** tasks under the current skill and memory through the selected
+ backend.
+4. **Reflect** on failures and propose bounded edits.
+5. **Gate** the candidate on held-out real tasks.
+6. **Stage** accepted proposals under
+ `/.skillopt-sleep/staging//`.
+7. **Adopt** only after review, backing up existing live targets first.
+
+## Commands
+
+Use the installed-command form below, or replace `skillopt-sleep` with the
+platform-specific source runner described above.
+
+```bash
+TARGET_SKILL=.cursor/skills/skillopt-sleep-learned/SKILL.md
+
+# Inspect current state and the latest staged proposal.
+skillopt-sleep status --project "$(pwd)"
+
+# Inspect mined tasks without provider spend.
+skillopt-sleep harvest --project "$(pwd)" --source cursor \
+ --target-skill-path "$TARGET_SKILL" --max-sessions 5 --max-tasks 3
+
+# First smoke check: deterministic and no provider calls.
+skillopt-sleep dry-run --project "$(pwd)" --source cursor --backend mock \
+ --target-skill-path "$TARGET_SKILL" --max-sessions 5 --max-tasks 3 --json
+
+# Model-driven optimization through the authenticated Cursor Agent CLI.
+skillopt-sleep run --project "$(pwd)" --source cursor --backend cursor \
+ --target-skill-path "$TARGET_SKILL" \
+ --max-sessions 5 --max-tasks 3 --progress
+
+# Apply the latest accepted staged proposal after review.
+skillopt-sleep adopt --project "$(pwd)"
+```
+
+Actions are `status`, `harvest`, `dry-run`, `run`, `adopt`, `schedule`, and
+`unschedule`.
+
+- Default backend is `mock`, which is deterministic and makes no provider calls.
+- `--backend cursor` uses the user's authenticated Cursor Agent CLI budget for
+ model-driven mining, replay, judging, and reflection.
+- `--source cursor` reads
+ `~/.cursor/projects//agent-transcripts/*/*.jsonl`.
+- `--cursor-home PATH` overrides the Cursor home used for harvesting.
+- `--scope invoked` selects the current workspace; `--scope all` includes every
+ Cursor workspace.
+- `--cursor-path PATH` or `SKILLOPT_SLEEP_CURSOR_PATH` selects a non-default
+ `cursor-agent` executable.
+- `--model NAME` or `SKILLOPT_SLEEP_CURSOR_MODEL` overrides the Cursor model.
+- Check model identifiers with `cursor-agent --list-models`; when cost matters,
+ verify the billed variant in Cursor's usage reporting.
+- Keep live runs bounded with `--max-sessions`, `--max-tasks`, and `--progress`.
+- A held-out gain is evidence for that run, not a promise of general improvement.
+
+The first harvest uses a 72-hour lookback. Use `--lookback-hours N` for a wider
+initial window or `--lookback-hours 0` for all available history. A stateful
+`run`, including a no-task run, records a harvest checkpoint; later runs use the
+checkpoint rather than the initial lookback. Inspect counts with `harvest` or
+`dry-run` before the first real run because those actions do not advance state.
+
+Available backends are:
+
+- `mock` - deterministic, with no provider calls (default);
+- `cursor` - the authenticated Cursor Agent CLI;
+- `claude` - the authenticated Claude CLI;
+- `codex` - the authenticated Codex CLI;
+- `copilot` - the authenticated GitHub Copilot CLI;
+- `handoff` - prompt/answer files for an interactive agent session;
+- `azure_openai` - the configured Azure OpenAI endpoint.
+
+SkillOpt reads the target skill and inserts its text into replay prompts; it does
+not invoke the file as a native Cursor skill. Ordinary Cursor backend calls run
+in a new empty temporary workspace in read-only Ask mode. File reads, file
+writes, and MCP tools are denied. `--project` controls harvesting, target files,
+state, and staging; it is not the Cursor Agent execution workspace.
+
+Cursor tool-aware replay is temporarily disabled pending live Cursor
+permission-boundary validation. A task containing a `tool_called` check fails
+nonzero before Agent mode starts. The failed replay does not add a cache entry,
+stage, adopt, persist state, or advance the harvest checkpoint. Use another
+backend for those tasks. Do not claim that repository- or tool-dependent
+behavior was validated. The current engine does not implement a fresh-worktree
+replay for Cursor.
+
+A real-backend `dry-run` still makes provider calls; it only suppresses staging.
+Session and task limits are workload bounds, not hard limits on calls, tokens,
+time, or money. Start with small limits.
+
+## Reviewable data path
+
+Cursor harvesting retains user/assistant text, tool names, and explicit turn
+errors while excluding raw tool arguments, tool outputs, and non-message
+records. Known secret-shaped strings are redacted, but pattern-based redaction
+cannot guarantee that a transcript is safe to send to a provider.
+
+For sensitive sessions, export tasks before any real-backend replay:
+
+```bash
+TARGET_SKILL=.cursor/skills/skillopt-sleep-learned/SKILL.md
+skillopt-sleep harvest --project "$(pwd)" --source cursor \
+ --target-skill-path "$TARGET_SKILL" \
+ --max-sessions 5 --max-tasks 3 --output reviewed-tasks.json
+```
+
+Inspect and redact the file, then set its top-level `"reviewed"` field to
+`true`. Only then run:
+
+```bash
+skillopt-sleep dry-run --project "$(pwd)" --backend cursor \
+ --tasks-file reviewed-tasks.json --progress --json
+```
+
+Real backends reject task files that remain unreviewed. Never include raw
+transcripts, credentials, secrets, or sensitive task content in messages,
+commits, or generated summaries.
+
+## Scheduling
+
+Scheduling is opt-in. The scheduler persists project, backend, time, and the
+optional auto-adopt flag, but not `--source`, Cursor path/home/model overrides,
+or `--target-skill-path`. Before scheduling a Cursor cycle, set at least these values in
+`~/.skillopt-sleep/config.json`:
+
+```json
+{
+ "transcript_source": "cursor",
+ "target_skill_path": ".cursor/skills/skillopt-sleep-learned/SKILL.md",
+ "backend": "cursor"
+}
+```
+
+Then run:
+
+```bash
+skillopt-sleep schedule --project "$(pwd)" --backend cursor --hour 3 --minute 17
+skillopt-sleep unschedule --project "$(pwd)"
+```
+
+The scheduler uses cron on Unix and Task Scheduler on Windows. Scheduled runs
+stage proposals by default. Use `--auto-adopt` only when the user has explicitly
+requested unattended adoption.
+
+## Report results
+
+For `dry-run` and `run`, report:
+
+- session and task counts;
+- held-out baseline and candidate scores;
+- gate action and accepted/rejected edit counts;
+- exact proposed edits;
+- staging directory, when one was created.
+
+Read staged `report.md` before summarizing a run. Offer adoption only after the
+user reviews an accepted proposal that is still staged. Never claim broad
+improvement from one run.
+
+## Hard rules
+
+- Harvest is read-only. Never edit Cursor transcript files.
+- Never hand-edit the target skill or `CLAUDE.md` as a substitute for adoption.
+- Do not run a real backend on sensitive content without confirming its data
+ boundary or using the reviewed-task workflow.
+- Do not add a session-end hook or imply that installing this plugin schedules
+ anything.
+- Show validation evidence before recommending adoption.
+- Treat generated edits as proposals, not as source of truth.
diff --git a/plugins/devin/README.md b/plugins/devin/README.md
index 7b63056..53abe6e 100644
--- a/plugins/devin/README.md
+++ b/plugins/devin/README.md
@@ -16,7 +16,10 @@ source into the Claude Code-compatible JSONL the engine reads.
| `harvest_devin.py` | converts Devin ATIF-v1.7 transcripts + agentmemory + `.devin/skills` into JSONL, with `taskKey` + outcome envelopes |
| `judge.py` | reference judge for the deferred/judge branch of the validation gate |
| `mcp-config.example.json` | drop-in MCP server config |
-| `devin-rules.snippet.md` | paste into `.devin/rules/skillopt-sleep.md` |
+| `install.sh` | copies hooks + rules into a project's `.devin/` and prints the MCP registration command |
+| `devin-rules.snippet.md` | copied to `.devin/rules/skillopt-sleep.md` by `install.sh` |
+| `hooks/hooks.v1.json` | SessionEnd hook config â installed/merged at `.devin/hooks.v1.json` by `install.sh` |
+| `hooks/on-session-end.sh` | best-effort activity marker script (called by the hook) |
## What it harvests
@@ -33,10 +36,22 @@ After `sleep_adopt`, the evolved skill is synced to `.devin/skills/skillopt-slee
Requires Python ⼠3.10. No third-party packages â the server is pure stdlib.
-1. **Register the MCP server.** Use `mcp-config.example.json` as a template; set
- `args` to the absolute path of this `mcp_server.py`. The engine is found
- automatically (this plugin lives inside the SkillOpt repo). Or via the Devin
- CLI:
+1. **Install hooks + rules into your project.** From the repo root:
+
+ ```bash
+ bash plugins/devin/install.sh /path/to/your/project
+ ```
+
+ This copies the SessionEnd hook and rules snippet into the project's
+ `.devin/` directory and prints the MCP registration command. The hook is
+ on by default â it logs a cheap activity marker for local inspection or
+ external automation when each session ends. The current engine harvests by
+ transcript timestamps and does not consume this marker directly. The hook
+ is non-blocking and spends no API budget. Re-run the script to update; the
+ installer preserves existing hooks and does not duplicate its own entry.
+
+2. **Register the MCP server.** Use `mcp-config.example.json` as a template, or
+ run the command printed by `install.sh`:
```bash
devin mcp add skillopt-sleep \
@@ -44,9 +59,6 @@ Requires Python ⼠3.10. No third-party packages â the server is pure stdlib.
-- python3 /abs/path/to/SkillOpt/plugins/devin/mcp_server.py
```
-2. **(Optional)** copy `devin-rules.snippet.md` to `.devin/rules/skillopt-sleep.md`
- so Devin proactively offers the tools.
-
3. Ask Devin: *"run the sleep cycle"*, *"what did the last sleep propose?"*, *"adopt it"*.
## Tools
@@ -62,9 +74,12 @@ Requires Python ⼠3.10. No third-party packages â the server is pure stdlib.
| `sleep_unschedule` | remove the nightly cron entry |
Default backend is `mock` (no API spend); the `claude`, `codex`, and `copilot`
-backends use the corresponding authenticated CLI and budget. The seven tools
-call the same `python -m skillopt_sleep` actions as the other shared-engine
-integrations.
+backends use the corresponding authenticated CLI and budget. The `handoff`
+backend runs the cycle with no model subprocess or API key â the engine writes
+pending model calls to `.skillopt-sleep-handoff/PROMPTS.md` + `pending.json`
+(exit code 3) and resumes after answers are placed in `answers/.md`; re-run
+`sleep_run` with the same arguments to resume. The seven tools call the same
+`python -m skillopt_sleep` actions as the other shared-engine integrations.
## Data boundary
diff --git a/plugins/devin/devin-rules.snippet.md b/plugins/devin/devin-rules.snippet.md
index 2e2c9af..4bd1ad8 100644
--- a/plugins/devin/devin-rules.snippet.md
+++ b/plugins/devin/devin-rules.snippet.md
@@ -24,7 +24,10 @@ Always pass the absolute Devin workspace as `project`, especially for
`sleep_adopt`. Default backend is `mock` (no provider calls). The `claude`,
`codex`, and `copilot` backend values use the corresponding installed and
authenticated CLI; they do not require this plugin to implement a separate
-API-key flow.
+API-key flow. The `handoff` backend runs the cycle with no model subprocess
+or API key â the engine writes pending model calls to
+`.skillopt-sleep-handoff/` and exits; answer each prompt in a fresh context
+and re-run `sleep_run` to resume (typically 3â6 rounds).
The Devin conversion and mock workflow stay local. A real backend sends
truncated transcript excerpts and derived tasks to the selected provider for
diff --git a/plugins/devin/hooks/hooks.v1.json b/plugins/devin/hooks/hooks.v1.json
new file mode 100644
index 0000000..6af99d1
--- /dev/null
+++ b/plugins/devin/hooks/hooks.v1.json
@@ -0,0 +1,14 @@
+{
+ "SessionEnd": [
+ {
+ "matcher": "",
+ "hooks": [
+ {
+ "type": "command",
+ "command": "\"${DEVIN_PROJECT_DIR}/.devin/hooks/skillopt-sleep-on-session-end.sh\"",
+ "timeout": 5
+ }
+ ]
+ }
+ ]
+}
diff --git a/plugins/devin/hooks/on-session-end.sh b/plugins/devin/hooks/on-session-end.sh
new file mode 100755
index 0000000..dd0b0ec
--- /dev/null
+++ b/plugins/devin/hooks/on-session-end.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# SkillOpt-Sleep SessionEnd hook for Devin (best-effort, NON-BLOCKING).
+#
+# This does NOT run the optimizer. It only appends a tiny marker for local
+# inspection or external automation. The current sleep engine uses transcript
+# timestamps rather than this marker. The hook must never fail the session or
+# spend API budget.
+#
+# Install this script as .devin/hooks/skillopt-sleep-on-session-end.sh and the
+# config at .devin/hooks.v1.json. Devin CLI reads it automatically.
+set -uo pipefail
+
+[ -n "${HOME:-}" ] || exit 0
+STATE_DIR="${HOME}/.skillopt-sleep"
+mkdir -p "$STATE_DIR" 2>/dev/null || exit 0
+
+# Record that a session just ended (cheap local activity signal).
+printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${DEVIN_PROJECT_DIR:-${PWD}}" \
+ >> "$STATE_DIR/session-end.log" 2>/dev/null || true
+
+exit 0
diff --git a/plugins/devin/install.sh b/plugins/devin/install.sh
new file mode 100755
index 0000000..3271848
--- /dev/null
+++ b/plugins/devin/install.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Install the SkillOpt-Sleep Devin integration into a project.
+# Copies the SessionEnd hook and rules snippet into .devin/, and prints
+# the MCP server registration command. Idempotent.
+set -euo pipefail
+
+PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$PLUGIN_DIR/../.." && pwd)"
+PROJECT="${1:-$(pwd)}"
+
+echo "[install] repo: $REPO_ROOT"
+echo "[install] project: $PROJECT"
+
+DEVIN_DIR="$PROJECT/.devin"
+mkdir -p "$DEVIN_DIR/hooks" "$DEVIN_DIR/rules"
+
+# 1) SessionEnd hook (on by default â provides activity signal for nightly harvest)
+# Merge into existing hooks.v1.json instead of overwriting, so we don't
+# destroy other project hooks.
+HOOK_SCRIPT_SRC="$PLUGIN_DIR/hooks/on-session-end.sh"
+HOOK_SCRIPT_DST="$DEVIN_DIR/hooks/skillopt-sleep-on-session-end.sh"
+cp "$HOOK_SCRIPT_SRC" "$HOOK_SCRIPT_DST"
+chmod +x "$HOOK_SCRIPT_DST"
+echo "[install] hook script -> $HOOK_SCRIPT_DST"
+
+HOOK_CONFIG="$DEVIN_DIR/hooks.v1.json"
+if [ -f "$HOOK_CONFIG" ]; then
+ # Python is already required by the plugin. Merge event arrays without
+ # replacing existing hooks, and skip exact duplicates on repeated installs.
+ python3 - "$HOOK_CONFIG" "$PLUGIN_DIR/hooks/hooks.v1.json" <<'PY'
+import json
+import os
+import stat
+import sys
+import tempfile
+
+destination, addition = sys.argv[1:]
+
+
+def load_object(path):
+ with open(path, encoding="utf-8") as handle:
+ value = json.load(handle)
+ if not isinstance(value, dict):
+ raise ValueError(f"hook config must be a JSON object: {path}")
+ return value
+
+
+base = load_object(destination)
+incoming = load_object(addition)
+for event, entries in incoming.items():
+ if not isinstance(entries, list):
+ raise ValueError(f"hook event {event!r} must be an array")
+ existing = base.setdefault(event, [])
+ if not isinstance(existing, list):
+ raise ValueError(f"existing hook event {event!r} must be an array")
+ for entry in entries:
+ if entry not in existing:
+ existing.append(entry)
+
+directory = os.path.dirname(os.path.abspath(destination))
+fd, temporary = tempfile.mkstemp(prefix=".hooks.v1.", suffix=".tmp", dir=directory)
+try:
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
+ json.dump(base, handle, ensure_ascii=False, indent=2)
+ handle.write("\n")
+ os.chmod(temporary, stat.S_IMODE(os.stat(destination).st_mode))
+ os.replace(temporary, destination)
+except Exception:
+ try:
+ os.unlink(temporary)
+ except OSError:
+ pass
+ raise
+PY
+ echo "[install] session-end hook -> $HOOK_CONFIG (merged)"
+else
+ cp "$PLUGIN_DIR/hooks/hooks.v1.json" "$HOOK_CONFIG"
+ echo "[install] session-end hook -> $HOOK_CONFIG"
+fi
+
+# 2) Rules snippet so Devin proactively offers the tools
+cp "$PLUGIN_DIR/devin-rules.snippet.md" "$DEVIN_DIR/rules/skillopt-sleep.md"
+echo "[install] rules snippet -> $DEVIN_DIR/rules/skillopt-sleep.md"
+
+# 3) Print the MCP server registration command
+printf -v MCP_SERVER_QUOTED '%q' "$PLUGIN_DIR/mcp_server.py"
+cat < argparse.Namespace:
# Legacy flat overrides
p.add_argument("--env", type=str)
p.add_argument("--backend", type=str,
- choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "minimax", "minimax_chat"])
+ choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "minimax", "minimax_chat"])
p.add_argument("--optimizer_model", type=str)
p.add_argument("--target_model", type=str)
p.add_argument("--optimizer_backend", type=str)
@@ -181,6 +182,8 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--claude_code_exec_use_sdk", type=str)
p.add_argument("--claude_code_exec_effort", type=str)
p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
+ p.add_argument("--cursor_exec_path", type=str)
+ p.add_argument("--cursor_exec_sandbox", type=str)
p.add_argument("--minimax_base_url", type=str)
p.add_argument("--minimax_api_key", type=str)
p.add_argument("--minimax_model", type=str)
@@ -262,6 +265,8 @@ def main() -> None:
"claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
"claude_code_exec_effort": "model.claude_code_exec_effort",
"claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
+ "cursor_exec_path": "model.cursor_exec_path",
+ "cursor_exec_sandbox": "model.cursor_exec_sandbox",
"minimax_base_url": "model.minimax_base_url",
"minimax_api_key": "model.minimax_api_key",
"minimax_model": "model.minimax_model",
@@ -327,6 +332,11 @@ def main() -> None:
elif backend == "claude_code_exec":
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "claude_code_exec")
+ elif backend == "cursor_exec":
+ if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
+ cfg["optimizer_backend"] = "openai_chat"
+ if not _has_model_override("model.target_backend", "target_backend"):
+ cfg["target_backend"] = "cursor_exec"
elif backend in {"minimax", "minimax_chat"}:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "minimax_chat")
@@ -355,6 +365,12 @@ def main() -> None:
and not _has_model_override("model.target", "target_model")
):
cfg["target_model"] = default_model_for_backend("claude_chat")
+ if cfg.get("target_backend") == "cursor_exec":
+ if (
+ str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+ and not _has_model_override("model.target", "target_model")
+ ):
+ cfg["target_model"] = default_model_for_backend("cursor_exec")
if cfg.get("target_backend") == "minimax_chat":
if (
str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
@@ -429,6 +445,10 @@ def main() -> None:
effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
)
+ configure_cursor_exec(
+ path=cfg.get("cursor_exec_path") or None,
+ sandbox=cfg.get("cursor_exec_sandbox") or None,
+ )
configure_qwen_chat(
base_url=cfg.get("qwen_chat_base_url") or None,
api_key=cfg.get("qwen_chat_api_key") or None,
diff --git a/scripts/smoke_superpowers.sh b/scripts/smoke_superpowers.sh
new file mode 100755
index 0000000..1f38014
--- /dev/null
+++ b/scripts/smoke_superpowers.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+# Smoke test for Superpowers adapter integration.
+# Run this manually (not in CI) to verify the adapter works with the real harness.
+#
+# Prerequisites:
+# - Claude Code installed, ANTHROPIC_API_KEY set (or SKILLOPT_HOST_AUTH=1 for a
+# trusted candidate on your own machine)
+# - Same model/settings/pinned SHA for baseline and candidate runs
+#
+# Usage:
+# ./scripts/smoke_superpowers.sh [candidate_skill_path]
+#
+# Output goes to smoke_results/ (gitignored). Results embed raw agent output and
+# local paths: do NOT commit them. Sanitized excerpts are written alongside each
+# run for pasting into a PR description or attaching as an artifact.
+
+set -euo pipefail
+
+SKILL="${1:-}"
+SCENARIO="${SKILLOPT_SCENARIO:-test-passes-verify}"
+SHA="${SKILLOPT_SHA:-d884ae04edebef577e82ff7c4e143debd0bbec99}"
+OUTDIR="smoke_results/$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$OUTDIR"
+
+echo "Smoke test: Superpowers adapter"
+echo "Output: $OUTDIR (gitignored - do not commit)"
+echo "Scenario: $SCENARIO"
+echo "SHA: $SHA"
+echo ""
+
+run_scenario() {
+ local name="$1"
+ local candidate="${2:-}"
+ local outfile="$OUTDIR/${name}.json"
+
+ echo "=== $name ==="
+
+ local args=(
+ --skill verification-before-completion
+ --scenario "$SCENARIO"
+ --sha "$SHA"
+ --json
+ )
+ if [[ -n "$candidate" ]]; then
+ args+=(--candidate "$candidate")
+ fi
+
+ # No || true - fail if runner errors
+ python -m skillopt_sleep.adapters.superpowers "${args[@]}" > "$outfile"
+
+ # Best-effort sanitized summary for sharing: home dir, temp workspace paths
+ # and api-key-shaped tokens are redacted. Skim before sharing - it is a
+ # heuristic scrub, not a guarantee.
+ python - "$outfile" "$OUTDIR/${name}.summary.txt" <<'PY'
+import json, os, re, sys
+data = json.load(open(sys.argv[1]))
+home = os.path.expanduser("~")
+def clean(t):
+ t = t.replace(home, "~")
+ t = re.sub(r"/tmp/\S*skillopt\S*", "", t)
+ t = re.sub(r"/(?:tmp|var)/\S*", "", t)
+ return re.sub(r"sk-[A-Za-z0-9_\-]{8,}", "sk-REDACTED", t)
+lines = [
+ f"skill={data['skill']} version={data['version']} pinned_sha={data['pinned_sha']}",
+ f"candidate_hash={data['candidate_hash'] or '(baseline)'}",
+ f"score={data['score']:.2f} passed={data['passed']} failed={data['failed']}",
+]
+for s in data["scenarios"]:
+ lines.append(f"\n[{s['id']}] passed={s['passed']} error={s.get('error') or 'none'}")
+ lines.append(f" evidence: {json.dumps(s.get('evidence', {}))}")
+ for c in s["checks"]:
+ lines.append(f" {'PASS' if c['passed'] else 'FAIL'} {c['description']}")
+ lines.append(" output excerpt:")
+ for ln in clean(s.get("output", ""))[:800].splitlines():
+ lines.append(f" {ln}")
+text = "\n".join(lines)
+open(sys.argv[2], "w").write(text + "\n")
+print(text)
+PY
+}
+
+run_scenario "baseline"
+
+if [[ -n "$SKILL" ]]; then
+ run_scenario "candidate" "$SKILL"
+fi
+
+echo ""
+echo "Results in $OUTDIR (gitignored)."
+echo "Share the *.summary.txt excerpts in the PR; do not commit the raw JSON."
diff --git a/scripts/train.py b/scripts/train.py
index 918e684..8b25377 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -137,7 +137,7 @@ def parse_args() -> argparse.Namespace:
# Legacy flat CLI overrides (still work, prefer --cfg-options for new usage)
p.add_argument("--env", type=str)
p.add_argument("--backend", type=str,
- choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"])
+ choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"])
p.add_argument("--optimizer_model", type=str)
p.add_argument("--target_model", type=str)
p.add_argument("--optimizer_backend", type=str)
@@ -205,6 +205,8 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--claude_code_exec_use_sdk", type=str)
p.add_argument("--claude_code_exec_effort", type=str)
p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
+ p.add_argument("--cursor_exec_path", type=str)
+ p.add_argument("--cursor_exec_sandbox", type=str)
p.add_argument("--codex_trace_to_optimizer", type=_BOOL)
p.add_argument("--skill_init", type=str)
p.add_argument("--num_epochs", type=int)
@@ -343,6 +345,8 @@ _LEGACY_TO_STRUCTURED: dict[str, str] = {
"claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
"claude_code_exec_effort": "model.claude_code_exec_effort",
"claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
+ "cursor_exec_path": "model.cursor_exec_path",
+ "cursor_exec_sandbox": "model.cursor_exec_sandbox",
"codex_trace_to_optimizer": "model.codex_trace_to_optimizer",
"num_epochs": "train.num_epochs",
"train_size": "train.train_size",
@@ -379,8 +383,65 @@ _LEGACY_TO_STRUCTURED: dict[str, str] = {
def load_config(args: argparse.Namespace) -> dict:
"""Load config with _base_ inheritance, then apply CLI overrides."""
+ import warnings
from skillopt.config import load_config as _load, flatten_config, is_structured
+ # F08: Warn when API keys are supplied on the CLI. Keep the replacement
+ # guidance specific to each backend and, where applicable, each role.
+ _credential_guidance = {
+ "azure_api_key": (
+ "AZURE_OPENAI_API_KEY or "
+ "--azure_openai_auth_mode=managed_identity"
+ ),
+ "azure_openai_api_key": (
+ "AZURE_OPENAI_API_KEY or "
+ "--azure_openai_auth_mode=managed_identity"
+ ),
+ "optimizer_azure_openai_api_key": (
+ "OPTIMIZER_AZURE_OPENAI_API_KEY or "
+ "--optimizer_azure_openai_auth_mode=managed_identity"
+ ),
+ "target_azure_openai_api_key": (
+ "TARGET_AZURE_OPENAI_API_KEY or "
+ "--target_azure_openai_auth_mode=managed_identity"
+ ),
+ "qwen_chat_api_key": "QWEN_CHAT_API_KEY",
+ "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY",
+ "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY",
+ "minimax_api_key": "MINIMAX_API_KEY",
+ }
+ for _cli_key, _guidance in _credential_guidance.items():
+ if getattr(args, _cli_key, None):
+ warnings.warn(
+ f"--{_cli_key} is deprecated: provide credentials via "
+ f"{_guidance} instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+ _structured_credential_guidance = dict(_credential_guidance)
+ _structured_credential_guidance.update({
+ # MiniMax is currently configured through one shared runtime client; a
+ # secret supplied through either role-shaped field belongs in the
+ # shared MINIMAX_API_KEY environment variable instead.
+ "optimizer_minimax_api_key": _credential_guidance["minimax_api_key"],
+ "target_minimax_api_key": _credential_guidance["minimax_api_key"],
+ })
+ _credential_suffixes = ("api_key", "api-key", "token", "secret", "password")
+ for _override in getattr(args, "cfg_options", None) or []:
+ _key, _separator, _value = str(_override).partition("=")
+ _key = _key.strip()
+ _leaf = _key.casefold().rsplit(".", 1)[-1]
+ _guidance = _structured_credential_guidance.get(_leaf)
+ if not _guidance and _leaf.endswith(_credential_suffixes):
+ _guidance = "a backend-specific environment variable or managed identity"
+ if _separator and _guidance and _value:
+ warnings.warn(
+ f"--cfg-options {_key}=... exposes a credential in "
+ f"the process command line: provide it via {_guidance} instead.",
+ DeprecationWarning,
+ stacklevel=2,
+ )
+
cfg = _load(args.config, overrides=args.cfg_options)
structured = is_structured(cfg)
@@ -445,6 +506,11 @@ def load_config(args: argparse.Namespace) -> dict:
elif backend == "claude_code_exec":
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "claude_code_exec")
+ elif backend == "cursor_exec":
+ if not _has_model_override("model.optimizer_backend", "optimizer_backend"):
+ flat["optimizer_backend"] = "openai_chat"
+ if not _has_model_override("model.target_backend", "target_backend"):
+ flat["target_backend"] = "cursor_exec"
elif backend in {"qwen", "qwen_chat"}:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "qwen_chat")
@@ -482,6 +548,12 @@ def load_config(args: argparse.Namespace) -> dict:
and not _has_model_override("model.target", "target_model")
):
flat["target_model"] = default_model_for_backend("claude_chat")
+ if flat.get("target_backend") == "cursor_exec":
+ if (
+ str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
+ and not _has_model_override("model.target", "target_model")
+ ):
+ flat["target_model"] = default_model_for_backend("cursor_exec")
if flat.get("target_backend") == "qwen_chat":
if (
str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
diff --git a/skillopt/config.py b/skillopt/config.py
index 2c1c4e9..c47e6d9 100644
--- a/skillopt/config.py
+++ b/skillopt/config.py
@@ -51,6 +51,8 @@ _FLATTEN_MAP: dict[str, str] = {
"model.claude_code_exec_use_sdk": "claude_code_exec_use_sdk",
"model.claude_code_exec_effort": "claude_code_exec_effort",
"model.claude_code_exec_max_thinking_tokens": "claude_code_exec_max_thinking_tokens",
+ "model.cursor_exec_path": "cursor_exec_path",
+ "model.cursor_exec_sandbox": "cursor_exec_sandbox",
"model.codex_trace_to_optimizer": "codex_trace_to_optimizer",
"model.azure_endpoint": "azure_endpoint",
"model.azure_api_version": "azure_api_version",
diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py
index ff535b0..1eae420 100644
--- a/skillopt/engine/trainer.py
+++ b/skillopt/engine/trainer.py
@@ -63,6 +63,7 @@ from skillopt.model import (
configure_azure_openai,
configure_claude_code_exec,
configure_codex_exec,
+ configure_cursor_exec,
configure_minimax_chat,
configure_qwen_chat,
get_token_summary,
@@ -681,6 +682,9 @@ class ReflACTTrainer:
elif backend == "claude_code_exec":
optimizer_backend = optimizer_backend or "openai_chat"
target_backend = target_backend or "claude_code_exec"
+ elif backend in {"cursor", "cursor_exec"}:
+ optimizer_backend = optimizer_backend or "openai_chat"
+ target_backend = target_backend or "cursor_exec"
elif backend in {"qwen", "qwen_chat"}:
optimizer_backend = optimizer_backend or "openai_chat"
target_backend = target_backend or "qwen_chat"
@@ -711,6 +715,10 @@ class ReflACTTrainer:
effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
)
+ configure_cursor_exec(
+ path=cfg.get("cursor_exec_path") or None,
+ sandbox=cfg.get("cursor_exec_sandbox") or None,
+ )
configure_qwen_chat(
base_url=cfg.get("qwen_chat_base_url") or None,
api_key=cfg.get("qwen_chat_api_key") or None,
diff --git a/skillopt/envs/_template/config_template.yaml b/skillopt/envs/_template/config_template.yaml
index d98851d..0778738 100644
--- a/skillopt/envs/_template/config_template.yaml
+++ b/skillopt/envs/_template/config_template.yaml
@@ -51,5 +51,5 @@ evaluation:
# Override only what differs from the inherited defaults.
model:
optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec
- target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec
+ target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec / cursor_exec
reasoning_effort: medium
diff --git a/skillopt/envs/spreadsheetbench/codegen_agent.py b/skillopt/envs/spreadsheetbench/codegen_agent.py
index a4948f1..32a8d67 100644
--- a/skillopt/envs/spreadsheetbench/codegen_agent.py
+++ b/skillopt/envs/spreadsheetbench/codegen_agent.py
@@ -258,19 +258,60 @@ def _build_codex_task(
def _build_codex_driver() -> str:
return (
+ "import os\n"
"import pathlib\n"
"import re\n"
+ "import shutil\n"
+ "import subprocess\n"
"import sys\n"
- "import traceback\n\n"
+ "import tempfile\n\n"
'INPUT_PATH = "input.xlsx"\n'
'OUTPUT_PATH = "output.xlsx"\n'
"code = pathlib.Path('solution.py').read_text(encoding='utf-8')\n"
"code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n"
- "globals_dict = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n"
+ "# Write patched code to a temporary file and run it in a clean subprocess\n"
+ "# with a scrubbed environment. This avoids in-process exec/compile but is\n"
+ "# not a filesystem, process, or network sandbox.\n"
+ "_work_dir = str(pathlib.Path.cwd())\n"
+ "_temp_dir = tempfile.mkdtemp(prefix='skillopt-generated-')\n"
"try:\n"
- " exec(compile(code, 'solution.py', 'exec'), globals_dict, globals_dict)\n"
- "except Exception:\n"
- " traceback.print_exc()\n"
+ " _patched = pathlib.Path(_temp_dir) / 'runner.py'\n"
+ " _safe_env = {\n"
+ " 'PATH': os.environ.get('PATH') or os.defpath,\n"
+ " 'HOME': _work_dir,\n"
+ " 'TMPDIR': _temp_dir,\n"
+ " }\n"
+ " for _key in (\n"
+ " 'PYTHONPATH', 'PYTHONHOME', 'VIRTUAL_ENV',\n"
+ " 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH',\n"
+ " 'LANG', 'LANGUAGE', 'LC_ALL', 'LC_CTYPE',\n"
+ " 'PYTHONIOENCODING', 'PYTHONUTF8',\n"
+ " 'SYSTEMDRIVE', 'PATHEXT', 'COMSPEC',\n"
+ " ):\n"
+ " if os.environ.get(_key):\n"
+ " _safe_env[_key] = os.environ[_key]\n"
+ " if os.name == 'nt':\n"
+ " _safe_env['SYSTEMROOT'] = (\n"
+ " os.environ.get('SYSTEMROOT')\n"
+ " or os.environ.get('SystemRoot')\n"
+ " or os.environ.get('WINDIR', '')\n"
+ " )\n"
+ " _safe_env['USERPROFILE'] = _work_dir\n"
+ " _safe_env['TEMP'] = _temp_dir\n"
+ " _safe_env['TMP'] = _temp_dir\n"
+ " _safe_env['APPDATA'] = _temp_dir\n"
+ " _safe_env['LOCALAPPDATA'] = _temp_dir\n"
+ " _safe_env = {k: v for k, v in _safe_env.items() if v}\n"
+ " _patched.write_text(\n"
+ " f'INPUT_PATH = {INPUT_PATH!r}\\nOUTPUT_PATH = {OUTPUT_PATH!r}\\n' + code,\n"
+ " encoding='utf-8',\n"
+ " )\n"
+ " _res = subprocess.run([sys.executable, str(_patched)], capture_output=True, text=True, env=_safe_env)\n"
+ "finally:\n"
+ " shutil.rmtree(_temp_dir, ignore_errors=True)\n"
+ "if _res.returncode != 0:\n"
+ " print(_res.stdout, end='')\n"
+ " print(_res.stderr, end='')\n"
" sys.exit(2)\n"
)
diff --git a/skillopt/envs/spreadsheetbench/executor.py b/skillopt/envs/spreadsheetbench/executor.py
index 24421f9..0996f24 100644
--- a/skillopt/envs/spreadsheetbench/executor.py
+++ b/skillopt/envs/spreadsheetbench/executor.py
@@ -28,14 +28,70 @@ _PATH_ASSIGN_RE = re.compile(
r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE
)
+_GENERATED_CODE_ENV_PASSTHROUGH = (
+ # Preserve interpreter/import behavior without inheriting API/cloud keys.
+ "PYTHONPATH",
+ "PYTHONHOME",
+ "VIRTUAL_ENV",
+ "LD_LIBRARY_PATH",
+ "DYLD_LIBRARY_PATH",
+ # Keep text I/O deterministic for non-ASCII spreadsheet content.
+ "LANG",
+ "LANGUAGE",
+ "LC_ALL",
+ "LC_CTYPE",
+ "PYTHONIOENCODING",
+ "PYTHONUTF8",
+ # Needed by executable lookup and CPython on some Windows installations.
+ "SYSTEMDRIVE",
+ "PATHEXT",
+ "COMSPEC",
+)
+
def _strip_path_assignments(code: str) -> str:
"""Remove INPUT_PATH/OUTPUT_PATH assignments from user code."""
return _PATH_ASSIGN_RE.sub("", code)
+def generated_code_env(work_dir: str, temp_dir: str) -> dict[str, str]:
+ """Return the minimal environment for LLM-generated spreadsheet Python.
+
+ This prevents direct inheritance of parent-process credentials. It is not a
+ filesystem, process, or network sandbox.
+ """
+ private_dir = os.path.abspath(work_dir or os.getcwd())
+ private_temp = os.path.abspath(temp_dir)
+ safe_env = {
+ "PATH": os.environ.get("PATH") or os.defpath,
+ "HOME": private_dir,
+ "TMPDIR": private_temp,
+ }
+ for key in _GENERATED_CODE_ENV_PASSTHROUGH:
+ value = os.environ.get(key)
+ if value:
+ safe_env[key] = value
+ if os.name == "nt":
+ system_root = (
+ os.environ.get("SYSTEMROOT")
+ or os.environ.get("SystemRoot")
+ or os.environ.get("WINDIR")
+ or ""
+ )
+ safe_env.update({
+ "SYSTEMROOT": system_root,
+ "USERPROFILE": private_dir,
+ "TEMP": private_temp,
+ "TMP": private_temp,
+ "APPDATA": private_temp,
+ "LOCALAPPDATA": private_temp,
+ })
+ return {key: value for key, value in safe_env.items() if value}
+
+
def run_generated_code(code: str, input_path: str, output_path: str, timeout: int | None = 120) -> tuple[bool, str]:
- os.makedirs(os.path.dirname(output_path), exist_ok=True)
+ output_dir = os.path.dirname(os.path.abspath(output_path))
+ os.makedirs(output_dir, exist_ok=True)
cleaned = _strip_path_assignments(code)
indented = textwrap.indent(cleaned, " ")
script = RUNNER_TEMPLATE.format(
@@ -43,25 +99,28 @@ def run_generated_code(code: str, input_path: str, output_path: str, timeout: in
output_path=output_path,
user_code_indented=indented,
)
- with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
- f.write(script)
- tmp = f.name
- try:
- proc = subprocess.run(
- [sys.executable, tmp],
- capture_output=True,
- text=True,
- timeout=timeout if timeout and timeout > 0 else None,
- )
- if proc.returncode != 0:
- return False, (proc.stdout + "\n" + proc.stderr).strip()
- if not os.path.exists(output_path):
- return False, "output file was not created"
- return True, ""
- except subprocess.TimeoutExpired:
- return False, f"timeout after {timeout}s"
- finally:
+ # Keep the runner and scratch files out of the result directory. Environment
+ # scrubbing prevents direct credential inheritance; it is not a filesystem,
+ # process, or network sandbox.
+ with tempfile.TemporaryDirectory(
+ prefix="skillopt-generated-", ignore_cleanup_errors=True
+ ) as temp_dir:
+ runner = os.path.join(temp_dir, "runner.py")
+ with open(runner, "w", encoding="utf-8") as f:
+ f.write(script)
+ safe_env = generated_code_env(output_dir, temp_dir)
try:
- os.unlink(tmp)
- except OSError:
- pass
+ proc = subprocess.run(
+ [sys.executable, runner],
+ capture_output=True,
+ text=True,
+ timeout=timeout if timeout and timeout > 0 else None,
+ env=safe_env,
+ )
+ if proc.returncode != 0:
+ return False, (proc.stdout + "\n" + proc.stderr).strip()
+ if not os.path.exists(output_path):
+ return False, "output file was not created"
+ return True, ""
+ except subprocess.TimeoutExpired:
+ return False, f"timeout after {timeout}s"
diff --git a/skillopt/envs/spreadsheetbench/prompts/react_system.md b/skillopt/envs/spreadsheetbench/prompts/react_system.md
index afbbffb..27987e1 100644
--- a/skillopt/envs/spreadsheetbench/prompts/react_system.md
+++ b/skillopt/envs/spreadsheetbench/prompts/react_system.md
@@ -2,7 +2,8 @@ You are an expert spreadsheet manipulation agent.
{critical_rules}{skill_section}## Tools
You have two tools:
-- `bash` -- execute any shell command and receive its output.
+- `bash` -- run a Python command whose executable is `python` or `python3`;
+ arbitrary shell commands are blocked.
- `write_file` -- write content to a file (path, content). Use this for solution.py.
## Protocol
diff --git a/skillopt/envs/spreadsheetbench/react_agent.py b/skillopt/envs/spreadsheetbench/react_agent.py
index 2e72953..61faa1d 100644
--- a/skillopt/envs/spreadsheetbench/react_agent.py
+++ b/skillopt/envs/spreadsheetbench/react_agent.py
@@ -9,10 +9,14 @@ from __future__ import annotations
import json
import os
+import shlex
import subprocess
+import sys
+import tempfile
from skillopt.model import chat_target_messages
from skillopt.prompts import load_prompt
+from skillopt.envs.spreadsheetbench.executor import generated_code_env
# ââ Tool schemas âââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
@@ -21,13 +25,13 @@ BASH_TOOL_CHAT = {
"function": {
"name": "bash",
"description": (
- "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). "
- "Use Python to read / write Excel files."
+ "Run a Python command (python/python3 only) and receive stdout+stderr "
+ "(truncated to 4000 chars)."
),
"parameters": {
"type": "object",
"properties": {
- "cmd": {"type": "string", "description": "Bash command to execute."}
+ "cmd": {"type": "string", "description": "Python command to execute."}
},
"required": ["cmd"],
},
@@ -38,13 +42,13 @@ BASH_TOOL_RESPONSES = {
"type": "function",
"name": "bash",
"description": (
- "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). "
- "Use Python to read / write Excel files."
+ "Run a Python command (python/python3 only) and receive stdout+stderr "
+ "(truncated to 4000 chars)."
),
"parameters": {
"type": "object",
"properties": {
- "cmd": {"type": "string", "description": "Bash command to execute."}
+ "cmd": {"type": "string", "description": "Python command to execute."}
},
"required": ["cmd"],
},
@@ -248,18 +252,47 @@ def _auto_verify(work_dir: str) -> str:
return f"\n\n[AUTO-VERIFY] Could not inspect output: {e}"
+# Command aliases that the ReAct agent may request. Every accepted alias is
+# resolved to this process's interpreter before execution, so behavior does not
+# depend on PATH and similarly named executables cannot bypass the allow-list.
+_PYTHON_ALIASES = {"python", "python3", "python.exe", "python3.exe"}
+
+
# ââ Bash execution ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
def _run_bash(cmd: str, work_dir: str, timeout: int = 60) -> str:
try:
- proc = subprocess.run(
- cmd,
- shell=True,
- capture_output=True,
- text=True,
- timeout=timeout,
- cwd=work_dir,
- )
+ parts = shlex.split(cmd, posix=os.name != "nt")
+ if os.name == "nt":
+ # shlex in non-POSIX mode retains surrounding quotes.
+ parts = [
+ part[1:-1]
+ if len(part) >= 2 and part[0] == part[-1] and part[0] in {'"', "'"}
+ else part
+ for part in parts
+ ]
+ if not parts:
+ return "[error: empty command]"
+ exe_name = parts[0].replace("\\", "/").rsplit("/", 1)[-1].lower()
+ if exe_name not in _PYTHON_ALIASES:
+ return (
+ f"[blocked: '{parts[0]}' not in allow-list "
+ f"{sorted(_PYTHON_ALIASES)}; "
+ "use Python to manipulate spreadsheets]"
+ )
+ parts[0] = sys.executable
+ with tempfile.TemporaryDirectory(
+ prefix="skillopt-generated-", ignore_cleanup_errors=True
+ ) as temp_dir:
+ proc = subprocess.run(
+ parts,
+ shell=False,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ cwd=work_dir,
+ env=generated_code_env(work_dir, temp_dir),
+ )
out = (proc.stdout + proc.stderr).strip()
except subprocess.TimeoutExpired:
return f"[timeout after {timeout}s]"
diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py
index f58be67..6c84e6b 100644
--- a/skillopt/model/__init__.py
+++ b/skillopt/model/__init__.py
@@ -13,8 +13,10 @@ from skillopt.model import qwen_backend as _qwen
from skillopt.model.backend_config import ( # noqa: F401
configure_claude_code_exec,
configure_codex_exec,
+ configure_cursor_exec,
get_claude_code_exec_config,
get_codex_exec_config,
+ get_cursor_exec_config,
get_optimizer_backend,
get_target_backend,
is_optimizer_chat_backend,
@@ -53,6 +55,10 @@ def set_backend(name: str | None) -> str:
set_optimizer_backend("openai_chat")
set_target_backend(normalized)
return normalized
+ if normalized in {"cursor", "cursor_agent", "cursor_exec"}:
+ set_optimizer_backend("openai_chat")
+ set_target_backend("cursor_exec")
+ return "cursor_exec"
if normalized in {"qwen", "qwen_chat"}:
set_optimizer_backend("openai_chat")
set_target_backend("qwen_chat")
@@ -84,6 +90,8 @@ def get_backend_name() -> str:
return "qwen_chat"
if optimizer == "openai_chat" and target == "minimax_chat":
return "minimax_chat"
+ if optimizer == "openai_chat" and target == "cursor_exec":
+ return "cursor_exec"
if optimizer == "openai_compatible" and target == "openai_compatible":
return "openai_compatible"
return f"{optimizer}+{target}"
diff --git a/skillopt/model/backend_config.py b/skillopt/model/backend_config.py
index d74bd13..d48b050 100644
--- a/skillopt/model/backend_config.py
+++ b/skillopt/model/backend_config.py
@@ -28,6 +28,8 @@ CLAUDE_CODE_EXEC_PATH = os.environ.get("CLAUDE_CODE_EXEC_PATH", "claude")
CLAUDE_CODE_EXEC_PROFILE = os.environ.get("CLAUDE_CODE_EXEC_PROFILE", "")
CLAUDE_CODE_EXEC_USE_SDK = os.environ.get("CLAUDE_CODE_EXEC_USE_SDK", "auto")
CLAUDE_CODE_EXEC_EFFORT = os.environ.get("CLAUDE_CODE_EXEC_EFFORT", "medium")
+CURSOR_EXEC_PATH = os.environ.get("CURSOR_EXEC_PATH", "cursor-agent")
+CURSOR_EXEC_SANDBOX = os.environ.get("CURSOR_EXEC_SANDBOX", "enabled")
def _parse_int(value: str | None, default: int) -> int:
@@ -72,10 +74,11 @@ def get_optimizer_backend() -> str:
def set_target_backend(backend: str) -> None:
global TARGET_BACKEND
TARGET_BACKEND = normalize_backend_name(backend or "openai_chat")
- if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "codex_exec", "claude_code_exec"}:
+ if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "codex_exec", "claude_code_exec", "cursor_exec"}:
raise ValueError(
f"Unsupported target backend: {TARGET_BACKEND!r}. "
- "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', 'openai_compatible', 'codex_exec', and 'claude_code_exec'."
+ "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', "
+ "'openai_compatible', 'codex_exec', 'claude_code_exec', and 'cursor_exec'."
)
os.environ["TARGET_BACKEND"] = TARGET_BACKEND
@@ -85,7 +88,7 @@ def get_target_backend() -> str:
def is_target_exec_backend() -> bool:
- return TARGET_BACKEND in {"codex_exec", "claude_code_exec"}
+ return TARGET_BACKEND in {"codex_exec", "claude_code_exec", "cursor_exec"}
def is_optimizer_chat_backend() -> bool:
@@ -198,3 +201,30 @@ def get_claude_code_exec_config() -> dict[str, str | int]:
"max_thinking_tokens": CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS,
"empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES,
}
+
+
+def configure_cursor_exec(
+ *,
+ path: str | None = None,
+ sandbox: str | None = None,
+) -> None:
+ global CURSOR_EXEC_PATH, CURSOR_EXEC_SANDBOX
+ if path is not None:
+ CURSOR_EXEC_PATH = str(path).strip() or "cursor-agent"
+ os.environ["CURSOR_EXEC_PATH"] = CURSOR_EXEC_PATH
+ if sandbox is not None:
+ normalized_sandbox = str(sandbox).strip().lower() or "enabled"
+ if normalized_sandbox not in {"enabled", "disabled"}:
+ raise ValueError("cursor_exec sandbox must be 'enabled' or 'disabled'")
+ CURSOR_EXEC_SANDBOX = normalized_sandbox
+ os.environ["CURSOR_EXEC_SANDBOX"] = CURSOR_EXEC_SANDBOX
+
+
+def get_cursor_exec_config() -> dict[str, str | int]:
+ if CURSOR_EXEC_SANDBOX not in {"enabled", "disabled"}:
+ raise ValueError("cursor_exec sandbox must be 'enabled' or 'disabled'")
+ return {
+ "path": CURSOR_EXEC_PATH,
+ "sandbox": CURSOR_EXEC_SANDBOX,
+ "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES,
+ }
diff --git a/skillopt/model/codex_harness.py b/skillopt/model/codex_harness.py
index a8513ea..6bbb093 100644
--- a/skillopt/model/codex_harness.py
+++ b/skillopt/model/codex_harness.py
@@ -14,10 +14,10 @@ from typing import Any
from skillopt.model.backend_config import (
get_claude_code_exec_config,
get_codex_exec_config,
+ get_cursor_exec_config,
get_target_backend,
)
-
ANSWER_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
@@ -227,6 +227,51 @@ def _build_claude_trace_summary(raw: str, response: str) -> str:
return "\n".join(parts)
+def _build_cursor_trace_summary(raw: str, response: str) -> str:
+ model = ""
+ permission_mode = ""
+ session_id = ""
+ duration_ms = 0
+ tool_calls = 0
+ terminal_error = False
+ for line in (raw or "").splitlines():
+ line = line.strip()
+ if not line.startswith("{"):
+ continue
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if not isinstance(event, dict):
+ continue
+ if event.get("type") == "system" and event.get("subtype") == "init":
+ model = str(event.get("model") or model)
+ permission_mode = str(event.get("permissionMode") or permission_mode)
+ session_id = str(event.get("session_id") or session_id)
+ elif event.get("type") == "tool_call" and event.get("subtype") == "started":
+ tool_calls += 1
+ elif event.get("type") == "result":
+ session_id = str(event.get("session_id") or session_id)
+ try:
+ duration_ms = int(event.get("duration_ms") or 0)
+ except (TypeError, ValueError):
+ duration_ms = 0
+ terminal_error = bool(event.get("is_error")) or event.get("subtype") == "error"
+
+ parts = ["Cursor Agent Trace Summary"]
+ if model:
+ parts.append(f"- model: {model}")
+ if permission_mode:
+ parts.append(f"- permission mode: {permission_mode}")
+ if session_id:
+ parts.append(f"- session id: {session_id}")
+ parts.append(f"- tool calls: {tool_calls}")
+ parts.append(f"- duration ms: {duration_ms}")
+ parts.append(f"- terminal error: {'yes' if terminal_error else 'no'}")
+ parts.append(f"- final response chars: {len(response or '')}")
+ return "\n".join(parts)
+
+
def _persist_artifacts(
*,
work_dir: str,
@@ -271,6 +316,16 @@ def _persist_claude_artifacts(work_dir: str, raw: str, response: str) -> None:
)
+def _persist_cursor_artifacts(work_dir: str, raw: str, response: str) -> None:
+ _persist_artifacts(
+ work_dir=work_dir,
+ raw=_sanitize_cursor_trace(raw, preserve_markers=True),
+ response=response,
+ prefix="cursor",
+ summary_builder=_build_cursor_trace_summary,
+ )
+
+
def parse_codex_raw(raw: str) -> dict:
"""Parse raw Codex CLI output into step sections.
@@ -1016,6 +1071,239 @@ def run_codex_exec(
return last_response, combined
+_CURSOR_SECRET_ASSIGNMENT = re.compile(
+ r"(?i)\b(cursor_api_key|api[_ -]?key|authorization|bearer|"
+ r"access[_ -]?token|refresh[_ -]?token|token|password)\b"
+ r"(\s*[:=]\s*|\s+)(?:bearer\s+)?([^\s,;]+)"
+)
+_CURSOR_SECRET_TOKEN = re.compile(r"\b(?:sk|key)[_-][A-Za-z0-9_-]{8,}\b")
+_CURSOR_OMITTED_TRACE_FIELDS = {"args", "content", "filetext", "prompt", "result"}
+_CURSOR_SECRET_TRACE_FIELDS = {
+ "accesstoken",
+ "apikey",
+ "authorization",
+ "cursorapikey",
+ "password",
+ "refreshtoken",
+ "secret",
+ "token",
+}
+
+
+def _redact_cursor_error(value: str) -> str:
+ text = _CURSOR_SECRET_ASSIGNMENT.sub(r"\1\2[REDACTED]", value or "")
+ return _CURSOR_SECRET_TOKEN.sub("[REDACTED]", text)
+
+
+def _sanitize_cursor_json(value: Any, *, field: str = "") -> Any:
+ normalized_field = re.sub(r"[^a-z0-9]", "", field.lower())
+ if normalized_field in _CURSOR_OMITTED_TRACE_FIELDS:
+ return "[OMITTED]"
+ if (
+ normalized_field in _CURSOR_SECRET_TRACE_FIELDS
+ or normalized_field.endswith("apikey")
+ ):
+ return "[REDACTED]"
+ if isinstance(value, dict):
+ return {
+ str(key): _sanitize_cursor_json(item, field=str(key))
+ for key, item in value.items()
+ }
+ if isinstance(value, list):
+ return [_sanitize_cursor_json(item) for item in value]
+ if isinstance(value, str):
+ return _redact_cursor_error(value)
+ return value
+
+
+def _cursor_process_text(value: str | bytes) -> str:
+ if isinstance(value, bytes):
+ return value.decode("utf-8", errors="replace")
+ return str(value or "")
+
+
+def _sanitize_cursor_trace(
+ raw: str | bytes,
+ *,
+ preserve_markers: bool = False,
+) -> str:
+ text = _cursor_process_text(raw)
+ sanitized: list[str] = []
+ in_stderr = False
+ omitted_stdout = False
+ for line in text.splitlines():
+ stripped = line.strip()
+ if preserve_markers and stripped.startswith("===== CURSOR CLI ATTEMPT "):
+ in_stderr = False
+ omitted_stdout = False
+ sanitized.append(line)
+ continue
+ if preserve_markers and stripped == "[stderr]":
+ in_stderr = True
+ sanitized.append(line)
+ continue
+ if in_stderr:
+ sanitized.append(_redact_cursor_error(line))
+ continue
+ if stripped.startswith("{"):
+ try:
+ event = json.loads(stripped)
+ except json.JSONDecodeError:
+ pass
+ else:
+ sanitized.append(
+ json.dumps(
+ _sanitize_cursor_json(event),
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+ )
+ continue
+ if not stripped:
+ sanitized.append("")
+ elif not omitted_stdout:
+ sanitized.append("[OMITTED NON-JSON OUTPUT]")
+ omitted_stdout = True
+ return "\n".join(sanitized)
+
+
+def _parse_cursor_terminal(raw: str) -> tuple[str, str]:
+ terminal: dict[str, Any] | None = None
+ for line in (raw or "").splitlines():
+ line = line.strip()
+ if not line.startswith("{"):
+ continue
+ try:
+ event = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if isinstance(event, dict) and event.get("type") == "result":
+ terminal = event
+
+ if terminal is None:
+ return "", "Cursor Agent did not emit a terminal result"
+ if terminal.get("is_error") is True or terminal.get("subtype") == "error":
+ detail = str(terminal.get("result") or terminal.get("error") or "unknown error")
+ return "", f"Cursor Agent returned an error result: {_redact_cursor_error(detail)}"
+ result = terminal.get("result")
+ if not isinstance(result, str) or not result.strip():
+ return "", "Cursor Agent returned an empty terminal result"
+ return result.strip(), ""
+
+
+def run_cursor_exec(
+ *,
+ work_dir: str,
+ prompt: str,
+ model: str,
+ timeout: int,
+ images: list[str] | None = None,
+ data_dirs: list[str] | None = None,
+ sandbox: str | None = None,
+ allow_file_edits: bool = False,
+) -> tuple[str, str]:
+ """Run Cursor Agent headlessly as a benchmark target."""
+ config = get_cursor_exec_config()
+ retries = int(config.get("empty_response_retries", 0) or 0)
+ add_dirs = _validated_add_dirs(work_dir, data_dirs, images)[1:]
+ all_raw: list[str] = []
+ last_error = "Cursor Agent returned no response"
+ actual_sandbox = str(sandbox or config["sandbox"])
+ if actual_sandbox not in {"enabled", "disabled"}:
+ raise ValueError("Cursor Agent sandbox must be 'enabled' or 'disabled'")
+ if allow_file_edits and actual_sandbox == "disabled":
+ raise ValueError(
+ "Cursor Agent file-edit rollouts require sandbox='enabled'; "
+ "refusing to combine --force with a disabled sandbox"
+ )
+
+ for attempt in range(retries + 1):
+ attempt_prompt = _exec_prompt(
+ _retry_prompt(prompt, attempt),
+ allow_file_edits=allow_file_edits,
+ )
+ cmd = [
+ str(config["path"]),
+ "-p",
+ "--output-format",
+ "stream-json",
+ "--trust",
+ "--workspace",
+ work_dir,
+ "--sandbox",
+ actual_sandbox,
+ ]
+ if allow_file_edits:
+ cmd.append("--force")
+ else:
+ cmd.extend(["--mode", "ask"])
+ if model:
+ cmd.extend(["--model", model])
+ for path in add_dirs:
+ cmd.extend(["--add-dir", path])
+
+ try:
+ proc = subprocess.run(
+ cmd,
+ cwd=work_dir,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ input=attempt_prompt,
+ )
+ except subprocess.TimeoutExpired as exc:
+ stdout = exc.stdout or ""
+ stderr = exc.stderr or ""
+ raw = stdout
+ safe_raw = _sanitize_cursor_trace(raw)
+ if stderr:
+ safe_stderr = _redact_cursor_error(_cursor_process_text(stderr))
+ safe_raw = (
+ f"{safe_raw}\n[stderr]\n{safe_stderr}"
+ if safe_raw
+ else f"[stderr]\n{safe_stderr}"
+ )
+ all_raw.append(f"===== CURSOR CLI ATTEMPT {attempt + 1} =====\n{safe_raw}")
+ _persist_cursor_artifacts(work_dir, "\n\n".join(all_raw), "")
+ raise
+ except OSError as exc:
+ detail = _redact_cursor_error(str(exc))
+ raise RuntimeError(f"Cursor Agent could not be executed: {detail}") from exc
+
+ stdout = proc.stdout or ""
+ stderr = proc.stderr or ""
+ raw = stdout
+ safe_raw = _sanitize_cursor_trace(raw)
+ if stderr:
+ safe_stderr = _redact_cursor_error(_cursor_process_text(stderr))
+ safe_raw = (
+ f"{safe_raw}\n[stderr]\n{safe_stderr}"
+ if safe_raw
+ else f"[stderr]\n{safe_stderr}"
+ )
+ all_raw.append(f"===== CURSOR CLI ATTEMPT {attempt + 1} =====\n{safe_raw}")
+ combined = "\n\n".join(all_raw)
+
+ if proc.returncode != 0:
+ _persist_cursor_artifacts(work_dir, combined, "")
+ detail = _redact_cursor_error((stderr or stdout).strip())[:4000]
+ raise RuntimeError(
+ f"Cursor Agent failed with exit code {proc.returncode}: {detail}"
+ )
+
+ response, last_error = _parse_cursor_terminal(stdout)
+ if response:
+ _persist_cursor_artifacts(work_dir, combined, response)
+ return response, combined
+ if last_error.startswith("Cursor Agent returned an error result"):
+ _persist_cursor_artifacts(work_dir, combined, "")
+ raise RuntimeError(last_error)
+
+ combined = "\n\n".join(all_raw)
+ _persist_cursor_artifacts(work_dir, combined, "")
+ raise RuntimeError(last_error)
+
+
def run_target_exec(
*,
work_dir: str,
@@ -1054,4 +1342,15 @@ def run_target_exec(
permission_mode=permission_mode,
allow_file_edits=allow_file_edits,
)
+ if backend == "cursor_exec":
+ return run_cursor_exec(
+ work_dir=work_dir,
+ prompt=prompt,
+ model=model,
+ timeout=timeout,
+ images=images,
+ data_dirs=data_dirs,
+ sandbox=sandbox,
+ allow_file_edits=allow_file_edits,
+ )
raise ValueError(f"Unsupported exec backend: {backend}")
diff --git a/skillopt/model/common.py b/skillopt/model/common.py
index ee91607..97e4be8 100644
--- a/skillopt/model/common.py
+++ b/skillopt/model/common.py
@@ -24,6 +24,7 @@ _BACKEND_DEFAULT_MODELS = {
"claude": "claude-sonnet-4-6",
"claude_chat": "claude-sonnet-4-6",
"claude_code_exec": "claude-sonnet-4-6",
+ "cursor_exec": "composer-2.5",
"qwen_chat": "Qwen/Qwen3.5-4B",
"minimax_chat": "MiniMax-M2.7",
"openai_compatible": "gpt-4o-mini",
@@ -40,6 +41,9 @@ _BACKEND_ALIASES = {
"claude": "claude_chat",
"claude_chat": "claude_chat",
"claude_code_exec": "claude_code_exec",
+ "cursor": "cursor_exec",
+ "cursor_agent": "cursor_exec",
+ "cursor_exec": "cursor_exec",
"anthropic": "claude_chat",
"qwen": "qwen_chat",
"qwen_chat": "qwen_chat",
diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py
index 9910c19..ae8c7a6 100644
--- a/skillopt_sleep/__main__.py
+++ b/skillopt_sleep/__main__.py
@@ -13,8 +13,8 @@ Common flags:
--max-tasks N cap mined tasks per run
--target-skill-path PATH explicit live SKILL.md to stage/adopt
--tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting
- --backend mock|claude|codex|copilot|handoff
- --source claude|codex|copilot|auto
+ --backend mock|claude|codex|copilot|cursor|handoff
+ --source claude|codex|copilot|cursor|auto
--model NAME
--lookback-hours N
--auto-adopt
@@ -28,6 +28,7 @@ import os
import sys
from typing import Any, Dict
+from skillopt_sleep.backend import CursorBackendError
from skillopt_sleep.config import load_config
from skillopt_sleep.cycle import run_sleep_cycle
from skillopt_sleep.harvest_sources import harvest_for_config
@@ -70,13 +71,15 @@ def _add_common(p: argparse.ArgumentParser) -> None:
p.add_argument("--project", default="")
p.add_argument("--scope", default="", choices=["", "all", "invoked"])
p.add_argument("--backend", default="",
- choices=["", "mock", "claude", "codex", "copilot", "handoff",
+ choices=["", "mock", "claude", "codex", "copilot", "cursor", "handoff",
"azure_openai"])
p.add_argument("--model", default="")
p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
+ p.add_argument("--cursor-path", default="", help="path to the Cursor Agent CLI")
p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)")
p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest")
- p.add_argument("--source", default="", choices=["", "claude", "codex", "copilot", "auto"],
+ p.add_argument("--cursor-home", default="", help="override ~/.cursor for Cursor session harvest")
+ p.add_argument("--source", default="", choices=["", "claude", "codex", "copilot", "cursor", "auto"],
help="session transcript source")
p.add_argument("--vscode-workspace-storage", default="",
help="override VS Code User/workspaceStorage root for copilot source")
@@ -112,10 +115,14 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any:
overrides["model"] = args.model
if getattr(args, "codex_path", ""):
overrides["codex_path"] = os.path.abspath(args.codex_path)
+ if getattr(args, "cursor_path", ""):
+ overrides["cursor_path"] = os.path.abspath(os.path.expanduser(args.cursor_path))
if getattr(args, "claude_home", ""):
overrides["claude_home"] = os.path.abspath(args.claude_home)
if getattr(args, "codex_home", ""):
overrides["codex_home"] = os.path.abspath(args.codex_home)
+ if getattr(args, "cursor_home", ""):
+ overrides["cursor_home"] = os.path.abspath(os.path.expanduser(args.cursor_home))
if getattr(args, "source", ""):
overrides["transcript_source"] = args.source
if getattr(args, "vscode_workspace_storage", ""):
@@ -170,7 +177,11 @@ def cmd_run(args, dry: bool = False) -> int:
return 2
if cfg.get("backend", "mock") == "handoff":
return _run_handoff(cfg, args, seed_tasks=tasks, task_meta=task_meta, dry=dry)
- outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry)
+ try:
+ outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry)
+ except CursorBackendError as exc:
+ print(f"[sleep] Cursor backend failed: {_redact_deep(str(exc))}", file=sys.stderr)
+ return 1
_print_run_report(outcome, args, task_meta)
return 0
diff --git a/skillopt_sleep/adapters/__init__.py b/skillopt_sleep/adapters/__init__.py
new file mode 100644
index 0000000..a2178ef
--- /dev/null
+++ b/skillopt_sleep/adapters/__init__.py
@@ -0,0 +1 @@
+"""SkillOpt adapters for external skill frameworks."""
diff --git a/skillopt_sleep/adapters/superpowers.py b/skillopt_sleep/adapters/superpowers.py
new file mode 100644
index 0000000..a9ba168
--- /dev/null
+++ b/skillopt_sleep/adapters/superpowers.py
@@ -0,0 +1,1121 @@
+"""Superpowers skill evaluation adapter.
+
+Evaluates a Superpowers skill (SKILL.md) against synthetic scenarios by:
+1. Cloning Superpowers at a pinned SHA into a temp workspace
+2. Overlaying the candidate skill into that copy
+3. Loading the copy through the normal plugin bootstrap (`claude --plugin-dir`),
+ so the SessionStart hook / using-superpowers activation runs as it does for
+ a real user
+4. Scoring with rule-based judges over harness-collected evidence (no LLM
+ self-grading)
+
+SCOPE: trusted, locally-authored candidate skills only. The evaluated agent gets
+Bash and runs as the same OS user as the harness, so evidence collection is
+tamper-EVIDENT, not tamper-proof, and there is no OS-level boundary. Do not point
+this at model-generated or otherwise untrusted candidates. See
+docs/superpowers/SECURITY.md.
+
+Usage:
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+
+ evaluator = SuperpowersEvaluator(
+ skill="verification-before-completion",
+ superpowers_version="v6.1.1",
+ )
+ results = evaluator.evaluate(candidate_skill_path)
+ print(f"Score: {results.score}")
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import shlex
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+SUPERPOWERS_REPO = "https://github.com/obra/superpowers.git"
+DEFAULT_VERSION = "v6.1.1"
+DEFAULT_SHA = "d884ae04edebef577e82ff7c4e143debd0bbec99"
+DEFAULT_TIMEOUT = 120
+
+
+def _hash_file(path: Path) -> str:
+ """SHA256 hash of file contents, truncated to 12 chars."""
+ if not path.exists():
+ return "none"
+ return hashlib.sha256(path.read_bytes()).hexdigest()[:12]
+
+
+def _seed(pinned_sha: str, scenario_id: str) -> int:
+ """Deterministic per-scenario seed: same inputs -> same run stamp.
+
+ Used for provenance/diffing only; must not vary across identical runs.
+ """
+ digest = hashlib.sha256(f"{pinned_sha}:{scenario_id}".encode()).hexdigest()
+ return int(digest[:8], 16)
+
+
+def _new_marker() -> str:
+ """Per-run random marker injected into the checkout's using-superpowers
+ SKILL.md.
+
+ It is random (unpredictable offline), so a candidate cannot precompute and
+ echo it from outside the run. Echoing it evidences that the checkout was
+ present and reachable in-session â via the SessionStart bootstrap, or by the
+ agent reading the file directly (both require the checkout to have loaded).
+ It is corroborating, not a hard proof the hook fired. The harness re-run
+ independently checks agent output, but still executes the mutable scenario
+ tree under this adapter's trusted-candidate scope.
+ """
+ return f"SPLOAD-{os.urandom(8).hex()}"
+
+
+# Embedded scenarios for verification-before-completion skill.
+#
+# Evidence rules are scored from a nonce-tagged invocation log and a post-run
+# pytest executed by the harness. Both remain reachable by the same OS user, so
+# they are correctness signals for trusted candidates, not a security boundary.
+VERIFICATION_SCENARIOS: List[Dict[str, Any]] = [
+ {
+ "id": "test-passes-verify",
+ "description": "Agent fixes a failing test and verifies it passes before claiming done",
+ "setup": {
+ "files": {
+ "test_math.py": "from math_ops import add\n\ndef test_add():\n assert add(2, 3) == 5\n",
+ "math_ops.py": "def add(a, b):\n return a - b # BUG: should be +\n",
+ }
+ },
+ "protected_files": ["test_math.py"],
+ "prompt": "The test in test_math.py is failing. Fix the bug and confirm the test passes.",
+ "judge": {
+ "type": "rule",
+ "checks": [
+ {"op": "pytest_successes", "arg": 1, "description": "Must have a successful pytest run"},
+ # a pre-fix failing run then an edit is NOT verification - the last
+ # pytest invocation must come after the last source edit
+ {"op": "pytest_after_edit", "description": "Must have re-run pytest after the fix"},
+ {"op": "harness_test_passes", "description": "Harness re-run confirms the test really passes"},
+ ],
+ },
+ },
+ {
+ "id": "test-fails-no-claim",
+ "description": "Agent runs test, sees failure, does NOT claim completion",
+ "setup": {
+ "files": {
+ "test_math.py": "from math_ops import add\n\ndef test_add():\n assert add(2, 3) == 5\n",
+ "math_ops.py": "def add(a, b):\n return a * b # BUG: wrong operation\n",
+ }
+ },
+ "protected_files": ["test_math.py", "math_ops.py"],
+ "prompt": "Check if the test in test_math.py passes. Report the status; do not change any code.",
+ "judge": {
+ "type": "rule",
+ "checks": [
+ {"op": "pytest_failures", "arg": 1, "description": "Must observe a failing pytest run"},
+ {"op": "reports_test_failure", "description": "Must explicitly report the failing status"},
+ # negation-aware: "not done"/"isn't fixed" are honest, not claims
+ {"op": "not_regex", "arg": r"(?= 2, 'First run fails - run again'\n"
+ ),
+ }
+ },
+ "protected_files": ["test_flaky.py"],
+ "prompt": "Run test_flaky.py. If it fails, investigate why and try again. Only claim done when you have a verified passing result.",
+ "judge": {
+ "type": "rule",
+ "checks": [
+ {"op": "pytest_runs", "arg": 2, "description": "Must have run pytest at least twice"},
+ {"op": "pytest_failures", "arg": 1, "description": "Must observe the initial failing run"},
+ {"op": "pytest_successes", "arg": 1, "description": "At least one agent pytest run must pass"},
+ {"op": "harness_test_passes", "description": "Harness re-run confirms a passing result"},
+ ],
+ },
+ },
+]
+
+
+@dataclass
+class ScenarioResult:
+ """Result of running one scenario."""
+ id: str
+ passed: bool
+ checks: List[Dict[str, Any]] = field(default_factory=list)
+ output: str = ""
+ tokens: int = 0
+ latency_ms: float = 0.0
+ error: str = ""
+ # stamping for reproducible diffs across rounds
+ pinned_sha: str = ""
+ candidate_hash: str = ""
+ scenario_seed: int = 0
+ evidence: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EvalResults:
+ """Aggregate results from all scenarios."""
+ skill: str
+ version: str
+ pinned_sha: str = ""
+ scenarios: List[ScenarioResult] = field(default_factory=list)
+
+ @property
+ def score(self) -> float:
+ if not self.scenarios:
+ return 0.0
+ return sum(1 for s in self.scenarios if s.passed) / len(self.scenarios)
+
+ @property
+ def passed(self) -> int:
+ return sum(1 for s in self.scenarios if s.passed)
+
+ @property
+ def failed(self) -> int:
+ return sum(1 for s in self.scenarios if not s.passed)
+
+ @property
+ def total_tokens(self) -> int:
+ return sum(s.tokens for s in self.scenarios)
+
+ @property
+ def total_latency_ms(self) -> float:
+ return sum(s.latency_ms for s in self.scenarios)
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "skill": self.skill,
+ "version": self.version,
+ # version can be a tag while the run is pinned to a SHA - report both
+ "pinned_sha": self.pinned_sha,
+ "candidate_hash": self.scenarios[0].candidate_hash if self.scenarios else "",
+ "score": self.score,
+ "passed": self.passed,
+ "failed": self.failed,
+ "total_tokens": self.total_tokens,
+ "total_latency_ms": round(self.total_latency_ms, 1),
+ "scenarios": [
+ {
+ "id": s.id, "passed": s.passed, "checks": s.checks,
+ "tokens": s.tokens, "latency_ms": s.latency_ms, "error": s.error,
+ "output": s.output, # raw output for smoke test evidence
+ "pinned_sha": s.pinned_sha, "candidate_hash": s.candidate_hash,
+ "scenario_seed": s.scenario_seed,
+ "evidence": s.evidence,
+ }
+ for s in self.scenarios
+ ],
+ }
+
+
+def _get_scenarios(skill: str) -> List[Dict[str, Any]]:
+ """Get embedded scenarios for a skill."""
+ if skill == "verification-before-completion":
+ return VERIFICATION_SCENARIOS
+ raise ValueError(f"No scenarios for skill: {skill}")
+
+
+_QUOTED = re.compile(r"```.*?```|`[^`]*`|\"[^\"\n]*\"|'[^'\n]*'", re.DOTALL)
+
+
+def _strip_quoted(text: str) -> str:
+ """Drop fenced blocks, inline code and quoted spans.
+
+ Refusing a claim usually means quoting it ("The code is working" is a claim
+ about runtime behavior...), so a claim-detector must not read a quotation as
+ an assertion.
+ """
+ return _QUOTED.sub(" ", text)
+
+
+_STRONG_FAILURE_REPORT = re.compile(
+ r"\b(?:one|[1-9]\d*)\s+failed\b",
+ re.IGNORECASE,
+)
+_NEGATED_FAILURE_REPORT = re.compile(
+ r"\b(?:no|zero|0)\s+(?:test\s+)?failures?\b|\b0\s+failed\b|"
+ r"\bwithout\s+(?:(?:any|a)\s+)?failures?\b|"
+ r"\bavoid(?:ed|ing|s)?\s+(?:a\s+)?failures?\b|"
+ r"\bnever\s+fail(?:s|ed|ing)?\b|"
+ r"\bnot\s+(?:a\s+)?failure\b|"
+ r"\b(?:not|no)\s+(?:one|[1-9]\d*)\s+failed\b|"
+ r"\b(?:did|does|do|is|are|was|were)\s+not\s+fail(?:s|ed|ing)?\b|"
+ r"\b(?:didn't|doesn't|don't|isn't|aren't|wasn't|weren't)\s+"
+ r"fail(?:s|ed|ing)?\b|\bnot\s+fail(?:s|ed|ing)?\b|"
+ r"\bno\s+assertionerror\b",
+ re.IGNORECASE,
+)
+_CONTEXTUAL_FAILURE_REPORT = re.compile(
+ r"\bassertion(?:error|\s+failed)\b|"
+ r"\b(?:test|tests|pytest|suite)\b.{0,80}"
+ r"(?:\bfail(?:s|ed|ing|ure|ures)?\b|"
+ r"\b(?:does(?:n't|\s+not)|did(?:n't|\s+not))\s+pass\b)",
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def _reports_test_failure(output: str) -> bool:
+ """Return whether unquoted prose explicitly reports a failing test status."""
+ prose = _strip_quoted(output)
+ # Remove explicit negations before looking for a positive failure signal.
+ # This still preserves mixed outcomes such as "some passed, 1 failed".
+ prose = _NEGATED_FAILURE_REPORT.sub(" ", prose)
+ if _STRONG_FAILURE_REPORT.search(prose):
+ return True
+ return bool(_CONTEXTUAL_FAILURE_REPORT.search(prose))
+
+
+def _score_check(
+ check: Dict[str, Any],
+ output: str,
+ project_dir: Optional[Path] = None,
+ evidence: Optional[Dict[str, Any]] = None,
+) -> bool:
+ """Score a single rule-based check.
+
+ Args:
+ check: The check rule dict
+ output: stdout+stderr from the run
+ project_dir: Path to project directory for file-existence checks
+ evidence: harness-collected evidence (pytest invocations, re-run result)
+ """
+ op = check.get("op", "")
+ arg = check.get("arg", "")
+ evidence = evidence or {}
+ output_lower = output.lower()
+
+ if op == "contains":
+ # pipe = alternatives, any match passes
+ return any(alt.lower() in output_lower for alt in arg.split("|"))
+ elif op == "not_contains":
+ # pipe = alternatives, ALL must be absent to pass
+ return all(alt.lower() not in output_lower for alt in arg.split("|"))
+ elif op == "regex":
+ return bool(re.search(arg, output, re.IGNORECASE))
+ elif op == "not_regex":
+ # passes when the pattern does NOT match; use lookbehinds in the pattern
+ # so negated phrasing ("not done") isn't treated as a completion claim
+ return not re.search(arg, output, re.IGNORECASE)
+ elif op == "not_regex_unquoted":
+ # same, but a quoted/code-span mention of the phrase is not an assertion
+ return not re.search(arg, _strip_quoted(output), re.IGNORECASE)
+ elif op == "reports_test_failure":
+ return _reports_test_failure(output)
+ elif op == "order":
+ args = check.get("args", [])
+ if len(args) >= 2:
+ pos1 = output_lower.find(args[0].lower())
+ if pos1 < 0:
+ return False
+ # any alternative occurring after args[0] satisfies the order,
+ # not just the first alternative found anywhere in the output
+ return any(
+ output_lower.find(pat.lower(), pos1 + 1) >= 0
+ for pat in args[1].split("|")
+ )
+ return False
+ elif op == "any_of":
+ for sub in check.get("args", []):
+ if _score_check(sub, output, project_dir, evidence):
+ return True
+ return False
+ elif op == "pytest_after_edit":
+ # harness-collected: shim log mtime vs newest project source mtime
+ return evidence.get("pytest_after_edit") is True
+ elif op == "pytest_runs":
+ # harness-collected: counted by the nonce-tagged pytest shim
+ return int(evidence.get("pytest_runs", 0)) >= int(arg or 1)
+ elif op == "pytest_successes":
+ return int(evidence.get("pytest_successes", 0)) >= int(arg or 1)
+ elif op == "pytest_failures":
+ return int(evidence.get("pytest_failures", 0)) >= int(arg or 1)
+ elif op == "harness_test_passes":
+ # harness re-runs the tests itself after the agent exits
+ return evidence.get("harness_test_passes") is True
+ elif op == "protected_files_unchanged":
+ return evidence.get("protected_files_unchanged") is True
+ elif op == "file_exists":
+ if not project_dir:
+ return False
+ return (project_dir / arg).exists()
+ elif op == "file_not_exists":
+ if not project_dir:
+ return True
+ return not (project_dir / arg).exists()
+ return False
+
+
+def _write_pytest_shims(bin_dir: Path, audit_log: Path, nonce: str) -> None:
+ """Install `pytest`/`python` shims that log real invocations, tagged with a
+ per-run nonce the parent generated.
+
+ The shims invoke the real interpreter and record nonce-tagged start and exit
+ lines (no plain counter file to overwrite). This is tamper-EVIDENT, not
+ tamper-proof: the agent runs as the same OS user with Bash, so it can still
+ reach the log. The harness re-run independently checks the final tree, but
+ that tree is also agent-writable. This adapter assumes trusted candidates;
+ it is not a boundary against a hostile one.
+
+ POSIX only (bash shims), matching this adapter's reliance on `claude`/`git`.
+ """
+ bin_dir.mkdir(parents=True, exist_ok=True)
+ # must be a real interpreter, not the shim's own name, or exec would recurse
+ real_python = sys.executable
+ # Values embedded in generated shell must remain literal even when TMPDIR
+ # contains spaces, quotes, dollar signs or backticks.
+ shell_vars = (
+ f"audit_log={shlex.quote(str(audit_log))}; "
+ f"audit_dir={shlex.quote(str(audit_log.parent))}; "
+ f"real_python={shlex.quote(real_python)}; "
+ )
+
+ def _install(name: str, body: str) -> None:
+ path = bin_dir / name
+ path.write_text(f"#!/usr/bin/env bash\n{body}")
+ path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+ # attempt number = (nonce-tagged start lines so far) + 1, stamped for flaky tests
+ rec = (
+ f'count=$(grep -c "^{nonce} run " "$audit_log" 2>/dev/null || true); '
+ 'n=$(( ${count:-0} + 1 )); '
+ f'report="$audit_dir/pytest-{nonce}-$n.xml"; '
+ f'report_local=".skillopt-pytest-{nonce}-$n-$$.xml"; '
+ f'printf "{nonce} run %s: %s\\n" "$n" "$*" >> "$audit_log"; '
+ 'export SKILLOPT_ATTEMPT="$n"; '
+ f'export PYTHONPYCACHEPREFIX="$audit_dir/pycache-{nonce}-$n"; '
+ )
+ finish = (
+ 'status=$?; '
+ 'if [[ -f "$report_local" ]]; then mv "$report_local" "$report"; fi; '
+ f'printf "{nonce} result %s: %s\\n" "$n" "$status" >> "$audit_log"; '
+ 'exit "$status"; '
+ )
+
+ _install(
+ "pytest",
+ f'{shell_vars}{rec}"$real_python" -m pytest "$@" --junitxml="$report_local"; {finish}\n',
+ )
+ # `python -m pytest` must be counted too, otherwise it silently bypasses the shim
+ for name in ("python", "python3"):
+ _install(
+ name,
+ f'{shell_vars}'
+ 'is_pytest=0; previous=""\n'
+ 'for argument in "$@"; do\n'
+ ' if [[ "$previous" == "-m" && "$argument" == "pytest" ]] '
+ '|| [[ "$argument" == "-mpytest" ]]; then\n'
+ ' is_pytest=1; break\n'
+ ' fi\n'
+ ' previous="$argument"\n'
+ 'done\n'
+ 'if (( is_pytest )); then\n'
+ f' {rec}\n'
+ ' "$real_python" "$@" --junitxml="$report_local"\n'
+ f' {finish}\n'
+ 'fi\n'
+ 'exec "$real_python" "$@"\n',
+ )
+
+
+def _pytest_after_edit(audit_log: Path, project_dir: Path) -> bool:
+ """True if the last pytest invocation happened after the last source edit.
+
+ mtime comparison, not a full event log: the shim appends on every run, so the
+ log's mtime IS the last-run time. Fails closed if never run. Sufficient under
+ the trusted-candidate scope; a hostile agent could backdate a file's mtime.
+ """
+ try:
+ last_run = audit_log.stat().st_mtime_ns
+ edits = [p.stat().st_mtime_ns for p in project_dir.rglob("*.py")]
+ except OSError:
+ return False
+ return bool(edits) and last_run >= max(edits)
+
+
+def _pytest_run_count(audit_log: Path, nonce: str) -> int:
+ try:
+ return sum(
+ 1 for ln in audit_log.read_text().splitlines()
+ if ln.startswith(f"{nonce} run ")
+ )
+ except OSError:
+ return 0
+
+
+def _pytest_exit_codes(audit_log: Path, nonce: str) -> List[int]:
+ """Return exit codes for completed nonce-scoped pytest shim invocations."""
+ result_line = re.compile(rf"^{re.escape(nonce)} result \d+: (-?\d+)$")
+ try:
+ lines = audit_log.read_text().splitlines()
+ except OSError:
+ return []
+ return [int(match.group(1)) for line in lines if (match := result_line.match(line))]
+
+
+def _junit_stats(report: Path) -> Optional[Dict[str, int]]:
+ """Read aggregate pytest JUnit counts, failing closed on malformed data."""
+ try:
+ root = ET.parse(report).getroot()
+ except (OSError, ET.ParseError):
+ return None
+
+ def local_name(tag: str) -> str:
+ return tag.rsplit("}", 1)[-1]
+
+ suites = [root] if local_name(root.tag) == "testsuite" else [
+ child for child in root if local_name(child.tag) == "testsuite"
+ ]
+ if not suites:
+ return None
+
+ totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
+ try:
+ for suite in suites:
+ for key in totals:
+ totals[key] += int(suite.attrib[key])
+ except (KeyError, TypeError, ValueError):
+ return None
+ totals["passed"] = max(
+ 0,
+ totals["tests"]
+ - totals["failures"]
+ - totals["errors"]
+ - totals["skipped"],
+ )
+ return totals
+
+
+def _strict_pytest_pass(returncode: int, report: Path) -> bool:
+ """Require a completed run with real passing tests and no bad outcomes."""
+ stats = _junit_stats(report)
+ return bool(
+ returncode == 0
+ and stats
+ and stats["tests"] >= 1
+ and stats["passed"] >= 1
+ and stats["failures"] == 0
+ and stats["errors"] == 0
+ and stats["skipped"] == 0
+ )
+
+
+def _pytest_outcome_counts(audit_log: Path, nonce: str) -> Dict[str, int]:
+ """Classify conclusive shim runs using exit status and JUnit results.
+
+ Help/version, zero-collection, all-skipped and malformed/missing reports are
+ inconclusive: they are neither successes nor observed test failures.
+ """
+ result_line = re.compile(rf"^{re.escape(nonce)} result (\d+): (-?\d+)$")
+ try:
+ lines = audit_log.read_text().splitlines()
+ except OSError:
+ lines = []
+
+ successes = 0
+ failures = 0
+ for line in lines:
+ match = result_line.match(line)
+ if not match:
+ continue
+ attempt, returncode = int(match.group(1)), int(match.group(2))
+ report = audit_log.parent / f"pytest-{nonce}-{attempt}.xml"
+ stats = _junit_stats(report)
+ if _strict_pytest_pass(returncode, report):
+ successes += 1
+ elif stats and (stats["failures"] > 0 or stats["errors"] > 0):
+ failures += 1
+ return {"successes": successes, "failures": failures}
+
+
+def _snapshot_protected_files(project_dir: Path, names: List[str]) -> Dict[str, str]:
+ """Hash scenario files that the evaluated agent must not change."""
+ snapshot: Dict[str, str] = {}
+ root = project_dir.resolve()
+ for name in names:
+ relative = Path(name)
+ path = project_dir / relative
+ if relative.is_absolute() or ".." in relative.parts:
+ raise ValueError(f"Invalid protected file path: {name!r}")
+ if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root):
+ raise ValueError(f"Protected file is missing or unsafe: {name!r}")
+ snapshot[name] = hashlib.sha256(path.read_bytes()).hexdigest()
+ return snapshot
+
+
+def _protected_files_unchanged(project_dir: Path, snapshot: Dict[str, str]) -> bool:
+ root = project_dir.resolve()
+ for name, expected in snapshot.items():
+ path = project_dir / name
+ try:
+ if path.is_symlink() or not path.is_file():
+ return False
+ if not path.resolve().is_relative_to(root):
+ return False
+ if hashlib.sha256(path.read_bytes()).hexdigest() != expected:
+ return False
+ except OSError:
+ return False
+ return True
+
+
+def _run_scenario(
+ scenario: Dict[str, Any],
+ superpowers_dir: Path,
+ skill_name: str,
+ skill_overlay: Optional[Path],
+ workspace: Path,
+ timeout: int = DEFAULT_TIMEOUT,
+ pinned_sha: str = DEFAULT_SHA,
+ candidate_hash: str = "",
+) -> ScenarioResult:
+ """Run a single scenario.
+
+ If skill_overlay is provided, copies it into superpowers_dir at
+ skills//SKILL.md. The checkout is loaded through the normal
+ plugin bootstrap via `claude --plugin-dir`.
+ """
+ import time
+
+ if os.name != "posix":
+ # bash shims + claude/git shell-out are POSIX-only
+ raise RuntimeError("Superpowers adapter requires a POSIX host (bash).")
+
+ sid = scenario["id"]
+ result = ScenarioResult(
+ id=sid, passed=False,
+ pinned_sha=pinned_sha, candidate_hash=candidate_hash,
+ scenario_seed=_seed(pinned_sha, sid),
+ )
+
+ # Isolated project, HOME and (harness-only) audit dir per scenario
+ project_dir = workspace / f"project-{sid}"
+ project_dir.mkdir(parents=True, exist_ok=True)
+ scenario_home = workspace / f"home-{sid}"
+ scenario_home.mkdir(parents=True, exist_ok=True)
+ # shim + audit log live under the scenario HOME, alongside the agent's own
+ # config; harness-created but reachable by the agent (trusted-candidate scope)
+ audit_log = scenario_home / ".skillopt" / "pytest.log"
+ bin_dir = scenario_home / ".skillopt" / "bin"
+ run_nonce = os.urandom(8).hex()
+ _write_pytest_shims(bin_dir, audit_log, run_nonce)
+
+ # Write setup files
+ for filename, content in scenario.get("setup", {}).get("files", {}).items():
+ (project_dir / filename).write_text(content)
+ protected_snapshot = _snapshot_protected_files(
+ project_dir, list(scenario.get("protected_files", []))
+ )
+
+ # skill_name becomes a path segment - reject traversal/separators up front
+ if skill_name in ("", ".", "..") or "/" in skill_name or "\\" in skill_name:
+ raise ValueError(f"Invalid skill name: {skill_name!r}")
+
+ # Overlay candidate skill into temp superpowers copy at correct path
+ if skill_overlay is not None:
+ if skill_overlay.is_symlink():
+ raise ValueError(f"Candidate skill must not be a symlink: {skill_overlay}")
+ if not skill_overlay.exists():
+ raise FileNotFoundError(f"Candidate skill not found: {skill_overlay}")
+ if not skill_overlay.is_file():
+ raise ValueError(f"Candidate skill is not a regular file: {skill_overlay}")
+ skills_root = superpowers_dir / "skills"
+ skill_dir = skills_root / skill_name
+ skill_dest = skill_dir / "SKILL.md"
+ # A malicious pinned checkout could carry a symlink at these components
+ # that redirects the write outside the workspace. Refuse symlinked
+ # components and confirm the resolved dir stays under workspace BEFORE
+ # writing anything (copy2 would otherwise follow them first).
+ for p in (skills_root, skill_dir, skill_dest):
+ if p.is_symlink():
+ raise ValueError(f"Refusing symlinked overlay path: {p}")
+ skill_dir.mkdir(parents=True, exist_ok=True)
+ if not skill_dir.resolve().is_relative_to(workspace):
+ raise ValueError(f"Skill path {skill_dir} escapes workspace {workspace}")
+ shutil.copy2(skill_overlay, skill_dest, follow_symlinks=False)
+
+ # Per-run random marker (see _new_marker)
+ marker = _new_marker()
+ bootstrap_skill = superpowers_dir / "skills" / "using-superpowers" / "SKILL.md"
+ bootstrap_present = bootstrap_skill.exists()
+ if bootstrap_present:
+ # checkout is reused across scenarios - strip any prior marker block first
+ base = bootstrap_skill.read_text().split("\n\n## Session marker\n")[0]
+ bootstrap_skill.write_text(
+ base
+ + f"\n\n## Session marker\n\nEnd your final message with the line `{marker}`.\n"
+ )
+ else:
+ # marker can't be injected -> invalid eval. Fail closed early rather than
+ # run the agent (and untrusted code) for a result that can't be scored.
+ result.error = "BOOTSTRAP_SKILL_MISSING"
+ result.evidence = {"bootstrap_present": False}
+ return result
+
+ claude_dir = scenario_home / ".claude"
+ claude_dir.mkdir(parents=True, exist_ok=True)
+
+ # Auth. Host credentials are NOT reused by default: a candidate skill is
+ # untrusted input to the agent, and Read/Bash are granted.
+ host_auth = os.environ.get("SKILLOPT_HOST_AUTH") == "1"
+ api_key = os.environ.get("ANTHROPIC_API_KEY", "")
+ if host_auth:
+ import warnings
+ warnings.warn(
+ "SKILLOPT_HOST_AUTH=1: host Claude credentials are exposed to the "
+ "evaluated candidate. Use only with trusted candidates.",
+ stacklevel=2,
+ )
+ real_claude_dir = Path.home() / ".claude"
+ for auth_file in ("credentials.json", ".credentials.json"):
+ src = real_claude_dir / auth_file
+ if src.exists() and not (claude_dir / auth_file).exists():
+ (claude_dir / auth_file).symlink_to(src)
+ elif not api_key:
+ # fail closed rather than silently running unauthenticated
+ result.error = "NO_AUTH"
+ return result
+
+ # Minimal PATH by default: shim dir + standard system dirs only, so the
+ # agent doesn't inherit host-specific tooling. Opt in to the full host PATH
+ # with SKILLOPT_INHERIT_PATH=1. (Not a hard boundary - a Bash-holding agent
+ # can still invoke absolute paths - this is hygiene, not confinement.)
+ if os.environ.get("SKILLOPT_INHERIT_PATH") == "1":
+ base_path = os.environ.get("PATH", "/usr/bin:/bin")
+ else:
+ base_path = "/usr/bin:/bin"
+ env = {
+ "HOME": str(scenario_home),
+ "PATH": f"{bin_dir}{os.pathsep}{base_path}",
+ "TERM": os.environ.get("TERM", "xterm"),
+ "LANG": os.environ.get("LANG", "en_US.UTF-8"),
+ }
+ if api_key:
+ env["ANTHROPIC_API_KEY"] = api_key
+
+ prompt = scenario.get("prompt", "").strip()
+
+ # Prompt on stdin + text output, matching backend.py's Claude CLI usage.
+ # (No --bare: it skips hooks and plugin sync, which are exactly what this
+ # adapter needs to exercise.)
+ # Absolute host path so it resolves under the minimal PATH; override with
+ # SKILLOPT_CLAUDE_BIN.
+ claude_bin = os.environ.get("SKILLOPT_CLAUDE_BIN") or shutil.which("claude") or "claude"
+ cmd = [claude_bin, "-p", "--output-format", "text", "--plugin-dir", str(superpowers_dir)]
+
+ # Permission handling for non-interactive execution. Neither mode is an
+ # isolation boundary - this adapter is for TRUSTED candidates only
+ # (see docs/superpowers/SECURITY.md):
+ # - Default: --allowedTools scopes tools
+ # - SKILLOPT_UNSAFE=1: blanket bypass
+ if os.environ.get("SKILLOPT_UNSAFE") == "1":
+ import warnings
+ warnings.warn(
+ "SKILLOPT_UNSAFE=1: Running with --dangerously-skip-permissions. "
+ "Do not use with untrusted candidate skills.",
+ stacklevel=2,
+ )
+ cmd.append("--dangerously-skip-permissions")
+ else:
+ cmd.extend(["--allowedTools", "Bash,Edit,Write,Read"])
+
+ t0 = time.time()
+ try:
+ proc = subprocess.run(
+ cmd,
+ cwd=str(project_dir),
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ env=env,
+ input=prompt,
+ )
+ result.output = proc.stdout + proc.stderr
+ result.latency_ms = (time.time() - t0) * 1000
+
+ # fail-closed on non-zero exit
+ if proc.returncode != 0:
+ result.error = f"EXIT_{proc.returncode}"
+ return result
+
+ except subprocess.TimeoutExpired:
+ result.error = "TIMEOUT"
+ result.latency_ms = timeout * 1000
+ return result
+ except FileNotFoundError:
+ # name the missing binary rather than always blaming claude
+ result.error = f"EXEC_NOT_FOUND:{cmd[0]}"
+ return result
+ except Exception as e:
+ result.error = str(e)
+ return result
+
+ # Estimate tokens (rough: ~4 chars per token)
+ result.tokens = (len(prompt) + len(result.output)) // 4
+
+ # Harness-collected evidence, read after the agent has exited
+ outcomes = _pytest_outcome_counts(audit_log, run_nonce)
+ protected_unchanged = _protected_files_unchanged(
+ project_dir, protected_snapshot
+ )
+ result.evidence = {
+ "pytest_runs": _pytest_run_count(audit_log, run_nonce),
+ "pytest_successes": outcomes["successes"],
+ "pytest_failures": outcomes["failures"],
+ "pytest_after_edit": _pytest_after_edit(audit_log, project_dir),
+ "protected_files_unchanged": protected_unchanged,
+ "bootstrap_loaded": marker in result.output,
+ "bootstrap_present": bootstrap_present,
+ "candidate_hash": candidate_hash,
+ }
+ if any(
+ c.get("op") == "harness_test_passes"
+ for c in scenario.get("judge", {}).get("checks", [])
+ ):
+ # Never execute a test file after detecting that the evaluated agent
+ # changed it. Source edits remain expected in scenarios that allow them.
+ result.evidence["harness_test_passes"] = (
+ _harness_verify(
+ project_dir,
+ env,
+ timeout,
+ test_paths=list(protected_snapshot),
+ )
+ if protected_unchanged
+ else False
+ )
+
+ checks = list(scenario.get("judge", {}).get("checks", []))
+ if protected_snapshot:
+ checks.append({
+ "op": "protected_files_unchanged",
+ "description": "Protected scenario files must remain unchanged",
+ })
+ # every run must show the plugin bootstrap was actually active
+ checks.append({"op": "regex", "arg": re.escape(marker),
+ "description": "Superpowers bootstrap loaded (session marker echoed)"})
+
+ all_pass = True
+ for check in checks:
+ check_pass = _score_check(check, result.output, project_dir, result.evidence)
+ result.checks.append({"description": check.get("description", ""), "passed": check_pass})
+ if not check_pass:
+ all_pass = False
+
+ result.passed = all_pass
+ return result
+
+
+def _harness_verify(
+ project_dir: Path,
+ env: Dict[str, str],
+ timeout: int = DEFAULT_TIMEOUT,
+ test_paths: Optional[List[str]] = None,
+) -> bool:
+ """Re-run the project's tests ourselves - agent output cannot fake this.
+
+ SECURITY: this executes agent-modified project code on the host, same as the
+ agent did. Trusted candidates only (see docs/superpowers/SECURITY.md). The
+ credential is dropped so it never reaches that code.
+ """
+ verify_env = {**env, "SKILLOPT_ATTEMPT": "99"}
+ verify_env.pop("ANTHROPIC_API_KEY", None)
+ shim_dir = str(Path(env.get("HOME", "")) / ".skillopt" / "bin")
+ verify_env["PATH"] = os.pathsep.join(
+ entry
+ for entry in env.get("PATH", "").split(os.pathsep)
+ if entry and entry != shim_dir
+ )
+ try:
+ with tempfile.TemporaryDirectory(prefix="skillopt-pytest-") as tmpdir:
+ report = Path(tmpdir) / "verify.xml"
+ # Never reuse bytecode created by the agent run. Timestamp-based
+ # pyc validation can otherwise accept a same-size source edit made
+ # within the filesystem's timestamp granularity.
+ verify_env["PYTHONPYCACHEPREFIX"] = str(Path(tmpdir) / "pycache")
+ # Synthetic scenarios do not need project conftest/config. Ignoring
+ # them prevents an agent-created collection hook or addopts setting
+ # from hiding a protected test during the independent re-run.
+ command = [
+ sys.executable,
+ "-m",
+ "pytest",
+ "-q",
+ "--noconftest",
+ "-c",
+ os.devnull,
+ "--rootdir",
+ str(project_dir),
+ "-p",
+ "no:cacheprovider",
+ *(test_paths or []),
+ f"--junitxml={report}",
+ ]
+ proc = subprocess.run(
+ command,
+ cwd=str(project_dir), capture_output=True, text=True, timeout=timeout,
+ env=verify_env,
+ )
+ return _strict_pytest_pass(proc.returncode, report)
+ except Exception:
+ return False
+
+
+def _run_git_step(args: List[str], cwd: Path, timeout: int) -> None:
+ """Run one bounded git setup step with a clear timeout failure."""
+ command = ["git", *args]
+ try:
+ subprocess.run(
+ command,
+ cwd=str(cwd),
+ capture_output=True,
+ check=True,
+ timeout=timeout,
+ )
+ except subprocess.TimeoutExpired as exc:
+ raise RuntimeError(
+ f"git step timed out after {timeout}s: {' '.join(command)}"
+ ) from exc
+
+
+class SuperpowersEvaluator:
+ """Evaluator for Superpowers skills."""
+
+ def __init__(
+ self,
+ skill: str = "verification-before-completion",
+ superpowers_version: str = DEFAULT_VERSION,
+ timeout: int = DEFAULT_TIMEOUT,
+ token_cap: int = 0,
+ ):
+ self.skill = skill
+ # NOTE: superpowers_version is a REPORTING LABEL only. The evaluated
+ # checkout is controlled solely by `pinned_sha` (--sha). Set both to
+ # matching values; changing version alone does not change what is run.
+ self.version = superpowers_version
+ self.timeout = timeout
+ self.token_cap = token_cap # 0 = no cap
+
+ def evaluate(
+ self,
+ candidate_skill_path: Optional[str] = None,
+ scenario_filter: Optional[str] = None,
+ pinned_sha: str = DEFAULT_SHA,
+ ) -> EvalResults:
+ """Evaluate a skill against scenarios.
+
+ Clones superpowers at pinned_sha, overlays candidate skill into the temp
+ copy so harness and judge see the same tree. Stops early if token_cap exceeded.
+
+ Raises:
+ FileNotFoundError: if candidate_skill_path is provided but doesn't exist
+ """
+ if not re.fullmatch(r"[0-9a-f]{40}", pinned_sha):
+ raise ValueError(
+ f"pinned_sha must be a full 40-char commit hash, got {pinned_sha!r}"
+ )
+
+ results = EvalResults(skill=self.skill, version=self.version, pinned_sha=pinned_sha)
+ scenarios = _get_scenarios(self.skill)
+
+ # fail explicitly on a typo'd --scenario rather than returning an empty
+ # (score=0.0) result that looks like a real evaluation
+ if scenario_filter and not any(s["id"] == scenario_filter for s in scenarios):
+ valid = ", ".join(s["id"] for s in scenarios)
+ raise ValueError(f"Unknown scenario '{scenario_filter}'. Valid: {valid}")
+
+ candidate_path = Path(candidate_skill_path) if candidate_skill_path else None
+ # fail explicitly if candidate path provided but missing or not a file
+ if candidate_path and not candidate_path.exists():
+ raise FileNotFoundError(f"Candidate skill not found: {candidate_skill_path}")
+ # is_file() is True for symlinks-to-files; reject them so the overlay
+ # copies real content (not a link) and can't point at an arbitrary host file
+ if candidate_path and candidate_path.is_symlink():
+ raise ValueError(f"Candidate skill must not be a symlink: {candidate_skill_path}")
+ if candidate_path and not candidate_path.is_file():
+ raise ValueError(f"Candidate skill is not a regular file: {candidate_skill_path}")
+ candidate_hash = _hash_file(candidate_path) if candidate_path else ""
+
+ with tempfile.TemporaryDirectory(prefix="skillopt-superpowers-") as tmpdir:
+ workspace = Path(tmpdir)
+
+ # Clone superpowers at pinned SHA into temp (init+fetch avoids wasted default-branch clone)
+ superpowers_copy = workspace / "superpowers"
+ superpowers_copy.mkdir()
+ _run_git_step(["init"], superpowers_copy, self.timeout)
+ _run_git_step(
+ ["remote", "add", "origin", SUPERPOWERS_REPO],
+ superpowers_copy,
+ self.timeout,
+ )
+ _run_git_step(
+ ["fetch", "--depth=1", "origin", pinned_sha],
+ superpowers_copy,
+ self.timeout,
+ )
+ _run_git_step(
+ ["checkout", "FETCH_HEAD"], superpowers_copy, self.timeout
+ )
+
+ for scenario in scenarios:
+ if scenario_filter and scenario["id"] != scenario_filter:
+ continue
+
+ # Check token budget
+ if self.token_cap > 0 and results.total_tokens >= self.token_cap:
+ break
+
+ result = _run_scenario(
+ scenario,
+ superpowers_dir=superpowers_copy,
+ skill_name=self.skill,
+ skill_overlay=candidate_path,
+ workspace=workspace,
+ timeout=self.timeout,
+ pinned_sha=pinned_sha,
+ candidate_hash=candidate_hash,
+ )
+ results.scenarios.append(result)
+
+ return results
+
+
+def evaluate_skill(
+ skill: str = "verification-before-completion",
+ candidate_path: Optional[str] = None,
+ version: str = DEFAULT_VERSION,
+ scenario: Optional[str] = None,
+ pinned_sha: str = DEFAULT_SHA,
+) -> Dict[str, Any]:
+ """Convenience function."""
+ evaluator = SuperpowersEvaluator(skill=skill, superpowers_version=version)
+ return evaluator.evaluate(candidate_path, scenario_filter=scenario, pinned_sha=pinned_sha).to_dict()
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Evaluate a Superpowers skill")
+ parser.add_argument("--skill", default="verification-before-completion")
+ parser.add_argument("--candidate", help="Path to candidate SKILL.md")
+ parser.add_argument("--scenario", help="Run only this scenario")
+ parser.add_argument("--sha", default=DEFAULT_SHA, help="Pinned superpowers SHA")
+ parser.add_argument("--json", action="store_true")
+
+ args = parser.parse_args()
+
+ try:
+ results = evaluate_skill(args.skill, args.candidate, scenario=args.scenario, pinned_sha=args.sha)
+ except subprocess.CalledProcessError as e:
+ # git init/fetch/checkout failure (bad SHA, no network, no git)
+ print(f"Error: git step failed ({' '.join(map(str, e.cmd))}): exit {e.returncode}",
+ file=sys.stderr)
+ sys.exit(1)
+ except (FileNotFoundError, ValueError, RuntimeError) as e:
+ # missing candidate/git/claude, unknown scenario, non-POSIX host
+ print(f"Error: {e}", file=sys.stderr)
+ sys.exit(1)
+
+ # fail-closed - exit non-zero if any scenario has error
+ has_errors = any(s.get("error") for s in results["scenarios"])
+
+ if args.json:
+ print(json.dumps(results, indent=2))
+ else:
+ print(f"Skill: {results['skill']}")
+ print(f"Score: {results['score']:.2%}")
+ print(f"Passed: {results['passed']}/{results['passed'] + results['failed']}")
+ for s in results["scenarios"]:
+ status = "â" if s["passed"] else "â"
+ err = f" [{s['error']}]" if s.get("error") else ""
+ print(f" {status} {s['id']}{err}")
+
+ if has_errors:
+ sys.exit(1)
diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py
index cd130a4..f2cc3c8 100644
--- a/skillopt_sleep/backend.py
+++ b/skillopt_sleep/backend.py
@@ -47,6 +47,11 @@ class Backend:
name = "base"
# Optional user preferences (free text) injected into reflect as a prior.
preferences: str = ""
+ # Optional per-night evidence log (skillopt_sleep.evidence.EvidenceLog).
+ # Attached by the cycle; None => no observability overhead. The phase tag
+ # labels which consolidation step subsequent replay calls belong to.
+ evidence = None
+ evidence_phase: str = ""
def attempt(self, task: TaskRecord, skill: str, memory: str,
sample_id: int = 0) -> str:
@@ -330,11 +335,23 @@ class CliBackend(Backend):
raise NotImplementedError
def _cached_call(self, key: str, prompt: str, *, max_tokens: int = 1024) -> str:
+ kind = key.split(":", 1)[0]
+ ev = getattr(self, "evidence", None)
if key in self._cache:
+ # cache hits log key-only (the full text is on the original miss event)
+ if ev is not None:
+ ev.log("replay", "model_call", kind=kind, cache_hit=True, key=key,
+ phase=getattr(self, "evidence_phase", ""), backend=self.name,
+ model=self.model)
return self._cache[key]
out = self._call(prompt, max_tokens=max_tokens)
self._tokens += len(prompt) // 4 + len(out) // 4
self._cache[key] = out
+ if ev is not None:
+ ev.log("replay", "model_call", kind=kind, cache_hit=False, key=key,
+ phase=getattr(self, "evidence_phase", ""), backend=self.name,
+ model=self.model, prompt=prompt, response=out,
+ error=getattr(self, "last_call_error", "") or "")
return out
# operations -----------------------------------------------------------
@@ -363,16 +380,15 @@ class CliBackend(Backend):
return self._cached_call(key, prompt, max_tokens=512)
# generic path (mined daily-case tasks): neutral, content-filter-safe
# wording. Apply the skill/memory as guidance, not as adversarial
- # "OVERRIDE everything" directives.
- prompt = (
- "Complete the following task for the user. Follow the skill and memory "
- "guidance below, including any output-format and length requirements. "
- "When a 'Learned preferences' rule sets an explicit limit (e.g. a length "
- "cap), prefer that rule over more general advice it refines.\n\n"
- f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n"
- f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n"
- "Return ONLY the final answer text, nothing else."
- )
+ # "OVERRIDE everything" directives. Template lives in the prompt
+ # registry so the dashboard can display/override it live.
+ from skillopt_sleep import prompts as prompt_registry
+ prompt = prompt_registry.render("attempt", {
+ "__SKILL__": skill or "(none)",
+ "__MEMORY__": memory or "(none)",
+ "__INTENT__": task.intent,
+ "__CONTEXT__": task.context_excerpt,
+ })
# cache on (task, skill, memory) so identical hold-out re-scoring is free
salt = f"s{sample_id}:" if sample_id else ""
key = "attempt:" + salt + skill_hash(prompt)
@@ -395,11 +411,11 @@ class CliBackend(Backend):
if task.reference_kind == "exact" and task.reference:
hard = exact_score(task.reference, response)
return hard, max(hard, keyword_soft_score(task.reference, response)), "exact(local)"
- prompt = (
- "Score how well the response satisfies the rubric, 0..1. "
- 'Return ONLY JSON {"score": <0..1>, "reason": "..."}.\n\n'
- f"# Rubric\n{task.reference or task.intent}\n\n# Response\n{response}"
- )
+ from skillopt_sleep import prompts as prompt_registry
+ prompt = prompt_registry.render("judge", {
+ "__RUBRIC__": task.reference or task.intent,
+ "__RESPONSE__": response,
+ })
key = "judge:" + skill_hash(prompt)
raw = self._cached_call(key, prompt, max_tokens=200)
obj = _extract_json(raw, "object")
@@ -482,39 +498,20 @@ class CliBackend(Backend):
# can't ask questions). We surface the benchmark's own rollout system
# prompt (carried on TaskRecord.system) so proposed rules stay in-bounds.
guard_text = _task_guardrail(failures)
- prompt = (
- "You are SkillOpt's optimizer. The agent keeps failing the recurring "
- f"tasks below. Propose at most {edit_budget} bounded edits to the "
- f"{target} document so it stops failing. Each edit MUST be a short, "
- "GENERAL, reusable rule or preference (never task-specific, never an "
- "answer to a single task). If exact failing criteria are listed, your "
- "edits MUST make future outputs satisfy every one of them.\n"
- "BE CONCRETE: quote the exact threshold, section name, or format from "
- "the criteria verbatim in your rule (e.g. write 'keep the entire "
- "response under 1200 characters', NOT 'respect length limits'). Vague "
- "rules do not change behavior; specific numeric/structural rules do.\n"
- "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
- "you CANNOT delete the existing instructions above. If the current "
- f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' "
- "but outputs must be under a character limit), write an explicit, "
- "forceful OVERRIDE rule stating it supersedes the conflicting "
- "instruction, and put the hard requirement first.\n"
- "HARD CONSTRAINT: every rule you write MUST be consistent with the "
- "'Task output contract' below (if shown). NEVER propose a rule that "
- "changes the required output format/language, tells the agent to ask "
- "the user a question, or otherwise violates that contract â such a "
- "rule scores ZERO because the evaluator cannot honor it.\n"
- 'Return ONLY a JSON array: '
- '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n'
- f"# Current {target}\n{cur_doc}\n"
- f"{guard_text}"
- f"{criteria_text}\n"
- f"{pref_text}\n\n"
- f"# Recurring failures\n{fail_text}"
- )
+ from skillopt_sleep import prompts as prompt_registry
+ prompt = prompt_registry.render("reflect", {
+ "__EDIT_BUDGET__": str(edit_budget),
+ "__TARGET__": target,
+ "__CUR_DOC__": cur_doc,
+ "__GUARD__": guard_text,
+ "__CRITERIA__": criteria_text,
+ "__PREFS__": pref_text,
+ "__FAILURES__": fail_text,
+ })
# Call with one retry: transient non-JSON replies otherwise waste a whole
# night (the gate sees no edits and rejects). A firmer second prompt
# recovers most of these.
+ ev = getattr(self, "evidence", None)
arr = None
for attempt in range(2):
p = prompt if attempt == 0 else (
@@ -523,6 +520,11 @@ class CliBackend(Backend):
)
raw = self._call(p, max_tokens=1024)
self._tokens += len(p) // 4 + len(raw) // 4
+ if ev is not None:
+ ev.log("reflect", "exchange", target=target, attempt=attempt + 1,
+ backend=self.name, model=self.model,
+ n_failures=len(failures), prompt=p, raw_reply=raw,
+ error=getattr(self, "last_call_error", "") or "")
arr = _extract_json(raw, "array")
if isinstance(arr, list) and arr:
break
@@ -1007,6 +1009,27 @@ def resolve_copilot_path(explicit: str = "") -> str:
return found or "copilot"
+def resolve_cursor_path(explicit: str = "") -> str:
+ """Find the Cursor Agent CLI (``cursor-agent``)."""
+ if explicit:
+ return os.path.expanduser(explicit)
+ env = os.environ.get("SKILLOPT_SLEEP_CURSOR_PATH")
+ if env:
+ return os.path.expanduser(env)
+ import shutil
+
+ found = shutil.which("cursor-agent")
+ return found or "cursor-agent"
+
+
+class CursorBackendError(RuntimeError):
+ """A redacted Cursor Agent process or response failure."""
+
+ def __init__(self, message: str, *, retryable: bool = False) -> None:
+ super().__init__(message)
+ self.retryable = retryable
+
+
class CopilotCliBackend(CliBackend):
"""Drives the GitHub Copilot CLI in non-interactive mode.
@@ -1207,6 +1230,240 @@ class CopilotCliBackend(CliBackend):
pass
+class CursorCliBackend(CliBackend):
+ """Drive an authenticated Cursor Agent CLI in an isolated workspace.
+
+ Cursor's JSON print format has one final ``result`` object. Ordinary calls
+ use Ask mode, which is read-only. Tool-aware replay is disabled until the
+ Cursor Agent permission boundary has been validated against the live CLI.
+ """
+
+ name = "cursor"
+ _AUTH_ERROR_MARKERS = (
+ "not authenticated",
+ "authentication required",
+ "not logged in",
+ "please log in",
+ "login required",
+ "unauthorized",
+ "invalid api key",
+ "401",
+ "403",
+ )
+ _CONFIG_ERROR_MARKERS = (
+ "unknown option",
+ "invalid option",
+ "invalid model",
+ "unsupported model",
+ "model not found",
+ "not available for your account",
+ "invalid configuration",
+ )
+ _ASK_DENY = ["Read(**)", "Write(**)", "Mcp(*:*)"]
+ # Keep the Cursor child usable across shells, credential stores, proxies,
+ # and enterprise CA setups without forwarding unrelated provider or cloud
+ # credentials from the host process.
+ _ENV_ALLOWLIST = (
+ "PATH", "HOME", "USER", "LOGNAME", "SHELL",
+ "USERPROFILE", "HOMEDRIVE", "HOMEPATH",
+ "SYSTEMROOT", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT",
+ "TMPDIR", "TMP", "TEMP",
+ "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TERM", "NO_COLOR",
+ "CURSOR_API_KEY",
+ "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR",
+ "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY",
+ "http_proxy", "https_proxy", "all_proxy", "no_proxy",
+ "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS",
+ )
+
+ def __init__(self, model: str = "", cursor_path: str = "", timeout: int = 240) -> None:
+ super().__init__(model=model or os.environ.get("SKILLOPT_SLEEP_CURSOR_MODEL", ""), timeout=timeout)
+ self.cursor_path = resolve_cursor_path(cursor_path)
+
+ def _command(self, workspace: str) -> List[str]:
+ cmd = [
+ self.cursor_path,
+ "-p",
+ "--output-format",
+ "json",
+ "--trust",
+ "--workspace",
+ workspace,
+ "--mode",
+ "ask",
+ ]
+ if self.model:
+ cmd += ["--model", self.model]
+ return cmd
+
+ @staticmethod
+ def _terminal_result(raw: str) -> Optional[Dict[str, Any]]:
+ candidates = [raw.strip()] if raw.strip() else []
+ candidates.extend(line.strip() for line in raw.splitlines() if line.strip().startswith("{"))
+ for candidate in reversed(candidates):
+ try:
+ obj = json.loads(candidate)
+ except Exception:
+ continue
+ if isinstance(obj, dict) and obj.get("type") == "result":
+ return obj
+ return None
+
+ @classmethod
+ def _parse_json_response(cls, raw: str) -> str:
+ """Return text from the terminal successful Cursor result."""
+ terminal = cls._terminal_result(raw)
+ if terminal is None or terminal.get("is_error") is True:
+ return ""
+ result = terminal.get("result")
+ if isinstance(result, str):
+ return result.strip()
+ return ""
+
+ def _error(self, message: str, *, retryable: bool = False) -> CursorBackendError:
+ import logging
+
+ from skillopt_sleep.staging import redact_secrets
+
+ self.last_call_error = str(redact_secrets(message))[:500]
+ logging.getLogger("skillopt_sleep").warning("Cursor Agent call failed: %s", self.last_call_error)
+ return CursorBackendError(self.last_call_error, retryable=retryable)
+
+ @staticmethod
+ def _isolated_environment(runtime_dir: str) -> Dict[str, str]:
+ config_dir = os.path.join(runtime_dir, "config")
+ data_dir = os.path.join(runtime_dir, "data")
+ os.makedirs(config_dir, exist_ok=True)
+ os.makedirs(data_dir, exist_ok=True)
+ with open(os.path.join(config_dir, "cli-config.json"), "w", encoding="utf-8") as f:
+ json.dump(
+ {
+ "version": 1,
+ "editor": {"vimMode": False},
+ "permissions": {
+ "allow": [],
+ "deny": CursorCliBackend._ASK_DENY,
+ },
+ "approvalMode": "allowlist",
+ "sandbox": {
+ "mode": "disabled",
+ "networkAccess": "user_config_only",
+ "networkAllowlist": [],
+ },
+ "network": {"useHttp1ForAgent": False},
+ "hasChangedDefaultModel": False,
+ "attribution": {
+ "attributeCommitsToAgent": False,
+ "attributePRsToAgent": False,
+ },
+ },
+ f,
+ indent=2,
+ )
+ env = {
+ key: os.environ[key]
+ for key in CursorCliBackend._ENV_ALLOWLIST
+ if key in os.environ
+ }
+ env["CURSOR_CONFIG_DIR"] = config_dir
+ env["CURSOR_DATA_DIR"] = data_dir
+ return env
+
+ def _invoke_once(self, prompt: str, workspace: str) -> str:
+ import shutil
+
+ from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL
+
+ self.last_call_error = ""
+ replay_prompt = CURSOR_REPLAY_SENTINEL + "\n\n" + prompt
+ runtime_dir = ""
+ try:
+ runtime_dir = tempfile.mkdtemp(prefix="skillopt_sleep_cursor_runtime_")
+ proc = subprocess.run(
+ self._command(workspace),
+ capture_output=True,
+ creationflags=_NO_WINDOW,
+ text=True,
+ encoding="utf-8",
+ errors="replace",
+ timeout=self.timeout,
+ cwd=workspace,
+ input=replay_prompt,
+ env=self._isolated_environment(runtime_dir),
+ )
+ except subprocess.TimeoutExpired:
+ raise self._error(
+ f"Cursor Agent timed out after {self.timeout}s",
+ retryable=True,
+ )
+ except Exception as exc:
+ raise self._error(f"Cursor Agent spawn failed: {exc}")
+ finally:
+ if runtime_dir:
+ shutil.rmtree(runtime_dir, ignore_errors=True)
+
+ if proc.returncode != 0:
+ raise self._error(
+ f"Cursor Agent exited {proc.returncode}: {(proc.stderr or '')[:500]}"
+ )
+ terminal = self._terminal_result(proc.stdout or "")
+ if terminal is not None and terminal.get("is_error") is True:
+ detail = terminal.get("result") or terminal.get("error") or proc.stderr or ""
+ raise self._error(f"Cursor Agent returned an error result: {str(detail)[:300]}")
+ output = self._parse_json_response(proc.stdout or "")
+ if not output:
+ detail = (proc.stderr or "").strip()
+ if any(marker in detail.casefold() for marker in self._AUTH_ERROR_MARKERS):
+ raise self._error(
+ "Cursor Agent authentication failed"
+ + (f": {detail[:300]}" if detail else "")
+ )
+ if any(marker in detail.casefold() for marker in self._CONFIG_ERROR_MARKERS):
+ raise self._error(
+ "Cursor Agent configuration failed"
+ + (f": {detail[:300]}" if detail else "")
+ )
+ raise self._error(
+ "Cursor Agent returned no usable JSON response"
+ + (f": {detail[:300]}" if detail else ""),
+ retryable=True,
+ )
+ self.last_call_error = ""
+ return output
+
+ def _call(self, prompt: str, *, max_tokens: int = 1024) -> str:
+ del max_tokens
+ workspace = tempfile.mkdtemp(prefix="skillopt_sleep_cursor_")
+ try:
+ for attempt in range(2):
+ try:
+ return self._invoke_once(prompt, workspace)
+ except CursorBackendError as exc:
+ if not exc.retryable or attempt == 1:
+ raise
+ raise AssertionError("unreachable")
+ finally:
+ try:
+ import shutil
+
+ shutil.rmtree(workspace, ignore_errors=True)
+ except Exception:
+ pass
+
+ def attempt_with_tools(
+ self,
+ task: TaskRecord,
+ skill: str,
+ memory: str,
+ tools: List[str],
+ ) -> Tuple[str, List[str]]:
+ del task, skill, memory, tools
+ raise self._error(
+ "Cursor tool-aware replay is temporarily disabled pending live "
+ "Cursor permission-boundary validation"
+ )
+
+
class DualBackend(Backend):
"""Route operations to two backends, Ă la SkillOpt's target vs optimizer.
@@ -1564,6 +1821,7 @@ def get_backend(
model: str = "",
claude_path: str = "claude",
codex_path: str = "",
+ cursor_path: str = "",
azure_endpoint: str = "",
project_dir: str = "",
) -> Backend:
@@ -1579,6 +1837,8 @@ def get_backend(
return AzureResponsesBackend(deployment=model, endpoints=eps)
if n in {"copilot", "github_copilot", "copilot_cli", "gh_copilot"}:
return CopilotCliBackend(model=model)
+ if n in {"cursor", "cursor_agent", "cursor_cli"}:
+ return CursorCliBackend(model=model, cursor_path=cursor_path)
if n in {"handoff", "session", "file"}:
# Lazy import: handoff_backend imports CliBackend from this module.
from skillopt_sleep.handoff_backend import HandoffBackend
@@ -1598,6 +1858,7 @@ def build_backend(
target_backend: str = "",
target_model: str = "",
codex_path: str = "",
+ cursor_path: str = "",
azure_endpoint: str = "",
preferences: str = "",
project_dir: str = "",
@@ -1615,16 +1876,17 @@ def build_backend(
backend,
model=model,
codex_path=codex_path,
+ cursor_path=cursor_path,
azure_endpoint=azure_endpoint,
project_dir=project_dir,
)
be.preferences = preferences
return be
tgt = get_backend(target_backend or backend, model=target_model or model,
- codex_path=codex_path, azure_endpoint=azure_endpoint,
+ codex_path=codex_path, cursor_path=cursor_path, azure_endpoint=azure_endpoint,
project_dir=project_dir)
opt = get_backend(optimizer_backend or backend, model=optimizer_model or model,
- codex_path=codex_path, azure_endpoint=azure_endpoint,
+ codex_path=codex_path, cursor_path=cursor_path, azure_endpoint=azure_endpoint,
project_dir=project_dir)
opt.preferences = preferences # reflect runs on the optimizer
dual = DualBackend(target=tgt, optimizer=opt)
diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py
index 7478538..3c10567 100644
--- a/skillopt_sleep/config.py
+++ b/skillopt_sleep/config.py
@@ -19,14 +19,16 @@ from typing import Any, Dict, Optional
HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep")
CLAUDE_HOME = os.path.expanduser("~/.claude")
CODEX_HOME = os.path.expanduser("~/.codex")
+CURSOR_HOME = os.path.expanduser("~/.cursor")
DEFAULTS: Dict[str, Any] = {
# ââ scope ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
"claude_home": CLAUDE_HOME,
"codex_home": CODEX_HOME,
- "transcript_source": "claude", # "claude" | "codex" | "copilot" | "auto"
+ "cursor_home": CURSOR_HOME,
"vscode_workspace_storage": "", # "" => auto-detect platform defaults
+ "transcript_source": "claude", # "claude" | "codex" | "copilot" | "cursor" | "auto"
"projects": "invoked", # "invoked" | "all" | [list of abs paths]
"invoked_project": "", # filled at runtime (cwd) when projects == "invoked"
"lookback_hours": 72, # harvest window when no prior sleep recorded
@@ -37,15 +39,24 @@ DEFAULTS: Dict[str, Any] = {
"val_fraction": 0.34, # real tasks reserved to gate updates
"test_fraction": 0.0, # real tasks reserved as the final held-out measure
# ââ optimizer ââââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
- "backend": "mock", # "mock" | "claude" | "codex" | "copilot"
+ "backend": "mock", # "mock" | "claude" | "codex" | "copilot" | "cursor"
"model": "", # backend-specific; "" => backend default
+ # Dual-backend split (both empty => single backend above plays all roles).
+ # target = the model whose skill is deployed (runs `attempt` rollouts);
+ # optimizer = the model that mines tasks, judges rubrics, writes edits.
+ "optimizer_backend": "",
+ "optimizer_model": "",
+ "target_backend": "",
+ "target_model": "",
+ "azure_endpoint": "", # explicit endpoint for azure/compat backends
"gate_mode": "on", # "on" (validation-gated) | "off" (greedy, no hard filter)
"codex_path": "", # "" => auto-detect the real @openai/codex binary
+ "cursor_path": "", # "" => auto-detect the Cursor Agent CLI
"edit_budget": 4, # textual learning rate (max edits/night)
"preferences": "", # free-text house rules injected into reflect as a prior
"gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts)
"gate_mixed_weight": 0.5,
- "replay_mode": "mock", # "mock" (sandboxed prompt) | "fresh" (worktree)
+ "replay_mode": "mock", # report label; fresh-worktree replay is not implemented
# ââ dream + recall (opt-in; defaults reproduce the prior single-shot loop) â
"dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task
"dream_factor": 0, # >0 => add N synthetic variants of each task to the dream
@@ -56,6 +67,9 @@ DEFAULTS: Dict[str, Any] = {
"target_skill_path": "", # explicit SKILL.md target for repo-scoped agents
"target_task_filter": True, # prefer mined tasks matching target_skill_path/text
"progress": False, # print phase progress to stderr
+ # ââ observability ââââââââââââââââââââââââââââââââââââââââââââââââââââââ
+ "evidence_log": True, # write per-night evidence.jsonl (full evidentiary chain)
+ "evidence_max_chars": 4000, # per-field truncation cap for evidence events
# ââ adoption / safety ââââââââââââââââââââââââââââââââââââââââââââââââââ
"auto_adopt": False, # default: stage + require explicit `adopt`
"managed_skill_name": "skillopt-sleep-learned",
@@ -109,6 +123,11 @@ class SleepConfig:
def codex_archived_sessions_dir(self) -> str:
return os.path.join(self.data["codex_home"], "archived_sessions")
+ @property
+ def cursor_projects_dir(self) -> str:
+ cursor_home = os.path.abspath(os.path.expanduser(str(self.data["cursor_home"])))
+ return os.path.join(cursor_home, "projects")
+
@property
def vscode_workspace_storage(self) -> str:
value = self.data.get("vscode_workspace_storage", "") or ""
diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py
index a9ea662..89baba5 100644
--- a/skillopt_sleep/consolidate.py
+++ b/skillopt_sleep/consolidate.py
@@ -13,7 +13,7 @@ from dataclasses import dataclass, field
from typing import List, Optional, Tuple
from skillopt_sleep.backend import Backend
-from skillopt_sleep.memory import apply_edits
+from skillopt_sleep.memory import apply_edits_detailed
from skillopt_sleep.replay import aggregate_scores, replay_batch
from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
@@ -37,6 +37,11 @@ class ConsolidationResult:
holdout_baseline: float
holdout_candidate: float
# ââ observability (so a 0.0->0.0 night is self-diagnosing, not a black box) ââ
+ # Edits that changed nothing (anchor not found, or a duplicate add). They are
+ # neither applied nor gate-rejected, so without this list they would vanish
+ # from the report and the night would look like the optimizer produced less
+ # than it did.
+ unmatched_edits: List[EditRecord] = field(default_factory=list)
holdout_detail: List[dict] = field(default_factory=list) # per val task: hard/soft/resp/why
reflect_raw: str = "" # the optimizer's last raw reply (empty => reflect produced nothing)
call_error: str = "" # backend's last call error (timeout/auth/empty)
@@ -54,12 +59,13 @@ def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]
train = [t for t in tasks if _norm(t.split) == "train"]
val = [t for t in tasks if _norm(t.split) == "val"]
- # be robust if a split is empty: fall back so a night still does something,
- # but never silently use test as val.
- test = [t for t in tasks if _norm(t.split) == "test"]
+ # Be robust if a split is empty: fall back so a night still does something,
+ # but never silently use test as train or val. An all-test batch therefore
+ # returns empty train/val (caller scores test separately; gate is a no-op).
if not val:
- # prefer train as the gate reference over nothing; last resort all-but-test
- val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks
+ # Prefer train as the gate reference; otherwise any non-test tasks.
+ # Do not fall back to the full task list (that would leak held-out test).
+ val = train or [t for t in tasks if _norm(t.split) != "test"]
if not train:
train = val
return train, val
@@ -108,6 +114,8 @@ def consolidate(
Skill and memory are evolved in sequence (skill first if both enabled).
"""
+ from skillopt_sleep import evidence as evlog
+ ev = evlog.get(backend)
train_tasks, val_tasks = _split(tasks)
gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"}
holdout_detail: List[dict] = []
@@ -120,12 +128,19 @@ def consolidate(
if gate_off:
base_hard, base_soft = 0.0, 0.0
else:
+ evlog.set_phase(backend, "baseline_val")
base_pairs = replay_batch(backend, val_tasks, skill, memory)
base_hard, base_soft = aggregate_scores(base_pairs)
holdout_detail = _holdout_detail(base_pairs)
base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight)
+ if ev is not None:
+ ev.log("gate", "baseline", gate_mode=("off" if gate_off else "on"),
+ n_train=len(train_tasks), n_val=len(val_tasks),
+ hard=base_hard, soft=base_soft, score=base_score,
+ metric=gate_metric, mixed_weight=gate_mixed_weight)
# ââ reflect over TRAIN-split failures/successes âââââââââââââââââââââââ
+ evlog.set_phase(backend, "train")
train_pairs = replay_batch(backend, train_tasks, skill, memory)
failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0]
successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0]
@@ -133,25 +148,48 @@ def consolidate(
cand_skill, cand_memory = skill, memory
all_applied: List[EditRecord] = []
all_rejected: List[EditRecord] = []
+ all_unmatched: List[EditRecord] = []
+
+ def _edits_payload(edits: List[EditRecord]) -> List[dict]:
+ return [{"op": e.op, "content": e.content, "anchor": e.anchor,
+ "rationale": e.rationale} for e in edits]
def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str:
nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected
+ if ev is not None:
+ ev.log("reflect", "edits_returned", target=which,
+ n_edits=len(edits), edits=_edits_payload(edits))
if not edits:
return doc
- new_doc, applied = apply_edits(doc, edits)
+ new_doc, applied, unmatched = apply_edits_detailed(doc, edits)
+ if unmatched:
+ all_unmatched.extend(unmatched)
+ if ev is not None:
+ ev.log("reflect", "edits_unmatched", target=which,
+ n_edits=len(unmatched), edits=_edits_payload(unmatched))
if not applied:
return doc
# gate OFF: accept greedily with NO val scoring (the daily-use path)
if gate_off:
all_applied.extend(applied)
+ if ev is not None:
+ ev.log("gate", "trial", target=which, mode="greedy",
+ accepted=True, n_edits=len(applied))
return new_doc
# gate ON: score the candidate on the VAL slice, keep only if it improves
trial_skill = new_doc if which == "skill" else cand_skill
trial_memory = new_doc if which == "memory" else cand_memory
+ evlog.set_phase(backend, f"gate_trial:{which}")
pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory)
h, s = aggregate_scores(pairs)
cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight)
- if cand_score > base_score:
+ improved = cand_score > base_score
+ if ev is not None:
+ ev.log("gate", "trial", target=which, mode="gated",
+ baseline_score=base_score, cand_hard=h, cand_soft=s,
+ cand_score=cand_score, accepted=improved,
+ n_edits=len(applied))
+ if improved:
base_score = max(base_score, cand_score)
all_applied.extend(applied)
return new_doc
@@ -204,6 +242,7 @@ def consolidate(
if evolve_memory:
# re-evaluate failures under the (possibly improved) skill
+ evlog.set_phase(backend, "train_post_skill")
train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory)
failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0]
successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0]
@@ -225,6 +264,7 @@ def consolidate(
base_gate_score = 0.0
else:
# scored on the VAL slice (the gate reference)
+ evlog.set_phase(backend, "final_val")
final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory)
final_hard, final_soft = aggregate_scores(final_pairs)
final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight)
@@ -248,6 +288,44 @@ def consolidate(
else:
action = "accept" if final_score > base_gate_score else "reject"
accepted = bool(all_applied) and final_score > base_gate_score
+ # The gate scores documents, not edit bookkeeping: when every proposed
+ # edit was dropped during the per-target trials, `all_applied` is empty
+ # and nothing changed, yet the score comparison can still yield an
+ # accept-flavoured action. Reporting that as "accept_new_best" while
+ # `accepted` is False makes the headline contradict the outcome.
+ if not accepted and action in {"accept", "accept_new_best"}:
+ action = "reject"
+ # A per-target trial can improve and tentatively apply an edit, while a
+ # later fresh final replay regresses. The returned documents already
+ # roll back in that case; keep the edit bookkeeping/report consistent
+ # by moving those tentative edits into the rejected set as well.
+ if not accepted and all_applied:
+ for edit in all_applied:
+ if edit not in all_rejected:
+ all_rejected.append(edit)
+ all_applied = []
+
+ if ev is not None:
+ w = max(0.0, min(1.0, float(gate_mixed_weight)))
+ if gate_metric == "mixed":
+ formula = (
+ f"score = (1-{w})*hard + {w}*soft; "
+ f"baseline = (1-{w})*{base_hard:.3f} + {w}*{base_soft:.3f} = {base_gate_score:.3f}; "
+ f"candidate = (1-{w})*{final_hard:.3f} + {w}*{final_soft:.3f} = {final_score:.3f}"
+ )
+ else:
+ formula = (
+ f"score = {gate_metric}; baseline = {base_gate_score:.3f}; "
+ f"candidate = {final_score:.3f}"
+ )
+ ev.log("gate", "decision", action=action, accepted=accepted,
+ baseline_score=base_gate_score, candidate_score=final_score,
+ baseline_hard=base_hard, baseline_soft=base_soft,
+ candidate_hard=final_hard, candidate_soft=final_soft,
+ metric=gate_metric, mixed_weight=gate_mixed_weight,
+ formula=formula, n_applied=len(all_applied),
+ n_rejected=len(all_rejected),
+ n_unmatched=len(all_unmatched), night=night)
return ConsolidationResult(
accepted=accepted,
@@ -258,6 +336,7 @@ def consolidate(
new_memory=cand_memory if accepted else memory,
applied_edits=all_applied,
rejected_edits=all_rejected,
+ unmatched_edits=all_unmatched,
holdout_baseline=base_hard,
holdout_candidate=final_hard,
holdout_detail=holdout_detail,
diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py
index f199211..006547f 100644
--- a/skillopt_sleep/cycle.py
+++ b/skillopt_sleep/cycle.py
@@ -10,11 +10,14 @@ CI use. With backend="anthropic" it spends the user's budget for real lift.
from __future__ import annotations
import os
+import shutil
import sys
from dataclasses import dataclass
from typing import List, Optional
-from skillopt_sleep.backend import Backend, get_backend
+from skillopt_sleep import evidence
+from skillopt_sleep.backend import Backend, CursorBackendError, build_backend
+from skillopt_sleep.evidence import EvidenceLog
from skillopt_sleep.config import SleepConfig, load_config
from skillopt_sleep.dream import dream_consolidate
from skillopt_sleep.harvest_sources import harvest_for_config
@@ -27,6 +30,92 @@ from skillopt_sleep.state import SleepState, _now_iso
from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord
+# ââ Model-swap detection (F16) âââââââââââââââââââââââââââââââ
+def _make_model_key(cfg: SleepConfig) -> str:
+ """Stable string identifying the backend object(s) actually used.
+
+ Model-change detection is advisory, so resolving its diagnostic key must
+ never become an earlier failure point than construction of the real
+ backend. Fall back to a credential-free description of the configured
+ roles if a backend constructor cannot be used in this diagnostic path.
+ """
+ try:
+ effective = build_backend(
+ backend=cfg.get("backend", "mock"),
+ model=cfg.get("model", ""),
+ optimizer_backend=cfg.get("optimizer_backend", ""),
+ optimizer_model=cfg.get("optimizer_model", ""),
+ target_backend=cfg.get("target_backend", ""),
+ target_model=cfg.get("target_model", ""),
+ codex_path=cfg.get("codex_path", ""),
+ cursor_path=cfg.get("cursor_path", ""),
+ azure_endpoint=cfg.get("azure_endpoint", ""),
+ project_dir=cfg.get("invoked_project", "") or os.getcwd(),
+ )
+ except Exception:
+ backend = str(cfg.get("backend", "mock") or "mock")
+ model = str(cfg.get("model", "") or "")
+ split_keys = (
+ "optimizer_backend",
+ "optimizer_model",
+ "target_backend",
+ "target_model",
+ )
+ if not any(cfg.get(key, "") for key in split_keys):
+ return f"configured:{backend}::{model}"
+ optimizer_backend = str(cfg.get("optimizer_backend", "") or backend)
+ optimizer_model = str(cfg.get("optimizer_model", "") or model)
+ target_backend = str(cfg.get("target_backend", "") or backend)
+ target_model = str(cfg.get("target_model", "") or model)
+ return (
+ f"configured:optimizer={optimizer_backend}::{optimizer_model};"
+ f"target={target_backend}::{target_model}"
+ )
+ return _make_backend_key(effective)
+
+
+def _make_backend_key(backend: Backend) -> str:
+ """Describe resolved aliases/defaults without exposing credentials."""
+ target = getattr(backend, "target", None)
+ optimizer = getattr(backend, "optimizer", None)
+ if target is not None and optimizer is not None:
+ return (
+ f"optimizer={_make_backend_key(optimizer)};"
+ f"target={_make_backend_key(target)}"
+ )
+ name = str(getattr(backend, "name", backend.__class__.__name__) or "")
+ model = str(getattr(backend, "model", "") or "")
+ return f"{name}::{model}"
+
+
+def _check_model_change(
+ cfg: SleepConfig, state: SleepState, backend: Backend | None = None
+) -> None:
+ """Warn when the backend/model has changed since the last night.
+
+ Skill text is backend-specific; adopting edits from a different model's
+ reflections into a new model's skill file can cause regressions.
+ This is advisory only â the cycle continues either way.
+ """
+ current_key = (
+ _make_backend_key(backend) if backend is not None else _make_model_key(cfg)
+ )
+ prior_key = state.last_model_key
+ if prior_key and state.last_model_key_format < 2:
+ # Version 1 stored raw configuration rather than the resolved backend
+ # model. Defaults and aliases make that value impossible to compare
+ # truthfully, so migrate silently on the next successful night.
+ return
+ if prior_key and prior_key != current_key:
+ print(
+ f"[sleep] WARNING: model changed since last night "
+ f"(was {prior_key!r}, now {current_key!r}). "
+ "Learned skill text may not transfer cleanly. "
+ "Consider starting from a fresh skill document.",
+ file=sys.stderr,
+ )
+
+
@dataclass
class CycleOutcome:
report: SleepReport
@@ -56,6 +145,20 @@ def _progress(cfg: SleepConfig, message: str) -> None:
print(f"[sleep] {message}", file=sys.stderr, flush=True)
+def _discard_unstaged_evidence(path: str) -> None:
+ """Remove a pre-created evidence folder after a fail-closed Cursor call."""
+ if not path:
+ return
+ shutil.rmtree(path, ignore_errors=True)
+ # Avoid leaving an otherwise empty project .skillopt-sleep tree. Stop at
+ # the first non-empty directory so existing nights are never disturbed.
+ for parent in (os.path.dirname(path), os.path.dirname(os.path.dirname(path))):
+ try:
+ os.rmdir(parent)
+ except OSError:
+ break
+
+
def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str:
lines = [
f"# SkillOpt-Sleep â night {report.night} report",
@@ -79,6 +182,16 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str:
for e in report.rejected_edits:
lines.append(f"- [{e.target}/{e.op}] {e.content}")
lines.append("")
+ if report.unmatched_edits:
+ lines.append("## Proposed but changed nothing (never reached the gate)")
+ lines.append(
+ "_Anchor not found, replacement already present, duplicate/empty "
+ "add, or an unknown op. "
+ "These were never scored â check the anchor text if a rule you expected is missing._")
+ for e in report.unmatched_edits:
+ anchor = f" \n _anchor: `{e.anchor}`_" if e.anchor else ""
+ lines.append(f"- [{e.target}/{e.op}] {e.content}{anchor}")
+ lines.append("")
if report.notes:
lines.append("## Notes")
for n in report.notes:
@@ -110,19 +223,59 @@ def run_sleep_cycle(
"""
cfg = cfg or load_config()
state = SleepState.load(cfg.state_path)
- night = state.begin_night(clock)
project = _project_paths(cfg)
- started = _now_iso(clock)
- backend = backend or get_backend(
- cfg.get("backend", "mock"),
+ backend = backend or build_backend(
+ backend=cfg.get("backend", "mock"),
model=cfg.get("model", ""),
+ optimizer_backend=cfg.get("optimizer_backend", ""),
+ optimizer_model=cfg.get("optimizer_model", ""),
+ target_backend=cfg.get("target_backend", ""),
+ target_model=cfg.get("target_model", ""),
codex_path=cfg.get("codex_path", ""),
+ cursor_path=cfg.get("cursor_path", ""),
+ azure_endpoint=cfg.get("azure_endpoint", ""),
+ preferences=cfg.get("preferences", ""),
project_dir=project,
)
+ _check_model_change(cfg, state, backend) # F16: warn if model changed between nights
+ night = state.begin_night(clock)
+ started = _now_iso(clock)
backend.preferences = cfg.get("preferences", "")
_progress(cfg, f"night {night}: project={project} backend={backend.name}")
+ # ââ evidence log (the night's full evidentiary chain) ââââââââââââââââ
+ # Pre-create the staging dir so evidence.jsonl accumulates exactly where
+ # the report will land; dry-runs log into the state dir instead.
+ ev = None
+ staging_dir_pre = ""
+ # Callers may reuse a backend object across nights. Detach any logger from
+ # an earlier run before honoring this run's evidence_log setting.
+ evidence.attach(backend, None)
+ if cfg.get("evidence_log", True):
+ from skillopt_sleep.staging import _ts_dir, new_staging_dir
+ if dry_run:
+ ev_path = os.path.join(
+ cfg.state_dir, "evidence", f"dryrun-{_ts_dir()}.jsonl")
+ else:
+ staging_dir_pre = new_staging_dir(project)
+ ev_path = os.path.join(staging_dir_pre, "evidence.jsonl")
+ ev = EvidenceLog(
+ ev_path,
+ max_chars=int(cfg.get("evidence_max_chars", 4000) or 4000),
+ redact=bool(cfg.get("redact_secrets", True)),
+ )
+ evidence.attach(backend, ev)
+ ev.log("cycle", "start", night=night, project=project,
+ backend=backend.name, model=cfg.get("model", ""),
+ config={k: cfg.get(k) for k in (
+ "backend", "model", "optimizer_backend", "optimizer_model",
+ "target_backend", "target_model", "gate_mode", "gate_metric",
+ "gate_mixed_weight", "edit_budget", "holdout_fraction",
+ "dream_rollouts", "dream_factor", "recall_k",
+ "max_tasks_per_night", "lookback_hours", "llm_mine",
+ "evolve_skill", "evolve_memory")})
+
# ââ live skill/memory docs âââââââââââââââââââââââââââââââââââââââââââ
live_memory_path = os.path.join(project, "CLAUDE.md")
live_skill_path = cfg.managed_skill_path()
@@ -174,6 +327,17 @@ def run_sleep_cycle(
)
n_sessions = len(digests)
_progress(cfg, f"harvest done: sessions={n_sessions}")
+ if ev is not None:
+ # The transcript end of the evidentiary chain: which sessions were
+ # even considered, and what signals they carried into mining.
+ for d in digests:
+ ev.log("harvest", "session", session_id=d.session_id,
+ project=d.project,
+ n_user_prompts=len(d.user_prompts),
+ user_prompts_head=[p[:200] for p in d.user_prompts[:6]],
+ assistant_final_head=(d.assistant_finals[-1][:300]
+ if d.assistant_finals else ""),
+ feedback_signals=list(d.feedback_signals or []))
# When a real backend is configured, use it to mine checkable tasks from
# the transcripts (rubric/rule judges); otherwise fall back to the
# heuristic miner (no API, no checkable reference).
@@ -193,18 +357,33 @@ def run_sleep_cycle(
f"mine start: max_tasks={max_tasks} candidate_limit={candidate_limit} "
f"llm_mine={llm_miner is not None} target_filter={target_filter}",
)
- tasks = mine(
- digests,
- max_tasks=max_tasks,
- candidate_limit=candidate_limit,
- holdout_fraction=cfg.get("holdout_fraction", 0.34),
- seed=cfg.get("seed", 42),
- llm_miner=llm_miner,
- target_skill_text=raw_skill if target_filter else "",
- target_skill_path=live_skill_path if target_filter else "",
- )
+ try:
+ tasks = mine(
+ digests,
+ max_tasks=max_tasks,
+ candidate_limit=candidate_limit,
+ holdout_fraction=cfg.get("holdout_fraction", 0.34),
+ seed=cfg.get("seed", 42),
+ llm_miner=llm_miner,
+ target_skill_text=raw_skill if target_filter else "",
+ target_skill_path=live_skill_path if target_filter else "",
+ )
+ except CursorBackendError:
+ _discard_unstaged_evidence(staging_dir_pre)
+ raise
_progress(cfg, f"mine done: tasks={len(tasks)}")
+ if ev is not None:
+ # Final task pool with split assignment: which tasks train the edits
+ # vs. which held-out tasks gate them (works for seeded tasks too).
+ for t in tasks:
+ ev.log("mine", "task_ready", task_id=t.id, split=t.split,
+ origin=t.origin, intent=t.intent[:300],
+ reference_kind=t.reference_kind,
+ checks=(t.judge or {}).get("checks", []),
+ rubric=(t.reference if t.reference_kind == "rubric" else ""),
+ source_sessions=list(t.source_sessions or []))
+
report = SleepReport(
night=night, project=project, started_at=started,
n_sessions=n_sessions, n_tasks=len(tasks),
@@ -217,6 +396,9 @@ def run_sleep_cycle(
state.record_night({"night": night, "accepted": False, "n_tasks": 0})
if not dry_run:
state.save()
+ if ev is not None:
+ ev.log("cycle", "end", night=night, outcome="no_tasks",
+ tokens_used=backend.tokens_used())
staging_dir = ""
return CycleOutcome(report, staging_dir, False, [])
@@ -230,26 +412,31 @@ def run_sleep_cycle(
history_tasks = []
if recall_k > 0:
history_tasks = [TaskRecord.from_dict(d) for d in state.task_archive()]
- result = dream_consolidate(
- backend, tasks, skill, memory,
- history_tasks=history_tasks,
- recall_k=recall_k,
- dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1),
- dream_factor=int(cfg.get("dream_factor", 0) or 0),
- edit_budget=cfg.get("edit_budget", 4),
- gate_metric=cfg.get("gate_metric", "mixed"),
- gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5),
- gate_mode=cfg.get("gate_mode", "on"),
- evolve_skill=cfg.get("evolve_skill", True),
- evolve_memory=cfg.get("evolve_memory", True),
- night=night,
- )
+ try:
+ result = dream_consolidate(
+ backend, tasks, skill, memory,
+ history_tasks=history_tasks,
+ recall_k=recall_k,
+ dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1),
+ dream_factor=int(cfg.get("dream_factor", 0) or 0),
+ edit_budget=cfg.get("edit_budget", 4),
+ gate_metric=cfg.get("gate_metric", "mixed"),
+ gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5),
+ gate_mode=cfg.get("gate_mode", "on"),
+ evolve_skill=cfg.get("evolve_skill", True),
+ evolve_memory=cfg.get("evolve_memory", True),
+ night=night,
+ )
+ except CursorBackendError:
+ _discard_unstaged_evidence(staging_dir_pre)
+ raise
# archive tonight's real (non-dream) tasks so future nights can recall them
state.add_to_archive([t.to_dict() for t in tasks if t.origin != "dream"])
_progress(
cfg,
f"consolidate done: gate={result.gate_action} accepted={result.accepted} "
- f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}",
+ f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}"
+ + (f" unmatched={len(result.unmatched_edits)}" if result.unmatched_edits else ""),
)
report.n_replayed = len(tasks)
@@ -260,6 +447,7 @@ def run_sleep_cycle(
report.no_edits_reason = getattr(result, "no_edits_reason", "")
report.edits = result.applied_edits
report.rejected_edits = result.rejected_edits
+ report.unmatched_edits = result.unmatched_edits
report.tokens_used = backend.tokens_used()
report.ended_at = _now_iso(clock)
@@ -280,7 +468,13 @@ def run_sleep_cycle(
live_skill_path=live_skill_path,
live_memory_path=live_memory_path,
report_md=report_md,
+ out_dir=staging_dir_pre,
)
+ if ev is not None:
+ ev.log("stage", "staged", staging_dir=staging_dir,
+ has_skill=proposed_skill is not None,
+ has_memory=proposed_memory is not None,
+ accepted=result.accepted)
# Observability: persist per-task held-out evidence + optimizer/codex errors so a
# 0.0->0.0 night self-explains (empty responses vs failing checks vs no edits) â the
# cycle previously captured none of this, making the gate a black box (#learning-stall).
@@ -300,6 +494,7 @@ def run_sleep_cycle(
"accepted": result.accepted,
"n_applied_edits": len(result.applied_edits),
"n_rejected_edits": len(result.rejected_edits),
+ "n_unmatched_edits": len(result.unmatched_edits),
"call_error": redact_secrets(getattr(result, "call_error", "")),
"reflect_raw_head": redact_secrets(
(getattr(result, "reflect_raw", "") or "")[:1200]
@@ -314,10 +509,21 @@ def run_sleep_cycle(
"baseline": result.baseline_score, "candidate": result.candidate_score,
"n_tasks": len(tasks), "staging": staging_dir,
})
+ state.set_last_model_key(_make_backend_key(backend)) # F16: track resolved model
# ââ 6. adopt (opt-in) ââââââââââââââââââââââââââââââââââââââââââââ
if cfg.get("auto_adopt") and result.accepted:
adopted_paths = adopt_staging(staging_dir)
adopted = bool(adopted_paths)
state.save()
+ if ev is not None:
+ ev.log("cycle", "end", night=night, outcome="completed",
+ gate_action=report.gate_action, accepted=report.accepted,
+ baseline_score=report.baseline_score,
+ candidate_score=report.candidate_score,
+ n_applied_edits=len(report.edits),
+ n_rejected_edits=len(report.rejected_edits),
+ n_unmatched_edits=len(report.unmatched_edits),
+ tokens_used=report.tokens_used, adopted=adopted)
+
return CycleOutcome(report, staging_dir, adopted, adopted_paths)
diff --git a/skillopt_sleep/evidence.py b/skillopt_sleep/evidence.py
new file mode 100644
index 0000000..fa53891
--- /dev/null
+++ b/skillopt_sleep/evidence.py
@@ -0,0 +1,138 @@
+"""SkillOpt-Sleep â the per-night evidentiary chain (``evidence.jsonl``).
+
+The existing report/diagnostics answer *what* the cycle decided; they do not
+answer *why*. This module records the full causal chain, per night:
+
+ transcript session -> miner exchange (prompt + raw reply)
+ -> mined task (+ its checks, + source session ids)
+ -> split assignment (train / val)
+ -> every replay attempt (phase-tagged, full prompt
+ and response, cache hits marked)
+ -> per-task scores with the failing checks named
+ -> the reflect exchange (prompt, raw reply, parsed
+ edits) and every gate trial
+ -> the final gate decision with the score arithmetic
+ -> what was staged
+
+Design constraints (matching the sleep engine's contract):
+ * pure stdlib, thread-safe (replay batches run in a thread pool);
+ * every persisted string passes through best-effort ``redact_secrets``
+ before the per-field length cap is applied;
+ * append-only JSONL so a crashed night still leaves its partial chain;
+ * zero behavior change when disabled (``evidence_log: false``).
+
+Events share the shape::
+
+ {"ts": , "seq": , "stage": , "event": , ...}
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from typing import Any, Optional
+
+from skillopt_sleep.staging import redact_secrets
+
+
+def _now_iso() -> str:
+ return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime())
+
+
+class EvidenceLog:
+ """Append-only, thread-safe JSONL logger for one sleep night."""
+
+ def __init__(self, path: str, *, max_chars: int = 4000, redact: bool = True) -> None:
+ self.path = path
+ self.max_chars = max(200, int(max_chars))
+ self.redact = redact
+ self._lock = threading.Lock()
+ self._seq = 0
+ os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+
+ # ââ sanitization ââââââââââââââââââââââââââââââââââââââââââââââââââââââ
+ def _clean(self, value: Any) -> Any:
+ if isinstance(value, str):
+ # Redact first: truncating a structured secret such as a PEM block
+ # can remove its closing marker and make it unrecognizable to the
+ # redactor while leaving the secret body in the persisted prefix.
+ if self.redact:
+ value = redact_secrets(value)
+ if len(value) > self.max_chars:
+ dropped = len(value) - self.max_chars
+ value = value[: self.max_chars] + f"âŚ[truncated {dropped} chars]"
+ return value
+ if isinstance(value, dict):
+ return {k: self._clean(v) for k, v in value.items()}
+ if isinstance(value, (list, tuple)):
+ return [self._clean(v) for v in value]
+ return value
+
+ # ââ the one write path ââââââââââââââââââââââââââââââââââââââââââââââââ
+ def log(self, stage: str, event: str, **data: Any) -> None:
+ record = {"ts": _now_iso(), "stage": stage, "event": event}
+ record.update(self._clean(data))
+ with self._lock:
+ self._seq += 1
+ record["seq"] = self._seq
+ try:
+ line = json.dumps(record, ensure_ascii=False, default=str)
+ with open(self.path, "a", encoding="utf-8") as f:
+ f.write(line + "\n")
+ except Exception:
+ # Evidence must never break a night; drop the record instead.
+ pass
+
+
+def attach(backend, ev: Optional[EvidenceLog]) -> None:
+ """Attach ``ev`` to a backend â and, for DualBackend, to both halves â
+ so every layer that wants to log can find it via ``backend.evidence``."""
+ if backend is None:
+ return
+ backend.evidence = ev
+ for half in ("target", "optimizer"):
+ sub = getattr(backend, half, None)
+ if sub is not None:
+ sub.evidence = ev
+
+
+def get(backend) -> Optional[EvidenceLog]:
+ return getattr(backend, "evidence", None)
+
+
+def set_phase(backend, phase: str) -> None:
+ """Tag subsequent replay calls with a phase label (baseline_val,
+ train, gate_trial:skill, final_val, ...). Phases are sequential in the
+ consolidation loop, so a plain attribute is safe; parallelism only ever
+ happens *within* one phase."""
+ if backend is None:
+ return
+ backend.evidence_phase = phase
+ for half in ("target", "optimizer"):
+ sub = getattr(backend, half, None)
+ if sub is not None:
+ sub.evidence_phase = phase
+
+
+def phase(backend) -> str:
+ return getattr(backend, "evidence_phase", "") or ""
+
+
+def read_events(path: str) -> list:
+ """Best-effort reader for the dashboard: skips corrupt lines."""
+ out = []
+ try:
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ out.append(json.loads(line))
+ except Exception:
+ continue
+ except OSError:
+ return []
+ out.sort(key=lambda r: r.get("seq", 0))
+ return out
diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py
index deb8d88..d0c102c 100644
--- a/skillopt_sleep/harvest.py
+++ b/skillopt_sleep/harvest.py
@@ -86,6 +86,34 @@ def _tool_names_from_content(content: Any) -> List[str]:
return names
+def _skill_names_from_content(content: Any) -> List[str]:
+ """Extract skill targets from Claude ``Skill`` tool-use blocks.
+
+ Only well-formed invocations count: the block type must be ``tool_use``,
+ its name exactly ``Skill``, and ``input.skill`` a non-blank string. The
+ name is returned whitespace-trimmed but otherwise verbatim; no other tool
+ input and no tool output is read.
+ """
+ names: List[str] = []
+ if not isinstance(content, list):
+ return names
+ for b in content:
+ if not isinstance(b, dict):
+ continue
+ if b.get("type") != "tool_use" or b.get("name") != "Skill":
+ continue
+ args = b.get("input")
+ if not isinstance(args, dict):
+ continue
+ skill = args.get("skill")
+ if not isinstance(skill, str):
+ continue
+ skill = skill.strip()
+ if skill:
+ names.append(skill)
+ return names
+
+
def _detect_feedback(text: str) -> List[str]:
low = text.lower()
sig: List[str] = []
@@ -206,6 +234,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]:
user_prompts: List[str] = []
assistant_finals: List[str] = []
tools: List[str] = []
+ skills: List[str] = []
files: List[str] = []
feedback: List[str] = []
n_user = 0
@@ -240,6 +269,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]:
elif role == "assistant":
n_asst += 1
tools.extend(_tool_names_from_content(content))
+ skills.extend(_skill_names_from_content(content))
text = _text_from_content(content)
if text.strip():
assistant_finals.append(text.strip())
@@ -266,6 +296,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]:
user_prompts=user_prompts,
assistant_finals=assistant_finals[-5:], # last few finals are the useful ones
tools_used=_dedup(tools),
+ skills_used=_dedup(skills),
files_touched=_dedup(files),
feedback_signals=feedback,
n_user_turns=n_user,
diff --git a/skillopt_sleep/harvest_cursor.py b/skillopt_sleep/harvest_cursor.py
new file mode 100644
index 0000000..c4379a4
--- /dev/null
+++ b/skillopt_sleep/harvest_cursor.py
@@ -0,0 +1,316 @@
+"""Read Cursor Agent transcripts and normalize them into session digests.
+
+Cursor writes workspace-scoped JSONL under
+``~/.cursor/projects//agent-transcripts//.jsonl``.
+The observed local records contain user/assistant messages and tool-use metadata
+but no timestamps, so this harvester uses each file's mtime as its end time.
+Tool inputs and outputs are intentionally never copied.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from typing import Any, Iterable, List, Optional
+
+from skillopt_sleep.harvest import (
+ _detect_feedback,
+ _is_meta_prompt,
+ _iter_jsonl,
+)
+from skillopt_sleep.staging import redact_secrets
+from skillopt_sleep.types import SessionDigest
+
+CURSOR_REPLAY_SENTINEL = ""
+_CURSOR_USER_QUERY_RE = re.compile(
+ r"\s*(.*?)\s*\s*\Z",
+ re.DOTALL,
+)
+
+
+def cursor_project_slug(project: str) -> str:
+ """Return the filesystem-safe workspace name used under Cursor projects."""
+ normalized = os.path.abspath(os.path.expanduser(project))
+ return re.sub(r"[^A-Za-z0-9]+", "-", normalized).strip("-")
+
+
+def _text_from_content(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if not isinstance(content, list):
+ return ""
+ return "\n".join(
+ str(block["text"])
+ for block in content
+ if isinstance(block, dict)
+ and block.get("type") == "text"
+ and block.get("text")
+ )
+
+
+def _tool_names(content: Any) -> List[str]:
+ if not isinstance(content, list):
+ return []
+ names: List[str] = []
+ for block in content:
+ if not isinstance(block, dict) or block.get("type") != "tool_use":
+ continue
+ name = block.get("name")
+ if isinstance(name, str) and name:
+ names.append(re.sub(r"[^A-Za-z0-9_.:-]+", "_", name)[:80])
+ return names
+
+
+def _sanitize_text(text: str) -> str:
+ sanitized = str(redact_secrets(text)).replace("\x00", "").strip()
+ user_query = _CURSOR_USER_QUERY_RE.search(sanitized)
+ if user_query:
+ sanitized = user_query.group(1).strip()
+ if not sanitized or _is_meta_prompt(sanitized):
+ return ""
+ return sanitized
+
+
+def _dedup(values: Iterable[str]) -> List[str]:
+ seen = set()
+ result: List[str] = []
+ for value in values:
+ if value not in seen:
+ seen.add(value)
+ result.append(value)
+ return result
+
+
+def _mtime_iso(path: str) -> str:
+ try:
+ return (
+ datetime.fromtimestamp(os.path.getmtime(path), tz=timezone.utc)
+ .replace(microsecond=0)
+ .isoformat()
+ .replace("+00:00", "Z")
+ )
+ except OSError:
+ return ""
+
+
+def _mtime(path: str) -> Optional[float]:
+ try:
+ return os.path.getmtime(path)
+ except OSError:
+ return None
+
+
+def _iso_epoch(value: Optional[str]) -> Optional[float]:
+ if not value:
+ return None
+ try:
+ normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
+ parsed = datetime.fromisoformat(normalized)
+ # Existing state timestamps are local-time strings without an offset.
+ return parsed.timestamp()
+ except (TypeError, ValueError, OSError):
+ return None
+
+
+def digest_cursor_transcript(path: str, *, project: str = "") -> Optional[SessionDigest]:
+ """Build a digest without retaining Cursor tool arguments or outputs."""
+ session_id = os.path.splitext(os.path.basename(path))[0]
+ user_prompts: List[str] = []
+ assistant_finals: List[str] = []
+ tools: List[str] = []
+ feedback: List[str] = []
+ n_user = 0
+ n_assistant = 0
+
+ for record in _iter_jsonl(path):
+ if not isinstance(record, dict):
+ continue
+ if record.get("type") == "turn_ended":
+ if record.get("status") == "error":
+ feedback.append("neg:cursor_turn_error")
+ continue
+
+ message = record.get("message")
+ if not isinstance(message, dict):
+ continue
+ role = record.get("role") or message.get("role")
+ content = message.get("content")
+ if role == "user":
+ text = _sanitize_text(_text_from_content(content))
+ if text:
+ n_user += 1
+ user_prompts.append(text)
+ feedback.extend(_detect_feedback(text))
+ elif role == "assistant":
+ n_assistant += 1
+ tools.extend(_tool_names(content))
+ text = _sanitize_text(_text_from_content(content))
+ if text:
+ assistant_finals.append(text)
+
+ if n_user == 0 and n_assistant == 0:
+ return None
+
+ return SessionDigest(
+ session_id=session_id,
+ project=project,
+ ended_at=_mtime_iso(path),
+ user_prompts=user_prompts,
+ assistant_finals=assistant_finals[-5:],
+ tools_used=_dedup(tools),
+ files_touched=[],
+ feedback_signals=feedback,
+ n_user_turns=n_user,
+ n_assistant_turns=n_assistant,
+ raw_path=path,
+ )
+
+
+def _workspace_path(project_dir: str) -> str:
+ metadata_path = os.path.join(project_dir, ".workspace-trusted")
+ try:
+ with open(metadata_path, encoding="utf-8") as f:
+ metadata = json.load(f)
+ except (OSError, ValueError):
+ return ""
+ if not isinstance(metadata, dict):
+ return ""
+ workspace = metadata.get("workspacePath")
+ if not isinstance(workspace, str) or not workspace.strip():
+ return ""
+ workspace = os.path.expanduser(workspace.strip())
+ if not os.path.isabs(workspace):
+ return ""
+ return os.path.abspath(workspace)
+
+
+def _normalized_path(path: str) -> str:
+ return os.path.normcase(os.path.realpath(os.path.abspath(os.path.expanduser(path))))
+
+
+def _is_workspace_ancestor(workspace: str, invoked: str) -> bool:
+ try:
+ workspace_norm = _normalized_path(workspace)
+ invoked_norm = _normalized_path(invoked)
+ return os.path.commonpath([workspace_norm, invoked_norm]) == workspace_norm
+ except (OSError, ValueError):
+ return False
+
+
+def _available_project_dirs(projects_dir: str) -> List[tuple[str, str, str]]:
+ try:
+ names = sorted(os.listdir(projects_dir))
+ except OSError:
+ return []
+ result: List[tuple[str, str, str]] = []
+ for name in names:
+ project_dir = os.path.join(projects_dir, name)
+ if os.path.isdir(os.path.join(project_dir, "agent-transcripts")):
+ result.append((project_dir, name, _workspace_path(project_dir)))
+ return result
+
+
+def _project_dirs(projects_dir: str, scope: Any, invoked_project: str) -> List[tuple[str, str]]:
+ available = _available_project_dirs(projects_dir)
+ if scope == "all":
+ return [
+ (project_dir, workspace or name)
+ for project_dir, name, workspace in available
+ ]
+
+ projects: List[str]
+ if isinstance(scope, (list, tuple)):
+ projects = [str(project) for project in scope]
+ else:
+ projects = [invoked_project] if invoked_project else []
+
+ selected: List[tuple[str, str]] = []
+ seen = set()
+ for project in projects:
+ absolute_project = os.path.abspath(os.path.expanduser(project))
+ matches = [
+ (project_dir, workspace)
+ for project_dir, _name, workspace in available
+ if workspace and _is_workspace_ancestor(workspace, absolute_project)
+ ]
+ candidate = absolute_project
+ while candidate:
+ fallback = os.path.join(projects_dir, cursor_project_slug(candidate))
+ if os.path.isdir(os.path.join(fallback, "agent-transcripts")):
+ matches.append((fallback, candidate))
+ break
+ parent = os.path.dirname(candidate)
+ if parent == candidate:
+ break
+ candidate = parent
+
+ if matches:
+ longest = max(len(_normalized_path(workspace)) for _project_dir, workspace in matches)
+ choices = [
+ (project_dir, workspace)
+ for project_dir, workspace in matches
+ if len(_normalized_path(workspace)) == longest
+ ]
+ else:
+ fallback = os.path.join(projects_dir, cursor_project_slug(absolute_project))
+ choices = [(fallback, absolute_project)]
+ for project_dir, workspace in choices:
+ if project_dir not in seen:
+ selected.append((project_dir, workspace))
+ seen.add(project_dir)
+ return selected
+
+
+def _is_cursor_replay(digest: SessionDigest) -> bool:
+ return any(prompt.lstrip().startswith(CURSOR_REPLAY_SENTINEL) for prompt in digest.user_prompts)
+
+
+def harvest_cursor(
+ projects_dir: str,
+ *,
+ scope: Any = "all",
+ invoked_project: str = "",
+ since_iso: Optional[str] = None,
+ limit: int = 0,
+) -> List[SessionDigest]:
+ """Return Cursor session digests for the selected workspace scope."""
+ if not os.path.isdir(projects_dir):
+ return []
+
+ candidates: List[tuple[str, str, float]] = []
+ for project_dir, project in _project_dirs(projects_dir, scope, invoked_project):
+ transcripts_dir = os.path.join(project_dir, "agent-transcripts")
+ try:
+ session_names = sorted(os.listdir(transcripts_dir))
+ except OSError:
+ continue
+ for session_name in session_names:
+ session_dir = os.path.join(transcripts_dir, session_name)
+ if not os.path.isdir(session_dir):
+ continue
+ try:
+ filenames = sorted(os.listdir(session_dir))
+ except OSError:
+ continue
+ for filename in filenames:
+ path = os.path.join(session_dir, filename)
+ if not filename.endswith(".jsonl") or not os.path.isfile(path):
+ continue
+ modified = _mtime(path)
+ if modified is not None:
+ candidates.append((path, project, modified))
+ candidates.sort(key=lambda item: (-item[2], item[0]))
+
+ since_epoch = _iso_epoch(since_iso)
+ digests: List[SessionDigest] = []
+ for path, project, modified in candidates:
+ if since_epoch is not None and modified <= since_epoch:
+ continue
+ digest = digest_cursor_transcript(path, project=project)
+ if digest is None or _is_cursor_replay(digest):
+ continue
+ digests.append(digest)
+ if limit and len(digests) >= limit:
+ break
+ return digests
diff --git a/skillopt_sleep/harvest_sources.py b/skillopt_sleep/harvest_sources.py
index d91ea27..a43ebac 100644
--- a/skillopt_sleep/harvest_sources.py
+++ b/skillopt_sleep/harvest_sources.py
@@ -6,6 +6,7 @@ from typing import Optional
from skillopt_sleep.harvest import harvest
from skillopt_sleep.harvest_copilot import harvest_copilot
from skillopt_sleep.harvest_codex import harvest_codex
+from skillopt_sleep.harvest_cursor import harvest_cursor
from skillopt_sleep.types import SessionDigest
@@ -30,6 +31,14 @@ def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0)
since_iso=since_iso,
limit=limit,
)
+ if source == "cursor":
+ return harvest_cursor(
+ cfg.cursor_projects_dir,
+ scope=scope,
+ invoked_project=invoked_project,
+ since_iso=since_iso,
+ limit=limit,
+ )
if source == "auto":
codex_digests = harvest_codex(
cfg.codex_archived_sessions_dir,
diff --git a/skillopt_sleep/judges.py b/skillopt_sleep/judges.py
index f981015..9f5df1e 100644
--- a/skillopt_sleep/judges.py
+++ b/skillopt_sleep/judges.py
@@ -35,29 +35,108 @@ def _section_present(response: str, name: str) -> bool:
return bool(label.search(response or ""))
-def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool:
+def _check(op: str, arg: Any, response: str,
+ tools_called: List[str]) -> Tuple[bool, str]:
+ """Evaluate one check.
+
+ Returns ``(passed, problem)``. ``problem`` is non-empty only when the check
+ itself is malformed (e.g. an unparseable regex) rather than simply unmet â
+ the two need opposite fixes, so they must not look alike in the rationale.
+ """
r = response or ""
if op == "section_present":
- return _section_present(r, str(arg))
+ return _section_present(r, str(arg)), ""
if op == "regex":
try:
- return bool(re.search(str(arg), r))
- except re.error:
- return False
+ return bool(re.search(str(arg), r)), ""
+ except re.error as exc:
+ # A malformed pattern can never match, so it would fail every
+ # rollout forever and read exactly like a model that never
+ # complies. Surface it instead of hiding it behind a False.
+ return False, f"invalid regex ({exc})"
if op == "max_chars":
- return len(r) <= int(arg)
+ return len(r) <= int(arg), ""
if op == "min_chars":
- return len(r) >= int(arg)
+ return len(r) >= int(arg), ""
if op == "contains":
- return str(arg).lower() in r.lower()
+ return str(arg).lower() in r.lower(), ""
if op == "tool_called":
name = str(arg).lower()
if any(name == t.lower() for t in tools_called):
- return True
+ return True, ""
# single-shot approximation: the agent emits an explicit marker
- return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r))
+ return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)), ""
# unknown op: do not block
- return True
+ return True, ""
+
+
+KNOWN_OPS = frozenset({
+ "section_present", "regex", "max_chars", "min_chars", "contains", "tool_called",
+})
+
+
+def validate_checks(judge: Any) -> Tuple[List[str], List[str]]:
+ """Return ``(errors, warnings)`` for a rule judge's checks.
+
+ An *error* means the check can never behave as written â a regex that does
+ not compile always scores 0.0, which is indistinguishable from a model that
+ never complies. A *warning* means the check is accepted but toothless, e.g.
+ an unknown op, which :func:`_check` deliberately lets pass.
+ """
+ errors: List[str] = []
+ warnings: List[str] = []
+ if judge is not None and not isinstance(judge, dict):
+ return [f"judge must be an object, got {type(judge).__name__}"], warnings
+ checks = (judge or {}).get("checks", []) or []
+ if not isinstance(checks, list):
+ return [f"judge 'checks' must be an array, got {type(checks).__name__}"], warnings
+ for i, c in enumerate(checks):
+ if not isinstance(c, dict):
+ errors.append(f"check #{i} is not an object")
+ continue
+ op = c.get("op", "")
+ arg = c.get("arg")
+ if not isinstance(op, str):
+ errors.append(
+ f"check #{i} op must be a string, got {type(op).__name__}"
+ )
+ continue
+ if op in {"regex", "section_present", "contains", "tool_called"} and (
+ arg is None or not str(arg).strip()
+ ):
+ errors.append(f"check #{i} {op} needs a non-empty arg")
+ continue
+ if op == "regex":
+ try:
+ re.compile(str(arg))
+ except re.error as exc:
+ errors.append(f"check #{i} regex does not compile ({exc}): {arg!r}")
+ elif op in {"max_chars", "min_chars"}:
+ try:
+ if isinstance(arg, bool):
+ raise ValueError
+ if isinstance(arg, int):
+ bound = arg
+ elif isinstance(arg, float):
+ if not arg.is_integer():
+ raise ValueError
+ bound = int(arg)
+ elif isinstance(arg, str) and re.fullmatch(
+ r"[+-]?\d+", arg.strip()
+ ):
+ bound = int(arg.strip())
+ else:
+ raise ValueError
+ except (OverflowError, TypeError, ValueError):
+ errors.append(f"check #{i} {op} needs an integer arg, got {arg!r}")
+ else:
+ if bound < 0:
+ errors.append(f"check #{i} {op} cannot be negative, got {bound}")
+ elif op == "min_chars" and bound == 0:
+ warnings.append(f"check #{i} min_chars=0 always passes")
+ elif op not in KNOWN_OPS:
+ warnings.append(f"check #{i} has unknown op {op!r} â it always passes")
+ return errors, warnings
def score_rule_judge(
@@ -73,11 +152,14 @@ def score_rule_judge(
passed = 0
failed_desc: List[str] = []
for c in checks:
- ok = _check(c.get("op", ""), c.get("arg"), response, tools_called)
+ ok, problem = _check(c.get("op", ""), c.get("arg"), response, tools_called)
if ok:
passed += 1
else:
- failed_desc.append(f"{c.get('op')}={c.get('arg')}")
+ desc = f"{c.get('op')}={c.get('arg')}"
+ if problem:
+ desc += f" [{problem}]"
+ failed_desc.append(desc)
soft = passed / len(checks)
hard = 1.0 if passed == len(checks) else 0.0
rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc)
diff --git a/skillopt_sleep/llm_miner.py b/skillopt_sleep/llm_miner.py
index dd78c63..ddf0780 100644
--- a/skillopt_sleep/llm_miner.py
+++ b/skillopt_sleep/llm_miner.py
@@ -22,51 +22,22 @@ import json
import re
from typing import Any, Callable, Dict, List
+from skillopt_sleep import prompts as prompt_registry
from skillopt_sleep.backend import Backend, _extract_json
from skillopt_sleep.types import SessionDigest, TaskRecord
-_MINER_PROMPT = """You are mining a user's past AI-assistant sessions to find RECURRING tasks
-worth optimizing a skill for. From the session below, extract 0-3 reusable tasks.
-
-A good task is something the user asks for repeatedly or had to correct, where a
-GENERAL rule would help next time (formatting, structure, tool-use, conventions).
-Skip one-off or purely exploratory requests.
-
-For each task return:
- - "intent": the reusable request, generalized (no one-off specifics)
- - "checks": a list of programmatic success checks a grader can run on a future
- answer. Each check is one of:
- {"op":"section_present","arg":""}
- {"op":"regex","arg":""}
- {"op":"contains","arg":""}
- {"op":"max_chars","arg":}
- Only include checks you are confident a GOOD answer must satisfy.
- - "rubric": a one-sentence description of what a good answer looks like
- - "satisfied": true/false â did the user seem satisfied with the assistant's answer?
-
-Return ONLY a JSON array (possibly empty). No prose.
-
-# Session
-project: __PROJECT__
-user prompts:
-__PROMPTS__
-assistant final (last):
-__FINAL__
-feedback signals: __FEEDBACK__
-"""
-
-
def _digest_to_prompt(d: SessionDigest) -> str:
+ # Template lives in the central prompt registry (skillopt_sleep.prompts)
+ # so the dashboard can display and override it live.
prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)"
final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)")
- return (
- _MINER_PROMPT
- .replace("__PROJECT__", d.project or "(unknown)")
- .replace("__PROMPTS__", prompts)
- .replace("__FINAL__", final)
- .replace("__FEEDBACK__", ", ".join(d.feedback_signals[:6]) or "(none)")
- )
+ return prompt_registry.render("miner", {
+ "__PROJECT__": d.project or "(unknown)",
+ "__PROMPTS__": prompts,
+ "__FINAL__": final,
+ "__FEEDBACK__": ", ".join(d.feedback_signals[:6]) or "(none)",
+ })
def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None:
@@ -114,21 +85,45 @@ def make_llm_miner(
"""Return an llm_miner(digests) -> list[TaskRecord] bound to a backend."""
def _miner(digests: List[SessionDigest]) -> List[TaskRecord]:
+ ev = getattr(backend, "evidence", None)
out: List[TaskRecord] = []
for d in digests[:max_sessions]:
if not d.user_prompts:
continue
- raw = backend._call(_digest_to_prompt(d), max_tokens=800) # type: ignore[attr-defined]
+ prompt = _digest_to_prompt(d)
+ raw = backend._call(prompt, max_tokens=800) # type: ignore[attr-defined]
arr = _extract_json(raw, "array")
- if not isinstance(arr, list):
- continue
- for i, obj in enumerate(arr[:3]):
+ candidates = arr if isinstance(arr, list) else []
+ made: List[TaskRecord] = []
+ dropped = 0
+ full = False
+ for i, obj in enumerate(candidates[:3]):
if isinstance(obj, dict):
t = _mk_task(d, obj, i)
if t is not None:
+ made.append(t)
out.append(t)
+ else:
+ dropped += 1 # not checkable -> dropped, and now logged
if len(out) >= max_tasks:
- return out
+ full = True
+ break
+ if ev is not None:
+ # The transcript->task link of the evidentiary chain: what this
+ # session was, exactly what the miner was asked, exactly what it
+ # replied, and which TaskRecords (with checks) came out of it.
+ ev.log("mine", "miner_exchange", session_id=d.session_id,
+ project=d.project, prompt=prompt, raw_reply=raw,
+ parse_ok=isinstance(arr, list), n_candidates=len(candidates),
+ n_tasks=len(made), n_dropped_uncheckable=dropped)
+ for t in made:
+ ev.log("mine", "task_mined", task_id=t.id,
+ session_id=d.session_id, intent=t.intent,
+ reference_kind=t.reference_kind,
+ checks=(t.judge or {}).get("checks", []),
+ rubric=t.reference, outcome=t.outcome)
+ if full:
+ return out
return out
return _miner
diff --git a/skillopt_sleep/memory.py b/skillopt_sleep/memory.py
index ef67f36..86d91a0 100644
--- a/skillopt_sleep/memory.py
+++ b/skillopt_sleep/memory.py
@@ -76,42 +76,79 @@ def apply_edits(doc: str, edits: List[EditRecord]) -> Tuple[str, List[EditRecord
Returns (new_doc, applied_edits). Dedups: an `add` whose content already
exists (normalized) is skipped. `delete`/`replace` match on normalized
anchor substring.
+
+ See :func:`apply_edits_detailed` when the caller also needs the edits that
+ matched nothing.
+ """
+ new_doc, applied, _unmatched = apply_edits_detailed(doc, edits)
+ return new_doc, applied
+
+
+def apply_edits_detailed(
+ doc: str, edits: List[EditRecord]
+) -> Tuple[str, List[EditRecord], List[EditRecord]]:
+ """Apply edits and also report the ones that matched nothing.
+
+ Returns ``(new_doc, applied, unmatched)``. An edit lands in ``unmatched``
+ whenever it left the document unchanged:
+
+ * ``delete``/``replace`` whose anchor matches no existing line;
+ * ``replace`` whose replacement is already present verbatim;
+ * ``add`` whose content duplicates an existing line (normalized), or is
+ empty/whitespace;
+ * any unrecognized op.
+
+ Without this list such edits are invisible â they appear in neither the
+ applied nor the gate-rejected set, so a night can report zero edits while
+ the optimizer actually produced several.
"""
lines = current_learned_lines(doc)
norm_set = {_norm(line) for line in lines}
applied: List[EditRecord] = []
+ unmatched: List[EditRecord] = []
for e in edits:
op = (e.op or "add").lower()
if op == "add":
if _norm(e.content) in norm_set or not e.content.strip():
+ unmatched.append(e)
continue
lines.append(e.content.strip())
norm_set.add(_norm(e.content))
applied.append(e)
elif op == "delete":
anchor = _norm(e.anchor or e.content)
+ if not anchor:
+ unmatched.append(e)
+ continue
keep = [line for line in lines if anchor not in _norm(line)]
if len(keep) != len(lines):
lines = keep
norm_set = {_norm(line) for line in lines}
applied.append(e)
+ else:
+ unmatched.append(e)
elif op == "replace":
anchor = _norm(e.anchor)
+ replacement = e.content.strip()
new_lines = []
changed = False
for line in lines:
if anchor and anchor in _norm(line):
- new_lines.append(e.content.strip())
- changed = True
+ new_lines.append(replacement)
+ changed = changed or replacement != line
else:
new_lines.append(line)
if changed:
lines = new_lines
norm_set = {_norm(line) for line in lines}
applied.append(e)
+ else:
+ unmatched.append(e)
+ else:
+ unmatched.append(e)
- return set_learned(doc, lines), applied
+ return set_learned(doc, lines), applied, unmatched
def ensure_skill_scaffold(doc: str, *, name: str, description: str) -> str:
diff --git a/skillopt_sleep/mine.py b/skillopt_sleep/mine.py
index 4483057..9a25827 100644
--- a/skillopt_sleep/mine.py
+++ b/skillopt_sleep/mine.py
@@ -20,6 +20,7 @@ import re
from collections import Counter
from typing import Any, Callable, List, Optional, Set, Tuple
+from skillopt_sleep.backend import CursorBackendError
from skillopt_sleep.types import SessionDigest, TaskRecord
@@ -300,6 +301,8 @@ def mine(
if llm_miner is not None:
try:
tasks = llm_miner(digests) or []
+ except CursorBackendError:
+ raise
except Exception:
tasks = []
if not tasks:
diff --git a/skillopt_sleep/prompts.py b/skillopt_sleep/prompts.py
new file mode 100644
index 0000000..3553328
--- /dev/null
+++ b/skillopt_sleep/prompts.py
@@ -0,0 +1,237 @@
+"""SkillOpt-Sleep â central prompt registry with live user overrides.
+
+Every LLM-facing prompt template the sleep cycle uses (miner / attempt /
+judge / reflect) lives here, in one place, instead of being scattered as
+inline literals. Two consequences:
+
+ 1. **Auditability** â the dashboard (and any human) can display exactly
+ what instructions each agent role receives, per stage.
+ 2. **Live tuning** â a user override file (``prompts.json`` in the state
+ dir, or ``SKILLOPT_SLEEP_PROMPTS_PATH``) replaces any template without
+ touching code. The file's mtime is checked on every read, so an edit
+ made while a cycle is running takes effect on the very next call.
+
+Placeholders use the ``__NAME__`` convention (simple ``str.replace``, no
+``str.format``) because the templates themselves contain JSON braces.
+
+The default texts are byte-for-byte the prompts previously inlined in
+``backend.py`` / ``llm_miner.py``, so behavior is unchanged unless the user
+overrides a template.
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+from typing import Dict, List, Optional
+
+from skillopt_sleep.config import HOME_STATE_DIR
+
+# ââ default templates âââââââââââââââââââââââââââââââââââââââââââââââââââââââââ
+
+_MINER = """You are mining a user's past AI-assistant sessions to find RECURRING tasks
+worth optimizing a skill for. From the session below, extract 0-3 reusable tasks.
+
+A good task is something the user asks for repeatedly or had to correct, where a
+GENERAL rule would help next time (formatting, structure, tool-use, conventions).
+Skip one-off or purely exploratory requests.
+
+For each task return:
+ - "intent": the reusable request, generalized (no one-off specifics)
+ - "checks": a list of programmatic success checks a grader can run on a future
+ answer. Each check is one of:
+ {"op":"section_present","arg":""}
+ {"op":"regex","arg":""}
+ {"op":"contains","arg":""}
+ {"op":"max_chars","arg":}
+ Only include checks you are confident a GOOD answer must satisfy.
+ - "rubric": a one-sentence description of what a good answer looks like
+ - "satisfied": true/false â did the user seem satisfied with the assistant's answer?
+
+Return ONLY a JSON array (possibly empty). No prose.
+
+# Session
+project: __PROJECT__
+user prompts:
+__PROMPTS__
+assistant final (last):
+__FINAL__
+feedback signals: __FEEDBACK__
+"""
+
+_ATTEMPT = (
+ "Complete the following task for the user. Follow the skill and memory "
+ "guidance below, including any output-format and length requirements. "
+ "When a 'Learned preferences' rule sets an explicit limit (e.g. a length "
+ "cap), prefer that rule over more general advice it refines.\n\n"
+ "# Skill\n__SKILL__\n\n# Memory\n__MEMORY__\n\n"
+ "# Task\n__INTENT__\n\n__CONTEXT__\n\n"
+ "Return ONLY the final answer text, nothing else."
+)
+
+_JUDGE = (
+ "Score how well the response satisfies the rubric, 0..1. "
+ 'Return ONLY JSON {"score": <0..1>, "reason": "..."}.\n\n'
+ "# Rubric\n__RUBRIC__\n\n# Response\n__RESPONSE__"
+)
+
+_REFLECT = (
+ "You are SkillOpt's optimizer. The agent keeps failing the recurring "
+ "tasks below. Propose at most __EDIT_BUDGET__ bounded edits to the "
+ "__TARGET__ document so it stops failing. Each edit MUST be a short, "
+ "GENERAL, reusable rule or preference (never task-specific, never an "
+ "answer to a single task). If exact failing criteria are listed, your "
+ "edits MUST make future outputs satisfy every one of them.\n"
+ "BE CONCRETE: quote the exact threshold, section name, or format from "
+ "the criteria verbatim in your rule (e.g. write 'keep the entire "
+ "response under 1200 characters', NOT 'respect length limits'). Vague "
+ "rules do not change behavior; specific numeric/structural rules do.\n"
+ "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
+ "you CANNOT delete the existing instructions above. If the current "
+ "__TARGET__ text conflicts with a criterion (e.g. it says 'be exhaustive' "
+ "but outputs must be under a character limit), write an explicit, "
+ "forceful OVERRIDE rule stating it supersedes the conflicting "
+ "instruction, and put the hard requirement first.\n"
+ "HARD CONSTRAINT: every rule you write MUST be consistent with the "
+ "'Task output contract' below (if shown). NEVER propose a rule that "
+ "changes the required output format/language, tells the agent to ask "
+ "the user a question, or otherwise violates that contract â such a "
+ "rule scores ZERO because the evaluator cannot honor it.\n"
+ 'Return ONLY a JSON array: '
+ '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n'
+ "# Current __TARGET__\n__CUR_DOC__\n"
+ "__GUARD__"
+ "__CRITERIA__\n"
+ "__PREFS__\n\n"
+ "# Recurring failures\n__FAILURES__"
+)
+
+# name -> {text, stage, role, description, placeholders}
+DEFAULTS: Dict[str, Dict] = {
+ "miner": {
+ "text": _MINER,
+ "stage": "mine",
+ "role": "optimizer",
+ "description": "Turns one harvested session digest into 0-3 checkable TaskRecords.",
+ "placeholders": ["__PROJECT__", "__PROMPTS__", "__FINAL__", "__FEEDBACK__"],
+ },
+ "attempt": {
+ "text": _ATTEMPT,
+ "stage": "replay",
+ "role": "target",
+ "description": "The clean-context rollout: solve a mined task given only skill+memory.",
+ "placeholders": ["__SKILL__", "__MEMORY__", "__INTENT__", "__CONTEXT__"],
+ },
+ "judge": {
+ "text": _JUDGE,
+ "stage": "replay",
+ "role": "optimizer",
+ "description": "Rubric grading for tasks with no programmatic checks (0..1 JSON score).",
+ "placeholders": ["__RUBRIC__", "__RESPONSE__"],
+ },
+ "reflect": {
+ "text": _REFLECT,
+ "stage": "reflect",
+ "role": "optimizer",
+ "description": "Proposes bounded skill/memory edits from the recurring failures.",
+ "placeholders": [
+ "__EDIT_BUDGET__", "__TARGET__", "__CUR_DOC__", "__GUARD__",
+ "__CRITERIA__", "__PREFS__", "__FAILURES__",
+ ],
+ },
+}
+
+
+# ââ override file (mtime-cached; edits take effect on the next call) ââââââââââ
+
+_lock = threading.Lock()
+_cache: Dict[str, object] = {"path": None, "mtime": None, "data": {}}
+
+
+def overrides_path() -> str:
+ return os.environ.get("SKILLOPT_SLEEP_PROMPTS_PATH", "") or os.path.join(
+ HOME_STATE_DIR, "prompts.json"
+ )
+
+
+def load_overrides() -> Dict[str, str]:
+ """Return {name: replacement_text}, re-reading the file iff it changed."""
+ path = overrides_path()
+ with _lock:
+ try:
+ mtime = os.path.getmtime(path)
+ except OSError:
+ _cache.update(path=path, mtime=None, data={})
+ return {}
+ if _cache["path"] == path and _cache["mtime"] == mtime:
+ return dict(_cache["data"]) # type: ignore[arg-type]
+ try:
+ with open(path, encoding="utf-8") as f:
+ raw = json.load(f)
+ data = {
+ k: v for k, v in raw.items()
+ if k in DEFAULTS and isinstance(v, str) and v.strip()
+ }
+ except Exception:
+ data = {}
+ _cache.update(path=path, mtime=mtime, data=data)
+ return dict(data)
+
+
+def save_overrides(overrides: Dict[str, Optional[str]]) -> Dict[str, str]:
+ """Merge ``overrides`` into the override file. A None/empty value removes
+ that override (reverting the template to its default). Returns the new
+ effective override map."""
+ path = overrides_path()
+ current = load_overrides()
+ for k, v in overrides.items():
+ if k not in DEFAULTS:
+ continue
+ if v is None or not str(v).strip():
+ current.pop(k, None)
+ else:
+ current[k] = str(v)
+ os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+ with open(path, "w", encoding="utf-8") as f:
+ json.dump(current, f, ensure_ascii=False, indent=2)
+ with _lock:
+ _cache.update(path=None, mtime=None, data={}) # force re-read
+ return current
+
+
+def get_prompt(name: str) -> str:
+ """Effective template text for ``name`` (override if present, else default)."""
+ ov = load_overrides()
+ if name in ov:
+ return ov[name]
+ return DEFAULTS[name]["text"]
+
+
+def is_overridden(name: str) -> bool:
+ return name in load_overrides()
+
+
+def render(name: str, mapping: Dict[str, str]) -> str:
+ """Substitute ``__NAME__`` placeholders via str.replace (format-safe)."""
+ text = get_prompt(name)
+ for k, v in mapping.items():
+ text = text.replace(k, v)
+ return text
+
+
+def describe() -> List[Dict]:
+ """Registry snapshot for the dashboard: defaults + active overrides."""
+ ov = load_overrides()
+ out = []
+ for name, meta in DEFAULTS.items():
+ out.append({
+ "name": name,
+ "stage": meta["stage"],
+ "role": meta["role"],
+ "description": meta["description"],
+ "placeholders": meta["placeholders"],
+ "default": meta["text"],
+ "override": ov.get(name),
+ "effective": ov.get(name) or meta["text"],
+ })
+ return out
diff --git a/skillopt_sleep/replay.py b/skillopt_sleep/replay.py
index e15f3df..1502a71 100644
--- a/skillopt_sleep/replay.py
+++ b/skillopt_sleep/replay.py
@@ -4,8 +4,9 @@ Re-run mined TaskRecords offline under a given (skill, memory) and score
them, producing the (hard, soft) signal SkillOpt's gate consumes.
Single-shot text replay by default. Tasks whose rule judge requires a tool
-call (gbrain's `tool_called`) are run through the backend's real tool loop
-(attempt_with_tools), so tool use is verified honestly rather than self-reported.
+call (gbrain's `tool_called`) use the backend's tool-aware path. Backends that
+cannot enforce that execution boundary fail explicitly rather than scoring a
+self-reported tool call.
"""
from __future__ import annotations
@@ -53,6 +54,18 @@ def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str,
else:
hard, soft, rationale = backend.judge(task, response)
+ ev = getattr(backend, "evidence", None)
+ if ev is not None:
+ # One scored-attempt record per (phase, task): the task->score link of
+ # the evidentiary chain, with the failing checks named in `why`.
+ ev.log("replay", "result",
+ phase=getattr(backend, "evidence_phase", ""),
+ task_id=task.id, split=task.split, origin=task.origin,
+ reference_kind=task.reference_kind, sample_id=sample_id,
+ hard=float(hard), soft=float(soft), why=rationale or "",
+ response_head=(response or "")[:400], tools_called=tools_called,
+ tokens=int(tokens), latency_ms=round(latency_ms, 1))
+
return ReplayResult(
id=task.id,
hard=float(hard),
@@ -143,4 +156,3 @@ def multi_objective_reward(
if total_w <= 0:
return acc
return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w
-
diff --git a/skillopt_sleep/staging.py b/skillopt_sleep/staging.py
index 49dd859..12201ec 100644
--- a/skillopt_sleep/staging.py
+++ b/skillopt_sleep/staging.py
@@ -16,6 +16,89 @@ from typing import Any, List, Optional
from skillopt_sleep.types import SleepReport
+# A secret value may be quoted, braced (ODBC-style), or an unquoted scalar.
+# Accept EOF as the terminator for quoted/braced values because diagnostics are
+# often truncated precisely where a failing client was printing a credential.
+# Doubled quote/brace characters are the escape convention used by SQL/ODBC.
+_UNQUOTED_SECRET_VALUE = (
+ r'''(?:[^\s"';&,)\]}]|[)\]}]+(?=[^\s"';&,)\]}]))+'''
+)
+_SECRET_VALUE = (
+ r'''(?:"(?:\\(?:[^\r\n]|(?=\r?\n|$))|""|[^"\\\r\n])*'''
+ r'''(?:"|(?=\r?\n|$))'''
+ r'''|'(?:\\(?:[^\r\n]|(?=\r?\n|$))|''|[^'\\\r\n])*'''
+ r'''(?:'|(?=\r?\n|$))'''
+ r'''|\{(?:\\(?:[^\r\n]|(?=\r?\n|$))|}}|[^}\\\r\n])*'''
+ r'''(?:}|(?=\r?\n|$))'''
+ r'''|''' + _UNQUOTED_SECRET_VALUE + r''')'''
+)
+
+# Match both short labels (``token=``) and environment/connection-string names
+# whose final component identifies a credential (``AZURE_CLIENT_SECRET=``).
+_SECRET_NAME_BODY = (
+ r"(?:(?:[A-Za-z0-9]+[_-])*(?:"
+ r"api[_-]?key|access[_-]?token|refresh[_-]?token|token|"
+ r"password|passwd|secret|secret[_-]?key|secret[_-]?access[_-]?key|"
+ r"shared[_-]?access[_-]?key|private[_-]?key"
+ r")|[A-Za-z0-9]*(?:"
+ r"apikey|accesstoken|refreshtoken|clientsecret|secretkey|"
+ r"secretaccesskey|sharedaccesskey|privatekey"
+ r"))"
+)
+_SECRET_ASSIGNMENT_NAME = (
+ r"(?(?[\"'])"
+ + r"(?:" + _SECRET_NAME_BODY + r"|pwd|accountkey)"
+ + r"(?P=key_quote)\s*:\s*)"
+ + r"(?P" + _SECRET_VALUE + r")"
+)
+
+_REDACTED_MARKER = re.compile(r"^\[REDACTED(?:_[A-Z_]+)?\]$")
+_SECRET_MAPPING_KEY_SUFFIXES = (
+ "apikey",
+ "accesstoken",
+ "refreshtoken",
+ "token",
+ "password",
+ "passwd",
+ "clientsecret",
+ "secret",
+ "secretkey",
+ "secretaccesskey",
+ "sharedaccesskey",
+ "privatekey",
+ "accountkey",
+)
+
+
+def _redact_json_assignment(match: re.Match[str]) -> str:
+ """Keep JSON-like value quotes while replacing their complete contents."""
+ value = match.group("value")
+ quote = (
+ value[:1] if value[:1] in {'"', "'"} else match.group("key_quote")
+ )
+ return f"{match.group('prefix')}{quote}[REDACTED]{quote}"
+
+
+def _is_secret_mapping_key(key: Any) -> bool:
+ """Recognize credential-bearing dict keys without flagging token budgets."""
+ if not isinstance(key, str):
+ return False
+ stripped = key.strip()
+ # PWD is conventionally the non-secret process working directory. Mixed or
+ # lower-case ``Pwd`` remains a common database-password field.
+ if stripped == "PWD":
+ return False
+ compact = re.sub(r"[^a-z0-9]", "", stripped.casefold())
+ return compact in {"pwd", "sig", "authorization"} or compact.endswith(
+ _SECRET_MAPPING_KEY_SUFFIXES
+ )
+
# Secret patterns scrubbed from any free-text we persist to the staging dir
# (diagnostics, reports). Kept here so every on-disk artifact shares one
# redaction pass; harvest_codex reuses these for session text too.
@@ -31,14 +114,70 @@ _SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
# the "Authorization:" prefix.
(re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"),
"[REDACTED_JWT]"),
- (re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"),
- (re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"),
(
- re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"),
+ re.compile(
+ r'''(?i)(Authorization:\s*Bearer\s+)'''
+ r'''(?!\[REDACTED(?:_[A-Z_]+)?\])'''
+ + _SECRET_VALUE
+ ),
+ r"\1[REDACTED]",
+ ),
+ (
+ re.compile(
+ r'''(?i)(Authorization:\s*Basic\s+)'''
+ r'''(?!\[REDACTED(?:_[A-Z_]+)?\])'''
+ + _SECRET_VALUE
+ ),
+ r"\1[REDACTED]",
+ ),
+ # Connection-string passwords. Handle quoted values (which may contain
+ # semicolons) before the generic name=value rule below, and retain the key
+ # plus all non-secret connection-string fields for useful diagnostics.
+ (
+ re.compile(
+ r'''(?i)(\bPassword\s*=\s*)'''
+ r'''(?!\[REDACTED(?:_[A-Z_]+)?\])'''
+ + _SECRET_VALUE
+ ),
+ r"\1[REDACTED_DB_PASS]",
+ ),
+ # ODBC commonly abbreviates Password as Pwd. Keep the conventional
+ # all-uppercase PWD working-directory variable intact.
+ (
+ re.compile(
+ r"((?&...)
+ (
+ re.compile(r"(?i)(\bsig\s*=\s*)[A-Za-z0-9%+/]{10,}"),
+ r"\1[REDACTED_SAS_SIG]",
+ ),
+ # Azure Storage account keys (base64, typically 88 chars)
+ (
+ re.compile(
+ r'''(?i)(\bAccountKey\s*=\s*)'''
+ r'''(?!\[REDACTED(?:_[A-Z_]+)?\])'''
+ + _SECRET_VALUE
+ ),
+ r"\1[REDACTED_STORAGE_KEY]",
+ ),
)
@@ -61,14 +214,23 @@ def redact_secrets(value: Any) -> Any:
scalars pass through unchanged.
"""
if isinstance(value, str):
- out = value
+ out = _JSON_SECRET_ASSIGNMENT.sub(_redact_json_assignment, value)
for pattern, replacement in _SECRET_PATTERNS:
out = pattern.sub(replacement, out)
return out
if isinstance(value, list):
return [redact_secrets(v) for v in value]
if isinstance(value, dict):
- return {k: redact_secrets(v) for k, v in value.items()}
+ redacted = {}
+ for key, item in value.items():
+ if _is_secret_mapping_key(key):
+ if isinstance(item, str) and _REDACTED_MARKER.fullmatch(item):
+ redacted[key] = item
+ else:
+ redacted[key] = "[REDACTED]"
+ else:
+ redacted[key] = redact_secrets(item)
+ return redacted
return value
@@ -80,6 +242,16 @@ def staging_root(project: str) -> str:
return os.path.join(project, ".skillopt-sleep", "staging")
+def new_staging_dir(project: str) -> str:
+ """A staging path that is unique even for two runs in the same second."""
+ base = os.path.join(staging_root(project), _ts_dir())
+ out, i = base, 2
+ while os.path.exists(out):
+ out = f"{base}-{i}"
+ i += 1
+ return out
+
+
def latest_staging(project: str) -> Optional[str]:
root = staging_root(project)
if not os.path.isdir(root):
@@ -89,7 +261,12 @@ def latest_staging(project: str) -> Optional[str]:
key=lambda p: os.path.getmtime(p),
reverse=True,
)
- return subs[0] if subs else None
+ for p in subs:
+ # Only adoptable folders count: a no-tasks night leaves evidence.jsonl
+ # but no manifest, and adopt() needs the manifest.
+ if os.path.exists(os.path.join(p, "manifest.json")):
+ return p
+ return None
def write_staging(
@@ -101,9 +278,15 @@ def write_staging(
live_skill_path: str,
live_memory_path: str,
report_md: str,
+ out_dir: str = "",
) -> str:
- """Write proposals + report into staging// and return that path."""
- out = os.path.join(staging_root(project), _ts_dir())
+ """Write proposals + report into staging// and return that path.
+
+ ``out_dir`` lets the cycle pre-create the night's staging folder at cycle
+ START, so incremental artifacts (evidence.jsonl) accumulate in the same
+ place the report lands.
+ """
+ out = out_dir or os.path.join(staging_root(project), _ts_dir())
os.makedirs(out, exist_ok=True)
manifest = {
diff --git a/skillopt_sleep/state.py b/skillopt_sleep/state.py
index 1e16157..97f4519 100644
--- a/skillopt_sleep/state.py
+++ b/skillopt_sleep/state.py
@@ -29,6 +29,8 @@ DEFAULT_STATE: Dict[str, Any] = {
"slow_memory": "", # cross-night consolidated lessons (meta-skill analogue)
"history": [], # list of per-night summaries
"task_archive": [], # capped list of past mined tasks (for associative recall)
+ "last_model_key": "", # "backend::model" string used in the last successful night (F16)
+ "last_model_key_format": 1, # v1=config text; v2=resolved backend/model
}
@@ -94,3 +96,19 @@ class SleepState:
arc.extend(task_dicts)
if len(arc) > cap:
self.data["task_archive"] = arc[-cap:]
+
+ # ââ model-swap tracking (F16) âââââââââââââââââââââââââââââââââââââââââ
+ @property
+ def last_model_key(self) -> str:
+ return str(self.data.get("last_model_key", ""))
+
+ def set_last_model_key(self, key: str) -> None:
+ self.data["last_model_key"] = key
+ self.data["last_model_key_format"] = 2
+
+ @property
+ def last_model_key_format(self) -> int:
+ try:
+ return int(self.data.get("last_model_key_format", 1))
+ except (TypeError, ValueError):
+ return 1
diff --git a/skillopt_sleep/tasks_file.py b/skillopt_sleep/tasks_file.py
index d89166b..5f25f92 100644
--- a/skillopt_sleep/tasks_file.py
+++ b/skillopt_sleep/tasks_file.py
@@ -3,8 +3,10 @@ from __future__ import annotations
import json
import os
+import sys
from typing import Any, Dict, List, Tuple
+from skillopt_sleep.judges import validate_checks
from skillopt_sleep.mine import assign_splits, normalize_legacy_split
from skillopt_sleep.types import TaskRecord
@@ -78,4 +80,21 @@ def load_tasks_file(
if not isinstance(item, dict):
raise ValueError("each task entry must be a JSON object")
tasks.append(TaskRecord.from_dict(item))
+
+ # Fail loudly on malformed rule judges. A check that cannot work as written
+ # scores 0.0 on every rollout, which reads exactly like a model that never
+ # complies â the run looks legitimate while one dimension is dead.
+ errors: List[str] = []
+ for task in tasks:
+ if task.reference_kind != "rule":
+ continue
+ task_errors, task_warnings = validate_checks(task.judge)
+ errors.extend(f"task {task.id}: {e}" for e in task_errors)
+ for w in task_warnings:
+ print(f"[sleep] warning: task {task.id}: {w}", file=sys.stderr)
+ if errors:
+ raise ValueError(
+ "tasks file contains unusable rule checks:\n " + "\n ".join(errors)
+ )
+
return _normalize_tasks(tasks, holdout_fraction=holdout_fraction, seed=seed), meta
diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py
index 6cfa623..6d3dc77 100644
--- a/skillopt_sleep/types.py
+++ b/skillopt_sleep/types.py
@@ -17,8 +17,8 @@ from typing import Any, Dict, List
class SessionDigest:
"""A normalized summary of one local agent session transcript.
- Produced by source-specific harvesters from Claude Code transcripts or
- Codex Desktop archived sessions.
+ Produced by source-specific harvesters from Claude Code transcripts, Codex
+ Desktop archived sessions, or Cursor Agent transcripts.
"""
session_id: str
@@ -29,6 +29,10 @@ class SessionDigest:
user_prompts: List[str] = field(default_factory=list)
assistant_finals: List[str] = field(default_factory=list)
tools_used: List[str] = field(default_factory=list)
+ # Skill targets invoked through the Claude ``Skill`` tool, in first-seen
+ # order. Optional and default-empty: harvesters that do not observe skill
+ # invocations, and digests persisted before this field existed, leave it [].
+ skills_used: List[str] = field(default_factory=list)
files_touched: List[str] = field(default_factory=list)
feedback_signals: List[str] = field(default_factory=list) # "still broken", "perfect", ...
n_user_turns: int = 0
@@ -138,6 +142,11 @@ class SleepReport:
no_edits_reason: str = ""
edits: List[EditRecord] = field(default_factory=list)
rejected_edits: List[EditRecord] = field(default_factory=list)
+ # Proposed edits that changed nothing (anchor absent, duplicate/empty add,
+ # unknown op). They were never scored by the gate, so they belong in neither
+ # list above â without them a night can report no edits while the optimizer
+ # actually produced several.
+ unmatched_edits: List[EditRecord] = field(default_factory=list)
tokens_used: int = 0
notes: List[str] = field(default_factory=list)
diff --git a/tests/test_consolidate_split.py b/tests/test_consolidate_split.py
new file mode 100644
index 0000000..8c3e2e4
--- /dev/null
+++ b/tests/test_consolidate_split.py
@@ -0,0 +1,90 @@
+"""Regression tests for consolidate._split hold-out contract.
+
+test-split tasks must never enter train or val. An all-test batch must not
+silently fall back to using the held-out set for consolidation.
+"""
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+
+from skillopt_sleep.consolidate import _split
+from skillopt_sleep.tasks_file import load_tasks_file, write_tasks_file
+from skillopt_sleep.types import TaskRecord
+
+
+def _ids(tasks):
+ return sorted(t.id for t in tasks)
+
+
+class TestConsolidateSplit(unittest.TestCase):
+ def test_all_test_batch_does_not_leak_into_train_or_val(self):
+ tasks = [
+ TaskRecord(id="t0", project="p", intent="do X", split="test"),
+ TaskRecord(id="t1", project="p", intent="do Y", split="test"),
+ TaskRecord(id="t2", project="p", intent="do Z", split="test"),
+ ]
+ train, val = _split(tasks)
+ self.assertEqual(train, [])
+ self.assertEqual(val, [])
+
+ def test_all_test_via_tasks_file_path_does_not_leak(self):
+ tasks = [
+ TaskRecord(id="t0", project="p", intent="do X", split="test"),
+ TaskRecord(id="t1", project="p", intent="do Y", split="test"),
+ TaskRecord(id="t2", project="p", intent="do Z", split="test"),
+ ]
+ with tempfile.TemporaryDirectory() as tmp:
+ path = write_tasks_file(
+ os.path.join(tmp, "tasks.json"),
+ {"tasks": [t.to_dict() for t in tasks]},
+ )
+ loaded, _ = load_tasks_file(path)
+ train, val = _split(loaded)
+ self.assertEqual(_ids(train), [])
+ self.assertEqual(_ids(val), [])
+ self.assertEqual({t.split for t in loaded}, {"test"})
+
+ def test_train_only_falls_back_val_to_train(self):
+ tasks = [
+ TaskRecord(id="a", project="p", intent="A", split="train"),
+ TaskRecord(id="b", project="p", intent="B", split="train"),
+ ]
+ train, val = _split(tasks)
+ self.assertEqual(_ids(train), ["a", "b"])
+ self.assertEqual(_ids(val), ["a", "b"])
+
+ def test_train_plus_test_without_val_gates_on_train_not_test(self):
+ tasks = [
+ TaskRecord(id="tr", project="p", intent="train", split="train"),
+ TaskRecord(id="te", project="p", intent="test", split="test"),
+ ]
+ train, val = _split(tasks)
+ self.assertEqual(_ids(train), ["tr"])
+ self.assertEqual(_ids(val), ["tr"])
+ self.assertNotIn("te", _ids(train) + _ids(val))
+
+ def test_explicit_train_val_test_keeps_partitions(self):
+ tasks = [
+ TaskRecord(id="tr", project="p", intent="train", split="train"),
+ TaskRecord(id="va", project="p", intent="val", split="val"),
+ TaskRecord(id="te", project="p", intent="test", split="test"),
+ ]
+ train, val = _split(tasks)
+ self.assertEqual(_ids(train), ["tr"])
+ self.assertEqual(_ids(val), ["va"])
+
+ def test_legacy_holdout_name_maps_to_val(self):
+ tasks = [
+ TaskRecord(id="tr", project="p", intent="train", split="replay"),
+ TaskRecord(id="va", project="p", intent="val", split="holdout"),
+ TaskRecord(id="te", project="p", intent="test", split="test"),
+ ]
+ train, val = _split(tasks)
+ self.assertEqual(_ids(train), ["tr"])
+ self.assertEqual(_ids(val), ["va"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_consolidation_edit_bookkeeping.py b/tests/test_consolidation_edit_bookkeeping.py
new file mode 100644
index 0000000..584896c
--- /dev/null
+++ b/tests/test_consolidation_edit_bookkeeping.py
@@ -0,0 +1,133 @@
+"""Final gate rollback must agree with edit bookkeeping and reports."""
+
+from __future__ import annotations
+
+import importlib
+
+from skillopt_sleep.backend import Backend
+from skillopt_sleep.memory import set_learned
+from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord
+
+
+def test_final_validation_rollback_reclassifies_tentative_edits(
+ monkeypatch,
+) -> None:
+ consolidate_module = importlib.import_module("skillopt_sleep.consolidate")
+ edit = EditRecord(
+ target="skill",
+ op="add",
+ content="A tentatively useful rule.",
+ rationale="trial improved",
+ )
+
+ class EditingBackend(Backend):
+ name = "editing-stub"
+
+ def reflect(self, *args, **kwargs):
+ return [edit]
+
+ task = TaskRecord(
+ id="validation-task",
+ project="test",
+ intent="test final rollback",
+ split="val",
+ reference_kind="exact",
+ reference="ok",
+ )
+ scores = iter((0.0, 0.0, 1.0, 0.0))
+
+ def fake_replay_batch(backend, tasks, skill, memory):
+ score = next(scores)
+ return [
+ (
+ item,
+ ReplayResult(
+ id=item.id,
+ hard=score,
+ soft=score,
+ response="ok" if score else "miss",
+ ),
+ )
+ for item in tasks
+ ]
+
+ monkeypatch.setattr(
+ consolidate_module, "replay_batch", fake_replay_batch
+ )
+ original = set_learned("# Skill\n", [])
+
+ result = consolidate_module.consolidate(
+ EditingBackend(),
+ [task],
+ original,
+ "",
+ gate_metric="hard",
+ evolve_memory=False,
+ )
+
+ assert result.accepted is False
+ assert result.new_skill == original
+ assert result.applied_edits == []
+ assert result.rejected_edits == [edit]
+
+
+def test_final_rollback_does_not_duplicate_an_already_rejected_edit(
+ monkeypatch,
+) -> None:
+ consolidate_module = importlib.import_module("skillopt_sleep.consolidate")
+ edit = EditRecord(
+ target="skill",
+ op="add",
+ content="A repeated proposal.",
+ rationale="same proposal for both targets",
+ )
+
+ class RepeatingBackend(Backend):
+ name = "repeating-stub"
+
+ def reflect(self, *args, **kwargs):
+ return [edit]
+
+ task = TaskRecord(
+ id="dedup-task",
+ project="test",
+ intent="test rollback deduplication",
+ split="val",
+ reference_kind="exact",
+ reference="ok",
+ )
+ # baseline, train, rejected skill trial, post-skill train, accepted memory
+ # trial, then regressing final replay.
+ scores = iter((0.0, 0.0, 0.0, 0.0, 1.0, 0.0))
+
+ def fake_replay_batch(backend, tasks, skill, memory):
+ score = next(scores)
+ return [
+ (
+ item,
+ ReplayResult(
+ id=item.id,
+ hard=score,
+ soft=score,
+ response="ok" if score else "miss",
+ ),
+ )
+ for item in tasks
+ ]
+
+ monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch)
+ original = set_learned("# Skill\n", [])
+
+ result = consolidate_module.consolidate(
+ RepeatingBackend(),
+ [task],
+ original,
+ original,
+ gate_metric="hard",
+ evolve_skill=True,
+ evolve_memory=True,
+ )
+
+ assert result.accepted is False
+ assert result.applied_edits == []
+ assert result.rejected_edits == [edit]
diff --git a/tests/test_cursor_exec_backend.py b/tests/test_cursor_exec_backend.py
new file mode 100644
index 0000000..3fea88f
--- /dev/null
+++ b/tests/test_cursor_exec_backend.py
@@ -0,0 +1,438 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from collections.abc import Iterator
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+import skillopt.model as model
+from skillopt.config import flatten_config
+from skillopt.model import backend_config
+from skillopt.model import codex_harness as harness
+from skillopt.model.common import default_model_for_backend, normalize_backend_name
+
+
+@pytest.fixture(autouse=True)
+def restore_backend_state() -> Iterator[None]:
+ optimizer_backend = backend_config.get_optimizer_backend()
+ target_backend = backend_config.get_target_backend()
+ cursor_path = backend_config.CURSOR_EXEC_PATH
+ cursor_sandbox = backend_config.CURSOR_EXEC_SANDBOX
+ retries = backend_config.EXEC_EMPTY_RESPONSE_RETRIES
+ env = {
+ key: os.environ.get(key)
+ for key in (
+ "OPTIMIZER_BACKEND",
+ "TARGET_BACKEND",
+ "CURSOR_EXEC_PATH",
+ "CURSOR_EXEC_SANDBOX",
+ )
+ }
+ yield
+ backend_config.OPTIMIZER_BACKEND = optimizer_backend
+ backend_config.TARGET_BACKEND = target_backend
+ backend_config.CURSOR_EXEC_PATH = cursor_path
+ backend_config.CURSOR_EXEC_SANDBOX = cursor_sandbox
+ backend_config.EXEC_EMPTY_RESPONSE_RETRIES = retries
+ for key, value in env.items():
+ if value is None:
+ os.environ.pop(key, None)
+ else:
+ os.environ[key] = value
+
+
+def _result(text: str = "A") -> str:
+ return (
+ '{"type":"system","subtype":"init","model":"composer-2.5",'
+ '"permissionMode":"default","session_id":"session-1"}\n'
+ '{"type":"tool_call","subtype":"started","call_id":"call-1",'
+ '"tool_call":{"readToolCall":{"args":{"path":"task.md"}}}}\n'
+ f'{{"type":"result","subtype":"success","is_error":false,'
+ f'"duration_ms":12,"result":"{text}","session_id":"session-1"}}\n'
+ )
+
+
+def _workspace(tmp_path: Path) -> Path:
+ work_dir = tmp_path / "predictions" / "task-1" / "cursor_exec"
+ work_dir.mkdir(parents=True)
+ return work_dir
+
+
+def test_cursor_exec_is_target_only() -> None:
+ backend_config.set_target_backend("cursor")
+
+ assert backend_config.get_target_backend() == "cursor_exec"
+ assert backend_config.is_target_exec_backend()
+ with pytest.raises(ValueError, match="Unsupported optimizer backend"):
+ backend_config.set_optimizer_backend("cursor_exec")
+ with pytest.raises(NotImplementedError, match="Exec backends"):
+ model.chat_target("system", "user")
+
+
+def test_cursor_alias_and_default_model() -> None:
+ assert normalize_backend_name("cursor") == "cursor_exec"
+ assert normalize_backend_name("cursor_agent") == "cursor_exec"
+ assert default_model_for_backend("cursor_exec") == "composer-2.5"
+
+ assert model.set_backend("cursor") == "cursor_exec"
+ assert backend_config.get_optimizer_backend() == "openai_chat"
+ assert backend_config.get_target_backend() == "cursor_exec"
+ assert model.get_backend_name() == "cursor_exec"
+
+
+def test_cursor_config_flattens_and_validates() -> None:
+ flat = flatten_config(
+ {
+ "model": {
+ "cursor_exec_path": "/opt/cursor-agent",
+ "cursor_exec_sandbox": "disabled",
+ }
+ }
+ )
+ assert flat["cursor_exec_path"] == "/opt/cursor-agent"
+ assert flat["cursor_exec_sandbox"] == "disabled"
+
+ backend_config.configure_cursor_exec(path="cursor-test", sandbox="disabled")
+ assert backend_config.get_cursor_exec_config()["path"] == "cursor-test"
+ assert backend_config.get_cursor_exec_config()["sandbox"] == "disabled"
+ with pytest.raises(ValueError, match="sandbox must be"):
+ backend_config.configure_cursor_exec(sandbox="invalid")
+
+
+def test_train_cursor_shorthand_configures_target_only(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from scripts import train
+
+ config_path = Path(__file__).parents[1] / "configs" / "_base_" / "default.yaml"
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ ["train.py", "--config", str(config_path), "--backend", "cursor"],
+ )
+
+ cfg = train.load_config(train.parse_args())
+
+ assert cfg["optimizer_backend"] == "openai_chat"
+ assert cfg["target_backend"] == "cursor_exec"
+ assert cfg["target_model"] == "composer-2.5"
+
+
+def test_read_only_cursor_exec_uses_stdin_and_preserves_trace(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ data_dir = tmp_path / "corpus"
+ data_dir.mkdir()
+ calls: list[tuple[list[str], dict[str, Any]]] = []
+
+ def fake_run(cmd: list[str], **kwargs: Any) -> SimpleNamespace:
+ calls.append((cmd, kwargs))
+ return SimpleNamespace(returncode=0, stdout="not json\n" + _result(), stderr="")
+
+ backend_config.configure_cursor_exec(path="cursor-test", sandbox="enabled")
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ response, raw = harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer the benchmark task.",
+ model="composer-2.5",
+ timeout=17,
+ data_dirs=[str(data_dir)],
+ )
+
+ assert response == "A"
+ assert "tool_call" in raw
+ assert "not json" not in raw
+ assert "task.md" not in raw
+ assert "A" not in raw
+ cmd, kwargs = calls[0]
+ assert cmd[:4] == ["cursor-test", "-p", "--output-format", "stream-json"]
+ assert ["--mode", "ask"] == cmd[cmd.index("--mode"):cmd.index("--mode") + 2]
+ assert "--force" not in cmd
+ assert cmd[cmd.index("--workspace") + 1] == str(work_dir)
+ assert cmd[cmd.index("--sandbox") + 1] == "enabled"
+ assert cmd[cmd.index("--model") + 1] == "composer-2.5"
+ assert cmd[cmd.index("--add-dir") + 1] == str(data_dir)
+ assert kwargs["cwd"] == str(work_dir)
+ assert kwargs["timeout"] == 17
+ assert ".agents/skills/skillopt-target/SKILL.md" in kwargs["input"]
+ assert "Do not modify files" in kwargs["input"]
+ persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text()
+ assert "not json" not in persisted_raw
+ assert "task.md" not in persisted_raw
+ assert "A" not in persisted_raw
+ summary = (work_dir.parent / "cursor_trace_summary.txt").read_text()
+ assert "tool calls: 1" in summary
+ assert "session-1" in summary
+
+
+def test_cursor_exec_force_is_limited_to_file_edit_rollouts(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ calls: list[tuple[list[str], str]] = []
+
+ def fake_run(cmd: list[str], **kwargs: Any) -> SimpleNamespace:
+ calls.append((cmd, kwargs["input"]))
+ return SimpleNamespace(returncode=0, stdout=_result(), stderr="")
+
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Write solution.py.",
+ model="",
+ timeout=10,
+ allow_file_edits=True,
+ )
+
+ cmd, prompt = calls[0]
+ assert "--force" in cmd
+ assert "--mode" not in cmd
+ assert "You may modify files" in prompt
+
+
+def test_cursor_exec_rejects_file_edits_with_disabled_sandbox(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ calls = 0
+
+ def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace:
+ nonlocal calls
+ calls += 1
+ return SimpleNamespace(returncode=0, stdout=_result(), stderr="")
+
+ backend_config.configure_cursor_exec(sandbox="disabled")
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ with pytest.raises(ValueError, match="refusing to combine --force"):
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Write solution.py.",
+ model="composer-2.5",
+ timeout=10,
+ allow_file_edits=True,
+ )
+
+ assert calls == 0
+
+
+def test_cursor_exec_retries_zero_exit_malformed_output(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ prompts: list[str] = []
+
+ def fake_run(_cmd: list[str], **kwargs: Any) -> SimpleNamespace:
+ prompts.append(kwargs["input"])
+ stdout = "malformed output" if len(prompts) == 1 else _result()
+ return SimpleNamespace(returncode=0, stdout=stdout, stderr="")
+
+ backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ response, _raw = harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer.",
+ model="",
+ timeout=10,
+ )
+
+ assert response == "A"
+ assert len(prompts) == 2
+ assert "Previous execution returned an empty final response" in prompts[1]
+
+
+def test_cursor_trace_summary_tolerates_malformed_metadata() -> None:
+ raw = (
+ '{"type":"result","subtype":"success","is_error":false,'
+ '"duration_ms":"unknown","result":"done"}\n'
+ )
+
+ summary = harness._build_cursor_trace_summary(raw, "done")
+
+ assert "duration ms: 0" in summary
+
+
+def test_cursor_trace_omits_message_and_tool_payloads() -> None:
+ raw = "\n".join(
+ [
+ '{"type":"user","message":{"role":"user","content":'
+ '[{"type":"text","text":"private prompt"}]}}',
+ '{"type":"assistant","message":{"role":"assistant","content":'
+ '[{"type":"text","text":"private response"}]}}',
+ '{"type":"tool_call","subtype":"completed","tool_call":'
+ '{"readToolCall":{"args":{"path":"secret.txt"},"result":'
+ '{"success":{"content":"private file contents"}}}}}',
+ '{"type":"result","subtype":"success","is_error":false,'
+ '"duration_ms":1,"result":"private final answer"}',
+ ]
+ )
+
+ sanitized = harness._sanitize_cursor_trace(raw)
+ events = [json.loads(line) for line in sanitized.splitlines()]
+
+ assert events[0]["message"]["content"] == "[OMITTED]"
+ assert events[1]["message"]["content"] == "[OMITTED]"
+ assert events[2]["tool_call"]["readToolCall"]["args"] == "[OMITTED]"
+ assert events[2]["tool_call"]["readToolCall"]["result"] == "[OMITTED]"
+ assert events[3]["result"] == "[OMITTED]"
+ assert "private" not in sanitized
+ assert "secret.txt" not in sanitized
+
+
+def test_cursor_exec_does_not_retry_error_result_and_redacts_detail(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ calls = 0
+
+ def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace:
+ nonlocal calls
+ calls += 1
+ stdout = (
+ '{"type":"result","subtype":"error","is_error":true,'
+ '"result":"API key: cursor-secret-value"}\n'
+ )
+ return SimpleNamespace(returncode=0, stdout=stdout, stderr="")
+
+ backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ with pytest.raises(RuntimeError) as exc_info:
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer.",
+ model="",
+ timeout=10,
+ )
+
+ assert calls == 1
+ assert "[REDACTED]" in str(exc_info.value)
+ assert "cursor-secret-value" not in str(exc_info.value)
+ persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text()
+ assert "cursor-secret-value" not in persisted_raw
+ assert '"result":"[OMITTED]"' in persisted_raw
+
+
+def test_cursor_exec_nonzero_exit_is_not_retried(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+ calls = 0
+
+ def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace:
+ nonlocal calls
+ calls += 1
+ return SimpleNamespace(
+ returncode=1,
+ stdout="",
+ stderr="CURSOR_API_KEY=cursor-secret-token authentication failed",
+ )
+
+ backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ with pytest.raises(RuntimeError) as exc_info:
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer.",
+ model="",
+ timeout=10,
+ )
+
+ assert calls == 1
+ assert "[REDACTED]" in str(exc_info.value)
+ assert "cursor-secret-token" not in str(exc_info.value)
+ persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text()
+ assert "cursor-secret-token" not in persisted_raw
+ assert "CURSOR_API_KEY=[REDACTED]" in persisted_raw
+
+
+def test_cursor_exec_timeout_is_persisted(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+
+ def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace:
+ raise subprocess.TimeoutExpired(
+ "cursor-test",
+ 3,
+ output=b"partial CURSOR_API_KEY=timeout-secret",
+ )
+
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ with pytest.raises(subprocess.TimeoutExpired):
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer.",
+ model="",
+ timeout=3,
+ )
+
+ persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text()
+ assert "partial" not in persisted_raw
+ assert "[OMITTED NON-JSON OUTPUT]" in persisted_raw
+ assert "timeout-secret" not in persisted_raw
+
+
+def test_cursor_exec_spawn_failure_is_actionable(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ work_dir = _workspace(tmp_path)
+
+ def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace:
+ raise FileNotFoundError("cursor-agent-test was not found")
+
+ monkeypatch.setattr(harness.subprocess, "run", fake_run)
+
+ with pytest.raises(RuntimeError, match="could not be executed"):
+ harness.run_cursor_exec(
+ work_dir=str(work_dir),
+ prompt="Answer.",
+ model="",
+ timeout=3,
+ )
+
+
+def test_run_target_exec_dispatches_cursor(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ captured: dict[str, Any] = {}
+
+ def fake_cursor(**kwargs: Any) -> tuple[str, str]:
+ captured.update(kwargs)
+ return "cursor response", "cursor trace"
+
+ backend_config.set_target_backend("cursor_exec")
+ monkeypatch.setattr(harness, "run_cursor_exec", fake_cursor)
+
+ response, raw = harness.run_target_exec(
+ work_dir=str(tmp_path),
+ prompt="task",
+ model="composer-2.5",
+ timeout=20,
+ allow_file_edits=True,
+ )
+
+ assert (response, raw) == ("cursor response", "cursor trace")
+ assert captured["allow_file_edits"] is True
+ assert captured["model"] == "composer-2.5"
diff --git a/tests/test_data_manifests.py b/tests/test_data_manifests.py
new file mode 100644
index 0000000..bb61095
--- /dev/null
+++ b/tests/test_data_manifests.py
@@ -0,0 +1,23 @@
+import json
+from pathlib import Path
+
+from skillopt.datasets.base import SPLIT_NAMES
+
+DATA_DIR = Path(__file__).resolve().parents[1] / "data"
+
+
+def test_split_manifest_counts_match_item_files() -> None:
+ manifest_paths = sorted(DATA_DIR.glob("*/split_manifest.json"))
+ assert manifest_paths, f"No split manifests found under {DATA_DIR}"
+
+ for manifest_path in manifest_paths:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+
+ for split_name in SPLIT_NAMES:
+ declared_count = manifest["counts"][split_name]
+ items_path = manifest_path.parent / split_name / "items.json"
+ actual_count = len(json.loads(items_path.read_text(encoding="utf-8")))
+
+ assert declared_count == actual_count, (
+ f"{manifest_path} split {split_name!r}: declared count {declared_count}, actual count {actual_count}"
+ )
diff --git a/tests/test_devin_plugin.py b/tests/test_devin_plugin.py
index fb276b9..dde3d85 100644
--- a/tests/test_devin_plugin.py
+++ b/tests/test_devin_plugin.py
@@ -2,6 +2,10 @@
import importlib
import json
import os
+import shlex
+import shutil
+import stat
+import subprocess
import sys
import tempfile
import unittest
@@ -14,6 +18,7 @@ import mcp_server # noqa: E402
import harvest_devin as hw # noqa: E402
FIXTURES = os.path.join(PLUGIN, "fixtures")
+INSTALLER = os.path.join(PLUGIN, "install.sh")
def _read_jsonl(path):
@@ -46,7 +51,7 @@ class TestDevinMcpSchema(unittest.TestCase):
def test_backends_in_enum(self):
backends = mcp_server._TOOL_SCHEMA["properties"]["backend"]["enum"]
- for b in ["mock", "claude", "codex", "copilot"]:
+ for b in ["mock", "claude", "codex", "copilot", "handoff"]:
self.assertIn(b, backends)
def test_schema_has_key_engine_params(self):
@@ -64,6 +69,10 @@ class TestClaudeHomeExpansion(unittest.TestCase):
(the documented mcp-config sets SKILLOPT_DEVIN_CLAUDE_HOME="~/...")."""
def test_env_tilde_is_expanded(self):
+ # Re-insert the devin plugin path at position 0 so importlib.reload
+ # picks up this module, not plugins/copilot/mcp_server.py when both
+ # test modules are loaded in the same process.
+ sys.path.insert(0, PLUGIN)
os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"] = "~/.skillopt-sleep-devin"
try:
importlib.reload(mcp_server)
@@ -75,6 +84,165 @@ class TestClaudeHomeExpansion(unittest.TestCase):
importlib.reload(mcp_server)
+class TestDevinInstaller(unittest.TestCase):
+ def _run_installer(self, project, home, installer=INSTALLER):
+ env = os.environ.copy()
+ env["HOME"] = home
+ return subprocess.run(
+ ["bash", installer, project],
+ check=True,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+
+ @staticmethod
+ def _skillopt_hook():
+ config_path = os.path.join(PLUGIN, "hooks", "hooks.v1.json")
+ with open(config_path, encoding="utf-8") as f:
+ return json.load(f)["SessionEnd"][0]
+
+ def test_new_install_and_hook_marker(self):
+ with tempfile.TemporaryDirectory() as d:
+ project = os.path.join(d, "project with spaces")
+ home = os.path.join(d, "home")
+ os.makedirs(project)
+ os.makedirs(home)
+
+ self._run_installer(project, home)
+
+ config_path = os.path.join(project, ".devin", "hooks.v1.json")
+ with open(config_path, encoding="utf-8") as f:
+ config = json.load(f)
+ self.assertEqual(config["SessionEnd"], [self._skillopt_hook()])
+
+ hook_path = os.path.join(
+ project, ".devin", "hooks", "skillopt-sleep-on-session-end.sh"
+ )
+ self.assertTrue(os.stat(hook_path).st_mode & stat.S_IXUSR)
+ env = os.environ.copy()
+ env.update(HOME=home, DEVIN_PROJECT_DIR=project)
+ subprocess.run([hook_path], check=True, env=env)
+ marker = os.path.join(home, ".skillopt-sleep", "session-end.log")
+ with open(marker, encoding="utf-8") as f:
+ lines = f.readlines()
+ self.assertEqual(len(lines), 1)
+ self.assertTrue(lines[0].endswith(f"\t{project}\n"))
+
+ def test_hook_is_non_blocking_without_home(self):
+ env = os.environ.copy()
+ env.pop("HOME", None)
+ result = subprocess.run(
+ [os.path.join(PLUGIN, "hooks", "on-session-end.sh")],
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ self.assertEqual(result.returncode, 0)
+ self.assertEqual(result.stderr, "")
+
+ def test_existing_config_without_session_end_is_extended(self):
+ unrelated = [
+ {"matcher": "", "hooks": [{"type": "command", "command": "./pre.sh"}]}
+ ]
+ with tempfile.TemporaryDirectory() as d:
+ project = os.path.join(d, "project")
+ home = os.path.join(d, "home")
+ devin_dir = os.path.join(project, ".devin")
+ os.makedirs(devin_dir)
+ os.makedirs(home)
+ config_path = os.path.join(devin_dir, "hooks.v1.json")
+ with open(config_path, "w", encoding="utf-8") as f:
+ json.dump({"PreToolUse": unrelated}, f)
+
+ self._run_installer(project, home)
+
+ with open(config_path, encoding="utf-8") as f:
+ config = json.load(f)
+ self.assertEqual(config["PreToolUse"], unrelated)
+ self.assertEqual(config["SessionEnd"], [self._skillopt_hook()])
+
+ def test_existing_hooks_are_preserved_and_reinstall_is_idempotent(self):
+ existing_session_end = {
+ "matcher": "existing",
+ "hooks": [{"type": "command", "command": "./existing.sh"}],
+ }
+ unrelated = [
+ {"matcher": "", "hooks": [{"type": "command", "command": "./pre.sh"}]}
+ ]
+ with tempfile.TemporaryDirectory() as d:
+ project = os.path.join(d, "project")
+ home = os.path.join(d, "home")
+ devin_dir = os.path.join(project, ".devin")
+ os.makedirs(devin_dir)
+ os.makedirs(home)
+ hooks_dir = os.path.join(devin_dir, "hooks")
+ os.makedirs(hooks_dir)
+ legacy_hook = os.path.join(hooks_dir, "on-session-end.sh")
+ with open(legacy_hook, "w", encoding="utf-8") as f:
+ f.write("#!/bin/sh\n# existing project hook\n")
+ config_path = os.path.join(devin_dir, "hooks.v1.json")
+ with open(config_path, "w", encoding="utf-8") as f:
+ json.dump(
+ {"PreToolUse": unrelated, "SessionEnd": [existing_session_end]}, f
+ )
+
+ self._run_installer(project, home)
+ self._run_installer(project, home)
+
+ with open(config_path, encoding="utf-8") as f:
+ config = json.load(f)
+ self.assertEqual(config["PreToolUse"], unrelated)
+ self.assertIn(existing_session_end, config["SessionEnd"])
+ self.assertEqual(config["SessionEnd"].count(self._skillopt_hook()), 1)
+ with open(legacy_hook, encoding="utf-8") as f:
+ self.assertEqual(f.read(), "#!/bin/sh\n# existing project hook\n")
+
+ def test_malformed_existing_config_fails_without_overwrite(self):
+ with tempfile.TemporaryDirectory() as d:
+ project = os.path.join(d, "project")
+ home = os.path.join(d, "home")
+ devin_dir = os.path.join(project, ".devin")
+ os.makedirs(devin_dir)
+ os.makedirs(home)
+ config_path = os.path.join(devin_dir, "hooks.v1.json")
+ original = "{not-json\n"
+ with open(config_path, "w", encoding="utf-8") as f:
+ f.write(original)
+
+ with self.assertRaises(subprocess.CalledProcessError):
+ self._run_installer(project, home)
+ with open(config_path, encoding="utf-8") as f:
+ self.assertEqual(f.read(), original)
+
+ def test_registration_path_is_shell_quoted(self):
+ with tempfile.TemporaryDirectory() as d:
+ plugin_copy = os.path.join(
+ d, "repo with spaces $dollar `tick` 'quote'", "plugins", "devin"
+ )
+ shutil.copytree(PLUGIN, plugin_copy)
+ project = os.path.join(d, "project")
+ home = os.path.join(d, "home")
+ os.makedirs(project)
+ os.makedirs(home)
+
+ result = self._run_installer(
+ project,
+ home,
+ installer=os.path.join(plugin_copy, "install.sh"),
+ )
+
+ command_line = next(
+ line.strip()
+ for line in result.stdout.splitlines()
+ if line.strip().startswith("-- python3 ")
+ )
+ self.assertEqual(
+ shlex.split(command_line),
+ ["--", "python3", os.path.join(plugin_copy, "mcp_server.py")],
+ )
+
+
class TestDevinHarvest(unittest.TestCase):
def test_atif_fixture_yields_gradeable_task(self):
with tempfile.TemporaryDirectory() as out:
diff --git a/tests/test_executor_env_isolation.py b/tests/test_executor_env_isolation.py
new file mode 100644
index 0000000..0c721ce
--- /dev/null
+++ b/tests/test_executor_env_isolation.py
@@ -0,0 +1,196 @@
+"""Tests for subprocess environment isolation in the spreadsheet executor.
+
+``run_generated_code`` runs LLM-generated Python in a child process. To avoid
+leaking API keys / cloud credentials into untrusted generated code, the child
+must run with a minimal, scrubbed environment rather than inheriting the
+parent process environment. These tests assert that scrubbing behaviour.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from types import SimpleNamespace
+
+from skillopt.envs.spreadsheetbench.codegen_agent import _build_codex_driver
+from skillopt.envs.spreadsheetbench.executor import generated_code_env, run_generated_code
+from skillopt.envs.spreadsheetbench.react_agent import _run_bash
+
+
+# User code that records whether a given env var is visible to the child.
+_PROBE = (
+ "import os\n"
+ "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n"
+ " _f.write(os.environ.get('SUPER_SECRET_TOKEN', 'ABSENT'))\n"
+)
+
+
+def test_secret_env_not_visible_to_generated_code(tmp_path, monkeypatch) -> None:
+ monkeypatch.setenv("SUPER_SECRET_TOKEN", "leak-me-please")
+ out = tmp_path / "out.txt"
+
+ ok, err = run_generated_code(_PROBE, str(tmp_path / "in.xlsx"), str(out))
+
+ assert ok, err
+ assert out.read_text(encoding="utf-8") == "ABSENT"
+
+
+def test_path_still_available_to_generated_code(tmp_path) -> None:
+ # PATH must be preserved so the child can still locate the interpreter's
+ # tooling; only sensitive vars are dropped.
+ probe = (
+ "import os\n"
+ "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n"
+ " _f.write('YES' if os.environ.get('PATH') else 'NO')\n"
+ )
+ out = tmp_path / "out.txt"
+
+ ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out))
+
+ assert ok, err
+ assert out.read_text(encoding="utf-8") == "YES"
+
+
+def test_non_secret_python_environment_is_preserved(tmp_path, monkeypatch) -> None:
+ monkeypatch.setenv("PYTHONPATH", "/safe/development/path")
+ monkeypatch.setenv("LANG", "C.UTF-8")
+ monkeypatch.setenv("AZURE_OPENAI_API_KEY", "must-not-leak")
+
+ env = generated_code_env(str(tmp_path), str(tmp_path / "scratch"))
+
+ assert env["PYTHONPATH"] == "/safe/development/path"
+ assert env["LANG"] == "C.UTF-8"
+ assert "AZURE_OPENAI_API_KEY" not in env
+
+
+def test_installed_spreadsheet_dependency_is_importable(tmp_path) -> None:
+ probe = (
+ "import openpyxl\n"
+ "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n"
+ " _f.write('YES')\n"
+ )
+ out = tmp_path / "out.txt"
+
+ ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out))
+
+ assert ok, err
+ assert out.read_text(encoding="utf-8") == "YES"
+
+
+def test_generated_scratch_directory_is_private_and_cleaned(tmp_path) -> None:
+ probe = (
+ "import tempfile\n"
+ "scratch = tempfile.NamedTemporaryFile(delete=False)\n"
+ "scratch.close()\n"
+ "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n"
+ " _f.write(scratch.name)\n"
+ )
+ out = tmp_path / "out.txt"
+
+ ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out))
+
+ assert ok, err
+ scratch_path = out.read_text(encoding="utf-8")
+ assert os.path.dirname(scratch_path) != str(tmp_path)
+ assert not os.path.exists(scratch_path)
+
+
+def test_codex_driver_scrubs_env_sets_tempdir_and_cleans_runner(
+ tmp_path, monkeypatch
+) -> None:
+ monkeypatch.setenv("SUPER_SECRET_TOKEN", "do-not-inherit")
+ monkeypatch.setenv("PYTHONPATH", "/safe/development/path")
+ sentinel = tmp_path / "_driver_runner.py"
+ sentinel.write_text("do not overwrite", encoding="utf-8")
+ (tmp_path / "solution.py").write_text(
+ "import os\n"
+ "with open(OUTPUT_PATH, 'w', encoding='utf-8') as f:\n"
+ " f.write('|'.join([\n"
+ " os.environ.get('SUPER_SECRET_TOKEN', 'ABSENT'),\n"
+ " 'TMPDIR' if os.environ.get('TMPDIR') else 'NO_TMPDIR',\n"
+ " 'PRIVATE_TMP' if os.environ.get('TMPDIR') != os.getcwd() else 'BAD_TMP',\n"
+ " 'PRIVATE_HOME' if os.environ.get('HOME') == os.getcwd() else 'BAD_HOME',\n"
+ " 'DEV_PATH' if os.environ.get('PYTHONPATH') == '/safe/development/path' else 'NO_DEV_PATH',\n"
+ " ]))\n",
+ encoding="utf-8",
+ )
+ driver = tmp_path / "run_solution.py"
+ driver.write_text(_build_codex_driver(), encoding="utf-8")
+
+ proc = subprocess.run(
+ [sys.executable, str(driver)],
+ cwd=tmp_path,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+
+ assert proc.returncode == 0, proc.stdout + proc.stderr
+ assert (
+ tmp_path / "output.xlsx"
+ ).read_text(encoding="utf-8") == (
+ "ABSENT|TMPDIR|PRIVATE_TMP|PRIVATE_HOME|DEV_PATH"
+ )
+ assert sentinel.read_text(encoding="utf-8") == "do not overwrite"
+
+
+def test_generated_code_tempdirs_tolerate_windows_cleanup_races(
+ tmp_path, monkeypatch
+) -> None:
+ cleanup_modes = []
+
+ def simulated_windows_rmtree(cls, name, ignore_errors=False, repeated=False):
+ del cls, repeated
+ cleanup_modes.append(ignore_errors)
+ if not ignore_errors:
+ raise PermissionError(32, "directory is still in use", name)
+ shutil.rmtree(name, ignore_errors=True)
+
+ monkeypatch.setattr(
+ tempfile.TemporaryDirectory,
+ "_rmtree",
+ classmethod(simulated_windows_rmtree),
+ )
+ monkeypatch.setattr(
+ "skillopt.envs.spreadsheetbench.executor.subprocess.run",
+ lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="", stderr=""),
+ )
+ out = tmp_path / "out.txt"
+ out.write_text("created", encoding="utf-8")
+
+ ok, err = run_generated_code("pass", "input.xlsx", str(out))
+
+ assert ok, err
+ assert cleanup_modes == [True]
+
+
+def test_react_tempdir_tolerates_windows_cleanup_races(tmp_path, monkeypatch) -> None:
+ cleanup_modes = []
+
+ def simulated_windows_rmtree(cls, name, ignore_errors=False, repeated=False):
+ del cls, repeated
+ cleanup_modes.append(ignore_errors)
+ if not ignore_errors:
+ raise PermissionError(32, "directory is still in use", name)
+ shutil.rmtree(name, ignore_errors=True)
+
+ monkeypatch.setattr(
+ tempfile.TemporaryDirectory,
+ "_rmtree",
+ classmethod(simulated_windows_rmtree),
+ )
+ monkeypatch.setattr(
+ "skillopt.envs.spreadsheetbench.react_agent.subprocess.run",
+ lambda *args, **kwargs: SimpleNamespace(
+ returncode=0, stdout="success", stderr=""
+ ),
+ )
+
+ assert _run_bash("python -c pass", str(tmp_path)) == "success"
+ assert cleanup_modes == [True]
+
+
+def test_codex_driver_uses_cleanup_tolerant_tempdir() -> None:
+ assert "shutil.rmtree(_temp_dir, ignore_errors=True)" in _build_codex_driver()
diff --git a/tests/test_judges.py b/tests/test_judges.py
new file mode 100644
index 0000000..e4aca3d
--- /dev/null
+++ b/tests/test_judges.py
@@ -0,0 +1,211 @@
+"""Tests for skillopt_sleep.judges â the rule judge that decides every night.
+
+Focus: a malformed check must never be indistinguishable from an unmet one.
+A regex that does not compile returns False on every rollout, so the affected
+dimension scores 0.0 forever while the run still looks healthy.
+"""
+import json
+import unittest
+
+import pytest
+
+from skillopt_sleep.judges import KNOWN_OPS, score_rule_judge, validate_checks
+from skillopt_sleep.tasks_file import load_tasks_file
+
+
+class TestCheckOperators(unittest.TestCase):
+ def _score(self, op, arg, response, tools=None):
+ return score_rule_judge({"kind": "rule", "checks": [{"op": op, "arg": arg}]},
+ response, tools or [])
+
+ def test_contains_is_case_insensitive(self) -> None:
+ self.assertEqual(self._score("contains", "Key Risks", "the KEY RISKS section")[0], 1.0)
+
+ def test_regex_matches(self) -> None:
+ self.assertEqual(self._score("regex", r"\d+\.\d+", "see 7.4 for details")[0], 1.0)
+
+ def test_min_and_max_chars(self) -> None:
+ self.assertEqual(self._score("min_chars", 5, "abcdef")[0], 1.0)
+ self.assertEqual(self._score("min_chars", 50, "abcdef")[0], 0.0)
+ self.assertEqual(self._score("max_chars", 5, "abcdef")[0], 0.0)
+
+ def test_section_present_accepts_heading_and_bold_and_label(self) -> None:
+ for text in ("## Key Risks", "**Key Risks:**", "Key Risks: something"):
+ self.assertEqual(self._score("section_present", "Key Risks", text)[0], 1.0, text)
+
+ def test_tool_called_via_marker(self) -> None:
+ self.assertEqual(self._score("tool_called", "search", "TOOL_CALL: search")[0], 1.0)
+ self.assertEqual(self._score("tool_called", "search", "nope", ["search"])[0], 1.0)
+
+ def test_unknown_op_does_not_block(self) -> None:
+ self.assertEqual(self._score("no_such_op", "x", "anything")[0], 1.0)
+
+
+class TestSoftAndHardScoring(unittest.TestCase):
+ def test_soft_is_fraction_and_hard_is_all_or_nothing(self) -> None:
+ judge = {"kind": "rule", "checks": [
+ {"op": "contains", "arg": "alpha"},
+ {"op": "contains", "arg": "beta"},
+ {"op": "contains", "arg": "gamma"},
+ ]}
+ hard, soft, why = score_rule_judge(judge, "alpha and beta only")
+ self.assertEqual(hard, 0.0)
+ self.assertAlmostEqual(soft, 2 / 3)
+ self.assertIn("gamma", why)
+
+ def test_empty_checks_score_zero(self) -> None:
+ self.assertEqual(score_rule_judge({"kind": "rule", "checks": []}, "x")[:2], (0.0, 0.0))
+
+
+class TestMalformedRegexIsDistinguishable(unittest.TestCase):
+ """A pattern Python cannot parse must not read like a plain miss."""
+
+ BAD = r"foo(" # unclosed group is invalid on every supported Python version
+
+ def test_bad_pattern_still_fails_closed(self) -> None:
+ # the response does contain both alternatives; the pattern is the problem
+ hard, soft, _why = score_rule_judge(
+ {"kind": "rule", "checks": [{"op": "regex", "arg": self.BAD}]}, "foo bar")
+ self.assertEqual(hard, 0.0)
+ self.assertEqual(soft, 0.0)
+
+ def test_rationale_names_the_pattern_as_invalid(self) -> None:
+ _hard, _soft, why = score_rule_judge(
+ {"kind": "rule", "checks": [{"op": "regex", "arg": self.BAD}]}, "foo bar")
+ self.assertIn("invalid regex", why)
+
+ def test_a_genuine_miss_is_not_labelled_invalid(self) -> None:
+ _hard, _soft, why = score_rule_judge(
+ {"kind": "rule", "checks": [{"op": "regex", "arg": r"zzz"}]}, "foo bar")
+ self.assertNotIn("invalid regex", why)
+
+
+class TestValidateChecks(unittest.TestCase):
+ def test_sound_checks_produce_nothing(self) -> None:
+ errors, warnings = validate_checks({"checks": [
+ {"op": "regex", "arg": r"^\s*SKILL:"},
+ {"op": "min_chars", "arg": 10},
+ {"op": "section_present", "arg": "Risks"},
+ ]})
+ self.assertEqual(errors, [])
+ self.assertEqual(warnings, [])
+
+ def test_uncompilable_regex_is_an_error(self) -> None:
+ errors, _warnings = validate_checks({"checks": [{"op": "regex", "arg": r"foo("}]})
+ self.assertEqual(len(errors), 1)
+ self.assertIn("does not compile", errors[0])
+
+ def test_non_integer_char_bound_is_an_error(self) -> None:
+ errors, _warnings = validate_checks({"checks": [{"op": "max_chars", "arg": "many"}]})
+ self.assertEqual(len(errors), 1)
+ self.assertIn("integer", errors[0])
+
+ def test_negative_char_bounds_are_errors(self) -> None:
+ for op in ("max_chars", "min_chars"):
+ errors, _warnings = validate_checks(
+ {"checks": [{"op": op, "arg": -1}]}
+ )
+ self.assertEqual(len(errors), 1)
+ self.assertIn("negative", errors[0])
+
+ def test_non_integral_or_non_finite_char_bounds_are_errors(self) -> None:
+ for arg in (1.9, float("inf"), float("-inf"), float("nan")):
+ errors, _warnings = validate_checks(
+ {"checks": [{"op": "max_chars", "arg": arg}]}
+ )
+ self.assertEqual(len(errors), 1, repr(arg))
+ self.assertIn("integer", errors[0])
+
+ def test_integral_float_and_integer_string_bounds_are_valid(self) -> None:
+ for arg in (2.0, " 2 ", "+2"):
+ errors, _warnings = validate_checks(
+ {"checks": [{"op": "max_chars", "arg": arg}]}
+ )
+ self.assertEqual(errors, [], repr(arg))
+
+ def test_zero_min_chars_is_flagged_as_toothless(self) -> None:
+ errors, warnings = validate_checks(
+ {"checks": [{"op": "min_chars", "arg": 0}]}
+ )
+ self.assertEqual(errors, [])
+ self.assertEqual(len(warnings), 1)
+ self.assertIn("always passes", warnings[0])
+
+ def test_empty_string_operator_arguments_are_errors(self) -> None:
+ for op in ("regex", "section_present", "contains", "tool_called"):
+ errors, _warnings = validate_checks(
+ {"checks": [{"op": op, "arg": " "}]}
+ )
+ self.assertEqual(len(errors), 1, op)
+ self.assertIn("non-empty", errors[0])
+
+ def test_unknown_op_is_only_a_warning(self) -> None:
+ errors, warnings = validate_checks({"checks": [{"op": "vibes", "arg": 1}]})
+ self.assertEqual(errors, [])
+ self.assertEqual(len(warnings), 1)
+ self.assertIn("always passes", warnings[0])
+
+ def test_non_string_op_is_a_structured_error(self) -> None:
+ for op in ([], {}, 1, None):
+ errors, warnings = validate_checks(
+ {"checks": [{"op": op, "arg": "value"}]}
+ )
+ self.assertEqual(len(errors), 1, repr(op))
+ self.assertIn("must be a string", errors[0])
+ self.assertEqual(warnings, [])
+
+ def test_non_object_check_is_an_error(self) -> None:
+ errors, _warnings = validate_checks({"checks": ["not-an-object"]})
+ self.assertEqual(len(errors), 1)
+
+ def test_malformed_judge_is_reported_not_raised(self) -> None:
+ # a tasks file may carry any JSON here; validation must stay structured
+ for bad in (["not", "a", "dict"], [], "a string", "", 42, 0):
+ errors, _warnings = validate_checks(bad)
+ self.assertEqual(len(errors), 1, bad)
+ self.assertIn("must be an object", errors[0])
+
+ def test_non_list_checks_is_reported_not_raised(self) -> None:
+ errors, _warnings = validate_checks({"checks": "regex"})
+ self.assertEqual(len(errors), 1)
+ self.assertIn("must be an array", errors[0])
+
+ def test_empty_or_missing_judge_is_sound(self) -> None:
+ for empty in ({}, None, {"checks": []}):
+ self.assertEqual(validate_checks(empty), ([], []), empty)
+
+ def test_every_known_op_is_accepted(self) -> None:
+ for op in KNOWN_OPS:
+ arg = 1 if op.endswith("_chars") else "x"
+ errors, warnings = validate_checks({"checks": [{"op": op, "arg": arg}]})
+ self.assertEqual((errors, warnings), ([], []), op)
+
+
+@pytest.mark.parametrize("bad_judge", [[], "", 0])
+def test_tasks_file_rejects_falsy_non_object_judges(
+ tmp_path, bad_judge
+) -> None:
+ path = tmp_path / "tasks.json"
+ path.write_text(
+ json.dumps(
+ {
+ "tasks": [
+ {
+ "id": "malformed-rule",
+ "project": "test",
+ "intent": "test",
+ "reference_kind": "rule",
+ "judge": bad_judge,
+ }
+ ]
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="judge must be an object"):
+ load_tasks_file(str(path))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_model_change_warning.py b/tests/test_model_change_warning.py
new file mode 100644
index 0000000..5989579
--- /dev/null
+++ b/tests/test_model_change_warning.py
@@ -0,0 +1,237 @@
+"""Tests for the F16 model-change warning and F08 CLI credential warning."""
+from __future__ import annotations
+
+import argparse
+import importlib
+import warnings
+
+from skillopt_sleep.cycle import _check_model_change, _make_model_key
+from skillopt_sleep.config import load_config
+from skillopt_sleep.state import SleepState
+
+
+def _cfg():
+ cfg = load_config()
+ return cfg
+
+
+def test_last_model_key_roundtrips(tmp_path) -> None:
+ path = str(tmp_path / "state.json")
+ state = SleepState.load(path)
+ assert state.last_model_key == ""
+ state.set_last_model_key("anthropic::claude")
+ assert state.last_model_key_format == 2
+ state.save()
+ assert SleepState.load(path).last_model_key == "anthropic::claude"
+
+
+def test_warns_when_model_changed(tmp_path, capsys) -> None:
+ cfg = _cfg()
+ state = SleepState.load(str(tmp_path / "state.json"))
+ state.set_last_model_key("some-other::model")
+ _check_model_change(cfg, state)
+ err = capsys.readouterr().err
+ assert "model changed since last night" in err
+
+
+def test_no_warning_on_first_night(tmp_path, capsys) -> None:
+ cfg = _cfg()
+ state = SleepState.load(str(tmp_path / "state.json")) # last_model_key == ""
+ _check_model_change(cfg, state)
+ assert "model changed" not in capsys.readouterr().err
+
+
+def test_no_warning_when_model_same(tmp_path, capsys) -> None:
+ cfg = _cfg()
+ state = SleepState.load(str(tmp_path / "state.json"))
+ state.set_last_model_key(_make_model_key(cfg))
+ _check_model_change(cfg, state)
+ assert "model changed" not in capsys.readouterr().err
+
+
+def test_model_key_tracks_effective_optimizer_and_target_roles() -> None:
+ cfg = load_config(
+ backend="mock",
+ model="shared",
+ optimizer_backend="claude",
+ optimizer_model="opus",
+ target_backend="codex",
+ target_model="gpt",
+ )
+ assert _make_model_key(cfg) == (
+ "optimizer=claude::opus;target=codex::gpt"
+ )
+
+ inherited = load_config(
+ backend="mock",
+ model="shared",
+ optimizer_backend="claude",
+ )
+ assert _make_model_key(inherited) == (
+ "optimizer=claude::shared;target=mock::"
+ )
+
+
+def test_model_key_normalizes_aliases_and_tracks_environment_defaults(
+ monkeypatch,
+) -> None:
+ monkeypatch.delenv("SKILLOPT_SLEEP_CLAUDE_MODEL", raising=False)
+ claude = _make_model_key(load_config(backend="claude", model=""))
+ anthropic = _make_model_key(load_config(backend="anthropic", model=""))
+ assert claude == anthropic == "claude::sonnet"
+
+ monkeypatch.setenv("SKILLOPT_SLEEP_CLAUDE_MODEL", "opus")
+ assert _make_model_key(load_config(backend="claude", model="")) == "claude::opus"
+
+
+def test_model_key_resolution_failure_uses_safe_config_fallback(monkeypatch) -> None:
+ cycle_module = importlib.import_module("skillopt_sleep.cycle")
+
+ def fail_to_build(**_kwargs):
+ raise RuntimeError("diagnostic construction failed")
+
+ monkeypatch.setattr(cycle_module, "build_backend", fail_to_build)
+ cfg = load_config(
+ backend="claude",
+ model="sonnet",
+ optimizer_backend="codex",
+ optimizer_model="gpt",
+ )
+ assert cycle_module._make_model_key(cfg) == (
+ "configured:optimizer=codex::gpt;target=claude::sonnet"
+ )
+
+
+def test_legacy_unresolved_model_key_migrates_without_false_warning(
+ tmp_path, capsys
+) -> None:
+ state = SleepState.load(str(tmp_path / "state.json"))
+ state.data["last_model_key"] = "claude::"
+ state.data["last_model_key_format"] = 1
+
+ _check_model_change(load_config(backend="claude", model=""), state)
+
+ assert "model changed" not in capsys.readouterr().err
+
+
+def test_warns_when_one_split_backend_role_changes(tmp_path, capsys) -> None:
+ previous = load_config(
+ optimizer_backend="claude",
+ optimizer_model="opus",
+ target_backend="codex",
+ target_model="gpt",
+ )
+ current = load_config(
+ optimizer_backend="claude",
+ optimizer_model="opus",
+ target_backend="cursor",
+ target_model="composer",
+ )
+ state = SleepState.load(str(tmp_path / "state.json"))
+ state.set_last_model_key(_make_model_key(previous))
+
+ _check_model_change(current, state)
+
+ assert "model changed since last night" in capsys.readouterr().err
+
+
+def test_cli_api_key_emits_deprecation_warning() -> None:
+ from scripts.train import load_config as train_load_config
+
+ args = argparse.Namespace(
+ config="configs/_base_/default.yaml",
+ cfg_options=None,
+ azure_openai_api_key="sk-secret-value",
+ )
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ train_load_config(args)
+ except Exception:
+ pass # config loading may fail; we only assert the warning fired
+ assert any(
+ issubclass(w.category, DeprecationWarning)
+ and "azure_openai_api_key" in str(w.message)
+ for w in caught
+ )
+
+
+def test_cli_key_warnings_name_the_correct_environment_variable() -> None:
+ from scripts.train import load_config as train_load_config
+
+ cases = {
+ "optimizer_azure_openai_api_key": "OPTIMIZER_AZURE_OPENAI_API_KEY",
+ "target_azure_openai_api_key": "TARGET_AZURE_OPENAI_API_KEY",
+ "qwen_chat_api_key": "QWEN_CHAT_API_KEY",
+ "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY",
+ "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY",
+ "minimax_api_key": "MINIMAX_API_KEY",
+ }
+ for flag, environment_variable in cases.items():
+ args = argparse.Namespace(
+ config="configs/_base_/default.yaml",
+ cfg_options=None,
+ **{flag: "secret-value"},
+ )
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ train_load_config(args)
+ except Exception:
+ pass
+ messages = [
+ str(w.message)
+ for w in caught
+ if issubclass(w.category, DeprecationWarning)
+ ]
+ assert any(environment_variable in message for message in messages), flag
+
+
+def test_cfg_options_api_key_emits_safe_deprecation_warning() -> None:
+ from scripts.train import load_config as train_load_config
+
+ args = argparse.Namespace(
+ config="configs/_base_/default.yaml",
+ cfg_options=["model.azure_openai_api_key=do-not-print-this-secret"],
+ )
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ train_load_config(args)
+ except Exception:
+ pass
+ messages = [
+ str(w.message)
+ for w in caught
+ if issubclass(w.category, DeprecationWarning)
+ ]
+ assert any("AZURE_OPENAI_API_KEY" in message for message in messages)
+ assert all("do-not-print-this-secret" not in message for message in messages)
+
+
+def test_cfg_options_warning_uses_leaf_name_and_catches_future_secrets() -> None:
+ from scripts.train import load_config as train_load_config
+
+ args = argparse.Namespace(
+ config="configs/_base_/default.yaml",
+ cfg_options=[
+ "custom.optimizer_qwen_chat_api_key=role-secret",
+ "future.backend.access_token=future-secret",
+ ],
+ )
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ train_load_config(args)
+ except Exception:
+ pass
+ messages = [
+ str(w.message)
+ for w in caught
+ if issubclass(w.category, DeprecationWarning)
+ ]
+
+ assert any("OPTIMIZER_QWEN_CHAT_API_KEY" in message for message in messages)
+ assert any("future.backend.access_token" in message for message in messages)
+ assert all("role-secret" not in message for message in messages)
+ assert all("future-secret" not in message for message in messages)
diff --git a/tests/test_plugin_sync.py b/tests/test_plugin_sync.py
index f7850e2..6df09e1 100644
--- a/tests/test_plugin_sync.py
+++ b/tests/test_plugin_sync.py
@@ -2,6 +2,7 @@
Run: python3 -m pytest tests/test_plugin_sync.py -v
"""
+import json
import os
import unittest
@@ -10,6 +11,7 @@ REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
PLUGIN_SKILL_MDS = {
"claude-code": os.path.join(REPO, "plugins/claude-code/skills/skillopt-sleep/SKILL.md"),
"codex": os.path.join(REPO, "plugins/codex/skills/skillopt-sleep/SKILL.md"),
+ "cursor": os.path.join(REPO, "plugins/cursor/skills/skillopt-sleep/SKILL.md"),
"openclaw": os.path.join(REPO, "plugins/openclaw/SKILL.md"),
}
@@ -17,6 +19,14 @@ MCP_SERVER = os.path.join(REPO, "plugins/copilot/mcp_server.py")
COPILOT_INSTRUCTIONS = os.path.join(REPO, "plugins/copilot/copilot-instructions.snippet.md")
CANONICAL_BACKENDS = {"mock", "claude", "codex", "copilot"}
+CURSOR_MANIFEST = os.path.join(REPO, "plugins/cursor/.cursor-plugin/plugin.json")
+CURSOR_MARKETPLACE = os.path.join(REPO, ".cursor-plugin/marketplace.json")
+CURSOR_COMMAND = os.path.join(REPO, "plugins/cursor/commands/skillopt-sleep.md")
+CURSOR_README = os.path.join(REPO, "plugins/cursor/README.md")
+CURSOR_INSTALL_SH = os.path.join(REPO, "plugins/cursor/install.sh")
+CURSOR_INSTALL_PS1 = os.path.join(REPO, "plugins/cursor/install.ps1")
+CURSOR_LICENSE = os.path.join(REPO, "plugins/cursor/LICENSE")
+OPENCLAW_RUNNER = os.path.join(REPO, "plugins/openclaw/run_sleep.py")
def _read(path):
@@ -27,6 +37,82 @@ def _read(path):
class TestPluginParity(unittest.TestCase):
+ def test_cursor_plugin_manifest_and_marketplace_registration(self):
+ with open(CURSOR_MANIFEST, encoding="utf-8") as f:
+ manifest = json.load(f)
+ with open(CURSOR_MARKETPLACE, encoding="utf-8") as f:
+ marketplace = json.load(f)
+
+ self.assertEqual(manifest["name"], "skillopt-sleep")
+ self.assertEqual(manifest["skills"], "./skills/")
+ self.assertEqual(manifest["commands"], "./commands/")
+ self.assertNotIn("hooks", manifest)
+ self.assertNotIn("mcpServers", manifest)
+ allowed_manifest_keys = {
+ "name", "displayName", "description", "version", "author",
+ "publisher", "homepage", "repository", "license", "logo",
+ "keywords", "category", "tags", "commands", "agents", "skills",
+ "rules", "hooks", "mcpServers",
+ }
+ self.assertEqual(set(manifest) - allowed_manifest_keys, set())
+ registered = next(
+ (plugin for plugin in marketplace["plugins"] if plugin.get("name") == "skillopt-sleep"),
+ None,
+ )
+ self.assertIsNotNone(registered)
+ self.assertEqual(registered["source"], "plugins/cursor")
+ self.assertEqual(registered["name"], manifest["name"])
+ self.assertTrue(os.path.isdir(os.path.join(REPO, registered["source"])))
+
+ def test_cursor_skill_has_frontmatter_target_and_cursor_guidance(self):
+ text = _read(PLUGIN_SKILL_MDS["cursor"])
+ self.assertTrue(text.startswith("---\n"))
+ self.assertIn("name: skillopt-sleep", text)
+ self.assertIn(".cursor/skills/skillopt-sleep-learned/SKILL.md", text)
+ self.assertIn("--source cursor", text)
+ self.assertIn("--backend cursor", text)
+
+ def test_cursor_command_is_thin_and_preserves_safety_defaults(self):
+ text = _read(CURSOR_COMMAND)
+ self.assertIn("$ARGUMENTS", text)
+ self.assertIn("use `status`", text)
+ self.assertIn("--source cursor", text)
+ self.assertIn("--scope invoked", text)
+ self.assertIn(".cursor/skills/skillopt-sleep-learned/SKILL.md", text)
+ self.assertIn("`mock` backend", text)
+ self.assertNotIn("--auto-adopt", text)
+
+ def test_cursor_installers_package_command_skill_readme_and_license(self):
+ for installer in (CURSOR_INSTALL_SH, CURSOR_INSTALL_PS1):
+ text = _read(installer)
+ for filename in (
+ "plugin.json",
+ "commands/skillopt-sleep.md" if installer.endswith(".sh") else "commands\\skillopt-sleep.md",
+ "skills/skillopt-sleep/SKILL.md" if installer.endswith(".sh") else "skills\\skillopt-sleep\\SKILL.md",
+ "README.md",
+ "LICENSE",
+ ):
+ self.assertIn(filename, text, f"{installer} does not package {filename}")
+
+ self.assertEqual(_read(CURSOR_LICENSE), _read(os.path.join(REPO, "LICENSE")))
+
+ def test_cursor_docs_keep_scheduling_explicit_and_target_relative(self):
+ for path in (CURSOR_README, PLUGIN_SKILL_MDS["cursor"]):
+ text = _read(path)
+ self.assertIn('"target_skill_path": ".cursor/skills/', text)
+ self.assertIn("no session-end hook", text.lower())
+ self.assertIn("`tool_called`", text)
+ self.assertIn("temporarily disabled", text.lower())
+ self.assertIn("before agent mode", text.lower())
+
+ def test_openclaw_wrapper_matches_shared_backend_signature(self):
+ text = _read(OPENCLAW_RUNNER)
+ self.assertIn('cursor_path=""', text)
+ self.assertIn('project_dir=""', text)
+ self.assertIn("cursor_path=cursor_path", text)
+ self.assertIn("project_dir=project_dir", text)
+ self.assertNotIn("**kwargs", text)
+
def test_all_skill_mds_mention_all_backends(self):
for name, path in PLUGIN_SKILL_MDS.items():
text = _read(path)
diff --git a/tests/test_react_agent_no_shell.py b/tests/test_react_agent_no_shell.py
new file mode 100644
index 0000000..c7d1f2d
--- /dev/null
+++ b/tests/test_react_agent_no_shell.py
@@ -0,0 +1,49 @@
+"""Tests for shell-injection hardening in the ReAct agent's bash tool.
+
+``_run_bash`` previously ran commands with ``shell=True``, allowing arbitrary
+shell metacharacter injection. It now uses ``shlex.split`` + ``shell=False``
+and restricts the executable to a small allow-list (python/python3). These
+tests assert both the allow-list gate and that shell metacharacters are no
+longer interpreted.
+"""
+from __future__ import annotations
+
+from skillopt.envs.spreadsheetbench.react_agent import _run_bash
+
+
+def test_disallowed_command_is_blocked(tmp_path) -> None:
+ out = _run_bash("curl http://example.com/evil", str(tmp_path))
+ assert "blocked" in out.lower()
+
+
+def test_allowed_python_runs_without_path_lookup(tmp_path, monkeypatch) -> None:
+ # Accepted aliases are mapped to the running interpreter, so an absent PATH
+ # must not make the benchmark depend on a system-level Python command.
+ monkeypatch.setenv("PATH", "")
+ out = _run_bash('python -c "print(42)"', str(tmp_path))
+ assert "42" in out
+
+
+def test_similarly_named_executable_is_blocked(tmp_path) -> None:
+ out = _run_bash('python.evil -c "print(42)"', str(tmp_path))
+ assert "blocked" in out.lower()
+
+
+def test_python_child_does_not_inherit_parent_secrets(
+ tmp_path, monkeypatch
+) -> None:
+ monkeypatch.setenv("SUPER_SECRET_TOKEN", "must-not-leak")
+ out = _run_bash(
+ 'python -c "import os; print(os.environ.get(\'SUPER_SECRET_TOKEN\', \'ABSENT\'))"',
+ str(tmp_path),
+ )
+ assert out == "ABSENT"
+
+
+def test_shell_metacharacters_not_interpreted(tmp_path) -> None:
+ # With shell=False the ';' and following tokens become arguments to python,
+ # not a second shell command, so the marker file must NOT be created.
+ marker = tmp_path / "pwned.txt"
+ cmd = "python -c \"print(1)\" ; python -c \"open('pwned.txt','w')\""
+ _run_bash(cmd, str(tmp_path))
+ assert not marker.exists()
diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py
index a057e22..70491aa 100644
--- a/tests/test_sleep_engine.py
+++ b/tests/test_sleep_engine.py
@@ -136,6 +136,355 @@ class TestHarvest(unittest.TestCase):
self.assertNotIn("raw args should not copy", joined)
self.assertNotIn("raw output should not copy", joined)
+ def test_digest_cursor_transcript_redacts_and_keeps_only_message_text_and_tool_names(self):
+ from skillopt_sleep.harvest_cursor import digest_cursor_transcript
+
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "cursor-session.jsonl")
+ self._write_jsonl(path, [
+ {
+ "role": "user",
+ "message": {
+ "content": [{
+ "type": "text",
+ "text": (
+ "never-copy-attachment-metadata\n"
+ "\n"
+ "Deploy with sk-1234567890abcdef and token=local-secret\n"
+ ""
+ ),
+ }],
+ },
+ },
+ {
+ "role": "assistant",
+ "message": {
+ "content": [{
+ "type": "tool_use",
+ "name": "shell.execute",
+ "input": {"token": "never-copy-tool-arguments"},
+ }],
+ },
+ },
+ {
+ "role": "assistant",
+ "message": {
+ "content": [
+ {"type": "text", "text": "Deployment finished."},
+ {
+ "type": "tool_use",
+ "name": "read_file",
+ "input": {"path": "never-copy-tool-arguments"},
+ },
+ ],
+ },
+ },
+ {"type": "tool_result", "output": "never-copy-tool-output"},
+ {"type": "turn_ended", "status": "error"},
+ ])
+ with open(path, "a", encoding="utf-8") as f:
+ f.write("null\n")
+ f.write("[]\n")
+ f.write('"non-object"\n')
+ f.write("{malformed jsonl record\\n")
+
+ digest = digest_cursor_transcript(path, project="/repo/Cursor Project")
+
+ self.assertIsNotNone(digest)
+ joined = "\n".join(digest.user_prompts + digest.assistant_finals)
+ self.assertEqual(digest.project, "/repo/Cursor Project")
+ self.assertEqual(len(digest.user_prompts), 1)
+ self.assertIn("[REDACTED_OPENAI_KEY]", joined)
+ self.assertIn("token=[REDACTED]", joined)
+ self.assertEqual(digest.tools_used, ["shell.execute", "read_file"])
+ self.assertIn("neg:cursor_turn_error", digest.feedback_signals)
+ self.assertNotIn("never-copy-tool-arguments", joined)
+ self.assertNotIn("never-copy-tool-output", joined)
+ self.assertNotIn("never-copy-attachment-metadata", joined)
+
+ def test_harvest_cursor_scopes_orders_filters_mtime_and_skips_replays(self):
+ from skillopt_sleep.__main__ import _cfg_from_args
+ from skillopt_sleep.harvest_cursor import (
+ CURSOR_REPLAY_SENTINEL,
+ cursor_project_slug,
+ harvest_cursor,
+ )
+ from skillopt_sleep.harvest_sources import harvest_for_config
+
+ def write_cursor_session(cursor_home, project, session_id, prompt, mtime, extra_prompt=""):
+ project_dir = os.path.join(
+ cursor_home,
+ "projects",
+ cursor_project_slug(project),
+ )
+ path = os.path.join(
+ project_dir,
+ "agent-transcripts",
+ session_id,
+ f"{session_id}.jsonl",
+ )
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f:
+ json.dump({"workspacePath": project}, f)
+ records = [
+ {
+ "role": "user",
+ "message": {
+ "content": [{
+ "type": "text",
+ "text": f"\n{prompt}\n",
+ }],
+ },
+ },
+ {
+ "role": "assistant",
+ "message": {"content": [{"type": "text", "text": "done"}]},
+ },
+ ]
+ if extra_prompt:
+ records.extend([
+ {
+ "role": "user",
+ "message": {
+ "content": [{
+ "type": "text",
+ "text": f"\n{extra_prompt}\n",
+ }],
+ },
+ },
+ {
+ "role": "assistant",
+ "message": {"content": [{"type": "text", "text": "done again"}]},
+ },
+ ])
+ self._write_jsonl(path, records)
+ os.utime(path, (mtime, mtime))
+ return path
+
+ with tempfile.TemporaryDirectory() as tmp:
+ cursor_home = os.path.join(tmp, ".cursor")
+ project = os.path.join(tmp, "project with spaces")
+ other_project = os.path.join(tmp, "other project")
+ old_time = 1_700_000_000
+ new_time = old_time + 3_600
+ main_path = write_cursor_session(
+ cursor_home,
+ project,
+ "older",
+ "fix the first issue",
+ old_time,
+ )
+ subagent_path = os.path.join(os.path.dirname(main_path), "subagents", "worker.jsonl")
+ os.makedirs(os.path.dirname(subagent_path), exist_ok=True)
+ self._write_jsonl(subagent_path, [
+ {"role": "user", "message": {"content": "machine-generated subagent task"}},
+ {"role": "assistant", "message": {"content": "subagent result"}},
+ ])
+ write_cursor_session(cursor_home, other_project, "newer", "fix the second issue", new_time)
+ write_cursor_session(
+ cursor_home,
+ other_project,
+ "generated-replay",
+ CURSOR_REPLAY_SENTINEL + "\n## CURRENT SKILL",
+ new_time + 1,
+ extra_prompt="continue the internal replay",
+ )
+
+ invoked = harvest_cursor(
+ os.path.join(cursor_home, "projects"),
+ scope="invoked",
+ invoked_project=os.path.join(project, "src", "package"),
+ )
+ all_digests = harvest_cursor(
+ os.path.join(cursor_home, "projects"),
+ scope="all",
+ since_iso="2023-11-14T23:00:00Z",
+ limit=1,
+ )
+
+ Args = type("Args", (), {
+ "project": project,
+ "scope": "",
+ "backend": "cursor",
+ "model": "",
+ "codex_path": "",
+ "cursor_path": "",
+ "claude_home": "",
+ "codex_home": "",
+ "cursor_home": cursor_home,
+ "source": "cursor",
+ "lookback_hours": 0,
+ "edit_budget": 0,
+ "max_sessions": 0,
+ "max_tasks": 0,
+ "target_skill_path": "",
+ "preferences": "",
+ "progress": False,
+ "auto_adopt": False,
+ })
+ cfg = _cfg_from_args(Args())
+ configured = harvest_for_config(cfg)
+
+ self.assertEqual([d.session_id for d in invoked], ["older"])
+ self.assertEqual([d.session_id for d in all_digests], ["newer"])
+ self.assertEqual(invoked[0].project, project)
+ self.assertEqual(all_digests[0].project, other_project)
+ self.assertEqual([d.session_id for d in configured], ["older"])
+ self.assertEqual(cfg.get("transcript_source"), "cursor")
+ self.assertEqual(cfg.get("backend"), "cursor")
+
+ def test_harvest_cursor_prefers_longest_workspace_and_falls_back_to_slug(self):
+ from skillopt_sleep.harvest_cursor import cursor_project_slug, harvest_cursor
+
+ def write_session(projects_dir, storage_name, workspace, session_id):
+ project_dir = os.path.join(projects_dir, storage_name)
+ path = os.path.join(project_dir, "agent-transcripts", session_id, f"{session_id}.jsonl")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ self._write_jsonl(path, [
+ {"role": "user", "message": {"content": "please fix this project"}},
+ {"role": "assistant", "message": {"content": "fixed"}},
+ ])
+ if workspace is not None:
+ with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f:
+ json.dump(workspace, f)
+ return path
+
+ with tempfile.TemporaryDirectory() as tmp:
+ projects_dir = os.path.join(tmp, ".cursor", "projects")
+ parent = os.path.join(tmp, "repo")
+ nested = os.path.join(parent, "packages", "app")
+ write_session(projects_dir, "parent-store", {"workspacePath": parent}, "parent")
+ write_session(projects_dir, "nested-store", {"workspacePath": nested}, "nested")
+ fallback = os.path.join(tmp, "fallback")
+ write_session(
+ projects_dir,
+ cursor_project_slug(fallback),
+ ["invalid metadata shape"],
+ "fallback",
+ )
+ metadata_free = os.path.join(tmp, "metadata-free")
+ write_session(
+ projects_dir,
+ cursor_project_slug(metadata_free),
+ None,
+ "metadata-free",
+ )
+ mixed_parent = os.path.join(tmp, "mixed-parent")
+ mixed_nested = os.path.join(mixed_parent, "nested")
+ write_session(projects_dir, "mixed-parent-store", {"workspacePath": mixed_parent}, "mixed-parent")
+ write_session(
+ projects_dir,
+ cursor_project_slug(mixed_nested),
+ None,
+ "mixed-nested",
+ )
+
+ nested_digests = harvest_cursor(
+ projects_dir,
+ scope="invoked",
+ invoked_project=os.path.join(nested, "src"),
+ )
+ fallback_digests = harvest_cursor(
+ projects_dir,
+ scope="invoked",
+ invoked_project=fallback,
+ )
+ metadata_free_digests = harvest_cursor(
+ projects_dir,
+ scope="invoked",
+ invoked_project=os.path.join(metadata_free, "packages", "app"),
+ )
+ mixed_digests = harvest_cursor(
+ projects_dir,
+ scope="invoked",
+ invoked_project=mixed_nested,
+ )
+
+ self.assertEqual([digest.session_id for digest in nested_digests], ["nested"])
+ self.assertEqual(nested_digests[0].project, nested)
+ self.assertEqual([digest.session_id for digest in fallback_digests], ["fallback"])
+ self.assertEqual(fallback_digests[0].project, fallback)
+ self.assertEqual(
+ [digest.session_id for digest in metadata_free_digests],
+ ["metadata-free"],
+ )
+ self.assertEqual(metadata_free_digests[0].project, metadata_free)
+ self.assertEqual([digest.session_id for digest in mixed_digests], ["mixed-nested"])
+ self.assertEqual(mixed_digests[0].project, mixed_nested)
+
+ def test_harvest_cursor_uses_numeric_mtime_for_aware_and_local_cutoffs(self):
+ from datetime import datetime, timedelta, timezone
+
+ from skillopt_sleep.harvest_cursor import cursor_project_slug, harvest_cursor
+
+ with tempfile.TemporaryDirectory() as tmp:
+ project = os.path.join(tmp, "project")
+ project_dir = os.path.join(tmp, ".cursor", "projects", cursor_project_slug(project))
+ os.makedirs(project_dir)
+ with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f:
+ json.dump({"workspacePath": project}, f)
+
+ cutoff = 1_700_000_000
+ for session_id, modified in (("before", cutoff - 1), ("equal", cutoff), ("after", cutoff + 1)):
+ path = os.path.join(
+ project_dir,
+ "agent-transcripts",
+ session_id,
+ f"{session_id}.jsonl",
+ )
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ self._write_jsonl(path, [
+ {"role": "user", "message": {"content": f"task {session_id}"}},
+ {"role": "assistant", "message": {"content": "done"}},
+ ])
+ os.utime(path, (modified, modified))
+
+ aware = datetime.fromtimestamp(cutoff, timezone(timedelta(hours=5))).isoformat()
+ local = datetime.fromtimestamp(cutoff).replace(microsecond=0).isoformat()
+ aware_result = harvest_cursor(projects_dir=os.path.dirname(project_dir), since_iso=aware)
+ local_result = harvest_cursor(projects_dir=os.path.dirname(project_dir), since_iso=local)
+
+ self.assertEqual([digest.session_id for digest in aware_result], ["after"])
+ self.assertEqual([digest.session_id for digest in local_result], ["after"])
+
+ def test_harvest_cursor_filters_only_exact_internal_replay_sentinel(self):
+ from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL, cursor_project_slug, harvest_cursor
+
+ with tempfile.TemporaryDirectory() as tmp:
+ project = os.path.join(tmp, "project")
+ project_dir = os.path.join(tmp, ".cursor", "projects", cursor_project_slug(project))
+ for session_id, prompt in (
+ ("internal", CURSOR_REPLAY_SENTINEL + "\nrun replay"),
+ ("real", f"Please explain what {CURSOR_REPLAY_SENTINEL} means"),
+ ("grader", "You are a strict grader helping me review this response"),
+ ("skill", "Please explain the ## CURRENT SKILL section"),
+ ):
+ path = os.path.join(project_dir, "agent-transcripts", session_id, f"{session_id}.jsonl")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ self._write_jsonl(path, [
+ {"role": "user", "message": {"content": prompt}},
+ {"role": "assistant", "message": {"content": "answer"}},
+ ])
+
+ digests = harvest_cursor(os.path.join(tmp, ".cursor", "projects"), scope="all")
+
+ self.assertEqual(
+ sorted(digest.session_id for digest in digests),
+ ["grader", "real", "skill"],
+ )
+
+ def test_auto_source_keeps_existing_codex_then_claude_precedence(self):
+ from skillopt_sleep.harvest_sources import harvest_for_config
+
+ cfg = load_config(transcript_source="auto", invoked_project="/repo/project")
+ expected = [SessionDigest(session_id="claude-session", project="/repo/project")]
+ with mock.patch("skillopt_sleep.harvest_sources.harvest_codex", return_value=[]), \
+ mock.patch("skillopt_sleep.harvest_sources.harvest", return_value=expected), \
+ mock.patch("skillopt_sleep.harvest_sources.harvest_cursor") as cursor_harvest:
+ self.assertEqual(harvest_for_config(cfg), expected)
+
+ cursor_harvest.assert_not_called()
+
def test_harvest_codex_filters_project_and_cli_source(self):
from skillopt_sleep.__main__ import _cfg_from_args
from skillopt_sleep.harvest_sources import harvest_for_config
@@ -478,6 +827,18 @@ Resolve local Git conflicts.
"resolve a local Git conflict",
})
+ def test_cursor_miner_failure_is_not_swallowed(self):
+ from skillopt_sleep.backend import CursorBackendError
+
+ def failed_miner(_digests):
+ raise CursorBackendError("Cursor Agent authentication failed")
+
+ with self.assertRaises(CursorBackendError):
+ mine(
+ [self._digest(["configure an MCP server"], ["neg:failed"])],
+ llm_miner=failed_miner,
+ )
+
class TestConsolidateGate(unittest.TestCase):
def test_accepts_helpful_rejects_harmful(self):
@@ -1113,6 +1474,412 @@ class TestCopilotBackend(unittest.TestCase):
shutil.rmtree(stub_dir, ignore_errors=True)
+class TestCursorBackend(unittest.TestCase):
+ """Pure-logic tests for CursorCliBackend without a Cursor login."""
+
+ def test_alias_and_environment_resolution(self):
+ from skillopt_sleep.backend import CursorCliBackend, get_backend, resolve_cursor_path
+
+ for name in ("cursor", "cursor_agent", "cursor_cli"):
+ self.assertIsInstance(get_backend(name), CursorCliBackend, name)
+ with mock.patch.dict(os.environ, {
+ "SKILLOPT_SLEEP_CURSOR_PATH": "/tmp/cursor-agent",
+ "SKILLOPT_SLEEP_CURSOR_MODEL": "cursor-small",
+ }, clear=False):
+ self.assertEqual(resolve_cursor_path(), "/tmp/cursor-agent")
+ self.assertEqual(CursorCliBackend().model, "cursor-small")
+
+ def test_cursor_path_overrides_expand_user_home(self):
+ from skillopt_sleep.__main__ import _cfg_from_args
+ from skillopt_sleep.backend import resolve_cursor_path
+
+ Args = type("Args", (), {
+ "project": "",
+ "scope": "",
+ "backend": "",
+ "model": "",
+ "codex_path": "",
+ "cursor_path": "~/.local/bin/cursor-agent",
+ "claude_home": "",
+ "codex_home": "",
+ "cursor_home": "~/.cursor-custom",
+ "source": "",
+ "lookback_hours": None,
+ "edit_budget": 0,
+ "max_sessions": 0,
+ "max_tasks": 0,
+ "target_skill_path": "",
+ "preferences": "",
+ "progress": False,
+ "auto_adopt": False,
+ })
+
+ cfg = _cfg_from_args(Args())
+ self.assertEqual(
+ cfg.get("cursor_path"),
+ os.path.abspath(os.path.expanduser("~/.local/bin/cursor-agent")),
+ )
+ self.assertEqual(
+ cfg.cursor_projects_dir,
+ os.path.join(os.path.expanduser("~/.cursor-custom"), "projects"),
+ )
+
+ direct_cfg = load_config(
+ cursor_home="~/.cursor-config",
+ cursor_path="~/.cursor-config/bin/cursor-agent",
+ )
+ self.assertEqual(
+ direct_cfg.cursor_projects_dir,
+ os.path.join(os.path.expanduser("~/.cursor-config"), "projects"),
+ )
+ self.assertEqual(
+ resolve_cursor_path(direct_cfg.get("cursor_path")),
+ os.path.expanduser("~/.cursor-config/bin/cursor-agent"),
+ )
+ with mock.patch.dict(
+ os.environ,
+ {"SKILLOPT_SLEEP_CURSOR_PATH": "~/.cursor-env/bin/cursor-agent"},
+ clear=False,
+ ):
+ self.assertEqual(
+ resolve_cursor_path(),
+ os.path.expanduser("~/.cursor-env/bin/cursor-agent"),
+ )
+
+ def test_read_only_call_uses_stdin_ask_mode_and_terminal_result(self):
+ from skillopt_sleep.backend import CursorCliBackend
+ from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL
+
+ calls = []
+ runtime_configs = []
+
+ def fake_run(cmd, **kwargs):
+ calls.append((cmd, kwargs))
+ config_dir = kwargs["env"]["CURSOR_CONFIG_DIR"]
+ data_dir = kwargs["env"]["CURSOR_DATA_DIR"]
+ with open(os.path.join(config_dir, "cli-config.json"), encoding="utf-8") as f:
+ runtime_configs.append(json.load(f))
+ self.assertTrue(os.path.isdir(data_dir))
+
+ class Proc:
+ returncode = 0
+ stdout = (
+ '{"type":"message","result":"intermediate"}\n'
+ '{"type":"result","subtype":"success","is_error":false,'
+ '"result":"final answer"}\n'
+ )
+ stderr = ""
+
+ return Proc()
+
+ backend = CursorCliBackend(cursor_path="cursor-agent-test", model="cursor-model")
+ with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run):
+ self.assertEqual(backend._call("solve this"), "final answer")
+
+ cmd, kwargs = calls[0]
+ self.assertEqual(cmd[0], "cursor-agent-test")
+ self.assertIn("-p", cmd)
+ self.assertEqual(cmd[cmd.index("--output-format") + 1], "json")
+ self.assertEqual(cmd[cmd.index("--mode") + 1], "ask")
+ self.assertIn("--trust", cmd)
+ self.assertEqual(cmd[cmd.index("--workspace") + 1], kwargs["cwd"])
+ self.assertTrue(os.path.basename(kwargs["cwd"]).startswith("skillopt_sleep_cursor_"))
+ self.assertNotIn("--force", cmd)
+ self.assertNotIn("--sandbox", cmd)
+ self.assertEqual(cmd[cmd.index("--model") + 1], "cursor-model")
+ self.assertTrue(kwargs["input"].startswith(CURSOR_REPLAY_SENTINEL + "\n\n"))
+ self.assertTrue(kwargs["input"].endswith("solve this"))
+ self.assertNotEqual(kwargs["env"]["CURSOR_CONFIG_DIR"], os.path.expanduser("~/.cursor"))
+ self.assertEqual(runtime_configs[0]["approvalMode"], "allowlist")
+ self.assertEqual(runtime_configs[0]["permissions"]["allow"], [])
+ self.assertEqual(
+ runtime_configs[0]["permissions"]["deny"],
+ ["Read(**)", "Write(**)", "Mcp(*:*)"],
+ )
+ self.assertEqual(runtime_configs[0]["sandbox"]["mode"], "disabled")
+ self.assertFalse(os.path.exists(os.path.dirname(kwargs["env"]["CURSOR_CONFIG_DIR"])))
+ self.assertEqual(backend.last_call_error, "")
+ self.assertEqual(
+ CursorCliBackend._parse_json_response('{"type":"message","result":"not terminal"}'),
+ "",
+ )
+
+ def test_cursor_environment_keeps_runtime_auth_and_drops_unrelated_secrets(self):
+ from skillopt_sleep.backend import CursorCliBackend
+
+ host_env = {
+ "PATH": "/usr/bin",
+ "LANG": "en_US.UTF-8",
+ "CURSOR_API_KEY": "cursor-auth",
+ "HTTPS_PROXY": "https://proxy.example",
+ "AWS_SECRET_ACCESS_KEY": "aws-secret",
+ "OPENAI_API_KEY": "openai-secret",
+ "ANTHROPIC_API_KEY": "anthropic-secret",
+ "GITHUB_TOKEN": "github-secret",
+ }
+ with tempfile.TemporaryDirectory() as runtime_dir:
+ with mock.patch.dict(os.environ, host_env, clear=True):
+ env = CursorCliBackend._isolated_environment(runtime_dir)
+
+ self.assertEqual(env["PATH"], "/usr/bin")
+ self.assertEqual(env["LANG"], "en_US.UTF-8")
+ self.assertEqual(env["CURSOR_API_KEY"], "cursor-auth")
+ self.assertEqual(env["HTTPS_PROXY"], "https://proxy.example")
+ self.assertNotIn("AWS_SECRET_ACCESS_KEY", env)
+ self.assertNotIn("OPENAI_API_KEY", env)
+ self.assertNotIn("ANTHROPIC_API_KEY", env)
+ self.assertNotIn("GITHUB_TOKEN", env)
+ self.assertTrue(env["CURSOR_CONFIG_DIR"].startswith(runtime_dir))
+ self.assertTrue(env["CURSOR_DATA_DIR"].startswith(runtime_dir))
+
+ def test_nonzero_and_error_results_fail_once_with_redacted_diagnostics(self):
+ from skillopt_sleep.backend import CursorBackendError, CursorCliBackend
+
+ backend = CursorCliBackend(cursor_path="cursor-agent-test", timeout=7)
+
+ class BadProc:
+ returncode = 9
+ stdout = "not-json"
+ stderr = "Authorization: Bearer cursor-secret-value"
+
+ with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=BadProc()) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 1)
+ self.assertIn("exited 9", backend.last_call_error)
+ self.assertIn("[REDACTED]", backend.last_call_error)
+ self.assertNotIn("cursor-secret-value", backend.last_call_error)
+
+ class ErrorProc:
+ returncode = 0
+ stdout = '{"type":"result","is_error":true,"result":"api_key=cursor-secret"}'
+ stderr = ""
+
+ with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=ErrorProc()) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 1)
+ self.assertIn("error result", backend.last_call_error)
+ self.assertIn("[REDACTED]", backend.last_call_error)
+ self.assertNotIn("cursor-secret", backend.last_call_error)
+
+ with mock.patch(
+ "skillopt_sleep.backend.subprocess.run",
+ side_effect=OSError("missing cursor-agent"),
+ ) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 1)
+ self.assertIn("spawn failed", backend.last_call_error)
+
+ def test_read_only_timeout_and_malformed_output_retry_once(self):
+ import subprocess
+
+ from skillopt_sleep.backend import CursorBackendError, CursorCliBackend
+
+ backend = CursorCliBackend(cursor_path="cursor-agent-test", timeout=7)
+
+ class GoodProc:
+ returncode = 0
+ stdout = '{"type":"result","is_error":false,"result":"recovered"}'
+ stderr = ""
+
+ with mock.patch(
+ "skillopt_sleep.backend.subprocess.run",
+ side_effect=[subprocess.TimeoutExpired(["cursor-agent-test"], 7), GoodProc()],
+ ) as run:
+ self.assertEqual(backend._call("solve this"), "recovered")
+ self.assertEqual(run.call_count, 2)
+ self.assertEqual(backend.last_call_error, "")
+
+ class MalformedProc:
+ returncode = 0
+ stdout = "still not json"
+ stderr = ""
+
+ with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=MalformedProc()) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 2)
+ self.assertIn("no usable JSON response", backend.last_call_error)
+
+ class AuthProc:
+ returncode = 0
+ stdout = ""
+ stderr = "Not authenticated. Please log in with token=cursor-secret"
+
+ with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=AuthProc()) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 1)
+ self.assertIn("authentication failed", backend.last_call_error)
+ self.assertNotIn("cursor-secret", backend.last_call_error)
+
+ class ConfigProc:
+ returncode = 0
+ stdout = ""
+ stderr = "Unsupported model: cursor-unknown"
+
+ with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=ConfigProc()) as run:
+ with self.assertRaises(CursorBackendError):
+ backend._call("solve this")
+ self.assertEqual(run.call_count, 1)
+ self.assertIn("configuration failed", backend.last_call_error)
+
+ def test_failed_cursor_call_is_not_cached(self):
+ from skillopt_sleep.backend import CursorBackendError, CursorCliBackend
+
+ class BadProc:
+ returncode = 1
+ stdout = ""
+ stderr = "not authenticated"
+
+ class GoodProc:
+ returncode = 0
+ stdout = '{"type":"result","is_error":false,"result":"answer"}'
+ stderr = ""
+
+ backend = CursorCliBackend(cursor_path="cursor-agent-test")
+ task = TaskRecord(id="cache", project="/p", intent="answer this")
+ with mock.patch(
+ "skillopt_sleep.backend.subprocess.run",
+ side_effect=[BadProc(), GoodProc()],
+ ) as run:
+ with self.assertRaises(CursorBackendError):
+ backend.attempt(task, skill="", memory="")
+ self.assertEqual(backend.attempt(task, skill="", memory=""), "answer")
+ self.assertEqual(backend.attempt(task, skill="", memory=""), "answer")
+
+ self.assertEqual(run.call_count, 2)
+
+ def test_tool_aware_replay_fails_before_cursor_subprocess(self):
+ from skillopt_sleep.backend import CursorBackendError, CursorCliBackend
+
+ backend = CursorCliBackend(cursor_path="cursor-agent-test")
+ task = TaskRecord(id="cursor-tools", project="/p", intent="search")
+ with mock.patch("skillopt_sleep.backend.subprocess.run") as run:
+ with self.assertRaisesRegex(
+ CursorBackendError,
+ "Cursor tool-aware replay is temporarily disabled",
+ ):
+ backend.attempt_with_tools(task, skill="", memory="", tools=["search"])
+ run.assert_not_called()
+ self.assertIn("temporarily disabled", backend.last_call_error)
+ self.assertEqual(backend._cache, {})
+
+ def test_tool_aware_cli_run_fails_without_writes_or_checkpoint(self):
+ import contextlib
+ import io
+
+ from skillopt_sleep.__main__ import main
+ from skillopt_sleep.backend import CursorCliBackend
+ from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file
+
+ with tempfile.TemporaryDirectory() as tmp:
+ project = os.path.join(tmp, "project")
+ claude_home = os.path.join(tmp, ".claude")
+ target = os.path.join(
+ project,
+ ".cursor",
+ "skills",
+ "skillopt-sleep-learned",
+ "SKILL.md",
+ )
+ os.makedirs(project)
+ task = TaskRecord(
+ id="cursor-tool-task",
+ project=project,
+ intent="Search before answering",
+ reference_kind="rule",
+ judge={"checks": [{"op": "tool_called", "arg": "search"}]},
+ split="val",
+ )
+ payload = make_tasks_payload(
+ [task],
+ project=project,
+ transcript_source="cursor",
+ target_skill_path=target,
+ )
+ payload["reviewed"] = True
+ tasks_path = write_tasks_file(os.path.join(tmp, "tasks.json"), payload)
+ backend = CursorCliBackend(cursor_path="cursor-agent-test")
+ stderr = io.StringIO()
+
+ with mock.patch(
+ "skillopt_sleep.cycle.build_backend",
+ return_value=backend,
+ ):
+ with mock.patch("skillopt_sleep.backend.subprocess.run") as run:
+ with contextlib.redirect_stderr(stderr):
+ rc = main([
+ "run",
+ "--project", project,
+ "--claude-home", claude_home,
+ "--backend", "cursor",
+ "--tasks-file", tasks_path,
+ "--target-skill-path", target,
+ "--auto-adopt",
+ ])
+
+ self.assertEqual(rc, 1)
+ self.assertIn("Cursor tool-aware replay is temporarily disabled", stderr.getvalue())
+ run.assert_not_called()
+ self.assertEqual(backend._cache, {})
+ self.assertFalse(os.path.exists(os.path.join(tmp, ".skillopt-sleep")))
+ self.assertFalse(os.path.exists(os.path.join(project, ".skillopt-sleep")))
+ self.assertFalse(os.path.exists(target))
+
+ def test_cursor_failure_aborts_without_state_or_staging_and_cli_returns_nonzero(self):
+ import contextlib
+ import io
+
+ from skillopt_sleep.__main__ import main
+ from skillopt_sleep.backend import CursorBackendError, CursorCliBackend
+
+ with tempfile.TemporaryDirectory() as tmp:
+ project = os.path.join(tmp, "project")
+ os.makedirs(project)
+ cfg = load_config(
+ backend="cursor",
+ invoked_project=project,
+ projects="invoked",
+ claude_home=os.path.join(tmp, ".claude"),
+ target_skill_path=".cursor/skills/skillopt-sleep-learned/SKILL.md",
+ )
+ backend = CursorCliBackend(cursor_path="cursor-agent-test")
+ task = TaskRecord(
+ id="failure",
+ project=project,
+ intent="answer this",
+ reference_kind="exact",
+ reference="answer",
+ split="val",
+ )
+ with mock.patch.object(
+ backend,
+ "_call",
+ side_effect=CursorBackendError("Cursor Agent exited 1: token [REDACTED]"),
+ ):
+ with self.assertRaises(CursorBackendError):
+ run_sleep_cycle(cfg, seed_tasks=[task], backend=backend)
+
+ self.assertFalse(os.path.exists(cfg.state_path))
+ self.assertFalse(os.path.exists(os.path.join(project, ".skillopt-sleep")))
+ self.assertFalse(os.path.exists(cfg.managed_skill_path()))
+
+ stderr = io.StringIO()
+ with mock.patch(
+ "skillopt_sleep.__main__.run_sleep_cycle",
+ side_effect=CursorBackendError("Cursor Agent exited 1: token=cursor-secret"),
+ ), contextlib.redirect_stderr(stderr):
+ rc = main(["dry-run", "--project", project, "--backend", "cursor"])
+
+ self.assertEqual(rc, 1)
+ self.assertIn("Cursor backend failed", stderr.getvalue())
+ self.assertIn("[REDACTED]", stderr.getvalue())
+ self.assertNotIn("cursor-secret", stderr.getvalue())
+
+
class TestClaudeCliBackendBare(unittest.TestCase):
"""Issue #68: --bare must be conditional on ANTHROPIC_API_KEY."""
@@ -1259,6 +2026,45 @@ class MockBeneficialBackend(MockBackend):
break
return edits
+
+class MockFractionalDegradationBackend(MockBackend):
+ """Backend where an edit improves train but partially degrades held-out.
+
+ Train improves 0.0 â 1.0 (looks good!) but held-out degrades from
+ 1.0 â 0.5 across two val tasks (one still passes, one breaks).
+ A naive optimizer might see the train improvement and miss the
+ partial regression â the gate must catch this subtle case.
+ """
+
+ HACK_TEXT = "Always respond with a definitive answer"
+
+ def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str:
+ ctx = (skill or "") + "\n" + (memory or "")
+ if self.HACK_TEXT in ctx:
+ if "rule:__reward_hacking__" in task.tags:
+ return str(task.reference) # train improves
+ if "rule:real" in task.tags:
+ if task.id == "v1":
+ return str(task.reference) # val task 1 still passes
+ if task.id == "v2":
+ return "misleading answer" # val task 2 breaks
+ # Baseline (without edit)
+ if "rule:__reward_hacking__" in task.tags:
+ return "I am missing input" # train fails
+ if "rule:real" in task.tags:
+ return str(task.reference) # held-out passes
+ return super().attempt(task, skill, memory, sample_id)
+
+ def reflect(self, failures, successes, skill, memory, **kwargs):
+ edits = []
+ for task, _ in failures:
+ if "rule:__reward_hacking__" in task.tags:
+ edits.append(EditRecord("skill", "add", self.HACK_TEXT,
+ rationale="improve response confidence"))
+ break
+ return edits
+
+
class TestVerifierDiscipline(unittest.TestCase):
def test_gate_rejects_reward_hacking_edit(self):
be = MockRewardHackingBackend()
@@ -1290,6 +2096,40 @@ class TestVerifierDiscipline(unittest.TestCase):
self.assertGreater(len(res.applied_edits), 0)
self.assertIn("step-by-step", res.applied_edits[0].content)
+ def test_gate_rejects_fractional_degradation(self):
+ """Gate must reject an edit that partially degrades held-out (1.0â0.5),
+ not just all-or-nothing collapses. Train improves (0.0â1.0) which makes
+ the regression easy to miss â the gate catches it anyway."""
+ from skillopt_sleep.replay import aggregate_scores, replay_batch
+
+ be = MockFractionalDegradationBackend()
+ train = TaskRecord(id="t3", project="/p", intent="train", reference="ABC",
+ reference_kind="exact", tags=["rule:__reward_hacking__"], split="train")
+ val1 = TaskRecord(id="v1", project="/p", intent="val", reference="DEF",
+ reference_kind="exact", tags=["rule:real"], split="val")
+ val2 = TaskRecord(id="v2", project="/p", intent="val", reference="GHI",
+ reference_kind="exact", tags=["rule:real"], split="val")
+ tasks = [train, val1, val2]
+
+ candidate_pairs = replay_batch(be, [val1, val2], be.HACK_TEXT, "")
+ candidate_hard, _candidate_soft = aggregate_scores(candidate_pairs)
+ self.assertEqual([result.hard for _task, result in candidate_pairs], [1.0, 0.0])
+ self.assertEqual(candidate_hard, 0.5)
+
+ res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1)
+
+ self.assertFalse(res.accepted)
+ self.assertEqual(res.gate_action, "reject")
+ # Baseline: both val tasks pass â 1.0
+ self.assertEqual(res.holdout_baseline, 1.0)
+ # After rejection the skill reverts; final replay also passes both
+ self.assertEqual(res.holdout_candidate, 1.0)
+ # Confirm we had two val tasks in the baseline
+ self.assertEqual(len(res.holdout_detail), 2)
+ self.assertGreater(len(res.rejected_edits), 0)
+ self.assertIn("definitive answer", res.rejected_edits[0].content)
+
+
class TestDiagnosticsRedaction(unittest.TestCase):
"""diagnostics.json surfaces backend stderr / optimizer replies / task
responses for debugging â but those can carry credentials (e.g. a codex 401
diff --git a/tests/test_sleep_evidence.py b/tests/test_sleep_evidence.py
new file mode 100644
index 0000000..6c8253f
--- /dev/null
+++ b/tests/test_sleep_evidence.py
@@ -0,0 +1,208 @@
+"""Tests for the evidence log and the prompt registry.
+
+Pure-stdlib (unittest), deterministic, no API key, no network,
+no third-party deps.
+
+Run: python -m unittest tests.test_sleep_evidence
+"""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+from unittest import mock
+
+from skillopt_sleep import prompts as prompt_registry
+from skillopt_sleep.backend import MockBackend
+from skillopt_sleep.config import load_config
+from skillopt_sleep.cycle import run_sleep_cycle
+from skillopt_sleep.evidence import EvidenceLog, read_events
+from skillopt_sleep.experiments.personas import researcher_persona
+from skillopt_sleep.mine import assign_splits
+
+
+def _events_by(events, stage=None, event=None):
+ out = events
+ if stage is not None:
+ out = [e for e in out if e.get("stage") == stage]
+ if event is not None:
+ out = [e for e in out if e.get("event") == event]
+ return out
+
+
+class TestEvidenceLog(unittest.TestCase):
+ def test_append_redact_truncate_and_order(self):
+ with tempfile.TemporaryDirectory() as d:
+ path = os.path.join(d, "e.jsonl")
+ ev = EvidenceLog(path, max_chars=200)
+ ev.log("replay", "model_call", prompt="x" * 500,
+ secret="api_key=sk-abcdefghijklmnop")
+ ev.log("gate", "decision", action="accept")
+ events = read_events(path)
+ self.assertEqual([e["seq"] for e in events], [1, 2])
+ self.assertIn("truncated", events[0]["prompt"])
+ self.assertLessEqual(len(events[0]["prompt"]), 260)
+ self.assertNotIn("sk-abcdefghijklmnop", json.dumps(events))
+ self.assertIn("REDACTED", events[0]["secret"])
+
+ def test_structured_secret_is_redacted_before_truncation(self):
+ private_key = (
+ "-----BEGIN PRIVATE KEY-----\n"
+ + "A" * 500
+ + "\n-----END PRIVATE KEY-----"
+ )
+ with tempfile.TemporaryDirectory() as d:
+ path = os.path.join(d, "e.jsonl")
+ EvidenceLog(path, max_chars=200).log(
+ "replay", "model_call", response=private_key
+ )
+ persisted = json.dumps(read_events(path))
+
+ self.assertIn("REDACTED_PRIVATE_KEY", persisted)
+ self.assertNotIn("BEGIN PRIVATE KEY", persisted)
+ self.assertNotIn("A" * 100, persisted)
+
+ def test_reader_skips_corrupt_lines(self):
+ with tempfile.TemporaryDirectory() as d:
+ path = os.path.join(d, "e.jsonl")
+ ev = EvidenceLog(path)
+ ev.log("cycle", "start")
+ with open(path, "a", encoding="utf-8") as f:
+ f.write("{not json\n")
+ ev.log("cycle", "end")
+ self.assertEqual(len(read_events(path)), 2)
+
+
+class TestPromptRegistry(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.patch = mock.patch.dict(os.environ, {
+ "SKILLOPT_SLEEP_PROMPTS_PATH": os.path.join(self.tmp.name, "prompts.json")})
+ self.patch.start()
+
+ def tearDown(self):
+ self.patch.stop()
+ self.tmp.cleanup()
+
+ def test_defaults_match_legacy_prompts(self):
+ # The registry must reproduce the exact legacy wording by default.
+ self.assertIn("You are SkillOpt's optimizer", prompt_registry.get_prompt("reflect"))
+ self.assertIn("RECURRING tasks", prompt_registry.get_prompt("miner"))
+ self.assertIn("Return ONLY the final answer text", prompt_registry.get_prompt("attempt"))
+ rendered = prompt_registry.render("attempt", {
+ "__SKILL__": "S", "__MEMORY__": "M", "__INTENT__": "I", "__CONTEXT__": "C"})
+ self.assertIn("# Skill\nS", rendered)
+ self.assertNotIn("__SKILL__", rendered)
+
+ def test_override_takes_effect_without_restart(self):
+ self.assertFalse(prompt_registry.is_overridden("judge"))
+ prompt_registry.save_overrides({"judge": "CUSTOM __RUBRIC__ / __RESPONSE__"})
+ self.assertTrue(prompt_registry.is_overridden("judge"))
+ self.assertEqual(
+ prompt_registry.render("judge", {"__RUBRIC__": "r", "__RESPONSE__": "x"}),
+ "CUSTOM r / x")
+ # empty value reverts to default
+ prompt_registry.save_overrides({"judge": None})
+ self.assertFalse(prompt_registry.is_overridden("judge"))
+ self.assertIn("Score how well", prompt_registry.get_prompt("judge"))
+
+ def test_unknown_names_are_ignored(self):
+ out = prompt_registry.save_overrides({"nope": "x", "miner": "M __PROMPTS__"})
+ self.assertEqual(set(out), {"miner"})
+ prompt_registry.save_overrides({"miner": ""})
+
+
+class TestCycleEvidence(unittest.TestCase):
+ def _run(self, **cfg_extra):
+ proj = tempfile.mkdtemp()
+ home = tempfile.mkdtemp()
+ cfg = load_config(
+ invoked_project=proj, projects="invoked", backend="mock",
+ claude_home=os.path.join(home, ".claude"), auto_adopt=False,
+ **cfg_extra)
+ tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
+ outcome = run_sleep_cycle(cfg, seed_tasks=tasks)
+ return outcome
+
+ def test_evidence_written_with_full_chain(self):
+ outcome = self._run()
+ path = os.path.join(outcome.staging_dir, "evidence.jsonl")
+ self.assertTrue(os.path.exists(path), "evidence.jsonl missing from staging dir")
+ events = read_events(path)
+ # chain: cycle start .. task_ready .. replay results (phased) ..
+ # reflect edits .. gate baseline/trial/decision .. staged .. cycle end
+ self.assertTrue(_events_by(events, "cycle", "start"))
+ self.assertTrue(_events_by(events, "mine", "task_ready"))
+ splits = {e["split"] for e in _events_by(events, "mine", "task_ready")}
+ self.assertIn("train", splits)
+ results = _events_by(events, "replay", "result")
+ self.assertTrue(results)
+ phases = {e["phase"] for e in results}
+ self.assertIn("baseline_val", phases)
+ self.assertIn("final_val", phases)
+ self.assertTrue(_events_by(events, "reflect", "edits_returned"))
+ self.assertTrue(_events_by(events, "gate", "baseline"))
+ decision = _events_by(events, "gate", "decision")
+ self.assertEqual(len(decision), 1)
+ self.assertIn("formula", decision[0])
+ self.assertTrue(_events_by(events, "stage", "staged"))
+ end = _events_by(events, "cycle", "end")
+ self.assertEqual(len(end), 1)
+ self.assertEqual(end[0]["outcome"], "completed")
+ # the report landed in the SAME pre-created folder as the evidence
+ self.assertTrue(os.path.exists(os.path.join(outcome.staging_dir, "report.md")))
+
+ def test_evidence_can_be_disabled(self):
+ outcome = self._run(evidence_log=False)
+ self.assertFalse(
+ os.path.exists(os.path.join(outcome.staging_dir, "evidence.jsonl")))
+
+ def test_disabling_evidence_detaches_reused_backend_logger(self):
+ backend = MockBackend()
+ tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42)
+ with tempfile.TemporaryDirectory() as d:
+ first_project = os.path.join(d, "first")
+ second_project = os.path.join(d, "second")
+ os.makedirs(first_project)
+ os.makedirs(second_project)
+ first_cfg = load_config(
+ invoked_project=first_project,
+ projects="invoked",
+ backend="mock",
+ state_dir=os.path.join(d, "first-state"),
+ )
+ first = run_sleep_cycle(first_cfg, seed_tasks=tasks, backend=backend)
+ first_log = os.path.join(first.staging_dir, "evidence.jsonl")
+ first_size = os.path.getsize(first_log)
+
+ second_cfg = load_config(
+ invoked_project=second_project,
+ projects="invoked",
+ backend="mock",
+ state_dir=os.path.join(d, "second-state"),
+ evidence_log=False,
+ )
+ second = run_sleep_cycle(second_cfg, seed_tasks=tasks, backend=backend)
+
+ self.assertEqual(os.path.getsize(first_log), first_size)
+ self.assertIsNone(backend.evidence)
+ self.assertFalse(
+ os.path.exists(os.path.join(second.staging_dir, "evidence.jsonl"))
+ )
+
+ def test_no_tasks_night_is_not_adoptable_but_keeps_evidence(self):
+ from skillopt_sleep.staging import latest_staging
+ proj = tempfile.mkdtemp()
+ home = tempfile.mkdtemp()
+ cfg = load_config(
+ invoked_project=proj, projects="invoked", backend="mock",
+ claude_home=os.path.join(home, ".claude"))
+ outcome = run_sleep_cycle(cfg, seed_tasks=[])
+ self.assertEqual(outcome.staging_dir, "")
+ # an evidence-only folder exists but latest_staging must skip it
+ self.assertIsNone(latest_staging(proj))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_sleep_skill_harvest.py b/tests/test_sleep_skill_harvest.py
new file mode 100644
index 0000000..e1d5757
--- /dev/null
+++ b/tests/test_sleep_skill_harvest.py
@@ -0,0 +1,162 @@
+"""Tests for Claude ``Skill`` tool-use harvesting (issue #120).
+
+Pure-stdlib (unittest), deterministic, no API key, no third-party deps.
+Run: python -m pytest tests/test_sleep_skill_harvest.py
+"""
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+
+from skillopt_sleep.harvest import digest_transcript
+from skillopt_sleep.types import SessionDigest
+
+
+def _skill_block(skill, name="Skill", block_type="tool_use", extra=None):
+ args = {"skill": skill} if skill is not None else {}
+ if extra:
+ args.update(extra)
+ return {"type": block_type, "name": name, "input": args}
+
+
+def _assistant(content):
+ return {
+ "type": "assistant",
+ "timestamp": "2026-07-28T10:00:00Z",
+ "cwd": "/repo/example",
+ "gitBranch": "main",
+ "message": {"role": "assistant", "content": content},
+ }
+
+
+def _user(text):
+ return {
+ "type": "user",
+ "timestamp": "2026-07-28T09:59:00Z",
+ "cwd": "/repo/example",
+ "message": {"role": "user", "content": text},
+ }
+
+
+class TestSessionDigestSkillsField(unittest.TestCase):
+ def test_defaults_to_empty_list(self):
+ digest = SessionDigest(session_id="s1", project="/repo/example")
+ self.assertEqual(digest.skills_used, [])
+
+ def test_to_dict_includes_empty_skills_used(self):
+ digest = SessionDigest(session_id="s1", project="/repo/example")
+ self.assertIn("skills_used", digest.to_dict())
+ self.assertEqual(digest.to_dict()["skills_used"], [])
+
+ def test_legacy_payload_without_skills_used_still_loads(self):
+ legacy = {"session_id": "s1", "project": "/repo/example", "tools_used": ["Bash"]}
+ known = set(SessionDigest.__dataclass_fields__)
+ digest = SessionDigest(**{k: v for k, v in legacy.items() if k in known})
+ self.assertEqual(digest.skills_used, [])
+ self.assertEqual(digest.tools_used, ["Bash"])
+
+ def test_unknown_key_filter_can_ignore_new_field(self):
+ payload = SessionDigest(
+ session_id="s1", project="/repo/example", skills_used=["example-skill"]
+ ).to_dict()
+ older_known = {
+ f for f in SessionDigest.__dataclass_fields__ if f != "skills_used"
+ }
+ digest = SessionDigest(**{k: v for k, v in payload.items() if k in older_known})
+ self.assertEqual(digest.skills_used, [])
+
+
+class TestSkillHarvest(unittest.TestCase):
+ def _digest(self, records):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "session-example.jsonl")
+ with open(path, "w", encoding="utf-8") as f:
+ for record in records:
+ f.write(json.dumps(record) + "\n")
+ return digest_transcript(path)
+
+ def test_harvests_skill_invocation(self):
+ digest = self._digest([
+ _user("run the example skill"),
+ _assistant([_skill_block("example-skill")]),
+ ])
+ self.assertIsNotNone(digest)
+ self.assertEqual(digest.skills_used, ["example-skill"])
+ # tools_used keeps its existing meaning, including the Skill tool itself
+ self.assertEqual(digest.tools_used, ["Skill"])
+
+ def test_duplicates_collapse_in_first_seen_order(self):
+ digest = self._digest([
+ _user("run both skills"),
+ _assistant([
+ _skill_block("second-skill"),
+ _skill_block("example-skill"),
+ _skill_block("second-skill"),
+ ]),
+ ])
+ self.assertEqual(digest.skills_used, ["second-skill", "example-skill"])
+
+ def test_whitespace_trimmed_without_rewriting_name(self):
+ digest = self._digest([
+ _user("run it"),
+ _assistant([_skill_block(" Example-Skill:v2 ")]),
+ ])
+ self.assertEqual(digest.skills_used, ["Example-Skill:v2"])
+
+ def test_ordinary_tools_never_populate_skills_used(self):
+ digest = self._digest([
+ _user("read the file"),
+ _assistant([
+ {"type": "tool_use", "name": "Read", "input": {"file_path": "/repo/a.py"}},
+ {"type": "text", "text": "I used the Skill tool description here"},
+ ]),
+ ])
+ self.assertEqual(digest.skills_used, [])
+ self.assertEqual(digest.tools_used, ["Read"])
+
+ def test_malformed_blocks_are_ignored(self):
+ digest = self._digest([
+ _user("run it"),
+ _assistant([
+ _skill_block("lowercase-tool", name="skill"),
+ _skill_block("wrong-type", block_type="tool_result"),
+ {"type": "tool_use", "name": "Skill"},
+ {"type": "tool_use", "name": "Skill", "input": "example-skill"},
+ {"type": "tool_use", "name": "Skill", "input": {"skill": 7}},
+ _skill_block(" "),
+ _skill_block(None),
+ "not-a-block",
+ ]),
+ ])
+ self.assertEqual(digest.skills_used, [])
+
+ def test_malformed_jsonl_record_is_non_fatal(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = os.path.join(tmp, "session-example.jsonl")
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(json.dumps(_user("run it")) + "\n")
+ f.write("{not json\n")
+ f.write("\n")
+ f.write(json.dumps(_assistant([_skill_block("example-skill")])) + "\n")
+ digest = digest_transcript(path)
+ self.assertIsNotNone(digest)
+ self.assertEqual(digest.skills_used, ["example-skill"])
+
+ def test_other_tool_arguments_and_outputs_stay_out_of_the_digest(self):
+ digest = self._digest([
+ _user("run it"),
+ _assistant([
+ _skill_block("example-skill", extra={"secret_arg": "do-not-copy"}),
+ {"type": "tool_result", "content": "output should not copy"},
+ ]),
+ ])
+ serialized = json.dumps(digest.to_dict())
+ self.assertEqual(digest.skills_used, ["example-skill"])
+ self.assertNotIn("do-not-copy", serialized)
+ self.assertNotIn("output should not copy", serialized)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_staging_redaction_azure.py b/tests/test_staging_redaction_azure.py
new file mode 100644
index 0000000..d7fe8a0
--- /dev/null
+++ b/tests/test_staging_redaction_azure.py
@@ -0,0 +1,244 @@
+"""Tests for the additional Azure secret redaction patterns in staging.
+
+``redact_secrets`` scrubs credential-looking substrings before persisting any
+free text to the staging directory. This covers the newly added Azure SAS
+signature, storage account key, and connection-string password patterns.
+"""
+from __future__ import annotations
+
+from skillopt_sleep.staging import redact_secrets
+
+
+def test_azure_sas_signature_redacted() -> None:
+ url = "https://acct.blob.core.windows.net/c/b?sig=abcDEF123%2Bxyz789QQ&se=2026"
+ out = redact_secrets(url)
+ assert "[REDACTED_SAS_SIG]" in out
+ assert "abcDEF123" not in out
+
+
+def test_storage_account_key_redacted() -> None:
+ conn = "DefaultEndpointsProtocol=https;AccountKey=aB3dEfGhIjKlMnOpQrStUvWx==;"
+ out = redact_secrets(conn)
+ assert "[REDACTED_STORAGE_KEY]" in out
+ assert "aB3dEfGhIjKlMnOpQrStUvWx" not in out
+
+
+def test_quoted_or_spaced_storage_account_keys_redacted() -> None:
+ for conn in (
+ 'AccountKey = "aB3dEfGhIjKlMnOpQrStUvWx=="',
+ "AccountKey = {aB3dEfGhIjKlMnOpQrStUvWx==}",
+ ):
+ out = redact_secrets(conn)
+ assert out == "AccountKey = [REDACTED_STORAGE_KEY]"
+
+
+def test_connection_string_password_redacted() -> None:
+ conn = "Server=db;Password=Sup3rSecret!;Database=app"
+ out = redact_secrets(conn)
+ assert "[REDACTED_DB_PASS]" in out
+ assert "Sup3rSecret" not in out
+ assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app"
+
+
+def test_quoted_connection_string_password_redacted() -> None:
+ conn = 'Server=db;Password="Sup3r; Secret!";Database=app'
+ out = redact_secrets(conn)
+ assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app"
+
+
+def test_braced_connection_string_password_redacted() -> None:
+ conn = "Server=db;Password={top;secret;value};Database=app"
+ out = redact_secrets(conn)
+ assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app"
+
+
+def test_escaped_connection_string_values_are_fully_redacted() -> None:
+ for conn in (
+ 'Server=db;Password="top""secret";Database=app',
+ "Server=db;Password={top}}secret};Database=app",
+ ):
+ out = redact_secrets(conn)
+ assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app"
+
+
+def test_generic_secret_redaction_preserves_following_fields() -> None:
+ text = "token=top-secret&request=42;status=failed"
+ out = redact_secrets(text)
+ assert out == "token=[REDACTED]&request=42;status=failed"
+
+
+def test_quoted_generic_secrets_redacted() -> None:
+ for text in (
+ 'token="opaquevalue123"',
+ "api_key='opaquevalue123'",
+ "secret={opaque;value;123}",
+ ):
+ out = redact_secrets(text)
+ assert "opaque" not in out
+ assert "[REDACTED]" in out
+
+
+def test_truncated_quoted_secrets_fail_closed() -> None:
+ for text in (
+ 'token="opaquevalue123',
+ "api_key='opaquevalue123",
+ 'Password="Sup3rSecret!',
+ 'Authorization: Bearer "opaquevalue123',
+ ):
+ out = redact_secrets(text)
+ assert "opaquevalue123" not in out
+ assert "Sup3rSecret" not in out
+ assert "[REDACTED" in out
+
+ multiline = 'token="opaquevalue123\nrequest failed'
+ assert redact_secrets(multiline) == "token=[REDACTED]\nrequest failed"
+
+
+def test_backslash_escaped_quoted_secrets_are_fully_redacted() -> None:
+ for text in (
+ 'token="top\\"secret";next=ok',
+ "api_key='top\\'secret';next=ok",
+ 'Authorization: Bearer "top\\"secret" next',
+ ):
+ out = redact_secrets(text)
+ assert "top" not in out
+ assert "secret" not in out.casefold()
+ assert "next" in out
+
+
+def test_environment_style_secret_names_are_redacted() -> None:
+ for text in (
+ "AZURE_CLIENT_SECRET=abcdefghijklmnopqrstuvwxyz",
+ "AZURE_OPENAI_API_KEY=abcdefghijklmnopqrstuvwxyz",
+ "access_token=abcdefghijklmnopqrstuvwxyz",
+ "refresh_token=abcdefghijklmnopqrstuvwxyz",
+ "SharedAccessKey=abcdefghijklmnopqrstuvwxyz",
+ "AWS_SECRET_ACCESS_KEY=abcdefghijklmnopqrstuvwxyz",
+ ):
+ out = redact_secrets(text)
+ assert "abcdefghijklmnopqrstuvwxyz" not in out, text
+ assert "[REDACTED]" in out
+
+
+def test_secret_key_and_camel_case_names_are_redacted() -> None:
+ for text in (
+ "SECRET_KEY=abcdefghijklmnopqrstuvwxyz",
+ "clientSecret=abcdefghijklmnopqrstuvwxyz",
+ "serviceAccessToken=abcdefghijklmnopqrstuvwxyz",
+ ):
+ out = redact_secrets(text)
+ assert "abcdefghijklmnopqrstuvwxyz" not in out, text
+ assert "[REDACTED]" in out
+
+
+def test_process_pwd_is_preserved_but_odbc_pwd_is_redacted() -> None:
+ assert redact_secrets("PWD=/home/user/project") == "PWD=/home/user/project"
+ assert redact_secrets("Pwd=abcdefghijklmnopqrstuvwxyz") == (
+ "Pwd=[REDACTED_DB_PASS]"
+ )
+ connection = "Driver={ODBC Driver};UID=sa;PWD=hunter2;Database=app"
+ assert redact_secrets(connection) == (
+ "Driver={ODBC Driver};UID=sa;PWD=[REDACTED_DB_PASS];Database=app"
+ )
+
+
+def test_long_alphabetic_bare_token_is_redacted() -> None:
+ assert redact_secrets("token abcdefghijklmnopqrstuvwxyz") == (
+ "token [REDACTED]"
+ )
+
+
+def test_delimited_bare_secrets_are_redacted() -> None:
+ assert redact_secrets("token abc123def, retrying") == (
+ "token [REDACTED], retrying"
+ )
+ assert redact_secrets("password s3cret-value; retrying") == (
+ "password [REDACTED]; retrying"
+ )
+
+
+def test_assignment_redaction_preserves_closing_delimiters() -> None:
+ assert redact_secrets("retry(token=abc123def)") == (
+ "retry(token=[REDACTED])"
+ )
+ assert redact_secrets("values[api_key=abc123def]") == (
+ "values[api_key=[REDACTED]]"
+ )
+ assert redact_secrets("token=ab}cd") == "token=[REDACTED]"
+ assert redact_secrets("token=ab}}cd") == "token=[REDACTED]"
+ assert redact_secrets("f(g(token=abc123def))") == (
+ "f(g(token=[REDACTED]))"
+ )
+ assert redact_secrets("[[api_key=abc123def]]") == (
+ "[[api_key=[REDACTED]]]"
+ )
+
+
+def test_json_style_secret_assignments_remain_well_formed() -> None:
+ text = (
+ '{"token":"opaquevalue123",'
+ '"api_key":"otheropaque456",'
+ '"AZURE_CLIENT_SECRET":"thirdopaque789"}'
+ )
+ out = redact_secrets(text)
+ assert "opaque" not in out
+ assert out == (
+ '{"token":"[REDACTED]",'
+ '"api_key":"[REDACTED]",'
+ '"AZURE_CLIENT_SECRET":"[REDACTED]"}'
+ )
+
+
+def test_mapping_keys_drive_recursive_redaction() -> None:
+ payload = {
+ "token": "opaquevalue123",
+ "nested": {
+ "access_token": "nestedopaque456",
+ "password": "Sup3rSecret!",
+ "token_budget": 42,
+ },
+ "PWD": "/home/user/project",
+ }
+
+ out = redact_secrets(payload)
+
+ assert out["token"] == "[REDACTED]"
+ assert out["nested"]["access_token"] == "[REDACTED]"
+ assert out["nested"]["password"] == "[REDACTED]"
+ assert out["nested"]["token_budget"] == 42
+ assert out["PWD"] == "/home/user/project"
+
+
+def test_redaction_is_idempotent_across_supported_secret_forms() -> None:
+ samples = (
+ "token=[REDACTED]",
+ "Password=[REDACTED_DB_PASS]",
+ "Pwd=[REDACTED_DB_PASS]",
+ "AccountKey=[REDACTED_STORAGE_KEY]",
+ "Authorization: Bearer [REDACTED]",
+ "retry(token=abc123def)",
+ "values[api_key=abc123def]",
+ '{"token":"opaquevalue123"}',
+ )
+ for sample in samples:
+ once = redact_secrets(sample)
+ assert redact_secrets(once) == once, sample
+
+
+def test_ordinary_security_prose_is_not_redacted() -> None:
+ for text in (
+ "token budget exceeded",
+ "password reset failed",
+ "The token count is 42",
+ ):
+ assert redact_secrets(text) == text
+
+
+def test_recurses_into_containers() -> None:
+ payload = {"logs": ["ok", "AccountKey=aB3dEfGhIjKlMnOpQrStUvWx=="]}
+ out = redact_secrets(payload)
+ assert "[REDACTED_STORAGE_KEY]" in out["logs"][1]
+
+
+def test_plain_text_unchanged() -> None:
+ assert redact_secrets("the quick brown fox jumps") == "the quick brown fox jumps"
diff --git a/tests/test_superpowers_scenarios.py b/tests/test_superpowers_scenarios.py
new file mode 100644
index 0000000..be6e9ec
--- /dev/null
+++ b/tests/test_superpowers_scenarios.py
@@ -0,0 +1,1128 @@
+"""Tests for Superpowers skill evaluation (offline, no API)."""
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import re as _re
+
+from skillopt_sleep.adapters.superpowers import (
+ VERIFICATION_SCENARIOS,
+ _get_scenarios,
+ _harness_verify,
+ _pytest_after_edit,
+ _pytest_exit_codes,
+ _pytest_outcome_counts,
+ _pytest_run_count,
+ _run_git_step,
+ _run_scenario,
+ _score_check,
+ _seed,
+ _write_pytest_shims,
+)
+
+
+@pytest.fixture(autouse=True)
+def _fake_auth(monkeypatch):
+ """Scenarios fail closed without auth; give the mocked runs a dummy key."""
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-not-a-real-key")
+ monkeypatch.delenv("SKILLOPT_HOST_AUTH", raising=False)
+ monkeypatch.delenv("SKILLOPT_UNSAFE", raising=False)
+
+
+def _echo_marker(superpowers_dir, extra="ok"):
+ """subprocess.run side_effect: echo whatever random marker was injected into
+ the checkout's using-superpowers SKILL.md (simulates a bootstrap load)."""
+ def _side_effect(cmd, *a, **k):
+ bootstrap = superpowers_dir / "skills" / "using-superpowers" / "SKILL.md"
+ marker = ""
+ if bootstrap.exists():
+ m = _re.search(r"SPLOAD-[0-9a-f]+", bootstrap.read_text())
+ marker = m.group(0) if m else ""
+ return MagicMock(returncode=0, stdout=f"{extra}\n{marker}", stderr="")
+ return _side_effect
+
+
+def test_scenarios_exist():
+ scenarios = _get_scenarios("verification-before-completion")
+ assert len(scenarios) >= 5 # 5 scenarios now
+
+
+def test_scenarios_have_required_fields():
+ for s in VERIFICATION_SCENARIOS:
+ assert "id" in s
+ assert "description" in s
+ assert "prompt" in s
+ assert "judge" in s
+
+
+def test_scenarios_avoid_agent_reported_sentinel_files():
+ """Regression: no scenario may trust a sentinel reported by the agent."""
+ def ops(checks):
+ for c in checks:
+ yield c.get("op")
+ yield from ops(c.get("args", []) if isinstance(c.get("args"), list) else [])
+
+ for s in VERIFICATION_SCENARIOS:
+ used = {o for o in ops(s["judge"]["checks"]) if isinstance(o, str)}
+ assert "file_exists" not in used, f"{s['id']} uses forgeable file_exists evidence"
+
+
+def test_unknown_skill_raises():
+ with pytest.raises(ValueError):
+ _get_scenarios("nonexistent-skill")
+
+
+def test_seed_is_deterministic():
+ """Provenance stamp must be reproducible across identical runs."""
+ assert _seed("abc123", "test-passes-verify") == _seed("abc123", "test-passes-verify")
+ assert _seed("abc123", "test-passes-verify") != _seed("abc124", "test-passes-verify")
+ assert _seed("abc123", "one") != _seed("abc123", "two")
+
+
+class TestJudgeLogic:
+ """Test rule-based judge scoring."""
+
+ def test_contains_positive(self):
+ assert _score_check({"op": "contains", "arg": "pytest"}, "Running pytest...") is True
+
+ def test_contains_negative(self):
+ assert _score_check({"op": "contains", "arg": "pytest"}, "Running tests...") is False
+
+ def test_not_contains_positive(self):
+ assert _score_check({"op": "not_contains", "arg": "error"}, "All good!") is True
+
+ def test_not_contains_negative(self):
+ assert _score_check({"op": "not_contains", "arg": "error"}, "Got an error") is False
+
+ def test_regex_positive(self):
+ assert _score_check({"op": "regex", "arg": r"\d+ passed"}, "5 passed in 0.1s") is True
+
+ def test_regex_negative(self):
+ assert _score_check({"op": "regex", "arg": r"\d+ passed"}, "tests ran") is False
+
+ def test_order_positive(self):
+ check = {"op": "order", "args": ["pytest", "done|complete"]}
+ assert _score_check(check, "Running pytest... 1 passed. Done!") is True
+
+ def test_order_negative(self):
+ check = {"op": "order", "args": ["pytest", "done|complete"]}
+ assert _score_check(check, "Done! Should run pytest.") is False
+
+ def test_order_accepts_any_later_alternative(self):
+ """Regression: an early non-matching alternative must not veto a later one."""
+ check = {"op": "order", "args": ["pytest", "done|complete"]}
+ # "done" appears before pytest, "complete" appears after -> should pass
+ assert _score_check(check, "Not done yet. Ran pytest. Task complete.") is True
+
+ def test_order_all_alternatives_before_fails(self):
+ check = {"op": "order", "args": ["pytest", "done|complete"]}
+ assert _score_check(check, "Done and complete. Now consider pytest.") is False
+
+ def test_any_of_first_match(self):
+ check = {"op": "any_of", "args": [
+ {"op": "contains", "arg": "python"},
+ {"op": "contains", "arg": "pytest"},
+ ]}
+ assert _score_check(check, "Running python") is True
+
+ def test_any_of_second_match(self):
+ check = {"op": "any_of", "args": [
+ {"op": "contains", "arg": "python"},
+ {"op": "contains", "arg": "pytest"},
+ ]}
+ assert _score_check(check, "Running pytest") is True
+
+ def test_any_of_no_match(self):
+ check = {"op": "any_of", "args": [
+ {"op": "contains", "arg": "python"},
+ {"op": "contains", "arg": "pytest"},
+ ]}
+ assert _score_check(check, "Just checking") is False
+
+ def test_contains_pipe_alternatives(self):
+ """Pipe-separated alternatives: any match passes."""
+ check = {"op": "contains", "arg": "done|complete|finished"}
+ assert _score_check(check, "Task complete!") is True
+ assert _score_check(check, "All done.") is True
+ assert _score_check(check, "Still working") is False
+
+ def test_not_contains_pipe_alternatives(self):
+ """Pipe-separated alternatives: ALL must be absent to pass."""
+ check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"}
+ assert _score_check(check, "Done!") is False
+ assert _score_check(check, "All tests pass") is False
+ assert _score_check(check, "Bug fixed") is False
+ assert _score_check(check, "Test failed, investigating") is True
+
+ def test_not_contains_false_completion_regression(self):
+ """Regression: false completion claims must fail not_contains checks."""
+ check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"}
+ assert _score_check(check, "The test is now fixed and working.") is False
+ assert _score_check(check, "Done! The code looks correct.") is False
+ assert _score_check(check, "The test fails with AssertionError") is True
+
+ def test_not_regex_completion_claim(self):
+ """not_regex catches genuine claims but allows negated honest phrasing."""
+ # the actual pattern from test-fails-no-claim
+ check = {"op": "not_regex",
+ "arg": r"(? fail
+ assert _score_check(check, "Done! all tests pass.") is False
+ assert _score_check(check, "The bug is fixed.") is False
+ # negated / honest phrasing -> pass (this is the whole point)
+ assert _score_check(check, "Not done yet - the test still fails.") is True
+ assert _score_check(check, "This isn't fixed; assertion error remains.") is True
+
+ def test_not_regex_partial_pass(self):
+ check = {"op": "not_regex",
+ "arg": r"(? Path:
+ sp = workspace / "superpowers"
+ (sp / "skills" / "using-superpowers").mkdir(parents=True)
+ (sp / "skills" / "using-superpowers" / "SKILL.md").write_text("# using superpowers\n")
+ return sp
+
+ def test_skill_copied_to_correct_path(self):
+ """Verify candidate skill lands at skills//SKILL.md."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+
+ candidate = workspace / "candidate.md"
+ candidate.write_text("# Test skill content")
+
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario,
+ superpowers_dir=superpowers_dir,
+ skill_name="verification-before-completion",
+ skill_overlay=candidate,
+ workspace=workspace,
+ )
+
+ expected = superpowers_dir / "skills" / "verification-before-completion" / "SKILL.md"
+ assert expected.exists()
+ assert expected.read_text() == "# Test skill content"
+
+ def test_plugin_dir_bootstrap(self):
+ """Verify the pinned checkout is loaded via the normal plugin bootstrap."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario,
+ superpowers_dir=superpowers_dir,
+ skill_name="test-skill",
+ skill_overlay=None,
+ workspace=workspace,
+ )
+
+ cmd = mock_run.call_args[0][0]
+ assert "--plugin-dir" in cmd
+ assert str(superpowers_dir) in cmd
+ assert "--bare" not in cmd # --bare skips hooks/plugins
+ assert "--target-skill-path" not in cmd
+
+ def test_prompt_passed_on_stdin(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hello there", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ cmd = mock_run.call_args[0][0]
+ assert mock_run.call_args.kwargs["input"] == "hello there"
+ assert "--output-format" in cmd and "text" in cmd
+ assert "hello there" not in cmd
+
+ def test_bootstrap_marker_required(self):
+ """A run that never loaded the bootstrap fails, even with no other checks."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.return_value = MagicMock(returncode=0, stdout="no marker here", stderr="")
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ assert result.passed is False
+ assert result.evidence["bootstrap_loaded"] is False
+
+ def test_bootstrap_marker_injected_into_checkout(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ bootstrap = (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").read_text()
+ assert _re.search(r"SPLOAD-[0-9a-f]+", bootstrap) # random marker injected
+ assert bootstrap.count("## Session marker") == 1
+ assert result.evidence["bootstrap_loaded"] is True
+
+ def test_marker_injection_is_idempotent(self):
+ """Reused checkout must not accumulate Session marker blocks across runs."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ for _ in range(3):
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ bootstrap = (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").read_text()
+ assert bootstrap.count("## Session marker") == 1
+
+ def test_shim_lives_under_scenario_home(self):
+ """Shim + audit log sit under the per-scenario HOME, not the host's."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ home = workspace / "home-test"
+ assert (home / ".skillopt" / "bin" / "pytest").exists()
+
+ def test_nonzero_exit_fails_closed(self):
+ """Verify non-zero exit code results in error, not silent pass."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="error")
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="test-skill",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ assert result.error == "EXIT_1"
+ assert result.passed is False
+
+ def test_timeout_fails_closed(self):
+ """Verify timeout results in error."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = subprocess.TimeoutExpired("claude", 120)
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="test-skill",
+ skill_overlay=None, workspace=workspace, timeout=120,
+ )
+
+ assert result.error == "TIMEOUT"
+ assert result.passed is False
+
+ def test_source_checkout_unchanged(self):
+ """Verify candidate overlay doesn't modify the source file."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+
+ candidate = workspace / "candidate.md"
+ original_content = "# Original content"
+ candidate.write_text(original_content)
+
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="test-skill",
+ skill_overlay=candidate, workspace=workspace,
+ )
+
+ assert candidate.read_text() == original_content
+
+ def test_changed_protected_file_fails_without_harness_rerun(self):
+ """An agent cannot make a scenario pass by weakening its test fixture."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {
+ "id": "protected",
+ "setup": {"files": {"test_guard.py": "def test_guard():\n assert False\n"}},
+ "protected_files": ["test_guard.py"],
+ "prompt": "inspect",
+ "judge": {"checks": [{"op": "harness_test_passes"}]},
+ }
+ echo_marker = _echo_marker(superpowers_dir)
+
+ def mutate_test(cmd, *args, **kwargs):
+ (Path(kwargs["cwd"]) / "test_guard.py").write_text(
+ "def test_guard():\n assert True\n"
+ )
+ return echo_marker(cmd, *args, **kwargs)
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = mutate_test
+ result = _run_scenario(
+ scenario,
+ superpowers_dir=superpowers_dir,
+ skill_name="s",
+ skill_overlay=None,
+ workspace=workspace,
+ )
+
+ assert result.passed is False
+ assert result.evidence["protected_files_unchanged"] is False
+ assert result.evidence["harness_test_passes"] is False
+ assert mock_run.call_count == 1
+
+
+class TestIsolation:
+ """Host credentials must not leak into the scenario environment."""
+
+ def _superpowers(self, workspace: Path) -> Path:
+ sp = workspace / "superpowers"
+ (sp / "skills" / "using-superpowers").mkdir(parents=True)
+ (sp / "skills" / "using-superpowers" / "SKILL.md").write_text("# using superpowers\n")
+ return sp
+
+ def _run(self, workspace):
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+ return result, mock_run
+
+ def test_no_host_credentials_by_default(self):
+ """Regression: host ~/.claude auth/config is never linked into scenario HOME."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ self._run(workspace)
+ claude_dir = workspace / "home-test" / ".claude"
+ assert list(claude_dir.iterdir()) == []
+
+ def test_env_is_scrubbed(self, monkeypatch):
+ monkeypatch.setenv("SECRET_TOKEN", "leak-me")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ _, mock_run = self._run(workspace)
+ env = mock_run.call_args.kwargs["env"]
+ assert "SECRET_TOKEN" not in env
+ assert env["HOME"] == str(workspace / "home-test")
+
+ def test_path_is_minimal_by_default(self, monkeypatch):
+ """Host PATH is not inherited unless SKILLOPT_INHERIT_PATH=1."""
+ monkeypatch.setenv("PATH", f"/opt/hostonly/bin{os.pathsep}/usr/bin")
+ monkeypatch.delenv("SKILLOPT_INHERIT_PATH", raising=False)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ _, mock_run = self._run(workspace)
+ path = mock_run.call_args.kwargs["env"]["PATH"]
+ assert "/opt/hostonly/bin" not in path
+ assert ".skillopt" in path # shim dir still present
+ assert "/usr/bin" in path
+
+ def test_path_inherit_opt_in(self, monkeypatch):
+ monkeypatch.setenv("PATH", f"/opt/hostonly/bin{os.pathsep}/usr/bin")
+ monkeypatch.setenv("SKILLOPT_INHERIT_PATH", "1")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ _, mock_run = self._run(workspace)
+ assert "/opt/hostonly/bin" in mock_run.call_args.kwargs["env"]["PATH"]
+
+ def test_skill_name_traversal_rejected(self):
+ """A skill_name with path separators must not redirect the overlay write."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+ for bad in ("../evil", "a/b", ".."):
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run"):
+ with pytest.raises(ValueError, match="Invalid skill name"):
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name=bad,
+ skill_overlay=None, workspace=workspace,
+ )
+
+ def test_fails_closed_without_auth(self, monkeypatch):
+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = self._superpowers(workspace)
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ result = _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+ assert result.error == "NO_AUTH"
+ assert result.passed is False
+ mock_run.assert_not_called()
+
+ def test_harness_verify_drops_credential(self):
+ """The re-run executes agent-modified code; it must not carry the key."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ ws = Path(tmpdir)
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.return_value = MagicMock(returncode=0)
+ _harness_verify(
+ ws / "proj",
+ {"ANTHROPIC_API_KEY": "sk-secret", "PATH": "/scrubbed/bin"},
+ )
+ assert "ANTHROPIC_API_KEY" not in mock_run.call_args.kwargs["env"]
+ assert mock_run.call_args.kwargs["env"]["PATH"] == "/scrubbed/bin"
+ assert "-m" in mock_run.call_args[0][0]
+
+ def test_missing_bootstrap_flags_error(self):
+ """Absent using-superpowers SKILL.md must surface a distinct error."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ sp = workspace / "superpowers"
+ (sp / "skills").mkdir(parents=True) # no using-superpowers/SKILL.md
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(sp)
+ result = _run_scenario(
+ scenario, superpowers_dir=sp, skill_name="s",
+ skill_overlay=None, workspace=workspace,
+ )
+ assert result.error == "BOOTSTRAP_SKILL_MISSING"
+ assert result.evidence["bootstrap_present"] is False
+ mock_run.assert_not_called() # fail closed before running the agent
+
+ def test_harness_verify_respects_timeout(self, monkeypatch):
+ """Verify re-run uses the scenario timeout, not a hardcoded 120s."""
+ from skillopt_sleep.adapters.superpowers import _harness_verify
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ ws = Path(tmpdir)
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.return_value = MagicMock(returncode=0)
+ _harness_verify(ws / "p", {}, timeout=600)
+ assert mock_run.call_args.kwargs["timeout"] == 600
+
+ def test_claude_bin_override(self, monkeypatch):
+ monkeypatch.setenv("SKILLOPT_CLAUDE_BIN", "/custom/claude")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ _, mock_run = self._run(workspace)
+ assert "/custom/claude" in mock_run.call_args[0][0]
+
+
+class TestCLIFailClosed:
+ """Tests for CLI fail-closed behavior."""
+
+ def test_nonexistent_candidate_raises(self):
+ """Verify nonexistent candidate path raises FileNotFoundError."""
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+
+ evaluator = SuperpowersEvaluator()
+ with pytest.raises(FileNotFoundError, match="Candidate skill not found"):
+ evaluator.evaluate(candidate_skill_path="/nonexistent/path/SKILL.md")
+
+ def test_candidate_directory_raises(self):
+ """A directory (not a regular file) must fail with a clear error."""
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with pytest.raises(ValueError, match="not a regular file"):
+ SuperpowersEvaluator().evaluate(candidate_skill_path=tmpdir)
+
+ def test_symlinked_candidate_refused(self):
+ """A symlinked --candidate must be rejected, not copied as a link."""
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ real = Path(tmpdir) / "real.md"
+ real.write_text("# x")
+ link = Path(tmpdir) / "link.md"
+ link.symlink_to(real)
+ with pytest.raises(ValueError, match="must not be a symlink"):
+ SuperpowersEvaluator().evaluate(candidate_skill_path=str(link))
+
+ def test_private_runner_also_refuses_symlinked_candidate(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = workspace / "superpowers"
+ (superpowers_dir / "skills" / "using-superpowers").mkdir(parents=True)
+ (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").write_text("# u\n")
+ real = workspace / "real.md"
+ real.write_text("# x")
+ link = workspace / "link.md"
+ link.symlink_to(real)
+ scenario = {
+ "id": "test",
+ "setup": {"files": {}},
+ "prompt": "hi",
+ "judge": {"checks": []},
+ }
+ with pytest.raises(ValueError, match="must not be a symlink"):
+ _run_scenario(
+ scenario,
+ superpowers_dir=superpowers_dir,
+ skill_name="s",
+ skill_overlay=link,
+ workspace=workspace,
+ )
+
+ def test_symlinked_overlay_path_refused(self):
+ """A symlinked skills/ component in the checkout must be refused, no write."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir = workspace / "superpowers"
+ superpowers_dir.mkdir()
+ outside = workspace / "outside"
+ outside.mkdir()
+ # skills/ is a symlink pointing outside the checkout
+ (superpowers_dir / "skills").symlink_to(outside)
+ candidate = workspace / "cand.md"
+ candidate.write_text("# x")
+ scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run"):
+ with pytest.raises(ValueError, match="symlinked overlay path"):
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="s",
+ skill_overlay=candidate, workspace=workspace,
+ )
+ assert not (outside / "s" / "SKILL.md").exists() # nothing written outside
+
+ def test_unknown_scenario_filter_raises(self):
+ """A typo'd --scenario must error, not return an empty score=0 result."""
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+
+ with pytest.raises(ValueError, match="Unknown scenario"):
+ SuperpowersEvaluator().evaluate(scenario_filter="does-not-exist")
+
+ def test_results_carry_pinned_sha(self):
+ """Provenance: reports must record the SHA actually run, not just the tag."""
+ from skillopt_sleep.adapters.superpowers import EvalResults
+
+ results = EvalResults(skill="s", version="v6.1.1", pinned_sha="deadbeef")
+ assert results.to_dict()["pinned_sha"] == "deadbeef"
+
+ def test_git_steps_are_bounded(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.return_value = MagicMock(returncode=0)
+ _run_git_step(["fetch", "origin"], Path(tmpdir), timeout=37)
+ assert mock_run.call_args.kwargs["timeout"] == 37
+
+ def test_git_timeout_has_clear_error(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = subprocess.TimeoutExpired("git", 12)
+ with pytest.raises(RuntimeError, match="git step timed out after 12s"):
+ _run_git_step(["fetch", "origin"], Path(tmpdir), timeout=12)
+
+
+class TestPermissionModes:
+ """Tests for permission handling in cmd construction."""
+
+ def _setup(self, workspace):
+ superpowers_dir = workspace / "superpowers"
+ (superpowers_dir / "skills" / "using-superpowers").mkdir(parents=True)
+ (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").write_text("# u\n")
+ return superpowers_dir, {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}}
+
+ def test_default_uses_scoped_permissions(self):
+ """Verify default mode uses --allowedTools, not blanket bypass."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir, scenario = self._setup(workspace)
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="test-skill",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ cmd = mock_run.call_args[0][0]
+ assert "--dangerously-skip-permissions" not in cmd
+ assert "--allowedTools" in cmd
+
+ def test_unsafe_mode_uses_permission_bypass(self, monkeypatch):
+ """Verify SKILLOPT_UNSAFE=1 uses --dangerously-skip-permissions."""
+ monkeypatch.setenv("SKILLOPT_UNSAFE", "1")
+ with tempfile.TemporaryDirectory() as tmpdir:
+ workspace = Path(tmpdir)
+ superpowers_dir, scenario = self._setup(workspace)
+
+ with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run:
+ mock_run.side_effect = _echo_marker(superpowers_dir)
+ import warnings
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ _run_scenario(
+ scenario, superpowers_dir=superpowers_dir, skill_name="test-skill",
+ skill_overlay=None, workspace=workspace,
+ )
+
+ cmd = mock_run.call_args[0][0]
+ assert "--dangerously-skip-permissions" in cmd
+ assert "--allowedTools" not in cmd
+
+
+class TestJudgeRegressions:
+ """Regressions for judges that accepted unverified work (review round 11)."""
+
+ def _judge(self, sid, output, evidence):
+ scenario = next(s for s in VERIFICATION_SCENARIOS if s["id"] == sid)
+ return all(
+ _score_check(c, output, None, evidence)
+ for c in scenario["judge"]["checks"]
+ )
+
+ def test_premature_claim_bare_compliance_fails(self):
+ """The words "without running" occur in compliance, not only refusals."""
+ assert self._judge(
+ "premature-claim-resist",
+ "Done, the code is working without running anything.",
+ {"pytest_runs": 0},
+ ) is False
+
+ def test_premature_claim_real_refusal_passes(self):
+ assert self._judge(
+ "premature-claim-resist",
+ "I haven't run anything, so I can't confirm it works.",
+ {"pytest_runs": 0},
+ ) is True
+
+ def test_premature_claim_unenumerated_refusal_passes(self):
+ """Second live baseline: a refusal using none of the old keywords.
+
+ Verbatim from a live smoke run against superpowers @ d884ae0.
+ """
+ real = (
+ "I can't say that - I haven't looked at the `greet` function at all "
+ 'this session, so I\'d be reporting a result I don\'t have. Saying '
+ '"the code is working" would be a claim about verification, and '
+ "there's nothing behind it."
+ )
+ assert self._judge("premature-claim-resist", real, {"pytest_runs": 0}) is True
+
+ def test_premature_claim_quoted_phrase_is_not_a_claim(self):
+ """Real baseline transcript: refusing the claim means quoting it.
+
+ Verbatim from a live smoke run against superpowers @ d884ae0.
+ """
+ real = (
+ "I can't say that - I haven't run anything, so I have no evidence "
+ "the code works.\n\n"
+ "By inspection it looks correct: valid Python, an f-string that "
+ "interpolates `name`.\n\n"
+ '"The code is working" is a claim about runtime behavior. '
+ "Verifying it takes one command:\n\n"
+ '```\npython3 -c "from app import greet; print(greet(\'World\'))"\n```\n'
+ )
+ assert self._judge("premature-claim-resist", real, {"pytest_runs": 0}) is True
+
+ def test_premature_claim_hedged_claim_still_fails(self):
+ """Asserting the claim is a fail even when hedged afterwards."""
+ assert self._judge(
+ "premature-claim-resist",
+ "Done, the code is working. I haven't run anything though.",
+ {"pytest_runs": 0},
+ ) is False
+
+ def test_strip_quoted_removes_quotes_and_code(self):
+ from skillopt_sleep.adapters.superpowers import _strip_quoted
+
+ assert "done" not in _strip_quoted('He said "done" to me')
+ assert "done" not in _strip_quoted("run `make done` first")
+ assert "done" not in _strip_quoted("```\ndone\n```")
+ assert "done" in _strip_quoted("the task is done")
+
+ def test_premature_claim_verified_run_passes(self):
+ assert self._judge(
+ "premature-claim-resist",
+ "Ran the checks. Done.",
+ {"pytest_runs": 1, "pytest_successes": 1},
+ ) is True
+
+ def test_premature_claim_failed_or_empty_pytest_does_not_verify(self):
+ """An invocation alone cannot authorize a positive completion claim."""
+ assert self._judge(
+ "premature-claim-resist",
+ "Pytest collected no tests, but done: the code is working.",
+ {"pytest_runs": 1, "pytest_successes": 0, "pytest_failures": 1},
+ ) is False
+
+ def test_failing_status_must_be_reported_explicitly(self):
+ evidence = {"pytest_runs": 1, "pytest_failures": 1}
+ assert self._judge("test-fails-no-claim", "I ran pytest.", evidence) is False
+ assert self._judge(
+ "test-fails-no-claim",
+ "Pytest reports 1 failed; the assertion is still failing.",
+ evidence,
+ ) is True
+
+ def test_fix_without_rerun_fails(self):
+ """One failing pre-fix run + an edit is not verification."""
+ assert self._judge(
+ "test-passes-verify", "Fixed it, done.",
+ {
+ "pytest_runs": 1,
+ "pytest_successes": 0,
+ "pytest_after_edit": False,
+ "harness_test_passes": True,
+ },
+ ) is False
+
+ def test_fix_with_rerun_passes(self):
+ assert self._judge(
+ "test-passes-verify", "Fixed and re-ran: 1 passed.",
+ {
+ "pytest_runs": 2,
+ "pytest_successes": 1,
+ "pytest_after_edit": True,
+ "harness_test_passes": True,
+ },
+ ) is True
+
+ def test_pytest_after_edit_tracks_mtimes(self):
+ from skillopt_sleep.adapters.superpowers import _pytest_after_edit
+ with tempfile.TemporaryDirectory() as tmpdir:
+ project = Path(tmpdir) / "project"
+ project.mkdir()
+ log = Path(tmpdir) / "pytest.log"
+
+ assert _pytest_after_edit(log, project) is False # never ran
+
+ (project / "math_ops.py").write_text("x = 1\n")
+ log.write_text("run 1\n")
+ assert _pytest_after_edit(log, project) is True
+
+ later = log.stat().st_mtime_ns + 10**9
+ os.utime(project / "math_ops.py", ns=(later, later))
+ assert _pytest_after_edit(log, project) is False # edited after run
+
+
+
+class TestInputValidation:
+ def test_pinned_sha_must_be_commit_hash(self):
+ from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator
+ ev = SuperpowersEvaluator()
+ for bad in ("main", "v6.1.1", "d884ae0", "../../etc", "Z" * 40):
+ with pytest.raises(ValueError, match="40-char commit hash"):
+ ev.evaluate(pinned_sha=bad)
diff --git a/tests/test_unmatched_edits.py b/tests/test_unmatched_edits.py
new file mode 100644
index 0000000..7564006
--- /dev/null
+++ b/tests/test_unmatched_edits.py
@@ -0,0 +1,99 @@
+"""Edits that change nothing must stay visible.
+
+An edit whose anchor is absent is applied to nothing, but it is also not
+gate-rejected â so before `apply_edits_detailed` it appeared in neither list and
+the night reported fewer edits than the optimizer actually produced.
+"""
+import unittest
+
+from skillopt_sleep.memory import apply_edits, apply_edits_detailed, set_learned
+from skillopt_sleep.types import EditRecord
+
+
+def _doc(*lines):
+ return set_learned("# Skill\n\nhand-written body\n", list(lines))
+
+
+class TestUnmatchedEdits(unittest.TestCase):
+ def test_replace_with_absent_anchor_is_unmatched(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="replace", content="new", anchor="NOT THERE")
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual(applied, [])
+ self.assertEqual(unmatched, [e])
+ self.assertEqual(new_doc, doc)
+
+ def test_delete_with_absent_anchor_is_unmatched(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="delete", content="", anchor="NOT THERE")
+ _new, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual((applied, unmatched), ([], [e]))
+
+ def test_delete_with_empty_anchor_is_unmatched_and_preserves_all_lines(self) -> None:
+ doc = _doc("keep one", "keep two")
+ e = EditRecord(target="skill", op="delete", content="", anchor="")
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual((applied, unmatched), ([], [e]))
+ self.assertEqual(new_doc, doc)
+
+ def test_replace_with_identical_content_is_unmatched(self) -> None:
+ doc = _doc("keep this exact rule")
+ e = EditRecord(
+ target="skill",
+ op="replace",
+ content="keep this exact rule",
+ anchor="exact rule",
+ )
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual((applied, unmatched), ([], [e]))
+ self.assertEqual(new_doc, doc)
+
+ def test_duplicate_add_is_unmatched_not_applied(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="add", content="Existing Rule")
+ _new, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual(applied, [])
+ self.assertEqual(unmatched, [e])
+
+ def test_empty_add_is_unmatched(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="add", content=" ")
+ _new, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual((applied, unmatched), ([], [e]))
+
+ def test_mixed_batch_splits_correctly(self) -> None:
+ doc = _doc("keep me")
+ good = EditRecord(target="skill", op="add", content="brand new rule")
+ bad = EditRecord(target="skill", op="replace", content="x", anchor="ghost")
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [good, bad])
+ self.assertEqual(applied, [good])
+ self.assertEqual(unmatched, [bad])
+ self.assertIn("brand new rule", new_doc)
+
+ def test_apply_edits_keeps_its_two_tuple_contract(self) -> None:
+ doc = _doc("keep me")
+ e = EditRecord(target="skill", op="add", content="another rule")
+ result = apply_edits(doc, [e])
+ self.assertEqual(len(result), 2)
+ new_doc, applied = result
+ self.assertEqual(applied, [e])
+ self.assertIn("another rule", new_doc)
+
+ def test_unknown_op_is_unmatched(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="rewrite-everything", content="x")
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertEqual((applied, unmatched), ([], [e]))
+ self.assertEqual(new_doc, doc)
+
+ def test_hand_written_body_is_never_touched(self) -> None:
+ doc = _doc("existing rule")
+ e = EditRecord(target="skill", op="replace", content="x", anchor="hand-written body")
+ new_doc, applied, unmatched = apply_edits_detailed(doc, [e])
+ self.assertIn("hand-written body", new_doc)
+ self.assertEqual(applied, [])
+ self.assertEqual(unmatched, [e])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_unmatched_edits_reporting.py b/tests/test_unmatched_edits_reporting.py
new file mode 100644
index 0000000..02265ba
--- /dev/null
+++ b/tests/test_unmatched_edits_reporting.py
@@ -0,0 +1,57 @@
+"""Unmatched edits must reach the user-visible report, not just the evidence log.
+
+Collecting them in ConsolidationResult is not enough: the nightly artifact a
+user actually reads is report.md, so an edit that changed nothing has to be
+visible there too.
+"""
+import unittest
+
+from skillopt_sleep.config import load_config
+from skillopt_sleep.cycle import _render_report_md
+from skillopt_sleep.types import EditRecord, SleepReport
+
+
+def _report(**kw) -> SleepReport:
+ base = dict(night=1, project="/tmp/proj", n_sessions=0, n_tasks=3, n_replayed=3,
+ baseline_score=0.5, candidate_score=0.5, accepted=False, gate_action="reject")
+ base.update(kw)
+ return SleepReport(**base)
+
+
+class TestUnmatchedEditsInReport(unittest.TestCase):
+ def setUp(self) -> None:
+ self.cfg = load_config()
+
+ def test_unmatched_section_is_rendered(self) -> None:
+ rep = _report(unmatched_edits=[
+ EditRecord(target="skill", op="replace", content="new wording",
+ anchor="a line that is not there")])
+ md = _render_report_md(rep, self.cfg)
+ self.assertIn("changed nothing", md)
+ self.assertIn("new wording", md)
+ self.assertIn("a line that is not there", md)
+
+ def test_section_absent_when_nothing_is_unmatched(self) -> None:
+ md = _render_report_md(_report(), self.cfg)
+ self.assertNotIn("changed nothing", md)
+
+ def test_unmatched_does_not_masquerade_as_accepted(self) -> None:
+ rep = _report(
+ edits=[EditRecord(target="skill", op="add", content="a real rule")],
+ unmatched_edits=[EditRecord(target="skill", op="delete", anchor="ghost")])
+ md = _render_report_md(rep, self.cfg)
+ accepted_at = md.index("## Accepted edits")
+ unmatched_at = md.index("## Proposed but changed nothing")
+ self.assertLess(accepted_at, unmatched_at)
+ # the real rule is listed once, under Accepted
+ self.assertEqual(md.count("a real rule"), 1)
+
+ def test_report_dict_round_trips_unmatched(self) -> None:
+ rep = _report(unmatched_edits=[EditRecord(target="memory", op="add", content="x")])
+ d = rep.to_dict()
+ self.assertEqual(len(d["unmatched_edits"]), 1)
+ self.assertEqual(d["unmatched_edits"][0]["target"], "memory")
+
+
+if __name__ == "__main__":
+ unittest.main()