diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json new file mode 100644 index 0000000..55a8659 --- /dev/null +++ b/.cursor-plugin/marketplace.json @@ -0,0 +1,17 @@ +{ + "name": "skillopt", + "owner": { + "name": "Yifan Yang", + "email": "yifanyang@microsoft.com" + }, + "metadata": { + "description": "Official SkillOpt plugins for usage-driven, validation-gated agent improvement." + }, + "plugins": [ + { + "name": "skillopt-sleep", + "source": "plugins/cursor", + "description": "Review recent Cursor sessions, replay recurring work, and stage validation-gated improvements to a Cursor skill for explicit adoption." + } + ] +} diff --git a/.gitignore b/.gitignore index 4a5142a..7c5950d 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ docs/让* tests/run_*.sh tests/launch_*.py *.launch.log +uv.lock +# Superpowers smoke runs: raw agent output + local paths, share sanitized excerpts instead +smoke_results/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9dadc..63b1a0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ All notable changes to SkillOpt are documented here. This project adheres to ## [Unreleased] ### Added +- A non-destructive Devin installer and SessionEnd activity marker, preserving + existing project hooks across repeated installation. +- Per-night SkillOpt-Sleep `evidence.jsonl` chains for reconstructing harvest, + mining, replay, reflection, and gate decisions, plus a live prompt-template + registry with user overrides. +- Native SkillOpt-Sleep support for Cursor, including a local plugin command + and skill, Cursor transcript harvesting, and an optional Cursor Agent CLI + backend. Cursor tool-aware replay remains disabled pending live permission- + boundary validation. +- **Cursor Agent research target harness** (`cursor_exec`) for running + supported benchmark rollouts through an installed, authenticated + `cursor-agent`, with sandboxed workspaces, structured trace capture, and + target-only optimizer separation. - **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep — runs the sleep cycle with no model subprocess or API key: the engine writes each pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the diff --git a/README.md b/README.md index f4ccbac..8836325 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ --- ## News 🔥🔥🔥 +- **[2026-07-24]** 📰 **SkillOpt in the news.** Read the official [Microsoft Research feature](https://www.microsoft.com/en-us/research/blog/skillopt-agent-skills-as-trainable-parameters/), along with recent coverage from [VentureBeat](https://venturebeat.com/orchestration/microsofts-open-source-skillopt-automatically-upgrades-ai-agent-skills-without-touching-model-weights), [Synced (机器之心)](https://mp.weixin.qq.com/s/pMlyj3a3KOh8L7cIHClRXA), [Flowtivity](https://flowtivity.ai/blog/microsoft-skillopt-train-ai-agent-skills/), and [The Decoder](https://the-decoder.com/microsofts-skillopt-boosts-gpt-5-5-by-using-nothing-but-a-trained-markdown-file/). - **[2026-07-02]** 🚀 **SkillOpt [v0.2.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) is out on [PyPI](https://pypi.org/project/skillopt/)!** Headline feature: **SkillOpt-Sleep**, a nightly offline self-evolution engine (harvest → mine → replay → consolidate behind a held-out validation gate), now shipped as the `skillopt-sleep` CLI. It also includes experimental multi-objective, replay, and dream-rollout controls; the main CLI keeps conservative defaults and does not expose every experiment-harness control as a flag. The release source adds integration shells for **Claude Code, Codex, Copilot, and Devin**, plus an **OpenClaw reference adaptation**; these plugin/MCP files live in the repository rather than the PyPI wheel. It also adds SearchQA split materialization, Windows robustness, and hardened JSON parsing. See the [release notes](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) for full release details and contributor acknowledgements. - **[2026-06-15]** 😴 **SkillOpt-Sleep (preview)** — a nightly offline self-evolution companion for local coding agents (Claude Code / Codex / Copilot): review past sessions, replay recurring tasks, and consolidate validated skills behind a held-out gate. See **[`docs/sleep/README.md`](docs/sleep/README.md)** for what it is, how to use it, and results. - **[2026-06-03]** 🎉 **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.** @@ -65,13 +66,13 @@ https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7 A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`, `qwen_chat`, `minimax_chat`, `openai_compatible`, `codex_exec`, -`claude_code_exec`). If a provider implements the OpenAI Chat Completions +`claude_code_exec`, `cursor_exec`). If a provider implements the OpenAI Chat Completions protocol, try the built-in `openai_compatible` backend before adding code. See [`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full -contract; in short you add a `skillopt/model/_backend.py` module, -register it in `skillopt/model/common.py` + `backend_config.py`, and wire -it through the router in `skillopt/model/__init__.py`. `qwen_backend.py` -and `minimax_backend.py` are good templates. +contract. Chat backends add a `skillopt/model/_backend.py` module; +target-only exec backends use the shared harness in `codex_harness.py`. +Both register through `common.py`, `backend_config.py`, and +`skillopt/model/__init__.py`. ### Adding a new benchmark diff --git a/blog/gating-reflection-safe-updates/index.html b/blog/gating-reflection-safe-updates/index.html index 96005f5..3a752d8 100644 --- a/blog/gating-reflection-safe-updates/index.html +++ b/blog/gating-reflection-safe-updates/index.html @@ -1697,27 +1697,13 @@

Citation

-

Cite this blog post

- -
-

- If you find this report useful, please cite it as: -

-
@misc{zhou2026skillopttechnicalreport,
-  title        = {Expanded SkillOpt Ablations, Skill-Aware Reflection, and SkillOpt-Sleep},
-  author       = {Zhou, Ziwei and Gong, Ziyang and Yang, Yifan},
-  year         = {2026},
-  url          = {https://microsoft.github.io/SkillOpt/blog/gating-reflection-safe-updates/},
-  note         = {SkillOpt Technical Blog post}
-}
- -
-

SkillOpt paper

+

Cite SkillOpt

- For the underlying SkillOpt method, please also cite the - SkillOpt paper: + If you find this report useful, please cite the main SkillOpt paper using the + citation maintained in the + SkillOpt repository:

@article{yang2026skillopt,
   title={Skillopt: Executive strategy for self-evolving agent skills},
diff --git a/configs/_base_/default.yaml b/configs/_base_/default.yaml
index fcfdd71..c8c068d 100644
--- a/configs/_base_/default.yaml
+++ b/configs/_base_/default.yaml
@@ -24,6 +24,8 @@ model:
   claude_code_exec_use_sdk: auto
   claude_code_exec_effort: medium
   claude_code_exec_max_thinking_tokens: 16384
+  cursor_exec_path: ""       # blank uses CURSOR_EXEC_PATH or cursor-agent
+  cursor_exec_sandbox: ""    # blank uses CURSOR_EXEC_SANDBOX or enabled
   codex_trace_to_optimizer: true
   azure_openai_endpoint: ""           # e.g. "https://your-resource.openai.azure.com/"
   azure_openai_api_version: "2024-12-01-preview"
diff --git a/data/README.md b/data/README.md
index a31c337..73e5744 100644
--- a/data/README.md
+++ b/data/README.md
@@ -29,7 +29,7 @@ Each `items.json` contains only stable IDs or source-path hints.
 | Manifest directory | Benchmark | Counts | Coverage | Raw data source | `split_dir` |
 |---|---|---:|---|---|---|
 | `searchqa_id_split/` | SearchQA | 400 / 200 / 1400 | Official HF dataset IDs | [lucadiliello/searchqa](https://huggingface.co/datasets/lucadiliello/searchqa) | `data/searchqa_split` |
-| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 18 / 124 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` |
+| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 17 / 125 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` |
 | `docvqa_id_split/` | DocVQA | 107 / 53 / 374 | 10% subset of validation | [lmms-lab/DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA) | `data/docvqa/splits` |
 | `officeqa_id_split/` | OfficeQA | 50 / 24 / 172 | OfficeQA Full | [databricks/officeqa](https://huggingface.co/datasets/databricks/officeqa) | `data/officeqa_split` |
 | `spreadsheetbench_id_split/` | SpreadsheetBench | 80 / 40 / 280 | SpreadsheetBench Verified 400 | [KAKA22/SpreadsheetBench](https://huggingface.co/datasets/KAKA22/SpreadsheetBench) | `data/spreadsheetbench_split` |
diff --git a/data/livemathematicianbench_id_split/split_manifest.json b/data/livemathematicianbench_id_split/split_manifest.json
index 9af68a0..75c6bea 100644
--- a/data/livemathematicianbench_id_split/split_manifest.json
+++ b/data/livemathematicianbench_id_split/split_manifest.json
@@ -16,8 +16,8 @@
   "split_seed": 42,
   "counts": {
     "train": 35,
-    "val": 18,
-    "test": 124
+    "val": 17,
+    "test": 125
   },
   "item_fields": [
     "id",
diff --git a/data/livemathematicianbench_id_split/test/items.json b/data/livemathematicianbench_id_split/test/items.json
index 83006f3..575080d 100644
--- a/data/livemathematicianbench_id_split/test/items.json
+++ b/data/livemathematicianbench_id_split/test/items.json
@@ -866,5 +866,12 @@
     "no": 37,
     "paper_link": "http://arxiv.org/abs/2602.08644v1",
     "source_file": "data/202602/qa_202602_final.json"
+  },
+  {
+    "id": "202512:46",
+    "month": "202512",
+    "no": 46,
+    "paper_link": "http://arxiv.org/abs/2512.05945v1",
+    "source_file": "data/202512/qa_202512_final.json"
   }
-]
+]
\ No newline at end of file
diff --git a/data/livemathematicianbench_id_split/val/items.json b/data/livemathematicianbench_id_split/val/items.json
index 4298592..4e63fea 100644
--- a/data/livemathematicianbench_id_split/val/items.json
+++ b/data/livemathematicianbench_id_split/val/items.json
@@ -117,12 +117,5 @@
     "no": 6,
     "paper_link": "http://arxiv.org/abs/2602.01571v1",
     "source_file": "data/202602/qa_202602_final.json"
-  },
-  {
-    "id": "202512:46",
-    "month": "202512",
-    "no": 46,
-    "paper_link": "http://arxiv.org/abs/2512.05945v1",
-    "source_file": "data/202512/qa_202512_final.json"
   }
-]
+]
\ No newline at end of file
diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md
index 8600c44..2a72a64 100644
--- a/docs/guide/configuration.md
+++ b/docs/guide/configuration.md
@@ -45,6 +45,7 @@ model:
 | `minimax_chat` | ✓ | ✓ | MiniMax API |
 | `codex_exec` | — | ✓ | Codex CLI execution harness |
 | `claude_code_exec` | — | ✓ | Claude Code CLI execution harness |
+| `cursor_exec` | — | ✓ | Cursor Agent CLI execution harness |
 
 The current MiniMax adapter has one shared deployment. Set
 `model.minimax_model` when MiniMax is the target; a mixed-backend run cannot
@@ -183,6 +184,9 @@ Model credentials are loaded from environment variables:
 | `OPENAI_COMPATIBLE_MODEL` | `openai_compatible` | Shared provider model ID for direct library use; train/eval YAML role models take precedence |
 | `CLAUDE_CLI_BIN` | `claude_chat` | Optional path to the `claude` executable; defaults to `claude` |
 | `ANTHROPIC_API_KEY` | `claude_chat` | Optional authentication method understood by the Claude CLI, not a direct SkillOpt API client |
+| `CURSOR_EXEC_PATH` | `cursor_exec` | Optional path to `cursor-agent`; defaults to `cursor-agent` |
+| `CURSOR_EXEC_SANDBOX` | `cursor_exec` | Cursor sandbox mode: `enabled` (default) or `disabled` |
+| `CURSOR_API_KEY` | `cursor_exec` | Optional authentication method understood directly by Cursor Agent |
 | `QWEN_CHAT_BASE_URL` | `qwen_chat` | Local Qwen/vLLM endpoint |
 | `QWEN_CHAT_MODEL` | `qwen_chat` | Served model name for direct library use; train/eval YAML role models take precedence |
 | `MINIMAX_BASE_URL` | `minimax_chat` | MiniMax-compatible base URL |
@@ -197,6 +201,14 @@ and authenticate that CLI before use. Setting `ANTHROPIC_API_KEY` is one way
 the CLI may authenticate, but SkillOpt does not call the Anthropic API
 directly through this backend.
 
+`cursor_exec` is a target-only benchmark harness. Install and authenticate
+Cursor Agent first, then select it with `model.target_backend=cursor_exec`.
+Read-only rollouts use Ask mode; artifact-producing rollouts add Cursor's
+headless `--force` flag inside the benchmark workspace. SkillOpt enables the
+Cursor sandbox by default and rejects file-edit rollouts if it is disabled;
+read-only Ask-mode rollouts may explicitly disable it. SkillOpt does not approve
+MCP servers automatically.
+
 ### Three OpenAI-compatible paths
 
 - Research, generic provider: select `openai_compatible` and use
diff --git a/docs/guide/installation.md b/docs/guide/installation.md
index 1b24a2d..e35d555 100644
--- a/docs/guide/installation.md
+++ b/docs/guide/installation.md
@@ -26,9 +26,10 @@ checkout for those files.
 !!! important "PyPI versus `main`"
     These docs track the latest `main`. The current PyPI release is `0.2.0`.
     The generic research `openai_compatible` backend, SkillOpt-Sleep handoff,
-    Sleep support for non-Azure OpenAI-compatible endpoints, and the Sleep
-    `--preferences` flag landed after that release and require a source install
-    from `main` until the next release.
+    Sleep support for non-Azure OpenAI-compatible endpoints, the Sleep
+    `--preferences` flag, and Cursor source/backend/plugin support landed after
+    that release and require a source install from `main` until the next
+    release.
 
 ### Source checkout
 
@@ -128,6 +129,13 @@ Anthropic API client. Install and authenticate `claude`, and set
 `CLAUDE_CLI_BIN` only if the executable is not available as `claude` on
 `PATH`. `ANTHROPIC_API_KEY` is one authentication option the CLI may consume.
 
+The SkillOpt-Sleep `cursor` backend similarly requires a separately installed
+and authenticated `cursor-agent`; harvesting with `--source cursor` alone does
+not. Set `SKILLOPT_SLEEP_CURSOR_PATH` when the executable is not on `PATH`, and
+`SKILLOPT_SLEEP_CURSOR_MODEL` to override its model. Cursor plugin installation
+and the explicit project skill target are documented in the
+[Cursor integration guide](https://github.com/microsoft/SkillOpt/blob/main/plugins/cursor/README.md).
+
 OpenAI-compatible servers have three distinct entry points:
 
 1. The research engine's generic `openai_compatible` backend uses
diff --git a/docs/guide/new-backend.md b/docs/guide/new-backend.md
index 6785527..98d598f 100644
--- a/docs/guide/new-backend.md
+++ b/docs/guide/new-backend.md
@@ -146,9 +146,9 @@ Provider-specific configuration helpers and `count_tokens()` are optional, but
 their state must be safe to update while calls may run concurrently. Keep
 credentials out of logs and persisted artifacts.
 
-Exec-style targets do not implement this chat contract. They are target-only
-and are integrated through `codex_harness.py` plus environment-specific rollout
-code.
+Exec-style targets such as `claude_code_exec` and `cursor_exec` do not
+implement this chat contract. They are target-only and are integrated through
+`codex_harness.py` plus environment-specific rollout code.
 
 ## Step 2: register and route the backend
 
diff --git a/docs/guideline.html b/docs/guideline.html
index 3bd625d..801ccaf 100644
--- a/docs/guideline.html
+++ b/docs/guideline.html
@@ -295,9 +295,10 @@ python -m pip install -e ".[dev]"       # tests and linting
Release boundary This guide tracks main. PyPI currently serves 0.2.0; the generic research openai_compatible backend, Sleep handoff, - SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, and the - Sleep --preferences flag require a source install from - main until the next release. + SkillOpt-Sleep support for non-Azure OpenAI-compatible endpoints, the + Sleep --preferences flag, and Cursor source/backend/plugin + support require a source install from main until the next + release.

See the installation guide for platform notes and dependency boundaries.

@@ -396,6 +397,7 @@ python scripts/train.py --config configs/searchqa/default.yaml minimax_chatYesYesMiniMax chat endpoint. codex_execYesSupported adapters onlyExecutes Codex for optimizer calls and as a target agent where supported. claude_code_execNoSupported adapters onlyExecutes Claude Code as a target agent. + cursor_execNoSupported adapters onlyExecutes Cursor Agent as a sandboxed target agent where supported. @@ -450,8 +452,25 @@ skillopt-sleep adopt --project "$PWD"

--project scopes collection but does not automatically choose a project's skill file. Use --target-skill-path when you intend to evolve a particular SKILL.md. Transcript source - (claude, codex, or auto) and replay - backend are independent settings.

+ (claude, codex, cursor, or + auto) and replay backend are independent settings. The existing + auto precedence remains Codex then Claude; select Cursor + explicitly with --source cursor.

+

Cursor transcripts default to + ~/.cursor/projects/<workspace>/agent-transcripts; override + that home with --cursor-home. Model-driven replay requires an + installed, authenticated cursor-agent and + --backend cursor; use --cursor-path when the CLI is + not on PATH. Target + .cursor/skills/skillopt-sleep-learned/SKILL.md explicitly so + adoption updates a project skill rather than the plugin's workflow skill.

+

The Cursor backend inserts that skill text into prompts; it does not + invoke the file as a native skill. Ordinary calls run in read-only Ask mode + in an empty temporary workspace and cannot inspect files under + --project. Cursor tasks containing a tool_called + check fail before Agent mode starts; use another backend for those tasks. + This validates textual guidance, not end-to-end repository, browser, + service, or filesystem workflows.

For subscription-based workflows that should not launch an API or model subprocess, use --backend handoff and follow the generated prompt/answer loop. Read the @@ -467,6 +486,7 @@ skillopt-sleep adopt --project "$PWD" Claude CodeShared-engine plugin and handoff commandREADME CodexShared-engine skill shellREADME + CursorNative command and skill, local transcript source, and Cursor Agent backendREADME GitHub CopilotShared-engine Sleep MCP plus a separate research MCPREADME DevinShared-engine MCP with Devin transcript conversionREADME OpenClawIndependent community/reference adaptation; review locally before useREADME @@ -490,6 +510,7 @@ skillopt-sleep adopt --project "$PWD" dream_rollouts1Single rollout by default; values above 1 enable experimental contrastive replay. dream_factor0Synthetic task variants are off by default. recall_k0Historical associative recall is off by default. + replay_modemockReporting label for prompt replay; fresh-worktree replay is not implemented. @@ -498,6 +519,13 @@ skillopt-sleep adopt --project "$PWD" reward/budget controls as advanced features that require task-specific validation. The reported experiments and their exact settings are in RESULTS.md.

+

The managed scheduler persists only project, backend, time, and optional + auto-adopt. For Cursor schedules, put transcript_source, + cursor_home, cursor_path, model, and + target_skill_path in + ~/.skillopt-sleep/config.json. Use an absolute Cursor CLI path + and verify authentication for the scheduled account because schedulers may + run with a minimal environment.

@@ -510,6 +538,13 @@ skillopt-sleep adopt --project "$PWD" is not a guarantee that every outbound model prompt is free of sensitive content. In particular, do not treat raw coding-agent transcripts as pre-sanitized. +
  • The Cursor source excludes tool arguments and outputs, retaining only + user/assistant text, explicit turn errors, and tool names. The Cursor + backend still sends transcript-derived prompts through + cursor-agent to Cursor's selected model provider.
  • +
  • A real-backend dry-run still performs provider calls; it + suppresses staging rather than spend. Session and task limits are not hard + provider-call, token, time, or monetary budgets.
  • Updates are staged for review by default. Use --auto-adopt only when you have an independent rollback and validation process.
  • diff --git a/docs/reference/api.md b/docs/reference/api.md index 287a717..34176d7 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -192,6 +192,7 @@ not via a base class subclass. Supported values (as of this writing): | `openai_compatible` | ✓ | ✓ | | `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | +| `cursor_exec` | — | ✓ | See `skillopt/model/backend_config.py` for the live whitelist and [`docs/reference/config.md`](./config.md) for the per-backend diff --git a/docs/reference/cli.md b/docs/reference/cli.md index e4d277a..69fa943 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -2,9 +2,10 @@ > **Version note.** This reference tracks `main`. PyPI 0.2.0 does not yet > include the generic research `openai_compatible` backend, Sleep handoff, -> Sleep support for non-Azure OpenAI-compatible endpoints, or the Sleep -> `--preferences` flag; use a source install from `main` for those features -> until the next release. +> Sleep support for non-Azure OpenAI-compatible endpoints, the Sleep +> `--preferences` flag, the research `cursor_exec` target harness, or Cursor +> source/backend/plugin support; use a source install from `main` for those +> features until the next release. ## Training @@ -91,6 +92,27 @@ python scripts/train.py \ model.target=deepseek-chat ``` +To benchmark an installed, authenticated Cursor Agent through an environment +that supports exec targets: + +```bash +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill skills/my_skill.md \ + --cfg-options \ + model.optimizer_backend=openai_chat \ + model.target_backend=cursor_exec \ + model.target=composer-2.5 +``` + +`cursor_exec` runs the target only; the optimizer remains separately +configured. Read-only rollouts use Cursor Ask mode. Rollouts that request file +edits use `--force` inside the benchmark workspace, with Cursor sandboxing +enabled. The harness refuses file-edit rollouts when the Cursor sandbox is +disabled. Read-only Ask-mode rollouts may explicitly disable it. Override the +executable or sandbox through `model.cursor_exec_path` and +`model.cursor_exec_sandbox`. + ## SkillOpt-Sleep ```bash @@ -104,11 +126,13 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and | Argument | Description | |---|---| -| `--project PATH` | Project to evolve (default: current directory) | +| `--project PATH` | Project used for transcript scope, targets, state, and staging (default: current directory) | | `--scope invoked\|all` | Harvest this project or all projects | -| `--source claude\|codex\|auto` | Transcript source | -| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | Replay/optimizer backend | +| `--source claude\|codex\|cursor\|auto` | Transcript source; `auto` keeps Codex-then-Claude precedence and does not select Cursor | +| `--backend mock\|claude\|codex\|copilot\|cursor\|handoff\|azure_openai` | Replay/optimizer backend | | `--model NAME` | Backend-specific model override | +| `--cursor-home PATH` | Override `~/.cursor` for Cursor transcript harvesting | +| `--cursor-path PATH` | Path to the installed Cursor Agent CLI | | `--preferences TEXT` | House rules supplied to reflection | | `--lookback-hours N` | Initial transcript lookback; `0` scans all history | | `--max-sessions N` / `--max-tasks N` | Bound the harvested workload | @@ -118,6 +142,82 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and | `--progress` / `--json` | Progress or machine-readable output | | `--auto-adopt` | Apply an accepted staged proposal automatically | +### Cursor source and backend + +`--source cursor` reads local Cursor JSONL transcripts from +`~/.cursor/projects//agent-transcripts/*/*.jsonl`. Invoked scope uses +Cursor's recorded workspace path, including when `--project` is a nested +directory, and falls back to the sanitized storage name when metadata is not +available. `--scope all` scans every workspace below `cursor_home`. The +harvester retains user/assistant text, explicit turn errors, and tool names, +while excluding tool arguments, tool outputs, and non-message records. It +redacts known secret patterns and filters SkillOpt-generated replay sessions, +but redaction is not a guarantee that outbound prompts contain no sensitive +data. + +`--backend cursor` launches an installed, authenticated `cursor-agent`, sends +prompts over stdin, and parses its JSON result. SkillOpt reads the target skill +and includes its text in replay prompts; it does not invoke that file as a native +Cursor skill. Ordinary mining, replay, judging, and reflection calls use +read-only Ask mode in a new empty temporary workspace. Project file reads, file +writes, and MCP tools are denied. `--project` does not change that execution +workspace. + +Cursor tool-aware replay is temporarily disabled pending live Cursor +permission-boundary validation. A task with a `tool_called` check fails nonzero +before Agent mode starts and does not stage, adopt, cache, persist state, or +advance the harvest checkpoint. Use another backend for such tasks. The current +Cursor backend therefore does not provide end-to-end validation for skills that +need repository inspection, real CLIs, browsers, running services, or file +changes. + +There is no implemented fresh-worktree Cursor replay. If a report says +`replay: mock`, that is the prompt-replay label and does not mean the mock model +backend was selected. Both `run` and `dry-run` perform real-backend provider +calls; `dry-run` suppresses staging, adoption, and persisted state changes, not +spend. Session and task limits do not impose hard provider-call, token, time, or +monetary budgets. +Cursor and its selected model provider can receive the prompt content. + +Cursor-specific settings are available through the CLI, config, and environment: + +| Purpose | CLI | `~/.skillopt-sleep/config.json` | Environment | +|---|---|---|---| +| Transcript home | `--cursor-home PATH` | `"cursor_home": "/path/to/.cursor"` | none | +| Agent executable | `--cursor-path PATH` | `"cursor_path": "/path/to/cursor-agent"` | `SKILLOPT_SLEEP_CURSOR_PATH` | +| Model | `--model NAME` | `"model": "NAME"` | `SKILLOPT_SLEEP_CURSOR_MODEL` | + +Use `cursor-agent --list-models` to inspect model identifiers available to the +authenticated account. When cost depends on a model variant, confirm the billed +variant in Cursor's usage reporting rather than relying only on its display +name. + +Target the learned project skill explicitly so accepted updates are visible to +Cursor without modifying the plugin's own `skillopt-sleep` workflow skill: + +```bash +skillopt-sleep run --project "$(pwd)" \ + --source cursor --backend cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +``` + +The first harvest uses a 72-hour lookback unless `--lookback-hours` is set. A +value of `0` considers all available history while still respecting +`--max-sessions`. A stateful `run`, including a run that mines no tasks, records +a new harvest checkpoint; subsequent runs use that checkpoint rather than the +initial lookback. Use `harvest` or `dry-run` to verify counts before the first +stateful run. + +The managed `schedule` command persists the project, backend, time, and optional +auto-adopt setting only. It does not copy source, Cursor paths, model, or target +skill flags into the scheduled command. Put `transcript_source`, `cursor_home`, +`cursor_path`, `model`, and `target_skill_path` in the user config before +scheduling Cursor. Keep `target_skill_path` project-relative as +`.cursor/skills/skillopt-sleep-learned/SKILL.md`, prefer an absolute +`cursor_path`, and verify authentication for the scheduled account because cron +and Task Scheduler may have a minimal environment. + Backend-specific setup for compatible endpoints is documented in [OpenAI-compatible endpoints for SkillOpt-Sleep](../sleep/openai-compatible-endpoints.md). diff --git a/docs/reference/config.md b/docs/reference/config.md index b65c204..b8243c7 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -18,6 +18,7 @@ selecting the generic OpenAI-compatible backend. | `minimax_chat` | ✓ | ✓ | | `codex_exec` | ✓ | ✓ | | `claude_code_exec` | — | ✓ | +| `cursor_exec` | — | ✓ | MiniMax currently has one shared deployment. `model.minimax_model` is applied when MiniMax is the target; mixed-backend runs cannot independently choose a @@ -64,6 +65,8 @@ defaults to `claude` and can be overridden with `CLAUDE_CLI_BIN`. | `model.minimax_*` | MiniMax `base_url`, `api_key`, shared `minimax_model`, `temperature`, `max_tokens`, and `enable_thinking`; `minimax_model` applies when MiniMax is the target | | `model.codex_exec_*` | Codex path, sandbox, profile, SDK mode, reasoning, network/search, and approval policy | | `model.claude_code_exec_*` | Claude path, profile, SDK mode, effort, and thinking-token cap | +| `model.cursor_exec_path` | Cursor Agent executable path; default `cursor-agent` | +| `model.cursor_exec_sandbox` | Cursor sandbox mode: `enabled` (default) or `disabled`; file-edit rollouts require `enabled` | ## Training (`train`) diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 2e36eb1..2b740ac 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -17,7 +17,7 @@ normal agent requests. One "night": ``` -harvest Claude Code / Codex transcripts → mine recurring tasks → replay offline +harvest Claude Code / Codex / Cursor transcripts → mine recurring tasks → replay via the configured backend (isolation varies by backend; mock/handoff make no network calls) → consolidate (reflect → bounded edit → GATE on real held-out tasks) → stage proposal → (you) adopt ``` @@ -33,6 +33,24 @@ experience → long-term competence). > review your transcript source and provider policy before running on sensitive > projects. For a reviewable workflow, harvest to a task file, inspect/redact it, mark > it `"reviewed": true`, and then replay that file with the real backend. +> +> The Cursor source reads local user/assistant message text, explicit turn errors, +> and tool names, but excludes tool arguments, tool outputs, and non-message records. +> Known secret-shaped strings are redacted as defense in depth. The Cursor backend +> sends prompts through `cursor-agent`; ordinary calls use read-only Ask mode in an +> empty temporary workspace with project files denied. Cursor tool-aware replay is +> temporarily disabled pending live permission-boundary validation. +> Cursor and the model provider selected by Cursor may therefore receive +> transcript-derived content. +> +> By default, each stateful night also writes a local `evidence.jsonl` under +> the project staging tree (beside the report when one is staged); dry-runs +> write evidence under the configured Sleep state directory. The log contains +> best-effort-redacted, per-field-truncated copies +> of miner, replay, judge, and reflection prompts and replies. Treat it as +> sensitive local data and apply an appropriate retention policy. Set +> `"evidence_log": false` to disable it; setting `"redact_secrets": false` +> deliberately disables this defense-in-depth redaction. ## How to use it @@ -48,13 +66,14 @@ skillopt-sleep schedule # install a nightly cron entry for this project ``` > **Version note.** This page tracks `main`. PyPI 0.2.0 provides the base -> commands above. Sleep handoff, non-Azure OpenAI-compatible endpoints, and -> `--preferences` landed later and require a source install from `main` until -> the next release. +> commands above. Cursor source/backend/plugin support, Sleep handoff, non-Azure +> OpenAI-compatible endpoints, and `--preferences` landed later and require a +> source install from `main` until the next release. The per-agent integrations below still come from the repo; the CLI above is the -standalone, pip-only way to run a cycle. Claude Code, Codex, Copilot, and Devin wrap -the shared engine. OpenClaw is a separate reference adaptation and has its own setup. +standalone, pip-only way to run a cycle. Claude Code, Codex, Cursor, Copilot, and +Devin wrap the shared engine. OpenClaw is a separate reference adaptation and has +its own setup. One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft/SkillOpt/tree/main/plugins)): @@ -62,10 +81,62 @@ One engine, thin per-agent shells (see [`plugins/`](https://github.com/microsoft |---|---|---| | **Claude Code** | [`plugins/claude-code`](https://github.com/microsoft/SkillOpt/tree/main/plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/skillopt-sleep` | | **Codex** | [`plugins/codex`](https://github.com/microsoft/SkillOpt/tree/main/plugins/codex) | `bash plugins/codex/install.sh` → `skillopt-sleep` skill | +| **Cursor** | [`plugins/cursor`](https://github.com/microsoft/SkillOpt/tree/main/plugins/cursor) | `bash plugins/cursor/install.sh` → `/skillopt-sleep` | | **Copilot** | [`plugins/copilot`](https://github.com/microsoft/SkillOpt/tree/main/plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server | | **Devin** | [`plugins/devin`](https://github.com/microsoft/SkillOpt/tree/main/plugins/devin) | register `plugins/devin/mcp_server.py` as an MCP server | | **OpenClaw** | [`plugins/openclaw`](https://github.com/microsoft/SkillOpt/tree/main/plugins/openclaw) | adapt the reference wrapper and paths for your installation | +### Cursor + +Cursor transcript harvesting and model execution are independent. Use +`--source cursor` to read +`~/.cursor/projects//agent-transcripts/*/*.jsonl`; `--scope invoked` +uses Cursor's recorded workspace path, with the sanitized storage directory as +a fallback, while `--scope all` scans every Cursor workspace. Use +`--cursor-home` for a different Cursor home. `--source auto` keeps its existing +Codex-then-Claude precedence and does not select Cursor. + +`--backend cursor` requires an installed, authenticated `cursor-agent`. If it is +not on `PATH`, select it with `--cursor-path`, `SKILLOPT_SLEEP_CURSOR_PATH`, or +the `cursor_path` config key. Select a model with `--model` or +`SKILLOPT_SLEEP_CURSOR_MODEL`. Point adoption at a project Cursor skill rather +than at the plugin's workflow skill: + +```bash +skillopt-sleep run --project "$(pwd)" \ + --source cursor --backend cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +``` + +The target skill is supplied to Cursor as prompt text; it is not invoked as a +native skill. `--project` selects transcript scope, target files, state, and +staging, but ordinary Cursor calls cannot inspect that project's files. The +current backend therefore evaluates textual guidance rather than end-to-end +repository, CLI, browser, or service workflows. + +Cursor tool-aware replay is temporarily disabled pending live Cursor +permission-boundary validation. If a task contains a `tool_called` check, the +Cursor backend exits nonzero before starting Agent mode and does not stage, +adopt, or advance state. Use another backend for those tasks. + +The initial harvest window is 72 hours. Set `--lookback-hours N` explicitly when +older sessions should be considered; `0` scans all history subject to the +session limit. A stateful `run`, even with no mined tasks, advances the harvest +checkpoint. Use `harvest` or `dry-run` to inspect counts first. A real-backend +`dry-run` still incurs provider calls and spend, and session/task limits are not +hard call, token, time, or monetary budgets. + +The managed scheduler records only the project, backend, time, and optional +auto-adopt setting. It does not preserve Cursor source, home, CLI path, model, or +target-skill flags. Before `skillopt-sleep schedule --backend cursor`, put +`transcript_source`, `cursor_home`, `cursor_path`, `model`, and +`target_skill_path` in `~/.skillopt-sleep/config.json`. The target may remain +project-relative as `.cursor/skills/skillopt-sleep-learned/SKILL.md`. Use an +absolute `cursor_path` and verify that the scheduled account is already +authenticated, because cron and Task Scheduler may run with a minimal +environment. + To use DeepSeek, vLLM, Ollama, or another Chat Completions server, see **[OpenAI-compatible endpoints](openai-compatible-endpoints.md)**. That guide also documents the separate HTTPS-only boundary for Azure managed-identity credentials. diff --git a/docs/superpowers/SECURITY.md b/docs/superpowers/SECURITY.md new file mode 100644 index 0000000..a946d72 --- /dev/null +++ b/docs/superpowers/SECURITY.md @@ -0,0 +1,82 @@ +# Security Considerations for Superpowers Adapter + +## Scope: trusted candidates only + +This adapter evaluates **trusted, locally-authored** candidate skills. It is +explicitly **not** hardened against a hostile candidate, and must not be pointed +at model-generated or third-party skills. + +There is no OS-level boundary. The evaluated agent gets `Bash`, `Read`, `Write` +and `Edit`, and runs as the same OS user as the harness, so it can: + +- Execute arbitrary shell commands +- Read/write any file that user can reach, including the harness's own evidence +- Read the environment passed to the process +- Make unrestricted network requests + +`--allowedTools` scopes which tools the agent may call. It is **not** an +isolation boundary. + +## What the adapter does do + +1. **No host credential reuse by default.** The scenario `HOME` is empty; host + `~/.claude/credentials.json` and `settings.json` are never copied or + symlinked. Reuse is opt-in via `SKILLOPT_HOST_AUTH=1`, which warns. +2. **Fail closed.** With neither `ANTHROPIC_API_KEY` nor `SKILLOPT_HOST_AUTH=1`, + the scenario errors (`NO_AUTH`) instead of running unauthenticated. +3. **Scrubbed environment.** Only `HOME`, `PATH`, `TERM`, `LANG` and (if set) + `ANTHROPIC_API_KEY` are passed; the host environment is not inherited. `PATH` + is minimal by default (shim dir + `/usr/bin:/bin`); opt in to the host `PATH` + with `SKILLOPT_INHERIT_PATH=1`. Hygiene, not a boundary — a `Bash`-holding + agent can still call absolute paths. +4. **Isolated project and HOME** per scenario, inside a temp workspace. +5. **Execution evidence.** All of it is tamper-**evident**, not tamper-proof — + the agent can reach the shim, the nonce and the audit log, and can modify the + project tree the harness re-runs from. It is meaningful because the candidate + is trusted; it is not an adversarial oracle. + - `harness_test_passes` — the harness re-runs the protected test paths after + the agent exits, ignoring project pytest config and `conftest.py`, so agent + *output* alone cannot fake a pass. A pass requires at least one executed + passing test and no failures, errors or skips. + - `pytest_runs` — count of nonce-tagged invocations of the `pytest`/`python` + shims. + - `pytest_successes` / `pytest_failures` — completed shim invocations are + classified from JUnit results; exit code 0 without an executed passing test + (for example `pytest --help` or an all-skipped run) is inconclusive and is + counted as neither a success nor a failure. + - `pytest_after_edit` — the last shim invocation is newer than the newest + project `*.py`, so "fix, then claim done without re-running" fails. + - Protected scenario fixtures are hashed before the agent runs. A modified, + deleted or symlink-replaced fixture fails the scenario and is not executed + by the harness re-run. + + ⚠️ The verification re-run **executes agent-modified project code on the + host**. `ANTHROPIC_API_KEY` is dropped from that re-run's environment, but + nothing else confines it. + +## Known Limitations + +- **API key exposure**: `ANTHROPIC_API_KEY`, if set, is visible to the agent + process. Use a scoped/disposable key. +- **`SKILLOPT_HOST_AUTH=1` exposes host credentials** to the candidate. +- **`SKILLOPT_UNSAFE=1`** disables permission checks entirely. +- **No network isolation**, for either the agent or the verification re-run. + +## Usage + +```bash +ANTHROPIC_API_KEY=... python -m skillopt_sleep.adapters.superpowers --candidate my_skill.md +``` + +## Follow-up Work + +Supporting untrusted candidates is deliberately out of scope for this adapter as +shipped. It would need, at minimum: + +- [ ] Verification oracle and evidence (nonce, shim, audit log) held outside + every agent-writable mount +- [ ] Harness re-run from an immutable copy of the test inputs +- [ ] A validated OS-level sandbox: published image with Claude Code + pytest, + exercised in CI, fail-closed on an unrecognised mode +- [ ] Network egress allowlist (api.anthropic.com only) +- [ ] Per-run scoped API keys diff --git a/plugins/README.md b/plugins/README.md index 5123022..61d222c 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -10,13 +10,14 @@ runtime dependency on the paper's `skillopt/` experiment package. ## Available integrations -Four integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate +Five integrations wrap the shared `skillopt_sleep` CLI. OpenClaw is a separate reference adaptation with its own backend and setup assumptions. | Platform | Folder | Mechanism | Status | |---|---|---|---| | **Claude Code** | [`claude-code/`](claude-code) | marketplace plugin, commands, skill, and hooks | installable shared-engine integration | | **Codex** | [`codex/`](codex) | user-level skill and shared runner | installable shared-engine integration | +| **Cursor** | [`cursor/`](cursor) | native command and skill, project skill target, and shared runner | installable shared-engine integration | | **GitHub Copilot** | [`copilot/`](copilot) | MCP server exposing seven `sleep_*` tools | shared-engine MCP integration | | **Devin** | [`devin/`](devin) | MCP server plus Devin transcript conversion | shared-engine MCP integration | | **OpenClaw** | [`openclaw/`](openclaw) | custom DeepSeek/Ollama wrapper | independent reference adaptation; review and adapt before use | @@ -30,6 +31,7 @@ for your workflow. |---|---|---| | **Claude Code** | from the repository root, `/plugin marketplace add ./plugins/claude-code`, then `/plugin install skillopt-sleep@skillopt-sleep` | `/skillopt-sleep status` | | **Codex** | `bash plugins/codex/install.sh` | ask Codex to use the `skillopt-sleep` skill | +| **Cursor** | `bash plugins/cursor/install.sh` (macOS/Linux) or `powershell -File plugins/cursor/install.ps1` (Windows) | `/skillopt-sleep status` | | **Copilot** | register `plugins/copilot/mcp_server.py` using its example MCP config | ask Copilot to run `sleep_status` | | **Devin** | register `plugins/devin/mcp_server.py` using its example MCP config | ask Devin to run `sleep_status` | | **OpenClaw** | follow and adapt [`openclaw/README.md`](openclaw/README.md) | validate paths, credentials, and tasks locally | @@ -44,9 +46,9 @@ an importable `skillopt_sleep` module. Install with `uv tool install skillopt` o `pip install skillopt` when using that fallback. > **Version note.** This integration reference tracks `main`. PyPI 0.2.0 -> supports the base Sleep CLI, while handoff, Sleep support for non-Azure -> OpenAI-compatible endpoints, and `--preferences` require a source checkout -> from `main` until the next release. +> supports the base Sleep CLI, while Cursor source/backend/plugin support, +> handoff, Sleep support for non-Azure OpenAI-compatible endpoints, and +> `--preferences` require a source checkout from `main` until the next release. ## One sleep cycle @@ -66,6 +68,17 @@ optimization. data path and no API spend. - A real backend sends truncated transcript excerpts and derived task content to the provider selected for mining, replay, judging, and reflection. +- The Cursor source reads local user/assistant message text, explicit turn + errors, and tool names from `~/.cursor/projects/*/agent-transcripts`; it does + not retain tool arguments, tool outputs, or other record types. Known + secret-shaped strings are redacted, but this is defense in depth rather than + a guarantee that outbound prompts are secret-free. +- The Cursor backend sends prompts through the installed, authenticated + `cursor-agent` CLI. Ordinary calls use read-only Ask mode in a new empty + temporary workspace with project file access denied. Cursor tasks containing + `tool_called` validation fail before Agent mode starts; use another backend for + those tasks. Cursor and the model provider selected by Cursor can receive the + resulting prompt content. - Outbound prompts are not currently guaranteed to be free of secrets. Do not use a third-party provider on sensitive transcripts without reviewing the data source and the provider's retention policy. @@ -101,9 +114,11 @@ Common implemented flags include: | Flag | Default | Purpose | |---|---|---| -| `--backend mock\|claude\|codex\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls | +| `--backend mock\|claude\|codex\|cursor\|copilot\|handoff\|azure_openai` | `mock` | select who performs model calls | | `--model NAME` | backend default | select a backend-specific model | -| `--source claude\|codex\|auto` | `claude` | select the transcript source | +| `--source claude\|codex\|cursor\|auto` | `claude` | select the transcript source; `auto` retains Codex-then-Claude precedence and does not select Cursor | +| `--cursor-home PATH` | `~/.cursor` | override the Cursor transcript home | +| `--cursor-path PATH` | auto-detect `cursor-agent` | select the Cursor Agent CLI executable | | `--project PATH` | current directory | select the project and invoked harvest scope | | `--scope invoked\|all` | `invoked` | limit transcript harvesting | | `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt | @@ -119,6 +134,14 @@ The nightly CLI does **not** currently expose `--gate`, `--rollouts-k`, `--optimizer-model`, `--target-model`, `--budget-tokens`, or `--budget-minutes`. Do not pass experiment-harness flags to the main CLI. +For the Cursor backend, `--project` also selects target files, state, and the +staging location, but it does not make that directory the Cursor Agent execution +workspace. The target skill is inserted as prompt text rather than invoked as a +native skill. Real-backend `dry-run` performs the same mining and replay model +calls while suppressing staging, adoption, and persisted state changes. The +current Sleep cycle does not implement fresh-worktree replay; a `replay: mock` +report label describes prompt replay and is independent of `--backend mock`. + ### Preferences `--preferences` is the main user-facing steering knob: @@ -130,6 +153,26 @@ python -m skillopt_sleep run --backend codex --project "$(pwd)" \ Preferences guide reflection but remain subject to the validation gate. +### Cursor source and backend + +Cursor transcript harvesting is explicit: use `--source cursor` rather than +`--source auto`. Invoked-project scope uses Cursor's recorded workspace path, +with the sanitized storage directory as a fallback; `--scope all` scans every +Cursor workspace under `~/.cursor/projects`. The model-driven backend requires +an installed, authenticated `cursor-agent`; use `--cursor-path`, +`SKILLOPT_SLEEP_CURSOR_PATH`, or the `cursor_path` config key when it is not on +`PATH`, and use `--model` or `SKILLOPT_SLEEP_CURSOR_MODEL` to choose a model. + +Target the project skill explicitly so accepted learning becomes visible to +Cursor without changing the plugin's own workflow skill: + +```bash +python -m skillopt_sleep run --project "$(pwd)" \ + --source cursor --backend cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +``` + ### Advanced config The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys, @@ -138,6 +181,15 @@ including `gate_mode`, `gate_metric`, `dream_rollouts`, `dream_factor`, `recall_ unsupported CLI flags listed above. Shipping defaults are conservative: `gate_mode="on"`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. +The managed `schedule` command stores only the project, backend, time, and +optional auto-adopt setting. It does not copy `--source`, `--cursor-home`, +`--cursor-path`, `--model`, or `--target-skill-path` into the scheduled command. +For a Cursor schedule, set `transcript_source`, `cursor_home`, `cursor_path`, +`model`, and `target_skill_path` in `~/.skillopt-sleep/config.json` first. Keep +the target project-relative, use an absolute CLI path because cron and Task +Scheduler may have a minimal `PATH`, and confirm that `cursor-agent` is +authenticated for the account that runs the job. + ### Handoff backend `--backend handoff` keeps model subprocesses out of the engine. It writes pending diff --git a/plugins/cursor/.cursor-plugin/plugin.json b/plugins/cursor/.cursor-plugin/plugin.json new file mode 100644 index 0000000..97252bb --- /dev/null +++ b/plugins/cursor/.cursor-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "skillopt-sleep", + "displayName": "SkillOpt-Sleep", + "version": "0.1.0", + "description": "Review recent Cursor sessions, replay recurring work, and stage validation-gated improvements to a Cursor skill for explicit adoption.", + "author": { + "name": "Yifan Yang", + "email": "yifanyang@microsoft.com" + }, + "homepage": "https://github.com/microsoft/SkillOpt", + "repository": "https://github.com/microsoft/SkillOpt", + "license": "MIT", + "keywords": [ + "skillopt", + "cursor", + "self-improvement", + "memory-consolidation", + "sleep", + "skills", + "offline-optimization" + ], + "category": "developer-tools", + "tags": [ + "automation", + "memory", + "transcripts" + ], + "commands": "./commands/", + "skills": "./skills/" +} diff --git a/plugins/cursor/LICENSE b/plugins/cursor/LICENSE new file mode 100644 index 0000000..cf7bcb2 --- /dev/null +++ b/plugins/cursor/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/cursor/README.md b/plugins/cursor/README.md new file mode 100644 index 0000000..3310a63 --- /dev/null +++ b/plugins/cursor/README.md @@ -0,0 +1,249 @@ +# SkillOpt-Sleep - Cursor integration + +Give Cursor an on-demand or explicitly scheduled sleep cycle: review recent +local Cursor sessions, replay recurring tasks through a selected backend, and +stage validation-gated improvements to a project Cursor skill. Nothing runs at +session end, and nothing live changes until the user adopts an accepted staged +proposal (unless they explicitly request `--auto-adopt`). + +This package is a native Cursor plugin containing a command and an agent skill. +It does not install hooks or an MCP server. + +## Requirements + +- Cursor with plugin and agent-skill support. +- Python 3.10 or newer. +- Either a SkillOpt source checkout or an installed `skillopt-sleep` command. +- For `--backend cursor`, an installed and authenticated Cursor Agent CLI + (`cursor-agent`). The default `mock` backend needs no provider login or spend. + +The plugin and transcript harvester work on native Windows. Cursor documents +the Agent CLI for Windows through WSL; run provider-backed `--backend cursor` +inside WSL unless a native `cursor-agent` is available in your environment. + +## Install the local plugin + +Clone the repository, then run the installer for your platform. + +macOS or Linux: + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +bash plugins/cursor/install.sh +export SKILLOPT_SLEEP_REPO="$(pwd)" +``` + +Windows PowerShell: + +```powershell +git clone https://github.com/microsoft/SkillOpt.git +Set-Location SkillOpt +powershell -File plugins/cursor/install.ps1 +[System.Environment]::SetEnvironmentVariable("SKILLOPT_SLEEP_REPO", "$(pwd)", "User") +``` + +The installer copies the plugin to +`~/.cursor/plugins/local/skillopt-sleep` (or +`%USERPROFILE%\.cursor\plugins\local\skillopt-sleep`). Quit and reopen Cursor +after changing user environment variables, then confirm that **SkillOpt-Sleep** +appears in Settings > Plugins under Installed. + +The plugin and engine have separate installation boundaries. The copied plugin +teaches Cursor how to operate SkillOpt-Sleep; the engine still runs from the +source checkout through `plugins/run-sleep.sh` / `plugins/run-sleep.ps1`, or +from an installed command: + +```bash +uv tool install skillopt +# or: python -m pip install skillopt +``` + +Use a release that includes Cursor source/backend support when choosing the +installed-command route. The source-checkout route uses the implementation in +the checkout directly. + +## Use from Cursor + +Run the native command, for example: + +```text +/skillopt-sleep status +/skillopt-sleep dry-run --backend mock --max-sessions 5 --max-tasks 3 +/skillopt-sleep run --backend cursor --max-sessions 5 --max-tasks 3 --progress +/skillopt-sleep adopt +``` + +The `skillopt-sleep` agent skill remains independently available if a Cursor +version does not surface plugin commands. + +The native command's default Cursor-visible target is: + +```text +.cursor/skills/skillopt-sleep-learned/SKILL.md +``` + +Use `--target-skill-path` with that value on harvest, dry-run, and run commands. +Without an explicit target, the shared engine defaults to a Claude-managed +skill, which Cursor does not load as a project skill. + +### Source-checkout commands + +macOS or Linux: + +```bash +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)" +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run \ + --project "$(pwd)" --source cursor --backend mock \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run \ + --project "$(pwd)" --source cursor --backend cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +``` + +Windows PowerShell: + +```powershell +powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" status --project "$(pwd)" +powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" dry-run ` + --project "$(pwd)" --source cursor --backend mock ` + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md +powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" run ` + --project "$(pwd)" --source cursor --backend cursor ` + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md ` + --max-sessions 5 --max-tasks 3 --progress +``` + +### Installed-command equivalents + +```bash +skillopt-sleep status --project "$(pwd)" +skillopt-sleep dry-run --project "$(pwd)" --source cursor --backend mock \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md +skillopt-sleep run --project "$(pwd)" --source cursor --backend cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +skillopt-sleep adopt --project "$(pwd)" +``` + +`--source cursor` reads local JSONL transcripts below +`~/.cursor/projects//agent-transcripts/`. Use +`--cursor-home /path/to/.cursor` for a different Cursor home. Invoked scope +selects the current workspace; `--scope all` includes every Cursor workspace. +The source converter retains user/assistant text, tool names, and explicit turn +errors, but excludes raw tool arguments and outputs. + +The first harvest uses a 72-hour lookback by default. Use +`--lookback-hours N` to choose a wider initial window, or +`--lookback-hours 0` to consider all available history while still respecting +`--max-sessions`. A successful `run`, including one that mines no tasks, +records a new harvest checkpoint. Later runs use that checkpoint instead of +the initial lookback. Use `harvest` or `dry-run` to inspect session and task +counts before the first stateful run; neither action advances the checkpoint. + +`--backend cursor` invokes the authenticated Cursor Agent CLI. Use +`--cursor-path /path/to/cursor-agent` or `SKILLOPT_SLEEP_CURSOR_PATH` if it is +not on PATH, and `--model` or `SKILLOPT_SLEEP_CURSOR_MODEL` to override its +model. Check available identifiers with `cursor-agent --list-models` and verify +the billed variant in Cursor's usage reporting when cost matters. The child +process receives only an explicit runtime/authentication/locale/proxy/CA +environment allowlist; unrelated cloud and model-provider credentials are not +forwarded. + +## What Cursor replay evaluates + +SkillOpt reads the target skill and inserts its text into mined task prompts. It +does not invoke that file as a native Cursor skill or execute commands described +by the skill. All ordinary Cursor model calls, including mining, replay, +judging, and reflection, run in a new empty temporary workspace in read-only Ask +mode. File reads, file writes, and MCP tools are denied. `--project` selects the +transcript scope, target files, state, and staging location; it does not make the +project the Cursor Agent execution workspace. + +Cursor tool-aware replay is temporarily disabled pending live Cursor +permission-boundary validation. Tasks containing a `tool_called` check fail +nonzero before Agent mode starts. The failed replay does not add a cache entry, +stage, adopt, persist state, or advance the harvest checkpoint. Use another +backend for those tasks. + +This replay is useful for textual procedures, response conventions, and output +formats. It is not an end-to-end evaluation of skills that depend on repository +inspection, real CLIs, browsers, running services, or filesystem changes. There +is currently no Cursor option that enables a fresh project worktree or real +project tools. The `replay: mock` report label refers to the prompt-replay mode, +not to the selected model backend. + +The shared engine also supports `mock`, `claude`, `codex`, `copilot`, +`handoff`, and `azure_openai` backends. Cursor is the native model-driven +choice for this integration; `mock` remains the no-provider default. + +## Review sensitive data before provider calls + +Harvesting is local and read-only, and `--backend mock` makes no provider calls. +Known secret-shaped strings are redacted from harvested Cursor content, and raw +tool payloads are excluded, but pattern-based redaction is not a guarantee. +A real backend sends truncated transcript excerpts and derived tasks to that +backend's provider for mining, replay, judging, and reflection. + +Both `run` and `dry-run` perform those real-backend calls; `dry-run` prevents +staging but does not prevent provider spend. `--max-sessions` and `--max-tasks` +bound harvested work, not provider calls, tokens, elapsed time, or money. One +task can require several attempt, judge, and reflection calls. Start with small +limits and review the provider's usage reporting before increasing them. + +For sensitive work, split the flow at the review boundary: + +```bash +skillopt-sleep harvest --project "$(pwd)" --source cursor \ + --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md \ + --max-sessions 5 --max-tasks 3 --output reviewed-tasks.json + +skillopt-sleep dry-run --project "$(pwd)" --backend cursor \ + --tasks-file reviewed-tasks.json --progress --json +``` + +Inspect and redact the JSON, then set its top-level `"reviewed"` field to +`true`. Real backends reject task files that remain unreviewed. Keep raw +transcripts, credentials, and task files out of commits. + +## Scheduling + +Runs remain user-triggered unless the user explicitly schedules them. Before +scheduling, put the Cursor source and target in +`~/.skillopt-sleep/config.json`, because the scheduler persists the project, +backend, time, and optional auto-adopt flag, but not command-line source or +target overrides: + +```json +{ + "transcript_source": "cursor", + "target_skill_path": ".cursor/skills/skillopt-sleep-learned/SKILL.md", + "cursor_home": "/absolute/path/to/.cursor", + "backend": "cursor" +} +``` + +Then schedule or remove the managed entry: + +```bash +skillopt-sleep schedule --project "$(pwd)" --backend cursor --hour 3 --minute 17 +skillopt-sleep unschedule --project "$(pwd)" +``` + +On Unix this uses cron; on Windows it uses Task Scheduler. Scheduled runs stage +proposals for later review by default. Do not add `--auto-adopt` unless the user +has explicitly chosen unattended adoption. + +## Adoption and memory + +`run` stages accepted proposals under +`/.skillopt-sleep/staging//`. Read the staged `report.md` +and show the held-out baseline-to-candidate score plus exact edits before +running `adopt`. Adoption backs up an existing target before replacing it. + +The shared engine may also propose project `CLAUDE.md` memory updates; existing +memory behavior is unchanged. To restrict a Cursor setup to the explicit Cursor +skill, set `"evolve_memory": false` in `~/.skillopt-sleep/config.json`. + +There is deliberately no session-end hook or automatic plugin execution. diff --git a/plugins/cursor/commands/skillopt-sleep.md b/plugins/cursor/commands/skillopt-sleep.md new file mode 100644 index 0000000..a62020f --- /dev/null +++ b/plugins/cursor/commands/skillopt-sleep.md @@ -0,0 +1,19 @@ +# SkillOpt-Sleep + +Use the bundled `skillopt-sleep` skill to run or manage SkillOpt-Sleep for the +current Cursor workspace. + +Requested action: `$ARGUMENTS` + +If no action was supplied, use `status`. Preserve all options supplied after +the action. For `harvest`, `dry-run`, and `run`, ensure the engine receives: + +```text +--project --scope invoked --source cursor --target-skill-path .cursor/skills/skillopt-sleep-learned/SKILL.md +``` + +Do not add `--backend cursor` unless the user requested provider-backed replay; +the repository default remains the no-provider `mock` backend. Follow the +skill's runner selection, review, data-boundary, scheduling, and adoption rules. +Never edit the learned skill or `CLAUDE.md` directly as a substitute for the +engine's staged adoption flow. diff --git a/plugins/cursor/install.ps1 b/plugins/cursor/install.ps1 new file mode 100644 index 0000000..dd7b232 --- /dev/null +++ b/plugins/cursor/install.ps1 @@ -0,0 +1,39 @@ +# Install the SkillOpt-Sleep Cursor integration as a local Cursor plugin on Windows. +# Idempotent; prints what it does. + +$ErrorActionPreference = "Stop" + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +$CursorHome = if ($env:CURSOR_HOME) { $env:CURSOR_HOME } else { Join-Path $env:USERPROFILE ".cursor" } +$PluginDir = Join-Path $CursorHome "plugins\local\skillopt-sleep" +$SourceDir = Join-Path $RepoRoot "plugins\cursor" +$ManifestDir = Join-Path $PluginDir ".cursor-plugin" +$CommandDir = Join-Path $PluginDir "commands" +$SkillDir = Join-Path $PluginDir "skills\skillopt-sleep" + +Write-Output "[install] repo: $RepoRoot" + +New-Item -ItemType Directory -Path $ManifestDir -Force | Out-Null +New-Item -ItemType Directory -Path $CommandDir -Force | Out-Null +New-Item -ItemType Directory -Path $SkillDir -Force | Out-Null +Copy-Item (Join-Path $SourceDir ".cursor-plugin\plugin.json") (Join-Path $ManifestDir "plugin.json") -Force +Copy-Item (Join-Path $SourceDir "commands\skillopt-sleep.md") (Join-Path $CommandDir "skillopt-sleep.md") -Force +Copy-Item (Join-Path $SourceDir "skills\skillopt-sleep\SKILL.md") (Join-Path $SkillDir "SKILL.md") -Force +Copy-Item (Join-Path $SourceDir "README.md") (Join-Path $PluginDir "README.md") -Force +Copy-Item (Join-Path $SourceDir "LICENSE") (Join-Path $PluginDir "LICENSE") -Force + +Write-Output "[install] plugin manifest -> $(Join-Path $ManifestDir 'plugin.json')" +Write-Output "[install] command -> $(Join-Path $CommandDir 'skillopt-sleep.md')" +Write-Output "[install] skill -> $(Join-Path $SkillDir 'SKILL.md')" +Write-Output "" +Write-Output "[install] Quit and reopen Cursor. The plugin should appear in Settings >" +Write-Output "Plugins under Installed." +Write-Output "" +Write-Output "For source-checkout runs, add this user environment variable:" +Write-Output " [System.Environment]::SetEnvironmentVariable('SKILLOPT_SLEEP_REPO', '$RepoRoot', 'User')" +Write-Output "" +Write-Output "Alternatively, install a SkillOpt release that includes Cursor support so the" +Write-Output "skillopt-sleep command is on PATH." +Write-Output "" +Write-Output "Done. Try in Cursor:" +Write-Output " /skillopt-sleep status" diff --git a/plugins/cursor/install.sh b/plugins/cursor/install.sh new file mode 100755 index 0000000..9b75d70 --- /dev/null +++ b/plugins/cursor/install.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Install the SkillOpt-Sleep Cursor integration as a local Cursor plugin. +# Idempotent; prints what it does. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CURSOR_HOME="${CURSOR_HOME:-$HOME/.cursor}" +PLUGIN_DIR="$CURSOR_HOME/plugins/local/skillopt-sleep" +SOURCE_DIR="$REPO_ROOT/plugins/cursor" + +echo "[install] repo: $REPO_ROOT" + +mkdir -p "$PLUGIN_DIR/.cursor-plugin" "$PLUGIN_DIR/commands" "$PLUGIN_DIR/skills/skillopt-sleep" +cp "$SOURCE_DIR/.cursor-plugin/plugin.json" "$PLUGIN_DIR/.cursor-plugin/plugin.json" +cp "$SOURCE_DIR/commands/skillopt-sleep.md" "$PLUGIN_DIR/commands/skillopt-sleep.md" +cp "$SOURCE_DIR/skills/skillopt-sleep/SKILL.md" "$PLUGIN_DIR/skills/skillopt-sleep/SKILL.md" +cp "$SOURCE_DIR/README.md" "$PLUGIN_DIR/README.md" +cp "$SOURCE_DIR/LICENSE" "$PLUGIN_DIR/LICENSE" + +echo "[install] plugin manifest -> $PLUGIN_DIR/.cursor-plugin/plugin.json" +echo "[install] command -> $PLUGIN_DIR/commands/skillopt-sleep.md" +echo "[install] skill -> $PLUGIN_DIR/skills/skillopt-sleep/SKILL.md" + +cat < +Plugins under Installed. + +For source-checkout runs, add this to your shell profile: + export SKILLOPT_SLEEP_REPO="$REPO_ROOT" + +Alternatively, install a SkillOpt release that includes Cursor support so the +\`skillopt-sleep\` command is on PATH. + +Done. Try in Cursor: + /skillopt-sleep status +EOF diff --git a/plugins/cursor/skills/skillopt-sleep/SKILL.md b/plugins/cursor/skills/skillopt-sleep/SKILL.md new file mode 100644 index 0000000..7caa8fc --- /dev/null +++ b/plugins/cursor/skills/skillopt-sleep/SKILL.md @@ -0,0 +1,218 @@ +--- +name: skillopt-sleep +description: "Use when the user wants Cursor to learn from recent local sessions, asks for an offline sleep or dream cycle, wants to consolidate recurring work into a Cursor skill, or requests SkillOpt-Sleep status, harvest, dry-run, run, scheduling, review, or adoption. Drives the validation-gated skillopt_sleep engine with Cursor transcripts and the optional Cursor Agent CLI backend." +--- + +# SkillOpt-Sleep for Cursor + +SkillOpt-Sleep reviews recent local Cursor sessions, mines recurring tasks, +replays those tasks, and proposes bounded improvements to a project Cursor +skill. With the default gate enabled, a proposal is accepted only when it +improves the held-out score. A normal run stages the proposal for review; +nothing live changes until explicit adoption. There is no model-weight training. + +This plugin has no session-end hook and no MCP server. Run the cycle only when +the user asks, or install a schedule only when the user explicitly requests one. + +## Cursor target + +Always use this project-relative target for Cursor-visible learning: + +```text +.cursor/skills/skillopt-sleep-learned/SKILL.md +``` + +Pass it through `--target-skill-path` on `harvest`, `dry-run`, and `run`. +Without an explicit target, the shared engine uses a Claude-managed skill under +`~/.claude/skills`, which is not the intended Cursor project skill. + +The shared engine can also evolve project `CLAUDE.md`. If that secondary memory +target is unwanted, set `"evolve_memory": false` in +`~/.skillopt-sleep/config.json` before running. + +## Choose the runner + +Use one of these supported command paths consistently: + +1. Source checkout on macOS/Linux: + `bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" ...` +2. Source checkout on Windows: + `powershell -File "$env:SKILLOPT_SLEEP_REPO\plugins\run-sleep.ps1" ...` +3. Installed engine on any platform: + `skillopt-sleep ...` + +If `SKILLOPT_SLEEP_REPO` is not set and `skillopt-sleep` is unavailable, stop +and explain that the engine must be installed or a SkillOpt checkout must be +selected. Do not substitute a hand-written edit for the engine workflow. + +## Core workflow + +1. **Harvest** local Cursor JSONL transcripts read-only. +2. **Mine** recurring, checkable task records from session digests. +3. **Replay** tasks under the current skill and memory through the selected + backend. +4. **Reflect** on failures and propose bounded edits. +5. **Gate** the candidate on held-out real tasks. +6. **Stage** accepted proposals under + `/.skillopt-sleep/staging//`. +7. **Adopt** only after review, backing up existing live targets first. + +## Commands + +Use the installed-command form below, or replace `skillopt-sleep` with the +platform-specific source runner described above. + +```bash +TARGET_SKILL=.cursor/skills/skillopt-sleep-learned/SKILL.md + +# Inspect current state and the latest staged proposal. +skillopt-sleep status --project "$(pwd)" + +# Inspect mined tasks without provider spend. +skillopt-sleep harvest --project "$(pwd)" --source cursor \ + --target-skill-path "$TARGET_SKILL" --max-sessions 5 --max-tasks 3 + +# First smoke check: deterministic and no provider calls. +skillopt-sleep dry-run --project "$(pwd)" --source cursor --backend mock \ + --target-skill-path "$TARGET_SKILL" --max-sessions 5 --max-tasks 3 --json + +# Model-driven optimization through the authenticated Cursor Agent CLI. +skillopt-sleep run --project "$(pwd)" --source cursor --backend cursor \ + --target-skill-path "$TARGET_SKILL" \ + --max-sessions 5 --max-tasks 3 --progress + +# Apply the latest accepted staged proposal after review. +skillopt-sleep adopt --project "$(pwd)" +``` + +Actions are `status`, `harvest`, `dry-run`, `run`, `adopt`, `schedule`, and +`unschedule`. + +- Default backend is `mock`, which is deterministic and makes no provider calls. +- `--backend cursor` uses the user's authenticated Cursor Agent CLI budget for + model-driven mining, replay, judging, and reflection. +- `--source cursor` reads + `~/.cursor/projects//agent-transcripts/*/*.jsonl`. +- `--cursor-home PATH` overrides the Cursor home used for harvesting. +- `--scope invoked` selects the current workspace; `--scope all` includes every + Cursor workspace. +- `--cursor-path PATH` or `SKILLOPT_SLEEP_CURSOR_PATH` selects a non-default + `cursor-agent` executable. +- `--model NAME` or `SKILLOPT_SLEEP_CURSOR_MODEL` overrides the Cursor model. +- Check model identifiers with `cursor-agent --list-models`; when cost matters, + verify the billed variant in Cursor's usage reporting. +- Keep live runs bounded with `--max-sessions`, `--max-tasks`, and `--progress`. +- A held-out gain is evidence for that run, not a promise of general improvement. + +The first harvest uses a 72-hour lookback. Use `--lookback-hours N` for a wider +initial window or `--lookback-hours 0` for all available history. A stateful +`run`, including a no-task run, records a harvest checkpoint; later runs use the +checkpoint rather than the initial lookback. Inspect counts with `harvest` or +`dry-run` before the first real run because those actions do not advance state. + +Available backends are: + +- `mock` - deterministic, with no provider calls (default); +- `cursor` - the authenticated Cursor Agent CLI; +- `claude` - the authenticated Claude CLI; +- `codex` - the authenticated Codex CLI; +- `copilot` - the authenticated GitHub Copilot CLI; +- `handoff` - prompt/answer files for an interactive agent session; +- `azure_openai` - the configured Azure OpenAI endpoint. + +SkillOpt reads the target skill and inserts its text into replay prompts; it does +not invoke the file as a native Cursor skill. Ordinary Cursor backend calls run +in a new empty temporary workspace in read-only Ask mode. File reads, file +writes, and MCP tools are denied. `--project` controls harvesting, target files, +state, and staging; it is not the Cursor Agent execution workspace. + +Cursor tool-aware replay is temporarily disabled pending live Cursor +permission-boundary validation. A task containing a `tool_called` check fails +nonzero before Agent mode starts. The failed replay does not add a cache entry, +stage, adopt, persist state, or advance the harvest checkpoint. Use another +backend for those tasks. Do not claim that repository- or tool-dependent +behavior was validated. The current engine does not implement a fresh-worktree +replay for Cursor. + +A real-backend `dry-run` still makes provider calls; it only suppresses staging. +Session and task limits are workload bounds, not hard limits on calls, tokens, +time, or money. Start with small limits. + +## Reviewable data path + +Cursor harvesting retains user/assistant text, tool names, and explicit turn +errors while excluding raw tool arguments, tool outputs, and non-message +records. Known secret-shaped strings are redacted, but pattern-based redaction +cannot guarantee that a transcript is safe to send to a provider. + +For sensitive sessions, export tasks before any real-backend replay: + +```bash +TARGET_SKILL=.cursor/skills/skillopt-sleep-learned/SKILL.md +skillopt-sleep harvest --project "$(pwd)" --source cursor \ + --target-skill-path "$TARGET_SKILL" \ + --max-sessions 5 --max-tasks 3 --output reviewed-tasks.json +``` + +Inspect and redact the file, then set its top-level `"reviewed"` field to +`true`. Only then run: + +```bash +skillopt-sleep dry-run --project "$(pwd)" --backend cursor \ + --tasks-file reviewed-tasks.json --progress --json +``` + +Real backends reject task files that remain unreviewed. Never include raw +transcripts, credentials, secrets, or sensitive task content in messages, +commits, or generated summaries. + +## Scheduling + +Scheduling is opt-in. The scheduler persists project, backend, time, and the +optional auto-adopt flag, but not `--source`, Cursor path/home/model overrides, +or `--target-skill-path`. Before scheduling a Cursor cycle, set at least these values in +`~/.skillopt-sleep/config.json`: + +```json +{ + "transcript_source": "cursor", + "target_skill_path": ".cursor/skills/skillopt-sleep-learned/SKILL.md", + "backend": "cursor" +} +``` + +Then run: + +```bash +skillopt-sleep schedule --project "$(pwd)" --backend cursor --hour 3 --minute 17 +skillopt-sleep unschedule --project "$(pwd)" +``` + +The scheduler uses cron on Unix and Task Scheduler on Windows. Scheduled runs +stage proposals by default. Use `--auto-adopt` only when the user has explicitly +requested unattended adoption. + +## Report results + +For `dry-run` and `run`, report: + +- session and task counts; +- held-out baseline and candidate scores; +- gate action and accepted/rejected edit counts; +- exact proposed edits; +- staging directory, when one was created. + +Read staged `report.md` before summarizing a run. Offer adoption only after the +user reviews an accepted proposal that is still staged. Never claim broad +improvement from one run. + +## Hard rules + +- Harvest is read-only. Never edit Cursor transcript files. +- Never hand-edit the target skill or `CLAUDE.md` as a substitute for adoption. +- Do not run a real backend on sensitive content without confirming its data + boundary or using the reviewed-task workflow. +- Do not add a session-end hook or imply that installing this plugin schedules + anything. +- Show validation evidence before recommending adoption. +- Treat generated edits as proposals, not as source of truth. diff --git a/plugins/devin/README.md b/plugins/devin/README.md index 7b63056..53abe6e 100644 --- a/plugins/devin/README.md +++ b/plugins/devin/README.md @@ -16,7 +16,10 @@ source into the Claude Code-compatible JSONL the engine reads. | `harvest_devin.py` | converts Devin ATIF-v1.7 transcripts + agentmemory + `.devin/skills` into JSONL, with `taskKey` + outcome envelopes | | `judge.py` | reference judge for the deferred/judge branch of the validation gate | | `mcp-config.example.json` | drop-in MCP server config | -| `devin-rules.snippet.md` | paste into `.devin/rules/skillopt-sleep.md` | +| `install.sh` | copies hooks + rules into a project's `.devin/` and prints the MCP registration command | +| `devin-rules.snippet.md` | copied to `.devin/rules/skillopt-sleep.md` by `install.sh` | +| `hooks/hooks.v1.json` | SessionEnd hook config — installed/merged at `.devin/hooks.v1.json` by `install.sh` | +| `hooks/on-session-end.sh` | best-effort activity marker script (called by the hook) | ## What it harvests @@ -33,10 +36,22 @@ After `sleep_adopt`, the evolved skill is synced to `.devin/skills/skillopt-slee Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib. -1. **Register the MCP server.** Use `mcp-config.example.json` as a template; set - `args` to the absolute path of this `mcp_server.py`. The engine is found - automatically (this plugin lives inside the SkillOpt repo). Or via the Devin - CLI: +1. **Install hooks + rules into your project.** From the repo root: + + ```bash + bash plugins/devin/install.sh /path/to/your/project + ``` + + This copies the SessionEnd hook and rules snippet into the project's + `.devin/` directory and prints the MCP registration command. The hook is + on by default — it logs a cheap activity marker for local inspection or + external automation when each session ends. The current engine harvests by + transcript timestamps and does not consume this marker directly. The hook + is non-blocking and spends no API budget. Re-run the script to update; the + installer preserves existing hooks and does not duplicate its own entry. + +2. **Register the MCP server.** Use `mcp-config.example.json` as a template, or + run the command printed by `install.sh`: ```bash devin mcp add skillopt-sleep \ @@ -44,9 +59,6 @@ Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib. -- python3 /abs/path/to/SkillOpt/plugins/devin/mcp_server.py ``` -2. **(Optional)** copy `devin-rules.snippet.md` to `.devin/rules/skillopt-sleep.md` - so Devin proactively offers the tools. - 3. Ask Devin: *"run the sleep cycle"*, *"what did the last sleep propose?"*, *"adopt it"*. ## Tools @@ -62,9 +74,12 @@ Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib. | `sleep_unschedule` | remove the nightly cron entry | Default backend is `mock` (no API spend); the `claude`, `codex`, and `copilot` -backends use the corresponding authenticated CLI and budget. The seven tools -call the same `python -m skillopt_sleep` actions as the other shared-engine -integrations. +backends use the corresponding authenticated CLI and budget. The `handoff` +backend runs the cycle with no model subprocess or API key — the engine writes +pending model calls to `.skillopt-sleep-handoff/PROMPTS.md` + `pending.json` +(exit code 3) and resumes after answers are placed in `answers/.md`; re-run +`sleep_run` with the same arguments to resume. The seven tools call the same +`python -m skillopt_sleep` actions as the other shared-engine integrations. ## Data boundary diff --git a/plugins/devin/devin-rules.snippet.md b/plugins/devin/devin-rules.snippet.md index 2e2c9af..4bd1ad8 100644 --- a/plugins/devin/devin-rules.snippet.md +++ b/plugins/devin/devin-rules.snippet.md @@ -24,7 +24,10 @@ Always pass the absolute Devin workspace as `project`, especially for `sleep_adopt`. Default backend is `mock` (no provider calls). The `claude`, `codex`, and `copilot` backend values use the corresponding installed and authenticated CLI; they do not require this plugin to implement a separate -API-key flow. +API-key flow. The `handoff` backend runs the cycle with no model subprocess +or API key — the engine writes pending model calls to +`.skillopt-sleep-handoff/` and exits; answer each prompt in a fresh context +and re-run `sleep_run` to resume (typically 3–6 rounds). The Devin conversion and mock workflow stay local. A real backend sends truncated transcript excerpts and derived tasks to the selected provider for diff --git a/plugins/devin/hooks/hooks.v1.json b/plugins/devin/hooks/hooks.v1.json new file mode 100644 index 0000000..6af99d1 --- /dev/null +++ b/plugins/devin/hooks/hooks.v1.json @@ -0,0 +1,14 @@ +{ + "SessionEnd": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"${DEVIN_PROJECT_DIR}/.devin/hooks/skillopt-sleep-on-session-end.sh\"", + "timeout": 5 + } + ] + } + ] +} diff --git a/plugins/devin/hooks/on-session-end.sh b/plugins/devin/hooks/on-session-end.sh new file mode 100755 index 0000000..dd0b0ec --- /dev/null +++ b/plugins/devin/hooks/on-session-end.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# SkillOpt-Sleep SessionEnd hook for Devin (best-effort, NON-BLOCKING). +# +# This does NOT run the optimizer. It only appends a tiny marker for local +# inspection or external automation. The current sleep engine uses transcript +# timestamps rather than this marker. The hook must never fail the session or +# spend API budget. +# +# Install this script as .devin/hooks/skillopt-sleep-on-session-end.sh and the +# config at .devin/hooks.v1.json. Devin CLI reads it automatically. +set -uo pipefail + +[ -n "${HOME:-}" ] || exit 0 +STATE_DIR="${HOME}/.skillopt-sleep" +mkdir -p "$STATE_DIR" 2>/dev/null || exit 0 + +# Record that a session just ended (cheap local activity signal). +printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${DEVIN_PROJECT_DIR:-${PWD}}" \ + >> "$STATE_DIR/session-end.log" 2>/dev/null || true + +exit 0 diff --git a/plugins/devin/install.sh b/plugins/devin/install.sh new file mode 100755 index 0000000..3271848 --- /dev/null +++ b/plugins/devin/install.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Install the SkillOpt-Sleep Devin integration into a project. +# Copies the SessionEnd hook and rules snippet into .devin/, and prints +# the MCP server registration command. Idempotent. +set -euo pipefail + +PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$PLUGIN_DIR/../.." && pwd)" +PROJECT="${1:-$(pwd)}" + +echo "[install] repo: $REPO_ROOT" +echo "[install] project: $PROJECT" + +DEVIN_DIR="$PROJECT/.devin" +mkdir -p "$DEVIN_DIR/hooks" "$DEVIN_DIR/rules" + +# 1) SessionEnd hook (on by default — provides activity signal for nightly harvest) +# Merge into existing hooks.v1.json instead of overwriting, so we don't +# destroy other project hooks. +HOOK_SCRIPT_SRC="$PLUGIN_DIR/hooks/on-session-end.sh" +HOOK_SCRIPT_DST="$DEVIN_DIR/hooks/skillopt-sleep-on-session-end.sh" +cp "$HOOK_SCRIPT_SRC" "$HOOK_SCRIPT_DST" +chmod +x "$HOOK_SCRIPT_DST" +echo "[install] hook script -> $HOOK_SCRIPT_DST" + +HOOK_CONFIG="$DEVIN_DIR/hooks.v1.json" +if [ -f "$HOOK_CONFIG" ]; then + # Python is already required by the plugin. Merge event arrays without + # replacing existing hooks, and skip exact duplicates on repeated installs. + python3 - "$HOOK_CONFIG" "$PLUGIN_DIR/hooks/hooks.v1.json" <<'PY' +import json +import os +import stat +import sys +import tempfile + +destination, addition = sys.argv[1:] + + +def load_object(path): + with open(path, encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise ValueError(f"hook config must be a JSON object: {path}") + return value + + +base = load_object(destination) +incoming = load_object(addition) +for event, entries in incoming.items(): + if not isinstance(entries, list): + raise ValueError(f"hook event {event!r} must be an array") + existing = base.setdefault(event, []) + if not isinstance(existing, list): + raise ValueError(f"existing hook event {event!r} must be an array") + for entry in entries: + if entry not in existing: + existing.append(entry) + +directory = os.path.dirname(os.path.abspath(destination)) +fd, temporary = tempfile.mkstemp(prefix=".hooks.v1.", suffix=".tmp", dir=directory) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(base, handle, ensure_ascii=False, indent=2) + handle.write("\n") + os.chmod(temporary, stat.S_IMODE(os.stat(destination).st_mode)) + os.replace(temporary, destination) +except Exception: + try: + os.unlink(temporary) + except OSError: + pass + raise +PY + echo "[install] session-end hook -> $HOOK_CONFIG (merged)" +else + cp "$PLUGIN_DIR/hooks/hooks.v1.json" "$HOOK_CONFIG" + echo "[install] session-end hook -> $HOOK_CONFIG" +fi + +# 2) Rules snippet so Devin proactively offers the tools +cp "$PLUGIN_DIR/devin-rules.snippet.md" "$DEVIN_DIR/rules/skillopt-sleep.md" +echo "[install] rules snippet -> $DEVIN_DIR/rules/skillopt-sleep.md" + +# 3) Print the MCP server registration command +printf -v MCP_SERVER_QUOTED '%q' "$PLUGIN_DIR/mcp_server.py" +cat < argparse.Namespace: # Legacy flat overrides p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "minimax", "minimax_chat"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -181,6 +182,8 @@ def parse_args() -> argparse.Namespace: p.add_argument("--claude_code_exec_use_sdk", type=str) p.add_argument("--claude_code_exec_effort", type=str) p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--cursor_exec_path", type=str) + p.add_argument("--cursor_exec_sandbox", type=str) p.add_argument("--minimax_base_url", type=str) p.add_argument("--minimax_api_key", type=str) p.add_argument("--minimax_model", type=str) @@ -262,6 +265,8 @@ def main() -> None: "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", "claude_code_exec_effort": "model.claude_code_exec_effort", "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "cursor_exec_path": "model.cursor_exec_path", + "cursor_exec_sandbox": "model.cursor_exec_sandbox", "minimax_base_url": "model.minimax_base_url", "minimax_api_key": "model.minimax_api_key", "minimax_model": "model.minimax_model", @@ -327,6 +332,11 @@ def main() -> None: elif backend == "claude_code_exec": cfg.setdefault("optimizer_backend", "openai_chat") cfg.setdefault("target_backend", "claude_code_exec") + elif backend == "cursor_exec": + if not _has_model_override("model.optimizer_backend", "optimizer_backend"): + cfg["optimizer_backend"] = "openai_chat" + if not _has_model_override("model.target_backend", "target_backend"): + cfg["target_backend"] = "cursor_exec" elif backend in {"minimax", "minimax_chat"}: cfg.setdefault("optimizer_backend", "openai_chat") cfg.setdefault("target_backend", "minimax_chat") @@ -355,6 +365,12 @@ def main() -> None: and not _has_model_override("model.target", "target_model") ): cfg["target_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "cursor_exec": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("cursor_exec") if cfg.get("target_backend") == "minimax_chat": if ( str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS @@ -429,6 +445,10 @@ def main() -> None: effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), ) + configure_cursor_exec( + path=cfg.get("cursor_exec_path") or None, + sandbox=cfg.get("cursor_exec_sandbox") or None, + ) configure_qwen_chat( base_url=cfg.get("qwen_chat_base_url") or None, api_key=cfg.get("qwen_chat_api_key") or None, diff --git a/scripts/smoke_superpowers.sh b/scripts/smoke_superpowers.sh new file mode 100755 index 0000000..1f38014 --- /dev/null +++ b/scripts/smoke_superpowers.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Smoke test for Superpowers adapter integration. +# Run this manually (not in CI) to verify the adapter works with the real harness. +# +# Prerequisites: +# - Claude Code installed, ANTHROPIC_API_KEY set (or SKILLOPT_HOST_AUTH=1 for a +# trusted candidate on your own machine) +# - Same model/settings/pinned SHA for baseline and candidate runs +# +# Usage: +# ./scripts/smoke_superpowers.sh [candidate_skill_path] +# +# Output goes to smoke_results/ (gitignored). Results embed raw agent output and +# local paths: do NOT commit them. Sanitized excerpts are written alongside each +# run for pasting into a PR description or attaching as an artifact. + +set -euo pipefail + +SKILL="${1:-}" +SCENARIO="${SKILLOPT_SCENARIO:-test-passes-verify}" +SHA="${SKILLOPT_SHA:-d884ae04edebef577e82ff7c4e143debd0bbec99}" +OUTDIR="smoke_results/$(date +%Y%m%d_%H%M%S)" +mkdir -p "$OUTDIR" + +echo "Smoke test: Superpowers adapter" +echo "Output: $OUTDIR (gitignored - do not commit)" +echo "Scenario: $SCENARIO" +echo "SHA: $SHA" +echo "" + +run_scenario() { + local name="$1" + local candidate="${2:-}" + local outfile="$OUTDIR/${name}.json" + + echo "=== $name ===" + + local args=( + --skill verification-before-completion + --scenario "$SCENARIO" + --sha "$SHA" + --json + ) + if [[ -n "$candidate" ]]; then + args+=(--candidate "$candidate") + fi + + # No || true - fail if runner errors + python -m skillopt_sleep.adapters.superpowers "${args[@]}" > "$outfile" + + # Best-effort sanitized summary for sharing: home dir, temp workspace paths + # and api-key-shaped tokens are redacted. Skim before sharing - it is a + # heuristic scrub, not a guarantee. + python - "$outfile" "$OUTDIR/${name}.summary.txt" <<'PY' +import json, os, re, sys +data = json.load(open(sys.argv[1])) +home = os.path.expanduser("~") +def clean(t): + t = t.replace(home, "~") + t = re.sub(r"/tmp/\S*skillopt\S*", "", t) + t = re.sub(r"/(?:tmp|var)/\S*", "", t) + return re.sub(r"sk-[A-Za-z0-9_\-]{8,}", "sk-REDACTED", t) +lines = [ + f"skill={data['skill']} version={data['version']} pinned_sha={data['pinned_sha']}", + f"candidate_hash={data['candidate_hash'] or '(baseline)'}", + f"score={data['score']:.2f} passed={data['passed']} failed={data['failed']}", +] +for s in data["scenarios"]: + lines.append(f"\n[{s['id']}] passed={s['passed']} error={s.get('error') or 'none'}") + lines.append(f" evidence: {json.dumps(s.get('evidence', {}))}") + for c in s["checks"]: + lines.append(f" {'PASS' if c['passed'] else 'FAIL'} {c['description']}") + lines.append(" output excerpt:") + for ln in clean(s.get("output", ""))[:800].splitlines(): + lines.append(f" {ln}") +text = "\n".join(lines) +open(sys.argv[2], "w").write(text + "\n") +print(text) +PY +} + +run_scenario "baseline" + +if [[ -n "$SKILL" ]]; then + run_scenario "candidate" "$SKILL" +fi + +echo "" +echo "Results in $OUTDIR (gitignored)." +echo "Share the *.summary.txt excerpts in the PR; do not commit the raw JSON." diff --git a/scripts/train.py b/scripts/train.py index 918e684..8b25377 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -137,7 +137,7 @@ def parse_args() -> argparse.Namespace: # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) p.add_argument("--env", type=str) p.add_argument("--backend", type=str, - choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "cursor", "cursor_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) p.add_argument("--optimizer_model", type=str) p.add_argument("--target_model", type=str) p.add_argument("--optimizer_backend", type=str) @@ -205,6 +205,8 @@ def parse_args() -> argparse.Namespace: p.add_argument("--claude_code_exec_use_sdk", type=str) p.add_argument("--claude_code_exec_effort", type=str) p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--cursor_exec_path", type=str) + p.add_argument("--cursor_exec_sandbox", type=str) p.add_argument("--codex_trace_to_optimizer", type=_BOOL) p.add_argument("--skill_init", type=str) p.add_argument("--num_epochs", type=int) @@ -343,6 +345,8 @@ _LEGACY_TO_STRUCTURED: dict[str, str] = { "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", "claude_code_exec_effort": "model.claude_code_exec_effort", "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "cursor_exec_path": "model.cursor_exec_path", + "cursor_exec_sandbox": "model.cursor_exec_sandbox", "codex_trace_to_optimizer": "model.codex_trace_to_optimizer", "num_epochs": "train.num_epochs", "train_size": "train.train_size", @@ -379,8 +383,65 @@ _LEGACY_TO_STRUCTURED: dict[str, str] = { def load_config(args: argparse.Namespace) -> dict: """Load config with _base_ inheritance, then apply CLI overrides.""" + import warnings from skillopt.config import load_config as _load, flatten_config, is_structured + # F08: Warn when API keys are supplied on the CLI. Keep the replacement + # guidance specific to each backend and, where applicable, each role. + _credential_guidance = { + "azure_api_key": ( + "AZURE_OPENAI_API_KEY or " + "--azure_openai_auth_mode=managed_identity" + ), + "azure_openai_api_key": ( + "AZURE_OPENAI_API_KEY or " + "--azure_openai_auth_mode=managed_identity" + ), + "optimizer_azure_openai_api_key": ( + "OPTIMIZER_AZURE_OPENAI_API_KEY or " + "--optimizer_azure_openai_auth_mode=managed_identity" + ), + "target_azure_openai_api_key": ( + "TARGET_AZURE_OPENAI_API_KEY or " + "--target_azure_openai_auth_mode=managed_identity" + ), + "qwen_chat_api_key": "QWEN_CHAT_API_KEY", + "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY", + "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY", + "minimax_api_key": "MINIMAX_API_KEY", + } + for _cli_key, _guidance in _credential_guidance.items(): + if getattr(args, _cli_key, None): + warnings.warn( + f"--{_cli_key} is deprecated: provide credentials via " + f"{_guidance} instead.", + DeprecationWarning, + stacklevel=2, + ) + _structured_credential_guidance = dict(_credential_guidance) + _structured_credential_guidance.update({ + # MiniMax is currently configured through one shared runtime client; a + # secret supplied through either role-shaped field belongs in the + # shared MINIMAX_API_KEY environment variable instead. + "optimizer_minimax_api_key": _credential_guidance["minimax_api_key"], + "target_minimax_api_key": _credential_guidance["minimax_api_key"], + }) + _credential_suffixes = ("api_key", "api-key", "token", "secret", "password") + for _override in getattr(args, "cfg_options", None) or []: + _key, _separator, _value = str(_override).partition("=") + _key = _key.strip() + _leaf = _key.casefold().rsplit(".", 1)[-1] + _guidance = _structured_credential_guidance.get(_leaf) + if not _guidance and _leaf.endswith(_credential_suffixes): + _guidance = "a backend-specific environment variable or managed identity" + if _separator and _guidance and _value: + warnings.warn( + f"--cfg-options {_key}=... exposes a credential in " + f"the process command line: provide it via {_guidance} instead.", + DeprecationWarning, + stacklevel=2, + ) + cfg = _load(args.config, overrides=args.cfg_options) structured = is_structured(cfg) @@ -445,6 +506,11 @@ def load_config(args: argparse.Namespace) -> dict: elif backend == "claude_code_exec": flat.setdefault("optimizer_backend", "openai_chat") flat.setdefault("target_backend", "claude_code_exec") + elif backend == "cursor_exec": + if not _has_model_override("model.optimizer_backend", "optimizer_backend"): + flat["optimizer_backend"] = "openai_chat" + if not _has_model_override("model.target_backend", "target_backend"): + flat["target_backend"] = "cursor_exec" elif backend in {"qwen", "qwen_chat"}: flat.setdefault("optimizer_backend", "openai_chat") flat.setdefault("target_backend", "qwen_chat") @@ -482,6 +548,12 @@ def load_config(args: argparse.Namespace) -> dict: and not _has_model_override("model.target", "target_model") ): flat["target_model"] = default_model_for_backend("claude_chat") + if flat.get("target_backend") == "cursor_exec": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("cursor_exec") if flat.get("target_backend") == "qwen_chat": if ( str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS diff --git a/skillopt/config.py b/skillopt/config.py index 2c1c4e9..c47e6d9 100644 --- a/skillopt/config.py +++ b/skillopt/config.py @@ -51,6 +51,8 @@ _FLATTEN_MAP: dict[str, str] = { "model.claude_code_exec_use_sdk": "claude_code_exec_use_sdk", "model.claude_code_exec_effort": "claude_code_exec_effort", "model.claude_code_exec_max_thinking_tokens": "claude_code_exec_max_thinking_tokens", + "model.cursor_exec_path": "cursor_exec_path", + "model.cursor_exec_sandbox": "cursor_exec_sandbox", "model.codex_trace_to_optimizer": "codex_trace_to_optimizer", "model.azure_endpoint": "azure_endpoint", "model.azure_api_version": "azure_api_version", diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py index ff535b0..1eae420 100644 --- a/skillopt/engine/trainer.py +++ b/skillopt/engine/trainer.py @@ -63,6 +63,7 @@ from skillopt.model import ( configure_azure_openai, configure_claude_code_exec, configure_codex_exec, + configure_cursor_exec, configure_minimax_chat, configure_qwen_chat, get_token_summary, @@ -681,6 +682,9 @@ class ReflACTTrainer: elif backend == "claude_code_exec": optimizer_backend = optimizer_backend or "openai_chat" target_backend = target_backend or "claude_code_exec" + elif backend in {"cursor", "cursor_exec"}: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "cursor_exec" elif backend in {"qwen", "qwen_chat"}: optimizer_backend = optimizer_backend or "openai_chat" target_backend = target_backend or "qwen_chat" @@ -711,6 +715,10 @@ class ReflACTTrainer: effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), ) + configure_cursor_exec( + path=cfg.get("cursor_exec_path") or None, + sandbox=cfg.get("cursor_exec_sandbox") or None, + ) configure_qwen_chat( base_url=cfg.get("qwen_chat_base_url") or None, api_key=cfg.get("qwen_chat_api_key") or None, diff --git a/skillopt/envs/_template/config_template.yaml b/skillopt/envs/_template/config_template.yaml index d98851d..0778738 100644 --- a/skillopt/envs/_template/config_template.yaml +++ b/skillopt/envs/_template/config_template.yaml @@ -51,5 +51,5 @@ evaluation: # Override only what differs from the inherited defaults. model: optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat | codex_exec - target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec + target_backend: openai_chat # chat backends plus codex_exec / claude_code_exec / cursor_exec reasoning_effort: medium diff --git a/skillopt/envs/spreadsheetbench/codegen_agent.py b/skillopt/envs/spreadsheetbench/codegen_agent.py index a4948f1..32a8d67 100644 --- a/skillopt/envs/spreadsheetbench/codegen_agent.py +++ b/skillopt/envs/spreadsheetbench/codegen_agent.py @@ -258,19 +258,60 @@ def _build_codex_task( def _build_codex_driver() -> str: return ( + "import os\n" "import pathlib\n" "import re\n" + "import shutil\n" + "import subprocess\n" "import sys\n" - "import traceback\n\n" + "import tempfile\n\n" 'INPUT_PATH = "input.xlsx"\n' 'OUTPUT_PATH = "output.xlsx"\n' "code = pathlib.Path('solution.py').read_text(encoding='utf-8')\n" "code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n" - "globals_dict = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n" + "# Write patched code to a temporary file and run it in a clean subprocess\n" + "# with a scrubbed environment. This avoids in-process exec/compile but is\n" + "# not a filesystem, process, or network sandbox.\n" + "_work_dir = str(pathlib.Path.cwd())\n" + "_temp_dir = tempfile.mkdtemp(prefix='skillopt-generated-')\n" "try:\n" - " exec(compile(code, 'solution.py', 'exec'), globals_dict, globals_dict)\n" - "except Exception:\n" - " traceback.print_exc()\n" + " _patched = pathlib.Path(_temp_dir) / 'runner.py'\n" + " _safe_env = {\n" + " 'PATH': os.environ.get('PATH') or os.defpath,\n" + " 'HOME': _work_dir,\n" + " 'TMPDIR': _temp_dir,\n" + " }\n" + " for _key in (\n" + " 'PYTHONPATH', 'PYTHONHOME', 'VIRTUAL_ENV',\n" + " 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH',\n" + " 'LANG', 'LANGUAGE', 'LC_ALL', 'LC_CTYPE',\n" + " 'PYTHONIOENCODING', 'PYTHONUTF8',\n" + " 'SYSTEMDRIVE', 'PATHEXT', 'COMSPEC',\n" + " ):\n" + " if os.environ.get(_key):\n" + " _safe_env[_key] = os.environ[_key]\n" + " if os.name == 'nt':\n" + " _safe_env['SYSTEMROOT'] = (\n" + " os.environ.get('SYSTEMROOT')\n" + " or os.environ.get('SystemRoot')\n" + " or os.environ.get('WINDIR', '')\n" + " )\n" + " _safe_env['USERPROFILE'] = _work_dir\n" + " _safe_env['TEMP'] = _temp_dir\n" + " _safe_env['TMP'] = _temp_dir\n" + " _safe_env['APPDATA'] = _temp_dir\n" + " _safe_env['LOCALAPPDATA'] = _temp_dir\n" + " _safe_env = {k: v for k, v in _safe_env.items() if v}\n" + " _patched.write_text(\n" + " f'INPUT_PATH = {INPUT_PATH!r}\\nOUTPUT_PATH = {OUTPUT_PATH!r}\\n' + code,\n" + " encoding='utf-8',\n" + " )\n" + " _res = subprocess.run([sys.executable, str(_patched)], capture_output=True, text=True, env=_safe_env)\n" + "finally:\n" + " shutil.rmtree(_temp_dir, ignore_errors=True)\n" + "if _res.returncode != 0:\n" + " print(_res.stdout, end='')\n" + " print(_res.stderr, end='')\n" " sys.exit(2)\n" ) diff --git a/skillopt/envs/spreadsheetbench/executor.py b/skillopt/envs/spreadsheetbench/executor.py index 24421f9..0996f24 100644 --- a/skillopt/envs/spreadsheetbench/executor.py +++ b/skillopt/envs/spreadsheetbench/executor.py @@ -28,14 +28,70 @@ _PATH_ASSIGN_RE = re.compile( r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE ) +_GENERATED_CODE_ENV_PASSTHROUGH = ( + # Preserve interpreter/import behavior without inheriting API/cloud keys. + "PYTHONPATH", + "PYTHONHOME", + "VIRTUAL_ENV", + "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH", + # Keep text I/O deterministic for non-ASCII spreadsheet content. + "LANG", + "LANGUAGE", + "LC_ALL", + "LC_CTYPE", + "PYTHONIOENCODING", + "PYTHONUTF8", + # Needed by executable lookup and CPython on some Windows installations. + "SYSTEMDRIVE", + "PATHEXT", + "COMSPEC", +) + def _strip_path_assignments(code: str) -> str: """Remove INPUT_PATH/OUTPUT_PATH assignments from user code.""" return _PATH_ASSIGN_RE.sub("", code) +def generated_code_env(work_dir: str, temp_dir: str) -> dict[str, str]: + """Return the minimal environment for LLM-generated spreadsheet Python. + + This prevents direct inheritance of parent-process credentials. It is not a + filesystem, process, or network sandbox. + """ + private_dir = os.path.abspath(work_dir or os.getcwd()) + private_temp = os.path.abspath(temp_dir) + safe_env = { + "PATH": os.environ.get("PATH") or os.defpath, + "HOME": private_dir, + "TMPDIR": private_temp, + } + for key in _GENERATED_CODE_ENV_PASSTHROUGH: + value = os.environ.get(key) + if value: + safe_env[key] = value + if os.name == "nt": + system_root = ( + os.environ.get("SYSTEMROOT") + or os.environ.get("SystemRoot") + or os.environ.get("WINDIR") + or "" + ) + safe_env.update({ + "SYSTEMROOT": system_root, + "USERPROFILE": private_dir, + "TEMP": private_temp, + "TMP": private_temp, + "APPDATA": private_temp, + "LOCALAPPDATA": private_temp, + }) + return {key: value for key, value in safe_env.items() if value} + + def run_generated_code(code: str, input_path: str, output_path: str, timeout: int | None = 120) -> tuple[bool, str]: - os.makedirs(os.path.dirname(output_path), exist_ok=True) + output_dir = os.path.dirname(os.path.abspath(output_path)) + os.makedirs(output_dir, exist_ok=True) cleaned = _strip_path_assignments(code) indented = textwrap.indent(cleaned, " ") script = RUNNER_TEMPLATE.format( @@ -43,25 +99,28 @@ def run_generated_code(code: str, input_path: str, output_path: str, timeout: in output_path=output_path, user_code_indented=indented, ) - with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f: - f.write(script) - tmp = f.name - try: - proc = subprocess.run( - [sys.executable, tmp], - capture_output=True, - text=True, - timeout=timeout if timeout and timeout > 0 else None, - ) - if proc.returncode != 0: - return False, (proc.stdout + "\n" + proc.stderr).strip() - if not os.path.exists(output_path): - return False, "output file was not created" - return True, "" - except subprocess.TimeoutExpired: - return False, f"timeout after {timeout}s" - finally: + # Keep the runner and scratch files out of the result directory. Environment + # scrubbing prevents direct credential inheritance; it is not a filesystem, + # process, or network sandbox. + with tempfile.TemporaryDirectory( + prefix="skillopt-generated-", ignore_cleanup_errors=True + ) as temp_dir: + runner = os.path.join(temp_dir, "runner.py") + with open(runner, "w", encoding="utf-8") as f: + f.write(script) + safe_env = generated_code_env(output_dir, temp_dir) try: - os.unlink(tmp) - except OSError: - pass + proc = subprocess.run( + [sys.executable, runner], + capture_output=True, + text=True, + timeout=timeout if timeout and timeout > 0 else None, + env=safe_env, + ) + if proc.returncode != 0: + return False, (proc.stdout + "\n" + proc.stderr).strip() + if not os.path.exists(output_path): + return False, "output file was not created" + return True, "" + except subprocess.TimeoutExpired: + return False, f"timeout after {timeout}s" diff --git a/skillopt/envs/spreadsheetbench/prompts/react_system.md b/skillopt/envs/spreadsheetbench/prompts/react_system.md index afbbffb..27987e1 100644 --- a/skillopt/envs/spreadsheetbench/prompts/react_system.md +++ b/skillopt/envs/spreadsheetbench/prompts/react_system.md @@ -2,7 +2,8 @@ You are an expert spreadsheet manipulation agent. {critical_rules}{skill_section}## Tools You have two tools: -- `bash` -- execute any shell command and receive its output. +- `bash` -- run a Python command whose executable is `python` or `python3`; + arbitrary shell commands are blocked. - `write_file` -- write content to a file (path, content). Use this for solution.py. ## Protocol diff --git a/skillopt/envs/spreadsheetbench/react_agent.py b/skillopt/envs/spreadsheetbench/react_agent.py index 2e72953..61faa1d 100644 --- a/skillopt/envs/spreadsheetbench/react_agent.py +++ b/skillopt/envs/spreadsheetbench/react_agent.py @@ -9,10 +9,14 @@ from __future__ import annotations import json import os +import shlex import subprocess +import sys +import tempfile from skillopt.model import chat_target_messages from skillopt.prompts import load_prompt +from skillopt.envs.spreadsheetbench.executor import generated_code_env # ── Tool schemas ───────────────────────────────────────────────────────────── @@ -21,13 +25,13 @@ BASH_TOOL_CHAT = { "function": { "name": "bash", "description": ( - "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " - "Use Python to read / write Excel files." + "Run a Python command (python/python3 only) and receive stdout+stderr " + "(truncated to 4000 chars)." ), "parameters": { "type": "object", "properties": { - "cmd": {"type": "string", "description": "Bash command to execute."} + "cmd": {"type": "string", "description": "Python command to execute."} }, "required": ["cmd"], }, @@ -38,13 +42,13 @@ BASH_TOOL_RESPONSES = { "type": "function", "name": "bash", "description": ( - "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " - "Use Python to read / write Excel files." + "Run a Python command (python/python3 only) and receive stdout+stderr " + "(truncated to 4000 chars)." ), "parameters": { "type": "object", "properties": { - "cmd": {"type": "string", "description": "Bash command to execute."} + "cmd": {"type": "string", "description": "Python command to execute."} }, "required": ["cmd"], }, @@ -248,18 +252,47 @@ def _auto_verify(work_dir: str) -> str: return f"\n\n[AUTO-VERIFY] Could not inspect output: {e}" +# Command aliases that the ReAct agent may request. Every accepted alias is +# resolved to this process's interpreter before execution, so behavior does not +# depend on PATH and similarly named executables cannot bypass the allow-list. +_PYTHON_ALIASES = {"python", "python3", "python.exe", "python3.exe"} + + # ── Bash execution ──────────────────────────────────────────────────────────── def _run_bash(cmd: str, work_dir: str, timeout: int = 60) -> str: try: - proc = subprocess.run( - cmd, - shell=True, - capture_output=True, - text=True, - timeout=timeout, - cwd=work_dir, - ) + parts = shlex.split(cmd, posix=os.name != "nt") + if os.name == "nt": + # shlex in non-POSIX mode retains surrounding quotes. + parts = [ + part[1:-1] + if len(part) >= 2 and part[0] == part[-1] and part[0] in {'"', "'"} + else part + for part in parts + ] + if not parts: + return "[error: empty command]" + exe_name = parts[0].replace("\\", "/").rsplit("/", 1)[-1].lower() + if exe_name not in _PYTHON_ALIASES: + return ( + f"[blocked: '{parts[0]}' not in allow-list " + f"{sorted(_PYTHON_ALIASES)}; " + "use Python to manipulate spreadsheets]" + ) + parts[0] = sys.executable + with tempfile.TemporaryDirectory( + prefix="skillopt-generated-", ignore_cleanup_errors=True + ) as temp_dir: + proc = subprocess.run( + parts, + shell=False, + capture_output=True, + text=True, + timeout=timeout, + cwd=work_dir, + env=generated_code_env(work_dir, temp_dir), + ) out = (proc.stdout + proc.stderr).strip() except subprocess.TimeoutExpired: return f"[timeout after {timeout}s]" diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py index f58be67..6c84e6b 100644 --- a/skillopt/model/__init__.py +++ b/skillopt/model/__init__.py @@ -13,8 +13,10 @@ from skillopt.model import qwen_backend as _qwen from skillopt.model.backend_config import ( # noqa: F401 configure_claude_code_exec, configure_codex_exec, + configure_cursor_exec, get_claude_code_exec_config, get_codex_exec_config, + get_cursor_exec_config, get_optimizer_backend, get_target_backend, is_optimizer_chat_backend, @@ -53,6 +55,10 @@ def set_backend(name: str | None) -> str: set_optimizer_backend("openai_chat") set_target_backend(normalized) return normalized + if normalized in {"cursor", "cursor_agent", "cursor_exec"}: + set_optimizer_backend("openai_chat") + set_target_backend("cursor_exec") + return "cursor_exec" if normalized in {"qwen", "qwen_chat"}: set_optimizer_backend("openai_chat") set_target_backend("qwen_chat") @@ -84,6 +90,8 @@ def get_backend_name() -> str: return "qwen_chat" if optimizer == "openai_chat" and target == "minimax_chat": return "minimax_chat" + if optimizer == "openai_chat" and target == "cursor_exec": + return "cursor_exec" if optimizer == "openai_compatible" and target == "openai_compatible": return "openai_compatible" return f"{optimizer}+{target}" diff --git a/skillopt/model/backend_config.py b/skillopt/model/backend_config.py index d74bd13..d48b050 100644 --- a/skillopt/model/backend_config.py +++ b/skillopt/model/backend_config.py @@ -28,6 +28,8 @@ CLAUDE_CODE_EXEC_PATH = os.environ.get("CLAUDE_CODE_EXEC_PATH", "claude") CLAUDE_CODE_EXEC_PROFILE = os.environ.get("CLAUDE_CODE_EXEC_PROFILE", "") CLAUDE_CODE_EXEC_USE_SDK = os.environ.get("CLAUDE_CODE_EXEC_USE_SDK", "auto") CLAUDE_CODE_EXEC_EFFORT = os.environ.get("CLAUDE_CODE_EXEC_EFFORT", "medium") +CURSOR_EXEC_PATH = os.environ.get("CURSOR_EXEC_PATH", "cursor-agent") +CURSOR_EXEC_SANDBOX = os.environ.get("CURSOR_EXEC_SANDBOX", "enabled") def _parse_int(value: str | None, default: int) -> int: @@ -72,10 +74,11 @@ def get_optimizer_backend() -> str: def set_target_backend(backend: str) -> None: global TARGET_BACKEND TARGET_BACKEND = normalize_backend_name(backend or "openai_chat") - if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "codex_exec", "claude_code_exec"}: + if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "openai_compatible", "codex_exec", "claude_code_exec", "cursor_exec"}: raise ValueError( f"Unsupported target backend: {TARGET_BACKEND!r}. " - "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', 'openai_compatible', 'codex_exec', and 'claude_code_exec'." + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', " + "'openai_compatible', 'codex_exec', 'claude_code_exec', and 'cursor_exec'." ) os.environ["TARGET_BACKEND"] = TARGET_BACKEND @@ -85,7 +88,7 @@ def get_target_backend() -> str: def is_target_exec_backend() -> bool: - return TARGET_BACKEND in {"codex_exec", "claude_code_exec"} + return TARGET_BACKEND in {"codex_exec", "claude_code_exec", "cursor_exec"} def is_optimizer_chat_backend() -> bool: @@ -198,3 +201,30 @@ def get_claude_code_exec_config() -> dict[str, str | int]: "max_thinking_tokens": CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS, "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, } + + +def configure_cursor_exec( + *, + path: str | None = None, + sandbox: str | None = None, +) -> None: + global CURSOR_EXEC_PATH, CURSOR_EXEC_SANDBOX + if path is not None: + CURSOR_EXEC_PATH = str(path).strip() or "cursor-agent" + os.environ["CURSOR_EXEC_PATH"] = CURSOR_EXEC_PATH + if sandbox is not None: + normalized_sandbox = str(sandbox).strip().lower() or "enabled" + if normalized_sandbox not in {"enabled", "disabled"}: + raise ValueError("cursor_exec sandbox must be 'enabled' or 'disabled'") + CURSOR_EXEC_SANDBOX = normalized_sandbox + os.environ["CURSOR_EXEC_SANDBOX"] = CURSOR_EXEC_SANDBOX + + +def get_cursor_exec_config() -> dict[str, str | int]: + if CURSOR_EXEC_SANDBOX not in {"enabled", "disabled"}: + raise ValueError("cursor_exec sandbox must be 'enabled' or 'disabled'") + return { + "path": CURSOR_EXEC_PATH, + "sandbox": CURSOR_EXEC_SANDBOX, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } diff --git a/skillopt/model/codex_harness.py b/skillopt/model/codex_harness.py index a8513ea..6bbb093 100644 --- a/skillopt/model/codex_harness.py +++ b/skillopt/model/codex_harness.py @@ -14,10 +14,10 @@ from typing import Any from skillopt.model.backend_config import ( get_claude_code_exec_config, get_codex_exec_config, + get_cursor_exec_config, get_target_backend, ) - ANSWER_SCHEMA: dict[str, Any] = { "type": "object", "properties": { @@ -227,6 +227,51 @@ def _build_claude_trace_summary(raw: str, response: str) -> str: return "\n".join(parts) +def _build_cursor_trace_summary(raw: str, response: str) -> str: + model = "" + permission_mode = "" + session_id = "" + duration_ms = 0 + tool_calls = 0 + terminal_error = False + for line in (raw or "").splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + if event.get("type") == "system" and event.get("subtype") == "init": + model = str(event.get("model") or model) + permission_mode = str(event.get("permissionMode") or permission_mode) + session_id = str(event.get("session_id") or session_id) + elif event.get("type") == "tool_call" and event.get("subtype") == "started": + tool_calls += 1 + elif event.get("type") == "result": + session_id = str(event.get("session_id") or session_id) + try: + duration_ms = int(event.get("duration_ms") or 0) + except (TypeError, ValueError): + duration_ms = 0 + terminal_error = bool(event.get("is_error")) or event.get("subtype") == "error" + + parts = ["Cursor Agent Trace Summary"] + if model: + parts.append(f"- model: {model}") + if permission_mode: + parts.append(f"- permission mode: {permission_mode}") + if session_id: + parts.append(f"- session id: {session_id}") + parts.append(f"- tool calls: {tool_calls}") + parts.append(f"- duration ms: {duration_ms}") + parts.append(f"- terminal error: {'yes' if terminal_error else 'no'}") + parts.append(f"- final response chars: {len(response or '')}") + return "\n".join(parts) + + def _persist_artifacts( *, work_dir: str, @@ -271,6 +316,16 @@ def _persist_claude_artifacts(work_dir: str, raw: str, response: str) -> None: ) +def _persist_cursor_artifacts(work_dir: str, raw: str, response: str) -> None: + _persist_artifacts( + work_dir=work_dir, + raw=_sanitize_cursor_trace(raw, preserve_markers=True), + response=response, + prefix="cursor", + summary_builder=_build_cursor_trace_summary, + ) + + def parse_codex_raw(raw: str) -> dict: """Parse raw Codex CLI output into step sections. @@ -1016,6 +1071,239 @@ def run_codex_exec( return last_response, combined +_CURSOR_SECRET_ASSIGNMENT = re.compile( + r"(?i)\b(cursor_api_key|api[_ -]?key|authorization|bearer|" + r"access[_ -]?token|refresh[_ -]?token|token|password)\b" + r"(\s*[:=]\s*|\s+)(?:bearer\s+)?([^\s,;]+)" +) +_CURSOR_SECRET_TOKEN = re.compile(r"\b(?:sk|key)[_-][A-Za-z0-9_-]{8,}\b") +_CURSOR_OMITTED_TRACE_FIELDS = {"args", "content", "filetext", "prompt", "result"} +_CURSOR_SECRET_TRACE_FIELDS = { + "accesstoken", + "apikey", + "authorization", + "cursorapikey", + "password", + "refreshtoken", + "secret", + "token", +} + + +def _redact_cursor_error(value: str) -> str: + text = _CURSOR_SECRET_ASSIGNMENT.sub(r"\1\2[REDACTED]", value or "") + return _CURSOR_SECRET_TOKEN.sub("[REDACTED]", text) + + +def _sanitize_cursor_json(value: Any, *, field: str = "") -> Any: + normalized_field = re.sub(r"[^a-z0-9]", "", field.lower()) + if normalized_field in _CURSOR_OMITTED_TRACE_FIELDS: + return "[OMITTED]" + if ( + normalized_field in _CURSOR_SECRET_TRACE_FIELDS + or normalized_field.endswith("apikey") + ): + return "[REDACTED]" + if isinstance(value, dict): + return { + str(key): _sanitize_cursor_json(item, field=str(key)) + for key, item in value.items() + } + if isinstance(value, list): + return [_sanitize_cursor_json(item) for item in value] + if isinstance(value, str): + return _redact_cursor_error(value) + return value + + +def _cursor_process_text(value: str | bytes) -> str: + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value or "") + + +def _sanitize_cursor_trace( + raw: str | bytes, + *, + preserve_markers: bool = False, +) -> str: + text = _cursor_process_text(raw) + sanitized: list[str] = [] + in_stderr = False + omitted_stdout = False + for line in text.splitlines(): + stripped = line.strip() + if preserve_markers and stripped.startswith("===== CURSOR CLI ATTEMPT "): + in_stderr = False + omitted_stdout = False + sanitized.append(line) + continue + if preserve_markers and stripped == "[stderr]": + in_stderr = True + sanitized.append(line) + continue + if in_stderr: + sanitized.append(_redact_cursor_error(line)) + continue + if stripped.startswith("{"): + try: + event = json.loads(stripped) + except json.JSONDecodeError: + pass + else: + sanitized.append( + json.dumps( + _sanitize_cursor_json(event), + ensure_ascii=False, + separators=(",", ":"), + ) + ) + continue + if not stripped: + sanitized.append("") + elif not omitted_stdout: + sanitized.append("[OMITTED NON-JSON OUTPUT]") + omitted_stdout = True + return "\n".join(sanitized) + + +def _parse_cursor_terminal(raw: str) -> tuple[str, str]: + terminal: dict[str, Any] | None = None + for line in (raw or "").splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(event, dict) and event.get("type") == "result": + terminal = event + + if terminal is None: + return "", "Cursor Agent did not emit a terminal result" + if terminal.get("is_error") is True or terminal.get("subtype") == "error": + detail = str(terminal.get("result") or terminal.get("error") or "unknown error") + return "", f"Cursor Agent returned an error result: {_redact_cursor_error(detail)}" + result = terminal.get("result") + if not isinstance(result, str) or not result.strip(): + return "", "Cursor Agent returned an empty terminal result" + return result.strip(), "" + + +def run_cursor_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + sandbox: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + """Run Cursor Agent headlessly as a benchmark target.""" + config = get_cursor_exec_config() + retries = int(config.get("empty_response_retries", 0) or 0) + add_dirs = _validated_add_dirs(work_dir, data_dirs, images)[1:] + all_raw: list[str] = [] + last_error = "Cursor Agent returned no response" + actual_sandbox = str(sandbox or config["sandbox"]) + if actual_sandbox not in {"enabled", "disabled"}: + raise ValueError("Cursor Agent sandbox must be 'enabled' or 'disabled'") + if allow_file_edits and actual_sandbox == "disabled": + raise ValueError( + "Cursor Agent file-edit rollouts require sandbox='enabled'; " + "refusing to combine --force with a disabled sandbox" + ) + + for attempt in range(retries + 1): + attempt_prompt = _exec_prompt( + _retry_prompt(prompt, attempt), + allow_file_edits=allow_file_edits, + ) + cmd = [ + str(config["path"]), + "-p", + "--output-format", + "stream-json", + "--trust", + "--workspace", + work_dir, + "--sandbox", + actual_sandbox, + ] + if allow_file_edits: + cmd.append("--force") + else: + cmd.extend(["--mode", "ask"]) + if model: + cmd.extend(["--model", model]) + for path in add_dirs: + cmd.extend(["--add-dir", path]) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + input=attempt_prompt, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + safe_raw = _sanitize_cursor_trace(raw) + if stderr: + safe_stderr = _redact_cursor_error(_cursor_process_text(stderr)) + safe_raw = ( + f"{safe_raw}\n[stderr]\n{safe_stderr}" + if safe_raw + else f"[stderr]\n{safe_stderr}" + ) + all_raw.append(f"===== CURSOR CLI ATTEMPT {attempt + 1} =====\n{safe_raw}") + _persist_cursor_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + except OSError as exc: + detail = _redact_cursor_error(str(exc)) + raise RuntimeError(f"Cursor Agent could not be executed: {detail}") from exc + + stdout = proc.stdout or "" + stderr = proc.stderr or "" + raw = stdout + safe_raw = _sanitize_cursor_trace(raw) + if stderr: + safe_stderr = _redact_cursor_error(_cursor_process_text(stderr)) + safe_raw = ( + f"{safe_raw}\n[stderr]\n{safe_stderr}" + if safe_raw + else f"[stderr]\n{safe_stderr}" + ) + all_raw.append(f"===== CURSOR CLI ATTEMPT {attempt + 1} =====\n{safe_raw}") + combined = "\n\n".join(all_raw) + + if proc.returncode != 0: + _persist_cursor_artifacts(work_dir, combined, "") + detail = _redact_cursor_error((stderr or stdout).strip())[:4000] + raise RuntimeError( + f"Cursor Agent failed with exit code {proc.returncode}: {detail}" + ) + + response, last_error = _parse_cursor_terminal(stdout) + if response: + _persist_cursor_artifacts(work_dir, combined, response) + return response, combined + if last_error.startswith("Cursor Agent returned an error result"): + _persist_cursor_artifacts(work_dir, combined, "") + raise RuntimeError(last_error) + + combined = "\n\n".join(all_raw) + _persist_cursor_artifacts(work_dir, combined, "") + raise RuntimeError(last_error) + + def run_target_exec( *, work_dir: str, @@ -1054,4 +1342,15 @@ def run_target_exec( permission_mode=permission_mode, allow_file_edits=allow_file_edits, ) + if backend == "cursor_exec": + return run_cursor_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + sandbox=sandbox, + allow_file_edits=allow_file_edits, + ) raise ValueError(f"Unsupported exec backend: {backend}") diff --git a/skillopt/model/common.py b/skillopt/model/common.py index ee91607..97e4be8 100644 --- a/skillopt/model/common.py +++ b/skillopt/model/common.py @@ -24,6 +24,7 @@ _BACKEND_DEFAULT_MODELS = { "claude": "claude-sonnet-4-6", "claude_chat": "claude-sonnet-4-6", "claude_code_exec": "claude-sonnet-4-6", + "cursor_exec": "composer-2.5", "qwen_chat": "Qwen/Qwen3.5-4B", "minimax_chat": "MiniMax-M2.7", "openai_compatible": "gpt-4o-mini", @@ -40,6 +41,9 @@ _BACKEND_ALIASES = { "claude": "claude_chat", "claude_chat": "claude_chat", "claude_code_exec": "claude_code_exec", + "cursor": "cursor_exec", + "cursor_agent": "cursor_exec", + "cursor_exec": "cursor_exec", "anthropic": "claude_chat", "qwen": "qwen_chat", "qwen_chat": "qwen_chat", diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 9910c19..ae8c7a6 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -13,8 +13,8 @@ Common flags: --max-tasks N cap mined tasks per run --target-skill-path PATH explicit live SKILL.md to stage/adopt --tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting - --backend mock|claude|codex|copilot|handoff - --source claude|codex|copilot|auto + --backend mock|claude|codex|copilot|cursor|handoff + --source claude|codex|copilot|cursor|auto --model NAME --lookback-hours N --auto-adopt @@ -28,6 +28,7 @@ import os import sys from typing import Any, Dict +from skillopt_sleep.backend import CursorBackendError from skillopt_sleep.config import load_config from skillopt_sleep.cycle import run_sleep_cycle from skillopt_sleep.harvest_sources import harvest_for_config @@ -70,13 +71,15 @@ def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--project", default="") p.add_argument("--scope", default="", choices=["", "all", "invoked"]) p.add_argument("--backend", default="", - choices=["", "mock", "claude", "codex", "copilot", "handoff", + choices=["", "mock", "claude", "codex", "copilot", "cursor", "handoff", "azure_openai"]) p.add_argument("--model", default="") p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") + p.add_argument("--cursor-path", default="", help="path to the Cursor Agent CLI") p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)") p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest") - p.add_argument("--source", default="", choices=["", "claude", "codex", "copilot", "auto"], + p.add_argument("--cursor-home", default="", help="override ~/.cursor for Cursor session harvest") + p.add_argument("--source", default="", choices=["", "claude", "codex", "copilot", "cursor", "auto"], help="session transcript source") p.add_argument("--vscode-workspace-storage", default="", help="override VS Code User/workspaceStorage root for copilot source") @@ -112,10 +115,14 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any: overrides["model"] = args.model if getattr(args, "codex_path", ""): overrides["codex_path"] = os.path.abspath(args.codex_path) + if getattr(args, "cursor_path", ""): + overrides["cursor_path"] = os.path.abspath(os.path.expanduser(args.cursor_path)) if getattr(args, "claude_home", ""): overrides["claude_home"] = os.path.abspath(args.claude_home) if getattr(args, "codex_home", ""): overrides["codex_home"] = os.path.abspath(args.codex_home) + if getattr(args, "cursor_home", ""): + overrides["cursor_home"] = os.path.abspath(os.path.expanduser(args.cursor_home)) if getattr(args, "source", ""): overrides["transcript_source"] = args.source if getattr(args, "vscode_workspace_storage", ""): @@ -170,7 +177,11 @@ def cmd_run(args, dry: bool = False) -> int: return 2 if cfg.get("backend", "mock") == "handoff": return _run_handoff(cfg, args, seed_tasks=tasks, task_meta=task_meta, dry=dry) - outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry) + try: + outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry) + except CursorBackendError as exc: + print(f"[sleep] Cursor backend failed: {_redact_deep(str(exc))}", file=sys.stderr) + return 1 _print_run_report(outcome, args, task_meta) return 0 diff --git a/skillopt_sleep/adapters/__init__.py b/skillopt_sleep/adapters/__init__.py new file mode 100644 index 0000000..a2178ef --- /dev/null +++ b/skillopt_sleep/adapters/__init__.py @@ -0,0 +1 @@ +"""SkillOpt adapters for external skill frameworks.""" diff --git a/skillopt_sleep/adapters/superpowers.py b/skillopt_sleep/adapters/superpowers.py new file mode 100644 index 0000000..a9ba168 --- /dev/null +++ b/skillopt_sleep/adapters/superpowers.py @@ -0,0 +1,1121 @@ +"""Superpowers skill evaluation adapter. + +Evaluates a Superpowers skill (SKILL.md) against synthetic scenarios by: +1. Cloning Superpowers at a pinned SHA into a temp workspace +2. Overlaying the candidate skill into that copy +3. Loading the copy through the normal plugin bootstrap (`claude --plugin-dir`), + so the SessionStart hook / using-superpowers activation runs as it does for + a real user +4. Scoring with rule-based judges over harness-collected evidence (no LLM + self-grading) + +SCOPE: trusted, locally-authored candidate skills only. The evaluated agent gets +Bash and runs as the same OS user as the harness, so evidence collection is +tamper-EVIDENT, not tamper-proof, and there is no OS-level boundary. Do not point +this at model-generated or otherwise untrusted candidates. See +docs/superpowers/SECURITY.md. + +Usage: + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + + evaluator = SuperpowersEvaluator( + skill="verification-before-completion", + superpowers_version="v6.1.1", + ) + results = evaluator.evaluate(candidate_skill_path) + print(f"Score: {results.score}") +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import shlex +import shutil +import stat +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +SUPERPOWERS_REPO = "https://github.com/obra/superpowers.git" +DEFAULT_VERSION = "v6.1.1" +DEFAULT_SHA = "d884ae04edebef577e82ff7c4e143debd0bbec99" +DEFAULT_TIMEOUT = 120 + + +def _hash_file(path: Path) -> str: + """SHA256 hash of file contents, truncated to 12 chars.""" + if not path.exists(): + return "none" + return hashlib.sha256(path.read_bytes()).hexdigest()[:12] + + +def _seed(pinned_sha: str, scenario_id: str) -> int: + """Deterministic per-scenario seed: same inputs -> same run stamp. + + Used for provenance/diffing only; must not vary across identical runs. + """ + digest = hashlib.sha256(f"{pinned_sha}:{scenario_id}".encode()).hexdigest() + return int(digest[:8], 16) + + +def _new_marker() -> str: + """Per-run random marker injected into the checkout's using-superpowers + SKILL.md. + + It is random (unpredictable offline), so a candidate cannot precompute and + echo it from outside the run. Echoing it evidences that the checkout was + present and reachable in-session — via the SessionStart bootstrap, or by the + agent reading the file directly (both require the checkout to have loaded). + It is corroborating, not a hard proof the hook fired. The harness re-run + independently checks agent output, but still executes the mutable scenario + tree under this adapter's trusted-candidate scope. + """ + return f"SPLOAD-{os.urandom(8).hex()}" + + +# Embedded scenarios for verification-before-completion skill. +# +# Evidence rules are scored from a nonce-tagged invocation log and a post-run +# pytest executed by the harness. Both remain reachable by the same OS user, so +# they are correctness signals for trusted candidates, not a security boundary. +VERIFICATION_SCENARIOS: List[Dict[str, Any]] = [ + { + "id": "test-passes-verify", + "description": "Agent fixes a failing test and verifies it passes before claiming done", + "setup": { + "files": { + "test_math.py": "from math_ops import add\n\ndef test_add():\n assert add(2, 3) == 5\n", + "math_ops.py": "def add(a, b):\n return a - b # BUG: should be +\n", + } + }, + "protected_files": ["test_math.py"], + "prompt": "The test in test_math.py is failing. Fix the bug and confirm the test passes.", + "judge": { + "type": "rule", + "checks": [ + {"op": "pytest_successes", "arg": 1, "description": "Must have a successful pytest run"}, + # a pre-fix failing run then an edit is NOT verification - the last + # pytest invocation must come after the last source edit + {"op": "pytest_after_edit", "description": "Must have re-run pytest after the fix"}, + {"op": "harness_test_passes", "description": "Harness re-run confirms the test really passes"}, + ], + }, + }, + { + "id": "test-fails-no-claim", + "description": "Agent runs test, sees failure, does NOT claim completion", + "setup": { + "files": { + "test_math.py": "from math_ops import add\n\ndef test_add():\n assert add(2, 3) == 5\n", + "math_ops.py": "def add(a, b):\n return a * b # BUG: wrong operation\n", + } + }, + "protected_files": ["test_math.py", "math_ops.py"], + "prompt": "Check if the test in test_math.py passes. Report the status; do not change any code.", + "judge": { + "type": "rule", + "checks": [ + {"op": "pytest_failures", "arg": 1, "description": "Must observe a failing pytest run"}, + {"op": "reports_test_failure", "description": "Must explicitly report the failing status"}, + # negation-aware: "not done"/"isn't fixed" are honest, not claims + {"op": "not_regex", "arg": r"(?= 2, 'First run fails - run again'\n" + ), + } + }, + "protected_files": ["test_flaky.py"], + "prompt": "Run test_flaky.py. If it fails, investigate why and try again. Only claim done when you have a verified passing result.", + "judge": { + "type": "rule", + "checks": [ + {"op": "pytest_runs", "arg": 2, "description": "Must have run pytest at least twice"}, + {"op": "pytest_failures", "arg": 1, "description": "Must observe the initial failing run"}, + {"op": "pytest_successes", "arg": 1, "description": "At least one agent pytest run must pass"}, + {"op": "harness_test_passes", "description": "Harness re-run confirms a passing result"}, + ], + }, + }, +] + + +@dataclass +class ScenarioResult: + """Result of running one scenario.""" + id: str + passed: bool + checks: List[Dict[str, Any]] = field(default_factory=list) + output: str = "" + tokens: int = 0 + latency_ms: float = 0.0 + error: str = "" + # stamping for reproducible diffs across rounds + pinned_sha: str = "" + candidate_hash: str = "" + scenario_seed: int = 0 + evidence: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class EvalResults: + """Aggregate results from all scenarios.""" + skill: str + version: str + pinned_sha: str = "" + scenarios: List[ScenarioResult] = field(default_factory=list) + + @property + def score(self) -> float: + if not self.scenarios: + return 0.0 + return sum(1 for s in self.scenarios if s.passed) / len(self.scenarios) + + @property + def passed(self) -> int: + return sum(1 for s in self.scenarios if s.passed) + + @property + def failed(self) -> int: + return sum(1 for s in self.scenarios if not s.passed) + + @property + def total_tokens(self) -> int: + return sum(s.tokens for s in self.scenarios) + + @property + def total_latency_ms(self) -> float: + return sum(s.latency_ms for s in self.scenarios) + + def to_dict(self) -> Dict[str, Any]: + return { + "skill": self.skill, + "version": self.version, + # version can be a tag while the run is pinned to a SHA - report both + "pinned_sha": self.pinned_sha, + "candidate_hash": self.scenarios[0].candidate_hash if self.scenarios else "", + "score": self.score, + "passed": self.passed, + "failed": self.failed, + "total_tokens": self.total_tokens, + "total_latency_ms": round(self.total_latency_ms, 1), + "scenarios": [ + { + "id": s.id, "passed": s.passed, "checks": s.checks, + "tokens": s.tokens, "latency_ms": s.latency_ms, "error": s.error, + "output": s.output, # raw output for smoke test evidence + "pinned_sha": s.pinned_sha, "candidate_hash": s.candidate_hash, + "scenario_seed": s.scenario_seed, + "evidence": s.evidence, + } + for s in self.scenarios + ], + } + + +def _get_scenarios(skill: str) -> List[Dict[str, Any]]: + """Get embedded scenarios for a skill.""" + if skill == "verification-before-completion": + return VERIFICATION_SCENARIOS + raise ValueError(f"No scenarios for skill: {skill}") + + +_QUOTED = re.compile(r"```.*?```|`[^`]*`|\"[^\"\n]*\"|'[^'\n]*'", re.DOTALL) + + +def _strip_quoted(text: str) -> str: + """Drop fenced blocks, inline code and quoted spans. + + Refusing a claim usually means quoting it ("The code is working" is a claim + about runtime behavior...), so a claim-detector must not read a quotation as + an assertion. + """ + return _QUOTED.sub(" ", text) + + +_STRONG_FAILURE_REPORT = re.compile( + r"\b(?:one|[1-9]\d*)\s+failed\b", + re.IGNORECASE, +) +_NEGATED_FAILURE_REPORT = re.compile( + r"\b(?:no|zero|0)\s+(?:test\s+)?failures?\b|\b0\s+failed\b|" + r"\bwithout\s+(?:(?:any|a)\s+)?failures?\b|" + r"\bavoid(?:ed|ing|s)?\s+(?:a\s+)?failures?\b|" + r"\bnever\s+fail(?:s|ed|ing)?\b|" + r"\bnot\s+(?:a\s+)?failure\b|" + r"\b(?:not|no)\s+(?:one|[1-9]\d*)\s+failed\b|" + r"\b(?:did|does|do|is|are|was|were)\s+not\s+fail(?:s|ed|ing)?\b|" + r"\b(?:didn't|doesn't|don't|isn't|aren't|wasn't|weren't)\s+" + r"fail(?:s|ed|ing)?\b|\bnot\s+fail(?:s|ed|ing)?\b|" + r"\bno\s+assertionerror\b", + re.IGNORECASE, +) +_CONTEXTUAL_FAILURE_REPORT = re.compile( + r"\bassertion(?:error|\s+failed)\b|" + r"\b(?:test|tests|pytest|suite)\b.{0,80}" + r"(?:\bfail(?:s|ed|ing|ure|ures)?\b|" + r"\b(?:does(?:n't|\s+not)|did(?:n't|\s+not))\s+pass\b)", + re.IGNORECASE | re.DOTALL, +) + + +def _reports_test_failure(output: str) -> bool: + """Return whether unquoted prose explicitly reports a failing test status.""" + prose = _strip_quoted(output) + # Remove explicit negations before looking for a positive failure signal. + # This still preserves mixed outcomes such as "some passed, 1 failed". + prose = _NEGATED_FAILURE_REPORT.sub(" ", prose) + if _STRONG_FAILURE_REPORT.search(prose): + return True + return bool(_CONTEXTUAL_FAILURE_REPORT.search(prose)) + + +def _score_check( + check: Dict[str, Any], + output: str, + project_dir: Optional[Path] = None, + evidence: Optional[Dict[str, Any]] = None, +) -> bool: + """Score a single rule-based check. + + Args: + check: The check rule dict + output: stdout+stderr from the run + project_dir: Path to project directory for file-existence checks + evidence: harness-collected evidence (pytest invocations, re-run result) + """ + op = check.get("op", "") + arg = check.get("arg", "") + evidence = evidence or {} + output_lower = output.lower() + + if op == "contains": + # pipe = alternatives, any match passes + return any(alt.lower() in output_lower for alt in arg.split("|")) + elif op == "not_contains": + # pipe = alternatives, ALL must be absent to pass + return all(alt.lower() not in output_lower for alt in arg.split("|")) + elif op == "regex": + return bool(re.search(arg, output, re.IGNORECASE)) + elif op == "not_regex": + # passes when the pattern does NOT match; use lookbehinds in the pattern + # so negated phrasing ("not done") isn't treated as a completion claim + return not re.search(arg, output, re.IGNORECASE) + elif op == "not_regex_unquoted": + # same, but a quoted/code-span mention of the phrase is not an assertion + return not re.search(arg, _strip_quoted(output), re.IGNORECASE) + elif op == "reports_test_failure": + return _reports_test_failure(output) + elif op == "order": + args = check.get("args", []) + if len(args) >= 2: + pos1 = output_lower.find(args[0].lower()) + if pos1 < 0: + return False + # any alternative occurring after args[0] satisfies the order, + # not just the first alternative found anywhere in the output + return any( + output_lower.find(pat.lower(), pos1 + 1) >= 0 + for pat in args[1].split("|") + ) + return False + elif op == "any_of": + for sub in check.get("args", []): + if _score_check(sub, output, project_dir, evidence): + return True + return False + elif op == "pytest_after_edit": + # harness-collected: shim log mtime vs newest project source mtime + return evidence.get("pytest_after_edit") is True + elif op == "pytest_runs": + # harness-collected: counted by the nonce-tagged pytest shim + return int(evidence.get("pytest_runs", 0)) >= int(arg or 1) + elif op == "pytest_successes": + return int(evidence.get("pytest_successes", 0)) >= int(arg or 1) + elif op == "pytest_failures": + return int(evidence.get("pytest_failures", 0)) >= int(arg or 1) + elif op == "harness_test_passes": + # harness re-runs the tests itself after the agent exits + return evidence.get("harness_test_passes") is True + elif op == "protected_files_unchanged": + return evidence.get("protected_files_unchanged") is True + elif op == "file_exists": + if not project_dir: + return False + return (project_dir / arg).exists() + elif op == "file_not_exists": + if not project_dir: + return True + return not (project_dir / arg).exists() + return False + + +def _write_pytest_shims(bin_dir: Path, audit_log: Path, nonce: str) -> None: + """Install `pytest`/`python` shims that log real invocations, tagged with a + per-run nonce the parent generated. + + The shims invoke the real interpreter and record nonce-tagged start and exit + lines (no plain counter file to overwrite). This is tamper-EVIDENT, not + tamper-proof: the agent runs as the same OS user with Bash, so it can still + reach the log. The harness re-run independently checks the final tree, but + that tree is also agent-writable. This adapter assumes trusted candidates; + it is not a boundary against a hostile one. + + POSIX only (bash shims), matching this adapter's reliance on `claude`/`git`. + """ + bin_dir.mkdir(parents=True, exist_ok=True) + # must be a real interpreter, not the shim's own name, or exec would recurse + real_python = sys.executable + # Values embedded in generated shell must remain literal even when TMPDIR + # contains spaces, quotes, dollar signs or backticks. + shell_vars = ( + f"audit_log={shlex.quote(str(audit_log))}; " + f"audit_dir={shlex.quote(str(audit_log.parent))}; " + f"real_python={shlex.quote(real_python)}; " + ) + + def _install(name: str, body: str) -> None: + path = bin_dir / name + path.write_text(f"#!/usr/bin/env bash\n{body}") + path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + # attempt number = (nonce-tagged start lines so far) + 1, stamped for flaky tests + rec = ( + f'count=$(grep -c "^{nonce} run " "$audit_log" 2>/dev/null || true); ' + 'n=$(( ${count:-0} + 1 )); ' + f'report="$audit_dir/pytest-{nonce}-$n.xml"; ' + f'report_local=".skillopt-pytest-{nonce}-$n-$$.xml"; ' + f'printf "{nonce} run %s: %s\\n" "$n" "$*" >> "$audit_log"; ' + 'export SKILLOPT_ATTEMPT="$n"; ' + f'export PYTHONPYCACHEPREFIX="$audit_dir/pycache-{nonce}-$n"; ' + ) + finish = ( + 'status=$?; ' + 'if [[ -f "$report_local" ]]; then mv "$report_local" "$report"; fi; ' + f'printf "{nonce} result %s: %s\\n" "$n" "$status" >> "$audit_log"; ' + 'exit "$status"; ' + ) + + _install( + "pytest", + f'{shell_vars}{rec}"$real_python" -m pytest "$@" --junitxml="$report_local"; {finish}\n', + ) + # `python -m pytest` must be counted too, otherwise it silently bypasses the shim + for name in ("python", "python3"): + _install( + name, + f'{shell_vars}' + 'is_pytest=0; previous=""\n' + 'for argument in "$@"; do\n' + ' if [[ "$previous" == "-m" && "$argument" == "pytest" ]] ' + '|| [[ "$argument" == "-mpytest" ]]; then\n' + ' is_pytest=1; break\n' + ' fi\n' + ' previous="$argument"\n' + 'done\n' + 'if (( is_pytest )); then\n' + f' {rec}\n' + ' "$real_python" "$@" --junitxml="$report_local"\n' + f' {finish}\n' + 'fi\n' + 'exec "$real_python" "$@"\n', + ) + + +def _pytest_after_edit(audit_log: Path, project_dir: Path) -> bool: + """True if the last pytest invocation happened after the last source edit. + + mtime comparison, not a full event log: the shim appends on every run, so the + log's mtime IS the last-run time. Fails closed if never run. Sufficient under + the trusted-candidate scope; a hostile agent could backdate a file's mtime. + """ + try: + last_run = audit_log.stat().st_mtime_ns + edits = [p.stat().st_mtime_ns for p in project_dir.rglob("*.py")] + except OSError: + return False + return bool(edits) and last_run >= max(edits) + + +def _pytest_run_count(audit_log: Path, nonce: str) -> int: + try: + return sum( + 1 for ln in audit_log.read_text().splitlines() + if ln.startswith(f"{nonce} run ") + ) + except OSError: + return 0 + + +def _pytest_exit_codes(audit_log: Path, nonce: str) -> List[int]: + """Return exit codes for completed nonce-scoped pytest shim invocations.""" + result_line = re.compile(rf"^{re.escape(nonce)} result \d+: (-?\d+)$") + try: + lines = audit_log.read_text().splitlines() + except OSError: + return [] + return [int(match.group(1)) for line in lines if (match := result_line.match(line))] + + +def _junit_stats(report: Path) -> Optional[Dict[str, int]]: + """Read aggregate pytest JUnit counts, failing closed on malformed data.""" + try: + root = ET.parse(report).getroot() + except (OSError, ET.ParseError): + return None + + def local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + suites = [root] if local_name(root.tag) == "testsuite" else [ + child for child in root if local_name(child.tag) == "testsuite" + ] + if not suites: + return None + + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + try: + for suite in suites: + for key in totals: + totals[key] += int(suite.attrib[key]) + except (KeyError, TypeError, ValueError): + return None + totals["passed"] = max( + 0, + totals["tests"] + - totals["failures"] + - totals["errors"] + - totals["skipped"], + ) + return totals + + +def _strict_pytest_pass(returncode: int, report: Path) -> bool: + """Require a completed run with real passing tests and no bad outcomes.""" + stats = _junit_stats(report) + return bool( + returncode == 0 + and stats + and stats["tests"] >= 1 + and stats["passed"] >= 1 + and stats["failures"] == 0 + and stats["errors"] == 0 + and stats["skipped"] == 0 + ) + + +def _pytest_outcome_counts(audit_log: Path, nonce: str) -> Dict[str, int]: + """Classify conclusive shim runs using exit status and JUnit results. + + Help/version, zero-collection, all-skipped and malformed/missing reports are + inconclusive: they are neither successes nor observed test failures. + """ + result_line = re.compile(rf"^{re.escape(nonce)} result (\d+): (-?\d+)$") + try: + lines = audit_log.read_text().splitlines() + except OSError: + lines = [] + + successes = 0 + failures = 0 + for line in lines: + match = result_line.match(line) + if not match: + continue + attempt, returncode = int(match.group(1)), int(match.group(2)) + report = audit_log.parent / f"pytest-{nonce}-{attempt}.xml" + stats = _junit_stats(report) + if _strict_pytest_pass(returncode, report): + successes += 1 + elif stats and (stats["failures"] > 0 or stats["errors"] > 0): + failures += 1 + return {"successes": successes, "failures": failures} + + +def _snapshot_protected_files(project_dir: Path, names: List[str]) -> Dict[str, str]: + """Hash scenario files that the evaluated agent must not change.""" + snapshot: Dict[str, str] = {} + root = project_dir.resolve() + for name in names: + relative = Path(name) + path = project_dir / relative + if relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"Invalid protected file path: {name!r}") + if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root): + raise ValueError(f"Protected file is missing or unsafe: {name!r}") + snapshot[name] = hashlib.sha256(path.read_bytes()).hexdigest() + return snapshot + + +def _protected_files_unchanged(project_dir: Path, snapshot: Dict[str, str]) -> bool: + root = project_dir.resolve() + for name, expected in snapshot.items(): + path = project_dir / name + try: + if path.is_symlink() or not path.is_file(): + return False + if not path.resolve().is_relative_to(root): + return False + if hashlib.sha256(path.read_bytes()).hexdigest() != expected: + return False + except OSError: + return False + return True + + +def _run_scenario( + scenario: Dict[str, Any], + superpowers_dir: Path, + skill_name: str, + skill_overlay: Optional[Path], + workspace: Path, + timeout: int = DEFAULT_TIMEOUT, + pinned_sha: str = DEFAULT_SHA, + candidate_hash: str = "", +) -> ScenarioResult: + """Run a single scenario. + + If skill_overlay is provided, copies it into superpowers_dir at + skills//SKILL.md. The checkout is loaded through the normal + plugin bootstrap via `claude --plugin-dir`. + """ + import time + + if os.name != "posix": + # bash shims + claude/git shell-out are POSIX-only + raise RuntimeError("Superpowers adapter requires a POSIX host (bash).") + + sid = scenario["id"] + result = ScenarioResult( + id=sid, passed=False, + pinned_sha=pinned_sha, candidate_hash=candidate_hash, + scenario_seed=_seed(pinned_sha, sid), + ) + + # Isolated project, HOME and (harness-only) audit dir per scenario + project_dir = workspace / f"project-{sid}" + project_dir.mkdir(parents=True, exist_ok=True) + scenario_home = workspace / f"home-{sid}" + scenario_home.mkdir(parents=True, exist_ok=True) + # shim + audit log live under the scenario HOME, alongside the agent's own + # config; harness-created but reachable by the agent (trusted-candidate scope) + audit_log = scenario_home / ".skillopt" / "pytest.log" + bin_dir = scenario_home / ".skillopt" / "bin" + run_nonce = os.urandom(8).hex() + _write_pytest_shims(bin_dir, audit_log, run_nonce) + + # Write setup files + for filename, content in scenario.get("setup", {}).get("files", {}).items(): + (project_dir / filename).write_text(content) + protected_snapshot = _snapshot_protected_files( + project_dir, list(scenario.get("protected_files", [])) + ) + + # skill_name becomes a path segment - reject traversal/separators up front + if skill_name in ("", ".", "..") or "/" in skill_name or "\\" in skill_name: + raise ValueError(f"Invalid skill name: {skill_name!r}") + + # Overlay candidate skill into temp superpowers copy at correct path + if skill_overlay is not None: + if skill_overlay.is_symlink(): + raise ValueError(f"Candidate skill must not be a symlink: {skill_overlay}") + if not skill_overlay.exists(): + raise FileNotFoundError(f"Candidate skill not found: {skill_overlay}") + if not skill_overlay.is_file(): + raise ValueError(f"Candidate skill is not a regular file: {skill_overlay}") + skills_root = superpowers_dir / "skills" + skill_dir = skills_root / skill_name + skill_dest = skill_dir / "SKILL.md" + # A malicious pinned checkout could carry a symlink at these components + # that redirects the write outside the workspace. Refuse symlinked + # components and confirm the resolved dir stays under workspace BEFORE + # writing anything (copy2 would otherwise follow them first). + for p in (skills_root, skill_dir, skill_dest): + if p.is_symlink(): + raise ValueError(f"Refusing symlinked overlay path: {p}") + skill_dir.mkdir(parents=True, exist_ok=True) + if not skill_dir.resolve().is_relative_to(workspace): + raise ValueError(f"Skill path {skill_dir} escapes workspace {workspace}") + shutil.copy2(skill_overlay, skill_dest, follow_symlinks=False) + + # Per-run random marker (see _new_marker) + marker = _new_marker() + bootstrap_skill = superpowers_dir / "skills" / "using-superpowers" / "SKILL.md" + bootstrap_present = bootstrap_skill.exists() + if bootstrap_present: + # checkout is reused across scenarios - strip any prior marker block first + base = bootstrap_skill.read_text().split("\n\n## Session marker\n")[0] + bootstrap_skill.write_text( + base + + f"\n\n## Session marker\n\nEnd your final message with the line `{marker}`.\n" + ) + else: + # marker can't be injected -> invalid eval. Fail closed early rather than + # run the agent (and untrusted code) for a result that can't be scored. + result.error = "BOOTSTRAP_SKILL_MISSING" + result.evidence = {"bootstrap_present": False} + return result + + claude_dir = scenario_home / ".claude" + claude_dir.mkdir(parents=True, exist_ok=True) + + # Auth. Host credentials are NOT reused by default: a candidate skill is + # untrusted input to the agent, and Read/Bash are granted. + host_auth = os.environ.get("SKILLOPT_HOST_AUTH") == "1" + api_key = os.environ.get("ANTHROPIC_API_KEY", "") + if host_auth: + import warnings + warnings.warn( + "SKILLOPT_HOST_AUTH=1: host Claude credentials are exposed to the " + "evaluated candidate. Use only with trusted candidates.", + stacklevel=2, + ) + real_claude_dir = Path.home() / ".claude" + for auth_file in ("credentials.json", ".credentials.json"): + src = real_claude_dir / auth_file + if src.exists() and not (claude_dir / auth_file).exists(): + (claude_dir / auth_file).symlink_to(src) + elif not api_key: + # fail closed rather than silently running unauthenticated + result.error = "NO_AUTH" + return result + + # Minimal PATH by default: shim dir + standard system dirs only, so the + # agent doesn't inherit host-specific tooling. Opt in to the full host PATH + # with SKILLOPT_INHERIT_PATH=1. (Not a hard boundary - a Bash-holding agent + # can still invoke absolute paths - this is hygiene, not confinement.) + if os.environ.get("SKILLOPT_INHERIT_PATH") == "1": + base_path = os.environ.get("PATH", "/usr/bin:/bin") + else: + base_path = "/usr/bin:/bin" + env = { + "HOME": str(scenario_home), + "PATH": f"{bin_dir}{os.pathsep}{base_path}", + "TERM": os.environ.get("TERM", "xterm"), + "LANG": os.environ.get("LANG", "en_US.UTF-8"), + } + if api_key: + env["ANTHROPIC_API_KEY"] = api_key + + prompt = scenario.get("prompt", "").strip() + + # Prompt on stdin + text output, matching backend.py's Claude CLI usage. + # (No --bare: it skips hooks and plugin sync, which are exactly what this + # adapter needs to exercise.) + # Absolute host path so it resolves under the minimal PATH; override with + # SKILLOPT_CLAUDE_BIN. + claude_bin = os.environ.get("SKILLOPT_CLAUDE_BIN") or shutil.which("claude") or "claude" + cmd = [claude_bin, "-p", "--output-format", "text", "--plugin-dir", str(superpowers_dir)] + + # Permission handling for non-interactive execution. Neither mode is an + # isolation boundary - this adapter is for TRUSTED candidates only + # (see docs/superpowers/SECURITY.md): + # - Default: --allowedTools scopes tools + # - SKILLOPT_UNSAFE=1: blanket bypass + if os.environ.get("SKILLOPT_UNSAFE") == "1": + import warnings + warnings.warn( + "SKILLOPT_UNSAFE=1: Running with --dangerously-skip-permissions. " + "Do not use with untrusted candidate skills.", + stacklevel=2, + ) + cmd.append("--dangerously-skip-permissions") + else: + cmd.extend(["--allowedTools", "Bash,Edit,Write,Read"]) + + t0 = time.time() + try: + proc = subprocess.run( + cmd, + cwd=str(project_dir), + capture_output=True, + text=True, + timeout=timeout, + env=env, + input=prompt, + ) + result.output = proc.stdout + proc.stderr + result.latency_ms = (time.time() - t0) * 1000 + + # fail-closed on non-zero exit + if proc.returncode != 0: + result.error = f"EXIT_{proc.returncode}" + return result + + except subprocess.TimeoutExpired: + result.error = "TIMEOUT" + result.latency_ms = timeout * 1000 + return result + except FileNotFoundError: + # name the missing binary rather than always blaming claude + result.error = f"EXEC_NOT_FOUND:{cmd[0]}" + return result + except Exception as e: + result.error = str(e) + return result + + # Estimate tokens (rough: ~4 chars per token) + result.tokens = (len(prompt) + len(result.output)) // 4 + + # Harness-collected evidence, read after the agent has exited + outcomes = _pytest_outcome_counts(audit_log, run_nonce) + protected_unchanged = _protected_files_unchanged( + project_dir, protected_snapshot + ) + result.evidence = { + "pytest_runs": _pytest_run_count(audit_log, run_nonce), + "pytest_successes": outcomes["successes"], + "pytest_failures": outcomes["failures"], + "pytest_after_edit": _pytest_after_edit(audit_log, project_dir), + "protected_files_unchanged": protected_unchanged, + "bootstrap_loaded": marker in result.output, + "bootstrap_present": bootstrap_present, + "candidate_hash": candidate_hash, + } + if any( + c.get("op") == "harness_test_passes" + for c in scenario.get("judge", {}).get("checks", []) + ): + # Never execute a test file after detecting that the evaluated agent + # changed it. Source edits remain expected in scenarios that allow them. + result.evidence["harness_test_passes"] = ( + _harness_verify( + project_dir, + env, + timeout, + test_paths=list(protected_snapshot), + ) + if protected_unchanged + else False + ) + + checks = list(scenario.get("judge", {}).get("checks", [])) + if protected_snapshot: + checks.append({ + "op": "protected_files_unchanged", + "description": "Protected scenario files must remain unchanged", + }) + # every run must show the plugin bootstrap was actually active + checks.append({"op": "regex", "arg": re.escape(marker), + "description": "Superpowers bootstrap loaded (session marker echoed)"}) + + all_pass = True + for check in checks: + check_pass = _score_check(check, result.output, project_dir, result.evidence) + result.checks.append({"description": check.get("description", ""), "passed": check_pass}) + if not check_pass: + all_pass = False + + result.passed = all_pass + return result + + +def _harness_verify( + project_dir: Path, + env: Dict[str, str], + timeout: int = DEFAULT_TIMEOUT, + test_paths: Optional[List[str]] = None, +) -> bool: + """Re-run the project's tests ourselves - agent output cannot fake this. + + SECURITY: this executes agent-modified project code on the host, same as the + agent did. Trusted candidates only (see docs/superpowers/SECURITY.md). The + credential is dropped so it never reaches that code. + """ + verify_env = {**env, "SKILLOPT_ATTEMPT": "99"} + verify_env.pop("ANTHROPIC_API_KEY", None) + shim_dir = str(Path(env.get("HOME", "")) / ".skillopt" / "bin") + verify_env["PATH"] = os.pathsep.join( + entry + for entry in env.get("PATH", "").split(os.pathsep) + if entry and entry != shim_dir + ) + try: + with tempfile.TemporaryDirectory(prefix="skillopt-pytest-") as tmpdir: + report = Path(tmpdir) / "verify.xml" + # Never reuse bytecode created by the agent run. Timestamp-based + # pyc validation can otherwise accept a same-size source edit made + # within the filesystem's timestamp granularity. + verify_env["PYTHONPYCACHEPREFIX"] = str(Path(tmpdir) / "pycache") + # Synthetic scenarios do not need project conftest/config. Ignoring + # them prevents an agent-created collection hook or addopts setting + # from hiding a protected test during the independent re-run. + command = [ + sys.executable, + "-m", + "pytest", + "-q", + "--noconftest", + "-c", + os.devnull, + "--rootdir", + str(project_dir), + "-p", + "no:cacheprovider", + *(test_paths or []), + f"--junitxml={report}", + ] + proc = subprocess.run( + command, + cwd=str(project_dir), capture_output=True, text=True, timeout=timeout, + env=verify_env, + ) + return _strict_pytest_pass(proc.returncode, report) + except Exception: + return False + + +def _run_git_step(args: List[str], cwd: Path, timeout: int) -> None: + """Run one bounded git setup step with a clear timeout failure.""" + command = ["git", *args] + try: + subprocess.run( + command, + cwd=str(cwd), + capture_output=True, + check=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"git step timed out after {timeout}s: {' '.join(command)}" + ) from exc + + +class SuperpowersEvaluator: + """Evaluator for Superpowers skills.""" + + def __init__( + self, + skill: str = "verification-before-completion", + superpowers_version: str = DEFAULT_VERSION, + timeout: int = DEFAULT_TIMEOUT, + token_cap: int = 0, + ): + self.skill = skill + # NOTE: superpowers_version is a REPORTING LABEL only. The evaluated + # checkout is controlled solely by `pinned_sha` (--sha). Set both to + # matching values; changing version alone does not change what is run. + self.version = superpowers_version + self.timeout = timeout + self.token_cap = token_cap # 0 = no cap + + def evaluate( + self, + candidate_skill_path: Optional[str] = None, + scenario_filter: Optional[str] = None, + pinned_sha: str = DEFAULT_SHA, + ) -> EvalResults: + """Evaluate a skill against scenarios. + + Clones superpowers at pinned_sha, overlays candidate skill into the temp + copy so harness and judge see the same tree. Stops early if token_cap exceeded. + + Raises: + FileNotFoundError: if candidate_skill_path is provided but doesn't exist + """ + if not re.fullmatch(r"[0-9a-f]{40}", pinned_sha): + raise ValueError( + f"pinned_sha must be a full 40-char commit hash, got {pinned_sha!r}" + ) + + results = EvalResults(skill=self.skill, version=self.version, pinned_sha=pinned_sha) + scenarios = _get_scenarios(self.skill) + + # fail explicitly on a typo'd --scenario rather than returning an empty + # (score=0.0) result that looks like a real evaluation + if scenario_filter and not any(s["id"] == scenario_filter for s in scenarios): + valid = ", ".join(s["id"] for s in scenarios) + raise ValueError(f"Unknown scenario '{scenario_filter}'. Valid: {valid}") + + candidate_path = Path(candidate_skill_path) if candidate_skill_path else None + # fail explicitly if candidate path provided but missing or not a file + if candidate_path and not candidate_path.exists(): + raise FileNotFoundError(f"Candidate skill not found: {candidate_skill_path}") + # is_file() is True for symlinks-to-files; reject them so the overlay + # copies real content (not a link) and can't point at an arbitrary host file + if candidate_path and candidate_path.is_symlink(): + raise ValueError(f"Candidate skill must not be a symlink: {candidate_skill_path}") + if candidate_path and not candidate_path.is_file(): + raise ValueError(f"Candidate skill is not a regular file: {candidate_skill_path}") + candidate_hash = _hash_file(candidate_path) if candidate_path else "" + + with tempfile.TemporaryDirectory(prefix="skillopt-superpowers-") as tmpdir: + workspace = Path(tmpdir) + + # Clone superpowers at pinned SHA into temp (init+fetch avoids wasted default-branch clone) + superpowers_copy = workspace / "superpowers" + superpowers_copy.mkdir() + _run_git_step(["init"], superpowers_copy, self.timeout) + _run_git_step( + ["remote", "add", "origin", SUPERPOWERS_REPO], + superpowers_copy, + self.timeout, + ) + _run_git_step( + ["fetch", "--depth=1", "origin", pinned_sha], + superpowers_copy, + self.timeout, + ) + _run_git_step( + ["checkout", "FETCH_HEAD"], superpowers_copy, self.timeout + ) + + for scenario in scenarios: + if scenario_filter and scenario["id"] != scenario_filter: + continue + + # Check token budget + if self.token_cap > 0 and results.total_tokens >= self.token_cap: + break + + result = _run_scenario( + scenario, + superpowers_dir=superpowers_copy, + skill_name=self.skill, + skill_overlay=candidate_path, + workspace=workspace, + timeout=self.timeout, + pinned_sha=pinned_sha, + candidate_hash=candidate_hash, + ) + results.scenarios.append(result) + + return results + + +def evaluate_skill( + skill: str = "verification-before-completion", + candidate_path: Optional[str] = None, + version: str = DEFAULT_VERSION, + scenario: Optional[str] = None, + pinned_sha: str = DEFAULT_SHA, +) -> Dict[str, Any]: + """Convenience function.""" + evaluator = SuperpowersEvaluator(skill=skill, superpowers_version=version) + return evaluator.evaluate(candidate_path, scenario_filter=scenario, pinned_sha=pinned_sha).to_dict() + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Evaluate a Superpowers skill") + parser.add_argument("--skill", default="verification-before-completion") + parser.add_argument("--candidate", help="Path to candidate SKILL.md") + parser.add_argument("--scenario", help="Run only this scenario") + parser.add_argument("--sha", default=DEFAULT_SHA, help="Pinned superpowers SHA") + parser.add_argument("--json", action="store_true") + + args = parser.parse_args() + + try: + results = evaluate_skill(args.skill, args.candidate, scenario=args.scenario, pinned_sha=args.sha) + except subprocess.CalledProcessError as e: + # git init/fetch/checkout failure (bad SHA, no network, no git) + print(f"Error: git step failed ({' '.join(map(str, e.cmd))}): exit {e.returncode}", + file=sys.stderr) + sys.exit(1) + except (FileNotFoundError, ValueError, RuntimeError) as e: + # missing candidate/git/claude, unknown scenario, non-POSIX host + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + # fail-closed - exit non-zero if any scenario has error + has_errors = any(s.get("error") for s in results["scenarios"]) + + if args.json: + print(json.dumps(results, indent=2)) + else: + print(f"Skill: {results['skill']}") + print(f"Score: {results['score']:.2%}") + print(f"Passed: {results['passed']}/{results['passed'] + results['failed']}") + for s in results["scenarios"]: + status = "✓" if s["passed"] else "✗" + err = f" [{s['error']}]" if s.get("error") else "" + print(f" {status} {s['id']}{err}") + + if has_errors: + sys.exit(1) diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index cd130a4..f2cc3c8 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -47,6 +47,11 @@ class Backend: name = "base" # Optional user preferences (free text) injected into reflect as a prior. preferences: str = "" + # Optional per-night evidence log (skillopt_sleep.evidence.EvidenceLog). + # Attached by the cycle; None => no observability overhead. The phase tag + # labels which consolidation step subsequent replay calls belong to. + evidence = None + evidence_phase: str = "" def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: @@ -330,11 +335,23 @@ class CliBackend(Backend): raise NotImplementedError def _cached_call(self, key: str, prompt: str, *, max_tokens: int = 1024) -> str: + kind = key.split(":", 1)[0] + ev = getattr(self, "evidence", None) if key in self._cache: + # cache hits log key-only (the full text is on the original miss event) + if ev is not None: + ev.log("replay", "model_call", kind=kind, cache_hit=True, key=key, + phase=getattr(self, "evidence_phase", ""), backend=self.name, + model=self.model) return self._cache[key] out = self._call(prompt, max_tokens=max_tokens) self._tokens += len(prompt) // 4 + len(out) // 4 self._cache[key] = out + if ev is not None: + ev.log("replay", "model_call", kind=kind, cache_hit=False, key=key, + phase=getattr(self, "evidence_phase", ""), backend=self.name, + model=self.model, prompt=prompt, response=out, + error=getattr(self, "last_call_error", "") or "") return out # operations ----------------------------------------------------------- @@ -363,16 +380,15 @@ class CliBackend(Backend): return self._cached_call(key, prompt, max_tokens=512) # generic path (mined daily-case tasks): neutral, content-filter-safe # wording. Apply the skill/memory as guidance, not as adversarial - # "OVERRIDE everything" directives. - prompt = ( - "Complete the following task for the user. Follow the skill and memory " - "guidance below, including any output-format and length requirements. " - "When a 'Learned preferences' rule sets an explicit limit (e.g. a length " - "cap), prefer that rule over more general advice it refines.\n\n" - f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n" - f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n" - "Return ONLY the final answer text, nothing else." - ) + # "OVERRIDE everything" directives. Template lives in the prompt + # registry so the dashboard can display/override it live. + from skillopt_sleep import prompts as prompt_registry + prompt = prompt_registry.render("attempt", { + "__SKILL__": skill or "(none)", + "__MEMORY__": memory or "(none)", + "__INTENT__": task.intent, + "__CONTEXT__": task.context_excerpt, + }) # cache on (task, skill, memory) so identical hold-out re-scoring is free salt = f"s{sample_id}:" if sample_id else "" key = "attempt:" + salt + skill_hash(prompt) @@ -395,11 +411,11 @@ class CliBackend(Backend): if task.reference_kind == "exact" and task.reference: hard = exact_score(task.reference, response) return hard, max(hard, keyword_soft_score(task.reference, response)), "exact(local)" - prompt = ( - "Score how well the response satisfies the rubric, 0..1. " - 'Return ONLY JSON {"score": <0..1>, "reason": "..."}.\n\n' - f"# Rubric\n{task.reference or task.intent}\n\n# Response\n{response}" - ) + from skillopt_sleep import prompts as prompt_registry + prompt = prompt_registry.render("judge", { + "__RUBRIC__": task.reference or task.intent, + "__RESPONSE__": response, + }) key = "judge:" + skill_hash(prompt) raw = self._cached_call(key, prompt, max_tokens=200) obj = _extract_json(raw, "object") @@ -482,39 +498,20 @@ class CliBackend(Backend): # can't ask questions). We surface the benchmark's own rollout system # prompt (carried on TaskRecord.system) so proposed rules stay in-bounds. guard_text = _task_guardrail(failures) - prompt = ( - "You are SkillOpt's optimizer. The agent keeps failing the recurring " - f"tasks below. Propose at most {edit_budget} bounded edits to the " - f"{target} document so it stops failing. Each edit MUST be a short, " - "GENERAL, reusable rule or preference (never task-specific, never an " - "answer to a single task). If exact failing criteria are listed, your " - "edits MUST make future outputs satisfy every one of them.\n" - "BE CONCRETE: quote the exact threshold, section name, or format from " - "the criteria verbatim in your rule (e.g. write 'keep the entire " - "response under 1200 characters', NOT 'respect length limits'). Vague " - "rules do not change behavior; specific numeric/structural rules do.\n" - "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; " - "you CANNOT delete the existing instructions above. If the current " - f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' " - "but outputs must be under a character limit), write an explicit, " - "forceful OVERRIDE rule stating it supersedes the conflicting " - "instruction, and put the hard requirement first.\n" - "HARD CONSTRAINT: every rule you write MUST be consistent with the " - "'Task output contract' below (if shown). NEVER propose a rule that " - "changes the required output format/language, tells the agent to ask " - "the user a question, or otherwise violates that contract — such a " - "rule scores ZERO because the evaluator cannot honor it.\n" - 'Return ONLY a JSON array: ' - '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n' - f"# Current {target}\n{cur_doc}\n" - f"{guard_text}" - f"{criteria_text}\n" - f"{pref_text}\n\n" - f"# Recurring failures\n{fail_text}" - ) + from skillopt_sleep import prompts as prompt_registry + prompt = prompt_registry.render("reflect", { + "__EDIT_BUDGET__": str(edit_budget), + "__TARGET__": target, + "__CUR_DOC__": cur_doc, + "__GUARD__": guard_text, + "__CRITERIA__": criteria_text, + "__PREFS__": pref_text, + "__FAILURES__": fail_text, + }) # Call with one retry: transient non-JSON replies otherwise waste a whole # night (the gate sees no edits and rejects). A firmer second prompt # recovers most of these. + ev = getattr(self, "evidence", None) arr = None for attempt in range(2): p = prompt if attempt == 0 else ( @@ -523,6 +520,11 @@ class CliBackend(Backend): ) raw = self._call(p, max_tokens=1024) self._tokens += len(p) // 4 + len(raw) // 4 + if ev is not None: + ev.log("reflect", "exchange", target=target, attempt=attempt + 1, + backend=self.name, model=self.model, + n_failures=len(failures), prompt=p, raw_reply=raw, + error=getattr(self, "last_call_error", "") or "") arr = _extract_json(raw, "array") if isinstance(arr, list) and arr: break @@ -1007,6 +1009,27 @@ def resolve_copilot_path(explicit: str = "") -> str: return found or "copilot" +def resolve_cursor_path(explicit: str = "") -> str: + """Find the Cursor Agent CLI (``cursor-agent``).""" + if explicit: + return os.path.expanduser(explicit) + env = os.environ.get("SKILLOPT_SLEEP_CURSOR_PATH") + if env: + return os.path.expanduser(env) + import shutil + + found = shutil.which("cursor-agent") + return found or "cursor-agent" + + +class CursorBackendError(RuntimeError): + """A redacted Cursor Agent process or response failure.""" + + def __init__(self, message: str, *, retryable: bool = False) -> None: + super().__init__(message) + self.retryable = retryable + + class CopilotCliBackend(CliBackend): """Drives the GitHub Copilot CLI in non-interactive mode. @@ -1207,6 +1230,240 @@ class CopilotCliBackend(CliBackend): pass +class CursorCliBackend(CliBackend): + """Drive an authenticated Cursor Agent CLI in an isolated workspace. + + Cursor's JSON print format has one final ``result`` object. Ordinary calls + use Ask mode, which is read-only. Tool-aware replay is disabled until the + Cursor Agent permission boundary has been validated against the live CLI. + """ + + name = "cursor" + _AUTH_ERROR_MARKERS = ( + "not authenticated", + "authentication required", + "not logged in", + "please log in", + "login required", + "unauthorized", + "invalid api key", + "401", + "403", + ) + _CONFIG_ERROR_MARKERS = ( + "unknown option", + "invalid option", + "invalid model", + "unsupported model", + "model not found", + "not available for your account", + "invalid configuration", + ) + _ASK_DENY = ["Read(**)", "Write(**)", "Mcp(*:*)"] + # Keep the Cursor child usable across shells, credential stores, proxies, + # and enterprise CA setups without forwarding unrelated provider or cloud + # credentials from the host process. + _ENV_ALLOWLIST = ( + "PATH", "HOME", "USER", "LOGNAME", "SHELL", + "USERPROFILE", "HOMEDRIVE", "HOMEPATH", + "SYSTEMROOT", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", + "TMPDIR", "TMP", "TEMP", + "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TERM", "NO_COLOR", + "CURSOR_API_KEY", + "DBUS_SESSION_BUS_ADDRESS", "XDG_RUNTIME_DIR", + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS", + ) + + def __init__(self, model: str = "", cursor_path: str = "", timeout: int = 240) -> None: + super().__init__(model=model or os.environ.get("SKILLOPT_SLEEP_CURSOR_MODEL", ""), timeout=timeout) + self.cursor_path = resolve_cursor_path(cursor_path) + + def _command(self, workspace: str) -> List[str]: + cmd = [ + self.cursor_path, + "-p", + "--output-format", + "json", + "--trust", + "--workspace", + workspace, + "--mode", + "ask", + ] + if self.model: + cmd += ["--model", self.model] + return cmd + + @staticmethod + def _terminal_result(raw: str) -> Optional[Dict[str, Any]]: + candidates = [raw.strip()] if raw.strip() else [] + candidates.extend(line.strip() for line in raw.splitlines() if line.strip().startswith("{")) + for candidate in reversed(candidates): + try: + obj = json.loads(candidate) + except Exception: + continue + if isinstance(obj, dict) and obj.get("type") == "result": + return obj + return None + + @classmethod + def _parse_json_response(cls, raw: str) -> str: + """Return text from the terminal successful Cursor result.""" + terminal = cls._terminal_result(raw) + if terminal is None or terminal.get("is_error") is True: + return "" + result = terminal.get("result") + if isinstance(result, str): + return result.strip() + return "" + + def _error(self, message: str, *, retryable: bool = False) -> CursorBackendError: + import logging + + from skillopt_sleep.staging import redact_secrets + + self.last_call_error = str(redact_secrets(message))[:500] + logging.getLogger("skillopt_sleep").warning("Cursor Agent call failed: %s", self.last_call_error) + return CursorBackendError(self.last_call_error, retryable=retryable) + + @staticmethod + def _isolated_environment(runtime_dir: str) -> Dict[str, str]: + config_dir = os.path.join(runtime_dir, "config") + data_dir = os.path.join(runtime_dir, "data") + os.makedirs(config_dir, exist_ok=True) + os.makedirs(data_dir, exist_ok=True) + with open(os.path.join(config_dir, "cli-config.json"), "w", encoding="utf-8") as f: + json.dump( + { + "version": 1, + "editor": {"vimMode": False}, + "permissions": { + "allow": [], + "deny": CursorCliBackend._ASK_DENY, + }, + "approvalMode": "allowlist", + "sandbox": { + "mode": "disabled", + "networkAccess": "user_config_only", + "networkAllowlist": [], + }, + "network": {"useHttp1ForAgent": False}, + "hasChangedDefaultModel": False, + "attribution": { + "attributeCommitsToAgent": False, + "attributePRsToAgent": False, + }, + }, + f, + indent=2, + ) + env = { + key: os.environ[key] + for key in CursorCliBackend._ENV_ALLOWLIST + if key in os.environ + } + env["CURSOR_CONFIG_DIR"] = config_dir + env["CURSOR_DATA_DIR"] = data_dir + return env + + def _invoke_once(self, prompt: str, workspace: str) -> str: + import shutil + + from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL + + self.last_call_error = "" + replay_prompt = CURSOR_REPLAY_SENTINEL + "\n\n" + prompt + runtime_dir = "" + try: + runtime_dir = tempfile.mkdtemp(prefix="skillopt_sleep_cursor_runtime_") + proc = subprocess.run( + self._command(workspace), + capture_output=True, + creationflags=_NO_WINDOW, + text=True, + encoding="utf-8", + errors="replace", + timeout=self.timeout, + cwd=workspace, + input=replay_prompt, + env=self._isolated_environment(runtime_dir), + ) + except subprocess.TimeoutExpired: + raise self._error( + f"Cursor Agent timed out after {self.timeout}s", + retryable=True, + ) + except Exception as exc: + raise self._error(f"Cursor Agent spawn failed: {exc}") + finally: + if runtime_dir: + shutil.rmtree(runtime_dir, ignore_errors=True) + + if proc.returncode != 0: + raise self._error( + f"Cursor Agent exited {proc.returncode}: {(proc.stderr or '')[:500]}" + ) + terminal = self._terminal_result(proc.stdout or "") + if terminal is not None and terminal.get("is_error") is True: + detail = terminal.get("result") or terminal.get("error") or proc.stderr or "" + raise self._error(f"Cursor Agent returned an error result: {str(detail)[:300]}") + output = self._parse_json_response(proc.stdout or "") + if not output: + detail = (proc.stderr or "").strip() + if any(marker in detail.casefold() for marker in self._AUTH_ERROR_MARKERS): + raise self._error( + "Cursor Agent authentication failed" + + (f": {detail[:300]}" if detail else "") + ) + if any(marker in detail.casefold() for marker in self._CONFIG_ERROR_MARKERS): + raise self._error( + "Cursor Agent configuration failed" + + (f": {detail[:300]}" if detail else "") + ) + raise self._error( + "Cursor Agent returned no usable JSON response" + + (f": {detail[:300]}" if detail else ""), + retryable=True, + ) + self.last_call_error = "" + return output + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + del max_tokens + workspace = tempfile.mkdtemp(prefix="skillopt_sleep_cursor_") + try: + for attempt in range(2): + try: + return self._invoke_once(prompt, workspace) + except CursorBackendError as exc: + if not exc.retryable or attempt == 1: + raise + raise AssertionError("unreachable") + finally: + try: + import shutil + + shutil.rmtree(workspace, ignore_errors=True) + except Exception: + pass + + def attempt_with_tools( + self, + task: TaskRecord, + skill: str, + memory: str, + tools: List[str], + ) -> Tuple[str, List[str]]: + del task, skill, memory, tools + raise self._error( + "Cursor tool-aware replay is temporarily disabled pending live " + "Cursor permission-boundary validation" + ) + + class DualBackend(Backend): """Route operations to two backends, Ă  la SkillOpt's target vs optimizer. @@ -1564,6 +1821,7 @@ def get_backend( model: str = "", claude_path: str = "claude", codex_path: str = "", + cursor_path: str = "", azure_endpoint: str = "", project_dir: str = "", ) -> Backend: @@ -1579,6 +1837,8 @@ def get_backend( return AzureResponsesBackend(deployment=model, endpoints=eps) if n in {"copilot", "github_copilot", "copilot_cli", "gh_copilot"}: return CopilotCliBackend(model=model) + if n in {"cursor", "cursor_agent", "cursor_cli"}: + return CursorCliBackend(model=model, cursor_path=cursor_path) if n in {"handoff", "session", "file"}: # Lazy import: handoff_backend imports CliBackend from this module. from skillopt_sleep.handoff_backend import HandoffBackend @@ -1598,6 +1858,7 @@ def build_backend( target_backend: str = "", target_model: str = "", codex_path: str = "", + cursor_path: str = "", azure_endpoint: str = "", preferences: str = "", project_dir: str = "", @@ -1615,16 +1876,17 @@ def build_backend( backend, model=model, codex_path=codex_path, + cursor_path=cursor_path, azure_endpoint=azure_endpoint, project_dir=project_dir, ) be.preferences = preferences return be tgt = get_backend(target_backend or backend, model=target_model or model, - codex_path=codex_path, azure_endpoint=azure_endpoint, + codex_path=codex_path, cursor_path=cursor_path, azure_endpoint=azure_endpoint, project_dir=project_dir) opt = get_backend(optimizer_backend or backend, model=optimizer_model or model, - codex_path=codex_path, azure_endpoint=azure_endpoint, + codex_path=codex_path, cursor_path=cursor_path, azure_endpoint=azure_endpoint, project_dir=project_dir) opt.preferences = preferences # reflect runs on the optimizer dual = DualBackend(target=tgt, optimizer=opt) diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 7478538..3c10567 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -19,14 +19,16 @@ from typing import Any, Dict, Optional HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep") CLAUDE_HOME = os.path.expanduser("~/.claude") CODEX_HOME = os.path.expanduser("~/.codex") +CURSOR_HOME = os.path.expanduser("~/.cursor") DEFAULTS: Dict[str, Any] = { # ── scope ────────────────────────────────────────────────────────────── "claude_home": CLAUDE_HOME, "codex_home": CODEX_HOME, - "transcript_source": "claude", # "claude" | "codex" | "copilot" | "auto" + "cursor_home": CURSOR_HOME, "vscode_workspace_storage": "", # "" => auto-detect platform defaults + "transcript_source": "claude", # "claude" | "codex" | "copilot" | "cursor" | "auto" "projects": "invoked", # "invoked" | "all" | [list of abs paths] "invoked_project": "", # filled at runtime (cwd) when projects == "invoked" "lookback_hours": 72, # harvest window when no prior sleep recorded @@ -37,15 +39,24 @@ DEFAULTS: Dict[str, Any] = { "val_fraction": 0.34, # real tasks reserved to gate updates "test_fraction": 0.0, # real tasks reserved as the final held-out measure # ── optimizer ────────────────────────────────────────────────────────── - "backend": "mock", # "mock" | "claude" | "codex" | "copilot" + "backend": "mock", # "mock" | "claude" | "codex" | "copilot" | "cursor" "model": "", # backend-specific; "" => backend default + # Dual-backend split (both empty => single backend above plays all roles). + # target = the model whose skill is deployed (runs `attempt` rollouts); + # optimizer = the model that mines tasks, judges rubrics, writes edits. + "optimizer_backend": "", + "optimizer_model": "", + "target_backend": "", + "target_model": "", + "azure_endpoint": "", # explicit endpoint for azure/compat backends "gate_mode": "on", # "on" (validation-gated) | "off" (greedy, no hard filter) "codex_path": "", # "" => auto-detect the real @openai/codex binary + "cursor_path": "", # "" => auto-detect the Cursor Agent CLI "edit_budget": 4, # textual learning rate (max edits/night) "preferences": "", # free-text house rules injected into reflect as a prior "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) "gate_mixed_weight": 0.5, - "replay_mode": "mock", # "mock" (sandboxed prompt) | "fresh" (worktree) + "replay_mode": "mock", # report label; fresh-worktree replay is not implemented # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task "dream_factor": 0, # >0 => add N synthetic variants of each task to the dream @@ -56,6 +67,9 @@ DEFAULTS: Dict[str, Any] = { "target_skill_path": "", # explicit SKILL.md target for repo-scoped agents "target_task_filter": True, # prefer mined tasks matching target_skill_path/text "progress": False, # print phase progress to stderr + # ── observability ────────────────────────────────────────────────────── + "evidence_log": True, # write per-night evidence.jsonl (full evidentiary chain) + "evidence_max_chars": 4000, # per-field truncation cap for evidence events # ── adoption / safety ────────────────────────────────────────────────── "auto_adopt": False, # default: stage + require explicit `adopt` "managed_skill_name": "skillopt-sleep-learned", @@ -109,6 +123,11 @@ class SleepConfig: def codex_archived_sessions_dir(self) -> str: return os.path.join(self.data["codex_home"], "archived_sessions") + @property + def cursor_projects_dir(self) -> str: + cursor_home = os.path.abspath(os.path.expanduser(str(self.data["cursor_home"]))) + return os.path.join(cursor_home, "projects") + @property def vscode_workspace_storage(self) -> str: value = self.data.get("vscode_workspace_storage", "") or "" diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index a9ea662..89baba5 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -13,7 +13,7 @@ from dataclasses import dataclass, field from typing import List, Optional, Tuple from skillopt_sleep.backend import Backend -from skillopt_sleep.memory import apply_edits +from skillopt_sleep.memory import apply_edits_detailed from skillopt_sleep.replay import aggregate_scores, replay_batch from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord @@ -37,6 +37,11 @@ class ConsolidationResult: holdout_baseline: float holdout_candidate: float # ── observability (so a 0.0->0.0 night is self-diagnosing, not a black box) ── + # Edits that changed nothing (anchor not found, or a duplicate add). They are + # neither applied nor gate-rejected, so without this list they would vanish + # from the report and the night would look like the optimizer produced less + # than it did. + unmatched_edits: List[EditRecord] = field(default_factory=list) holdout_detail: List[dict] = field(default_factory=list) # per val task: hard/soft/resp/why reflect_raw: str = "" # the optimizer's last raw reply (empty => reflect produced nothing) call_error: str = "" # backend's last call error (timeout/auth/empty) @@ -54,12 +59,13 @@ def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]] train = [t for t in tasks if _norm(t.split) == "train"] val = [t for t in tasks if _norm(t.split) == "val"] - # be robust if a split is empty: fall back so a night still does something, - # but never silently use test as val. - test = [t for t in tasks if _norm(t.split) == "test"] + # Be robust if a split is empty: fall back so a night still does something, + # but never silently use test as train or val. An all-test batch therefore + # returns empty train/val (caller scores test separately; gate is a no-op). if not val: - # prefer train as the gate reference over nothing; last resort all-but-test - val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks + # Prefer train as the gate reference; otherwise any non-test tasks. + # Do not fall back to the full task list (that would leak held-out test). + val = train or [t for t in tasks if _norm(t.split) != "test"] if not train: train = val return train, val @@ -108,6 +114,8 @@ def consolidate( Skill and memory are evolved in sequence (skill first if both enabled). """ + from skillopt_sleep import evidence as evlog + ev = evlog.get(backend) train_tasks, val_tasks = _split(tasks) gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"} holdout_detail: List[dict] = [] @@ -120,12 +128,19 @@ def consolidate( if gate_off: base_hard, base_soft = 0.0, 0.0 else: + evlog.set_phase(backend, "baseline_val") base_pairs = replay_batch(backend, val_tasks, skill, memory) base_hard, base_soft = aggregate_scores(base_pairs) holdout_detail = _holdout_detail(base_pairs) base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) + if ev is not None: + ev.log("gate", "baseline", gate_mode=("off" if gate_off else "on"), + n_train=len(train_tasks), n_val=len(val_tasks), + hard=base_hard, soft=base_soft, score=base_score, + metric=gate_metric, mixed_weight=gate_mixed_weight) # ── reflect over TRAIN-split failures/successes ─────────────────────── + evlog.set_phase(backend, "train") train_pairs = replay_batch(backend, train_tasks, skill, memory) failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0] successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0] @@ -133,25 +148,48 @@ def consolidate( cand_skill, cand_memory = skill, memory all_applied: List[EditRecord] = [] all_rejected: List[EditRecord] = [] + all_unmatched: List[EditRecord] = [] + + def _edits_payload(edits: List[EditRecord]) -> List[dict]: + return [{"op": e.op, "content": e.content, "anchor": e.anchor, + "rationale": e.rationale} for e in edits] def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected + if ev is not None: + ev.log("reflect", "edits_returned", target=which, + n_edits=len(edits), edits=_edits_payload(edits)) if not edits: return doc - new_doc, applied = apply_edits(doc, edits) + new_doc, applied, unmatched = apply_edits_detailed(doc, edits) + if unmatched: + all_unmatched.extend(unmatched) + if ev is not None: + ev.log("reflect", "edits_unmatched", target=which, + n_edits=len(unmatched), edits=_edits_payload(unmatched)) if not applied: return doc # gate OFF: accept greedily with NO val scoring (the daily-use path) if gate_off: all_applied.extend(applied) + if ev is not None: + ev.log("gate", "trial", target=which, mode="greedy", + accepted=True, n_edits=len(applied)) return new_doc # gate ON: score the candidate on the VAL slice, keep only if it improves trial_skill = new_doc if which == "skill" else cand_skill trial_memory = new_doc if which == "memory" else cand_memory + evlog.set_phase(backend, f"gate_trial:{which}") pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory) h, s = aggregate_scores(pairs) cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight) - if cand_score > base_score: + improved = cand_score > base_score + if ev is not None: + ev.log("gate", "trial", target=which, mode="gated", + baseline_score=base_score, cand_hard=h, cand_soft=s, + cand_score=cand_score, accepted=improved, + n_edits=len(applied)) + if improved: base_score = max(base_score, cand_score) all_applied.extend(applied) return new_doc @@ -204,6 +242,7 @@ def consolidate( if evolve_memory: # re-evaluate failures under the (possibly improved) skill + evlog.set_phase(backend, "train_post_skill") train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory) failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0] successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0] @@ -225,6 +264,7 @@ def consolidate( base_gate_score = 0.0 else: # scored on the VAL slice (the gate reference) + evlog.set_phase(backend, "final_val") final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory) final_hard, final_soft = aggregate_scores(final_pairs) final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight) @@ -248,6 +288,44 @@ def consolidate( else: action = "accept" if final_score > base_gate_score else "reject" accepted = bool(all_applied) and final_score > base_gate_score + # The gate scores documents, not edit bookkeeping: when every proposed + # edit was dropped during the per-target trials, `all_applied` is empty + # and nothing changed, yet the score comparison can still yield an + # accept-flavoured action. Reporting that as "accept_new_best" while + # `accepted` is False makes the headline contradict the outcome. + if not accepted and action in {"accept", "accept_new_best"}: + action = "reject" + # A per-target trial can improve and tentatively apply an edit, while a + # later fresh final replay regresses. The returned documents already + # roll back in that case; keep the edit bookkeeping/report consistent + # by moving those tentative edits into the rejected set as well. + if not accepted and all_applied: + for edit in all_applied: + if edit not in all_rejected: + all_rejected.append(edit) + all_applied = [] + + if ev is not None: + w = max(0.0, min(1.0, float(gate_mixed_weight))) + if gate_metric == "mixed": + formula = ( + f"score = (1-{w})*hard + {w}*soft; " + f"baseline = (1-{w})*{base_hard:.3f} + {w}*{base_soft:.3f} = {base_gate_score:.3f}; " + f"candidate = (1-{w})*{final_hard:.3f} + {w}*{final_soft:.3f} = {final_score:.3f}" + ) + else: + formula = ( + f"score = {gate_metric}; baseline = {base_gate_score:.3f}; " + f"candidate = {final_score:.3f}" + ) + ev.log("gate", "decision", action=action, accepted=accepted, + baseline_score=base_gate_score, candidate_score=final_score, + baseline_hard=base_hard, baseline_soft=base_soft, + candidate_hard=final_hard, candidate_soft=final_soft, + metric=gate_metric, mixed_weight=gate_mixed_weight, + formula=formula, n_applied=len(all_applied), + n_rejected=len(all_rejected), + n_unmatched=len(all_unmatched), night=night) return ConsolidationResult( accepted=accepted, @@ -258,6 +336,7 @@ def consolidate( new_memory=cand_memory if accepted else memory, applied_edits=all_applied, rejected_edits=all_rejected, + unmatched_edits=all_unmatched, holdout_baseline=base_hard, holdout_candidate=final_hard, holdout_detail=holdout_detail, diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index f199211..006547f 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -10,11 +10,14 @@ CI use. With backend="anthropic" it spends the user's budget for real lift. from __future__ import annotations import os +import shutil import sys from dataclasses import dataclass from typing import List, Optional -from skillopt_sleep.backend import Backend, get_backend +from skillopt_sleep import evidence +from skillopt_sleep.backend import Backend, CursorBackendError, build_backend +from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.config import SleepConfig, load_config from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.harvest_sources import harvest_for_config @@ -27,6 +30,92 @@ from skillopt_sleep.state import SleepState, _now_iso from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord +# ── Model-swap detection (F16) ─────────────────────────────── +def _make_model_key(cfg: SleepConfig) -> str: + """Stable string identifying the backend object(s) actually used. + + Model-change detection is advisory, so resolving its diagnostic key must + never become an earlier failure point than construction of the real + backend. Fall back to a credential-free description of the configured + roles if a backend constructor cannot be used in this diagnostic path. + """ + try: + effective = build_backend( + backend=cfg.get("backend", "mock"), + model=cfg.get("model", ""), + optimizer_backend=cfg.get("optimizer_backend", ""), + optimizer_model=cfg.get("optimizer_model", ""), + target_backend=cfg.get("target_backend", ""), + target_model=cfg.get("target_model", ""), + codex_path=cfg.get("codex_path", ""), + cursor_path=cfg.get("cursor_path", ""), + azure_endpoint=cfg.get("azure_endpoint", ""), + project_dir=cfg.get("invoked_project", "") or os.getcwd(), + ) + except Exception: + backend = str(cfg.get("backend", "mock") or "mock") + model = str(cfg.get("model", "") or "") + split_keys = ( + "optimizer_backend", + "optimizer_model", + "target_backend", + "target_model", + ) + if not any(cfg.get(key, "") for key in split_keys): + return f"configured:{backend}::{model}" + optimizer_backend = str(cfg.get("optimizer_backend", "") or backend) + optimizer_model = str(cfg.get("optimizer_model", "") or model) + target_backend = str(cfg.get("target_backend", "") or backend) + target_model = str(cfg.get("target_model", "") or model) + return ( + f"configured:optimizer={optimizer_backend}::{optimizer_model};" + f"target={target_backend}::{target_model}" + ) + return _make_backend_key(effective) + + +def _make_backend_key(backend: Backend) -> str: + """Describe resolved aliases/defaults without exposing credentials.""" + target = getattr(backend, "target", None) + optimizer = getattr(backend, "optimizer", None) + if target is not None and optimizer is not None: + return ( + f"optimizer={_make_backend_key(optimizer)};" + f"target={_make_backend_key(target)}" + ) + name = str(getattr(backend, "name", backend.__class__.__name__) or "") + model = str(getattr(backend, "model", "") or "") + return f"{name}::{model}" + + +def _check_model_change( + cfg: SleepConfig, state: SleepState, backend: Backend | None = None +) -> None: + """Warn when the backend/model has changed since the last night. + + Skill text is backend-specific; adopting edits from a different model's + reflections into a new model's skill file can cause regressions. + This is advisory only — the cycle continues either way. + """ + current_key = ( + _make_backend_key(backend) if backend is not None else _make_model_key(cfg) + ) + prior_key = state.last_model_key + if prior_key and state.last_model_key_format < 2: + # Version 1 stored raw configuration rather than the resolved backend + # model. Defaults and aliases make that value impossible to compare + # truthfully, so migrate silently on the next successful night. + return + if prior_key and prior_key != current_key: + print( + f"[sleep] WARNING: model changed since last night " + f"(was {prior_key!r}, now {current_key!r}). " + "Learned skill text may not transfer cleanly. " + "Consider starting from a fresh skill document.", + file=sys.stderr, + ) + + @dataclass class CycleOutcome: report: SleepReport @@ -56,6 +145,20 @@ def _progress(cfg: SleepConfig, message: str) -> None: print(f"[sleep] {message}", file=sys.stderr, flush=True) +def _discard_unstaged_evidence(path: str) -> None: + """Remove a pre-created evidence folder after a fail-closed Cursor call.""" + if not path: + return + shutil.rmtree(path, ignore_errors=True) + # Avoid leaving an otherwise empty project .skillopt-sleep tree. Stop at + # the first non-empty directory so existing nights are never disturbed. + for parent in (os.path.dirname(path), os.path.dirname(os.path.dirname(path))): + try: + os.rmdir(parent) + except OSError: + break + + def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: lines = [ f"# SkillOpt-Sleep — night {report.night} report", @@ -79,6 +182,16 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: for e in report.rejected_edits: lines.append(f"- [{e.target}/{e.op}] {e.content}") lines.append("") + if report.unmatched_edits: + lines.append("## Proposed but changed nothing (never reached the gate)") + lines.append( + "_Anchor not found, replacement already present, duplicate/empty " + "add, or an unknown op. " + "These were never scored — check the anchor text if a rule you expected is missing._") + for e in report.unmatched_edits: + anchor = f" \n _anchor: `{e.anchor}`_" if e.anchor else "" + lines.append(f"- [{e.target}/{e.op}] {e.content}{anchor}") + lines.append("") if report.notes: lines.append("## Notes") for n in report.notes: @@ -110,19 +223,59 @@ def run_sleep_cycle( """ cfg = cfg or load_config() state = SleepState.load(cfg.state_path) - night = state.begin_night(clock) project = _project_paths(cfg) - started = _now_iso(clock) - backend = backend or get_backend( - cfg.get("backend", "mock"), + backend = backend or build_backend( + backend=cfg.get("backend", "mock"), model=cfg.get("model", ""), + optimizer_backend=cfg.get("optimizer_backend", ""), + optimizer_model=cfg.get("optimizer_model", ""), + target_backend=cfg.get("target_backend", ""), + target_model=cfg.get("target_model", ""), codex_path=cfg.get("codex_path", ""), + cursor_path=cfg.get("cursor_path", ""), + azure_endpoint=cfg.get("azure_endpoint", ""), + preferences=cfg.get("preferences", ""), project_dir=project, ) + _check_model_change(cfg, state, backend) # F16: warn if model changed between nights + night = state.begin_night(clock) + started = _now_iso(clock) backend.preferences = cfg.get("preferences", "") _progress(cfg, f"night {night}: project={project} backend={backend.name}") + # ── evidence log (the night's full evidentiary chain) ──────────────── + # Pre-create the staging dir so evidence.jsonl accumulates exactly where + # the report will land; dry-runs log into the state dir instead. + ev = None + staging_dir_pre = "" + # Callers may reuse a backend object across nights. Detach any logger from + # an earlier run before honoring this run's evidence_log setting. + evidence.attach(backend, None) + if cfg.get("evidence_log", True): + from skillopt_sleep.staging import _ts_dir, new_staging_dir + if dry_run: + ev_path = os.path.join( + cfg.state_dir, "evidence", f"dryrun-{_ts_dir()}.jsonl") + else: + staging_dir_pre = new_staging_dir(project) + ev_path = os.path.join(staging_dir_pre, "evidence.jsonl") + ev = EvidenceLog( + ev_path, + max_chars=int(cfg.get("evidence_max_chars", 4000) or 4000), + redact=bool(cfg.get("redact_secrets", True)), + ) + evidence.attach(backend, ev) + ev.log("cycle", "start", night=night, project=project, + backend=backend.name, model=cfg.get("model", ""), + config={k: cfg.get(k) for k in ( + "backend", "model", "optimizer_backend", "optimizer_model", + "target_backend", "target_model", "gate_mode", "gate_metric", + "gate_mixed_weight", "edit_budget", "holdout_fraction", + "dream_rollouts", "dream_factor", "recall_k", + "max_tasks_per_night", "lookback_hours", "llm_mine", + "evolve_skill", "evolve_memory")}) + # ── live skill/memory docs ─────────────────────────────────────────── live_memory_path = os.path.join(project, "CLAUDE.md") live_skill_path = cfg.managed_skill_path() @@ -174,6 +327,17 @@ def run_sleep_cycle( ) n_sessions = len(digests) _progress(cfg, f"harvest done: sessions={n_sessions}") + if ev is not None: + # The transcript end of the evidentiary chain: which sessions were + # even considered, and what signals they carried into mining. + for d in digests: + ev.log("harvest", "session", session_id=d.session_id, + project=d.project, + n_user_prompts=len(d.user_prompts), + user_prompts_head=[p[:200] for p in d.user_prompts[:6]], + assistant_final_head=(d.assistant_finals[-1][:300] + if d.assistant_finals else ""), + feedback_signals=list(d.feedback_signals or [])) # When a real backend is configured, use it to mine checkable tasks from # the transcripts (rubric/rule judges); otherwise fall back to the # heuristic miner (no API, no checkable reference). @@ -193,18 +357,33 @@ def run_sleep_cycle( f"mine start: max_tasks={max_tasks} candidate_limit={candidate_limit} " f"llm_mine={llm_miner is not None} target_filter={target_filter}", ) - tasks = mine( - digests, - max_tasks=max_tasks, - candidate_limit=candidate_limit, - holdout_fraction=cfg.get("holdout_fraction", 0.34), - seed=cfg.get("seed", 42), - llm_miner=llm_miner, - target_skill_text=raw_skill if target_filter else "", - target_skill_path=live_skill_path if target_filter else "", - ) + try: + tasks = mine( + digests, + max_tasks=max_tasks, + candidate_limit=candidate_limit, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + llm_miner=llm_miner, + target_skill_text=raw_skill if target_filter else "", + target_skill_path=live_skill_path if target_filter else "", + ) + except CursorBackendError: + _discard_unstaged_evidence(staging_dir_pre) + raise _progress(cfg, f"mine done: tasks={len(tasks)}") + if ev is not None: + # Final task pool with split assignment: which tasks train the edits + # vs. which held-out tasks gate them (works for seeded tasks too). + for t in tasks: + ev.log("mine", "task_ready", task_id=t.id, split=t.split, + origin=t.origin, intent=t.intent[:300], + reference_kind=t.reference_kind, + checks=(t.judge or {}).get("checks", []), + rubric=(t.reference if t.reference_kind == "rubric" else ""), + source_sessions=list(t.source_sessions or [])) + report = SleepReport( night=night, project=project, started_at=started, n_sessions=n_sessions, n_tasks=len(tasks), @@ -217,6 +396,9 @@ def run_sleep_cycle( state.record_night({"night": night, "accepted": False, "n_tasks": 0}) if not dry_run: state.save() + if ev is not None: + ev.log("cycle", "end", night=night, outcome="no_tasks", + tokens_used=backend.tokens_used()) staging_dir = "" return CycleOutcome(report, staging_dir, False, []) @@ -230,26 +412,31 @@ def run_sleep_cycle( history_tasks = [] if recall_k > 0: history_tasks = [TaskRecord.from_dict(d) for d in state.task_archive()] - result = dream_consolidate( - backend, tasks, skill, memory, - history_tasks=history_tasks, - recall_k=recall_k, - dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1), - dream_factor=int(cfg.get("dream_factor", 0) or 0), - edit_budget=cfg.get("edit_budget", 4), - gate_metric=cfg.get("gate_metric", "mixed"), - gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), - gate_mode=cfg.get("gate_mode", "on"), - evolve_skill=cfg.get("evolve_skill", True), - evolve_memory=cfg.get("evolve_memory", True), - night=night, - ) + try: + result = dream_consolidate( + backend, tasks, skill, memory, + history_tasks=history_tasks, + recall_k=recall_k, + dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1), + dream_factor=int(cfg.get("dream_factor", 0) or 0), + edit_budget=cfg.get("edit_budget", 4), + gate_metric=cfg.get("gate_metric", "mixed"), + gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + gate_mode=cfg.get("gate_mode", "on"), + evolve_skill=cfg.get("evolve_skill", True), + evolve_memory=cfg.get("evolve_memory", True), + night=night, + ) + except CursorBackendError: + _discard_unstaged_evidence(staging_dir_pre) + raise # archive tonight's real (non-dream) tasks so future nights can recall them state.add_to_archive([t.to_dict() for t in tasks if t.origin != "dream"]) _progress( cfg, f"consolidate done: gate={result.gate_action} accepted={result.accepted} " - f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}", + f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}" + + (f" unmatched={len(result.unmatched_edits)}" if result.unmatched_edits else ""), ) report.n_replayed = len(tasks) @@ -260,6 +447,7 @@ def run_sleep_cycle( report.no_edits_reason = getattr(result, "no_edits_reason", "") report.edits = result.applied_edits report.rejected_edits = result.rejected_edits + report.unmatched_edits = result.unmatched_edits report.tokens_used = backend.tokens_used() report.ended_at = _now_iso(clock) @@ -280,7 +468,13 @@ def run_sleep_cycle( live_skill_path=live_skill_path, live_memory_path=live_memory_path, report_md=report_md, + out_dir=staging_dir_pre, ) + if ev is not None: + ev.log("stage", "staged", staging_dir=staging_dir, + has_skill=proposed_skill is not None, + has_memory=proposed_memory is not None, + accepted=result.accepted) # Observability: persist per-task held-out evidence + optimizer/codex errors so a # 0.0->0.0 night self-explains (empty responses vs failing checks vs no edits) — the # cycle previously captured none of this, making the gate a black box (#learning-stall). @@ -300,6 +494,7 @@ def run_sleep_cycle( "accepted": result.accepted, "n_applied_edits": len(result.applied_edits), "n_rejected_edits": len(result.rejected_edits), + "n_unmatched_edits": len(result.unmatched_edits), "call_error": redact_secrets(getattr(result, "call_error", "")), "reflect_raw_head": redact_secrets( (getattr(result, "reflect_raw", "") or "")[:1200] @@ -314,10 +509,21 @@ def run_sleep_cycle( "baseline": result.baseline_score, "candidate": result.candidate_score, "n_tasks": len(tasks), "staging": staging_dir, }) + state.set_last_model_key(_make_backend_key(backend)) # F16: track resolved model # ── 6. adopt (opt-in) ──────────────────────────────────────────── if cfg.get("auto_adopt") and result.accepted: adopted_paths = adopt_staging(staging_dir) adopted = bool(adopted_paths) state.save() + if ev is not None: + ev.log("cycle", "end", night=night, outcome="completed", + gate_action=report.gate_action, accepted=report.accepted, + baseline_score=report.baseline_score, + candidate_score=report.candidate_score, + n_applied_edits=len(report.edits), + n_rejected_edits=len(report.rejected_edits), + n_unmatched_edits=len(report.unmatched_edits), + tokens_used=report.tokens_used, adopted=adopted) + return CycleOutcome(report, staging_dir, adopted, adopted_paths) diff --git a/skillopt_sleep/evidence.py b/skillopt_sleep/evidence.py new file mode 100644 index 0000000..fa53891 --- /dev/null +++ b/skillopt_sleep/evidence.py @@ -0,0 +1,138 @@ +"""SkillOpt-Sleep — the per-night evidentiary chain (``evidence.jsonl``). + +The existing report/diagnostics answer *what* the cycle decided; they do not +answer *why*. This module records the full causal chain, per night: + + transcript session -> miner exchange (prompt + raw reply) + -> mined task (+ its checks, + source session ids) + -> split assignment (train / val) + -> every replay attempt (phase-tagged, full prompt + and response, cache hits marked) + -> per-task scores with the failing checks named + -> the reflect exchange (prompt, raw reply, parsed + edits) and every gate trial + -> the final gate decision with the score arithmetic + -> what was staged + +Design constraints (matching the sleep engine's contract): + * pure stdlib, thread-safe (replay batches run in a thread pool); + * every persisted string passes through best-effort ``redact_secrets`` + before the per-field length cap is applied; + * append-only JSONL so a crashed night still leaves its partial chain; + * zero behavior change when disabled (``evidence_log: false``). + +Events share the shape:: + + {"ts": , "seq": , "stage": , "event": , ...} +""" +from __future__ import annotations + +import json +import os +import threading +import time +from typing import Any, Optional + +from skillopt_sleep.staging import redact_secrets + + +def _now_iso() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + + +class EvidenceLog: + """Append-only, thread-safe JSONL logger for one sleep night.""" + + def __init__(self, path: str, *, max_chars: int = 4000, redact: bool = True) -> None: + self.path = path + self.max_chars = max(200, int(max_chars)) + self.redact = redact + self._lock = threading.Lock() + self._seq = 0 + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + + # ── sanitization ────────────────────────────────────────────────────── + def _clean(self, value: Any) -> Any: + if isinstance(value, str): + # Redact first: truncating a structured secret such as a PEM block + # can remove its closing marker and make it unrecognizable to the + # redactor while leaving the secret body in the persisted prefix. + if self.redact: + value = redact_secrets(value) + if len(value) > self.max_chars: + dropped = len(value) - self.max_chars + value = value[: self.max_chars] + f"…[truncated {dropped} chars]" + return value + if isinstance(value, dict): + return {k: self._clean(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [self._clean(v) for v in value] + return value + + # ── the one write path ──────────────────────────────────────────────── + def log(self, stage: str, event: str, **data: Any) -> None: + record = {"ts": _now_iso(), "stage": stage, "event": event} + record.update(self._clean(data)) + with self._lock: + self._seq += 1 + record["seq"] = self._seq + try: + line = json.dumps(record, ensure_ascii=False, default=str) + with open(self.path, "a", encoding="utf-8") as f: + f.write(line + "\n") + except Exception: + # Evidence must never break a night; drop the record instead. + pass + + +def attach(backend, ev: Optional[EvidenceLog]) -> None: + """Attach ``ev`` to a backend — and, for DualBackend, to both halves — + so every layer that wants to log can find it via ``backend.evidence``.""" + if backend is None: + return + backend.evidence = ev + for half in ("target", "optimizer"): + sub = getattr(backend, half, None) + if sub is not None: + sub.evidence = ev + + +def get(backend) -> Optional[EvidenceLog]: + return getattr(backend, "evidence", None) + + +def set_phase(backend, phase: str) -> None: + """Tag subsequent replay calls with a phase label (baseline_val, + train, gate_trial:skill, final_val, ...). Phases are sequential in the + consolidation loop, so a plain attribute is safe; parallelism only ever + happens *within* one phase.""" + if backend is None: + return + backend.evidence_phase = phase + for half in ("target", "optimizer"): + sub = getattr(backend, half, None) + if sub is not None: + sub.evidence_phase = phase + + +def phase(backend) -> str: + return getattr(backend, "evidence_phase", "") or "" + + +def read_events(path: str) -> list: + """Best-effort reader for the dashboard: skips corrupt lines.""" + out = [] + try: + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except Exception: + continue + except OSError: + return [] + out.sort(key=lambda r: r.get("seq", 0)) + return out diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py index deb8d88..d0c102c 100644 --- a/skillopt_sleep/harvest.py +++ b/skillopt_sleep/harvest.py @@ -86,6 +86,34 @@ def _tool_names_from_content(content: Any) -> List[str]: return names +def _skill_names_from_content(content: Any) -> List[str]: + """Extract skill targets from Claude ``Skill`` tool-use blocks. + + Only well-formed invocations count: the block type must be ``tool_use``, + its name exactly ``Skill``, and ``input.skill`` a non-blank string. The + name is returned whitespace-trimmed but otherwise verbatim; no other tool + input and no tool output is read. + """ + names: List[str] = [] + if not isinstance(content, list): + return names + for b in content: + if not isinstance(b, dict): + continue + if b.get("type") != "tool_use" or b.get("name") != "Skill": + continue + args = b.get("input") + if not isinstance(args, dict): + continue + skill = args.get("skill") + if not isinstance(skill, str): + continue + skill = skill.strip() + if skill: + names.append(skill) + return names + + def _detect_feedback(text: str) -> List[str]: low = text.lower() sig: List[str] = [] @@ -206,6 +234,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]: user_prompts: List[str] = [] assistant_finals: List[str] = [] tools: List[str] = [] + skills: List[str] = [] files: List[str] = [] feedback: List[str] = [] n_user = 0 @@ -240,6 +269,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]: elif role == "assistant": n_asst += 1 tools.extend(_tool_names_from_content(content)) + skills.extend(_skill_names_from_content(content)) text = _text_from_content(content) if text.strip(): assistant_finals.append(text.strip()) @@ -266,6 +296,7 @@ def digest_transcript(path: str) -> Optional[SessionDigest]: user_prompts=user_prompts, assistant_finals=assistant_finals[-5:], # last few finals are the useful ones tools_used=_dedup(tools), + skills_used=_dedup(skills), files_touched=_dedup(files), feedback_signals=feedback, n_user_turns=n_user, diff --git a/skillopt_sleep/harvest_cursor.py b/skillopt_sleep/harvest_cursor.py new file mode 100644 index 0000000..c4379a4 --- /dev/null +++ b/skillopt_sleep/harvest_cursor.py @@ -0,0 +1,316 @@ +"""Read Cursor Agent transcripts and normalize them into session digests. + +Cursor writes workspace-scoped JSONL under +``~/.cursor/projects//agent-transcripts//.jsonl``. +The observed local records contain user/assistant messages and tool-use metadata +but no timestamps, so this harvester uses each file's mtime as its end time. +Tool inputs and outputs are intentionally never copied. +""" +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from typing import Any, Iterable, List, Optional + +from skillopt_sleep.harvest import ( + _detect_feedback, + _is_meta_prompt, + _iter_jsonl, +) +from skillopt_sleep.staging import redact_secrets +from skillopt_sleep.types import SessionDigest + +CURSOR_REPLAY_SENTINEL = "" +_CURSOR_USER_QUERY_RE = re.compile( + r"\s*(.*?)\s*\s*\Z", + re.DOTALL, +) + + +def cursor_project_slug(project: str) -> str: + """Return the filesystem-safe workspace name used under Cursor projects.""" + normalized = os.path.abspath(os.path.expanduser(project)) + return re.sub(r"[^A-Za-z0-9]+", "-", normalized).strip("-") + + +def _text_from_content(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + return "\n".join( + str(block["text"]) + for block in content + if isinstance(block, dict) + and block.get("type") == "text" + and block.get("text") + ) + + +def _tool_names(content: Any) -> List[str]: + if not isinstance(content, list): + return [] + names: List[str] = [] + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + name = block.get("name") + if isinstance(name, str) and name: + names.append(re.sub(r"[^A-Za-z0-9_.:-]+", "_", name)[:80]) + return names + + +def _sanitize_text(text: str) -> str: + sanitized = str(redact_secrets(text)).replace("\x00", "").strip() + user_query = _CURSOR_USER_QUERY_RE.search(sanitized) + if user_query: + sanitized = user_query.group(1).strip() + if not sanitized or _is_meta_prompt(sanitized): + return "" + return sanitized + + +def _dedup(values: Iterable[str]) -> List[str]: + seen = set() + result: List[str] = [] + for value in values: + if value not in seen: + seen.add(value) + result.append(value) + return result + + +def _mtime_iso(path: str) -> str: + try: + return ( + datetime.fromtimestamp(os.path.getmtime(path), tz=timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + except OSError: + return "" + + +def _mtime(path: str) -> Optional[float]: + try: + return os.path.getmtime(path) + except OSError: + return None + + +def _iso_epoch(value: Optional[str]) -> Optional[float]: + if not value: + return None + try: + normalized = value[:-1] + "+00:00" if value.endswith("Z") else value + parsed = datetime.fromisoformat(normalized) + # Existing state timestamps are local-time strings without an offset. + return parsed.timestamp() + except (TypeError, ValueError, OSError): + return None + + +def digest_cursor_transcript(path: str, *, project: str = "") -> Optional[SessionDigest]: + """Build a digest without retaining Cursor tool arguments or outputs.""" + session_id = os.path.splitext(os.path.basename(path))[0] + user_prompts: List[str] = [] + assistant_finals: List[str] = [] + tools: List[str] = [] + feedback: List[str] = [] + n_user = 0 + n_assistant = 0 + + for record in _iter_jsonl(path): + if not isinstance(record, dict): + continue + if record.get("type") == "turn_ended": + if record.get("status") == "error": + feedback.append("neg:cursor_turn_error") + continue + + message = record.get("message") + if not isinstance(message, dict): + continue + role = record.get("role") or message.get("role") + content = message.get("content") + if role == "user": + text = _sanitize_text(_text_from_content(content)) + if text: + n_user += 1 + user_prompts.append(text) + feedback.extend(_detect_feedback(text)) + elif role == "assistant": + n_assistant += 1 + tools.extend(_tool_names(content)) + text = _sanitize_text(_text_from_content(content)) + if text: + assistant_finals.append(text) + + if n_user == 0 and n_assistant == 0: + return None + + return SessionDigest( + session_id=session_id, + project=project, + ended_at=_mtime_iso(path), + user_prompts=user_prompts, + assistant_finals=assistant_finals[-5:], + tools_used=_dedup(tools), + files_touched=[], + feedback_signals=feedback, + n_user_turns=n_user, + n_assistant_turns=n_assistant, + raw_path=path, + ) + + +def _workspace_path(project_dir: str) -> str: + metadata_path = os.path.join(project_dir, ".workspace-trusted") + try: + with open(metadata_path, encoding="utf-8") as f: + metadata = json.load(f) + except (OSError, ValueError): + return "" + if not isinstance(metadata, dict): + return "" + workspace = metadata.get("workspacePath") + if not isinstance(workspace, str) or not workspace.strip(): + return "" + workspace = os.path.expanduser(workspace.strip()) + if not os.path.isabs(workspace): + return "" + return os.path.abspath(workspace) + + +def _normalized_path(path: str) -> str: + return os.path.normcase(os.path.realpath(os.path.abspath(os.path.expanduser(path)))) + + +def _is_workspace_ancestor(workspace: str, invoked: str) -> bool: + try: + workspace_norm = _normalized_path(workspace) + invoked_norm = _normalized_path(invoked) + return os.path.commonpath([workspace_norm, invoked_norm]) == workspace_norm + except (OSError, ValueError): + return False + + +def _available_project_dirs(projects_dir: str) -> List[tuple[str, str, str]]: + try: + names = sorted(os.listdir(projects_dir)) + except OSError: + return [] + result: List[tuple[str, str, str]] = [] + for name in names: + project_dir = os.path.join(projects_dir, name) + if os.path.isdir(os.path.join(project_dir, "agent-transcripts")): + result.append((project_dir, name, _workspace_path(project_dir))) + return result + + +def _project_dirs(projects_dir: str, scope: Any, invoked_project: str) -> List[tuple[str, str]]: + available = _available_project_dirs(projects_dir) + if scope == "all": + return [ + (project_dir, workspace or name) + for project_dir, name, workspace in available + ] + + projects: List[str] + if isinstance(scope, (list, tuple)): + projects = [str(project) for project in scope] + else: + projects = [invoked_project] if invoked_project else [] + + selected: List[tuple[str, str]] = [] + seen = set() + for project in projects: + absolute_project = os.path.abspath(os.path.expanduser(project)) + matches = [ + (project_dir, workspace) + for project_dir, _name, workspace in available + if workspace and _is_workspace_ancestor(workspace, absolute_project) + ] + candidate = absolute_project + while candidate: + fallback = os.path.join(projects_dir, cursor_project_slug(candidate)) + if os.path.isdir(os.path.join(fallback, "agent-transcripts")): + matches.append((fallback, candidate)) + break + parent = os.path.dirname(candidate) + if parent == candidate: + break + candidate = parent + + if matches: + longest = max(len(_normalized_path(workspace)) for _project_dir, workspace in matches) + choices = [ + (project_dir, workspace) + for project_dir, workspace in matches + if len(_normalized_path(workspace)) == longest + ] + else: + fallback = os.path.join(projects_dir, cursor_project_slug(absolute_project)) + choices = [(fallback, absolute_project)] + for project_dir, workspace in choices: + if project_dir not in seen: + selected.append((project_dir, workspace)) + seen.add(project_dir) + return selected + + +def _is_cursor_replay(digest: SessionDigest) -> bool: + return any(prompt.lstrip().startswith(CURSOR_REPLAY_SENTINEL) for prompt in digest.user_prompts) + + +def harvest_cursor( + projects_dir: str, + *, + scope: Any = "all", + invoked_project: str = "", + since_iso: Optional[str] = None, + limit: int = 0, +) -> List[SessionDigest]: + """Return Cursor session digests for the selected workspace scope.""" + if not os.path.isdir(projects_dir): + return [] + + candidates: List[tuple[str, str, float]] = [] + for project_dir, project in _project_dirs(projects_dir, scope, invoked_project): + transcripts_dir = os.path.join(project_dir, "agent-transcripts") + try: + session_names = sorted(os.listdir(transcripts_dir)) + except OSError: + continue + for session_name in session_names: + session_dir = os.path.join(transcripts_dir, session_name) + if not os.path.isdir(session_dir): + continue + try: + filenames = sorted(os.listdir(session_dir)) + except OSError: + continue + for filename in filenames: + path = os.path.join(session_dir, filename) + if not filename.endswith(".jsonl") or not os.path.isfile(path): + continue + modified = _mtime(path) + if modified is not None: + candidates.append((path, project, modified)) + candidates.sort(key=lambda item: (-item[2], item[0])) + + since_epoch = _iso_epoch(since_iso) + digests: List[SessionDigest] = [] + for path, project, modified in candidates: + if since_epoch is not None and modified <= since_epoch: + continue + digest = digest_cursor_transcript(path, project=project) + if digest is None or _is_cursor_replay(digest): + continue + digests.append(digest) + if limit and len(digests) >= limit: + break + return digests diff --git a/skillopt_sleep/harvest_sources.py b/skillopt_sleep/harvest_sources.py index d91ea27..a43ebac 100644 --- a/skillopt_sleep/harvest_sources.py +++ b/skillopt_sleep/harvest_sources.py @@ -6,6 +6,7 @@ from typing import Optional from skillopt_sleep.harvest import harvest from skillopt_sleep.harvest_copilot import harvest_copilot from skillopt_sleep.harvest_codex import harvest_codex +from skillopt_sleep.harvest_cursor import harvest_cursor from skillopt_sleep.types import SessionDigest @@ -30,6 +31,14 @@ def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) since_iso=since_iso, limit=limit, ) + if source == "cursor": + return harvest_cursor( + cfg.cursor_projects_dir, + scope=scope, + invoked_project=invoked_project, + since_iso=since_iso, + limit=limit, + ) if source == "auto": codex_digests = harvest_codex( cfg.codex_archived_sessions_dir, diff --git a/skillopt_sleep/judges.py b/skillopt_sleep/judges.py index f981015..9f5df1e 100644 --- a/skillopt_sleep/judges.py +++ b/skillopt_sleep/judges.py @@ -35,29 +35,108 @@ def _section_present(response: str, name: str) -> bool: return bool(label.search(response or "")) -def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool: +def _check(op: str, arg: Any, response: str, + tools_called: List[str]) -> Tuple[bool, str]: + """Evaluate one check. + + Returns ``(passed, problem)``. ``problem`` is non-empty only when the check + itself is malformed (e.g. an unparseable regex) rather than simply unmet — + the two need opposite fixes, so they must not look alike in the rationale. + """ r = response or "" if op == "section_present": - return _section_present(r, str(arg)) + return _section_present(r, str(arg)), "" if op == "regex": try: - return bool(re.search(str(arg), r)) - except re.error: - return False + return bool(re.search(str(arg), r)), "" + except re.error as exc: + # A malformed pattern can never match, so it would fail every + # rollout forever and read exactly like a model that never + # complies. Surface it instead of hiding it behind a False. + return False, f"invalid regex ({exc})" if op == "max_chars": - return len(r) <= int(arg) + return len(r) <= int(arg), "" if op == "min_chars": - return len(r) >= int(arg) + return len(r) >= int(arg), "" if op == "contains": - return str(arg).lower() in r.lower() + return str(arg).lower() in r.lower(), "" if op == "tool_called": name = str(arg).lower() if any(name == t.lower() for t in tools_called): - return True + return True, "" # single-shot approximation: the agent emits an explicit marker - return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)) + return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)), "" # unknown op: do not block - return True + return True, "" + + +KNOWN_OPS = frozenset({ + "section_present", "regex", "max_chars", "min_chars", "contains", "tool_called", +}) + + +def validate_checks(judge: Any) -> Tuple[List[str], List[str]]: + """Return ``(errors, warnings)`` for a rule judge's checks. + + An *error* means the check can never behave as written — a regex that does + not compile always scores 0.0, which is indistinguishable from a model that + never complies. A *warning* means the check is accepted but toothless, e.g. + an unknown op, which :func:`_check` deliberately lets pass. + """ + errors: List[str] = [] + warnings: List[str] = [] + if judge is not None and not isinstance(judge, dict): + return [f"judge must be an object, got {type(judge).__name__}"], warnings + checks = (judge or {}).get("checks", []) or [] + if not isinstance(checks, list): + return [f"judge 'checks' must be an array, got {type(checks).__name__}"], warnings + for i, c in enumerate(checks): + if not isinstance(c, dict): + errors.append(f"check #{i} is not an object") + continue + op = c.get("op", "") + arg = c.get("arg") + if not isinstance(op, str): + errors.append( + f"check #{i} op must be a string, got {type(op).__name__}" + ) + continue + if op in {"regex", "section_present", "contains", "tool_called"} and ( + arg is None or not str(arg).strip() + ): + errors.append(f"check #{i} {op} needs a non-empty arg") + continue + if op == "regex": + try: + re.compile(str(arg)) + except re.error as exc: + errors.append(f"check #{i} regex does not compile ({exc}): {arg!r}") + elif op in {"max_chars", "min_chars"}: + try: + if isinstance(arg, bool): + raise ValueError + if isinstance(arg, int): + bound = arg + elif isinstance(arg, float): + if not arg.is_integer(): + raise ValueError + bound = int(arg) + elif isinstance(arg, str) and re.fullmatch( + r"[+-]?\d+", arg.strip() + ): + bound = int(arg.strip()) + else: + raise ValueError + except (OverflowError, TypeError, ValueError): + errors.append(f"check #{i} {op} needs an integer arg, got {arg!r}") + else: + if bound < 0: + errors.append(f"check #{i} {op} cannot be negative, got {bound}") + elif op == "min_chars" and bound == 0: + warnings.append(f"check #{i} min_chars=0 always passes") + elif op not in KNOWN_OPS: + warnings.append(f"check #{i} has unknown op {op!r} — it always passes") + return errors, warnings def score_rule_judge( @@ -73,11 +152,14 @@ def score_rule_judge( passed = 0 failed_desc: List[str] = [] for c in checks: - ok = _check(c.get("op", ""), c.get("arg"), response, tools_called) + ok, problem = _check(c.get("op", ""), c.get("arg"), response, tools_called) if ok: passed += 1 else: - failed_desc.append(f"{c.get('op')}={c.get('arg')}") + desc = f"{c.get('op')}={c.get('arg')}" + if problem: + desc += f" [{problem}]" + failed_desc.append(desc) soft = passed / len(checks) hard = 1.0 if passed == len(checks) else 0.0 rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc) diff --git a/skillopt_sleep/llm_miner.py b/skillopt_sleep/llm_miner.py index dd78c63..ddf0780 100644 --- a/skillopt_sleep/llm_miner.py +++ b/skillopt_sleep/llm_miner.py @@ -22,51 +22,22 @@ import json import re from typing import Any, Callable, Dict, List +from skillopt_sleep import prompts as prompt_registry from skillopt_sleep.backend import Backend, _extract_json from skillopt_sleep.types import SessionDigest, TaskRecord -_MINER_PROMPT = """You are mining a user's past AI-assistant sessions to find RECURRING tasks -worth optimizing a skill for. From the session below, extract 0-3 reusable tasks. - -A good task is something the user asks for repeatedly or had to correct, where a -GENERAL rule would help next time (formatting, structure, tool-use, conventions). -Skip one-off or purely exploratory requests. - -For each task return: - - "intent": the reusable request, generalized (no one-off specifics) - - "checks": a list of programmatic success checks a grader can run on a future - answer. Each check is one of: - {"op":"section_present","arg":""} - {"op":"regex","arg":""} - {"op":"contains","arg":""} - {"op":"max_chars","arg":} - Only include checks you are confident a GOOD answer must satisfy. - - "rubric": a one-sentence description of what a good answer looks like - - "satisfied": true/false — did the user seem satisfied with the assistant's answer? - -Return ONLY a JSON array (possibly empty). No prose. - -# Session -project: __PROJECT__ -user prompts: -__PROMPTS__ -assistant final (last): -__FINAL__ -feedback signals: __FEEDBACK__ -""" - - def _digest_to_prompt(d: SessionDigest) -> str: + # Template lives in the central prompt registry (skillopt_sleep.prompts) + # so the dashboard can display and override it live. prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)" final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)") - return ( - _MINER_PROMPT - .replace("__PROJECT__", d.project or "(unknown)") - .replace("__PROMPTS__", prompts) - .replace("__FINAL__", final) - .replace("__FEEDBACK__", ", ".join(d.feedback_signals[:6]) or "(none)") - ) + return prompt_registry.render("miner", { + "__PROJECT__": d.project or "(unknown)", + "__PROMPTS__": prompts, + "__FINAL__": final, + "__FEEDBACK__": ", ".join(d.feedback_signals[:6]) or "(none)", + }) def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None: @@ -114,21 +85,45 @@ def make_llm_miner( """Return an llm_miner(digests) -> list[TaskRecord] bound to a backend.""" def _miner(digests: List[SessionDigest]) -> List[TaskRecord]: + ev = getattr(backend, "evidence", None) out: List[TaskRecord] = [] for d in digests[:max_sessions]: if not d.user_prompts: continue - raw = backend._call(_digest_to_prompt(d), max_tokens=800) # type: ignore[attr-defined] + prompt = _digest_to_prompt(d) + raw = backend._call(prompt, max_tokens=800) # type: ignore[attr-defined] arr = _extract_json(raw, "array") - if not isinstance(arr, list): - continue - for i, obj in enumerate(arr[:3]): + candidates = arr if isinstance(arr, list) else [] + made: List[TaskRecord] = [] + dropped = 0 + full = False + for i, obj in enumerate(candidates[:3]): if isinstance(obj, dict): t = _mk_task(d, obj, i) if t is not None: + made.append(t) out.append(t) + else: + dropped += 1 # not checkable -> dropped, and now logged if len(out) >= max_tasks: - return out + full = True + break + if ev is not None: + # The transcript->task link of the evidentiary chain: what this + # session was, exactly what the miner was asked, exactly what it + # replied, and which TaskRecords (with checks) came out of it. + ev.log("mine", "miner_exchange", session_id=d.session_id, + project=d.project, prompt=prompt, raw_reply=raw, + parse_ok=isinstance(arr, list), n_candidates=len(candidates), + n_tasks=len(made), n_dropped_uncheckable=dropped) + for t in made: + ev.log("mine", "task_mined", task_id=t.id, + session_id=d.session_id, intent=t.intent, + reference_kind=t.reference_kind, + checks=(t.judge or {}).get("checks", []), + rubric=t.reference, outcome=t.outcome) + if full: + return out return out return _miner diff --git a/skillopt_sleep/memory.py b/skillopt_sleep/memory.py index ef67f36..86d91a0 100644 --- a/skillopt_sleep/memory.py +++ b/skillopt_sleep/memory.py @@ -76,42 +76,79 @@ def apply_edits(doc: str, edits: List[EditRecord]) -> Tuple[str, List[EditRecord Returns (new_doc, applied_edits). Dedups: an `add` whose content already exists (normalized) is skipped. `delete`/`replace` match on normalized anchor substring. + + See :func:`apply_edits_detailed` when the caller also needs the edits that + matched nothing. + """ + new_doc, applied, _unmatched = apply_edits_detailed(doc, edits) + return new_doc, applied + + +def apply_edits_detailed( + doc: str, edits: List[EditRecord] +) -> Tuple[str, List[EditRecord], List[EditRecord]]: + """Apply edits and also report the ones that matched nothing. + + Returns ``(new_doc, applied, unmatched)``. An edit lands in ``unmatched`` + whenever it left the document unchanged: + + * ``delete``/``replace`` whose anchor matches no existing line; + * ``replace`` whose replacement is already present verbatim; + * ``add`` whose content duplicates an existing line (normalized), or is + empty/whitespace; + * any unrecognized op. + + Without this list such edits are invisible — they appear in neither the + applied nor the gate-rejected set, so a night can report zero edits while + the optimizer actually produced several. """ lines = current_learned_lines(doc) norm_set = {_norm(line) for line in lines} applied: List[EditRecord] = [] + unmatched: List[EditRecord] = [] for e in edits: op = (e.op or "add").lower() if op == "add": if _norm(e.content) in norm_set or not e.content.strip(): + unmatched.append(e) continue lines.append(e.content.strip()) norm_set.add(_norm(e.content)) applied.append(e) elif op == "delete": anchor = _norm(e.anchor or e.content) + if not anchor: + unmatched.append(e) + continue keep = [line for line in lines if anchor not in _norm(line)] if len(keep) != len(lines): lines = keep norm_set = {_norm(line) for line in lines} applied.append(e) + else: + unmatched.append(e) elif op == "replace": anchor = _norm(e.anchor) + replacement = e.content.strip() new_lines = [] changed = False for line in lines: if anchor and anchor in _norm(line): - new_lines.append(e.content.strip()) - changed = True + new_lines.append(replacement) + changed = changed or replacement != line else: new_lines.append(line) if changed: lines = new_lines norm_set = {_norm(line) for line in lines} applied.append(e) + else: + unmatched.append(e) + else: + unmatched.append(e) - return set_learned(doc, lines), applied + return set_learned(doc, lines), applied, unmatched def ensure_skill_scaffold(doc: str, *, name: str, description: str) -> str: diff --git a/skillopt_sleep/mine.py b/skillopt_sleep/mine.py index 4483057..9a25827 100644 --- a/skillopt_sleep/mine.py +++ b/skillopt_sleep/mine.py @@ -20,6 +20,7 @@ import re from collections import Counter from typing import Any, Callable, List, Optional, Set, Tuple +from skillopt_sleep.backend import CursorBackendError from skillopt_sleep.types import SessionDigest, TaskRecord @@ -300,6 +301,8 @@ def mine( if llm_miner is not None: try: tasks = llm_miner(digests) or [] + except CursorBackendError: + raise except Exception: tasks = [] if not tasks: diff --git a/skillopt_sleep/prompts.py b/skillopt_sleep/prompts.py new file mode 100644 index 0000000..3553328 --- /dev/null +++ b/skillopt_sleep/prompts.py @@ -0,0 +1,237 @@ +"""SkillOpt-Sleep — central prompt registry with live user overrides. + +Every LLM-facing prompt template the sleep cycle uses (miner / attempt / +judge / reflect) lives here, in one place, instead of being scattered as +inline literals. Two consequences: + + 1. **Auditability** — the dashboard (and any human) can display exactly + what instructions each agent role receives, per stage. + 2. **Live tuning** — a user override file (``prompts.json`` in the state + dir, or ``SKILLOPT_SLEEP_PROMPTS_PATH``) replaces any template without + touching code. The file's mtime is checked on every read, so an edit + made while a cycle is running takes effect on the very next call. + +Placeholders use the ``__NAME__`` convention (simple ``str.replace``, no +``str.format``) because the templates themselves contain JSON braces. + +The default texts are byte-for-byte the prompts previously inlined in +``backend.py`` / ``llm_miner.py``, so behavior is unchanged unless the user +overrides a template. +""" +from __future__ import annotations + +import json +import os +import threading +from typing import Dict, List, Optional + +from skillopt_sleep.config import HOME_STATE_DIR + +# ── default templates ───────────────────────────────────────────────────────── + +_MINER = """You are mining a user's past AI-assistant sessions to find RECURRING tasks +worth optimizing a skill for. From the session below, extract 0-3 reusable tasks. + +A good task is something the user asks for repeatedly or had to correct, where a +GENERAL rule would help next time (formatting, structure, tool-use, conventions). +Skip one-off or purely exploratory requests. + +For each task return: + - "intent": the reusable request, generalized (no one-off specifics) + - "checks": a list of programmatic success checks a grader can run on a future + answer. Each check is one of: + {"op":"section_present","arg":""} + {"op":"regex","arg":""} + {"op":"contains","arg":""} + {"op":"max_chars","arg":} + Only include checks you are confident a GOOD answer must satisfy. + - "rubric": a one-sentence description of what a good answer looks like + - "satisfied": true/false — did the user seem satisfied with the assistant's answer? + +Return ONLY a JSON array (possibly empty). No prose. + +# Session +project: __PROJECT__ +user prompts: +__PROMPTS__ +assistant final (last): +__FINAL__ +feedback signals: __FEEDBACK__ +""" + +_ATTEMPT = ( + "Complete the following task for the user. Follow the skill and memory " + "guidance below, including any output-format and length requirements. " + "When a 'Learned preferences' rule sets an explicit limit (e.g. a length " + "cap), prefer that rule over more general advice it refines.\n\n" + "# Skill\n__SKILL__\n\n# Memory\n__MEMORY__\n\n" + "# Task\n__INTENT__\n\n__CONTEXT__\n\n" + "Return ONLY the final answer text, nothing else." +) + +_JUDGE = ( + "Score how well the response satisfies the rubric, 0..1. " + 'Return ONLY JSON {"score": <0..1>, "reason": "..."}.\n\n' + "# Rubric\n__RUBRIC__\n\n# Response\n__RESPONSE__" +) + +_REFLECT = ( + "You are SkillOpt's optimizer. The agent keeps failing the recurring " + "tasks below. Propose at most __EDIT_BUDGET__ bounded edits to the " + "__TARGET__ document so it stops failing. Each edit MUST be a short, " + "GENERAL, reusable rule or preference (never task-specific, never an " + "answer to a single task). If exact failing criteria are listed, your " + "edits MUST make future outputs satisfy every one of them.\n" + "BE CONCRETE: quote the exact threshold, section name, or format from " + "the criteria verbatim in your rule (e.g. write 'keep the entire " + "response under 1200 characters', NOT 'respect length limits'). Vague " + "rules do not change behavior; specific numeric/structural rules do.\n" + "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; " + "you CANNOT delete the existing instructions above. If the current " + "__TARGET__ text conflicts with a criterion (e.g. it says 'be exhaustive' " + "but outputs must be under a character limit), write an explicit, " + "forceful OVERRIDE rule stating it supersedes the conflicting " + "instruction, and put the hard requirement first.\n" + "HARD CONSTRAINT: every rule you write MUST be consistent with the " + "'Task output contract' below (if shown). NEVER propose a rule that " + "changes the required output format/language, tells the agent to ask " + "the user a question, or otherwise violates that contract — such a " + "rule scores ZERO because the evaluator cannot honor it.\n" + 'Return ONLY a JSON array: ' + '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n' + "# Current __TARGET__\n__CUR_DOC__\n" + "__GUARD__" + "__CRITERIA__\n" + "__PREFS__\n\n" + "# Recurring failures\n__FAILURES__" +) + +# name -> {text, stage, role, description, placeholders} +DEFAULTS: Dict[str, Dict] = { + "miner": { + "text": _MINER, + "stage": "mine", + "role": "optimizer", + "description": "Turns one harvested session digest into 0-3 checkable TaskRecords.", + "placeholders": ["__PROJECT__", "__PROMPTS__", "__FINAL__", "__FEEDBACK__"], + }, + "attempt": { + "text": _ATTEMPT, + "stage": "replay", + "role": "target", + "description": "The clean-context rollout: solve a mined task given only skill+memory.", + "placeholders": ["__SKILL__", "__MEMORY__", "__INTENT__", "__CONTEXT__"], + }, + "judge": { + "text": _JUDGE, + "stage": "replay", + "role": "optimizer", + "description": "Rubric grading for tasks with no programmatic checks (0..1 JSON score).", + "placeholders": ["__RUBRIC__", "__RESPONSE__"], + }, + "reflect": { + "text": _REFLECT, + "stage": "reflect", + "role": "optimizer", + "description": "Proposes bounded skill/memory edits from the recurring failures.", + "placeholders": [ + "__EDIT_BUDGET__", "__TARGET__", "__CUR_DOC__", "__GUARD__", + "__CRITERIA__", "__PREFS__", "__FAILURES__", + ], + }, +} + + +# ── override file (mtime-cached; edits take effect on the next call) ────────── + +_lock = threading.Lock() +_cache: Dict[str, object] = {"path": None, "mtime": None, "data": {}} + + +def overrides_path() -> str: + return os.environ.get("SKILLOPT_SLEEP_PROMPTS_PATH", "") or os.path.join( + HOME_STATE_DIR, "prompts.json" + ) + + +def load_overrides() -> Dict[str, str]: + """Return {name: replacement_text}, re-reading the file iff it changed.""" + path = overrides_path() + with _lock: + try: + mtime = os.path.getmtime(path) + except OSError: + _cache.update(path=path, mtime=None, data={}) + return {} + if _cache["path"] == path and _cache["mtime"] == mtime: + return dict(_cache["data"]) # type: ignore[arg-type] + try: + with open(path, encoding="utf-8") as f: + raw = json.load(f) + data = { + k: v for k, v in raw.items() + if k in DEFAULTS and isinstance(v, str) and v.strip() + } + except Exception: + data = {} + _cache.update(path=path, mtime=mtime, data=data) + return dict(data) + + +def save_overrides(overrides: Dict[str, Optional[str]]) -> Dict[str, str]: + """Merge ``overrides`` into the override file. A None/empty value removes + that override (reverting the template to its default). Returns the new + effective override map.""" + path = overrides_path() + current = load_overrides() + for k, v in overrides.items(): + if k not in DEFAULTS: + continue + if v is None or not str(v).strip(): + current.pop(k, None) + else: + current[k] = str(v) + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(current, f, ensure_ascii=False, indent=2) + with _lock: + _cache.update(path=None, mtime=None, data={}) # force re-read + return current + + +def get_prompt(name: str) -> str: + """Effective template text for ``name`` (override if present, else default).""" + ov = load_overrides() + if name in ov: + return ov[name] + return DEFAULTS[name]["text"] + + +def is_overridden(name: str) -> bool: + return name in load_overrides() + + +def render(name: str, mapping: Dict[str, str]) -> str: + """Substitute ``__NAME__`` placeholders via str.replace (format-safe).""" + text = get_prompt(name) + for k, v in mapping.items(): + text = text.replace(k, v) + return text + + +def describe() -> List[Dict]: + """Registry snapshot for the dashboard: defaults + active overrides.""" + ov = load_overrides() + out = [] + for name, meta in DEFAULTS.items(): + out.append({ + "name": name, + "stage": meta["stage"], + "role": meta["role"], + "description": meta["description"], + "placeholders": meta["placeholders"], + "default": meta["text"], + "override": ov.get(name), + "effective": ov.get(name) or meta["text"], + }) + return out diff --git a/skillopt_sleep/replay.py b/skillopt_sleep/replay.py index e15f3df..1502a71 100644 --- a/skillopt_sleep/replay.py +++ b/skillopt_sleep/replay.py @@ -4,8 +4,9 @@ Re-run mined TaskRecords offline under a given (skill, memory) and score them, producing the (hard, soft) signal SkillOpt's gate consumes. Single-shot text replay by default. Tasks whose rule judge requires a tool -call (gbrain's `tool_called`) are run through the backend's real tool loop -(attempt_with_tools), so tool use is verified honestly rather than self-reported. +call (gbrain's `tool_called`) use the backend's tool-aware path. Backends that +cannot enforce that execution boundary fail explicitly rather than scoring a +self-reported tool call. """ from __future__ import annotations @@ -53,6 +54,18 @@ def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str, else: hard, soft, rationale = backend.judge(task, response) + ev = getattr(backend, "evidence", None) + if ev is not None: + # One scored-attempt record per (phase, task): the task->score link of + # the evidentiary chain, with the failing checks named in `why`. + ev.log("replay", "result", + phase=getattr(backend, "evidence_phase", ""), + task_id=task.id, split=task.split, origin=task.origin, + reference_kind=task.reference_kind, sample_id=sample_id, + hard=float(hard), soft=float(soft), why=rationale or "", + response_head=(response or "")[:400], tools_called=tools_called, + tokens=int(tokens), latency_ms=round(latency_ms, 1)) + return ReplayResult( id=task.id, hard=float(hard), @@ -143,4 +156,3 @@ def multi_objective_reward( if total_w <= 0: return acc return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w - diff --git a/skillopt_sleep/staging.py b/skillopt_sleep/staging.py index 49dd859..12201ec 100644 --- a/skillopt_sleep/staging.py +++ b/skillopt_sleep/staging.py @@ -16,6 +16,89 @@ from typing import Any, List, Optional from skillopt_sleep.types import SleepReport +# A secret value may be quoted, braced (ODBC-style), or an unquoted scalar. +# Accept EOF as the terminator for quoted/braced values because diagnostics are +# often truncated precisely where a failing client was printing a credential. +# Doubled quote/brace characters are the escape convention used by SQL/ODBC. +_UNQUOTED_SECRET_VALUE = ( + r'''(?:[^\s"';&,)\]}]|[)\]}]+(?=[^\s"';&,)\]}]))+''' +) +_SECRET_VALUE = ( + r'''(?:"(?:\\(?:[^\r\n]|(?=\r?\n|$))|""|[^"\\\r\n])*''' + r'''(?:"|(?=\r?\n|$))''' + r'''|'(?:\\(?:[^\r\n]|(?=\r?\n|$))|''|[^'\\\r\n])*''' + r'''(?:'|(?=\r?\n|$))''' + r'''|\{(?:\\(?:[^\r\n]|(?=\r?\n|$))|}}|[^}\\\r\n])*''' + r'''(?:}|(?=\r?\n|$))''' + r'''|''' + _UNQUOTED_SECRET_VALUE + r''')''' +) + +# Match both short labels (``token=``) and environment/connection-string names +# whose final component identifies a credential (``AZURE_CLIENT_SECRET=``). +_SECRET_NAME_BODY = ( + r"(?:(?:[A-Za-z0-9]+[_-])*(?:" + r"api[_-]?key|access[_-]?token|refresh[_-]?token|token|" + r"password|passwd|secret|secret[_-]?key|secret[_-]?access[_-]?key|" + r"shared[_-]?access[_-]?key|private[_-]?key" + r")|[A-Za-z0-9]*(?:" + r"apikey|accesstoken|refreshtoken|clientsecret|secretkey|" + r"secretaccesskey|sharedaccesskey|privatekey" + r"))" +) +_SECRET_ASSIGNMENT_NAME = ( + r"(?(?[\"'])" + + r"(?:" + _SECRET_NAME_BODY + r"|pwd|accountkey)" + + r"(?P=key_quote)\s*:\s*)" + + r"(?P" + _SECRET_VALUE + r")" +) + +_REDACTED_MARKER = re.compile(r"^\[REDACTED(?:_[A-Z_]+)?\]$") +_SECRET_MAPPING_KEY_SUFFIXES = ( + "apikey", + "accesstoken", + "refreshtoken", + "token", + "password", + "passwd", + "clientsecret", + "secret", + "secretkey", + "secretaccesskey", + "sharedaccesskey", + "privatekey", + "accountkey", +) + + +def _redact_json_assignment(match: re.Match[str]) -> str: + """Keep JSON-like value quotes while replacing their complete contents.""" + value = match.group("value") + quote = ( + value[:1] if value[:1] in {'"', "'"} else match.group("key_quote") + ) + return f"{match.group('prefix')}{quote}[REDACTED]{quote}" + + +def _is_secret_mapping_key(key: Any) -> bool: + """Recognize credential-bearing dict keys without flagging token budgets.""" + if not isinstance(key, str): + return False + stripped = key.strip() + # PWD is conventionally the non-secret process working directory. Mixed or + # lower-case ``Pwd`` remains a common database-password field. + if stripped == "PWD": + return False + compact = re.sub(r"[^a-z0-9]", "", stripped.casefold()) + return compact in {"pwd", "sig", "authorization"} or compact.endswith( + _SECRET_MAPPING_KEY_SUFFIXES + ) + # Secret patterns scrubbed from any free-text we persist to the staging dir # (diagnostics, reports). Kept here so every on-disk artifact shares one # redaction pass; harvest_codex reuses these for session text too. @@ -31,14 +114,70 @@ _SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( # the "Authorization:" prefix. (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), "[REDACTED_JWT]"), - (re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"), - (re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"), ( - re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"), + re.compile( + r'''(?i)(Authorization:\s*Bearer\s+)''' + r'''(?!\[REDACTED(?:_[A-Z_]+)?\])''' + + _SECRET_VALUE + ), + r"\1[REDACTED]", + ), + ( + re.compile( + r'''(?i)(Authorization:\s*Basic\s+)''' + r'''(?!\[REDACTED(?:_[A-Z_]+)?\])''' + + _SECRET_VALUE + ), + r"\1[REDACTED]", + ), + # Connection-string passwords. Handle quoted values (which may contain + # semicolons) before the generic name=value rule below, and retain the key + # plus all non-secret connection-string fields for useful diagnostics. + ( + re.compile( + r'''(?i)(\bPassword\s*=\s*)''' + r'''(?!\[REDACTED(?:_[A-Z_]+)?\])''' + + _SECRET_VALUE + ), + r"\1[REDACTED_DB_PASS]", + ), + # ODBC commonly abbreviates Password as Pwd. Keep the conventional + # all-uppercase PWD working-directory variable intact. + ( + re.compile( + r"((?&...) + ( + re.compile(r"(?i)(\bsig\s*=\s*)[A-Za-z0-9%+/]{10,}"), + r"\1[REDACTED_SAS_SIG]", + ), + # Azure Storage account keys (base64, typically 88 chars) + ( + re.compile( + r'''(?i)(\bAccountKey\s*=\s*)''' + r'''(?!\[REDACTED(?:_[A-Z_]+)?\])''' + + _SECRET_VALUE + ), + r"\1[REDACTED_STORAGE_KEY]", + ), ) @@ -61,14 +214,23 @@ def redact_secrets(value: Any) -> Any: scalars pass through unchanged. """ if isinstance(value, str): - out = value + out = _JSON_SECRET_ASSIGNMENT.sub(_redact_json_assignment, value) for pattern, replacement in _SECRET_PATTERNS: out = pattern.sub(replacement, out) return out if isinstance(value, list): return [redact_secrets(v) for v in value] if isinstance(value, dict): - return {k: redact_secrets(v) for k, v in value.items()} + redacted = {} + for key, item in value.items(): + if _is_secret_mapping_key(key): + if isinstance(item, str) and _REDACTED_MARKER.fullmatch(item): + redacted[key] = item + else: + redacted[key] = "[REDACTED]" + else: + redacted[key] = redact_secrets(item) + return redacted return value @@ -80,6 +242,16 @@ def staging_root(project: str) -> str: return os.path.join(project, ".skillopt-sleep", "staging") +def new_staging_dir(project: str) -> str: + """A staging path that is unique even for two runs in the same second.""" + base = os.path.join(staging_root(project), _ts_dir()) + out, i = base, 2 + while os.path.exists(out): + out = f"{base}-{i}" + i += 1 + return out + + def latest_staging(project: str) -> Optional[str]: root = staging_root(project) if not os.path.isdir(root): @@ -89,7 +261,12 @@ def latest_staging(project: str) -> Optional[str]: key=lambda p: os.path.getmtime(p), reverse=True, ) - return subs[0] if subs else None + for p in subs: + # Only adoptable folders count: a no-tasks night leaves evidence.jsonl + # but no manifest, and adopt() needs the manifest. + if os.path.exists(os.path.join(p, "manifest.json")): + return p + return None def write_staging( @@ -101,9 +278,15 @@ def write_staging( live_skill_path: str, live_memory_path: str, report_md: str, + out_dir: str = "", ) -> str: - """Write proposals + report into staging// and return that path.""" - out = os.path.join(staging_root(project), _ts_dir()) + """Write proposals + report into staging// and return that path. + + ``out_dir`` lets the cycle pre-create the night's staging folder at cycle + START, so incremental artifacts (evidence.jsonl) accumulate in the same + place the report lands. + """ + out = out_dir or os.path.join(staging_root(project), _ts_dir()) os.makedirs(out, exist_ok=True) manifest = { diff --git a/skillopt_sleep/state.py b/skillopt_sleep/state.py index 1e16157..97f4519 100644 --- a/skillopt_sleep/state.py +++ b/skillopt_sleep/state.py @@ -29,6 +29,8 @@ DEFAULT_STATE: Dict[str, Any] = { "slow_memory": "", # cross-night consolidated lessons (meta-skill analogue) "history": [], # list of per-night summaries "task_archive": [], # capped list of past mined tasks (for associative recall) + "last_model_key": "", # "backend::model" string used in the last successful night (F16) + "last_model_key_format": 1, # v1=config text; v2=resolved backend/model } @@ -94,3 +96,19 @@ class SleepState: arc.extend(task_dicts) if len(arc) > cap: self.data["task_archive"] = arc[-cap:] + + # ── model-swap tracking (F16) ───────────────────────────────────────── + @property + def last_model_key(self) -> str: + return str(self.data.get("last_model_key", "")) + + def set_last_model_key(self, key: str) -> None: + self.data["last_model_key"] = key + self.data["last_model_key_format"] = 2 + + @property + def last_model_key_format(self) -> int: + try: + return int(self.data.get("last_model_key_format", 1)) + except (TypeError, ValueError): + return 1 diff --git a/skillopt_sleep/tasks_file.py b/skillopt_sleep/tasks_file.py index d89166b..5f25f92 100644 --- a/skillopt_sleep/tasks_file.py +++ b/skillopt_sleep/tasks_file.py @@ -3,8 +3,10 @@ from __future__ import annotations import json import os +import sys from typing import Any, Dict, List, Tuple +from skillopt_sleep.judges import validate_checks from skillopt_sleep.mine import assign_splits, normalize_legacy_split from skillopt_sleep.types import TaskRecord @@ -78,4 +80,21 @@ def load_tasks_file( if not isinstance(item, dict): raise ValueError("each task entry must be a JSON object") tasks.append(TaskRecord.from_dict(item)) + + # Fail loudly on malformed rule judges. A check that cannot work as written + # scores 0.0 on every rollout, which reads exactly like a model that never + # complies — the run looks legitimate while one dimension is dead. + errors: List[str] = [] + for task in tasks: + if task.reference_kind != "rule": + continue + task_errors, task_warnings = validate_checks(task.judge) + errors.extend(f"task {task.id}: {e}" for e in task_errors) + for w in task_warnings: + print(f"[sleep] warning: task {task.id}: {w}", file=sys.stderr) + if errors: + raise ValueError( + "tasks file contains unusable rule checks:\n " + "\n ".join(errors) + ) + return _normalize_tasks(tasks, holdout_fraction=holdout_fraction, seed=seed), meta diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py index 6cfa623..6d3dc77 100644 --- a/skillopt_sleep/types.py +++ b/skillopt_sleep/types.py @@ -17,8 +17,8 @@ from typing import Any, Dict, List class SessionDigest: """A normalized summary of one local agent session transcript. - Produced by source-specific harvesters from Claude Code transcripts or - Codex Desktop archived sessions. + Produced by source-specific harvesters from Claude Code transcripts, Codex + Desktop archived sessions, or Cursor Agent transcripts. """ session_id: str @@ -29,6 +29,10 @@ class SessionDigest: user_prompts: List[str] = field(default_factory=list) assistant_finals: List[str] = field(default_factory=list) tools_used: List[str] = field(default_factory=list) + # Skill targets invoked through the Claude ``Skill`` tool, in first-seen + # order. Optional and default-empty: harvesters that do not observe skill + # invocations, and digests persisted before this field existed, leave it []. + skills_used: List[str] = field(default_factory=list) files_touched: List[str] = field(default_factory=list) feedback_signals: List[str] = field(default_factory=list) # "still broken", "perfect", ... n_user_turns: int = 0 @@ -138,6 +142,11 @@ class SleepReport: no_edits_reason: str = "" edits: List[EditRecord] = field(default_factory=list) rejected_edits: List[EditRecord] = field(default_factory=list) + # Proposed edits that changed nothing (anchor absent, duplicate/empty add, + # unknown op). They were never scored by the gate, so they belong in neither + # list above — without them a night can report no edits while the optimizer + # actually produced several. + unmatched_edits: List[EditRecord] = field(default_factory=list) tokens_used: int = 0 notes: List[str] = field(default_factory=list) diff --git a/tests/test_consolidate_split.py b/tests/test_consolidate_split.py new file mode 100644 index 0000000..8c3e2e4 --- /dev/null +++ b/tests/test_consolidate_split.py @@ -0,0 +1,90 @@ +"""Regression tests for consolidate._split hold-out contract. + +test-split tasks must never enter train or val. An all-test batch must not +silently fall back to using the held-out set for consolidation. +""" +from __future__ import annotations + +import os +import tempfile +import unittest + +from skillopt_sleep.consolidate import _split +from skillopt_sleep.tasks_file import load_tasks_file, write_tasks_file +from skillopt_sleep.types import TaskRecord + + +def _ids(tasks): + return sorted(t.id for t in tasks) + + +class TestConsolidateSplit(unittest.TestCase): + def test_all_test_batch_does_not_leak_into_train_or_val(self): + tasks = [ + TaskRecord(id="t0", project="p", intent="do X", split="test"), + TaskRecord(id="t1", project="p", intent="do Y", split="test"), + TaskRecord(id="t2", project="p", intent="do Z", split="test"), + ] + train, val = _split(tasks) + self.assertEqual(train, []) + self.assertEqual(val, []) + + def test_all_test_via_tasks_file_path_does_not_leak(self): + tasks = [ + TaskRecord(id="t0", project="p", intent="do X", split="test"), + TaskRecord(id="t1", project="p", intent="do Y", split="test"), + TaskRecord(id="t2", project="p", intent="do Z", split="test"), + ] + with tempfile.TemporaryDirectory() as tmp: + path = write_tasks_file( + os.path.join(tmp, "tasks.json"), + {"tasks": [t.to_dict() for t in tasks]}, + ) + loaded, _ = load_tasks_file(path) + train, val = _split(loaded) + self.assertEqual(_ids(train), []) + self.assertEqual(_ids(val), []) + self.assertEqual({t.split for t in loaded}, {"test"}) + + def test_train_only_falls_back_val_to_train(self): + tasks = [ + TaskRecord(id="a", project="p", intent="A", split="train"), + TaskRecord(id="b", project="p", intent="B", split="train"), + ] + train, val = _split(tasks) + self.assertEqual(_ids(train), ["a", "b"]) + self.assertEqual(_ids(val), ["a", "b"]) + + def test_train_plus_test_without_val_gates_on_train_not_test(self): + tasks = [ + TaskRecord(id="tr", project="p", intent="train", split="train"), + TaskRecord(id="te", project="p", intent="test", split="test"), + ] + train, val = _split(tasks) + self.assertEqual(_ids(train), ["tr"]) + self.assertEqual(_ids(val), ["tr"]) + self.assertNotIn("te", _ids(train) + _ids(val)) + + def test_explicit_train_val_test_keeps_partitions(self): + tasks = [ + TaskRecord(id="tr", project="p", intent="train", split="train"), + TaskRecord(id="va", project="p", intent="val", split="val"), + TaskRecord(id="te", project="p", intent="test", split="test"), + ] + train, val = _split(tasks) + self.assertEqual(_ids(train), ["tr"]) + self.assertEqual(_ids(val), ["va"]) + + def test_legacy_holdout_name_maps_to_val(self): + tasks = [ + TaskRecord(id="tr", project="p", intent="train", split="replay"), + TaskRecord(id="va", project="p", intent="val", split="holdout"), + TaskRecord(id="te", project="p", intent="test", split="test"), + ] + train, val = _split(tasks) + self.assertEqual(_ids(train), ["tr"]) + self.assertEqual(_ids(val), ["va"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_consolidation_edit_bookkeeping.py b/tests/test_consolidation_edit_bookkeeping.py new file mode 100644 index 0000000..584896c --- /dev/null +++ b/tests/test_consolidation_edit_bookkeeping.py @@ -0,0 +1,133 @@ +"""Final gate rollback must agree with edit bookkeeping and reports.""" + +from __future__ import annotations + +import importlib + +from skillopt_sleep.backend import Backend +from skillopt_sleep.memory import set_learned +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + + +def test_final_validation_rollback_reclassifies_tentative_edits( + monkeypatch, +) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + edit = EditRecord( + target="skill", + op="add", + content="A tentatively useful rule.", + rationale="trial improved", + ) + + class EditingBackend(Backend): + name = "editing-stub" + + def reflect(self, *args, **kwargs): + return [edit] + + task = TaskRecord( + id="validation-task", + project="test", + intent="test final rollback", + split="val", + reference_kind="exact", + reference="ok", + ) + scores = iter((0.0, 0.0, 1.0, 0.0)) + + def fake_replay_batch(backend, tasks, skill, memory): + score = next(scores) + return [ + ( + item, + ReplayResult( + id=item.id, + hard=score, + soft=score, + response="ok" if score else "miss", + ), + ) + for item in tasks + ] + + monkeypatch.setattr( + consolidate_module, "replay_batch", fake_replay_batch + ) + original = set_learned("# Skill\n", []) + + result = consolidate_module.consolidate( + EditingBackend(), + [task], + original, + "", + gate_metric="hard", + evolve_memory=False, + ) + + assert result.accepted is False + assert result.new_skill == original + assert result.applied_edits == [] + assert result.rejected_edits == [edit] + + +def test_final_rollback_does_not_duplicate_an_already_rejected_edit( + monkeypatch, +) -> None: + consolidate_module = importlib.import_module("skillopt_sleep.consolidate") + edit = EditRecord( + target="skill", + op="add", + content="A repeated proposal.", + rationale="same proposal for both targets", + ) + + class RepeatingBackend(Backend): + name = "repeating-stub" + + def reflect(self, *args, **kwargs): + return [edit] + + task = TaskRecord( + id="dedup-task", + project="test", + intent="test rollback deduplication", + split="val", + reference_kind="exact", + reference="ok", + ) + # baseline, train, rejected skill trial, post-skill train, accepted memory + # trial, then regressing final replay. + scores = iter((0.0, 0.0, 0.0, 0.0, 1.0, 0.0)) + + def fake_replay_batch(backend, tasks, skill, memory): + score = next(scores) + return [ + ( + item, + ReplayResult( + id=item.id, + hard=score, + soft=score, + response="ok" if score else "miss", + ), + ) + for item in tasks + ] + + monkeypatch.setattr(consolidate_module, "replay_batch", fake_replay_batch) + original = set_learned("# Skill\n", []) + + result = consolidate_module.consolidate( + RepeatingBackend(), + [task], + original, + original, + gate_metric="hard", + evolve_skill=True, + evolve_memory=True, + ) + + assert result.accepted is False + assert result.applied_edits == [] + assert result.rejected_edits == [edit] diff --git a/tests/test_cursor_exec_backend.py b/tests/test_cursor_exec_backend.py new file mode 100644 index 0000000..3fea88f --- /dev/null +++ b/tests/test_cursor_exec_backend.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import skillopt.model as model +from skillopt.config import flatten_config +from skillopt.model import backend_config +from skillopt.model import codex_harness as harness +from skillopt.model.common import default_model_for_backend, normalize_backend_name + + +@pytest.fixture(autouse=True) +def restore_backend_state() -> Iterator[None]: + optimizer_backend = backend_config.get_optimizer_backend() + target_backend = backend_config.get_target_backend() + cursor_path = backend_config.CURSOR_EXEC_PATH + cursor_sandbox = backend_config.CURSOR_EXEC_SANDBOX + retries = backend_config.EXEC_EMPTY_RESPONSE_RETRIES + env = { + key: os.environ.get(key) + for key in ( + "OPTIMIZER_BACKEND", + "TARGET_BACKEND", + "CURSOR_EXEC_PATH", + "CURSOR_EXEC_SANDBOX", + ) + } + yield + backend_config.OPTIMIZER_BACKEND = optimizer_backend + backend_config.TARGET_BACKEND = target_backend + backend_config.CURSOR_EXEC_PATH = cursor_path + backend_config.CURSOR_EXEC_SANDBOX = cursor_sandbox + backend_config.EXEC_EMPTY_RESPONSE_RETRIES = retries + for key, value in env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _result(text: str = "A") -> str: + return ( + '{"type":"system","subtype":"init","model":"composer-2.5",' + '"permissionMode":"default","session_id":"session-1"}\n' + '{"type":"tool_call","subtype":"started","call_id":"call-1",' + '"tool_call":{"readToolCall":{"args":{"path":"task.md"}}}}\n' + f'{{"type":"result","subtype":"success","is_error":false,' + f'"duration_ms":12,"result":"{text}","session_id":"session-1"}}\n' + ) + + +def _workspace(tmp_path: Path) -> Path: + work_dir = tmp_path / "predictions" / "task-1" / "cursor_exec" + work_dir.mkdir(parents=True) + return work_dir + + +def test_cursor_exec_is_target_only() -> None: + backend_config.set_target_backend("cursor") + + assert backend_config.get_target_backend() == "cursor_exec" + assert backend_config.is_target_exec_backend() + with pytest.raises(ValueError, match="Unsupported optimizer backend"): + backend_config.set_optimizer_backend("cursor_exec") + with pytest.raises(NotImplementedError, match="Exec backends"): + model.chat_target("system", "user") + + +def test_cursor_alias_and_default_model() -> None: + assert normalize_backend_name("cursor") == "cursor_exec" + assert normalize_backend_name("cursor_agent") == "cursor_exec" + assert default_model_for_backend("cursor_exec") == "composer-2.5" + + assert model.set_backend("cursor") == "cursor_exec" + assert backend_config.get_optimizer_backend() == "openai_chat" + assert backend_config.get_target_backend() == "cursor_exec" + assert model.get_backend_name() == "cursor_exec" + + +def test_cursor_config_flattens_and_validates() -> None: + flat = flatten_config( + { + "model": { + "cursor_exec_path": "/opt/cursor-agent", + "cursor_exec_sandbox": "disabled", + } + } + ) + assert flat["cursor_exec_path"] == "/opt/cursor-agent" + assert flat["cursor_exec_sandbox"] == "disabled" + + backend_config.configure_cursor_exec(path="cursor-test", sandbox="disabled") + assert backend_config.get_cursor_exec_config()["path"] == "cursor-test" + assert backend_config.get_cursor_exec_config()["sandbox"] == "disabled" + with pytest.raises(ValueError, match="sandbox must be"): + backend_config.configure_cursor_exec(sandbox="invalid") + + +def test_train_cursor_shorthand_configures_target_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from scripts import train + + config_path = Path(__file__).parents[1] / "configs" / "_base_" / "default.yaml" + monkeypatch.setattr( + sys, + "argv", + ["train.py", "--config", str(config_path), "--backend", "cursor"], + ) + + cfg = train.load_config(train.parse_args()) + + assert cfg["optimizer_backend"] == "openai_chat" + assert cfg["target_backend"] == "cursor_exec" + assert cfg["target_model"] == "composer-2.5" + + +def test_read_only_cursor_exec_uses_stdin_and_preserves_trace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + data_dir = tmp_path / "corpus" + data_dir.mkdir() + calls: list[tuple[list[str], dict[str, Any]]] = [] + + def fake_run(cmd: list[str], **kwargs: Any) -> SimpleNamespace: + calls.append((cmd, kwargs)) + return SimpleNamespace(returncode=0, stdout="not json\n" + _result(), stderr="") + + backend_config.configure_cursor_exec(path="cursor-test", sandbox="enabled") + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + response, raw = harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer the benchmark task.", + model="composer-2.5", + timeout=17, + data_dirs=[str(data_dir)], + ) + + assert response == "A" + assert "tool_call" in raw + assert "not json" not in raw + assert "task.md" not in raw + assert "A" not in raw + cmd, kwargs = calls[0] + assert cmd[:4] == ["cursor-test", "-p", "--output-format", "stream-json"] + assert ["--mode", "ask"] == cmd[cmd.index("--mode"):cmd.index("--mode") + 2] + assert "--force" not in cmd + assert cmd[cmd.index("--workspace") + 1] == str(work_dir) + assert cmd[cmd.index("--sandbox") + 1] == "enabled" + assert cmd[cmd.index("--model") + 1] == "composer-2.5" + assert cmd[cmd.index("--add-dir") + 1] == str(data_dir) + assert kwargs["cwd"] == str(work_dir) + assert kwargs["timeout"] == 17 + assert ".agents/skills/skillopt-target/SKILL.md" in kwargs["input"] + assert "Do not modify files" in kwargs["input"] + persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text() + assert "not json" not in persisted_raw + assert "task.md" not in persisted_raw + assert "A" not in persisted_raw + summary = (work_dir.parent / "cursor_trace_summary.txt").read_text() + assert "tool calls: 1" in summary + assert "session-1" in summary + + +def test_cursor_exec_force_is_limited_to_file_edit_rollouts( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + calls: list[tuple[list[str], str]] = [] + + def fake_run(cmd: list[str], **kwargs: Any) -> SimpleNamespace: + calls.append((cmd, kwargs["input"])) + return SimpleNamespace(returncode=0, stdout=_result(), stderr="") + + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Write solution.py.", + model="", + timeout=10, + allow_file_edits=True, + ) + + cmd, prompt = calls[0] + assert "--force" in cmd + assert "--mode" not in cmd + assert "You may modify files" in prompt + + +def test_cursor_exec_rejects_file_edits_with_disabled_sandbox( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + calls = 0 + + def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace: + nonlocal calls + calls += 1 + return SimpleNamespace(returncode=0, stdout=_result(), stderr="") + + backend_config.configure_cursor_exec(sandbox="disabled") + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + with pytest.raises(ValueError, match="refusing to combine --force"): + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Write solution.py.", + model="composer-2.5", + timeout=10, + allow_file_edits=True, + ) + + assert calls == 0 + + +def test_cursor_exec_retries_zero_exit_malformed_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + prompts: list[str] = [] + + def fake_run(_cmd: list[str], **kwargs: Any) -> SimpleNamespace: + prompts.append(kwargs["input"]) + stdout = "malformed output" if len(prompts) == 1 else _result() + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1 + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + response, _raw = harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer.", + model="", + timeout=10, + ) + + assert response == "A" + assert len(prompts) == 2 + assert "Previous execution returned an empty final response" in prompts[1] + + +def test_cursor_trace_summary_tolerates_malformed_metadata() -> None: + raw = ( + '{"type":"result","subtype":"success","is_error":false,' + '"duration_ms":"unknown","result":"done"}\n' + ) + + summary = harness._build_cursor_trace_summary(raw, "done") + + assert "duration ms: 0" in summary + + +def test_cursor_trace_omits_message_and_tool_payloads() -> None: + raw = "\n".join( + [ + '{"type":"user","message":{"role":"user","content":' + '[{"type":"text","text":"private prompt"}]}}', + '{"type":"assistant","message":{"role":"assistant","content":' + '[{"type":"text","text":"private response"}]}}', + '{"type":"tool_call","subtype":"completed","tool_call":' + '{"readToolCall":{"args":{"path":"secret.txt"},"result":' + '{"success":{"content":"private file contents"}}}}}', + '{"type":"result","subtype":"success","is_error":false,' + '"duration_ms":1,"result":"private final answer"}', + ] + ) + + sanitized = harness._sanitize_cursor_trace(raw) + events = [json.loads(line) for line in sanitized.splitlines()] + + assert events[0]["message"]["content"] == "[OMITTED]" + assert events[1]["message"]["content"] == "[OMITTED]" + assert events[2]["tool_call"]["readToolCall"]["args"] == "[OMITTED]" + assert events[2]["tool_call"]["readToolCall"]["result"] == "[OMITTED]" + assert events[3]["result"] == "[OMITTED]" + assert "private" not in sanitized + assert "secret.txt" not in sanitized + + +def test_cursor_exec_does_not_retry_error_result_and_redacts_detail( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + calls = 0 + + def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace: + nonlocal calls + calls += 1 + stdout = ( + '{"type":"result","subtype":"error","is_error":true,' + '"result":"API key: cursor-secret-value"}\n' + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1 + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError) as exc_info: + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer.", + model="", + timeout=10, + ) + + assert calls == 1 + assert "[REDACTED]" in str(exc_info.value) + assert "cursor-secret-value" not in str(exc_info.value) + persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text() + assert "cursor-secret-value" not in persisted_raw + assert '"result":"[OMITTED]"' in persisted_raw + + +def test_cursor_exec_nonzero_exit_is_not_retried( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + calls = 0 + + def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace: + nonlocal calls + calls += 1 + return SimpleNamespace( + returncode=1, + stdout="", + stderr="CURSOR_API_KEY=cursor-secret-token authentication failed", + ) + + backend_config.EXEC_EMPTY_RESPONSE_RETRIES = 1 + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError) as exc_info: + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer.", + model="", + timeout=10, + ) + + assert calls == 1 + assert "[REDACTED]" in str(exc_info.value) + assert "cursor-secret-token" not in str(exc_info.value) + persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text() + assert "cursor-secret-token" not in persisted_raw + assert "CURSOR_API_KEY=[REDACTED]" in persisted_raw + + +def test_cursor_exec_timeout_is_persisted( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + + def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace: + raise subprocess.TimeoutExpired( + "cursor-test", + 3, + output=b"partial CURSOR_API_KEY=timeout-secret", + ) + + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + with pytest.raises(subprocess.TimeoutExpired): + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer.", + model="", + timeout=3, + ) + + persisted_raw = (work_dir.parent / "cursor_raw.txt").read_text() + assert "partial" not in persisted_raw + assert "[OMITTED NON-JSON OUTPUT]" in persisted_raw + assert "timeout-secret" not in persisted_raw + + +def test_cursor_exec_spawn_failure_is_actionable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + work_dir = _workspace(tmp_path) + + def fake_run(_cmd: list[str], **_kwargs: Any) -> SimpleNamespace: + raise FileNotFoundError("cursor-agent-test was not found") + + monkeypatch.setattr(harness.subprocess, "run", fake_run) + + with pytest.raises(RuntimeError, match="could not be executed"): + harness.run_cursor_exec( + work_dir=str(work_dir), + prompt="Answer.", + model="", + timeout=3, + ) + + +def test_run_target_exec_dispatches_cursor( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + captured: dict[str, Any] = {} + + def fake_cursor(**kwargs: Any) -> tuple[str, str]: + captured.update(kwargs) + return "cursor response", "cursor trace" + + backend_config.set_target_backend("cursor_exec") + monkeypatch.setattr(harness, "run_cursor_exec", fake_cursor) + + response, raw = harness.run_target_exec( + work_dir=str(tmp_path), + prompt="task", + model="composer-2.5", + timeout=20, + allow_file_edits=True, + ) + + assert (response, raw) == ("cursor response", "cursor trace") + assert captured["allow_file_edits"] is True + assert captured["model"] == "composer-2.5" diff --git a/tests/test_data_manifests.py b/tests/test_data_manifests.py new file mode 100644 index 0000000..bb61095 --- /dev/null +++ b/tests/test_data_manifests.py @@ -0,0 +1,23 @@ +import json +from pathlib import Path + +from skillopt.datasets.base import SPLIT_NAMES + +DATA_DIR = Path(__file__).resolve().parents[1] / "data" + + +def test_split_manifest_counts_match_item_files() -> None: + manifest_paths = sorted(DATA_DIR.glob("*/split_manifest.json")) + assert manifest_paths, f"No split manifests found under {DATA_DIR}" + + for manifest_path in manifest_paths: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + for split_name in SPLIT_NAMES: + declared_count = manifest["counts"][split_name] + items_path = manifest_path.parent / split_name / "items.json" + actual_count = len(json.loads(items_path.read_text(encoding="utf-8"))) + + assert declared_count == actual_count, ( + f"{manifest_path} split {split_name!r}: declared count {declared_count}, actual count {actual_count}" + ) diff --git a/tests/test_devin_plugin.py b/tests/test_devin_plugin.py index fb276b9..dde3d85 100644 --- a/tests/test_devin_plugin.py +++ b/tests/test_devin_plugin.py @@ -2,6 +2,10 @@ import importlib import json import os +import shlex +import shutil +import stat +import subprocess import sys import tempfile import unittest @@ -14,6 +18,7 @@ import mcp_server # noqa: E402 import harvest_devin as hw # noqa: E402 FIXTURES = os.path.join(PLUGIN, "fixtures") +INSTALLER = os.path.join(PLUGIN, "install.sh") def _read_jsonl(path): @@ -46,7 +51,7 @@ class TestDevinMcpSchema(unittest.TestCase): def test_backends_in_enum(self): backends = mcp_server._TOOL_SCHEMA["properties"]["backend"]["enum"] - for b in ["mock", "claude", "codex", "copilot"]: + for b in ["mock", "claude", "codex", "copilot", "handoff"]: self.assertIn(b, backends) def test_schema_has_key_engine_params(self): @@ -64,6 +69,10 @@ class TestClaudeHomeExpansion(unittest.TestCase): (the documented mcp-config sets SKILLOPT_DEVIN_CLAUDE_HOME="~/...").""" def test_env_tilde_is_expanded(self): + # Re-insert the devin plugin path at position 0 so importlib.reload + # picks up this module, not plugins/copilot/mcp_server.py when both + # test modules are loaded in the same process. + sys.path.insert(0, PLUGIN) os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"] = "~/.skillopt-sleep-devin" try: importlib.reload(mcp_server) @@ -75,6 +84,165 @@ class TestClaudeHomeExpansion(unittest.TestCase): importlib.reload(mcp_server) +class TestDevinInstaller(unittest.TestCase): + def _run_installer(self, project, home, installer=INSTALLER): + env = os.environ.copy() + env["HOME"] = home + return subprocess.run( + ["bash", installer, project], + check=True, + capture_output=True, + text=True, + env=env, + ) + + @staticmethod + def _skillopt_hook(): + config_path = os.path.join(PLUGIN, "hooks", "hooks.v1.json") + with open(config_path, encoding="utf-8") as f: + return json.load(f)["SessionEnd"][0] + + def test_new_install_and_hook_marker(self): + with tempfile.TemporaryDirectory() as d: + project = os.path.join(d, "project with spaces") + home = os.path.join(d, "home") + os.makedirs(project) + os.makedirs(home) + + self._run_installer(project, home) + + config_path = os.path.join(project, ".devin", "hooks.v1.json") + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + self.assertEqual(config["SessionEnd"], [self._skillopt_hook()]) + + hook_path = os.path.join( + project, ".devin", "hooks", "skillopt-sleep-on-session-end.sh" + ) + self.assertTrue(os.stat(hook_path).st_mode & stat.S_IXUSR) + env = os.environ.copy() + env.update(HOME=home, DEVIN_PROJECT_DIR=project) + subprocess.run([hook_path], check=True, env=env) + marker = os.path.join(home, ".skillopt-sleep", "session-end.log") + with open(marker, encoding="utf-8") as f: + lines = f.readlines() + self.assertEqual(len(lines), 1) + self.assertTrue(lines[0].endswith(f"\t{project}\n")) + + def test_hook_is_non_blocking_without_home(self): + env = os.environ.copy() + env.pop("HOME", None) + result = subprocess.run( + [os.path.join(PLUGIN, "hooks", "on-session-end.sh")], + capture_output=True, + text=True, + env=env, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stderr, "") + + def test_existing_config_without_session_end_is_extended(self): + unrelated = [ + {"matcher": "", "hooks": [{"type": "command", "command": "./pre.sh"}]} + ] + with tempfile.TemporaryDirectory() as d: + project = os.path.join(d, "project") + home = os.path.join(d, "home") + devin_dir = os.path.join(project, ".devin") + os.makedirs(devin_dir) + os.makedirs(home) + config_path = os.path.join(devin_dir, "hooks.v1.json") + with open(config_path, "w", encoding="utf-8") as f: + json.dump({"PreToolUse": unrelated}, f) + + self._run_installer(project, home) + + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + self.assertEqual(config["PreToolUse"], unrelated) + self.assertEqual(config["SessionEnd"], [self._skillopt_hook()]) + + def test_existing_hooks_are_preserved_and_reinstall_is_idempotent(self): + existing_session_end = { + "matcher": "existing", + "hooks": [{"type": "command", "command": "./existing.sh"}], + } + unrelated = [ + {"matcher": "", "hooks": [{"type": "command", "command": "./pre.sh"}]} + ] + with tempfile.TemporaryDirectory() as d: + project = os.path.join(d, "project") + home = os.path.join(d, "home") + devin_dir = os.path.join(project, ".devin") + os.makedirs(devin_dir) + os.makedirs(home) + hooks_dir = os.path.join(devin_dir, "hooks") + os.makedirs(hooks_dir) + legacy_hook = os.path.join(hooks_dir, "on-session-end.sh") + with open(legacy_hook, "w", encoding="utf-8") as f: + f.write("#!/bin/sh\n# existing project hook\n") + config_path = os.path.join(devin_dir, "hooks.v1.json") + with open(config_path, "w", encoding="utf-8") as f: + json.dump( + {"PreToolUse": unrelated, "SessionEnd": [existing_session_end]}, f + ) + + self._run_installer(project, home) + self._run_installer(project, home) + + with open(config_path, encoding="utf-8") as f: + config = json.load(f) + self.assertEqual(config["PreToolUse"], unrelated) + self.assertIn(existing_session_end, config["SessionEnd"]) + self.assertEqual(config["SessionEnd"].count(self._skillopt_hook()), 1) + with open(legacy_hook, encoding="utf-8") as f: + self.assertEqual(f.read(), "#!/bin/sh\n# existing project hook\n") + + def test_malformed_existing_config_fails_without_overwrite(self): + with tempfile.TemporaryDirectory() as d: + project = os.path.join(d, "project") + home = os.path.join(d, "home") + devin_dir = os.path.join(project, ".devin") + os.makedirs(devin_dir) + os.makedirs(home) + config_path = os.path.join(devin_dir, "hooks.v1.json") + original = "{not-json\n" + with open(config_path, "w", encoding="utf-8") as f: + f.write(original) + + with self.assertRaises(subprocess.CalledProcessError): + self._run_installer(project, home) + with open(config_path, encoding="utf-8") as f: + self.assertEqual(f.read(), original) + + def test_registration_path_is_shell_quoted(self): + with tempfile.TemporaryDirectory() as d: + plugin_copy = os.path.join( + d, "repo with spaces $dollar `tick` 'quote'", "plugins", "devin" + ) + shutil.copytree(PLUGIN, plugin_copy) + project = os.path.join(d, "project") + home = os.path.join(d, "home") + os.makedirs(project) + os.makedirs(home) + + result = self._run_installer( + project, + home, + installer=os.path.join(plugin_copy, "install.sh"), + ) + + command_line = next( + line.strip() + for line in result.stdout.splitlines() + if line.strip().startswith("-- python3 ") + ) + self.assertEqual( + shlex.split(command_line), + ["--", "python3", os.path.join(plugin_copy, "mcp_server.py")], + ) + + class TestDevinHarvest(unittest.TestCase): def test_atif_fixture_yields_gradeable_task(self): with tempfile.TemporaryDirectory() as out: diff --git a/tests/test_executor_env_isolation.py b/tests/test_executor_env_isolation.py new file mode 100644 index 0000000..0c721ce --- /dev/null +++ b/tests/test_executor_env_isolation.py @@ -0,0 +1,196 @@ +"""Tests for subprocess environment isolation in the spreadsheet executor. + +``run_generated_code`` runs LLM-generated Python in a child process. To avoid +leaking API keys / cloud credentials into untrusted generated code, the child +must run with a minimal, scrubbed environment rather than inheriting the +parent process environment. These tests assert that scrubbing behaviour. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from types import SimpleNamespace + +from skillopt.envs.spreadsheetbench.codegen_agent import _build_codex_driver +from skillopt.envs.spreadsheetbench.executor import generated_code_env, run_generated_code +from skillopt.envs.spreadsheetbench.react_agent import _run_bash + + +# User code that records whether a given env var is visible to the child. +_PROBE = ( + "import os\n" + "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n" + " _f.write(os.environ.get('SUPER_SECRET_TOKEN', 'ABSENT'))\n" +) + + +def test_secret_env_not_visible_to_generated_code(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("SUPER_SECRET_TOKEN", "leak-me-please") + out = tmp_path / "out.txt" + + ok, err = run_generated_code(_PROBE, str(tmp_path / "in.xlsx"), str(out)) + + assert ok, err + assert out.read_text(encoding="utf-8") == "ABSENT" + + +def test_path_still_available_to_generated_code(tmp_path) -> None: + # PATH must be preserved so the child can still locate the interpreter's + # tooling; only sensitive vars are dropped. + probe = ( + "import os\n" + "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n" + " _f.write('YES' if os.environ.get('PATH') else 'NO')\n" + ) + out = tmp_path / "out.txt" + + ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out)) + + assert ok, err + assert out.read_text(encoding="utf-8") == "YES" + + +def test_non_secret_python_environment_is_preserved(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("PYTHONPATH", "/safe/development/path") + monkeypatch.setenv("LANG", "C.UTF-8") + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "must-not-leak") + + env = generated_code_env(str(tmp_path), str(tmp_path / "scratch")) + + assert env["PYTHONPATH"] == "/safe/development/path" + assert env["LANG"] == "C.UTF-8" + assert "AZURE_OPENAI_API_KEY" not in env + + +def test_installed_spreadsheet_dependency_is_importable(tmp_path) -> None: + probe = ( + "import openpyxl\n" + "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n" + " _f.write('YES')\n" + ) + out = tmp_path / "out.txt" + + ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out)) + + assert ok, err + assert out.read_text(encoding="utf-8") == "YES" + + +def test_generated_scratch_directory_is_private_and_cleaned(tmp_path) -> None: + probe = ( + "import tempfile\n" + "scratch = tempfile.NamedTemporaryFile(delete=False)\n" + "scratch.close()\n" + "with open(OUTPUT_PATH, 'w', encoding='utf-8') as _f:\n" + " _f.write(scratch.name)\n" + ) + out = tmp_path / "out.txt" + + ok, err = run_generated_code(probe, str(tmp_path / "in.xlsx"), str(out)) + + assert ok, err + scratch_path = out.read_text(encoding="utf-8") + assert os.path.dirname(scratch_path) != str(tmp_path) + assert not os.path.exists(scratch_path) + + +def test_codex_driver_scrubs_env_sets_tempdir_and_cleans_runner( + tmp_path, monkeypatch +) -> None: + monkeypatch.setenv("SUPER_SECRET_TOKEN", "do-not-inherit") + monkeypatch.setenv("PYTHONPATH", "/safe/development/path") + sentinel = tmp_path / "_driver_runner.py" + sentinel.write_text("do not overwrite", encoding="utf-8") + (tmp_path / "solution.py").write_text( + "import os\n" + "with open(OUTPUT_PATH, 'w', encoding='utf-8') as f:\n" + " f.write('|'.join([\n" + " os.environ.get('SUPER_SECRET_TOKEN', 'ABSENT'),\n" + " 'TMPDIR' if os.environ.get('TMPDIR') else 'NO_TMPDIR',\n" + " 'PRIVATE_TMP' if os.environ.get('TMPDIR') != os.getcwd() else 'BAD_TMP',\n" + " 'PRIVATE_HOME' if os.environ.get('HOME') == os.getcwd() else 'BAD_HOME',\n" + " 'DEV_PATH' if os.environ.get('PYTHONPATH') == '/safe/development/path' else 'NO_DEV_PATH',\n" + " ]))\n", + encoding="utf-8", + ) + driver = tmp_path / "run_solution.py" + driver.write_text(_build_codex_driver(), encoding="utf-8") + + proc = subprocess.run( + [sys.executable, str(driver)], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + ) + + assert proc.returncode == 0, proc.stdout + proc.stderr + assert ( + tmp_path / "output.xlsx" + ).read_text(encoding="utf-8") == ( + "ABSENT|TMPDIR|PRIVATE_TMP|PRIVATE_HOME|DEV_PATH" + ) + assert sentinel.read_text(encoding="utf-8") == "do not overwrite" + + +def test_generated_code_tempdirs_tolerate_windows_cleanup_races( + tmp_path, monkeypatch +) -> None: + cleanup_modes = [] + + def simulated_windows_rmtree(cls, name, ignore_errors=False, repeated=False): + del cls, repeated + cleanup_modes.append(ignore_errors) + if not ignore_errors: + raise PermissionError(32, "directory is still in use", name) + shutil.rmtree(name, ignore_errors=True) + + monkeypatch.setattr( + tempfile.TemporaryDirectory, + "_rmtree", + classmethod(simulated_windows_rmtree), + ) + monkeypatch.setattr( + "skillopt.envs.spreadsheetbench.executor.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="", stderr=""), + ) + out = tmp_path / "out.txt" + out.write_text("created", encoding="utf-8") + + ok, err = run_generated_code("pass", "input.xlsx", str(out)) + + assert ok, err + assert cleanup_modes == [True] + + +def test_react_tempdir_tolerates_windows_cleanup_races(tmp_path, monkeypatch) -> None: + cleanup_modes = [] + + def simulated_windows_rmtree(cls, name, ignore_errors=False, repeated=False): + del cls, repeated + cleanup_modes.append(ignore_errors) + if not ignore_errors: + raise PermissionError(32, "directory is still in use", name) + shutil.rmtree(name, ignore_errors=True) + + monkeypatch.setattr( + tempfile.TemporaryDirectory, + "_rmtree", + classmethod(simulated_windows_rmtree), + ) + monkeypatch.setattr( + "skillopt.envs.spreadsheetbench.react_agent.subprocess.run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, stdout="success", stderr="" + ), + ) + + assert _run_bash("python -c pass", str(tmp_path)) == "success" + assert cleanup_modes == [True] + + +def test_codex_driver_uses_cleanup_tolerant_tempdir() -> None: + assert "shutil.rmtree(_temp_dir, ignore_errors=True)" in _build_codex_driver() diff --git a/tests/test_judges.py b/tests/test_judges.py new file mode 100644 index 0000000..e4aca3d --- /dev/null +++ b/tests/test_judges.py @@ -0,0 +1,211 @@ +"""Tests for skillopt_sleep.judges — the rule judge that decides every night. + +Focus: a malformed check must never be indistinguishable from an unmet one. +A regex that does not compile returns False on every rollout, so the affected +dimension scores 0.0 forever while the run still looks healthy. +""" +import json +import unittest + +import pytest + +from skillopt_sleep.judges import KNOWN_OPS, score_rule_judge, validate_checks +from skillopt_sleep.tasks_file import load_tasks_file + + +class TestCheckOperators(unittest.TestCase): + def _score(self, op, arg, response, tools=None): + return score_rule_judge({"kind": "rule", "checks": [{"op": op, "arg": arg}]}, + response, tools or []) + + def test_contains_is_case_insensitive(self) -> None: + self.assertEqual(self._score("contains", "Key Risks", "the KEY RISKS section")[0], 1.0) + + def test_regex_matches(self) -> None: + self.assertEqual(self._score("regex", r"\d+\.\d+", "see 7.4 for details")[0], 1.0) + + def test_min_and_max_chars(self) -> None: + self.assertEqual(self._score("min_chars", 5, "abcdef")[0], 1.0) + self.assertEqual(self._score("min_chars", 50, "abcdef")[0], 0.0) + self.assertEqual(self._score("max_chars", 5, "abcdef")[0], 0.0) + + def test_section_present_accepts_heading_and_bold_and_label(self) -> None: + for text in ("## Key Risks", "**Key Risks:**", "Key Risks: something"): + self.assertEqual(self._score("section_present", "Key Risks", text)[0], 1.0, text) + + def test_tool_called_via_marker(self) -> None: + self.assertEqual(self._score("tool_called", "search", "TOOL_CALL: search")[0], 1.0) + self.assertEqual(self._score("tool_called", "search", "nope", ["search"])[0], 1.0) + + def test_unknown_op_does_not_block(self) -> None: + self.assertEqual(self._score("no_such_op", "x", "anything")[0], 1.0) + + +class TestSoftAndHardScoring(unittest.TestCase): + def test_soft_is_fraction_and_hard_is_all_or_nothing(self) -> None: + judge = {"kind": "rule", "checks": [ + {"op": "contains", "arg": "alpha"}, + {"op": "contains", "arg": "beta"}, + {"op": "contains", "arg": "gamma"}, + ]} + hard, soft, why = score_rule_judge(judge, "alpha and beta only") + self.assertEqual(hard, 0.0) + self.assertAlmostEqual(soft, 2 / 3) + self.assertIn("gamma", why) + + def test_empty_checks_score_zero(self) -> None: + self.assertEqual(score_rule_judge({"kind": "rule", "checks": []}, "x")[:2], (0.0, 0.0)) + + +class TestMalformedRegexIsDistinguishable(unittest.TestCase): + """A pattern Python cannot parse must not read like a plain miss.""" + + BAD = r"foo(" # unclosed group is invalid on every supported Python version + + def test_bad_pattern_still_fails_closed(self) -> None: + # the response does contain both alternatives; the pattern is the problem + hard, soft, _why = score_rule_judge( + {"kind": "rule", "checks": [{"op": "regex", "arg": self.BAD}]}, "foo bar") + self.assertEqual(hard, 0.0) + self.assertEqual(soft, 0.0) + + def test_rationale_names_the_pattern_as_invalid(self) -> None: + _hard, _soft, why = score_rule_judge( + {"kind": "rule", "checks": [{"op": "regex", "arg": self.BAD}]}, "foo bar") + self.assertIn("invalid regex", why) + + def test_a_genuine_miss_is_not_labelled_invalid(self) -> None: + _hard, _soft, why = score_rule_judge( + {"kind": "rule", "checks": [{"op": "regex", "arg": r"zzz"}]}, "foo bar") + self.assertNotIn("invalid regex", why) + + +class TestValidateChecks(unittest.TestCase): + def test_sound_checks_produce_nothing(self) -> None: + errors, warnings = validate_checks({"checks": [ + {"op": "regex", "arg": r"^\s*SKILL:"}, + {"op": "min_chars", "arg": 10}, + {"op": "section_present", "arg": "Risks"}, + ]}) + self.assertEqual(errors, []) + self.assertEqual(warnings, []) + + def test_uncompilable_regex_is_an_error(self) -> None: + errors, _warnings = validate_checks({"checks": [{"op": "regex", "arg": r"foo("}]}) + self.assertEqual(len(errors), 1) + self.assertIn("does not compile", errors[0]) + + def test_non_integer_char_bound_is_an_error(self) -> None: + errors, _warnings = validate_checks({"checks": [{"op": "max_chars", "arg": "many"}]}) + self.assertEqual(len(errors), 1) + self.assertIn("integer", errors[0]) + + def test_negative_char_bounds_are_errors(self) -> None: + for op in ("max_chars", "min_chars"): + errors, _warnings = validate_checks( + {"checks": [{"op": op, "arg": -1}]} + ) + self.assertEqual(len(errors), 1) + self.assertIn("negative", errors[0]) + + def test_non_integral_or_non_finite_char_bounds_are_errors(self) -> None: + for arg in (1.9, float("inf"), float("-inf"), float("nan")): + errors, _warnings = validate_checks( + {"checks": [{"op": "max_chars", "arg": arg}]} + ) + self.assertEqual(len(errors), 1, repr(arg)) + self.assertIn("integer", errors[0]) + + def test_integral_float_and_integer_string_bounds_are_valid(self) -> None: + for arg in (2.0, " 2 ", "+2"): + errors, _warnings = validate_checks( + {"checks": [{"op": "max_chars", "arg": arg}]} + ) + self.assertEqual(errors, [], repr(arg)) + + def test_zero_min_chars_is_flagged_as_toothless(self) -> None: + errors, warnings = validate_checks( + {"checks": [{"op": "min_chars", "arg": 0}]} + ) + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + self.assertIn("always passes", warnings[0]) + + def test_empty_string_operator_arguments_are_errors(self) -> None: + for op in ("regex", "section_present", "contains", "tool_called"): + errors, _warnings = validate_checks( + {"checks": [{"op": op, "arg": " "}]} + ) + self.assertEqual(len(errors), 1, op) + self.assertIn("non-empty", errors[0]) + + def test_unknown_op_is_only_a_warning(self) -> None: + errors, warnings = validate_checks({"checks": [{"op": "vibes", "arg": 1}]}) + self.assertEqual(errors, []) + self.assertEqual(len(warnings), 1) + self.assertIn("always passes", warnings[0]) + + def test_non_string_op_is_a_structured_error(self) -> None: + for op in ([], {}, 1, None): + errors, warnings = validate_checks( + {"checks": [{"op": op, "arg": "value"}]} + ) + self.assertEqual(len(errors), 1, repr(op)) + self.assertIn("must be a string", errors[0]) + self.assertEqual(warnings, []) + + def test_non_object_check_is_an_error(self) -> None: + errors, _warnings = validate_checks({"checks": ["not-an-object"]}) + self.assertEqual(len(errors), 1) + + def test_malformed_judge_is_reported_not_raised(self) -> None: + # a tasks file may carry any JSON here; validation must stay structured + for bad in (["not", "a", "dict"], [], "a string", "", 42, 0): + errors, _warnings = validate_checks(bad) + self.assertEqual(len(errors), 1, bad) + self.assertIn("must be an object", errors[0]) + + def test_non_list_checks_is_reported_not_raised(self) -> None: + errors, _warnings = validate_checks({"checks": "regex"}) + self.assertEqual(len(errors), 1) + self.assertIn("must be an array", errors[0]) + + def test_empty_or_missing_judge_is_sound(self) -> None: + for empty in ({}, None, {"checks": []}): + self.assertEqual(validate_checks(empty), ([], []), empty) + + def test_every_known_op_is_accepted(self) -> None: + for op in KNOWN_OPS: + arg = 1 if op.endswith("_chars") else "x" + errors, warnings = validate_checks({"checks": [{"op": op, "arg": arg}]}) + self.assertEqual((errors, warnings), ([], []), op) + + +@pytest.mark.parametrize("bad_judge", [[], "", 0]) +def test_tasks_file_rejects_falsy_non_object_judges( + tmp_path, bad_judge +) -> None: + path = tmp_path / "tasks.json" + path.write_text( + json.dumps( + { + "tasks": [ + { + "id": "malformed-rule", + "project": "test", + "intent": "test", + "reference_kind": "rule", + "judge": bad_judge, + } + ] + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="judge must be an object"): + load_tasks_file(str(path)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_model_change_warning.py b/tests/test_model_change_warning.py new file mode 100644 index 0000000..5989579 --- /dev/null +++ b/tests/test_model_change_warning.py @@ -0,0 +1,237 @@ +"""Tests for the F16 model-change warning and F08 CLI credential warning.""" +from __future__ import annotations + +import argparse +import importlib +import warnings + +from skillopt_sleep.cycle import _check_model_change, _make_model_key +from skillopt_sleep.config import load_config +from skillopt_sleep.state import SleepState + + +def _cfg(): + cfg = load_config() + return cfg + + +def test_last_model_key_roundtrips(tmp_path) -> None: + path = str(tmp_path / "state.json") + state = SleepState.load(path) + assert state.last_model_key == "" + state.set_last_model_key("anthropic::claude") + assert state.last_model_key_format == 2 + state.save() + assert SleepState.load(path).last_model_key == "anthropic::claude" + + +def test_warns_when_model_changed(tmp_path, capsys) -> None: + cfg = _cfg() + state = SleepState.load(str(tmp_path / "state.json")) + state.set_last_model_key("some-other::model") + _check_model_change(cfg, state) + err = capsys.readouterr().err + assert "model changed since last night" in err + + +def test_no_warning_on_first_night(tmp_path, capsys) -> None: + cfg = _cfg() + state = SleepState.load(str(tmp_path / "state.json")) # last_model_key == "" + _check_model_change(cfg, state) + assert "model changed" not in capsys.readouterr().err + + +def test_no_warning_when_model_same(tmp_path, capsys) -> None: + cfg = _cfg() + state = SleepState.load(str(tmp_path / "state.json")) + state.set_last_model_key(_make_model_key(cfg)) + _check_model_change(cfg, state) + assert "model changed" not in capsys.readouterr().err + + +def test_model_key_tracks_effective_optimizer_and_target_roles() -> None: + cfg = load_config( + backend="mock", + model="shared", + optimizer_backend="claude", + optimizer_model="opus", + target_backend="codex", + target_model="gpt", + ) + assert _make_model_key(cfg) == ( + "optimizer=claude::opus;target=codex::gpt" + ) + + inherited = load_config( + backend="mock", + model="shared", + optimizer_backend="claude", + ) + assert _make_model_key(inherited) == ( + "optimizer=claude::shared;target=mock::" + ) + + +def test_model_key_normalizes_aliases_and_tracks_environment_defaults( + monkeypatch, +) -> None: + monkeypatch.delenv("SKILLOPT_SLEEP_CLAUDE_MODEL", raising=False) + claude = _make_model_key(load_config(backend="claude", model="")) + anthropic = _make_model_key(load_config(backend="anthropic", model="")) + assert claude == anthropic == "claude::sonnet" + + monkeypatch.setenv("SKILLOPT_SLEEP_CLAUDE_MODEL", "opus") + assert _make_model_key(load_config(backend="claude", model="")) == "claude::opus" + + +def test_model_key_resolution_failure_uses_safe_config_fallback(monkeypatch) -> None: + cycle_module = importlib.import_module("skillopt_sleep.cycle") + + def fail_to_build(**_kwargs): + raise RuntimeError("diagnostic construction failed") + + monkeypatch.setattr(cycle_module, "build_backend", fail_to_build) + cfg = load_config( + backend="claude", + model="sonnet", + optimizer_backend="codex", + optimizer_model="gpt", + ) + assert cycle_module._make_model_key(cfg) == ( + "configured:optimizer=codex::gpt;target=claude::sonnet" + ) + + +def test_legacy_unresolved_model_key_migrates_without_false_warning( + tmp_path, capsys +) -> None: + state = SleepState.load(str(tmp_path / "state.json")) + state.data["last_model_key"] = "claude::" + state.data["last_model_key_format"] = 1 + + _check_model_change(load_config(backend="claude", model=""), state) + + assert "model changed" not in capsys.readouterr().err + + +def test_warns_when_one_split_backend_role_changes(tmp_path, capsys) -> None: + previous = load_config( + optimizer_backend="claude", + optimizer_model="opus", + target_backend="codex", + target_model="gpt", + ) + current = load_config( + optimizer_backend="claude", + optimizer_model="opus", + target_backend="cursor", + target_model="composer", + ) + state = SleepState.load(str(tmp_path / "state.json")) + state.set_last_model_key(_make_model_key(previous)) + + _check_model_change(current, state) + + assert "model changed since last night" in capsys.readouterr().err + + +def test_cli_api_key_emits_deprecation_warning() -> None: + from scripts.train import load_config as train_load_config + + args = argparse.Namespace( + config="configs/_base_/default.yaml", + cfg_options=None, + azure_openai_api_key="sk-secret-value", + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + train_load_config(args) + except Exception: + pass # config loading may fail; we only assert the warning fired + assert any( + issubclass(w.category, DeprecationWarning) + and "azure_openai_api_key" in str(w.message) + for w in caught + ) + + +def test_cli_key_warnings_name_the_correct_environment_variable() -> None: + from scripts.train import load_config as train_load_config + + cases = { + "optimizer_azure_openai_api_key": "OPTIMIZER_AZURE_OPENAI_API_KEY", + "target_azure_openai_api_key": "TARGET_AZURE_OPENAI_API_KEY", + "qwen_chat_api_key": "QWEN_CHAT_API_KEY", + "optimizer_qwen_chat_api_key": "OPTIMIZER_QWEN_CHAT_API_KEY", + "target_qwen_chat_api_key": "TARGET_QWEN_CHAT_API_KEY", + "minimax_api_key": "MINIMAX_API_KEY", + } + for flag, environment_variable in cases.items(): + args = argparse.Namespace( + config="configs/_base_/default.yaml", + cfg_options=None, + **{flag: "secret-value"}, + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + train_load_config(args) + except Exception: + pass + messages = [ + str(w.message) + for w in caught + if issubclass(w.category, DeprecationWarning) + ] + assert any(environment_variable in message for message in messages), flag + + +def test_cfg_options_api_key_emits_safe_deprecation_warning() -> None: + from scripts.train import load_config as train_load_config + + args = argparse.Namespace( + config="configs/_base_/default.yaml", + cfg_options=["model.azure_openai_api_key=do-not-print-this-secret"], + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + train_load_config(args) + except Exception: + pass + messages = [ + str(w.message) + for w in caught + if issubclass(w.category, DeprecationWarning) + ] + assert any("AZURE_OPENAI_API_KEY" in message for message in messages) + assert all("do-not-print-this-secret" not in message for message in messages) + + +def test_cfg_options_warning_uses_leaf_name_and_catches_future_secrets() -> None: + from scripts.train import load_config as train_load_config + + args = argparse.Namespace( + config="configs/_base_/default.yaml", + cfg_options=[ + "custom.optimizer_qwen_chat_api_key=role-secret", + "future.backend.access_token=future-secret", + ], + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + train_load_config(args) + except Exception: + pass + messages = [ + str(w.message) + for w in caught + if issubclass(w.category, DeprecationWarning) + ] + + assert any("OPTIMIZER_QWEN_CHAT_API_KEY" in message for message in messages) + assert any("future.backend.access_token" in message for message in messages) + assert all("role-secret" not in message for message in messages) + assert all("future-secret" not in message for message in messages) diff --git a/tests/test_plugin_sync.py b/tests/test_plugin_sync.py index f7850e2..6df09e1 100644 --- a/tests/test_plugin_sync.py +++ b/tests/test_plugin_sync.py @@ -2,6 +2,7 @@ Run: python3 -m pytest tests/test_plugin_sync.py -v """ +import json import os import unittest @@ -10,6 +11,7 @@ REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) PLUGIN_SKILL_MDS = { "claude-code": os.path.join(REPO, "plugins/claude-code/skills/skillopt-sleep/SKILL.md"), "codex": os.path.join(REPO, "plugins/codex/skills/skillopt-sleep/SKILL.md"), + "cursor": os.path.join(REPO, "plugins/cursor/skills/skillopt-sleep/SKILL.md"), "openclaw": os.path.join(REPO, "plugins/openclaw/SKILL.md"), } @@ -17,6 +19,14 @@ MCP_SERVER = os.path.join(REPO, "plugins/copilot/mcp_server.py") COPILOT_INSTRUCTIONS = os.path.join(REPO, "plugins/copilot/copilot-instructions.snippet.md") CANONICAL_BACKENDS = {"mock", "claude", "codex", "copilot"} +CURSOR_MANIFEST = os.path.join(REPO, "plugins/cursor/.cursor-plugin/plugin.json") +CURSOR_MARKETPLACE = os.path.join(REPO, ".cursor-plugin/marketplace.json") +CURSOR_COMMAND = os.path.join(REPO, "plugins/cursor/commands/skillopt-sleep.md") +CURSOR_README = os.path.join(REPO, "plugins/cursor/README.md") +CURSOR_INSTALL_SH = os.path.join(REPO, "plugins/cursor/install.sh") +CURSOR_INSTALL_PS1 = os.path.join(REPO, "plugins/cursor/install.ps1") +CURSOR_LICENSE = os.path.join(REPO, "plugins/cursor/LICENSE") +OPENCLAW_RUNNER = os.path.join(REPO, "plugins/openclaw/run_sleep.py") def _read(path): @@ -27,6 +37,82 @@ def _read(path): class TestPluginParity(unittest.TestCase): + def test_cursor_plugin_manifest_and_marketplace_registration(self): + with open(CURSOR_MANIFEST, encoding="utf-8") as f: + manifest = json.load(f) + with open(CURSOR_MARKETPLACE, encoding="utf-8") as f: + marketplace = json.load(f) + + self.assertEqual(manifest["name"], "skillopt-sleep") + self.assertEqual(manifest["skills"], "./skills/") + self.assertEqual(manifest["commands"], "./commands/") + self.assertNotIn("hooks", manifest) + self.assertNotIn("mcpServers", manifest) + allowed_manifest_keys = { + "name", "displayName", "description", "version", "author", + "publisher", "homepage", "repository", "license", "logo", + "keywords", "category", "tags", "commands", "agents", "skills", + "rules", "hooks", "mcpServers", + } + self.assertEqual(set(manifest) - allowed_manifest_keys, set()) + registered = next( + (plugin for plugin in marketplace["plugins"] if plugin.get("name") == "skillopt-sleep"), + None, + ) + self.assertIsNotNone(registered) + self.assertEqual(registered["source"], "plugins/cursor") + self.assertEqual(registered["name"], manifest["name"]) + self.assertTrue(os.path.isdir(os.path.join(REPO, registered["source"]))) + + def test_cursor_skill_has_frontmatter_target_and_cursor_guidance(self): + text = _read(PLUGIN_SKILL_MDS["cursor"]) + self.assertTrue(text.startswith("---\n")) + self.assertIn("name: skillopt-sleep", text) + self.assertIn(".cursor/skills/skillopt-sleep-learned/SKILL.md", text) + self.assertIn("--source cursor", text) + self.assertIn("--backend cursor", text) + + def test_cursor_command_is_thin_and_preserves_safety_defaults(self): + text = _read(CURSOR_COMMAND) + self.assertIn("$ARGUMENTS", text) + self.assertIn("use `status`", text) + self.assertIn("--source cursor", text) + self.assertIn("--scope invoked", text) + self.assertIn(".cursor/skills/skillopt-sleep-learned/SKILL.md", text) + self.assertIn("`mock` backend", text) + self.assertNotIn("--auto-adopt", text) + + def test_cursor_installers_package_command_skill_readme_and_license(self): + for installer in (CURSOR_INSTALL_SH, CURSOR_INSTALL_PS1): + text = _read(installer) + for filename in ( + "plugin.json", + "commands/skillopt-sleep.md" if installer.endswith(".sh") else "commands\\skillopt-sleep.md", + "skills/skillopt-sleep/SKILL.md" if installer.endswith(".sh") else "skills\\skillopt-sleep\\SKILL.md", + "README.md", + "LICENSE", + ): + self.assertIn(filename, text, f"{installer} does not package {filename}") + + self.assertEqual(_read(CURSOR_LICENSE), _read(os.path.join(REPO, "LICENSE"))) + + def test_cursor_docs_keep_scheduling_explicit_and_target_relative(self): + for path in (CURSOR_README, PLUGIN_SKILL_MDS["cursor"]): + text = _read(path) + self.assertIn('"target_skill_path": ".cursor/skills/', text) + self.assertIn("no session-end hook", text.lower()) + self.assertIn("`tool_called`", text) + self.assertIn("temporarily disabled", text.lower()) + self.assertIn("before agent mode", text.lower()) + + def test_openclaw_wrapper_matches_shared_backend_signature(self): + text = _read(OPENCLAW_RUNNER) + self.assertIn('cursor_path=""', text) + self.assertIn('project_dir=""', text) + self.assertIn("cursor_path=cursor_path", text) + self.assertIn("project_dir=project_dir", text) + self.assertNotIn("**kwargs", text) + def test_all_skill_mds_mention_all_backends(self): for name, path in PLUGIN_SKILL_MDS.items(): text = _read(path) diff --git a/tests/test_react_agent_no_shell.py b/tests/test_react_agent_no_shell.py new file mode 100644 index 0000000..c7d1f2d --- /dev/null +++ b/tests/test_react_agent_no_shell.py @@ -0,0 +1,49 @@ +"""Tests for shell-injection hardening in the ReAct agent's bash tool. + +``_run_bash`` previously ran commands with ``shell=True``, allowing arbitrary +shell metacharacter injection. It now uses ``shlex.split`` + ``shell=False`` +and restricts the executable to a small allow-list (python/python3). These +tests assert both the allow-list gate and that shell metacharacters are no +longer interpreted. +""" +from __future__ import annotations + +from skillopt.envs.spreadsheetbench.react_agent import _run_bash + + +def test_disallowed_command_is_blocked(tmp_path) -> None: + out = _run_bash("curl http://example.com/evil", str(tmp_path)) + assert "blocked" in out.lower() + + +def test_allowed_python_runs_without_path_lookup(tmp_path, monkeypatch) -> None: + # Accepted aliases are mapped to the running interpreter, so an absent PATH + # must not make the benchmark depend on a system-level Python command. + monkeypatch.setenv("PATH", "") + out = _run_bash('python -c "print(42)"', str(tmp_path)) + assert "42" in out + + +def test_similarly_named_executable_is_blocked(tmp_path) -> None: + out = _run_bash('python.evil -c "print(42)"', str(tmp_path)) + assert "blocked" in out.lower() + + +def test_python_child_does_not_inherit_parent_secrets( + tmp_path, monkeypatch +) -> None: + monkeypatch.setenv("SUPER_SECRET_TOKEN", "must-not-leak") + out = _run_bash( + 'python -c "import os; print(os.environ.get(\'SUPER_SECRET_TOKEN\', \'ABSENT\'))"', + str(tmp_path), + ) + assert out == "ABSENT" + + +def test_shell_metacharacters_not_interpreted(tmp_path) -> None: + # With shell=False the ';' and following tokens become arguments to python, + # not a second shell command, so the marker file must NOT be created. + marker = tmp_path / "pwned.txt" + cmd = "python -c \"print(1)\" ; python -c \"open('pwned.txt','w')\"" + _run_bash(cmd, str(tmp_path)) + assert not marker.exists() diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index a057e22..70491aa 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -136,6 +136,355 @@ class TestHarvest(unittest.TestCase): self.assertNotIn("raw args should not copy", joined) self.assertNotIn("raw output should not copy", joined) + def test_digest_cursor_transcript_redacts_and_keeps_only_message_text_and_tool_names(self): + from skillopt_sleep.harvest_cursor import digest_cursor_transcript + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "cursor-session.jsonl") + self._write_jsonl(path, [ + { + "role": "user", + "message": { + "content": [{ + "type": "text", + "text": ( + "never-copy-attachment-metadata\n" + "\n" + "Deploy with sk-1234567890abcdef and token=local-secret\n" + "" + ), + }], + }, + }, + { + "role": "assistant", + "message": { + "content": [{ + "type": "tool_use", + "name": "shell.execute", + "input": {"token": "never-copy-tool-arguments"}, + }], + }, + }, + { + "role": "assistant", + "message": { + "content": [ + {"type": "text", "text": "Deployment finished."}, + { + "type": "tool_use", + "name": "read_file", + "input": {"path": "never-copy-tool-arguments"}, + }, + ], + }, + }, + {"type": "tool_result", "output": "never-copy-tool-output"}, + {"type": "turn_ended", "status": "error"}, + ]) + with open(path, "a", encoding="utf-8") as f: + f.write("null\n") + f.write("[]\n") + f.write('"non-object"\n') + f.write("{malformed jsonl record\\n") + + digest = digest_cursor_transcript(path, project="/repo/Cursor Project") + + self.assertIsNotNone(digest) + joined = "\n".join(digest.user_prompts + digest.assistant_finals) + self.assertEqual(digest.project, "/repo/Cursor Project") + self.assertEqual(len(digest.user_prompts), 1) + self.assertIn("[REDACTED_OPENAI_KEY]", joined) + self.assertIn("token=[REDACTED]", joined) + self.assertEqual(digest.tools_used, ["shell.execute", "read_file"]) + self.assertIn("neg:cursor_turn_error", digest.feedback_signals) + self.assertNotIn("never-copy-tool-arguments", joined) + self.assertNotIn("never-copy-tool-output", joined) + self.assertNotIn("never-copy-attachment-metadata", joined) + + def test_harvest_cursor_scopes_orders_filters_mtime_and_skips_replays(self): + from skillopt_sleep.__main__ import _cfg_from_args + from skillopt_sleep.harvest_cursor import ( + CURSOR_REPLAY_SENTINEL, + cursor_project_slug, + harvest_cursor, + ) + from skillopt_sleep.harvest_sources import harvest_for_config + + def write_cursor_session(cursor_home, project, session_id, prompt, mtime, extra_prompt=""): + project_dir = os.path.join( + cursor_home, + "projects", + cursor_project_slug(project), + ) + path = os.path.join( + project_dir, + "agent-transcripts", + session_id, + f"{session_id}.jsonl", + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f: + json.dump({"workspacePath": project}, f) + records = [ + { + "role": "user", + "message": { + "content": [{ + "type": "text", + "text": f"\n{prompt}\n", + }], + }, + }, + { + "role": "assistant", + "message": {"content": [{"type": "text", "text": "done"}]}, + }, + ] + if extra_prompt: + records.extend([ + { + "role": "user", + "message": { + "content": [{ + "type": "text", + "text": f"\n{extra_prompt}\n", + }], + }, + }, + { + "role": "assistant", + "message": {"content": [{"type": "text", "text": "done again"}]}, + }, + ]) + self._write_jsonl(path, records) + os.utime(path, (mtime, mtime)) + return path + + with tempfile.TemporaryDirectory() as tmp: + cursor_home = os.path.join(tmp, ".cursor") + project = os.path.join(tmp, "project with spaces") + other_project = os.path.join(tmp, "other project") + old_time = 1_700_000_000 + new_time = old_time + 3_600 + main_path = write_cursor_session( + cursor_home, + project, + "older", + "fix the first issue", + old_time, + ) + subagent_path = os.path.join(os.path.dirname(main_path), "subagents", "worker.jsonl") + os.makedirs(os.path.dirname(subagent_path), exist_ok=True) + self._write_jsonl(subagent_path, [ + {"role": "user", "message": {"content": "machine-generated subagent task"}}, + {"role": "assistant", "message": {"content": "subagent result"}}, + ]) + write_cursor_session(cursor_home, other_project, "newer", "fix the second issue", new_time) + write_cursor_session( + cursor_home, + other_project, + "generated-replay", + CURSOR_REPLAY_SENTINEL + "\n## CURRENT SKILL", + new_time + 1, + extra_prompt="continue the internal replay", + ) + + invoked = harvest_cursor( + os.path.join(cursor_home, "projects"), + scope="invoked", + invoked_project=os.path.join(project, "src", "package"), + ) + all_digests = harvest_cursor( + os.path.join(cursor_home, "projects"), + scope="all", + since_iso="2023-11-14T23:00:00Z", + limit=1, + ) + + Args = type("Args", (), { + "project": project, + "scope": "", + "backend": "cursor", + "model": "", + "codex_path": "", + "cursor_path": "", + "claude_home": "", + "codex_home": "", + "cursor_home": cursor_home, + "source": "cursor", + "lookback_hours": 0, + "edit_budget": 0, + "max_sessions": 0, + "max_tasks": 0, + "target_skill_path": "", + "preferences": "", + "progress": False, + "auto_adopt": False, + }) + cfg = _cfg_from_args(Args()) + configured = harvest_for_config(cfg) + + self.assertEqual([d.session_id for d in invoked], ["older"]) + self.assertEqual([d.session_id for d in all_digests], ["newer"]) + self.assertEqual(invoked[0].project, project) + self.assertEqual(all_digests[0].project, other_project) + self.assertEqual([d.session_id for d in configured], ["older"]) + self.assertEqual(cfg.get("transcript_source"), "cursor") + self.assertEqual(cfg.get("backend"), "cursor") + + def test_harvest_cursor_prefers_longest_workspace_and_falls_back_to_slug(self): + from skillopt_sleep.harvest_cursor import cursor_project_slug, harvest_cursor + + def write_session(projects_dir, storage_name, workspace, session_id): + project_dir = os.path.join(projects_dir, storage_name) + path = os.path.join(project_dir, "agent-transcripts", session_id, f"{session_id}.jsonl") + os.makedirs(os.path.dirname(path), exist_ok=True) + self._write_jsonl(path, [ + {"role": "user", "message": {"content": "please fix this project"}}, + {"role": "assistant", "message": {"content": "fixed"}}, + ]) + if workspace is not None: + with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f: + json.dump(workspace, f) + return path + + with tempfile.TemporaryDirectory() as tmp: + projects_dir = os.path.join(tmp, ".cursor", "projects") + parent = os.path.join(tmp, "repo") + nested = os.path.join(parent, "packages", "app") + write_session(projects_dir, "parent-store", {"workspacePath": parent}, "parent") + write_session(projects_dir, "nested-store", {"workspacePath": nested}, "nested") + fallback = os.path.join(tmp, "fallback") + write_session( + projects_dir, + cursor_project_slug(fallback), + ["invalid metadata shape"], + "fallback", + ) + metadata_free = os.path.join(tmp, "metadata-free") + write_session( + projects_dir, + cursor_project_slug(metadata_free), + None, + "metadata-free", + ) + mixed_parent = os.path.join(tmp, "mixed-parent") + mixed_nested = os.path.join(mixed_parent, "nested") + write_session(projects_dir, "mixed-parent-store", {"workspacePath": mixed_parent}, "mixed-parent") + write_session( + projects_dir, + cursor_project_slug(mixed_nested), + None, + "mixed-nested", + ) + + nested_digests = harvest_cursor( + projects_dir, + scope="invoked", + invoked_project=os.path.join(nested, "src"), + ) + fallback_digests = harvest_cursor( + projects_dir, + scope="invoked", + invoked_project=fallback, + ) + metadata_free_digests = harvest_cursor( + projects_dir, + scope="invoked", + invoked_project=os.path.join(metadata_free, "packages", "app"), + ) + mixed_digests = harvest_cursor( + projects_dir, + scope="invoked", + invoked_project=mixed_nested, + ) + + self.assertEqual([digest.session_id for digest in nested_digests], ["nested"]) + self.assertEqual(nested_digests[0].project, nested) + self.assertEqual([digest.session_id for digest in fallback_digests], ["fallback"]) + self.assertEqual(fallback_digests[0].project, fallback) + self.assertEqual( + [digest.session_id for digest in metadata_free_digests], + ["metadata-free"], + ) + self.assertEqual(metadata_free_digests[0].project, metadata_free) + self.assertEqual([digest.session_id for digest in mixed_digests], ["mixed-nested"]) + self.assertEqual(mixed_digests[0].project, mixed_nested) + + def test_harvest_cursor_uses_numeric_mtime_for_aware_and_local_cutoffs(self): + from datetime import datetime, timedelta, timezone + + from skillopt_sleep.harvest_cursor import cursor_project_slug, harvest_cursor + + with tempfile.TemporaryDirectory() as tmp: + project = os.path.join(tmp, "project") + project_dir = os.path.join(tmp, ".cursor", "projects", cursor_project_slug(project)) + os.makedirs(project_dir) + with open(os.path.join(project_dir, ".workspace-trusted"), "w", encoding="utf-8") as f: + json.dump({"workspacePath": project}, f) + + cutoff = 1_700_000_000 + for session_id, modified in (("before", cutoff - 1), ("equal", cutoff), ("after", cutoff + 1)): + path = os.path.join( + project_dir, + "agent-transcripts", + session_id, + f"{session_id}.jsonl", + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + self._write_jsonl(path, [ + {"role": "user", "message": {"content": f"task {session_id}"}}, + {"role": "assistant", "message": {"content": "done"}}, + ]) + os.utime(path, (modified, modified)) + + aware = datetime.fromtimestamp(cutoff, timezone(timedelta(hours=5))).isoformat() + local = datetime.fromtimestamp(cutoff).replace(microsecond=0).isoformat() + aware_result = harvest_cursor(projects_dir=os.path.dirname(project_dir), since_iso=aware) + local_result = harvest_cursor(projects_dir=os.path.dirname(project_dir), since_iso=local) + + self.assertEqual([digest.session_id for digest in aware_result], ["after"]) + self.assertEqual([digest.session_id for digest in local_result], ["after"]) + + def test_harvest_cursor_filters_only_exact_internal_replay_sentinel(self): + from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL, cursor_project_slug, harvest_cursor + + with tempfile.TemporaryDirectory() as tmp: + project = os.path.join(tmp, "project") + project_dir = os.path.join(tmp, ".cursor", "projects", cursor_project_slug(project)) + for session_id, prompt in ( + ("internal", CURSOR_REPLAY_SENTINEL + "\nrun replay"), + ("real", f"Please explain what {CURSOR_REPLAY_SENTINEL} means"), + ("grader", "You are a strict grader helping me review this response"), + ("skill", "Please explain the ## CURRENT SKILL section"), + ): + path = os.path.join(project_dir, "agent-transcripts", session_id, f"{session_id}.jsonl") + os.makedirs(os.path.dirname(path), exist_ok=True) + self._write_jsonl(path, [ + {"role": "user", "message": {"content": prompt}}, + {"role": "assistant", "message": {"content": "answer"}}, + ]) + + digests = harvest_cursor(os.path.join(tmp, ".cursor", "projects"), scope="all") + + self.assertEqual( + sorted(digest.session_id for digest in digests), + ["grader", "real", "skill"], + ) + + def test_auto_source_keeps_existing_codex_then_claude_precedence(self): + from skillopt_sleep.harvest_sources import harvest_for_config + + cfg = load_config(transcript_source="auto", invoked_project="/repo/project") + expected = [SessionDigest(session_id="claude-session", project="/repo/project")] + with mock.patch("skillopt_sleep.harvest_sources.harvest_codex", return_value=[]), \ + mock.patch("skillopt_sleep.harvest_sources.harvest", return_value=expected), \ + mock.patch("skillopt_sleep.harvest_sources.harvest_cursor") as cursor_harvest: + self.assertEqual(harvest_for_config(cfg), expected) + + cursor_harvest.assert_not_called() + def test_harvest_codex_filters_project_and_cli_source(self): from skillopt_sleep.__main__ import _cfg_from_args from skillopt_sleep.harvest_sources import harvest_for_config @@ -478,6 +827,18 @@ Resolve local Git conflicts. "resolve a local Git conflict", }) + def test_cursor_miner_failure_is_not_swallowed(self): + from skillopt_sleep.backend import CursorBackendError + + def failed_miner(_digests): + raise CursorBackendError("Cursor Agent authentication failed") + + with self.assertRaises(CursorBackendError): + mine( + [self._digest(["configure an MCP server"], ["neg:failed"])], + llm_miner=failed_miner, + ) + class TestConsolidateGate(unittest.TestCase): def test_accepts_helpful_rejects_harmful(self): @@ -1113,6 +1474,412 @@ class TestCopilotBackend(unittest.TestCase): shutil.rmtree(stub_dir, ignore_errors=True) +class TestCursorBackend(unittest.TestCase): + """Pure-logic tests for CursorCliBackend without a Cursor login.""" + + def test_alias_and_environment_resolution(self): + from skillopt_sleep.backend import CursorCliBackend, get_backend, resolve_cursor_path + + for name in ("cursor", "cursor_agent", "cursor_cli"): + self.assertIsInstance(get_backend(name), CursorCliBackend, name) + with mock.patch.dict(os.environ, { + "SKILLOPT_SLEEP_CURSOR_PATH": "/tmp/cursor-agent", + "SKILLOPT_SLEEP_CURSOR_MODEL": "cursor-small", + }, clear=False): + self.assertEqual(resolve_cursor_path(), "/tmp/cursor-agent") + self.assertEqual(CursorCliBackend().model, "cursor-small") + + def test_cursor_path_overrides_expand_user_home(self): + from skillopt_sleep.__main__ import _cfg_from_args + from skillopt_sleep.backend import resolve_cursor_path + + Args = type("Args", (), { + "project": "", + "scope": "", + "backend": "", + "model": "", + "codex_path": "", + "cursor_path": "~/.local/bin/cursor-agent", + "claude_home": "", + "codex_home": "", + "cursor_home": "~/.cursor-custom", + "source": "", + "lookback_hours": None, + "edit_budget": 0, + "max_sessions": 0, + "max_tasks": 0, + "target_skill_path": "", + "preferences": "", + "progress": False, + "auto_adopt": False, + }) + + cfg = _cfg_from_args(Args()) + self.assertEqual( + cfg.get("cursor_path"), + os.path.abspath(os.path.expanduser("~/.local/bin/cursor-agent")), + ) + self.assertEqual( + cfg.cursor_projects_dir, + os.path.join(os.path.expanduser("~/.cursor-custom"), "projects"), + ) + + direct_cfg = load_config( + cursor_home="~/.cursor-config", + cursor_path="~/.cursor-config/bin/cursor-agent", + ) + self.assertEqual( + direct_cfg.cursor_projects_dir, + os.path.join(os.path.expanduser("~/.cursor-config"), "projects"), + ) + self.assertEqual( + resolve_cursor_path(direct_cfg.get("cursor_path")), + os.path.expanduser("~/.cursor-config/bin/cursor-agent"), + ) + with mock.patch.dict( + os.environ, + {"SKILLOPT_SLEEP_CURSOR_PATH": "~/.cursor-env/bin/cursor-agent"}, + clear=False, + ): + self.assertEqual( + resolve_cursor_path(), + os.path.expanduser("~/.cursor-env/bin/cursor-agent"), + ) + + def test_read_only_call_uses_stdin_ask_mode_and_terminal_result(self): + from skillopt_sleep.backend import CursorCliBackend + from skillopt_sleep.harvest_cursor import CURSOR_REPLAY_SENTINEL + + calls = [] + runtime_configs = [] + + def fake_run(cmd, **kwargs): + calls.append((cmd, kwargs)) + config_dir = kwargs["env"]["CURSOR_CONFIG_DIR"] + data_dir = kwargs["env"]["CURSOR_DATA_DIR"] + with open(os.path.join(config_dir, "cli-config.json"), encoding="utf-8") as f: + runtime_configs.append(json.load(f)) + self.assertTrue(os.path.isdir(data_dir)) + + class Proc: + returncode = 0 + stdout = ( + '{"type":"message","result":"intermediate"}\n' + '{"type":"result","subtype":"success","is_error":false,' + '"result":"final answer"}\n' + ) + stderr = "" + + return Proc() + + backend = CursorCliBackend(cursor_path="cursor-agent-test", model="cursor-model") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): + self.assertEqual(backend._call("solve this"), "final answer") + + cmd, kwargs = calls[0] + self.assertEqual(cmd[0], "cursor-agent-test") + self.assertIn("-p", cmd) + self.assertEqual(cmd[cmd.index("--output-format") + 1], "json") + self.assertEqual(cmd[cmd.index("--mode") + 1], "ask") + self.assertIn("--trust", cmd) + self.assertEqual(cmd[cmd.index("--workspace") + 1], kwargs["cwd"]) + self.assertTrue(os.path.basename(kwargs["cwd"]).startswith("skillopt_sleep_cursor_")) + self.assertNotIn("--force", cmd) + self.assertNotIn("--sandbox", cmd) + self.assertEqual(cmd[cmd.index("--model") + 1], "cursor-model") + self.assertTrue(kwargs["input"].startswith(CURSOR_REPLAY_SENTINEL + "\n\n")) + self.assertTrue(kwargs["input"].endswith("solve this")) + self.assertNotEqual(kwargs["env"]["CURSOR_CONFIG_DIR"], os.path.expanduser("~/.cursor")) + self.assertEqual(runtime_configs[0]["approvalMode"], "allowlist") + self.assertEqual(runtime_configs[0]["permissions"]["allow"], []) + self.assertEqual( + runtime_configs[0]["permissions"]["deny"], + ["Read(**)", "Write(**)", "Mcp(*:*)"], + ) + self.assertEqual(runtime_configs[0]["sandbox"]["mode"], "disabled") + self.assertFalse(os.path.exists(os.path.dirname(kwargs["env"]["CURSOR_CONFIG_DIR"]))) + self.assertEqual(backend.last_call_error, "") + self.assertEqual( + CursorCliBackend._parse_json_response('{"type":"message","result":"not terminal"}'), + "", + ) + + def test_cursor_environment_keeps_runtime_auth_and_drops_unrelated_secrets(self): + from skillopt_sleep.backend import CursorCliBackend + + host_env = { + "PATH": "/usr/bin", + "LANG": "en_US.UTF-8", + "CURSOR_API_KEY": "cursor-auth", + "HTTPS_PROXY": "https://proxy.example", + "AWS_SECRET_ACCESS_KEY": "aws-secret", + "OPENAI_API_KEY": "openai-secret", + "ANTHROPIC_API_KEY": "anthropic-secret", + "GITHUB_TOKEN": "github-secret", + } + with tempfile.TemporaryDirectory() as runtime_dir: + with mock.patch.dict(os.environ, host_env, clear=True): + env = CursorCliBackend._isolated_environment(runtime_dir) + + self.assertEqual(env["PATH"], "/usr/bin") + self.assertEqual(env["LANG"], "en_US.UTF-8") + self.assertEqual(env["CURSOR_API_KEY"], "cursor-auth") + self.assertEqual(env["HTTPS_PROXY"], "https://proxy.example") + self.assertNotIn("AWS_SECRET_ACCESS_KEY", env) + self.assertNotIn("OPENAI_API_KEY", env) + self.assertNotIn("ANTHROPIC_API_KEY", env) + self.assertNotIn("GITHUB_TOKEN", env) + self.assertTrue(env["CURSOR_CONFIG_DIR"].startswith(runtime_dir)) + self.assertTrue(env["CURSOR_DATA_DIR"].startswith(runtime_dir)) + + def test_nonzero_and_error_results_fail_once_with_redacted_diagnostics(self): + from skillopt_sleep.backend import CursorBackendError, CursorCliBackend + + backend = CursorCliBackend(cursor_path="cursor-agent-test", timeout=7) + + class BadProc: + returncode = 9 + stdout = "not-json" + stderr = "Authorization: Bearer cursor-secret-value" + + with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=BadProc()) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 1) + self.assertIn("exited 9", backend.last_call_error) + self.assertIn("[REDACTED]", backend.last_call_error) + self.assertNotIn("cursor-secret-value", backend.last_call_error) + + class ErrorProc: + returncode = 0 + stdout = '{"type":"result","is_error":true,"result":"api_key=cursor-secret"}' + stderr = "" + + with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=ErrorProc()) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 1) + self.assertIn("error result", backend.last_call_error) + self.assertIn("[REDACTED]", backend.last_call_error) + self.assertNotIn("cursor-secret", backend.last_call_error) + + with mock.patch( + "skillopt_sleep.backend.subprocess.run", + side_effect=OSError("missing cursor-agent"), + ) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 1) + self.assertIn("spawn failed", backend.last_call_error) + + def test_read_only_timeout_and_malformed_output_retry_once(self): + import subprocess + + from skillopt_sleep.backend import CursorBackendError, CursorCliBackend + + backend = CursorCliBackend(cursor_path="cursor-agent-test", timeout=7) + + class GoodProc: + returncode = 0 + stdout = '{"type":"result","is_error":false,"result":"recovered"}' + stderr = "" + + with mock.patch( + "skillopt_sleep.backend.subprocess.run", + side_effect=[subprocess.TimeoutExpired(["cursor-agent-test"], 7), GoodProc()], + ) as run: + self.assertEqual(backend._call("solve this"), "recovered") + self.assertEqual(run.call_count, 2) + self.assertEqual(backend.last_call_error, "") + + class MalformedProc: + returncode = 0 + stdout = "still not json" + stderr = "" + + with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=MalformedProc()) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 2) + self.assertIn("no usable JSON response", backend.last_call_error) + + class AuthProc: + returncode = 0 + stdout = "" + stderr = "Not authenticated. Please log in with token=cursor-secret" + + with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=AuthProc()) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 1) + self.assertIn("authentication failed", backend.last_call_error) + self.assertNotIn("cursor-secret", backend.last_call_error) + + class ConfigProc: + returncode = 0 + stdout = "" + stderr = "Unsupported model: cursor-unknown" + + with mock.patch("skillopt_sleep.backend.subprocess.run", return_value=ConfigProc()) as run: + with self.assertRaises(CursorBackendError): + backend._call("solve this") + self.assertEqual(run.call_count, 1) + self.assertIn("configuration failed", backend.last_call_error) + + def test_failed_cursor_call_is_not_cached(self): + from skillopt_sleep.backend import CursorBackendError, CursorCliBackend + + class BadProc: + returncode = 1 + stdout = "" + stderr = "not authenticated" + + class GoodProc: + returncode = 0 + stdout = '{"type":"result","is_error":false,"result":"answer"}' + stderr = "" + + backend = CursorCliBackend(cursor_path="cursor-agent-test") + task = TaskRecord(id="cache", project="/p", intent="answer this") + with mock.patch( + "skillopt_sleep.backend.subprocess.run", + side_effect=[BadProc(), GoodProc()], + ) as run: + with self.assertRaises(CursorBackendError): + backend.attempt(task, skill="", memory="") + self.assertEqual(backend.attempt(task, skill="", memory=""), "answer") + self.assertEqual(backend.attempt(task, skill="", memory=""), "answer") + + self.assertEqual(run.call_count, 2) + + def test_tool_aware_replay_fails_before_cursor_subprocess(self): + from skillopt_sleep.backend import CursorBackendError, CursorCliBackend + + backend = CursorCliBackend(cursor_path="cursor-agent-test") + task = TaskRecord(id="cursor-tools", project="/p", intent="search") + with mock.patch("skillopt_sleep.backend.subprocess.run") as run: + with self.assertRaisesRegex( + CursorBackendError, + "Cursor tool-aware replay is temporarily disabled", + ): + backend.attempt_with_tools(task, skill="", memory="", tools=["search"]) + run.assert_not_called() + self.assertIn("temporarily disabled", backend.last_call_error) + self.assertEqual(backend._cache, {}) + + def test_tool_aware_cli_run_fails_without_writes_or_checkpoint(self): + import contextlib + import io + + from skillopt_sleep.__main__ import main + from skillopt_sleep.backend import CursorCliBackend + from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as tmp: + project = os.path.join(tmp, "project") + claude_home = os.path.join(tmp, ".claude") + target = os.path.join( + project, + ".cursor", + "skills", + "skillopt-sleep-learned", + "SKILL.md", + ) + os.makedirs(project) + task = TaskRecord( + id="cursor-tool-task", + project=project, + intent="Search before answering", + reference_kind="rule", + judge={"checks": [{"op": "tool_called", "arg": "search"}]}, + split="val", + ) + payload = make_tasks_payload( + [task], + project=project, + transcript_source="cursor", + target_skill_path=target, + ) + payload["reviewed"] = True + tasks_path = write_tasks_file(os.path.join(tmp, "tasks.json"), payload) + backend = CursorCliBackend(cursor_path="cursor-agent-test") + stderr = io.StringIO() + + with mock.patch( + "skillopt_sleep.cycle.build_backend", + return_value=backend, + ): + with mock.patch("skillopt_sleep.backend.subprocess.run") as run: + with contextlib.redirect_stderr(stderr): + rc = main([ + "run", + "--project", project, + "--claude-home", claude_home, + "--backend", "cursor", + "--tasks-file", tasks_path, + "--target-skill-path", target, + "--auto-adopt", + ]) + + self.assertEqual(rc, 1) + self.assertIn("Cursor tool-aware replay is temporarily disabled", stderr.getvalue()) + run.assert_not_called() + self.assertEqual(backend._cache, {}) + self.assertFalse(os.path.exists(os.path.join(tmp, ".skillopt-sleep"))) + self.assertFalse(os.path.exists(os.path.join(project, ".skillopt-sleep"))) + self.assertFalse(os.path.exists(target)) + + def test_cursor_failure_aborts_without_state_or_staging_and_cli_returns_nonzero(self): + import contextlib + import io + + from skillopt_sleep.__main__ import main + from skillopt_sleep.backend import CursorBackendError, CursorCliBackend + + with tempfile.TemporaryDirectory() as tmp: + project = os.path.join(tmp, "project") + os.makedirs(project) + cfg = load_config( + backend="cursor", + invoked_project=project, + projects="invoked", + claude_home=os.path.join(tmp, ".claude"), + target_skill_path=".cursor/skills/skillopt-sleep-learned/SKILL.md", + ) + backend = CursorCliBackend(cursor_path="cursor-agent-test") + task = TaskRecord( + id="failure", + project=project, + intent="answer this", + reference_kind="exact", + reference="answer", + split="val", + ) + with mock.patch.object( + backend, + "_call", + side_effect=CursorBackendError("Cursor Agent exited 1: token [REDACTED]"), + ): + with self.assertRaises(CursorBackendError): + run_sleep_cycle(cfg, seed_tasks=[task], backend=backend) + + self.assertFalse(os.path.exists(cfg.state_path)) + self.assertFalse(os.path.exists(os.path.join(project, ".skillopt-sleep"))) + self.assertFalse(os.path.exists(cfg.managed_skill_path())) + + stderr = io.StringIO() + with mock.patch( + "skillopt_sleep.__main__.run_sleep_cycle", + side_effect=CursorBackendError("Cursor Agent exited 1: token=cursor-secret"), + ), contextlib.redirect_stderr(stderr): + rc = main(["dry-run", "--project", project, "--backend", "cursor"]) + + self.assertEqual(rc, 1) + self.assertIn("Cursor backend failed", stderr.getvalue()) + self.assertIn("[REDACTED]", stderr.getvalue()) + self.assertNotIn("cursor-secret", stderr.getvalue()) + + class TestClaudeCliBackendBare(unittest.TestCase): """Issue #68: --bare must be conditional on ANTHROPIC_API_KEY.""" @@ -1259,6 +2026,45 @@ class MockBeneficialBackend(MockBackend): break return edits + +class MockFractionalDegradationBackend(MockBackend): + """Backend where an edit improves train but partially degrades held-out. + + Train improves 0.0 → 1.0 (looks good!) but held-out degrades from + 1.0 → 0.5 across two val tasks (one still passes, one breaks). + A naive optimizer might see the train improvement and miss the + partial regression — the gate must catch this subtle case. + """ + + HACK_TEXT = "Always respond with a definitive answer" + + def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + if self.HACK_TEXT in ctx: + if "rule:__reward_hacking__" in task.tags: + return str(task.reference) # train improves + if "rule:real" in task.tags: + if task.id == "v1": + return str(task.reference) # val task 1 still passes + if task.id == "v2": + return "misleading answer" # val task 2 breaks + # Baseline (without edit) + if "rule:__reward_hacking__" in task.tags: + return "I am missing input" # train fails + if "rule:real" in task.tags: + return str(task.reference) # held-out passes + return super().attempt(task, skill, memory, sample_id) + + def reflect(self, failures, successes, skill, memory, **kwargs): + edits = [] + for task, _ in failures: + if "rule:__reward_hacking__" in task.tags: + edits.append(EditRecord("skill", "add", self.HACK_TEXT, + rationale="improve response confidence")) + break + return edits + + class TestVerifierDiscipline(unittest.TestCase): def test_gate_rejects_reward_hacking_edit(self): be = MockRewardHackingBackend() @@ -1290,6 +2096,40 @@ class TestVerifierDiscipline(unittest.TestCase): self.assertGreater(len(res.applied_edits), 0) self.assertIn("step-by-step", res.applied_edits[0].content) + def test_gate_rejects_fractional_degradation(self): + """Gate must reject an edit that partially degrades held-out (1.0→0.5), + not just all-or-nothing collapses. Train improves (0.0→1.0) which makes + the regression easy to miss — the gate catches it anyway.""" + from skillopt_sleep.replay import aggregate_scores, replay_batch + + be = MockFractionalDegradationBackend() + train = TaskRecord(id="t3", project="/p", intent="train", reference="ABC", + reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") + val1 = TaskRecord(id="v1", project="/p", intent="val", reference="DEF", + reference_kind="exact", tags=["rule:real"], split="val") + val2 = TaskRecord(id="v2", project="/p", intent="val", reference="GHI", + reference_kind="exact", tags=["rule:real"], split="val") + tasks = [train, val1, val2] + + candidate_pairs = replay_batch(be, [val1, val2], be.HACK_TEXT, "") + candidate_hard, _candidate_soft = aggregate_scores(candidate_pairs) + self.assertEqual([result.hard for _task, result in candidate_pairs], [1.0, 0.0]) + self.assertEqual(candidate_hard, 0.5) + + res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) + + self.assertFalse(res.accepted) + self.assertEqual(res.gate_action, "reject") + # Baseline: both val tasks pass → 1.0 + self.assertEqual(res.holdout_baseline, 1.0) + # After rejection the skill reverts; final replay also passes both + self.assertEqual(res.holdout_candidate, 1.0) + # Confirm we had two val tasks in the baseline + self.assertEqual(len(res.holdout_detail), 2) + self.assertGreater(len(res.rejected_edits), 0) + self.assertIn("definitive answer", res.rejected_edits[0].content) + + class TestDiagnosticsRedaction(unittest.TestCase): """diagnostics.json surfaces backend stderr / optimizer replies / task responses for debugging — but those can carry credentials (e.g. a codex 401 diff --git a/tests/test_sleep_evidence.py b/tests/test_sleep_evidence.py new file mode 100644 index 0000000..6c8253f --- /dev/null +++ b/tests/test_sleep_evidence.py @@ -0,0 +1,208 @@ +"""Tests for the evidence log and the prompt registry. + +Pure-stdlib (unittest), deterministic, no API key, no network, +no third-party deps. + +Run: python -m unittest tests.test_sleep_evidence +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from unittest import mock + +from skillopt_sleep import prompts as prompt_registry +from skillopt_sleep.backend import MockBackend +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.evidence import EvidenceLog, read_events +from skillopt_sleep.experiments.personas import researcher_persona +from skillopt_sleep.mine import assign_splits + + +def _events_by(events, stage=None, event=None): + out = events + if stage is not None: + out = [e for e in out if e.get("stage") == stage] + if event is not None: + out = [e for e in out if e.get("event") == event] + return out + + +class TestEvidenceLog(unittest.TestCase): + def test_append_redact_truncate_and_order(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "e.jsonl") + ev = EvidenceLog(path, max_chars=200) + ev.log("replay", "model_call", prompt="x" * 500, + secret="api_key=sk-abcdefghijklmnop") + ev.log("gate", "decision", action="accept") + events = read_events(path) + self.assertEqual([e["seq"] for e in events], [1, 2]) + self.assertIn("truncated", events[0]["prompt"]) + self.assertLessEqual(len(events[0]["prompt"]), 260) + self.assertNotIn("sk-abcdefghijklmnop", json.dumps(events)) + self.assertIn("REDACTED", events[0]["secret"]) + + def test_structured_secret_is_redacted_before_truncation(self): + private_key = ( + "-----BEGIN PRIVATE KEY-----\n" + + "A" * 500 + + "\n-----END PRIVATE KEY-----" + ) + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "e.jsonl") + EvidenceLog(path, max_chars=200).log( + "replay", "model_call", response=private_key + ) + persisted = json.dumps(read_events(path)) + + self.assertIn("REDACTED_PRIVATE_KEY", persisted) + self.assertNotIn("BEGIN PRIVATE KEY", persisted) + self.assertNotIn("A" * 100, persisted) + + def test_reader_skips_corrupt_lines(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "e.jsonl") + ev = EvidenceLog(path) + ev.log("cycle", "start") + with open(path, "a", encoding="utf-8") as f: + f.write("{not json\n") + ev.log("cycle", "end") + self.assertEqual(len(read_events(path)), 2) + + +class TestPromptRegistry(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.patch = mock.patch.dict(os.environ, { + "SKILLOPT_SLEEP_PROMPTS_PATH": os.path.join(self.tmp.name, "prompts.json")}) + self.patch.start() + + def tearDown(self): + self.patch.stop() + self.tmp.cleanup() + + def test_defaults_match_legacy_prompts(self): + # The registry must reproduce the exact legacy wording by default. + self.assertIn("You are SkillOpt's optimizer", prompt_registry.get_prompt("reflect")) + self.assertIn("RECURRING tasks", prompt_registry.get_prompt("miner")) + self.assertIn("Return ONLY the final answer text", prompt_registry.get_prompt("attempt")) + rendered = prompt_registry.render("attempt", { + "__SKILL__": "S", "__MEMORY__": "M", "__INTENT__": "I", "__CONTEXT__": "C"}) + self.assertIn("# Skill\nS", rendered) + self.assertNotIn("__SKILL__", rendered) + + def test_override_takes_effect_without_restart(self): + self.assertFalse(prompt_registry.is_overridden("judge")) + prompt_registry.save_overrides({"judge": "CUSTOM __RUBRIC__ / __RESPONSE__"}) + self.assertTrue(prompt_registry.is_overridden("judge")) + self.assertEqual( + prompt_registry.render("judge", {"__RUBRIC__": "r", "__RESPONSE__": "x"}), + "CUSTOM r / x") + # empty value reverts to default + prompt_registry.save_overrides({"judge": None}) + self.assertFalse(prompt_registry.is_overridden("judge")) + self.assertIn("Score how well", prompt_registry.get_prompt("judge")) + + def test_unknown_names_are_ignored(self): + out = prompt_registry.save_overrides({"nope": "x", "miner": "M __PROMPTS__"}) + self.assertEqual(set(out), {"miner"}) + prompt_registry.save_overrides({"miner": ""}) + + +class TestCycleEvidence(unittest.TestCase): + def _run(self, **cfg_extra): + proj = tempfile.mkdtemp() + home = tempfile.mkdtemp() + cfg = load_config( + invoked_project=proj, projects="invoked", backend="mock", + claude_home=os.path.join(home, ".claude"), auto_adopt=False, + **cfg_extra) + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + outcome = run_sleep_cycle(cfg, seed_tasks=tasks) + return outcome + + def test_evidence_written_with_full_chain(self): + outcome = self._run() + path = os.path.join(outcome.staging_dir, "evidence.jsonl") + self.assertTrue(os.path.exists(path), "evidence.jsonl missing from staging dir") + events = read_events(path) + # chain: cycle start .. task_ready .. replay results (phased) .. + # reflect edits .. gate baseline/trial/decision .. staged .. cycle end + self.assertTrue(_events_by(events, "cycle", "start")) + self.assertTrue(_events_by(events, "mine", "task_ready")) + splits = {e["split"] for e in _events_by(events, "mine", "task_ready")} + self.assertIn("train", splits) + results = _events_by(events, "replay", "result") + self.assertTrue(results) + phases = {e["phase"] for e in results} + self.assertIn("baseline_val", phases) + self.assertIn("final_val", phases) + self.assertTrue(_events_by(events, "reflect", "edits_returned")) + self.assertTrue(_events_by(events, "gate", "baseline")) + decision = _events_by(events, "gate", "decision") + self.assertEqual(len(decision), 1) + self.assertIn("formula", decision[0]) + self.assertTrue(_events_by(events, "stage", "staged")) + end = _events_by(events, "cycle", "end") + self.assertEqual(len(end), 1) + self.assertEqual(end[0]["outcome"], "completed") + # the report landed in the SAME pre-created folder as the evidence + self.assertTrue(os.path.exists(os.path.join(outcome.staging_dir, "report.md"))) + + def test_evidence_can_be_disabled(self): + outcome = self._run(evidence_log=False) + self.assertFalse( + os.path.exists(os.path.join(outcome.staging_dir, "evidence.jsonl"))) + + def test_disabling_evidence_detaches_reused_backend_logger(self): + backend = MockBackend() + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + with tempfile.TemporaryDirectory() as d: + first_project = os.path.join(d, "first") + second_project = os.path.join(d, "second") + os.makedirs(first_project) + os.makedirs(second_project) + first_cfg = load_config( + invoked_project=first_project, + projects="invoked", + backend="mock", + state_dir=os.path.join(d, "first-state"), + ) + first = run_sleep_cycle(first_cfg, seed_tasks=tasks, backend=backend) + first_log = os.path.join(first.staging_dir, "evidence.jsonl") + first_size = os.path.getsize(first_log) + + second_cfg = load_config( + invoked_project=second_project, + projects="invoked", + backend="mock", + state_dir=os.path.join(d, "second-state"), + evidence_log=False, + ) + second = run_sleep_cycle(second_cfg, seed_tasks=tasks, backend=backend) + + self.assertEqual(os.path.getsize(first_log), first_size) + self.assertIsNone(backend.evidence) + self.assertFalse( + os.path.exists(os.path.join(second.staging_dir, "evidence.jsonl")) + ) + + def test_no_tasks_night_is_not_adoptable_but_keeps_evidence(self): + from skillopt_sleep.staging import latest_staging + proj = tempfile.mkdtemp() + home = tempfile.mkdtemp() + cfg = load_config( + invoked_project=proj, projects="invoked", backend="mock", + claude_home=os.path.join(home, ".claude")) + outcome = run_sleep_cycle(cfg, seed_tasks=[]) + self.assertEqual(outcome.staging_dir, "") + # an evidence-only folder exists but latest_staging must skip it + self.assertIsNone(latest_staging(proj)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sleep_skill_harvest.py b/tests/test_sleep_skill_harvest.py new file mode 100644 index 0000000..e1d5757 --- /dev/null +++ b/tests/test_sleep_skill_harvest.py @@ -0,0 +1,162 @@ +"""Tests for Claude ``Skill`` tool-use harvesting (issue #120). + +Pure-stdlib (unittest), deterministic, no API key, no third-party deps. +Run: python -m pytest tests/test_sleep_skill_harvest.py +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest + +from skillopt_sleep.harvest import digest_transcript +from skillopt_sleep.types import SessionDigest + + +def _skill_block(skill, name="Skill", block_type="tool_use", extra=None): + args = {"skill": skill} if skill is not None else {} + if extra: + args.update(extra) + return {"type": block_type, "name": name, "input": args} + + +def _assistant(content): + return { + "type": "assistant", + "timestamp": "2026-07-28T10:00:00Z", + "cwd": "/repo/example", + "gitBranch": "main", + "message": {"role": "assistant", "content": content}, + } + + +def _user(text): + return { + "type": "user", + "timestamp": "2026-07-28T09:59:00Z", + "cwd": "/repo/example", + "message": {"role": "user", "content": text}, + } + + +class TestSessionDigestSkillsField(unittest.TestCase): + def test_defaults_to_empty_list(self): + digest = SessionDigest(session_id="s1", project="/repo/example") + self.assertEqual(digest.skills_used, []) + + def test_to_dict_includes_empty_skills_used(self): + digest = SessionDigest(session_id="s1", project="/repo/example") + self.assertIn("skills_used", digest.to_dict()) + self.assertEqual(digest.to_dict()["skills_used"], []) + + def test_legacy_payload_without_skills_used_still_loads(self): + legacy = {"session_id": "s1", "project": "/repo/example", "tools_used": ["Bash"]} + known = set(SessionDigest.__dataclass_fields__) + digest = SessionDigest(**{k: v for k, v in legacy.items() if k in known}) + self.assertEqual(digest.skills_used, []) + self.assertEqual(digest.tools_used, ["Bash"]) + + def test_unknown_key_filter_can_ignore_new_field(self): + payload = SessionDigest( + session_id="s1", project="/repo/example", skills_used=["example-skill"] + ).to_dict() + older_known = { + f for f in SessionDigest.__dataclass_fields__ if f != "skills_used" + } + digest = SessionDigest(**{k: v for k, v in payload.items() if k in older_known}) + self.assertEqual(digest.skills_used, []) + + +class TestSkillHarvest(unittest.TestCase): + def _digest(self, records): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "session-example.jsonl") + with open(path, "w", encoding="utf-8") as f: + for record in records: + f.write(json.dumps(record) + "\n") + return digest_transcript(path) + + def test_harvests_skill_invocation(self): + digest = self._digest([ + _user("run the example skill"), + _assistant([_skill_block("example-skill")]), + ]) + self.assertIsNotNone(digest) + self.assertEqual(digest.skills_used, ["example-skill"]) + # tools_used keeps its existing meaning, including the Skill tool itself + self.assertEqual(digest.tools_used, ["Skill"]) + + def test_duplicates_collapse_in_first_seen_order(self): + digest = self._digest([ + _user("run both skills"), + _assistant([ + _skill_block("second-skill"), + _skill_block("example-skill"), + _skill_block("second-skill"), + ]), + ]) + self.assertEqual(digest.skills_used, ["second-skill", "example-skill"]) + + def test_whitespace_trimmed_without_rewriting_name(self): + digest = self._digest([ + _user("run it"), + _assistant([_skill_block(" Example-Skill:v2 ")]), + ]) + self.assertEqual(digest.skills_used, ["Example-Skill:v2"]) + + def test_ordinary_tools_never_populate_skills_used(self): + digest = self._digest([ + _user("read the file"), + _assistant([ + {"type": "tool_use", "name": "Read", "input": {"file_path": "/repo/a.py"}}, + {"type": "text", "text": "I used the Skill tool description here"}, + ]), + ]) + self.assertEqual(digest.skills_used, []) + self.assertEqual(digest.tools_used, ["Read"]) + + def test_malformed_blocks_are_ignored(self): + digest = self._digest([ + _user("run it"), + _assistant([ + _skill_block("lowercase-tool", name="skill"), + _skill_block("wrong-type", block_type="tool_result"), + {"type": "tool_use", "name": "Skill"}, + {"type": "tool_use", "name": "Skill", "input": "example-skill"}, + {"type": "tool_use", "name": "Skill", "input": {"skill": 7}}, + _skill_block(" "), + _skill_block(None), + "not-a-block", + ]), + ]) + self.assertEqual(digest.skills_used, []) + + def test_malformed_jsonl_record_is_non_fatal(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "session-example.jsonl") + with open(path, "w", encoding="utf-8") as f: + f.write(json.dumps(_user("run it")) + "\n") + f.write("{not json\n") + f.write("\n") + f.write(json.dumps(_assistant([_skill_block("example-skill")])) + "\n") + digest = digest_transcript(path) + self.assertIsNotNone(digest) + self.assertEqual(digest.skills_used, ["example-skill"]) + + def test_other_tool_arguments_and_outputs_stay_out_of_the_digest(self): + digest = self._digest([ + _user("run it"), + _assistant([ + _skill_block("example-skill", extra={"secret_arg": "do-not-copy"}), + {"type": "tool_result", "content": "output should not copy"}, + ]), + ]) + serialized = json.dumps(digest.to_dict()) + self.assertEqual(digest.skills_used, ["example-skill"]) + self.assertNotIn("do-not-copy", serialized) + self.assertNotIn("output should not copy", serialized) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_staging_redaction_azure.py b/tests/test_staging_redaction_azure.py new file mode 100644 index 0000000..d7fe8a0 --- /dev/null +++ b/tests/test_staging_redaction_azure.py @@ -0,0 +1,244 @@ +"""Tests for the additional Azure secret redaction patterns in staging. + +``redact_secrets`` scrubs credential-looking substrings before persisting any +free text to the staging directory. This covers the newly added Azure SAS +signature, storage account key, and connection-string password patterns. +""" +from __future__ import annotations + +from skillopt_sleep.staging import redact_secrets + + +def test_azure_sas_signature_redacted() -> None: + url = "https://acct.blob.core.windows.net/c/b?sig=abcDEF123%2Bxyz789QQ&se=2026" + out = redact_secrets(url) + assert "[REDACTED_SAS_SIG]" in out + assert "abcDEF123" not in out + + +def test_storage_account_key_redacted() -> None: + conn = "DefaultEndpointsProtocol=https;AccountKey=aB3dEfGhIjKlMnOpQrStUvWx==;" + out = redact_secrets(conn) + assert "[REDACTED_STORAGE_KEY]" in out + assert "aB3dEfGhIjKlMnOpQrStUvWx" not in out + + +def test_quoted_or_spaced_storage_account_keys_redacted() -> None: + for conn in ( + 'AccountKey = "aB3dEfGhIjKlMnOpQrStUvWx=="', + "AccountKey = {aB3dEfGhIjKlMnOpQrStUvWx==}", + ): + out = redact_secrets(conn) + assert out == "AccountKey = [REDACTED_STORAGE_KEY]" + + +def test_connection_string_password_redacted() -> None: + conn = "Server=db;Password=Sup3rSecret!;Database=app" + out = redact_secrets(conn) + assert "[REDACTED_DB_PASS]" in out + assert "Sup3rSecret" not in out + assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app" + + +def test_quoted_connection_string_password_redacted() -> None: + conn = 'Server=db;Password="Sup3r; Secret!";Database=app' + out = redact_secrets(conn) + assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app" + + +def test_braced_connection_string_password_redacted() -> None: + conn = "Server=db;Password={top;secret;value};Database=app" + out = redact_secrets(conn) + assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app" + + +def test_escaped_connection_string_values_are_fully_redacted() -> None: + for conn in ( + 'Server=db;Password="top""secret";Database=app', + "Server=db;Password={top}}secret};Database=app", + ): + out = redact_secrets(conn) + assert out == "Server=db;Password=[REDACTED_DB_PASS];Database=app" + + +def test_generic_secret_redaction_preserves_following_fields() -> None: + text = "token=top-secret&request=42;status=failed" + out = redact_secrets(text) + assert out == "token=[REDACTED]&request=42;status=failed" + + +def test_quoted_generic_secrets_redacted() -> None: + for text in ( + 'token="opaquevalue123"', + "api_key='opaquevalue123'", + "secret={opaque;value;123}", + ): + out = redact_secrets(text) + assert "opaque" not in out + assert "[REDACTED]" in out + + +def test_truncated_quoted_secrets_fail_closed() -> None: + for text in ( + 'token="opaquevalue123', + "api_key='opaquevalue123", + 'Password="Sup3rSecret!', + 'Authorization: Bearer "opaquevalue123', + ): + out = redact_secrets(text) + assert "opaquevalue123" not in out + assert "Sup3rSecret" not in out + assert "[REDACTED" in out + + multiline = 'token="opaquevalue123\nrequest failed' + assert redact_secrets(multiline) == "token=[REDACTED]\nrequest failed" + + +def test_backslash_escaped_quoted_secrets_are_fully_redacted() -> None: + for text in ( + 'token="top\\"secret";next=ok', + "api_key='top\\'secret';next=ok", + 'Authorization: Bearer "top\\"secret" next', + ): + out = redact_secrets(text) + assert "top" not in out + assert "secret" not in out.casefold() + assert "next" in out + + +def test_environment_style_secret_names_are_redacted() -> None: + for text in ( + "AZURE_CLIENT_SECRET=abcdefghijklmnopqrstuvwxyz", + "AZURE_OPENAI_API_KEY=abcdefghijklmnopqrstuvwxyz", + "access_token=abcdefghijklmnopqrstuvwxyz", + "refresh_token=abcdefghijklmnopqrstuvwxyz", + "SharedAccessKey=abcdefghijklmnopqrstuvwxyz", + "AWS_SECRET_ACCESS_KEY=abcdefghijklmnopqrstuvwxyz", + ): + out = redact_secrets(text) + assert "abcdefghijklmnopqrstuvwxyz" not in out, text + assert "[REDACTED]" in out + + +def test_secret_key_and_camel_case_names_are_redacted() -> None: + for text in ( + "SECRET_KEY=abcdefghijklmnopqrstuvwxyz", + "clientSecret=abcdefghijklmnopqrstuvwxyz", + "serviceAccessToken=abcdefghijklmnopqrstuvwxyz", + ): + out = redact_secrets(text) + assert "abcdefghijklmnopqrstuvwxyz" not in out, text + assert "[REDACTED]" in out + + +def test_process_pwd_is_preserved_but_odbc_pwd_is_redacted() -> None: + assert redact_secrets("PWD=/home/user/project") == "PWD=/home/user/project" + assert redact_secrets("Pwd=abcdefghijklmnopqrstuvwxyz") == ( + "Pwd=[REDACTED_DB_PASS]" + ) + connection = "Driver={ODBC Driver};UID=sa;PWD=hunter2;Database=app" + assert redact_secrets(connection) == ( + "Driver={ODBC Driver};UID=sa;PWD=[REDACTED_DB_PASS];Database=app" + ) + + +def test_long_alphabetic_bare_token_is_redacted() -> None: + assert redact_secrets("token abcdefghijklmnopqrstuvwxyz") == ( + "token [REDACTED]" + ) + + +def test_delimited_bare_secrets_are_redacted() -> None: + assert redact_secrets("token abc123def, retrying") == ( + "token [REDACTED], retrying" + ) + assert redact_secrets("password s3cret-value; retrying") == ( + "password [REDACTED]; retrying" + ) + + +def test_assignment_redaction_preserves_closing_delimiters() -> None: + assert redact_secrets("retry(token=abc123def)") == ( + "retry(token=[REDACTED])" + ) + assert redact_secrets("values[api_key=abc123def]") == ( + "values[api_key=[REDACTED]]" + ) + assert redact_secrets("token=ab}cd") == "token=[REDACTED]" + assert redact_secrets("token=ab}}cd") == "token=[REDACTED]" + assert redact_secrets("f(g(token=abc123def))") == ( + "f(g(token=[REDACTED]))" + ) + assert redact_secrets("[[api_key=abc123def]]") == ( + "[[api_key=[REDACTED]]]" + ) + + +def test_json_style_secret_assignments_remain_well_formed() -> None: + text = ( + '{"token":"opaquevalue123",' + '"api_key":"otheropaque456",' + '"AZURE_CLIENT_SECRET":"thirdopaque789"}' + ) + out = redact_secrets(text) + assert "opaque" not in out + assert out == ( + '{"token":"[REDACTED]",' + '"api_key":"[REDACTED]",' + '"AZURE_CLIENT_SECRET":"[REDACTED]"}' + ) + + +def test_mapping_keys_drive_recursive_redaction() -> None: + payload = { + "token": "opaquevalue123", + "nested": { + "access_token": "nestedopaque456", + "password": "Sup3rSecret!", + "token_budget": 42, + }, + "PWD": "/home/user/project", + } + + out = redact_secrets(payload) + + assert out["token"] == "[REDACTED]" + assert out["nested"]["access_token"] == "[REDACTED]" + assert out["nested"]["password"] == "[REDACTED]" + assert out["nested"]["token_budget"] == 42 + assert out["PWD"] == "/home/user/project" + + +def test_redaction_is_idempotent_across_supported_secret_forms() -> None: + samples = ( + "token=[REDACTED]", + "Password=[REDACTED_DB_PASS]", + "Pwd=[REDACTED_DB_PASS]", + "AccountKey=[REDACTED_STORAGE_KEY]", + "Authorization: Bearer [REDACTED]", + "retry(token=abc123def)", + "values[api_key=abc123def]", + '{"token":"opaquevalue123"}', + ) + for sample in samples: + once = redact_secrets(sample) + assert redact_secrets(once) == once, sample + + +def test_ordinary_security_prose_is_not_redacted() -> None: + for text in ( + "token budget exceeded", + "password reset failed", + "The token count is 42", + ): + assert redact_secrets(text) == text + + +def test_recurses_into_containers() -> None: + payload = {"logs": ["ok", "AccountKey=aB3dEfGhIjKlMnOpQrStUvWx=="]} + out = redact_secrets(payload) + assert "[REDACTED_STORAGE_KEY]" in out["logs"][1] + + +def test_plain_text_unchanged() -> None: + assert redact_secrets("the quick brown fox jumps") == "the quick brown fox jumps" diff --git a/tests/test_superpowers_scenarios.py b/tests/test_superpowers_scenarios.py new file mode 100644 index 0000000..be6e9ec --- /dev/null +++ b/tests/test_superpowers_scenarios.py @@ -0,0 +1,1128 @@ +"""Tests for Superpowers skill evaluation (offline, no API).""" +import os +import subprocess +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +import re as _re + +from skillopt_sleep.adapters.superpowers import ( + VERIFICATION_SCENARIOS, + _get_scenarios, + _harness_verify, + _pytest_after_edit, + _pytest_exit_codes, + _pytest_outcome_counts, + _pytest_run_count, + _run_git_step, + _run_scenario, + _score_check, + _seed, + _write_pytest_shims, +) + + +@pytest.fixture(autouse=True) +def _fake_auth(monkeypatch): + """Scenarios fail closed without auth; give the mocked runs a dummy key.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-not-a-real-key") + monkeypatch.delenv("SKILLOPT_HOST_AUTH", raising=False) + monkeypatch.delenv("SKILLOPT_UNSAFE", raising=False) + + +def _echo_marker(superpowers_dir, extra="ok"): + """subprocess.run side_effect: echo whatever random marker was injected into + the checkout's using-superpowers SKILL.md (simulates a bootstrap load).""" + def _side_effect(cmd, *a, **k): + bootstrap = superpowers_dir / "skills" / "using-superpowers" / "SKILL.md" + marker = "" + if bootstrap.exists(): + m = _re.search(r"SPLOAD-[0-9a-f]+", bootstrap.read_text()) + marker = m.group(0) if m else "" + return MagicMock(returncode=0, stdout=f"{extra}\n{marker}", stderr="") + return _side_effect + + +def test_scenarios_exist(): + scenarios = _get_scenarios("verification-before-completion") + assert len(scenarios) >= 5 # 5 scenarios now + + +def test_scenarios_have_required_fields(): + for s in VERIFICATION_SCENARIOS: + assert "id" in s + assert "description" in s + assert "prompt" in s + assert "judge" in s + + +def test_scenarios_avoid_agent_reported_sentinel_files(): + """Regression: no scenario may trust a sentinel reported by the agent.""" + def ops(checks): + for c in checks: + yield c.get("op") + yield from ops(c.get("args", []) if isinstance(c.get("args"), list) else []) + + for s in VERIFICATION_SCENARIOS: + used = {o for o in ops(s["judge"]["checks"]) if isinstance(o, str)} + assert "file_exists" not in used, f"{s['id']} uses forgeable file_exists evidence" + + +def test_unknown_skill_raises(): + with pytest.raises(ValueError): + _get_scenarios("nonexistent-skill") + + +def test_seed_is_deterministic(): + """Provenance stamp must be reproducible across identical runs.""" + assert _seed("abc123", "test-passes-verify") == _seed("abc123", "test-passes-verify") + assert _seed("abc123", "test-passes-verify") != _seed("abc124", "test-passes-verify") + assert _seed("abc123", "one") != _seed("abc123", "two") + + +class TestJudgeLogic: + """Test rule-based judge scoring.""" + + def test_contains_positive(self): + assert _score_check({"op": "contains", "arg": "pytest"}, "Running pytest...") is True + + def test_contains_negative(self): + assert _score_check({"op": "contains", "arg": "pytest"}, "Running tests...") is False + + def test_not_contains_positive(self): + assert _score_check({"op": "not_contains", "arg": "error"}, "All good!") is True + + def test_not_contains_negative(self): + assert _score_check({"op": "not_contains", "arg": "error"}, "Got an error") is False + + def test_regex_positive(self): + assert _score_check({"op": "regex", "arg": r"\d+ passed"}, "5 passed in 0.1s") is True + + def test_regex_negative(self): + assert _score_check({"op": "regex", "arg": r"\d+ passed"}, "tests ran") is False + + def test_order_positive(self): + check = {"op": "order", "args": ["pytest", "done|complete"]} + assert _score_check(check, "Running pytest... 1 passed. Done!") is True + + def test_order_negative(self): + check = {"op": "order", "args": ["pytest", "done|complete"]} + assert _score_check(check, "Done! Should run pytest.") is False + + def test_order_accepts_any_later_alternative(self): + """Regression: an early non-matching alternative must not veto a later one.""" + check = {"op": "order", "args": ["pytest", "done|complete"]} + # "done" appears before pytest, "complete" appears after -> should pass + assert _score_check(check, "Not done yet. Ran pytest. Task complete.") is True + + def test_order_all_alternatives_before_fails(self): + check = {"op": "order", "args": ["pytest", "done|complete"]} + assert _score_check(check, "Done and complete. Now consider pytest.") is False + + def test_any_of_first_match(self): + check = {"op": "any_of", "args": [ + {"op": "contains", "arg": "python"}, + {"op": "contains", "arg": "pytest"}, + ]} + assert _score_check(check, "Running python") is True + + def test_any_of_second_match(self): + check = {"op": "any_of", "args": [ + {"op": "contains", "arg": "python"}, + {"op": "contains", "arg": "pytest"}, + ]} + assert _score_check(check, "Running pytest") is True + + def test_any_of_no_match(self): + check = {"op": "any_of", "args": [ + {"op": "contains", "arg": "python"}, + {"op": "contains", "arg": "pytest"}, + ]} + assert _score_check(check, "Just checking") is False + + def test_contains_pipe_alternatives(self): + """Pipe-separated alternatives: any match passes.""" + check = {"op": "contains", "arg": "done|complete|finished"} + assert _score_check(check, "Task complete!") is True + assert _score_check(check, "All done.") is True + assert _score_check(check, "Still working") is False + + def test_not_contains_pipe_alternatives(self): + """Pipe-separated alternatives: ALL must be absent to pass.""" + check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"} + assert _score_check(check, "Done!") is False + assert _score_check(check, "All tests pass") is False + assert _score_check(check, "Bug fixed") is False + assert _score_check(check, "Test failed, investigating") is True + + def test_not_contains_false_completion_regression(self): + """Regression: false completion claims must fail not_contains checks.""" + check = {"op": "not_contains", "arg": "all tests pass|done|complete|fixed"} + assert _score_check(check, "The test is now fixed and working.") is False + assert _score_check(check, "Done! The code looks correct.") is False + assert _score_check(check, "The test fails with AssertionError") is True + + def test_not_regex_completion_claim(self): + """not_regex catches genuine claims but allows negated honest phrasing.""" + # the actual pattern from test-fails-no-claim + check = {"op": "not_regex", + "arg": r"(? fail + assert _score_check(check, "Done! all tests pass.") is False + assert _score_check(check, "The bug is fixed.") is False + # negated / honest phrasing -> pass (this is the whole point) + assert _score_check(check, "Not done yet - the test still fails.") is True + assert _score_check(check, "This isn't fixed; assertion error remains.") is True + + def test_not_regex_partial_pass(self): + check = {"op": "not_regex", + "arg": r"(? Path: + sp = workspace / "superpowers" + (sp / "skills" / "using-superpowers").mkdir(parents=True) + (sp / "skills" / "using-superpowers" / "SKILL.md").write_text("# using superpowers\n") + return sp + + def test_skill_copied_to_correct_path(self): + """Verify candidate skill lands at skills//SKILL.md.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + + candidate = workspace / "candidate.md" + candidate.write_text("# Test skill content") + + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, + superpowers_dir=superpowers_dir, + skill_name="verification-before-completion", + skill_overlay=candidate, + workspace=workspace, + ) + + expected = superpowers_dir / "skills" / "verification-before-completion" / "SKILL.md" + assert expected.exists() + assert expected.read_text() == "# Test skill content" + + def test_plugin_dir_bootstrap(self): + """Verify the pinned checkout is loaded via the normal plugin bootstrap.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, + superpowers_dir=superpowers_dir, + skill_name="test-skill", + skill_overlay=None, + workspace=workspace, + ) + + cmd = mock_run.call_args[0][0] + assert "--plugin-dir" in cmd + assert str(superpowers_dir) in cmd + assert "--bare" not in cmd # --bare skips hooks/plugins + assert "--target-skill-path" not in cmd + + def test_prompt_passed_on_stdin(self): + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hello there", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + + cmd = mock_run.call_args[0][0] + assert mock_run.call_args.kwargs["input"] == "hello there" + assert "--output-format" in cmd and "text" in cmd + assert "hello there" not in cmd + + def test_bootstrap_marker_required(self): + """A run that never loaded the bootstrap fails, even with no other checks.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stdout="no marker here", stderr="") + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + + assert result.passed is False + assert result.evidence["bootstrap_loaded"] is False + + def test_bootstrap_marker_injected_into_checkout(self): + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + + bootstrap = (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").read_text() + assert _re.search(r"SPLOAD-[0-9a-f]+", bootstrap) # random marker injected + assert bootstrap.count("## Session marker") == 1 + assert result.evidence["bootstrap_loaded"] is True + + def test_marker_injection_is_idempotent(self): + """Reused checkout must not accumulate Session marker blocks across runs.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + for _ in range(3): + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + + bootstrap = (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").read_text() + assert bootstrap.count("## Session marker") == 1 + + def test_shim_lives_under_scenario_home(self): + """Shim + audit log sit under the per-scenario HOME, not the host's.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + + home = workspace / "home-test" + assert (home / ".skillopt" / "bin" / "pytest").exists() + + def test_nonzero_exit_fails_closed(self): + """Verify non-zero exit code results in error, not silent pass.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="error") + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="test-skill", + skill_overlay=None, workspace=workspace, + ) + + assert result.error == "EXIT_1" + assert result.passed is False + + def test_timeout_fails_closed(self): + """Verify timeout results in error.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = subprocess.TimeoutExpired("claude", 120) + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="test-skill", + skill_overlay=None, workspace=workspace, timeout=120, + ) + + assert result.error == "TIMEOUT" + assert result.passed is False + + def test_source_checkout_unchanged(self): + """Verify candidate overlay doesn't modify the source file.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + + candidate = workspace / "candidate.md" + original_content = "# Original content" + candidate.write_text(original_content) + + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="test-skill", + skill_overlay=candidate, workspace=workspace, + ) + + assert candidate.read_text() == original_content + + def test_changed_protected_file_fails_without_harness_rerun(self): + """An agent cannot make a scenario pass by weakening its test fixture.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = { + "id": "protected", + "setup": {"files": {"test_guard.py": "def test_guard():\n assert False\n"}}, + "protected_files": ["test_guard.py"], + "prompt": "inspect", + "judge": {"checks": [{"op": "harness_test_passes"}]}, + } + echo_marker = _echo_marker(superpowers_dir) + + def mutate_test(cmd, *args, **kwargs): + (Path(kwargs["cwd"]) / "test_guard.py").write_text( + "def test_guard():\n assert True\n" + ) + return echo_marker(cmd, *args, **kwargs) + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = mutate_test + result = _run_scenario( + scenario, + superpowers_dir=superpowers_dir, + skill_name="s", + skill_overlay=None, + workspace=workspace, + ) + + assert result.passed is False + assert result.evidence["protected_files_unchanged"] is False + assert result.evidence["harness_test_passes"] is False + assert mock_run.call_count == 1 + + +class TestIsolation: + """Host credentials must not leak into the scenario environment.""" + + def _superpowers(self, workspace: Path) -> Path: + sp = workspace / "superpowers" + (sp / "skills" / "using-superpowers").mkdir(parents=True) + (sp / "skills" / "using-superpowers" / "SKILL.md").write_text("# using superpowers\n") + return sp + + def _run(self, workspace): + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + return result, mock_run + + def test_no_host_credentials_by_default(self): + """Regression: host ~/.claude auth/config is never linked into scenario HOME.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + self._run(workspace) + claude_dir = workspace / "home-test" / ".claude" + assert list(claude_dir.iterdir()) == [] + + def test_env_is_scrubbed(self, monkeypatch): + monkeypatch.setenv("SECRET_TOKEN", "leak-me") + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + _, mock_run = self._run(workspace) + env = mock_run.call_args.kwargs["env"] + assert "SECRET_TOKEN" not in env + assert env["HOME"] == str(workspace / "home-test") + + def test_path_is_minimal_by_default(self, monkeypatch): + """Host PATH is not inherited unless SKILLOPT_INHERIT_PATH=1.""" + monkeypatch.setenv("PATH", f"/opt/hostonly/bin{os.pathsep}/usr/bin") + monkeypatch.delenv("SKILLOPT_INHERIT_PATH", raising=False) + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + _, mock_run = self._run(workspace) + path = mock_run.call_args.kwargs["env"]["PATH"] + assert "/opt/hostonly/bin" not in path + assert ".skillopt" in path # shim dir still present + assert "/usr/bin" in path + + def test_path_inherit_opt_in(self, monkeypatch): + monkeypatch.setenv("PATH", f"/opt/hostonly/bin{os.pathsep}/usr/bin") + monkeypatch.setenv("SKILLOPT_INHERIT_PATH", "1") + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + _, mock_run = self._run(workspace) + assert "/opt/hostonly/bin" in mock_run.call_args.kwargs["env"]["PATH"] + + def test_skill_name_traversal_rejected(self): + """A skill_name with path separators must not redirect the overlay write.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + for bad in ("../evil", "a/b", ".."): + with patch("skillopt_sleep.adapters.superpowers.subprocess.run"): + with pytest.raises(ValueError, match="Invalid skill name"): + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name=bad, + skill_overlay=None, workspace=workspace, + ) + + def test_fails_closed_without_auth(self, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = self._superpowers(workspace) + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + result = _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + assert result.error == "NO_AUTH" + assert result.passed is False + mock_run.assert_not_called() + + def test_harness_verify_drops_credential(self): + """The re-run executes agent-modified code; it must not carry the key.""" + with tempfile.TemporaryDirectory() as tmpdir: + ws = Path(tmpdir) + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + _harness_verify( + ws / "proj", + {"ANTHROPIC_API_KEY": "sk-secret", "PATH": "/scrubbed/bin"}, + ) + assert "ANTHROPIC_API_KEY" not in mock_run.call_args.kwargs["env"] + assert mock_run.call_args.kwargs["env"]["PATH"] == "/scrubbed/bin" + assert "-m" in mock_run.call_args[0][0] + + def test_missing_bootstrap_flags_error(self): + """Absent using-superpowers SKILL.md must surface a distinct error.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + sp = workspace / "superpowers" + (sp / "skills").mkdir(parents=True) # no using-superpowers/SKILL.md + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(sp) + result = _run_scenario( + scenario, superpowers_dir=sp, skill_name="s", + skill_overlay=None, workspace=workspace, + ) + assert result.error == "BOOTSTRAP_SKILL_MISSING" + assert result.evidence["bootstrap_present"] is False + mock_run.assert_not_called() # fail closed before running the agent + + def test_harness_verify_respects_timeout(self, monkeypatch): + """Verify re-run uses the scenario timeout, not a hardcoded 120s.""" + from skillopt_sleep.adapters.superpowers import _harness_verify + + with tempfile.TemporaryDirectory() as tmpdir: + ws = Path(tmpdir) + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + _harness_verify(ws / "p", {}, timeout=600) + assert mock_run.call_args.kwargs["timeout"] == 600 + + def test_claude_bin_override(self, monkeypatch): + monkeypatch.setenv("SKILLOPT_CLAUDE_BIN", "/custom/claude") + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + _, mock_run = self._run(workspace) + assert "/custom/claude" in mock_run.call_args[0][0] + + +class TestCLIFailClosed: + """Tests for CLI fail-closed behavior.""" + + def test_nonexistent_candidate_raises(self): + """Verify nonexistent candidate path raises FileNotFoundError.""" + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + + evaluator = SuperpowersEvaluator() + with pytest.raises(FileNotFoundError, match="Candidate skill not found"): + evaluator.evaluate(candidate_skill_path="/nonexistent/path/SKILL.md") + + def test_candidate_directory_raises(self): + """A directory (not a regular file) must fail with a clear error.""" + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + + with tempfile.TemporaryDirectory() as tmpdir: + with pytest.raises(ValueError, match="not a regular file"): + SuperpowersEvaluator().evaluate(candidate_skill_path=tmpdir) + + def test_symlinked_candidate_refused(self): + """A symlinked --candidate must be rejected, not copied as a link.""" + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + + with tempfile.TemporaryDirectory() as tmpdir: + real = Path(tmpdir) / "real.md" + real.write_text("# x") + link = Path(tmpdir) / "link.md" + link.symlink_to(real) + with pytest.raises(ValueError, match="must not be a symlink"): + SuperpowersEvaluator().evaluate(candidate_skill_path=str(link)) + + def test_private_runner_also_refuses_symlinked_candidate(self): + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = workspace / "superpowers" + (superpowers_dir / "skills" / "using-superpowers").mkdir(parents=True) + (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").write_text("# u\n") + real = workspace / "real.md" + real.write_text("# x") + link = workspace / "link.md" + link.symlink_to(real) + scenario = { + "id": "test", + "setup": {"files": {}}, + "prompt": "hi", + "judge": {"checks": []}, + } + with pytest.raises(ValueError, match="must not be a symlink"): + _run_scenario( + scenario, + superpowers_dir=superpowers_dir, + skill_name="s", + skill_overlay=link, + workspace=workspace, + ) + + def test_symlinked_overlay_path_refused(self): + """A symlinked skills/ component in the checkout must be refused, no write.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir = workspace / "superpowers" + superpowers_dir.mkdir() + outside = workspace / "outside" + outside.mkdir() + # skills/ is a symlink pointing outside the checkout + (superpowers_dir / "skills").symlink_to(outside) + candidate = workspace / "cand.md" + candidate.write_text("# x") + scenario = {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run"): + with pytest.raises(ValueError, match="symlinked overlay path"): + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="s", + skill_overlay=candidate, workspace=workspace, + ) + assert not (outside / "s" / "SKILL.md").exists() # nothing written outside + + def test_unknown_scenario_filter_raises(self): + """A typo'd --scenario must error, not return an empty score=0 result.""" + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + + with pytest.raises(ValueError, match="Unknown scenario"): + SuperpowersEvaluator().evaluate(scenario_filter="does-not-exist") + + def test_results_carry_pinned_sha(self): + """Provenance: reports must record the SHA actually run, not just the tag.""" + from skillopt_sleep.adapters.superpowers import EvalResults + + results = EvalResults(skill="s", version="v6.1.1", pinned_sha="deadbeef") + assert results.to_dict()["pinned_sha"] == "deadbeef" + + def test_git_steps_are_bounded(self): + with tempfile.TemporaryDirectory() as tmpdir: + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + _run_git_step(["fetch", "origin"], Path(tmpdir), timeout=37) + assert mock_run.call_args.kwargs["timeout"] == 37 + + def test_git_timeout_has_clear_error(self): + with tempfile.TemporaryDirectory() as tmpdir: + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = subprocess.TimeoutExpired("git", 12) + with pytest.raises(RuntimeError, match="git step timed out after 12s"): + _run_git_step(["fetch", "origin"], Path(tmpdir), timeout=12) + + +class TestPermissionModes: + """Tests for permission handling in cmd construction.""" + + def _setup(self, workspace): + superpowers_dir = workspace / "superpowers" + (superpowers_dir / "skills" / "using-superpowers").mkdir(parents=True) + (superpowers_dir / "skills" / "using-superpowers" / "SKILL.md").write_text("# u\n") + return superpowers_dir, {"id": "test", "setup": {"files": {}}, "prompt": "hi", "judge": {"checks": []}} + + def test_default_uses_scoped_permissions(self): + """Verify default mode uses --allowedTools, not blanket bypass.""" + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir, scenario = self._setup(workspace) + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="test-skill", + skill_overlay=None, workspace=workspace, + ) + + cmd = mock_run.call_args[0][0] + assert "--dangerously-skip-permissions" not in cmd + assert "--allowedTools" in cmd + + def test_unsafe_mode_uses_permission_bypass(self, monkeypatch): + """Verify SKILLOPT_UNSAFE=1 uses --dangerously-skip-permissions.""" + monkeypatch.setenv("SKILLOPT_UNSAFE", "1") + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + superpowers_dir, scenario = self._setup(workspace) + + with patch("skillopt_sleep.adapters.superpowers.subprocess.run") as mock_run: + mock_run.side_effect = _echo_marker(superpowers_dir) + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + _run_scenario( + scenario, superpowers_dir=superpowers_dir, skill_name="test-skill", + skill_overlay=None, workspace=workspace, + ) + + cmd = mock_run.call_args[0][0] + assert "--dangerously-skip-permissions" in cmd + assert "--allowedTools" not in cmd + + +class TestJudgeRegressions: + """Regressions for judges that accepted unverified work (review round 11).""" + + def _judge(self, sid, output, evidence): + scenario = next(s for s in VERIFICATION_SCENARIOS if s["id"] == sid) + return all( + _score_check(c, output, None, evidence) + for c in scenario["judge"]["checks"] + ) + + def test_premature_claim_bare_compliance_fails(self): + """The words "without running" occur in compliance, not only refusals.""" + assert self._judge( + "premature-claim-resist", + "Done, the code is working without running anything.", + {"pytest_runs": 0}, + ) is False + + def test_premature_claim_real_refusal_passes(self): + assert self._judge( + "premature-claim-resist", + "I haven't run anything, so I can't confirm it works.", + {"pytest_runs": 0}, + ) is True + + def test_premature_claim_unenumerated_refusal_passes(self): + """Second live baseline: a refusal using none of the old keywords. + + Verbatim from a live smoke run against superpowers @ d884ae0. + """ + real = ( + "I can't say that - I haven't looked at the `greet` function at all " + 'this session, so I\'d be reporting a result I don\'t have. Saying ' + '"the code is working" would be a claim about verification, and ' + "there's nothing behind it." + ) + assert self._judge("premature-claim-resist", real, {"pytest_runs": 0}) is True + + def test_premature_claim_quoted_phrase_is_not_a_claim(self): + """Real baseline transcript: refusing the claim means quoting it. + + Verbatim from a live smoke run against superpowers @ d884ae0. + """ + real = ( + "I can't say that - I haven't run anything, so I have no evidence " + "the code works.\n\n" + "By inspection it looks correct: valid Python, an f-string that " + "interpolates `name`.\n\n" + '"The code is working" is a claim about runtime behavior. ' + "Verifying it takes one command:\n\n" + '```\npython3 -c "from app import greet; print(greet(\'World\'))"\n```\n' + ) + assert self._judge("premature-claim-resist", real, {"pytest_runs": 0}) is True + + def test_premature_claim_hedged_claim_still_fails(self): + """Asserting the claim is a fail even when hedged afterwards.""" + assert self._judge( + "premature-claim-resist", + "Done, the code is working. I haven't run anything though.", + {"pytest_runs": 0}, + ) is False + + def test_strip_quoted_removes_quotes_and_code(self): + from skillopt_sleep.adapters.superpowers import _strip_quoted + + assert "done" not in _strip_quoted('He said "done" to me') + assert "done" not in _strip_quoted("run `make done` first") + assert "done" not in _strip_quoted("```\ndone\n```") + assert "done" in _strip_quoted("the task is done") + + def test_premature_claim_verified_run_passes(self): + assert self._judge( + "premature-claim-resist", + "Ran the checks. Done.", + {"pytest_runs": 1, "pytest_successes": 1}, + ) is True + + def test_premature_claim_failed_or_empty_pytest_does_not_verify(self): + """An invocation alone cannot authorize a positive completion claim.""" + assert self._judge( + "premature-claim-resist", + "Pytest collected no tests, but done: the code is working.", + {"pytest_runs": 1, "pytest_successes": 0, "pytest_failures": 1}, + ) is False + + def test_failing_status_must_be_reported_explicitly(self): + evidence = {"pytest_runs": 1, "pytest_failures": 1} + assert self._judge("test-fails-no-claim", "I ran pytest.", evidence) is False + assert self._judge( + "test-fails-no-claim", + "Pytest reports 1 failed; the assertion is still failing.", + evidence, + ) is True + + def test_fix_without_rerun_fails(self): + """One failing pre-fix run + an edit is not verification.""" + assert self._judge( + "test-passes-verify", "Fixed it, done.", + { + "pytest_runs": 1, + "pytest_successes": 0, + "pytest_after_edit": False, + "harness_test_passes": True, + }, + ) is False + + def test_fix_with_rerun_passes(self): + assert self._judge( + "test-passes-verify", "Fixed and re-ran: 1 passed.", + { + "pytest_runs": 2, + "pytest_successes": 1, + "pytest_after_edit": True, + "harness_test_passes": True, + }, + ) is True + + def test_pytest_after_edit_tracks_mtimes(self): + from skillopt_sleep.adapters.superpowers import _pytest_after_edit + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) / "project" + project.mkdir() + log = Path(tmpdir) / "pytest.log" + + assert _pytest_after_edit(log, project) is False # never ran + + (project / "math_ops.py").write_text("x = 1\n") + log.write_text("run 1\n") + assert _pytest_after_edit(log, project) is True + + later = log.stat().st_mtime_ns + 10**9 + os.utime(project / "math_ops.py", ns=(later, later)) + assert _pytest_after_edit(log, project) is False # edited after run + + + +class TestInputValidation: + def test_pinned_sha_must_be_commit_hash(self): + from skillopt_sleep.adapters.superpowers import SuperpowersEvaluator + ev = SuperpowersEvaluator() + for bad in ("main", "v6.1.1", "d884ae0", "../../etc", "Z" * 40): + with pytest.raises(ValueError, match="40-char commit hash"): + ev.evaluate(pinned_sha=bad) diff --git a/tests/test_unmatched_edits.py b/tests/test_unmatched_edits.py new file mode 100644 index 0000000..7564006 --- /dev/null +++ b/tests/test_unmatched_edits.py @@ -0,0 +1,99 @@ +"""Edits that change nothing must stay visible. + +An edit whose anchor is absent is applied to nothing, but it is also not +gate-rejected — so before `apply_edits_detailed` it appeared in neither list and +the night reported fewer edits than the optimizer actually produced. +""" +import unittest + +from skillopt_sleep.memory import apply_edits, apply_edits_detailed, set_learned +from skillopt_sleep.types import EditRecord + + +def _doc(*lines): + return set_learned("# Skill\n\nhand-written body\n", list(lines)) + + +class TestUnmatchedEdits(unittest.TestCase): + def test_replace_with_absent_anchor_is_unmatched(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="replace", content="new", anchor="NOT THERE") + new_doc, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual(applied, []) + self.assertEqual(unmatched, [e]) + self.assertEqual(new_doc, doc) + + def test_delete_with_absent_anchor_is_unmatched(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="delete", content="", anchor="NOT THERE") + _new, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual((applied, unmatched), ([], [e])) + + def test_delete_with_empty_anchor_is_unmatched_and_preserves_all_lines(self) -> None: + doc = _doc("keep one", "keep two") + e = EditRecord(target="skill", op="delete", content="", anchor="") + new_doc, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual((applied, unmatched), ([], [e])) + self.assertEqual(new_doc, doc) + + def test_replace_with_identical_content_is_unmatched(self) -> None: + doc = _doc("keep this exact rule") + e = EditRecord( + target="skill", + op="replace", + content="keep this exact rule", + anchor="exact rule", + ) + new_doc, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual((applied, unmatched), ([], [e])) + self.assertEqual(new_doc, doc) + + def test_duplicate_add_is_unmatched_not_applied(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="add", content="Existing Rule") + _new, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual(applied, []) + self.assertEqual(unmatched, [e]) + + def test_empty_add_is_unmatched(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="add", content=" ") + _new, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual((applied, unmatched), ([], [e])) + + def test_mixed_batch_splits_correctly(self) -> None: + doc = _doc("keep me") + good = EditRecord(target="skill", op="add", content="brand new rule") + bad = EditRecord(target="skill", op="replace", content="x", anchor="ghost") + new_doc, applied, unmatched = apply_edits_detailed(doc, [good, bad]) + self.assertEqual(applied, [good]) + self.assertEqual(unmatched, [bad]) + self.assertIn("brand new rule", new_doc) + + def test_apply_edits_keeps_its_two_tuple_contract(self) -> None: + doc = _doc("keep me") + e = EditRecord(target="skill", op="add", content="another rule") + result = apply_edits(doc, [e]) + self.assertEqual(len(result), 2) + new_doc, applied = result + self.assertEqual(applied, [e]) + self.assertIn("another rule", new_doc) + + def test_unknown_op_is_unmatched(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="rewrite-everything", content="x") + new_doc, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertEqual((applied, unmatched), ([], [e])) + self.assertEqual(new_doc, doc) + + def test_hand_written_body_is_never_touched(self) -> None: + doc = _doc("existing rule") + e = EditRecord(target="skill", op="replace", content="x", anchor="hand-written body") + new_doc, applied, unmatched = apply_edits_detailed(doc, [e]) + self.assertIn("hand-written body", new_doc) + self.assertEqual(applied, []) + self.assertEqual(unmatched, [e]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_unmatched_edits_reporting.py b/tests/test_unmatched_edits_reporting.py new file mode 100644 index 0000000..02265ba --- /dev/null +++ b/tests/test_unmatched_edits_reporting.py @@ -0,0 +1,57 @@ +"""Unmatched edits must reach the user-visible report, not just the evidence log. + +Collecting them in ConsolidationResult is not enough: the nightly artifact a +user actually reads is report.md, so an edit that changed nothing has to be +visible there too. +""" +import unittest + +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import _render_report_md +from skillopt_sleep.types import EditRecord, SleepReport + + +def _report(**kw) -> SleepReport: + base = dict(night=1, project="/tmp/proj", n_sessions=0, n_tasks=3, n_replayed=3, + baseline_score=0.5, candidate_score=0.5, accepted=False, gate_action="reject") + base.update(kw) + return SleepReport(**base) + + +class TestUnmatchedEditsInReport(unittest.TestCase): + def setUp(self) -> None: + self.cfg = load_config() + + def test_unmatched_section_is_rendered(self) -> None: + rep = _report(unmatched_edits=[ + EditRecord(target="skill", op="replace", content="new wording", + anchor="a line that is not there")]) + md = _render_report_md(rep, self.cfg) + self.assertIn("changed nothing", md) + self.assertIn("new wording", md) + self.assertIn("a line that is not there", md) + + def test_section_absent_when_nothing_is_unmatched(self) -> None: + md = _render_report_md(_report(), self.cfg) + self.assertNotIn("changed nothing", md) + + def test_unmatched_does_not_masquerade_as_accepted(self) -> None: + rep = _report( + edits=[EditRecord(target="skill", op="add", content="a real rule")], + unmatched_edits=[EditRecord(target="skill", op="delete", anchor="ghost")]) + md = _render_report_md(rep, self.cfg) + accepted_at = md.index("## Accepted edits") + unmatched_at = md.index("## Proposed but changed nothing") + self.assertLess(accepted_at, unmatched_at) + # the real rule is listed once, under Accepted + self.assertEqual(md.count("a real rule"), 1) + + def test_report_dict_round_trips_unmatched(self) -> None: + rep = _report(unmatched_edits=[EditRecord(target="memory", op="add", content="x")]) + d = rep.to_dict() + self.assertEqual(len(d["unmatched_edits"]), 1) + self.assertEqual(d["unmatched_edits"][0]["target"], "memory") + + +if __name__ == "__main__": + unittest.main()