mirror of
https://github.com/microsoft/SkillOpt.git
synced 2026-07-07 08:22:44 +08:00
devin plugin: expand ~ in CLAUDE_HOME from env + add tests & ATIF fixture
Review fixes: - Path bug: SKILLOPT_DEVIN_CLAUDE_HOME (and SKILLOPT_SLEEP_REPO) read from the env are now wrapped in os.path.expanduser, so the documented "~/..." config no longer passes a literal ~ to --claude-home (which yielded zero mined sessions). expanduser on an absolute default is a no-op. - tests/test_devin_plugin.py: tool-schema completeness, action→subcommand map, backend enum, the CLAUDE_HOME expansion regression, and an ATIF-v1.7 harvest shape test against a bundled fixture. - plugins/devin/fixtures/devin_sample.json: sample ATIF-v1.7 transcript. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
21
plugins/devin/fixtures/devin_sample.json
Normal file
21
plugins/devin/fixtures/devin_sample.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"schema_version": "ATIF-v1.7",
|
||||
"session_id": "demo-001",
|
||||
"steps": [
|
||||
{
|
||||
"source": "user",
|
||||
"message": "Fix the failing NullPointerException in OrderService.persist() in the dutch-kis project",
|
||||
"timestamp": "2026-06-20T10:00:00Z"
|
||||
},
|
||||
{
|
||||
"source": "agent",
|
||||
"message": "The repository call returns an Optional that is being unwrapped with .get(). I'll switch to orElseThrow(NotFoundException::new) so the missing-row case is handled.",
|
||||
"timestamp": "2026-06-20T10:00:05Z"
|
||||
},
|
||||
{
|
||||
"source": "agent",
|
||||
"message": "Applied the fix and ran the suite: rtk mvn test -Dtest=OrderServiceTest -> BUILD SUCCESS, 142 passed, 0 failed.",
|
||||
"timestamp": "2026-06-20T10:01:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -36,14 +36,13 @@ import sys
|
||||
|
||||
# ── constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
REPO_ROOT = (
|
||||
REPO_ROOT = os.path.expanduser(
|
||||
os.environ.get("SKILLOPT_SLEEP_REPO")
|
||||
or os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
)
|
||||
PLUGIN_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
CLAUDE_HOME = os.environ.get(
|
||||
"SKILLOPT_DEVIN_CLAUDE_HOME",
|
||||
os.path.expanduser("~/.skillopt-sleep-devin"),
|
||||
CLAUDE_HOME = os.path.expanduser(
|
||||
os.environ.get("SKILLOPT_DEVIN_CLAUDE_HOME", "~/.skillopt-sleep-devin")
|
||||
)
|
||||
MANAGED_SKILL_NAME = os.environ.get("SKILLOPT_MANAGED_SKILL", "skillopt-sleep-learned")
|
||||
PROTOCOL_VERSION = "2024-11-05"
|
||||
|
||||
87
tests/test_devin_plugin.py
Normal file
87
tests/test_devin_plugin.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""Tests for the Devin MCP plugin: tool schema, ATIF-v1.7 harvest, path expansion."""
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
# Allow importing from the plugin directory (mirrors tests/test_mcp_schema.py)
|
||||
PLUGIN = os.path.join(os.path.dirname(__file__), "..", "plugins", "devin")
|
||||
sys.path.insert(0, PLUGIN)
|
||||
|
||||
import mcp_server # noqa: E402
|
||||
import harvest_devin as hw # noqa: E402
|
||||
|
||||
FIXTURES = os.path.join(PLUGIN, "fixtures")
|
||||
|
||||
|
||||
def _read_jsonl(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [json.loads(line) for line in f if line.strip()]
|
||||
|
||||
|
||||
def _find_session_jsonl(out_dir):
|
||||
for root, _dirs, files in os.walk(os.path.join(out_dir, "projects")):
|
||||
for name in files:
|
||||
if name.endswith(".jsonl"):
|
||||
return _read_jsonl(os.path.join(root, name))
|
||||
raise AssertionError("no session jsonl written")
|
||||
|
||||
|
||||
class TestDevinMcpSchema(unittest.TestCase):
|
||||
def test_tools_are_the_sleep_interface(self):
|
||||
names = {t["name"] for t in mcp_server.TOOLS}
|
||||
self.assertEqual(names, {"sleep_status", "sleep_dry_run", "sleep_run",
|
||||
"sleep_adopt", "sleep_harvest"})
|
||||
|
||||
def test_actions_map_to_engine_subcommands(self):
|
||||
expected = {"sleep_status": "status", "sleep_dry_run": "dry-run",
|
||||
"sleep_run": "run", "sleep_adopt": "adopt",
|
||||
"sleep_harvest": "harvest"}
|
||||
for t in mcp_server.TOOLS:
|
||||
self.assertEqual(t["action"], expected[t["name"]])
|
||||
|
||||
def test_backends_in_enum(self):
|
||||
backends = mcp_server._TOOL_SCHEMA["properties"]["backend"]["enum"]
|
||||
for b in ["mock", "claude", "codex"]:
|
||||
self.assertIn(b, backends)
|
||||
|
||||
|
||||
class TestClaudeHomeExpansion(unittest.TestCase):
|
||||
"""Regression: ~ must be expanded even when CLAUDE_HOME comes from the env
|
||||
(the documented mcp-config sets SKILLOPT_DEVIN_CLAUDE_HOME="~/...")."""
|
||||
|
||||
def test_env_tilde_is_expanded(self):
|
||||
os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"] = "~/.skillopt-sleep-devin"
|
||||
try:
|
||||
importlib.reload(mcp_server)
|
||||
self.assertFalse(mcp_server.CLAUDE_HOME.startswith("~"))
|
||||
self.assertEqual(mcp_server.CLAUDE_HOME,
|
||||
os.path.expanduser("~/.skillopt-sleep-devin"))
|
||||
finally:
|
||||
del os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"]
|
||||
importlib.reload(mcp_server)
|
||||
|
||||
|
||||
class TestDevinHarvest(unittest.TestCase):
|
||||
def test_atif_fixture_yields_gradeable_task(self):
|
||||
with tempfile.TemporaryDirectory() as out:
|
||||
n = hw.harvest_devin_transcripts(FIXTURES, out, ["/tmp/proj"])
|
||||
self.assertEqual(n, 1)
|
||||
|
||||
outcomes = _read_jsonl(os.path.join(out, "outcomes.jsonl"))
|
||||
self.assertEqual(len(outcomes), 1)
|
||||
o = outcomes[0]
|
||||
self.assertEqual(o["verifier"], "tests")
|
||||
self.assertTrue(o["success"])
|
||||
self.assertIn("repro", o["reference"])
|
||||
|
||||
# the converted transcript carries the grouping key on the user turn
|
||||
session = _find_session_jsonl(out)
|
||||
user_turn = next(r for r in session if r["type"] == "user")
|
||||
self.assertIn("taskKey", user_turn)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user