github/microsoft-SkillOpt
1
0
Fork 0
mirror of https://github.com/microsoft/SkillOpt.git synced 2026-07-07 16:40:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(sleep): isolate claude CLI calls; concrete+override-aware reflect; honor hard constraints

Browse Source 
Critical correctness fix found by debugging the thorough-analyst failure:

* `claude -p` was running with the AMBIENT Claude Code project context (the
  repo's CLAUDE.md, installed skills, tools). The optimizer/target calls were
  polluted — reflect once replied with a list of the user's installed skills
  instead of JSON edits. Now ClaudeCliBackend._call runs ISOLATED: a clean temp
  cwd, --disallowedTools '*', --exclude-dynamic-system-prompt-sections. This is
  essential for the backend to be trustworthy and reproducible.

* reflect prompt: translate failing rule-judge criteria into plain English
  (max_chars=1200 -> "the ENTIRE response must be at most 1200 characters") and
  require CONCRETE, verbatim thresholds in proposed rules (not "respect limits").

* attempt prompt: treat the Learned-preferences block as HARD CONSTRAINTS that
  override earlier conflicting skill text.

Earlier Claude results predate this fix and are being re-validated clean; the
Codex backend was never affected (it runs in its own exec context).

Co-Authored-By: Claude Opus 4 <noreply@anthropic.com>
This commit is contained in:
Yifan Yang
2026-06-08 14:31:51 +00:00
parent 233b619555
commit defb4566ea
1 changed files with 54 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
skillopt/sleep/backend.py
View File

@@ -253,7 +253,12 @@ class CliBackend(Backend):
def attempt(self, task: TaskRecord, skill: str, memory: str) -> str:
prompt = (
"You are completing a recurring task for a user. Apply the skill and "
"memory rules EXACTLY, including any output-format requirements.\n\n"
"memory rules EXACTLY, including any output-format requirements. If the "
"skill contains a 'Learned preferences' block, treat those rules as "
"HARD CONSTRAINTS that OVERRIDE anything earlier in the skill they "
"conflict with (e.g. an explicit length limit overrides 'be "
"exhaustive'). Satisfy every such constraint even at the cost of "
"brevity or detail.\n\n"
f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n"
f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n"
"Return ONLY the final answer text, nothing else."
@@ -319,11 +324,31 @@ class CliBackend(Backend):
part = part.strip()
if part:
crit[part] += 1
def _explain(c: str) -> str:
# translate an "op=arg" criterion into a plain-English requirement
if "=" in c:
op, _, arg = c.partition("=")
op = op.strip(); arg = arg.strip()
if op == "max_chars":
return f"the ENTIRE response must be at most {arg} characters long"
if op == "min_chars":
return f"the response must be at least {arg} characters long"
if op == "section_present":
return f"the response must contain a section/heading titled '{arg}'"
if op == "regex":
return f"the response must match the pattern /{arg}/ (e.g. include that label)"
if op == "contains":
return f"the response must contain the text '{arg}'"
if op == "tool_called":
return f"the agent must actually call the '{arg}' tool"
return c
criteria_text = ""
if crit:
criteria_text = (
"\n# Exact criteria the outputs are FAILING (fix these directly)\n"
+ "\n".join(f"- {c} (failed {n}x)" for c, n in crit.most_common())
+ "\n".join(f"- {_explain(c)} [{c}, failed {n}x]" for c, n in crit.most_common())
)
prompt = (
"You are SkillOpt's optimizer. The agent keeps failing the recurring "
@@ -332,12 +357,16 @@ class CliBackend(Backend):
"GENERAL, reusable rule or preference (never task-specific, never an "
"answer to a single task). If exact failing criteria are listed, your "
"edits MUST make future outputs satisfy every one of them.\n"
"BE CONCRETE: quote the exact threshold, section name, or format from "
"the criteria verbatim in your rule (e.g. write 'keep the entire "
"response under 1200 characters', NOT 'respect length limits'). Vague "
"rules do not change behavior; specific numeric/structural rules do.\n"
"IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; "
"you CANNOT delete the existing instructions above. If the current "
f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' "
"but outputs must be under a character limit), write an explicit, "
"forceful OVERRIDE rule that says it supersedes the conflicting "
"instruction. "
"forceful OVERRIDE rule stating it supersedes the conflicting "
"instruction, and put the hard requirement first.\n"
'Return ONLY a JSON array: '
'[{"op":"add|replace|delete","content":"<rule>","anchor":"<text to replace/delete, optional>","rationale":"<why>"}].\n\n'
f"# Current {target}\n{cur_doc}\n"
@@ -381,14 +410,33 @@ class ClaudeCliBackend(CliBackend):
self.claude_path = claude_path
def _call(self, prompt: str, *, max_tokens: int = 1024) -> str:
cmd = [self.claude_path, "-p", "--output-format", "text"]
# Run ISOLATED: a clean temp cwd so the ambient project's CLAUDE.md /
# skills / tools do not leak into the optimizer/target call, no tools,
# and per-machine dynamic system-prompt sections excluded. Without this,
# `claude -p` answers with full Claude Code context and ignores our
# prompt (e.g. it lists the user's installed skills).
import tempfile
cmd = [
self.claude_path, "-p", "--output-format", "text",
"--disallowedTools", "*",
"--exclude-dynamic-system-prompt-sections",
]
if self.model:
cmd += ["--model", self.model]
cmd += ["--", prompt]
clean_cwd = tempfile.mkdtemp(prefix="skillopt_sleep_claude_")
try:
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=self.timeout)
proc = subprocess.run(
cmd, capture_output=True, text=True, timeout=self.timeout, cwd=clean_cwd,
)
except Exception:
return ""
finally:
try:
import shutil
shutil.rmtree(clean_cwd, ignore_errors=True)
except Exception:
pass
return (proc.stdout or "").strip()