Files
github-spec-kit/tests/integrations/test_integration_base_toml.py
Quratulain-bilal 94c7ec288f fix(toml): escape control characters so generated command files parse (#3341)
* fix(toml): escape control characters so generated command files parse

both toml renderers (TomlIntegration._render_toml_string for gemini/tabnine
and CommandRegistrar.render_toml_command for extension/preset commands) wrote
control characters raw into a multiline or basic string. toml forbids literal
control chars (U+0000-U+001F except tab/newline, and U+007F) in every string
form, and a bare CR that is not part of a CRLF pair, so a description or body
containing one produced a .toml file that fails to parse.

route any value with such a character to a fully-escaped basic string that
emits the leftover control chars as \uXXXX. added regression tests that
round-trip through tomllib.

* refactor(toml): centralize control-char escaping in one shared helper

the control-char detection and basic-string escaping added for both toml
renderers were copy-pasted into agents.py and integrations/base.py. move the
two functions into specify_cli/_toml_string.py and have both renderers
delegate to it, so the escaping rules can't drift apart later.

no behavior change; both renderers now reference the same implementation.
2026-07-08 07:42:34 -05:00

598 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Reusable test mixin for standard TomlIntegration subclasses.
Each per-agent test file sets ``KEY``, ``FOLDER``, ``COMMANDS_SUBDIR``,
and ``REGISTRAR_DIR``, then inherits all verification logic from
``TomlIntegrationTests``.
Mirrors ``MarkdownIntegrationTests`` closely — same test structure,
adapted for TOML output format.
"""
import os
import tomllib
import pytest
from specify_cli.integrations import INTEGRATION_REGISTRY, get_integration
from specify_cli.integrations.base import TomlIntegration
from specify_cli.integrations.manifest import IntegrationManifest
class TomlIntegrationTests:
"""Mixin — set class-level constants and inherit these tests.
Required class attrs on subclass::
KEY: str — integration registry key
FOLDER: str — e.g. ".gemini/"
COMMANDS_SUBDIR: str — e.g. "commands"
REGISTRAR_DIR: str — e.g. ".gemini/commands"
"""
KEY: str
FOLDER: str
COMMANDS_SUBDIR: str
REGISTRAR_DIR: str
# -- Registration -----------------------------------------------------
def test_registered(self):
assert self.KEY in INTEGRATION_REGISTRY
assert get_integration(self.KEY) is not None
def test_is_toml_integration(self):
assert isinstance(get_integration(self.KEY), TomlIntegration)
# -- Config -----------------------------------------------------------
def test_config_folder(self):
i = get_integration(self.KEY)
assert i.config["folder"] == self.FOLDER
def test_config_commands_subdir(self):
i = get_integration(self.KEY)
assert i.config["commands_subdir"] == self.COMMANDS_SUBDIR
def test_registrar_config(self):
i = get_integration(self.KEY)
assert i.registrar_config["dir"] == self.REGISTRAR_DIR
assert i.registrar_config["format"] == "toml"
assert i.registrar_config["args"] == "{{args}}"
assert i.registrar_config["extension"] == ".toml"
# -- Setup / teardown -------------------------------------------------
def test_setup_creates_files(self, tmp_path):
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
assert len(created) > 0
cmd_files = [f for f in created if "scripts" not in f.parts]
for f in cmd_files:
assert f.exists()
assert f.name.startswith("speckit.")
assert f.name.endswith(".toml")
def test_setup_writes_to_correct_directory(self, tmp_path):
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
expected_dir = i.commands_dest(tmp_path)
assert expected_dir.exists(), (
f"Expected directory {expected_dir} was not created"
)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) > 0, "No command files were created"
for f in cmd_files:
assert f.resolve().parent == expected_dir.resolve(), (
f"{f} is not under {expected_dir}"
)
def test_templates_are_processed(self, tmp_path):
"""Command files must have placeholders replaced and be valid TOML."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) > 0
for f in cmd_files:
content = f.read_text(encoding="utf-8")
assert "{SCRIPT}" not in content, f"{f.name} has unprocessed {{SCRIPT}}"
assert "__AGENT__" not in content, f"{f.name} has unprocessed __AGENT__"
assert "{ARGS}" not in content, f"{f.name} has unprocessed {{ARGS}}"
assert "__SPECKIT_COMMAND_" not in content, f"{f.name} has unprocessed __SPECKIT_COMMAND_*__"
def test_toml_has_description(self, tmp_path):
"""Every TOML command file should have a description key."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
for f in cmd_files:
content = f.read_text(encoding="utf-8")
assert 'description = "' in content, f"{f.name} missing description key"
def test_toml_has_prompt(self, tmp_path):
"""Every TOML command file should have a prompt key."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
for f in cmd_files:
content = f.read_text(encoding="utf-8")
assert "prompt = " in content, f"{f.name} missing prompt key"
def test_toml_uses_correct_arg_placeholder(self, tmp_path):
"""TOML commands must use {{args}} (from {ARGS} replacement)."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
# At least one file should contain {{args}} from the {ARGS} placeholder
has_args = any("{{args}}" in f.read_text(encoding="utf-8") for f in cmd_files)
assert has_args, "No TOML command file contains {{args}} placeholder"
has_dollar_args = any(
"$ARGUMENTS" in f.read_text(encoding="utf-8") for f in cmd_files
)
assert not has_dollar_args, (
"TOML command still contains $ARGUMENTS instead of {{args}}"
)
@pytest.mark.parametrize(
("frontmatter", "expected"),
[
(
"---\ndescription: |\n First line\n Second line\n---\nBody\n",
"First line\nSecond line\n",
),
(
"---\ndescription: >\n First line\n Second line\n---\nBody\n",
"First line Second line\n",
),
(
"---\ndescription: |-\n First line\n Second line\n---\nBody\n",
"First line\nSecond line",
),
(
"---\ndescription: >-\n First line\n Second line\n---\nBody\n",
"First line Second line",
),
],
)
def test_toml_extract_description_supports_block_scalars(
self, frontmatter, expected
):
assert TomlIntegration._extract_description(frontmatter) == expected
def test_split_frontmatter_ignores_indented_delimiters(self):
content = "---\ndescription: |\n line one\n ---\n line two\n---\nBody\n"
frontmatter, body = TomlIntegration._split_frontmatter(content)
assert "line two" in frontmatter
assert body == "Body\n"
def test_toml_prompt_excludes_frontmatter(self, tmp_path, monkeypatch):
i = get_integration(self.KEY)
template = tmp_path / "sample.md"
template.write_text(
"---\n"
"description: Summary line one\n"
"scripts:\n"
" sh: scripts/bash/example.sh\n"
"---\n"
"Body line one\n"
"Body line two\n",
encoding="utf-8",
)
monkeypatch.setattr(i, "list_command_templates", lambda: [template])
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) == 1
generated = cmd_files[0].read_text(encoding="utf-8")
parsed = tomllib.loads(generated)
assert parsed["description"] == "Summary line one"
assert parsed["prompt"] == "Body line one\nBody line two"
assert "description:" not in parsed["prompt"]
assert "scripts:" not in parsed["prompt"]
assert "---" not in parsed["prompt"]
def test_toml_no_ambiguous_closing_quotes(self, tmp_path, monkeypatch):
"""Multiline body ending with a double quote must not produce an ambiguous TOML multiline-string closing delimiter (#2113)."""
i = get_integration(self.KEY)
template = tmp_path / "sample.md"
template.write_text(
"---\n"
"description: Test\n"
"scripts:\n"
" sh: echo ok\n"
"---\n"
"Check the following:\n"
'- Correct: "Is X clearly specified?"\n',
encoding="utf-8",
)
monkeypatch.setattr(i, "list_command_templates", lambda: [template])
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) == 1
raw = cmd_files[0].read_text(encoding="utf-8")
assert '""""' not in raw, "closing delimiter must not merge with body quote"
assert '"""\n' in raw, "body must use multiline basic string"
parsed = tomllib.loads(raw)
assert parsed["prompt"].endswith('specified?"')
assert not parsed["prompt"].endswith("\n"), (
"parsed value must not gain a trailing newline"
)
def test_toml_triple_double_and_single_quote_ending(self, tmp_path, monkeypatch):
"""Body containing `\"\"\"` and ending with `'` falls back to escaped basic string."""
i = get_integration(self.KEY)
template = tmp_path / "sample.md"
template.write_text(
"---\n"
"description: Test\n"
"scripts:\n"
" sh: echo ok\n"
"---\n"
'Use """triple""" quotes\n'
"and end with 'single'\n",
encoding="utf-8",
)
monkeypatch.setattr(i, "list_command_templates", lambda: [template])
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) == 1
raw = cmd_files[0].read_text(encoding="utf-8")
assert "''''" not in raw, (
"literal string must not produce ambiguous closing quotes"
)
parsed = tomllib.loads(raw)
assert parsed["prompt"].endswith("'single'")
assert '"""triple"""' in parsed["prompt"]
assert not parsed["prompt"].endswith("\n"), (
"parsed value must not gain a trailing newline"
)
def test_toml_closing_delimiter_inline_when_safe(self, tmp_path, monkeypatch):
"""Body NOT ending with `"` keeps closing `\"\"\"` inline (no extra newline)."""
i = get_integration(self.KEY)
template = tmp_path / "sample.md"
template.write_text(
"---\n"
"description: Test\n"
"scripts:\n"
" sh: echo ok\n"
"---\n"
"Line one\n"
"Plain body content\n",
encoding="utf-8",
)
monkeypatch.setattr(i, "list_command_templates", lambda: [template])
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
assert len(cmd_files) == 1
raw = cmd_files[0].read_text(encoding="utf-8")
parsed = tomllib.loads(raw)
assert parsed["prompt"] == "Line one\nPlain body content"
assert raw.rstrip().endswith('content"""'), (
"closing delimiter should be inline when body does not end with a quote"
)
def test_toml_string_escapes_control_characters(self):
"""A value with control chars / a bare CR must render as parseable TOML.
TOML forbids literal control characters (U+0000U+001F except tab and
newline, plus U+007F) in every string form, and a bare CR that is not
part of a CRLF pair. The renderer used to emit these raw into a basic or
``\"\"\"`` multiline string, producing a config file that fails to parse."""
value = "start\x00null\x01ctrl\x1besc\x7fdel\rlone-cr end"
rendered = TomlIntegration._render_toml_string(value)
parsed = tomllib.loads(f"prompt = {rendered}")
assert parsed["prompt"] == value
def test_toml_is_valid(self, tmp_path):
"""Every generated TOML file must parse without errors."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
cmd_files = [f for f in created if "scripts" not in f.parts]
for f in cmd_files:
raw = f.read_bytes()
try:
parsed = tomllib.loads(raw.decode("utf-8"))
except Exception as exc:
raise AssertionError(f"{f.name} is not valid TOML: {exc}") from exc
assert "prompt" in parsed, f"{f.name} parsed TOML has no 'prompt' key"
def test_plan_command_has_no_context_placeholder(self, tmp_path):
"""The generated plan command must not carry a context-file placeholder.
Agent context files are owned entirely by the opt-in agent-context
extension, so the core plan command must not reference one.
"""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
i.setup(tmp_path, m)
plan_file = i.commands_dest(tmp_path) / i.command_filename("plan")
assert plan_file.exists(), f"Plan file {plan_file} not created"
content = plan_file.read_text(encoding="utf-8")
assert "__CONTEXT_FILE__" not in content, (
f"Plan command has unprocessed __CONTEXT_FILE__ placeholder in {plan_file.name}"
)
def test_all_files_tracked_in_manifest(self, tmp_path):
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.setup(tmp_path, m)
for f in created:
rel = f.resolve().relative_to(tmp_path.resolve()).as_posix()
assert rel in m.files, f"{rel} not tracked in manifest"
def test_install_uninstall_roundtrip(self, tmp_path):
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.install(tmp_path, m)
assert len(created) > 0
m.save()
for f in created:
assert f.exists()
removed, skipped = i.uninstall(tmp_path, m)
assert len(removed) == len(created)
assert skipped == []
def test_modified_file_survives_uninstall(self, tmp_path):
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
created = i.install(tmp_path, m)
m.save()
modified_file = created[0]
modified_file.write_text("user modified this", encoding="utf-8")
removed, skipped = i.uninstall(tmp_path, m)
assert modified_file.exists()
assert modified_file in skipped
# -- Context file ownership (extension-owned, opt-in) -----------------
def test_setup_does_not_write_context_section(self, tmp_path):
"""Setup must not create or manage any agent context file — that is
owned entirely by the opt-in agent-context extension."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
i.setup(tmp_path, m)
for path in tmp_path.rglob("*"):
if path.is_file():
text = path.read_text(encoding="utf-8", errors="ignore")
assert "<!-- SPECKIT START -->" not in text, (
f"Setup wrote a managed context section into {path} for {self.KEY}"
)
def test_teardown_leaves_existing_context_file_intact(self, tmp_path):
"""A user-authored context file must survive setup + teardown untouched."""
i = get_integration(self.KEY)
m = IntegrationManifest(self.KEY, tmp_path)
ctx_path = tmp_path / "AGENTS.md"
original = "# My Rules\n\nUser content.\n"
ctx_path.write_text(original, encoding="utf-8")
i.setup(tmp_path, m)
m.save()
i.teardown(tmp_path, m)
assert ctx_path.read_text(encoding="utf-8") == original
# -- CLI integration flag -------------------------------------------------
def test_integration_flag_auto_promotes(self, tmp_path):
from typer.testing import CliRunner
from specify_cli import app
project = tmp_path / f"promote-{self.KEY}"
project.mkdir()
old_cwd = os.getcwd()
try:
os.chdir(project)
runner = CliRunner()
result = runner.invoke(
app,
[
"init",
"--here",
"--integration",
self.KEY,
"--script",
"sh",
"--ignore-agent-tools",
],
catch_exceptions=False,
)
finally:
os.chdir(old_cwd)
assert result.exit_code == 0, f"init --integration {self.KEY} failed: {result.output}"
i = get_integration(self.KEY)
cmd_dir = i.commands_dest(project)
assert cmd_dir.is_dir(), f"--integration {self.KEY} did not create commands directory"
def test_integration_flag_creates_files(self, tmp_path):
from typer.testing import CliRunner
from specify_cli import app
project = tmp_path / f"int-{self.KEY}"
project.mkdir()
old_cwd = os.getcwd()
try:
os.chdir(project)
runner = CliRunner()
result = runner.invoke(
app,
[
"init",
"--here",
"--integration",
self.KEY,
"--script",
"sh",
"--ignore-agent-tools",
],
catch_exceptions=False,
)
finally:
os.chdir(old_cwd)
assert result.exit_code == 0, (
f"init --integration {self.KEY} failed: {result.output}"
)
i = get_integration(self.KEY)
cmd_dir = i.commands_dest(project)
assert cmd_dir.is_dir(), f"Commands directory {cmd_dir} not created"
commands = sorted(cmd_dir.glob("speckit.*.toml"))
assert len(commands) > 0, f"No command files in {cmd_dir}"
# -- Complete file inventory ------------------------------------------
COMMAND_STEMS = [
"analyze",
"clarify",
"constitution",
"converge",
"implement",
"plan",
"checklist",
"specify",
"tasks",
"taskstoissues",
]
def _expected_files(self, script_variant: str) -> list[str]:
"""Build the expected file list for this integration + script variant."""
i = get_integration(self.KEY)
cmd_dir = i.registrar_config["dir"]
files = []
# Command files (.toml)
for stem in self.COMMAND_STEMS:
files.append(f"{cmd_dir}/speckit.{stem}.toml")
# Framework files
files.append(".specify/integration.json")
files.append(".specify/init-options.json")
files.append(f".specify/integrations/{self.KEY}.manifest.json")
files.append(".specify/integrations/speckit.manifest.json")
if script_variant == "sh":
for name in [
"check-prerequisites.sh",
"common.sh",
"create-new-feature.sh",
"setup-plan.sh",
"setup-tasks.sh",
]:
files.append(f".specify/scripts/bash/{name}")
else:
for name in [
"check-prerequisites.ps1",
"common.ps1",
"create-new-feature.ps1",
"setup-plan.ps1",
"setup-tasks.ps1",
]:
files.append(f".specify/scripts/powershell/{name}")
for name in [
"checklist-template.md",
"constitution-template.md",
"plan-template.md",
"spec-template.md",
"tasks-template.md",
]:
files.append(f".specify/templates/{name}")
files.append(".specify/memory/constitution.md")
# Bundled workflow
files.append(".specify/workflows/speckit/workflow.yml")
files.append(".specify/workflows/workflow-registry.json")
return sorted(files)
def test_complete_file_inventory_sh(self, tmp_path):
"""Every file produced by specify init --integration <key> --script sh."""
from typer.testing import CliRunner
from specify_cli import app
project = tmp_path / f"inventory-sh-{self.KEY}"
project.mkdir()
old_cwd = os.getcwd()
try:
os.chdir(project)
result = CliRunner().invoke(
app,
[
"init",
"--here",
"--integration",
self.KEY,
"--script",
"sh",
"--ignore-agent-tools",
],
catch_exceptions=False,
)
finally:
os.chdir(old_cwd)
assert result.exit_code == 0, f"init failed: {result.output}"
actual = sorted(
p.relative_to(project).as_posix() for p in project.rglob("*") if p.is_file() and ".git" not in p.parts
)
expected = self._expected_files("sh")
assert actual == expected, (
f"Missing: {sorted(set(expected) - set(actual))}\n"
f"Extra: {sorted(set(actual) - set(expected))}"
)
def test_complete_file_inventory_ps(self, tmp_path):
"""Every file produced by specify init --integration <key> --script ps."""
from typer.testing import CliRunner
from specify_cli import app
project = tmp_path / f"inventory-ps-{self.KEY}"
project.mkdir()
old_cwd = os.getcwd()
try:
os.chdir(project)
result = CliRunner().invoke(
app,
[
"init",
"--here",
"--integration",
self.KEY,
"--script",
"ps",
"--ignore-agent-tools",
],
catch_exceptions=False,
)
finally:
os.chdir(old_cwd)
assert result.exit_code == 0, f"init failed: {result.output}"
actual = sorted(
p.relative_to(project).as_posix() for p in project.rglob("*") if p.is_file() and ".git" not in p.parts
)
expected = self._expected_files("ps")
assert actual == expected, (
f"Missing: {sorted(set(expected) - set(actual))}\n"
f"Extra: {sorted(set(actual) - set(expected))}"
)