mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-03 16:31:58 +08:00
Completes the #30719 restart-loop defenses. Defenses 1-2 (the _HERMES_GATEWAY guard on `hermes gateway stop|restart` + terminal_tool, and the cron-creation lifecycle filter) already landed on main, but two gaps remained: - The agent's `cronjob` model tool calls cron.jobs.create_job directly, bypassing the hermes_cli.cron.cron_create CLI filter, so lifecycle commands scheduled via the model tool were only blocked at execution time (terminal_tool), not at creation. Moved the filter to a shared cron/lifecycle_guard.py enforced at create_job — the single chokepoint every job-creation path hits (CLI + model tool). Re-exported _contains_gateway_lifecycle_command from hermes_cli.cron so terminal_tool's import keeps working. - No breaker for the auto-resume loop itself. Defenses 1-2 cover the cron/CLI/terminal paths, but any other SIGTERM source (e.g. a raw terminal("launchctl kickstart ai.hermes.gateway")) still triggers the boot->auto-resume->re-run cycle. Added gateway/restart_loop_guard.py: counts restart-interrupted boots in a rolling window (config gateway.restart_loop_guard, default 3 boots / 60s) and skips auto-resume for that boot once tripped. The gateway still comes up and serves real inbound messages; it just stops replaying the session that keeps killing it, putting a human back in the loop. Also tightened the lifecycle regex over main's version: dropped `hermes gateway start` (benign), required the gateway identifier on the launchctl/systemctl branches (so `launchctl unload ai.hermes.update-checker.plist` and `systemctl restart hermes-meta.service` no longer false-positive), added the inverse pkill token order, and fixed the binary-script bypass (decode with errors='replace' instead of swallowing UnicodeDecodeError). The create_job guard resolves relative script paths under HERMES_HOME/scripts the same way the scheduler does, so a bare script name is scanned as the file that actually runs. Design and much of defense-2 originate from PR #33395 (@kshitijk4poor), which itself salvaged #30728 (@SimoKiihamaki). Rebuilt against current main since defenses 1-2 had already landed under different names. Closes #30719. Co-authored-by: SimoKiihamaki <simo.kiihamaki@gmail.com> Co-authored-by: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com>
142 lines
6.2 KiB
Python
142 lines
6.2 KiB
Python
"""Gateway lifecycle guard for cron job creation (#30719).
|
|
|
|
An agent running inside a gateway can schedule a cron job that calls
|
|
``hermes gateway restart`` (or ``launchctl kickstart ai.hermes.gateway``
|
|
or ``systemctl restart hermes-gateway``). When the cron fires, the
|
|
gateway dies, the supervisor (launchd KeepAlive / systemd Restart=)
|
|
revives it, auto-resume picks up the offending session, and the resumed
|
|
turn re-runs the same logic — a SIGTERM-respawn loop every ~10 seconds
|
|
until manually broken.
|
|
|
|
This module rejects cron job specs whose prompt or script contains a
|
|
direct shell-level gateway-lifecycle command. It is enforced at
|
|
``cron.jobs.create_job`` so it fires on every job-creation path: the
|
|
``hermes cron create`` CLI subcommand AND the agent's ``cronjob`` model
|
|
tool (which calls ``create_job`` directly, bypassing the CLI layer).
|
|
|
|
The pattern is intentionally command-shaped: it anchors on a concrete
|
|
command identifier (``hermes gateway``, ``launchctl ... hermes-gateway``,
|
|
``systemctl ... hermes-gateway``, ``pkill`` against the gateway) so it
|
|
cannot fire on prose. A cron ``prompt`` is fed to a future LLM, not a
|
|
shell, so an over-broad substring match on English ("Kong API gateway
|
|
autoscaling and restart behavior") would produce a high false-positive
|
|
rate without preventing the actual foot-gun, which requires a real
|
|
command shape.
|
|
|
|
This is a defence-in-depth layer. ``tools/terminal_tool.py`` already
|
|
blocks these commands at *execution* time when ``_HERMES_GATEWAY=1``, and
|
|
``hermes gateway stop|restart`` refuse to self-target from inside the
|
|
gateway. Blocking at *creation* time as well means the agent gets an
|
|
immediate, informative rejection instead of scheduling a job that will
|
|
only fail (silently) when it fires.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
class GatewayLifecycleBlocked(ValueError):
|
|
"""Raised when a cron job spec contains a gateway-lifecycle command."""
|
|
|
|
|
|
# Shell-level command shapes that target the gateway lifecycle. Each branch
|
|
# is anchored on a concrete command identifier so a match can only fire on
|
|
# actual shell-command-shaped strings, not on prose.
|
|
_GATEWAY_LIFECYCLE_PATTERN = re.compile(
|
|
r"(?i)"
|
|
# Branch A: `hermes gateway restart|stop` — the canonical foot-gun.
|
|
# `start` is intentionally excluded: starting a gateway from inside a
|
|
# gateway is benign (a no-op or "already running" error), and a
|
|
# legitimate cron job might start a sibling profile's gateway.
|
|
r"(?:hermes\s+gateway\s+(?:restart|stop))"
|
|
# Branch B: launchctl ops on a hermes-gateway label. macOS launchd
|
|
# labels look like `ai.hermes.gateway` / `hermes-gateway`. Requiring the
|
|
# gateway identifier prevents blocking unrelated hermes services (e.g.
|
|
# `launchctl unload ai.hermes.update-checker.plist`).
|
|
r"|(?:launchctl\s+(?:kickstart|unload|load|stop|restart)\b[^\n]*\bhermes[.\-]?gateway)"
|
|
# Branch C: systemctl ops on a hermes-gateway unit.
|
|
r"|(?:systemctl\s+(?:-\S+\s+)*(?:restart|stop|start)\b[^\n]*\bhermes[.\-]?gateway)"
|
|
# Branch D: pkill / kill targeting the hermes gateway process. Both
|
|
# token orders because real reproductions show both.
|
|
r"|(?:p?kill\b[^\n]*\bhermes\b[^\n]*\bgateway)"
|
|
r"|(?:p?kill\b[^\n]*\bgateway\b[^\n]*\bhermes)"
|
|
)
|
|
|
|
|
|
def contains_gateway_lifecycle_command(text: str) -> bool:
|
|
"""Return True if *text* contains a gateway lifecycle command pattern."""
|
|
if not text:
|
|
return False
|
|
return bool(_GATEWAY_LIFECYCLE_PATTERN.search(text))
|
|
|
|
|
|
def _resolve_script_path(script_path: str) -> Path:
|
|
"""Resolve a cron ``script`` value the same way the scheduler does.
|
|
|
|
The scheduler (``cron.scheduler``) resolves a bare/relative script path
|
|
under ``<HERMES_HOME>/scripts/`` and only accepts absolute paths as-is.
|
|
We MUST mirror that here so the guard scans the file that will actually
|
|
run — otherwise a job whose script lives at the scheduler's real location
|
|
(``~/.hermes/scripts/restart.sh``) but is passed as the bare name
|
|
``restart.sh`` would read as a nonexistent relative path and silently
|
|
scan prompt-only content, letting the command through.
|
|
"""
|
|
from hermes_constants import get_hermes_home
|
|
|
|
raw = Path(script_path).expanduser()
|
|
if raw.is_absolute():
|
|
return raw
|
|
return get_hermes_home() / "scripts" / raw
|
|
|
|
|
|
def _read_script_for_scanning(script_path: str) -> str:
|
|
"""Read a script file for lifecycle-pattern scanning.
|
|
|
|
Decodes with ``errors="replace"`` so binary or non-UTF-8 content does not
|
|
silently bypass the check — a plain text-mode read raises
|
|
``UnicodeDecodeError`` on such files, and swallowing that error would let
|
|
an attacker hide the command in binary noise. Returns an empty string
|
|
only when the file cannot be read at all.
|
|
"""
|
|
try:
|
|
return _resolve_script_path(script_path).read_bytes().decode(
|
|
"utf-8", errors="replace"
|
|
)
|
|
except OSError:
|
|
return ""
|
|
|
|
|
|
def check_gateway_lifecycle(
|
|
prompt: Optional[str],
|
|
script: Optional[str] = None,
|
|
) -> None:
|
|
"""Raise ``GatewayLifecycleBlocked`` if *prompt* or *script* contains a
|
|
gateway-lifecycle command pattern.
|
|
|
|
``prompt`` is scanned directly. ``script``, when supplied, is read from
|
|
disk and concatenated for the scan. Both are considered together so a
|
|
job cannot slip through by splitting the command across the prompt and
|
|
the script.
|
|
|
|
Callers should let the exception propagate when they want the create to
|
|
fail with a ``ValueError``-shaped error (the agent's ``cronjob`` tool
|
|
surfaces this as a tool error; the CLI prints it in red and exits 1).
|
|
"""
|
|
combined = prompt or ""
|
|
if script:
|
|
script_text = _read_script_for_scanning(script)
|
|
if script_text:
|
|
combined = f"{combined}\n{script_text}"
|
|
|
|
if contains_gateway_lifecycle_command(combined):
|
|
raise GatewayLifecycleBlocked(
|
|
"Blocked: cron job contains a gateway lifecycle command "
|
|
"(restart/stop/kill). This is blocked to prevent agent-driven "
|
|
"SIGTERM-respawn loops under launchd/systemd supervision "
|
|
"(#30719). Run `hermes gateway restart` from a shell outside "
|
|
"the running gateway instead."
|
|
)
|