Files
hermes-agent/tests/hermes_cli/test_setup.py

495 lines
17 KiB
Python
Raw Normal View History

"""Tests for setup.py configuration flows."""
import json
import sys
import types
import pytest
from hermes_cli.auth import get_active_provider
from hermes_cli.config import load_config, save_config
from hermes_cli import setup as setup_mod
from hermes_cli.setup import setup_model_provider
def _maybe_keep_current_tts(question, choices):
if question != "Select TTS provider:":
return None
assert choices[-1].startswith("Keep current (")
return len(choices) - 1
def _clear_provider_env(monkeypatch):
for key in (
"NOUS_API_KEY",
"OPENROUTER_API_KEY",
"OPENAI_BASE_URL",
"OPENAI_API_KEY",
"LLM_MODEL",
):
monkeypatch.delenv(key, raising=False)
def _stub_tts(monkeypatch):
"""Stub out TTS prompts so setup_model_provider doesn't block."""
monkeypatch.setattr("hermes_cli.setup.prompt_choice", lambda q, c, d=0: (
_maybe_keep_current_tts(q, c) if _maybe_keep_current_tts(q, c) is not None
else d
))
monkeypatch.setattr("hermes_cli.setup.prompt_yes_no", lambda *a, **kw: False)
def _write_model_config(tmp_path, provider, base_url="", model_name="test-model"):
"""Simulate what a _model_flow_* function writes to disk."""
cfg = load_config()
m = cfg.get("model")
if not isinstance(m, dict):
m = {"default": m} if m else {}
cfg["model"] = m
m["provider"] = provider
if base_url:
m["base_url"] = base_url
if model_name:
m["default"] = model_name
save_config(cfg)
def test_setup_delegates_to_select_provider_and_model(tmp_path, monkeypatch):
"""setup_model_provider calls select_provider_and_model and syncs config."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
def fake_select():
_write_model_config(tmp_path, "custom", "http://localhost:11434/v1", "qwen3.5:32b")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
reloaded = load_config()
assert isinstance(reloaded["model"], dict)
assert reloaded["model"]["provider"] == "custom"
assert reloaded["model"]["base_url"] == "http://localhost:11434/v1"
assert reloaded["model"]["default"] == "qwen3.5:32b"
def test_setup_syncs_openrouter_from_disk(tmp_path, monkeypatch):
"""When select_provider_and_model saves OpenRouter config to disk,
the wizard's config dict picks it up."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
assert isinstance(config.get("model"), str) # fresh install
def fake_select():
_write_model_config(tmp_path, "openrouter", model_name="anthropic/claude-opus-4.6")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
reloaded = load_config()
assert isinstance(reloaded["model"], dict)
assert reloaded["model"]["provider"] == "openrouter"
def test_setup_syncs_nous_from_disk(tmp_path, monkeypatch):
"""Nous OAuth writes config to disk; wizard config dict must pick it up."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
def fake_select():
_write_model_config(tmp_path, "nous", "https://inference.example.com/v1", "gemini-3-flash")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
feat: overhaul context length detection with models.dev and provider-aware resolution (#2158) Replace the fragile hardcoded context length system with a multi-source resolution chain that correctly identifies context windows per provider. Key changes: - New agent/models_dev.py: Fetches and caches the models.dev registry (3800+ models across 100+ providers with per-provider context windows). In-memory cache (1hr TTL) + disk cache for cold starts. - Rewritten get_model_context_length() resolution chain: 0. Config override (model.context_length) 1. Custom providers per-model context_length 2. Persistent disk cache 3. Endpoint /models (local servers) 4. Anthropic /v1/models API (max_input_tokens, API-key only) 5. OpenRouter live API (existing, unchanged) 6. Nous suffix-match via OpenRouter (dot/dash normalization) 7. models.dev registry lookup (provider-aware) 8. Thin hardcoded defaults (broad family patterns) 9. 128K fallback (was 2M) - Provider-aware context: same model now correctly resolves to different context windows per provider (e.g. claude-opus-4.6: 1M on Anthropic, 128K on GitHub Copilot). Provider name flows through ContextCompressor. - DEFAULT_CONTEXT_LENGTHS shrunk from 80+ entries to ~16 broad patterns. models.dev replaces the per-model hardcoding. - CONTEXT_PROBE_TIERS changed from [2M, 1M, 512K, 200K, 128K, 64K, 32K] to [128K, 64K, 32K, 16K, 8K]. Unknown models no longer start at 2M. - hermes model: prompts for context_length when configuring custom endpoints. Supports shorthand (32k, 128K). Saved to custom_providers per-model config. - custom_providers schema extended with optional models dict for per-model context_length (backward compatible). - Nous Portal: suffix-matches bare IDs (claude-opus-4-6) against OpenRouter's prefixed IDs (anthropic/claude-opus-4.6) with dot/dash normalization. Handles all 15 current Nous models. - Anthropic direct: queries /v1/models for max_input_tokens. Only works with regular API keys (sk-ant-api*), not OAuth tokens. Falls through to models.dev for OAuth users. Tests: 5574 passed (18 new tests for models_dev + updated probe tiers) Docs: Updated configuration.md context length section, AGENTS.md Co-authored-by: Test <test@test.com>
2026-03-20 06:04:33 -07:00
reloaded = load_config()
assert isinstance(reloaded["model"], dict)
assert reloaded["model"]["provider"] == "nous"
assert reloaded["model"]["base_url"] == "https://inference.example.com/v1"
def test_setup_custom_providers_synced(tmp_path, monkeypatch):
"""custom_providers written by select_provider_and_model must survive."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
def fake_select():
_write_model_config(tmp_path, "custom", "http://localhost:8080/v1", "llama3")
cfg = load_config()
cfg["custom_providers"] = [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
save_config(cfg)
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
feat: overhaul context length detection with models.dev and provider-aware resolution (#2158) Replace the fragile hardcoded context length system with a multi-source resolution chain that correctly identifies context windows per provider. Key changes: - New agent/models_dev.py: Fetches and caches the models.dev registry (3800+ models across 100+ providers with per-provider context windows). In-memory cache (1hr TTL) + disk cache for cold starts. - Rewritten get_model_context_length() resolution chain: 0. Config override (model.context_length) 1. Custom providers per-model context_length 2. Persistent disk cache 3. Endpoint /models (local servers) 4. Anthropic /v1/models API (max_input_tokens, API-key only) 5. OpenRouter live API (existing, unchanged) 6. Nous suffix-match via OpenRouter (dot/dash normalization) 7. models.dev registry lookup (provider-aware) 8. Thin hardcoded defaults (broad family patterns) 9. 128K fallback (was 2M) - Provider-aware context: same model now correctly resolves to different context windows per provider (e.g. claude-opus-4.6: 1M on Anthropic, 128K on GitHub Copilot). Provider name flows through ContextCompressor. - DEFAULT_CONTEXT_LENGTHS shrunk from 80+ entries to ~16 broad patterns. models.dev replaces the per-model hardcoding. - CONTEXT_PROBE_TIERS changed from [2M, 1M, 512K, 200K, 128K, 64K, 32K] to [128K, 64K, 32K, 16K, 8K]. Unknown models no longer start at 2M. - hermes model: prompts for context_length when configuring custom endpoints. Supports shorthand (32k, 128K). Saved to custom_providers per-model config. - custom_providers schema extended with optional models dict for per-model context_length (backward compatible). - Nous Portal: suffix-matches bare IDs (claude-opus-4-6) against OpenRouter's prefixed IDs (anthropic/claude-opus-4.6) with dot/dash normalization. Handles all 15 current Nous models. - Anthropic direct: queries /v1/models for max_input_tokens. Only works with regular API keys (sk-ant-api*), not OAuth tokens. Falls through to models.dev for OAuth users. Tests: 5574 passed (18 new tests for models_dev + updated probe tiers) Docs: Updated configuration.md context length section, AGENTS.md Co-authored-by: Test <test@test.com>
2026-03-20 06:04:33 -07:00
reloaded = load_config()
assert reloaded.get("custom_providers") == [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
2026-03-13 21:06:06 -07:00
def test_setup_gateway_skips_service_install_when_systemctl_missing(monkeypatch, capsys):
env = {
"TELEGRAM_BOT_TOKEN": "",
"TELEGRAM_HOME_CHANNEL": "",
"DISCORD_BOT_TOKEN": "",
"DISCORD_HOME_CHANNEL": "",
"SLACK_BOT_TOKEN": "",
"SLACK_HOME_CHANNEL": "",
"MATRIX_HOMESERVER": "https://matrix.example.com",
"MATRIX_USER_ID": "@alice:example.com",
"MATRIX_PASSWORD": "",
"MATRIX_ACCESS_TOKEN": "token",
"BLUEBUBBLES_SERVER_URL": "",
"BLUEBUBBLES_HOME_CHANNEL": "",
"WHATSAPP_ENABLED": "",
"WEBHOOK_ENABLED": "",
}
monkeypatch.setattr(setup_mod, "get_env_value", lambda key: env.get(key, ""))
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *args, **kwargs: False)
monkeypatch.setattr("platform.system", lambda: "Linux")
import hermes_cli.gateway as gateway_mod
monkeypatch.setattr(gateway_mod, "supports_systemd_services", lambda: False)
monkeypatch.setattr(gateway_mod, "is_macos", lambda: False)
monkeypatch.setattr(gateway_mod, "_is_service_installed", lambda: False)
monkeypatch.setattr(gateway_mod, "_is_service_running", lambda: False)
setup_mod.setup_gateway({})
out = capsys.readouterr().out
assert "Messaging platforms configured!" in out
assert "Start the gateway to bring your bots online:" in out
assert "hermes gateway" in out
def test_setup_gateway_in_container_shows_docker_guidance(monkeypatch, capsys):
"""setup_gateway() in a Docker container shows Docker-specific restart instructions."""
env = {
"TELEGRAM_BOT_TOKEN": "",
"TELEGRAM_HOME_CHANNEL": "",
"DISCORD_BOT_TOKEN": "",
"DISCORD_HOME_CHANNEL": "",
"SLACK_BOT_TOKEN": "",
"SLACK_HOME_CHANNEL": "",
"MATRIX_HOMESERVER": "https://matrix.example.com",
"MATRIX_USER_ID": "@alice:example.com",
"MATRIX_PASSWORD": "",
"MATRIX_ACCESS_TOKEN": "token",
"BLUEBUBBLES_SERVER_URL": "",
"BLUEBUBBLES_HOME_CHANNEL": "",
"WHATSAPP_ENABLED": "",
"WEBHOOK_ENABLED": "",
}
monkeypatch.setattr(setup_mod, "get_env_value", lambda key: env.get(key, ""))
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *args, **kwargs: False)
monkeypatch.setattr("platform.system", lambda: "Linux")
import hermes_cli.gateway as gateway_mod
monkeypatch.setattr(gateway_mod, "supports_systemd_services", lambda: False)
monkeypatch.setattr(gateway_mod, "is_macos", lambda: False)
monkeypatch.setattr(gateway_mod, "_is_service_installed", lambda: False)
monkeypatch.setattr(gateway_mod, "_is_service_running", lambda: False)
# Patch is_container at the import location in setup.py
import hermes_constants
monkeypatch.setattr(hermes_constants, "is_container", lambda: True)
setup_mod.setup_gateway({})
out = capsys.readouterr().out
assert "Messaging platforms configured!" in out
assert "docker" in out.lower() or "Docker" in out
assert "restart" in out.lower()
def test_setup_syncs_custom_provider_removal_from_disk(tmp_path, monkeypatch):
"""Removing the last custom provider in model setup should persist."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
config["custom_providers"] = [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
save_config(config)
def fake_select():
cfg = load_config()
cfg["model"] = {"provider": "openrouter", "default": "anthropic/claude-opus-4.6"}
cfg["custom_providers"] = []
save_config(cfg)
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
save_config(config)
reloaded = load_config()
assert reloaded.get("custom_providers") == []
def test_setup_cancel_preserves_existing_config(tmp_path, monkeypatch):
"""When the user cancels provider selection, existing config is preserved."""
2026-03-13 21:06:06 -07:00
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
# Pre-set a provider
_write_model_config(tmp_path, "openrouter", model_name="gpt-4o")
2026-03-13 21:06:06 -07:00
config = load_config()
assert config["model"]["provider"] == "openrouter"
2026-03-13 21:06:06 -07:00
def fake_select():
pass # user cancelled — nothing written to disk
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
2026-03-13 21:06:06 -07:00
setup_model_provider(config)
save_config(config)
reloaded = load_config()
assert isinstance(reloaded["model"], dict)
assert reloaded["model"]["provider"] == "openrouter"
assert reloaded["model"]["default"] == "gpt-4o"
def test_setup_exception_in_select_gracefully_handled(tmp_path, monkeypatch):
"""If select_provider_and_model raises, setup continues with existing config."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
def fake_select():
raise RuntimeError("something broke")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
# Should not raise
setup_model_provider(config)
def test_setup_keyboard_interrupt_gracefully_handled(tmp_path, monkeypatch):
"""KeyboardInterrupt during provider selection is handled."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
_stub_tts(monkeypatch)
config = load_config()
def fake_select():
raise KeyboardInterrupt()
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
2026-03-13 21:06:06 -07:00
def test_select_provider_and_model_warns_if_named_custom_provider_disappears(
tmp_path, monkeypatch, capsys
):
"""If a saved custom provider is deleted mid-selection, show a warning instead of silently doing nothing."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
_clear_provider_env(monkeypatch)
cfg = load_config()
cfg["custom_providers"] = [{"name": "Local", "base_url": "http://localhost:8080/v1"}]
save_config(cfg)
def fake_prompt_provider_choice(choices, default=0):
current = load_config()
current["custom_providers"] = []
save_config(current)
return next(i for i, label in enumerate(choices) if label.startswith("Local (localhost:8080/v1)"))
monkeypatch.setattr("hermes_cli.auth.resolve_provider", lambda provider: None)
monkeypatch.setattr("hermes_cli.main._prompt_provider_choice", fake_prompt_provider_choice)
monkeypatch.setattr(
"hermes_cli.main._model_flow_named_custom",
lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("named custom flow should not run")),
)
from hermes_cli.main import select_provider_and_model
select_provider_and_model()
out = capsys.readouterr().out
assert "selected saved custom provider is no longer available" in out
def test_codex_setup_uses_runtime_access_token_for_live_model_list(tmp_path, monkeypatch):
"""Codex model list fetching uses the runtime access token."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
2026-03-13 21:06:06 -07:00
monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key")
_clear_provider_env(monkeypatch)
2026-03-13 21:06:06 -07:00
monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key")
config = load_config()
_stub_tts(monkeypatch)
def fake_select():
_write_model_config(tmp_path, "openai-codex", "https://api.openai.com/v1", "gpt-4o")
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select)
setup_model_provider(config)
2026-03-13 21:06:06 -07:00
save_config(config)
2026-03-13 21:06:06 -07:00
reloaded = load_config()
assert isinstance(reloaded["model"], dict)
assert reloaded["model"]["provider"] == "openai-codex"
def test_modal_setup_can_use_nous_subscription_without_modal_creds(tmp_path, monkeypatch, capsys):
monkeypatch.setenv("HERMES_ENABLE_NOUS_MANAGED_TOOLS", "1")
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
config = load_config()
def fake_prompt_choice(question, choices, default=0):
if question == "Select terminal backend:":
return 2
if question == "Select how Modal execution should be billed:":
return 0
raise AssertionError(f"Unexpected prompt_choice call: {question}")
def fake_prompt(message, *args, **kwargs):
assert "Modal Token" not in message
raise AssertionError(f"Unexpected prompt call: {message}")
monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice)
monkeypatch.setattr("hermes_cli.setup.prompt", fake_prompt)
monkeypatch.setattr("hermes_cli.setup._prompt_container_resources", lambda config: None)
monkeypatch.setattr(
"hermes_cli.setup.get_nous_subscription_features",
lambda config: type("Features", (), {"nous_auth_present": True})(),
)
monkeypatch.setitem(
sys.modules,
"tools.managed_tool_gateway",
types.SimpleNamespace(
is_managed_tool_gateway_ready=lambda vendor: vendor == "modal",
resolve_managed_tool_gateway=lambda vendor: None,
),
)
from hermes_cli.setup import setup_terminal_backend
setup_terminal_backend(config)
out = capsys.readouterr().out
assert config["terminal"]["backend"] == "modal"
assert config["terminal"]["modal_mode"] == "managed"
assert "bill to your subscription" in out
def test_modal_setup_persists_direct_mode_when_user_chooses_their_own_account(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_ENABLE_NOUS_MANAGED_TOOLS", "1")
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.delenv("MODAL_TOKEN_ID", raising=False)
monkeypatch.delenv("MODAL_TOKEN_SECRET", raising=False)
config = load_config()
def fake_prompt_choice(question, choices, default=0):
if question == "Select terminal backend:":
return 2
if question == "Select how Modal execution should be billed:":
return 1
raise AssertionError(f"Unexpected prompt_choice call: {question}")
prompt_values = iter(["token-id", "token-secret", ""])
monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice)
monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: next(prompt_values))
monkeypatch.setattr("hermes_cli.setup._prompt_container_resources", lambda config: None)
monkeypatch.setattr(
"hermes_cli.setup.get_nous_subscription_features",
lambda config: type("Features", (), {"nous_auth_present": True})(),
)
monkeypatch.setitem(
sys.modules,
"tools.managed_tool_gateway",
types.SimpleNamespace(
is_managed_tool_gateway_ready=lambda vendor: vendor == "modal",
resolve_managed_tool_gateway=lambda vendor: None,
),
)
monkeypatch.setitem(sys.modules, "swe_rex", object())
from hermes_cli.setup import setup_terminal_backend
setup_terminal_backend(config)
assert config["terminal"]["backend"] == "modal"
assert config["terminal"]["modal_mode"] == "direct"
def test_resolve_hermes_chat_argv_prefers_which(monkeypatch):
from hermes_cli import setup as setup_mod
monkeypatch.setattr(setup_mod.shutil, "which", lambda name: "/usr/local/bin/hermes" if name == "hermes" else None)
assert setup_mod._resolve_hermes_chat_argv() == ["/usr/local/bin/hermes", "chat"]
def test_resolve_hermes_chat_argv_falls_back_to_module(monkeypatch):
from hermes_cli import setup as setup_mod
monkeypatch.setattr(setup_mod.shutil, "which", lambda _name: None)
monkeypatch.setattr(setup_mod.importlib.util, "find_spec", lambda name: object() if name == "hermes_cli" else None)
assert setup_mod._resolve_hermes_chat_argv() == [sys.executable, "-m", "hermes_cli.main", "chat"]
def test_offer_launch_chat_execs_fresh_process(monkeypatch):
from hermes_cli import setup as setup_mod
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
monkeypatch.setattr(setup_mod, "_resolve_hermes_chat_argv", lambda: ["/usr/local/bin/hermes", "chat"])
exec_calls = []
def fake_execvp(path, argv):
exec_calls.append((path, argv))
raise SystemExit(0)
monkeypatch.setattr(setup_mod.os, "execvp", fake_execvp)
with pytest.raises(SystemExit):
setup_mod._offer_launch_chat()
assert exec_calls == [("/usr/local/bin/hermes", ["/usr/local/bin/hermes", "chat"])]
def test_offer_launch_chat_manual_fallback_when_unresolvable(monkeypatch, capsys):
from hermes_cli import setup as setup_mod
monkeypatch.setattr(setup_mod, "prompt_yes_no", lambda *_args, **_kwargs: True)
monkeypatch.setattr(setup_mod, "_resolve_hermes_chat_argv", lambda: None)
setup_mod._offer_launch_chat()
captured = capsys.readouterr()
assert "Run 'hermes chat' manually" in captured.out