mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 06:51:16 +08:00
ea01bdcebe
The AIAgent.flush_memories pre-compression save, the gateway _flush_memories_for_session, and everything feeding them are obsolete now that the background memory/skill review handles persistent memory extraction. Problems with flush_memories: - Pre-dates the background review loop. It was the only memory-save path when introduced; the background review now fires every 10 user turns on CLI and gateway alike, which is far more frequent than compression or session reset ever triggered flush. - Blocking and synchronous. Pre-compression flush ran on the live agent before compression, blocking the user-visible response. - Cache-breaking. Flush built a temporary conversation prefix (system prompt + memory-only tool list) that diverged from the live conversation's cached prefix, invalidating prompt caching. The gateway variant spawned a fresh AIAgent with its own clean prompt for each finalized session — still cache-breaking, just in a different process. - Redundant. Background review runs in the live conversation's session context, gets the same content, writes to the same memory store, and doesn't break the cache. Everything flush_memories claimed to preserve is already covered. What this removes: - AIAgent.flush_memories() method (~248 LOC in run_agent.py) - Pre-compression flush call in _compress_context - flush_memories call sites in cli.py (/new + exit) - GatewayRunner._flush_memories_for_session + _async_flush_memories (and the 3 call sites: session expiry watcher, /new, /resume) - 'flush_memories' entry from DEFAULT_CONFIG auxiliary tasks, hermes tools UI task list, auxiliary_client docstrings - _memory_flush_min_turns config + init - #15631's headroom-deduction math in _check_compression_model_feasibility (headroom was only needed because flush dragged the full main-agent system prompt along; the compression summariser sends a single user-role prompt so new_threshold = aux_context is safe again) - The dedicated test files and assertions that exercised flush-specific paths What this renames (with read-time backcompat on sessions.json): - SessionEntry.memory_flushed -> SessionEntry.expiry_finalized. The session-expiry watcher still uses the flag to avoid re-running finalize/eviction on the same expired session; the new name reflects what it now actually gates. from_dict() reads 'expiry_finalized' first, falls back to the legacy 'memory_flushed' key so existing sessions.json files upgrade seamlessly. Supersedes #15631 and #15638. Tested: 383 targeted tests pass across run_agent/, agent/, cli/, and gateway/ session-boundary suites. No behavior regressions — background memory review continues to handle persistent memory extraction on both CLI and gateway.
430 lines
18 KiB
Python
430 lines
18 KiB
Python
"""Tests for named custom provider and 'main' alias resolution in auxiliary_client."""
|
|
|
|
import os
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate(tmp_path, monkeypatch):
|
|
"""Redirect HERMES_HOME and clear module caches."""
|
|
hermes_home = tmp_path / ".hermes"
|
|
hermes_home.mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
|
|
# Write a minimal config so load_config doesn't fail
|
|
(hermes_home / "config.yaml").write_text("model:\n default: test-model\n")
|
|
|
|
|
|
def _write_config(tmp_path, config_dict):
|
|
"""Write a config.yaml to the test HERMES_HOME."""
|
|
import yaml
|
|
config_path = tmp_path / ".hermes" / "config.yaml"
|
|
config_path.write_text(yaml.dump(config_dict))
|
|
|
|
|
|
class TestNormalizeVisionProvider:
|
|
"""_normalize_vision_provider should resolve 'main' to actual main provider."""
|
|
|
|
def test_main_resolves_to_named_custom(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "my-model", "provider": "custom:beans"},
|
|
"custom_providers": [{"name": "beans", "base_url": "http://localhost/v1"}],
|
|
})
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("main") == "custom:beans"
|
|
|
|
def test_main_resolves_to_openrouter(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "anthropic/claude-sonnet-4", "provider": "openrouter"},
|
|
})
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("main") == "openrouter"
|
|
|
|
def test_main_resolves_to_deepseek(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "deepseek-chat", "provider": "deepseek"},
|
|
})
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("main") == "deepseek"
|
|
|
|
def test_main_falls_back_to_custom_when_no_provider(self, tmp_path):
|
|
_write_config(tmp_path, {"model": {"default": "gpt-4o"}})
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("main") == "custom"
|
|
|
|
def test_bare_provider_name_unchanged(self):
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("beans") == "beans"
|
|
assert _normalize_vision_provider("deepseek") == "deepseek"
|
|
|
|
def test_custom_colon_named_provider_preserved(self):
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("custom:beans") == "beans"
|
|
|
|
def test_codex_alias_still_works(self):
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("codex") == "openai-codex"
|
|
|
|
def test_auto_unchanged(self):
|
|
from agent.auxiliary_client import _normalize_vision_provider
|
|
assert _normalize_vision_provider("auto") == "auto"
|
|
assert _normalize_vision_provider(None) == "auto"
|
|
|
|
|
|
class TestResolveProviderClientMainAlias:
|
|
"""resolve_provider_client('main', ...) should resolve to actual main provider."""
|
|
|
|
def test_main_resolves_to_named_custom_provider(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "my-model", "provider": "beans"},
|
|
"custom_providers": [
|
|
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
client, model = resolve_provider_client("main", "override-model")
|
|
assert client is not None
|
|
assert model == "override-model"
|
|
assert "beans.local" in str(client.base_url)
|
|
|
|
def test_main_with_custom_colon_prefix(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "my-model", "provider": "custom:beans"},
|
|
"custom_providers": [
|
|
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
client, model = resolve_provider_client("main", "test")
|
|
assert client is not None
|
|
assert "beans.local" in str(client.base_url)
|
|
|
|
def test_main_resolves_github_copilot_alias(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "gpt-5.4", "provider": "github-copilot"},
|
|
})
|
|
with (
|
|
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
|
"api_key": "ghu_test_token",
|
|
"base_url": "https://api.githubcopilot.com",
|
|
}),
|
|
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
|
):
|
|
mock_openai.return_value = MagicMock()
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
|
|
client, model = resolve_provider_client("main", "gpt-5.4")
|
|
|
|
assert client is not None
|
|
assert model == "gpt-5.4"
|
|
assert mock_openai.called
|
|
|
|
|
|
class TestResolveProviderClientNamedCustom:
|
|
"""resolve_provider_client should resolve named custom providers directly."""
|
|
|
|
def test_named_custom_provider(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "test-model"},
|
|
"custom_providers": [
|
|
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
client, model = resolve_provider_client("beans", "my-model")
|
|
assert client is not None
|
|
assert model == "my-model"
|
|
assert "beans.local" in str(client.base_url)
|
|
|
|
def test_named_custom_provider_default_model(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "main-model"},
|
|
"custom_providers": [
|
|
{"name": "beans", "base_url": "http://beans.local/v1", "api_key": "k"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
client, model = resolve_provider_client("beans")
|
|
assert client is not None
|
|
# Should use _read_main_model() fallback
|
|
assert model == "main-model"
|
|
|
|
def test_named_custom_no_api_key_uses_fallback(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "test"},
|
|
"custom_providers": [
|
|
{"name": "local", "base_url": "http://localhost:8080/v1"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
client, model = resolve_provider_client("local", "test")
|
|
assert client is not None
|
|
# no-key-required should be used
|
|
|
|
def test_nonexistent_named_custom_falls_through(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "test"},
|
|
"custom_providers": [
|
|
{"name": "beans", "base_url": "http://beans.local/v1"},
|
|
],
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
# "coffee" doesn't exist in custom_providers
|
|
client, model = resolve_provider_client("coffee", "test")
|
|
assert client is None
|
|
|
|
|
|
class TestResolveProviderClientModelNormalization:
|
|
"""Direct-provider auxiliary routing should normalize models like main runtime."""
|
|
|
|
def test_matching_native_prefix_is_stripped_for_main_provider(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
|
})
|
|
with (
|
|
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
|
"api_key": "glm-key",
|
|
"base_url": "https://api.z.ai/api/paas/v4",
|
|
}),
|
|
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
|
):
|
|
mock_openai.return_value = MagicMock()
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
|
|
client, model = resolve_provider_client("main", "zai/glm-5.1")
|
|
|
|
assert client is not None
|
|
assert model == "glm-5.1"
|
|
|
|
def test_non_matching_prefix_is_preserved_for_direct_provider(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
|
})
|
|
with (
|
|
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
|
"api_key": "glm-key",
|
|
"base_url": "https://api.z.ai/api/paas/v4",
|
|
}),
|
|
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
|
):
|
|
mock_openai.return_value = MagicMock()
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
|
|
client, model = resolve_provider_client("zai", "google/gemini-2.5-pro")
|
|
|
|
assert client is not None
|
|
assert model == "google/gemini-2.5-pro"
|
|
|
|
def test_aggregator_vendor_slug_is_preserved(self, monkeypatch):
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "or-key")
|
|
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
|
|
mock_openai.return_value = MagicMock()
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
|
|
client, model = resolve_provider_client(
|
|
"openrouter", "anthropic/claude-sonnet-4.6"
|
|
)
|
|
|
|
assert client is not None
|
|
assert model == "anthropic/claude-sonnet-4.6"
|
|
|
|
|
|
class TestResolveVisionProviderClientModelNormalization:
|
|
"""Vision auto-routing should reuse the same provider-specific normalization."""
|
|
|
|
def test_vision_auto_strips_matching_main_provider_prefix(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "zai/glm-5.1", "provider": "zai"},
|
|
})
|
|
with (
|
|
patch("agent.auxiliary_client._read_nous_auth", return_value=None),
|
|
patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={
|
|
"api_key": "glm-key",
|
|
"base_url": "https://api.z.ai/api/paas/v4",
|
|
}),
|
|
patch("agent.auxiliary_client.OpenAI") as mock_openai,
|
|
):
|
|
mock_openai.return_value = MagicMock()
|
|
from agent.auxiliary_client import resolve_vision_provider_client
|
|
|
|
provider, client, model = resolve_vision_provider_client()
|
|
|
|
assert provider == "zai"
|
|
assert client is not None
|
|
assert model == "glm-5v-turbo" # zai has dedicated vision model in _PROVIDER_VISION_MODELS
|
|
|
|
|
|
class TestVisionPathApiMode:
|
|
"""Vision path should propagate api_mode to _get_cached_client."""
|
|
|
|
def test_explicit_provider_passes_api_mode(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"model": {"default": "test-model"},
|
|
"auxiliary": {"vision": {"api_mode": "chat_completions"}},
|
|
})
|
|
with patch("agent.auxiliary_client._get_cached_client") as mock_gcc:
|
|
mock_gcc.return_value = (MagicMock(), "test-model")
|
|
from agent.auxiliary_client import resolve_vision_provider_client
|
|
|
|
provider, client, model = resolve_vision_provider_client(provider="deepseek")
|
|
|
|
mock_gcc.assert_called_once()
|
|
_, kwargs = mock_gcc.call_args
|
|
assert kwargs.get("api_mode") == "chat_completions"
|
|
|
|
|
|
class TestProvidersDictApiModeAnthropicMessages:
|
|
"""Regression guard for #15033.
|
|
|
|
Named providers declared under the ``providers:`` dict with
|
|
``api_mode: anthropic_messages`` must route auxiliary calls through
|
|
the Anthropic Messages API (via AnthropicAuxiliaryClient), not
|
|
through an OpenAI chat-completions client.
|
|
|
|
The bug had two halves: the providers-dict branch of
|
|
``_get_named_custom_provider`` dropped the ``api_mode`` field, and
|
|
``resolve_provider_client``'s named-custom branch never read it.
|
|
"""
|
|
|
|
def test_providers_dict_propagates_api_mode(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"myrelay": {
|
|
"name": "myrelay",
|
|
"base_url": "https://example-relay.test/anthropic",
|
|
"key_env": "MYRELAY_API_KEY",
|
|
"api_mode": "anthropic_messages",
|
|
"default_model": "claude-opus-4-7",
|
|
},
|
|
},
|
|
})
|
|
from hermes_cli.runtime_provider import _get_named_custom_provider
|
|
entry = _get_named_custom_provider("myrelay")
|
|
assert entry is not None
|
|
assert entry.get("api_mode") == "anthropic_messages"
|
|
assert entry.get("base_url") == "https://example-relay.test/anthropic"
|
|
assert entry.get("api_key") == "sk-test"
|
|
|
|
def test_providers_dict_invalid_api_mode_is_dropped(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"weird": {
|
|
"name": "weird",
|
|
"base_url": "https://example.test",
|
|
"api_mode": "bogus_nonsense",
|
|
"default_model": "x",
|
|
},
|
|
},
|
|
})
|
|
from hermes_cli.runtime_provider import _get_named_custom_provider
|
|
entry = _get_named_custom_provider("weird")
|
|
assert entry is not None
|
|
assert "api_mode" not in entry
|
|
|
|
def test_providers_dict_without_api_mode_is_unchanged(self, tmp_path):
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"localchat": {
|
|
"name": "localchat",
|
|
"base_url": "http://127.0.0.1:1234/v1",
|
|
"api_key": "local-key",
|
|
"default_model": "llama-3",
|
|
},
|
|
},
|
|
})
|
|
from hermes_cli.runtime_provider import _get_named_custom_provider
|
|
entry = _get_named_custom_provider("localchat")
|
|
assert entry is not None
|
|
assert "api_mode" not in entry
|
|
|
|
def test_resolve_provider_client_returns_anthropic_client(self, tmp_path, monkeypatch):
|
|
"""Named custom provider with api_mode=anthropic_messages must
|
|
route through AnthropicAuxiliaryClient."""
|
|
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"myrelay": {
|
|
"name": "myrelay",
|
|
"base_url": "https://example-relay.test/anthropic",
|
|
"key_env": "MYRELAY_API_KEY",
|
|
"api_mode": "anthropic_messages",
|
|
"default_model": "claude-opus-4-7",
|
|
},
|
|
},
|
|
})
|
|
from agent.auxiliary_client import (
|
|
resolve_provider_client,
|
|
AnthropicAuxiliaryClient,
|
|
AsyncAnthropicAuxiliaryClient,
|
|
)
|
|
sync_client, sync_model = resolve_provider_client("myrelay", async_mode=False)
|
|
assert isinstance(sync_client, AnthropicAuxiliaryClient), (
|
|
f"expected AnthropicAuxiliaryClient, got {type(sync_client).__name__}"
|
|
)
|
|
assert sync_model == "claude-opus-4-7"
|
|
|
|
async_client, async_model = resolve_provider_client("myrelay", async_mode=True)
|
|
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient), (
|
|
f"expected AsyncAnthropicAuxiliaryClient, got {type(async_client).__name__}"
|
|
)
|
|
assert async_model == "claude-opus-4-7"
|
|
|
|
def test_aux_task_override_routes_named_provider_to_anthropic(self, tmp_path, monkeypatch):
|
|
"""The full chain: auxiliary.<task>.provider: myrelay with
|
|
api_mode anthropic_messages must produce an Anthropic client."""
|
|
monkeypatch.setenv("MYRELAY_API_KEY", "sk-test")
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"myrelay": {
|
|
"name": "myrelay",
|
|
"base_url": "https://example-relay.test/anthropic",
|
|
"key_env": "MYRELAY_API_KEY",
|
|
"api_mode": "anthropic_messages",
|
|
"default_model": "claude-opus-4-7",
|
|
},
|
|
},
|
|
"auxiliary": {
|
|
"compression": {
|
|
"provider": "myrelay",
|
|
"model": "claude-sonnet-4.6",
|
|
},
|
|
},
|
|
"model": {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"},
|
|
})
|
|
from agent.auxiliary_client import (
|
|
get_async_text_auxiliary_client,
|
|
get_text_auxiliary_client,
|
|
AnthropicAuxiliaryClient,
|
|
AsyncAnthropicAuxiliaryClient,
|
|
)
|
|
async_client, async_model = get_async_text_auxiliary_client("compression")
|
|
assert isinstance(async_client, AsyncAnthropicAuxiliaryClient)
|
|
assert async_model == "claude-sonnet-4.6"
|
|
|
|
sync_client, sync_model = get_text_auxiliary_client("compression")
|
|
assert isinstance(sync_client, AnthropicAuxiliaryClient)
|
|
assert sync_model == "claude-sonnet-4.6"
|
|
|
|
def test_provider_without_api_mode_still_uses_openai(self, tmp_path):
|
|
"""Named providers that don't declare api_mode should still go
|
|
through the plain OpenAI-wire path (no regression)."""
|
|
_write_config(tmp_path, {
|
|
"providers": {
|
|
"localchat": {
|
|
"name": "localchat",
|
|
"base_url": "http://127.0.0.1:1234/v1",
|
|
"api_key": "local-key",
|
|
"default_model": "llama-3",
|
|
},
|
|
},
|
|
})
|
|
from agent.auxiliary_client import resolve_provider_client
|
|
from openai import OpenAI, AsyncOpenAI
|
|
sync_client, _ = resolve_provider_client("localchat", async_mode=False)
|
|
# sync returns the raw OpenAI client
|
|
assert isinstance(sync_client, OpenAI)
|
|
async_client, _ = resolve_provider_client("localchat", async_mode=True)
|
|
assert isinstance(async_client, AsyncOpenAI)
|