hermes-agent/tests/tools/test_skill_usage.py

"""Tests for tools/skill_usage.py — sidecar telemetry + provenance filtering."""

import json
import os
from pathlib import Path

import pytest


@pytest.fixture
def skills_home(tmp_path, monkeypatch):
    """Isolated HERMES_HOME with a clean skills/ dir for each test."""
    home = tmp_path / ".hermes"
    home.mkdir()
    (home / "skills").mkdir()
    monkeypatch.setattr(Path, "home", lambda: tmp_path)
    monkeypatch.setenv("HERMES_HOME", str(home))
    # Force skill_usage module to re-resolve paths per test
    import importlib
    import tools.skill_usage as mod
    importlib.reload(mod)
    return home


def _write_skill(skills_dir: Path, name: str, category: str = ""):
    """Create a minimal SKILL.md with a name: frontmatter field."""
    if category:
        d = skills_dir / category / name
    else:
        d = skills_dir / name
    d.mkdir(parents=True, exist_ok=True)
    (d / "SKILL.md").write_text(
        f"""---
name: {name}
description: test skill
---

# body
""",
        encoding="utf-8",
    )
    return d


# ---------------------------------------------------------------------------
# Round-trip
# ---------------------------------------------------------------------------

def test_empty_usage_returns_empty_dict(skills_home):
    from tools.skill_usage import load_usage
    assert load_usage() == {}


def test_save_and_load_roundtrip(skills_home):
    from tools.skill_usage import load_usage, save_usage
    data = {"skill-a": {"use_count": 3, "state": "active"}}
    save_usage(data)
    loaded = load_usage()
    assert loaded["skill-a"]["use_count"] == 3
    assert loaded["skill-a"]["state"] == "active"


def test_save_is_atomic_no_partial_tmp_files(skills_home):
    from tools.skill_usage import save_usage, _usage_file
    save_usage({"x": {"use_count": 1}})
    skills_dir = _usage_file().parent
    # No leftover tempfile
    for p in skills_dir.iterdir():
        assert not p.name.startswith(".usage_"), f"leftover tmp: {p.name}"


def test_get_record_missing_returns_empty_record(skills_home):
    from tools.skill_usage import get_record
    rec = get_record("nonexistent")
    assert rec["use_count"] == 0
    assert rec["view_count"] == 0
    assert rec["state"] == "active"
    assert rec["pinned"] is False
    assert rec["archived_at"] is None


def test_get_record_backfills_missing_keys(skills_home):
    from tools.skill_usage import get_record, save_usage
    save_usage({"legacy": {"use_count": 5}})  # old-format record
    rec = get_record("legacy")
    assert rec["use_count"] == 5
    assert "view_count" in rec  # backfilled
    assert "state" in rec


def test_load_usage_handles_corrupt_file(skills_home):
    from tools.skill_usage import load_usage, _usage_file
    _usage_file().write_text("{ not json }", encoding="utf-8")
    assert load_usage() == {}


# ---------------------------------------------------------------------------
# Counter bumps
# ---------------------------------------------------------------------------

def test_bump_view_increments_and_timestamps(skills_home):
    from tools.skill_usage import bump_view, get_record
    bump_view("my-skill")
    bump_view("my-skill")
    rec = get_record("my-skill")
    assert rec["view_count"] == 2
    assert rec["last_viewed_at"] is not None


def test_bump_use_increments_and_timestamps(skills_home):
    from tools.skill_usage import bump_use, get_record
    bump_use("my-skill")
    rec = get_record("my-skill")
    assert rec["use_count"] == 1
    assert rec["last_used_at"] is not None


def test_bump_patch_increments_and_timestamps(skills_home):
    from tools.skill_usage import bump_patch, get_record
    bump_patch("my-skill")
    rec = get_record("my-skill")
    assert rec["patch_count"] == 1
    assert rec["last_patched_at"] is not None


def test_bump_on_empty_name_is_noop(skills_home):
    from tools.skill_usage import bump_view, load_usage
    bump_view("")
    assert load_usage() == {}


def test_bumps_do_not_corrupt_other_skills(skills_home):
    from tools.skill_usage import bump_view, bump_use, get_record
    bump_view("skill-a")
    bump_use("skill-b")
    bump_view("skill-a")
    assert get_record("skill-a")["view_count"] == 2
    assert get_record("skill-a")["use_count"] == 0
    assert get_record("skill-b")["use_count"] == 1


# ---------------------------------------------------------------------------
# State transitions
# ---------------------------------------------------------------------------

def test_set_state_active(skills_home):
    from tools.skill_usage import set_state, get_record, STATE_ACTIVE
    set_state("x", STATE_ACTIVE)
    assert get_record("x")["state"] == "active"


def test_set_state_archived_records_timestamp(skills_home):
    from tools.skill_usage import set_state, get_record, STATE_ARCHIVED
    set_state("x", STATE_ARCHIVED)
    rec = get_record("x")
    assert rec["state"] == "archived"
    assert rec["archived_at"] is not None


def test_set_state_invalid_is_noop(skills_home):
    from tools.skill_usage import set_state, get_record
    set_state("x", "bogus")
    # No record created for invalid state
    rec = get_record("x")
    assert rec["state"] == "active"  # default


def test_restoring_from_archive_clears_timestamp(skills_home):
    from tools.skill_usage import set_state, get_record, STATE_ARCHIVED, STATE_ACTIVE
    set_state("x", STATE_ARCHIVED)
    assert get_record("x")["archived_at"] is not None
    set_state("x", STATE_ACTIVE)
    assert get_record("x")["archived_at"] is None


def test_set_pinned(skills_home):
    from tools.skill_usage import set_pinned, get_record
    set_pinned("x", True)
    assert get_record("x")["pinned"] is True
    set_pinned("x", False)
    assert get_record("x")["pinned"] is False


def test_forget_removes_record(skills_home):
    from tools.skill_usage import bump_view, forget, load_usage
    bump_view("x")
    assert "x" in load_usage()
    forget("x")
    assert "x" not in load_usage()


# ---------------------------------------------------------------------------
# Provenance filter — the load-bearing safety check
# ---------------------------------------------------------------------------

def test_agent_created_excludes_bundled(skills_home):
    from tools.skill_usage import list_agent_created_skill_names
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "bundled-skill", category="github")
    _write_skill(skills_dir, "my-skill")
    # Seed a bundled manifest marking bundled-skill as upstream
    (skills_dir / ".bundled_manifest").write_text(
        "bundled-skill:abc123\n", encoding="utf-8",
    )
    names = list_agent_created_skill_names()
    assert "my-skill" in names
    assert "bundled-skill" not in names


def test_agent_created_excludes_hub_installed(skills_home):
    from tools.skill_usage import list_agent_created_skill_names
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "hub-skill")
    _write_skill(skills_dir, "my-skill")
    hub_dir = skills_dir / ".hub"
    hub_dir.mkdir()
    (hub_dir / "lock.json").write_text(
        json.dumps({"version": 1, "installed": {"hub-skill": {"source": "taps/main"}}}),
        encoding="utf-8",
    )
    names = list_agent_created_skill_names()
    assert "my-skill" in names
    assert "hub-skill" not in names


def test_is_agent_created(skills_home):
    from tools.skill_usage import is_agent_created
    skills_dir = skills_home / "skills"
    (skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
    hub_dir = skills_dir / ".hub"
    hub_dir.mkdir()
    (hub_dir / "lock.json").write_text(
        json.dumps({"installed": {"hubbed": {}}}), encoding="utf-8",
    )
    assert is_agent_created("my-skill") is True
    assert is_agent_created("bundled") is False
    assert is_agent_created("hubbed") is False


def test_agent_created_skips_archive_and_hub_dirs(skills_home):
    from tools.skill_usage import list_agent_created_skill_names
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "real-skill")
    # Dot-prefixed dirs must be ignored even if they contain SKILL.md
    archive = skills_dir / ".archive" / "old-skill"
    archive.mkdir(parents=True)
    (archive / "SKILL.md").write_text(
        "---\nname: old-skill\n---\n", encoding="utf-8",
    )
    names = list_agent_created_skill_names()
    assert "real-skill" in names
    assert "old-skill" not in names


# ---------------------------------------------------------------------------
# Archive / restore
# ---------------------------------------------------------------------------

def test_archive_skill_moves_directory(skills_home):
    from tools.skill_usage import archive_skill, get_record, STATE_ARCHIVED
    skills_dir = skills_home / "skills"
    skill_dir = _write_skill(skills_dir, "old-skill")
    assert skill_dir.exists()

    ok, msg = archive_skill("old-skill")
    assert ok, msg
    assert not skill_dir.exists()
    assert (skills_dir / ".archive" / "old-skill" / "SKILL.md").exists()
    assert get_record("old-skill")["state"] == "archived"
    assert get_record("old-skill")["archived_at"] is not None


def test_archive_refuses_bundled_skill(skills_home):
    from tools.skill_usage import archive_skill
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "bundled")
    (skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")

    ok, msg = archive_skill("bundled")
    assert not ok
    assert "bundled" in msg.lower() or "hub" in msg.lower()


def test_archive_refuses_hub_skill(skills_home):
    from tools.skill_usage import archive_skill
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "hub-skill")
    hub_dir = skills_dir / ".hub"
    hub_dir.mkdir()
    (hub_dir / "lock.json").write_text(
        json.dumps({"installed": {"hub-skill": {}}}), encoding="utf-8",
    )

    ok, msg = archive_skill("hub-skill")
    assert not ok


def test_archive_missing_skill_returns_error(skills_home):
    from tools.skill_usage import archive_skill
    ok, msg = archive_skill("nonexistent")
    assert not ok
    assert "not found" in msg.lower()


def test_restore_skill_moves_back(skills_home):
    from tools.skill_usage import archive_skill, restore_skill, get_record
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "temp-skill")
    archive_skill("temp-skill")
    assert not (skills_dir / "temp-skill").exists()

    ok, msg = restore_skill("temp-skill")
    assert ok, msg
    assert (skills_dir / "temp-skill" / "SKILL.md").exists()
    assert get_record("temp-skill")["state"] == "active"


def test_archive_collision_gets_suffix(skills_home):
    from tools.skill_usage import archive_skill
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "dup")
    archive_skill("dup")
    _write_skill(skills_dir, "dup")  # recreate
    ok, msg = archive_skill("dup")
    assert ok
    # Two entries under .archive/ — second should have a timestamp suffix
    archived = sorted(p.name for p in (skills_dir / ".archive").iterdir() if p.is_dir())
    assert "dup" in archived
    assert any(n.startswith("dup-") and n != "dup" for n in archived)


# ---------------------------------------------------------------------------
# Reporting
# ---------------------------------------------------------------------------

def test_agent_created_report_includes_defaults(skills_home):
    from tools.skill_usage import agent_created_report, bump_view
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "a")
    _write_skill(skills_dir, "b")
    bump_view("a")
    rows = agent_created_report()
    by_name = {r["name"]: r for r in rows}
    assert "a" in by_name and "b" in by_name
    assert by_name["a"]["view_count"] == 1
    # b has no usage record yet — must still appear with defaults
    assert by_name["b"]["view_count"] == 0
    assert by_name["b"]["state"] == "active"


def test_agent_created_report_excludes_bundled_and_hub(skills_home):
    from tools.skill_usage import agent_created_report
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "mine")
    _write_skill(skills_dir, "bundled")
    _write_skill(skills_dir, "hubbed")
    (skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
    hub = skills_dir / ".hub"
    hub.mkdir()
    (hub / "lock.json").write_text(
        json.dumps({"installed": {"hubbed": {}}}), encoding="utf-8",
    )
    names = {r["name"] for r in agent_created_report()}
    assert "mine" in names
    assert "bundled" not in names
    assert "hubbed" not in names



# ---------------------------------------------------------------------------
# Provenance guard — telemetry must not leak records for bundled/hub skills
# ---------------------------------------------------------------------------

def test_bump_view_no_op_for_bundled_skill(skills_home):
    """Telemetry bumps on bundled skills are dropped — the sidecar must stay
    focused on agent-created skills only."""
    from tools.skill_usage import bump_view, load_usage
    skills_dir = skills_home / "skills"
    (skills_dir / ".bundled_manifest").write_text(
        "ship-bundled:abc\n", encoding="utf-8",
    )

    bump_view("ship-bundled")
    assert "ship-bundled" not in load_usage(), (
        "bundled skill leaked into .usage.json"
    )


def test_bump_patch_no_op_for_hub_skill(skills_home):
    from tools.skill_usage import bump_patch, load_usage
    skills_dir = skills_home / "skills"
    hub = skills_dir / ".hub"
    hub.mkdir()
    (hub / "lock.json").write_text(
        json.dumps({"installed": {"from-hub": {}}}), encoding="utf-8",
    )

    bump_patch("from-hub")
    assert "from-hub" not in load_usage()


def test_bump_use_no_op_for_hub_skill(skills_home):
    from tools.skill_usage import bump_use, load_usage
    skills_dir = skills_home / "skills"
    hub = skills_dir / ".hub"
    hub.mkdir()
    (hub / "lock.json").write_text(
        json.dumps({"installed": {"from-hub": {}}}), encoding="utf-8",
    )

    bump_use("from-hub")
    assert "from-hub" not in load_usage()


def test_set_state_no_op_for_bundled_skill(skills_home):
    """State transitions on bundled skills must not land in the sidecar."""
    from tools.skill_usage import set_state, load_usage, STATE_ARCHIVED
    skills_dir = skills_home / "skills"
    (skills_dir / ".bundled_manifest").write_text(
        "locked:abc\n", encoding="utf-8",
    )
    set_state("locked", STATE_ARCHIVED)
    assert "locked" not in load_usage()


def test_restore_refuses_to_shadow_bundled_skill(skills_home):
    """If a bundled skill now occupies the name, refuse to restore."""
    from tools.skill_usage import archive_skill, restore_skill
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "shared-name")
    archive_skill("shared-name")

    # Now a bundled skill appears with the same name
    (skills_dir / ".bundled_manifest").write_text(
        "shared-name:abc\n", encoding="utf-8",
    )
    _write_skill(skills_dir, "shared-name")  # bundled install landed

    ok, msg = restore_skill("shared-name")
    assert not ok
    assert "bundled" in msg.lower() or "shadow" in msg.lower()


def test_end_to_end_no_code_path_mutates_bundled_skill(skills_home):
    """The combined guarantee: no curator code path can archive, mark stale,
    set-state, or persist telemetry for a bundled or hub-installed skill."""
    from tools.skill_usage import (
        bump_view, bump_use, bump_patch, set_state, set_pinned,
        archive_skill, load_usage, STATE_STALE, STATE_ARCHIVED,
    )
    skills_dir = skills_home / "skills"
    _write_skill(skills_dir, "bundled-one")
    _write_skill(skills_dir, "hub-one")
    _write_skill(skills_dir, "mine")

    (skills_dir / ".bundled_manifest").write_text(
        "bundled-one:abc\n", encoding="utf-8",
    )
    hub = skills_dir / ".hub"
    hub.mkdir()
    (hub / "lock.json").write_text(
        json.dumps({"installed": {"hub-one": {}}}), encoding="utf-8",
    )

    # Hammer every mutator at the bundled/hub names
    for name in ("bundled-one", "hub-one"):
        bump_view(name)
        bump_use(name)
        bump_patch(name)
        set_state(name, STATE_STALE)
        set_state(name, STATE_ARCHIVED)
        set_pinned(name, True)
        ok, _msg = archive_skill(name)
        assert not ok, f"archive_skill(\"{name}\") should refuse"

    # Sidecar must be clean of all three
    data = load_usage()
    assert "bundled-one" not in data
    assert "hub-one" not in data

    # Directories must still be in place on disk
    assert (skills_dir / "bundled-one" / "SKILL.md").exists()
    assert (skills_dir / "hub-one" / "SKILL.md").exists()

    # The agent-created skill can still be mutated normally
    bump_view("mine")
    assert load_usage()["mine"]["view_count"] == 1