mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 23:11:37 +08:00
Compare commits
1 Commits
gemini-cli
...
opencode-p
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c2e4d6a0e5 |
@@ -5928,6 +5928,13 @@ Examples:
|
|||||||
sessions_export.add_argument("output", help="Output JSONL file path (use - for stdout)")
|
sessions_export.add_argument("output", help="Output JSONL file path (use - for stdout)")
|
||||||
sessions_export.add_argument("--source", help="Filter by source")
|
sessions_export.add_argument("--source", help="Filter by source")
|
||||||
sessions_export.add_argument("--session-id", help="Export a specific session")
|
sessions_export.add_argument("--session-id", help="Export a specific session")
|
||||||
|
sessions_export.add_argument(
|
||||||
|
"--sanitize",
|
||||||
|
action="store_true",
|
||||||
|
help="Redact user/model content (message text, reasoning, tool args/output, titles, "
|
||||||
|
"system prompt) before export. Structure and metrics are preserved. "
|
||||||
|
"Use when sharing exports for bug reports or training data.",
|
||||||
|
)
|
||||||
|
|
||||||
sessions_delete = sessions_subparsers.add_parser("delete", help="Delete a specific session")
|
sessions_delete = sessions_subparsers.add_parser("delete", help="Delete a specific session")
|
||||||
sessions_delete.add_argument("session_id", help="Session ID to delete")
|
sessions_delete.add_argument("session_id", help="Session ID to delete")
|
||||||
@@ -5997,6 +6004,19 @@ Examples:
|
|||||||
print(f"{preview:<50} {last_active:<13} {s['source']:<6} {sid}")
|
print(f"{preview:<50} {last_active:<13} {s['source']:<6} {sid}")
|
||||||
|
|
||||||
elif action == "export":
|
elif action == "export":
|
||||||
|
sanitize = getattr(args, "sanitize", False)
|
||||||
|
if sanitize:
|
||||||
|
try:
|
||||||
|
from hermes_state import sanitize_session_export as _sanitize_fn
|
||||||
|
except Exception:
|
||||||
|
_sanitize_fn = None
|
||||||
|
print("Warning: sanitize_session_export unavailable — exporting raw data.")
|
||||||
|
else:
|
||||||
|
_sanitize_fn = None
|
||||||
|
|
||||||
|
def _maybe_sanitize(d):
|
||||||
|
return _sanitize_fn(d) if _sanitize_fn else d
|
||||||
|
|
||||||
if args.session_id:
|
if args.session_id:
|
||||||
resolved_session_id = db.resolve_session_id(args.session_id)
|
resolved_session_id = db.resolve_session_id(args.session_id)
|
||||||
if not resolved_session_id:
|
if not resolved_session_id:
|
||||||
@@ -6006,6 +6026,7 @@ Examples:
|
|||||||
if not data:
|
if not data:
|
||||||
print(f"Session '{args.session_id}' not found.")
|
print(f"Session '{args.session_id}' not found.")
|
||||||
return
|
return
|
||||||
|
data = _maybe_sanitize(data)
|
||||||
line = _json.dumps(data, ensure_ascii=False) + "\n"
|
line = _json.dumps(data, ensure_ascii=False) + "\n"
|
||||||
if args.output == "-":
|
if args.output == "-":
|
||||||
import sys
|
import sys
|
||||||
@@ -6013,18 +6034,20 @@ Examples:
|
|||||||
else:
|
else:
|
||||||
with open(args.output, "w", encoding="utf-8") as f:
|
with open(args.output, "w", encoding="utf-8") as f:
|
||||||
f.write(line)
|
f.write(line)
|
||||||
print(f"Exported 1 session to {args.output}")
|
suffix = " (sanitized)" if sanitize and _sanitize_fn else ""
|
||||||
|
print(f"Exported 1 session to {args.output}{suffix}")
|
||||||
else:
|
else:
|
||||||
sessions = db.export_all(source=args.source)
|
sessions = db.export_all(source=args.source)
|
||||||
if args.output == "-":
|
if args.output == "-":
|
||||||
import sys
|
import sys
|
||||||
for s in sessions:
|
for s in sessions:
|
||||||
sys.stdout.write(_json.dumps(s, ensure_ascii=False) + "\n")
|
sys.stdout.write(_json.dumps(_maybe_sanitize(s), ensure_ascii=False) + "\n")
|
||||||
else:
|
else:
|
||||||
with open(args.output, "w", encoding="utf-8") as f:
|
with open(args.output, "w", encoding="utf-8") as f:
|
||||||
for s in sessions:
|
for s in sessions:
|
||||||
f.write(_json.dumps(s, ensure_ascii=False) + "\n")
|
f.write(_json.dumps(_maybe_sanitize(s), ensure_ascii=False) + "\n")
|
||||||
print(f"Exported {len(sessions)} sessions to {args.output}")
|
suffix = " (sanitized)" if sanitize and _sanitize_fn else ""
|
||||||
|
print(f"Exported {len(sessions)} sessions to {args.output}{suffix}")
|
||||||
|
|
||||||
elif action == "delete":
|
elif action == "delete":
|
||||||
resolved_session_id = db.resolve_session_id(args.session_id)
|
resolved_session_id = db.resolve_session_id(args.session_id)
|
||||||
|
|||||||
150
hermes_state.py
150
hermes_state.py
@@ -1160,6 +1160,23 @@ class SessionDB:
|
|||||||
results.append({**session, "messages": messages})
|
results.append({**session, "messages": messages})
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------
|
||||||
|
# Export sanitization
|
||||||
|
# ---------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# When users share session exports for debugging or training, the
|
||||||
|
# raw JSON contains every user message, tool output, and reasoning
|
||||||
|
# trace — which often includes file contents, command output, env
|
||||||
|
# variables, paths, and other confidential information.
|
||||||
|
#
|
||||||
|
# ``sanitize_session_export`` produces a deep copy of the export
|
||||||
|
# with all content fields replaced by opaque ``[redacted:<kind>:<id>]``
|
||||||
|
# tokens. Structural metadata (IDs, roles, timestamps, token counts,
|
||||||
|
# tool names, finish reasons, model info, cost data) is preserved
|
||||||
|
# so that the shape of a conversation is still analysable.
|
||||||
|
#
|
||||||
|
# Inspired by anomalyco/opencode#22489 (opencode's ``export --sanitize``).
|
||||||
|
|
||||||
def clear_messages(self, session_id: str) -> None:
|
def clear_messages(self, session_id: str) -> None:
|
||||||
"""Delete all messages for a session and reset its counters."""
|
"""Delete all messages for a session and reset its counters."""
|
||||||
def _do(conn):
|
def _do(conn):
|
||||||
@@ -1236,3 +1253,136 @@ class SessionDB:
|
|||||||
return len(session_ids)
|
return len(session_ids)
|
||||||
|
|
||||||
return self._execute_write(_do)
|
return self._execute_write(_do)
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# Session export sanitization
|
||||||
|
# =========================================================================
|
||||||
|
#
|
||||||
|
# Ported from anomalyco/opencode#22489 — users often want to share a
|
||||||
|
# session export for bug reports, feature requests, or training data
|
||||||
|
# collection, but the raw export contains every user prompt, tool
|
||||||
|
# output, file content, and reasoning trace. ``sanitize_session_export``
|
||||||
|
# replaces content fields with opaque tokens while preserving the
|
||||||
|
# conversation's structure and metrics.
|
||||||
|
|
||||||
|
# Message-level content fields that are always redacted on a message.
|
||||||
|
_REDACT_MSG_STRING_FIELDS = (
|
||||||
|
"content",
|
||||||
|
"reasoning",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Session-level fields that can contain user-facing text.
|
||||||
|
_REDACT_SESSION_STRING_FIELDS = (
|
||||||
|
"system_prompt",
|
||||||
|
"title",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_token(kind: str, id_: Any, value: Any) -> Any:
|
||||||
|
"""Produce an opaque redaction token. Preserves empty/None values."""
|
||||||
|
if value in (None, "", b""):
|
||||||
|
return value
|
||||||
|
return f"[redacted:{kind}:{id_}]"
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_tool_call(call: Any, msg_id: Any, index: int) -> Any:
|
||||||
|
"""Redact arguments inside a tool_call while preserving structure (id, name)."""
|
||||||
|
if not isinstance(call, dict):
|
||||||
|
return call
|
||||||
|
out = dict(call)
|
||||||
|
tcid = out.get("id") or f"{msg_id}-{index}"
|
||||||
|
fn = out.get("function")
|
||||||
|
if isinstance(fn, dict):
|
||||||
|
new_fn = dict(fn)
|
||||||
|
if "arguments" in new_fn and new_fn["arguments"] not in (None, "", "{}"):
|
||||||
|
new_fn["arguments"] = _redact_token("tool-input", tcid, new_fn["arguments"])
|
||||||
|
out["function"] = new_fn
|
||||||
|
# Some schemas put args at the top level rather than under ``function``.
|
||||||
|
if "arguments" in out and out["arguments"] not in (None, "", "{}"):
|
||||||
|
out["arguments"] = _redact_token("tool-input", tcid, out["arguments"])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_reasoning_details(details: Any, msg_id: Any) -> Any:
|
||||||
|
"""Redact text inside OpenAI / Anthropic reasoning_details blocks.
|
||||||
|
|
||||||
|
``reasoning_details`` is a list of dicts with shapes like::
|
||||||
|
|
||||||
|
{"type": "reasoning.text", "text": "..."}
|
||||||
|
{"type": "reasoning.encrypted", "data": "..."}
|
||||||
|
{"type": "reasoning.summary", "summary": "..."}
|
||||||
|
|
||||||
|
We preserve the block type/structure and redact the inner payload.
|
||||||
|
"""
|
||||||
|
if not isinstance(details, list):
|
||||||
|
return details
|
||||||
|
out = []
|
||||||
|
for idx, block in enumerate(details):
|
||||||
|
if not isinstance(block, dict):
|
||||||
|
out.append(block)
|
||||||
|
continue
|
||||||
|
new_block = dict(block)
|
||||||
|
for key in ("text", "data", "summary", "content"):
|
||||||
|
if key in new_block and new_block[key] not in (None, ""):
|
||||||
|
new_block[key] = _redact_token(f"reasoning-{key}", f"{msg_id}-{idx}", new_block[key])
|
||||||
|
out.append(new_block)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_message(msg: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Return a sanitized copy of a single message row."""
|
||||||
|
if not isinstance(msg, dict):
|
||||||
|
return msg
|
||||||
|
msg_id = msg.get("id", "msg")
|
||||||
|
out = dict(msg)
|
||||||
|
|
||||||
|
# Plain string content fields.
|
||||||
|
for field in _REDACT_MSG_STRING_FIELDS:
|
||||||
|
if field in out and out[field] not in (None, ""):
|
||||||
|
out[field] = _redact_token(field.replace("_", "-"), msg_id, out[field])
|
||||||
|
|
||||||
|
# Tool calls: keep structure (id, name) but redact arguments.
|
||||||
|
tcs = out.get("tool_calls")
|
||||||
|
if isinstance(tcs, list):
|
||||||
|
out["tool_calls"] = [_redact_tool_call(tc, msg_id, i) for i, tc in enumerate(tcs)]
|
||||||
|
|
||||||
|
# Reasoning details: preserve block structure, redact text/data.
|
||||||
|
if "reasoning_details" in out:
|
||||||
|
out["reasoning_details"] = _redact_reasoning_details(out["reasoning_details"], msg_id)
|
||||||
|
|
||||||
|
# Codex reasoning items follow the same shape as reasoning_details.
|
||||||
|
if "codex_reasoning_items" in out:
|
||||||
|
out["codex_reasoning_items"] = _redact_reasoning_details(out["codex_reasoning_items"], msg_id)
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize_session_export(session: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Return a deep-sanitized copy of a session export.
|
||||||
|
|
||||||
|
All user-facing content (message text, reasoning, tool arguments and
|
||||||
|
outputs, system prompt, title) is replaced by ``[redacted:<kind>:<id>]``
|
||||||
|
tokens. Structural metadata (ids, timestamps, token counts, tool names,
|
||||||
|
model/provider info, cost data, finish reasons) is preserved so the
|
||||||
|
export remains useful for debugging schema issues, analysing tool-use
|
||||||
|
patterns, or counting sessions without leaking confidential data.
|
||||||
|
|
||||||
|
The input dict is not mutated.
|
||||||
|
"""
|
||||||
|
if not isinstance(session, dict):
|
||||||
|
return session
|
||||||
|
sid = session.get("id", "session")
|
||||||
|
out = dict(session)
|
||||||
|
|
||||||
|
# Session-level text fields (title, system prompt).
|
||||||
|
for field in _REDACT_SESSION_STRING_FIELDS:
|
||||||
|
if field in out and out[field] not in (None, ""):
|
||||||
|
out[field] = _redact_token(field.replace("_", "-"), sid, out[field])
|
||||||
|
|
||||||
|
# Messages list: sanitize each row.
|
||||||
|
msgs = out.get("messages")
|
||||||
|
if isinstance(msgs, list):
|
||||||
|
out["messages"] = [_redact_message(m) for m in msgs]
|
||||||
|
|
||||||
|
return out
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Tests for hermes_state.py — SessionDB SQLite CRUD, FTS5 search, export."""
|
"""Tests for hermes_state.py — SessionDB SQLite CRUD, FTS5 search, export."""
|
||||||
|
|
||||||
|
import json
|
||||||
import time
|
import time
|
||||||
import pytest
|
import pytest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -609,6 +610,156 @@ class TestDeleteAndExport:
|
|||||||
assert exports[0]["source"] == "cli"
|
assert exports[0]["source"] == "cli"
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# Export sanitization (ported from anomalyco/opencode#22489)
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
class TestSanitizeSessionExport:
|
||||||
|
"""Validate that sanitize_session_export redacts user content while
|
||||||
|
preserving structural metadata useful for analysis."""
|
||||||
|
|
||||||
|
def test_redacts_message_content(self, db):
|
||||||
|
from hermes_state import sanitize_session_export
|
||||||
|
|
||||||
|
db.create_session(session_id="s1", source="cli", model="test", system_prompt="secret prompt")
|
||||||
|
db.set_session_title("s1", "my confidential task")
|
||||||
|
db.append_message("s1", role="user", content="what is my password?")
|
||||||
|
db.append_message("s1", role="assistant", content="Here's your secret: XYZ")
|
||||||
|
|
||||||
|
raw = db.export_session("s1")
|
||||||
|
sanitized = sanitize_session_export(raw)
|
||||||
|
|
||||||
|
# Structural / metric fields are preserved.
|
||||||
|
assert sanitized["id"] == "s1"
|
||||||
|
assert sanitized["source"] == "cli"
|
||||||
|
assert sanitized["model"] == "test"
|
||||||
|
assert len(sanitized["messages"]) == 2
|
||||||
|
for msg in sanitized["messages"]:
|
||||||
|
assert "role" in msg
|
||||||
|
assert msg["role"] in ("user", "assistant")
|
||||||
|
assert "id" in msg
|
||||||
|
assert "timestamp" in msg
|
||||||
|
|
||||||
|
# Content is redacted.
|
||||||
|
assert "password" not in json.dumps(sanitized)
|
||||||
|
assert "XYZ" not in json.dumps(sanitized)
|
||||||
|
assert "confidential" not in json.dumps(sanitized)
|
||||||
|
assert "secret prompt" not in json.dumps(sanitized)
|
||||||
|
for msg in sanitized["messages"]:
|
||||||
|
assert msg["content"].startswith("[redacted:content:")
|
||||||
|
|
||||||
|
# Title and system_prompt are redacted.
|
||||||
|
assert sanitized["title"].startswith("[redacted:title:")
|
||||||
|
assert sanitized["system_prompt"].startswith("[redacted:system-prompt:")
|
||||||
|
|
||||||
|
def test_redacts_reasoning_and_tool_calls(self, db):
|
||||||
|
from hermes_state import sanitize_session_export
|
||||||
|
|
||||||
|
db.create_session(session_id="s1", source="cli")
|
||||||
|
db.append_message(
|
||||||
|
"s1",
|
||||||
|
role="assistant",
|
||||||
|
content="let me search",
|
||||||
|
reasoning="user asked about their private API key",
|
||||||
|
tool_calls=[{
|
||||||
|
"id": "tc_1",
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "terminal",
|
||||||
|
"arguments": '{"command": "cat /etc/passwd"}',
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
db.append_message(
|
||||||
|
"s1",
|
||||||
|
role="tool",
|
||||||
|
content="root:x:0:0:root:/root:/bin/bash",
|
||||||
|
tool_call_id="tc_1",
|
||||||
|
tool_name="terminal",
|
||||||
|
)
|
||||||
|
|
||||||
|
raw = db.export_session("s1")
|
||||||
|
sanitized = sanitize_session_export(raw)
|
||||||
|
dumped = json.dumps(sanitized)
|
||||||
|
|
||||||
|
# No leaked content.
|
||||||
|
assert "private API key" not in dumped
|
||||||
|
assert "/etc/passwd" not in dumped
|
||||||
|
assert "root:x:0:0" not in dumped
|
||||||
|
assert "cat" not in dumped # the command body should not leak
|
||||||
|
|
||||||
|
# Tool call structure preserved (id, type, function name).
|
||||||
|
asst = sanitized["messages"][0]
|
||||||
|
assert asst["tool_calls"][0]["id"] == "tc_1"
|
||||||
|
assert asst["tool_calls"][0]["type"] == "function"
|
||||||
|
assert asst["tool_calls"][0]["function"]["name"] == "terminal"
|
||||||
|
assert asst["tool_calls"][0]["function"]["arguments"].startswith("[redacted:tool-input:")
|
||||||
|
|
||||||
|
# Reasoning field redacted but present.
|
||||||
|
assert asst["reasoning"].startswith("[redacted:reasoning:")
|
||||||
|
|
||||||
|
# Tool response metadata preserved (tool_call_id, tool_name).
|
||||||
|
tool_msg = sanitized["messages"][1]
|
||||||
|
assert tool_msg["tool_call_id"] == "tc_1"
|
||||||
|
assert tool_msg["tool_name"] == "terminal"
|
||||||
|
assert tool_msg["content"].startswith("[redacted:content:")
|
||||||
|
|
||||||
|
def test_preserves_empty_values(self, db):
|
||||||
|
"""Empty/None content should pass through untouched so consumers
|
||||||
|
don't treat sanitization as 'there was hidden data here'."""
|
||||||
|
from hermes_state import sanitize_session_export
|
||||||
|
|
||||||
|
db.create_session(session_id="s1", source="cli")
|
||||||
|
db.append_message("s1", role="user", content="")
|
||||||
|
raw = db.export_session("s1")
|
||||||
|
sanitized = sanitize_session_export(raw)
|
||||||
|
|
||||||
|
# Empty content stays empty (not a fake redaction token).
|
||||||
|
assert sanitized["messages"][0]["content"] in ("", None)
|
||||||
|
|
||||||
|
def test_does_not_mutate_input(self, db):
|
||||||
|
from hermes_state import sanitize_session_export
|
||||||
|
|
||||||
|
db.create_session(session_id="s1", source="cli")
|
||||||
|
db.append_message("s1", role="user", content="original text")
|
||||||
|
raw = db.export_session("s1")
|
||||||
|
original_content = raw["messages"][0]["content"]
|
||||||
|
|
||||||
|
sanitize_session_export(raw)
|
||||||
|
|
||||||
|
# Original dict is unchanged.
|
||||||
|
assert raw["messages"][0]["content"] == original_content
|
||||||
|
|
||||||
|
def test_redacts_reasoning_details_blocks(self):
|
||||||
|
"""reasoning_details is a list of typed blocks — preserve type, redact payload."""
|
||||||
|
from hermes_state import sanitize_session_export
|
||||||
|
|
||||||
|
session = {
|
||||||
|
"id": "s1",
|
||||||
|
"source": "cli",
|
||||||
|
"messages": [{
|
||||||
|
"id": "m1",
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "done",
|
||||||
|
"reasoning_details": [
|
||||||
|
{"type": "reasoning.text", "text": "sensitive internal thought"},
|
||||||
|
{"type": "reasoning.encrypted", "data": "encrypted_blob_XYZ"},
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
sanitized = sanitize_session_export(session)
|
||||||
|
dumped = json.dumps(sanitized)
|
||||||
|
|
||||||
|
assert "sensitive internal thought" not in dumped
|
||||||
|
assert "encrypted_blob_XYZ" not in dumped
|
||||||
|
# Block types preserved.
|
||||||
|
blocks = sanitized["messages"][0]["reasoning_details"]
|
||||||
|
assert blocks[0]["type"] == "reasoning.text"
|
||||||
|
assert blocks[0]["text"].startswith("[redacted:reasoning-text:")
|
||||||
|
assert blocks[1]["type"] == "reasoning.encrypted"
|
||||||
|
assert blocks[1]["data"].startswith("[redacted:reasoning-data:")
|
||||||
|
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# Prune
|
# Prune
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user