diff --git a/ui-tui/src/__tests__/text.test.ts b/ui-tui/src/__tests__/text.test.ts index d4a2469e8fd..9c0a886d7d7 100644 --- a/ui-tui/src/__tests__/text.test.ts +++ b/ui-tui/src/__tests__/text.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { + boundedLiveRenderText, edgePreview, estimateRows, estimateTokensRough, @@ -106,3 +107,25 @@ describe('estimateRows', () => { expect(estimateRows(snake, w)).toBe(estimateRows(plain, w)) }) }) + +describe('boundedLiveRenderText', () => { + it('keeps short text unchanged', () => { + expect(boundedLiveRenderText('alpha\nbeta', { maxChars: 50, maxLines: 5 })).toBe('alpha\nbeta') + }) + + it('keeps the tail of long live text', () => { + const text = Array.from({ length: 6 }, (_, i) => `line-${i + 1}`).join('\n') + const out = boundedLiveRenderText(text, { maxChars: 100, maxLines: 3 }) + + expect(out).toContain('omitted 3 lines') + expect(out.endsWith('line-4\nline-5\nline-6')).toBe(true) + expect(out).not.toContain('line-1') + }) + + it('bounds very long single-line text by chars', () => { + const out = boundedLiveRenderText('a'.repeat(60), { maxChars: 12, maxLines: 5 }) + + expect(out).toContain('omitted 48 chars') + expect(out.endsWith('a'.repeat(12))).toBe(true) + }) +}) diff --git a/ui-tui/src/app/turnController.ts b/ui-tui/src/app/turnController.ts index 1041b4d4f5f..332fb0297d8 100644 --- a/ui-tui/src/app/turnController.ts +++ b/ui-tui/src/app/turnController.ts @@ -2,6 +2,7 @@ import { REASONING_PULSE_MS, STREAM_BATCH_MS } from '../config/timing.js' import type { SessionInterruptResponse, SubagentEventPayload } from '../gatewayTypes.js' import { hasReasoningTag, splitReasoning } from '../lib/reasoning.js' import { + boundedLiveRenderText, buildToolTrailLine, estimateTokensRough, isTransientTrailLine, @@ -492,7 +493,7 @@ class TurnController { this.streamTimer = null const raw = this.bufRef.trimStart() const visible = hasReasoningTag(raw) ? splitReasoning(raw).text : raw - patchTurnState({ streaming: visible }) + patchTurnState({ streaming: boundedLiveRenderText(visible) }) }, STREAM_BATCH_MS) } diff --git a/ui-tui/src/components/messageLine.tsx b/ui-tui/src/components/messageLine.tsx index fc6f78e9245..8556f29b800 100644 --- a/ui-tui/src/components/messageLine.tsx +++ b/ui-tui/src/components/messageLine.tsx @@ -5,7 +5,7 @@ import { LONG_MSG } from '../config/limits.js' import { sectionMode } from '../domain/details.js' import { userDisplay } from '../domain/messages.js' import { ROLE } from '../domain/roles.js' -import { compactPreview, hasAnsi, isPasteBackedText, stripAnsi } from '../lib/text.js' +import { boundedLiveRenderText, compactPreview, hasAnsi, isPasteBackedText, stripAnsi } from '../lib/text.js' import type { Theme } from '../theme.js' import type { DetailsMode, Msg, SectionVisibility } from '../types.js' @@ -84,7 +84,11 @@ export const MessageLine = memo(function MessageLine({ } if (msg.role === 'assistant') { - return isStreaming ? {msg.text} : + return isStreaming ? ( + {boundedLiveRenderText(msg.text)} + ) : ( + + ) } if (msg.role === 'user' && msg.text.length > LONG_MSG && isPasteBackedText(msg.text)) { diff --git a/ui-tui/src/components/thinking.tsx b/ui-tui/src/components/thinking.tsx index 2d52102b516..c50dd03c517 100644 --- a/ui-tui/src/components/thinking.tsx +++ b/ui-tui/src/components/thinking.tsx @@ -16,6 +16,7 @@ import { widthByDepth } from '../lib/subagentTree.js' import { + boundedLiveRenderText, compactPreview, estimateTokensRough, fmtK, @@ -633,7 +634,12 @@ export const Thinking = memo(function Thinking({ streaming?: boolean t: Theme }) { - const preview = useMemo(() => thinkingPreview(reasoning, mode, THINKING_COT_MAX), [mode, reasoning]) + const preview = useMemo(() => { + const raw = thinkingPreview(reasoning, mode, THINKING_COT_MAX) + + return mode === 'full' ? boundedLiveRenderText(raw) : raw + }, [mode, reasoning]) + const lines = useMemo(() => preview.split('\n').map(line => line.replace(/\t/g, ' ')), [preview]) if (!preview && !active) { @@ -868,8 +874,8 @@ export const ToolTrail = memo(function ToolTrail({ const hasTools = groups.length > 0 const hasSubagents = subagents.length > 0 const hasMeta = meta.length > 0 - const hasThinking = !!cot || reasoningActive || busy const thinkingLive = reasoningActive || reasoningStreaming + const hasThinking = !!cot || thinkingLive const tokenCount = reasoningTokens && reasoningTokens > 0 ? reasoningTokens : reasoning ? estimateTokensRough(reasoning) : 0 @@ -1002,7 +1008,7 @@ export const ToolTrail = memo(function ToolTrail({ open: openThinking, render: rails => ( - `hermes-tui: ${snap.level} memory (${formatBytes(snap.heapUsed)}) — auto heap dump → ${dump?.heapPath ?? '(failed)'}\n` + snap.source === 'heap' + ? `hermes-tui: ${snap.level} heap (${formatBytes(snap.heapUsed)}, rss ${formatBytes(snap.rss)}) — auto heap dump → ${dump?.heapPath ?? '(failed)'}\n` + : `hermes-tui: ${snap.level} rss (${formatBytes(snap.rss)}, native ${formatBytes(snap.nativeUsed)}) — auto diagnostics → ${dump?.diagPath ?? '(failed)'}\n` setupGracefulExit({ cleanups: [() => gw.kill()], diff --git a/ui-tui/src/lib/memory.ts b/ui-tui/src/lib/memory.ts index 9f157adffc8..38450b3f9c7 100644 --- a/ui-tui/src/lib/memory.ts +++ b/ui-tui/src/lib/memory.ts @@ -145,11 +145,11 @@ export async function performHeapDump(trigger: MemoryTrigger = 'manual'): Promis // Diagnostics first — heap-snapshot serialization can crash on very large // heaps, and the JSON sidecar is the most actionable artifact if so. const diagnostics = await captureMemoryDiagnostics(trigger) - const dir = process.env.HERMES_HEAPDUMP_DIR?.trim() || join(homedir() || tmpdir(), '.hermes', 'heapdumps') + const dir = memoryDumpDir() await mkdir(dir, { recursive: true }) - const base = `hermes-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}-${trigger}` + const base = memoryDumpBase(trigger) const heapPath = join(dir, `${base}.heapsnapshot`) const diagPath = join(dir, `${base}.diagnostics.json`) @@ -162,6 +162,23 @@ export async function performHeapDump(trigger: MemoryTrigger = 'manual'): Promis } } +export async function performDiagnosticsDump(trigger: MemoryTrigger = 'manual'): Promise { + try { + const diagnostics = await captureMemoryDiagnostics(trigger) + const dir = memoryDumpDir() + + await mkdir(dir, { recursive: true }) + + const diagPath = join(dir, `${memoryDumpBase(trigger)}.diagnostics.json`) + + await writeFile(diagPath, JSON.stringify(diagnostics, null, 2), { mode: 0o600 }) + + return { diagPath, success: true } + } catch (e) { + return { error: e instanceof Error ? e.message : String(e), success: false } + } +} + export function formatBytes(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) { return '0B' @@ -177,6 +194,11 @@ const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] const STARTED_AT = { rss: process.memoryUsage().rss, uptime: process.uptime() } +const memoryDumpDir = () => process.env.HERMES_HEAPDUMP_DIR?.trim() || join(homedir() || tmpdir(), '.hermes', 'heapdumps') + +const memoryDumpBase = (trigger: MemoryTrigger) => + `hermes-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}-${trigger}` + // Returns undefined when the probe isn't available (non-Linux paths, sandboxed FS). const swallow = async (fn: () => Promise): Promise => { try { diff --git a/ui-tui/src/lib/memoryMonitor.test.ts b/ui-tui/src/lib/memoryMonitor.test.ts new file mode 100644 index 00000000000..62f5bce98f6 --- /dev/null +++ b/ui-tui/src/lib/memoryMonitor.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const memory = vi.hoisted(() => ({ + performDiagnosticsDump: vi.fn(async () => ({ diagPath: '/tmp/diag.json', success: true })), + performHeapDump: vi.fn(async () => ({ heapPath: '/tmp/heap.heapsnapshot', success: true })) +})) + +vi.mock('./memory.js', () => memory) + +import { type MemorySnapshot, startMemoryMonitor } from './memoryMonitor.js' + +const GB = 1024 ** 3 + +const usage = (heapUsed: number, rss: number): NodeJS.MemoryUsage => + ({ + arrayBuffers: 0, + external: 0, + heapTotal: heapUsed, + heapUsed, + rss + }) as NodeJS.MemoryUsage + +describe('startMemoryMonitor', () => { + let memoryUsageSpy: ReturnType + + beforeEach(() => { + vi.useFakeTimers() + memory.performDiagnosticsDump.mockClear() + memory.performHeapDump.mockClear() + }) + + afterEach(() => { + memoryUsageSpy?.mockRestore() + vi.useRealTimers() + }) + + it('captures diagnostics only for native RSS pressure', async () => { + memoryUsageSpy = vi.spyOn(process, 'memoryUsage').mockReturnValue(usage(100 * 1024 ** 2, 5 * GB)) + + const snaps: MemorySnapshot[] = [] + + const stop = startMemoryMonitor({ + intervalMs: 1000, + onHigh: snap => snaps.push(snap), + rssHighBytes: 4 * GB + }) + + await vi.advanceTimersByTimeAsync(1000) + stop() + + expect(memory.performDiagnosticsDump).toHaveBeenCalledWith('auto-high') + expect(memory.performHeapDump).not.toHaveBeenCalled() + expect(snaps[0]).toMatchObject({ level: 'high', source: 'rss' }) + expect(snaps[0]?.nativeUsed).toBeGreaterThan(4 * GB) + }) + + it('keeps heap dumps for V8 heap pressure', async () => { + memoryUsageSpy = vi.spyOn(process, 'memoryUsage').mockReturnValue(usage(3 * GB, 3.5 * GB)) + + const snaps: MemorySnapshot[] = [] + + const stop = startMemoryMonitor({ + intervalMs: 1000, + onCritical: snap => snaps.push(snap) + }) + + await vi.advanceTimersByTimeAsync(1000) + stop() + + expect(memory.performHeapDump).toHaveBeenCalledWith('auto-critical') + expect(memory.performDiagnosticsDump).not.toHaveBeenCalled() + expect(snaps[0]).toMatchObject({ level: 'critical', source: 'heap' }) + }) +}) diff --git a/ui-tui/src/lib/memoryMonitor.ts b/ui-tui/src/lib/memoryMonitor.ts index 6655819b5a5..b649a6ab773 100644 --- a/ui-tui/src/lib/memoryMonitor.ts +++ b/ui-tui/src/lib/memoryMonitor.ts @@ -1,11 +1,14 @@ -import { type HeapDumpResult, performHeapDump } from './memory.js' +import { type HeapDumpResult, performDiagnosticsDump, performHeapDump } from './memory.js' export type MemoryLevel = 'critical' | 'high' | 'normal' +export type MemoryTriggerSource = 'heap' | 'rss' export interface MemorySnapshot { heapUsed: number level: MemoryLevel + nativeUsed: number rss: number + source: MemoryTriggerSource } export interface MemoryMonitorOptions { @@ -14,35 +17,61 @@ export interface MemoryMonitorOptions { intervalMs?: number onCritical?: (snap: MemorySnapshot, dump: HeapDumpResult | null) => void onHigh?: (snap: MemorySnapshot, dump: HeapDumpResult | null) => void + rssCriticalBytes?: number + rssHighBytes?: number } const GB = 1024 ** 3 +const maxLevel = (heapLevel: MemoryLevel, rssLevel: MemoryLevel): MemoryLevel => { + if (heapLevel === 'critical' || rssLevel === 'critical') { + return 'critical' + } + + return heapLevel === 'high' || rssLevel === 'high' ? 'high' : 'normal' +} + export function startMemoryMonitor({ criticalBytes = 2.5 * GB, highBytes = 1.5 * GB, intervalMs = 10_000, onCritical, - onHigh + onHigh, + rssCriticalBytes = 8 * GB, + rssHighBytes = 4 * GB }: MemoryMonitorOptions = {}): () => void { - const dumped = new Set>() + const dumped = new Set<`${MemoryTriggerSource}:${Exclude}`>() const tick = async () => { const { heapUsed, rss } = process.memoryUsage() - const level: MemoryLevel = heapUsed >= criticalBytes ? 'critical' : heapUsed >= highBytes ? 'high' : 'normal' + const nativeUsed = Math.max(0, rss - heapUsed) + const heapLevel: MemoryLevel = heapUsed >= criticalBytes ? 'critical' : heapUsed >= highBytes ? 'high' : 'normal' + const rssLevel: MemoryLevel = rss >= rssCriticalBytes ? 'critical' : rss >= rssHighBytes ? 'high' : 'normal' + const level = maxLevel(heapLevel, rssLevel) if (level === 'normal') { return void dumped.clear() } - if (dumped.has(level)) { + const source: MemoryTriggerSource = + heapLevel === level || (heapLevel !== 'normal' && rssLevel === level) ? 'heap' : 'rss' + + const key = `${source}:${level}` as const + + if (dumped.has(key)) { return } - dumped.add(level) - const dump = await performHeapDump(level === 'critical' ? 'auto-critical' : 'auto-high').catch(() => null) + dumped.add(key) - const snap: MemorySnapshot = { heapUsed, level, rss } + const trigger = level === 'critical' ? 'auto-critical' : 'auto-high' + + const dump = + source === 'heap' + ? await performHeapDump(trigger).catch(() => null) + : await performDiagnosticsDump(trigger).catch(() => null) + + const snap: MemorySnapshot = { heapUsed, level, nativeUsed, rss, source } ;(level === 'critical' ? onCritical : onHigh)?.(snap, dump) } diff --git a/ui-tui/src/lib/text.ts b/ui-tui/src/lib/text.ts index 8541ac3f685..4f363a2c0aa 100644 --- a/ui-tui/src/lib/text.ts +++ b/ui-tui/src/lib/text.ts @@ -1,4 +1,4 @@ -import { THINKING_COT_MAX } from '../config/limits.js' +import { LIVE_RENDER_MAX_CHARS, LIVE_RENDER_MAX_LINES, THINKING_COT_MAX } from '../config/limits.js' import type { ThinkingMode } from '../types.js' const ESC = String.fromCharCode(27) @@ -76,6 +76,61 @@ export const thinkingPreview = (reasoning: string, mode: ThinkingMode, max: numb return !raw || mode === 'collapsed' ? '' : mode === 'full' ? raw : compactPreview(raw.replace(WS_RE, ' '), max) } +export const boundedLiveRenderText = ( + text: string, + { maxChars = LIVE_RENDER_MAX_CHARS, maxLines = LIVE_RENDER_MAX_LINES } = {} +) => { + if (text.length <= maxChars && text.split('\n', maxLines + 1).length <= maxLines) { + return text + } + + let start = 0 + let idx = text.length + + for (let seen = 0; seen < maxLines && idx > 0; seen++) { + idx = text.lastIndexOf('\n', idx - 1) + start = idx < 0 ? 0 : idx + 1 + + if (idx < 0) { + break + } + } + + const lineStart = start + start = Math.max(lineStart, text.length - maxChars) + + if (start > lineStart) { + const nextBreak = text.indexOf('\n', start) + + if (nextBreak >= 0 && nextBreak < text.length - 1) { + start = nextBreak + 1 + } + } + + const tail = text.slice(start).trimStart() + const omittedLines = countNewlines(text, start) + const omittedChars = Math.max(0, text.length - tail.length) + + const label = + omittedLines > 0 + ? `[showing live tail; omitted ${fmtK(omittedLines)} lines / ${fmtK(omittedChars)} chars]\n` + : `[showing live tail; omitted ${fmtK(omittedChars)} chars]\n` + + return `${label}${tail.trimStart()}` +} + +const countNewlines = (text: string, end: number) => { + let count = 0 + + for (let i = 0; i < end; i++) { + if (text.charCodeAt(i) === 10) { + count++ + } + } + + return count +} + export const stripTrailingPasteNewlines = (text: string) => (/[^\n]/.test(text) ? text.replace(/\n+$/, '') : text) export const toolTrailLabel = (name: string) =>