"""Scale benchmarks for the Kanban kernel. Measures: - dispatch_once latency at 100, 1000, 10000 tasks - recompute_ready latency at 100, 1000, 10000 todo tasks with wide parent graphs - build_worker_context latency with 1, 10, 50 parent dependencies - board list/stats query latency - task_runs query latency at scale Results printed as a table. Saved to JSON for regression-diffing in CI or future reviews. Not a pass/fail test — records numbers so we know when a change regresses latency by 10x and can decide whether to care. """ import json import os import random import sys import tempfile import time from pathlib import Path WT = str(Path(__file__).resolve().parents[2]) def bench(label, fn, iterations=5): """Time fn over `iterations` runs, return (min, median, max) in ms.""" times = [] for _ in range(iterations): t0 = time.perf_counter() fn() times.append((time.perf_counter() - t0) * 1000) times.sort() mn = times[0] md = times[len(times) // 2] mx = times[-1] return {"label": label, "iter": iterations, "min_ms": mn, "median_ms": md, "max_ms": mx} def seed_tasks(conn, kb, n, assignee="bench-worker", with_parents=False): """Seed n tasks. Optionally give each task 5 parents.""" ids = [] for i in range(n): if with_parents and i >= 5: parents = random.sample(ids[:i], 5) else: parents = () tid = kb.create_task( conn, title=f"bench {i}", assignee=assignee, tenant="bench", parents=parents, ) ids.append(tid) return ids def main(): home = tempfile.mkdtemp(prefix="hermes_bench_") os.environ["HERMES_HOME"] = home os.environ["HOME"] = home sys.path.insert(0, WT) from hermes_cli import kanban_db as kb kb.init_db() results = [] # ============ dispatch_once latency ============ for n in [100, 1000, 10000]: print(f"\n== dispatch_once @ {n} tasks ==") # Fresh DB each time so we're not measuring cumulative effects import shutil shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() seed_tasks(conn, kb, n, assignee=None) # no assignee → won't spawn r = bench( f"dispatch_once (n={n}, no spawn)", lambda: kb.dispatch_once(conn, spawn_fn=lambda *_: None), iterations=5, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["n"] = n results.append(r) conn.close() # ============ recompute_ready at scale with parent graphs ============ for n in [100, 1000, 10000]: print(f"\n== recompute_ready @ {n} tasks (5 parents each) ==") shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() ids = seed_tasks(conn, kb, n, assignee=None, with_parents=True) # Complete the first 100 so some todo tasks might get promoted for tid in ids[:min(100, n // 10)]: kb.complete_task(conn, tid, result="bench") r = bench( f"recompute_ready (n={n}, with parents)", lambda: kb.recompute_ready(conn), iterations=5, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["n"] = n results.append(r) conn.close() # ============ build_worker_context with N parents ============ for parent_count in [1, 10, 50]: print(f"\n== build_worker_context with {parent_count} parents ==") shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() # Create parents, complete them with summaries+metadata parent_ids = [] for i in range(parent_count): pid = kb.create_task(conn, title=f"parent {i}", assignee="p") kb.claim_task(conn, pid) kb.complete_task( conn, pid, summary=f"parent {i} result that is longer than a single token " f"so we actually measure the IO", metadata={"files": [f"file_{j}.py" for j in range(5)], "i": i}, ) parent_ids.append(pid) child_id = kb.create_task( conn, title="child", assignee="c", parents=parent_ids, ) r = bench( f"build_worker_context (parents={parent_count})", lambda: kb.build_worker_context(conn, child_id), iterations=10, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["parent_count"] = parent_count results.append(r) conn.close() # ============ list_tasks at scale ============ for n in [100, 1000, 10000]: print(f"\n== list_tasks @ {n} ==") shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() seed_tasks(conn, kb, n) r = bench( f"list_tasks (n={n})", lambda: kb.list_tasks(conn), iterations=5, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["n"] = n results.append(r) conn.close() # ============ board_stats at scale ============ for n in [100, 1000, 10000]: print(f"\n== board_stats @ {n} ==") shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() seed_tasks(conn, kb, n) r = bench( f"board_stats (n={n})", lambda: kb.board_stats(conn), iterations=5, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["n"] = n results.append(r) conn.close() # ============ list_runs at scale ============ for n in [100, 1000]: print(f"\n== list_runs for task with {n} attempts ==") shutil.rmtree(home, ignore_errors=True) os.makedirs(home) kb._INITIALIZED_PATHS.clear() kb.init_db() conn = kb.connect() tid = kb.create_task(conn, title="x", assignee="w") # Create N attempts via claim/release for i in range(n): kb.claim_task(conn, tid, ttl_seconds=0) kb.release_stale_claims(conn) r = bench( f"list_runs (runs={n})", lambda: kb.list_runs(conn, tid), iterations=10, ) print(f" min={r['min_ms']:.1f} median={r['median_ms']:.1f} max={r['max_ms']:.1f} ms") r["run_count"] = n results.append(r) conn.close() # ============ SUMMARY TABLE ============ print() print("=" * 60) print("SUMMARY") print("=" * 60) print(f"{'Benchmark':<50} {'min':>8} {'median':>8} {'max':>8}") for r in results: print(f"{r['label']:<50} {r['min_ms']:>7.1f}ms {r['median_ms']:>7.1f}ms {r['max_ms']:>7.1f}ms") # Save for future diffing. out_path = "/tmp/kanban_bench_results.json" with open(out_path, "w") as f: json.dump(results, f, indent=2) print(f"\nResults saved to {out_path}") if __name__ == "__main__": main()