mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-04-28 23:11:37 +08:00
feat: just more cleaning
This commit is contained in:
@@ -29,8 +29,16 @@ const dim = (s: string) => DIM + s + DIM_OFF
|
||||
|
||||
let _seg: Intl.Segmenter | null = null
|
||||
const seg = () => (_seg ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' }))
|
||||
const STOP_CACHE_MAX = 32
|
||||
const stopCache = new Map<string, number[]>()
|
||||
|
||||
function graphemeStops(s: string) {
|
||||
const hit = stopCache.get(s)
|
||||
|
||||
if (hit) {
|
||||
return hit
|
||||
}
|
||||
|
||||
const stops = [0]
|
||||
|
||||
for (const { index } of seg().segment(s)) {
|
||||
@@ -43,6 +51,16 @@ function graphemeStops(s: string) {
|
||||
stops.push(s.length)
|
||||
}
|
||||
|
||||
stopCache.set(s, stops)
|
||||
|
||||
if (stopCache.size > STOP_CACHE_MAX) {
|
||||
const oldest = stopCache.keys().next().value
|
||||
|
||||
if (oldest !== undefined) {
|
||||
stopCache.delete(oldest)
|
||||
}
|
||||
}
|
||||
|
||||
return stops
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user