fix: estimator cache memory, RestorationTracker bounds, remove dead ParallelExecutor

CHANGELOG.md

@@ -16,11 +16,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed
+- **BPE estimator cache no longer retains input texts** (`internal/core`).
+  The 8192-entry sharded LRU previously stored the full original string per
+  entry for collision checks and copied every input to `[]byte` on each hash.
+  It now hashes with `hash/maphash` (zero-copy, random per-process seed),
+  validates hits by `(hash, length)`, and skips caching texts above a 1 MiB
+  cutoff. This bounds the cache at small fixed-size records regardless of
+  prompt size — previously, `Compress` estimating the original plus every
+  intermediate layer output could pin hundreds of MB in the cache.
+
+### Fixed
+- **`RestorationTracker` no longer retains file contents unboundedly.**
+  `Track` previously did an O(n) linear dedup scan per call (O(n²) over a
+  session) and kept every tracked file's full content in memory until
+  `Clear()`. Dedup is now O(1) via a path-indexed map, and total retained
+  content is capped by a token budget (`DefaultRetainedTokenBudget`,
+  200_000 tokens — one default context window, ~800 KB) that evicts the
+  least recently tracked entries; configurable via
+  `WithRetainedTokenBudget`. The most recently tracked entry is always kept.
 - **Version re-baselined to `0.1.0`** across `CITATION.cff` and the embedded
   `Version` constant, aligning tok with the rest of the hawk-eco ecosystem
   (`hawk`, `eyrie`, `yaad`, `sight`, `inspect`). No git tag has been cut yet.
 
 ### Removed
+- Dead `ParallelExecutor` (`internal/filter`): ran every layer on the full
+  input concurrently and kept the shortest output. No production callers in
+  tok or hawk, no tests, and it measured output size in bytes rather than
+  tokens, so it could never be wired into the token-budget pipeline as-is.
 - Unused internal config structs `QuestionAwareLayerConfig`,
   `DensityAdaptiveLayerConfig`, `NumericalQuantLayerConfig`, and
   `DynamicRatioLayerConfig` (and their fields on the internal `LayerConfig`)

internal/core/estimator.go

@@ -3,7 +3,7 @@ package core
 import (
 	"container/list"
 	"fmt"
-	"hash/fnv"
+	"hash/maphash"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -21,12 +21,28 @@ type BPETokenizer struct {
 }
 
 // cacheEntry stores cached token counts with LRU metadata.
+// It deliberately does NOT retain the original text: Compress estimates the
+// original prompt plus every intermediate layer output, so pinning full texts
+// in the LRU could retain hundreds of MB. Hit validation relies on the
+// 64-bit maphash plus the text length (see hashText).
 type cacheEntry struct {
-	count int
-	text  string        // original text for collision detection
-	elem  *list.Element // pointer to list element for O(1) removal
+	count  int
+	length int           // text length, secondary collision check
+	elem   *list.Element // pointer to list element for O(1) removal
 }
 
+// cacheSeed is generated once per process. The token cache is purely
+// in-process, so a random seed is sufficient for correctness and avoids
+// fixed-seed hash-flooding concerns.
+var cacheSeed = maphash.MakeSeed()
+
+// maxCacheableTextBytes is the size cutoff above which texts bypass the
+// cache entirely. Giant inputs are rare in steady state, dominate the cache's
+// memory footprint if admitted, and their Count cost is dominated by the BPE
+// encode itself, so caching them buys little. 1 MiB bounds the worst case
+// while still covering every realistic prompt, layer output, or file read.
+const maxCacheableTextBytes = 1 << 20 // 1 MiB
+
 // lruItem holds the key for list element tracking.
 type lruItem struct {
 	key uint64
@@ -69,21 +85,31 @@ func (c *tokenCache) getShard(key uint64) *tokenCacheShard {
 	return &c.shards[key%64]
 }
 
+// hashText returns a 64-bit hash of text without copying it.
+// maphash.String reads the string's backing bytes directly (no []byte(text)
+// conversion) and is seeded per process. Collision handling: entries are
+// additionally validated by text length, so a false hit requires two distinct
+// texts with identical length AND identical 64-bit hash. With <=8192 cached
+// entries the birthday-bound probability of that is ~1e-12 per process —
+// negligible next to the heuristic fallback path — so no stronger check
+// (e.g. content prefix) is retained.
 func hashText(text string) uint64 {
-	h := fnv.New64a()
-	_, _ = h.Write([]byte(text)) // #nosec G104 -- hash.Hash.Write never returns an error
-	return h.Sum64()
+	return maphash.String(cacheSeed, text)
 }
 
 // get retrieves a cached token count and promotes the entry in the LRU list.
 // Uses per-shard locking to reduce contention across concurrent goroutines.
-// Compares original text to handle FNV-64a hash collisions.
+// Validates (hash, length) — the cache never stores the text itself.
+// Texts larger than maxCacheableTextBytes always miss.
 func (c *tokenCache) get(text string) (int, bool) {
+	if len(text) > maxCacheableTextBytes {
+		return 0, false
+	}
 	key := hashText(text)
 	shard := c.getShard(key)
 	shard.mu.Lock()
 	entry, ok := shard.items[key]
-	if !ok || entry.text != text {
+	if !ok || entry.length != len(text) {
 		shard.mu.Unlock()
 		return 0, false
 	}
@@ -96,12 +122,15 @@ func (c *tokenCache) get(text string) (int, bool) {
 }
 
 func (c *tokenCache) set(text string, count int) {
+	if len(text) > maxCacheableTextBytes {
+		return // giant inputs never enter the LRU
+	}
 	key := hashText(text)
 	shard := c.getShard(key)
 	shard.mu.Lock()
 	defer shard.mu.Unlock()
 
-	if entry, ok := shard.items[key]; ok && entry.text == text {
+	if entry, ok := shard.items[key]; ok && entry.length == len(text) {
 		entry.count = count
 		shard.ll.MoveToFront(entry.elem)
 		return
@@ -113,9 +142,9 @@ func (c *tokenCache) set(text string, count int) {
 
 	elem := shard.ll.PushFront(&lruItem{key: key})
 	shard.items[key] = &cacheEntry{
-		count: count,
-		text:  text,
-		elem:  elem,
+		count:  count,
+		length: len(text),
+		elem:   elem,
 	}
 }
 

internal/core/estimator_test.go

@@ -1,6 +1,11 @@
 package core
 
-import "testing"
+import (
+	"reflect"
+	"strings"
+	"sync/atomic"
+	"testing"
+)
 
 func TestEstimateTokensExact(t *testing.T) {
 	t.Parallel()
@@ -115,3 +120,77 @@ func TestEstimateTokensPreciseShortString(t *testing.T) {
 	t.Logf("short=%q fast=%d precise=%d (difference acceptable; precise is authoritative)",
 		short, fast, precise)
 }
+
+// TestTokenCacheHitAcrossCalls verifies the LRU cache returns a cached count
+// for identical text across separate calls (and bumps the hit counter).
+func TestTokenCacheHitAcrossCalls(t *testing.T) {
+	t.Parallel()
+	c := newTokenCache(64)
+	c.set("hello token cache world", 5)
+
+	got, ok := c.get("hello token cache world")
+	if !ok {
+		t.Fatal("expected cache hit for identical text")
+	}
+	if got != 5 {
+		t.Errorf("cached count = %d, want 5", got)
+	}
+	if hits := atomic.LoadInt64(&c.hits); hits != 1 {
+		t.Errorf("hits = %d, want 1", hits)
+	}
+}
+
+// TestTokenCacheSameLengthDifferentTextMisses verifies hit validation:
+// two distinct texts of identical length must not collide.
+func TestTokenCacheSameLengthDifferentTextMisses(t *testing.T) {
+	t.Parallel()
+	c := newTokenCache(64)
+	a := "aaaa first text"
+	b := "bbbb second one" // same length, different content
+	if len(a) != len(b) {
+		t.Fatalf("test bug: lengths differ (%d vs %d)", len(a), len(b))
+	}
+	c.set(a, 7)
+
+	if _, ok := c.get(b); ok {
+		t.Error("different text with same length must be a cache miss")
+	}
+	if _, ok := c.get(a); !ok {
+		t.Error("original text must still hit after miss probe")
+	}
+}
+
+// TestTokenCacheOversizeTextBypassed verifies texts above the size cutoff
+// never enter the LRU, so giant inputs cannot pin cache memory.
+func TestTokenCacheOversizeTextBypassed(t *testing.T) {
+	t.Parallel()
+	c := newTokenCache(64)
+	huge := strings.Repeat("x", maxCacheableTextBytes+1)
+
+	c.set(huge, 12345)
+	if _, ok := c.get(huge); ok {
+		t.Errorf("text of %d bytes (> cutoff %d) must bypass the cache",
+			len(huge), maxCacheableTextBytes)
+	}
+
+	// The boundary itself must still be cacheable.
+	boundary := strings.Repeat("y", maxCacheableTextBytes)
+	c.set(boundary, 42)
+	if got, ok := c.get(boundary); !ok || got != 42 {
+		t.Errorf("text exactly at cutoff must hit, got (count=%d, ok=%v)", got, ok)
+	}
+}
+
+// TestCacheEntryDoesNotRetainText pins the memory fix: cacheEntry must not
+// carry any string field. Storing the original text meant the 8192-entry LRU
+// retained full prompts and every intermediate layer output.
+func TestCacheEntryDoesNotRetainText(t *testing.T) {
+	t.Parallel()
+	typ := reflect.TypeOf(cacheEntry{})
+	for i := 0; i < typ.NumField(); i++ {
+		if typ.Field(i).Type.Kind() == reflect.String {
+			t.Errorf("cacheEntry field %q is a string — the cache must not retain text",
+				typ.Field(i).Name)
+		}
+	}
+}