From 23b51b227e37d135aba2a4fe5a18e019517e0944 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 11 Jun 2026 12:19:44 +0530 Subject: [PATCH 1/3] refactor(filter): remove unused reserved layer config types Remove QuestionAwareLayerConfig, DensityAdaptiveLayerConfig, NumericalQuantLayerConfig, and DynamicRatioLayerConfig (all marked "reserved for future implementation") plus their never-read fields on LayerConfig. Verified unreferenced via grep across the repo. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 ++++ internal/filter/pipeline_types.go | 33 ------------------------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd1f89c1..85f87c320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`hawk`, `eyrie`, `yaad`, `sight`, `inspect`). No git tag has been cut yet. ### Removed +- Unused internal config structs `QuestionAwareLayerConfig`, + `DensityAdaptiveLayerConfig`, `NumericalQuantLayerConfig`, and + `DynamicRatioLayerConfig` (and their fields on the internal `LayerConfig`) + that were marked "reserved for future implementation" and never referenced. - `scripts/deploy.sh` — referenced a Dockerfile, k8s manifests, and an `internal/integration/` package that never existed. tok is a Go library (no binary, no HTTP server); deployment happens via `go get`. diff --git a/internal/filter/pipeline_types.go b/internal/filter/pipeline_types.go index ca7ceb26a..7844d3a5b 100644 --- a/internal/filter/pipeline_types.go +++ b/internal/filter/pipeline_types.go @@ -225,41 +225,12 @@ type AgentMemoryLayerConfig struct { ConsolidationMax int } -// QuestionAwareLayerConfig groups T12 settings. -// NOTE: Currently unused - reserved for future implementation. -type QuestionAwareLayerConfig struct { - Enabled bool - Threshold float64 -} - -// DensityAdaptiveLayerConfig groups T17 settings. -// NOTE: Currently unused - reserved for future implementation. -type DensityAdaptiveLayerConfig struct { - Enabled bool - TargetRatio float64 - Threshold float64 -} - // TFIDFLayerConfig groups TF-IDF filter settings. type TFIDFLayerConfig struct { Enabled bool Threshold float64 } -// NumericalQuantLayerConfig groups numerical quantization settings. -// NOTE: Currently unused - reserved for future implementation. -type NumericalQuantLayerConfig struct { - Enabled bool - DecimalPlaces int -} - -// DynamicRatioLayerConfig groups dynamic compression ratio settings. -// NOTE: Currently unused - reserved for future implementation. -type DynamicRatioLayerConfig struct { - Enabled bool - Base float64 -} - // LayerConfig groups per-layer config structs. type LayerConfig struct { Core CoreLayersConfig @@ -273,11 +244,7 @@ type LayerConfig struct { LazyPruner LazyPrunerLayerConfig SemanticAnchor SemanticAnchorLayerConfig AgentMemory AgentMemoryLayerConfig - QuestionAware QuestionAwareLayerConfig - DensityAdaptive DensityAdaptiveLayerConfig TFIDF TFIDFLayerConfig - NumericalQuant NumericalQuantLayerConfig - DynamicRatio DynamicRatioLayerConfig SymbolicCompress bool PhraseGrouping bool Hypernym bool From 64be20385c7951491e041e26e652da7f35a2df6e Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 11 Jun 2026 12:20:40 +0530 Subject: [PATCH 2/3] fix(tracking): include db path and operation in wrapped errors All error paths in tracking.go now report the database path and the operation that failed (open, create dir, init schema, enable WAL, record, aggregate, recent, scan, prune), making failures diagnosable without enabling debug logging. Co-Authored-By: Claude Fable 5 --- internal/tracking/tracking.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/tracking/tracking.go b/internal/tracking/tracking.go index 45c90baf1..bd0aa8219 100644 --- a/internal/tracking/tracking.go +++ b/internal/tracking/tracking.go @@ -44,6 +44,7 @@ type Event struct { // Tracker is the public type. Construct with New() or NewAt(path). type Tracker struct { db *sql.DB + path string mu sync.Mutex retenDays int closed bool @@ -77,16 +78,16 @@ func NewAt(ctx context.Context, path string) (*Tracker, error) { return nil, errors.New("tracking: empty path") } if err := mkdirAll(filepath.Dir(path), 0o700); err != nil { - return nil, fmt.Errorf("tracking: create dir: %w", err) + return nil, fmt.Errorf("tracking: create dir %q: %w", filepath.Dir(path), err) } db, err := sql.Open("sqlite", path) if err != nil { - return nil, fmt.Errorf("tracking: open db: %w", err) + return nil, fmt.Errorf("tracking: open db at %q: %w", path, err) } // SQLite is single-writer; use a per-connection mutex to serialize // writes without serializing reads. db.SetMaxOpenConns(1) - t := &Tracker{db: db, retenDays: 90} + t := &Tracker{db: db, path: path, retenDays: 90} if err := t.initSchema(ctx); err != nil { _ = db.Close() return nil, err @@ -127,7 +128,7 @@ func (t *Tracker) initSchema(ctx context.Context) error { CREATE INDEX IF NOT EXISTS idx_events_command ON events(command); `) if err != nil { - return fmt.Errorf("tracking: init schema: %w", err) + return fmt.Errorf("tracking: init schema at %q: %w", t.path, err) } return nil } @@ -137,7 +138,7 @@ func (t *Tracker) initSchema(ctx context.Context) error { func (t *Tracker) enableWAL(ctx context.Context) error { _, err := t.db.ExecContext(ctx, `PRAGMA journal_mode=WAL;`) if err != nil { - return fmt.Errorf("tracking: enable WAL: %w", err) + return fmt.Errorf("tracking: enable WAL at %q: %w", t.path, err) } return nil } @@ -166,7 +167,7 @@ func (t *Tracker) Record(ctx context.Context, ev Event) error { ev.Mode, ev.Tier, ev.Model, ) if err != nil { - return fmt.Errorf("tracking: record: %w", err) + return fmt.Errorf("tracking: record event in db at %q: %w", t.path, err) } return nil } @@ -209,7 +210,7 @@ func (t *Tracker) Aggregate(ctx context.Context, days int) (Aggregate, error) { WHERE ts >= ? `, since) if err := row.Scan(&agg.EventCount, &agg.TotalBytesSaved, &agg.AvgSavingsPct); err != nil { - return agg, fmt.Errorf("tracking: aggregate: %w", err) + return agg, fmt.Errorf("tracking: aggregate query on db at %q: %w", t.path, err) } agg.PeriodStart = time.Unix(since, 0) agg.PeriodEnd = time.Now() @@ -235,7 +236,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) { LIMIT ? `, n) if err != nil { - return nil, fmt.Errorf("tracking: recent: %w", err) + return nil, fmt.Errorf("tracking: recent query on db at %q: %w", t.path, err) } defer func() { _ = rows.Close() }() var out []Event @@ -246,7 +247,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) { &ev.OriginalBytes, &ev.CompressedBytes, &ev.OriginalTokens, &ev.CompressedTokens, &ev.Mode, &ev.Tier, &ev.Model); err != nil { - return nil, err + return nil, fmt.Errorf("tracking: scan event row from db at %q: %w", t.path, err) } ev.Timestamp = time.Unix(ts, 0) if ev.OriginalBytes > 0 { @@ -266,7 +267,7 @@ func (t *Tracker) Prune(ctx context.Context) (int64, error) { cutoff := time.Now().Add(-time.Duration(t.retenDays) * 24 * time.Hour).Unix() res, err := t.db.ExecContext(ctx, `DELETE FROM events WHERE ts < ?`, cutoff) if err != nil { - return 0, fmt.Errorf("tracking: prune: %w", err) + return 0, fmt.Errorf("tracking: prune db at %q: %w", t.path, err) } return res.RowsAffected() } From b0e9740e854e0e3d2c3893037ceec40b14ff9a2d Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Thu, 11 Jun 2026 12:22:34 +0530 Subject: [PATCH 3/3] docs(unsafe): add safety-invariant comments at unsafe conversions Each unsafe string<->[]byte conversion in internal/fastops/simd.go and internal/filter/zerocopy.go now carries an explicit "safe: ..." comment stating the aliasing invariant (backing array not mutated while the alias is live). Also migrated the legacy *(*string)(unsafe.Pointer(&b)) header-punning pattern to the supported unsafe.String/unsafe.Slice + unsafe.SliceData/StringData APIs, which are valid under the Go memory model (the old slice->string pun read a slice header as a string header). Co-Authored-By: Claude Fable 5 --- internal/fastops/simd.go | 9 ++++++--- internal/filter/zerocopy.go | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/fastops/simd.go b/internal/fastops/simd.go index c49f594fc..7edf368fd 100644 --- a/internal/fastops/simd.go +++ b/internal/fastops/simd.go @@ -168,15 +168,18 @@ func FastLower(s string) string { return s } - // Convert using unsafe for zero-copy when possible - b := []byte(s) + // Convert using unsafe to avoid a second copy on the way back to string. + b := []byte(s) // fresh copy of s; we own this backing array exclusively for i := 0; i < len(b); i++ { c := b[i] if c >= 'A' && c <= 'Z' { b[i] = c + ('a' - 'A') } } - return *(*string)(unsafe.Pointer(&b)) + // safe: b is a private copy created above; no other reference to its + // backing array exists and it is never mutated after this point, so + // aliasing it as a string preserves string immutability. + return unsafe.String(unsafe.SliceData(b), len(b)) } // FastEqual compares strings with early exit diff --git a/internal/filter/zerocopy.go b/internal/filter/zerocopy.go index 1c11cc66b..4dbb27d3b 100644 --- a/internal/filter/zerocopy.go +++ b/internal/filter/zerocopy.go @@ -32,7 +32,10 @@ func (z *ZeroCopyBuffer) String() string { if len(z.data) == 0 { return "" } - return *(*string)(unsafe.Pointer(&z.data)) + // safe: the string header aliases z.data's backing array. The invariant + // (documented above) is that the backing array is not mutated while the + // returned string is live — Append/Reset after String() violate it. + return unsafe.String(unsafe.SliceData(z.data), len(z.data)) } func (z *ZeroCopyBuffer) Reset() { @@ -51,7 +54,11 @@ func StringToBytes(s string) []byte { if len(s) == 0 { return nil } - return *(*[]byte)(unsafe.Pointer(&s)) + // safe: the slice aliases the string's backing array (len == cap == len(s)). + // The invariant is that the backing array is never mutated through the + // returned slice — callers treat it as read-only, preserving string + // immutability. + return unsafe.Slice(unsafe.StringData(s), len(s)) } // BytesToString converts []byte to string without allocation. @@ -62,5 +69,8 @@ func BytesToString(b []byte) string { if len(b) == 0 { return "" } - return *(*string)(unsafe.Pointer(&b)) + // safe: the string header aliases b's backing array. The invariant is that + // the backing array is not mutated (through b or any other alias) while the + // returned string is live, preserving string immutability. + return unsafe.String(unsafe.SliceData(b), len(b)) }