diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd1f89c..85f87c32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`hawk`, `eyrie`, `yaad`, `sight`, `inspect`). No git tag has been cut yet. ### Removed +- Unused internal config structs `QuestionAwareLayerConfig`, + `DensityAdaptiveLayerConfig`, `NumericalQuantLayerConfig`, and + `DynamicRatioLayerConfig` (and their fields on the internal `LayerConfig`) + that were marked "reserved for future implementation" and never referenced. - `scripts/deploy.sh` — referenced a Dockerfile, k8s manifests, and an `internal/integration/` package that never existed. tok is a Go library (no binary, no HTTP server); deployment happens via `go get`. diff --git a/internal/fastops/simd.go b/internal/fastops/simd.go index c49f594f..7edf368f 100644 --- a/internal/fastops/simd.go +++ b/internal/fastops/simd.go @@ -168,15 +168,18 @@ func FastLower(s string) string { return s } - // Convert using unsafe for zero-copy when possible - b := []byte(s) + // Convert using unsafe to avoid a second copy on the way back to string. + b := []byte(s) // fresh copy of s; we own this backing array exclusively for i := 0; i < len(b); i++ { c := b[i] if c >= 'A' && c <= 'Z' { b[i] = c + ('a' - 'A') } } - return *(*string)(unsafe.Pointer(&b)) + // safe: b is a private copy created above; no other reference to its + // backing array exists and it is never mutated after this point, so + // aliasing it as a string preserves string immutability. + return unsafe.String(unsafe.SliceData(b), len(b)) } // FastEqual compares strings with early exit diff --git a/internal/filter/pipeline_types.go b/internal/filter/pipeline_types.go index ca7ceb26..7844d3a5 100644 --- a/internal/filter/pipeline_types.go +++ b/internal/filter/pipeline_types.go @@ -225,41 +225,12 @@ type AgentMemoryLayerConfig struct { ConsolidationMax int } -// QuestionAwareLayerConfig groups T12 settings. -// NOTE: Currently unused - reserved for future implementation. -type QuestionAwareLayerConfig struct { - Enabled bool - Threshold float64 -} - -// DensityAdaptiveLayerConfig groups T17 settings. -// NOTE: Currently unused - reserved for future implementation. -type DensityAdaptiveLayerConfig struct { - Enabled bool - TargetRatio float64 - Threshold float64 -} - // TFIDFLayerConfig groups TF-IDF filter settings. type TFIDFLayerConfig struct { Enabled bool Threshold float64 } -// NumericalQuantLayerConfig groups numerical quantization settings. -// NOTE: Currently unused - reserved for future implementation. -type NumericalQuantLayerConfig struct { - Enabled bool - DecimalPlaces int -} - -// DynamicRatioLayerConfig groups dynamic compression ratio settings. -// NOTE: Currently unused - reserved for future implementation. -type DynamicRatioLayerConfig struct { - Enabled bool - Base float64 -} - // LayerConfig groups per-layer config structs. type LayerConfig struct { Core CoreLayersConfig @@ -273,11 +244,7 @@ type LayerConfig struct { LazyPruner LazyPrunerLayerConfig SemanticAnchor SemanticAnchorLayerConfig AgentMemory AgentMemoryLayerConfig - QuestionAware QuestionAwareLayerConfig - DensityAdaptive DensityAdaptiveLayerConfig TFIDF TFIDFLayerConfig - NumericalQuant NumericalQuantLayerConfig - DynamicRatio DynamicRatioLayerConfig SymbolicCompress bool PhraseGrouping bool Hypernym bool diff --git a/internal/filter/zerocopy.go b/internal/filter/zerocopy.go index 1c11cc66..4dbb27d3 100644 --- a/internal/filter/zerocopy.go +++ b/internal/filter/zerocopy.go @@ -32,7 +32,10 @@ func (z *ZeroCopyBuffer) String() string { if len(z.data) == 0 { return "" } - return *(*string)(unsafe.Pointer(&z.data)) + // safe: the string header aliases z.data's backing array. The invariant + // (documented above) is that the backing array is not mutated while the + // returned string is live — Append/Reset after String() violate it. + return unsafe.String(unsafe.SliceData(z.data), len(z.data)) } func (z *ZeroCopyBuffer) Reset() { @@ -51,7 +54,11 @@ func StringToBytes(s string) []byte { if len(s) == 0 { return nil } - return *(*[]byte)(unsafe.Pointer(&s)) + // safe: the slice aliases the string's backing array (len == cap == len(s)). + // The invariant is that the backing array is never mutated through the + // returned slice — callers treat it as read-only, preserving string + // immutability. + return unsafe.Slice(unsafe.StringData(s), len(s)) } // BytesToString converts []byte to string without allocation. @@ -62,5 +69,8 @@ func BytesToString(b []byte) string { if len(b) == 0 { return "" } - return *(*string)(unsafe.Pointer(&b)) + // safe: the string header aliases b's backing array. The invariant is that + // the backing array is not mutated (through b or any other alias) while the + // returned string is live, preserving string immutability. + return unsafe.String(unsafe.SliceData(b), len(b)) } diff --git a/internal/tracking/tracking.go b/internal/tracking/tracking.go index 45c90baf..bd0aa821 100644 --- a/internal/tracking/tracking.go +++ b/internal/tracking/tracking.go @@ -44,6 +44,7 @@ type Event struct { // Tracker is the public type. Construct with New() or NewAt(path). type Tracker struct { db *sql.DB + path string mu sync.Mutex retenDays int closed bool @@ -77,16 +78,16 @@ func NewAt(ctx context.Context, path string) (*Tracker, error) { return nil, errors.New("tracking: empty path") } if err := mkdirAll(filepath.Dir(path), 0o700); err != nil { - return nil, fmt.Errorf("tracking: create dir: %w", err) + return nil, fmt.Errorf("tracking: create dir %q: %w", filepath.Dir(path), err) } db, err := sql.Open("sqlite", path) if err != nil { - return nil, fmt.Errorf("tracking: open db: %w", err) + return nil, fmt.Errorf("tracking: open db at %q: %w", path, err) } // SQLite is single-writer; use a per-connection mutex to serialize // writes without serializing reads. db.SetMaxOpenConns(1) - t := &Tracker{db: db, retenDays: 90} + t := &Tracker{db: db, path: path, retenDays: 90} if err := t.initSchema(ctx); err != nil { _ = db.Close() return nil, err @@ -127,7 +128,7 @@ func (t *Tracker) initSchema(ctx context.Context) error { CREATE INDEX IF NOT EXISTS idx_events_command ON events(command); `) if err != nil { - return fmt.Errorf("tracking: init schema: %w", err) + return fmt.Errorf("tracking: init schema at %q: %w", t.path, err) } return nil } @@ -137,7 +138,7 @@ func (t *Tracker) initSchema(ctx context.Context) error { func (t *Tracker) enableWAL(ctx context.Context) error { _, err := t.db.ExecContext(ctx, `PRAGMA journal_mode=WAL;`) if err != nil { - return fmt.Errorf("tracking: enable WAL: %w", err) + return fmt.Errorf("tracking: enable WAL at %q: %w", t.path, err) } return nil } @@ -166,7 +167,7 @@ func (t *Tracker) Record(ctx context.Context, ev Event) error { ev.Mode, ev.Tier, ev.Model, ) if err != nil { - return fmt.Errorf("tracking: record: %w", err) + return fmt.Errorf("tracking: record event in db at %q: %w", t.path, err) } return nil } @@ -209,7 +210,7 @@ func (t *Tracker) Aggregate(ctx context.Context, days int) (Aggregate, error) { WHERE ts >= ? `, since) if err := row.Scan(&agg.EventCount, &agg.TotalBytesSaved, &agg.AvgSavingsPct); err != nil { - return agg, fmt.Errorf("tracking: aggregate: %w", err) + return agg, fmt.Errorf("tracking: aggregate query on db at %q: %w", t.path, err) } agg.PeriodStart = time.Unix(since, 0) agg.PeriodEnd = time.Now() @@ -235,7 +236,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) { LIMIT ? `, n) if err != nil { - return nil, fmt.Errorf("tracking: recent: %w", err) + return nil, fmt.Errorf("tracking: recent query on db at %q: %w", t.path, err) } defer func() { _ = rows.Close() }() var out []Event @@ -246,7 +247,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) { &ev.OriginalBytes, &ev.CompressedBytes, &ev.OriginalTokens, &ev.CompressedTokens, &ev.Mode, &ev.Tier, &ev.Model); err != nil { - return nil, err + return nil, fmt.Errorf("tracking: scan event row from db at %q: %w", t.path, err) } ev.Timestamp = time.Unix(ts, 0) if ev.OriginalBytes > 0 { @@ -266,7 +267,7 @@ func (t *Tracker) Prune(ctx context.Context) (int64, error) { cutoff := time.Now().Add(-time.Duration(t.retenDays) * 24 * time.Hour).Unix() res, err := t.db.ExecContext(ctx, `DELETE FROM events WHERE ts < ?`, cutoff) if err != nil { - return 0, fmt.Errorf("tracking: prune: %w", err) + return 0, fmt.Errorf("tracking: prune db at %q: %w", t.path, err) } return res.RowsAffected() }