From da63aff9bb0e86860ee2ad41044c3b4de29e69e8 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Wed, 22 Jul 2026 21:50:42 +0200 Subject: [PATCH] fix(llm): parse DeepSeek cache metrics + honest 'n/a' when unreported The Telegram stats line showed 'cache: 0 write / 0 read / 0 total' on every turn. Two issues: 1. odek parsed Anthropic (cache_creation/read_input_tokens) and OpenAI (prompt_tokens_details.cached_tokens) cache fields, but not DeepSeek's native prompt_cache_hit_tokens / prompt_cache_miss_tokens, which many DeepSeek endpoints and gateways return INSTEAD of the OpenAI-style details. DeepSeek hits now map to cache read, misses to cache write (DeepSeek caches missed prefixes automatically). 2. Zeros were displayed even when the provider returned NO cache metrics at all, falsely implying caching ran and missed. A new CacheReported flag (llm.CallResult -> loop totals -> IterationInfo) distinguishes 'no data' from '0 tokens', and the Telegram stats line now shows 'cache: n/a (provider reports no cache metrics)' instead of fake zeros. --- cmd/odek/telegram.go | 13 ++++++++++--- cmd/odek/telegram_test.go | 13 +++++++++++++ internal/llm/client.go | 21 ++++++++++++++++++++ internal/llm/client_test.go | 39 +++++++++++++++++++++++++++++++++++++ internal/loop/loop.go | 13 ++++++++++--- 5 files changed, 93 insertions(+), 6 deletions(-) diff --git a/cmd/odek/telegram.go b/cmd/odek/telegram.go index 65a6fea8..f8ec0aae 100644 --- a/cmd/odek/telegram.go +++ b/cmd/odek/telegram.go @@ -2138,9 +2138,16 @@ func formatTelegramStats(info loop.IterationInfo, toolList []string) string { iters += "s" } - // Always include cache stats so the user can see them even when zero. - cacheStr := fmt.Sprintf(" · cache: %d write / %d read / %d total", - info.CacheCreationTokens, info.CacheReadTokens, info.CachedTokens) + // Show real numbers when the provider reports cache metrics; otherwise + // say so — "0 write / 0 read" would wrongly imply caching ran and + // missed, when in fact the provider returned no data at all. + var cacheStr string + if info.CacheReported { + cacheStr = fmt.Sprintf(" · cache: %d write / %d read / %d total", + info.CacheCreationTokens, info.CacheReadTokens, info.CachedTokens) + } else { + cacheStr = " · cache: n/a (provider reports no cache metrics)" + } return fmt.Sprintf( "```\n✅ Done · %s · %d in / %d out%s · %s — tools: %s\n```", diff --git a/cmd/odek/telegram_test.go b/cmd/odek/telegram_test.go index b7aca8fc..1b7f7771 100644 --- a/cmd/odek/telegram_test.go +++ b/cmd/odek/telegram_test.go @@ -1163,6 +1163,7 @@ func TestFormatTelegramStats(t *testing.T) { CacheCreationTokens: 10, CacheReadTokens: 20, CachedTokens: 30, + CacheReported: true, TotalLatency: 5*time.Second + 500*time.Millisecond, } out := formatTelegramStats(info, []string{"read_file", "shell"}) @@ -1189,6 +1190,18 @@ func TestFormatTelegramStats_SingularTurn(t *testing.T) { } } +// TestFormatTelegramStats_CacheNotReported verifies the honest display when +// the provider returned no cache metrics at all (vs. real zeros). +func TestFormatTelegramStats_CacheNotReported(t *testing.T) { + out := formatTelegramStats(loop.IterationInfo{Turn: 1}, nil) + if !strings.Contains(out, "cache: n/a") { + t.Errorf("expected cache n/a when provider reports nothing, got: %s", out) + } + if strings.Contains(out, "write") || strings.Contains(out, "read /") { + t.Errorf("must not show fake zero cache numbers when unreported: %s", out) + } +} + // TestFormatStopSummary verifies the /stop summary formatting. func TestFormatStopSummary(t *testing.T) { info := loop.IterationInfo{ diff --git a/internal/llm/client.go b/internal/llm/client.go index 62dfe3cd..cb4f9673 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -135,9 +135,13 @@ type CallResult struct { // Cache metrics. Only populated when the provider returns them. // Anthropic: cache_creation_input_tokens, cache_read_input_tokens // OpenAI: prompt_tokens_details.cached_tokens + // DeepSeek: prompt_cache_hit_tokens (read), prompt_cache_miss_tokens (write) CacheCreationTokens int // Anthropic — tokens written to cache CacheReadTokens int // Anthropic — tokens read from cache hit CachedTokens int // OpenAI — cached tokens in prompt + // CacheReported is true when the provider returned any cache metrics at + // all; false means "no data", which is different from "0 tokens cached". + CacheReported bool } // toolChoiceNone forces the model to not call tools. @@ -451,6 +455,11 @@ func parseResponse(data []byte) (*CallResult, error) { PromptTokensDetails *struct { CachedTokens int `json:"cached_tokens"` } `json:"prompt_tokens_details"` + // DeepSeek native prompt caching (always present on DeepSeek + // endpoints, unlike prompt_tokens_details which varies by + // gateway/proxy). + PromptCacheHitTokens int `json:"prompt_cache_hit_tokens"` + PromptCacheMissTokens int `json:"prompt_cache_miss_tokens"` } `json:"usage"` } if err := json.Unmarshal(data, &raw); err != nil { @@ -472,6 +481,18 @@ func parseResponse(data []byte) (*CallResult, error) { result.CacheReadTokens = raw.Usage.CacheReadTokens if raw.Usage.PromptTokensDetails != nil { result.CachedTokens = raw.Usage.PromptTokensDetails.CachedTokens + result.CacheReported = true + } + if raw.Usage.CacheCreationTokens > 0 || raw.Usage.CacheReadTokens > 0 { + result.CacheReported = true + } + // DeepSeek native fields: a hit is prompt content read from cache; + // a miss is newly processed content that DeepSeek then caches for + // future requests, i.e. a cache write. + if raw.Usage.PromptCacheHitTokens > 0 || raw.Usage.PromptCacheMissTokens > 0 { + result.CacheReadTokens += raw.Usage.PromptCacheHitTokens + result.CacheCreationTokens += raw.Usage.PromptCacheMissTokens + result.CacheReported = true } } for _, tc := range msg.ToolCalls { diff --git a/internal/llm/client_test.go b/internal/llm/client_test.go index bc3f28a9..df13adec 100644 --- a/internal/llm/client_test.go +++ b/internal/llm/client_test.go @@ -787,6 +787,45 @@ func TestParseResponse_OpenAICacheMetrics(t *testing.T) { } } +func TestParseResponse_DeepSeekCacheMetrics(t *testing.T) { + raw := `{ + "choices": [{"message": {"content": "deepseek response"}}], + "usage": { + "prompt_tokens": 1000, + "completion_tokens": 40, + "prompt_cache_hit_tokens": 750, + "prompt_cache_miss_tokens": 250 + } + }` + result, err := parseResponse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + if result.CacheReadTokens != 750 { + t.Errorf("CacheReadTokens = %d, want 750 (deepseek hit)", result.CacheReadTokens) + } + if result.CacheCreationTokens != 250 { + t.Errorf("CacheCreationTokens = %d, want 250 (deepseek miss)", result.CacheCreationTokens) + } + if !result.CacheReported { + t.Error("CacheReported should be true when deepseek cache fields are present") + } +} + +func TestParseResponse_CacheNotReported(t *testing.T) { + raw := `{ + "choices": [{"message": {"content": "plain response"}}], + "usage": {"prompt_tokens": 100, "completion_tokens": 10} + }` + result, err := parseResponse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + if result.CacheReported { + t.Error("CacheReported should be false when no cache fields are present") + } +} + func TestApplyCacheMarkers_WithSystemPrompt(t *testing.T) { messages := []Message{ {Role: "system", Content: "You are a helpful assistant."}, diff --git a/internal/loop/loop.go b/internal/loop/loop.go index 09249134..c29f630d 100644 --- a/internal/loop/loop.go +++ b/internal/loop/loop.go @@ -91,6 +91,7 @@ type IterationInfo struct { CacheCreationTokens int // cumulative cache creation tokens CacheReadTokens int // cumulative cache read tokens CachedTokens int // cumulative cached tokens (OpenAI) + CacheReported bool // provider returned cache metrics at least once TotalLatency time.Duration // cumulative wall time HasFinalAnswer bool // true when the agent reached a final answer ReasoningContent string // LLM reasoning before tool calls (empty if none) @@ -185,9 +186,10 @@ type Engine struct { TotalOutputTokens int // Cache metrics accumulated across all iterations. - TotalCacheCreationTokens int // Anthropic: tokens written to cache - TotalCacheReadTokens int // Anthropic: tokens read from cache - TotalCachedTokens int // OpenAI: cached prompt tokens + TotalCacheCreationTokens int // Anthropic: tokens written to cache + TotalCacheReadTokens int // Anthropic: tokens read from cache + TotalCachedTokens int // OpenAI: cached prompt tokens + TotalCacheReported bool // provider returned cache metrics at least once } // New creates a new loop Engine. @@ -587,6 +589,7 @@ func (e *Engine) RunWithMessages(ctx context.Context, messages []llm.Message) (s e.TotalCacheCreationTokens = 0 e.TotalCacheReadTokens = 0 e.TotalCachedTokens = 0 + e.TotalCacheReported = false return e.runLoop(ctx, messages) } @@ -867,6 +870,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [ e.TotalCacheCreationTokens += result.CacheCreationTokens e.TotalCacheReadTokens += result.CacheReadTokens e.TotalCachedTokens += result.CachedTokens + e.TotalCacheReported = e.TotalCacheReported || result.CacheReported // No tool calls = final answer if len(result.ToolCalls) == 0 { @@ -901,6 +905,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [ CacheCreationTokens: e.TotalCacheCreationTokens, CacheReadTokens: e.TotalCacheReadTokens, CachedTokens: e.TotalCachedTokens, + CacheReported: e.TotalCacheReported, TotalLatency: time.Since(startTime), HasFinalAnswer: true, ReasoningContent: result.ReasoningContent, @@ -955,6 +960,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [ CacheCreationTokens: e.TotalCacheCreationTokens, CacheReadTokens: e.TotalCacheReadTokens, CachedTokens: e.TotalCachedTokens, + CacheReported: e.TotalCacheReported, TotalLatency: time.Since(startTime), HasFinalAnswer: false, ReasoningContent: result.ReasoningContent, @@ -1244,6 +1250,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [ CacheCreationTokens: e.TotalCacheCreationTokens, CacheReadTokens: e.TotalCacheReadTokens, CachedTokens: e.TotalCachedTokens, + CacheReported: e.TotalCacheReported, TotalLatency: time.Since(startTime), HasFinalAnswer: false, })