Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions cmd/odek/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -2138,9 +2138,16 @@ func formatTelegramStats(info loop.IterationInfo, toolList []string) string {
iters += "s"
}

// Always include cache stats so the user can see them even when zero.
cacheStr := fmt.Sprintf(" · cache: %d write / %d read / %d total",
info.CacheCreationTokens, info.CacheReadTokens, info.CachedTokens)
// Show real numbers when the provider reports cache metrics; otherwise
// say so — "0 write / 0 read" would wrongly imply caching ran and
// missed, when in fact the provider returned no data at all.
var cacheStr string
if info.CacheReported {
cacheStr = fmt.Sprintf(" · cache: %d write / %d read / %d total",
info.CacheCreationTokens, info.CacheReadTokens, info.CachedTokens)
} else {
cacheStr = " · cache: n/a (provider reports no cache metrics)"
}

return fmt.Sprintf(
"```\n✅ Done · %s · %d in / %d out%s · %s — tools: %s\n```",
Expand Down
13 changes: 13 additions & 0 deletions cmd/odek/telegram_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1163,6 +1163,7 @@ func TestFormatTelegramStats(t *testing.T) {
CacheCreationTokens: 10,
CacheReadTokens: 20,
CachedTokens: 30,
CacheReported: true,
TotalLatency: 5*time.Second + 500*time.Millisecond,
}
out := formatTelegramStats(info, []string{"read_file", "shell"})
Expand All @@ -1189,6 +1190,18 @@ func TestFormatTelegramStats_SingularTurn(t *testing.T) {
}
}

// TestFormatTelegramStats_CacheNotReported verifies the honest display when
// the provider returned no cache metrics at all (vs. real zeros).
func TestFormatTelegramStats_CacheNotReported(t *testing.T) {
out := formatTelegramStats(loop.IterationInfo{Turn: 1}, nil)
if !strings.Contains(out, "cache: n/a") {
t.Errorf("expected cache n/a when provider reports nothing, got: %s", out)
}
if strings.Contains(out, "write") || strings.Contains(out, "read /") {
t.Errorf("must not show fake zero cache numbers when unreported: %s", out)
}
}

// TestFormatStopSummary verifies the /stop summary formatting.
func TestFormatStopSummary(t *testing.T) {
info := loop.IterationInfo{
Expand Down
21 changes: 21 additions & 0 deletions internal/llm/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,13 @@ type CallResult struct {
// Cache metrics. Only populated when the provider returns them.
// Anthropic: cache_creation_input_tokens, cache_read_input_tokens
// OpenAI: prompt_tokens_details.cached_tokens
// DeepSeek: prompt_cache_hit_tokens (read), prompt_cache_miss_tokens (write)
CacheCreationTokens int // Anthropic — tokens written to cache
CacheReadTokens int // Anthropic — tokens read from cache hit
CachedTokens int // OpenAI — cached tokens in prompt
// CacheReported is true when the provider returned any cache metrics at
// all; false means "no data", which is different from "0 tokens cached".
CacheReported bool
}

// toolChoiceNone forces the model to not call tools.
Expand Down Expand Up @@ -451,6 +455,11 @@ func parseResponse(data []byte) (*CallResult, error) {
PromptTokensDetails *struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details"`
// DeepSeek native prompt caching (always present on DeepSeek
// endpoints, unlike prompt_tokens_details which varies by
// gateway/proxy).
PromptCacheHitTokens int `json:"prompt_cache_hit_tokens"`
PromptCacheMissTokens int `json:"prompt_cache_miss_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(data, &raw); err != nil {
Expand All @@ -472,6 +481,18 @@ func parseResponse(data []byte) (*CallResult, error) {
result.CacheReadTokens = raw.Usage.CacheReadTokens
if raw.Usage.PromptTokensDetails != nil {
result.CachedTokens = raw.Usage.PromptTokensDetails.CachedTokens
result.CacheReported = true
}
if raw.Usage.CacheCreationTokens > 0 || raw.Usage.CacheReadTokens > 0 {
result.CacheReported = true
}
// DeepSeek native fields: a hit is prompt content read from cache;
// a miss is newly processed content that DeepSeek then caches for
// future requests, i.e. a cache write.
if raw.Usage.PromptCacheHitTokens > 0 || raw.Usage.PromptCacheMissTokens > 0 {
result.CacheReadTokens += raw.Usage.PromptCacheHitTokens
result.CacheCreationTokens += raw.Usage.PromptCacheMissTokens
result.CacheReported = true
}
}
for _, tc := range msg.ToolCalls {
Expand Down
39 changes: 39 additions & 0 deletions internal/llm/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,45 @@ func TestParseResponse_OpenAICacheMetrics(t *testing.T) {
}
}

func TestParseResponse_DeepSeekCacheMetrics(t *testing.T) {
raw := `{
"choices": [{"message": {"content": "deepseek response"}}],
"usage": {
"prompt_tokens": 1000,
"completion_tokens": 40,
"prompt_cache_hit_tokens": 750,
"prompt_cache_miss_tokens": 250
}
}`
result, err := parseResponse([]byte(raw))
if err != nil {
t.Fatal(err)
}
if result.CacheReadTokens != 750 {
t.Errorf("CacheReadTokens = %d, want 750 (deepseek hit)", result.CacheReadTokens)
}
if result.CacheCreationTokens != 250 {
t.Errorf("CacheCreationTokens = %d, want 250 (deepseek miss)", result.CacheCreationTokens)
}
if !result.CacheReported {
t.Error("CacheReported should be true when deepseek cache fields are present")
}
}

func TestParseResponse_CacheNotReported(t *testing.T) {
raw := `{
"choices": [{"message": {"content": "plain response"}}],
"usage": {"prompt_tokens": 100, "completion_tokens": 10}
}`
result, err := parseResponse([]byte(raw))
if err != nil {
t.Fatal(err)
}
if result.CacheReported {
t.Error("CacheReported should be false when no cache fields are present")
}
}

func TestApplyCacheMarkers_WithSystemPrompt(t *testing.T) {
messages := []Message{
{Role: "system", Content: "You are a helpful assistant."},
Expand Down
13 changes: 10 additions & 3 deletions internal/loop/loop.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ type IterationInfo struct {
CacheCreationTokens int // cumulative cache creation tokens
CacheReadTokens int // cumulative cache read tokens
CachedTokens int // cumulative cached tokens (OpenAI)
CacheReported bool // provider returned cache metrics at least once
TotalLatency time.Duration // cumulative wall time
HasFinalAnswer bool // true when the agent reached a final answer
ReasoningContent string // LLM reasoning before tool calls (empty if none)
Expand Down Expand Up @@ -185,9 +186,10 @@ type Engine struct {
TotalOutputTokens int

// Cache metrics accumulated across all iterations.
TotalCacheCreationTokens int // Anthropic: tokens written to cache
TotalCacheReadTokens int // Anthropic: tokens read from cache
TotalCachedTokens int // OpenAI: cached prompt tokens
TotalCacheCreationTokens int // Anthropic: tokens written to cache
TotalCacheReadTokens int // Anthropic: tokens read from cache
TotalCachedTokens int // OpenAI: cached prompt tokens
TotalCacheReported bool // provider returned cache metrics at least once
}

// New creates a new loop Engine.
Expand Down Expand Up @@ -587,6 +589,7 @@ func (e *Engine) RunWithMessages(ctx context.Context, messages []llm.Message) (s
e.TotalCacheCreationTokens = 0
e.TotalCacheReadTokens = 0
e.TotalCachedTokens = 0
e.TotalCacheReported = false
return e.runLoop(ctx, messages)
}

Expand Down Expand Up @@ -867,6 +870,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
e.TotalCacheCreationTokens += result.CacheCreationTokens
e.TotalCacheReadTokens += result.CacheReadTokens
e.TotalCachedTokens += result.CachedTokens
e.TotalCacheReported = e.TotalCacheReported || result.CacheReported

// No tool calls = final answer
if len(result.ToolCalls) == 0 {
Expand Down Expand Up @@ -901,6 +905,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
CacheCreationTokens: e.TotalCacheCreationTokens,
CacheReadTokens: e.TotalCacheReadTokens,
CachedTokens: e.TotalCachedTokens,
CacheReported: e.TotalCacheReported,
TotalLatency: time.Since(startTime),
HasFinalAnswer: true,
ReasoningContent: result.ReasoningContent,
Expand Down Expand Up @@ -955,6 +960,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
CacheCreationTokens: e.TotalCacheCreationTokens,
CacheReadTokens: e.TotalCacheReadTokens,
CachedTokens: e.TotalCachedTokens,
CacheReported: e.TotalCacheReported,
TotalLatency: time.Since(startTime),
HasFinalAnswer: false,
ReasoningContent: result.ReasoningContent,
Expand Down Expand Up @@ -1244,6 +1250,7 @@ func (e *Engine) runLoop(ctx context.Context, messages []llm.Message) (string, [
CacheCreationTokens: e.TotalCacheCreationTokens,
CacheReadTokens: e.TotalCacheReadTokens,
CachedTokens: e.TotalCachedTokens,
CacheReported: e.TotalCacheReported,
TotalLatency: time.Since(startTime),
HasFinalAnswer: false,
})
Expand Down
Loading