Merge pull request #86 from cuipengfei/fix-token-counting

Fix hardcoded token counts in streaming responses

Author: ericc-ch

src/routes/messages/anthropic-types.ts

@@ -157,7 +157,12 @@ export interface AnthropicMessageDeltaEvent {
     stop_reason?: AnthropicResponse["stop_reason"]
     stop_sequence?: string | null
   }
-  usage?: { output_tokens: number }
+  usage?: {
+    input_tokens?: number
+    output_tokens: number
+    cache_creation_input_tokens?: number
+    cache_read_input_tokens?: number
+  }
 }
 
 export interface AnthropicMessageStopEvent {

src/routes/messages/stream-translation.ts

@@ -42,8 +42,8 @@ export function translateChunkToAnthropicEvents(
         stop_reason: null,
         stop_sequence: null,
         usage: {
-          input_tokens: 1,
-          output_tokens: 1, // Anthropic requires this to be > 0
+          input_tokens: chunk.usage?.prompt_tokens ?? 0,
+          output_tokens: 0, // Will be updated in message_delta when finished
         },
       },
     })
@@ -152,7 +152,13 @@ export function translateChunkToAnthropicEvents(
           stop_sequence: null,
         },
         usage: {
-          output_tokens: 1,
+          input_tokens: chunk.usage?.prompt_tokens ?? 0,
+          output_tokens: chunk.usage?.completion_tokens ?? 0,
+          ...(chunk.usage?.prompt_tokens_details?.cached_tokens
+            !== undefined && {
+            cache_read_input_tokens:
+              chunk.usage.prompt_tokens_details.cached_tokens,
+          }),
         },
       },
       {

src/services/copilot/create-chat-completions.ts

@@ -55,6 +55,18 @@ export interface ChatCompletionChunk {
   model: string
   choices: Array<Choice>
   system_fingerprint?: string
+  usage?: {
+    prompt_tokens: number
+    completion_tokens: number
+    total_tokens: number
+    prompt_tokens_details?: {
+      cached_tokens: number
+    }
+    completion_tokens_details?: {
+      accepted_prediction_tokens: number
+      rejected_prediction_tokens: number
+    }
+  }
 }
 
 interface Delta {