[Bugfix][Responses API] Complete cache-write token accounting (#60973)

Signed-off-by: Kay Yan <kay.yan@daocloud.io>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Kay Yan
2026-10-10 09:08:36 +00:00
committed by GitHub
co-authored by Codex
parent a98247ab4d
commit c41b2639e2
4 changed files with 34 additions and 2 deletions
@@ -315,6 +315,7 @@ class ParsableContext(ConversationContext):
self.num_prompt_tokens = 0
self.num_output_tokens = 0
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
self.num_reasoning_tokens = 0
# not implemented yet for ParsableContext
self.all_turn_metrics: list[TurnMetrics] = []
@@ -347,6 +348,7 @@ class ParsableContext(ConversationContext):
def append_output(self, output: RequestOutput) -> None:
self.num_prompt_tokens = len(output.prompt_token_ids or [])
self.num_cached_tokens = output.num_cached_tokens or 0
self.num_cache_creation_tokens += output.num_cache_creation_tokens or 0
self.num_output_tokens += len(output.outputs[0].token_ids or [])
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
@@ -651,6 +653,7 @@ class HarmonyContext(ConversationContext):
self.num_prompt_tokens = 0
self.num_output_tokens = 0
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
self.num_reasoning_tokens = 0
self.num_tool_output_tokens = 0
@@ -773,6 +776,8 @@ class HarmonyContext(ConversationContext):
self.num_cached_tokens += num_cached_token
self.current_turn_metrics.cached_input_tokens = num_cached_token
self.num_cache_creation_tokens += output.num_cache_creation_tokens or 0
def _update_decode_token_usage(self, output: RequestOutput) -> int:
"""Update token usage statistics for the decode phase of generation.
+1 -1
View File
@@ -881,7 +881,7 @@ class OpenAIServingResponses(GenerateBaseServing):
output_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
input_tokens_details=InputTokensDetails(
cache_write_tokens=getattr(context, "num_cache_creation_tokens", 0),
cache_write_tokens=context.num_cache_creation_tokens,
cached_tokens=num_cached_tokens,
input_tokens_per_turn=[
turn.input_tokens for turn in context.all_turn_metrics