[Bugfix] Add Responses cache_write_tokens for API compat and CC parity (#57222)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
yzong-rh
2026-09-16 21:52:34 +00:00
committed by GitHub
co-authored by Flora Feng
parent 929aa2a7ed
commit 5a6ccc5892
5 changed files with 17 additions and 0 deletions
@@ -189,6 +189,7 @@ class SimpleContext(ConversationContext):
self.num_prompt_tokens = 0
self.num_output_tokens = 0
self.num_cached_tokens = 0
self.num_cache_creation_tokens = 0
# todo num_reasoning_tokens is not implemented yet.
self.num_reasoning_tokens = 0
# not implemented yet for SimpleContext
@@ -204,6 +205,8 @@ class SimpleContext(ConversationContext):
raise ValueError("SimpleContext only supports RequestOutput.")
self.num_prompt_tokens = len(output.prompt_token_ids or [])
self.num_cached_tokens = output.num_cached_tokens or 0
if output.num_cache_creation_tokens is not None:
self.num_cache_creation_tokens = output.num_cache_creation_tokens
self.num_output_tokens += len(output.outputs[0].token_ids or [])
if output.kv_transfer_params is not None:
self.kv_transfer_params = output.kv_transfer_params
@@ -80,6 +80,7 @@ _INT64_MAX = 2**63 - 1
class InputTokensDetails(OpenAIBaseModel):
cached_tokens: int
cache_write_tokens: int
input_tokens_per_turn: list[int] = Field(default_factory=list)
cached_tokens_per_turn: list[int] = Field(default_factory=list)
@@ -837,6 +837,7 @@ class OpenAIServingResponses(GenerateBaseServing):
output_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
input_tokens_details=InputTokensDetails(
cache_write_tokens=getattr(context, "num_cache_creation_tokens", 0),
cached_tokens=num_cached_tokens,
input_tokens_per_turn=[
turn.input_tokens for turn in context.all_turn_metrics