[Bugfix] Add Responses cache_write_tokens for API compat and CC parity (#57222)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -189,6 +189,7 @@ class SimpleContext(ConversationContext):
|
||||
self.num_prompt_tokens = 0
|
||||
self.num_output_tokens = 0
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
# todo num_reasoning_tokens is not implemented yet.
|
||||
self.num_reasoning_tokens = 0
|
||||
# not implemented yet for SimpleContext
|
||||
@@ -204,6 +205,8 @@ class SimpleContext(ConversationContext):
|
||||
raise ValueError("SimpleContext only supports RequestOutput.")
|
||||
self.num_prompt_tokens = len(output.prompt_token_ids or [])
|
||||
self.num_cached_tokens = output.num_cached_tokens or 0
|
||||
if output.num_cache_creation_tokens is not None:
|
||||
self.num_cache_creation_tokens = output.num_cache_creation_tokens
|
||||
self.num_output_tokens += len(output.outputs[0].token_ids or [])
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
|
||||
@@ -80,6 +80,7 @@ _INT64_MAX = 2**63 - 1
|
||||
|
||||
class InputTokensDetails(OpenAIBaseModel):
|
||||
cached_tokens: int
|
||||
cache_write_tokens: int
|
||||
input_tokens_per_turn: list[int] = Field(default_factory=list)
|
||||
cached_tokens_per_turn: list[int] = Field(default_factory=list)
|
||||
|
||||
|
||||
@@ -837,6 +837,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
output_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cache_write_tokens=getattr(context, "num_cache_creation_tokens", 0),
|
||||
cached_tokens=num_cached_tokens,
|
||||
input_tokens_per_turn=[
|
||||
turn.input_tokens for turn in context.all_turn_metrics
|
||||
|
||||
Reference in New Issue
Block a user