[Bugfix][Responses API] Complete cache-write token accounting (#60973)
Signed-off-by: Kay Yan <kay.yan@daocloud.io> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -315,6 +315,7 @@ class ParsableContext(ConversationContext):
|
||||
self.num_prompt_tokens = 0
|
||||
self.num_output_tokens = 0
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
self.num_reasoning_tokens = 0
|
||||
# not implemented yet for ParsableContext
|
||||
self.all_turn_metrics: list[TurnMetrics] = []
|
||||
@@ -347,6 +348,7 @@ class ParsableContext(ConversationContext):
|
||||
def append_output(self, output: RequestOutput) -> None:
|
||||
self.num_prompt_tokens = len(output.prompt_token_ids or [])
|
||||
self.num_cached_tokens = output.num_cached_tokens or 0
|
||||
self.num_cache_creation_tokens += output.num_cache_creation_tokens or 0
|
||||
self.num_output_tokens += len(output.outputs[0].token_ids or [])
|
||||
if output.kv_transfer_params is not None:
|
||||
self.kv_transfer_params = output.kv_transfer_params
|
||||
@@ -651,6 +653,7 @@ class HarmonyContext(ConversationContext):
|
||||
self.num_prompt_tokens = 0
|
||||
self.num_output_tokens = 0
|
||||
self.num_cached_tokens = 0
|
||||
self.num_cache_creation_tokens = 0
|
||||
self.num_reasoning_tokens = 0
|
||||
self.num_tool_output_tokens = 0
|
||||
|
||||
@@ -773,6 +776,8 @@ class HarmonyContext(ConversationContext):
|
||||
self.num_cached_tokens += num_cached_token
|
||||
self.current_turn_metrics.cached_input_tokens = num_cached_token
|
||||
|
||||
self.num_cache_creation_tokens += output.num_cache_creation_tokens or 0
|
||||
|
||||
def _update_decode_token_usage(self, output: RequestOutput) -> int:
|
||||
"""Update token usage statistics for the decode phase of generation.
|
||||
|
||||
|
||||
@@ -881,7 +881,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
output_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cache_write_tokens=getattr(context, "num_cache_creation_tokens", 0),
|
||||
cache_write_tokens=context.num_cache_creation_tokens,
|
||||
cached_tokens=num_cached_tokens,
|
||||
input_tokens_per_turn=[
|
||||
turn.input_tokens for turn in context.all_turn_metrics
|
||||
|
||||
Reference in New Issue
Block a user