[Bugfix] Fix --enable-prompt-tokens-details omitting zero cached tokens (#44383)

Signed-off-by: Sasindharan Sankar <sasindharansankar@email.com>
Co-authored-by: Sasindharan Sankar <sasindharansankar@email.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
sasindharan
2026-06-11 20:37:42 -07:00
committed by GitHub
co-authored by Sasindharan Sankar Chauncey
parent 2263f8a3de
commit 42ae5e7ac6
5 changed files with 61 additions and 9 deletions
@@ -732,7 +732,7 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
@@ -1023,7 +1023,10 @@ class OpenAIServingChat(OpenAIServing):
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
)
@@ -443,7 +443,7 @@ class OpenAIServingCompletion(OpenAIServing):
total_tokens=total_prompt_tokens + total_completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)
@@ -583,7 +583,7 @@ class OpenAIServingCompletion(OpenAIServing):
if (
self.enable_prompt_tokens_details
and last_final_res
and last_final_res.num_cached_tokens
and last_final_res.num_cached_tokens is not None
):
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=last_final_res.num_cached_tokens
+5 -2
View File
@@ -307,7 +307,10 @@ class ServingTokens(OpenAIServing):
completion_tokens=num_generated_tokens,
total_tokens=num_prompt_tokens + num_generated_tokens,
)
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
if (
self.enable_prompt_tokens_details
and final_res.num_cached_tokens is not None
):
# This info is not available at the /coordinator level
usage.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=final_res.num_cached_tokens
@@ -424,7 +427,7 @@ class ServingTokens(OpenAIServing):
total_tokens=num_prompt_tokens + total_completion_tokens,
)
if self.enable_prompt_tokens_details and num_cached_tokens:
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
cached_tokens=num_cached_tokens
)