[Bugfix] Fix --enable-prompt-tokens-details omitting zero cached tokens (#44383)
Signed-off-by: Sasindharan Sankar <sasindharansankar@email.com> Co-authored-by: Sasindharan Sankar <sasindharansankar@email.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Sasindharan Sankar
Chauncey
parent
2263f8a3de
commit
42ae5e7ac6
@@ -732,7 +732,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=num_prompt_tokens + completion_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
@@ -1023,7 +1023,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
)
|
||||
|
||||
@@ -443,7 +443,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
@@ -583,7 +583,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and last_final_res
|
||||
and last_final_res.num_cached_tokens
|
||||
and last_final_res.num_cached_tokens is not None
|
||||
):
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=last_final_res.num_cached_tokens
|
||||
|
||||
@@ -307,7 +307,10 @@ class ServingTokens(OpenAIServing):
|
||||
completion_tokens=num_generated_tokens,
|
||||
total_tokens=num_prompt_tokens + num_generated_tokens,
|
||||
)
|
||||
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
|
||||
if (
|
||||
self.enable_prompt_tokens_details
|
||||
and final_res.num_cached_tokens is not None
|
||||
):
|
||||
# This info is not available at the /coordinator level
|
||||
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=final_res.num_cached_tokens
|
||||
@@ -424,7 +427,7 @@ class ServingTokens(OpenAIServing):
|
||||
total_tokens=num_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens:
|
||||
if self.enable_prompt_tokens_details and num_cached_tokens is not None:
|
||||
final_usage_info.prompt_tokens_details = PromptTokenUsageInfo(
|
||||
cached_tokens=num_cached_tokens
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user