Fix vllm:prompt_tokens_total metric calculation (#2869)

This commit is contained in:
Ronen Schaffer
2024-02-18 23:55:41 -08:00
committed by GitHub
parent 86fd8bb0ac
commit e433c115bc
3 changed files with 41 additions and 6 deletions
+3 -1
View File
@@ -867,7 +867,9 @@ class LLMEngine:
# Number of Tokens.
if prompt_run:
num_prompt_tokens = scheduler_outputs.num_batched_tokens
num_prompt_tokens = sum(
len(seq_group.prompt_token_ids)
for seq_group in scheduler_outputs.scheduled_seq_groups)
else:
num_generation_tokens = scheduler_outputs.num_batched_tokens