Include tokens from prompt phase in counter_generation_tokens (#2802)

This commit is contained in:
Ronen Schaffer
2024-02-22 14:00:12 -08:00
committed by GitHub
parent 6f32cddf1c
commit 4caf7044e0
3 changed files with 39 additions and 1 deletions
+3
View File
@@ -872,6 +872,9 @@ class LLMEngine:
num_prompt_tokens = sum(
len(seq_group.prompt_token_ids)
for seq_group in scheduler_outputs.scheduled_seq_groups)
num_generation_tokens = sum(
seq_group.num_seqs()
for seq_group in scheduler_outputs.scheduled_seq_groups)
else:
num_generation_tokens = scheduler_outputs.num_batched_tokens