[Refactor] Move iteration logging to the frontend (#46647)

Signed-off-by: maxyanghu <hyoung2991@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Shang Wang <shangw@nvidia.com>
This commit is contained in:
Max Hu
2026-07-15 17:59:05 -07:00
committed by GitHub
co-authored by Roger Wang Shang Wang
parent 9d1c695be5
commit 5a65ba5f17
9 changed files with 399 additions and 41 deletions
+14 -1
View File
@@ -782,12 +782,16 @@ class IterationDetails:
num_ctx_tokens: int
num_generation_requests: int
num_generation_tokens: int
num_encoder_inputs: int = 0
num_encoder_output_tokens: int = 0
def __repr__(self) -> str:
return f"IterationDetails(num_ctx_requests={self.num_ctx_requests},\
num_ctx_tokens={self.num_ctx_tokens}, \
num_generation_requests={self.num_generation_requests}, \
num_generation_tokens={self.num_generation_tokens})"
num_generation_tokens={self.num_generation_tokens}, \
num_encoder_inputs={self.num_encoder_inputs}, \
num_encoder_output_tokens={self.num_encoder_output_tokens})"
def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDetails:
@@ -818,9 +822,18 @@ def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDet
else:
num_generation_requests += 1
num_generation_tokens += num_tokens
scheduled_encoder_input_stats = scheduler_output.scheduled_encoder_input_stats
num_encoder_inputs = 0
num_encoder_output_tokens = 0
if scheduled_encoder_input_stats is not None:
num_encoder_inputs = scheduled_encoder_input_stats.num_inputs
num_encoder_output_tokens = scheduled_encoder_input_stats.output_tokens
return IterationDetails(
num_context_requests,
num_context_tokens,
num_generation_requests,
num_generation_tokens,
num_encoder_inputs,
num_encoder_output_tokens,
)