[Refactor] Move iteration logging to the frontend (#46647)
Signed-off-by: maxyanghu <hyoung2991@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Shang Wang <shangw@nvidia.com>
This commit is contained in:
co-authored by
Roger Wang
Shang Wang
parent
9d1c695be5
commit
5a65ba5f17
+14
-1
@@ -782,12 +782,16 @@ class IterationDetails:
|
||||
num_ctx_tokens: int
|
||||
num_generation_requests: int
|
||||
num_generation_tokens: int
|
||||
num_encoder_inputs: int = 0
|
||||
num_encoder_output_tokens: int = 0
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"IterationDetails(num_ctx_requests={self.num_ctx_requests},\
|
||||
num_ctx_tokens={self.num_ctx_tokens}, \
|
||||
num_generation_requests={self.num_generation_requests}, \
|
||||
num_generation_tokens={self.num_generation_tokens})"
|
||||
num_generation_tokens={self.num_generation_tokens}, \
|
||||
num_encoder_inputs={self.num_encoder_inputs}, \
|
||||
num_encoder_output_tokens={self.num_encoder_output_tokens})"
|
||||
|
||||
|
||||
def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDetails:
|
||||
@@ -818,9 +822,18 @@ def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDet
|
||||
else:
|
||||
num_generation_requests += 1
|
||||
num_generation_tokens += num_tokens
|
||||
scheduled_encoder_input_stats = scheduler_output.scheduled_encoder_input_stats
|
||||
num_encoder_inputs = 0
|
||||
num_encoder_output_tokens = 0
|
||||
if scheduled_encoder_input_stats is not None:
|
||||
num_encoder_inputs = scheduled_encoder_input_stats.num_inputs
|
||||
num_encoder_output_tokens = scheduled_encoder_input_stats.output_tokens
|
||||
|
||||
return IterationDetails(
|
||||
num_context_requests,
|
||||
num_context_tokens,
|
||||
num_generation_requests,
|
||||
num_generation_tokens,
|
||||
num_encoder_inputs,
|
||||
num_encoder_output_tokens,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user