[Core] Add span metrics for model_forward, scheduler and sampler time (#7089)
This commit is contained in:
@@ -267,6 +267,7 @@ class LLMEngine:
|
||||
speculative_config=speculative_config,
|
||||
load_config=load_config,
|
||||
prompt_adapter_config=prompt_adapter_config,
|
||||
observability_config=self.observability_config,
|
||||
)
|
||||
|
||||
if not self.model_config.embedding_mode:
|
||||
@@ -1183,6 +1184,22 @@ class LLMEngine:
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
seq_group.update_num_computed_tokens(
|
||||
scheduled_seq_group.token_chunk_size)
|
||||
if output is not None and len(output) > 0:
|
||||
for o in output:
|
||||
if (isinstance(o, SamplerOutput)
|
||||
and seq_group.metrics is not None):
|
||||
if seq_group.metrics.model_forward_time is not None:
|
||||
seq_group.metrics.model_forward_time += (
|
||||
o.model_forward_time)
|
||||
else:
|
||||
seq_group.metrics.model_forward_time = (
|
||||
o.model_forward_time)
|
||||
if seq_group.metrics.model_execute_time is not None:
|
||||
seq_group.metrics.model_execute_time += (
|
||||
o.model_execute_time)
|
||||
else:
|
||||
seq_group.metrics.model_execute_time = (
|
||||
o.model_execute_time)
|
||||
if self.model_config.embedding_mode:
|
||||
self._process_sequence_group_outputs(seq_group, outputs)
|
||||
continue
|
||||
@@ -1575,6 +1592,18 @@ class LLMEngine:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
|
||||
seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
|
||||
if metrics.scheduler_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER,
|
||||
metrics.scheduler_time)
|
||||
if metrics.model_forward_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD,
|
||||
metrics.model_forward_time / 1000.0)
|
||||
if metrics.model_execute_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
|
||||
metrics.model_execute_time)
|
||||
|
||||
def is_encoder_decoder_model(self):
|
||||
return self.model_config.is_encoder_decoder_model
|
||||
|
||||
Reference in New Issue
Block a user