[Core] Add span metrics for model_forward, scheduler and sampler time (#7089)
This commit is contained in:
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
|
||||
|
||||
|
||||
def nullable_str(val: str):
|
||||
if not val or val == "None":
|
||||
@@ -117,6 +119,7 @@ class EngineArgs:
|
||||
disable_logprobs_during_spec_decoding: Optional[bool] = None
|
||||
|
||||
otlp_traces_endpoint: Optional[str] = None
|
||||
collect_detailed_traces: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tokenizer is None:
|
||||
@@ -660,6 +663,16 @@ class EngineArgs:
|
||||
type=str,
|
||||
default=None,
|
||||
help='Target URL to which OpenTelemetry traces will be sent.')
|
||||
parser.add_argument(
|
||||
'--collect-detailed-traces',
|
||||
type=str,
|
||||
default=None,
|
||||
help="Valid choices are " +
|
||||
",".join(ALLOWED_DETAILED_TRACE_MODULES) +
|
||||
". It makes sense to set this only if --otlp-traces-endpoint is"
|
||||
" set. If set, it will collect detailed traces for the specified "
|
||||
"modules. This involves use of possibly costly and or blocking "
|
||||
"operations and hence might have a performance impact.")
|
||||
|
||||
return parser
|
||||
|
||||
@@ -852,8 +865,26 @@ class EngineArgs:
|
||||
decoding_config = DecodingConfig(
|
||||
guided_decoding_backend=self.guided_decoding_backend)
|
||||
|
||||
detailed_trace_modules = []
|
||||
if self.collect_detailed_traces is not None:
|
||||
detailed_trace_modules = self.collect_detailed_traces.split(",")
|
||||
for m in detailed_trace_modules:
|
||||
if m not in ALLOWED_DETAILED_TRACE_MODULES:
|
||||
raise ValueError(
|
||||
f"Invalid module {m} in collect_detailed_traces. "
|
||||
f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}")
|
||||
if (m == "model"
|
||||
or m == "all") and self.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"Collection of detailed traces for the 'model' module is "
|
||||
"not yet supported with pipeline parallelism.")
|
||||
observability_config = ObservabilityConfig(
|
||||
otlp_traces_endpoint=self.otlp_traces_endpoint)
|
||||
otlp_traces_endpoint=self.otlp_traces_endpoint,
|
||||
collect_model_forward_time="model" in detailed_trace_modules
|
||||
or "all" in detailed_trace_modules,
|
||||
collect_model_execute_time="worker" in detailed_trace_modules
|
||||
or "all" in detailed_trace_modules,
|
||||
)
|
||||
|
||||
if (model_config.get_sliding_window() is not None
|
||||
and scheduler_config.chunked_prefill_enabled
|
||||
|
||||
@@ -267,6 +267,7 @@ class LLMEngine:
|
||||
speculative_config=speculative_config,
|
||||
load_config=load_config,
|
||||
prompt_adapter_config=prompt_adapter_config,
|
||||
observability_config=self.observability_config,
|
||||
)
|
||||
|
||||
if not self.model_config.embedding_mode:
|
||||
@@ -1183,6 +1184,22 @@ class LLMEngine:
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
seq_group.update_num_computed_tokens(
|
||||
scheduled_seq_group.token_chunk_size)
|
||||
if output is not None and len(output) > 0:
|
||||
for o in output:
|
||||
if (isinstance(o, SamplerOutput)
|
||||
and seq_group.metrics is not None):
|
||||
if seq_group.metrics.model_forward_time is not None:
|
||||
seq_group.metrics.model_forward_time += (
|
||||
o.model_forward_time)
|
||||
else:
|
||||
seq_group.metrics.model_forward_time = (
|
||||
o.model_forward_time)
|
||||
if seq_group.metrics.model_execute_time is not None:
|
||||
seq_group.metrics.model_execute_time += (
|
||||
o.model_execute_time)
|
||||
else:
|
||||
seq_group.metrics.model_execute_time = (
|
||||
o.model_execute_time)
|
||||
if self.model_config.embedding_mode:
|
||||
self._process_sequence_group_outputs(seq_group, outputs)
|
||||
continue
|
||||
@@ -1575,6 +1592,18 @@ class LLMEngine:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
|
||||
seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
|
||||
if metrics.scheduler_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER,
|
||||
metrics.scheduler_time)
|
||||
if metrics.model_forward_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD,
|
||||
metrics.model_forward_time / 1000.0)
|
||||
if metrics.model_execute_time is not None:
|
||||
seq_span.set_attribute(
|
||||
SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
|
||||
metrics.model_execute_time)
|
||||
|
||||
def is_encoder_decoder_model(self):
|
||||
return self.model_config.is_encoder_decoder_model
|
||||
|
||||
Reference in New Issue
Block a user