[Feature] OTEL tracing during loading (#31162)
This commit is contained in:
@@ -20,7 +20,12 @@ from vllm.outputs import (
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context
|
||||
from vllm.tracing import (
|
||||
SpanAttributes,
|
||||
SpanKind,
|
||||
extract_trace_context,
|
||||
instrument_manual,
|
||||
)
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
@@ -422,7 +427,7 @@ class OutputProcessor:
|
||||
self.parent_requests: dict[str, ParentRequest] = {}
|
||||
self.external_req_ids: defaultdict[str, list[str]] = defaultdict(list)
|
||||
self.lora_states = LoRARequestStates(log_stats)
|
||||
self.tracer: Tracer | None = None
|
||||
self.tracing_enabled: bool = False
|
||||
self._requests_drained = asyncio.Event()
|
||||
self._requests_drained.set()
|
||||
|
||||
@@ -678,7 +683,7 @@ class OutputProcessor:
|
||||
self._update_stats_from_finished(
|
||||
req_state, finish_reason, iteration_stats
|
||||
)
|
||||
if self.tracer:
|
||||
if self.tracing_enabled:
|
||||
self.do_tracing(engine_core_output, req_state, iteration_stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
@@ -714,62 +719,59 @@ class OutputProcessor:
|
||||
) -> None:
|
||||
assert req_state.stats is not None
|
||||
assert iteration_stats is not None
|
||||
assert self.tracer is not None
|
||||
|
||||
arrival_time_nano_seconds = int(req_state.stats.arrival_time * 1e9)
|
||||
metrics = req_state.stats
|
||||
arrival_time_ns = int(metrics.arrival_time * 1e9)
|
||||
trace_context = extract_trace_context(engine_core_output.trace_headers)
|
||||
prompt_length = length_from_prompt_token_ids_or_embeds(
|
||||
req_state.prompt_token_ids, req_state.prompt_embeds
|
||||
)
|
||||
with self.tracer.start_as_current_span(
|
||||
"llm_request",
|
||||
kind=SpanKind.SERVER,
|
||||
context=trace_context,
|
||||
start_time=arrival_time_nano_seconds,
|
||||
) as span:
|
||||
metrics = req_state.stats
|
||||
e2e_time = iteration_stats.iteration_timestamp - metrics.arrival_time
|
||||
queued_time = metrics.scheduled_ts - metrics.queued_ts
|
||||
prefill_time = metrics.first_token_ts - metrics.scheduled_ts
|
||||
decode_time = metrics.last_token_ts - metrics.first_token_ts
|
||||
inference_time = metrics.last_token_ts - metrics.scheduled_ts
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN,
|
||||
metrics.first_token_latency,
|
||||
)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_LATENCY_E2E, e2e_time)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE, queued_time)
|
||||
span.set_attribute(SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS, prompt_length)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS,
|
||||
metrics.num_generation_tokens,
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL, prefill_time
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE, decode_time
|
||||
)
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE, inference_time
|
||||
)
|
||||
|
||||
# meta
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_REQUEST_ID, req_state.external_req_id
|
||||
# Calculate timing metrics
|
||||
e2e_time = iteration_stats.iteration_timestamp - metrics.arrival_time
|
||||
queued_time = metrics.scheduled_ts - metrics.queued_ts
|
||||
prefill_time = metrics.first_token_ts - metrics.scheduled_ts
|
||||
decode_time = metrics.last_token_ts - metrics.first_token_ts
|
||||
inference_time = metrics.last_token_ts - metrics.scheduled_ts
|
||||
|
||||
# Build attributes dict
|
||||
attributes: dict[str, Any] = {
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN: (
|
||||
metrics.first_token_latency
|
||||
),
|
||||
SpanAttributes.GEN_AI_LATENCY_E2E: e2e_time,
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE: queued_time,
|
||||
SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS: prompt_length,
|
||||
SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS: (
|
||||
metrics.num_generation_tokens
|
||||
),
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_PREFILL: prefill_time,
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_DECODE: decode_time,
|
||||
SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_INFERENCE: inference_time,
|
||||
SpanAttributes.GEN_AI_REQUEST_ID: req_state.external_req_id,
|
||||
}
|
||||
|
||||
# Add optional request parameters
|
||||
if req_state.top_p:
|
||||
attributes[SpanAttributes.GEN_AI_REQUEST_TOP_P] = req_state.top_p
|
||||
if req_state.max_tokens_param:
|
||||
attributes[SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS] = (
|
||||
req_state.max_tokens_param
|
||||
)
|
||||
if req_state.top_p:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P, req_state.top_p)
|
||||
if req_state.max_tokens_param:
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS, req_state.max_tokens_param
|
||||
)
|
||||
if req_state.temperature:
|
||||
span.set_attribute(
|
||||
SpanAttributes.GEN_AI_REQUEST_TEMPERATURE, req_state.temperature
|
||||
)
|
||||
if req_state.n:
|
||||
span.set_attribute(SpanAttributes.GEN_AI_REQUEST_N, req_state.n)
|
||||
if req_state.temperature:
|
||||
attributes[SpanAttributes.GEN_AI_REQUEST_TEMPERATURE] = (
|
||||
req_state.temperature
|
||||
)
|
||||
if req_state.n:
|
||||
attributes[SpanAttributes.GEN_AI_REQUEST_N] = req_state.n
|
||||
|
||||
instrument_manual(
|
||||
span_name="llm_request",
|
||||
start_time=arrival_time_ns,
|
||||
attributes=attributes,
|
||||
context=trace_context,
|
||||
kind=SpanKind.SERVER,
|
||||
)
|
||||
|
||||
def _update_stats_from_output(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user