[V1][Metrics] Add several request timing histograms (#12644)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import enum
|
||||
import time
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import msgspec
|
||||
@@ -60,6 +61,30 @@ class EngineCoreRequest(
|
||||
lora_request: Optional[LoRARequest]
|
||||
|
||||
|
||||
class EngineCoreEventType(enum.IntEnum):
|
||||
"""The type of engine core request event."""
|
||||
QUEUED = 1
|
||||
SCHEDULED = 2
|
||||
|
||||
|
||||
class EngineCoreEvent(msgspec.Struct):
|
||||
"""A timestamped engine core event associated with a request.
|
||||
|
||||
The timestamp is a monotonic timestamps and is used for by the engine
|
||||
frontend to calculate intervals between engine core events. These
|
||||
timestamps should not be compared with timestamps from other processes.
|
||||
"""
|
||||
type: EngineCoreEventType
|
||||
timestamp: float
|
||||
|
||||
@classmethod
|
||||
def new_event(cls,
|
||||
event_type: EngineCoreEventType,
|
||||
timestamp: Optional[float] = None) -> "EngineCoreEvent":
|
||||
timestamp = time.monotonic() if timestamp is None else timestamp
|
||||
return cls(event_type, timestamp)
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
msgspec.Struct,
|
||||
array_like=True, # type: ignore[call-arg]
|
||||
@@ -74,6 +99,7 @@ class EngineCoreOutput(
|
||||
|
||||
finish_reason: Optional[FinishReason] = None
|
||||
stop_reason: Union[int, str, None] = None
|
||||
events: Optional[List[EngineCoreEvent]] = None
|
||||
|
||||
@property
|
||||
def finished(self) -> bool:
|
||||
@@ -91,7 +117,12 @@ class EngineCoreOutputs(
|
||||
|
||||
# [num_reqs]
|
||||
outputs: List[EngineCoreOutput]
|
||||
scheduler_stats: SchedulerStats
|
||||
scheduler_stats: Optional[SchedulerStats]
|
||||
timestamp: float = 0.0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.timestamp == 0.0:
|
||||
self.timestamp = time.monotonic()
|
||||
|
||||
|
||||
class EngineCoreRequestType(enum.Enum):
|
||||
|
||||
+14
-10
@@ -53,10 +53,12 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
self.log_requests = log_requests
|
||||
self.log_stats = log_stats
|
||||
self.stat_loggers: List[StatLoggerBase] = [
|
||||
LoggingStatLogger(),
|
||||
PrometheusStatLogger(vllm_config.model_config),
|
||||
]
|
||||
self.stat_loggers: List[StatLoggerBase] = []
|
||||
if self.log_stats:
|
||||
self.stat_loggers.extend([
|
||||
LoggingStatLogger(),
|
||||
PrometheusStatLogger(vllm_config.model_config),
|
||||
])
|
||||
|
||||
# Tokenizer (+ ensure liveness if running in another process).
|
||||
self.tokenizer = init_tokenizer_from_configs(
|
||||
@@ -85,6 +87,7 @@ class AsyncLLM(EngineClient):
|
||||
asyncio_mode=True,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
|
||||
self.output_handler: Optional[asyncio.Task] = None
|
||||
@@ -246,6 +249,8 @@ class AsyncLLM(EngineClient):
|
||||
# 1) Pull EngineCoreOutputs from the EngineCore.
|
||||
outputs = await self.engine_core.get_output_async()
|
||||
|
||||
iteration_stats = IterationStats() if self.log_stats else None
|
||||
|
||||
# Split outputs into chunks of at most
|
||||
# VLLM_V1_OUTPUT_PROC_CHUNK_SIZE, so that we don't block the
|
||||
# event loop for too long.
|
||||
@@ -257,14 +262,12 @@ class AsyncLLM(EngineClient):
|
||||
outputs.outputs,
|
||||
cdiv(num_outputs, VLLM_V1_OUTPUT_PROC_CHUNK_SIZE))
|
||||
|
||||
iteration_stats = None
|
||||
for i, outputs_slice in enumerate(slices):
|
||||
# 2) Process EngineCoreOutputs.
|
||||
processed_outputs = self.output_processor.process_outputs(
|
||||
outputs_slice, iteration_stats)
|
||||
outputs_slice, outputs.timestamp, iteration_stats)
|
||||
# NOTE: RequestOutputs are pushed to their queues.
|
||||
assert not processed_outputs.request_outputs
|
||||
iteration_stats = processed_outputs.iteration_stats
|
||||
|
||||
# Allow other asyncio tasks to run between chunks
|
||||
if i + 1 < len(slices):
|
||||
@@ -277,7 +280,6 @@ class AsyncLLM(EngineClient):
|
||||
# 4) Logging.
|
||||
# TODO(rob): make into a coroutine and launch it in
|
||||
# background thread once Prometheus overhead is non-trivial.
|
||||
assert iteration_stats is not None
|
||||
self._log_stats(
|
||||
scheduler_stats=outputs.scheduler_stats,
|
||||
iteration_stats=iteration_stats,
|
||||
@@ -299,12 +301,14 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
def _log_stats(
|
||||
self,
|
||||
scheduler_stats: SchedulerStats,
|
||||
iteration_stats: IterationStats,
|
||||
scheduler_stats: Optional[SchedulerStats],
|
||||
iteration_stats: Optional[IterationStats],
|
||||
):
|
||||
if not self.log_stats:
|
||||
return
|
||||
|
||||
assert scheduler_stats is not None
|
||||
assert iteration_stats is not None
|
||||
for logger in self.stat_loggers:
|
||||
logger.log(scheduler_stats=scheduler_stats,
|
||||
iteration_stats=iteration_stats)
|
||||
|
||||
@@ -38,12 +38,15 @@ class EngineCore:
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: Type[Executor],
|
||||
log_stats: bool,
|
||||
):
|
||||
assert vllm_config.model_config.runner_type != "pooling"
|
||||
|
||||
logger.info("Initializing a V1 LLM engine (v%s) with config: %s",
|
||||
VLLM_VERSION, vllm_config)
|
||||
|
||||
self.log_stats = log_stats
|
||||
|
||||
# Setup Model.
|
||||
self.model_executor = executor_class(vllm_config)
|
||||
|
||||
@@ -59,6 +62,7 @@ class EngineCore:
|
||||
model_config=vllm_config.model_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
lora_config=vllm_config.lora_config,
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
|
||||
self.mm_input_mapper_server = MMInputMapperServer(
|
||||
@@ -148,11 +152,9 @@ class EngineCoreProc(EngineCore):
|
||||
ready_pipe: Connection,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: Type[Executor],
|
||||
log_stats: bool = False,
|
||||
log_stats: bool,
|
||||
):
|
||||
super().__init__(vllm_config, executor_class)
|
||||
|
||||
self.log_stats = log_stats
|
||||
super().__init__(vllm_config, executor_class, log_stats)
|
||||
|
||||
# Background Threads and Queues for IO. These enable us to
|
||||
# overlap ZMQ socket IO with GPU since they release the GIL,
|
||||
|
||||
@@ -41,6 +41,7 @@ class EngineCoreClient(ABC):
|
||||
asyncio_mode: bool,
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: Type[Executor],
|
||||
log_stats: bool,
|
||||
) -> "EngineCoreClient":
|
||||
|
||||
# TODO: support this for debugging purposes.
|
||||
@@ -50,12 +51,12 @@ class EngineCoreClient(ABC):
|
||||
"is not currently supported.")
|
||||
|
||||
if multiprocess_mode and asyncio_mode:
|
||||
return AsyncMPClient(vllm_config, executor_class)
|
||||
return AsyncMPClient(vllm_config, executor_class, log_stats)
|
||||
|
||||
if multiprocess_mode and not asyncio_mode:
|
||||
return SyncMPClient(vllm_config, executor_class)
|
||||
return SyncMPClient(vllm_config, executor_class, log_stats)
|
||||
|
||||
return InprocClient(vllm_config, executor_class)
|
||||
return InprocClient(vllm_config, executor_class, log_stats)
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self):
|
||||
@@ -204,13 +205,13 @@ class MPClient(EngineCoreClient):
|
||||
class SyncMPClient(MPClient):
|
||||
"""Synchronous client for multi-proc EngineCore."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig,
|
||||
executor_class: Type[Executor]):
|
||||
def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
|
||||
log_stats: bool):
|
||||
super().__init__(
|
||||
asyncio_mode=False,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
def get_output(self) -> EngineCoreOutputs:
|
||||
@@ -245,13 +246,13 @@ class SyncMPClient(MPClient):
|
||||
class AsyncMPClient(MPClient):
|
||||
"""Asyncio-compatible client for multi-proc EngineCore."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig,
|
||||
executor_class: Type[Executor]):
|
||||
def __init__(self, vllm_config: VllmConfig, executor_class: Type[Executor],
|
||||
log_stats: bool):
|
||||
super().__init__(
|
||||
asyncio_mode=True,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=True,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
self.outputs_queue: Optional[asyncio.Queue[bytes]] = None
|
||||
|
||||
@@ -73,6 +73,7 @@ class LLMEngine:
|
||||
asyncio_mode=False,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False, # FIXME: implement
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -19,7 +19,6 @@ class OutputProcessorOutput:
|
||||
|
||||
request_outputs: List[RequestOutput]
|
||||
reqs_to_abort: List[str]
|
||||
iteration_stats: IterationStats
|
||||
|
||||
|
||||
class RequestState:
|
||||
@@ -34,6 +33,7 @@ class RequestState:
|
||||
detokenizer: IncrementalDetokenizer,
|
||||
arrival_time: float,
|
||||
queue: Optional[asyncio.Queue[RequestOutput]],
|
||||
log_stats: bool,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.output_kind = output_kind
|
||||
@@ -45,14 +45,16 @@ class RequestState:
|
||||
self.is_prefilling = True
|
||||
self.queue = queue
|
||||
|
||||
self.stats = RequestStateStats(last_token_time=arrival_time)
|
||||
self.stats = RequestStateStats(
|
||||
arrival_time=arrival_time) if log_stats else None
|
||||
|
||||
@classmethod
|
||||
def from_new_request(
|
||||
cls,
|
||||
tokenizer: AnyTokenizer,
|
||||
request: EngineCoreRequest,
|
||||
queue: Optional[asyncio.Queue[RequestOutput]] = None,
|
||||
queue: Optional[asyncio.Queue[RequestOutput]],
|
||||
log_stats: bool,
|
||||
) -> "RequestState":
|
||||
return cls(
|
||||
request_id=request.request_id,
|
||||
@@ -69,6 +71,7 @@ class RequestState:
|
||||
),
|
||||
arrival_time=request.arrival_time,
|
||||
queue=queue,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
|
||||
|
||||
@@ -112,11 +115,13 @@ class OutputProcessor:
|
||||
self.request_states[request_id] = RequestState.from_new_request(
|
||||
tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
|
||||
request=request,
|
||||
queue=queue)
|
||||
queue=queue,
|
||||
log_stats=self.log_stats)
|
||||
|
||||
def process_outputs(
|
||||
self,
|
||||
engine_core_outputs: List[EngineCoreOutput],
|
||||
engine_core_timestamp: Optional[float] = None,
|
||||
iteration_stats: Optional[IterationStats] = None,
|
||||
) -> OutputProcessorOutput:
|
||||
"""
|
||||
@@ -145,8 +150,6 @@ class OutputProcessor:
|
||||
|
||||
request_outputs: List[RequestOutput] = []
|
||||
reqs_to_abort: List[str] = []
|
||||
if not iteration_stats:
|
||||
iteration_stats = IterationStats(self.log_stats)
|
||||
for engine_core_output in engine_core_outputs:
|
||||
req_id = engine_core_output.request_id
|
||||
req_state = self.request_states.get(req_id)
|
||||
@@ -155,10 +158,9 @@ class OutputProcessor:
|
||||
continue
|
||||
|
||||
# 1) Compute stats for this iteration.
|
||||
iteration_stats.update_from_output(engine_core_output,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats)
|
||||
self._update_stats_from_output(req_state, engine_core_output,
|
||||
engine_core_timestamp,
|
||||
iteration_stats)
|
||||
|
||||
new_token_ids = engine_core_output.new_token_ids
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
@@ -205,17 +207,44 @@ class OutputProcessor:
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats.
|
||||
assert finish_reason is not None
|
||||
iteration_stats.update_from_finished_request(
|
||||
finish_reason, request_output, req_state.stats)
|
||||
# Track per-request stats
|
||||
self._update_stats_from_finished(req_state, request_output,
|
||||
finish_reason,
|
||||
iteration_stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
reqs_to_abort=reqs_to_abort,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
|
||||
def _update_stats_from_output(self, req_state: RequestState,
|
||||
engine_core_output: EngineCoreOutput,
|
||||
engine_core_timestamp: Optional[float],
|
||||
iteration_stats: Optional[IterationStats]):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
assert engine_core_timestamp is not None
|
||||
assert req_state.stats is not None
|
||||
iteration_stats.update_from_output(engine_core_output,
|
||||
engine_core_timestamp,
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats)
|
||||
|
||||
def _update_stats_from_finished(self, req_state: RequestState,
|
||||
request_output: RequestOutput,
|
||||
finish_reason: Optional[FinishReason],
|
||||
iteration_stats: Optional[IterationStats]):
|
||||
if iteration_stats is None:
|
||||
return
|
||||
|
||||
assert finish_reason is not None
|
||||
assert req_state.stats is not None
|
||||
iteration_stats.update_from_finished_request(finish_reason,
|
||||
request_output,
|
||||
req_state.stats)
|
||||
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
request_state: RequestState,
|
||||
|
||||
Reference in New Issue
Block a user