[Bugfix][MRV2] Support encoder timing stats in model runner V2 (#50020)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5426311d91
commit
457a5f34db
@@ -224,3 +224,19 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
|
||||
assert not cache.encoder_outputs
|
||||
state.encoder_runner.execute_mm_encoder.assert_not_called()
|
||||
|
||||
|
||||
def test_encoder_timing_stats_registry():
|
||||
runner = _make_runner([], [])
|
||||
runner.enable_timing = True
|
||||
|
||||
with runner.timed_encoder_operation({"r1"}):
|
||||
pass
|
||||
with runner.timed_encoder_operation({"r1"}):
|
||||
pass
|
||||
|
||||
stats = runner.get_encoder_timing_stats()
|
||||
assert set(stats) == {"r1"}
|
||||
assert stats["r1"]["num_encoder_calls"] == 2
|
||||
assert stats["r1"]["encoder_forward_secs"] >= 0
|
||||
assert runner.get_encoder_timing_stats() == {}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Collection
|
||||
from contextlib import contextmanager
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
@@ -13,7 +18,10 @@ from vllm.multimodal.utils import (
|
||||
set_mm_embedding_modality,
|
||||
)
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
|
||||
from vllm.v1.worker.utils import (
|
||||
EncoderTimingStats,
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -27,6 +35,7 @@ class EncoderRunner:
|
||||
encoder_cache: EncoderCache,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
enable_timing: bool = False,
|
||||
):
|
||||
self.model = model
|
||||
self.max_num_tokens = max_num_tokens
|
||||
@@ -35,6 +44,9 @@ class EncoderRunner:
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.is_realtime = supports_realtime(model)
|
||||
self.enable_timing = enable_timing
|
||||
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
|
||||
self._timing_lock = threading.Lock()
|
||||
|
||||
self.inputs_embeds = torch.zeros(
|
||||
max_num_tokens, hidden_size, dtype=dtype, device=device
|
||||
@@ -110,6 +122,36 @@ class EncoderRunner:
|
||||
encoder_outputs.extend(batch_outputs)
|
||||
return encoder_outputs
|
||||
|
||||
@contextmanager
|
||||
def timed_encoder_operation(self, request_ids: Collection[str]):
|
||||
if not (self.enable_timing and request_ids):
|
||||
yield
|
||||
return
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.accelerator.synchronize()
|
||||
per_request_time = (time.perf_counter() - start_time) / len(request_ids)
|
||||
with self._timing_lock:
|
||||
for req_id in request_ids:
|
||||
stats = self.encoder_timing_registry.setdefault(
|
||||
req_id, EncoderTimingStats()
|
||||
)
|
||||
stats.encoder_forward_secs += per_request_time
|
||||
stats.num_encoder_calls += 1
|
||||
|
||||
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
|
||||
with self._timing_lock:
|
||||
stats = {
|
||||
req_id: stats_obj.to_dict()
|
||||
for req_id, stats_obj in self.encoder_timing_registry.items()
|
||||
}
|
||||
self.encoder_timing_registry.clear()
|
||||
return stats
|
||||
|
||||
def gather_mm_embeddings(
|
||||
self,
|
||||
req_ids: list[str],
|
||||
|
||||
@@ -453,6 +453,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Cache the default CUDA stream to avoid lookup overhead.
|
||||
return torch.cuda.current_stream(self.device)
|
||||
|
||||
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
|
||||
encoder_runner = getattr(self.model_state, "encoder_runner", None)
|
||||
if encoder_runner is None:
|
||||
return {}
|
||||
return encoder_runner.get_encoder_timing_stats()
|
||||
|
||||
def get_kv_cache_spec(self):
|
||||
if self.is_encoder_only:
|
||||
return {}
|
||||
|
||||
@@ -86,7 +86,8 @@ class EncoderDecoderModelState(ModelState):
|
||||
# so execute_mm_encoder preserves request order; use its return value
|
||||
# directly. No need to store in encoder_cache: cross-attention K/V are
|
||||
# written to the KV cache on the first step; decode steps use the cache.
|
||||
self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
with self.encoder_runner.timed_encoder_operation(encoder_inputs.keys()):
|
||||
self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
else:
|
||||
# Decode steps: encoder K/V are in cross-attention KV cache.
|
||||
self.encoder_outputs = []
|
||||
|
||||
@@ -60,6 +60,7 @@ class ModelState(ABC):
|
||||
self.supports_mm_inputs = encoder_cache is not None
|
||||
if encoder_cache is not None:
|
||||
self.encoder_cache = encoder_cache
|
||||
observability_config = vllm_config.observability_config
|
||||
self.encoder_runner = EncoderRunner(
|
||||
model=self.model,
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
@@ -67,6 +68,10 @@ class ModelState(ABC):
|
||||
encoder_cache=encoder_cache,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
enable_timing=bool(
|
||||
observability_config
|
||||
and observability_config.enable_mm_processor_stats
|
||||
),
|
||||
)
|
||||
|
||||
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
|
||||
@@ -150,7 +155,10 @@ class ModelState(ABC):
|
||||
scheduled_encoder_inputs
|
||||
)
|
||||
if mm_kwargs:
|
||||
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
with self.encoder_runner.timed_encoder_operation(
|
||||
scheduled_encoder_inputs.keys()
|
||||
):
|
||||
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
|
||||
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
|
||||
|
||||
def gather_mm_embeddings(
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from copy import copy, deepcopy
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import replace
|
||||
from functools import reduce
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
|
||||
|
||||
@@ -230,7 +230,11 @@ from vllm.v1.worker.ubatch_utils import (
|
||||
maybe_create_ubatch_slices,
|
||||
split_attn_metadata,
|
||||
)
|
||||
from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits
|
||||
from vllm.v1.worker.utils import (
|
||||
EncoderTimingStats,
|
||||
is_residual_scattered_for_sp,
|
||||
raise_if_nan_logits,
|
||||
)
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
from .utils import (
|
||||
@@ -7948,20 +7952,3 @@ class GPUModelRunner(
|
||||
stats = self.encoder_timing_registry[req_id]
|
||||
stats.encoder_forward_secs += per_request_time
|
||||
stats.num_encoder_calls += 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class EncoderTimingStats:
|
||||
"""Per-request timing statistics for encoder forward pass."""
|
||||
|
||||
encoder_forward_secs: float = 0.0
|
||||
"""Time spent in vision encoder forward pass (seconds)."""
|
||||
|
||||
num_encoder_calls: int = 0
|
||||
"""Number of times encoder was called for this request."""
|
||||
|
||||
def to_dict(self) -> dict[str, float | int]:
|
||||
return {
|
||||
"encoder_forward_secs": self.encoder_forward_secs,
|
||||
"num_encoder_calls": self.num_encoder_calls,
|
||||
}
|
||||
|
||||
@@ -639,3 +639,20 @@ def is_residual_scattered_for_sp(
|
||||
assert num_input_tokens % tp == 0
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EncoderTimingStats:
|
||||
"""Per-request timing statistics for encoder forward pass."""
|
||||
|
||||
encoder_forward_secs: float = 0.0
|
||||
"""Time spent in vision encoder forward pass (seconds)."""
|
||||
|
||||
num_encoder_calls: int = 0
|
||||
"""Number of times encoder was called for this request."""
|
||||
|
||||
def to_dict(self) -> dict[str, float | int]:
|
||||
return {
|
||||
"encoder_forward_secs": self.encoder_forward_secs,
|
||||
"num_encoder_calls": self.num_encoder_calls,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user