[Bugfix][MRV2] Support encoder timing stats in model runner V2 (#50020)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Guan-Ming Chiu
2026-08-12 00:00:49 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5426311d91
commit 457a5f34db
7 changed files with 99 additions and 22 deletions
+16
View File
@@ -224,3 +224,19 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
assert not cache.encoder_outputs
state.encoder_runner.execute_mm_encoder.assert_not_called()
def test_encoder_timing_stats_registry():
runner = _make_runner([], [])
runner.enable_timing = True
with runner.timed_encoder_operation({"r1"}):
pass
with runner.timed_encoder_operation({"r1"}):
pass
stats = runner.get_encoder_timing_stats()
assert set(stats) == {"r1"}
assert stats["r1"]["num_encoder_calls"] == 2
assert stats["r1"]["encoder_forward_secs"] >= 0
assert runner.get_encoder_timing_stats() == {}
+43 -1
View File
@@ -1,5 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
from collections.abc import Collection
from contextlib import contextmanager
import numpy as np
import torch
@@ -13,7 +18,10 @@ from vllm.multimodal.utils import (
set_mm_embedding_modality,
)
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
from vllm.v1.worker.utils import (
EncoderTimingStats,
sanity_check_mm_encoder_outputs,
)
logger = init_logger(__name__)
@@ -27,6 +35,7 @@ class EncoderRunner:
encoder_cache: EncoderCache,
dtype: torch.dtype,
device: torch.device,
enable_timing: bool = False,
):
self.model = model
self.max_num_tokens = max_num_tokens
@@ -35,6 +44,9 @@ class EncoderRunner:
self.dtype = dtype
self.device = device
self.is_realtime = supports_realtime(model)
self.enable_timing = enable_timing
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
self._timing_lock = threading.Lock()
self.inputs_embeds = torch.zeros(
max_num_tokens, hidden_size, dtype=dtype, device=device
@@ -110,6 +122,36 @@ class EncoderRunner:
encoder_outputs.extend(batch_outputs)
return encoder_outputs
@contextmanager
def timed_encoder_operation(self, request_ids: Collection[str]):
if not (self.enable_timing and request_ids):
yield
return
torch.accelerator.synchronize()
start_time = time.perf_counter()
try:
yield
finally:
torch.accelerator.synchronize()
per_request_time = (time.perf_counter() - start_time) / len(request_ids)
with self._timing_lock:
for req_id in request_ids:
stats = self.encoder_timing_registry.setdefault(
req_id, EncoderTimingStats()
)
stats.encoder_forward_secs += per_request_time
stats.num_encoder_calls += 1
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
with self._timing_lock:
stats = {
req_id: stats_obj.to_dict()
for req_id, stats_obj in self.encoder_timing_registry.items()
}
self.encoder_timing_registry.clear()
return stats
def gather_mm_embeddings(
self,
req_ids: list[str],
+6
View File
@@ -453,6 +453,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Cache the default CUDA stream to avoid lookup overhead.
return torch.cuda.current_stream(self.device)
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
encoder_runner = getattr(self.model_state, "encoder_runner", None)
if encoder_runner is None:
return {}
return encoder_runner.get_encoder_timing_stats()
def get_kv_cache_spec(self):
if self.is_encoder_only:
return {}
@@ -86,7 +86,8 @@ class EncoderDecoderModelState(ModelState):
# so execute_mm_encoder preserves request order; use its return value
# directly. No need to store in encoder_cache: cross-attention K/V are
# written to the KV cache on the first step; decode steps use the cache.
self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
with self.encoder_runner.timed_encoder_operation(encoder_inputs.keys()):
self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
else:
# Decode steps: encoder K/V are in cross-attention KV cache.
self.encoder_outputs = []
+9 -1
View File
@@ -60,6 +60,7 @@ class ModelState(ABC):
self.supports_mm_inputs = encoder_cache is not None
if encoder_cache is not None:
self.encoder_cache = encoder_cache
observability_config = vllm_config.observability_config
self.encoder_runner = EncoderRunner(
model=self.model,
max_num_tokens=self.max_num_tokens,
@@ -67,6 +68,10 @@ class ModelState(ABC):
encoder_cache=encoder_cache,
dtype=self.dtype,
device=self.device,
enable_timing=bool(
observability_config
and observability_config.enable_mm_processor_stats
),
)
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
@@ -150,7 +155,10 @@ class ModelState(ABC):
scheduled_encoder_inputs
)
if mm_kwargs:
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
with self.encoder_runner.timed_encoder_operation(
scheduled_encoder_inputs.keys()
):
encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs)
self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs))
def gather_mm_embeddings(
+6 -19
View File
@@ -10,7 +10,7 @@ from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from contextlib import AbstractContextManager, contextmanager, nullcontext
from copy import copy, deepcopy
from dataclasses import dataclass, replace
from dataclasses import replace
from functools import reduce
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
@@ -230,7 +230,11 @@ from vllm.v1.worker.ubatch_utils import (
maybe_create_ubatch_slices,
split_attn_metadata,
)
from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits
from vllm.v1.worker.utils import (
EncoderTimingStats,
is_residual_scattered_for_sp,
raise_if_nan_logits,
)
from vllm.v1.worker.workspace import lock_workspace
from .utils import (
@@ -7948,20 +7952,3 @@ class GPUModelRunner(
stats = self.encoder_timing_registry[req_id]
stats.encoder_forward_secs += per_request_time
stats.num_encoder_calls += 1
@dataclass
class EncoderTimingStats:
"""Per-request timing statistics for encoder forward pass."""
encoder_forward_secs: float = 0.0
"""Time spent in vision encoder forward pass (seconds)."""
num_encoder_calls: int = 0
"""Number of times encoder was called for this request."""
def to_dict(self) -> dict[str, float | int]:
return {
"encoder_forward_secs": self.encoder_forward_secs,
"num_encoder_calls": self.num_encoder_calls,
}
+17
View File
@@ -639,3 +639,20 @@ def is_residual_scattered_for_sp(
assert num_input_tokens % tp == 0
return True
@dataclass
class EncoderTimingStats:
"""Per-request timing statistics for encoder forward pass."""
encoder_forward_secs: float = 0.0
"""Time spent in vision encoder forward pass (seconds)."""
num_encoder_calls: int = 0
"""Number of times encoder was called for this request."""
def to_dict(self) -> dict[str, float | int]:
return {
"encoder_forward_secs": self.encoder_forward_secs,
"num_encoder_calls": self.num_encoder_calls,
}