From 457a5f34db04b082239c11ee34601e5ee5754a7c Mon Sep 17 00:00:00 2001 From: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:00:49 +0800 Subject: [PATCH] [Bugfix][MRV2] Support encoder timing stats in model runner V2 (#50020) Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/worker/test_encoder_runner.py | 16 +++++++ vllm/v1/worker/gpu/mm/encoder_runner.py | 44 ++++++++++++++++++- vllm/v1/worker/gpu/model_runner.py | 6 +++ .../gpu/model_states/encoder_decoder.py | 3 +- vllm/v1/worker/gpu/model_states/interface.py | 10 ++++- vllm/v1/worker/gpu_model_runner.py | 25 +++-------- vllm/v1/worker/utils.py | 17 +++++++ 7 files changed, 99 insertions(+), 22 deletions(-) diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py index a1f5555359..914985b829 100644 --- a/tests/v1/worker/test_encoder_runner.py +++ b/tests/v1/worker/test_encoder_runner.py @@ -224,3 +224,19 @@ def test_execute_mm_encoder_is_a_noop_without_scheduled_items(): assert not cache.encoder_outputs state.encoder_runner.execute_mm_encoder.assert_not_called() + + +def test_encoder_timing_stats_registry(): + runner = _make_runner([], []) + runner.enable_timing = True + + with runner.timed_encoder_operation({"r1"}): + pass + with runner.timed_encoder_operation({"r1"}): + pass + + stats = runner.get_encoder_timing_stats() + assert set(stats) == {"r1"} + assert stats["r1"]["num_encoder_calls"] == 2 + assert stats["r1"]["encoder_forward_secs"] >= 0 + assert runner.get_encoder_timing_stats() == {} diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index df4edbe7a5..e255a8675c 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -1,5 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import threading +import time +from collections.abc import Collection +from contextlib import contextmanager + import numpy as np import torch @@ -13,7 +18,10 @@ from vllm.multimodal.utils import ( set_mm_embedding_modality, ) from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache -from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs +from vllm.v1.worker.utils import ( + EncoderTimingStats, + sanity_check_mm_encoder_outputs, +) logger = init_logger(__name__) @@ -27,6 +35,7 @@ class EncoderRunner: encoder_cache: EncoderCache, dtype: torch.dtype, device: torch.device, + enable_timing: bool = False, ): self.model = model self.max_num_tokens = max_num_tokens @@ -35,6 +44,9 @@ class EncoderRunner: self.dtype = dtype self.device = device self.is_realtime = supports_realtime(model) + self.enable_timing = enable_timing + self.encoder_timing_registry: dict[str, EncoderTimingStats] = {} + self._timing_lock = threading.Lock() self.inputs_embeds = torch.zeros( max_num_tokens, hidden_size, dtype=dtype, device=device @@ -110,6 +122,36 @@ class EncoderRunner: encoder_outputs.extend(batch_outputs) return encoder_outputs + @contextmanager + def timed_encoder_operation(self, request_ids: Collection[str]): + if not (self.enable_timing and request_ids): + yield + return + + torch.accelerator.synchronize() + start_time = time.perf_counter() + try: + yield + finally: + torch.accelerator.synchronize() + per_request_time = (time.perf_counter() - start_time) / len(request_ids) + with self._timing_lock: + for req_id in request_ids: + stats = self.encoder_timing_registry.setdefault( + req_id, EncoderTimingStats() + ) + stats.encoder_forward_secs += per_request_time + stats.num_encoder_calls += 1 + + def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]: + with self._timing_lock: + stats = { + req_id: stats_obj.to_dict() + for req_id, stats_obj in self.encoder_timing_registry.items() + } + self.encoder_timing_registry.clear() + return stats + def gather_mm_embeddings( self, req_ids: list[str], diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a6ac6eb67e..3bf89ee156 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -453,6 +453,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Cache the default CUDA stream to avoid lookup overhead. return torch.cuda.current_stream(self.device) + def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]: + encoder_runner = getattr(self.model_state, "encoder_runner", None) + if encoder_runner is None: + return {} + return encoder_runner.get_encoder_timing_stats() + def get_kv_cache_spec(self): if self.is_encoder_only: return {} diff --git a/vllm/v1/worker/gpu/model_states/encoder_decoder.py b/vllm/v1/worker/gpu/model_states/encoder_decoder.py index 618984c97f..7a461466d0 100644 --- a/vllm/v1/worker/gpu/model_states/encoder_decoder.py +++ b/vllm/v1/worker/gpu/model_states/encoder_decoder.py @@ -86,7 +86,8 @@ class EncoderDecoderModelState(ModelState): # so execute_mm_encoder preserves request order; use its return value # directly. No need to store in encoder_cache: cross-attention K/V are # written to the KV cache on the first step; decode steps use the cache. - self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs) + with self.encoder_runner.timed_encoder_operation(encoder_inputs.keys()): + self.encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs) else: # Decode steps: encoder K/V are in cross-attention KV cache. self.encoder_outputs = [] diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index a91d23fe92..722b709bb5 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -60,6 +60,7 @@ class ModelState(ABC): self.supports_mm_inputs = encoder_cache is not None if encoder_cache is not None: self.encoder_cache = encoder_cache + observability_config = vllm_config.observability_config self.encoder_runner = EncoderRunner( model=self.model, max_num_tokens=self.max_num_tokens, @@ -67,6 +68,10 @@ class ModelState(ABC): encoder_cache=encoder_cache, dtype=self.dtype, device=self.device, + enable_timing=bool( + observability_config + and observability_config.enable_mm_processor_stats + ), ) def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]: @@ -150,7 +155,10 @@ class ModelState(ABC): scheduled_encoder_inputs ) if mm_kwargs: - encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs) + with self.encoder_runner.timed_encoder_operation( + scheduled_encoder_inputs.keys() + ): + encoder_outputs = self.encoder_runner.execute_mm_encoder(mm_kwargs) self.encoder_cache.encoder_outputs.update(zip(mm_hashes, encoder_outputs)) def gather_mm_embeddings( diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 43f5c45323..d11cb858e9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -10,7 +10,7 @@ from collections import defaultdict from collections.abc import Callable, Iterable, Iterator, Sequence from contextlib import AbstractContextManager, contextmanager, nullcontext from copy import copy, deepcopy -from dataclasses import dataclass, replace +from dataclasses import replace from functools import reduce from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast @@ -230,7 +230,11 @@ from vllm.v1.worker.ubatch_utils import ( maybe_create_ubatch_slices, split_attn_metadata, ) -from vllm.v1.worker.utils import is_residual_scattered_for_sp, raise_if_nan_logits +from vllm.v1.worker.utils import ( + EncoderTimingStats, + is_residual_scattered_for_sp, + raise_if_nan_logits, +) from vllm.v1.worker.workspace import lock_workspace from .utils import ( @@ -7948,20 +7952,3 @@ class GPUModelRunner( stats = self.encoder_timing_registry[req_id] stats.encoder_forward_secs += per_request_time stats.num_encoder_calls += 1 - - -@dataclass -class EncoderTimingStats: - """Per-request timing statistics for encoder forward pass.""" - - encoder_forward_secs: float = 0.0 - """Time spent in vision encoder forward pass (seconds).""" - - num_encoder_calls: int = 0 - """Number of times encoder was called for this request.""" - - def to_dict(self) -> dict[str, float | int]: - return { - "encoder_forward_secs": self.encoder_forward_secs, - "num_encoder_calls": self.num_encoder_calls, - } diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index 55f93860ad..b10a236ae0 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -639,3 +639,20 @@ def is_residual_scattered_for_sp( assert num_input_tokens % tp == 0 return True + + +@dataclass +class EncoderTimingStats: + """Per-request timing statistics for encoder forward pass.""" + + encoder_forward_secs: float = 0.0 + """Time spent in vision encoder forward pass (seconds).""" + + num_encoder_calls: int = 0 + """Number of times encoder was called for this request.""" + + def to_dict(self) -> dict[str, float | int]: + return { + "encoder_forward_secs": self.encoder_forward_secs, + "num_encoder_calls": self.num_encoder_calls, + }