[MRV2] Keep warmup out of the EPLB load stats (#56716)

Signed-off-by: almogtavor <almogtavor@gmail.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
This commit is contained in:
Almog Tavor
2026-10-14 00:00:48 +00:00
committed by GitHub
co-authored by Itay Alroy
parent f43d7e1a0f
commit 87e7dce3a7
5 changed files with 40 additions and 7 deletions
@@ -23,6 +23,7 @@ from vllm.compilation.counter import compilation_counter
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu import cudagraph_utils as cgu
from vllm.v1.worker.gpu import model_runner as mrv2
from vllm.v1.worker.gpu.eplb_utils import EPLBController
GLOBAL_POOL = "global-pool"
THROWAWAY_POOL = "throwaway-pool"
@@ -224,6 +225,9 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
def test_model_runner_delegates_to_cudagraph_utils(monkeypatch):
runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
runner.eplb = EPLBController(
SimpleNamespace(enable_eplb=False), torch.device("cpu")
)
monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42)
assert runner.profile_cudagraph_memory() == 42
+4 -4
View File
@@ -584,15 +584,15 @@ class EplbState:
"""
ep_group = get_ep_group().device_group
if is_profile:
self.rearrange(is_profile=True)
return
if is_dummy:
# Do not record load metrics for dummy steps
for eplb_model_state in self.model_states.values():
eplb_model_state.expert_load_pass.zero_()
if is_profile:
self.rearrange(is_profile=True)
return
if (
log_stats
and self.expert_rearrangement_step
+22 -2
View File
@@ -49,10 +49,12 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable:
def decorator(fn: Callable) -> Callable:
@wraps(fn)
def wrapper(self: Any, *args, **kwargs) -> Any:
result = fn(self, *args, **kwargs)
if kwargs.get("skip_eplb", False):
return result
# The router kernel records load before step() is reached.
with self.eplb.suppress():
return fn(self, *args, **kwargs)
result = fn(self, *args, **kwargs)
is_profile = kwargs.get("is_profile", False) if is_dummy else False
self.eplb.step(is_dummy=is_dummy, is_profile=is_profile)
return result
@@ -91,6 +93,24 @@ class EPLBController:
self.suppressed = False
self._has_registered_models = False
@contextmanager
def suppress(self) -> Iterator[None]:
"""Keep warmup forwards from stepping EPLB or recording expert load."""
should_record = (
self.state.should_record_tensor if self.state is not None else None
)
was_suppressed = self.suppressed
was_recording = should_record.clone() if should_record is not None else None
self.suppressed = True
if should_record is not None:
should_record.fill_(False)
try:
yield
finally:
if should_record is not None and was_recording is not None:
should_record.copy_(was_recording)
self.suppressed = was_suppressed
def prepare_load(self) -> None:
self.state = None
self._has_registered_models = False
+3 -1
View File
@@ -1037,7 +1037,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
@torch.inference_mode()
def profile_cudagraph_memory(self) -> int:
"""Estimate the GPU memory required to capture CUDA graphs."""
return _profile_cudagraph_memory(self)
# Runs before the KV cache exists, outside compile_or_warm_up_model.
with self.eplb.suppress():
return _profile_cudagraph_memory(self)
def needs_cudagraph_capture(self) -> bool:
"""Whether capture_model() has any CUDA graphs left to capture."""
+7
View File
@@ -906,6 +906,13 @@ class Worker(WorkerBase):
@instrument(span_name="Warmup (GPU)")
def compile_or_warm_up_model(self) -> CompilationTimes:
if not self.use_v2_model_runner:
return self._compile_or_warm_up_model()
# MRV2 warmup forwards record expert load like real ones.
with self.model_runner.eplb.suppress(): # type: ignore
return self._compile_or_warm_up_model()
def _compile_or_warm_up_model(self) -> CompilationTimes:
warmup_sizes: list[int] = []
if (