[MRV2] Keep warmup out of the EPLB load stats (#56716)
Signed-off-by: almogtavor <almogtavor@gmail.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
This commit is contained in:
co-authored by
Itay Alroy
parent
f43d7e1a0f
commit
87e7dce3a7
@@ -23,6 +23,7 @@ from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.worker.gpu import cudagraph_utils as cgu
|
||||
from vllm.v1.worker.gpu import model_runner as mrv2
|
||||
from vllm.v1.worker.gpu.eplb_utils import EPLBController
|
||||
|
||||
GLOBAL_POOL = "global-pool"
|
||||
THROWAWAY_POOL = "throwaway-pool"
|
||||
@@ -224,6 +225,9 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
|
||||
|
||||
def test_model_runner_delegates_to_cudagraph_utils(monkeypatch):
|
||||
runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
|
||||
runner.eplb = EPLBController(
|
||||
SimpleNamespace(enable_eplb=False), torch.device("cpu")
|
||||
)
|
||||
monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42)
|
||||
assert runner.profile_cudagraph_memory() == 42
|
||||
|
||||
|
||||
@@ -584,15 +584,15 @@ class EplbState:
|
||||
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
if is_profile:
|
||||
self.rearrange(is_profile=True)
|
||||
return
|
||||
|
||||
if is_dummy:
|
||||
# Do not record load metrics for dummy steps
|
||||
for eplb_model_state in self.model_states.values():
|
||||
eplb_model_state.expert_load_pass.zero_()
|
||||
|
||||
if is_profile:
|
||||
self.rearrange(is_profile=True)
|
||||
return
|
||||
|
||||
if (
|
||||
log_stats
|
||||
and self.expert_rearrangement_step
|
||||
|
||||
@@ -49,10 +49,12 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable:
|
||||
def decorator(fn: Callable) -> Callable:
|
||||
@wraps(fn)
|
||||
def wrapper(self: Any, *args, **kwargs) -> Any:
|
||||
result = fn(self, *args, **kwargs)
|
||||
if kwargs.get("skip_eplb", False):
|
||||
return result
|
||||
# The router kernel records load before step() is reached.
|
||||
with self.eplb.suppress():
|
||||
return fn(self, *args, **kwargs)
|
||||
|
||||
result = fn(self, *args, **kwargs)
|
||||
is_profile = kwargs.get("is_profile", False) if is_dummy else False
|
||||
self.eplb.step(is_dummy=is_dummy, is_profile=is_profile)
|
||||
return result
|
||||
@@ -91,6 +93,24 @@ class EPLBController:
|
||||
self.suppressed = False
|
||||
self._has_registered_models = False
|
||||
|
||||
@contextmanager
|
||||
def suppress(self) -> Iterator[None]:
|
||||
"""Keep warmup forwards from stepping EPLB or recording expert load."""
|
||||
should_record = (
|
||||
self.state.should_record_tensor if self.state is not None else None
|
||||
)
|
||||
was_suppressed = self.suppressed
|
||||
was_recording = should_record.clone() if should_record is not None else None
|
||||
self.suppressed = True
|
||||
if should_record is not None:
|
||||
should_record.fill_(False)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if should_record is not None and was_recording is not None:
|
||||
should_record.copy_(was_recording)
|
||||
self.suppressed = was_suppressed
|
||||
|
||||
def prepare_load(self) -> None:
|
||||
self.state = None
|
||||
self._has_registered_models = False
|
||||
|
||||
@@ -1037,7 +1037,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
@torch.inference_mode()
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
"""Estimate the GPU memory required to capture CUDA graphs."""
|
||||
return _profile_cudagraph_memory(self)
|
||||
# Runs before the KV cache exists, outside compile_or_warm_up_model.
|
||||
with self.eplb.suppress():
|
||||
return _profile_cudagraph_memory(self)
|
||||
|
||||
def needs_cudagraph_capture(self) -> bool:
|
||||
"""Whether capture_model() has any CUDA graphs left to capture."""
|
||||
|
||||
@@ -906,6 +906,13 @@ class Worker(WorkerBase):
|
||||
|
||||
@instrument(span_name="Warmup (GPU)")
|
||||
def compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
if not self.use_v2_model_runner:
|
||||
return self._compile_or_warm_up_model()
|
||||
# MRV2 warmup forwards record expert load like real ones.
|
||||
with self.model_runner.eplb.suppress(): # type: ignore
|
||||
return self._compile_or_warm_up_model()
|
||||
|
||||
def _compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
warmup_sizes: list[int] = []
|
||||
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user