diff --git a/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py index 74191206d3..a2aca98dd8 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py @@ -23,6 +23,7 @@ from vllm.compilation.counter import compilation_counter from vllm.config.compilation import CUDAGraphMode from vllm.v1.worker.gpu import cudagraph_utils as cgu from vllm.v1.worker.gpu import model_runner as mrv2 +from vllm.v1.worker.gpu.eplb_utils import EPLBController GLOBAL_POOL = "global-pool" THROWAWAY_POOL = "throwaway-pool" @@ -224,6 +225,9 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch): def test_model_runner_delegates_to_cudagraph_utils(monkeypatch): runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner) + runner.eplb = EPLBController( + SimpleNamespace(enable_eplb=False), torch.device("cpu") + ) monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42) assert runner.profile_cudagraph_memory() == 42 diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 2d782c4179..303d69ba23 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -584,15 +584,15 @@ class EplbState: """ ep_group = get_ep_group().device_group - if is_profile: - self.rearrange(is_profile=True) - return - if is_dummy: # Do not record load metrics for dummy steps for eplb_model_state in self.model_states.values(): eplb_model_state.expert_load_pass.zero_() + if is_profile: + self.rearrange(is_profile=True) + return + if ( log_stats and self.expert_rearrangement_step diff --git a/vllm/v1/worker/gpu/eplb_utils.py b/vllm/v1/worker/gpu/eplb_utils.py index 5c2c5d1412..e7706d7087 100644 --- a/vllm/v1/worker/gpu/eplb_utils.py +++ b/vllm/v1/worker/gpu/eplb_utils.py @@ -49,10 +49,12 @@ def step_eplb_after(*, is_dummy: bool = False) -> Callable: def decorator(fn: Callable) -> Callable: @wraps(fn) def wrapper(self: Any, *args, **kwargs) -> Any: - result = fn(self, *args, **kwargs) if kwargs.get("skip_eplb", False): - return result + # The router kernel records load before step() is reached. + with self.eplb.suppress(): + return fn(self, *args, **kwargs) + result = fn(self, *args, **kwargs) is_profile = kwargs.get("is_profile", False) if is_dummy else False self.eplb.step(is_dummy=is_dummy, is_profile=is_profile) return result @@ -91,6 +93,24 @@ class EPLBController: self.suppressed = False self._has_registered_models = False + @contextmanager + def suppress(self) -> Iterator[None]: + """Keep warmup forwards from stepping EPLB or recording expert load.""" + should_record = ( + self.state.should_record_tensor if self.state is not None else None + ) + was_suppressed = self.suppressed + was_recording = should_record.clone() if should_record is not None else None + self.suppressed = True + if should_record is not None: + should_record.fill_(False) + try: + yield + finally: + if should_record is not None and was_recording is not None: + should_record.copy_(was_recording) + self.suppressed = was_suppressed + def prepare_load(self) -> None: self.state = None self._has_registered_models = False diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 57cef586f8..08e70f3380 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1037,7 +1037,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): @torch.inference_mode() def profile_cudagraph_memory(self) -> int: """Estimate the GPU memory required to capture CUDA graphs.""" - return _profile_cudagraph_memory(self) + # Runs before the KV cache exists, outside compile_or_warm_up_model. + with self.eplb.suppress(): + return _profile_cudagraph_memory(self) def needs_cudagraph_capture(self) -> bool: """Whether capture_model() has any CUDA graphs left to capture.""" diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 04feb5da02..4fcac2d6c1 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -906,6 +906,13 @@ class Worker(WorkerBase): @instrument(span_name="Warmup (GPU)") def compile_or_warm_up_model(self) -> CompilationTimes: + if not self.use_v2_model_runner: + return self._compile_or_warm_up_model() + # MRV2 warmup forwards record expert load like real ones. + with self.model_runner.eplb.suppress(): # type: ignore + return self._compile_or_warm_up_model() + + def _compile_or_warm_up_model(self) -> CompilationTimes: warmup_sizes: list[int] = [] if (