[Bugfix][V2] Warm up kernels before capturing CUDA graphs (#55341)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-09-04 22:44:18 +00:00
committed by GitHub
co-authored by aoshen02 Nick Hill
parent 8f269a93bb
commit 685074cd61
7 changed files with 129 additions and 16 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
from types import SimpleNamespace
import pytest
@@ -213,3 +214,84 @@ def test_append_block_ids_rejects_write_past_row_capacity():
)
assert block_tables.num_blocks.np[0, 1] == 3
def _make_capture_runner(captured: bool) -> GPUModelRunner:
"""Minimal V2 runner for capture_model: fakes everything except the
cudagraph_manager's needs_capture decision."""
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.model_state = SimpleNamespace(supports_mm_inputs=False)
runner.cudagraph_manager = SimpleNamespace(
needs_capture=lambda: captured,
capture=lambda *args, **kwargs: None,
)
runner.lora_config = None
runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext()
runner.speculator = None
runner.adaptive_verification = None
runner.model = None
runner.input_buffers = None
runner.pcp_manager = None
runner.intermediate_tensors = None
runner.block_tables = None
runner.attn_groups = None
runner.kv_cache_config = None
runner.use_aux_hidden_state_outputs = False
return runner
def test_capture_model_locks_workspace_after_capture(monkeypatch):
"""A workspace resize after capture frees the buffer the captured graphs
baked in, so capture_model must lock the workspace before returning
(https://github.com/vllm-project/vllm/issues/55336)."""
runner = _make_capture_runner(captured=True)
monkeypatch.setattr(
model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext
)
monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None)
monkeypatch.setattr(
torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
)
lock_calls = []
monkeypatch.setattr(
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
)
runner.capture_model()
assert lock_calls == ["lock"]
def test_capture_model_skips_lock_when_nothing_captured(monkeypatch):
"""With no graphs to capture (e.g. enforce_eager) there is nothing baked
into the workspace, so the early return must not lock it."""
runner = _make_capture_runner(captured=False)
lock_calls = []
monkeypatch.setattr(
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
)
assert runner.capture_model() == 0
assert lock_calls == []
def test_capture_model_profile_only_skips_lock(monkeypatch):
"""The memory-profiling capture pass runs before kernel warmup and the
real capture; locking there would stop the warmup from growing the
workspace to its scheduler-realistic size."""
runner = _make_capture_runner(captured=True)
monkeypatch.setattr(
model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext
)
monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None)
monkeypatch.setattr(
torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
)
lock_calls = []
monkeypatch.setattr(
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
)
runner.capture_model(profile_only=True)
assert lock_calls == []
@@ -67,7 +67,8 @@ def _make_profiling_runner(
events: list[str] = []
runner.events = events
def _capture_model() -> int:
def _capture_model(*, profile_only: bool = False) -> int:
assert profile_only
events.append("capture")
# Simulate the manager's per-FULL-graph memory sampling.
samples = runner.cudagraph_manager._capture_mem_samples
@@ -183,7 +184,7 @@ def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _boom() -> int:
def _boom(*, profile_only: bool = False) -> int:
runner.events.append("capture")
raise RuntimeError("capture failed")
@@ -204,7 +205,7 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _capture_model() -> int:
def _capture_model(*, profile_only: bool = False) -> int:
compilation_counter.num_cudagraph_captured += 5
compilation_counter.num_gpu_runner_capture_triggers += 1
return 1 << 30
@@ -288,9 +289,9 @@ def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch):
try:
capture_model = runner.capture_model
def _capture_model() -> int:
def _capture_model(*, profile_only: bool = False) -> int:
wrapper.pool_during_capture = wrapper.graph_pool
return capture_model()
return capture_model(profile_only=profile_only)
runner.capture_model = _capture_model
@@ -328,9 +329,9 @@ def test_profile_cudagraph_memory_swaps_and_drops_speculator_managers(monkeypatc
pools_seen: list[Any] = []
capture_model = runner.capture_model
def _capture_model() -> int:
def _capture_model(*, profile_only: bool = False) -> int:
pools_seen.append(runner.speculator.cudagraph_manager.pool)
return capture_model()
return capture_model(profile_only=profile_only)
runner.capture_model = _capture_model
@@ -379,7 +380,7 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch):
manager.pool = cgu.current_platform.get_global_graph_pool()
r.speculator = SimpleNamespace(cudagraph_manager=manager)
def _capture_model() -> int:
def _capture_model(*, profile_only: bool = False) -> int:
for owner in (
runner.cudagraph_manager,
runner.speculator.cudagraph_manager,
+24
View File
@@ -82,6 +82,30 @@ def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
assert len(pointers) == 4
def test_workspace_lock_blocks_growth_and_unlock_restores(monkeypatch) -> None:
"""Once locked, oversized requests fail loudly instead of reallocating the
buffer that captured CUDA graphs point at; unlock restores growth."""
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
(buf,) = manager.get_simultaneous(((256,), torch.uint8))
manager.lock()
assert manager.is_locked()
# Requests within the reserved size still reuse the same buffer.
(same,) = manager.get_simultaneous(((256,), torch.uint8))
(smaller,) = manager.get_simultaneous(((8,), torch.uint8))
assert same.data_ptr() == buf.data_ptr()
assert smaller.data_ptr() == buf.data_ptr()
with pytest.raises(AssertionError, match="Workspace is locked"):
manager.get_simultaneous(((512,), torch.uint8))
manager.unlock()
(grown,) = manager.get_simultaneous(((512,), torch.uint8))
assert grown.numel() == 512
def test_workspace_lane_validation(monkeypatch) -> None:
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
+1 -1
View File
@@ -828,7 +828,7 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
mem_samples: list[int] = []
manager._capture_mem_samples = mem_samples
measured = int(runner.capture_model())
measured = int(runner.capture_model(profile_only=True))
# The measured delta covers PIECEWISE, encoder and speculator graphs
# plus the sampled FULL graphs; swap the sampled FULL cost for the
+7 -2
View File
@@ -173,7 +173,7 @@ from vllm.v1.worker.utils import (
copy_kv_cache_blocks_inplace,
get_uniform_decode_token_count,
)
from vllm.v1.worker.workspace import use_workspace_lane
from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane
logger = init_logger(__name__)
@@ -928,7 +928,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return _profile_cudagraph_memory(self)
@torch.inference_mode()
def capture_model(self) -> int:
def capture_model(self, *, profile_only: bool = False) -> int:
assert self.cudagraph_manager is not None
capture_encoder = (
self.model_state.supports_mm_inputs
@@ -986,6 +986,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
if not profile_only:
# Lock workspace to prevent resizing during execution. A resize after
# capture frees the static cuda graph buffer.
lock_workspace()
end_time = time.perf_counter()
elapsed_time = end_time - start_time
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
+5 -4
View File
@@ -790,6 +790,10 @@ class Worker(WorkerBase):
# cuda graph capture.
kernel_warmup(self)
if self.use_v2_model_runner:
# A workspace resize after capture frees what the graphs point at.
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
cuda_graph_memory_bytes = 0
if not self.model_config.enforce_eager:
cuda_graph_memory_bytes = self.model_runner.capture_model()
@@ -868,10 +872,7 @@ class Worker(WorkerBase):
maybe_save_startup_plan(self, kv_cache_memory_bytes_to_requested_limit)
if self.use_v2_model_runner:
# V2: Run full execute_model + sample_tokens to JIT compile triton kernels.
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
elif get_pp_group().is_last_rank:
if not self.use_v2_model_runner and get_pp_group().is_last_rank:
# V1: Warm up sampler and preallocate memory buffer for logits and other
# sampling related tensors of max possible shape to avoid memory
# fragmentation issue.
+1 -1
View File
@@ -67,7 +67,7 @@ class MMEncoderModelRunner(GPUModelRunner):
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
return {}
def capture_model(self) -> int:
def capture_model(self, *, profile_only: bool = False) -> int:
return 0
def _dummy_run(