[Bugfix][V2] Warm up kernels before capturing CUDA graphs (#55341)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
aoshen02
Nick Hill
parent
8f269a93bb
commit
685074cd61
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
@@ -213,3 +214,84 @@ def test_append_block_ids_rejects_write_past_row_capacity():
|
||||
)
|
||||
|
||||
assert block_tables.num_blocks.np[0, 1] == 3
|
||||
|
||||
|
||||
def _make_capture_runner(captured: bool) -> GPUModelRunner:
|
||||
"""Minimal V2 runner for capture_model: fakes everything except the
|
||||
cudagraph_manager's needs_capture decision."""
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.model_state = SimpleNamespace(supports_mm_inputs=False)
|
||||
runner.cudagraph_manager = SimpleNamespace(
|
||||
needs_capture=lambda: captured,
|
||||
capture=lambda *args, **kwargs: None,
|
||||
)
|
||||
runner.lora_config = None
|
||||
runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext()
|
||||
runner.speculator = None
|
||||
runner.adaptive_verification = None
|
||||
runner.model = None
|
||||
runner.input_buffers = None
|
||||
runner.pcp_manager = None
|
||||
runner.intermediate_tensors = None
|
||||
runner.block_tables = None
|
||||
runner.attn_groups = None
|
||||
runner.kv_cache_config = None
|
||||
runner.use_aux_hidden_state_outputs = False
|
||||
return runner
|
||||
|
||||
|
||||
def test_capture_model_locks_workspace_after_capture(monkeypatch):
|
||||
"""A workspace resize after capture frees the buffer the captured graphs
|
||||
baked in, so capture_model must lock the workspace before returning
|
||||
(https://github.com/vllm-project/vllm/issues/55336)."""
|
||||
runner = _make_capture_runner(captured=True)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
|
||||
)
|
||||
lock_calls = []
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
|
||||
)
|
||||
|
||||
runner.capture_model()
|
||||
|
||||
assert lock_calls == ["lock"]
|
||||
|
||||
|
||||
def test_capture_model_skips_lock_when_nothing_captured(monkeypatch):
|
||||
"""With no graphs to capture (e.g. enforce_eager) there is nothing baked
|
||||
into the workspace, so the early return must not lock it."""
|
||||
runner = _make_capture_runner(captured=False)
|
||||
lock_calls = []
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
|
||||
)
|
||||
|
||||
assert runner.capture_model() == 0
|
||||
assert lock_calls == []
|
||||
|
||||
|
||||
def test_capture_model_profile_only_skips_lock(monkeypatch):
|
||||
"""The memory-profiling capture pass runs before kernel warmup and the
|
||||
real capture; locking there would stop the warmup from growing the
|
||||
workspace to its scheduler-realistic size."""
|
||||
runner = _make_capture_runner(captured=True)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
|
||||
)
|
||||
lock_calls = []
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "lock_workspace", lambda: lock_calls.append("lock")
|
||||
)
|
||||
|
||||
runner.capture_model(profile_only=True)
|
||||
|
||||
assert lock_calls == []
|
||||
|
||||
@@ -67,7 +67,8 @@ def _make_profiling_runner(
|
||||
events: list[str] = []
|
||||
runner.events = events
|
||||
|
||||
def _capture_model() -> int:
|
||||
def _capture_model(*, profile_only: bool = False) -> int:
|
||||
assert profile_only
|
||||
events.append("capture")
|
||||
# Simulate the manager's per-FULL-graph memory sampling.
|
||||
samples = runner.cudagraph_manager._capture_mem_samples
|
||||
@@ -183,7 +184,7 @@ def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch):
|
||||
_patch_module(monkeypatch)
|
||||
runner = _make_profiling_runner(CUDAGraphMode.FULL)
|
||||
|
||||
def _boom() -> int:
|
||||
def _boom(*, profile_only: bool = False) -> int:
|
||||
runner.events.append("capture")
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
@@ -204,7 +205,7 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
|
||||
_patch_module(monkeypatch)
|
||||
runner = _make_profiling_runner(CUDAGraphMode.FULL)
|
||||
|
||||
def _capture_model() -> int:
|
||||
def _capture_model(*, profile_only: bool = False) -> int:
|
||||
compilation_counter.num_cudagraph_captured += 5
|
||||
compilation_counter.num_gpu_runner_capture_triggers += 1
|
||||
return 1 << 30
|
||||
@@ -288,9 +289,9 @@ def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch):
|
||||
try:
|
||||
capture_model = runner.capture_model
|
||||
|
||||
def _capture_model() -> int:
|
||||
def _capture_model(*, profile_only: bool = False) -> int:
|
||||
wrapper.pool_during_capture = wrapper.graph_pool
|
||||
return capture_model()
|
||||
return capture_model(profile_only=profile_only)
|
||||
|
||||
runner.capture_model = _capture_model
|
||||
|
||||
@@ -328,9 +329,9 @@ def test_profile_cudagraph_memory_swaps_and_drops_speculator_managers(monkeypatc
|
||||
pools_seen: list[Any] = []
|
||||
capture_model = runner.capture_model
|
||||
|
||||
def _capture_model() -> int:
|
||||
def _capture_model(*, profile_only: bool = False) -> int:
|
||||
pools_seen.append(runner.speculator.cudagraph_manager.pool)
|
||||
return capture_model()
|
||||
return capture_model(profile_only=profile_only)
|
||||
|
||||
runner.capture_model = _capture_model
|
||||
|
||||
@@ -379,7 +380,7 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch):
|
||||
manager.pool = cgu.current_platform.get_global_graph_pool()
|
||||
r.speculator = SimpleNamespace(cudagraph_manager=manager)
|
||||
|
||||
def _capture_model() -> int:
|
||||
def _capture_model(*, profile_only: bool = False) -> int:
|
||||
for owner in (
|
||||
runner.cudagraph_manager,
|
||||
runner.speculator.cudagraph_manager,
|
||||
|
||||
@@ -82,6 +82,30 @@ def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
|
||||
assert len(pointers) == 4
|
||||
|
||||
|
||||
def test_workspace_lock_blocks_growth_and_unlock_restores(monkeypatch) -> None:
|
||||
"""Once locked, oversized requests fail loudly instead of reallocating the
|
||||
buffer that captured CUDA graphs point at; unlock restores growth."""
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
|
||||
|
||||
(buf,) = manager.get_simultaneous(((256,), torch.uint8))
|
||||
manager.lock()
|
||||
assert manager.is_locked()
|
||||
|
||||
# Requests within the reserved size still reuse the same buffer.
|
||||
(same,) = manager.get_simultaneous(((256,), torch.uint8))
|
||||
(smaller,) = manager.get_simultaneous(((8,), torch.uint8))
|
||||
assert same.data_ptr() == buf.data_ptr()
|
||||
assert smaller.data_ptr() == buf.data_ptr()
|
||||
|
||||
with pytest.raises(AssertionError, match="Workspace is locked"):
|
||||
manager.get_simultaneous(((512,), torch.uint8))
|
||||
|
||||
manager.unlock()
|
||||
(grown,) = manager.get_simultaneous(((512,), torch.uint8))
|
||||
assert grown.numel() == 512
|
||||
|
||||
|
||||
def test_workspace_lane_validation(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
|
||||
|
||||
@@ -828,7 +828,7 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||
mem_samples: list[int] = []
|
||||
manager._capture_mem_samples = mem_samples
|
||||
|
||||
measured = int(runner.capture_model())
|
||||
measured = int(runner.capture_model(profile_only=True))
|
||||
|
||||
# The measured delta covers PIECEWISE, encoder and speculator graphs
|
||||
# plus the sampled FULL graphs; swap the sampled FULL cost for the
|
||||
|
||||
@@ -173,7 +173,7 @@ from vllm.v1.worker.utils import (
|
||||
copy_kv_cache_blocks_inplace,
|
||||
get_uniform_decode_token_count,
|
||||
)
|
||||
from vllm.v1.worker.workspace import use_workspace_lane
|
||||
from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -928,7 +928,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return _profile_cudagraph_memory(self)
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self) -> int:
|
||||
def capture_model(self, *, profile_only: bool = False) -> int:
|
||||
assert self.cudagraph_manager is not None
|
||||
capture_encoder = (
|
||||
self.model_state.supports_mm_inputs
|
||||
@@ -986,6 +986,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
if not profile_only:
|
||||
# Lock workspace to prevent resizing during execution. A resize after
|
||||
# capture frees the static cuda graph buffer.
|
||||
lock_workspace()
|
||||
|
||||
end_time = time.perf_counter()
|
||||
elapsed_time = end_time - start_time
|
||||
cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory
|
||||
|
||||
@@ -790,6 +790,10 @@ class Worker(WorkerBase):
|
||||
# cuda graph capture.
|
||||
kernel_warmup(self)
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# A workspace resize after capture frees what the graphs point at.
|
||||
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
|
||||
|
||||
cuda_graph_memory_bytes = 0
|
||||
if not self.model_config.enforce_eager:
|
||||
cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
@@ -868,10 +872,7 @@ class Worker(WorkerBase):
|
||||
|
||||
maybe_save_startup_plan(self, kv_cache_memory_bytes_to_requested_limit)
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# V2: Run full execute_model + sample_tokens to JIT compile triton kernels.
|
||||
warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
|
||||
elif get_pp_group().is_last_rank:
|
||||
if not self.use_v2_model_runner and get_pp_group().is_last_rank:
|
||||
# V1: Warm up sampler and preallocate memory buffer for logits and other
|
||||
# sampling related tensors of max possible shape to avoid memory
|
||||
# fragmentation issue.
|
||||
|
||||
@@ -67,7 +67,7 @@ class MMEncoderModelRunner(GPUModelRunner):
|
||||
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
|
||||
return {}
|
||||
|
||||
def capture_model(self) -> int:
|
||||
def capture_model(self, *, profile_only: bool = False) -> int:
|
||||
return 0
|
||||
|
||||
def _dummy_run(
|
||||
|
||||
Reference in New Issue
Block a user