diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 291cbbb312..eb05b9858a 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib from types import SimpleNamespace import pytest @@ -213,3 +214,84 @@ def test_append_block_ids_rejects_write_past_row_capacity(): ) assert block_tables.num_blocks.np[0, 1] == 3 + + +def _make_capture_runner(captured: bool) -> GPUModelRunner: + """Minimal V2 runner for capture_model: fakes everything except the + cudagraph_manager's needs_capture decision.""" + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.model_state = SimpleNamespace(supports_mm_inputs=False) + runner.cudagraph_manager = SimpleNamespace( + needs_capture=lambda: captured, + capture=lambda *args, **kwargs: None, + ) + runner.lora_config = None + runner.maybe_setup_dummy_loras = lambda _cfg: contextlib.nullcontext() + runner.speculator = None + runner.adaptive_verification = None + runner.model = None + runner.input_buffers = None + runner.pcp_manager = None + runner.intermediate_tensors = None + runner.block_tables = None + runner.attn_groups = None + runner.kv_cache_config = None + runner.use_aux_hidden_state_outputs = False + return runner + + +def test_capture_model_locks_workspace_after_capture(monkeypatch): + """A workspace resize after capture frees the buffer the captured graphs + baked in, so capture_model must lock the workspace before returning + (https://github.com/vllm-project/vllm/issues/55336).""" + runner = _make_capture_runner(captured=True) + monkeypatch.setattr( + model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext + ) + monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None) + monkeypatch.setattr( + torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30) + ) + lock_calls = [] + monkeypatch.setattr( + model_runner_module, "lock_workspace", lambda: lock_calls.append("lock") + ) + + runner.capture_model() + + assert lock_calls == ["lock"] + + +def test_capture_model_skips_lock_when_nothing_captured(monkeypatch): + """With no graphs to capture (e.g. enforce_eager) there is nothing baked + into the workspace, so the early return must not lock it.""" + runner = _make_capture_runner(captured=False) + lock_calls = [] + monkeypatch.setattr( + model_runner_module, "lock_workspace", lambda: lock_calls.append("lock") + ) + + assert runner.capture_model() == 0 + assert lock_calls == [] + + +def test_capture_model_profile_only_skips_lock(monkeypatch): + """The memory-profiling capture pass runs before kernel warmup and the + real capture; locking there would stop the warmup from growing the + workspace to its scheduler-realistic size.""" + runner = _make_capture_runner(captured=True) + monkeypatch.setattr( + model_runner_module, "freeze_gc_for_cudagraph_capture", contextlib.nullcontext + ) + monkeypatch.setattr(torch.accelerator, "empty_cache", lambda: None) + monkeypatch.setattr( + torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30) + ) + lock_calls = [] + monkeypatch.setattr( + model_runner_module, "lock_workspace", lambda: lock_calls.append("lock") + ) + + runner.capture_model(profile_only=True) + + assert lock_calls == [] diff --git a/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py index 0b5609dc21..74191206d3 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py @@ -67,7 +67,8 @@ def _make_profiling_runner( events: list[str] = [] runner.events = events - def _capture_model() -> int: + def _capture_model(*, profile_only: bool = False) -> int: + assert profile_only events.append("capture") # Simulate the manager's per-FULL-graph memory sampling. samples = runner.cudagraph_manager._capture_mem_samples @@ -183,7 +184,7 @@ def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch): _patch_module(monkeypatch) runner = _make_profiling_runner(CUDAGraphMode.FULL) - def _boom() -> int: + def _boom(*, profile_only: bool = False) -> int: runner.events.append("capture") raise RuntimeError("capture failed") @@ -204,7 +205,7 @@ def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch): _patch_module(monkeypatch) runner = _make_profiling_runner(CUDAGraphMode.FULL) - def _capture_model() -> int: + def _capture_model(*, profile_only: bool = False) -> int: compilation_counter.num_cudagraph_captured += 5 compilation_counter.num_gpu_runner_capture_triggers += 1 return 1 << 30 @@ -288,9 +289,9 @@ def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch): try: capture_model = runner.capture_model - def _capture_model() -> int: + def _capture_model(*, profile_only: bool = False) -> int: wrapper.pool_during_capture = wrapper.graph_pool - return capture_model() + return capture_model(profile_only=profile_only) runner.capture_model = _capture_model @@ -328,9 +329,9 @@ def test_profile_cudagraph_memory_swaps_and_drops_speculator_managers(monkeypatc pools_seen: list[Any] = [] capture_model = runner.capture_model - def _capture_model() -> int: + def _capture_model(*, profile_only: bool = False) -> int: pools_seen.append(runner.speculator.cudagraph_manager.pool) - return capture_model() + return capture_model(profile_only=profile_only) runner.capture_model = _capture_model @@ -379,7 +380,7 @@ def test_profile_cudagraph_memory_frees_throwaway_pool(monkeypatch): manager.pool = cgu.current_platform.get_global_graph_pool() r.speculator = SimpleNamespace(cudagraph_manager=manager) - def _capture_model() -> int: + def _capture_model(*, profile_only: bool = False) -> int: for owner in ( runner.cudagraph_manager, runner.speculator.cudagraph_manager, diff --git a/tests/v1/worker/test_workspace.py b/tests/v1/worker/test_workspace.py index a909ea9af8..7824c8000d 100644 --- a/tests/v1/worker/test_workspace.py +++ b/tests/v1/worker/test_workspace.py @@ -82,6 +82,30 @@ def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None: assert len(pointers) == 4 +def test_workspace_lock_blocks_growth_and_unlock_restores(monkeypatch) -> None: + """Once locked, oversized requests fail loudly instead of reallocating the + buffer that captured CUDA graphs point at; unlock restores growth.""" + monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0) + manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1) + + (buf,) = manager.get_simultaneous(((256,), torch.uint8)) + manager.lock() + assert manager.is_locked() + + # Requests within the reserved size still reuse the same buffer. + (same,) = manager.get_simultaneous(((256,), torch.uint8)) + (smaller,) = manager.get_simultaneous(((8,), torch.uint8)) + assert same.data_ptr() == buf.data_ptr() + assert smaller.data_ptr() == buf.data_ptr() + + with pytest.raises(AssertionError, match="Workspace is locked"): + manager.get_simultaneous(((512,), torch.uint8)) + + manager.unlock() + (grown,) = manager.get_simultaneous(((512,), torch.uint8)) + assert grown.numel() == 512 + + def test_workspace_lane_validation(monkeypatch) -> None: monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0) manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 084429b3f8..66051a9fe2 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -828,7 +828,7 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int: mem_samples: list[int] = [] manager._capture_mem_samples = mem_samples - measured = int(runner.capture_model()) + measured = int(runner.capture_model(profile_only=True)) # The measured delta covers PIECEWISE, encoder and speculator graphs # plus the sampled FULL graphs; swap the sampled FULL cost for the diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 901bd68754..a0584e90c6 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -173,7 +173,7 @@ from vllm.v1.worker.utils import ( copy_kv_cache_blocks_inplace, get_uniform_decode_token_count, ) -from vllm.v1.worker.workspace import use_workspace_lane +from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane logger = init_logger(__name__) @@ -928,7 +928,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): return _profile_cudagraph_memory(self) @torch.inference_mode() - def capture_model(self) -> int: + def capture_model(self, *, profile_only: bool = False) -> int: assert self.cudagraph_manager is not None capture_encoder = ( self.model_state.supports_mm_inputs @@ -986,6 +986,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): end_free_gpu_memory = torch.accelerator.get_memory_info()[0] + if not profile_only: + # Lock workspace to prevent resizing during execution. A resize after + # capture frees the static cuda graph buffer. + lock_workspace() + end_time = time.perf_counter() elapsed_time = end_time - start_time cuda_graph_size = start_free_gpu_memory - end_free_gpu_memory diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 70ab3ab95f..e832e4acaf 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -790,6 +790,10 @@ class Worker(WorkerBase): # cuda graph capture. kernel_warmup(self) + if self.use_v2_model_runner: + # A workspace resize after capture frees what the graphs point at. + warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens) + cuda_graph_memory_bytes = 0 if not self.model_config.enforce_eager: cuda_graph_memory_bytes = self.model_runner.capture_model() @@ -868,10 +872,7 @@ class Worker(WorkerBase): maybe_save_startup_plan(self, kv_cache_memory_bytes_to_requested_limit) - if self.use_v2_model_runner: - # V2: Run full execute_model + sample_tokens to JIT compile triton kernels. - warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens) - elif get_pp_group().is_last_rank: + if not self.use_v2_model_runner and get_pp_group().is_last_rank: # V1: Warm up sampler and preallocate memory buffer for logits and other # sampling related tensors of max possible shape to avoid memory # fragmentation issue. diff --git a/vllm/v1/worker/mm_encoder_model_runner.py b/vllm/v1/worker/mm_encoder_model_runner.py index 2a68ab37a9..8208a6ce3d 100644 --- a/vllm/v1/worker/mm_encoder_model_runner.py +++ b/vllm/v1/worker/mm_encoder_model_runner.py @@ -67,7 +67,7 @@ class MMEncoderModelRunner(GPUModelRunner): def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: return {} - def capture_model(self) -> int: + def capture_model(self, *, profile_only: bool = False) -> int: return 0 def _dummy_run(