diff --git a/tests/v1/cudagraph/test_encoder_cudagraph.py b/tests/v1/cudagraph/test_encoder_cudagraph.py index 9c315f293d..61134a4f5a 100644 --- a/tests/v1/cudagraph/test_encoder_cudagraph.py +++ b/tests/v1/cudagraph/test_encoder_cudagraph.py @@ -109,6 +109,7 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager: mgr.max_batch_size = 16 mgr.use_dp = False mgr.budget_graphs = {} + mgr.graph_pool = None mgr.graph_hits = 0 mgr.graph_misses = 0 mgr.log_stats_interval = 100 @@ -180,6 +181,10 @@ class TestFindBudgetGraph: # Budget selection still works correctly after sorting assert mgr._find_smallest_fitting_budget_given_tokens(3000) == 4096 + def test_num_graphs_to_capture_tracks_budgets(self): + mgr = _make_manager_with_budgets([8192, 2048, 4096]) + assert mgr.get_num_graphs_to_capture() == 3 + # --------------------------------------------------------------------------- # get_cumulative_stats @@ -409,6 +414,7 @@ def _make_manager_for_gpu( ) mgr.use_dp = False mgr.budget_graphs = {} + mgr.graph_pool = None mgr.graph_hits = 0 mgr.graph_misses = 0 mgr.log_stats_interval = 100 @@ -467,7 +473,8 @@ class TestEncoderCudaGraphCaptureReplay: self.mgr = _make_manager_for_gpu( self.model, _BUDGETS, _MAX_BATCH, self.device, self.dtype ) - self.mgr.capture() + self.graph_pool = current_platform.graph_pool_handle() + self.mgr.capture(graph_pool=self.graph_pool) # --- capture --- @@ -475,6 +482,14 @@ class TestEncoderCudaGraphCaptureReplay: assert len(self.mgr.budget_graphs) == len(_BUDGETS) assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS) + def test_capture_uses_supplied_graph_pool(self): + assert self.mgr.graph_pool is self.graph_pool + + def test_clear_releases_graphs_and_pool(self): + self.mgr.clear() + assert self.mgr.budget_graphs == {} + assert self.mgr.graph_pool is None + # --- output shape --- def test_execute_returns_one_tensor_per_image(self): @@ -742,7 +757,8 @@ class TestEncoderCudaGraphVideoReplay: self.dtype, max_frames_per_batch=_VIDEO_MAX_FRAMES, ) - self.mgr.capture() + self.graph_pool = current_platform.graph_pool_handle() + self.mgr.capture(graph_pool=self.graph_pool) # --- capture --- diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index e008e17d80..c37b3b6c70 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -404,7 +404,7 @@ def _has_module(module_name: str) -> bool: if importlib.util.find_spec(module_name) is None: return False importlib.import_module(module_name) - except ImportError: + except Exception: logger.warning( "Module %s was found but failed to import", module_name, exc_info=True ) diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index ae9e8d58b8..583fd78ced 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -153,6 +153,7 @@ class EncoderCudaGraphManager: ) self.budget_graphs: dict[int, BudgetGraphMetadata] = {} + self.graph_pool: Any | None = None self.graph_hits = 0 self.graph_misses = 0 self.log_stats_interval = 100 @@ -183,9 +184,16 @@ class EncoderCudaGraphManager: """Check if a modality is supported by this manager.""" return modality in self.config.modalities - def capture(self): + def clear(self) -> None: + """Release captured encoder CUDA graphs and the manager-local pool.""" + self.budget_graphs.clear() + self.graph_pool = None + + def capture(self, graph_pool: Any): """Capture CUDA graphs for all token budgets.""" - for token_budget in self.token_budgets: + self.graph_pool = graph_pool + + for token_budget in sorted(self.token_budgets, reverse=True): self._capture_budget_graph(token_budget) logger.info( @@ -193,6 +201,9 @@ class EncoderCudaGraphManager: len(self.budget_graphs), ) + def get_num_graphs_to_capture(self) -> int: + return len(self.token_budgets) + def _capture_budget_graph(self, token_budget: int): """Capture CUDA graph for a single token budget.""" logger.debug( @@ -218,7 +229,7 @@ class EncoderCudaGraphManager: output_buffer = torch.empty_like(output) graph = torch.cuda.CUDAGraph() - with torch.inference_mode(), torch.cuda.graph(graph): + with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool): output = self.model.encoder_cudagraph_forward({**values}) output_buffer.copy_(output) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 9a05c76589..46a273f456 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -6304,6 +6304,42 @@ class GPUModelRunner( logger.debug("Cleaned up profiling KV cache and CUDA graphs") + @torch.inference_mode() + def _create_encoder_cudagraph_manager(self) -> "EncoderCudaGraphManager | None": + if not ( + self.compilation_config.cudagraph_mm_encoder and self.supports_mm_inputs + ): + return None + + # Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper, because + # @runtime_checkable Protocol isinstance() checks do not work through + # __getattr__ forwarding. + from vllm.model_executor.models.interfaces import ( + SupportsEncoderCudaGraph, + supports_encoder_cudagraph, + ) + from vllm.v1.worker.encoder_cudagraph import ( + EncoderCudaGraphManager, + ) + + raw_model = self.get_model() + if not supports_encoder_cudagraph(raw_model): + return None + + return EncoderCudaGraphManager( + vllm_config=self.vllm_config, + device=self.device, + dtype=self.dtype, + model=cast(SupportsEncoderCudaGraph, raw_model), + ) + + @torch.inference_mode() + def _maybe_init_encoder_cudagraph_manager(self) -> None: + if self.encoder_cudagraph_manager is None: + self.encoder_cudagraph_manager = self._create_encoder_cudagraph_manager() + if self.encoder_cudagraph_manager is not None: + logger.info("Initialized EncoderCudaGraphManager for vision encoder") + @torch.inference_mode() def profile_cudagraph_memory(self) -> int: with set_current_vllm_config(self.vllm_config): @@ -6312,24 +6348,40 @@ class GPUModelRunner( saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured capture_descs = self.cudagraph_dispatcher.get_capture_descs() + # Use a temporary manager for memory profiling. The persistent manager + # is initialized later so it does not keep profiling-only graph state. + encoder_cudagraph_manager = self._create_encoder_cudagraph_manager() - total_graphs = sum(len(descs) for _, descs in capture_descs) + decoder_graphs = sum(len(descs) for _, descs in capture_descs) + encoder_graphs = ( + encoder_cudagraph_manager.get_num_graphs_to_capture() + if encoder_cudagraph_manager is not None + else 0 + ) + total_graphs = decoder_graphs + encoder_graphs if total_graphs == 0: logger.debug("No CUDA graphs will be captured, skipping profiling") self._cleanup_profiling_kv_cache() return 0 - logger.info( - "Profiling CUDA graph memory: %s", - ", ".join( + graph_groups = [ + *( f"{mode.name}={len(descs)} (largest={descs[0].num_tokens})" for mode, descs in capture_descs if descs ), - ) + ] + if encoder_graphs > 0: + graph_groups.append( + f"ENCODER={encoder_graphs} " + f"(largest={encoder_cudagraph_manager.token_budgets[-1]})" + ) + + logger.info("Profiling CUDA graph memory: %s", ", ".join(graph_groups)) # Use a temporary pool for profiling to avoid fragmentation in the main pool. profiling_pool = current_platform.graph_pool_handle() + encoder_profiling_pool = current_platform.graph_pool_handle() original_pools: dict[int, Any] = {} all_wrappers = list(CUDAGraphWrapper._all_instances) + list( BreakableCUDAGraphWrapper._all_instances @@ -6338,73 +6390,98 @@ class GPUModelRunner( original_pools[id(instance)] = instance.graph_pool instance.graph_pool = profiling_pool - set_cudagraph_capturing_enabled(True) - with self._freeze_gc(), graph_capture(device=self.device): - shared_memory_estimate = {} - per_graph_estimate = {} - torch.accelerator.synchronize() - torch.accelerator.empty_cache() + shared_memory_estimate = {} + per_graph_estimate = {} + encoder_memory_estimate = 0 - for mode, descs in capture_descs: - profile_descs = descs[:2] - mem_samples: list[int] = [] + # Cleanup-only guard: CUDA graph capture errors should still propagate + # because encoder graph capture is opt-in. + try: + set_cudagraph_capturing_enabled(True) + with self._freeze_gc(), graph_capture(device=self.device): + torch.accelerator.synchronize() + torch.accelerator.empty_cache() - for i, desc in enumerate(profile_descs): - mem_before = torch.cuda.mem_get_info()[0] - self._warmup_and_capture( - desc, - cudagraph_runtime_mode=mode, - profile_seq_lens=( - min( - self.max_model_len, - self.max_num_tokens // desc.num_tokens, - ) - if mode == CUDAGraphMode.FULL and i == 0 - else None - ), + for mode, descs in capture_descs: + profile_descs = descs[:2] + mem_samples: list[int] = [] + + for i, desc in enumerate(profile_descs): + mem_before = torch.cuda.mem_get_info()[0] + self._warmup_and_capture( + desc, + cudagraph_runtime_mode=mode, + profile_seq_lens=( + min( + self.max_model_len, + self.max_num_tokens // desc.num_tokens, + ) + if mode == CUDAGraphMode.FULL and i == 0 + else None + ), + ) + torch.accelerator.synchronize() + free_after = torch.cuda.mem_get_info()[0] + mem_samples.append(mem_before - free_after) + + first_capture = mem_samples[0] + # Use at least 1 MiB per graph for driver overhead + per_graph = max( + mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20 ) + + shared_memory_estimate[mode] = first_capture + per_graph_estimate[mode] = per_graph * (len(descs) - 1) + + logger.debug( + "Estimated %s CUDA graph memory: " + "%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph", + mode.name, + first_capture / (1 << 20), + len(descs), + per_graph / (1 << 20), + ) + + if encoder_cudagraph_manager is not None: + mem_before = torch.cuda.mem_get_info()[0] + encoder_cudagraph_manager.capture(graph_pool=encoder_profiling_pool) torch.accelerator.synchronize() free_after = torch.cuda.mem_get_info()[0] - mem_samples.append(mem_before - free_after) + encoder_memory_estimate = max(mem_before - free_after, 0) - first_capture = mem_samples[0] - # Use at least 1 MiB per graph for driver overhead - per_graph = max(mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20) - - shared_memory_estimate[mode] = first_capture - per_graph_estimate[mode] = per_graph * (len(descs) - 1) - - logger.debug( - "Estimated %s CUDA graph memory: " - "%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph", - mode.name, - first_capture / (1 << 20), - len(descs), - per_graph / (1 << 20), - ) - - set_cudagraph_capturing_enabled(False) - CUDAGraphWrapper.clear_all_graphs() - BreakableCUDAGraphWrapper.clear_all_graphs() - all_wrappers = list(CUDAGraphWrapper._all_instances) + list( - BreakableCUDAGraphWrapper._all_instances - ) - for instance in all_wrappers: - if id(instance) in original_pools: - instance.graph_pool = original_pools[id(instance)] - for key_set in self.cudagraph_dispatcher.cudagraph_keys.values(): - key_set.clear() - self.cudagraph_dispatcher.keys_initialized = False - self.maybe_remove_all_loras(self.lora_config) - self._cleanup_profiling_kv_cache() - compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured + logger.debug( + "Estimated encoder CUDA graph memory: %.2f MiB for %d graphs", + encoder_memory_estimate / (1 << 20), + encoder_graphs, + ) + finally: + set_cudagraph_capturing_enabled(False) + CUDAGraphWrapper.clear_all_graphs() + BreakableCUDAGraphWrapper.clear_all_graphs() + if encoder_cudagraph_manager is not None: + encoder_cudagraph_manager.clear() + all_wrappers = list(CUDAGraphWrapper._all_instances) + list( + BreakableCUDAGraphWrapper._all_instances + ) + for instance in all_wrappers: + if id(instance) in original_pools: + instance.graph_pool = original_pools[id(instance)] + for key_set in self.cudagraph_dispatcher.cudagraph_keys.values(): + key_set.clear() + self.cudagraph_dispatcher.keys_initialized = False + self.maybe_remove_all_loras(self.lora_config) + self._cleanup_profiling_kv_cache() + compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured # FULL and PIECEWISE graphs share the global pool at runtime and are # never replayed concurrently, so the pool overlays their memory. # Take the max to avoid double-counting the overlap. - total_estimate = max(shared_memory_estimate.values()) + sum( + decoder_estimate = max(shared_memory_estimate.values(), default=0) + sum( per_graph_estimate.values() ) + # Encoder graphs use a manager-local pool at runtime, separate from the + # decoder pool, so add their estimate instead of overlaying it. + total_estimate = decoder_estimate + encoder_memory_estimate logger.info( "Estimated CUDA graph memory: %.2f GiB total", total_estimate / (1 << 30), @@ -6422,31 +6499,7 @@ class GPUModelRunner( return 0 # Initialize encoder CUDA graph manager if enabled. - # Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper, - # because @runtime_checkable Protocol isinstance() checks do not - # work through __getattr__ forwarding. - if ( - self.compilation_config.cudagraph_mm_encoder - and self.supports_mm_inputs - and self.encoder_cudagraph_manager is None - ): - from vllm.model_executor.models.interfaces import ( - SupportsEncoderCudaGraph, - supports_encoder_cudagraph, - ) - from vllm.v1.worker.encoder_cudagraph import ( - EncoderCudaGraphManager, - ) - - raw_model = self.get_model() - if supports_encoder_cudagraph(raw_model): - self.encoder_cudagraph_manager = EncoderCudaGraphManager( - vllm_config=self.vllm_config, - device=self.device, - dtype=self.dtype, - model=cast(SupportsEncoderCudaGraph, raw_model), - ) - logger.info("Initialized EncoderCudaGraphManager for vision encoder") + self._maybe_init_encoder_cudagraph_manager() compilation_counter.num_gpu_runner_capture_triggers += 1 @@ -6473,7 +6526,8 @@ class GPUModelRunner( # Capture encoder CUDA graphs if enabled if self.encoder_cudagraph_manager is not None: - self.encoder_cudagraph_manager.capture() + encoder_graph_pool = current_platform.graph_pool_handle() + self.encoder_cudagraph_manager.capture(graph_pool=encoder_graph_pool) torch.accelerator.synchronize() end_free_gpu_memory = torch.cuda.mem_get_info()[0]