[MM][CG] Profile encoder CUDA graph pool memory (#41714)

Signed-off-by: JooHo Lee <jooho414@gmail.com>
This commit is contained in:
JooHo Lee
2026-06-02 12:27:34 +08:00
committed by GitHub
parent a3a5a5ece5
commit a045c7425f
4 changed files with 173 additions and 92 deletions
+18 -2
View File
@@ -109,6 +109,7 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
mgr.max_batch_size = 16
mgr.use_dp = False
mgr.budget_graphs = {}
mgr.graph_pool = None
mgr.graph_hits = 0
mgr.graph_misses = 0
mgr.log_stats_interval = 100
@@ -180,6 +181,10 @@ class TestFindBudgetGraph:
# Budget selection still works correctly after sorting
assert mgr._find_smallest_fitting_budget_given_tokens(3000) == 4096
def test_num_graphs_to_capture_tracks_budgets(self):
mgr = _make_manager_with_budgets([8192, 2048, 4096])
assert mgr.get_num_graphs_to_capture() == 3
# ---------------------------------------------------------------------------
# get_cumulative_stats
@@ -409,6 +414,7 @@ def _make_manager_for_gpu(
)
mgr.use_dp = False
mgr.budget_graphs = {}
mgr.graph_pool = None
mgr.graph_hits = 0
mgr.graph_misses = 0
mgr.log_stats_interval = 100
@@ -467,7 +473,8 @@ class TestEncoderCudaGraphCaptureReplay:
self.mgr = _make_manager_for_gpu(
self.model, _BUDGETS, _MAX_BATCH, self.device, self.dtype
)
self.mgr.capture()
self.graph_pool = current_platform.graph_pool_handle()
self.mgr.capture(graph_pool=self.graph_pool)
# --- capture ---
@@ -475,6 +482,14 @@ class TestEncoderCudaGraphCaptureReplay:
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
def test_capture_uses_supplied_graph_pool(self):
assert self.mgr.graph_pool is self.graph_pool
def test_clear_releases_graphs_and_pool(self):
self.mgr.clear()
assert self.mgr.budget_graphs == {}
assert self.mgr.graph_pool is None
# --- output shape ---
def test_execute_returns_one_tensor_per_image(self):
@@ -742,7 +757,8 @@ class TestEncoderCudaGraphVideoReplay:
self.dtype,
max_frames_per_batch=_VIDEO_MAX_FRAMES,
)
self.mgr.capture()
self.graph_pool = current_platform.graph_pool_handle()
self.mgr.capture(graph_pool=self.graph_pool)
# --- capture ---
+1 -1
View File
@@ -404,7 +404,7 @@ def _has_module(module_name: str) -> bool:
if importlib.util.find_spec(module_name) is None:
return False
importlib.import_module(module_name)
except ImportError:
except Exception:
logger.warning(
"Module %s was found but failed to import", module_name, exc_info=True
)
+14 -3
View File
@@ -153,6 +153,7 @@ class EncoderCudaGraphManager:
)
self.budget_graphs: dict[int, BudgetGraphMetadata] = {}
self.graph_pool: Any | None = None
self.graph_hits = 0
self.graph_misses = 0
self.log_stats_interval = 100
@@ -183,9 +184,16 @@ class EncoderCudaGraphManager:
"""Check if a modality is supported by this manager."""
return modality in self.config.modalities
def capture(self):
def clear(self) -> None:
"""Release captured encoder CUDA graphs and the manager-local pool."""
self.budget_graphs.clear()
self.graph_pool = None
def capture(self, graph_pool: Any):
"""Capture CUDA graphs for all token budgets."""
for token_budget in self.token_budgets:
self.graph_pool = graph_pool
for token_budget in sorted(self.token_budgets, reverse=True):
self._capture_budget_graph(token_budget)
logger.info(
@@ -193,6 +201,9 @@ class EncoderCudaGraphManager:
len(self.budget_graphs),
)
def get_num_graphs_to_capture(self) -> int:
return len(self.token_budgets)
def _capture_budget_graph(self, token_budget: int):
"""Capture CUDA graph for a single token budget."""
logger.debug(
@@ -218,7 +229,7 @@ class EncoderCudaGraphManager:
output_buffer = torch.empty_like(output)
graph = torch.cuda.CUDAGraph()
with torch.inference_mode(), torch.cuda.graph(graph):
with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool):
output = self.model.encoder_cudagraph_forward({**values})
output_buffer.copy_(output)
+140 -86
View File
@@ -6304,6 +6304,42 @@ class GPUModelRunner(
logger.debug("Cleaned up profiling KV cache and CUDA graphs")
@torch.inference_mode()
def _create_encoder_cudagraph_manager(self) -> "EncoderCudaGraphManager | None":
if not (
self.compilation_config.cudagraph_mm_encoder and self.supports_mm_inputs
):
return None
# Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper, because
# @runtime_checkable Protocol isinstance() checks do not work through
# __getattr__ forwarding.
from vllm.model_executor.models.interfaces import (
SupportsEncoderCudaGraph,
supports_encoder_cudagraph,
)
from vllm.v1.worker.encoder_cudagraph import (
EncoderCudaGraphManager,
)
raw_model = self.get_model()
if not supports_encoder_cudagraph(raw_model):
return None
return EncoderCudaGraphManager(
vllm_config=self.vllm_config,
device=self.device,
dtype=self.dtype,
model=cast(SupportsEncoderCudaGraph, raw_model),
)
@torch.inference_mode()
def _maybe_init_encoder_cudagraph_manager(self) -> None:
if self.encoder_cudagraph_manager is None:
self.encoder_cudagraph_manager = self._create_encoder_cudagraph_manager()
if self.encoder_cudagraph_manager is not None:
logger.info("Initialized EncoderCudaGraphManager for vision encoder")
@torch.inference_mode()
def profile_cudagraph_memory(self) -> int:
with set_current_vllm_config(self.vllm_config):
@@ -6312,24 +6348,40 @@ class GPUModelRunner(
saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured
capture_descs = self.cudagraph_dispatcher.get_capture_descs()
# Use a temporary manager for memory profiling. The persistent manager
# is initialized later so it does not keep profiling-only graph state.
encoder_cudagraph_manager = self._create_encoder_cudagraph_manager()
total_graphs = sum(len(descs) for _, descs in capture_descs)
decoder_graphs = sum(len(descs) for _, descs in capture_descs)
encoder_graphs = (
encoder_cudagraph_manager.get_num_graphs_to_capture()
if encoder_cudagraph_manager is not None
else 0
)
total_graphs = decoder_graphs + encoder_graphs
if total_graphs == 0:
logger.debug("No CUDA graphs will be captured, skipping profiling")
self._cleanup_profiling_kv_cache()
return 0
logger.info(
"Profiling CUDA graph memory: %s",
", ".join(
graph_groups = [
*(
f"{mode.name}={len(descs)} (largest={descs[0].num_tokens})"
for mode, descs in capture_descs
if descs
),
)
]
if encoder_graphs > 0:
graph_groups.append(
f"ENCODER={encoder_graphs} "
f"(largest={encoder_cudagraph_manager.token_budgets[-1]})"
)
logger.info("Profiling CUDA graph memory: %s", ", ".join(graph_groups))
# Use a temporary pool for profiling to avoid fragmentation in the main pool.
profiling_pool = current_platform.graph_pool_handle()
encoder_profiling_pool = current_platform.graph_pool_handle()
original_pools: dict[int, Any] = {}
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
@@ -6338,73 +6390,98 @@ class GPUModelRunner(
original_pools[id(instance)] = instance.graph_pool
instance.graph_pool = profiling_pool
set_cudagraph_capturing_enabled(True)
with self._freeze_gc(), graph_capture(device=self.device):
shared_memory_estimate = {}
per_graph_estimate = {}
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
shared_memory_estimate = {}
per_graph_estimate = {}
encoder_memory_estimate = 0
for mode, descs in capture_descs:
profile_descs = descs[:2]
mem_samples: list[int] = []
# Cleanup-only guard: CUDA graph capture errors should still propagate
# because encoder graph capture is opt-in.
try:
set_cudagraph_capturing_enabled(True)
with self._freeze_gc(), graph_capture(device=self.device):
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
for i, desc in enumerate(profile_descs):
mem_before = torch.cuda.mem_get_info()[0]
self._warmup_and_capture(
desc,
cudagraph_runtime_mode=mode,
profile_seq_lens=(
min(
self.max_model_len,
self.max_num_tokens // desc.num_tokens,
)
if mode == CUDAGraphMode.FULL and i == 0
else None
),
for mode, descs in capture_descs:
profile_descs = descs[:2]
mem_samples: list[int] = []
for i, desc in enumerate(profile_descs):
mem_before = torch.cuda.mem_get_info()[0]
self._warmup_and_capture(
desc,
cudagraph_runtime_mode=mode,
profile_seq_lens=(
min(
self.max_model_len,
self.max_num_tokens // desc.num_tokens,
)
if mode == CUDAGraphMode.FULL and i == 0
else None
),
)
torch.accelerator.synchronize()
free_after = torch.cuda.mem_get_info()[0]
mem_samples.append(mem_before - free_after)
first_capture = mem_samples[0]
# Use at least 1 MiB per graph for driver overhead
per_graph = max(
mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20
)
shared_memory_estimate[mode] = first_capture
per_graph_estimate[mode] = per_graph * (len(descs) - 1)
logger.debug(
"Estimated %s CUDA graph memory: "
"%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph",
mode.name,
first_capture / (1 << 20),
len(descs),
per_graph / (1 << 20),
)
if encoder_cudagraph_manager is not None:
mem_before = torch.cuda.mem_get_info()[0]
encoder_cudagraph_manager.capture(graph_pool=encoder_profiling_pool)
torch.accelerator.synchronize()
free_after = torch.cuda.mem_get_info()[0]
mem_samples.append(mem_before - free_after)
encoder_memory_estimate = max(mem_before - free_after, 0)
first_capture = mem_samples[0]
# Use at least 1 MiB per graph for driver overhead
per_graph = max(mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20)
shared_memory_estimate[mode] = first_capture
per_graph_estimate[mode] = per_graph * (len(descs) - 1)
logger.debug(
"Estimated %s CUDA graph memory: "
"%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph",
mode.name,
first_capture / (1 << 20),
len(descs),
per_graph / (1 << 20),
)
set_cudagraph_capturing_enabled(False)
CUDAGraphWrapper.clear_all_graphs()
BreakableCUDAGraphWrapper.clear_all_graphs()
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
)
for instance in all_wrappers:
if id(instance) in original_pools:
instance.graph_pool = original_pools[id(instance)]
for key_set in self.cudagraph_dispatcher.cudagraph_keys.values():
key_set.clear()
self.cudagraph_dispatcher.keys_initialized = False
self.maybe_remove_all_loras(self.lora_config)
self._cleanup_profiling_kv_cache()
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
logger.debug(
"Estimated encoder CUDA graph memory: %.2f MiB for %d graphs",
encoder_memory_estimate / (1 << 20),
encoder_graphs,
)
finally:
set_cudagraph_capturing_enabled(False)
CUDAGraphWrapper.clear_all_graphs()
BreakableCUDAGraphWrapper.clear_all_graphs()
if encoder_cudagraph_manager is not None:
encoder_cudagraph_manager.clear()
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
)
for instance in all_wrappers:
if id(instance) in original_pools:
instance.graph_pool = original_pools[id(instance)]
for key_set in self.cudagraph_dispatcher.cudagraph_keys.values():
key_set.clear()
self.cudagraph_dispatcher.keys_initialized = False
self.maybe_remove_all_loras(self.lora_config)
self._cleanup_profiling_kv_cache()
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
# FULL and PIECEWISE graphs share the global pool at runtime and are
# never replayed concurrently, so the pool overlays their memory.
# Take the max to avoid double-counting the overlap.
total_estimate = max(shared_memory_estimate.values()) + sum(
decoder_estimate = max(shared_memory_estimate.values(), default=0) + sum(
per_graph_estimate.values()
)
# Encoder graphs use a manager-local pool at runtime, separate from the
# decoder pool, so add their estimate instead of overlaying it.
total_estimate = decoder_estimate + encoder_memory_estimate
logger.info(
"Estimated CUDA graph memory: %.2f GiB total",
total_estimate / (1 << 30),
@@ -6422,31 +6499,7 @@ class GPUModelRunner(
return 0
# Initialize encoder CUDA graph manager if enabled.
# Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper,
# because @runtime_checkable Protocol isinstance() checks do not
# work through __getattr__ forwarding.
if (
self.compilation_config.cudagraph_mm_encoder
and self.supports_mm_inputs
and self.encoder_cudagraph_manager is None
):
from vllm.model_executor.models.interfaces import (
SupportsEncoderCudaGraph,
supports_encoder_cudagraph,
)
from vllm.v1.worker.encoder_cudagraph import (
EncoderCudaGraphManager,
)
raw_model = self.get_model()
if supports_encoder_cudagraph(raw_model):
self.encoder_cudagraph_manager = EncoderCudaGraphManager(
vllm_config=self.vllm_config,
device=self.device,
dtype=self.dtype,
model=cast(SupportsEncoderCudaGraph, raw_model),
)
logger.info("Initialized EncoderCudaGraphManager for vision encoder")
self._maybe_init_encoder_cudagraph_manager()
compilation_counter.num_gpu_runner_capture_triggers += 1
@@ -6473,7 +6526,8 @@ class GPUModelRunner(
# Capture encoder CUDA graphs if enabled
if self.encoder_cudagraph_manager is not None:
self.encoder_cudagraph_manager.capture()
encoder_graph_pool = current_platform.graph_pool_handle()
self.encoder_cudagraph_manager.capture(graph_pool=encoder_graph_pool)
torch.accelerator.synchronize()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]