[MM][CG] Profile encoder CUDA graph pool memory (#41714)
Signed-off-by: JooHo Lee <jooho414@gmail.com>
This commit is contained in:
@@ -109,6 +109,7 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
mgr.max_batch_size = 16
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {}
|
||||
mgr.graph_pool = None
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
@@ -180,6 +181,10 @@ class TestFindBudgetGraph:
|
||||
# Budget selection still works correctly after sorting
|
||||
assert mgr._find_smallest_fitting_budget_given_tokens(3000) == 4096
|
||||
|
||||
def test_num_graphs_to_capture_tracks_budgets(self):
|
||||
mgr = _make_manager_with_budgets([8192, 2048, 4096])
|
||||
assert mgr.get_num_graphs_to_capture() == 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_cumulative_stats
|
||||
@@ -409,6 +414,7 @@ def _make_manager_for_gpu(
|
||||
)
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {}
|
||||
mgr.graph_pool = None
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
@@ -467,7 +473,8 @@ class TestEncoderCudaGraphCaptureReplay:
|
||||
self.mgr = _make_manager_for_gpu(
|
||||
self.model, _BUDGETS, _MAX_BATCH, self.device, self.dtype
|
||||
)
|
||||
self.mgr.capture()
|
||||
self.graph_pool = current_platform.graph_pool_handle()
|
||||
self.mgr.capture(graph_pool=self.graph_pool)
|
||||
|
||||
# --- capture ---
|
||||
|
||||
@@ -475,6 +482,14 @@ class TestEncoderCudaGraphCaptureReplay:
|
||||
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
|
||||
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
|
||||
|
||||
def test_capture_uses_supplied_graph_pool(self):
|
||||
assert self.mgr.graph_pool is self.graph_pool
|
||||
|
||||
def test_clear_releases_graphs_and_pool(self):
|
||||
self.mgr.clear()
|
||||
assert self.mgr.budget_graphs == {}
|
||||
assert self.mgr.graph_pool is None
|
||||
|
||||
# --- output shape ---
|
||||
|
||||
def test_execute_returns_one_tensor_per_image(self):
|
||||
@@ -742,7 +757,8 @@ class TestEncoderCudaGraphVideoReplay:
|
||||
self.dtype,
|
||||
max_frames_per_batch=_VIDEO_MAX_FRAMES,
|
||||
)
|
||||
self.mgr.capture()
|
||||
self.graph_pool = current_platform.graph_pool_handle()
|
||||
self.mgr.capture(graph_pool=self.graph_pool)
|
||||
|
||||
# --- capture ---
|
||||
|
||||
|
||||
@@ -404,7 +404,7 @@ def _has_module(module_name: str) -> bool:
|
||||
if importlib.util.find_spec(module_name) is None:
|
||||
return False
|
||||
importlib.import_module(module_name)
|
||||
except ImportError:
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Module %s was found but failed to import", module_name, exc_info=True
|
||||
)
|
||||
|
||||
@@ -153,6 +153,7 @@ class EncoderCudaGraphManager:
|
||||
)
|
||||
|
||||
self.budget_graphs: dict[int, BudgetGraphMetadata] = {}
|
||||
self.graph_pool: Any | None = None
|
||||
self.graph_hits = 0
|
||||
self.graph_misses = 0
|
||||
self.log_stats_interval = 100
|
||||
@@ -183,9 +184,16 @@ class EncoderCudaGraphManager:
|
||||
"""Check if a modality is supported by this manager."""
|
||||
return modality in self.config.modalities
|
||||
|
||||
def capture(self):
|
||||
def clear(self) -> None:
|
||||
"""Release captured encoder CUDA graphs and the manager-local pool."""
|
||||
self.budget_graphs.clear()
|
||||
self.graph_pool = None
|
||||
|
||||
def capture(self, graph_pool: Any):
|
||||
"""Capture CUDA graphs for all token budgets."""
|
||||
for token_budget in self.token_budgets:
|
||||
self.graph_pool = graph_pool
|
||||
|
||||
for token_budget in sorted(self.token_budgets, reverse=True):
|
||||
self._capture_budget_graph(token_budget)
|
||||
|
||||
logger.info(
|
||||
@@ -193,6 +201,9 @@ class EncoderCudaGraphManager:
|
||||
len(self.budget_graphs),
|
||||
)
|
||||
|
||||
def get_num_graphs_to_capture(self) -> int:
|
||||
return len(self.token_budgets)
|
||||
|
||||
def _capture_budget_graph(self, token_budget: int):
|
||||
"""Capture CUDA graph for a single token budget."""
|
||||
logger.debug(
|
||||
@@ -218,7 +229,7 @@ class EncoderCudaGraphManager:
|
||||
output_buffer = torch.empty_like(output)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.inference_mode(), torch.cuda.graph(graph):
|
||||
with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool):
|
||||
output = self.model.encoder_cudagraph_forward({**values})
|
||||
output_buffer.copy_(output)
|
||||
|
||||
|
||||
@@ -6304,6 +6304,42 @@ class GPUModelRunner(
|
||||
|
||||
logger.debug("Cleaned up profiling KV cache and CUDA graphs")
|
||||
|
||||
@torch.inference_mode()
|
||||
def _create_encoder_cudagraph_manager(self) -> "EncoderCudaGraphManager | None":
|
||||
if not (
|
||||
self.compilation_config.cudagraph_mm_encoder and self.supports_mm_inputs
|
||||
):
|
||||
return None
|
||||
|
||||
# Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper, because
|
||||
# @runtime_checkable Protocol isinstance() checks do not work through
|
||||
# __getattr__ forwarding.
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEncoderCudaGraph,
|
||||
supports_encoder_cudagraph,
|
||||
)
|
||||
from vllm.v1.worker.encoder_cudagraph import (
|
||||
EncoderCudaGraphManager,
|
||||
)
|
||||
|
||||
raw_model = self.get_model()
|
||||
if not supports_encoder_cudagraph(raw_model):
|
||||
return None
|
||||
|
||||
return EncoderCudaGraphManager(
|
||||
vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
model=cast(SupportsEncoderCudaGraph, raw_model),
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _maybe_init_encoder_cudagraph_manager(self) -> None:
|
||||
if self.encoder_cudagraph_manager is None:
|
||||
self.encoder_cudagraph_manager = self._create_encoder_cudagraph_manager()
|
||||
if self.encoder_cudagraph_manager is not None:
|
||||
logger.info("Initialized EncoderCudaGraphManager for vision encoder")
|
||||
|
||||
@torch.inference_mode()
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
@@ -6312,24 +6348,40 @@ class GPUModelRunner(
|
||||
saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured
|
||||
|
||||
capture_descs = self.cudagraph_dispatcher.get_capture_descs()
|
||||
# Use a temporary manager for memory profiling. The persistent manager
|
||||
# is initialized later so it does not keep profiling-only graph state.
|
||||
encoder_cudagraph_manager = self._create_encoder_cudagraph_manager()
|
||||
|
||||
total_graphs = sum(len(descs) for _, descs in capture_descs)
|
||||
decoder_graphs = sum(len(descs) for _, descs in capture_descs)
|
||||
encoder_graphs = (
|
||||
encoder_cudagraph_manager.get_num_graphs_to_capture()
|
||||
if encoder_cudagraph_manager is not None
|
||||
else 0
|
||||
)
|
||||
total_graphs = decoder_graphs + encoder_graphs
|
||||
if total_graphs == 0:
|
||||
logger.debug("No CUDA graphs will be captured, skipping profiling")
|
||||
self._cleanup_profiling_kv_cache()
|
||||
return 0
|
||||
|
||||
logger.info(
|
||||
"Profiling CUDA graph memory: %s",
|
||||
", ".join(
|
||||
graph_groups = [
|
||||
*(
|
||||
f"{mode.name}={len(descs)} (largest={descs[0].num_tokens})"
|
||||
for mode, descs in capture_descs
|
||||
if descs
|
||||
),
|
||||
)
|
||||
]
|
||||
if encoder_graphs > 0:
|
||||
graph_groups.append(
|
||||
f"ENCODER={encoder_graphs} "
|
||||
f"(largest={encoder_cudagraph_manager.token_budgets[-1]})"
|
||||
)
|
||||
|
||||
logger.info("Profiling CUDA graph memory: %s", ", ".join(graph_groups))
|
||||
|
||||
# Use a temporary pool for profiling to avoid fragmentation in the main pool.
|
||||
profiling_pool = current_platform.graph_pool_handle()
|
||||
encoder_profiling_pool = current_platform.graph_pool_handle()
|
||||
original_pools: dict[int, Any] = {}
|
||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||
BreakableCUDAGraphWrapper._all_instances
|
||||
@@ -6338,73 +6390,98 @@ class GPUModelRunner(
|
||||
original_pools[id(instance)] = instance.graph_pool
|
||||
instance.graph_pool = profiling_pool
|
||||
|
||||
set_cudagraph_capturing_enabled(True)
|
||||
with self._freeze_gc(), graph_capture(device=self.device):
|
||||
shared_memory_estimate = {}
|
||||
per_graph_estimate = {}
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
shared_memory_estimate = {}
|
||||
per_graph_estimate = {}
|
||||
encoder_memory_estimate = 0
|
||||
|
||||
for mode, descs in capture_descs:
|
||||
profile_descs = descs[:2]
|
||||
mem_samples: list[int] = []
|
||||
# Cleanup-only guard: CUDA graph capture errors should still propagate
|
||||
# because encoder graph capture is opt-in.
|
||||
try:
|
||||
set_cudagraph_capturing_enabled(True)
|
||||
with self._freeze_gc(), graph_capture(device=self.device):
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
for i, desc in enumerate(profile_descs):
|
||||
mem_before = torch.cuda.mem_get_info()[0]
|
||||
self._warmup_and_capture(
|
||||
desc,
|
||||
cudagraph_runtime_mode=mode,
|
||||
profile_seq_lens=(
|
||||
min(
|
||||
self.max_model_len,
|
||||
self.max_num_tokens // desc.num_tokens,
|
||||
)
|
||||
if mode == CUDAGraphMode.FULL and i == 0
|
||||
else None
|
||||
),
|
||||
for mode, descs in capture_descs:
|
||||
profile_descs = descs[:2]
|
||||
mem_samples: list[int] = []
|
||||
|
||||
for i, desc in enumerate(profile_descs):
|
||||
mem_before = torch.cuda.mem_get_info()[0]
|
||||
self._warmup_and_capture(
|
||||
desc,
|
||||
cudagraph_runtime_mode=mode,
|
||||
profile_seq_lens=(
|
||||
min(
|
||||
self.max_model_len,
|
||||
self.max_num_tokens // desc.num_tokens,
|
||||
)
|
||||
if mode == CUDAGraphMode.FULL and i == 0
|
||||
else None
|
||||
),
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
free_after = torch.cuda.mem_get_info()[0]
|
||||
mem_samples.append(mem_before - free_after)
|
||||
|
||||
first_capture = mem_samples[0]
|
||||
# Use at least 1 MiB per graph for driver overhead
|
||||
per_graph = max(
|
||||
mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20
|
||||
)
|
||||
|
||||
shared_memory_estimate[mode] = first_capture
|
||||
per_graph_estimate[mode] = per_graph * (len(descs) - 1)
|
||||
|
||||
logger.debug(
|
||||
"Estimated %s CUDA graph memory: "
|
||||
"%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph",
|
||||
mode.name,
|
||||
first_capture / (1 << 20),
|
||||
len(descs),
|
||||
per_graph / (1 << 20),
|
||||
)
|
||||
|
||||
if encoder_cudagraph_manager is not None:
|
||||
mem_before = torch.cuda.mem_get_info()[0]
|
||||
encoder_cudagraph_manager.capture(graph_pool=encoder_profiling_pool)
|
||||
torch.accelerator.synchronize()
|
||||
free_after = torch.cuda.mem_get_info()[0]
|
||||
mem_samples.append(mem_before - free_after)
|
||||
encoder_memory_estimate = max(mem_before - free_after, 0)
|
||||
|
||||
first_capture = mem_samples[0]
|
||||
# Use at least 1 MiB per graph for driver overhead
|
||||
per_graph = max(mem_samples[1] if len(mem_samples) > 1 else 0, 1 << 20)
|
||||
|
||||
shared_memory_estimate[mode] = first_capture
|
||||
per_graph_estimate[mode] = per_graph * (len(descs) - 1)
|
||||
|
||||
logger.debug(
|
||||
"Estimated %s CUDA graph memory: "
|
||||
"%.2f MiB first-capture + (%d-1) × %.2f MiB per-graph",
|
||||
mode.name,
|
||||
first_capture / (1 << 20),
|
||||
len(descs),
|
||||
per_graph / (1 << 20),
|
||||
)
|
||||
|
||||
set_cudagraph_capturing_enabled(False)
|
||||
CUDAGraphWrapper.clear_all_graphs()
|
||||
BreakableCUDAGraphWrapper.clear_all_graphs()
|
||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||
BreakableCUDAGraphWrapper._all_instances
|
||||
)
|
||||
for instance in all_wrappers:
|
||||
if id(instance) in original_pools:
|
||||
instance.graph_pool = original_pools[id(instance)]
|
||||
for key_set in self.cudagraph_dispatcher.cudagraph_keys.values():
|
||||
key_set.clear()
|
||||
self.cudagraph_dispatcher.keys_initialized = False
|
||||
self.maybe_remove_all_loras(self.lora_config)
|
||||
self._cleanup_profiling_kv_cache()
|
||||
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
|
||||
logger.debug(
|
||||
"Estimated encoder CUDA graph memory: %.2f MiB for %d graphs",
|
||||
encoder_memory_estimate / (1 << 20),
|
||||
encoder_graphs,
|
||||
)
|
||||
finally:
|
||||
set_cudagraph_capturing_enabled(False)
|
||||
CUDAGraphWrapper.clear_all_graphs()
|
||||
BreakableCUDAGraphWrapper.clear_all_graphs()
|
||||
if encoder_cudagraph_manager is not None:
|
||||
encoder_cudagraph_manager.clear()
|
||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||
BreakableCUDAGraphWrapper._all_instances
|
||||
)
|
||||
for instance in all_wrappers:
|
||||
if id(instance) in original_pools:
|
||||
instance.graph_pool = original_pools[id(instance)]
|
||||
for key_set in self.cudagraph_dispatcher.cudagraph_keys.values():
|
||||
key_set.clear()
|
||||
self.cudagraph_dispatcher.keys_initialized = False
|
||||
self.maybe_remove_all_loras(self.lora_config)
|
||||
self._cleanup_profiling_kv_cache()
|
||||
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
|
||||
|
||||
# FULL and PIECEWISE graphs share the global pool at runtime and are
|
||||
# never replayed concurrently, so the pool overlays their memory.
|
||||
# Take the max to avoid double-counting the overlap.
|
||||
total_estimate = max(shared_memory_estimate.values()) + sum(
|
||||
decoder_estimate = max(shared_memory_estimate.values(), default=0) + sum(
|
||||
per_graph_estimate.values()
|
||||
)
|
||||
# Encoder graphs use a manager-local pool at runtime, separate from the
|
||||
# decoder pool, so add their estimate instead of overlaying it.
|
||||
total_estimate = decoder_estimate + encoder_memory_estimate
|
||||
logger.info(
|
||||
"Estimated CUDA graph memory: %.2f GiB total",
|
||||
total_estimate / (1 << 30),
|
||||
@@ -6422,31 +6499,7 @@ class GPUModelRunner(
|
||||
return 0
|
||||
|
||||
# Initialize encoder CUDA graph manager if enabled.
|
||||
# Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper,
|
||||
# because @runtime_checkable Protocol isinstance() checks do not
|
||||
# work through __getattr__ forwarding.
|
||||
if (
|
||||
self.compilation_config.cudagraph_mm_encoder
|
||||
and self.supports_mm_inputs
|
||||
and self.encoder_cudagraph_manager is None
|
||||
):
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEncoderCudaGraph,
|
||||
supports_encoder_cudagraph,
|
||||
)
|
||||
from vllm.v1.worker.encoder_cudagraph import (
|
||||
EncoderCudaGraphManager,
|
||||
)
|
||||
|
||||
raw_model = self.get_model()
|
||||
if supports_encoder_cudagraph(raw_model):
|
||||
self.encoder_cudagraph_manager = EncoderCudaGraphManager(
|
||||
vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
model=cast(SupportsEncoderCudaGraph, raw_model),
|
||||
)
|
||||
logger.info("Initialized EncoderCudaGraphManager for vision encoder")
|
||||
self._maybe_init_encoder_cudagraph_manager()
|
||||
|
||||
compilation_counter.num_gpu_runner_capture_triggers += 1
|
||||
|
||||
@@ -6473,7 +6526,8 @@ class GPUModelRunner(
|
||||
|
||||
# Capture encoder CUDA graphs if enabled
|
||||
if self.encoder_cudagraph_manager is not None:
|
||||
self.encoder_cudagraph_manager.capture()
|
||||
encoder_graph_pool = current_platform.graph_pool_handle()
|
||||
self.encoder_cudagraph_manager.capture(graph_pool=encoder_graph_pool)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
|
||||
Reference in New Issue
Block a user