[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+23 -5
View File
@@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
ModelCudaGraphManager,
)
from vllm.v1.worker.gpu.cudagraph_utils import (
profile_cudagraph_memory as _profile_cudagraph_memory,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
@@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_kv_cache_spec(self):
return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
def initialize_kv_cache(
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
@@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
kv_cache_config=self.kv_cache_config,
max_num_reqs=self.max_num_reqs,
is_profiling=is_profiling,
)
self.cudagraph_manager = ModelCudaGraphManager(
self.vllm_config,
@@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.kernel_block_sizes,
self.vllm_config,
)
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
else:
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
def _init_kv_zero_meta(self) -> None:
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
@@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None:
self.pooling_runner.clear()
@torch.inference_mode()
def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not.
return 0
"""Estimate the GPU memory required to capture CUDA graphs."""
return _profile_cudagraph_memory(self)
@torch.inference_mode()
def capture_model(self) -> int:
@@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings, self.kv_cache_config
)
assert block_tables is not None
attn_groups = self.attn_groups
if dummy_run and is_profile:
# Mamba layers take a cheap warmup path with no metadata;
# attention metadata is still built so those kernels tune.
attn_groups = [
[g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)]
for groups in attn_groups
]
attn_metadata = self.model_state.prepare_attn(
input_batch,
batch_desc.cg_mode,
block_tables,
slot_mappings,
self.attn_groups,
attn_groups,
self.kv_cache_config,
# FULL replay reads capture-time metadata buffers. Re-stage them
# from the zeroed dummy block tables instead of retaining state