[Model Runner V2] Reserve CUDA graph memory (#53306)
Signed-off-by: Anh Tran <anh.tran.3889@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Anh Tran <anh.tran.3889@gmail.com> Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
co-authored by
Anh Tran
anhtra3889
parent
8c2bbe00d5
commit
a4d70bef37
@@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
ModelCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
profile_cudagraph_memory as _profile_cudagraph_memory,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
|
||||
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
|
||||
@@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def get_kv_cache_spec(self):
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
|
||||
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
|
||||
) -> None:
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
@@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
|
||||
kv_cache_config=self.kv_cache_config,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
is_profiling=is_profiling,
|
||||
)
|
||||
self.cudagraph_manager = ModelCudaGraphManager(
|
||||
self.vllm_config,
|
||||
@@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kernel_block_sizes,
|
||||
self.vllm_config,
|
||||
)
|
||||
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
else:
|
||||
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
|
||||
|
||||
def _init_kv_zero_meta(self) -> None:
|
||||
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
|
||||
@@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.pooling_runner is not None:
|
||||
self.pooling_runner.clear()
|
||||
|
||||
@torch.inference_mode()
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
# NOTE(woosuk): It is TBD whether we keep this API or not.
|
||||
return 0
|
||||
"""Estimate the GPU memory required to capture CUDA graphs."""
|
||||
return _profile_cudagraph_memory(self)
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self) -> int:
|
||||
@@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
slot_mappings, self.kv_cache_config
|
||||
)
|
||||
assert block_tables is not None
|
||||
attn_groups = self.attn_groups
|
||||
if dummy_run and is_profile:
|
||||
# Mamba layers take a cheap warmup path with no metadata;
|
||||
# attention metadata is still built so those kernels tune.
|
||||
attn_groups = [
|
||||
[g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)]
|
||||
for groups in attn_groups
|
||||
]
|
||||
attn_metadata = self.model_state.prepare_attn(
|
||||
input_batch,
|
||||
batch_desc.cg_mode,
|
||||
block_tables,
|
||||
slot_mappings,
|
||||
self.attn_groups,
|
||||
attn_groups,
|
||||
self.kv_cache_config,
|
||||
# FULL replay reads capture-time metadata buffers. Re-stage them
|
||||
# from the zeroed dummy block tables instead of retaining state
|
||||
|
||||
Reference in New Issue
Block a user