lower memory required for capturing cudagraphs for large cudagraph sizes (#48483)

Signed-off-by: Omer Ullman Argov <118735753+omera-nv@users.noreply.github.com>
This commit is contained in:
Omer Ullman Argov
2026-07-13 10:14:25 -04:00
committed by GitHub
parent b3cfca996c
commit 1be6e937b2
+5 -1
View File
@@ -6426,7 +6426,11 @@ class GPUModelRunner(
kv_cache_spec = self.get_kv_cache_spec()
KVCacheSpecRegistry.check_kv_cache_spec_registry(kv_cache_spec)
kv_cache_groups = get_kv_cache_groups(self.vllm_config, kv_cache_spec)
min_blocks = self.compilation_config.max_cudagraph_capture_size or 1
# the minimum number of blocks required is 1 block *per sequence*
min_blocks = (
min(self.max_num_reqs, self.compilation_config.max_cudagraph_capture_size)
or 1
)
# Temporarily change num_gpu_blocks_override to allocate a minimal KV cache
saved_override = self.cache_config.num_gpu_blocks_override