[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+4 -3
View File
@@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
"facebook/opt-125m",
max_logprobs=5,
enable_prefix_caching=False,
# 2 other llms alive during whole session
gpu_memory_utilization=0.05,
# 2 other llms alive during whole session; must also cover the
# cudagraph memory reservation from startup profiling.
gpu_memory_utilization=0.1,
max_model_len=16,
logprobs_mode=logprobs_mode,
)
@@ -575,7 +576,7 @@ def test_prompt_logprobs_mode():
llm = LLM(
"facebook/opt-125m",
enable_prefix_caching=False,
gpu_memory_utilization=0.05,
gpu_memory_utilization=0.1,
max_model_len=16,
logprobs_mode=mode,
)