[Model Runner V2] Reserve CUDA graph memory (#53306)
Signed-off-by: Anh Tran <anh.tran.3889@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Anh Tran <anh.tran.3889@gmail.com> Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
co-authored by
Anh Tran
anhtra3889
parent
8c2bbe00d5
commit
a4d70bef37
@@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
|
||||
"facebook/opt-125m",
|
||||
max_logprobs=5,
|
||||
enable_prefix_caching=False,
|
||||
# 2 other llms alive during whole session
|
||||
gpu_memory_utilization=0.05,
|
||||
# 2 other llms alive during whole session; must also cover the
|
||||
# cudagraph memory reservation from startup profiling.
|
||||
gpu_memory_utilization=0.1,
|
||||
max_model_len=16,
|
||||
logprobs_mode=logprobs_mode,
|
||||
)
|
||||
@@ -575,7 +576,7 @@ def test_prompt_logprobs_mode():
|
||||
llm = LLM(
|
||||
"facebook/opt-125m",
|
||||
enable_prefix_caching=False,
|
||||
gpu_memory_utilization=0.05,
|
||||
gpu_memory_utilization=0.1,
|
||||
max_model_len=16,
|
||||
logprobs_mode=mode,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user