[Startup][UX] Enable CUDAGraph memory profiling by default (#38284)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
9db4650e5e
commit
96a85c5750
@@ -321,7 +321,7 @@ def test_speculators_model_integration(
|
||||
test_prompts = get_test_prompts(mm_enabled=False)
|
||||
|
||||
# First run: Direct speculator model (simplified integration)
|
||||
spec_llm = LLM(model=model_path, max_model_len=4096)
|
||||
spec_llm = LLM(model=model_path, max_model_len=4096, gpu_memory_utilization=0.92)
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_llm, expected_accuracy_threshold=expected_accuracy_threshold
|
||||
)
|
||||
@@ -351,7 +351,7 @@ def test_speculators_model_integration(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
# Second run: Reference without speculative decoding
|
||||
ref_llm = LLM(model=verifier_model, max_model_len=4096)
|
||||
ref_llm = LLM(model=verifier_model, max_model_len=4096, gpu_memory_utilization=0.92)
|
||||
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
Reference in New Issue
Block a user