[Startup][UX] Enable CUDAGraph memory profiling by default (#38284)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
parent
9db4650e5e
commit
96a85c5750
@@ -29,7 +29,7 @@ llm = LLM(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),
|
||||
distributed_executor_backend="external_launcher",
|
||||
gpu_memory_utilization=random.uniform(0.7, 0.9),
|
||||
gpu_memory_utilization=random.uniform(0.8, 0.92),
|
||||
seed=0,
|
||||
)
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ llm = LLM(
|
||||
pipeline_parallel_size=int(os.getenv("PP_SIZE", "1")),
|
||||
enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,
|
||||
distributed_executor_backend="external_launcher",
|
||||
gpu_memory_utilization=random.uniform(0.7, 0.9),
|
||||
gpu_memory_utilization=random.uniform(0.8, 0.92),
|
||||
seed=0,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=16,
|
||||
|
||||
@@ -65,7 +65,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
|
||||
assert max_batch_size >= 2, "Batch size should be >= 2 to mix needle."
|
||||
|
||||
# Keep GPU memory usage low to avoid startup allocation failures.
|
||||
gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.4"))
|
||||
gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.5"))
|
||||
max_model_len = int(os.getenv("VLLM_MAX_MODEL_LEN", "5120"))
|
||||
|
||||
# Sampling parameters: longer outputs with a more random-sounding
|
||||
|
||||
@@ -321,7 +321,7 @@ def test_speculators_model_integration(
|
||||
test_prompts = get_test_prompts(mm_enabled=False)
|
||||
|
||||
# First run: Direct speculator model (simplified integration)
|
||||
spec_llm = LLM(model=model_path, max_model_len=4096)
|
||||
spec_llm = LLM(model=model_path, max_model_len=4096, gpu_memory_utilization=0.92)
|
||||
evaluate_llm_for_gsm8k(
|
||||
spec_llm, expected_accuracy_threshold=expected_accuracy_threshold
|
||||
)
|
||||
@@ -351,7 +351,7 @@ def test_speculators_model_integration(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
# Second run: Reference without speculative decoding
|
||||
ref_llm = LLM(model=verifier_model, max_model_len=4096)
|
||||
ref_llm = LLM(model=verifier_model, max_model_len=4096, gpu_memory_utilization=0.92)
|
||||
ref_outputs = ref_llm.chat(test_prompts, sampling_config)
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
Reference in New Issue
Block a user