[CI/Build][AMD] Fix ROCm OOM in eagle_correctness_heavy by reserving CUDA graph memory (#47366)
Signed-off-by: pei.zhang <pei.zhang@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude
Andreas Karatzas
parent
feb384ada2
commit
88e5e2c57b
@@ -17,6 +17,7 @@ from tests.utils import (
|
||||
multi_gpu_marks,
|
||||
multi_gpu_only,
|
||||
single_gpu_only,
|
||||
wait_for_rocm_memory_to_settle,
|
||||
)
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
@@ -448,6 +449,9 @@ def _run_eagle_correctness(
|
||||
del ref_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so the spec engine started right after
|
||||
# does not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
spec_llm = LLM(
|
||||
model=model_name,
|
||||
@@ -485,6 +489,9 @@ def _run_eagle_correctness(
|
||||
del spec_llm
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so the next parametrization's engine does
|
||||
# not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle()
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
|
||||
@@ -478,11 +478,14 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
# Profile CUDA graph memory if graphs will be captured.
|
||||
# Skip on ROCm/HIP/XPU as graph pool handles and get_memory_info
|
||||
# behave differently and can produce incorrect/negative estimates.
|
||||
# ROCm is included: #44825 moved the profiler to
|
||||
# torch.accelerator.get_memory_info (reliable on ROCm, as used by
|
||||
# the AMD-CI mem tests), and graph_pool_handle resolves to the same
|
||||
# torch.cuda handle the live capture path already uses on ROCm.
|
||||
# XPU stays excluded (see #39977).
|
||||
cudagraph_memory_estimate = 0
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
current_platform.is_cuda_alike()
|
||||
and self.vllm_config.compilation_config.cudagraph_mode
|
||||
!= CUDAGraphMode.NONE
|
||||
):
|
||||
@@ -498,8 +501,7 @@ class Worker(WorkerBase):
|
||||
+ profile_result.weights_memory
|
||||
)
|
||||
|
||||
# On ROCm, cudagraph_memory_estimate is always 0 so this is a no-op.
|
||||
# On CUDA, respect the opt-in flag as originally designed.
|
||||
# Respect the opt-in flag as originally designed.
|
||||
cudagraph_memory_estimate_applied = (
|
||||
cudagraph_memory_estimate
|
||||
if envs.VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS
|
||||
|
||||
Reference in New Issue
Block a user