From 6a9f24aa8cb856235528d01a829a4ba85fc1c19d Mon Sep 17 00:00:00 2001 From: Charlie Fu Date: Wed, 15 Jul 2026 23:23:18 -0500 Subject: [PATCH] [ROCm][CI] Fix cuda graph mem profile issue (#48764) Signed-off-by: charlifu Co-authored-by: Andreas Karatzas --- vllm/v1/worker/gpu_model_runner.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index b104e4c5ca..ea8980a6e5 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -120,6 +120,7 @@ from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import ( PIN_MEMORY, async_tensor_h2d, + current_stream, get_dtype_size, is_quantized_kv_cache, kv_cache_dtype_str_to_dtype, @@ -6608,17 +6609,19 @@ class GPUModelRunner( per_graph_estimate = {} encoder_memory_estimate = 0 - # On ROCm, capture these throwaway profiling graphs on the current stream - # instead of the fresh side stream graph_capture() allocates by default. - # torch's allocator pools free blocks per stream, so a side-stream forward - # strands a persistent aiter scratch buffer in a separate pool, shifting - # the physical placement of the real KV cache allocated afterward and - # slowing bandwidth-bound decode ~20%. The graphs are discarded, so a - # side stream is unnecessary here. - # cap_ctx=None keeps the side-stream path on CUDA, where the current - # stream is the legacy default stream, on which capture cannot begin. + # On ROCm, capture these throwaway profiling graphs on vLLM's dedicated + # compute stream instead of the fresh side stream graph_capture() + # allocates by default. torch's allocator pools free blocks per stream, + # so a side-stream forward strands a persistent aiter scratch buffer in + # a separate pool, shifting the physical placement of the real KV cache + # allocated afterward and slowing bandwidth-bound decode ~20%. The + # graphs are discarded, so a side stream is unnecessary here. + # Use current_stream(), not torch.cuda.current_stream(): before vLLM + # initializes its dedicated stream, torch returns the per-thread default + # stream (cuda_stream=0), which cannot be used for cudagraph capture. + # cap_ctx=None keeps the side-stream path on CUDA. cap_ctx = ( - GraphCaptureContext(torch.cuda.current_stream(self.device)) + GraphCaptureContext(current_stream()) if current_platform.is_rocm() else None )