[ROCm][CI] Fix cuda graph mem profile issue (#48764)

Signed-off-by: charlifu <charlifu@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Charlie Fu
2026-07-16 04:23:18 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent ba47bb5be1
commit 6a9f24aa8c
+13 -10
View File
@@ -120,6 +120,7 @@ from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
current_stream,
get_dtype_size,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
@@ -6608,17 +6609,19 @@ class GPUModelRunner(
per_graph_estimate = {}
encoder_memory_estimate = 0
# On ROCm, capture these throwaway profiling graphs on the current stream
# instead of the fresh side stream graph_capture() allocates by default.
# torch's allocator pools free blocks per stream, so a side-stream forward
# strands a persistent aiter scratch buffer in a separate pool, shifting
# the physical placement of the real KV cache allocated afterward and
# slowing bandwidth-bound decode ~20%. The graphs are discarded, so a
# side stream is unnecessary here.
# cap_ctx=None keeps the side-stream path on CUDA, where the current
# stream is the legacy default stream, on which capture cannot begin.
# On ROCm, capture these throwaway profiling graphs on vLLM's dedicated
# compute stream instead of the fresh side stream graph_capture()
# allocates by default. torch's allocator pools free blocks per stream,
# so a side-stream forward strands a persistent aiter scratch buffer in
# a separate pool, shifting the physical placement of the real KV cache
# allocated afterward and slowing bandwidth-bound decode ~20%. The
# graphs are discarded, so a side stream is unnecessary here.
# Use current_stream(), not torch.cuda.current_stream(): before vLLM
# initializes its dedicated stream, torch returns the per-thread default
# stream (cuda_stream=0), which cannot be used for cudagraph capture.
# cap_ctx=None keeps the side-stream path on CUDA.
cap_ctx = (
GraphCaptureContext(torch.cuda.current_stream(self.device))
GraphCaptureContext(current_stream())
if current_platform.is_rocm()
else None
)