[ROCm][Bugfix] Reduce CUDA graph divergences (#57229)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-09-17 05:12:13 +00:00
committed by GitHub
parent 0bfc7a15d0
commit 2d8a5c5741
5 changed files with 8 additions and 90 deletions
+1 -23
View File
@@ -46,7 +46,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
copy_kv_blocks,
)
from vllm.distributed.parallel_state import (
GraphCaptureContext,
get_dcp_group,
get_pp_group,
get_tp_group,
@@ -128,7 +127,6 @@ from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
current_stream,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
@@ -6712,31 +6710,11 @@ class GPUModelRunner(
per_graph_estimate = {}
encoder_memory_estimate = 0
# On ROCm, capture these throwaway profiling graphs on vLLM's dedicated
# compute stream instead of the fresh side stream graph_capture()
# allocates by default. torch's allocator pools free blocks per stream,
# so a side-stream forward strands a persistent aiter scratch buffer in
# a separate pool, shifting the physical placement of the real KV cache
# allocated afterward and slowing bandwidth-bound decode ~20%. The
# graphs are discarded, so a side stream is unnecessary here.
# Use current_stream(), not torch.cuda.current_stream(): before vLLM
# initializes its dedicated stream, torch returns the per-thread default
# stream (cuda_stream=0), which cannot be used for cudagraph capture.
# cap_ctx=None keeps the side-stream path on CUDA.
cap_ctx = (
GraphCaptureContext(current_stream())
if current_platform.is_rocm()
else None
)
# Cleanup-only guard: CUDA graph capture errors should still propagate
# because encoder graph capture is opt-in.
try:
set_cudagraph_capturing_enabled(True)
with (
self._freeze_gc(),
graph_capture(device=self.device, graph_capture_context=cap_ctx),
):
with self._freeze_gc(), graph_capture(device=self.device):
torch.accelerator.synchronize()
torch.accelerator.empty_cache()