From 080a66a69c6fd1fe464756f88ab958baad66ce69 Mon Sep 17 00:00:00 2001 From: fanxingran Date: Wed, 26 Aug 2026 23:16:45 +0800 Subject: [PATCH] [Bugfix][ROCm] Capture CUDA graphs on the current stream (#53818) Signed-off-by: fanxingran --- vllm/v1/spec_decode/gemma4.py | 3 ++- vllm/v1/worker/encoder_cudagraph.py | 6 +++++- vllm/v1/worker/gpu/cudagraph_utils.py | 5 ++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/vllm/v1/spec_decode/gemma4.py b/vllm/v1/spec_decode/gemma4.py index 07cfaa2f4d..b94fa6b867 100644 --- a/vllm/v1/spec_decode/gemma4.py +++ b/vllm/v1/spec_decode/gemma4.py @@ -17,6 +17,7 @@ from vllm.config import VllmConfig, get_layers_from_vllm_config, replace from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.utils.torch_utils import current_stream from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.kv_cache_interface import ( KVCacheConfig, @@ -130,7 +131,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer): torch.accelerator.synchronize() g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g): + with torch.cuda.graph(g, stream=current_stream()): static_output = masked_emb.get_top_tokens( static_input, lm_head_weight, diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index 04a54a39a0..a1b955a741 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -20,6 +20,7 @@ from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.utils import scatter_output_slices from vllm.model_executor.models.vision import get_load_balance_assignment from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import current_stream from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphConfig, EncoderItemSpec, @@ -270,7 +271,10 @@ class EncoderCudaGraphManager: output_buffer = torch.empty_like(output) graph = torch.cuda.CUDAGraph() - with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool): + with ( + torch.inference_mode(), + torch.cuda.graph(graph, pool=self.graph_pool, stream=current_stream()), + ): output = self.model.encoder_cudagraph_forward({**values}, path=path) output_buffer.copy_(output) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index f271adc7ec..bb08241dee 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -31,6 +31,7 @@ from vllm.model_executor.offloader.base import get_offloader from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import round_up +from vllm.utils.torch_utils import current_stream from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables @@ -377,7 +378,9 @@ class CudaGraphManager: if self._capture_mem_samples is not None: torch.accelerator.synchronize() free_before = torch.accelerator.get_memory_info()[0] - with torch.cuda.graph(graph, self.pool): + with torch.cuda.graph( + graph, self.pool, stream=current_stream() + ): forward_fn(CUDAGraphMode.NONE) # Join offloader's copy stream after forward to avoid # unjoined stream error. The last layer's start_prefetch