[Bugfix][ROCm] Capture CUDA graphs on the current stream (#53818)

Signed-off-by: fanxingran <xingran.fan@amd.com>
This commit is contained in:
fanxingran
2026-08-26 23:16:45 +08:00
committed by GitHub
parent 28b484e5d7
commit 080a66a69c
3 changed files with 11 additions and 3 deletions
+2 -1
View File
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig, get_layers_from_vllm_config, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
@@ -130,7 +131,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
with torch.cuda.graph(g, stream=current_stream()):
static_output = masked_emb.get_top_tokens(
static_input,
lm_head_weight,
+5 -1
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.models.interfaces import (
from vllm.model_executor.models.utils import scatter_output_slices
from vllm.model_executor.models.vision import get_load_balance_assignment
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
EncoderItemSpec,
@@ -270,7 +271,10 @@ class EncoderCudaGraphManager:
output_buffer = torch.empty_like(output)
graph = torch.cuda.CUDAGraph()
with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool):
with (
torch.inference_mode(),
torch.cuda.graph(graph, pool=self.graph_pool, stream=current_stream()),
):
output = self.model.encoder_cudagraph_forward({**values}, path=path)
output_buffer.copy_(output)
+4 -1
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import current_stream
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
@@ -377,7 +378,9 @@ class CudaGraphManager:
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_before = torch.accelerator.get_memory_info()[0]
with torch.cuda.graph(graph, self.pool):
with torch.cuda.graph(
graph, self.pool, stream=current_stream()
):
forward_fn(CUDAGraphMode.NONE)
# Join offloader's copy stream after forward to avoid
# unjoined stream error. The last layer's start_prefetch