[Bugfix][ROCm] Capture CUDA graphs on the current stream (#53818)
Signed-off-by: fanxingran <xingran.fan@amd.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig, get_layers_from_vllm_config, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
@@ -130,7 +131,7 @@ class Gemma4Proposer(SpecDecodeBaseProposer):
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
with torch.cuda.graph(g, stream=current_stream()):
|
||||
static_output = masked_emb.get_top_tokens(
|
||||
static_input,
|
||||
lm_head_weight,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
from vllm.model_executor.models.utils import scatter_output_slices
|
||||
from vllm.model_executor.models.vision import get_load_balance_assignment
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderItemSpec,
|
||||
@@ -270,7 +271,10 @@ class EncoderCudaGraphManager:
|
||||
output_buffer = torch.empty_like(output)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.inference_mode(), torch.cuda.graph(graph, pool=self.graph_pool):
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
torch.cuda.graph(graph, pool=self.graph_pool, stream=current_stream()),
|
||||
):
|
||||
output = self.model.encoder_cudagraph_forward({**values}, path=path)
|
||||
output_buffer.copy_(output)
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
@@ -377,7 +378,9 @@ class CudaGraphManager:
|
||||
if self._capture_mem_samples is not None:
|
||||
torch.accelerator.synchronize()
|
||||
free_before = torch.accelerator.get_memory_info()[0]
|
||||
with torch.cuda.graph(graph, self.pool):
|
||||
with torch.cuda.graph(
|
||||
graph, self.pool, stream=current_stream()
|
||||
):
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
# Join offloader's copy stream after forward to avoid
|
||||
# unjoined stream error. The last layer's start_prefetch
|
||||
|
||||
Reference in New Issue
Block a user