[ROCm][Bugfix] Reduce CUDA graph divergences (#57229)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-09-17 05:12:13 +00:00
committed by GitHub
parent 0bfc7a15d0
commit 2d8a5c5741
5 changed files with 8 additions and 90 deletions
-39
View File
@@ -477,45 +477,6 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
default_breakable_cudagraph_architectures.cache_clear()
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
@pytest.mark.parametrize(
("architecture", "use_v2", "mode", "expected"),
[
("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE),
("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE),
("DeepseekV4ForCausalLM", False, None, None),
("LlamaForCausalLM", True, None, None),
(
"DeepseekV4ForCausalLM",
True,
CUDAGraphMode.FULL_DECODE_ONLY,
CUDAGraphMode.FULL_DECODE_ONLY,
),
],
)
def test_rocm_gfx950_deepseek_v4_cudagraph_default(
monkeypatch, architecture, use_v2, mode, expected
):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False)
monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False)
monkeypatch.setattr(
rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False
)
config = SimpleNamespace(
compilation_config=CompilationConfig(cudagraph_mode=mode),
model_config=SimpleNamespace(architecture=architecture),
use_v2_model_runner=use_v2,
)
rocm.RocmPlatform.apply_config_platform_defaults(config)
assert config.compilation_config.cudagraph_mode == expected
@pytest.mark.parametrize(
("model", "architecture"),
[
+2 -10
View File
@@ -1632,10 +1632,7 @@ def get_pcp_group() -> GroupCoordinator:
@contextmanager
def graph_capture(
device: torch.device,
graph_capture_context: GraphCaptureContext | None = None,
):
def graph_capture(device: torch.device):
"""`graph_capture` is a context manager which should surround the code that
is capturing the CUDA graph. Its main purpose is to ensure that some
operations will be run after the graph is captured, before the graph
@@ -1647,13 +1644,8 @@ def graph_capture(
the graph capture is running on a separate stream from the default stream,
in order to explicitly distinguish the kernels to capture
from other kernels possibly launched on background in the default stream.
A caller may pass an explicit ``graph_capture_context`` to control the
stream used (e.g. to capture on the default stream).
"""
context = graph_capture_context or GraphCaptureContext(
torch.cuda.Stream(device=device)
)
context = GraphCaptureContext(torch.cuda.Stream(device=device))
with (
get_tp_group().graph_capture(context),
get_pp_group().graph_capture(context),
-17
View File
@@ -865,25 +865,8 @@ class RocmPlatform(Platform):
@classmethod
def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.compilation import CUDAGraphMode
compilation_config = vllm_config.compilation_config
model_config = vllm_config.model_config
if (
compilation_config.cudagraph_mode is None
and model_config is not None
and on_gfx950()
and vllm_config.use_v2_model_runner
and model_config.architecture
in {
"DeepseekV4ForCausalLM",
"DeepseekV4ForConditionalGeneration",
}
):
# Default to eager after reported gfx950/MRV2 accuracy regressions:
# https://github.com/vllm-project/vllm/issues/52644
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled()
use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled()
use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
+1 -23
View File
@@ -46,7 +46,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
copy_kv_blocks,
)
from vllm.distributed.parallel_state import (
GraphCaptureContext,
get_dcp_group,
get_pp_group,
get_tp_group,
@@ -128,7 +127,6 @@ from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
current_stream,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
@@ -6712,31 +6710,11 @@ class GPUModelRunner(
per_graph_estimate = {}
encoder_memory_estimate = 0
# On ROCm, capture these throwaway profiling graphs on vLLM's dedicated
# compute stream instead of the fresh side stream graph_capture()
# allocates by default. torch's allocator pools free blocks per stream,
# so a side-stream forward strands a persistent aiter scratch buffer in
# a separate pool, shifting the physical placement of the real KV cache
# allocated afterward and slowing bandwidth-bound decode ~20%. The
# graphs are discarded, so a side stream is unnecessary here.
# Use current_stream(), not torch.cuda.current_stream(): before vLLM
# initializes its dedicated stream, torch returns the per-thread default
# stream (cuda_stream=0), which cannot be used for cudagraph capture.
# cap_ctx=None keeps the side-stream path on CUDA.
cap_ctx = (
GraphCaptureContext(current_stream())
if current_platform.is_rocm()
else None
)
# Cleanup-only guard: CUDA graph capture errors should still propagate
# because encoder graph capture is opt-in.
try:
set_cudagraph_capturing_enabled(True)
with (
self._freeze_gc(),
graph_capture(device=self.device, graph_capture_context=cap_ctx),
):
with self._freeze_gc(), graph_capture(device=self.device):
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
+5 -1
View File
@@ -98,6 +98,7 @@ from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase
from vllm.v1.worker.workspace import init_workspace_manager
from ...model_executor.model_loader import TensorizerLoader
from .gpu.cudagraph_utils import has_compiled_submodule
from .gpu.warmup import warmup_kernels
from .utils import request_memory
@@ -772,7 +773,10 @@ class Worker(WorkerBase):
def compile_or_warm_up_model(self) -> CompilationTimes:
warmup_sizes: list[int] = []
if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:
if (
self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE
and has_compiled_submodule(self.model_runner.get_model())
):
# warm up sizes that are not in cudagraph capture sizes,
# but users still want to compile for better performance,
# e.g. for the max-num-batched token size in chunked prefill.