[ROCm][Bugfix] Reduce CUDA graph divergences (#57229)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
@@ -477,45 +477,6 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "use_v2", "mode", "expected"),
|
||||
[
|
||||
("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE),
|
||||
("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE),
|
||||
("DeepseekV4ForCausalLM", False, None, None),
|
||||
("LlamaForCausalLM", True, None, None),
|
||||
(
|
||||
"DeepseekV4ForCausalLM",
|
||||
True,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rocm_gfx950_deepseek_v4_cudagraph_default(
|
||||
monkeypatch, architecture, use_v2, mode, expected
|
||||
):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(cudagraph_mode=mode),
|
||||
model_config=SimpleNamespace(architecture=architecture),
|
||||
use_v2_model_runner=use_v2,
|
||||
)
|
||||
|
||||
rocm.RocmPlatform.apply_config_platform_defaults(config)
|
||||
|
||||
assert config.compilation_config.cudagraph_mode == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "architecture"),
|
||||
[
|
||||
|
||||
@@ -1632,10 +1632,7 @@ def get_pcp_group() -> GroupCoordinator:
|
||||
|
||||
|
||||
@contextmanager
|
||||
def graph_capture(
|
||||
device: torch.device,
|
||||
graph_capture_context: GraphCaptureContext | None = None,
|
||||
):
|
||||
def graph_capture(device: torch.device):
|
||||
"""`graph_capture` is a context manager which should surround the code that
|
||||
is capturing the CUDA graph. Its main purpose is to ensure that some
|
||||
operations will be run after the graph is captured, before the graph
|
||||
@@ -1647,13 +1644,8 @@ def graph_capture(
|
||||
the graph capture is running on a separate stream from the default stream,
|
||||
in order to explicitly distinguish the kernels to capture
|
||||
from other kernels possibly launched on background in the default stream.
|
||||
|
||||
A caller may pass an explicit ``graph_capture_context`` to control the
|
||||
stream used (e.g. to capture on the default stream).
|
||||
"""
|
||||
context = graph_capture_context or GraphCaptureContext(
|
||||
torch.cuda.Stream(device=device)
|
||||
)
|
||||
context = GraphCaptureContext(torch.cuda.Stream(device=device))
|
||||
with (
|
||||
get_tp_group().graph_capture(context),
|
||||
get_pp_group().graph_capture(context),
|
||||
|
||||
@@ -865,25 +865,8 @@ class RocmPlatform(Platform):
|
||||
@classmethod
|
||||
def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
|
||||
compilation_config = vllm_config.compilation_config
|
||||
model_config = vllm_config.model_config
|
||||
if (
|
||||
compilation_config.cudagraph_mode is None
|
||||
and model_config is not None
|
||||
and on_gfx950()
|
||||
and vllm_config.use_v2_model_runner
|
||||
and model_config.architecture
|
||||
in {
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepseekV4ForConditionalGeneration",
|
||||
}
|
||||
):
|
||||
# Default to eager after reported gfx950/MRV2 accuracy regressions:
|
||||
# https://github.com/vllm-project/vllm/issues/52644
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled()
|
||||
use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
|
||||
@@ -46,7 +46,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
copy_kv_blocks,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
GraphCaptureContext,
|
||||
get_dcp_group,
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
@@ -128,7 +127,6 @@ from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
current_stream,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -6712,31 +6710,11 @@ class GPUModelRunner(
|
||||
per_graph_estimate = {}
|
||||
encoder_memory_estimate = 0
|
||||
|
||||
# On ROCm, capture these throwaway profiling graphs on vLLM's dedicated
|
||||
# compute stream instead of the fresh side stream graph_capture()
|
||||
# allocates by default. torch's allocator pools free blocks per stream,
|
||||
# so a side-stream forward strands a persistent aiter scratch buffer in
|
||||
# a separate pool, shifting the physical placement of the real KV cache
|
||||
# allocated afterward and slowing bandwidth-bound decode ~20%. The
|
||||
# graphs are discarded, so a side stream is unnecessary here.
|
||||
# Use current_stream(), not torch.cuda.current_stream(): before vLLM
|
||||
# initializes its dedicated stream, torch returns the per-thread default
|
||||
# stream (cuda_stream=0), which cannot be used for cudagraph capture.
|
||||
# cap_ctx=None keeps the side-stream path on CUDA.
|
||||
cap_ctx = (
|
||||
GraphCaptureContext(current_stream())
|
||||
if current_platform.is_rocm()
|
||||
else None
|
||||
)
|
||||
|
||||
# Cleanup-only guard: CUDA graph capture errors should still propagate
|
||||
# because encoder graph capture is opt-in.
|
||||
try:
|
||||
set_cudagraph_capturing_enabled(True)
|
||||
with (
|
||||
self._freeze_gc(),
|
||||
graph_capture(device=self.device, graph_capture_context=cap_ctx),
|
||||
):
|
||||
with self._freeze_gc(), graph_capture(device=self.device):
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from ...model_executor.model_loader import TensorizerLoader
|
||||
from .gpu.cudagraph_utils import has_compiled_submodule
|
||||
from .gpu.warmup import warmup_kernels
|
||||
from .utils import request_memory
|
||||
|
||||
@@ -772,7 +773,10 @@ class Worker(WorkerBase):
|
||||
def compile_or_warm_up_model(self) -> CompilationTimes:
|
||||
warmup_sizes: list[int] = []
|
||||
|
||||
if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE:
|
||||
if (
|
||||
self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE
|
||||
and has_compiled_submodule(self.model_runner.get_model())
|
||||
):
|
||||
# warm up sizes that are not in cudagraph capture sizes,
|
||||
# but users still want to compile for better performance,
|
||||
# e.g. for the max-num-batched token size in chunked prefill.
|
||||
|
||||
Reference in New Issue
Block a user