diff --git a/tests/test_config.py b/tests/test_config.py index 3d6a17f7a2..5c892da6c6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -477,45 +477,6 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): default_breakable_cudagraph_architectures.cache_clear() -@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test") -@pytest.mark.parametrize( - ("architecture", "use_v2", "mode", "expected"), - [ - ("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE), - ("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE), - ("DeepseekV4ForCausalLM", False, None, None), - ("LlamaForCausalLM", True, None, None), - ( - "DeepseekV4ForCausalLM", - True, - CUDAGraphMode.FULL_DECODE_ONLY, - CUDAGraphMode.FULL_DECODE_ONLY, - ), - ], -) -def test_rocm_gfx950_deepseek_v4_cudagraph_default( - monkeypatch, architecture, use_v2, mode, expected -): - from vllm._aiter_ops import rocm_aiter_ops - from vllm.platforms import rocm - - monkeypatch.setattr(rocm, "on_gfx950", lambda: True) - monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False) - monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False) - monkeypatch.setattr( - rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False - ) - config = SimpleNamespace( - compilation_config=CompilationConfig(cudagraph_mode=mode), - model_config=SimpleNamespace(architecture=architecture), - use_v2_model_runner=use_v2, - ) - - rocm.RocmPlatform.apply_config_platform_defaults(config) - - assert config.compilation_config.cudagraph_mode == expected - - @pytest.mark.parametrize( ("model", "architecture"), [ diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 60ed229bc4..20118649e1 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -1632,10 +1632,7 @@ def get_pcp_group() -> GroupCoordinator: @contextmanager -def graph_capture( - device: torch.device, - graph_capture_context: GraphCaptureContext | None = None, -): +def graph_capture(device: torch.device): """`graph_capture` is a context manager which should surround the code that is capturing the CUDA graph. Its main purpose is to ensure that some operations will be run after the graph is captured, before the graph @@ -1647,13 +1644,8 @@ def graph_capture( the graph capture is running on a separate stream from the default stream, in order to explicitly distinguish the kernels to capture from other kernels possibly launched on background in the default stream. - - A caller may pass an explicit ``graph_capture_context`` to control the - stream used (e.g. to capture on the default stream). """ - context = graph_capture_context or GraphCaptureContext( - torch.cuda.Stream(device=device) - ) + context = GraphCaptureContext(torch.cuda.Stream(device=device)) with ( get_tp_group().graph_capture(context), get_pp_group().graph_capture(context), diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 9474ec4835..248f41d20d 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -865,25 +865,8 @@ class RocmPlatform(Platform): @classmethod def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None: from vllm._aiter_ops import rocm_aiter_ops - from vllm.config.compilation import CUDAGraphMode compilation_config = vllm_config.compilation_config - model_config = vllm_config.model_config - if ( - compilation_config.cudagraph_mode is None - and model_config is not None - and on_gfx950() - and vllm_config.use_v2_model_runner - and model_config.architecture - in { - "DeepseekV4ForCausalLM", - "DeepseekV4ForConditionalGeneration", - } - ): - # Default to eager after reported gfx950/MRV2 accuracy regressions: - # https://github.com/vllm-project/vllm/issues/52644 - compilation_config.cudagraph_mode = CUDAGraphMode.NONE - use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled() use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled() use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 5c85257fa3..10f85be718 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -46,7 +46,6 @@ from vllm.distributed.kv_transfer.kv_connector.utils import ( copy_kv_blocks, ) from vllm.distributed.parallel_state import ( - GraphCaptureContext, get_dcp_group, get_pp_group, get_tp_group, @@ -128,7 +127,6 @@ from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import ( PIN_MEMORY, async_tensor_h2d, - current_stream, kv_cache_dtype_str_to_dtype, ) from vllm.v1.attention.backend import ( @@ -6712,31 +6710,11 @@ class GPUModelRunner( per_graph_estimate = {} encoder_memory_estimate = 0 - # On ROCm, capture these throwaway profiling graphs on vLLM's dedicated - # compute stream instead of the fresh side stream graph_capture() - # allocates by default. torch's allocator pools free blocks per stream, - # so a side-stream forward strands a persistent aiter scratch buffer in - # a separate pool, shifting the physical placement of the real KV cache - # allocated afterward and slowing bandwidth-bound decode ~20%. The - # graphs are discarded, so a side stream is unnecessary here. - # Use current_stream(), not torch.cuda.current_stream(): before vLLM - # initializes its dedicated stream, torch returns the per-thread default - # stream (cuda_stream=0), which cannot be used for cudagraph capture. - # cap_ctx=None keeps the side-stream path on CUDA. - cap_ctx = ( - GraphCaptureContext(current_stream()) - if current_platform.is_rocm() - else None - ) - # Cleanup-only guard: CUDA graph capture errors should still propagate # because encoder graph capture is opt-in. try: set_cudagraph_capturing_enabled(True) - with ( - self._freeze_gc(), - graph_capture(device=self.device, graph_capture_context=cap_ctx), - ): + with self._freeze_gc(), graph_capture(device=self.device): torch.accelerator.synchronize() torch.accelerator.empty_cache() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index acba1d9b67..a288a97cc1 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -98,6 +98,7 @@ from vllm.v1.worker.worker_base import CompilationTimes, WorkerBase from vllm.v1.worker.workspace import init_workspace_manager from ...model_executor.model_loader import TensorizerLoader +from .gpu.cudagraph_utils import has_compiled_submodule from .gpu.warmup import warmup_kernels from .utils import request_memory @@ -772,7 +773,10 @@ class Worker(WorkerBase): def compile_or_warm_up_model(self) -> CompilationTimes: warmup_sizes: list[int] = [] - if self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE: + if ( + self.vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE + and has_compiled_submodule(self.model_runner.get_model()) + ): # warm up sizes that are not in cudagraph capture sizes, # but users still want to compile for better performance, # e.g. for the max-num-batched token size in chunked prefill.