diff --git a/tests/test_config.py b/tests/test_config.py index 96ca8f36e4..827dbcd4f8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -273,6 +273,44 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): default_breakable_cudagraph_architectures.cache_clear() +@pytest.mark.parametrize( + ("architecture", "use_v2", "mode", "expected"), + [ + ("DeepseekV4ForCausalLM", True, None, CUDAGraphMode.NONE), + ("DeepseekV4ForConditionalGeneration", True, None, CUDAGraphMode.NONE), + ("DeepseekV4ForCausalLM", False, None, None), + ("LlamaForCausalLM", True, None, None), + ( + "DeepseekV4ForCausalLM", + True, + CUDAGraphMode.FULL_DECODE_ONLY, + CUDAGraphMode.FULL_DECODE_ONLY, + ), + ], +) +def test_rocm_gfx950_deepseek_v4_cudagraph_default( + monkeypatch, architecture, use_v2, mode, expected +): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.platforms import rocm + + monkeypatch.setattr(rocm, "on_gfx950", lambda: True) + monkeypatch.setattr(rocm_aiter_ops, "is_fused_moe_enabled", lambda: False) + monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: False) + monkeypatch.setattr( + rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", lambda: False + ) + config = SimpleNamespace( + compilation_config=CompilationConfig(cudagraph_mode=mode), + model_config=SimpleNamespace(architecture=architecture), + use_v2_model_runner=use_v2, + ) + + rocm.RocmPlatform.apply_config_platform_defaults(config) + + assert config.compilation_config.cudagraph_mode == expected + + @pytest.mark.parametrize( ("model", "architecture"), [ @@ -463,6 +501,66 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes +@pytest.mark.parametrize( + ("mode", "piecewise_capture_available", "attention_support", "expected"), + [ + ("PIECEWISE", False, "ALWAYS", "NONE"), + ("FULL_AND_PIECEWISE", False, "ALWAYS", "FULL_DECODE_ONLY"), + ("FULL_DECODE_ONLY", False, "ALWAYS", "FULL_DECODE_ONLY"), + ("FULL_DECODE_ONLY", False, "NEVER", "NONE"), + ], +) +def test_resolve_cudagraph_mode_uses_loaded_piecewise_provider( + mode, piecewise_capture_available, attention_support, expected +): + compilation_config = CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + cudagraph_mode=CUDAGraphMode[mode], + use_inductor_graph_partition=True, + ) + + resolved = compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport[attention_support], + "FakeAttentionBackend", + piecewise_capture_available=piecewise_capture_available, + ) + + assert resolved.name == expected + assert compilation_config.cudagraph_mode == resolved + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="Requires CUDA graph support" +) +@pytest.mark.parametrize( + "engine_kwargs", + [ + {"runner": "pooling", "convert": "embed"}, + pytest.param( + {"prefill_context_parallel_size": 2, "tensor_parallel_size": 2}, + marks=pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm PCP graph restriction" + ), + ), + ], +) +def test_late_piecewise_restrictions_without_compilation(monkeypatch, engine_kwargs): + """Late compatibility overrides must not restore unavailable piecewise graphs.""" + from vllm.engine.arg_utils import EngineArgs + + monkeypatch.setenv("VLLM_USE_BREAKABLE_CUDAGRAPH", "0") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + config = EngineArgs( + model="facebook/opt-125m", + compilation_config=CompilationConfig(mode=CompilationMode.NONE), + **engine_kwargs, + ).create_engine_config() + + assert config.compilation_config.cudagraph_mode == CUDAGraphMode.NONE + assert config.compilation_config.cudagraph_capture_sizes == [] + assert config.compilation_config.max_cudagraph_capture_size == 0 + + def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): """Cudagraph memory profiling uses a minimal KV cache, so the Mamba block-count guard must only fire for the real cache sizing.""" @@ -498,6 +596,30 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE +@pytest.mark.parametrize( + ("graph_mode", "should_raise"), + [ + (CUDAGraphMode.FULL_DECODE_ONLY, False), + (CUDAGraphMode.FULL_AND_PIECEWISE, False), + (CUDAGraphMode.NONE, True), + (CUDAGraphMode.PIECEWISE, True), + ], +) +def test_adaptive_verification_requires_full_cudagraphs(graph_mode, should_raise): + config = SimpleNamespace( + speculative_config=SimpleNamespace(enable_adaptive_verification=True), + lora_config=None, + compilation_config=CompilationConfig(cudagraph_mode=graph_mode), + parallel_config=SimpleNamespace(pipeline_parallel_size=1), + ) + + if should_raise: + with pytest.raises(ValueError, match="requires full CUDA graphs"): + VllmConfig._validate_adaptive_verification(config) + else: + VllmConfig._validate_adaptive_verification(config) + + @pytest.mark.parametrize( ("model_config", "expected"), [ diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index d92f78a715..b14fc126bb 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -4,7 +4,9 @@ from types import SimpleNamespace import numpy as np +import pytest +from vllm.config.compilation import CUDAGraphMode from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.worker.gpu.async_utils import StepTimingSample from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo @@ -12,6 +14,7 @@ from vllm.v1.worker.gpu.spec_decode import adaptive_verification as adaptive_mod from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, maybe_create_adaptive_verification_manager, + resolve_adaptive_cudagraph_mode, ) from vllm.v1.worker.gpu.structured_outputs import _build_grammar_mapping @@ -35,6 +38,25 @@ def make_manager( return manager +@pytest.mark.parametrize( + ("mode", "piecewise_capture_available", "expected"), + [ + ("FULL_DECODE_ONLY", True, "FULL_DECODE_ONLY"), + ("FULL", True, "FULL_AND_PIECEWISE"), + ("FULL", False, "FULL_DECODE_ONLY"), + ("FULL_AND_PIECEWISE", True, "FULL_AND_PIECEWISE"), + ("FULL_AND_PIECEWISE", False, "FULL_DECODE_ONLY"), + ], +) +def test_resolve_adaptive_cudagraph_mode(mode, piecewise_capture_available, expected): + assert ( + resolve_adaptive_cudagraph_mode( + CUDAGraphMode[mode], piecewise_capture_available=piecewise_capture_available + ) + == CUDAGraphMode[expected] + ) + + def test_manager_scopes_varlen_check_without_weakening_runner_cg_mode(monkeypatch): class Backend: @classmethod diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 1219607d3a..2cd5fae775 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -77,6 +77,13 @@ class CUDAGraphMode(enum.Enum): def requires_piecewise_compilation(self) -> bool: return self.has_mode(CUDAGraphMode.PIECEWISE) + def without_piecewise(self) -> "CUDAGraphMode": + if self == CUDAGraphMode.PIECEWISE: + return CUDAGraphMode.NONE + if self == CUDAGraphMode.FULL_AND_PIECEWISE: + return CUDAGraphMode.FULL_DECODE_ONLY + return self + def max_cudagraph_mode(self) -> "CUDAGraphMode": return CUDAGraphMode(max(self.value)) if self.separate_routine() else self @@ -1382,6 +1389,7 @@ class CompilationConfig: kv_cache_config: "KVCacheConfig | None" = None, max_num_reqs: int | None = None, is_profiling: bool = False, + piecewise_capture_available: bool = True, ) -> CUDAGraphMode: from vllm.v1.attention.backend import AttentionCGSupport @@ -1464,6 +1472,20 @@ class CompilationConfig: cudagraph_mode = CUDAGraphMode.NONE logger.warning(msg) + if ( + not piecewise_capture_available + and cudagraph_mode.requires_piecewise_compilation() + ): + fallback_mode = cudagraph_mode.without_piecewise() + logger.warning_once( + "Cudagraph mode %s requires piecewise capture, but the loaded " + "model provides neither a compiled submodule nor breakable CUDA " + "graphs. Overriding to %s.", + cudagraph_mode, + fallback_mode, + ) + cudagraph_mode = fallback_mode + # double check that we can support full cudagraph if they are requested # even after automatic downgrades if ( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index cc443a14dd..5bd6af20de 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -991,6 +991,28 @@ class VllmConfig: ) speculative_config.num_speculative_tokens_per_batch_size = None + def _normalize_piecewise_cudagraph_mode( + self, *, breakable_cudagraph_enabled: bool + ) -> None: + compilation_config = self.compilation_config + if ( + compilation_config.cudagraph_mode.requires_piecewise_compilation() + and compilation_config.mode != CompilationMode.VLLM_COMPILE + and not breakable_cudagraph_enabled + ): + fallback_mode = compilation_config.cudagraph_mode.without_piecewise() + logger.info_once( + "Cudagraph mode %s is not compatible with compilation mode %s. " + "Overriding to %s.", + compilation_config.cudagraph_mode, + compilation_config.mode, + fallback_mode, + ) + compilation_config.cudagraph_mode = fallback_mode + if fallback_mode == CUDAGraphMode.NONE: + compilation_config.max_cudagraph_capture_size = 0 + compilation_config.cudagraph_capture_sizes = [] + def _post_init_kv_transfer_config(self) -> None: """Update KVTransferConfig based on top-level configs in VllmConfig. @@ -1536,18 +1558,9 @@ class VllmConfig: self._maybe_disable_dynamic_sd_for_data_parallel() self._maybe_override_dynamic_sd_cudagraph_mode() - if ( - self.compilation_config.cudagraph_mode.requires_piecewise_compilation() - and self.compilation_config.mode != CompilationMode.VLLM_COMPILE - and not envs.VLLM_USE_BREAKABLE_CUDAGRAPH - ): - logger.info_once( - "Cudagraph mode %s is not compatible with compilation mode %s." - "Overriding to NONE.", - self.compilation_config.cudagraph_mode, - self.compilation_config.mode, - ) - self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + self._normalize_piecewise_cudagraph_mode( + breakable_cudagraph_enabled=breakable_cudagraph_enabled + ) # async tp is built on top of sequence parallelism and requires it. pass_config = self.compilation_config.pass_config @@ -1716,6 +1729,10 @@ class VllmConfig: ) current_platform.check_and_update_config(self) + self._normalize_piecewise_cudagraph_mode( + breakable_cudagraph_enabled=breakable_cudagraph_enabled + ) + self._resolve_mm_embedding_inputs() self._resolve_mm_processor_device() self._validate_mm_processor_device() @@ -2728,12 +2745,11 @@ class VllmConfig: "Adaptive verification is not currently compatible with LoRA" ) - if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE: - # The draft budget divides by step costs profiled from captured - # cudagraphs; eager execution captures none. + if not self.compilation_config.cudagraph_mode.has_full_cudagraphs(): raise ValueError( - "Adaptive verification is not currently compatible with " - "enforce_eager/cudagraph_mode=none" + "Adaptive verification requires full CUDA graphs. Use cudagraph " + "mode FULL, FULL_DECODE_ONLY, or FULL_AND_PIECEWISE, or disable " + "adaptive verification." ) if self.parallel_config.pipeline_parallel_size > 1: diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index bdaa86c5de..297efc632f 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -880,8 +880,25 @@ class RocmPlatform(Platform): @classmethod def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None: from vllm._aiter_ops import rocm_aiter_ops + from vllm.config.compilation import CUDAGraphMode compilation_config = vllm_config.compilation_config + model_config = vllm_config.model_config + if ( + compilation_config.cudagraph_mode is None + and model_config is not None + and on_gfx950() + and vllm_config.use_v2_model_runner + and model_config.architecture + in { + "DeepseekV4ForCausalLM", + "DeepseekV4ForConditionalGeneration", + } + ): + # Default to eager after reported gfx950/MRV2 accuracy regressions: + # https://github.com/vllm-project/vllm/issues/52644 + compilation_config.cudagraph_mode = CUDAGraphMode.NONE + use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled() use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled() use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 6a3d2e34e9..f41a5a8d3a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -104,6 +104,7 @@ from vllm.v1.worker.gpu.cp_utils import maybe_prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, ModelCudaGraphManager, + has_compiled_submodule, make_cudagraph_stats, ) from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -152,6 +153,7 @@ from vllm.v1.worker.gpu.spec_decode import init_speculator from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, maybe_create_adaptive_verification_manager, + resolve_adaptive_cudagraph_mode, ) from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, @@ -674,8 +676,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.kv_cache_config, use_replayssm=self.cache_config.use_replayssm, ) + piecewise_capture_available = bool( + envs.VLLM_USE_BREAKABLE_CUDAGRAPH or has_compiled_submodule(self.model) + ) if self.adaptive_verification is not None: - self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE + self.compilation_config.cudagraph_mode = resolve_adaptive_cudagraph_mode( + self.compilation_config.cudagraph_mode, + piecewise_capture_available=piecewise_capture_available, + ) cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes( attn_cg_support.min_cg_support, attn_cg_support.min_cg_attn_backend, @@ -685,6 +693,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): kv_cache_config=self.kv_cache_config, max_num_reqs=self.max_num_reqs, is_profiling=is_profiling, + piecewise_capture_available=piecewise_capture_available, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index d3b3743d67..af58b8993b 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -10,6 +10,7 @@ import numpy as np import torch import vllm.envs as envs +from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -33,6 +34,16 @@ if TYPE_CHECKING: from vllm.v1.worker.utils import AttentionGroup +def resolve_adaptive_cudagraph_mode( + mode: CUDAGraphMode, *, piecewise_capture_available: bool +) -> CUDAGraphMode: + """Select a separate decode route for variable-length verification.""" + assert mode.has_full_cudagraphs() + if mode == CUDAGraphMode.FULL_DECODE_ONLY or not piecewise_capture_available: + return CUDAGraphMode.FULL_DECODE_ONLY + return CUDAGraphMode.FULL_AND_PIECEWISE + + def _assign_draft_token_budget( confidence_probs: torch.Tensor, idx_mapping: torch.Tensor,