From 4aab2b0ebed20343efe543c633f71b3c1336d5b8 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 27 Aug 2026 11:59:34 -0700 Subject: [PATCH] [Model Runner V2] Use MRV2 for all models by default (#53183) Signed-off-by: Nick Hill --- .../layers/test_fused_shared_expert.py | 3 + tests/test_config.py | 85 +++++++---- vllm/compilation/decorators.py | 9 ++ vllm/config/vllm.py | 137 +++++++----------- vllm/v1/attention/backends/utils.py | 5 +- 5 files changed, 122 insertions(+), 117 deletions(-) diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 3ec77d36ff..eac71564a2 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -487,6 +487,7 @@ def test_models_fse_init( quantization_config=None, runner_type="generate", is_moe=True, + logits_processors=None, ) vllm_config.parallel_config.enable_expert_parallel = False if model_type == "deepseek_v4": @@ -545,6 +546,8 @@ def test_models_fse_init( vllm_config.speculative_config = SimpleNamespace( draft_model_config=SimpleNamespace(hf_config=config), method="mtp", + parallel_drafting=False, + enable_adaptive_verification=False, ) mtp = DeepSeekV4MTP(vllm_config=vllm_config) assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude) diff --git a/tests/test_config.py b/tests/test_config.py index 0f01cf4541..df77d33e0c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -132,25 +132,30 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): """ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths.""" from vllm.config.vllm import ( + ROCM_DEFAULT_MRV1_ARCHITECTURES, default_breakable_cudagraph_architectures, - default_v2_model_runner_architectures, ) from vllm.platforms import current_platform monkeypatch.setattr(current_platform, "is_rocm", lambda: True) # The lookup is lru_cached against a fixed platform. - default_v2_model_runner_architectures.cache_clear() default_breakable_cudagraph_architectures.cache_clear() try: - v2_architectures = default_v2_model_runner_architectures() - breakable_architectures = default_breakable_cudagraph_architectures() + assert "DeepseekV32ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES + assert "DeepseekV4ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES - assert "DeepseekV32ForCausalLM" not in v2_architectures - assert "DeepseekV4ForCausalLM" not in v2_architectures + breakable_architectures = default_breakable_cudagraph_architectures() assert "DeepseekV32ForCausalLM" not in breakable_architectures assert "DeepseekV32MTPModel" not in breakable_architectures + + # The carve-out takes effect via the runner-selection property + # (warning_once args must be hashable for its lru_cache). + monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) + config = SimpleNamespace( + model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]) + ) + assert VllmConfig.use_v2_model_runner.fget(config) is False finally: - default_v2_model_runner_architectures.cache_clear() default_breakable_cudagraph_architectures.cache_clear() @@ -169,17 +174,13 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, ) - from vllm.config.vllm import ( - default_breakable_cudagraph_architectures, - default_v2_model_runner_architectures, - ) + from vllm.config.vllm import default_breakable_cudagraph_architectures from vllm.platforms import current_platform monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) monkeypatch.setattr(current_platform, "is_rocm", lambda: False) - default_v2_model_runner_architectures.cache_clear() default_breakable_cudagraph_architectures.cache_clear() model_config = SimpleNamespace( @@ -200,10 +201,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( ), ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False - config._is_default_v2_model_runner_model = lambda: ( - VllmConfig._is_default_v2_model_runner_model(config) - ) config._get_v2_model_runner_unsupported_features = lambda: [] config._uses_breakable_cudagraph_by_default = lambda: ( VllmConfig._uses_breakable_cudagraph_by_default(config) @@ -217,7 +214,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() finally: os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) - default_v2_model_runner_architectures.cache_clear() default_breakable_cudagraph_architectures.cache_clear() @@ -387,7 +383,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): [ ( SimpleNamespace( - model="Qwen/Qwen3-1.7B-Base", + model="Qwen/Qwen3-32B", architectures=["Qwen3ForCausalLM"], runner_type="generate", is_moe=False, @@ -397,8 +393,8 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): ), ( SimpleNamespace( - model="Qwen/Qwen3-32B", - architectures=["Qwen3ForCausalLM"], + model="Qwen/Qwen2-7B-Instruct", + architectures=["Qwen2ForCausalLM"], runner_type="generate", is_moe=False, is_quantized=False, @@ -465,6 +461,16 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): ), True, ), + ( + SimpleNamespace( + model="deepseek-ai/DeepSeek-V3", + architectures=["DeepseekV3ForCausalLM"], + runner_type="generate", + is_moe=True, + is_quantized=False, + ), + True, + ), ( SimpleNamespace( model="deepseek-ai/DeepSeek-V4-Flash", @@ -533,7 +539,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): is_moe=True, is_quantized=False, ), - False, + True, ), ( SimpleNamespace( @@ -554,7 +560,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): is_quantized=False, is_hybrid=True, ), - False, + True, ), ( SimpleNamespace( @@ -565,7 +571,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): is_quantized=False, is_attention_free=True, ), - False, + True, ), ( SimpleNamespace( @@ -602,20 +608,37 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): ), ], ) -def test_is_default_v2_model_runner_model(model_config, expected, monkeypatch): - from vllm.config.vllm import default_v2_model_runner_architectures +def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch): from vllm.platforms import current_platform # The expectations below are the platform-independent defaults; ROCm's - # DeepSeek V4 carve-out is covered by test_rocm_defaults_deepseek_v4_to_mrv1. + # DeepSeek carve-out is covered by test_rocm_keeps_compiled_deepseek_defaults. + monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) + monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) monkeypatch.setattr(current_platform, "is_rocm", lambda: False) - default_v2_model_runner_architectures.cache_clear() config = SimpleNamespace(model_config=model_config) + config._get_v2_model_runner_unsupported_features = lambda: [] - try: - assert VllmConfig._is_default_v2_model_runner_model(config) is expected - finally: - default_v2_model_runner_architectures.cache_clear() + assert VllmConfig.use_v2_model_runner.fget(config) is expected + + +def test_v1_model_runner_rejects_v2_only_features(): + config = SimpleNamespace( + parallel_config=SimpleNamespace( + prefill_context_parallel_size=2, + enable_batch_sharded_sampling=False, + ), + speculative_config=None, + model_config=None, + ) + config._dflash_needs_multi_kv_group = lambda: False + config._is_dflash2_draft = lambda: False + config._get_v1_model_runner_unsupported_features = lambda: ( + VllmConfig._get_v1_model_runner_unsupported_features(config) + ) + + with pytest.raises(ValueError, match="prefill context parallel"): + VllmConfig._validate_v1_model_runner(config) @pytest.mark.skip_global_cleanup diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index ceb3ad3170..d5a2ac7e68 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -591,6 +591,15 @@ def _support_torch_compile( **kwargs, ) + # Tensors that enter the graph through the forward context rather than + # model args (e.g. the MoE padding mask) need their batch dim marked + # dynamic too, otherwise AOT-compiled artifacts bake a static guard + # for the compile-time batch size. + if is_forward_context_available(): + is_padding = get_forward_context().is_padding + if is_padding is not None: + torch._dynamo.mark_dynamic(is_padding, 0) + original_code_object = self.original_code_object() logger.debug("Start compiling function %s", original_code_object) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 51b8b1019c..e88611ab0c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -66,19 +66,10 @@ else: logger = init_logger(__name__) -DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset( - { - "DeepseekV2ForCausalLM", - "DeepseekV32ForCausalLM", - "DeepseekV4ForCausalLM", - "GlmMoeDsaForCausalLM", - "GraniteMoeForCausalLM", - "InklingForCausalLM", - "InklingForConditionalGeneration", - "KimiK3ForConditionalGeneration", - "LongcatFlashNgramForCausalLM", - "Qwen2MoeForCausalLM", - } +# TODO(rocm): These models are either unsupported by MRV2 or slower with +# MRV2 on AMD GPUs. +ROCM_DEFAULT_MRV1_ARCHITECTURES = frozenset( + {"DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM"} ) DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( @@ -99,21 +90,6 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( ) -@lru_cache -def default_v2_model_runner_architectures() -> frozenset[str]: - """Architectures defaulting to the V2 model runner on this platform.""" - from vllm.platforms import current_platform - - if current_platform.is_rocm(): - # TODO(rocm): These models are either unsupported by MRV2 or slower with - # MRV2 on AMD GPUs. - return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - { - "DeepseekV32ForCausalLM", - "DeepseekV4ForCausalLM", - } - return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - - @lru_cache def default_breakable_cudagraph_architectures() -> frozenset[str]: """Architectures defaulting to breakable CUDA graphs on this platform.""" @@ -646,36 +622,17 @@ class VllmConfig: if use_v2_model_runner is not None: return use_v2_model_runner - # PCP runtime support is implemented only by the V2 model runner. - if self.parallel_config.prefill_context_parallel_size > 1: - return True + from vllm.platforms import current_platform - # DSpark is implemented only by the V2 GPU model runner, and DeepSeek-V4 - # is not otherwise a default-V2 architecture, so force V2 for it. If V2 - # is unsupported for the rest of the config, _validate_v2_model_runner - # raises rather than silently falling back to V1 (which can't run dspark). - if ( - self.speculative_config is not None - and self.speculative_config.method == "dspark" - ): - return True - - # Mixed sliding/full DFlash drafts need multiple KV groups (V2 only); - # force V2 as for dspark, since a hybrid target otherwise defaults to V1. - if self._dflash_needs_multi_kv_group(): - return True - - # The DFlash2 candidate selector exists only in the V2 speculator. On V1 - # the same checkpoint drafts through DFlashProposer, which never calls - # it, so the draft degrades to DFlash1 silently. Force V2 as for dspark. - if self._is_dflash2_draft(): - return True - - if self.model_config is not None and self.model_config.is_diffusion: - return True - - if not self._is_default_v2_model_runner_model(): - return False + model_config = self.model_config + if model_config is not None and current_platform.is_rocm(): + architectures = getattr(model_config, "architectures", ()) + if any(arch in ROCM_DEFAULT_MRV1_ARCHITECTURES for arch in architectures): + logger.warning_once( + "Defaulting to V1 model runner on ROCm for model architectures: %s", + ", ".join(architectures), + ) + return False if not HAS_TRITON: logger.warning_once( @@ -717,26 +674,6 @@ class VllmConfig: num_sliding = sum(lt == "sliding_attention" for lt in layer_types) return 0 < num_sliding < len(layer_types) - def _is_default_v2_model_runner_model(self) -> bool: - model_config = self.model_config - if model_config is None: - return False - - architectures = getattr(model_config, "architectures", []) - default_architectures = default_v2_model_runner_architectures() - is_default_v2_architecture = any( - arch in default_architectures for arch in architectures - ) - - if getattr(model_config, "is_hybrid", False) and ( - not is_default_v2_architecture - ): - return False - - if getattr(model_config, "is_attention_free", False): - return False - return is_default_v2_architecture or not model_config.is_moe - def _uses_breakable_cudagraph_by_default(self) -> bool: model_config = self.model_config if model_config is None: @@ -1666,11 +1603,8 @@ class VllmConfig: if self.use_v2_model_runner: self._validate_v2_model_runner() - elif self.parallel_config.prefill_context_parallel_size > 1: - raise ValueError( - "Prefill context parallelism requires Model Runner V2. " - "Remove VLLM_USE_V2_MODEL_RUNNER=0." - ) + else: + self._validate_v1_model_runner() self._validate_batch_sharded_sampling() @@ -2561,6 +2495,35 @@ class VllmConfig: return unsupported + def _get_v1_model_runner_unsupported_features(self) -> list[str]: + unsupported: list[str] = [] + + # PCP runtime support is implemented only by the V2 model runner. + if self.parallel_config.prefill_context_parallel_size > 1: + unsupported.append("prefill context parallel") + + # DSpark is implemented only by the V2 GPU model runner. + if self.speculative_config and self.speculative_config.method == "dspark": + unsupported.append("dspark speculative decoding") + + # Mixed sliding/full DFlash drafts need multiple KV groups (V2 only). + if self._dflash_needs_multi_kv_group(): + unsupported.append("mixed sliding/full dflash drafts") + + # The DFlash2 candidate selector exists only in the V2 speculator. On + # V1 the same checkpoint drafts through DFlashProposer, which never + # calls it, so the draft would degrade to DFlash1 silently. + if self._is_dflash2_draft(): + unsupported.append("dflash2 drafts") + + if self.model_config is not None and self.model_config.is_diffusion: + unsupported.append("diffusion models") + + if self.parallel_config.enable_batch_sharded_sampling: + unsupported.append("batch-sharded sampling") + + return unsupported + def _validate_batch_sharded_sampling(self) -> None: """Validate `enable_batch_sharded_sampling` against the rest of the config.""" if not self.parallel_config.enable_batch_sharded_sampling: @@ -2571,9 +2534,6 @@ class VllmConfig: blockers: list[str] = [] tp_size = self.parallel_config.tensor_parallel_size - if not self.use_v2_model_runner: - blockers.append("it is only implemented by Model Runner V2") - if tp_size <= 1: blockers.append("tensor_parallel_size is 1, so there is nothing to shard") elif self.scheduler_config.max_num_seqs < tp_size: @@ -2623,6 +2583,13 @@ class VllmConfig: f"Model Runner V2 does not yet support: {', '.join(unsupported)}" ) + def _validate_v1_model_runner(self) -> None: + unsupported = self._get_v1_model_runner_unsupported_features() + if unsupported: + raise ValueError( + f"Model Runner V1 does not support: {', '.join(unsupported)}" + ) + def validate_block_size(self) -> None: """Validate block_size against DCP and mamba constraints. diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 29f1ccdfd0..c9596e1c9c 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -17,6 +17,7 @@ from typing_extensions import runtime_checkable from vllm.config import CacheConfig, VllmConfig, get_layers_from_vllm_config from vllm.config.cache import _layout_from_name +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec @@ -475,7 +476,9 @@ def make_local_attention_virtual_batches( block_size: int = 0, ) -> tuple[CommonAttentionMetadata, Callable[[torch.Tensor], torch.Tensor]]: query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy() - seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy() + with gpu_sync_allowed(): + # TODO see https://github.com/vllm-project/vllm/pull/31852 + seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy() block_table = common_attn_metadata.block_table_tensor device = common_attn_metadata.query_start_loc.device