[Model Runner V2] Use MRV2 for all models by default (#53183)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-27 11:59:34 -07:00
committed by GitHub
parent a18dbe49ad
commit 4aab2b0ebe
5 changed files with 122 additions and 117 deletions
@@ -487,6 +487,7 @@ def test_models_fse_init(
quantization_config=None,
runner_type="generate",
is_moe=True,
logits_processors=None,
)
vllm_config.parallel_config.enable_expert_parallel = False
if model_type == "deepseek_v4":
@@ -545,6 +546,8 @@ def test_models_fse_init(
vllm_config.speculative_config = SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config),
method="mtp",
parallel_drafting=False,
enable_adaptive_verification=False,
)
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
+54 -31
View File
@@ -132,25 +132,30 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
"""ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths."""
from vllm.config.vllm import (
ROCM_DEFAULT_MRV1_ARCHITECTURES,
default_breakable_cudagraph_architectures,
default_v2_model_runner_architectures,
)
from vllm.platforms import current_platform
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
# The lookup is lru_cached against a fixed platform.
default_v2_model_runner_architectures.cache_clear()
default_breakable_cudagraph_architectures.cache_clear()
try:
v2_architectures = default_v2_model_runner_architectures()
breakable_architectures = default_breakable_cudagraph_architectures()
assert "DeepseekV32ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES
assert "DeepseekV4ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES
assert "DeepseekV32ForCausalLM" not in v2_architectures
assert "DeepseekV4ForCausalLM" not in v2_architectures
breakable_architectures = default_breakable_cudagraph_architectures()
assert "DeepseekV32ForCausalLM" not in breakable_architectures
assert "DeepseekV32MTPModel" not in breakable_architectures
# The carve-out takes effect via the runner-selection property
# (warning_once args must be hashable for its lru_cache).
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
config = SimpleNamespace(
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
)
assert VllmConfig.use_v2_model_runner.fget(config) is False
finally:
default_v2_model_runner_architectures.cache_clear()
default_breakable_cudagraph_architectures.cache_clear()
@@ -169,17 +174,13 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
from vllm.compilation.breakable_cudagraph import (
is_breakable_cudagraph_enabled,
)
from vllm.config.vllm import (
default_breakable_cudagraph_architectures,
default_v2_model_runner_architectures,
)
from vllm.config.vllm import default_breakable_cudagraph_architectures
from vllm.platforms import current_platform
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
default_v2_model_runner_architectures.cache_clear()
default_breakable_cudagraph_architectures.cache_clear()
model_config = SimpleNamespace(
@@ -200,10 +201,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
),
)
config._dflash_needs_multi_kv_group = lambda: False
config._is_dflash2_draft = lambda: False
config._is_default_v2_model_runner_model = lambda: (
VllmConfig._is_default_v2_model_runner_model(config)
)
config._get_v2_model_runner_unsupported_features = lambda: []
config._uses_breakable_cudagraph_by_default = lambda: (
VllmConfig._uses_breakable_cudagraph_by_default(config)
@@ -217,7 +214,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
finally:
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
default_v2_model_runner_architectures.cache_clear()
default_breakable_cudagraph_architectures.cache_clear()
@@ -387,7 +383,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
[
(
SimpleNamespace(
model="Qwen/Qwen3-1.7B-Base",
model="Qwen/Qwen3-32B",
architectures=["Qwen3ForCausalLM"],
runner_type="generate",
is_moe=False,
@@ -397,8 +393,8 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
),
(
SimpleNamespace(
model="Qwen/Qwen3-32B",
architectures=["Qwen3ForCausalLM"],
model="Qwen/Qwen2-7B-Instruct",
architectures=["Qwen2ForCausalLM"],
runner_type="generate",
is_moe=False,
is_quantized=False,
@@ -465,6 +461,16 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
),
True,
),
(
SimpleNamespace(
model="deepseek-ai/DeepSeek-V3",
architectures=["DeepseekV3ForCausalLM"],
runner_type="generate",
is_moe=True,
is_quantized=False,
),
True,
),
(
SimpleNamespace(
model="deepseek-ai/DeepSeek-V4-Flash",
@@ -533,7 +539,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
is_moe=True,
is_quantized=False,
),
False,
True,
),
(
SimpleNamespace(
@@ -554,7 +560,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
is_quantized=False,
is_hybrid=True,
),
False,
True,
),
(
SimpleNamespace(
@@ -565,7 +571,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
is_quantized=False,
is_attention_free=True,
),
False,
True,
),
(
SimpleNamespace(
@@ -602,20 +608,37 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
),
],
)
def test_is_default_v2_model_runner_model(model_config, expected, monkeypatch):
from vllm.config.vllm import default_v2_model_runner_architectures
def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch):
from vllm.platforms import current_platform
# The expectations below are the platform-independent defaults; ROCm's
# DeepSeek V4 carve-out is covered by test_rocm_defaults_deepseek_v4_to_mrv1.
# DeepSeek carve-out is covered by test_rocm_keeps_compiled_deepseek_defaults.
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
default_v2_model_runner_architectures.cache_clear()
config = SimpleNamespace(model_config=model_config)
config._get_v2_model_runner_unsupported_features = lambda: []
try:
assert VllmConfig._is_default_v2_model_runner_model(config) is expected
finally:
default_v2_model_runner_architectures.cache_clear()
assert VllmConfig.use_v2_model_runner.fget(config) is expected
def test_v1_model_runner_rejects_v2_only_features():
config = SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=2,
enable_batch_sharded_sampling=False,
),
speculative_config=None,
model_config=None,
)
config._dflash_needs_multi_kv_group = lambda: False
config._is_dflash2_draft = lambda: False
config._get_v1_model_runner_unsupported_features = lambda: (
VllmConfig._get_v1_model_runner_unsupported_features(config)
)
with pytest.raises(ValueError, match="prefill context parallel"):
VllmConfig._validate_v1_model_runner(config)
@pytest.mark.skip_global_cleanup
+9
View File
@@ -591,6 +591,15 @@ def _support_torch_compile(
**kwargs,
)
# Tensors that enter the graph through the forward context rather than
# model args (e.g. the MoE padding mask) need their batch dim marked
# dynamic too, otherwise AOT-compiled artifacts bake a static guard
# for the compile-time batch size.
if is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
torch._dynamo.mark_dynamic(is_padding, 0)
original_code_object = self.original_code_object()
logger.debug("Start compiling function %s", original_code_object)
+52 -85
View File
@@ -66,19 +66,10 @@ else:
logger = init_logger(__name__)
DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
{
"DeepseekV2ForCausalLM",
"DeepseekV32ForCausalLM",
"DeepseekV4ForCausalLM",
"GlmMoeDsaForCausalLM",
"GraniteMoeForCausalLM",
"InklingForCausalLM",
"InklingForConditionalGeneration",
"KimiK3ForConditionalGeneration",
"LongcatFlashNgramForCausalLM",
"Qwen2MoeForCausalLM",
}
# TODO(rocm): These models are either unsupported by MRV2 or slower with
# MRV2 on AMD GPUs.
ROCM_DEFAULT_MRV1_ARCHITECTURES = frozenset(
{"DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM"}
)
DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
@@ -99,21 +90,6 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
)
@lru_cache
def default_v2_model_runner_architectures() -> frozenset[str]:
"""Architectures defaulting to the V2 model runner on this platform."""
from vllm.platforms import current_platform
if current_platform.is_rocm():
# TODO(rocm): These models are either unsupported by MRV2 or slower with
# MRV2 on AMD GPUs.
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {
"DeepseekV32ForCausalLM",
"DeepseekV4ForCausalLM",
}
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES
@lru_cache
def default_breakable_cudagraph_architectures() -> frozenset[str]:
"""Architectures defaulting to breakable CUDA graphs on this platform."""
@@ -646,36 +622,17 @@ class VllmConfig:
if use_v2_model_runner is not None:
return use_v2_model_runner
# PCP runtime support is implemented only by the V2 model runner.
if self.parallel_config.prefill_context_parallel_size > 1:
return True
from vllm.platforms import current_platform
# DSpark is implemented only by the V2 GPU model runner, and DeepSeek-V4
# is not otherwise a default-V2 architecture, so force V2 for it. If V2
# is unsupported for the rest of the config, _validate_v2_model_runner
# raises rather than silently falling back to V1 (which can't run dspark).
if (
self.speculative_config is not None
and self.speculative_config.method == "dspark"
):
return True
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only);
# force V2 as for dspark, since a hybrid target otherwise defaults to V1.
if self._dflash_needs_multi_kv_group():
return True
# The DFlash2 candidate selector exists only in the V2 speculator. On V1
# the same checkpoint drafts through DFlashProposer, which never calls
# it, so the draft degrades to DFlash1 silently. Force V2 as for dspark.
if self._is_dflash2_draft():
return True
if self.model_config is not None and self.model_config.is_diffusion:
return True
if not self._is_default_v2_model_runner_model():
return False
model_config = self.model_config
if model_config is not None and current_platform.is_rocm():
architectures = getattr(model_config, "architectures", ())
if any(arch in ROCM_DEFAULT_MRV1_ARCHITECTURES for arch in architectures):
logger.warning_once(
"Defaulting to V1 model runner on ROCm for model architectures: %s",
", ".join(architectures),
)
return False
if not HAS_TRITON:
logger.warning_once(
@@ -717,26 +674,6 @@ class VllmConfig:
num_sliding = sum(lt == "sliding_attention" for lt in layer_types)
return 0 < num_sliding < len(layer_types)
def _is_default_v2_model_runner_model(self) -> bool:
model_config = self.model_config
if model_config is None:
return False
architectures = getattr(model_config, "architectures", [])
default_architectures = default_v2_model_runner_architectures()
is_default_v2_architecture = any(
arch in default_architectures for arch in architectures
)
if getattr(model_config, "is_hybrid", False) and (
not is_default_v2_architecture
):
return False
if getattr(model_config, "is_attention_free", False):
return False
return is_default_v2_architecture or not model_config.is_moe
def _uses_breakable_cudagraph_by_default(self) -> bool:
model_config = self.model_config
if model_config is None:
@@ -1666,11 +1603,8 @@ class VllmConfig:
if self.use_v2_model_runner:
self._validate_v2_model_runner()
elif self.parallel_config.prefill_context_parallel_size > 1:
raise ValueError(
"Prefill context parallelism requires Model Runner V2. "
"Remove VLLM_USE_V2_MODEL_RUNNER=0."
)
else:
self._validate_v1_model_runner()
self._validate_batch_sharded_sampling()
@@ -2561,6 +2495,35 @@ class VllmConfig:
return unsupported
def _get_v1_model_runner_unsupported_features(self) -> list[str]:
unsupported: list[str] = []
# PCP runtime support is implemented only by the V2 model runner.
if self.parallel_config.prefill_context_parallel_size > 1:
unsupported.append("prefill context parallel")
# DSpark is implemented only by the V2 GPU model runner.
if self.speculative_config and self.speculative_config.method == "dspark":
unsupported.append("dspark speculative decoding")
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only).
if self._dflash_needs_multi_kv_group():
unsupported.append("mixed sliding/full dflash drafts")
# The DFlash2 candidate selector exists only in the V2 speculator. On
# V1 the same checkpoint drafts through DFlashProposer, which never
# calls it, so the draft would degrade to DFlash1 silently.
if self._is_dflash2_draft():
unsupported.append("dflash2 drafts")
if self.model_config is not None and self.model_config.is_diffusion:
unsupported.append("diffusion models")
if self.parallel_config.enable_batch_sharded_sampling:
unsupported.append("batch-sharded sampling")
return unsupported
def _validate_batch_sharded_sampling(self) -> None:
"""Validate `enable_batch_sharded_sampling` against the rest of the config."""
if not self.parallel_config.enable_batch_sharded_sampling:
@@ -2571,9 +2534,6 @@ class VllmConfig:
blockers: list[str] = []
tp_size = self.parallel_config.tensor_parallel_size
if not self.use_v2_model_runner:
blockers.append("it is only implemented by Model Runner V2")
if tp_size <= 1:
blockers.append("tensor_parallel_size is 1, so there is nothing to shard")
elif self.scheduler_config.max_num_seqs < tp_size:
@@ -2623,6 +2583,13 @@ class VllmConfig:
f"Model Runner V2 does not yet support: {', '.join(unsupported)}"
)
def _validate_v1_model_runner(self) -> None:
unsupported = self._get_v1_model_runner_unsupported_features()
if unsupported:
raise ValueError(
f"Model Runner V1 does not support: {', '.join(unsupported)}"
)
def validate_block_size(self) -> None:
"""Validate block_size against DCP and mamba constraints.
+4 -1
View File
@@ -17,6 +17,7 @@ from typing_extensions import runtime_checkable
from vllm.config import CacheConfig, VllmConfig, get_layers_from_vllm_config
from vllm.config.cache import _layout_from_name
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec
@@ -475,7 +476,9 @@ def make_local_attention_virtual_batches(
block_size: int = 0,
) -> tuple[CommonAttentionMetadata, Callable[[torch.Tensor], torch.Tensor]]:
query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy()
seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()
with gpu_sync_allowed():
# TODO see https://github.com/vllm-project/vllm/pull/31852
seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()
block_table = common_attn_metadata.block_table_tensor
device = common_attn_metadata.query_start_loc.device