[Model Runner V2] Use MRV2 for all models by default (#53183)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -487,6 +487,7 @@ def test_models_fse_init(
|
||||
quantization_config=None,
|
||||
runner_type="generate",
|
||||
is_moe=True,
|
||||
logits_processors=None,
|
||||
)
|
||||
vllm_config.parallel_config.enable_expert_parallel = False
|
||||
if model_type == "deepseek_v4":
|
||||
@@ -545,6 +546,8 @@ def test_models_fse_init(
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config),
|
||||
method="mtp",
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=False,
|
||||
)
|
||||
mtp = DeepSeekV4MTP(vllm_config=vllm_config)
|
||||
assert model.is_fused_shared_expert_enabled is (fse_enabled and not exclude)
|
||||
|
||||
+54
-31
@@ -132,25 +132,30 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
"""ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths."""
|
||||
from vllm.config.vllm import (
|
||||
ROCM_DEFAULT_MRV1_ARCHITECTURES,
|
||||
default_breakable_cudagraph_architectures,
|
||||
default_v2_model_runner_architectures,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
# The lookup is lru_cached against a fixed platform.
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
try:
|
||||
v2_architectures = default_v2_model_runner_architectures()
|
||||
breakable_architectures = default_breakable_cudagraph_architectures()
|
||||
assert "DeepseekV32ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES
|
||||
assert "DeepseekV4ForCausalLM" in ROCM_DEFAULT_MRV1_ARCHITECTURES
|
||||
|
||||
assert "DeepseekV32ForCausalLM" not in v2_architectures
|
||||
assert "DeepseekV4ForCausalLM" not in v2_architectures
|
||||
breakable_architectures = default_breakable_cudagraph_architectures()
|
||||
assert "DeepseekV32ForCausalLM" not in breakable_architectures
|
||||
assert "DeepseekV32MTPModel" not in breakable_architectures
|
||||
|
||||
# The carve-out takes effect via the runner-selection property
|
||||
# (warning_once args must be hashable for its lru_cache).
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
|
||||
)
|
||||
assert VllmConfig.use_v2_model_runner.fget(config) is False
|
||||
finally:
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@@ -169,17 +174,13 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
from vllm.config.vllm import (
|
||||
default_breakable_cudagraph_architectures,
|
||||
default_v2_model_runner_architectures,
|
||||
)
|
||||
from vllm.config.vllm import default_breakable_cudagraph_architectures
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
@@ -200,10 +201,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
),
|
||||
)
|
||||
config._dflash_needs_multi_kv_group = lambda: False
|
||||
config._is_dflash2_draft = lambda: False
|
||||
config._is_default_v2_model_runner_model = lambda: (
|
||||
VllmConfig._is_default_v2_model_runner_model(config)
|
||||
)
|
||||
config._get_v2_model_runner_unsupported_features = lambda: []
|
||||
config._uses_breakable_cudagraph_by_default = lambda: (
|
||||
VllmConfig._uses_breakable_cudagraph_by_default(config)
|
||||
@@ -217,7 +214,6 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
|
||||
finally:
|
||||
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@@ -387,7 +383,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
[
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="Qwen/Qwen3-1.7B-Base",
|
||||
model="Qwen/Qwen3-32B",
|
||||
architectures=["Qwen3ForCausalLM"],
|
||||
runner_type="generate",
|
||||
is_moe=False,
|
||||
@@ -397,8 +393,8 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="Qwen/Qwen3-32B",
|
||||
architectures=["Qwen3ForCausalLM"],
|
||||
model="Qwen/Qwen2-7B-Instruct",
|
||||
architectures=["Qwen2ForCausalLM"],
|
||||
runner_type="generate",
|
||||
is_moe=False,
|
||||
is_quantized=False,
|
||||
@@ -465,6 +461,16 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
),
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="deepseek-ai/DeepSeek-V3",
|
||||
architectures=["DeepseekV3ForCausalLM"],
|
||||
runner_type="generate",
|
||||
is_moe=True,
|
||||
is_quantized=False,
|
||||
),
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
model="deepseek-ai/DeepSeek-V4-Flash",
|
||||
@@ -533,7 +539,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
is_moe=True,
|
||||
is_quantized=False,
|
||||
),
|
||||
False,
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
@@ -554,7 +560,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
is_quantized=False,
|
||||
is_hybrid=True,
|
||||
),
|
||||
False,
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
@@ -565,7 +571,7 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
is_quantized=False,
|
||||
is_attention_free=True,
|
||||
),
|
||||
False,
|
||||
True,
|
||||
),
|
||||
(
|
||||
SimpleNamespace(
|
||||
@@ -602,20 +608,37 @@ def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_is_default_v2_model_runner_model(model_config, expected, monkeypatch):
|
||||
from vllm.config.vllm import default_v2_model_runner_architectures
|
||||
def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# The expectations below are the platform-independent defaults; ROCm's
|
||||
# DeepSeek V4 carve-out is covered by test_rocm_defaults_deepseek_v4_to_mrv1.
|
||||
# DeepSeek carve-out is covered by test_rocm_keeps_compiled_deepseek_defaults.
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
config = SimpleNamespace(model_config=model_config)
|
||||
config._get_v2_model_runner_unsupported_features = lambda: []
|
||||
|
||||
try:
|
||||
assert VllmConfig._is_default_v2_model_runner_model(config) is expected
|
||||
finally:
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
assert VllmConfig.use_v2_model_runner.fget(config) is expected
|
||||
|
||||
|
||||
def test_v1_model_runner_rejects_v2_only_features():
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=2,
|
||||
enable_batch_sharded_sampling=False,
|
||||
),
|
||||
speculative_config=None,
|
||||
model_config=None,
|
||||
)
|
||||
config._dflash_needs_multi_kv_group = lambda: False
|
||||
config._is_dflash2_draft = lambda: False
|
||||
config._get_v1_model_runner_unsupported_features = lambda: (
|
||||
VllmConfig._get_v1_model_runner_unsupported_features(config)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="prefill context parallel"):
|
||||
VllmConfig._validate_v1_model_runner(config)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -591,6 +591,15 @@ def _support_torch_compile(
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Tensors that enter the graph through the forward context rather than
|
||||
# model args (e.g. the MoE padding mask) need their batch dim marked
|
||||
# dynamic too, otherwise AOT-compiled artifacts bake a static guard
|
||||
# for the compile-time batch size.
|
||||
if is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
torch._dynamo.mark_dynamic(is_padding, 0)
|
||||
|
||||
original_code_object = self.original_code_object()
|
||||
logger.debug("Start compiling function %s", original_code_object)
|
||||
|
||||
|
||||
+52
-85
@@ -66,19 +66,10 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV2ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"GraniteMoeForCausalLM",
|
||||
"InklingForCausalLM",
|
||||
"InklingForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
"Qwen2MoeForCausalLM",
|
||||
}
|
||||
# TODO(rocm): These models are either unsupported by MRV2 or slower with
|
||||
# MRV2 on AMD GPUs.
|
||||
ROCM_DEFAULT_MRV1_ARCHITECTURES = frozenset(
|
||||
{"DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM"}
|
||||
)
|
||||
|
||||
DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
@@ -99,21 +90,6 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def default_v2_model_runner_architectures() -> frozenset[str]:
|
||||
"""Architectures defaulting to the V2 model runner on this platform."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# TODO(rocm): These models are either unsupported by MRV2 or slower with
|
||||
# MRV2 on AMD GPUs.
|
||||
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
}
|
||||
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES
|
||||
|
||||
|
||||
@lru_cache
|
||||
def default_breakable_cudagraph_architectures() -> frozenset[str]:
|
||||
"""Architectures defaulting to breakable CUDA graphs on this platform."""
|
||||
@@ -646,36 +622,17 @@ class VllmConfig:
|
||||
if use_v2_model_runner is not None:
|
||||
return use_v2_model_runner
|
||||
|
||||
# PCP runtime support is implemented only by the V2 model runner.
|
||||
if self.parallel_config.prefill_context_parallel_size > 1:
|
||||
return True
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# DSpark is implemented only by the V2 GPU model runner, and DeepSeek-V4
|
||||
# is not otherwise a default-V2 architecture, so force V2 for it. If V2
|
||||
# is unsupported for the rest of the config, _validate_v2_model_runner
|
||||
# raises rather than silently falling back to V1 (which can't run dspark).
|
||||
if (
|
||||
self.speculative_config is not None
|
||||
and self.speculative_config.method == "dspark"
|
||||
):
|
||||
return True
|
||||
|
||||
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only);
|
||||
# force V2 as for dspark, since a hybrid target otherwise defaults to V1.
|
||||
if self._dflash_needs_multi_kv_group():
|
||||
return True
|
||||
|
||||
# The DFlash2 candidate selector exists only in the V2 speculator. On V1
|
||||
# the same checkpoint drafts through DFlashProposer, which never calls
|
||||
# it, so the draft degrades to DFlash1 silently. Force V2 as for dspark.
|
||||
if self._is_dflash2_draft():
|
||||
return True
|
||||
|
||||
if self.model_config is not None and self.model_config.is_diffusion:
|
||||
return True
|
||||
|
||||
if not self._is_default_v2_model_runner_model():
|
||||
return False
|
||||
model_config = self.model_config
|
||||
if model_config is not None and current_platform.is_rocm():
|
||||
architectures = getattr(model_config, "architectures", ())
|
||||
if any(arch in ROCM_DEFAULT_MRV1_ARCHITECTURES for arch in architectures):
|
||||
logger.warning_once(
|
||||
"Defaulting to V1 model runner on ROCm for model architectures: %s",
|
||||
", ".join(architectures),
|
||||
)
|
||||
return False
|
||||
|
||||
if not HAS_TRITON:
|
||||
logger.warning_once(
|
||||
@@ -717,26 +674,6 @@ class VllmConfig:
|
||||
num_sliding = sum(lt == "sliding_attention" for lt in layer_types)
|
||||
return 0 < num_sliding < len(layer_types)
|
||||
|
||||
def _is_default_v2_model_runner_model(self) -> bool:
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
return False
|
||||
|
||||
architectures = getattr(model_config, "architectures", [])
|
||||
default_architectures = default_v2_model_runner_architectures()
|
||||
is_default_v2_architecture = any(
|
||||
arch in default_architectures for arch in architectures
|
||||
)
|
||||
|
||||
if getattr(model_config, "is_hybrid", False) and (
|
||||
not is_default_v2_architecture
|
||||
):
|
||||
return False
|
||||
|
||||
if getattr(model_config, "is_attention_free", False):
|
||||
return False
|
||||
return is_default_v2_architecture or not model_config.is_moe
|
||||
|
||||
def _uses_breakable_cudagraph_by_default(self) -> bool:
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
@@ -1666,11 +1603,8 @@ class VllmConfig:
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
self._validate_v2_model_runner()
|
||||
elif self.parallel_config.prefill_context_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"Prefill context parallelism requires Model Runner V2. "
|
||||
"Remove VLLM_USE_V2_MODEL_RUNNER=0."
|
||||
)
|
||||
else:
|
||||
self._validate_v1_model_runner()
|
||||
|
||||
self._validate_batch_sharded_sampling()
|
||||
|
||||
@@ -2561,6 +2495,35 @@ class VllmConfig:
|
||||
|
||||
return unsupported
|
||||
|
||||
def _get_v1_model_runner_unsupported_features(self) -> list[str]:
|
||||
unsupported: list[str] = []
|
||||
|
||||
# PCP runtime support is implemented only by the V2 model runner.
|
||||
if self.parallel_config.prefill_context_parallel_size > 1:
|
||||
unsupported.append("prefill context parallel")
|
||||
|
||||
# DSpark is implemented only by the V2 GPU model runner.
|
||||
if self.speculative_config and self.speculative_config.method == "dspark":
|
||||
unsupported.append("dspark speculative decoding")
|
||||
|
||||
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only).
|
||||
if self._dflash_needs_multi_kv_group():
|
||||
unsupported.append("mixed sliding/full dflash drafts")
|
||||
|
||||
# The DFlash2 candidate selector exists only in the V2 speculator. On
|
||||
# V1 the same checkpoint drafts through DFlashProposer, which never
|
||||
# calls it, so the draft would degrade to DFlash1 silently.
|
||||
if self._is_dflash2_draft():
|
||||
unsupported.append("dflash2 drafts")
|
||||
|
||||
if self.model_config is not None and self.model_config.is_diffusion:
|
||||
unsupported.append("diffusion models")
|
||||
|
||||
if self.parallel_config.enable_batch_sharded_sampling:
|
||||
unsupported.append("batch-sharded sampling")
|
||||
|
||||
return unsupported
|
||||
|
||||
def _validate_batch_sharded_sampling(self) -> None:
|
||||
"""Validate `enable_batch_sharded_sampling` against the rest of the config."""
|
||||
if not self.parallel_config.enable_batch_sharded_sampling:
|
||||
@@ -2571,9 +2534,6 @@ class VllmConfig:
|
||||
blockers: list[str] = []
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
|
||||
if not self.use_v2_model_runner:
|
||||
blockers.append("it is only implemented by Model Runner V2")
|
||||
|
||||
if tp_size <= 1:
|
||||
blockers.append("tensor_parallel_size is 1, so there is nothing to shard")
|
||||
elif self.scheduler_config.max_num_seqs < tp_size:
|
||||
@@ -2623,6 +2583,13 @@ class VllmConfig:
|
||||
f"Model Runner V2 does not yet support: {', '.join(unsupported)}"
|
||||
)
|
||||
|
||||
def _validate_v1_model_runner(self) -> None:
|
||||
unsupported = self._get_v1_model_runner_unsupported_features()
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
f"Model Runner V1 does not support: {', '.join(unsupported)}"
|
||||
)
|
||||
|
||||
def validate_block_size(self) -> None:
|
||||
"""Validate block_size against DCP and mamba constraints.
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from typing_extensions import runtime_checkable
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.cache import _layout_from_name
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d, np_to_pinned_tensor
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout, KVCacheSpec, MambaSpec
|
||||
@@ -475,7 +476,9 @@ def make_local_attention_virtual_batches(
|
||||
block_size: int = 0,
|
||||
) -> tuple[CommonAttentionMetadata, Callable[[torch.Tensor], torch.Tensor]]:
|
||||
query_start_loc_np = common_attn_metadata.query_start_loc_cpu.numpy()
|
||||
seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()
|
||||
with gpu_sync_allowed():
|
||||
# TODO see https://github.com/vllm-project/vllm/pull/31852
|
||||
seq_lens_np = common_attn_metadata.seq_lens_cpu.numpy()
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
device = common_attn_metadata.query_start_loc.device
|
||||
|
||||
|
||||
Reference in New Issue
Block a user