[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)
This commit is contained in:
+140
-4
@@ -113,18 +113,154 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
|
||||
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
|
||||
|
||||
|
||||
def test_rocm_defaults_deepseek_v4_to_mrv1(monkeypatch):
|
||||
"""ROCm keeps DeepSeek V4 on MRV1, which is still faster there."""
|
||||
from vllm.config.vllm import default_v2_model_runner_architectures
|
||||
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
|
||||
"""ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths."""
|
||||
from vllm.config.vllm import (
|
||||
default_breakable_cudagraph_architectures,
|
||||
default_v2_model_runner_architectures,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
|
||||
# The lookup is lru_cached against a fixed platform.
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
try:
|
||||
assert "DeepseekV4ForCausalLM" not in default_v2_model_runner_architectures()
|
||||
v2_architectures = default_v2_model_runner_architectures()
|
||||
breakable_architectures = default_breakable_cudagraph_architectures()
|
||||
|
||||
assert "DeepseekV32ForCausalLM" not in v2_architectures
|
||||
assert "DeepseekV4ForCausalLM" not in v2_architectures
|
||||
assert "DeepseekV32ForCausalLM" not in breakable_architectures
|
||||
assert "DeepseekV32MTPModel" not in breakable_architectures
|
||||
finally:
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "architecture"),
|
||||
[
|
||||
("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM"),
|
||||
("zai-org/GLM-5.2-FP8", "GlmMoeDsaForCausalLM"),
|
||||
("nvidia/DeepSeek-V3.2-NVFP4", "DeepseekV32ForCausalLM"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("with_mtp", [False, True], ids=["no-mtp", "mtp"])
|
||||
def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
monkeypatch, model, architecture, with_mtp
|
||||
):
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
from vllm.config.vllm import (
|
||||
default_breakable_cudagraph_architectures,
|
||||
default_v2_model_runner_architectures,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
|
||||
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
|
||||
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=model,
|
||||
architectures=[architecture],
|
||||
runner_type="generate",
|
||||
is_moe=True,
|
||||
is_hybrid=False,
|
||||
is_attention_free=False,
|
||||
is_diffusion=False,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
model_config=model_config,
|
||||
speculative_config=SimpleNamespace(method="mtp") if with_mtp else None,
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=1),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
),
|
||||
)
|
||||
config._dflash_needs_multi_kv_group = lambda: False
|
||||
config._is_default_v2_model_runner_model = lambda: (
|
||||
VllmConfig._is_default_v2_model_runner_model(config)
|
||||
)
|
||||
config._get_v2_model_runner_unsupported_features = lambda: []
|
||||
config._uses_breakable_cudagraph_by_default = lambda: (
|
||||
VllmConfig._uses_breakable_cudagraph_by_default(config)
|
||||
)
|
||||
|
||||
try:
|
||||
assert VllmConfig.use_v2_model_runner.fget(config)
|
||||
assert VllmConfig._maybe_enable_breakable_cudagraph(config)
|
||||
assert is_breakable_cudagraph_enabled()
|
||||
assert config.compilation_config.mode == CompilationMode.NONE
|
||||
assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()
|
||||
finally:
|
||||
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
|
||||
default_v2_model_runner_architectures.cache_clear()
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "is_rocm", "expected"),
|
||||
[
|
||||
("DeepseekV32ForCausalLM", False, True),
|
||||
("DeepseekV32ForCausalLM", True, False),
|
||||
("DeepseekV32MTPModel", False, True),
|
||||
("DeepseekV32MTPModel", True, False),
|
||||
("GlmMoeDsaForCausalLM", False, True),
|
||||
("GlmMoeDsaForCausalLM", True, True),
|
||||
],
|
||||
)
|
||||
def test_dsa_breakable_cudagraph_platform_default(
|
||||
monkeypatch, architecture, is_rocm, expected
|
||||
):
|
||||
from vllm.config.vllm import default_breakable_cudagraph_architectures
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False)
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: is_rocm)
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(architectures=[architecture]),
|
||||
compilation_config=CompilationConfig(),
|
||||
)
|
||||
config._uses_breakable_cudagraph_by_default = lambda: (
|
||||
VllmConfig._uses_breakable_cudagraph_by_default(config)
|
||||
)
|
||||
|
||||
try:
|
||||
assert VllmConfig._maybe_enable_breakable_cudagraph(config) is expected
|
||||
if expected:
|
||||
assert config.compilation_config.mode == CompilationMode.NONE
|
||||
finally:
|
||||
os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None)
|
||||
default_breakable_cudagraph_architectures.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_type", "expected_architecture"),
|
||||
[
|
||||
("deepseek_v32", "DeepseekV32MTPModel"),
|
||||
("glm_moe_dsa", "DeepseekV32MTPModel"),
|
||||
("deepseek_v3", "DeepSeekMTPModel"),
|
||||
],
|
||||
)
|
||||
def test_dsa_models_select_matching_mtp(model_type, expected_architecture):
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
hf_config = PretrainedConfig(
|
||||
architectures=["DeepseekV32ForCausalLM"],
|
||||
num_nextn_predict_layers=1,
|
||||
)
|
||||
hf_config.model_type = model_type
|
||||
|
||||
SpeculativeConfig.hf_config_override(hf_config)
|
||||
|
||||
assert hf_config.architectures == [expected_architecture]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user