[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)
This commit is contained in:
@@ -341,6 +341,7 @@ class SpeculativeConfig:
|
||||
@staticmethod
|
||||
def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
|
||||
initial_architecture = hf_config.architectures[0]
|
||||
use_v32_mtp = hf_config.model_type in ("deepseek_v32", "glm_moe_dsa")
|
||||
if hf_config.model_type == "dots3_note":
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
|
||||
mtp_layer_types = getattr(hf_config, "mtp_layer_types", None)
|
||||
@@ -365,7 +366,12 @@ class SpeculativeConfig:
|
||||
if hf_config.model_type == "deepseek_mtp":
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]}
|
||||
{
|
||||
"n_predict": n_predict,
|
||||
"architectures": [
|
||||
"DeepseekV32MTPModel" if use_v32_mtp else "DeepSeekMTPModel"
|
||||
],
|
||||
}
|
||||
)
|
||||
if hf_config.model_type == "deepseek_v4":
|
||||
hf_config.model_type = "deepseek_mtp"
|
||||
@@ -759,10 +765,6 @@ class SpeculativeConfig:
|
||||
if self.method == "mtp":
|
||||
if self.target_model_config is None:
|
||||
raise ValueError("target_model_config must be present for mtp")
|
||||
if self.target_model_config.hf_text_config.model_type == "deepseek_v32":
|
||||
# FIXME(luccafong): cudagraph with v32 MTP is not supported,
|
||||
# remove this when the issue is fixed.
|
||||
self.enforce_eager = True
|
||||
# use the draft model from the same model:
|
||||
self.model = self.target_model_config.model
|
||||
# Align the quantization of draft model for cases such as
|
||||
|
||||
+67
-39
@@ -69,7 +69,9 @@ logger = init_logger(__name__)
|
||||
DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV2ForCausalLM",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"GraniteMoeForCausalLM",
|
||||
"InklingForCausalLM",
|
||||
"InklingForConditionalGeneration",
|
||||
@@ -79,6 +81,23 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"DeepseekV32MTPModel",
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTPModel",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"InklingForCausalLM",
|
||||
"InklingForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3MTPModel",
|
||||
"KimiLinearForCausalLM",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def default_v2_model_runner_architectures() -> frozenset[str]:
|
||||
@@ -86,13 +105,28 @@ def default_v2_model_runner_architectures() -> frozenset[str]:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# TODO(rocm): DeepSeek V4 is still faster on MRV1 on ROCm. The
|
||||
# attention layer picks the eager cudagraph region MRV1 needs, so
|
||||
# this is a perf default only; drop it once MRV2 catches up.
|
||||
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {"DeepseekV4ForCausalLM"}
|
||||
# TODO(rocm): These models are either unsupported by MRV2 or slower with
|
||||
# MRV2 on AMD GPUs.
|
||||
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV4ForCausalLM",
|
||||
}
|
||||
return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES
|
||||
|
||||
|
||||
@lru_cache
|
||||
def default_breakable_cudagraph_architectures() -> frozenset[str]:
|
||||
"""Architectures defaulting to breakable CUDA graphs on this platform."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES - {
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV32MTPModel",
|
||||
}
|
||||
return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES
|
||||
|
||||
|
||||
class OptimizationLevel(IntEnum):
|
||||
"""Optimization level enum."""
|
||||
|
||||
@@ -691,6 +725,34 @@ class VllmConfig:
|
||||
return False
|
||||
return is_default_v2_architecture or not model_config.is_moe
|
||||
|
||||
def _uses_breakable_cudagraph_by_default(self) -> bool:
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
return False
|
||||
|
||||
architectures = set(model_config.architectures)
|
||||
return bool(architectures & default_breakable_cudagraph_architectures())
|
||||
|
||||
def _maybe_enable_breakable_cudagraph(self) -> bool:
|
||||
if (
|
||||
"VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
|
||||
and self._uses_breakable_cudagraph_by_default()
|
||||
):
|
||||
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
||||
logger.info_once(
|
||||
"Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
|
||||
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
|
||||
)
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
|
||||
enabled = is_breakable_cudagraph_enabled()
|
||||
if enabled:
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
return enabled
|
||||
|
||||
@property
|
||||
def needs_dp_coordinator(self) -> bool:
|
||||
"""
|
||||
@@ -1288,41 +1350,7 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
# For model classes don't carry @support_torch_compile —
|
||||
# the breakable cudagraph is the supported PIECEWISE path. Auto-enable
|
||||
# it unless the user has explicitly opted out via the env var.
|
||||
if (
|
||||
self.model_config is not None
|
||||
and "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ
|
||||
and any(
|
||||
a
|
||||
in (
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTPModel",
|
||||
"InklingForCausalLM",
|
||||
"InklingForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
"KimiK3MTPModel",
|
||||
"KimiLinearForCausalLM",
|
||||
"MiniMaxM3SparseForCausalLM",
|
||||
"MiniMaxM3SparseForConditionalGeneration",
|
||||
)
|
||||
for a in self.model_config.architectures
|
||||
)
|
||||
):
|
||||
os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1"
|
||||
logger.info_once(
|
||||
"Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. "
|
||||
"Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out."
|
||||
)
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import (
|
||||
is_breakable_cudagraph_enabled,
|
||||
)
|
||||
|
||||
breakable_cudagraph_enabled = is_breakable_cudagraph_enabled()
|
||||
if breakable_cudagraph_enabled:
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph()
|
||||
|
||||
if not breakable_cudagraph_enabled and (
|
||||
self.compilation_config.backend == "eager"
|
||||
|
||||
@@ -92,7 +92,10 @@ _TEXT_GENERATION_MODELS = {
|
||||
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
|
||||
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
|
||||
"DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"DeepseekV32ForCausalLM": (
|
||||
"vllm.models.deepseek_v32",
|
||||
"DeepseekV32ForCausalLM",
|
||||
),
|
||||
"DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"),
|
||||
"Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"),
|
||||
"Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
|
||||
@@ -115,7 +118,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
|
||||
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
|
||||
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
|
||||
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
@@ -661,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
|
||||
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
|
||||
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
|
||||
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
|
||||
"Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"),
|
||||
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
|
||||
"BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"),
|
||||
|
||||
@@ -1,26 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V3.2 (``deepseek_v32``) model — hardware-isolated entry point.
|
||||
"""DeepSeek V3.2 (``deepseek_v32``) platform entry point.
|
||||
|
||||
DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
|
||||
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
|
||||
attend. The same model code serves any DSA checkpoint, including GLM-5.2
|
||||
(``glm_moe_dsa``), which reuses this architecture.
|
||||
|
||||
The CUDA implementation selects capability-specific kernels internally and
|
||||
falls back when an optimization is unavailable. Other platforms use the
|
||||
generic implementation by default.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from .amd.model import DeepseekV32ForCausalLM
|
||||
from .amd.mtp import DeepseekV32MTP
|
||||
elif current_platform.is_xpu():
|
||||
raise NotImplementedError("deepseek_v32 does not yet support XPU.")
|
||||
else:
|
||||
# Covers Blackwell (sm100) and all other CUDA devices.
|
||||
if current_platform.is_cuda():
|
||||
# GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized
|
||||
# kernels remain gated on the device capabilities they support.
|
||||
from .nvidia.model import DeepseekV32ForCausalLM
|
||||
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
|
||||
from .nvidia.mtp import DeepseekV32MTP
|
||||
else:
|
||||
# ROCm, XPU, and CPU keep the generic implementation.
|
||||
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
|
||||
from vllm.model_executor.models.deepseek_v2 import (
|
||||
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
|
||||
|
||||
__all__ = [
|
||||
"DeepseekV32ForCausalLM",
|
||||
"DeepseekV32MTP",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
]
|
||||
|
||||
@@ -38,7 +38,6 @@ class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
|
||||
|
||||
|
||||
class DeepseekV32MLAAttention(DeepseekV32Attention):
|
||||
require_fp8_kv_cache: bool = False
|
||||
indexer_cls = DeepseekV32ROCmIndexer
|
||||
|
||||
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
|
||||
|
||||
@@ -170,7 +170,6 @@ class DeepseekV32Attention(MLAAttention):
|
||||
indexer: "DeepseekV32Indexer | None"
|
||||
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
|
||||
|
||||
require_fp8_kv_cache: bool = True
|
||||
supports_dense_mha_prefill = False
|
||||
|
||||
def __init__(
|
||||
@@ -283,21 +282,9 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self.layer_name if enable_short_prefill_scoring_skip else ""
|
||||
)
|
||||
|
||||
if self.require_fp8_kv_cache:
|
||||
assert is_quantized_kv_cache(self.kv_cache_dtype), (
|
||||
"deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse "
|
||||
"MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) "
|
||||
"or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)."
|
||||
)
|
||||
self._fp8_query = self.impl.supports_quant_query_input
|
||||
if not self._fp8_query:
|
||||
assert self.kv_cache_dtype == "fp8_ds_mla", (
|
||||
"deepseek_v32 (nvidia) on a bf16-query sparse MLA backend "
|
||||
"(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. "
|
||||
"Launch with --kv-cache-dtype fp8_ds_mla."
|
||||
)
|
||||
|
||||
self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla"
|
||||
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
|
||||
self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla"
|
||||
|
||||
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
|
||||
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
|
||||
# has_indexer=False path so the indexer args don't allocate every call.
|
||||
@@ -473,7 +474,7 @@ def fused_norm_rope(
|
||||
|
||||
# --- MLA KV cache setup ---
|
||||
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
|
||||
mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla")
|
||||
mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla
|
||||
mla_num_tiles = 1
|
||||
mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device)
|
||||
mla_ds_rope_view = torch.empty(0, dtype=torch.bfloat16, device=device)
|
||||
|
||||
@@ -1018,7 +1018,11 @@ class SpecDecodeBaseProposer:
|
||||
# feedback into the next draft step.
|
||||
architectures = self.draft_model_config.hf_config.architectures or []
|
||||
return bool(
|
||||
{"DeepSeekMTPModel", "KimiK3MTPModel"}.intersection(architectures)
|
||||
{
|
||||
"DeepSeekMTPModel",
|
||||
"DeepseekV32MTPModel",
|
||||
"KimiK3MTPModel",
|
||||
}.intersection(architectures)
|
||||
)
|
||||
return self.method not in ("mtp", "draft_model", "dflash")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user