[Model][NVIDIA] Route DSA models to the CUDA non-compiled path (#52861)

This commit is contained in:
Woosuk Kwon
2026-08-19 00:13:52 -07:00
committed by GitHub
parent ee11730751
commit b09bd69b5b
16 changed files with 260 additions and 146 deletions
+17 -8
View File
@@ -1,26 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V3.2 (``deepseek_v32``) model — hardware-isolated entry point.
"""DeepSeek V3.2 (``deepseek_v32``) platform entry point.
DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture:
MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA
attend. The same model code serves any DSA checkpoint, including GLM-5.2
(``glm_moe_dsa``), which reuses this architecture.
The CUDA implementation selects capability-specific kernels internally and
falls back when an optimization is unavailable. Other platforms use the
generic implementation by default.
"""
from vllm.platforms import current_platform
if current_platform.is_rocm():
from .amd.model import DeepseekV32ForCausalLM
from .amd.mtp import DeepseekV32MTP
elif current_platform.is_xpu():
raise NotImplementedError("deepseek_v32 does not yet support XPU.")
else:
# Covers Blackwell (sm100) and all other CUDA devices.
if current_platform.is_cuda():
# GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized
# kernels remain gated on the device capabilities they support.
from .nvidia.model import DeepseekV32ForCausalLM
from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM
from .nvidia.mtp import DeepseekV32MTP
else:
# ROCm, XPU, and CPU keep the generic implementation.
from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV3ForCausalLM as DeepseekV32ForCausalLM,
)
from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM
__all__ = [
"DeepseekV32ForCausalLM",
"DeepseekV32MTP",
"GlmMoeDsaForCausalLM",
]
-1
View File
@@ -38,7 +38,6 @@ class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
class DeepseekV32MLAAttention(DeepseekV32Attention):
require_fp8_kv_cache: bool = False
indexer_cls = DeepseekV32ROCmIndexer
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
+3 -16
View File
@@ -170,7 +170,6 @@ class DeepseekV32Attention(MLAAttention):
indexer: "DeepseekV32Indexer | None"
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
require_fp8_kv_cache: bool = True
supports_dense_mha_prefill = False
def __init__(
@@ -283,21 +282,9 @@ class DeepseekV32Attention(MLAAttention):
self.layer_name if enable_short_prefill_scoring_skip else ""
)
if self.require_fp8_kv_cache:
assert is_quantized_kv_cache(self.kv_cache_dtype), (
"deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse "
"MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) "
"or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)."
)
self._fp8_query = self.impl.supports_quant_query_input
if not self._fp8_query:
assert self.kv_cache_dtype == "fp8_ds_mla", (
"deepseek_v32 (nvidia) on a bf16-query sparse MLA backend "
"(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. "
"Launch with --kv-cache-dtype fp8_ds_mla."
)
self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla"
fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype)
self._fp8_query = fp8_attention and self.impl.supports_quant_query_input
self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla"
self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False)
+2 -1
View File
@@ -6,6 +6,7 @@ import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import is_quantized_kv_cache
# Cache of tiny 1-element dummy tensors (per device, dtype) reused by the
# has_indexer=False path so the indexer args don't allocate every call.
@@ -473,7 +474,7 @@ def fused_norm_rope(
# --- MLA KV cache setup ---
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla")
mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla
mla_num_tiles = 1
mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device)
mla_ds_rope_view = torch.empty(0, dtype=torch.bfloat16, device=device)