From ab666069935c1f23e8ef56038b4659ac9e8f19f8 Mon Sep 17 00:00:00 2001 From: Jared Wen Date: Fri, 19 Jun 2026 09:57:51 +0800 Subject: [PATCH] [bugfix]Indexer init skip and MTP TopK share for iteration (#45895) Signed-off-by: JaredforReal --- .../layers/attention/mla_attention.py | 6 +++ vllm/model_executor/layers/mla.py | 1 + vllm/model_executor/models/deepseek_mtp.py | 8 +++- vllm/model_executor/models/deepseek_v2.py | 39 +++++++++++-------- .../backends/mla/flashinfer_mla_sparse.py | 10 +++-- .../attention/backends/mla/flashmla_sparse.py | 8 +++- .../backends/mla/rocm_aiter_mla_sparse.py | 10 +++-- .../attention/backends/mla/xpu_mla_sparse.py | 10 +++-- vllm/v1/spec_decode/llm_base_proposer.py | 7 ++++ 9 files changed, 69 insertions(+), 30 deletions(-) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 21e3215479..ab3874c5da 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -349,6 +349,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): attn_backend: type[AttentionBackend] | None = None, use_sparse: bool = False, indexer: object | None = None, + topk_indices_buffer: torch.Tensor | None = None, **extra_impl_args, ): super().__init__() @@ -437,6 +438,11 @@ class MLAAttention(nn.Module, AttentionLayerBase): ) cache_config.enable_prefix_caching = False + # Sparse MLA reads top-k indices from a shared buffer. Pass it + # explicitly so backbone "skip" layers (indexer=None) still find it. + if use_sparse: + extra_impl_args["topk_indices_buffer"] = topk_indices_buffer + impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls()) self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass num_heads=self.num_heads, diff --git a/vllm/model_executor/layers/mla.py b/vllm/model_executor/layers/mla.py index 856f6bb8a3..66a95b43c7 100644 --- a/vllm/model_executor/layers/mla.py +++ b/vllm/model_executor/layers/mla.py @@ -112,6 +112,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer): kv_b_proj=self.kv_b_proj, use_sparse=self.is_sparse, indexer=self.indexer, + topk_indices_buffer=mla_modules.topk_indices_buffer, ) self.prefix = prefix diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index d46eb67c5e..88f33ac021 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -119,8 +119,12 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module): hidden_states=hidden_states, residual=None, ) - hidden_states = residual + hidden_states - return hidden_states + hidden_states = residual + hidden_states # pre-final-norm (logits hidden) + # Recycle the post-final-norm hidden into the next draft step. + # compute_logits applies shared_head (== final norm) to the pre-norm + # element, so logits and the recycle each get exactly one final-norm. + # Matches SGLang's deepseek_nextn. + return hidden_states, self.shared_head(hidden_states) class DeepSeekMultiTokenPredictor(nn.Module): diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 80d518dacb..22c4003d3f 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -998,8 +998,29 @@ class DeepseekV2MLAAttention(nn.Module): self.is_v32 = hasattr(config, "index_topk") + # IndexCache config + # Refer: https://arxiv.org/abs/2603.12201 for more details. _skip_topk = False - if self.is_v32: + _index_topk_freq = getattr(config, "index_topk_freq", 1) + _index_topk_pattern = getattr(config, "index_topk_pattern", None) + _index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2) + layer_id = extract_layer_index(prefix) + + if _index_topk_pattern is None: + _skip_topk = ( + max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq != 0 + ) + elif 0 <= layer_id < len(_index_topk_pattern): + _skip_topk = _index_topk_pattern[layer_id] == "S" + + # The skip pattern only governs backbone layers. MTP/nextn layers + # (layer_id >= num_hidden_layers) always build a full indexer: they + # compute indices at draft step 0 and toggle at runtime via + # set_skip_topk (index_share_for_mtp_iteration). + _num_hidden_layers = getattr(config, "num_hidden_layers", None) + is_mtp_layer = _num_hidden_layers is not None and layer_id >= _num_hidden_layers + + if self.is_v32 and (not _skip_topk or is_mtp_layer): self.indexer_rope_emb = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, @@ -1017,22 +1038,6 @@ class DeepseekV2MLAAttention(nn.Module): f"{prefix}.indexer", is_inplace_rope=self.indexer_rope_emb.enabled(), ) - - # IndexCache config - # Refer: https://arxiv.org/abs/2603.12201 for more details. - _index_topk_freq = getattr(config, "index_topk_freq", 1) - _index_topk_pattern = getattr(config, "index_topk_pattern", None) - _index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2) - layer_id = extract_layer_index(prefix) - - if _index_topk_pattern is None: - _skip_topk = ( - max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq - != 0 - ) - elif 0 <= layer_id < len(_index_topk_pattern): - _skip_topk = _index_topk_pattern[layer_id] == "S" - else: self.indexer_rope_emb = None self.indexer = None diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index aa6301c13b..01716f567d 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -271,7 +271,7 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata attn_type: str, kv_sharing_target_layer_name: str | None, # MLA Specific Arguments - topk_indice_buffer: torch.Tensor | None = None, + topk_indices_buffer: torch.Tensor | None = None, indexer: "Indexer | None" = None, **mla_args, ) -> None: @@ -301,8 +301,12 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata self.qk_nope_head_dim: int = mla_args["qk_nope_head_dim"] self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"] - assert indexer is not None, "Indexer required for sparse MLA" - self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer + # The indexer carries the shared buffer for normal layers and tests; + # the explicitly-passed buffer covers backbone skip layers, whose + # indexer is not constructed (see deepseek_v2.py). + self.topk_indices_buffer: torch.Tensor | None = ( + indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer + ) self._workspace_buffer: torch.Tensor | None = None self.bmm1_scale: float | None = None diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 2da71f9d2c..6d8dfe1312 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -568,8 +568,12 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): self.kv_cache_dtype = kv_cache_dtype self.kv_lora_rank: int = mla_args["kv_lora_rank"] self.softmax_scale = scale - assert indexer is not None - self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer + # The indexer carries the shared buffer for normal layers and tests; + # the explicitly-passed buffer covers backbone skip layers, whose + # indexer is not constructed (see deepseek_v2.py). + self.topk_indices_buffer: torch.Tensor | None = ( + indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer + ) # Prefill BF16 kernel requires 64 on Hopper, 128 on Blackwell self.prefill_padding = ( 128 if current_platform.is_device_capability_family(100) else 64 diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 705ac167f2..1225352ace 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -629,7 +629,7 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) attn_type: str, kv_sharing_target_layer_name: str | None, # MLA Specific Arguments - topk_indice_buffer: torch.Tensor | None = None, + topk_indices_buffer: torch.Tensor | None = None, indexer: "Indexer | None" = None, **mla_args, ) -> None: @@ -642,8 +642,12 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) self.kv_cache_dtype = kv_cache_dtype self.kv_lora_rank: int = mla_args["kv_lora_rank"] self.softmax_scale = scale - assert indexer is not None - self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer + # The indexer carries the shared buffer for normal layers and tests; + # the explicitly-passed buffer covers backbone skip layers, whose + # indexer is not constructed (see deepseek_v2.py). + self.topk_indices_buffer: torch.Tensor | None = ( + indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer + ) vllm_config = get_current_vllm_config() max_tokens = vllm_config.scheduler_config.max_num_batched_tokens diff --git a/vllm/v1/attention/backends/mla/xpu_mla_sparse.py b/vllm/v1/attention/backends/mla/xpu_mla_sparse.py index 2fa91d0183..9aad453210 100644 --- a/vllm/v1/attention/backends/mla/xpu_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/xpu_mla_sparse.py @@ -184,7 +184,7 @@ class XPUMLASparseImpl(SparseMLAAttentionImpl[XPUMLASparseMetadata]): attn_type: str, kv_sharing_target_layer_name: str | None, # MLA Specific Arguments - topk_indice_buffer: torch.Tensor | None = None, + topk_indices_buffer: torch.Tensor | None = None, indexer: Optional["Indexer"] = None, **mla_args, ) -> None: @@ -195,8 +195,12 @@ class XPUMLASparseImpl(SparseMLAAttentionImpl[XPUMLASparseMetadata]): self.kv_cache_dtype = kv_cache_dtype self.kv_lora_rank: int = mla_args["kv_lora_rank"] self.softmax_scale = scale - assert indexer is not None - self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer + # The indexer carries the shared buffer for normal layers and tests; + # the explicitly-passed buffer covers backbone skip layers, whose + # indexer is not constructed (see deepseek_v2.py). + self.topk_indices_buffer: torch.Tensor | None = ( + indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer + ) def _forward_bf16_kv( self, diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index d4f2c1007b..b7c01d3ec1 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -918,6 +918,13 @@ class SpecDecodeBaseProposer: return per_group_attn_metadata, per_layer_attn_metadata def model_returns_tuple(self) -> bool: + if self.method == "mtp": + # DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final- + # norm hidden, so its forward returns (logit_hidden, + # recycle_hidden). Other MTP families return a single tensor. + return "DeepSeekMTPModel" in ( + self.draft_model_config.hf_config.architectures or [] + ) return self.method not in ("mtp", "draft_model", "dflash") def prepare_next_token_ids_cpu(