[bugfix]Indexer init skip and MTP TopK share for iteration (#45895)
Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
@@ -349,6 +349,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
attn_backend: type[AttentionBackend] | None = None,
|
||||
use_sparse: bool = False,
|
||||
indexer: object | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
**extra_impl_args,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -437,6 +438,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
)
|
||||
cache_config.enable_prefix_caching = False
|
||||
|
||||
# Sparse MLA reads top-k indices from a shared buffer. Pass it
|
||||
# explicitly so backbone "skip" layers (indexer=None) still find it.
|
||||
if use_sparse:
|
||||
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
|
||||
|
||||
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
|
||||
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
|
||||
num_heads=self.num_heads,
|
||||
|
||||
@@ -112,6 +112,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.prefix = prefix
|
||||
|
||||
@@ -119,8 +119,12 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
|
||||
# Recycle the post-final-norm hidden into the next draft step.
|
||||
# compute_logits applies shared_head (== final norm) to the pre-norm
|
||||
# element, so logits and the recycle each get exactly one final-norm.
|
||||
# Matches SGLang's deepseek_nextn.
|
||||
return hidden_states, self.shared_head(hidden_states)
|
||||
|
||||
|
||||
class DeepSeekMultiTokenPredictor(nn.Module):
|
||||
|
||||
@@ -998,8 +998,29 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
|
||||
self.is_v32 = hasattr(config, "index_topk")
|
||||
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_skip_topk = False
|
||||
if self.is_v32:
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = (
|
||||
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq != 0
|
||||
)
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
|
||||
# The skip pattern only governs backbone layers. MTP/nextn layers
|
||||
# (layer_id >= num_hidden_layers) always build a full indexer: they
|
||||
# compute indices at draft step 0 and toggle at runtime via
|
||||
# set_skip_topk (index_share_for_mtp_iteration).
|
||||
_num_hidden_layers = getattr(config, "num_hidden_layers", None)
|
||||
is_mtp_layer = _num_hidden_layers is not None and layer_id >= _num_hidden_layers
|
||||
|
||||
if self.is_v32 and (not _skip_topk or is_mtp_layer):
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
@@ -1017,22 +1038,6 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
f"{prefix}.indexer",
|
||||
is_inplace_rope=self.indexer_rope_emb.enabled(),
|
||||
)
|
||||
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = (
|
||||
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq
|
||||
!= 0
|
||||
)
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
|
||||
else:
|
||||
self.indexer_rope_emb = None
|
||||
self.indexer = None
|
||||
|
||||
@@ -271,7 +271,7 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
topk_indice_buffer: torch.Tensor | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
indexer: "Indexer | None" = None,
|
||||
**mla_args,
|
||||
) -> None:
|
||||
@@ -301,8 +301,12 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
||||
self.qk_nope_head_dim: int = mla_args["qk_nope_head_dim"]
|
||||
self.qk_rope_head_dim: int = mla_args["qk_rope_head_dim"]
|
||||
|
||||
assert indexer is not None, "Indexer required for sparse MLA"
|
||||
self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
||||
)
|
||||
|
||||
self._workspace_buffer: torch.Tensor | None = None
|
||||
self.bmm1_scale: float | None = None
|
||||
|
||||
@@ -568,8 +568,12 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]):
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
||||
self.softmax_scale = scale
|
||||
assert indexer is not None
|
||||
self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
||||
)
|
||||
# Prefill BF16 kernel requires 64 on Hopper, 128 on Blackwell
|
||||
self.prefill_padding = (
|
||||
128 if current_platform.is_device_capability_family(100) else 64
|
||||
|
||||
@@ -629,7 +629,7 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
topk_indice_buffer: torch.Tensor | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
indexer: "Indexer | None" = None,
|
||||
**mla_args,
|
||||
) -> None:
|
||||
@@ -642,8 +642,12 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
||||
self.softmax_scale = scale
|
||||
assert indexer is not None
|
||||
self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
@@ -184,7 +184,7 @@ class XPUMLASparseImpl(SparseMLAAttentionImpl[XPUMLASparseMetadata]):
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
topk_indice_buffer: torch.Tensor | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
indexer: Optional["Indexer"] = None,
|
||||
**mla_args,
|
||||
) -> None:
|
||||
@@ -195,8 +195,12 @@ class XPUMLASparseImpl(SparseMLAAttentionImpl[XPUMLASparseMetadata]):
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.kv_lora_rank: int = mla_args["kv_lora_rank"]
|
||||
self.softmax_scale = scale
|
||||
assert indexer is not None
|
||||
self.topk_indices_buffer: torch.Tensor | None = indexer.topk_indices_buffer
|
||||
# The indexer carries the shared buffer for normal layers and tests;
|
||||
# the explicitly-passed buffer covers backbone skip layers, whose
|
||||
# indexer is not constructed (see deepseek_v2.py).
|
||||
self.topk_indices_buffer: torch.Tensor | None = (
|
||||
indexer.topk_indices_buffer if indexer is not None else topk_indices_buffer
|
||||
)
|
||||
|
||||
def _forward_bf16_kv(
|
||||
self,
|
||||
|
||||
@@ -918,6 +918,13 @@ class SpecDecodeBaseProposer:
|
||||
return per_group_attn_metadata, per_layer_attn_metadata
|
||||
|
||||
def model_returns_tuple(self) -> bool:
|
||||
if self.method == "mtp":
|
||||
# DeepSeek-family MTP (deepseek_mtp.py) recycles the post-final-
|
||||
# norm hidden, so its forward returns (logit_hidden,
|
||||
# recycle_hidden). Other MTP families return a single tensor.
|
||||
return "DeepSeekMTPModel" in (
|
||||
self.draft_model_config.hf_config.architectures or []
|
||||
)
|
||||
return self.method not in ("mtp", "draft_model", "dflash")
|
||||
|
||||
def prepare_next_token_ids_cpu(
|
||||
|
||||
Reference in New Issue
Block a user