[bugfix]Indexer init skip and MTP TopK share for iteration (#45895)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
This commit is contained in:
Jared Wen
2026-06-19 09:57:51 +08:00
committed by GitHub
parent 9ea3a4015b
commit ab66606993
9 changed files with 69 additions and 30 deletions
@@ -349,6 +349,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
attn_backend: type[AttentionBackend] | None = None,
use_sparse: bool = False,
indexer: object | None = None,
topk_indices_buffer: torch.Tensor | None = None,
**extra_impl_args,
):
super().__init__()
@@ -437,6 +438,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
cache_config.enable_prefix_caching = False
# Sparse MLA reads top-k indices from a shared buffer. Pass it
# explicitly so backbone "skip" layers (indexer=None) still find it.
if use_sparse:
extra_impl_args["topk_indices_buffer"] = topk_indices_buffer
impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls())
self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass
num_heads=self.num_heads,
+1
View File
@@ -112,6 +112,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
kv_b_proj=self.kv_b_proj,
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=mla_modules.topk_indices_buffer,
)
self.prefix = prefix
+6 -2
View File
@@ -119,8 +119,12 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
hidden_states=hidden_states,
residual=None,
)
hidden_states = residual + hidden_states
return hidden_states
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
# Recycle the post-final-norm hidden into the next draft step.
# compute_logits applies shared_head (== final norm) to the pre-norm
# element, so logits and the recycle each get exactly one final-norm.
# Matches SGLang's deepseek_nextn.
return hidden_states, self.shared_head(hidden_states)
class DeepSeekMultiTokenPredictor(nn.Module):
+22 -17
View File
@@ -998,8 +998,29 @@ class DeepseekV2MLAAttention(nn.Module):
self.is_v32 = hasattr(config, "index_topk")
# IndexCache config
# Refer: https://arxiv.org/abs/2603.12201 for more details.
_skip_topk = False
if self.is_v32:
_index_topk_freq = getattr(config, "index_topk_freq", 1)
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
layer_id = extract_layer_index(prefix)
if _index_topk_pattern is None:
_skip_topk = (
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq != 0
)
elif 0 <= layer_id < len(_index_topk_pattern):
_skip_topk = _index_topk_pattern[layer_id] == "S"
# The skip pattern only governs backbone layers. MTP/nextn layers
# (layer_id >= num_hidden_layers) always build a full indexer: they
# compute indices at draft step 0 and toggle at runtime via
# set_skip_topk (index_share_for_mtp_iteration).
_num_hidden_layers = getattr(config, "num_hidden_layers", None)
is_mtp_layer = _num_hidden_layers is not None and layer_id >= _num_hidden_layers
if self.is_v32 and (not _skip_topk or is_mtp_layer):
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
@@ -1017,22 +1038,6 @@ class DeepseekV2MLAAttention(nn.Module):
f"{prefix}.indexer",
is_inplace_rope=self.indexer_rope_emb.enabled(),
)
# IndexCache config
# Refer: https://arxiv.org/abs/2603.12201 for more details.
_index_topk_freq = getattr(config, "index_topk_freq", 1)
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
layer_id = extract_layer_index(prefix)
if _index_topk_pattern is None:
_skip_topk = (
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq
!= 0
)
elif 0 <= layer_id < len(_index_topk_pattern):
_skip_topk = _index_topk_pattern[layer_id] == "S"
else:
self.indexer_rope_emb = None
self.indexer = None