[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA (#34552)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-03-03 07:21:57 -08:00
committed by GitHub
co-authored by Matthew Bonanni
parent fb7fdc49c4
commit 28ef9ba399
7 changed files with 260 additions and 197 deletions
+13 -9
View File
@@ -1936,7 +1936,7 @@ class GPUModelRunner(
if self.speculative_config and spec_decode_common_attn_metadata is None:
if isinstance(self.drafter, EagleProposer):
if self.drafter.attn_layer_names[0] in kv_cache_group.layer_names:
if self.drafter.kv_cache_gid == kv_cache_gid:
spec_decode_common_attn_metadata = cm
else:
spec_decode_common_attn_metadata = cm
@@ -5559,6 +5559,14 @@ class GPUModelRunner(
# because some of them change the threshold at init time.
self.calculate_reorder_batch_threshold()
# Initialize drafter attention backend
if self.speculative_config and (
self.speculative_config.use_eagle()
or self.speculative_config.uses_draft_model()
):
assert isinstance(self.drafter, EagleProposer | DraftModelProposer)
self.drafter.initialize_attn_backend(kv_cache_config, kernel_block_sizes)
def _check_and_update_cudagraph_mode(
self,
attention_backends: list[set[type[AttentionBackend]]],
@@ -6079,15 +6087,11 @@ class GPUModelRunner(
kv_cache_config, kernel_block_sizes
)
if self.speculative_config and (
self.speculative_config.use_eagle()
or self.speculative_config.uses_draft_model()
or self.speculative_config.uses_extract_hidden_states()
if (
self.speculative_config
and self.speculative_config.uses_extract_hidden_states()
):
assert isinstance(
self.drafter,
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
)
assert isinstance(self.drafter, ExtractHiddenStatesProposer)
# validate all draft model layers belong to the same kv cache
# group
self.drafter.validate_same_kv_cache_group(kv_cache_config)