[BugFix] Add support for MTP num_speculative_tokens > 1 with sparse MLA (#34552)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
fb7fdc49c4
commit
28ef9ba399
@@ -1936,7 +1936,7 @@ class GPUModelRunner(
|
||||
|
||||
if self.speculative_config and spec_decode_common_attn_metadata is None:
|
||||
if isinstance(self.drafter, EagleProposer):
|
||||
if self.drafter.attn_layer_names[0] in kv_cache_group.layer_names:
|
||||
if self.drafter.kv_cache_gid == kv_cache_gid:
|
||||
spec_decode_common_attn_metadata = cm
|
||||
else:
|
||||
spec_decode_common_attn_metadata = cm
|
||||
@@ -5559,6 +5559,14 @@ class GPUModelRunner(
|
||||
# because some of them change the threshold at init time.
|
||||
self.calculate_reorder_batch_threshold()
|
||||
|
||||
# Initialize drafter attention backend
|
||||
if self.speculative_config and (
|
||||
self.speculative_config.use_eagle()
|
||||
or self.speculative_config.uses_draft_model()
|
||||
):
|
||||
assert isinstance(self.drafter, EagleProposer | DraftModelProposer)
|
||||
self.drafter.initialize_attn_backend(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
def _check_and_update_cudagraph_mode(
|
||||
self,
|
||||
attention_backends: list[set[type[AttentionBackend]]],
|
||||
@@ -6079,15 +6087,11 @@ class GPUModelRunner(
|
||||
kv_cache_config, kernel_block_sizes
|
||||
)
|
||||
|
||||
if self.speculative_config and (
|
||||
self.speculative_config.use_eagle()
|
||||
or self.speculative_config.uses_draft_model()
|
||||
or self.speculative_config.uses_extract_hidden_states()
|
||||
if (
|
||||
self.speculative_config
|
||||
and self.speculative_config.uses_extract_hidden_states()
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DraftModelProposer | ExtractHiddenStatesProposer,
|
||||
)
|
||||
assert isinstance(self.drafter, ExtractHiddenStatesProposer)
|
||||
# validate all draft model layers belong to the same kv cache
|
||||
# group
|
||||
self.drafter.validate_same_kv_cache_group(kv_cache_config)
|
||||
|
||||
Reference in New Issue
Block a user