[Feature] Support MTP speculative decoding for Bailing hybrid models (#44880)
Signed-off-by: zc02384840 <zc02384840@antgroup.com> Co-authored-by: zc02384840 <zc02384840@antgroup.com>
This commit is contained in:
co-authored by
zc02384840
parent
95a248faed
commit
f2aaf59151
@@ -132,6 +132,9 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.linear_attn import (
|
||||
BailingLinearAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
NULL_BLOCK_ID,
|
||||
@@ -2442,9 +2445,16 @@ class GPUModelRunner(
|
||||
|
||||
extra_attn_metadata_args = {}
|
||||
if use_spec_decode and isinstance(
|
||||
builder, (Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder)
|
||||
builder,
|
||||
(
|
||||
Mamba2AttentionMetadataBuilder,
|
||||
GDNAttentionMetadataBuilder,
|
||||
BailingLinearAttentionMetadataBuilder,
|
||||
),
|
||||
):
|
||||
assert ubid is None, "UBatching not supported with GDN yet"
|
||||
assert ubid is None, (
|
||||
"UBatching not supported with GDN or linear attn yet"
|
||||
)
|
||||
extra_attn_metadata_args = dict(
|
||||
num_accepted_tokens=self.num_accepted_tokens.gpu[:num_reqs_padded],
|
||||
num_decode_draft_tokens_cpu=self.num_decode_draft_tokens.cpu[
|
||||
|
||||
Reference in New Issue
Block a user