[Feature] Support MTP speculative decoding for Bailing hybrid models (#44880)
Signed-off-by: zc02384840 <zc02384840@antgroup.com> Co-authored-by: zc02384840 <zc02384840@antgroup.com>
This commit is contained in:
co-authored by
zc02384840
parent
95a248faed
commit
f2aaf59151
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.speculative import MTPModelTypes, SpeculativeConfig
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
BailingHybridMTPModelArchConfigConvertor,
|
||||
)
|
||||
|
||||
|
||||
def _bailing_config() -> PretrainedConfig:
|
||||
config = PretrainedConfig(
|
||||
architectures=["BailingMoeV2_5ForCausalLM"],
|
||||
hidden_size=4096,
|
||||
kv_lora_rank=512,
|
||||
num_attention_heads=32,
|
||||
num_experts=256,
|
||||
num_hidden_layers=32,
|
||||
num_key_value_heads=32,
|
||||
num_nextn_predict_layers=1,
|
||||
qk_rope_head_dim=64,
|
||||
vocab_size=157184,
|
||||
)
|
||||
config.model_type = "bailing_hybrid"
|
||||
return config
|
||||
|
||||
|
||||
def test_bailing_hybrid_mtp_hf_config_override():
|
||||
config = _bailing_config()
|
||||
|
||||
overridden = SpeculativeConfig.hf_config_override(config)
|
||||
|
||||
assert overridden.model_type == "bailing_hybrid_mtp"
|
||||
assert overridden.architectures == ["BailingMoeV25MTPModel"]
|
||||
assert overridden.n_predict == 1
|
||||
assert "bailing_hybrid_mtp" in MTPModelTypes.__args__
|
||||
|
||||
|
||||
def test_bailing_hybrid_mtp_model_arch_config():
|
||||
config = _bailing_config()
|
||||
config.model_type = "bailing_hybrid_mtp"
|
||||
config.architectures = ["BailingMoeV25MTPModel"]
|
||||
|
||||
model_arch_config = BailingHybridMTPModelArchConfigConvertor(
|
||||
config, config
|
||||
).convert()
|
||||
|
||||
assert model_arch_config.model_type == "bailing_hybrid_mtp"
|
||||
assert model_arch_config.architectures == ["BailingMoeV25MTPModel"]
|
||||
assert model_arch_config.total_num_hidden_layers == 1
|
||||
assert model_arch_config.is_deepseek_mla
|
||||
@@ -1557,6 +1557,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
use_original_num_layers=True,
|
||||
),
|
||||
# [MTP]
|
||||
"BailingMoeV25MTPModel": _HfExamplesInfo(
|
||||
"inclusionAI/Ring-2.5-1T",
|
||||
speculative_model="inclusionAI/Ring-2.5-1T",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"DeepSeekMTPModel": _HfExamplesInfo(
|
||||
"luccafong/deepseek_mtp_main_random",
|
||||
speculative_model="luccafong/deepseek_mtp_draft_random",
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import CUDAGraphMode, SpeculativeConfig
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.linear_attn import (
|
||||
BailingLinearAttentionMetadataBuilder,
|
||||
LinearAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
|
||||
|
||||
def _create_mamba_spec(num_speculative_blocks: int = 1) -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
|
||||
def test_bailing_linear_attention_reports_uniform_batch_cudagraph_support():
|
||||
vllm_config = create_vllm_config(
|
||||
hf_config_override={
|
||||
"architectures": ["BailingMoeV2_5ForCausalLM"],
|
||||
"model_type": "bailing_hybrid",
|
||||
}
|
||||
)
|
||||
|
||||
support = BailingLinearAttentionMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, _create_mamba_spec()
|
||||
)
|
||||
|
||||
assert support == AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
|
||||
def test_non_bailing_linear_attention_keeps_single_token_cudagraph_support():
|
||||
vllm_config = create_vllm_config(
|
||||
hf_config_override={
|
||||
"architectures": ["MiniMaxText01ForCausalLM"],
|
||||
"model_type": "minimax_text_01",
|
||||
}
|
||||
)
|
||||
|
||||
support = LinearAttentionMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, _create_mamba_spec()
|
||||
)
|
||||
|
||||
assert support == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
|
||||
def test_linear_attention_spec_decode_full_graph_metadata_pads_cache_slots():
|
||||
vllm_config = create_vllm_config(
|
||||
hf_config_override={
|
||||
"architectures": ["BailingMoeV2_5ForCausalLM"],
|
||||
"model_type": "bailing_hybrid",
|
||||
}
|
||||
)
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
|
||||
builder = BailingLinearAttentionMetadataBuilder(
|
||||
kv_cache_spec=_create_mamba_spec(),
|
||||
layer_names=["model.layers.0.self_attn"],
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[20, 20, 0], query_lens=[2, 2, 0]),
|
||||
BLOCK_SIZE,
|
||||
DEVICE,
|
||||
)
|
||||
common.block_table_tensor[2].fill_(-1)
|
||||
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
num_accepted_tokens=torch.tensor([1, 2, 1], dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert metadata.num_decodes == 3
|
||||
assert metadata.num_prefills == 0
|
||||
assert metadata.num_decode_tokens == 4
|
||||
assert metadata.state_indices_tensor_d is not None
|
||||
assert metadata.state_indices_tensor_d.shape == (3, 2)
|
||||
assert torch.equal(
|
||||
metadata.state_indices_tensor_d[2],
|
||||
torch.full((2,), PAD_SLOT_ID, dtype=torch.int32),
|
||||
)
|
||||
assert torch.equal(
|
||||
metadata.state_indices_tensor[2],
|
||||
torch.tensor(PAD_SLOT_ID, dtype=torch.int32),
|
||||
)
|
||||
assert metadata.query_start_loc_d is not None
|
||||
assert metadata.query_start_loc_d.tolist() == [0, 2, 4, 4]
|
||||
assert metadata.num_accepted_tokens is not None
|
||||
assert metadata.num_accepted_tokens.tolist() == [1, 2, 1]
|
||||
|
||||
|
||||
def test_linear_attention_full_graph_metadata_uses_stable_decode_buffers():
|
||||
vllm_config = create_vllm_config(
|
||||
hf_config_override={
|
||||
"architectures": ["BailingMoeV2_5ForCausalLM"],
|
||||
"model_type": "bailing_hybrid",
|
||||
}
|
||||
)
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
|
||||
builder = BailingLinearAttentionMetadataBuilder(
|
||||
kv_cache_spec=_create_mamba_spec(),
|
||||
layer_names=["model.layers.0.self_attn"],
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[20, 20, 0], query_lens=[2, 2, 0]),
|
||||
BLOCK_SIZE,
|
||||
DEVICE,
|
||||
arange_block_indices=True,
|
||||
)
|
||||
common.block_table_tensor = torch.tensor(
|
||||
[[10, 11], [12, 13], [-1, -1]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
first = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common,
|
||||
num_accepted_tokens=torch.tensor([1, 2, 1], dtype=torch.int32),
|
||||
)
|
||||
assert first.state_indices_tensor_d is not None
|
||||
assert first.query_start_loc_d is not None
|
||||
assert first.num_accepted_tokens is not None
|
||||
state_ptr = first.state_indices_tensor_d.data_ptr()
|
||||
query_ptr = first.query_start_loc_d.data_ptr()
|
||||
accepted_ptr = first.num_accepted_tokens.data_ptr()
|
||||
|
||||
common2 = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=[36, 0, 0], query_lens=[2, 0, 0]),
|
||||
BLOCK_SIZE,
|
||||
DEVICE,
|
||||
arange_block_indices=True,
|
||||
)
|
||||
common2.block_table_tensor = torch.tensor(
|
||||
[[20, 21], [-1, -1], [-1, -1]],
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
second = builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common2,
|
||||
num_accepted_tokens=torch.tensor([2, 1, 1], dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert second.state_indices_tensor_d is not None
|
||||
assert second.query_start_loc_d is not None
|
||||
assert second.num_accepted_tokens is not None
|
||||
assert second.state_indices_tensor_d.data_ptr() == state_ptr
|
||||
assert second.query_start_loc_d.data_ptr() == query_ptr
|
||||
assert second.num_accepted_tokens.data_ptr() == accepted_ptr
|
||||
assert second.state_indices_tensor_d.tolist() == [
|
||||
[20, 21],
|
||||
[PAD_SLOT_ID, PAD_SLOT_ID],
|
||||
[PAD_SLOT_ID, PAD_SLOT_ID],
|
||||
]
|
||||
assert second.query_start_loc_d.tolist() == [0, 2, 2, 2]
|
||||
assert second.num_accepted_tokens.tolist() == [2, 1, 1]
|
||||
Reference in New Issue
Block a user