From f2aaf5915102cd56b3a60f8e6e59c4a7f31268dd Mon Sep 17 00:00:00 2001 From: alex101-ops Date: Mon, 6 Jul 2026 10:38:50 +0800 Subject: [PATCH] [Feature] Support MTP speculative decoding for Bailing hybrid models (#44880) Signed-off-by: zc02384840 Co-authored-by: zc02384840 --- tests/config/test_bailing_mtp_config.py | 52 +++ tests/models/registry.py | 6 + .../test_linear_attention_metadata_builder.py | 188 +++++++++ vllm/config/speculative.py | 16 + .../mamba/linear/bailing_linear_attn.py | 325 ++++++++++++++- .../models/bailing_moe_linear.py | 4 + vllm/model_executor/models/bailing_moe_mtp.py | 380 ++++++++++++++++++ vllm/model_executor/models/registry.py | 1 + .../model_arch_config_convertor.py | 7 + vllm/v1/attention/backends/linear_attn.py | 213 +++++++++- vllm/v1/worker/gpu_model_runner.py | 14 +- 11 files changed, 1200 insertions(+), 6 deletions(-) create mode 100644 tests/config/test_bailing_mtp_config.py create mode 100644 tests/v1/attention/test_linear_attention_metadata_builder.py create mode 100644 vllm/model_executor/models/bailing_moe_mtp.py diff --git a/tests/config/test_bailing_mtp_config.py b/tests/config/test_bailing_mtp_config.py new file mode 100644 index 0000000000..8fae29959f --- /dev/null +++ b/tests/config/test_bailing_mtp_config.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from transformers import PretrainedConfig + +from vllm.config.speculative import MTPModelTypes, SpeculativeConfig +from vllm.transformers_utils.model_arch_config_convertor import ( + BailingHybridMTPModelArchConfigConvertor, +) + + +def _bailing_config() -> PretrainedConfig: + config = PretrainedConfig( + architectures=["BailingMoeV2_5ForCausalLM"], + hidden_size=4096, + kv_lora_rank=512, + num_attention_heads=32, + num_experts=256, + num_hidden_layers=32, + num_key_value_heads=32, + num_nextn_predict_layers=1, + qk_rope_head_dim=64, + vocab_size=157184, + ) + config.model_type = "bailing_hybrid" + return config + + +def test_bailing_hybrid_mtp_hf_config_override(): + config = _bailing_config() + + overridden = SpeculativeConfig.hf_config_override(config) + + assert overridden.model_type == "bailing_hybrid_mtp" + assert overridden.architectures == ["BailingMoeV25MTPModel"] + assert overridden.n_predict == 1 + assert "bailing_hybrid_mtp" in MTPModelTypes.__args__ + + +def test_bailing_hybrid_mtp_model_arch_config(): + config = _bailing_config() + config.model_type = "bailing_hybrid_mtp" + config.architectures = ["BailingMoeV25MTPModel"] + + model_arch_config = BailingHybridMTPModelArchConfigConvertor( + config, config + ).convert() + + assert model_arch_config.model_type == "bailing_hybrid_mtp" + assert model_arch_config.architectures == ["BailingMoeV25MTPModel"] + assert model_arch_config.total_num_hidden_layers == 1 + assert model_arch_config.is_deepseek_mla diff --git a/tests/models/registry.py b/tests/models/registry.py index 5d075d5b39..ee99be3f62 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1557,6 +1557,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { use_original_num_layers=True, ), # [MTP] + "BailingMoeV25MTPModel": _HfExamplesInfo( + "inclusionAI/Ring-2.5-1T", + speculative_model="inclusionAI/Ring-2.5-1T", + trust_remote_code=True, + is_available_online=False, + ), "DeepSeekMTPModel": _HfExamplesInfo( "luccafong/deepseek_mtp_main_random", speculative_model="luccafong/deepseek_mtp_draft_random", diff --git a/tests/v1/attention/test_linear_attention_metadata_builder.py b/tests/v1/attention/test_linear_attention_metadata_builder.py new file mode 100644 index 0000000000..3ef811b3a6 --- /dev/null +++ b/tests/v1/attention/test_linear_attention_metadata_builder.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from tests.v1.attention.utils import ( + BatchSpec, + create_common_attn_metadata, + create_vllm_config, +) +from vllm.config import CUDAGraphMode, SpeculativeConfig +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.linear_attn import ( + BailingLinearAttentionMetadataBuilder, + LinearAttentionMetadataBuilder, +) +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.kv_cache_interface import MambaSpec + +BLOCK_SIZE = 16 +DEVICE = torch.device("cpu") + + +def _create_mamba_spec(num_speculative_blocks: int = 1) -> MambaSpec: + return MambaSpec( + block_size=BLOCK_SIZE, + shapes=((16, 64),), + dtypes=(torch.float16,), + num_speculative_blocks=num_speculative_blocks, + ) + + +def test_bailing_linear_attention_reports_uniform_batch_cudagraph_support(): + vllm_config = create_vllm_config( + hf_config_override={ + "architectures": ["BailingMoeV2_5ForCausalLM"], + "model_type": "bailing_hybrid", + } + ) + + support = BailingLinearAttentionMetadataBuilder.get_cudagraph_support( + vllm_config, _create_mamba_spec() + ) + + assert support == AttentionCGSupport.UNIFORM_BATCH + + +def test_non_bailing_linear_attention_keeps_single_token_cudagraph_support(): + vllm_config = create_vllm_config( + hf_config_override={ + "architectures": ["MiniMaxText01ForCausalLM"], + "model_type": "minimax_text_01", + } + ) + + support = LinearAttentionMetadataBuilder.get_cudagraph_support( + vllm_config, _create_mamba_spec() + ) + + assert support == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + + +def test_linear_attention_spec_decode_full_graph_metadata_pads_cache_slots(): + vllm_config = create_vllm_config( + hf_config_override={ + "architectures": ["BailingMoeV2_5ForCausalLM"], + "model_type": "bailing_hybrid", + } + ) + vllm_config.speculative_config = SpeculativeConfig( + method="ngram", + num_speculative_tokens=1, + ) + vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY + + builder = BailingLinearAttentionMetadataBuilder( + kv_cache_spec=_create_mamba_spec(), + layer_names=["model.layers.0.self_attn"], + vllm_config=vllm_config, + device=DEVICE, + ) + + common = create_common_attn_metadata( + BatchSpec(seq_lens=[20, 20, 0], query_lens=[2, 2, 0]), + BLOCK_SIZE, + DEVICE, + ) + common.block_table_tensor[2].fill_(-1) + + metadata = builder.build( + common_prefix_len=0, + common_attn_metadata=common, + num_accepted_tokens=torch.tensor([1, 2, 1], dtype=torch.int32), + ) + + assert metadata.num_decodes == 3 + assert metadata.num_prefills == 0 + assert metadata.num_decode_tokens == 4 + assert metadata.state_indices_tensor_d is not None + assert metadata.state_indices_tensor_d.shape == (3, 2) + assert torch.equal( + metadata.state_indices_tensor_d[2], + torch.full((2,), PAD_SLOT_ID, dtype=torch.int32), + ) + assert torch.equal( + metadata.state_indices_tensor[2], + torch.tensor(PAD_SLOT_ID, dtype=torch.int32), + ) + assert metadata.query_start_loc_d is not None + assert metadata.query_start_loc_d.tolist() == [0, 2, 4, 4] + assert metadata.num_accepted_tokens is not None + assert metadata.num_accepted_tokens.tolist() == [1, 2, 1] + + +def test_linear_attention_full_graph_metadata_uses_stable_decode_buffers(): + vllm_config = create_vllm_config( + hf_config_override={ + "architectures": ["BailingMoeV2_5ForCausalLM"], + "model_type": "bailing_hybrid", + } + ) + vllm_config.speculative_config = SpeculativeConfig( + method="ngram", + num_speculative_tokens=1, + ) + vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY + + builder = BailingLinearAttentionMetadataBuilder( + kv_cache_spec=_create_mamba_spec(), + layer_names=["model.layers.0.self_attn"], + vllm_config=vllm_config, + device=DEVICE, + ) + + common = create_common_attn_metadata( + BatchSpec(seq_lens=[20, 20, 0], query_lens=[2, 2, 0]), + BLOCK_SIZE, + DEVICE, + arange_block_indices=True, + ) + common.block_table_tensor = torch.tensor( + [[10, 11], [12, 13], [-1, -1]], + dtype=torch.int32, + device=DEVICE, + ) + + first = builder.build( + common_prefix_len=0, + common_attn_metadata=common, + num_accepted_tokens=torch.tensor([1, 2, 1], dtype=torch.int32), + ) + assert first.state_indices_tensor_d is not None + assert first.query_start_loc_d is not None + assert first.num_accepted_tokens is not None + state_ptr = first.state_indices_tensor_d.data_ptr() + query_ptr = first.query_start_loc_d.data_ptr() + accepted_ptr = first.num_accepted_tokens.data_ptr() + + common2 = create_common_attn_metadata( + BatchSpec(seq_lens=[36, 0, 0], query_lens=[2, 0, 0]), + BLOCK_SIZE, + DEVICE, + arange_block_indices=True, + ) + common2.block_table_tensor = torch.tensor( + [[20, 21], [-1, -1], [-1, -1]], + dtype=torch.int32, + device=DEVICE, + ) + second = builder.build( + common_prefix_len=0, + common_attn_metadata=common2, + num_accepted_tokens=torch.tensor([2, 1, 1], dtype=torch.int32), + ) + + assert second.state_indices_tensor_d is not None + assert second.query_start_loc_d is not None + assert second.num_accepted_tokens is not None + assert second.state_indices_tensor_d.data_ptr() == state_ptr + assert second.query_start_loc_d.data_ptr() == query_ptr + assert second.num_accepted_tokens.data_ptr() == accepted_ptr + assert second.state_indices_tensor_d.tolist() == [ + [20, 21], + [PAD_SLOT_ID, PAD_SLOT_ID], + [PAD_SLOT_ID, PAD_SLOT_ID], + ] + assert second.query_start_loc_d.tolist() == [0, 2, 2, 2] + assert second.num_accepted_tokens.tolist() == [2, 1, 1] diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 983759bc9a..7fc4700fa2 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -46,6 +46,7 @@ MTPModelTypes = Literal[ "qwen3_5_mtp", "longcat_flash_mtp", "minimax_m3_mtp", + "bailing_hybrid_mtp", "mtp", "pangu_ultra_moe_mtp", "step3p5_mtp", @@ -463,6 +464,21 @@ class SpeculativeConfig: {"n_predict": n_predict, "architectures": ["Qwen3NextMTP"]} ) + architectures = getattr(hf_config, "architectures", []) or [] + if ( + hf_config.model_type == "bailing_hybrid" + or "BailingMoeV2_5ForCausalLM" in architectures + ): + hf_config.model_type = "bailing_hybrid_mtp" + if hf_config.model_type == "bailing_hybrid_mtp": + n_predict = getattr(hf_config, "num_nextn_predict_layers", None) + hf_config.update( + { + "n_predict": n_predict, + "architectures": ["BailingMoeV25MTPModel"], + } + ) + if hf_config.model_type == "exaone_moe": hf_config.model_type = "exaone_moe_mtp" if hf_config.model_type == "exaone_moe_mtp": diff --git a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py index dd963f829d..a00fbc74bf 100644 --- a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py @@ -30,11 +30,10 @@ from vllm.model_executor.layers.mamba.linear.base import LinearAttention from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import ( MiniMaxText01LinearAttention, MiniMaxText01LinearKernel, - clear_linear_attention_cache_for_new_sequences, linear_attention_decode, - linear_attention_prefill_and_mix, ) from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata @@ -57,6 +56,290 @@ def _build_rope_parameters(config: PretrainedConfig) -> dict | None: return rope_parameters or None +def clear_linear_attention_cache_for_new_sequences( + kv_cache: torch.Tensor, + state_indices_tensor: torch.Tensor, + attn_metadata: LinearAttentionMetadata, +) -> None: + num_prefills = getattr(attn_metadata, "num_prefills", 0) + if num_prefills <= 0: + return + + num_decodes = getattr(attn_metadata, "num_decodes", 0) + prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None) + for prefill_idx in range(num_prefills): + if num_decodes + prefill_idx + 1 >= len(attn_metadata.query_start_loc): + break + q_start = attn_metadata.query_start_loc[num_decodes + prefill_idx] + q_end = attn_metadata.query_start_loc[num_decodes + prefill_idx + 1] + query_len = q_end - q_start + context_len = attn_metadata.seq_lens[num_decodes + prefill_idx] - query_len + if context_len == 0: + if prefill_state_indices is not None: + block_to_clear = prefill_state_indices[prefill_idx] + else: + block_to_clear = state_indices_tensor[num_decodes + prefill_idx] + kv_cache[block_to_clear, ...] = 0 + + +def linear_attention_prefill_and_mix( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_cache: torch.Tensor, + state_indices_tensor: torch.Tensor, + attn_metadata: LinearAttentionMetadata, + slope_rate: torch.Tensor, + block_size: int, + decode_fn, + prefix_fn, + layer_idx: int | None = None, +) -> torch.Tensor: + hidden = [] + req_offset = getattr(attn_metadata, "num_decodes", 0) + prefill_state_indices = getattr(attn_metadata, "state_indices_tensor_p", None) + for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)): + if req_offset + _prefill_idx + 1 >= len(attn_metadata.query_start_loc): + break + if prefill_state_indices is not None and _prefill_idx >= len( + prefill_state_indices + ): + break + if prefill_state_indices is None and _prefill_idx >= len(state_indices_tensor): + break + _start = attn_metadata.query_start_loc[req_offset + _prefill_idx] + _end = attn_metadata.query_start_loc[req_offset + _prefill_idx + 1] + if prefill_state_indices is not None: + slot_id = prefill_state_indices[_prefill_idx] + else: + slot_id = state_indices_tensor[req_offset + _prefill_idx] + qs = q[_start:_end].transpose(0, 1).contiguous() + ks = k[_start:_end].transpose(0, 1).contiguous() + vs = v[_start:_end].transpose(0, 1).contiguous() + slice_layer_cache = kv_cache[slot_id, ...] + out_slice = prefix_fn( + qs, + ks, + vs, + slice_layer_cache, + slope_rate, + block_size, + layer_idx=layer_idx, + ) + hidden.append(out_slice.contiguous()) + + if attn_metadata.num_decode_tokens > 0: + hidden_decode = decode_fn( + q, k, v, kv_cache, state_indices_tensor, attn_metadata + ) + hidden.insert(0, hidden_decode) + + if not hidden: + return torch.empty((0, q.size(1) * q.size(2)), device=q.device, dtype=q.dtype) + + hidden = torch.concat(hidden, dim=0).contiguous() + return hidden + + +@triton.jit +def _bailing_linear_attn_decode_spec_step_kernel( + q_ptr, + k_ptr, + v_ptr, + kv_cache_ptr, + slope_rate_ptr, + state_indices_ptr, + query_start_loc_ptr, + num_accepted_tokens_ptr, + output_ptr, + q_start: tl.constexpr, + D: tl.constexpr, + q_b_stride, + q_h_stride, + q_d_stride, + k_b_stride, + k_h_stride, + k_d_stride, + v_b_stride, + v_h_stride, + v_d_stride, + cache_b_stride, + cache_h_stride, + cache_d0_stride, + cache_d1_stride, + state_indices_b_stride, + state_indices_t_stride, + output_b_stride, + output_d_stride, + DRAFT_IDX: tl.constexpr, + STATE_WIDTH: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + req_id = tl.program_id(0) + head_id = tl.program_id(1) + block_id = tl.program_id(2) + + req_start = tl.load(query_start_loc_ptr + req_id).to(tl.int64) + req_end = tl.load(query_start_loc_ptr + req_id + 1).to(tl.int64) + query_len = req_end - req_start + if query_len <= DRAFT_IDX: + return + + dst_slot = tl.load( + state_indices_ptr + + req_id * state_indices_b_stride + + DRAFT_IDX * state_indices_t_stride + ).to(tl.int64) + if dst_slot == -1: + return + + if DRAFT_IDX == 0: + accepted_offset = tl.load(num_accepted_tokens_ptr + req_id).to(tl.int64) - 1 + accepted_offset = tl.maximum(accepted_offset, 0) + accepted_offset = tl.minimum(accepted_offset, STATE_WIDTH - 1) + src_slot = tl.load( + state_indices_ptr + + req_id * state_indices_b_stride + + accepted_offset * state_indices_t_stride + ).to(tl.int64) + else: + src_slot = tl.load( + state_indices_ptr + + req_id * state_indices_b_stride + + (DRAFT_IDX - 1) * state_indices_t_stride + ).to(tl.int64) + if src_slot == -1: + return + + token_idx = req_start - q_start + DRAFT_IDX + qk_offsets = tl.arange(0, D) + v_offsets = tl.arange(0, BLOCK_SIZE) + block_id * BLOCK_SIZE + qk_mask = qk_offsets < D + v_mask = v_offsets < D + kv_mask = qk_mask[:, None] & v_mask[None, :] + + q = tl.load( + q_ptr + token_idx * q_b_stride + head_id * q_h_stride + qk_offsets * q_d_stride, + mask=qk_mask, + other=0.0, + ) + k = tl.load( + k_ptr + token_idx * k_b_stride + head_id * k_h_stride + qk_offsets * k_d_stride, + mask=qk_mask, + other=0.0, + ) + v = tl.load( + v_ptr + token_idx * v_b_stride + head_id * v_h_stride + v_offsets * v_d_stride, + mask=v_mask, + other=0.0, + ) + + cache_offsets = ( + qk_offsets[:, None] * cache_d0_stride + v_offsets[None, :] * cache_d1_stride + ) + src_cache_ptr = ( + kv_cache_ptr + + src_slot * cache_b_stride + + head_id * cache_h_stride + + cache_offsets + ) + dst_cache_ptr = ( + kv_cache_ptr + + dst_slot * cache_b_stride + + head_id * cache_h_stride + + cache_offsets + ) + + slope = tl.load(slope_rate_ptr + head_id) + decay = tl.exp(-slope) + kv_old = tl.load(src_cache_ptr, mask=kv_mask, other=0.0) + kv_new = k[:, None] * v[None, :] + decay * kv_old + + output = tl.sum(q[:, None].to(tl.float32) * kv_new, axis=0) + tl.store(dst_cache_ptr, kv_new, mask=kv_mask) + tl.store( + output_ptr + + token_idx * output_b_stride + + (head_id * D + v_offsets) * output_d_stride, + output, + mask=v_mask, + ) + + +def bailing_linear_attention_decode_spec( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_cache: torch.Tensor, + slope_rate: torch.Tensor, + state_indices_tensor: torch.Tensor, + query_start_loc: torch.Tensor, + num_accepted_tokens: torch.Tensor, + q_start: int, + q_end: int | None, + slot_start: int, + slot_end: int | None, + block_size: int, +) -> torch.Tensor: + q_decode = q[q_start:q_end] + k_decode = k[q_start:q_end] + v_decode = v[q_start:q_end] + hidden = torch.empty( + (q_decode.shape[0], q.shape[1] * q.shape[2]), + device=q.device, + dtype=q.dtype, + ) + hidden.zero_() + + state_indices_tensor = state_indices_tensor[slot_start:slot_end] + query_start_loc = query_start_loc.to(device=q.device) + + batch_size = state_indices_tensor.shape[0] + num_heads = q_decode.shape[1] + head_dim = q_decode.shape[2] + assert k_decode.shape == (q_decode.shape[0], num_heads, head_dim) + assert v_decode.shape == (q_decode.shape[0], num_heads, head_dim) + state_width = state_indices_tensor.shape[1] + + grid = (batch_size, num_heads, triton.cdiv(head_dim, block_size)) + for draft_idx in range(state_width): + _bailing_linear_attn_decode_spec_step_kernel[grid]( + q_decode, + k_decode, + v_decode, + kv_cache, + slope_rate, + state_indices_tensor, + query_start_loc, + num_accepted_tokens[:batch_size], + hidden, + q_start, + head_dim, + q_decode.stride(0), + q_decode.stride(1), + q_decode.stride(2), + k_decode.stride(0), + k_decode.stride(1), + k_decode.stride(2), + v_decode.stride(0), + v_decode.stride(1), + v_decode.stride(2), + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + kv_cache.stride(3), + state_indices_tensor.stride(0), + state_indices_tensor.stride(1), + hidden.stride(0), + hidden.stride(1), + DRAFT_IDX=draft_idx, + STATE_WIDTH=state_width, + BLOCK_SIZE=block_size, + ) + + return hidden + + class BailingGroupRMSNormGate(RMSNormGated): def __init__( self, @@ -208,6 +491,13 @@ class BailingMoELinearAttention(LinearAttention): raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + def get_attn_backend(self): + from vllm.v1.attention.backends.linear_attn import ( + BailingLinearAttentionBackend, + ) + + return BailingLinearAttentionBackend + @staticmethod def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: """Load weight for linear attention layers. @@ -368,13 +658,42 @@ class BailingMoELinearAttention(LinearAttention): def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata): """Handle decode (single token per sequence).""" + decode_state_indices = getattr(attn_metadata, "state_indices_tensor_d", None) + num_accepted_tokens = getattr(attn_metadata, "num_accepted_tokens", None) + query_start_loc = getattr(attn_metadata, "query_start_loc_d", None) + if ( + decode_state_indices is not None + and decode_state_indices.dim() > 1 + and num_accepted_tokens is not None + and query_start_loc is not None + ): + return bailing_linear_attention_decode_spec( + q, + k, + v, + kv_cache, + self.tp_slope, + decode_state_indices, + query_start_loc, + num_accepted_tokens, + q_start=0, + q_end=attn_metadata.num_decode_tokens, + slot_start=0, + slot_end=attn_metadata.num_decodes, + block_size=32, + ) + decode_state_indices = ( + state_indices_tensor + if decode_state_indices is None + else decode_state_indices + ) hidden = linear_attention_decode( q, k, v, kv_cache, self.tp_slope, - state_indices_tensor, + decode_state_indices, q_start=0, q_end=attn_metadata.num_decode_tokens, slot_start=0, diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index 7bf5fb0407..5cc057c3b4 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -533,6 +533,10 @@ class BailingMoeV25Model(nn.Module): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.word_embeddings(input_ids) + @property + def embed_tokens(self) -> nn.Module: + return self.word_embeddings + def forward( self, input_ids: torch.Tensor | None, diff --git a/vllm/model_executor/models/bailing_moe_mtp.py b/vllm/model_executor/models/bailing_moe_mtp.py new file mode 100644 index 0000000000..da6b1ddb8b --- /dev/null +++ b/vllm/model_executor/models/bailing_moe_mtp.py @@ -0,0 +1,380 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only Bailing MoE v2.5 MTP model.""" + +from collections.abc import Iterable + +import torch +import torch.nn as nn +from transformers.configuration_utils import PretrainedConfig + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig +from vllm.distributed import get_pp_group +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.model_executor.models.bailing_moe_linear import ( + BailingMoeV25, + BailingMoeV25MLAAttention, +) +from vllm.sequence import IntermediateTensors + +from .utils import PPMissingLayer, is_pp_missing_parameter, maybe_prefix + + +def _get_draft_hf_config(vllm_config: VllmConfig) -> PretrainedConfig: + speculative_config = vllm_config.speculative_config + if speculative_config is not None: + draft_model_config = speculative_config.draft_model_config + if draft_model_config is not None: + return draft_model_config.hf_config + return vllm_config.model_config.hf_config + + +class BailingMTPSharedHead(nn.Module): + def __init__( + self, + config: PretrainedConfig, + prefix: str, + vllm_config: VllmConfig, + ) -> None: + super().__init__() + self.head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=vllm_config.quant_config, + prefix=maybe_prefix(prefix, "head"), + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states + + +class BailingMoeV25MultiTokenPredictorLayer(nn.Module): + def __init__( + self, + vllm_config: VllmConfig, + prefix: str, + layer_id: int, + ) -> None: + super().__init__() + config = _get_draft_hf_config(vllm_config) + self.config = config + self.layer_id = layer_id + self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.self_attn = BailingMoeV25MLAAttention( + config, + quant_config=vllm_config.quant_config, + layer_id=layer_id, + prefix=maybe_prefix(prefix, "self_attn"), + cache_config=vllm_config.cache_config, + ) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.mlp = BailingMoeV25( + config, + quant_config=vllm_config.quant_config, + layer_id=layer_id, + prefix=maybe_prefix(prefix, "mlp"), + ) + self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.shared_head = BailingMTPSharedHead( + config, + maybe_prefix(prefix, "shared_head"), + vllm_config, + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + spec_step_index: int = 0, + ) -> torch.Tensor: + assert inputs_embeds is not None + inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds) + inputs_embeds = self.enorm(inputs_embeds) + previous_hidden_states = self.hnorm(previous_hidden_states) + + hidden_states = self.eh_proj( + torch.cat([inputs_embeds, previous_hidden_states], dim=-1) + ) + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + hidden_states = self.self_attn(hidden_states, positions) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states.to(residual.device) + return self.final_layernorm(hidden_states) + + +class BailingMoeV25MultiTokenPredictor(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config = _get_draft_hf_config(vllm_config) + self.mtp_start_layer_idx = config.num_hidden_layers + self.num_mtp_layers = config.num_nextn_predict_layers + self.layers = nn.ModuleDict( + { + str(idx): BailingMoeV25MultiTokenPredictorLayer( + vllm_config, + f"{prefix}.layers.{idx}", + idx, + ) + for idx in range( + self.mtp_start_layer_idx, + self.mtp_start_layer_idx + self.num_mtp_layers, + ) + } + ) + if get_pp_group().is_first_rank: + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + prefix=maybe_prefix(prefix, "embed_tokens"), + ) + else: + self.embed_tokens = PPMissingLayer() + self.logits_processor = LogitsProcessor(config.vocab_size) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + current_step_idx = spec_step_idx % self.num_mtp_layers + return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( + input_ids, + positions, + previous_hidden_states, + inputs_embeds, + current_step_idx, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + lm_head: nn.Module | None = None, + ) -> torch.Tensor: + current_step_idx = spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + head = lm_head if lm_head is not None else mtp_layer.shared_head.head + return self.logits_processor( + head, + mtp_layer.shared_head(hidden_states), + ) + + +@support_torch_compile +class BailingMoeV25MTPModel(nn.Module): + packed_modules_mapping = { + "gate_up_proj": ["gate_proj", "up_proj"], + "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + self.config = _get_draft_hf_config(vllm_config) + self.lm_head: nn.Module | None = None + self.model = BailingMoeV25MultiTokenPredictor( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + hidden_states: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + return self.model( + input_ids, + positions, + hidden_states, + inputs_embeds, + spec_step_idx, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + return self.model.compute_logits(hidden_states, spec_step_idx, self.lm_head) + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts, + num_redundant_experts=0, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + (".fused_qkv_a_proj", ".q_a_proj", 0), + (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + expert_params_mapping = list(self.get_expert_mapping()) + params_dict = dict(self.named_parameters(remove_duplicate=False)) + loaded_params: set[str] = set() + loaded_mtp_layers: set[int] = set() + + def load_param( + name: str, + loaded_weight: torch.Tensor, + shard_id=None, + ) -> bool: + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + return False + if name not in params_dict or is_pp_missing_parameter(name, self): + return False + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + if shard_id is None: + weight_loader(param, loaded_weight) + elif isinstance(shard_id, int): + weight_loader(param, loaded_weight, shard_id) + else: + weight_loader( + param, + loaded_weight, + name, + expert_id=shard_id[0], + shard_id=shard_id[1], + ) + loaded_params.add(name) + return True + + def get_spec_layer_idx(name: str) -> int | None: + if not name.startswith("model.layers."): + return None + try: + layer_idx = int(name.split("model.layers.", 1)[1].split(".", 1)[0]) + except (IndexError, ValueError): + return None + mtp_idx = layer_idx - self.config.num_hidden_layers + if 0 <= mtp_idx < self.config.num_nextn_predict_layers: + return layer_idx + return None + + def normalize_name(name: str) -> str: + name = name.replace(".attention.dense", ".self_attn.o_proj") + name = name.replace(".attention.", ".self_attn.") + return name.replace( + "mlp.gate.e_score_correction_bias", + "mlp.gate.expert_bias", + ) + + def load_lm_head(loaded_weight: torch.Tensor) -> None: + for layer_idx in range( + self.model.mtp_start_layer_idx, + self.model.mtp_start_layer_idx + self.model.num_mtp_layers, + ): + name = f"model.layers.{layer_idx}.shared_head.head.weight" + load_param(name, loaded_weight) + + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + + if name == "model.word_embeddings.weight": + load_param("model.embed_tokens.weight", loaded_weight) + continue + if name == "lm_head.weight": + load_lm_head(loaded_weight) + continue + + spec_layer = get_spec_layer_idx(name) + if spec_layer is None: + continue + name = normalize_name(name) + + loaded = False + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + if "mlp.experts." in name and name not in params_dict: + continue + mapped_name = name.replace(weight_name, param_name) + if load_param(mapped_name, loaded_weight, shard_id): + loaded = True + break + if loaded: + loaded_mtp_layers.add(spec_layer) + continue + + if "mlp.experts" in name: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + mapped_name = name.replace(weight_name, param_name) + if load_param( + mapped_name, + loaded_weight, + (expert_id, shard_id), + ): + loaded = True + break + if loaded: + loaded_mtp_layers.add(spec_layer) + continue + + if load_param(name, loaded_weight): + loaded_mtp_layers.add(spec_layer) + + for layer_idx in range( + self.model.mtp_start_layer_idx, + self.model.mtp_start_layer_idx + self.model.num_mtp_layers, + ): + if layer_idx not in loaded_mtp_layers: + raise ValueError( + f"Bailing MTP speculative decoding layer {layer_idx} " + "weights are missing from checkpoint. Use a checkpoint " + "that includes MTP layer weights, or disable speculative " + "decoding." + ) + return loaded_params diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 44ce9a8817..07edf2a62d 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -612,6 +612,7 @@ _SPECULATIVE_DECODING_MODELS = { "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"), + "BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"), "Gemma4MTPModel": ("gemma4_mtp", "Gemma4MTP"), "ErnieMTPModel": ("ernie_mtp", "ErnieMTP"), "ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"), diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index 3b39c91109..7ced083afb 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -271,6 +271,7 @@ class ModelArchConfigConvertorBase: "pangu_ultra_moe", "pangu_ultra_moe_mtp", "bailing_hybrid", + "bailing_hybrid_mtp", ): # check is deepseek_v4 model if hasattr(self.hf_text_config, "compress_ratios"): @@ -539,6 +540,11 @@ class Qwen3NextMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): return getattr(self.hf_text_config, "num_nextn_predict_layers", 0) +class BailingHybridMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): + def get_num_hidden_layers(self) -> int: + return getattr(self.hf_text_config, "num_nextn_predict_layers", 0) + + class Qwen3_5MTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def get_num_hidden_layers(self) -> int: return getattr(self.hf_text_config, "mtp_num_hidden_layers", 0) @@ -633,6 +639,7 @@ class MossAudioModelArchConfigConvertor(ModelArchConfigConvertorBase): # hf_config.model_type -> convertor class MODEL_ARCH_CONFIG_CONVERTORS = { + "bailing_hybrid_mtp": BailingHybridMTPModelArchConfigConvertor, "cohere_asr": CohereAsrModelArchConfigConvertor, "dbrx": DbrxModelArchConfigConvertor, "deepseek_mtp": DeepSeekMTPModelArchConfigConvertor, diff --git a/vllm/v1/attention/backends/linear_attn.py b/vllm/v1/attention/backends/linear_attn.py index b2ca151986..9cdcf0e30e 100644 --- a/vllm/v1/attention/backends/linear_attn.py +++ b/vllm/v1/attention/backends/linear_attn.py @@ -4,7 +4,7 @@ from dataclasses import dataclass import torch -from vllm.config import VllmConfig +from vllm.config import CompilationConfig, VllmConfig from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, @@ -12,6 +12,7 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, ) from vllm.v1.attention.backends.utils import ( + PAD_SLOT_ID, mamba_get_block_table_tensor, split_decodes_and_prefills, ) @@ -91,3 +92,213 @@ class LinearAttentionMetadataBuilder(AttentionMetadataBuilder[LinearAttentionMet state_indices_tensor=state_indices_tensor, ) return attn_metadata + + +class BailingLinearAttentionBackend(LinearAttentionBackend): + @staticmethod + def get_name() -> str: + return "BAILING_LINEAR_ATTN" + + @staticmethod + def get_builder_cls() -> type["BailingLinearAttentionMetadataBuilder"]: + return BailingLinearAttentionMetadataBuilder + + +@dataclass +class BailingLinearAttentionMetadata(LinearAttentionMetadata): + state_indices_tensor_d: torch.Tensor | None = None + state_indices_tensor_p: torch.Tensor | None = None + num_accepted_tokens: torch.Tensor | None = None + query_start_loc_d: torch.Tensor | None = None + + +class BailingLinearAttentionMetadataBuilder(LinearAttentionMetadataBuilder): + supports_spec_decode_metadata = True + supports_update_block_table: bool = False + + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: AttentionSpec, + ) -> AttentionCGSupport: + return AttentionCGSupport.UNIFORM_BATCH + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ): + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + self.compilation_config: CompilationConfig = vllm_config.compilation_config + self.num_spec_tokens: int = vllm_config.num_speculative_tokens + self.use_spec_decode: bool = self.num_spec_tokens > 0 + self.decode_cudagraph_max_bs: int = vllm_config.scheduler_config.max_num_seqs + if self.compilation_config.max_cudagraph_capture_size is not None: + self.decode_cudagraph_max_bs = min( + self.decode_cudagraph_max_bs, + self.compilation_config.max_cudagraph_capture_size, + ) + self.decode_state_indices_tensor: torch.Tensor = torch.empty( + (self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens), + dtype=torch.int32, + device=device, + ) + self.decode_legacy_state_indices_tensor: torch.Tensor = torch.empty( + (self.decode_cudagraph_max_bs,), + dtype=torch.int32, + device=device, + ) + self.decode_query_start_loc: torch.Tensor = torch.empty( + (self.decode_cudagraph_max_bs + 1,), + dtype=torch.int32, + device=device, + ) + self.decode_num_accepted_tokens: torch.Tensor = torch.empty( + (self.decode_cudagraph_max_bs,), + dtype=torch.int32, + device=device, + ) + self._init_reorder_batch_threshold(1, self.use_spec_decode) + + def build_for_cudagraph_capture( + self, + common_attn_metadata: CommonAttentionMetadata, + ) -> BailingLinearAttentionMetadata: + num_accepted_tokens = None + if self.use_spec_decode: + assert common_attn_metadata.max_query_len <= 1 + self.num_spec_tokens, ( + "Bailing linear attention only supports speculative decoding " + "with query length <= 1 + number of speculative tokens." + ) + num_accepted_tokens = torch.diff(common_attn_metadata.query_start_loc) + return self.build( + common_prefix_len=0, + common_attn_metadata=common_attn_metadata, + num_accepted_tokens=num_accepted_tokens, + ) + + def build( # type: ignore[override] + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + *, + num_accepted_tokens: torch.Tensor | None = None, + num_decode_draft_tokens_cpu: torch.Tensor | None = None, + ) -> BailingLinearAttentionMetadata: + query_start_loc = common_attn_metadata.query_start_loc + seq_lens = common_attn_metadata.seq_lens + num_reqs = common_attn_metadata.num_reqs + use_spec_decode = self.use_spec_decode and num_accepted_tokens is not None + + state_indices_tensor = mamba_get_block_table_tensor( + common_attn_metadata.block_table_tensor, + common_attn_metadata.seq_lens, + self.kv_cache_spec, + self.vllm_config.cache_config.mamba_cache_mode, + ) + if state_indices_tensor.dim() == 1: + state_indices_tensor = state_indices_tensor.unsqueeze(-1) + + decode_threshold = self.reorder_batch_threshold if use_spec_decode else 1 + num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( + split_decodes_and_prefills( + common_attn_metadata, + decode_threshold=decode_threshold, + ) + ) + state_indices_tensor_d, state_indices_tensor_p = torch.split( + state_indices_tensor, + [num_decodes, num_prefills], + dim=0, + ) + state_indices_tensor_p = state_indices_tensor_p[:, 0] + + query_start_loc_d = None + if use_spec_decode: + assert num_accepted_tokens is not None + state_indices_tensor_d = state_indices_tensor_d[ + :, : 1 + self.num_spec_tokens + ] + query_start_loc_d = query_start_loc[: num_decodes + 1] + num_accepted_tokens = num_accepted_tokens[:num_decodes] + else: + state_indices_tensor_d = state_indices_tensor_d[:, 0] + num_accepted_tokens = None + + legacy_state_indices_tensor = state_indices_tensor[:, 0] + cudagraph_mode = self.compilation_config.cudagraph_mode + use_full_cudagraph = ( + cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs() + ) + if ( + num_prefills == 0 + and num_decodes <= self.decode_cudagraph_max_bs + and use_full_cudagraph + ): + padded_bs = num_reqs + is_padded_decode = seq_lens[:num_decodes] == 0 + if state_indices_tensor_d.dim() > 1: + state_indices_tensor_d = torch.where( + is_padded_decode.unsqueeze(1), + torch.full_like(state_indices_tensor_d, PAD_SLOT_ID), + state_indices_tensor_d, + ) + self.decode_state_indices_tensor[:num_decodes].copy_( + state_indices_tensor_d, + non_blocking=True, + ) + state_indices_tensor_d = self.decode_state_indices_tensor[:padded_bs] + state_indices_tensor_d[num_decodes:] = PAD_SLOT_ID + + self.decode_legacy_state_indices_tensor[:num_decodes].copy_( + torch.where( + is_padded_decode, + torch.full_like( + legacy_state_indices_tensor[:num_decodes], + PAD_SLOT_ID, + ), + legacy_state_indices_tensor[:num_decodes], + ), + non_blocking=True, + ) + legacy_state_indices_tensor = self.decode_legacy_state_indices_tensor[ + :padded_bs + ] + legacy_state_indices_tensor[num_decodes:] = PAD_SLOT_ID + if state_indices_tensor_d.dim() == 1: + state_indices_tensor_d = legacy_state_indices_tensor + + if use_spec_decode and num_accepted_tokens is not None: + assert query_start_loc_d is not None + self.decode_query_start_loc[: num_decodes + 1].copy_( + query_start_loc_d, + non_blocking=True, + ) + decode_num_query_tokens = query_start_loc_d[-1] + query_start_loc_d = self.decode_query_start_loc[: padded_bs + 1] + query_start_loc_d[num_decodes + 1 :] = decode_num_query_tokens + + self.decode_num_accepted_tokens[:num_decodes].copy_( + num_accepted_tokens, + non_blocking=True, + ) + num_accepted_tokens = self.decode_num_accepted_tokens[:padded_bs] + num_accepted_tokens[num_decodes:] = 1 + + return BailingLinearAttentionMetadata( + num_prefills=num_prefills, + num_prefill_tokens=num_prefill_tokens, + num_decodes=num_decodes, + num_decode_tokens=num_decode_tokens, + query_start_loc=query_start_loc, + seq_lens=seq_lens, + state_indices_tensor=legacy_state_indices_tensor, + state_indices_tensor_d=state_indices_tensor_d, + state_indices_tensor_p=state_indices_tensor_p, + num_accepted_tokens=num_accepted_tokens, + query_start_loc_d=query_start_loc_d, + ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 3930a07b24..f83ed8a8ac 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -132,6 +132,9 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, ) from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder +from vllm.v1.attention.backends.linear_attn import ( + BailingLinearAttentionMetadataBuilder, +) from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder from vllm.v1.attention.backends.utils import ( NULL_BLOCK_ID, @@ -2442,9 +2445,16 @@ class GPUModelRunner( extra_attn_metadata_args = {} if use_spec_decode and isinstance( - builder, (Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder) + builder, + ( + Mamba2AttentionMetadataBuilder, + GDNAttentionMetadataBuilder, + BailingLinearAttentionMetadataBuilder, + ), ): - assert ubid is None, "UBatching not supported with GDN yet" + assert ubid is None, ( + "UBatching not supported with GDN or linear attn yet" + ) extra_attn_metadata_args = dict( num_accepted_tokens=self.num_accepted_tokens.gpu[:num_reqs_padded], num_decode_draft_tokens_cpu=self.num_decode_draft_tokens.cpu[