[Bugfix] Fix speculative decoding for short_conv (LFM2) models (#50272)

Signed-off-by: jens.luecke <jens.luecke@liquid.ai>
Co-authored-by: Tonoken3 <sakamakismile@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jens
2026-08-22 13:34:22 +00:00
committed by GitHub
co-authored by Tonoken3 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 704f12aa1e
commit da329cc303
4 changed files with 43 additions and 11 deletions
@@ -235,9 +235,10 @@ class MambaStateShapeCalculator:
tp_world_size: int, tp_world_size: int,
intermediate_size: int, intermediate_size: int,
conv_kernel: int, conv_kernel: int,
num_spec: int = 0,
) -> tuple[tuple[int, int]]: ) -> tuple[tuple[int, int]]:
conv_dim = divide(intermediate_size, tp_world_size) conv_dim = divide(intermediate_size, tp_world_size)
conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1) conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1 + num_spec)
return (conv_state_shape,) return (conv_state_shape,)
@classmethod @classmethod
+33 -9
View File
@@ -80,7 +80,9 @@ class ShortConv(MambaBase, PluggableLayer):
prefix=f"{prefix}.out_proj", prefix=f"{prefix}.out_proj",
) )
compilation_config = get_current_vllm_config().compilation_config vllm_config = get_current_vllm_config()
self.num_spec = vllm_config.num_speculative_tokens
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context: if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}") raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self compilation_config.static_forward_context[prefix] = self
@@ -234,6 +236,8 @@ class ShortConv(MambaBase, PluggableLayer):
state_indices_tensor_d = attn_metadata.state_indices_tensor_d state_indices_tensor_d = attn_metadata.state_indices_tensor_d
has_initial_states_p = attn_metadata.has_initial_states_p has_initial_states_p = attn_metadata.has_initial_states_p
query_start_loc_p = attn_metadata.query_start_loc_p query_start_loc_p = attn_metadata.query_start_loc_p
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
BCx, _ = self.in_proj(hidden_states) BCx, _ = self.in_proj(hidden_states)
@@ -296,14 +300,33 @@ class ShortConv(MambaBase, PluggableLayer):
if has_decode: if has_decode:
Bx_d = (B_d * x_d).contiguous() Bx_d = (B_d * x_d).contiguous()
Bx = causal_conv1d_update( if num_accepted_tokens is not None:
Bx_d, # Speculative decode: the verify step feeds >1 query token per
conv_state, # decode request, so use the spec-aware conv update path
conv_weights, # (mirrors mamba_mixer2). state_indices_tensor_d is
self.conv.bias, # (num_decodes, 1 + num_spec_tokens) here.
activation=None, assert state_indices_tensor_d is not None
conv_state_indices=state_indices_tensor_d, Bx = causal_conv1d_update(
) Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
)
else:
# Non-spec decode
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
)
y = C_d * Bx y = C_d * Bx
conv_output_list.insert(0, y) conv_output_list.insert(0, y)
@@ -326,6 +349,7 @@ class ShortConv(MambaBase, PluggableLayer):
tp_world_size=get_tensor_model_parallel_world_size(), tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.conv_dim, intermediate_size=self.conv_dim,
conv_kernel=self.L_cache, conv_kernel=self.L_cache,
num_spec=self.num_spec,
) )
@property @property
@@ -12,6 +12,7 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.triton_utils import tl, triton from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder
from vllm.v1.core.sched.output import NewRequestData from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.utils import CpuGpuBuffer from vllm.v1.utils import CpuGpuBuffer
@@ -48,7 +49,11 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
) -> dict[str, Any]: ) -> dict[str, Any]:
if not isinstance( if not isinstance(
attn_metadata_builder, attn_metadata_builder,
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder), (
Mamba2AttentionMetadataBuilder,
GDNAttentionMetadataBuilder,
ShortConvAttentionMetadataBuilder,
),
): ):
return {} return {}
return { return {
+2
View File
@@ -148,6 +148,7 @@ from vllm.v1.attention.backends.linear_attn import (
BailingLinearAttentionMetadataBuilder, BailingLinearAttentionMetadataBuilder,
) )
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder
from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.backends.utils import (
NULL_BLOCK_ID, NULL_BLOCK_ID,
create_fast_prefill_custom_backend, create_fast_prefill_custom_backend,
@@ -2572,6 +2573,7 @@ class GPUModelRunner(
Mamba2AttentionMetadataBuilder, Mamba2AttentionMetadataBuilder,
GDNAttentionMetadataBuilder, GDNAttentionMetadataBuilder,
BailingLinearAttentionMetadataBuilder, BailingLinearAttentionMetadataBuilder,
ShortConvAttentionMetadataBuilder,
), ),
): ):
assert ubid is None, ( assert ubid is None, (