[Bugfix] Fix speculative decoding for short_conv (LFM2) models (#50272)

Signed-off-by: jens.luecke <jens.luecke@liquid.ai>
Co-authored-by: Tonoken3 <sakamakismile@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jens
2026-08-22 13:34:22 +00:00
committed by GitHub
co-authored by Tonoken3 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 704f12aa1e
commit da329cc303
4 changed files with 43 additions and 11 deletions
@@ -235,9 +235,10 @@ class MambaStateShapeCalculator:
tp_world_size: int,
intermediate_size: int,
conv_kernel: int,
num_spec: int = 0,
) -> tuple[tuple[int, int]]:
conv_dim = divide(intermediate_size, tp_world_size)
conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1)
conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1 + num_spec)
return (conv_state_shape,)
@classmethod
+33 -9
View File
@@ -80,7 +80,9 @@ class ShortConv(MambaBase, PluggableLayer):
prefix=f"{prefix}.out_proj",
)
compilation_config = get_current_vllm_config().compilation_config
vllm_config = get_current_vllm_config()
self.num_spec = vllm_config.num_speculative_tokens
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@@ -234,6 +236,8 @@ class ShortConv(MambaBase, PluggableLayer):
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
has_initial_states_p = attn_metadata.has_initial_states_p
query_start_loc_p = attn_metadata.query_start_loc_p
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
BCx, _ = self.in_proj(hidden_states)
@@ -296,14 +300,33 @@ class ShortConv(MambaBase, PluggableLayer):
if has_decode:
Bx_d = (B_d * x_d).contiguous()
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
)
if num_accepted_tokens is not None:
# Speculative decode: the verify step feeds >1 query token per
# decode request, so use the spec-aware conv update path
# (mirrors mamba_mixer2). state_indices_tensor_d is
# (num_decodes, 1 + num_spec_tokens) here.
assert state_indices_tensor_d is not None
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
)
else:
# Non-spec decode
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
)
y = C_d * Bx
conv_output_list.insert(0, y)
@@ -326,6 +349,7 @@ class ShortConv(MambaBase, PluggableLayer):
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.conv_dim,
conv_kernel=self.L_cache,
num_spec=self.num_spec,
)
@property