[Bugfix] Fix speculative decoding for short_conv (LFM2) models (#50272)
Signed-off-by: jens.luecke <jens.luecke@liquid.ai> Co-authored-by: Tonoken3 <sakamakismile@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Tonoken3
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
704f12aa1e
commit
da329cc303
@@ -235,9 +235,10 @@ class MambaStateShapeCalculator:
|
|||||||
tp_world_size: int,
|
tp_world_size: int,
|
||||||
intermediate_size: int,
|
intermediate_size: int,
|
||||||
conv_kernel: int,
|
conv_kernel: int,
|
||||||
|
num_spec: int = 0,
|
||||||
) -> tuple[tuple[int, int]]:
|
) -> tuple[tuple[int, int]]:
|
||||||
conv_dim = divide(intermediate_size, tp_world_size)
|
conv_dim = divide(intermediate_size, tp_world_size)
|
||||||
conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1)
|
conv_state_shape = cls._orient_conv_shape(conv_dim, conv_kernel - 1 + num_spec)
|
||||||
return (conv_state_shape,)
|
return (conv_state_shape,)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -80,7 +80,9 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
prefix=f"{prefix}.out_proj",
|
prefix=f"{prefix}.out_proj",
|
||||||
)
|
)
|
||||||
|
|
||||||
compilation_config = get_current_vllm_config().compilation_config
|
vllm_config = get_current_vllm_config()
|
||||||
|
self.num_spec = vllm_config.num_speculative_tokens
|
||||||
|
compilation_config = vllm_config.compilation_config
|
||||||
if prefix in compilation_config.static_forward_context:
|
if prefix in compilation_config.static_forward_context:
|
||||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||||
compilation_config.static_forward_context[prefix] = self
|
compilation_config.static_forward_context[prefix] = self
|
||||||
@@ -234,6 +236,8 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
||||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||||
|
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||||
|
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||||
|
|
||||||
BCx, _ = self.in_proj(hidden_states)
|
BCx, _ = self.in_proj(hidden_states)
|
||||||
|
|
||||||
@@ -296,14 +300,33 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
|
|
||||||
if has_decode:
|
if has_decode:
|
||||||
Bx_d = (B_d * x_d).contiguous()
|
Bx_d = (B_d * x_d).contiguous()
|
||||||
Bx = causal_conv1d_update(
|
if num_accepted_tokens is not None:
|
||||||
Bx_d,
|
# Speculative decode: the verify step feeds >1 query token per
|
||||||
conv_state,
|
# decode request, so use the spec-aware conv update path
|
||||||
conv_weights,
|
# (mirrors mamba_mixer2). state_indices_tensor_d is
|
||||||
self.conv.bias,
|
# (num_decodes, 1 + num_spec_tokens) here.
|
||||||
activation=None,
|
assert state_indices_tensor_d is not None
|
||||||
conv_state_indices=state_indices_tensor_d,
|
Bx = causal_conv1d_update(
|
||||||
)
|
Bx_d,
|
||||||
|
conv_state,
|
||||||
|
conv_weights,
|
||||||
|
self.conv.bias,
|
||||||
|
activation=None,
|
||||||
|
conv_state_indices=state_indices_tensor_d,
|
||||||
|
num_accepted_tokens=num_accepted_tokens,
|
||||||
|
query_start_loc=query_start_loc_d,
|
||||||
|
max_query_len=state_indices_tensor_d.size(-1),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Non-spec decode
|
||||||
|
Bx = causal_conv1d_update(
|
||||||
|
Bx_d,
|
||||||
|
conv_state,
|
||||||
|
conv_weights,
|
||||||
|
self.conv.bias,
|
||||||
|
activation=None,
|
||||||
|
conv_state_indices=state_indices_tensor_d,
|
||||||
|
)
|
||||||
y = C_d * Bx
|
y = C_d * Bx
|
||||||
conv_output_list.insert(0, y)
|
conv_output_list.insert(0, y)
|
||||||
|
|
||||||
@@ -326,6 +349,7 @@ class ShortConv(MambaBase, PluggableLayer):
|
|||||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||||
intermediate_size=self.conv_dim,
|
intermediate_size=self.conv_dim,
|
||||||
conv_kernel=self.L_cache,
|
conv_kernel=self.L_cache,
|
||||||
|
num_spec=self.num_spec,
|
||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from vllm.config.compilation import CUDAGraphMode
|
|||||||
from vllm.triton_utils import tl, triton
|
from vllm.triton_utils import tl, triton
|
||||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||||
|
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder
|
||||||
from vllm.v1.core.sched.output import NewRequestData
|
from vllm.v1.core.sched.output import NewRequestData
|
||||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||||
from vllm.v1.utils import CpuGpuBuffer
|
from vllm.v1.utils import CpuGpuBuffer
|
||||||
@@ -48,7 +49,11 @@ class MambaHybridAttnMetadata(ModelSpecificAttnMetadata):
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if not isinstance(
|
if not isinstance(
|
||||||
attn_metadata_builder,
|
attn_metadata_builder,
|
||||||
(Mamba2AttentionMetadataBuilder, GDNAttentionMetadataBuilder),
|
(
|
||||||
|
Mamba2AttentionMetadataBuilder,
|
||||||
|
GDNAttentionMetadataBuilder,
|
||||||
|
ShortConvAttentionMetadataBuilder,
|
||||||
|
),
|
||||||
):
|
):
|
||||||
return {}
|
return {}
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -148,6 +148,7 @@ from vllm.v1.attention.backends.linear_attn import (
|
|||||||
BailingLinearAttentionMetadataBuilder,
|
BailingLinearAttentionMetadataBuilder,
|
||||||
)
|
)
|
||||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||||
|
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadataBuilder
|
||||||
from vllm.v1.attention.backends.utils import (
|
from vllm.v1.attention.backends.utils import (
|
||||||
NULL_BLOCK_ID,
|
NULL_BLOCK_ID,
|
||||||
create_fast_prefill_custom_backend,
|
create_fast_prefill_custom_backend,
|
||||||
@@ -2572,6 +2573,7 @@ class GPUModelRunner(
|
|||||||
Mamba2AttentionMetadataBuilder,
|
Mamba2AttentionMetadataBuilder,
|
||||||
GDNAttentionMetadataBuilder,
|
GDNAttentionMetadataBuilder,
|
||||||
BailingLinearAttentionMetadataBuilder,
|
BailingLinearAttentionMetadataBuilder,
|
||||||
|
ShortConvAttentionMetadataBuilder,
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
assert ubid is None, (
|
assert ubid is None, (
|
||||||
|
|||||||
Reference in New Issue
Block a user