[Attention][Spec Decode] Enable varlen Mamba2 decode for adaptive verification (#53036)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
vllm-agent
parent
e7a5bdb454
commit
3dc6def360
@@ -6,7 +6,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config_or_none
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
@@ -15,12 +15,13 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba_attn import (
|
||||
BaseMambaAttentionMetadata,
|
||||
BaseMambaAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
|
||||
def compute_varlen_chunk_metadata(
|
||||
@@ -104,6 +105,14 @@ class Mamba2AttentionBackend(AttentionBackend):
|
||||
def is_ssm(cls) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# Decode kernels take device offsets; ReplaySSM plans off CPU counts.
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None:
|
||||
return True
|
||||
return not vllm_config.cache_config.use_replayssm
|
||||
|
||||
|
||||
@dataclass
|
||||
class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
|
||||
@@ -124,6 +133,14 @@ class Mamba2AttentionMetadataBuilder(
|
||||
):
|
||||
metadata_cls = Mamba2AttentionMetadata
|
||||
|
||||
@classmethod
|
||||
def get_varlen_cudagraph_max_query_len(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> int | None:
|
||||
return max_decode_query_len(vllm_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: MambaSpec,
|
||||
|
||||
Reference in New Issue
Block a user