[Mamba] Flashinfer selective_state_update (#36162)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -56,6 +56,9 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
initialize_mamba_ssu_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
MRotaryEmbedding,
|
||||
XDRotaryEmbedding,
|
||||
@@ -6750,6 +6753,7 @@ class GPUModelRunner(
|
||||
self.may_add_encoder_only_layers_to_kv_cache_config()
|
||||
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
|
||||
self.initialize_attn_backend(kv_cache_config, is_profiling=is_profiling)
|
||||
initialize_mamba_ssu_backend(self.vllm_config.mamba_config)
|
||||
# The kernel block size for all KV cache groups. For example, if
|
||||
# kv_cache_manager uses block_size 256 for a given group, but the attention
|
||||
# backends for that group only supports block_size 64, we will return
|
||||
|
||||
Reference in New Issue
Block a user