[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -7434,6 +7434,7 @@ class GPUModelRunner(
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_caches,
|
||||
num_attn_module,
|
||||
kv_cache_groups=kv_cache_config.kv_cache_groups,
|
||||
)
|
||||
return kv_caches
|
||||
|
||||
@@ -7485,7 +7486,9 @@ class GPUModelRunner(
|
||||
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
|
||||
self.initialize_attn_backend(kv_cache_config, is_profiling=is_profiling)
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config, self.kv_cache_config
|
||||
self.vllm_config.mamba_config,
|
||||
self.kv_cache_config,
|
||||
use_replayssm=self.vllm_config.cache_config.use_replayssm,
|
||||
)
|
||||
# The kernel block size for all KV cache groups. For example, if
|
||||
# kv_cache_manager uses block_size 256 for a given group, but the attention
|
||||
|
||||
Reference in New Issue
Block a user