[Model] Support Mamba (#6484)

This commit is contained in:
Tyler Michael Smith
2024-10-11 15:40:06 +00:00
committed by GitHub
parent df3dcdf49d
commit 7342a7d7f8
29 changed files with 1603 additions and 343 deletions
+1 -2
View File
@@ -56,13 +56,12 @@ class CPUCacheEngine:
# Get attention backend.
self.attn_backend = get_attn_backend(
self.model_config.get_num_attention_heads(self.parallel_config),
self.model_config.get_head_size(),
self.model_config.get_num_kv_heads(self.parallel_config),
self.model_config.get_sliding_window(),
self.model_config.dtype,
cache_config.cache_dtype,
self.block_size,
self.model_config.is_attention_free,
)
# Initialize the cache.