[V1][Hybrid] Mamba Prefix Caching with align mode (#30877)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Chen Zhang
parent
fec9da0af4
commit
5206e5e28c
@@ -60,6 +60,7 @@ from vllm.config.cache import (
|
||||
BlockSize,
|
||||
CacheDType,
|
||||
KVOffloadingBackend,
|
||||
MambaCacheMode,
|
||||
MambaDType,
|
||||
PrefixCachingHashAlgo,
|
||||
)
|
||||
@@ -556,6 +557,7 @@ class EngineArgs:
|
||||
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
|
||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
||||
|
||||
additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
|
||||
|
||||
@@ -939,6 +941,9 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--mamba-block-size", **cache_kwargs["mamba_block_size"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
|
||||
)
|
||||
@@ -1416,6 +1421,7 @@ class EngineArgs:
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
mamba_block_size=self.mamba_block_size,
|
||||
mamba_cache_mode=self.mamba_cache_mode,
|
||||
kv_offloading_size=self.kv_offloading_size,
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user