[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
@@ -693,6 +693,7 @@ class EngineArgs:
|
||||
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
|
||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
|
||||
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
||||
|
||||
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
|
||||
@@ -1194,6 +1195,9 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--mamba-block-size", **cache_kwargs["mamba_block_size"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--prefix-match-unit", **cache_kwargs["prefix_match_unit"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
||||
)
|
||||
@@ -1899,6 +1903,7 @@ class EngineArgs:
|
||||
mamba_cache_dtype=self.mamba_cache_dtype,
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
mamba_block_size=self.mamba_block_size,
|
||||
prefix_match_unit=self.prefix_match_unit,
|
||||
mamba_cache_mode=self.mamba_cache_mode,
|
||||
kv_offloading_size=self.kv_offloading_size,
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
|
||||
Reference in New Issue
Block a user