[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
+5
View File
@@ -693,6 +693,7 @@ class EngineArgs:
mamba_cache_dtype: MambaDType = CacheConfig.mamba_cache_dtype
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
@@ -1194,6 +1195,9 @@ class EngineArgs:
cache_group.add_argument(
"--mamba-block-size", **cache_kwargs["mamba_block_size"]
)
cache_group.add_argument(
"--prefix-match-unit", **cache_kwargs["prefix_match_unit"]
)
cache_group.add_argument(
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
)
@@ -1899,6 +1903,7 @@ class EngineArgs:
mamba_cache_dtype=self.mamba_cache_dtype,
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
mamba_block_size=self.mamba_block_size,
prefix_match_unit=self.prefix_match_unit,
mamba_cache_mode=self.mamba_cache_mode,
kv_offloading_size=self.kv_offloading_size,
kv_offloading_backend=self.kv_offloading_backend,