[Misc] Rename --enable-mamba-fine-grained-prefix-cache (#53945 follow-up) (#57382)

Signed-off-by: Roi Koren <roik@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
roikoren755
2026-09-20 09:50:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7d99c2c4fd
commit 49ee12d742
10 changed files with 38 additions and 38 deletions
+2 -2
View File
@@ -184,7 +184,7 @@ class CacheConfig:
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
"""
enable_mamba_fine_grained_prefix_cache: bool = False
enable_mamba_shared_prefix_checkpoint: bool = False
"""Also register a Mamba "align" checkpoint at the shared-prefix junction --
where an EAGLE/MTP sibling was observed to resume -- instead of only at the
prompt tail. Off by default; only takes effect with `mamba_cache_mode`
@@ -274,7 +274,7 @@ class CacheConfig:
"prefix_cache_retention_interval",
# Prefix-caching implementation detail (doesn't affect compiled graph).
"prefix_match_unit",
"enable_mamba_fine_grained_prefix_cache",
"enable_mamba_shared_prefix_checkpoint",
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
+6 -6
View File
@@ -753,8 +753,8 @@ class EngineArgs:
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
enable_mamba_fine_grained_prefix_cache: bool = (
CacheConfig.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint: bool = (
CacheConfig.enable_mamba_shared_prefix_checkpoint
)
replayssm_buffer_len: int = CacheConfig.replayssm_buffer_len
use_replayssm: bool = CacheConfig.use_replayssm
@@ -1330,8 +1330,8 @@ class EngineArgs:
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
)
cache_group.add_argument(
"--enable-mamba-fine-grained-prefix-cache",
**cache_kwargs["enable_mamba_fine_grained_prefix_cache"],
"--enable-mamba-shared-prefix-checkpoint",
**cache_kwargs["enable_mamba_shared_prefix_checkpoint"],
)
cache_group.add_argument(
"--replayssm-buffer-len", **cache_kwargs["replayssm_buffer_len"]
@@ -2128,8 +2128,8 @@ class EngineArgs:
mamba_block_size=self.mamba_block_size,
prefix_match_unit=self.prefix_match_unit,
mamba_cache_mode=self.mamba_cache_mode,
enable_mamba_fine_grained_prefix_cache=(
self.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint=(
self.enable_mamba_shared_prefix_checkpoint
),
replayssm_buffer_len=self.replayssm_buffer_len,
use_replayssm=self.use_replayssm,
+5 -5
View File
@@ -145,7 +145,7 @@ class KVCacheManager:
pcp_world_size: int = 1,
metrics_collector: KVCacheMetricsCollector | None = None,
watermark: float = 0.0,
enable_mamba_fine_grained_prefix_cache: bool = False,
enable_mamba_shared_prefix_checkpoint: bool = False,
) -> None:
self.max_model_len = max_model_len
# When unset, fall back to `max_model_len` so the recycling-aware cap
@@ -183,16 +183,16 @@ class KVCacheManager:
# junction costs a forward pass and displaces the block-boundary stop.
# Multi-module MTP is excluded because ``cache_blocks`` then hands the
# manager ``num_computed - num_reprefillable`` rather than the chunk end.
self.mamba_fine_grained_prefix_cache = (
enable_mamba_fine_grained_prefix_cache
self.mamba_shared_prefix_checkpoint = (
enable_mamba_shared_prefix_checkpoint
and bool(self.coordinator.eagle_group_ids)
and self.coordinator.enable_partial_hash_hits
and self.coordinator.num_reprefillable_tokens == 0
)
if self.mamba_fine_grained_prefix_cache:
if self.mamba_shared_prefix_checkpoint:
for manager in self.coordinator.single_type_managers:
if isinstance(manager, MambaManager):
manager.fine_grained_prefix_cache = True
manager.shared_prefix_checkpoint = True
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
self.block_pool = self.coordinator.block_pool
self.retained_hit_group_ids = tuple(
+5 -5
View File
@@ -320,8 +320,8 @@ class Scheduler(SchedulerInterface):
hash_block_size=hash_block_size,
metrics_collector=self.kv_metrics_collector,
watermark=self.scheduler_config.watermark,
enable_mamba_fine_grained_prefix_cache=(
self.cache_config.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint=(
self.cache_config.enable_mamba_shared_prefix_checkpoint
),
)
# Bind after construction so connectors can access the cache manager.
@@ -376,9 +376,9 @@ class Scheduler(SchedulerInterface):
# manager decides whether it can check-point there (per-group eagle bit,
# no MTP re-prefill tail); splitting for a stop it would refuse costs a
# forward pass and displaces the block-boundary stop.
self.mamba_fine_grained_prefix_cache = (
self.mamba_shared_prefix_checkpoint = (
self.mamba_partial_cache_hit
and self.kv_cache_manager.mamba_fine_grained_prefix_cache
and self.kv_cache_manager.mamba_shared_prefix_checkpoint
)
# Counts of non-empty steps scheduled / processed. update_from_output
@@ -509,7 +509,7 @@ class Scheduler(SchedulerInterface):
# output tokens can still observe a junction there.
junction_stop = (
junction
if self.mamba_fine_grained_prefix_cache
if self.mamba_shared_prefix_checkpoint
and junction <= request.num_prompt_tokens
else block_floored
)
+3 -3
View File
@@ -143,9 +143,9 @@ class SingleTypeKVCacheManager(ABC):
# aligned segment (SWA). Initialized lazily by the coordinator after
# determining the attention groups.
self.use_eagle = False
# ``CacheConfig.enable_mamba_fine_grained_prefix_cache``, narrowed and set
# ``CacheConfig.enable_mamba_shared_prefix_checkpoint``, narrowed and set
# by ``KVCacheManager``; only an EAGLE Mamba "align" group ever gets it.
self.fine_grained_prefix_cache = False
self.shared_prefix_checkpoint = False
# Partial-hit copy-on-write bookkeeping. Populated only by fine-grained
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
@@ -2090,7 +2090,7 @@ class MambaManager(SingleTypeKVCacheManager):
# running state block, mutated in place, which equals what its key
# promises only after that step's forward.
if num_tokens != latest_prompt_hash_boundary and not (
self.fine_grained_prefix_cache
self.shared_prefix_checkpoint
and num_tokens == request.shared_prefix_boundary
and request.num_computed_tokens < num_tokens <= request.num_prompt_tokens
):