[Misc] Rename --enable-mamba-fine-grained-prefix-cache (#53945 follow-up) (#57382)

Signed-off-by: Roi Koren <roik@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
roikoren755
2026-09-20 09:50:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7d99c2c4fd
commit 49ee12d742
10 changed files with 38 additions and 38 deletions
+2 -2
View File
@@ -15,7 +15,7 @@ Set `enable_prefix_caching=True` in vLLM engine to enable APC. Here is an exampl
## Hybrid Mamba models
Under `--mamba-cache-mode align`, Mamba state is stored only on the Mamba block grid, so a prefix-cache hit can resume only at a block boundary. `--enable-mamba-fine-grained-prefix-cache` also stores a checkpoint at the shared-prefix junction, the point where an earlier request with the same prefix stopped. Requests whose shared prefix ends inside a block can then reuse it.
Under `--mamba-cache-mode align`, Mamba state is stored only on the Mamba block grid, so a prefix-cache hit can resume only at a block boundary. `--enable-mamba-shared-prefix-checkpoint` also stores a checkpoint at the shared-prefix junction, the point where an earlier request with the same prefix stopped. Requests whose shared prefix ends inside a block can then reuse it.
This helps when many requests share a long system prompt and then diverge. It is off by default, and takes effect only when all of the following hold:
@@ -28,7 +28,7 @@ This helps when many requests share a long system prompt and then diverge. It is
vllm serve <hybrid-model> \
--mamba-cache-mode align \
--prefix-match-unit 64 \
--enable-mamba-fine-grained-prefix-cache
--enable-mamba-shared-prefix-checkpoint
```
`--prefix-match-unit` is required. It sets the granularity at which prefix-cache keys are computed. When unset it defaults to the greatest common divisor of the prefix-cacheable KV cache group block sizes. Under `align` that is the block size itself, so no sub-block boundary exists and the flag has no effect.
@@ -6,7 +6,7 @@ Full attention hits where it holds a key; EAGLE prunes one hash unit off that
candidate and drops it. Mamba materializes state only on its own block grid, so
the pruned position holds nothing and the hit floors back a whole Mamba block.
Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here
Gated by ``CacheConfig.enable_mamba_shared_prefix_checkpoint``. Every test here
drives the real ``Scheduler._mamba_block_aligned_split`` and the real
``KVCacheManager``; no chunk boundary is hard-coded.
"""
@@ -32,7 +32,7 @@ def _manager(
block_size,
hash_block_size,
*,
fine_grained=True,
shared_prefix_checkpoint=True,
num_blocks=8192,
eagle_group=None,
num_prefill_lookahead=0,
@@ -52,7 +52,7 @@ def _manager(
hash_block_size=hash_block_size,
use_eagle=True,
num_prefill_lookahead=num_prefill_lookahead,
enable_mamba_fine_grained_prefix_cache=fine_grained,
enable_mamba_shared_prefix_checkpoint=shared_prefix_checkpoint,
)
@@ -76,8 +76,8 @@ def _stub(manager, block_size, hash_block_size, *, block_drop=True):
hash_block_size=hash_block_size,
mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=(
partial_hit and manager.mamba_fine_grained_prefix_cache
mamba_shared_prefix_checkpoint=(
partial_hit and manager.mamba_shared_prefix_checkpoint
),
)
@@ -151,7 +151,7 @@ def test_sibling_resumes_from_the_observed_junction():
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed"
assert stub.mamba_shared_prefix_checkpoint, "the feature must be armed"
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
@@ -325,11 +325,11 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
"""
block_size, hash_block_size = 512, 32
def sibling_hit(fine_grained):
def sibling_hit(shared_prefix_checkpoint):
manager = _manager(
block_size,
hash_block_size,
fine_grained=fine_grained,
shared_prefix_checkpoint=shared_prefix_checkpoint,
eagle_group=eagle_group,
num_prefill_lookahead=num_prefill_lookahead,
)
@@ -348,7 +348,7 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
def test_disabled_by_default():
"""Off, the junction is block-floored and no resume point is registered."""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size, fine_grained=False)
manager = _manager(block_size, hash_block_size, shared_prefix_checkpoint=False)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, 2020)
@@ -188,7 +188,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
@@ -236,7 +236,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -277,7 +277,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -1826,7 +1826,7 @@ def test_mamba_align_split_stops_below_eagle_proof_boundary():
use_eagle_block_drop=True,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
@@ -144,7 +144,7 @@ def _split(
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
hash_block_size=ATTN_BLOCK_SIZE,
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
mamba_prefill_checkpoint_alignment=(
+1 -1
View File
@@ -2269,7 +2269,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
use_eagle_block_drop=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req_2.shared_prefix_boundary = shared_prefix_boundary
+2 -2
View File
@@ -184,7 +184,7 @@ class CacheConfig:
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
"""
enable_mamba_fine_grained_prefix_cache: bool = False
enable_mamba_shared_prefix_checkpoint: bool = False
"""Also register a Mamba "align" checkpoint at the shared-prefix junction --
where an EAGLE/MTP sibling was observed to resume -- instead of only at the
prompt tail. Off by default; only takes effect with `mamba_cache_mode`
@@ -274,7 +274,7 @@ class CacheConfig:
"prefix_cache_retention_interval",
# Prefix-caching implementation detail (doesn't affect compiled graph).
"prefix_match_unit",
"enable_mamba_fine_grained_prefix_cache",
"enable_mamba_shared_prefix_checkpoint",
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
+6 -6
View File
@@ -753,8 +753,8 @@ class EngineArgs:
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
enable_mamba_fine_grained_prefix_cache: bool = (
CacheConfig.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint: bool = (
CacheConfig.enable_mamba_shared_prefix_checkpoint
)
replayssm_buffer_len: int = CacheConfig.replayssm_buffer_len
use_replayssm: bool = CacheConfig.use_replayssm
@@ -1330,8 +1330,8 @@ class EngineArgs:
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
)
cache_group.add_argument(
"--enable-mamba-fine-grained-prefix-cache",
**cache_kwargs["enable_mamba_fine_grained_prefix_cache"],
"--enable-mamba-shared-prefix-checkpoint",
**cache_kwargs["enable_mamba_shared_prefix_checkpoint"],
)
cache_group.add_argument(
"--replayssm-buffer-len", **cache_kwargs["replayssm_buffer_len"]
@@ -2128,8 +2128,8 @@ class EngineArgs:
mamba_block_size=self.mamba_block_size,
prefix_match_unit=self.prefix_match_unit,
mamba_cache_mode=self.mamba_cache_mode,
enable_mamba_fine_grained_prefix_cache=(
self.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint=(
self.enable_mamba_shared_prefix_checkpoint
),
replayssm_buffer_len=self.replayssm_buffer_len,
use_replayssm=self.use_replayssm,
+5 -5
View File
@@ -145,7 +145,7 @@ class KVCacheManager:
pcp_world_size: int = 1,
metrics_collector: KVCacheMetricsCollector | None = None,
watermark: float = 0.0,
enable_mamba_fine_grained_prefix_cache: bool = False,
enable_mamba_shared_prefix_checkpoint: bool = False,
) -> None:
self.max_model_len = max_model_len
# When unset, fall back to `max_model_len` so the recycling-aware cap
@@ -183,16 +183,16 @@ class KVCacheManager:
# junction costs a forward pass and displaces the block-boundary stop.
# Multi-module MTP is excluded because ``cache_blocks`` then hands the
# manager ``num_computed - num_reprefillable`` rather than the chunk end.
self.mamba_fine_grained_prefix_cache = (
enable_mamba_fine_grained_prefix_cache
self.mamba_shared_prefix_checkpoint = (
enable_mamba_shared_prefix_checkpoint
and bool(self.coordinator.eagle_group_ids)
and self.coordinator.enable_partial_hash_hits
and self.coordinator.num_reprefillable_tokens == 0
)
if self.mamba_fine_grained_prefix_cache:
if self.mamba_shared_prefix_checkpoint:
for manager in self.coordinator.single_type_managers:
if isinstance(manager, MambaManager):
manager.fine_grained_prefix_cache = True
manager.shared_prefix_checkpoint = True
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
self.block_pool = self.coordinator.block_pool
self.retained_hit_group_ids = tuple(
+5 -5
View File
@@ -320,8 +320,8 @@ class Scheduler(SchedulerInterface):
hash_block_size=hash_block_size,
metrics_collector=self.kv_metrics_collector,
watermark=self.scheduler_config.watermark,
enable_mamba_fine_grained_prefix_cache=(
self.cache_config.enable_mamba_fine_grained_prefix_cache
enable_mamba_shared_prefix_checkpoint=(
self.cache_config.enable_mamba_shared_prefix_checkpoint
),
)
# Bind after construction so connectors can access the cache manager.
@@ -376,9 +376,9 @@ class Scheduler(SchedulerInterface):
# manager decides whether it can check-point there (per-group eagle bit,
# no MTP re-prefill tail); splitting for a stop it would refuse costs a
# forward pass and displaces the block-boundary stop.
self.mamba_fine_grained_prefix_cache = (
self.mamba_shared_prefix_checkpoint = (
self.mamba_partial_cache_hit
and self.kv_cache_manager.mamba_fine_grained_prefix_cache
and self.kv_cache_manager.mamba_shared_prefix_checkpoint
)
# Counts of non-empty steps scheduled / processed. update_from_output
@@ -509,7 +509,7 @@ class Scheduler(SchedulerInterface):
# output tokens can still observe a junction there.
junction_stop = (
junction
if self.mamba_fine_grained_prefix_cache
if self.mamba_shared_prefix_checkpoint
and junction <= request.num_prompt_tokens
else block_floored
)
+3 -3
View File
@@ -143,9 +143,9 @@ class SingleTypeKVCacheManager(ABC):
# aligned segment (SWA). Initialized lazily by the coordinator after
# determining the attention groups.
self.use_eagle = False
# ``CacheConfig.enable_mamba_fine_grained_prefix_cache``, narrowed and set
# ``CacheConfig.enable_mamba_shared_prefix_checkpoint``, narrowed and set
# by ``KVCacheManager``; only an EAGLE Mamba "align" group ever gets it.
self.fine_grained_prefix_cache = False
self.shared_prefix_checkpoint = False
# Partial-hit copy-on-write bookkeeping. Populated only by fine-grained
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
@@ -2090,7 +2090,7 @@ class MambaManager(SingleTypeKVCacheManager):
# running state block, mutated in place, which equals what its key
# promises only after that step's forward.
if num_tokens != latest_prompt_hash_boundary and not (
self.fine_grained_prefix_cache
self.shared_prefix_checkpoint
and num_tokens == request.shared_prefix_boundary
and request.num_computed_tokens < num_tokens <= request.num_prompt_tokens
):