diff --git a/docs/features/automatic_prefix_caching.md b/docs/features/automatic_prefix_caching.md index 03cca3b38d..cfc8c19acb 100644 --- a/docs/features/automatic_prefix_caching.md +++ b/docs/features/automatic_prefix_caching.md @@ -15,7 +15,7 @@ Set `enable_prefix_caching=True` in vLLM engine to enable APC. Here is an exampl ## Hybrid Mamba models -Under `--mamba-cache-mode align`, Mamba state is stored only on the Mamba block grid, so a prefix-cache hit can resume only at a block boundary. `--enable-mamba-fine-grained-prefix-cache` also stores a checkpoint at the shared-prefix junction, the point where an earlier request with the same prefix stopped. Requests whose shared prefix ends inside a block can then reuse it. +Under `--mamba-cache-mode align`, Mamba state is stored only on the Mamba block grid, so a prefix-cache hit can resume only at a block boundary. `--enable-mamba-shared-prefix-checkpoint` also stores a checkpoint at the shared-prefix junction, the point where an earlier request with the same prefix stopped. Requests whose shared prefix ends inside a block can then reuse it. This helps when many requests share a long system prompt and then diverge. It is off by default, and takes effect only when all of the following hold: @@ -28,7 +28,7 @@ This helps when many requests share a long system prompt and then diverge. It is vllm serve \ --mamba-cache-mode align \ --prefix-match-unit 64 \ - --enable-mamba-fine-grained-prefix-cache + --enable-mamba-shared-prefix-checkpoint ``` `--prefix-match-unit` is required. It sets the granularity at which prefix-cache keys are computed. When unset it defaults to the greatest common divisor of the prefix-cacheable KV cache group block sizes. Under `align` that is the block size itself, so no sub-block boundary exists and the flag has no effect. diff --git a/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py b/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py index 059b4005f0..4cb9afe871 100644 --- a/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py +++ b/tests/v1/core/prefix_cache/test_mamba_eagle_resume_checkpoint.py @@ -6,7 +6,7 @@ Full attention hits where it holds a key; EAGLE prunes one hash unit off that candidate and drops it. Mamba materializes state only on its own block grid, so the pruned position holds nothing and the hit floors back a whole Mamba block. -Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here +Gated by ``CacheConfig.enable_mamba_shared_prefix_checkpoint``. Every test here drives the real ``Scheduler._mamba_block_aligned_split`` and the real ``KVCacheManager``; no chunk boundary is hard-coded. """ @@ -32,7 +32,7 @@ def _manager( block_size, hash_block_size, *, - fine_grained=True, + shared_prefix_checkpoint=True, num_blocks=8192, eagle_group=None, num_prefill_lookahead=0, @@ -52,7 +52,7 @@ def _manager( hash_block_size=hash_block_size, use_eagle=True, num_prefill_lookahead=num_prefill_lookahead, - enable_mamba_fine_grained_prefix_cache=fine_grained, + enable_mamba_shared_prefix_checkpoint=shared_prefix_checkpoint, ) @@ -76,8 +76,8 @@ def _stub(manager, block_size, hash_block_size, *, block_drop=True): hash_block_size=hash_block_size, mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle mamba_partial_cache_hit=partial_hit, - mamba_fine_grained_prefix_cache=( - partial_hit and manager.mamba_fine_grained_prefix_cache + mamba_shared_prefix_checkpoint=( + partial_hit and manager.mamba_shared_prefix_checkpoint ), ) @@ -151,7 +151,7 @@ def test_sibling_resumes_from_the_observed_junction(): block_size, hash_block_size = 512, 32 manager = _manager(block_size, hash_block_size) stub = _stub(manager, block_size, hash_block_size) - assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed" + assert stub.mamba_shared_prefix_checkpoint, "the feature must be armed" _orphaned_full_attention_tail(manager, stub, 2020) consumer = make_request( @@ -325,11 +325,11 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead): """ block_size, hash_block_size = 512, 32 - def sibling_hit(fine_grained): + def sibling_hit(shared_prefix_checkpoint): manager = _manager( block_size, hash_block_size, - fine_grained=fine_grained, + shared_prefix_checkpoint=shared_prefix_checkpoint, eagle_group=eagle_group, num_prefill_lookahead=num_prefill_lookahead, ) @@ -348,7 +348,7 @@ def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead): def test_disabled_by_default(): """Off, the junction is block-floored and no resume point is registered.""" block_size, hash_block_size = 512, 32 - manager = _manager(block_size, hash_block_size, fine_grained=False) + manager = _manager(block_size, hash_block_size, shared_prefix_checkpoint=False) stub = _stub(manager, block_size, hash_block_size) _orphaned_full_attention_tail(manager, stub, 2020) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index 393ec48465..f1301d9288 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -188,7 +188,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int): dcp_world_size=dcp_world_size, scheduler_block_size=scheduler_block_size, mamba_partial_cache_hit=True, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, mamba_has_prefill_checkpoint_blocks=False, ) split = Scheduler._mamba_block_aligned_split @@ -236,7 +236,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget(): use_eagle_block_drop=False, hash_block_size=32, mamba_partial_cache_hit=False, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, mamba_has_prefill_checkpoint_blocks=False, ) req = make_request("0", [0] * prompt_length, 32, sha256) @@ -277,7 +277,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold(): use_eagle_block_drop=False, hash_block_size=32, mamba_partial_cache_hit=False, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, mamba_has_prefill_checkpoint_blocks=False, ) req = make_request("0", [0] * prompt_length, 32, sha256) @@ -1826,7 +1826,7 @@ def test_mamba_align_split_stops_below_eagle_proof_boundary(): use_eagle_block_drop=True, hash_block_size=hash_block_size, mamba_partial_cache_hit=True, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, mamba_has_prefill_checkpoint_blocks=False, ) split = Scheduler._mamba_block_aligned_split diff --git a/tests/v1/core/test_mamba_align_chunk_split.py b/tests/v1/core/test_mamba_align_chunk_split.py index 1bd7f13b59..b157523915 100644 --- a/tests/v1/core/test_mamba_align_chunk_split.py +++ b/tests/v1/core/test_mamba_align_chunk_split.py @@ -144,7 +144,7 @@ def _split( scheduler_config=SimpleNamespace(long_prefill_token_threshold=0), # `prefix_match_unit` finer than the block size (#46384). mamba_partial_cache_hit=partial_hit, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, hash_block_size=ATTN_BLOCK_SIZE, mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0), mamba_prefill_checkpoint_alignment=( diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index d6848a6a7f..0a34a6131d 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2269,7 +2269,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection(): use_eagle_block_drop=False, hash_block_size=block_size, mamba_partial_cache_hit=False, - mamba_fine_grained_prefix_cache=False, + mamba_shared_prefix_checkpoint=False, mamba_has_prefill_checkpoint_blocks=False, ) req_2.shared_prefix_boundary = shared_prefix_boundary diff --git a/vllm/config/cache.py b/vllm/config/cache.py index c12433a367..6315a1e317 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -184,7 +184,7 @@ class CacheConfig: when the token is at position i * block_size. This is the default when prefix caching is enabled. """ - enable_mamba_fine_grained_prefix_cache: bool = False + enable_mamba_shared_prefix_checkpoint: bool = False """Also register a Mamba "align" checkpoint at the shared-prefix junction -- where an EAGLE/MTP sibling was observed to resume -- instead of only at the prompt tail. Off by default; only takes effect with `mamba_cache_mode` @@ -274,7 +274,7 @@ class CacheConfig: "prefix_cache_retention_interval", # Prefix-caching implementation detail (doesn't affect compiled graph). "prefix_match_unit", - "enable_mamba_fine_grained_prefix_cache", + "enable_mamba_shared_prefix_checkpoint", "mamba_page_size_padded", "skip_page_size_padded", "user_specified_block_size", diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 67d8a2f2a0..0d33caf625 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -753,8 +753,8 @@ class EngineArgs: mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size") prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit") mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode - enable_mamba_fine_grained_prefix_cache: bool = ( - CacheConfig.enable_mamba_fine_grained_prefix_cache + enable_mamba_shared_prefix_checkpoint: bool = ( + CacheConfig.enable_mamba_shared_prefix_checkpoint ) replayssm_buffer_len: int = CacheConfig.replayssm_buffer_len use_replayssm: bool = CacheConfig.use_replayssm @@ -1330,8 +1330,8 @@ class EngineArgs: "--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"] ) cache_group.add_argument( - "--enable-mamba-fine-grained-prefix-cache", - **cache_kwargs["enable_mamba_fine_grained_prefix_cache"], + "--enable-mamba-shared-prefix-checkpoint", + **cache_kwargs["enable_mamba_shared_prefix_checkpoint"], ) cache_group.add_argument( "--replayssm-buffer-len", **cache_kwargs["replayssm_buffer_len"] @@ -2128,8 +2128,8 @@ class EngineArgs: mamba_block_size=self.mamba_block_size, prefix_match_unit=self.prefix_match_unit, mamba_cache_mode=self.mamba_cache_mode, - enable_mamba_fine_grained_prefix_cache=( - self.enable_mamba_fine_grained_prefix_cache + enable_mamba_shared_prefix_checkpoint=( + self.enable_mamba_shared_prefix_checkpoint ), replayssm_buffer_len=self.replayssm_buffer_len, use_replayssm=self.use_replayssm, diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 1adf386cab..f881fc9818 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -145,7 +145,7 @@ class KVCacheManager: pcp_world_size: int = 1, metrics_collector: KVCacheMetricsCollector | None = None, watermark: float = 0.0, - enable_mamba_fine_grained_prefix_cache: bool = False, + enable_mamba_shared_prefix_checkpoint: bool = False, ) -> None: self.max_model_len = max_model_len # When unset, fall back to `max_model_len` so the recycling-aware cap @@ -183,16 +183,16 @@ class KVCacheManager: # junction costs a forward pass and displaces the block-boundary stop. # Multi-module MTP is excluded because ``cache_blocks`` then hands the # manager ``num_computed - num_reprefillable`` rather than the chunk end. - self.mamba_fine_grained_prefix_cache = ( - enable_mamba_fine_grained_prefix_cache + self.mamba_shared_prefix_checkpoint = ( + enable_mamba_shared_prefix_checkpoint and bool(self.coordinator.eagle_group_ids) and self.coordinator.enable_partial_hash_hits and self.coordinator.num_reprefillable_tokens == 0 ) - if self.mamba_fine_grained_prefix_cache: + if self.mamba_shared_prefix_checkpoint: for manager in self.coordinator.single_type_managers: if isinstance(manager, MambaManager): - manager.fine_grained_prefix_cache = True + manager.shared_prefix_checkpoint = True self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) self.block_pool = self.coordinator.block_pool self.retained_hit_group_ids = tuple( diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 19ac16284c..a2638942f8 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -320,8 +320,8 @@ class Scheduler(SchedulerInterface): hash_block_size=hash_block_size, metrics_collector=self.kv_metrics_collector, watermark=self.scheduler_config.watermark, - enable_mamba_fine_grained_prefix_cache=( - self.cache_config.enable_mamba_fine_grained_prefix_cache + enable_mamba_shared_prefix_checkpoint=( + self.cache_config.enable_mamba_shared_prefix_checkpoint ), ) # Bind after construction so connectors can access the cache manager. @@ -376,9 +376,9 @@ class Scheduler(SchedulerInterface): # manager decides whether it can check-point there (per-group eagle bit, # no MTP re-prefill tail); splitting for a stop it would refuse costs a # forward pass and displaces the block-boundary stop. - self.mamba_fine_grained_prefix_cache = ( + self.mamba_shared_prefix_checkpoint = ( self.mamba_partial_cache_hit - and self.kv_cache_manager.mamba_fine_grained_prefix_cache + and self.kv_cache_manager.mamba_shared_prefix_checkpoint ) # Counts of non-empty steps scheduled / processed. update_from_output @@ -509,7 +509,7 @@ class Scheduler(SchedulerInterface): # output tokens can still observe a junction there. junction_stop = ( junction - if self.mamba_fine_grained_prefix_cache + if self.mamba_shared_prefix_checkpoint and junction <= request.num_prompt_tokens else block_floored ) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 9fe59dfdbe..70d56c6543 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -143,9 +143,9 @@ class SingleTypeKVCacheManager(ABC): # aligned segment (SWA). Initialized lazily by the coordinator after # determining the attention groups. self.use_eagle = False - # ``CacheConfig.enable_mamba_fine_grained_prefix_cache``, narrowed and set + # ``CacheConfig.enable_mamba_shared_prefix_checkpoint``, narrowed and set # by ``KVCacheManager``; only an EAGLE Mamba "align" group ever gets it. - self.fine_grained_prefix_cache = False + self.shared_prefix_checkpoint = False # Partial-hit copy-on-write bookkeeping. Populated only by fine-grained # managers (full attention, mamba "align"); harmlessly empty elsewhere. self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {} @@ -2090,7 +2090,7 @@ class MambaManager(SingleTypeKVCacheManager): # running state block, mutated in place, which equals what its key # promises only after that step's forward. if num_tokens != latest_prompt_hash_boundary and not ( - self.fine_grained_prefix_cache + self.shared_prefix_checkpoint and num_tokens == request.shared_prefix_boundary and request.num_computed_tokens < num_tokens <= request.num_prompt_tokens ):