[Bugfix][Spec Decode] Cache the Mamba state at the block-grid position of EAGLE resume (#53945)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Adam Shaver <ashaver@nvidia.com>
Signed-off-by: akshaver <168006157+akshaver@users.noreply.github.com>
Co-authored-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: roikoren755 <26850796+roikoren755@users.noreply.github.com>
This commit is contained in:
akshaver
2026-09-08 09:59:58 -04:00
committed by GitHub
co-authored by wzhao18 Claude Opus 5 roikoren755
parent 414057a3d3
commit 263c4ff95f
12 changed files with 710 additions and 21 deletions
+22
View File
@@ -13,6 +13,28 @@ Set `enable_prefix_caching=True` in vLLM engine to enable APC. Here is an exampl
[examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py](../../examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py)
## Hybrid Mamba models
Under `--mamba-cache-mode align`, Mamba state is stored only on the Mamba block grid, so a prefix-cache hit can resume only at a block boundary. `--enable-mamba-fine-grained-prefix-cache` also stores a checkpoint at the shared-prefix junction, the point where an earlier request with the same prefix stopped. Requests whose shared prefix ends inside a block can then reuse it.
This helps when many requests share a long system prompt and then diverge. It is off by default, and takes effect only when all of the following hold:
- `--mamba-cache-mode align`
- EAGLE/MTP speculative decoding on the Mamba group
- `--prefix-match-unit` smaller than the Mamba block size
- the model does not use multi-module MTP
```bash
vllm serve <hybrid-model> \
--mamba-cache-mode align \
--prefix-match-unit 64 \
--enable-mamba-fine-grained-prefix-cache
```
`--prefix-match-unit` is required. It sets the granularity at which prefix-cache keys are computed. When unset it defaults to the greatest common divisor of the prefix-cacheable KV cache group block sizes. Under `align` that is the block size itself, so no sub-block boundary exists and the flag has no effect.
Choose a value that divides the block size of every prefix-cacheable KV cache group, and that is a multiple of the per-state compression ratio for models that use one, such as sparse MLA. vLLM validates both at startup and names the offending sizes in the error. Read the served block size from the startup log. 64 is a reasonable starting point.
## Example workloads
We describe two example workloads, where APC can provide huge performance benefit:
@@ -0,0 +1,359 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba "align" check-points at the position an EAGLE sibling resumes from.
Full attention hits where it holds a key; EAGLE prunes one hash unit off that
candidate and drops it. Mamba materializes state only on its own block grid, so
the pruned position holds nothing and the hit floors back a whole Mamba block.
Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here
drives the real ``Scheduler._mamba_block_aligned_split`` and the real
``KVCacheManager``; no chunk boundary is hard-coded.
"""
from dataclasses import replace
from types import SimpleNamespace
import pytest
from tests.v1.core.test_prefix_caching import (
_make_hybrid_kv_cache_config,
make_kv_cache_manager,
make_request,
)
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import init_none_hash
from vllm.v1.core.sched.scheduler import Scheduler
PREFIX = list(range(1, 20_001))
def _manager(
block_size,
hash_block_size,
*,
fine_grained=True,
num_blocks=8192,
eagle_group=None,
num_prefill_lookahead=0,
):
init_none_hash(sha256)
config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
)
if eagle_group is not None:
groups = list(config.kv_cache_groups)
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
config = replace(config, kv_cache_groups=groups)
return make_kv_cache_manager(
kv_cache_config=config,
max_model_len=1 << 20,
enable_caching=True,
hash_block_size=hash_block_size,
use_eagle=True,
num_prefill_lookahead=num_prefill_lookahead,
enable_mamba_fine_grained_prefix_cache=fine_grained,
)
def _stub(manager, block_size, hash_block_size, *, block_drop=True):
"""``self`` for the real ``Scheduler._mamba_block_aligned_split``.
Derives both gates exactly as ``Scheduler.__init__`` does, so a manager that
cannot honour the junction stop also stops the split from making one.
"""
partial_hit = (
hash_block_size < block_size and manager.coordinator.enable_partial_hash_hits
)
return SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
max_num_scheduled_tokens=1 << 20,
use_eagle=True,
# The EAGLE adjustments key on the block-drop bit, not plain use_eagle:
# they exist only to compensate for the drop.
use_eagle_block_drop=block_drop,
hash_block_size=hash_block_size,
mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=(
partial_hit and manager.mamba_fine_grained_prefix_cache
),
)
def _prefill(manager, stub, request, *, external=0) -> list[int]:
"""Schedule ``request`` to completion the way ``Scheduler.schedule()`` does.
Returns the chunk ends. Every boundary comes from the real splitter, so no
test in this file hard-codes one.
"""
blocks, local, junction = manager.get_computed_blocks(request)
request.shared_prefix_boundary = junction
ends = []
first = True
while request.num_computed_tokens < request.num_tokens:
new_local, ext = (local, external) if first else (0, 0)
start = request.num_computed_tokens + new_local + ext
if start >= request.num_tokens:
break
num_new = Scheduler._mamba_block_aligned_split(
stub, request, request.num_tokens - start, new_local, ext
)
if num_new <= 0:
break
assert (
manager.allocate_slots(
request,
num_new,
num_new_computed_tokens=new_local,
new_computed_blocks=blocks if first else None,
num_external_computed_tokens=ext,
has_scheduled_reqs=False,
)
is not None
)
request.num_computed_tokens = start + num_new
ends.append(request.num_computed_tokens)
_, retained = manager.take_kv_cache_block_copies()
if retained:
manager.block_pool.free_blocks(retained)
manager.new_step_starts()
first = False
return ends
def _orphaned_full_attention_tail(manager, stub, prompt_len):
"""Produce the state a KV connector leaves behind, which creates a junction.
NIXL reports ``num_prompt_tokens - 1`` external computed tokens for Mamba
models (``nixl/base_scheduler.py::_get_remote_prefill_token_count``), so the
producer starts past its own tail stop: full attention still registers a
partial tail the Mamba group has no state for.
"""
producer = make_request(
"producer", PREFIX[:prompt_len], stub.hash_block_size, sha256
)
_prefill(manager, stub, producer, external=prompt_len - 1)
def _sibling_hit(manager, shared, suffix, hash_block_size):
request = make_request("sibling", PREFIX[:shared] + suffix, hash_block_size, sha256)
return manager.get_computed_blocks(request)[1]
# --------------------------------------------------------------------------
# The two positions a sibling resumes at
def test_sibling_resumes_from_the_observed_junction():
"""The scheduler splits at the junction; the manager must cache there."""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed"
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
assert junction, "expected a junction against the orphaned full-attention tail"
_prefill(manager, stub, consumer)
hit = _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
assert hit == junction, (
f"the chunk stopped at {junction} but nothing was cached there: "
f"sibling resumes at {hit}"
)
def test_sibling_resumes_below_the_block_grid_when_the_prefix_ends_early():
"""A system prompt followed by a per-request suffix -- the deployed shape.
The owner's prompt tail sits over its own suffix, which no sibling shares, so
a check-point there is unreachable. The next request's full-attention match
stops at the last shared block boundary and EAGLE drops one hash unit below
it, so state has to exist just under the block grid too.
"""
block_size, hash_block_size = 16, 4
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
shared = 24
owner = make_request("owner", PREFIX[:shared] + [-1] * 16, hash_block_size, sha256)
_prefill(manager, stub, owner)
# The first follower observes the junction (16 shared, dropped to 12) and
# registers state there; the second one gets to resume from it.
follower = make_request(
"follower", PREFIX[:shared] + [-2] * 16, hash_block_size, sha256
)
_prefill(manager, stub, follower)
resume = shared // block_size * block_size - hash_block_size
hit = _sibling_hit(manager, shared, [-3] * 16, hash_block_size)
assert hit == resume, f"expected the resume point at {resume}, got {hit}"
# --------------------------------------------------------------------------
# Invariants the two clauses have to keep
def _armed_configs():
"""``(block_size, hash_block_size)`` pairs the junction path can arm on.
The prefix match unit must divide every group's block size and be strictly
smaller, or ``enable_partial_hash_hits`` is False and the path is inert.
"""
for block_size in (16, 128, 512):
for hash_block_size in (4, 32):
if hash_block_size < block_size and block_size % hash_block_size == 0:
yield block_size, hash_block_size
def test_scheduler_never_stops_where_the_manager_refuses():
"""A junction stop must always be a position the manager caches.
"The scheduler schedules a number of tokens it thinks can be cached, but the
cache manager won't agree" is a property of the pair, not of one
configuration, so sweep the armed parameter space.
"""
evaluated, refused, skipped = 0, [], []
for block_size, hash_block_size in _armed_configs():
for prompt_len in (
2 * block_size - hash_block_size,
3 * block_size + hash_block_size + 1,
4 * block_size - 1,
):
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, prompt_len)
shared = prompt_len // hash_block_size * hash_block_size
suffix = [-1] * (block_size + 3)
consumer = make_request(
"consumer", PREFIX[:shared] + suffix, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
if not junction:
continue
case = (block_size, hash_block_size, prompt_len, junction)
# The stop and the check-point are both keyed on the junction, so it
# has to be on the hash grid -- nothing else keeps them together.
assert junction % hash_block_size == 0, case
if junction not in _prefill(manager, stub, consumer):
skipped.append(case)
continue
evaluated += 1
hit = _sibling_hit(
manager, shared, [-2] * (block_size + 3), hash_block_size
)
if hit < junction:
refused.append(case + (hit,))
assert not skipped, (
f"the scheduler declined to stop at {len(skipped)}: {skipped[:3]}"
)
assert evaluated >= 10, f"sweep did not exercise the junction: {evaluated} cases"
assert not refused, (
f"{len(refused)} of {evaluated} configs split at a junction the manager then "
f"refused (block, hash, prompt_len, junction, sibling_hit): {refused[:3]}"
)
def test_a_junction_past_the_prompt_falls_back_and_registers_nothing():
"""A resumed request's junction can land in its output tokens.
The manager writes nothing there -- during decode the target is the running
state block, mutated in place, which equals what its key promises only after
that step's forward. So the scheduler must not stop AT it, but it must still
take the block-floored stop stock vLLM would have made: dropping the stop
entirely loses a check-point a sibling could have resumed from.
"""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
request = make_request("r", PREFIX[:2000], hash_block_size, sha256)
for _ in range(2000):
request.append_output_token_ids(1)
assert request.num_prompt_tokens == 2000 and request.num_tokens == 4000
# 2208 is past the prompt: not used verbatim, but the block-floored stop
# (2048) still applies -- not the last cacheable boundary (3072), which is
# what dropping the stop would give.
request.shared_prefix_boundary = 2208
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 2048
# Inside the prompt it still stops, at the junction itself.
request.shared_prefix_boundary = 992
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 992
# And the manager refuses on its own at the position that matters: the
# RUNNING state block mid-decode. Interior blocks are nulled in align mode,
# so every other decode-side position is already refused for that reason --
# here the block is live and holds exactly what a key would claim, and only
# the prompt bound stops it being published a forward before it changes.
_prefill(manager, stub, request)
mamba = manager.coordinator.single_type_managers[1]
running = request.num_computed_tokens
assert not mamba.req_to_blocks["r"][running // block_size].is_null
request.shared_prefix_boundary = running
assert mamba._cache_partial_tail_block(request, running) is None
@pytest.mark.parametrize("eagle_group", [None, 0])
@pytest.mark.parametrize("num_prefill_lookahead", [0, 2, 33])
def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
"""Turning the flag on may add check-points; it must never move one away.
Two axes decide whether the manager can honour the junction stop the
scheduler makes for it, and both fail silently -- as lost reuse, not as an
error. ``is_eagle_group`` annotation: the manager takes the EAGLE drop per
group while the scheduler flag is model-wide, so on an annotated model the
Mamba group can be outside ``eagle_group_ids``. Multi-module MTP: the
coordinator hands the manager ``num_computed - num_reprefillable``, not the
chunk end. In either case the junction stop displaces the block-boundary
stop and the check-point it replaced is never written.
"""
block_size, hash_block_size = 512, 32
def sibling_hit(fine_grained):
manager = _manager(
block_size,
hash_block_size,
fine_grained=fine_grained,
eagle_group=eagle_group,
num_prefill_lookahead=num_prefill_lookahead,
)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
_prefill(manager, stub, consumer)
return _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
on, off = sibling_hit(True), sibling_hit(False)
assert on >= off, f"enabling the flag cut the sibling's hit from {off} to {on}"
def test_disabled_by_default():
"""Off, the junction is block-floored and no resume point is registered."""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size, fine_grained=False)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
assert junction, "expected a junction"
_prefill(manager, stub, consumer)
assert _sibling_hit(manager, 2016, [-2] * 584, hash_block_size) < junction
request = make_request("r", PREFIX[:10000], hash_block_size, sha256)
request.shared_prefix_boundary = 2000
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 1536
@@ -184,6 +184,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
@@ -231,6 +232,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -271,6 +273,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -1798,6 +1801,178 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
def test_mamba_align_split_stops_below_eagle_proof_boundary():
"""The chunk that registers the partial tail must end one hash unit below
the prompt's last hash boundary under Eagle, which drops one unit past the
candidate.
"""
block_size = 1536
hash_block_size = 128
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=True,
use_eagle_block_drop=True,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
for prompt_len in (24576, 24577):
req = make_request("0", [0] * prompt_len, hash_block_size, sha256)
req.num_computed_tokens = 23040
assert (
split(self=mock, request=req, num_new_tokens=prompt_len - 23040)
== 24448 - 23040
)
# Without the eagle block drop the tail stays at the prompt's own last hash
# boundary -- the shift exists only to compensate for the drop, so it keys
# on use_eagle_block_drop rather than plain use_eagle.
mock.use_eagle_block_drop = False
req = make_request("1", [0] * 24577, hash_block_size, sha256)
req.num_computed_tokens = 23040
assert split(self=mock, request=req, num_new_tokens=1537) == 24576 - 23040
def test_mamba_partial_tail_hits_below_eagle_proof_boundary():
"""Regression: a prompt whose last hash boundary coincides with its last
mamba block boundary registered its only fine-grained checkpoint at a
position Eagle can never resume from, collapsing the next turn to the
coarse block boundary (or, under sparse retention, to a full miss).
"""
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
use_eagle=True,
)
# 9 tokens: the last hash boundary (8) is also a mamba block boundary, so
# the prompt's two checkpoints collapse. The reachable tail is 6.
req0 = make_request("0", [7] * 9, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
manager.new_step_starts()
assert manager.allocate_slots(req0, 3) is not None
req0.num_computed_tokens = 9
manager.new_step_starts()
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_block = manager.block_pool.get_cached_block(
partial_hash, kv_cache_group_ids=[1]
)
assert partial_block is not None
assert partial_block[0].block_hash_num_tokens == 6
# Full attention matches to 8 and drops one hash unit to 6, where the
# mamba tail now exists -- instead of falling back to the block boundary 4.
req1 = make_request("1", [7] * 9 + [9] * 4, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
def test_mamba_coarse_checkpoint_retained_below_eagle_proof_boundary():
"""Without a prefix-match unit there is no partial tail, so the only
checkpoint comes from the retention mask. A prompt ending just past its
last block boundary pinned that checkpoint where eagle can never resume,
so under ``retention_interval=0`` the reachable block was masked out and
the next turn missed entirely.
"""
block_size = 4
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
prefix_cache_retention_interval=0,
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
use_eagle=True,
)
# 9 tokens: the last aligned boundary is 8, which eagle cannot resume
# from, so block 0 (ending at 4) must stay retained. Prefill in
# scheduler-split style -- Mamba keeps one rolling state, so a boundary is
# only materialized by a chunk that ends exactly on it.
req0 = make_request("0", [7] * 9, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 4
manager.new_step_starts()
assert manager.allocate_slots(req0, 5) is not None
req0.num_computed_tokens = 9
manager.free(req0)
manager.new_step_starts()
# The reachable state (4) is retained; the unreachable one (8) is not.
assert (
manager.block_pool.get_cached_block(
req0.block_hashes[0], kv_cache_group_ids=[1]
)
is not None
)
req1 = make_request("1", [7] * 9 + [9] * 4, block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 4
def test_hybrid_sliding_window_group_disables_partial_hash_hits():
hash_block_size = 2
sliding_window_block_size = 2 * hash_block_size
@@ -144,6 +144,7 @@ def _split(
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=False,
hash_block_size=ATTN_BLOCK_SIZE,
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
mamba_prefill_checkpoint_alignment=(
@@ -357,7 +358,12 @@ def test_unaligned_resume_never_runs_past_its_block(
"""
prompt_len = 3602
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
tail_boundary = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
# Under eagle the partial-tail stop sits one hash unit below the prompt's
# last hash boundary: eagle matches a unit past its candidate and drops it,
# so nothing proves that last boundary.
tail_stop = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
if use_eagle:
tail_stop -= ATTN_BLOCK_SIZE
pos, ends = resume_at, []
while pos < prompt_len:
@@ -382,7 +388,7 @@ def test_unaligned_resume_never_runs_past_its_block(
for end in ends[:-1]:
aligned = end % MAMBA_BLOCK_SIZE == 0
assert aligned or (partial_hit and end == tail_boundary), (
assert aligned or (partial_hit and end == tail_stop), (
f"intermediate chunk end {end} is neither block-aligned nor the "
f"partial-tail boundary"
f"partial-tail stop ({tail_stop})"
)
+8 -6
View File
@@ -1169,6 +1169,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
use_eagle_block_drop=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req_2.shared_prefix_boundary = shared_prefix_boundary
@@ -3607,9 +3608,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
use_eagle=True,
)
# 127 tokens: latest replay boundary is floor((127 - 1) / 32) * 32 = 96.
# The EAGLE/MTP SWA lookup group must cache the local tail ending at
# 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12.
# 127 tokens: eagle proves a boundary only if an aligned unit exists past
# it, so the replay boundary rewinds one unit to
# floor(127 / 32) * 32 - 32 = 64. The SWA tail plus its proof block then
# ends at 72, two 8-token blocks wide: hashes 7 and 8.
token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
@@ -3623,7 +3625,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
assert blocks is not None
pool = manager.block_pool
expected_swa_cached = {11, 12}
expected_swa_cached = {7, 8}
for i in range(15):
cached = pool.get_cached_block(req0.block_hashes[i], kv_cache_group_ids=[1])
if i in expected_swa_cached:
@@ -3635,8 +3637,8 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 12 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12]
assert num_computed_tokens == 8 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 8]
def test_block_lookup_cache_single_block_per_key():
@@ -192,7 +192,7 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
)
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
manager.cache_blocks(request_id, 1024)
manager.cache_blocks(request_id, 1024, replay_boundary=0)
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0
+7
View File
@@ -195,6 +195,12 @@ class CacheConfig:
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
"""
enable_mamba_fine_grained_prefix_cache: bool = False
"""Also register a Mamba "align" checkpoint at the shared-prefix junction --
where an EAGLE/MTP sibling was observed to resume -- instead of only at the
prompt tail. Off by default; only takes effect with `mamba_cache_mode`
"align", EAGLE on the Mamba group, and a prefix match unit smaller than the
Mamba block size."""
replayssm_buffer_len: int = Field(default=16, gt=0)
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
rows and FlashInfer uses B+1. Kimi-K3 speculative decode does not use B.
@@ -274,6 +280,7 @@ class CacheConfig:
"prefix_cache_retention_interval",
# Prefix-caching implementation detail (doesn't affect compiled graph).
"prefix_match_unit",
"enable_mamba_fine_grained_prefix_cache",
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
+10
View File
@@ -726,6 +726,9 @@ class EngineArgs:
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
prefix_match_unit: int | None = get_field(CacheConfig, "prefix_match_unit")
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
enable_mamba_fine_grained_prefix_cache: bool = (
CacheConfig.enable_mamba_fine_grained_prefix_cache
)
replayssm_buffer_len: int = CacheConfig.replayssm_buffer_len
use_replayssm: bool = CacheConfig.use_replayssm
@@ -1287,6 +1290,10 @@ class EngineArgs:
cache_group.add_argument(
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
)
cache_group.add_argument(
"--enable-mamba-fine-grained-prefix-cache",
**cache_kwargs["enable_mamba_fine_grained_prefix_cache"],
)
cache_group.add_argument(
"--replayssm-buffer-len", **cache_kwargs["replayssm_buffer_len"]
)
@@ -2072,6 +2079,9 @@ class EngineArgs:
mamba_block_size=self.mamba_block_size,
prefix_match_unit=self.prefix_match_unit,
mamba_cache_mode=self.mamba_cache_mode,
enable_mamba_fine_grained_prefix_cache=(
self.enable_mamba_fine_grained_prefix_cache
),
replayssm_buffer_len=self.replayssm_buffer_len,
use_replayssm=self.use_replayssm,
kv_offloading_size=self.kv_offloading_size,
+26
View File
@@ -309,6 +309,28 @@ class KVCacheCoordinator(ABC):
for manager in self.single_type_managers
)
def get_replay_boundary(self, request: Request) -> int:
"""Return the position a later request replaying this prompt resumes at.
A cache hit is the shortest hit across all groups, so this is a
model-level position: every group has to retain state here, whether or
not it is the group that drops. Groups differ only in how much they
keep around it -- EAGLE groups also keep the block above, which they
match and drop back from (see ``reachable_block_mask``).
Under EAGLE that block must exist, so the boundary sits one alignment
unit below the prompt's last aligned position; every group's block size
divides the alignment, so the block above always fits in the prompt.
"""
if not self.eagle_group_ids:
return request.num_prompt_tokens - 1
aligned = (
request.num_prompt_tokens
// self.scheduler_block_size
* self.scheduler_block_size
)
return max(aligned - self.scheduler_block_size, 0)
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
"""
Cache the blocks for the request.
@@ -319,6 +341,7 @@ class KVCacheCoordinator(ABC):
that need to be cached
(including tokens that are already cached).
"""
replay_boundary = self.get_replay_boundary(request)
for manager in self.single_type_managers:
# Only cache tokens with finalized KV. The last num_reprefillable_tokens
# tokens can be re-prefilled during multi-module MTP.
@@ -329,6 +352,7 @@ class KVCacheCoordinator(ABC):
request,
num_tokens_to_cache,
retention_interval=self.retention_interval,
replay_boundary=replay_boundary,
)
def free(self, request_id: str) -> None:
@@ -753,6 +777,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
cached_num_computed_tokens = self._align_cacheable(num_computed_tokens)
replay_boundary = self.get_replay_boundary(request)
for manager in self.single_type_managers:
num_tokens_to_cache = cached_num_computed_tokens
# EAGLE groups match one block past each aligned boundary and drop
@@ -779,6 +804,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
request,
num_tokens_to_cache,
retention_interval=self.retention_interval,
replay_boundary=replay_boundary,
)
def find_longest_cache_hit(
+17
View File
@@ -15,6 +15,7 @@ from vllm.v1.core.kv_cache_coordinator import (
)
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy
from vllm.v1.core.single_type_kv_cache_manager import MambaManager
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CrossAttentionSpec,
@@ -132,6 +133,7 @@ class KVCacheManager:
pcp_world_size: int = 1,
metrics_collector: KVCacheMetricsCollector | None = None,
watermark: float = 0.0,
enable_mamba_fine_grained_prefix_cache: bool = False,
) -> None:
self.max_model_len = max_model_len
# When unset, fall back to `max_model_len` so the recycling-aware cap
@@ -164,6 +166,21 @@ class KVCacheManager:
metrics_collector=self.metrics_collector,
num_prefill_lookahead=num_prefill_lookahead,
)
# One predicate, read by both sides of the feature, so the scheduler
# cannot end a chunk at a junction the manager would refuse -- a refused
# junction costs a forward pass and displaces the block-boundary stop.
# Multi-module MTP is excluded because ``cache_blocks`` then hands the
# manager ``num_computed - num_reprefillable`` rather than the chunk end.
self.mamba_fine_grained_prefix_cache = (
enable_mamba_fine_grained_prefix_cache
and bool(self.coordinator.eagle_group_ids)
and self.coordinator.enable_partial_hash_hits
and self.coordinator.num_reprefillable_tokens == 0
)
if self.mamba_fine_grained_prefix_cache:
for manager in self.coordinator.single_type_managers:
if isinstance(manager, MambaManager):
manager.fine_grained_prefix_cache = True
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
self.block_pool = self.coordinator.block_pool
self.kv_cache_config = kv_cache_config
+34 -6
View File
@@ -312,6 +312,9 @@ class Scheduler(SchedulerInterface):
hash_block_size=hash_block_size,
metrics_collector=self.kv_metrics_collector,
watermark=self.scheduler_config.watermark,
enable_mamba_fine_grained_prefix_cache=(
self.cache_config.enable_mamba_fine_grained_prefix_cache
),
)
# Bind GPU block pool to the KV connector. This must happen after
# kv_cache_manager is constructed so block_pool is available.
@@ -362,6 +365,14 @@ class Scheduler(SchedulerInterface):
and self.hash_block_size < self.block_size
and self.kv_cache_manager.coordinator.enable_partial_hash_hits
)
# Opt-in: also stop at the junction, where an eagle sibling resumes. The
# manager decides whether it can check-point there (per-group eagle bit,
# no MTP re-prefill tail); splitting for a stop it would refuse costs a
# forward pass and displaces the block-boundary stop.
self.mamba_fine_grained_prefix_cache = (
self.mamba_partial_cache_hit
and self.kv_cache_manager.mamba_fine_grained_prefix_cache
)
# Counts of non-empty steps scheduled / processed. update_from_output
# is called once per scheduled step in FIFO order, so these stay in sync.
@@ -475,6 +486,26 @@ class Scheduler(SchedulerInterface):
if self.mamba_partial_cache_hit and not use_internal_checkpoint
else 0
)
if tail_boundary and self.use_eagle_block_drop:
# Eagle matches one hash unit past the candidate and drops it, so
# nothing proves the prompt's own last hash boundary. Materialize
# the state one unit lower, where the hit can actually land. Keyed on
# the block-drop bit, not plain use_eagle: this shift exists only to
# compensate for the drop, and the Mamba manager's matching gate
# reads the same bit (the coordinator is handed use_eagle_block_drop).
tail_boundary = max(tail_boundary - self.hash_block_size, 0)
junction = request.shared_prefix_boundary
# Block-floored: a sub-block junction's state is not separately cacheable.
block_floored = start + (junction - start) // block_size * block_size
# Past the prompt the manager writes nothing, so fall back to the
# block-floored stop rather than dropping it: a resumed request replaying
# output tokens can still observe a junction there.
junction_stop = (
junction
if self.mamba_fine_grained_prefix_cache
and junction <= request.num_prompt_tokens
else block_floored
)
stops = (
# Same invariant: a chunk starting mid-block stops at the boundary
# rather than running past it.
@@ -488,12 +519,9 @@ class Scheduler(SchedulerInterface):
tail_boundary
if last_cache_position < tail_boundary < request.num_prompt_tokens
else 0,
# Marconi shared-prefix junction, block-floored (a sub-block
# junction's state is not separately cacheable): cache its state
# so sibling requests sharing the prefix can reuse it.
start + (request.shared_prefix_boundary - start) // block_size * block_size
if start < request.shared_prefix_boundary < end
else 0,
# Marconi shared-prefix junction: cache its state so sibling
# requests sharing the prefix can reuse it.
junction_stop if start < junction < end else 0,
)
# Stop at the earliest mandatory position strictly inside the chunk.
end = min((s for s in stops if start < s < end), default=end)
+42 -5
View File
@@ -119,7 +119,9 @@ class SingleTypeKVCacheManager(ABC):
# aligned segment (SWA). Initialized lazily by the coordinator after
# determining the attention groups.
self.use_eagle = False
# ``CacheConfig.enable_mamba_fine_grained_prefix_cache``, narrowed and set
# by ``KVCacheManager``; only an EAGLE Mamba "align" group ever gets it.
self.fine_grained_prefix_cache = False
# Partial-hit copy-on-write bookkeeping. Populated only by fine-grained
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
@@ -449,6 +451,8 @@ class SingleTypeKVCacheManager(ABC):
request: Request,
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
) -> None:
"""
Cache the blocks for the request.
@@ -471,7 +475,7 @@ class SingleTypeKVCacheManager(ABC):
# Token boundaries whose reachable tail must be retained under sparse
# retention: the replay boundary (``num_prompt - 1``, capped by
# ``get_computed_blocks``) and any detected shared-prefix junction.
reachable_boundaries = [request.num_prompt_tokens - 1]
reachable_boundaries = [replay_boundary]
if request.shared_prefix_boundary:
reachable_boundaries.append(request.shared_prefix_boundary)
@@ -813,8 +817,15 @@ class FullAttentionManager(SingleTypeKVCacheManager):
request: Request,
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
) -> None:
super().cache_blocks(request, num_tokens, retention_interval=retention_interval)
super().cache_blocks(
request,
num_tokens,
retention_interval=retention_interval,
replay_boundary=replay_boundary,
)
hash_block_size = self.block_pool.hash_block_size
if self.block_size == hash_block_size:
return
@@ -1216,6 +1227,8 @@ class CircularBufferManager(FullAttentionManager):
request: Request,
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
) -> None:
return
@@ -1926,9 +1939,16 @@ class MambaManager(SingleTypeKVCacheManager):
request: Request,
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
) -> None:
num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0)
super().cache_blocks(request, num_tokens, retention_interval=retention_interval)
super().cache_blocks(
request,
num_tokens,
retention_interval=retention_interval,
replay_boundary=replay_boundary,
)
num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0)
if self.mamba_cache_mode == "align":
partial_hash = self._cache_partial_tail_block(request, num_tokens)
@@ -1994,7 +2014,22 @@ class MambaManager(SingleTypeKVCacheManager):
latest_prompt_hash_boundary = (
request.num_prompt_tokens // hash_block_size
) * hash_block_size
if num_tokens != latest_prompt_hash_boundary:
if self.use_eagle:
# Eagle groups match one hash unit past the candidate and drop it,
# so register the tail one unit lower.
latest_prompt_hash_boundary = max(
latest_prompt_hash_boundary - hash_block_size, 0
)
# The junction is the other position a sibling resumes at: where one was
# observed to stop, and where the scheduler already ends a chunk. Bounded
# to the prompt chunk being computed -- during decode the target is the
# running state block, mutated in place, which equals what its key
# promises only after that step's forward.
if num_tokens != latest_prompt_hash_boundary and not (
self.fine_grained_prefix_cache
and num_tokens == request.shared_prefix_boundary
and request.num_computed_tokens < num_tokens <= request.num_prompt_tokens
):
return None
block_idx = num_tokens // self.block_size
@@ -2054,6 +2089,8 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
request: Request,
num_tokens: int,
retention_interval: int | None = None,
*,
replay_boundary: int,
) -> None:
# We do not cache blocks for cross-attention to be shared between
# requests, so this method is not relevant.