[Bugfix][Spec Decode] Cache the Mamba state at the block-grid position of EAGLE resume (#53945)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Adam Shaver <ashaver@nvidia.com> Signed-off-by: akshaver <168006157+akshaver@users.noreply.github.com> Co-authored-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: roikoren755 <26850796+roikoren755@users.noreply.github.com>
This commit is contained in:
co-authored by
wzhao18
Claude Opus 5
roikoren755
parent
414057a3d3
commit
263c4ff95f
@@ -0,0 +1,359 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mamba "align" check-points at the position an EAGLE sibling resumes from.
|
||||
|
||||
Full attention hits where it holds a key; EAGLE prunes one hash unit off that
|
||||
candidate and drops it. Mamba materializes state only on its own block grid, so
|
||||
the pruned position holds nothing and the hit floors back a whole Mamba block.
|
||||
|
||||
Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here
|
||||
drives the real ``Scheduler._mamba_block_aligned_split`` and the real
|
||||
``KVCacheManager``; no chunk boundary is hard-coded.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.core.test_prefix_caching import (
|
||||
_make_hybrid_kv_cache_config,
|
||||
make_kv_cache_manager,
|
||||
make_request,
|
||||
)
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import init_none_hash
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
PREFIX = list(range(1, 20_001))
|
||||
|
||||
|
||||
def _manager(
|
||||
block_size,
|
||||
hash_block_size,
|
||||
*,
|
||||
fine_grained=True,
|
||||
num_blocks=8192,
|
||||
eagle_group=None,
|
||||
num_prefill_lookahead=0,
|
||||
):
|
||||
init_none_hash(sha256)
|
||||
config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
)
|
||||
if eagle_group is not None:
|
||||
groups = list(config.kv_cache_groups)
|
||||
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
|
||||
config = replace(config, kv_cache_groups=groups)
|
||||
return make_kv_cache_manager(
|
||||
kv_cache_config=config,
|
||||
max_model_len=1 << 20,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=True,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
enable_mamba_fine_grained_prefix_cache=fine_grained,
|
||||
)
|
||||
|
||||
|
||||
def _stub(manager, block_size, hash_block_size, *, block_drop=True):
|
||||
"""``self`` for the real ``Scheduler._mamba_block_aligned_split``.
|
||||
|
||||
Derives both gates exactly as ``Scheduler.__init__`` does, so a manager that
|
||||
cannot honour the junction stop also stops the split from making one.
|
||||
"""
|
||||
partial_hit = (
|
||||
hash_block_size < block_size and manager.coordinator.enable_partial_hash_hits
|
||||
)
|
||||
return SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
max_num_scheduled_tokens=1 << 20,
|
||||
use_eagle=True,
|
||||
# The EAGLE adjustments key on the block-drop bit, not plain use_eagle:
|
||||
# they exist only to compensate for the drop.
|
||||
use_eagle_block_drop=block_drop,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle
|
||||
mamba_partial_cache_hit=partial_hit,
|
||||
mamba_fine_grained_prefix_cache=(
|
||||
partial_hit and manager.mamba_fine_grained_prefix_cache
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _prefill(manager, stub, request, *, external=0) -> list[int]:
|
||||
"""Schedule ``request`` to completion the way ``Scheduler.schedule()`` does.
|
||||
|
||||
Returns the chunk ends. Every boundary comes from the real splitter, so no
|
||||
test in this file hard-codes one.
|
||||
"""
|
||||
blocks, local, junction = manager.get_computed_blocks(request)
|
||||
request.shared_prefix_boundary = junction
|
||||
ends = []
|
||||
first = True
|
||||
while request.num_computed_tokens < request.num_tokens:
|
||||
new_local, ext = (local, external) if first else (0, 0)
|
||||
start = request.num_computed_tokens + new_local + ext
|
||||
if start >= request.num_tokens:
|
||||
break
|
||||
num_new = Scheduler._mamba_block_aligned_split(
|
||||
stub, request, request.num_tokens - start, new_local, ext
|
||||
)
|
||||
if num_new <= 0:
|
||||
break
|
||||
assert (
|
||||
manager.allocate_slots(
|
||||
request,
|
||||
num_new,
|
||||
num_new_computed_tokens=new_local,
|
||||
new_computed_blocks=blocks if first else None,
|
||||
num_external_computed_tokens=ext,
|
||||
has_scheduled_reqs=False,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
request.num_computed_tokens = start + num_new
|
||||
ends.append(request.num_computed_tokens)
|
||||
_, retained = manager.take_kv_cache_block_copies()
|
||||
if retained:
|
||||
manager.block_pool.free_blocks(retained)
|
||||
manager.new_step_starts()
|
||||
first = False
|
||||
return ends
|
||||
|
||||
|
||||
def _orphaned_full_attention_tail(manager, stub, prompt_len):
|
||||
"""Produce the state a KV connector leaves behind, which creates a junction.
|
||||
|
||||
NIXL reports ``num_prompt_tokens - 1`` external computed tokens for Mamba
|
||||
models (``nixl/base_scheduler.py::_get_remote_prefill_token_count``), so the
|
||||
producer starts past its own tail stop: full attention still registers a
|
||||
partial tail the Mamba group has no state for.
|
||||
"""
|
||||
producer = make_request(
|
||||
"producer", PREFIX[:prompt_len], stub.hash_block_size, sha256
|
||||
)
|
||||
_prefill(manager, stub, producer, external=prompt_len - 1)
|
||||
|
||||
|
||||
def _sibling_hit(manager, shared, suffix, hash_block_size):
|
||||
request = make_request("sibling", PREFIX[:shared] + suffix, hash_block_size, sha256)
|
||||
return manager.get_computed_blocks(request)[1]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The two positions a sibling resumes at
|
||||
|
||||
|
||||
def test_sibling_resumes_from_the_observed_junction():
|
||||
"""The scheduler splits at the junction; the manager must cache there."""
|
||||
block_size, hash_block_size = 512, 32
|
||||
manager = _manager(block_size, hash_block_size)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed"
|
||||
_orphaned_full_attention_tail(manager, stub, 2020)
|
||||
|
||||
consumer = make_request(
|
||||
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
|
||||
)
|
||||
junction = manager.get_computed_blocks(consumer)[2]
|
||||
assert junction, "expected a junction against the orphaned full-attention tail"
|
||||
_prefill(manager, stub, consumer)
|
||||
|
||||
hit = _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
|
||||
assert hit == junction, (
|
||||
f"the chunk stopped at {junction} but nothing was cached there: "
|
||||
f"sibling resumes at {hit}"
|
||||
)
|
||||
|
||||
|
||||
def test_sibling_resumes_below_the_block_grid_when_the_prefix_ends_early():
|
||||
"""A system prompt followed by a per-request suffix -- the deployed shape.
|
||||
|
||||
The owner's prompt tail sits over its own suffix, which no sibling shares, so
|
||||
a check-point there is unreachable. The next request's full-attention match
|
||||
stops at the last shared block boundary and EAGLE drops one hash unit below
|
||||
it, so state has to exist just under the block grid too.
|
||||
"""
|
||||
block_size, hash_block_size = 16, 4
|
||||
manager = _manager(block_size, hash_block_size)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
|
||||
shared = 24
|
||||
owner = make_request("owner", PREFIX[:shared] + [-1] * 16, hash_block_size, sha256)
|
||||
_prefill(manager, stub, owner)
|
||||
|
||||
# The first follower observes the junction (16 shared, dropped to 12) and
|
||||
# registers state there; the second one gets to resume from it.
|
||||
follower = make_request(
|
||||
"follower", PREFIX[:shared] + [-2] * 16, hash_block_size, sha256
|
||||
)
|
||||
_prefill(manager, stub, follower)
|
||||
|
||||
resume = shared // block_size * block_size - hash_block_size
|
||||
hit = _sibling_hit(manager, shared, [-3] * 16, hash_block_size)
|
||||
assert hit == resume, f"expected the resume point at {resume}, got {hit}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Invariants the two clauses have to keep
|
||||
|
||||
|
||||
def _armed_configs():
|
||||
"""``(block_size, hash_block_size)`` pairs the junction path can arm on.
|
||||
|
||||
The prefix match unit must divide every group's block size and be strictly
|
||||
smaller, or ``enable_partial_hash_hits`` is False and the path is inert.
|
||||
"""
|
||||
for block_size in (16, 128, 512):
|
||||
for hash_block_size in (4, 32):
|
||||
if hash_block_size < block_size and block_size % hash_block_size == 0:
|
||||
yield block_size, hash_block_size
|
||||
|
||||
|
||||
def test_scheduler_never_stops_where_the_manager_refuses():
|
||||
"""A junction stop must always be a position the manager caches.
|
||||
|
||||
"The scheduler schedules a number of tokens it thinks can be cached, but the
|
||||
cache manager won't agree" is a property of the pair, not of one
|
||||
configuration, so sweep the armed parameter space.
|
||||
"""
|
||||
evaluated, refused, skipped = 0, [], []
|
||||
for block_size, hash_block_size in _armed_configs():
|
||||
for prompt_len in (
|
||||
2 * block_size - hash_block_size,
|
||||
3 * block_size + hash_block_size + 1,
|
||||
4 * block_size - 1,
|
||||
):
|
||||
manager = _manager(block_size, hash_block_size)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
_orphaned_full_attention_tail(manager, stub, prompt_len)
|
||||
|
||||
shared = prompt_len // hash_block_size * hash_block_size
|
||||
suffix = [-1] * (block_size + 3)
|
||||
consumer = make_request(
|
||||
"consumer", PREFIX[:shared] + suffix, hash_block_size, sha256
|
||||
)
|
||||
junction = manager.get_computed_blocks(consumer)[2]
|
||||
if not junction:
|
||||
continue
|
||||
case = (block_size, hash_block_size, prompt_len, junction)
|
||||
# The stop and the check-point are both keyed on the junction, so it
|
||||
# has to be on the hash grid -- nothing else keeps them together.
|
||||
assert junction % hash_block_size == 0, case
|
||||
if junction not in _prefill(manager, stub, consumer):
|
||||
skipped.append(case)
|
||||
continue
|
||||
evaluated += 1
|
||||
hit = _sibling_hit(
|
||||
manager, shared, [-2] * (block_size + 3), hash_block_size
|
||||
)
|
||||
if hit < junction:
|
||||
refused.append(case + (hit,))
|
||||
|
||||
assert not skipped, (
|
||||
f"the scheduler declined to stop at {len(skipped)}: {skipped[:3]}"
|
||||
)
|
||||
assert evaluated >= 10, f"sweep did not exercise the junction: {evaluated} cases"
|
||||
assert not refused, (
|
||||
f"{len(refused)} of {evaluated} configs split at a junction the manager then "
|
||||
f"refused (block, hash, prompt_len, junction, sibling_hit): {refused[:3]}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_junction_past_the_prompt_falls_back_and_registers_nothing():
|
||||
"""A resumed request's junction can land in its output tokens.
|
||||
|
||||
The manager writes nothing there -- during decode the target is the running
|
||||
state block, mutated in place, which equals what its key promises only after
|
||||
that step's forward. So the scheduler must not stop AT it, but it must still
|
||||
take the block-floored stop stock vLLM would have made: dropping the stop
|
||||
entirely loses a check-point a sibling could have resumed from.
|
||||
"""
|
||||
block_size, hash_block_size = 512, 32
|
||||
manager = _manager(block_size, hash_block_size)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
|
||||
request = make_request("r", PREFIX[:2000], hash_block_size, sha256)
|
||||
for _ in range(2000):
|
||||
request.append_output_token_ids(1)
|
||||
assert request.num_prompt_tokens == 2000 and request.num_tokens == 4000
|
||||
|
||||
# 2208 is past the prompt: not used verbatim, but the block-floored stop
|
||||
# (2048) still applies -- not the last cacheable boundary (3072), which is
|
||||
# what dropping the stop would give.
|
||||
request.shared_prefix_boundary = 2208
|
||||
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 2048
|
||||
# Inside the prompt it still stops, at the junction itself.
|
||||
request.shared_prefix_boundary = 992
|
||||
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 992
|
||||
|
||||
# And the manager refuses on its own at the position that matters: the
|
||||
# RUNNING state block mid-decode. Interior blocks are nulled in align mode,
|
||||
# so every other decode-side position is already refused for that reason --
|
||||
# here the block is live and holds exactly what a key would claim, and only
|
||||
# the prompt bound stops it being published a forward before it changes.
|
||||
_prefill(manager, stub, request)
|
||||
mamba = manager.coordinator.single_type_managers[1]
|
||||
running = request.num_computed_tokens
|
||||
assert not mamba.req_to_blocks["r"][running // block_size].is_null
|
||||
request.shared_prefix_boundary = running
|
||||
assert mamba._cache_partial_tail_block(request, running) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eagle_group", [None, 0])
|
||||
@pytest.mark.parametrize("num_prefill_lookahead", [0, 2, 33])
|
||||
def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
|
||||
"""Turning the flag on may add check-points; it must never move one away.
|
||||
|
||||
Two axes decide whether the manager can honour the junction stop the
|
||||
scheduler makes for it, and both fail silently -- as lost reuse, not as an
|
||||
error. ``is_eagle_group`` annotation: the manager takes the EAGLE drop per
|
||||
group while the scheduler flag is model-wide, so on an annotated model the
|
||||
Mamba group can be outside ``eagle_group_ids``. Multi-module MTP: the
|
||||
coordinator hands the manager ``num_computed - num_reprefillable``, not the
|
||||
chunk end. In either case the junction stop displaces the block-boundary
|
||||
stop and the check-point it replaced is never written.
|
||||
"""
|
||||
block_size, hash_block_size = 512, 32
|
||||
|
||||
def sibling_hit(fine_grained):
|
||||
manager = _manager(
|
||||
block_size,
|
||||
hash_block_size,
|
||||
fine_grained=fine_grained,
|
||||
eagle_group=eagle_group,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
_orphaned_full_attention_tail(manager, stub, 2020)
|
||||
consumer = make_request(
|
||||
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
|
||||
)
|
||||
_prefill(manager, stub, consumer)
|
||||
return _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
|
||||
|
||||
on, off = sibling_hit(True), sibling_hit(False)
|
||||
assert on >= off, f"enabling the flag cut the sibling's hit from {off} to {on}"
|
||||
|
||||
|
||||
def test_disabled_by_default():
|
||||
"""Off, the junction is block-floored and no resume point is registered."""
|
||||
block_size, hash_block_size = 512, 32
|
||||
manager = _manager(block_size, hash_block_size, fine_grained=False)
|
||||
stub = _stub(manager, block_size, hash_block_size)
|
||||
_orphaned_full_attention_tail(manager, stub, 2020)
|
||||
|
||||
consumer = make_request(
|
||||
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
|
||||
)
|
||||
junction = manager.get_computed_blocks(consumer)[2]
|
||||
assert junction, "expected a junction"
|
||||
_prefill(manager, stub, consumer)
|
||||
|
||||
assert _sibling_hit(manager, 2016, [-2] * 584, hash_block_size) < junction
|
||||
|
||||
request = make_request("r", PREFIX[:10000], hash_block_size, sha256)
|
||||
request.shared_prefix_boundary = 2000
|
||||
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 1536
|
||||
@@ -184,6 +184,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
|
||||
dcp_world_size=dcp_world_size,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
@@ -231,6 +232,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
@@ -271,6 +273,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=32,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req = make_request("0", [0] * prompt_length, 32, sha256)
|
||||
@@ -1798,6 +1801,178 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
|
||||
|
||||
def test_mamba_align_split_stops_below_eagle_proof_boundary():
|
||||
"""The chunk that registers the partial tail must end one hash unit below
|
||||
the prompt's last hash boundary under Eagle, which drops one unit past the
|
||||
candidate.
|
||||
"""
|
||||
block_size = 1536
|
||||
hash_block_size = 128
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle=True,
|
||||
use_eagle_block_drop=True,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
for prompt_len in (24576, 24577):
|
||||
req = make_request("0", [0] * prompt_len, hash_block_size, sha256)
|
||||
req.num_computed_tokens = 23040
|
||||
assert (
|
||||
split(self=mock, request=req, num_new_tokens=prompt_len - 23040)
|
||||
== 24448 - 23040
|
||||
)
|
||||
|
||||
# Without the eagle block drop the tail stays at the prompt's own last hash
|
||||
# boundary -- the shift exists only to compensate for the drop, so it keys
|
||||
# on use_eagle_block_drop rather than plain use_eagle.
|
||||
mock.use_eagle_block_drop = False
|
||||
req = make_request("1", [0] * 24577, hash_block_size, sha256)
|
||||
req.num_computed_tokens = 23040
|
||||
assert split(self=mock, request=req, num_new_tokens=1537) == 24576 - 23040
|
||||
|
||||
|
||||
def test_mamba_partial_tail_hits_below_eagle_proof_boundary():
|
||||
"""Regression: a prompt whose last hash boundary coincides with its last
|
||||
mamba block boundary registered its only fine-grained checkpoint at a
|
||||
position Eagle can never resume from, collapsing the next turn to the
|
||||
coarse block boundary (or, under sparse retention, to a full miss).
|
||||
"""
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# 9 tokens: the last hash boundary (8) is also a mamba block boundary, so
|
||||
# the prompt's two checkpoints collapse. The reachable tail is 6.
|
||||
req0 = make_request("0", [7] * 9, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 3) is not None
|
||||
req0.num_computed_tokens = 9
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_block = manager.block_pool.get_cached_block(
|
||||
partial_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert partial_block is not None
|
||||
assert partial_block[0].block_hash_num_tokens == 6
|
||||
|
||||
# Full attention matches to 8 and drops one hash unit to 6, where the
|
||||
# mamba tail now exists -- instead of falling back to the block boundary 4.
|
||||
req1 = make_request("1", [7] * 9 + [9] * 4, hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
|
||||
|
||||
def test_mamba_coarse_checkpoint_retained_below_eagle_proof_boundary():
|
||||
"""Without a prefix-match unit there is no partial tail, so the only
|
||||
checkpoint comes from the retention mask. A prompt ending just past its
|
||||
last block boundary pinned that checkpoint where eagle can never resume,
|
||||
so under ``retention_interval=0`` the reachable block was masked out and
|
||||
the next turn missed entirely.
|
||||
"""
|
||||
block_size = 4
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
prefix_cache_retention_interval=0,
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# 9 tokens: the last aligned boundary is 8, which eagle cannot resume
|
||||
# from, so block 0 (ending at 4) must stay retained. Prefill in
|
||||
# scheduler-split style -- Mamba keeps one rolling state, so a boundary is
|
||||
# only materialized by a chunk that ends exactly on it.
|
||||
req0 = make_request("0", [7] * 9, block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 4
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 5) is not None
|
||||
req0.num_computed_tokens = 9
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
# The reachable state (4) is retained; the unreachable one (8) is not.
|
||||
assert (
|
||||
manager.block_pool.get_cached_block(
|
||||
req0.block_hashes[0], kv_cache_group_ids=[1]
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
req1 = make_request("1", [7] * 9 + [9] * 4, block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 4
|
||||
|
||||
|
||||
def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
hash_block_size = 2
|
||||
sliding_window_block_size = 2 * hash_block_size
|
||||
|
||||
@@ -144,6 +144,7 @@ def _split(
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
# `prefix_match_unit` finer than the block size (#46384).
|
||||
mamba_partial_cache_hit=partial_hit,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
hash_block_size=ATTN_BLOCK_SIZE,
|
||||
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
|
||||
mamba_prefill_checkpoint_alignment=(
|
||||
@@ -357,7 +358,12 @@ def test_unaligned_resume_never_runs_past_its_block(
|
||||
"""
|
||||
prompt_len = 3602
|
||||
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
|
||||
tail_boundary = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
|
||||
# Under eagle the partial-tail stop sits one hash unit below the prompt's
|
||||
# last hash boundary: eagle matches a unit past its candidate and drops it,
|
||||
# so nothing proves that last boundary.
|
||||
tail_stop = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
|
||||
if use_eagle:
|
||||
tail_stop -= ATTN_BLOCK_SIZE
|
||||
|
||||
pos, ends = resume_at, []
|
||||
while pos < prompt_len:
|
||||
@@ -382,7 +388,7 @@ def test_unaligned_resume_never_runs_past_its_block(
|
||||
|
||||
for end in ends[:-1]:
|
||||
aligned = end % MAMBA_BLOCK_SIZE == 0
|
||||
assert aligned or (partial_hit and end == tail_boundary), (
|
||||
assert aligned or (partial_hit and end == tail_stop), (
|
||||
f"intermediate chunk end {end} is neither block-aligned nor the "
|
||||
f"partial-tail boundary"
|
||||
f"partial-tail stop ({tail_stop})"
|
||||
)
|
||||
|
||||
@@ -1169,6 +1169,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
use_eagle_block_drop=False,
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
mamba_fine_grained_prefix_cache=False,
|
||||
mamba_has_prefill_checkpoint_blocks=False,
|
||||
)
|
||||
req_2.shared_prefix_boundary = shared_prefix_boundary
|
||||
@@ -3607,9 +3608,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# 127 tokens: latest replay boundary is floor((127 - 1) / 32) * 32 = 96.
|
||||
# The EAGLE/MTP SWA lookup group must cache the local tail ending at
|
||||
# 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12.
|
||||
# 127 tokens: eagle proves a boundary only if an aligned unit exists past
|
||||
# it, so the replay boundary rewinds one unit to
|
||||
# floor(127 / 32) * 32 - 32 = 64. The SWA tail plus its proof block then
|
||||
# ends at 72, two 8-token blocks wide: hashes 7 and 8.
|
||||
token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
@@ -3623,7 +3625,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
assert blocks is not None
|
||||
|
||||
pool = manager.block_pool
|
||||
expected_swa_cached = {11, 12}
|
||||
expected_swa_cached = {7, 8}
|
||||
for i in range(15):
|
||||
cached = pool.get_cached_block(req0.block_hashes[i], kv_cache_group_ids=[1])
|
||||
if i in expected_swa_cached:
|
||||
@@ -3635,8 +3637,8 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 12 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12]
|
||||
assert num_computed_tokens == 8 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 8]
|
||||
|
||||
|
||||
def test_block_lookup_cache_single_block_per_key():
|
||||
|
||||
@@ -192,7 +192,7 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
)
|
||||
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
|
||||
|
||||
manager.cache_blocks(request_id, 1024)
|
||||
manager.cache_blocks(request_id, 1024, replay_boundary=0)
|
||||
manager.remove_skipped_blocks(request_id, 1024)
|
||||
assert manager.get_num_common_prefix_blocks(request_id) == 0
|
||||
assert manager.get_num_skipped_tokens(1024) == 0
|
||||
|
||||
Reference in New Issue
Block a user