[Bugfix][Spec Decode] Cache the Mamba state at the block-grid position of EAGLE resume (#53945)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Signed-off-by: Adam Shaver <ashaver@nvidia.com>
Signed-off-by: akshaver <168006157+akshaver@users.noreply.github.com>
Co-authored-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: roikoren755 <26850796+roikoren755@users.noreply.github.com>
This commit is contained in:
akshaver
2026-09-08 09:59:58 -04:00
committed by GitHub
co-authored by wzhao18 Claude Opus 5 roikoren755
parent 414057a3d3
commit 263c4ff95f
12 changed files with 710 additions and 21 deletions
@@ -0,0 +1,359 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mamba "align" check-points at the position an EAGLE sibling resumes from.
Full attention hits where it holds a key; EAGLE prunes one hash unit off that
candidate and drops it. Mamba materializes state only on its own block grid, so
the pruned position holds nothing and the hit floors back a whole Mamba block.
Gated by ``CacheConfig.enable_mamba_fine_grained_prefix_cache``. Every test here
drives the real ``Scheduler._mamba_block_aligned_split`` and the real
``KVCacheManager``; no chunk boundary is hard-coded.
"""
from dataclasses import replace
from types import SimpleNamespace
import pytest
from tests.v1.core.test_prefix_caching import (
_make_hybrid_kv_cache_config,
make_kv_cache_manager,
make_request,
)
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import init_none_hash
from vllm.v1.core.sched.scheduler import Scheduler
PREFIX = list(range(1, 20_001))
def _manager(
block_size,
hash_block_size,
*,
fine_grained=True,
num_blocks=8192,
eagle_group=None,
num_prefill_lookahead=0,
):
init_none_hash(sha256)
config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
)
if eagle_group is not None:
groups = list(config.kv_cache_groups)
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
config = replace(config, kv_cache_groups=groups)
return make_kv_cache_manager(
kv_cache_config=config,
max_model_len=1 << 20,
enable_caching=True,
hash_block_size=hash_block_size,
use_eagle=True,
num_prefill_lookahead=num_prefill_lookahead,
enable_mamba_fine_grained_prefix_cache=fine_grained,
)
def _stub(manager, block_size, hash_block_size, *, block_drop=True):
"""``self`` for the real ``Scheduler._mamba_block_aligned_split``.
Derives both gates exactly as ``Scheduler.__init__`` does, so a manager that
cannot honour the junction stop also stops the split from making one.
"""
partial_hit = (
hash_block_size < block_size and manager.coordinator.enable_partial_hash_hits
)
return SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
max_num_scheduled_tokens=1 << 20,
use_eagle=True,
# The EAGLE adjustments key on the block-drop bit, not plain use_eagle:
# they exist only to compensate for the drop.
use_eagle_block_drop=block_drop,
hash_block_size=hash_block_size,
mamba_has_prefill_checkpoint_blocks=False, # forced False under eagle
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=(
partial_hit and manager.mamba_fine_grained_prefix_cache
),
)
def _prefill(manager, stub, request, *, external=0) -> list[int]:
"""Schedule ``request`` to completion the way ``Scheduler.schedule()`` does.
Returns the chunk ends. Every boundary comes from the real splitter, so no
test in this file hard-codes one.
"""
blocks, local, junction = manager.get_computed_blocks(request)
request.shared_prefix_boundary = junction
ends = []
first = True
while request.num_computed_tokens < request.num_tokens:
new_local, ext = (local, external) if first else (0, 0)
start = request.num_computed_tokens + new_local + ext
if start >= request.num_tokens:
break
num_new = Scheduler._mamba_block_aligned_split(
stub, request, request.num_tokens - start, new_local, ext
)
if num_new <= 0:
break
assert (
manager.allocate_slots(
request,
num_new,
num_new_computed_tokens=new_local,
new_computed_blocks=blocks if first else None,
num_external_computed_tokens=ext,
has_scheduled_reqs=False,
)
is not None
)
request.num_computed_tokens = start + num_new
ends.append(request.num_computed_tokens)
_, retained = manager.take_kv_cache_block_copies()
if retained:
manager.block_pool.free_blocks(retained)
manager.new_step_starts()
first = False
return ends
def _orphaned_full_attention_tail(manager, stub, prompt_len):
"""Produce the state a KV connector leaves behind, which creates a junction.
NIXL reports ``num_prompt_tokens - 1`` external computed tokens for Mamba
models (``nixl/base_scheduler.py::_get_remote_prefill_token_count``), so the
producer starts past its own tail stop: full attention still registers a
partial tail the Mamba group has no state for.
"""
producer = make_request(
"producer", PREFIX[:prompt_len], stub.hash_block_size, sha256
)
_prefill(manager, stub, producer, external=prompt_len - 1)
def _sibling_hit(manager, shared, suffix, hash_block_size):
request = make_request("sibling", PREFIX[:shared] + suffix, hash_block_size, sha256)
return manager.get_computed_blocks(request)[1]
# --------------------------------------------------------------------------
# The two positions a sibling resumes at
def test_sibling_resumes_from_the_observed_junction():
"""The scheduler splits at the junction; the manager must cache there."""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
assert stub.mamba_fine_grained_prefix_cache, "the feature must be armed"
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
assert junction, "expected a junction against the orphaned full-attention tail"
_prefill(manager, stub, consumer)
hit = _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
assert hit == junction, (
f"the chunk stopped at {junction} but nothing was cached there: "
f"sibling resumes at {hit}"
)
def test_sibling_resumes_below_the_block_grid_when_the_prefix_ends_early():
"""A system prompt followed by a per-request suffix -- the deployed shape.
The owner's prompt tail sits over its own suffix, which no sibling shares, so
a check-point there is unreachable. The next request's full-attention match
stops at the last shared block boundary and EAGLE drops one hash unit below
it, so state has to exist just under the block grid too.
"""
block_size, hash_block_size = 16, 4
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
shared = 24
owner = make_request("owner", PREFIX[:shared] + [-1] * 16, hash_block_size, sha256)
_prefill(manager, stub, owner)
# The first follower observes the junction (16 shared, dropped to 12) and
# registers state there; the second one gets to resume from it.
follower = make_request(
"follower", PREFIX[:shared] + [-2] * 16, hash_block_size, sha256
)
_prefill(manager, stub, follower)
resume = shared // block_size * block_size - hash_block_size
hit = _sibling_hit(manager, shared, [-3] * 16, hash_block_size)
assert hit == resume, f"expected the resume point at {resume}, got {hit}"
# --------------------------------------------------------------------------
# Invariants the two clauses have to keep
def _armed_configs():
"""``(block_size, hash_block_size)`` pairs the junction path can arm on.
The prefix match unit must divide every group's block size and be strictly
smaller, or ``enable_partial_hash_hits`` is False and the path is inert.
"""
for block_size in (16, 128, 512):
for hash_block_size in (4, 32):
if hash_block_size < block_size and block_size % hash_block_size == 0:
yield block_size, hash_block_size
def test_scheduler_never_stops_where_the_manager_refuses():
"""A junction stop must always be a position the manager caches.
"The scheduler schedules a number of tokens it thinks can be cached, but the
cache manager won't agree" is a property of the pair, not of one
configuration, so sweep the armed parameter space.
"""
evaluated, refused, skipped = 0, [], []
for block_size, hash_block_size in _armed_configs():
for prompt_len in (
2 * block_size - hash_block_size,
3 * block_size + hash_block_size + 1,
4 * block_size - 1,
):
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, prompt_len)
shared = prompt_len // hash_block_size * hash_block_size
suffix = [-1] * (block_size + 3)
consumer = make_request(
"consumer", PREFIX[:shared] + suffix, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
if not junction:
continue
case = (block_size, hash_block_size, prompt_len, junction)
# The stop and the check-point are both keyed on the junction, so it
# has to be on the hash grid -- nothing else keeps them together.
assert junction % hash_block_size == 0, case
if junction not in _prefill(manager, stub, consumer):
skipped.append(case)
continue
evaluated += 1
hit = _sibling_hit(
manager, shared, [-2] * (block_size + 3), hash_block_size
)
if hit < junction:
refused.append(case + (hit,))
assert not skipped, (
f"the scheduler declined to stop at {len(skipped)}: {skipped[:3]}"
)
assert evaluated >= 10, f"sweep did not exercise the junction: {evaluated} cases"
assert not refused, (
f"{len(refused)} of {evaluated} configs split at a junction the manager then "
f"refused (block, hash, prompt_len, junction, sibling_hit): {refused[:3]}"
)
def test_a_junction_past_the_prompt_falls_back_and_registers_nothing():
"""A resumed request's junction can land in its output tokens.
The manager writes nothing there -- during decode the target is the running
state block, mutated in place, which equals what its key promises only after
that step's forward. So the scheduler must not stop AT it, but it must still
take the block-floored stop stock vLLM would have made: dropping the stop
entirely loses a check-point a sibling could have resumed from.
"""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size)
stub = _stub(manager, block_size, hash_block_size)
request = make_request("r", PREFIX[:2000], hash_block_size, sha256)
for _ in range(2000):
request.append_output_token_ids(1)
assert request.num_prompt_tokens == 2000 and request.num_tokens == 4000
# 2208 is past the prompt: not used verbatim, but the block-floored stop
# (2048) still applies -- not the last cacheable boundary (3072), which is
# what dropping the stop would give.
request.shared_prefix_boundary = 2208
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 2048
# Inside the prompt it still stops, at the junction itself.
request.shared_prefix_boundary = 992
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 992
# And the manager refuses on its own at the position that matters: the
# RUNNING state block mid-decode. Interior blocks are nulled in align mode,
# so every other decode-side position is already refused for that reason --
# here the block is live and holds exactly what a key would claim, and only
# the prompt bound stops it being published a forward before it changes.
_prefill(manager, stub, request)
mamba = manager.coordinator.single_type_managers[1]
running = request.num_computed_tokens
assert not mamba.req_to_blocks["r"][running // block_size].is_null
request.shared_prefix_boundary = running
assert mamba._cache_partial_tail_block(request, running) is None
@pytest.mark.parametrize("eagle_group", [None, 0])
@pytest.mark.parametrize("num_prefill_lookahead", [0, 2, 33])
def test_enabling_never_reduces_reuse(eagle_group, num_prefill_lookahead):
"""Turning the flag on may add check-points; it must never move one away.
Two axes decide whether the manager can honour the junction stop the
scheduler makes for it, and both fail silently -- as lost reuse, not as an
error. ``is_eagle_group`` annotation: the manager takes the EAGLE drop per
group while the scheduler flag is model-wide, so on an annotated model the
Mamba group can be outside ``eagle_group_ids``. Multi-module MTP: the
coordinator hands the manager ``num_computed - num_reprefillable``, not the
chunk end. In either case the junction stop displaces the block-boundary
stop and the check-point it replaced is never written.
"""
block_size, hash_block_size = 512, 32
def sibling_hit(fine_grained):
manager = _manager(
block_size,
hash_block_size,
fine_grained=fine_grained,
eagle_group=eagle_group,
num_prefill_lookahead=num_prefill_lookahead,
)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
_prefill(manager, stub, consumer)
return _sibling_hit(manager, 2016, [-2] * 584, hash_block_size)
on, off = sibling_hit(True), sibling_hit(False)
assert on >= off, f"enabling the flag cut the sibling's hit from {off} to {on}"
def test_disabled_by_default():
"""Off, the junction is block-floored and no resume point is registered."""
block_size, hash_block_size = 512, 32
manager = _manager(block_size, hash_block_size, fine_grained=False)
stub = _stub(manager, block_size, hash_block_size)
_orphaned_full_attention_tail(manager, stub, 2020)
consumer = make_request(
"consumer", PREFIX[:2016] + [-1] * 584, hash_block_size, sha256
)
junction = manager.get_computed_blocks(consumer)[2]
assert junction, "expected a junction"
_prefill(manager, stub, consumer)
assert _sibling_hit(manager, 2016, [-2] * 584, hash_block_size) < junction
request = make_request("r", PREFIX[:10000], hash_block_size, sha256)
request.shared_prefix_boundary = 2000
assert Scheduler._mamba_block_aligned_split(stub, request, 8192) == 1536
@@ -184,6 +184,7 @@ def test_mamba_align_split_partial_tail_schedule(dcp_world_size: int):
dcp_world_size=dcp_world_size,
scheduler_block_size=scheduler_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
@@ -231,6 +232,7 @@ def test_mamba_align_split_when_block_exceeds_scheduling_budget():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -271,6 +273,7 @@ def test_mamba_align_split_when_block_exceeds_long_prefill_threshold():
use_eagle_block_drop=False,
hash_block_size=32,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req = make_request("0", [0] * prompt_length, 32, sha256)
@@ -1798,6 +1801,178 @@ def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
def test_mamba_align_split_stops_below_eagle_proof_boundary():
"""The chunk that registers the partial tail must end one hash unit below
the prompt's last hash boundary under Eagle, which drops one unit past the
candidate.
"""
block_size = 1536
hash_block_size = 128
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle=True,
use_eagle_block_drop=True,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
split = Scheduler._mamba_block_aligned_split
for prompt_len in (24576, 24577):
req = make_request("0", [0] * prompt_len, hash_block_size, sha256)
req.num_computed_tokens = 23040
assert (
split(self=mock, request=req, num_new_tokens=prompt_len - 23040)
== 24448 - 23040
)
# Without the eagle block drop the tail stays at the prompt's own last hash
# boundary -- the shift exists only to compensate for the drop, so it keys
# on use_eagle_block_drop rather than plain use_eagle.
mock.use_eagle_block_drop = False
req = make_request("1", [0] * 24577, hash_block_size, sha256)
req.num_computed_tokens = 23040
assert split(self=mock, request=req, num_new_tokens=1537) == 24576 - 23040
def test_mamba_partial_tail_hits_below_eagle_proof_boundary():
"""Regression: a prompt whose last hash boundary coincides with its last
mamba block boundary registered its only fine-grained checkpoint at a
position Eagle can never resume from, collapsing the next turn to the
coarse block boundary (or, under sparse retention, to a full miss).
"""
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
use_eagle=True,
)
# 9 tokens: the last hash boundary (8) is also a mamba block boundary, so
# the prompt's two checkpoints collapse. The reachable tail is 6.
req0 = make_request("0", [7] * 9, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
manager.new_step_starts()
assert manager.allocate_slots(req0, 3) is not None
req0.num_computed_tokens = 9
manager.new_step_starts()
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_block = manager.block_pool.get_cached_block(
partial_hash, kv_cache_group_ids=[1]
)
assert partial_block is not None
assert partial_block[0].block_hash_num_tokens == 6
# Full attention matches to 8 and drops one hash unit to 6, where the
# mamba tail now exists -- instead of falling back to the block boundary 4.
req1 = make_request("1", [7] * 9 + [9] * 4, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
def test_mamba_coarse_checkpoint_retained_below_eagle_proof_boundary():
"""Without a prefix-match unit there is no partial tail, so the only
checkpoint comes from the retention mask. A prompt ending just past its
last block boundary pinned that checkpoint where eagle can never resume,
so under ``retention_interval=0`` the reachable block was masked out and
the next turn missed entirely.
"""
block_size = 4
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
prefix_cache_retention_interval=0,
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
use_eagle=True,
)
# 9 tokens: the last aligned boundary is 8, which eagle cannot resume
# from, so block 0 (ending at 4) must stay retained. Prefill in
# scheduler-split style -- Mamba keeps one rolling state, so a boundary is
# only materialized by a chunk that ends exactly on it.
req0 = make_request("0", [7] * 9, block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 4
manager.new_step_starts()
assert manager.allocate_slots(req0, 5) is not None
req0.num_computed_tokens = 9
manager.free(req0)
manager.new_step_starts()
# The reachable state (4) is retained; the unreachable one (8) is not.
assert (
manager.block_pool.get_cached_block(
req0.block_hashes[0], kv_cache_group_ids=[1]
)
is not None
)
req1 = make_request("1", [7] * 9 + [9] * 4, block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 4
def test_hybrid_sliding_window_group_disables_partial_hash_hits():
hash_block_size = 2
sliding_window_block_size = 2 * hash_block_size
@@ -144,6 +144,7 @@ def _split(
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
# `prefix_match_unit` finer than the block size (#46384).
mamba_partial_cache_hit=partial_hit,
mamba_fine_grained_prefix_cache=False,
hash_block_size=ATTN_BLOCK_SIZE,
mamba_has_prefill_checkpoint_blocks=(num_prefill_checkpoint_blocks > 0),
mamba_prefill_checkpoint_alignment=(
@@ -357,7 +358,12 @@ def test_unaligned_resume_never_runs_past_its_block(
"""
prompt_len = 3602
(request,) = create_requests(1, num_tokens=prompt_len, block_size=ATTN_BLOCK_SIZE)
tail_boundary = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
# Under eagle the partial-tail stop sits one hash unit below the prompt's
# last hash boundary: eagle matches a unit past its candidate and drops it,
# so nothing proves that last boundary.
tail_stop = prompt_len // ATTN_BLOCK_SIZE * ATTN_BLOCK_SIZE
if use_eagle:
tail_stop -= ATTN_BLOCK_SIZE
pos, ends = resume_at, []
while pos < prompt_len:
@@ -382,7 +388,7 @@ def test_unaligned_resume_never_runs_past_its_block(
for end in ends[:-1]:
aligned = end % MAMBA_BLOCK_SIZE == 0
assert aligned or (partial_hit and end == tail_boundary), (
assert aligned or (partial_hit and end == tail_stop), (
f"intermediate chunk end {end} is neither block-aligned nor the "
f"partial-tail boundary"
f"partial-tail stop ({tail_stop})"
)
+8 -6
View File
@@ -1169,6 +1169,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
use_eagle_block_drop=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
mamba_fine_grained_prefix_cache=False,
mamba_has_prefill_checkpoint_blocks=False,
)
req_2.shared_prefix_boundary = shared_prefix_boundary
@@ -3607,9 +3608,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
use_eagle=True,
)
# 127 tokens: latest replay boundary is floor((127 - 1) / 32) * 32 = 96.
# The EAGLE/MTP SWA lookup group must cache the local tail ending at
# 104 tokens, and that tail is two 8-token blocks wide: hashes 11 and 12.
# 127 tokens: eagle proves a boundary only if an aligned unit exists past
# it, so the replay boundary rewinds one unit to
# floor(127 / 32) * 32 - 32 = 64. The SWA tail plus its proof block then
# ends at 72, two 8-token blocks wide: hashes 7 and 8.
token_ids = [i for i in range(15) for _ in range(block_size)] + [15] * 7
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
@@ -3623,7 +3625,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
assert blocks is not None
pool = manager.block_pool
expected_swa_cached = {11, 12}
expected_swa_cached = {7, 8}
for i in range(15):
cached = pool.get_cached_block(req0.block_hashes[i], kv_cache_group_ids=[1])
if i in expected_swa_cached:
@@ -3635,8 +3637,8 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 12 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 12]
assert num_computed_tokens == 8 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 8]
def test_block_lookup_cache_single_block_per_key():
@@ -192,7 +192,7 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
)
assert manager.allocate_new_blocks(request_id, num_tokens, num_tokens) == []
manager.cache_blocks(request_id, 1024)
manager.cache_blocks(request_id, 1024, replay_boundary=0)
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0