[Bugfix][Mooncake] Save exact Mamba boundary states (#51358)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Dao Le <daole@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-08-29 02:40:53 +00:00
committed by GitHub
co-authored by Claude Fable 5 OpenAI Codex Dao Le
parent 16d6c376bc
commit 6b110badbb
20 changed files with 1493 additions and 272 deletions
@@ -69,6 +69,19 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
manager.get_computed_blocks.assert_not_called()
def drain_boundary_state_offloads(manager):
"""Drain exact boundary-state block ids offered to a connector."""
return manager.take_boundary_state_offloads()
def _free_block_ids(manager):
"""Block ids the pool would hand out to the next allocation."""
return {
block.block_id
for block in manager.block_pool.free_block_queue.get_all_free_blocks()
}
def make_full_mamba_manager(
*,
dcp_world_size: int,
@@ -618,10 +631,9 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
assert moved[0].block_id == cow_copy.dst_block_id
def test_take_partial_tail_offloads_returns_cow_target():
"""The connector offload hand-off exposes the mamba CoW *target* block Y
(the durable boundary state), not the overwritten source X, and only at
the CoW step."""
def test_boundary_state_offloads_returns_cow_target():
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
target, never the overwritten CoW source."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -660,8 +672,18 @@ def test_take_partial_tail_offloads_returns_cow_target():
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# Step A registered the partial tail but has not CoW'd yet: no offload.
assert manager.take_partial_tail_offloads() == {}
# Step A offers the materialized aligned checkpoint immediately.
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
"0"
]
assert group_id == 1
assert boundary_tokens == 4
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
aligned_block = manager.block_pool.get_cached_block(
aligned_hash, kv_cache_group_ids=[1]
)
assert aligned_block is not None
assert block_id == aligned_block[0].block_id
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
source_block = manager.block_pool.get_cached_block(
@@ -675,34 +697,27 @@ def test_take_partial_tail_offloads_returns_cow_target():
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
offloads = manager.take_partial_tail_offloads()
offloads = drain_boundary_state_offloads(manager)
assert list(offloads.keys()) == ["0"]
assert len(offloads["0"]) == 1
group_id, block_id, boundary_tokens = offloads["0"][0]
assert group_id == 1 # the mamba group
assert boundary_tokens == 6
copies, _ = manager.take_kv_cache_block_copies()
copies, retained = manager.take_kv_cache_block_copies()
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
# The offload points at the durable CoW target Y, not the overwritten X.
assert block_id == cow_copy.dst_block_id
assert block_id != source_block_id
# Draining clears it.
assert manager.take_partial_tail_offloads() == {}
assert manager.take_boundary_state_offloads() == {}
# The hand-off pinned Y (its CoW retention is released after this step,
# and Y is off the request block table); freeing the request unpins it.
cow_block = manager.block_pool.blocks[block_id]
pinned_ref = cow_block.ref_cnt
assert pinned_ref >= 1
manager.block_pool.free_blocks(retained)
manager.free(req0)
assert cow_block.ref_cnt == pinned_ref - 1
def test_partial_tail_pin_survives_released_cow_retention():
"""If the CoW retention is released before the hand-off is drained
(immediate-free mode), the drain must rescue the cow block from the free
queue: a raw ref increment would leave a ref>0 block allocatable, and the
next allocation would pop it and assert."""
def test_block_pool_touch_pins_released_cow_target():
"""The connector can rescue an offered CoW target after its step-scoped
retention is released by using the bound BlockPool's touch method."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -746,13 +761,18 @@ def test_partial_tail_pin_survives_released_cow_retention():
_copies, retained = manager.take_kv_cache_block_copies()
manager.block_pool.free_blocks(retained)
offloads = manager.take_partial_tail_offloads()
offloads = drain_boundary_state_offloads(manager)
((_group_id, block_id, boundary_tokens),) = offloads["0"]
assert boundary_tokens == 6
cow_block = manager.block_pool.blocks[block_id]
assert cow_block.ref_cnt == 1
assert cow_block.ref_cnt == 0
assert block_id in _free_block_ids(manager)
# The pinned block is out of the free queue: draining every free block
manager.block_pool.touch([cow_block])
assert cow_block.ref_cnt == 1
assert block_id not in _free_block_ids(manager)
# The connector-pinned block is out of the free queue: draining every free block
# neither trips the allocator's ref_cnt assert nor hands it out.
new_blocks = manager.block_pool.get_new_blocks(
manager.block_pool.get_num_free_blocks()
@@ -760,7 +780,7 @@ def test_partial_tail_pin_survives_released_cow_retention():
assert block_id not in {b.block_id for b in new_blocks}
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
leak a pinned block."""
@@ -805,12 +825,12 @@ def test_partial_tail_offload_dropped_when_request_freed_before_drain():
# The request dies (preempt/abort) before the scheduler drains.
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
assert manager.take_partial_tail_offloads() == {}
assert manager.take_boundary_state_offloads() == {}
def test_take_partial_tail_offloads_empty_without_partial_tail():
"""A prompt ending on a block boundary registers no partial tail, so there
is nothing to offload."""
def test_boundary_state_offloads_block_aligned_prompt():
"""A prompt ending on a block boundary registers no CoW partial tail; its
boundary state block is handed off as a snapshot instead (once)."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
@@ -848,12 +868,17 @@ def test_take_partial_tail_offloads_empty_without_partial_tail():
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
assert manager.take_partial_tail_offloads() == {}
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
assert group_id == 1
assert boundary == block_size
assert block_id == mamba_blocks[0].block_id
req0.num_computed_tokens = 4
req0.append_output_token_ids([2])
assert manager.allocate_slots(req0, 1) is not None
assert manager.take_partial_tail_offloads() == {}
# The boundary was already handed off; decoding emits nothing new.
assert manager.take_boundary_state_offloads() == {}
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
@@ -1786,3 +1811,213 @@ def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
assert num_computed == 4
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
def _snapshot_offload_kv_cache_config(
hash_block_size: int, block_size: int, num_blocks: int = 24
):
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
def test_retention_snapshots_handed_off_with_exact_block_ids():
"""Under sparse retention, each retained mamba boundary state block is
handed to the connector with its exact block id. Connectors must not resolve
align-mode state blocks positionally because the block table is not
append-only."""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
manager.coordinator.retention_interval = 2 * block_size
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
offloads = drain_boundary_state_offloads(manager)
((group_id, block_id, boundary),) = offloads["0"]
assert group_id == 1
assert boundary == 2 * block_size
assert block_id == mamba_blocks[1].block_id
req0.num_computed_tokens = 8
manager.new_step_starts()
assert manager.allocate_slots(req0, 8) is not None
offloads = drain_boundary_state_offloads(manager)
((group_id, block_id2, boundary2),) = offloads["0"]
assert group_id == 1
assert boundary2 == 4 * block_size
assert block_id2 == mamba_blocks[3].block_id
# Interior align-mode positions stay null and are never handed off.
assert mamba_blocks[0].is_null and mamba_blocks[2].is_null
manager.free(req0)
def test_snapshot_handoff_dense_default_retention():
"""With dense (default) retention, every materialized mamba boundary
state block is handed off — regular mamba-align + prefix-match-unit
deployments get store-able boundary snapshots without setting
VLLM_PREFIX_CACHE_RETENTION_INTERVAL."""
hash_block_size = 2
block_size = 4
manager = make_kv_cache_manager(
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
assert manager.coordinator.retention_interval is None
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
assert group_id == 1
assert boundary == 2 * block_size
assert block_id == mamba_blocks[1].block_id
req0.num_computed_tokens = 8
manager.new_step_starts()
assert manager.allocate_slots(req0, 8) is not None
((group_id, block_id2, boundary2),) = drain_boundary_state_offloads(manager)["0"]
assert group_id == 1
assert boundary2 == 4 * block_size
assert block_id2 == mamba_blocks[3].block_id
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
boundary-state hand-offs offered (and claimed) at each step."""
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
for step in range(num_chunks):
manager.new_step_starts()
req.num_computed_tokens = step * chunk_size
if step == 0:
allocated = manager.allocate_slots(
req, chunk_size, num_computed, computed_blocks
)
else:
allocated = manager.allocate_slots(req, chunk_size)
assert allocated is not None
yield drain_boundary_state_offloads(manager).get(req.request_id, [])
def test_boundary_state_offer_includes_more_mamba_groups_than_two():
"""One offer batch carries one exact block entry per mamba group."""
block_size = 8
num_mamba_groups = 3
manager = make_kv_cache_manager(
kv_cache_config=KVCacheConfig(
num_blocks=200,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
*(
KVCacheGroupSpec(
[f"mamba{i}"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
)
for i in range(num_mamba_groups)
),
],
),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
req0 = make_request("0", [0] * (2 * block_size), block_size, sha256)
entries = next(_run_chunked_prefill(manager, req0, block_size, 1))
assert [group_id for group_id, _, _ in entries] == [1, 2, 3]
assert {boundary for _, _, boundary in entries} == {block_size}
def test_boundary_states_offered_past_prompt_for_resumed_prefill():
"""The core offers every committed boundary, including past
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
and every group re-saves them, so filtering on the original prompt length
would silently strip the mamba key for boundaries full attention still
stores; only the connector knows where its save window ends."""
block_size = 8
prompt_len = block_size
manager = make_kv_cache_manager(
kv_cache_config=_snapshot_offload_kv_cache_config(
hash_block_size=block_size, block_size=block_size, num_blocks=200
),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
)
req0 = make_request("0", [0] * prompt_len, block_size, sha256)
assert [
boundary
for entries in _run_chunked_prefill(manager, req0, block_size, 1)
for _, _, boundary in entries
] == [prompt_len]
# Generate past the next mamba boundary, then replay it as a resumed
# prefill: its committed boundary is offered even though it is past the
# prompt, matching what full attention saves for the same range.
for i in range(block_size):
manager.new_step_starts()
req0.num_computed_tokens = prompt_len + i
req0.append_output_token_ids([1])
assert manager.allocate_slots(req0, 1) is not None
drain_boundary_state_offloads(manager)
assert req0.num_tokens == 2 * block_size
manager.free(req0)
manager.new_step_starts()
req0.num_computed_tokens = 0
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, req0.num_tokens - num_computed, num_computed)
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
assert 2 * block_size in offered
assert req0.num_prompt_tokens < 2 * block_size
@@ -15,12 +15,14 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
FullAttentionManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
)
@@ -69,6 +71,36 @@ def test_external_computed_blocks_do_not_corrupt_free_pool():
assert len(manager.req_to_blocks[request_id]) == 3
def test_mamba_speculative_block_relocation_requires_exclusive_ownership():
spec = MambaSpec(
block_size=4,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=1,
)
block_pool = BlockPool(num_gpu_blocks=4, enable_caching=True, hash_block_size=4)
manager = MambaManager(
spec,
block_pool=block_pool,
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=4,
)
block = block_pool.get_new_blocks(1)[0]
blocks = [block]
manager._relocate_speculative_block(blocks, 0)
assert blocks == [block_pool.null_block, block]
assert block.ref_cnt == 1
pinned_block = block_pool.get_new_blocks(1)[0]
block_pool.touch((pinned_block,))
with pytest.raises(AssertionError, match="exclusively owned and unhashed"):
manager._relocate_speculative_block([pinned_block], 0)
def get_sliding_window_manager(
sliding_window_spec,
block_pool,
@@ -36,7 +36,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
is_store_reachable_swa_chunk,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock, KVCacheBlockCopy
from vllm.v1.core.sched.output import (
KVConnectorBlockState,
SchedulerOutput,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
@@ -107,11 +111,16 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
req_status = scheduler._req_status["req"]
req_status.group_states[0].block_ids[:] = [11, 12]
req_status.group_states[1].block_ids[:] = [0, 21]
scheduler.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
output = SimpleNamespace(partial_tail_offloads={"req": [(1, 99, 28)]})
output = SimpleNamespace(
kv_connector_block_state=KVConnectorBlockState(
block_ids={},
boundary_state_offloads={"req": [(1, 99, 28)]},
)
)
jobs = scheduler._build_partial_tail_store_jobs(output)
assert len(jobs) == 1
@@ -155,6 +164,83 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
assert recurrent_event.parent_block_hash is None
def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
scheduler = _make_partial_tail_scheduler()
_make_partial_tail_request(scheduler)
req_status = scheduler._req_status["req"]
req_status.group_states[0].block_ids[:] = [11, 12]
req_status.group_states[1].block_ids[:] = [0, 21]
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
output = SimpleNamespace(
kv_connector_block_state=KVConnectorBlockState(
block_ids={},
boundary_state_offloads={"req": [(1, 98, 16), (1, 99, 28)]},
)
)
jobs = scheduler._build_partial_tail_store_jobs(output)
assert len(jobs) == 2
src_spec = next(
job.src_spec for job in jobs.values() if len(job.src_spec.block_ids) == 1
)
assert isinstance(src_spec, GPULoadStoreSpec)
assert src_spec.block_ids.tolist() == [98]
assert src_spec.group_sizes == [0, 1]
assert src_spec.block_indices == [0, 0]
def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
scheduler = _make_partial_tail_scheduler()
_make_partial_tail_request(scheduler)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
output = SchedulerOutput.make_empty()
output.kv_connector_block_state = KVConnectorBlockState(
block_ids={},
boundary_state_offloads={"req": [(1, 99, 16)]},
)
meta = scheduler.build_connector_meta(output)
[job_id] = meta.store_jobs
output = SchedulerOutput.make_empty()
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
meta = scheduler.build_connector_meta(output)
assert meta.jobs_to_flush == {job_id}
def test_normal_store_excludes_align_mode_mamba_sources():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(scheduler)
request.num_computed_tokens = 0
request.status = RequestStatus.RUNNING
req_status = scheduler._req_status["req"]
req_status.group_states[0].block_ids[:] = [11]
req_status.group_states[1].block_ids[:] = [99]
req_status.update_offload_keys()
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
output = SimpleNamespace(
num_scheduled_tokens={"req": 16},
finished_req_ids=set(),
)
jobs = scheduler._build_store_jobs(output)
assert len(jobs) == 1
[job] = jobs.values()
src_spec = job.src_spec
assert isinstance(src_spec, GPULoadStoreSpec)
assert src_spec.block_ids.tolist() == [11]
assert src_spec.group_sizes == [1, 0]
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(scheduler)
@@ -5,6 +5,9 @@ import threading
import time
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -31,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MambaSpec,
)
from vllm.v1.outputs import KVConnectorOutput
@@ -62,6 +66,30 @@ def _make_kv_cache_config() -> KVCacheConfig:
)
def test_scheduler_requires_align_mode_for_mamba():
vllm_config = _make_vllm_config()
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
)
with (
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
),
pytest.raises(AssertionError, match="requires mamba_cache_mode='align'"),
):
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
def _make_block_stored() -> BlockStored:
return BlockStored(
block_hashes=[b"hash"],
@@ -97,6 +125,10 @@ def test_scheduler_role_initializes_store_scheduler_only():
mock_worker.assert_not_called()
assert connector.connector_scheduler is mock_scheduler.return_value
assert connector.connector_worker is None
block_pool = MagicMock()
connector.bind_gpu_block_pool(block_pool)
mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool)
def test_worker_methods_delegate_to_store_worker():
@@ -241,9 +273,13 @@ def test_update_connector_output_and_take_events():
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
kv_events.add_events([event])
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
output = KVConnectorOutput(kv_cache_events=kv_events)
connector.update_connector_output(output)
assert connector._kv_cache_events is kv_events
connector.connector_scheduler.update_connector_output.assert_called_once_with(
output
)
assert list(connector.take_events()) == [event]
assert connector._kv_cache_events is None
@@ -451,6 +451,27 @@ def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
assert longer[: len(shorter)] == shorter
def test_store_mask_excludes_mamba_groups_lookup_unaffected():
"""Align-mode mamba block tables are not append-only (interior states are
nulled/freed; speculative blocks relocate), so the positional normal save
must never cover mamba chunks — regardless of retention. Lookups still
probe mamba boundaries: their keys come from the pinned snapshot/CoW
hand-off path instead."""
groups = [
KVCacheGroupSpec(["L0"], _full(16)),
KVCacheGroupSpec(["L1"], _mamba_align(16)),
]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(64)
assert masks[0] is None # full attn stays dense
assert masks[1] == [False] * 4
coord = _make_coord(groups, hash_block_size=16, retention_interval=32)
assert coord.store_mask(64)[1] == [False] * 4
assert coord.lookup_mask(64)[1] is None
# ----- Eagle / MTP interaction with load_mask -----
@@ -236,6 +236,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
# 1-based: block 0 is the reserved null block and the save skips it.
save_req = ReqMeta(
req_id="r0",
token_len_chunk=64,
@@ -441,10 +442,10 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
boundary_state_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
send._maybe_offload_boundary_states(req)
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
partial_hash = hs[2]
@@ -512,8 +513,8 @@ def test_offload_syncs_event_before_put():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, 7, 12)],
store_job_id=1,
boundary_state_offloads=[(1, 7, 12)],
)
req.current_event = event
@@ -589,10 +590,10 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, mamba_cow_block, 12)],
boundary_state_offloads=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_partial_tail(req)
send._maybe_offload_boundary_states(req)
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
# block-end hashes; mamba (block 16): the partial boundary block under
@@ -16,6 +16,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
MooncakeStoreScheduler,
)
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.sched.output import KVConnectorBlockState
def _make_bare_scheduler(
@@ -47,9 +48,20 @@ def _make_bare_scheduler(
scheduler._num_workers = 1
scheduler._next_store_job_id = 0
scheduler._pinned_saves = {}
scheduler._boundary_state_group_ids = frozenset({1})
return scheduler
def _make_connector_block_state(
block_ids: tuple[list[int], ...] | None = None,
offloads: list[tuple[int, int, int]] | None = None,
) -> KVConnectorBlockState:
return KVConnectorBlockState(
block_ids={} if block_ids is None else {"req-0": block_ids},
boundary_state_offloads=({} if offloads is None else {"req-0": offloads}),
)
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
return SimpleNamespace(
finished_req_ids=set(),
@@ -65,6 +77,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
scheduled_spec_decode_tokens=(
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
),
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
)
@@ -84,6 +97,7 @@ def _make_decode_scheduler_output(
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
)
@@ -109,6 +123,7 @@ def _make_new_scheduler_output() -> SimpleNamespace:
),
num_scheduled_tokens={"req-0": 32},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1],)),
)
@@ -491,6 +506,7 @@ def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamesp
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
)
@@ -915,7 +931,10 @@ def _add_pending_partial_tail_request(
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
kv_connector_block_state=_make_connector_block_state(
block_ids=block_ids,
offloads=[(1, 7, 12)],
),
)
@@ -940,15 +959,262 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 0
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
assert req_meta.boundary_state_offloads == [(1, 7, 12)]
assert req_meta.num_prompt_tokens == 12
assert req_meta.block_ids == ([0],)
store_job_id = req_meta.store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [7]
assert scheduler._gpu_block_pool.blocks[0].ref_cnt == 0
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
def test_resumed_partial_tail_uses_handoff_boundary():
def test_decode_boundary_state_offload_dropped_unclaimed():
# A hand-off past the prefill end can never complete a joint hybrid hit
# (every other group stops saving there), so it is neither transferred nor
# claimed — leaving the core free to release the block immediately.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(24)),
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([0],),
num_saved_tokens=12,
token_ids=list(range(12)),
prefill_end_tokens=12,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
# Boundary 16 is a decode boundary for a 12-token prefill.
kv_connector_block_state=_make_connector_block_state(offloads=[(1, 7, 16)]),
)
meta = scheduler.build_connector_meta(out)
assert meta.requests == []
assert scheduler._pinned_saves == {}
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
assert scheduler._request_trackers["req-0"].has_pending_offload is False
def _register_offload_request(scheduler, *, prefill_end_tokens, num_prompt_tokens):
request = SimpleNamespace(
all_token_ids=list(range(64)),
block_hashes=[bytes([i]) for i in range(16)],
num_output_placeholders=0,
num_prompt_tokens=num_prompt_tokens,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=prefill_end_tokens,
allocated_block_ids=([0],),
num_saved_tokens=prefill_end_tokens,
token_ids=list(range(prefill_end_tokens)),
prefill_end_tokens=prefill_end_tokens,
)
def _make_offload_only_output(entries, block_ids=([0],)):
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(
block_ids=block_ids,
offloads=entries,
),
)
def test_resumed_prefill_claims_boundaries_past_prompt_length():
# A resumed request re-prefills its previously generated tokens, so its
# save window (`prefill_end_tokens`) extends past `num_prompt_tokens`.
# Boundaries in that range must still be claimed, or the mamba key would be
# missing for boundaries full attention does store and no joint hit could
# complete there.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
_register_offload_request(scheduler, prefill_end_tokens=20, num_prompt_tokens=12)
out = _make_offload_only_output([(1, 7, 16), (1, 9, 20), (1, 11, 24)])
meta = scheduler.build_connector_meta(out)
# 16 and 20 are inside the resumed prefill; 24 is past it.
assert meta.requests[0].boundary_state_offloads == [(1, 7, 16), (1, 9, 20)]
store_job_id = meta.requests[0].store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [7, 9]
def test_boundary_state_job_pins_exact_blocks_once():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
scheduler._request_trackers["req-0"].allocated_block_ids = ([7],)
out = _make_offload_only_output(
[(1, 7, 16), (1, 8, 32), (1, 9, 48)], block_ids=([7],)
)
meta = scheduler.build_connector_meta(out)
req_meta = meta.requests[0]
assert req_meta.boundary_state_offloads == [
(1, 7, 16),
(1, 8, 32),
(1, 9, 48),
]
store_job_id = req_meta.store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [7, 8, 9]
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
1,
1,
1,
]
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
0,
0,
0,
]
def test_store_job_pins_current_non_null_non_mamba_blocks():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[bytes([i]) for i in range(12)],
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, ([7, 2, 0], [21, 0, 22]))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=44,
allocated_block_ids=([7, 2, 0], [21, 0, 22]),
num_saved_tokens=32,
token_ids=list(range(44)),
prefill_end_tokens=48,
)
stale_block_ids = ([7, 2, 0, 5], [21, 0, 22, 23])
current_block_ids = ([7, 2, 0, 6], [24, 0, 25, 26])
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[([5], [23])],
num_computed_tokens=[44],
resumed_req_ids=set(),
),
num_scheduled_tokens={"req-0": 4},
scheduled_spec_decode_tokens={},
kv_connector_block_state=_make_connector_block_state(
block_ids=current_block_ids,
offloads=[(1, 8, 16)],
),
)
pool = scheduler._gpu_block_pool
meta = scheduler.build_connector_meta(out)
assert scheduler._request_trackers["req-0"].allocated_block_ids == stale_block_ids
req_meta = meta.requests[0]
assert req_meta.block_ids == current_block_ids
store_job_id = req_meta.store_job_id
assert scheduler._pinned_saves[store_job_id][0] == [8, 7, 2, 6]
assert pool.blocks[0].ref_cnt == 0
assert pool.blocks[5].ref_cnt == 0
assert [pool.blocks[i].ref_cnt for i in (21, 22, 23, 24, 25, 26)] == [
0,
0,
0,
0,
0,
0,
]
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [1, 1, 1, 1]
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [0, 0, 0, 0]
def test_boundary_state_release_is_per_store_job():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
first_job_id = first.requests[0].store_job_id
second_job_id = second.requests[0].store_job_id
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
assert first_job_id not in scheduler._pinned_saves
assert second_job_id in scheduler._pinned_saves
def test_preemption_and_request_id_reuse_do_not_release_inflight_job():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
first_job_id = first.requests[0].store_job_id
scheduler.build_connector_meta(_make_preemption_scheduler_output())
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
assert first_job_id in scheduler._pinned_saves
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
second_job_id = second.requests[0].store_job_id
assert second.requests[0].boundary_state_offloads == [(1, 8, 32)]
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
assert second_job_id in scheduler._pinned_saves
def test_boundary_state_never_claimed_without_a_send_thread():
# A kv_consumer has no store job that can acknowledge the block reference.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler.kv_role = "kv_consumer"
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
out = _make_offload_only_output([(1, 7, 16)])
assert scheduler.build_connector_meta(out).requests == []
assert scheduler._pinned_saves == {}
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
def test_resumed_partial_tail_uses_exact_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
# Resumption replays prompt + previously generated tokens.
out = _add_pending_partial_tail_request(
@@ -961,7 +1227,7 @@ def test_resumed_partial_tail_uses_handoff_boundary():
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
assert meta.requests[0].boundary_state_offloads == [(1, 7, 12)]
# Ordinary metadata retains the full resumed prefill range.
assert meta.requests[0].num_prompt_tokens == 20
tracker = scheduler._request_trackers["req-0"]
@@ -969,8 +1235,9 @@ def test_resumed_partial_tail_uses_handoff_boundary():
assert tracker.has_pending_offload is True
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
def test_resumed_partial_tail_attached_to_save_keeps_exact_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler._boundary_state_group_ids = frozenset({0})
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
@@ -987,13 +1254,13 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
prefill_end_tokens=48,
)
out = _make_scheduler_output(scheduled_spec_tokens=None)
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
out.kv_connector_block_state.boundary_state_offloads = {"req-0": [(0, 7, 36)]}
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
assert meta.requests[0].can_save is True
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
assert meta.requests[0].boundary_state_offloads == [(0, 7, 36)]
assert meta.requests[0].num_prompt_tokens == 48
# Ordinary saving still covers the full resumed prefill range.
tracker = scheduler._request_trackers["req-0"]
@@ -1019,7 +1286,8 @@ def test_partial_tail_cow_block_is_referenced_for_the_job():
store_job_id = meta.requests[0].store_job_id
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
# puts it last in eviction priority.
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
assert scheduler._pinned_saves[store_job_id][0] == [7]
assert pool.blocks[0].ref_cnt == 0
assert pool.blocks[7].ref_cnt == 1
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
@@ -181,6 +181,7 @@ def _run_store_req(thread, req_meta: ReqMeta) -> None:
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
# 1-based: block 0 is the reserved null block and the save skips it.
return ReqMeta(
req_id=req_id,
token_len_chunk=32,
@@ -651,6 +652,7 @@ def _make_partial_tail_send_thread(
enable_partial_hash_hits=True,
hash_block_size=4,
lcm_block_size=16,
mamba_group_ids={1},
)
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0),
@@ -659,10 +661,18 @@ def _make_partial_tail_send_thread(
)
db.set_kv_caches_base_addr([0x1000])
db.set_block_len([256])
# Group 1 models the mamba "align" group the hand-offs reference.
db_mamba = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
hash_block_size=4,
)
db_mamba.set_kv_caches_base_addr([0x2000])
db_mamba.set_block_len([256])
return _make_store_sending_thread(
store,
coord=coord,
token_databases=[db],
token_databases=[db, db_mamba],
replicate_config=replicate_config,
enable_group_semantics=enable_group_semantics,
supports_group_ids=supports_group_ids,
@@ -673,20 +683,20 @@ def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
return ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=(block_ids,),
block_ids=(block_ids, [1]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
partial_tail_offloads=[(1, 7, 12)],
boundary_state_offloads=[(1, 7, 12)],
)
def test_partial_tail_offload_skips_null_source_blocks():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, 256]
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_partial_tail_send_thread(store)
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([0, 2, 3]))
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
@@ -694,15 +704,21 @@ def test_partial_tail_offload_skips_null_source_blocks():
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
]
# FA reads blocks 2 and 3 (block 0 is null and skipped); the mamba
# boundary block reads the core-provided CoW block 7, not block_ids.
assert addrs == [
[0x1000 + 2 * 256],
[0x1000 + 3 * 256],
[0x2000 + 7 * 256],
]
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
def test_store_sending_thread_skips_null_sparse_group_blocks():
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
)
store = MagicMock()
@@ -711,16 +727,11 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
)
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
sparse = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[
KVCacheGroupSpec(["full"], full),
KVCacheGroupSpec(["mamba"], mamba),
KVCacheGroupSpec(["sparse"], sparse),
],
scheduler_block_size=16,
hash_block_size=16,
@@ -732,17 +743,17 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_mamba = ChunkedTokenDatabase(
db_sparse = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=16,
)
db_mamba.set_kv_caches_base_addr([0x2000])
db_mamba.set_block_len([512])
db_sparse.set_kv_caches_base_addr([0x2000])
db_sparse.set_block_len([512])
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_mamba],
token_databases=[db_full, db_sparse],
)
_run_store_req(
thread,
@@ -756,15 +767,86 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
)
keys, addrs, _, _ = store.batch_put_from_multi_buffers.call_args.args
mamba_keys = [key for key in keys if "@group:1" in key]
assert [key.rsplit("@", 1)[-1] for key in mamba_keys] == [b"a1".hex()]
sparse_keys = [key for key in keys if "@group:1" in key]
assert [key.rsplit("@", 1)[-1] for key in sparse_keys] == [b"a1".hex()]
assert all(addr[0] >= 0x2000 for key, addr in zip(keys, addrs) if "@group:1" in key)
def test_partial_tail_offload_skips_cap_omitted_mamba_group():
"""A Mamba group omitted by the handoff cap must not fall back to the
connector's positional block table."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
full_spec = FullAttentionSpec(
block_size=4, num_kv_heads=8, head_size=64, dtype=torch.float32
)
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[
KVCacheGroupSpec(["full"], full_spec),
KVCacheGroupSpec(["mamba-1"], mamba_spec),
KVCacheGroupSpec(["mamba-2"], mamba_spec),
],
scheduler_block_size=16,
hash_block_size=4,
)
def make_db(group_id: int, block_size: int, base_addr: int):
db = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
block_size=block_size,
hash_block_size=4,
)
db.set_kv_caches_base_addr([base_addr])
db.set_block_len([256])
return db
db_full = make_db(0, 4, 0x1000)
db_mamba_1 = make_db(1, 16, 0x2000)
db_mamba_2 = make_db(2, 16, 0x3000)
thread = _make_store_sending_thread(
store,
coord=coord,
token_databases=[db_full, db_mamba_1, db_mamba_2],
)
req = ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [5], [9]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
# The scheduler accepted group 1 and omitted group 2 at boundary 12.
boundary_state_offloads=[(1, 7, 12)],
)
assert thread._maybe_offload_boundary_states(req)
keys, addrs, _sizes, _replicate_config = (
store.batch_put_from_multi_buffers.call_args.args
)
puts = list(zip(keys, addrs, strict=True))
boundary_hash = BlockHash(b"a2")
assert (db_mamba_1.key_for(boundary_hash), [0x2000 + 7 * 256]) in puts
assert (db_mamba_2.key_for(boundary_hash), [0x3000 + 9 * 256]) not in puts
def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
store = MagicMock()
store.batch_is_exist.return_value = [1, 0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
store.batch_is_exist.return_value = [1, 0, 0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256, 256]
replicate_config = SimpleNamespace(group_ids=["stale"])
thread = _make_partial_tail_send_thread(
store,
@@ -773,17 +855,19 @@ def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
supports_group_ids=True,
)
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([1, 2, 3]))
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([1, 2, 3]))
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
]
assert config is replicate_config
assert config.group_ids == [
"vllm-mooncake-store:test-model@6131",
"vllm-mooncake-store:test-model@6132",
"vllm-mooncake-store:test-model@6132",
]
@@ -815,6 +899,165 @@ def test_partial_tail_put_failure_activates_pressure_gate():
assert store.batch_put_from_multi_buffers.call_count == 1
def test_normal_save_excludes_mamba_group_and_null_blocks():
"""The positional normal save must never cover mamba chunks (align-mode
block tables are not append-only) and must skip chunks whose block is the
reserved null block, for any group."""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
MambaSpec,
)
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=None
)
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
coord = mooncake_store_worker.MooncakeStoreCoordinator(
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], mamba_spec)],
scheduler_block_size=16,
hash_block_size=16,
)
db_full = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16
)
db_full.set_kv_caches_base_addr([0x1000])
db_full.set_block_len([256])
db_mamba = ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), block_size=16
)
db_mamba.set_kv_caches_base_addr([0x2000])
db_mamba.set_block_len([256])
thread = _make_store_sending_thread(
store, coord=coord, token_databases=[db_full, db_mamba]
)
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=64,
# FA has a null at chunk 1; the mamba table is align-mode shaped
# (interior nulls + one state block) and must not be read at all.
block_ids=([1, 0, 3, 4], [0, 0, 0, 9]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
),
)
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
assert all("@group:0@" in k for k in keys)
assert [k.rsplit("@", 1)[-1] for k in keys] == ["6130", "6132", "6133"]
assert addrs == [[0x1000 + 256], [0x1000 + 3 * 256], [0x1000 + 4 * 256]]
def test_block_aligned_snapshot_offload_uses_provided_block():
"""A block-aligned hand-off (sparse-retention mamba checkpoint) uploads
exactly the core-pinned block under the boundary-end hash key; the
positional block table entry for that chunk must be ignored."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_partial_tail_send_thread(store)
hs = [bytes([i + 1]) * 4 for i in range(8)] # 8 hash units = 32 tokens
req = ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [5]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, 7, 32)],
)
assert thread._maybe_offload_boundary_states(req)
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
# boundary 32 is block-aligned for the mamba group (block 16): one key,
# keyed by hs[32 // 4 - 1], read from the handed-off block 7 — not from
# block_ids[1] and with no FA gap coverage.
assert keys == [thread.token_databases[1].key_for(BlockHash(hs[7]))]
assert addrs == [[0x2000 + 7 * 256]]
def test_mixed_snapshot_and_sub_block_offloads():
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
in one hand-off; the sub-block path covers FA gap blocks but reads the
mamba boundary only from the CoW block, never positionally."""
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
thread = _make_partial_tail_send_thread(store)
hs = [bytes([i + 1]) * 4 for i in range(11)] # 11 hash units = 44 tokens
req = ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [0, 0, 0]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, 9, 32), (1, 7, 44)],
)
assert thread._maybe_offload_boundary_states(req)
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
db_full, db_mamba = thread.token_databases
assert keys == [
# Aligned snapshot: boundary 32 from the handed-off block 9.
db_mamba.key_for(BlockHash(hs[7])),
# Sub-block boundary 44: FA gap blocks ending at 4, 8, 12.
db_full.key_for(BlockHash(hs[0])),
db_full.key_for(BlockHash(hs[1])),
db_full.key_for(BlockHash(hs[2])),
# Mamba boundary block from the CoW hand-off (block 7).
db_mamba.key_for(BlockHash(hs[10])),
]
assert addrs == [
[0x2000 + 9 * 256],
[0x1000 + 1 * 256],
[0x1000 + 2 * 256],
[0x1000 + 3 * 256],
[0x2000 + 7 * 256],
]
def test_snapshot_offload_skips_null_handoff_block():
"""A hand-off the core could not materialize (null block) carries no
committed state; persisting it would poison the boundary key."""
store = MagicMock()
thread = _make_partial_tail_send_thread(store)
hs = [bytes([i + 1]) * 4 for i in range(8)]
assert thread._maybe_offload_boundary_states(
ReqMeta(
req_id="req-a",
token_len_chunk=0,
block_ids=([1, 2, 3], [5]),
block_hashes=hs,
can_save=True,
boundary_state_offloads=[(1, NULL_BLOCK_ID, 32)],
)
)
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
def test_worker_meta_is_absent_before_any_completed_job():
send_thread = _make_partial_tail_send_thread(MagicMock())
store_worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
store_worker.kv_send_thread = send_thread
assert store_worker.build_connector_worker_meta() is None
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -212,12 +212,21 @@ def _accuracy_test(llm: LLM, subscriber: MockSubscriber | None):
if subscriber is not None:
subscriber.get_new_cpu_stored_events()
# Pad prompt so its token count is a multiple of cpu_block_size.
# Use the tokenizer directly to avoid expensive llm.generate() calls.
# Align the reusable prefix; the final token must be recomputed for logits.
tokenizer = llm.get_tokenizer()
prompt = "Let's count to 10. One, two, three, four,"
while len(tokenizer.encode(prompt)) % cpu_block_size != 0:
prompt = ". " + prompt
prompt_token_ids = tokenizer.encode("Let's count to 10. One, two, three, four,")
reusable_prefix = prompt_token_ids[:-1]
final_token = prompt_token_ids[-1:]
[padding_token_id] = tokenizer.encode(".", add_special_tokens=False)
padding = [padding_token_id] * (-len(reusable_prefix) % cpu_block_size)
insert_at = int(reusable_prefix[0] == tokenizer.bos_token_id)
reusable_prefix[insert_at:insert_at] = padding
assert len(reusable_prefix) % cpu_block_size == 0
prompt = TokensPrompt(
prompt_token_ids=reusable_prefix + final_token,
)
# Seed the CPU cache with the prompt.
llm.generate(prompt, sampling_params, use_tqdm=False)
@@ -22,8 +22,13 @@ from vllm.v1.request import Request
class DummyConnectorMetadata(KVConnectorMetadata):
def __init__(self, block_hashes_by_req: dict[str, list[BlockHash]]):
def __init__(
self,
block_hashes_by_req: dict[str, list[BlockHash]],
block_ids_by_req: dict[str, tuple[list[int], ...]],
):
self.block_hashes_by_req = block_hashes_by_req
self.block_ids_by_req = block_ids_by_req
class DummyKVConnector(KVConnectorBase_V1):
@@ -47,8 +52,12 @@ class DummyKVConnector(KVConnectorBase_V1):
assert block_hashes_by_req is not None, (
"DummyKVConnector expected 'block_hashes_by_req' on scheduler_output"
)
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None
block_ids_by_req = block_state.block_ids
return DummyConnectorMetadata(
block_hashes_by_req=block_hashes_by_req,
block_ids_by_req=block_ids_by_req,
)
def start_load_kv(self, kv_caches, finished_req_ids):
@@ -129,3 +138,9 @@ def test_connector_receives_block_hashes(_load_plugin):
num_tokens // block_size
)
assert meta.block_hashes_by_req[req.request_id] == req.block_hashes
expected_block_ids = {
req.req_id: req.block_ids for req in output.scheduled_new_reqs
}
assert meta.block_ids_by_req == expected_block_ids
assert output.kv_connector_block_state is None
@@ -87,6 +87,11 @@ class MooncakeStoreCoordinator:
for g in kv_cache_groups
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.mamba_group_ids = {
group_id
for group_id, group in enumerate(kv_cache_groups)
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
}
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
self.enable_partial_hash_hits = partial_hash_hits_enabled(
@@ -205,12 +210,23 @@ class MooncakeStoreCoordinator:
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
Mamba groups are always all-False: the normal save resolves blocks
positionally from the connector's append-only block-ID snapshot, but
an align-mode mamba block table is not append-only (interior state
blocks are nulled/freed, and speculative decoding relocates the spec
blocks in place), so a positional read may hit a null, freed, or live
speculative-state block and persist wrong bytes under a valid prefix
hash. Mamba state is persisted only through the connector-pinned exact
block hand-off path
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
"""
return self._reachable_masks(
aligned_token_len,
start_token,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
exclude_mamba=True,
)
def lookup_mask(
@@ -237,6 +253,7 @@ class MooncakeStoreCoordinator:
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
exclude_mamba: bool = False,
) -> tuple[list[bool] | None, ...]:
mask_alignment = (
self.hash_block_size
@@ -252,6 +269,9 @@ class MooncakeStoreCoordinator:
spec = _unwrap_spec(g.kv_cache_spec)
end_chunk = aligned_token_len // spec.block_size
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
if exclude_mamba and isinstance(spec, MambaSpec):
masks.append([False] * (end_chunk - start_chunk))
continue
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
@@ -406,11 +406,11 @@ class ReqMeta:
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Core-provided per-mamba-group
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
# Present only on the producer's CoW step; drives the connector's offload
# (the FA group's block is derived from block_ids and boundary_tokens).
partial_tail_offloads: list[tuple[int, int, int]] | None = None
# Core-provided (group_id, block_id, boundary_tokens) mamba "align"
# boundary states. A block-aligned entry is a committed boundary snapshot;
# a non-aligned entry is the sub-block CoW tail. The store-job reference
# keeps each exact block alive until every worker rank finishes the job.
boundary_state_offloads: list[tuple[int, int, int]] | None = None
@staticmethod
def from_request_tracker(
@@ -23,11 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
@@ -80,12 +81,15 @@ class MooncakeStoreScheduler:
self._hash_block_size,
vllm_config.parallel_config.decode_context_parallel_size,
)
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
mamba_groups = {
group_id: group.kv_cache_spec
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
}
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._boundary_state_group_ids = frozenset(mamba_groups)
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
@@ -93,6 +97,12 @@ class MooncakeStoreScheduler:
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
@@ -381,47 +391,41 @@ class MooncakeStoreScheduler:
if req_meta is not None:
meta.add_request(req_meta)
# Flush partial-tail offloads in the step they arrive: the CoW copy is
# enqueued before the connector event records, so this step's event
# fences the cow block. Ride the request's save meta when present, else
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
# save; can_save=True takes the normal enqueue path).
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
if step_partial_tails and not is_consumer:
pending = dict(step_partial_tails)
for req_meta in meta.requests:
if req_meta.can_save:
groups = pending.pop(req_meta.req_id, None)
if groups:
req_meta.partial_tail_offloads = groups
tracker = self._request_trackers.get(req_meta.req_id)
if tracker is not None:
tracker.has_pending_offload = True
for req_id, groups in pending.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks
# are going away, so the offload is conservatively dropped.
logger.debug("Dropping partial-tail offload for request %s", req_id)
continue
assert len({boundary for _, _, boundary in groups}) == 1
tracker.has_pending_offload = True
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
partial_tail_offloads=groups,
)
)
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
if (
block_state is not None
and block_state.boundary_state_offloads
and not is_consumer
):
self._handle_boundary_state_offloads(
block_state.boundary_state_offloads, meta
)
self._apply_current_save_block_ids(meta, scheduler_output)
self._reference_save_blocks(meta)
return meta
def _apply_current_save_block_ids(
self,
meta: MooncakeStoreConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
"""Replace append-only mirrors with the core's current block tables."""
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
if not save_metas:
return
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None, (
"Current block tables are required for Mooncake store jobs"
)
for req_meta in save_metas:
block_ids = block_state.block_ids.get(req_meta.req_id)
assert block_ids is not None, (
f"Missing current block table for store request {req_meta.req_id}"
)
req_meta.block_ids = block_ids
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
@@ -439,21 +443,86 @@ class MooncakeStoreScheduler:
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids: list[int] = []
if req_meta.partial_tail_offloads:
# A partial-tail CoW block is deliberately kept out of the
# request's block table, so it is absent from `block_ids` even
# though the worker DMAs out of it just as asynchronously.
# It leads the list, as in `pop_blocks_for_free`.
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
if req_meta.boundary_state_offloads:
block_ids.extend(
block_id for _, block_id, _ in req_meta.boundary_state_offloads
)
assert NULL_BLOCK_ID not in block_ids, (
"A null block cannot back a boundary-state offload"
)
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids += [bid for group in req_meta.block_ids for bid in group]
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
)
# An aligned boundary block may also be present in the request's
# block table. Take and release exactly one reference per block.
block_ids = list(dict.fromkeys(block_ids))
assert NULL_BLOCK_ID not in block_ids
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[bid] for bid in block_ids])
pool.touch([pool.blocks[block_id] for block_id in block_ids])
def _handle_boundary_state_offloads(
self,
offloads: dict[str, list[tuple[int, int, int]]],
meta: MooncakeStoreConnectorMetadata,
) -> None:
"""Attach exact boundary-state blocks to store jobs for this step.
Flushed in the step they arrive: the CoW copy is enqueued before the
connector event records, so this step's event fences the exact block.
Entries ride the request's save meta when present, else an
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
``can_save=True`` takes the normal enqueue and store-job pinning path).
"""
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
for req_id, entries in offloads.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks are
# going away, so the offload is conservatively dropped.
logger.debug("Dropping boundary-state offload for request %s", req_id)
continue
accepted: list[tuple[int, int, int]] = []
for group_id, block_id, boundary_tokens in entries:
# Every other group stops saving at the end of this prefill, so
# a mamba-only key past it can never complete a joint hybrid
# hit. `prefill_end_tokens` — not the original prompt length —
# is the boundary: a resumed request re-prefills and re-saves
# its previously generated tokens for every group.
if boundary_tokens > tracker.prefill_end_tokens:
continue
if block_id == NULL_BLOCK_ID:
continue
if group_id not in self._boundary_state_group_ids:
continue
accepted.append((group_id, block_id, boundary_tokens))
if not accepted:
continue
tracker.has_pending_offload = True
if (req_meta := save_metas.get(req_id)) is not None:
req_meta.boundary_state_offloads = accepted
continue
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=tracker.prefill_end_tokens,
boundary_state_offloads=accepted,
)
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
@@ -648,48 +648,76 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.clear()
return True
def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool:
"""Offload the request's sub-block partial tail (its last prompt hash
boundary) so a later request can hit the sub-block prefix.
def _boundary_snapshot_puts(
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
"""Puts for committed mamba "align" boundary-state snapshots.
Covers every block from the normal save's lcm floor to the boundary:
the normal save floors to ``lcm_block_size``, so a smaller-block
group's full blocks in that gap are never persisted elsewhere, and
the consumer's lookup needs every group at every probed boundary.
Full blocks are keyed by their block-end hash, the partial boundary
block by the boundary sub-hash; the mamba "align" boundary block is
the core-provided CoW block. All keys are deduped against the store.
These are block-aligned boundaries, i.e. exactly what the normal save
would key — but ``store_mask`` masks mamba groups out of it entirely, so
this is their *only* writer. The exclusion is not an optimization: the
normal save resolves a chunk's address as
``req_meta.block_ids[g][start // block_size]``, and ``block_ids`` is the
connector's append-only mirror of the core's per-group table. An
align-mode table is mutated in place (a superseded state block is freed
and nulled; speculative blocks relocate), and the connector is never
told, so a stale mirror entry is indistinguishable from a live one — a
retry of a failed or pressure-skipped chunk would read a block that now
belongs to another request.
Returns:
True when no put is needed or every put succeeds, False otherwise.
Each entry's handed-off block *is* the boundary state and is pinned by
the core, so it is uploaded under its boundary-end hash key and never
resolved positionally.
"""
if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes:
return True
partial_tail_offloads = req_meta.partial_tail_offloads
if not partial_tail_offloads:
return True
hash_block_size = self.coord.hash_block_size
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
for group_id, block_id, boundary in entries:
if boundary == 0 or block_id == NULL_BLOCK_ID:
continue
hash_idx = boundary // hash_block_size - 1
if hash_idx >= len(req_meta.block_hashes):
continue
db = self.token_databases[group_id]
# Distribute across ranks by the same rule as normal chunks.
put_step = self.group_put_steps[group_id]
put_step_rank = (self.tp_rank + group_id) % put_step
if (boundary // db.block_size - 1) % put_step != put_step_rank:
continue
addr, size = db.prepare_value_for_block(block_id)
puts.append(
(db.key_for(req_meta.block_hashes[hash_idx]), addr, size, db.metadata)
)
return puts
def _sub_block_tail_puts(
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
"""Puts for the request's sub-block partial tail (its last prompt hash
boundary), so a later request can hit the sub-block prefix.
Covers every group's blocks from the normal save's lcm floor to the
boundary: the normal save floors to ``lcm_block_size``, so a
smaller-block group's full blocks in that gap are never persisted
elsewhere, and the consumer's lookup needs every group at every probed
boundary. Full blocks are keyed by their block-end hash and the partial
boundary block by the boundary sub-hash; a mamba "align" group
contributes only its boundary block, from the core-provided CoW block.
"""
boundaries = {boundary for _, _, boundary in entries}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
"Sub-block partial-tail offloads for one request must share a boundary"
)
boundary = boundaries.pop()
if boundary == 0:
return True
if boundary // hash_block_size - 1 >= len(req_meta.block_hashes):
return True
mamba_offloads = {
group_id: block_id for group_id, block_id, _ in partial_tail_offloads
}
hash_block_size = self.coord.hash_block_size
if boundary == 0 or boundary // hash_block_size - 1 >= len(
req_meta.block_hashes
):
return []
keys: list[str] = []
addrs: list[list[int]] = []
sizes: list[list[int]] = []
group_ids: list[str] | None = (
[] if self.enable_group_semantics and self.supports_group_ids else None
)
mamba_offloads = {group_id: block_id for group_id, block_id, _ in entries}
saved = self._saved_offset.get(req_meta.req_id, 0)
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
for g_idx, db in enumerate(self.token_databases):
group_blocks = req_meta.block_ids[g_idx]
# Distribute across ranks by the same rule as normal chunks.
@@ -705,16 +733,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
continue
valid_end = min((block_idx + 1) * db.block_size, boundary)
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
if (
g_idx in mamba_offloads
and valid_end == boundary
and boundary % db.block_size != 0
):
block_id = mamba_offloads[g_idx]
else:
if block_idx >= len(group_blocks):
if g_idx in mamba_offloads:
if valid_end != boundary:
# Interior align-mode state positions are null or
# stale (the block table is not append-only) and never
# valid gap content; only the boundary block is
# persisted, from the core-provided hand-off.
continue
block_id = mamba_offloads[g_idx]
elif g_idx in self.coord.mamba_group_ids:
continue
elif block_idx < len(group_blocks):
block_id = group_blocks[block_idx]
else:
continue
if block_id == NULL_BLOCK_ID:
logger.debug(
"Skipping unavailable partial-tail source block "
@@ -725,20 +757,58 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
continue
addr, size = db.prepare_value_for_block(block_id)
key = db.key_for(key_hash)
keys.append(key)
addrs.append(addr)
sizes.append(size)
if group_ids is not None:
group_ids.append(
_make_mooncake_group_id(
db.metadata,
key.rsplit("@", 1)[-1],
)
)
puts.append((db.key_for(key_hash), addr, size, db.metadata))
return puts
if not keys:
def _maybe_offload_boundary_states(self, req_meta: ReqMeta) -> bool:
"""Persist connector-pinned mamba "align" boundary states handed off
for this request, deduped against the store.
This is every mamba key the connector writes — ``store_mask`` excludes
mamba groups from the positional normal save, aligned boundaries
included (see :meth:`_boundary_snapshot_puts`).
The two entry kinds are keyed and sourced differently, so they are
prepared separately and put in one batch:
- block-aligned for its group: a committed boundary-state snapshot,
the handed-off block itself;
- not block-aligned: the sub-block CoW partial tail, which also has to
cover the other groups' blocks in the normal save's lcm gap.
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
offloads = req_meta.boundary_state_offloads
if not offloads or not req_meta.block_hashes:
return True
snapshots: list[tuple[int, int, int]] = []
sub_block: list[tuple[int, int, int]] = []
for group_id, block_id, boundary in offloads:
entry = (group_id, block_id, boundary)
if boundary % self.token_databases[group_id].block_size == 0:
snapshots.append(entry)
else:
sub_block.append(entry)
puts = self._boundary_snapshot_puts(req_meta, snapshots)
if sub_block and self.coord.enable_partial_hash_hits:
puts.extend(self._sub_block_tail_puts(req_meta, sub_block))
if not puts:
return True
keys = [key for key, _, _, _ in puts]
addrs = [addr for _, addr, _, _ in puts]
sizes = [size for _, _, size, _ in puts]
group_ids: list[str] | None = (
[
_make_mooncake_group_id(metadata, key.rsplit("@", 1)[-1])
for key, _, _, metadata in puts
]
if self.enable_group_semantics and self.supports_group_ids
else None
)
exists_start = time.perf_counter()
try:
exists = self.store.batch_is_exist(keys)
@@ -751,7 +821,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
num_failed_keys=len(keys),
)
logger.error(
"Failed to check partial-tail keys for request %s: %s",
"Failed to check boundary-state keys for request %s: %s",
req_meta.req_id,
e,
)
@@ -787,7 +857,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
num_failed_keys=len(keys),
)
logger.error(
"Failed to put partial-tail keys for request %s: %s",
"Failed to put boundary-state keys for request %s: %s",
req_meta.req_id,
e,
)
@@ -805,7 +875,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
if failed:
failed_codes = {res[i] for i in failed}
logger.warning(
"Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)",
"Boundary-state put failed for request %s: %d/%d keys failed "
"(codes=%s)",
req_meta.req_id,
len(failed),
len(keys),
@@ -818,7 +889,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared after a "
"successful partial-tail batch"
"successful boundary-state batch"
)
return True
@@ -857,10 +928,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
return
# Offload the sub-block partial tail (independent of the normal
# block-aligned save, which may be skipped this step).
if req_meta.partial_tail_offloads is not None and not (
self._maybe_offload_partial_tail(req_meta)
# Offload the handed-off mamba boundary states (independent of the
# normal positional save, which may be skipped this step).
if req_meta.boundary_state_offloads is not None and not (
self._maybe_offload_boundary_states(req_meta)
):
return
@@ -887,6 +958,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
# Rotate the stride phase per group to balance load across ranks.
put_step = self.group_put_steps[g_idx]
put_step_rank = (self.tp_rank + g_idx) % put_step
group_blocks = block_ids_per_group[g_idx]
for start, end, block_hash in db.process_tokens(
token_len,
req_meta.block_hashes,
@@ -1130,6 +1130,9 @@ class OffloadingConnectorScheduler:
if bid != 0:
self._current_batch_allocated_block_ids.add(bid)
for copy in scheduler_output.kv_cache_block_copies or ():
self._current_batch_allocated_block_ids.add(copy.dst_block_id)
# Zero out stale block_ids in sliding window groups' pending-store
# positions. Only sliding window groups can have stale entries (blocks
# freed by remove_skipped_blocks then reallocated). Only positions in
@@ -1151,18 +1154,89 @@ class OffloadingConnectorScheduler:
):
group_state.block_ids[j] = 0
def _build_aligned_boundary_store_jobs(
self, handoffs: dict[str, list[tuple[int, int, int]]]
) -> dict[int, TransferJob]:
store_jobs: dict[int, TransferJob] = {}
num_groups = len(self.config.kv_group_configs)
for req_id, entries in handoffs.items():
req_status = self._req_status.get(req_id)
if req_status is None:
continue
req = req_status.req
max_boundary = self._calc_num_offloadable_tokens(req_status, req.num_tokens)
for group_idx, block_id, boundary in entries:
group_config = self.config.kv_group_configs[group_idx]
if (
block_id == 0
or boundary > max_boundary
or boundary % group_config.tokens_per_chunk != 0
):
continue
key = self._make_boundary_key(req, group_idx, boundary)
store_output = self.manager.prepare_store([key], req_status.req_context)
if store_output is None:
self._connector_stats.increase_counter(
_ConnectorMetricName.ALLOCATION_FAILURE
)
continue
if not store_output.keys_to_store:
continue
job_id = self._generate_job_id()
req_status.transfer_jobs.add(job_id)
self._block_id_to_pending_jobs.setdefault(block_id, set()).add(job_id)
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys={key},
is_store=True,
fenced_block_ids=[block_id],
)
group_sizes = [0] * num_groups
group_sizes[group_idx] = 1
block_indices = [0] * num_groups
block_indices[group_idx] = boundary // group_config.tokens_per_block - 1
store_jobs[job_id] = TransferJob(
req_id=req_id,
src_spec=GPULoadStoreSpec(
[block_id],
group_sizes=group_sizes,
block_indices=block_indices,
),
dst_spec=store_output.store_spec,
)
self._events_tracker.record_store(
req,
group_config,
boundary // group_config.tokens_per_chunk - 1,
key,
)
return store_jobs
def _build_partial_tail_store_jobs(
self, scheduler_output: SchedulerOutput
) -> dict[int, TransferJob]:
handoffs = scheduler_output.partial_tail_offloads
if not self.config.supports_partial_tail or not handoffs:
block_state = scheduler_output.kv_connector_block_state
handoffs = block_state.boundary_state_offloads if block_state else None
if not handoffs:
return {}
store_jobs: dict[int, TransferJob] = {}
store_jobs = self._build_aligned_boundary_store_jobs(handoffs)
if not self.config.supports_partial_tail:
return store_jobs
for req_id, entries in handoffs.items():
entries = [
entry
for entry in entries
if entry[2] % self._partial_tail_block_size != 0
]
if not entries:
continue
req_status = self._req_status.get(req_id)
assert req_status is not None
assert entries
boundaries = {boundary for _, _, boundary in entries}
assert len(boundaries) == 1
boundary = boundaries.pop()
@@ -1282,6 +1356,9 @@ class OffloadingConnectorScheduler:
group_config, group_state, num_offloadable_tokens
)
if group_config.requires_cow_source:
continue
start_chunk_idx = group_state.next_stored_chunk_idx
if num_chunks <= start_chunk_idx:
continue
+4
View File
@@ -716,6 +716,10 @@ class BlockPool:
if self.metrics_collector:
self.metrics_collector.on_block_accessed(block)
def is_block_writable(self, block: KVCacheBlock) -> bool:
"""Return whether a block can be mutated by its sole owner."""
return not block.is_null and block.ref_cnt == 1 and block.block_hash is None
def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
"""Free a list of blocks. The blocks should be ordered by their
eviction priority, where the first block will be evicted first.
+12 -28
View File
@@ -189,10 +189,6 @@ class KVCacheManager:
tuple(() for _ in range(self.num_kv_cache_groups))
)
# Off-table cow blocks handed to a KV connector for partial-tail
# offload; pinned until the request's blocks are freed.
self._partial_tail_pins: dict[str, list[KVCacheBlock]] = {}
@property
def usage(self) -> float:
"""Get the KV cache usage.
@@ -575,9 +571,6 @@ class KVCacheManager:
Args:
request: The request to free the blocks.
"""
pins = self._partial_tail_pins.pop(request.request_id, None)
if pins:
self.block_pool.free_blocks(pins)
self.coordinator.free(request.request_id)
def remove_skipped_blocks(
@@ -610,14 +603,7 @@ class KVCacheManager:
Returns:
The request's blocks in allocation order.
"""
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
# Pins ride the same (possibly deferred) free as the request blocks.
# Preemption may release a pin under a still-queued offload — the same
# exposure normal saves of table blocks already have.
pins = self._partial_tail_pins.pop(request.request_id, None)
if pins:
blocks = pins + blocks
return blocks
return self.coordinator.pop_blocks_for_free(request.request_id)
def evict_blocks(self, block_ids: set[int]) -> None:
"""evict blocks from the prefix cache by their block IDs.
@@ -854,18 +840,18 @@ class KVCacheManager:
retained_blocks = [block for pair in pending_copies for block in pair]
return copies, retained_blocks
def take_partial_tail_offloads(self) -> dict[str, list[tuple[int, int, int]]]:
"""Drain producer partial-tail offload hand-offs per request.
def take_boundary_state_offloads(
self,
) -> dict[str, list[tuple[int, int, int]]]:
"""Drain this step's boundary-state hand-offs for a KV connector.
Returns ``{request_id: [(group_id, block_id, boundary_tokens), ...]}``
for the durable boundary blocks of producers' last-prompt-boundary
partial tails. Only mamba "align" groups contribute; empty otherwise.
A KV connector reads the referenced blocks and offloads them so a later
request can hit the sub-block prefix.
Each handed-off block lives off the request block table, so it is
pinned here and unpinned when the request's blocks are freed — for a
producer with saved tokens, after the connector reports sends done.
for mamba "align" boundary states: the
request's committed boundary-state snapshots and, on a sub-block partial
hit, the CoW copy of its last-prompt-boundary state. Only mamba "align"
groups contribute; empty otherwise. A connector reads the referenced
blocks — never resolving them positionally — and offloads them so a
later request can hit that prefix.
"""
offloads: dict[str, list[tuple[int, int, int]]] = {}
for mgr in self.coordinator.single_type_managers:
@@ -874,9 +860,7 @@ class KVCacheManager:
group_id,
block,
boundary_tokens,
) in mgr.take_pending_partial_tail_offloads():
self.block_pool.touch((block,))
self._partial_tail_pins.setdefault(req_id, []).append(block)
) in mgr.take_pending_boundary_state_offloads():
offloads.setdefault(req_id, []).append(
(group_id, block.block_id, boundary_tokens)
)
+12 -5
View File
@@ -205,6 +205,16 @@ class ScheduledEncoderInputStats:
output_tokens: int = 0
@dataclass
class KVConnectorBlockState:
"""Scheduler-local block state offered to a producer-side KV connector."""
# Authoritative current block-table snapshots.
block_ids: dict[str, tuple[list[int], ...]]
# Exact Mamba "align" boundary-state hand-offs.
boundary_state_offloads: dict[str, list[tuple[int, int, int]]]
@dataclass
class SchedulerOutput:
# list of the requests that are scheduled for the first time.
@@ -278,11 +288,8 @@ class SchedulerOutput:
# CoW copies to apply after zeroing new blocks and before forward.
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
# Producer partial-tail offload hand-off for external KV connectors:
# {request_id: [(group_id, block_id, boundary_tokens), ...]} pointing at
# the durable boundary block of a producer's last-prompt-boundary partial
# tail (mamba "align" CoW target). None unless partial hash hits are active.
partial_tail_offloads: dict[str, list[tuple[int, int, int]]] | None = None
# Scheduler-local; always None by the time this reaches a worker.
kv_connector_block_state: KVConnectorBlockState | None = None
# Dynamic speculative decoding: optimal K chosen by scheduler.
# Number of spec tokens to schedule for the next step.
+31 -15
View File
@@ -42,6 +42,7 @@ from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
from vllm.v1.core.sched.output import (
CachedRequestData,
GrammarOutput,
KVConnectorBlockState,
NewRequestData,
ScheduledEncoderInputStats,
SchedulerOutput,
@@ -1263,20 +1264,32 @@ class Scheduler(SchedulerInterface):
self.prev_step_scheduled_req_ids.clear()
self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys())
# Producer partial-tail hand-off for external KV connectors. Drained
# before the CoW retentions are released below, so the pin lands while
# the cow block still holds a retention ref. Without a producer-side
# connector nothing consumes the hand-off, so skip the drain (and its
# pin); the manager drops stale entries when the request's blocks are
# popped for free.
pending_partial_tail_offloads = None
if (
self.connector is not None
and self.vllm_config.kv_transfer_config is not None
and self.vllm_config.kv_transfer_config.is_kv_producer
):
pending_partial_tail_offloads = (
self.kv_cache_manager.take_partial_tail_offloads() or None
# Mamba "align" boundary states must be handed off with exact block ids;
# they cannot be reconstructed from a connector's append-only block
# table. Drained every step so stale offers cannot accumulate.
boundary_state_offloads = self.kv_cache_manager.take_boundary_state_offloads()
kv_connector_block_state = None
if self.connector is not None:
snapshot_req_ids = {req.req_id for req in new_reqs_data}
snapshot_req_ids.update(
req_id
for req_id, block_ids in zip(
cached_reqs_data.req_ids,
cached_reqs_data.new_block_ids,
strict=True,
)
if block_ids
)
snapshot_req_ids.update(
req_id for req_id in boundary_state_offloads if req_id in self.requests
)
kv_connector_block_state = KVConnectorBlockState(
block_ids={
req_id: self.kv_cache_manager.get_block_ids(req_id)
for req_id in snapshot_req_ids
},
boundary_state_offloads=boundary_state_offloads,
)
kv_cache_block_copies, cow_retained_blocks = (
@@ -1325,7 +1338,7 @@ class Scheduler(SchedulerInterface):
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
has_sync_kv_loads=has_sync_kv_loads,
kv_cache_block_copies=pending_kv_cache_block_copies,
partial_tail_offloads=pending_partial_tail_offloads,
kv_connector_block_state=kv_connector_block_state,
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
)
@@ -1345,6 +1358,9 @@ class Scheduler(SchedulerInterface):
)
scheduler_output.ec_connector_metadata = ec_meta
# Connector-only block state must not be dispatched to workers.
scheduler_output.kv_connector_block_state = None
# Advance the fence only for non-empty steps (those that actually
# write KV and have their output processed later in update_from_output).
if self.defer_block_free and total_num_scheduled_tokens > 0:
+52 -26
View File
@@ -112,13 +112,14 @@ class SingleTypeKVCacheManager(ABC):
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = []
# Partial-tail offload hand-off for external KV connectors: when a
# producer registers its last-prompt-boundary partial tail and the
# durable boundary block is not on the append-only request block table
# (mamba "align" CoW target), record the request, group, block, and
# exact token boundary so a connector can offload it under the right
# hash. Populated only by mamba "align".
self._pending_partial_tail_offloads: list[
# Boundary-state offload hand-off for external KV connectors. A mamba
# "align" block table is not append-only (interior states are
# nulled/freed and speculative blocks relocate in place), so a
# connector cannot resolve its state blocks positionally. Record
# (request, group, block, exact token boundary) for each committed
# boundary state so a connector can offload the right block under the
# right hash. Populated only by mamba "align".
self._pending_boundary_state_offloads: list[
tuple[str, int, KVCacheBlock, int]
] = []
@@ -385,19 +386,19 @@ class SingleTypeKVCacheManager(ABC):
self._pending_cow_copies = []
return pending_copies
def take_pending_partial_tail_offloads(
def take_pending_boundary_state_offloads(
self,
) -> list[tuple[str, int, KVCacheBlock, int]]:
"""Drain producer partial-tail hand-offs.
"""Drain producer boundary-state hand-offs.
Entries are ``(req_id, group_id, block, boundary_tokens)``.
Only mamba "align" populates this. The block lives off the request
block table, so the caller must pin it until the connector has read
it — nothing else keeps it alive once the CoW retention is released.
Only mamba "align" populates this. The blocks are not kept alive by
the request block table for the whole request, so a caller that reads
them asynchronously must pin them first.
"""
pending = self._pending_partial_tail_offloads
self._pending_partial_tail_offloads = []
pending = self._pending_boundary_state_offloads
self._pending_boundary_state_offloads = []
return pending
def _apply_cow(
@@ -1292,7 +1293,7 @@ class MambaManager(SingleTypeKVCacheManager):
# Requests that registered their own last-prompt-boundary partial
# tail (producers). On the next step's CoW the boundary state moves
# into a private cow_block; we record that block for connector
# offload (see _pending_partial_tail_offloads).
# offload (see _pending_boundary_state_offloads).
self._producer_partial_tail_reqs: dict[str, int] = {}
@classmethod
@@ -1625,13 +1626,12 @@ class MambaManager(SingleTypeKVCacheManager):
)
if blocks_allocated:
# reuse previous speculative blocks in this step
# Relocate exclusively owned speculative scratch blocks.
for block_idx in range(
prev_block_len - self.num_speculative_blocks, prev_block_len
):
if block_idx < num_skipped_blocks:
req_blocks.append(req_blocks[block_idx])
req_blocks[block_idx] = self._null_block
self._relocate_speculative_block(req_blocks, block_idx)
else:
break
num_new_blocks = num_required_blocks - len(req_blocks)
@@ -1664,7 +1664,7 @@ class MambaManager(SingleTypeKVCacheManager):
# This CoW preserved a producer's own boundary
# state in cow_block; hand it to the connector for
# partial-tail offload once the copy has run.
self._pending_partial_tail_offloads.append(
self._pending_boundary_state_offloads.append(
(
request_id,
self.kv_cache_group_id,
@@ -1685,17 +1685,31 @@ class MambaManager(SingleTypeKVCacheManager):
returned_blocks.extend(new_blocks)
return returned_blocks
def _relocate_speculative_block(
self, req_blocks: list[KVCacheBlock], block_idx: int
) -> None:
block = req_blocks[block_idx]
assert self.block_pool.is_block_writable(block), (
"Speculative Mamba blocks must be exclusively owned and unhashed "
"before relocation"
)
req_blocks.append(block)
req_blocks[block_idx] = self._null_block
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
if self.mamba_cache_mode == "align":
self._allocated_block_reqs.discard(request_id)
self.last_state_block_idx.pop(request_id, None)
self._num_checkpoint_blocks.pop(request_id, None)
self._producer_partial_tail_reqs.pop(request_id, None)
# A hand-off whose request died in this same scheduling pass must
# not reach the connector: its unpin hook (free) has already run.
self._pending_partial_tail_offloads = [
# An offer is only guaranteed to hold committed bytes until the end
# of the pass that made it. This request's blocks are going back to
# the pool now, so drop its not-yet-offered hand-offs rather than
# let a connector claim a block another request may already have
# been handed.
self._pending_boundary_state_offloads = [
entry
for entry in self._pending_partial_tail_offloads
for entry in self._pending_boundary_state_offloads
if entry[0] != request_id
]
return super().pop_blocks_for_free(request_id)
@@ -1722,9 +1736,9 @@ class MambaManager(SingleTypeKVCacheManager):
if partial_hash is not None:
self.cached_blocks_this_step.add(partial_hash)
if num_cached_blocks_after > num_cached_blocks_before:
for block in self.req_to_blocks[request.request_id][
num_cached_blocks_before:num_cached_blocks_after
]:
blocks = self.req_to_blocks[request.request_id]
for idx in range(num_cached_blocks_before, num_cached_blocks_after):
block = blocks[idx]
# Skip null blocks (align-mode skipped states) and blocks that
# were not cached this step — with sparse retention
# (reachable_block_mask) the intermediate state snapshots carry
@@ -1732,6 +1746,18 @@ class MambaManager(SingleTypeKVCacheManager):
if block.is_null or block.block_hash is None:
continue
self.cached_blocks_this_step.add(block.block_hash)
if self.mamba_cache_mode == "align":
# Offer every retained boundary with its exact block.
# The connector filters against its save window, which may
# extend past the original prompt during resumed prefill.
self._pending_boundary_state_offloads.append(
(
request.request_id,
self.kv_cache_group_id,
block,
(idx + 1) * self.block_size,
)
)
def new_step_starts(self) -> None:
self.cached_blocks_this_step.clear()