[Bugfix][Mooncake] Save exact Mamba boundary states (#51358)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Dao Le <daole@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
OpenAI Codex
Dao Le
parent
16d6c376bc
commit
6b110badbb
@@ -69,6 +69,19 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
manager.get_computed_blocks.assert_not_called()
|
||||
|
||||
|
||||
def drain_boundary_state_offloads(manager):
|
||||
"""Drain exact boundary-state block ids offered to a connector."""
|
||||
return manager.take_boundary_state_offloads()
|
||||
|
||||
|
||||
def _free_block_ids(manager):
|
||||
"""Block ids the pool would hand out to the next allocation."""
|
||||
return {
|
||||
block.block_id
|
||||
for block in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
}
|
||||
|
||||
|
||||
def make_full_mamba_manager(
|
||||
*,
|
||||
dcp_world_size: int,
|
||||
@@ -618,10 +631,9 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
assert moved[0].block_id == cow_copy.dst_block_id
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_returns_cow_target():
|
||||
"""The connector offload hand-off exposes the mamba CoW *target* block Y
|
||||
(the durable boundary state), not the overwritten source X, and only at
|
||||
the CoW step."""
|
||||
def test_boundary_state_offloads_returns_cow_target():
|
||||
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
|
||||
target, never the overwritten CoW source."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -660,8 +672,18 @@ def test_take_partial_tail_offloads_returns_cow_target():
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# Step A registered the partial tail but has not CoW'd yet: no offload.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
# Step A offers the materialized aligned checkpoint immediately.
|
||||
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
|
||||
"0"
|
||||
]
|
||||
assert group_id == 1
|
||||
assert boundary_tokens == 4
|
||||
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
|
||||
aligned_block = manager.block_pool.get_cached_block(
|
||||
aligned_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert aligned_block is not None
|
||||
assert block_id == aligned_block[0].block_id
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
source_block = manager.block_pool.get_cached_block(
|
||||
@@ -675,34 +697,27 @@ def test_take_partial_tail_offloads_returns_cow_target():
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
assert list(offloads.keys()) == ["0"]
|
||||
assert len(offloads["0"]) == 1
|
||||
group_id, block_id, boundary_tokens = offloads["0"][0]
|
||||
assert group_id == 1 # the mamba group
|
||||
assert boundary_tokens == 6
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
|
||||
# The offload points at the durable CoW target Y, not the overwritten X.
|
||||
assert block_id == cow_copy.dst_block_id
|
||||
assert block_id != source_block_id
|
||||
# Draining clears it.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
# The hand-off pinned Y (its CoW retention is released after this step,
|
||||
# and Y is off the request block table); freeing the request unpins it.
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
pinned_ref = cow_block.ref_cnt
|
||||
assert pinned_ref >= 1
|
||||
manager.block_pool.free_blocks(retained)
|
||||
manager.free(req0)
|
||||
assert cow_block.ref_cnt == pinned_ref - 1
|
||||
|
||||
|
||||
def test_partial_tail_pin_survives_released_cow_retention():
|
||||
"""If the CoW retention is released before the hand-off is drained
|
||||
(immediate-free mode), the drain must rescue the cow block from the free
|
||||
queue: a raw ref increment would leave a ref>0 block allocatable, and the
|
||||
next allocation would pop it and assert."""
|
||||
def test_block_pool_touch_pins_released_cow_target():
|
||||
"""The connector can rescue an offered CoW target after its step-scoped
|
||||
retention is released by using the bound BlockPool's touch method."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -746,13 +761,18 @@ def test_partial_tail_pin_survives_released_cow_retention():
|
||||
_copies, retained = manager.take_kv_cache_block_copies()
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((_group_id, block_id, boundary_tokens),) = offloads["0"]
|
||||
assert boundary_tokens == 6
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert cow_block.ref_cnt == 0
|
||||
assert block_id in _free_block_ids(manager)
|
||||
|
||||
# The pinned block is out of the free queue: draining every free block
|
||||
manager.block_pool.touch([cow_block])
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert block_id not in _free_block_ids(manager)
|
||||
|
||||
# The connector-pinned block is out of the free queue: draining every free block
|
||||
# neither trips the allocator's ref_cnt assert nor hands it out.
|
||||
new_blocks = manager.block_pool.get_new_blocks(
|
||||
manager.block_pool.get_num_free_blocks()
|
||||
@@ -760,7 +780,7 @@ def test_partial_tail_pin_survives_released_cow_retention():
|
||||
assert block_id not in {b.block_id for b in new_blocks}
|
||||
|
||||
|
||||
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block."""
|
||||
@@ -805,12 +825,12 @@ def test_partial_tail_offload_dropped_when_request_freed_before_drain():
|
||||
|
||||
# The request dies (preempt/abort) before the scheduler drains.
|
||||
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_empty_without_partial_tail():
|
||||
"""A prompt ending on a block boundary registers no partial tail, so there
|
||||
is nothing to offload."""
|
||||
def test_boundary_state_offloads_block_aligned_prompt():
|
||||
"""A prompt ending on a block boundary registers no CoW partial tail; its
|
||||
boundary state block is handed off as a snapshot instead (once)."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -848,12 +868,17 @@ def test_take_partial_tail_offloads_empty_without_partial_tail():
|
||||
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == block_size
|
||||
assert block_id == mamba_blocks[0].block_id
|
||||
|
||||
req0.num_computed_tokens = 4
|
||||
req0.append_output_token_ids([2])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
# The boundary was already handed off; decoding emits nothing new.
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
@@ -1786,3 +1811,213 @@ def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
|
||||
assert num_computed == 4
|
||||
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
|
||||
|
||||
def _snapshot_offload_kv_cache_config(
|
||||
hash_block_size: int, block_size: int, num_blocks: int = 24
|
||||
):
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_retention_snapshots_handed_off_with_exact_block_ids():
|
||||
"""Under sparse retention, each retained mamba boundary state block is
|
||||
handed to the connector with its exact block id. Connectors must not resolve
|
||||
align-mode state blocks positionally because the block table is not
|
||||
append-only."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
manager.coordinator.retention_interval = 2 * block_size
|
||||
|
||||
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((group_id, block_id, boundary),) = offloads["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == 2 * block_size
|
||||
assert block_id == mamba_blocks[1].block_id
|
||||
|
||||
req0.num_computed_tokens = 8
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 8) is not None
|
||||
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((group_id, block_id2, boundary2),) = offloads["0"]
|
||||
assert group_id == 1
|
||||
assert boundary2 == 4 * block_size
|
||||
assert block_id2 == mamba_blocks[3].block_id
|
||||
# Interior align-mode positions stay null and are never handed off.
|
||||
assert mamba_blocks[0].is_null and mamba_blocks[2].is_null
|
||||
|
||||
manager.free(req0)
|
||||
|
||||
|
||||
def test_snapshot_handoff_dense_default_retention():
|
||||
"""With dense (default) retention, every materialized mamba boundary
|
||||
state block is handed off — regular mamba-align + prefix-match-unit
|
||||
deployments get store-able boundary snapshots without setting
|
||||
VLLM_PREFIX_CACHE_RETENTION_INTERVAL."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
assert manager.coordinator.retention_interval is None
|
||||
|
||||
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == 2 * block_size
|
||||
assert block_id == mamba_blocks[1].block_id
|
||||
|
||||
req0.num_computed_tokens = 8
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 8) is not None
|
||||
((group_id, block_id2, boundary2),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary2 == 4 * block_size
|
||||
assert block_id2 == mamba_blocks[3].block_id
|
||||
|
||||
|
||||
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
|
||||
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
|
||||
boundary-state hand-offs offered (and claimed) at each step."""
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
for step in range(num_chunks):
|
||||
manager.new_step_starts()
|
||||
req.num_computed_tokens = step * chunk_size
|
||||
if step == 0:
|
||||
allocated = manager.allocate_slots(
|
||||
req, chunk_size, num_computed, computed_blocks
|
||||
)
|
||||
else:
|
||||
allocated = manager.allocate_slots(req, chunk_size)
|
||||
assert allocated is not None
|
||||
yield drain_boundary_state_offloads(manager).get(req.request_id, [])
|
||||
|
||||
|
||||
def test_boundary_state_offer_includes_more_mamba_groups_than_two():
|
||||
"""One offer batch carries one exact block entry per mamba group."""
|
||||
block_size = 8
|
||||
num_mamba_groups = 3
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=KVCacheConfig(
|
||||
num_blocks=200,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
*(
|
||||
KVCacheGroupSpec(
|
||||
[f"mamba{i}"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
)
|
||||
for i in range(num_mamba_groups)
|
||||
),
|
||||
],
|
||||
),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0] * (2 * block_size), block_size, sha256)
|
||||
entries = next(_run_chunked_prefill(manager, req0, block_size, 1))
|
||||
|
||||
assert [group_id for group_id, _, _ in entries] == [1, 2, 3]
|
||||
assert {boundary for _, _, boundary in entries} == {block_size}
|
||||
|
||||
|
||||
def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
"""The core offers every committed boundary, including past
|
||||
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
|
||||
and every group re-saves them, so filtering on the original prompt length
|
||||
would silently strip the mamba key for boundaries full attention still
|
||||
stores; only the connector knows where its save window ends."""
|
||||
block_size = 8
|
||||
prompt_len = block_size
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(
|
||||
hash_block_size=block_size, block_size=block_size, num_blocks=200
|
||||
),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0] * prompt_len, block_size, sha256)
|
||||
assert [
|
||||
boundary
|
||||
for entries in _run_chunked_prefill(manager, req0, block_size, 1)
|
||||
for _, _, boundary in entries
|
||||
] == [prompt_len]
|
||||
|
||||
# Generate past the next mamba boundary, then replay it as a resumed
|
||||
# prefill: its committed boundary is offered even though it is past the
|
||||
# prompt, matching what full attention saves for the same range.
|
||||
for i in range(block_size):
|
||||
manager.new_step_starts()
|
||||
req0.num_computed_tokens = prompt_len + i
|
||||
req0.append_output_token_ids([1])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
drain_boundary_state_offloads(manager)
|
||||
assert req0.num_tokens == 2 * block_size
|
||||
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
req0.num_computed_tokens = 0
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, req0.num_tokens - num_computed, num_computed)
|
||||
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
|
||||
assert 2 * block_size in offered
|
||||
assert req0.num_prompt_tokens < 2 * block_size
|
||||
|
||||
@@ -15,12 +15,14 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
FullAttentionManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
@@ -69,6 +71,36 @@ def test_external_computed_blocks_do_not_corrupt_free_pool():
|
||||
assert len(manager.req_to_blocks[request_id]) == 3
|
||||
|
||||
|
||||
def test_mamba_speculative_block_relocation_requires_exclusive_ownership():
|
||||
spec = MambaSpec(
|
||||
block_size=4,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=1,
|
||||
)
|
||||
block_pool = BlockPool(num_gpu_blocks=4, enable_caching=True, hash_block_size=4)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=block_pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=4,
|
||||
)
|
||||
block = block_pool.get_new_blocks(1)[0]
|
||||
blocks = [block]
|
||||
|
||||
manager._relocate_speculative_block(blocks, 0)
|
||||
|
||||
assert blocks == [block_pool.null_block, block]
|
||||
assert block.ref_cnt == 1
|
||||
|
||||
pinned_block = block_pool.get_new_blocks(1)[0]
|
||||
block_pool.touch((pinned_block,))
|
||||
with pytest.raises(AssertionError, match="exclusively owned and unhashed"):
|
||||
manager._relocate_speculative_block([pinned_block], 0)
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -36,7 +36,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
is_store_reachable_swa_chunk,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock, KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import (
|
||||
KVConnectorBlockState,
|
||||
SchedulerOutput,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -107,11 +111,16 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11, 12]
|
||||
req_status.group_states[1].block_ids[:] = [0, 21]
|
||||
scheduler.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(partial_tail_offloads={"req": [(1, 99, 28)]})
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
@@ -155,6 +164,83 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
assert recurrent_event.parent_block_hash is None
|
||||
|
||||
|
||||
def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11, 12]
|
||||
req_status.group_states[1].block_ids[:] = [0, 21]
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 98, 16), (1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 2
|
||||
src_spec = next(
|
||||
job.src_spec for job in jobs.values() if len(job.src_spec.block_ids) == 1
|
||||
)
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
assert src_spec.block_ids.tolist() == [98]
|
||||
assert src_spec.group_sizes == [0, 1]
|
||||
assert src_spec.block_indices == [0, 0]
|
||||
|
||||
|
||||
def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 99, 16)]},
|
||||
)
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
[job_id] = meta.store_jobs
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
|
||||
assert meta.jobs_to_flush == {job_id}
|
||||
|
||||
|
||||
def test_normal_store_excludes_align_mode_mamba_sources():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
request.num_computed_tokens = 0
|
||||
request.status = RequestStatus.RUNNING
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11]
|
||||
req_status.group_states[1].block_ids[:] = [99]
|
||||
req_status.update_offload_keys()
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(
|
||||
num_scheduled_tokens={"req": 16},
|
||||
finished_req_ids=set(),
|
||||
)
|
||||
jobs = scheduler._build_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
[job] = jobs.values()
|
||||
src_spec = job.src_spec
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
assert src_spec.block_ids.tolist() == [11]
|
||||
assert src_spec.group_sizes == [1, 0]
|
||||
|
||||
|
||||
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
|
||||
@@ -5,6 +5,9 @@ import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -31,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
@@ -62,6 +66,30 @@ def _make_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_requires_align_mode_for_mamba():
|
||||
vllm_config = _make_vllm_config()
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
),
|
||||
pytest.raises(AssertionError, match="requires mamba_cache_mode='align'"),
|
||||
):
|
||||
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
def _make_block_stored() -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[b"hash"],
|
||||
@@ -97,6 +125,10 @@ def test_scheduler_role_initializes_store_scheduler_only():
|
||||
mock_worker.assert_not_called()
|
||||
assert connector.connector_scheduler is mock_scheduler.return_value
|
||||
assert connector.connector_worker is None
|
||||
block_pool = MagicMock()
|
||||
|
||||
connector.bind_gpu_block_pool(block_pool)
|
||||
mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool)
|
||||
|
||||
|
||||
def test_worker_methods_delegate_to_store_worker():
|
||||
@@ -241,9 +273,13 @@ def test_update_connector_output_and_take_events():
|
||||
|
||||
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events.add_events([event])
|
||||
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
output = KVConnectorOutput(kv_cache_events=kv_events)
|
||||
connector.update_connector_output(output)
|
||||
|
||||
assert connector._kv_cache_events is kv_events
|
||||
connector.connector_scheduler.update_connector_output.assert_called_once_with(
|
||||
output
|
||||
)
|
||||
assert list(connector.take_events()) == [event]
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
|
||||
@@ -451,6 +451,27 @@ def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
def test_store_mask_excludes_mamba_groups_lookup_unaffected():
|
||||
"""Align-mode mamba block tables are not append-only (interior states are
|
||||
nulled/freed; speculative blocks relocate), so the positional normal save
|
||||
must never cover mamba chunks — regardless of retention. Lookups still
|
||||
probe mamba boundaries: their keys come from the pinned snapshot/CoW
|
||||
hand-off path instead."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(16)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks[0] is None # full attn stays dense
|
||||
assert masks[1] == [False] * 4
|
||||
|
||||
coord = _make_coord(groups, hash_block_size=16, retention_interval=32)
|
||||
assert coord.store_mask(64)[1] == [False] * 4
|
||||
|
||||
assert coord.lookup_mask(64)[1] is None
|
||||
|
||||
|
||||
# ----- Eagle / MTP interaction with load_mask -----
|
||||
|
||||
|
||||
|
||||
@@ -236,6 +236,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
# 1-based: block 0 is the reserved null block and the save skips it.
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
@@ -441,10 +442,10 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
send._maybe_offload_boundary_states(req)
|
||||
|
||||
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
|
||||
partial_hash = hs[2]
|
||||
@@ -512,8 +513,8 @@ def test_offload_syncs_event_before_put():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
store_job_id=1,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
req.current_event = event
|
||||
|
||||
@@ -589,10 +590,10 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
send._maybe_offload_boundary_states(req)
|
||||
|
||||
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
|
||||
# block-end hashes; mamba (block 16): the partial boundary block under
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
@@ -47,9 +48,20 @@ def _make_bare_scheduler(
|
||||
scheduler._num_workers = 1
|
||||
scheduler._next_store_job_id = 0
|
||||
scheduler._pinned_saves = {}
|
||||
scheduler._boundary_state_group_ids = frozenset({1})
|
||||
return scheduler
|
||||
|
||||
|
||||
def _make_connector_block_state(
|
||||
block_ids: tuple[list[int], ...] | None = None,
|
||||
offloads: list[tuple[int, int, int]] | None = None,
|
||||
) -> KVConnectorBlockState:
|
||||
return KVConnectorBlockState(
|
||||
block_ids={} if block_ids is None else {"req-0": block_ids},
|
||||
boundary_state_offloads=({} if offloads is None else {"req-0": offloads}),
|
||||
)
|
||||
|
||||
|
||||
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
@@ -65,6 +77,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
scheduled_spec_decode_tokens=(
|
||||
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
|
||||
),
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -84,6 +97,7 @@ def _make_decode_scheduler_output(
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -109,6 +123,7 @@ def _make_new_scheduler_output() -> SimpleNamespace:
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 32},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -491,6 +506,7 @@ def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamesp
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -915,7 +931,10 @@ def _add_pending_partial_tail_request(
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=block_ids,
|
||||
offloads=[(1, 7, 12)],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -940,15 +959,262 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 0
|
||||
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
|
||||
assert req_meta.boundary_state_offloads == [(1, 7, 12)]
|
||||
assert req_meta.num_prompt_tokens == 12
|
||||
assert req_meta.block_ids == ([0],)
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7]
|
||||
assert scheduler._gpu_block_pool.blocks[0].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
def test_decode_boundary_state_offload_dropped_unclaimed():
|
||||
# A hand-off past the prefill end can never complete a joint hybrid hit
|
||||
# (every other group stops saving there), so it is neither transferred nor
|
||||
# claimed — leaving the core free to release the block immediately.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(24)),
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([0],),
|
||||
num_saved_tokens=12,
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
# Boundary 16 is a decode boundary for a 12-token prefill.
|
||||
kv_connector_block_state=_make_connector_block_state(offloads=[(1, 7, 16)]),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert meta.requests == []
|
||||
assert scheduler._pinned_saves == {}
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._request_trackers["req-0"].has_pending_offload is False
|
||||
|
||||
|
||||
def _register_offload_request(scheduler, *, prefill_end_tokens, num_prompt_tokens):
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(64)),
|
||||
block_hashes=[bytes([i]) for i in range(16)],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=prefill_end_tokens,
|
||||
allocated_block_ids=([0],),
|
||||
num_saved_tokens=prefill_end_tokens,
|
||||
token_ids=list(range(prefill_end_tokens)),
|
||||
prefill_end_tokens=prefill_end_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _make_offload_only_output(entries, block_ids=([0],)):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=block_ids,
|
||||
offloads=entries,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_resumed_prefill_claims_boundaries_past_prompt_length():
|
||||
# A resumed request re-prefills its previously generated tokens, so its
|
||||
# save window (`prefill_end_tokens`) extends past `num_prompt_tokens`.
|
||||
# Boundaries in that range must still be claimed, or the mamba key would be
|
||||
# missing for boundaries full attention does store and no joint hit could
|
||||
# complete there.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=20, num_prompt_tokens=12)
|
||||
out = _make_offload_only_output([(1, 7, 16), (1, 9, 20), (1, 11, 24)])
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
# 16 and 20 are inside the resumed prefill; 24 is past it.
|
||||
assert meta.requests[0].boundary_state_offloads == [(1, 7, 16), (1, 9, 20)]
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 9]
|
||||
|
||||
|
||||
def test_boundary_state_job_pins_exact_blocks_once():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
scheduler._request_trackers["req-0"].allocated_block_ids = ([7],)
|
||||
out = _make_offload_only_output(
|
||||
[(1, 7, 16), (1, 8, 32), (1, 9, 48)], block_ids=([7],)
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.boundary_state_offloads == [
|
||||
(1, 7, 16),
|
||||
(1, 8, 32),
|
||||
(1, 9, 48),
|
||||
]
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 8, 9]
|
||||
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
]
|
||||
|
||||
|
||||
def test_store_job_pins_current_non_null_non_mamba_blocks():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[bytes([i]) for i in range(12)],
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([7, 2, 0], [21, 0, 22]))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=44,
|
||||
allocated_block_ids=([7, 2, 0], [21, 0, 22]),
|
||||
num_saved_tokens=32,
|
||||
token_ids=list(range(44)),
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
stale_block_ids = ([7, 2, 0, 5], [21, 0, 22, 23])
|
||||
current_block_ids = ([7, 2, 0, 6], [24, 0, 25, 26])
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([5], [23])],
|
||||
num_computed_tokens=[44],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=current_block_ids,
|
||||
offloads=[(1, 8, 16)],
|
||||
),
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert scheduler._request_trackers["req-0"].allocated_block_ids == stale_block_ids
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.block_ids == current_block_ids
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [8, 7, 2, 6]
|
||||
assert pool.blocks[0].ref_cnt == 0
|
||||
assert pool.blocks[5].ref_cnt == 0
|
||||
assert [pool.blocks[i].ref_cnt for i in (21, 22, 23, 24, 25, 26)] == [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
]
|
||||
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [1, 1, 1, 1]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
|
||||
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [0, 0, 0, 0]
|
||||
|
||||
|
||||
def test_boundary_state_release_is_per_store_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
|
||||
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
|
||||
first_job_id = first.requests[0].store_job_id
|
||||
second_job_id = second.requests[0].store_job_id
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
|
||||
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
|
||||
assert first_job_id not in scheduler._pinned_saves
|
||||
assert second_job_id in scheduler._pinned_saves
|
||||
|
||||
|
||||
def test_preemption_and_request_id_reuse_do_not_release_inflight_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
|
||||
first_job_id = first.requests[0].store_job_id
|
||||
|
||||
scheduler.build_connector_meta(_make_preemption_scheduler_output())
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
|
||||
assert first_job_id in scheduler._pinned_saves
|
||||
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
|
||||
second_job_id = second.requests[0].store_job_id
|
||||
assert second.requests[0].boundary_state_offloads == [(1, 8, 32)]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
|
||||
assert second_job_id in scheduler._pinned_saves
|
||||
|
||||
|
||||
def test_boundary_state_never_claimed_without_a_send_thread():
|
||||
# A kv_consumer has no store job that can acknowledge the block reference.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.kv_role = "kv_consumer"
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
out = _make_offload_only_output([(1, 7, 16)])
|
||||
|
||||
assert scheduler.build_connector_meta(out).requests == []
|
||||
assert scheduler._pinned_saves == {}
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_exact_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
out = _add_pending_partial_tail_request(
|
||||
@@ -961,7 +1227,7 @@ def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
|
||||
assert meta.requests[0].boundary_state_offloads == [(1, 7, 12)]
|
||||
# Ordinary metadata retains the full resumed prefill range.
|
||||
assert meta.requests[0].num_prompt_tokens == 20
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
@@ -969,8 +1235,9 @@ def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
def test_resumed_partial_tail_attached_to_save_keeps_exact_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler._boundary_state_group_ids = frozenset({0})
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
@@ -987,13 +1254,13 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
out = _make_scheduler_output(scheduled_spec_tokens=None)
|
||||
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
|
||||
out.kv_connector_block_state.boundary_state_offloads = {"req-0": [(0, 7, 36)]}
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].can_save is True
|
||||
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
|
||||
assert meta.requests[0].boundary_state_offloads == [(0, 7, 36)]
|
||||
assert meta.requests[0].num_prompt_tokens == 48
|
||||
# Ordinary saving still covers the full resumed prefill range.
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
@@ -1019,7 +1286,8 @@ def test_partial_tail_cow_block_is_referenced_for_the_job():
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
|
||||
# puts it last in eviction priority.
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7]
|
||||
assert pool.blocks[0].ref_cnt == 0
|
||||
assert pool.blocks[7].ref_cnt == 1
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
|
||||
@@ -181,6 +181,7 @@ def _run_store_req(thread, req_meta: ReqMeta) -> None:
|
||||
|
||||
|
||||
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
# 1-based: block 0 is the reserved null block and the save skips it.
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=32,
|
||||
@@ -651,6 +652,7 @@ def _make_partial_tail_send_thread(
|
||||
enable_partial_hash_hits=True,
|
||||
hash_block_size=4,
|
||||
lcm_block_size=16,
|
||||
mamba_group_ids={1},
|
||||
)
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
@@ -659,10 +661,18 @@ def _make_partial_tail_send_thread(
|
||||
)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([256])
|
||||
# Group 1 models the mamba "align" group the hand-offs reference.
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([256])
|
||||
return _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db],
|
||||
token_databases=[db, db_mamba],
|
||||
replicate_config=replicate_config,
|
||||
enable_group_semantics=enable_group_semantics,
|
||||
supports_group_ids=supports_group_ids,
|
||||
@@ -673,20 +683,20 @@ def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=(block_ids,),
|
||||
block_ids=(block_ids, [1]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
|
||||
|
||||
def test_partial_tail_offload_skips_null_source_blocks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
|
||||
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([0, 2, 3]))
|
||||
|
||||
keys, addrs, _sizes, _replicate_config = (
|
||||
store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -694,15 +704,21 @@ def test_partial_tail_offload_skips_null_source_blocks():
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
|
||||
]
|
||||
# FA reads blocks 2 and 3 (block 0 is null and skipped); the mamba
|
||||
# boundary block reads the core-provided CoW block 7, not block_ids.
|
||||
assert addrs == [
|
||||
[0x1000 + 2 * 256],
|
||||
[0x1000 + 3 * 256],
|
||||
[0x2000 + 7 * 256],
|
||||
]
|
||||
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
|
||||
|
||||
|
||||
def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
@@ -711,16 +727,11 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
sparse = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
KVCacheGroupSpec(["sparse"], sparse),
|
||||
],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
@@ -732,17 +743,17 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
db_sparse = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([512])
|
||||
db_sparse.set_kv_caches_base_addr([0x2000])
|
||||
db_sparse.set_block_len([512])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_mamba],
|
||||
token_databases=[db_full, db_sparse],
|
||||
)
|
||||
_run_store_req(
|
||||
thread,
|
||||
@@ -756,15 +767,86 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
)
|
||||
|
||||
keys, addrs, _, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
mamba_keys = [key for key in keys if "@group:1" in key]
|
||||
assert [key.rsplit("@", 1)[-1] for key in mamba_keys] == [b"a1".hex()]
|
||||
sparse_keys = [key for key in keys if "@group:1" in key]
|
||||
assert [key.rsplit("@", 1)[-1] for key in sparse_keys] == [b"a1".hex()]
|
||||
assert all(addr[0] >= 0x2000 for key, addr in zip(keys, addrs) if "@group:1" in key)
|
||||
|
||||
|
||||
def test_partial_tail_offload_skips_cap_omitted_mamba_group():
|
||||
"""A Mamba group omitted by the handoff cap must not fall back to the
|
||||
connector's positional block table."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=4, num_kv_heads=8, head_size=64, dtype=torch.float32
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[
|
||||
KVCacheGroupSpec(["full"], full_spec),
|
||||
KVCacheGroupSpec(["mamba-1"], mamba_spec),
|
||||
KVCacheGroupSpec(["mamba-2"], mamba_spec),
|
||||
],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
|
||||
def make_db(group_id: int, block_size: int, base_addr: int):
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
|
||||
block_size=block_size,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([base_addr])
|
||||
db.set_block_len([256])
|
||||
return db
|
||||
|
||||
db_full = make_db(0, 4, 0x1000)
|
||||
db_mamba_1 = make_db(1, 16, 0x2000)
|
||||
db_mamba_2 = make_db(2, 16, 0x3000)
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_mamba_1, db_mamba_2],
|
||||
)
|
||||
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5], [9]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
# The scheduler accepted group 1 and omitted group 2 at boundary 12.
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _replicate_config = (
|
||||
store.batch_put_from_multi_buffers.call_args.args
|
||||
)
|
||||
puts = list(zip(keys, addrs, strict=True))
|
||||
boundary_hash = BlockHash(b"a2")
|
||||
assert (db_mamba_1.key_for(boundary_hash), [0x2000 + 7 * 256]) in puts
|
||||
assert (db_mamba_2.key_for(boundary_hash), [0x3000 + 9 * 256]) not in puts
|
||||
|
||||
|
||||
def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [1, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
store.batch_is_exist.return_value = [1, 0, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256, 256]
|
||||
replicate_config = SimpleNamespace(group_ids=["stale"])
|
||||
thread = _make_partial_tail_send_thread(
|
||||
store,
|
||||
@@ -773,17 +855,19 @@ def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([1, 2, 3]))
|
||||
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
|
||||
]
|
||||
assert config is replicate_config
|
||||
assert config.group_ids == [
|
||||
"vllm-mooncake-store:test-model@6131",
|
||||
"vllm-mooncake-store:test-model@6132",
|
||||
"vllm-mooncake-store:test-model@6132",
|
||||
]
|
||||
|
||||
|
||||
@@ -815,6 +899,165 @@ def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
def test_normal_save_excludes_mamba_group_and_null_blocks():
|
||||
"""The positional normal save must never cover mamba chunks (align-mode
|
||||
block tables are not append-only) and must skip chunks whose block is the
|
||||
reserved null block, for any group."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], mamba_spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), block_size=16
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([256])
|
||||
thread = _make_store_sending_thread(
|
||||
store, coord=coord, token_databases=[db_full, db_mamba]
|
||||
)
|
||||
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
# FA has a null at chunk 1; the mamba table is align-mode shaped
|
||||
# (interior nulls + one state block) and must not be read at all.
|
||||
block_ids=([1, 0, 3, 4], [0, 0, 0, 9]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
),
|
||||
)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert all("@group:0@" in k for k in keys)
|
||||
assert [k.rsplit("@", 1)[-1] for k in keys] == ["6130", "6132", "6133"]
|
||||
assert addrs == [[0x1000 + 256], [0x1000 + 3 * 256], [0x1000 + 4 * 256]]
|
||||
|
||||
|
||||
def test_block_aligned_snapshot_offload_uses_provided_block():
|
||||
"""A block-aligned hand-off (sparse-retention mamba checkpoint) uploads
|
||||
exactly the core-pinned block under the boundary-end hash key; the
|
||||
positional block table entry for that chunk must be ignored."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)] # 8 hash units = 32 tokens
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, 7, 32)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
# boundary 32 is block-aligned for the mamba group (block 16): one key,
|
||||
# keyed by hs[32 // 4 - 1], read from the handed-off block 7 — not from
|
||||
# block_ids[1] and with no FA gap coverage.
|
||||
assert keys == [thread.token_databases[1].key_for(BlockHash(hs[7]))]
|
||||
assert addrs == [[0x2000 + 7 * 256]]
|
||||
|
||||
|
||||
def test_mixed_snapshot_and_sub_block_offloads():
|
||||
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
|
||||
in one hand-off; the sub-block path covers FA gap blocks but reads the
|
||||
mamba boundary only from the CoW block, never positionally."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(11)] # 11 hash units = 44 tokens
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [0, 0, 0]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, 9, 32), (1, 7, 44)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
db_full, db_mamba = thread.token_databases
|
||||
assert keys == [
|
||||
# Aligned snapshot: boundary 32 from the handed-off block 9.
|
||||
db_mamba.key_for(BlockHash(hs[7])),
|
||||
# Sub-block boundary 44: FA gap blocks ending at 4, 8, 12.
|
||||
db_full.key_for(BlockHash(hs[0])),
|
||||
db_full.key_for(BlockHash(hs[1])),
|
||||
db_full.key_for(BlockHash(hs[2])),
|
||||
# Mamba boundary block from the CoW hand-off (block 7).
|
||||
db_mamba.key_for(BlockHash(hs[10])),
|
||||
]
|
||||
assert addrs == [
|
||||
[0x2000 + 9 * 256],
|
||||
[0x1000 + 1 * 256],
|
||||
[0x1000 + 2 * 256],
|
||||
[0x1000 + 3 * 256],
|
||||
[0x2000 + 7 * 256],
|
||||
]
|
||||
|
||||
|
||||
def test_snapshot_offload_skips_null_handoff_block():
|
||||
"""A hand-off the core could not materialize (null block) carries no
|
||||
committed state; persisting it would poison the boundary key."""
|
||||
store = MagicMock()
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
assert thread._maybe_offload_boundary_states(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, NULL_BLOCK_ID, 32)],
|
||||
)
|
||||
)
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
|
||||
|
||||
def test_worker_meta_is_absent_before_any_completed_job():
|
||||
send_thread = _make_partial_tail_send_thread(MagicMock())
|
||||
store_worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
|
||||
store_worker.kv_send_thread = send_thread
|
||||
|
||||
assert store_worker.build_connector_worker_meta() is None
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
|
||||
@@ -212,12 +212,21 @@ def _accuracy_test(llm: LLM, subscriber: MockSubscriber | None):
|
||||
if subscriber is not None:
|
||||
subscriber.get_new_cpu_stored_events()
|
||||
|
||||
# Pad prompt so its token count is a multiple of cpu_block_size.
|
||||
# Use the tokenizer directly to avoid expensive llm.generate() calls.
|
||||
# Align the reusable prefix; the final token must be recomputed for logits.
|
||||
tokenizer = llm.get_tokenizer()
|
||||
prompt = "Let's count to 10. One, two, three, four,"
|
||||
while len(tokenizer.encode(prompt)) % cpu_block_size != 0:
|
||||
prompt = ". " + prompt
|
||||
prompt_token_ids = tokenizer.encode("Let's count to 10. One, two, three, four,")
|
||||
reusable_prefix = prompt_token_ids[:-1]
|
||||
final_token = prompt_token_ids[-1:]
|
||||
|
||||
[padding_token_id] = tokenizer.encode(".", add_special_tokens=False)
|
||||
padding = [padding_token_id] * (-len(reusable_prefix) % cpu_block_size)
|
||||
insert_at = int(reusable_prefix[0] == tokenizer.bos_token_id)
|
||||
reusable_prefix[insert_at:insert_at] = padding
|
||||
assert len(reusable_prefix) % cpu_block_size == 0
|
||||
|
||||
prompt = TokensPrompt(
|
||||
prompt_token_ids=reusable_prefix + final_token,
|
||||
)
|
||||
|
||||
# Seed the CPU cache with the prompt.
|
||||
llm.generate(prompt, sampling_params, use_tqdm=False)
|
||||
|
||||
@@ -22,8 +22,13 @@ from vllm.v1.request import Request
|
||||
|
||||
|
||||
class DummyConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self, block_hashes_by_req: dict[str, list[BlockHash]]):
|
||||
def __init__(
|
||||
self,
|
||||
block_hashes_by_req: dict[str, list[BlockHash]],
|
||||
block_ids_by_req: dict[str, tuple[list[int], ...]],
|
||||
):
|
||||
self.block_hashes_by_req = block_hashes_by_req
|
||||
self.block_ids_by_req = block_ids_by_req
|
||||
|
||||
|
||||
class DummyKVConnector(KVConnectorBase_V1):
|
||||
@@ -47,8 +52,12 @@ class DummyKVConnector(KVConnectorBase_V1):
|
||||
assert block_hashes_by_req is not None, (
|
||||
"DummyKVConnector expected 'block_hashes_by_req' on scheduler_output"
|
||||
)
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None
|
||||
block_ids_by_req = block_state.block_ids
|
||||
return DummyConnectorMetadata(
|
||||
block_hashes_by_req=block_hashes_by_req,
|
||||
block_ids_by_req=block_ids_by_req,
|
||||
)
|
||||
|
||||
def start_load_kv(self, kv_caches, finished_req_ids):
|
||||
@@ -129,3 +138,9 @@ def test_connector_receives_block_hashes(_load_plugin):
|
||||
num_tokens // block_size
|
||||
)
|
||||
assert meta.block_hashes_by_req[req.request_id] == req.block_hashes
|
||||
|
||||
expected_block_ids = {
|
||||
req.req_id: req.block_ids for req in output.scheduled_new_reqs
|
||||
}
|
||||
assert meta.block_ids_by_req == expected_block_ids
|
||||
assert output.kv_connector_block_state is None
|
||||
|
||||
@@ -87,6 +87,11 @@ class MooncakeStoreCoordinator:
|
||||
for g in kv_cache_groups
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.mamba_group_ids = {
|
||||
group_id
|
||||
for group_id, group in enumerate(kv_cache_groups)
|
||||
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
|
||||
}
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
@@ -205,12 +210,23 @@ class MooncakeStoreCoordinator:
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
|
||||
Mamba groups are always all-False: the normal save resolves blocks
|
||||
positionally from the connector's append-only block-ID snapshot, but
|
||||
an align-mode mamba block table is not append-only (interior state
|
||||
blocks are nulled/freed, and speculative decoding relocates the spec
|
||||
blocks in place), so a positional read may hit a null, freed, or live
|
||||
speculative-state block and persist wrong bytes under a valid prefix
|
||||
hash. Mamba state is persisted only through the connector-pinned exact
|
||||
block hand-off path
|
||||
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
start_token,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
exclude_mamba=True,
|
||||
)
|
||||
|
||||
def lookup_mask(
|
||||
@@ -237,6 +253,7 @@ class MooncakeStoreCoordinator:
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
exclude_mamba: bool = False,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
mask_alignment = (
|
||||
self.hash_block_size
|
||||
@@ -252,6 +269,9 @@ class MooncakeStoreCoordinator:
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
end_chunk = aligned_token_len // spec.block_size
|
||||
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
|
||||
if exclude_mamba and isinstance(spec, MambaSpec):
|
||||
masks.append([False] * (end_chunk - start_chunk))
|
||||
continue
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
|
||||
@@ -406,11 +406,11 @@ class ReqMeta:
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Core-provided per-mamba-group
|
||||
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
|
||||
# Present only on the producer's CoW step; drives the connector's offload
|
||||
# (the FA group's block is derived from block_ids and boundary_tokens).
|
||||
partial_tail_offloads: list[tuple[int, int, int]] | None = None
|
||||
# Core-provided (group_id, block_id, boundary_tokens) mamba "align"
|
||||
# boundary states. A block-aligned entry is a committed boundary snapshot;
|
||||
# a non-aligned entry is the sub-block CoW tail. The store-job reference
|
||||
# keeps each exact block alive until every worker rank finishes the job.
|
||||
boundary_state_offloads: list[tuple[int, int, int]] | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_request_tracker(
|
||||
|
||||
@@ -23,11 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -80,12 +81,15 @@ class MooncakeStoreScheduler:
|
||||
self._hash_block_size,
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
mamba_groups = {
|
||||
group_id: group.kv_cache_spec
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
}
|
||||
assert all(
|
||||
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._boundary_state_group_ids = frozenset(mamba_groups)
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
@@ -93,6 +97,12 @@ class MooncakeStoreScheduler:
|
||||
# store_job_id -> (referenced block ids, ranks yet to report completion)
|
||||
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
@@ -381,47 +391,41 @@ class MooncakeStoreScheduler:
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Flush partial-tail offloads in the step they arrive: the CoW copy is
|
||||
# enqueued before the connector event records, so this step's event
|
||||
# fences the cow block. Ride the request's save meta when present, else
|
||||
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
|
||||
# save; can_save=True takes the normal enqueue path).
|
||||
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
|
||||
if step_partial_tails and not is_consumer:
|
||||
pending = dict(step_partial_tails)
|
||||
for req_meta in meta.requests:
|
||||
if req_meta.can_save:
|
||||
groups = pending.pop(req_meta.req_id, None)
|
||||
if groups:
|
||||
req_meta.partial_tail_offloads = groups
|
||||
tracker = self._request_trackers.get(req_meta.req_id)
|
||||
if tracker is not None:
|
||||
tracker.has_pending_offload = True
|
||||
for req_id, groups in pending.items():
|
||||
tracker = self._request_trackers.get(req_id)
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if tracker is None or req_tuple is None:
|
||||
# Request finished/preempted within this step; its blocks
|
||||
# are going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping partial-tail offload for request %s", req_id)
|
||||
continue
|
||||
assert len({boundary for _, _, boundary in groups}) == 1
|
||||
tracker.has_pending_offload = True
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
partial_tail_offloads=groups,
|
||||
)
|
||||
)
|
||||
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
|
||||
if (
|
||||
block_state is not None
|
||||
and block_state.boundary_state_offloads
|
||||
and not is_consumer
|
||||
):
|
||||
self._handle_boundary_state_offloads(
|
||||
block_state.boundary_state_offloads, meta
|
||||
)
|
||||
|
||||
self._apply_current_save_block_ids(meta, scheduler_output)
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def _apply_current_save_block_ids(
|
||||
self,
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> None:
|
||||
"""Replace append-only mirrors with the core's current block tables."""
|
||||
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
|
||||
if not save_metas:
|
||||
return
|
||||
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None, (
|
||||
"Current block tables are required for Mooncake store jobs"
|
||||
)
|
||||
for req_meta in save_metas:
|
||||
block_ids = block_state.block_ids.get(req_meta.req_id)
|
||||
assert block_ids is not None, (
|
||||
f"Missing current block table for store request {req_meta.req_id}"
|
||||
)
|
||||
req_meta.block_ids = block_ids
|
||||
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
|
||||
@@ -439,21 +443,86 @@ class MooncakeStoreScheduler:
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids: list[int] = []
|
||||
if req_meta.partial_tail_offloads:
|
||||
# A partial-tail CoW block is deliberately kept out of the
|
||||
# request's block table, so it is absent from `block_ids` even
|
||||
# though the worker DMAs out of it just as asynchronously.
|
||||
# It leads the list, as in `pop_blocks_for_free`.
|
||||
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
|
||||
if req_meta.boundary_state_offloads:
|
||||
block_ids.extend(
|
||||
block_id for _, block_id, _ in req_meta.boundary_state_offloads
|
||||
)
|
||||
assert NULL_BLOCK_ID not in block_ids, (
|
||||
"A null block cannot back a boundary-state offload"
|
||||
)
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids += [bid for group in req_meta.block_ids for bid in group]
|
||||
block_ids.extend(
|
||||
block_id
|
||||
for group_id, group in enumerate(req_meta.block_ids)
|
||||
if group_id not in self._boundary_state_group_ids
|
||||
for block_id in group
|
||||
if block_id != NULL_BLOCK_ID
|
||||
)
|
||||
# An aligned boundary block may also be present in the request's
|
||||
# block table. Take and release exactly one reference per block.
|
||||
block_ids = list(dict.fromkeys(block_ids))
|
||||
assert NULL_BLOCK_ID not in block_ids
|
||||
if not block_ids:
|
||||
continue
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[bid] for bid in block_ids])
|
||||
pool.touch([pool.blocks[block_id] for block_id in block_ids])
|
||||
|
||||
def _handle_boundary_state_offloads(
|
||||
self,
|
||||
offloads: dict[str, list[tuple[int, int, int]]],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> None:
|
||||
"""Attach exact boundary-state blocks to store jobs for this step.
|
||||
|
||||
Flushed in the step they arrive: the CoW copy is enqueued before the
|
||||
connector event records, so this step's event fences the exact block.
|
||||
Entries ride the request's save meta when present, else an
|
||||
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
|
||||
``can_save=True`` takes the normal enqueue and store-job pinning path).
|
||||
"""
|
||||
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
|
||||
for req_id, entries in offloads.items():
|
||||
tracker = self._request_trackers.get(req_id)
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if tracker is None or req_tuple is None:
|
||||
# Request finished/preempted within this step; its blocks are
|
||||
# going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping boundary-state offload for request %s", req_id)
|
||||
continue
|
||||
accepted: list[tuple[int, int, int]] = []
|
||||
for group_id, block_id, boundary_tokens in entries:
|
||||
# Every other group stops saving at the end of this prefill, so
|
||||
# a mamba-only key past it can never complete a joint hybrid
|
||||
# hit. `prefill_end_tokens` — not the original prompt length —
|
||||
# is the boundary: a resumed request re-prefills and re-saves
|
||||
# its previously generated tokens for every group.
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
if group_id not in self._boundary_state_group_ids:
|
||||
continue
|
||||
accepted.append((group_id, block_id, boundary_tokens))
|
||||
if not accepted:
|
||||
continue
|
||||
tracker.has_pending_offload = True
|
||||
if (req_meta := save_metas.get(req_id)) is not None:
|
||||
req_meta.boundary_state_offloads = accepted
|
||||
continue
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
boundary_state_offloads=accepted,
|
||||
)
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Drop the block references of store jobs every rank has finished."""
|
||||
|
||||
@@ -648,48 +648,76 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._skip_store_requests.clear()
|
||||
return True
|
||||
|
||||
def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool:
|
||||
"""Offload the request's sub-block partial tail (its last prompt hash
|
||||
boundary) so a later request can hit the sub-block prefix.
|
||||
def _boundary_snapshot_puts(
|
||||
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
|
||||
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
|
||||
"""Puts for committed mamba "align" boundary-state snapshots.
|
||||
|
||||
Covers every block from the normal save's lcm floor to the boundary:
|
||||
the normal save floors to ``lcm_block_size``, so a smaller-block
|
||||
group's full blocks in that gap are never persisted elsewhere, and
|
||||
the consumer's lookup needs every group at every probed boundary.
|
||||
Full blocks are keyed by their block-end hash, the partial boundary
|
||||
block by the boundary sub-hash; the mamba "align" boundary block is
|
||||
the core-provided CoW block. All keys are deduped against the store.
|
||||
These are block-aligned boundaries, i.e. exactly what the normal save
|
||||
would key — but ``store_mask`` masks mamba groups out of it entirely, so
|
||||
this is their *only* writer. The exclusion is not an optimization: the
|
||||
normal save resolves a chunk's address as
|
||||
``req_meta.block_ids[g][start // block_size]``, and ``block_ids`` is the
|
||||
connector's append-only mirror of the core's per-group table. An
|
||||
align-mode table is mutated in place (a superseded state block is freed
|
||||
and nulled; speculative blocks relocate), and the connector is never
|
||||
told, so a stale mirror entry is indistinguishable from a live one — a
|
||||
retry of a failed or pressure-skipped chunk would read a block that now
|
||||
belongs to another request.
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
Each entry's handed-off block *is* the boundary state and is pinned by
|
||||
the core, so it is uploaded under its boundary-end hash key and never
|
||||
resolved positionally.
|
||||
"""
|
||||
if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes:
|
||||
return True
|
||||
partial_tail_offloads = req_meta.partial_tail_offloads
|
||||
if not partial_tail_offloads:
|
||||
return True
|
||||
hash_block_size = self.coord.hash_block_size
|
||||
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for group_id, block_id, boundary in entries:
|
||||
if boundary == 0 or block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
hash_idx = boundary // hash_block_size - 1
|
||||
if hash_idx >= len(req_meta.block_hashes):
|
||||
continue
|
||||
db = self.token_databases[group_id]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[group_id]
|
||||
put_step_rank = (self.tp_rank + group_id) % put_step
|
||||
if (boundary // db.block_size - 1) % put_step != put_step_rank:
|
||||
continue
|
||||
addr, size = db.prepare_value_for_block(block_id)
|
||||
puts.append(
|
||||
(db.key_for(req_meta.block_hashes[hash_idx]), addr, size, db.metadata)
|
||||
)
|
||||
return puts
|
||||
|
||||
def _sub_block_tail_puts(
|
||||
self, req_meta: ReqMeta, entries: list[tuple[int, int, int]]
|
||||
) -> list[tuple[str, list[int], list[int], KeyMetadata]]:
|
||||
"""Puts for the request's sub-block partial tail (its last prompt hash
|
||||
boundary), so a later request can hit the sub-block prefix.
|
||||
|
||||
Covers every group's blocks from the normal save's lcm floor to the
|
||||
boundary: the normal save floors to ``lcm_block_size``, so a
|
||||
smaller-block group's full blocks in that gap are never persisted
|
||||
elsewhere, and the consumer's lookup needs every group at every probed
|
||||
boundary. Full blocks are keyed by their block-end hash and the partial
|
||||
boundary block by the boundary sub-hash; a mamba "align" group
|
||||
contributes only its boundary block, from the core-provided CoW block.
|
||||
"""
|
||||
boundaries = {boundary for _, _, boundary in entries}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Partial-tail offloads for one request must share a boundary"
|
||||
"Sub-block partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary = boundaries.pop()
|
||||
if boundary == 0:
|
||||
return True
|
||||
if boundary // hash_block_size - 1 >= len(req_meta.block_hashes):
|
||||
return True
|
||||
mamba_offloads = {
|
||||
group_id: block_id for group_id, block_id, _ in partial_tail_offloads
|
||||
}
|
||||
hash_block_size = self.coord.hash_block_size
|
||||
if boundary == 0 or boundary // hash_block_size - 1 >= len(
|
||||
req_meta.block_hashes
|
||||
):
|
||||
return []
|
||||
|
||||
keys: list[str] = []
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
group_ids: list[str] | None = (
|
||||
[] if self.enable_group_semantics and self.supports_group_ids else None
|
||||
)
|
||||
mamba_offloads = {group_id: block_id for group_id, block_id, _ in entries}
|
||||
saved = self._saved_offset.get(req_meta.req_id, 0)
|
||||
puts: list[tuple[str, list[int], list[int], KeyMetadata]] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
group_blocks = req_meta.block_ids[g_idx]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
@@ -705,16 +733,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
continue
|
||||
valid_end = min((block_idx + 1) * db.block_size, boundary)
|
||||
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
|
||||
if (
|
||||
g_idx in mamba_offloads
|
||||
and valid_end == boundary
|
||||
and boundary % db.block_size != 0
|
||||
):
|
||||
block_id = mamba_offloads[g_idx]
|
||||
else:
|
||||
if block_idx >= len(group_blocks):
|
||||
if g_idx in mamba_offloads:
|
||||
if valid_end != boundary:
|
||||
# Interior align-mode state positions are null or
|
||||
# stale (the block table is not append-only) and never
|
||||
# valid gap content; only the boundary block is
|
||||
# persisted, from the core-provided hand-off.
|
||||
continue
|
||||
block_id = mamba_offloads[g_idx]
|
||||
elif g_idx in self.coord.mamba_group_ids:
|
||||
continue
|
||||
elif block_idx < len(group_blocks):
|
||||
block_id = group_blocks[block_idx]
|
||||
else:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
logger.debug(
|
||||
"Skipping unavailable partial-tail source block "
|
||||
@@ -725,20 +757,58 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
continue
|
||||
addr, size = db.prepare_value_for_block(block_id)
|
||||
key = db.key_for(key_hash)
|
||||
keys.append(key)
|
||||
addrs.append(addr)
|
||||
sizes.append(size)
|
||||
if group_ids is not None:
|
||||
group_ids.append(
|
||||
_make_mooncake_group_id(
|
||||
db.metadata,
|
||||
key.rsplit("@", 1)[-1],
|
||||
)
|
||||
)
|
||||
puts.append((db.key_for(key_hash), addr, size, db.metadata))
|
||||
return puts
|
||||
|
||||
if not keys:
|
||||
def _maybe_offload_boundary_states(self, req_meta: ReqMeta) -> bool:
|
||||
"""Persist connector-pinned mamba "align" boundary states handed off
|
||||
for this request, deduped against the store.
|
||||
|
||||
This is every mamba key the connector writes — ``store_mask`` excludes
|
||||
mamba groups from the positional normal save, aligned boundaries
|
||||
included (see :meth:`_boundary_snapshot_puts`).
|
||||
|
||||
The two entry kinds are keyed and sourced differently, so they are
|
||||
prepared separately and put in one batch:
|
||||
|
||||
- block-aligned for its group: a committed boundary-state snapshot,
|
||||
the handed-off block itself;
|
||||
- not block-aligned: the sub-block CoW partial tail, which also has to
|
||||
cover the other groups' blocks in the normal save's lcm gap.
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
"""
|
||||
offloads = req_meta.boundary_state_offloads
|
||||
if not offloads or not req_meta.block_hashes:
|
||||
return True
|
||||
|
||||
snapshots: list[tuple[int, int, int]] = []
|
||||
sub_block: list[tuple[int, int, int]] = []
|
||||
for group_id, block_id, boundary in offloads:
|
||||
entry = (group_id, block_id, boundary)
|
||||
if boundary % self.token_databases[group_id].block_size == 0:
|
||||
snapshots.append(entry)
|
||||
else:
|
||||
sub_block.append(entry)
|
||||
|
||||
puts = self._boundary_snapshot_puts(req_meta, snapshots)
|
||||
if sub_block and self.coord.enable_partial_hash_hits:
|
||||
puts.extend(self._sub_block_tail_puts(req_meta, sub_block))
|
||||
|
||||
if not puts:
|
||||
return True
|
||||
keys = [key for key, _, _, _ in puts]
|
||||
addrs = [addr for _, addr, _, _ in puts]
|
||||
sizes = [size for _, _, size, _ in puts]
|
||||
group_ids: list[str] | None = (
|
||||
[
|
||||
_make_mooncake_group_id(metadata, key.rsplit("@", 1)[-1])
|
||||
for key, _, _, metadata in puts
|
||||
]
|
||||
if self.enable_group_semantics and self.supports_group_ids
|
||||
else None
|
||||
)
|
||||
exists_start = time.perf_counter()
|
||||
try:
|
||||
exists = self.store.batch_is_exist(keys)
|
||||
@@ -751,7 +821,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error(
|
||||
"Failed to check partial-tail keys for request %s: %s",
|
||||
"Failed to check boundary-state keys for request %s: %s",
|
||||
req_meta.req_id,
|
||||
e,
|
||||
)
|
||||
@@ -787,7 +857,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error(
|
||||
"Failed to put partial-tail keys for request %s: %s",
|
||||
"Failed to put boundary-state keys for request %s: %s",
|
||||
req_meta.req_id,
|
||||
e,
|
||||
)
|
||||
@@ -805,7 +875,8 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if failed:
|
||||
failed_codes = {res[i] for i in failed}
|
||||
logger.warning(
|
||||
"Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)",
|
||||
"Boundary-state put failed for request %s: %d/%d keys failed "
|
||||
"(codes=%s)",
|
||||
req_meta.req_id,
|
||||
len(failed),
|
||||
len(keys),
|
||||
@@ -818,7 +889,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared after a "
|
||||
"successful partial-tail batch"
|
||||
"successful boundary-state batch"
|
||||
)
|
||||
return True
|
||||
|
||||
@@ -857,10 +928,10 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
return
|
||||
|
||||
# Offload the sub-block partial tail (independent of the normal
|
||||
# block-aligned save, which may be skipped this step).
|
||||
if req_meta.partial_tail_offloads is not None and not (
|
||||
self._maybe_offload_partial_tail(req_meta)
|
||||
# Offload the handed-off mamba boundary states (independent of the
|
||||
# normal positional save, which may be skipped this step).
|
||||
if req_meta.boundary_state_offloads is not None and not (
|
||||
self._maybe_offload_boundary_states(req_meta)
|
||||
):
|
||||
return
|
||||
|
||||
@@ -887,6 +958,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
# Rotate the stride phase per group to balance load across ranks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
group_blocks = block_ids_per_group[g_idx]
|
||||
for start, end, block_hash in db.process_tokens(
|
||||
token_len,
|
||||
req_meta.block_hashes,
|
||||
|
||||
@@ -1130,6 +1130,9 @@ class OffloadingConnectorScheduler:
|
||||
if bid != 0:
|
||||
self._current_batch_allocated_block_ids.add(bid)
|
||||
|
||||
for copy in scheduler_output.kv_cache_block_copies or ():
|
||||
self._current_batch_allocated_block_ids.add(copy.dst_block_id)
|
||||
|
||||
# Zero out stale block_ids in sliding window groups' pending-store
|
||||
# positions. Only sliding window groups can have stale entries (blocks
|
||||
# freed by remove_skipped_blocks then reallocated). Only positions in
|
||||
@@ -1151,18 +1154,89 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
group_state.block_ids[j] = 0
|
||||
|
||||
def _build_aligned_boundary_store_jobs(
|
||||
self, handoffs: dict[str, list[tuple[int, int, int]]]
|
||||
) -> dict[int, TransferJob]:
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
num_groups = len(self.config.kv_group_configs)
|
||||
for req_id, entries in handoffs.items():
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None:
|
||||
continue
|
||||
req = req_status.req
|
||||
max_boundary = self._calc_num_offloadable_tokens(req_status, req.num_tokens)
|
||||
for group_idx, block_id, boundary in entries:
|
||||
group_config = self.config.kv_group_configs[group_idx]
|
||||
if (
|
||||
block_id == 0
|
||||
or boundary > max_boundary
|
||||
or boundary % group_config.tokens_per_chunk != 0
|
||||
):
|
||||
continue
|
||||
|
||||
key = self._make_boundary_key(req, group_idx, boundary)
|
||||
store_output = self.manager.prepare_store([key], req_status.req_context)
|
||||
if store_output is None:
|
||||
self._connector_stats.increase_counter(
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
continue
|
||||
if not store_output.keys_to_store:
|
||||
continue
|
||||
|
||||
job_id = self._generate_job_id()
|
||||
req_status.transfer_jobs.add(job_id)
|
||||
self._block_id_to_pending_jobs.setdefault(block_id, set()).add(job_id)
|
||||
self._jobs[job_id] = TransferJobStatus(
|
||||
req_id=req_id,
|
||||
pending_count=self.config.num_workers,
|
||||
keys={key},
|
||||
is_store=True,
|
||||
fenced_block_ids=[block_id],
|
||||
)
|
||||
group_sizes = [0] * num_groups
|
||||
group_sizes[group_idx] = 1
|
||||
block_indices = [0] * num_groups
|
||||
block_indices[group_idx] = boundary // group_config.tokens_per_block - 1
|
||||
store_jobs[job_id] = TransferJob(
|
||||
req_id=req_id,
|
||||
src_spec=GPULoadStoreSpec(
|
||||
[block_id],
|
||||
group_sizes=group_sizes,
|
||||
block_indices=block_indices,
|
||||
),
|
||||
dst_spec=store_output.store_spec,
|
||||
)
|
||||
self._events_tracker.record_store(
|
||||
req,
|
||||
group_config,
|
||||
boundary // group_config.tokens_per_chunk - 1,
|
||||
key,
|
||||
)
|
||||
return store_jobs
|
||||
|
||||
def _build_partial_tail_store_jobs(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> dict[int, TransferJob]:
|
||||
handoffs = scheduler_output.partial_tail_offloads
|
||||
if not self.config.supports_partial_tail or not handoffs:
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
handoffs = block_state.boundary_state_offloads if block_state else None
|
||||
if not handoffs:
|
||||
return {}
|
||||
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
store_jobs = self._build_aligned_boundary_store_jobs(handoffs)
|
||||
if not self.config.supports_partial_tail:
|
||||
return store_jobs
|
||||
|
||||
for req_id, entries in handoffs.items():
|
||||
entries = [
|
||||
entry
|
||||
for entry in entries
|
||||
if entry[2] % self._partial_tail_block_size != 0
|
||||
]
|
||||
if not entries:
|
||||
continue
|
||||
req_status = self._req_status.get(req_id)
|
||||
assert req_status is not None
|
||||
assert entries
|
||||
boundaries = {boundary for _, _, boundary in entries}
|
||||
assert len(boundaries) == 1
|
||||
boundary = boundaries.pop()
|
||||
@@ -1282,6 +1356,9 @@ class OffloadingConnectorScheduler:
|
||||
group_config, group_state, num_offloadable_tokens
|
||||
)
|
||||
|
||||
if group_config.requires_cow_source:
|
||||
continue
|
||||
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
if num_chunks <= start_chunk_idx:
|
||||
continue
|
||||
|
||||
@@ -716,6 +716,10 @@ class BlockPool:
|
||||
if self.metrics_collector:
|
||||
self.metrics_collector.on_block_accessed(block)
|
||||
|
||||
def is_block_writable(self, block: KVCacheBlock) -> bool:
|
||||
"""Return whether a block can be mutated by its sole owner."""
|
||||
return not block.is_null and block.ref_cnt == 1 and block.block_hash is None
|
||||
|
||||
def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
|
||||
"""Free a list of blocks. The blocks should be ordered by their
|
||||
eviction priority, where the first block will be evicted first.
|
||||
|
||||
@@ -189,10 +189,6 @@ class KVCacheManager:
|
||||
tuple(() for _ in range(self.num_kv_cache_groups))
|
||||
)
|
||||
|
||||
# Off-table cow blocks handed to a KV connector for partial-tail
|
||||
# offload; pinned until the request's blocks are freed.
|
||||
self._partial_tail_pins: dict[str, list[KVCacheBlock]] = {}
|
||||
|
||||
@property
|
||||
def usage(self) -> float:
|
||||
"""Get the KV cache usage.
|
||||
@@ -575,9 +571,6 @@ class KVCacheManager:
|
||||
Args:
|
||||
request: The request to free the blocks.
|
||||
"""
|
||||
pins = self._partial_tail_pins.pop(request.request_id, None)
|
||||
if pins:
|
||||
self.block_pool.free_blocks(pins)
|
||||
self.coordinator.free(request.request_id)
|
||||
|
||||
def remove_skipped_blocks(
|
||||
@@ -610,14 +603,7 @@ class KVCacheManager:
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
"""
|
||||
blocks = self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
# Pins ride the same (possibly deferred) free as the request blocks.
|
||||
# Preemption may release a pin under a still-queued offload — the same
|
||||
# exposure normal saves of table blocks already have.
|
||||
pins = self._partial_tail_pins.pop(request.request_id, None)
|
||||
if pins:
|
||||
blocks = pins + blocks
|
||||
return blocks
|
||||
return self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
|
||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||
"""evict blocks from the prefix cache by their block IDs.
|
||||
@@ -854,18 +840,18 @@ class KVCacheManager:
|
||||
retained_blocks = [block for pair in pending_copies for block in pair]
|
||||
return copies, retained_blocks
|
||||
|
||||
def take_partial_tail_offloads(self) -> dict[str, list[tuple[int, int, int]]]:
|
||||
"""Drain producer partial-tail offload hand-offs per request.
|
||||
def take_boundary_state_offloads(
|
||||
self,
|
||||
) -> dict[str, list[tuple[int, int, int]]]:
|
||||
"""Drain this step's boundary-state hand-offs for a KV connector.
|
||||
|
||||
Returns ``{request_id: [(group_id, block_id, boundary_tokens), ...]}``
|
||||
for the durable boundary blocks of producers' last-prompt-boundary
|
||||
partial tails. Only mamba "align" groups contribute; empty otherwise.
|
||||
A KV connector reads the referenced blocks and offloads them so a later
|
||||
request can hit the sub-block prefix.
|
||||
|
||||
Each handed-off block lives off the request block table, so it is
|
||||
pinned here and unpinned when the request's blocks are freed — for a
|
||||
producer with saved tokens, after the connector reports sends done.
|
||||
for mamba "align" boundary states: the
|
||||
request's committed boundary-state snapshots and, on a sub-block partial
|
||||
hit, the CoW copy of its last-prompt-boundary state. Only mamba "align"
|
||||
groups contribute; empty otherwise. A connector reads the referenced
|
||||
blocks — never resolving them positionally — and offloads them so a
|
||||
later request can hit that prefix.
|
||||
"""
|
||||
offloads: dict[str, list[tuple[int, int, int]]] = {}
|
||||
for mgr in self.coordinator.single_type_managers:
|
||||
@@ -874,9 +860,7 @@ class KVCacheManager:
|
||||
group_id,
|
||||
block,
|
||||
boundary_tokens,
|
||||
) in mgr.take_pending_partial_tail_offloads():
|
||||
self.block_pool.touch((block,))
|
||||
self._partial_tail_pins.setdefault(req_id, []).append(block)
|
||||
) in mgr.take_pending_boundary_state_offloads():
|
||||
offloads.setdefault(req_id, []).append(
|
||||
(group_id, block.block_id, boundary_tokens)
|
||||
)
|
||||
|
||||
@@ -205,6 +205,16 @@ class ScheduledEncoderInputStats:
|
||||
output_tokens: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVConnectorBlockState:
|
||||
"""Scheduler-local block state offered to a producer-side KV connector."""
|
||||
|
||||
# Authoritative current block-table snapshots.
|
||||
block_ids: dict[str, tuple[list[int], ...]]
|
||||
# Exact Mamba "align" boundary-state hand-offs.
|
||||
boundary_state_offloads: dict[str, list[tuple[int, int, int]]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerOutput:
|
||||
# list of the requests that are scheduled for the first time.
|
||||
@@ -278,11 +288,8 @@ class SchedulerOutput:
|
||||
# CoW copies to apply after zeroing new blocks and before forward.
|
||||
kv_cache_block_copies: list[KVCacheBlockCopy] | None = None
|
||||
|
||||
# Producer partial-tail offload hand-off for external KV connectors:
|
||||
# {request_id: [(group_id, block_id, boundary_tokens), ...]} pointing at
|
||||
# the durable boundary block of a producer's last-prompt-boundary partial
|
||||
# tail (mamba "align" CoW target). None unless partial hash hits are active.
|
||||
partial_tail_offloads: dict[str, list[tuple[int, int, int]]] | None = None
|
||||
# Scheduler-local; always None by the time this reaches a worker.
|
||||
kv_connector_block_state: KVConnectorBlockState | None = None
|
||||
|
||||
# Dynamic speculative decoding: optimal K chosen by scheduler.
|
||||
# Number of spec tokens to schedule for the next step.
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
|
||||
from vllm.v1.core.sched.output import (
|
||||
CachedRequestData,
|
||||
GrammarOutput,
|
||||
KVConnectorBlockState,
|
||||
NewRequestData,
|
||||
ScheduledEncoderInputStats,
|
||||
SchedulerOutput,
|
||||
@@ -1263,20 +1264,32 @@ class Scheduler(SchedulerInterface):
|
||||
self.prev_step_scheduled_req_ids.clear()
|
||||
self.prev_step_scheduled_req_ids.update(num_scheduled_tokens.keys())
|
||||
|
||||
# Producer partial-tail hand-off for external KV connectors. Drained
|
||||
# before the CoW retentions are released below, so the pin lands while
|
||||
# the cow block still holds a retention ref. Without a producer-side
|
||||
# connector nothing consumes the hand-off, so skip the drain (and its
|
||||
# pin); the manager drops stale entries when the request's blocks are
|
||||
# popped for free.
|
||||
pending_partial_tail_offloads = None
|
||||
if (
|
||||
self.connector is not None
|
||||
and self.vllm_config.kv_transfer_config is not None
|
||||
and self.vllm_config.kv_transfer_config.is_kv_producer
|
||||
):
|
||||
pending_partial_tail_offloads = (
|
||||
self.kv_cache_manager.take_partial_tail_offloads() or None
|
||||
# Mamba "align" boundary states must be handed off with exact block ids;
|
||||
# they cannot be reconstructed from a connector's append-only block
|
||||
# table. Drained every step so stale offers cannot accumulate.
|
||||
boundary_state_offloads = self.kv_cache_manager.take_boundary_state_offloads()
|
||||
|
||||
kv_connector_block_state = None
|
||||
if self.connector is not None:
|
||||
snapshot_req_ids = {req.req_id for req in new_reqs_data}
|
||||
snapshot_req_ids.update(
|
||||
req_id
|
||||
for req_id, block_ids in zip(
|
||||
cached_reqs_data.req_ids,
|
||||
cached_reqs_data.new_block_ids,
|
||||
strict=True,
|
||||
)
|
||||
if block_ids
|
||||
)
|
||||
snapshot_req_ids.update(
|
||||
req_id for req_id in boundary_state_offloads if req_id in self.requests
|
||||
)
|
||||
kv_connector_block_state = KVConnectorBlockState(
|
||||
block_ids={
|
||||
req_id: self.kv_cache_manager.get_block_ids(req_id)
|
||||
for req_id in snapshot_req_ids
|
||||
},
|
||||
boundary_state_offloads=boundary_state_offloads,
|
||||
)
|
||||
|
||||
kv_cache_block_copies, cow_retained_blocks = (
|
||||
@@ -1325,7 +1338,7 @@ class Scheduler(SchedulerInterface):
|
||||
new_block_ids_to_zero=self._get_new_block_ids_to_zero(),
|
||||
has_sync_kv_loads=has_sync_kv_loads,
|
||||
kv_cache_block_copies=pending_kv_cache_block_copies,
|
||||
partial_tail_offloads=pending_partial_tail_offloads,
|
||||
kv_connector_block_state=kv_connector_block_state,
|
||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
||||
)
|
||||
@@ -1345,6 +1358,9 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
scheduler_output.ec_connector_metadata = ec_meta
|
||||
|
||||
# Connector-only block state must not be dispatched to workers.
|
||||
scheduler_output.kv_connector_block_state = None
|
||||
|
||||
# Advance the fence only for non-empty steps (those that actually
|
||||
# write KV and have their output processed later in update_from_output).
|
||||
if self.defer_block_free and total_num_scheduled_tokens > 0:
|
||||
|
||||
@@ -112,13 +112,14 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# managers (full attention, mamba "align"); harmlessly empty elsewhere.
|
||||
self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {}
|
||||
self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = []
|
||||
# Partial-tail offload hand-off for external KV connectors: when a
|
||||
# producer registers its last-prompt-boundary partial tail and the
|
||||
# durable boundary block is not on the append-only request block table
|
||||
# (mamba "align" CoW target), record the request, group, block, and
|
||||
# exact token boundary so a connector can offload it under the right
|
||||
# hash. Populated only by mamba "align".
|
||||
self._pending_partial_tail_offloads: list[
|
||||
# Boundary-state offload hand-off for external KV connectors. A mamba
|
||||
# "align" block table is not append-only (interior states are
|
||||
# nulled/freed and speculative blocks relocate in place), so a
|
||||
# connector cannot resolve its state blocks positionally. Record
|
||||
# (request, group, block, exact token boundary) for each committed
|
||||
# boundary state so a connector can offload the right block under the
|
||||
# right hash. Populated only by mamba "align".
|
||||
self._pending_boundary_state_offloads: list[
|
||||
tuple[str, int, KVCacheBlock, int]
|
||||
] = []
|
||||
|
||||
@@ -385,19 +386,19 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self._pending_cow_copies = []
|
||||
return pending_copies
|
||||
|
||||
def take_pending_partial_tail_offloads(
|
||||
def take_pending_boundary_state_offloads(
|
||||
self,
|
||||
) -> list[tuple[str, int, KVCacheBlock, int]]:
|
||||
"""Drain producer partial-tail hand-offs.
|
||||
"""Drain producer boundary-state hand-offs.
|
||||
|
||||
Entries are ``(req_id, group_id, block, boundary_tokens)``.
|
||||
|
||||
Only mamba "align" populates this. The block lives off the request
|
||||
block table, so the caller must pin it until the connector has read
|
||||
it — nothing else keeps it alive once the CoW retention is released.
|
||||
Only mamba "align" populates this. The blocks are not kept alive by
|
||||
the request block table for the whole request, so a caller that reads
|
||||
them asynchronously must pin them first.
|
||||
"""
|
||||
pending = self._pending_partial_tail_offloads
|
||||
self._pending_partial_tail_offloads = []
|
||||
pending = self._pending_boundary_state_offloads
|
||||
self._pending_boundary_state_offloads = []
|
||||
return pending
|
||||
|
||||
def _apply_cow(
|
||||
@@ -1292,7 +1293,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# Requests that registered their own last-prompt-boundary partial
|
||||
# tail (producers). On the next step's CoW the boundary state moves
|
||||
# into a private cow_block; we record that block for connector
|
||||
# offload (see _pending_partial_tail_offloads).
|
||||
# offload (see _pending_boundary_state_offloads).
|
||||
self._producer_partial_tail_reqs: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
@@ -1625,13 +1626,12 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
)
|
||||
|
||||
if blocks_allocated:
|
||||
# reuse previous speculative blocks in this step
|
||||
# Relocate exclusively owned speculative scratch blocks.
|
||||
for block_idx in range(
|
||||
prev_block_len - self.num_speculative_blocks, prev_block_len
|
||||
):
|
||||
if block_idx < num_skipped_blocks:
|
||||
req_blocks.append(req_blocks[block_idx])
|
||||
req_blocks[block_idx] = self._null_block
|
||||
self._relocate_speculative_block(req_blocks, block_idx)
|
||||
else:
|
||||
break
|
||||
num_new_blocks = num_required_blocks - len(req_blocks)
|
||||
@@ -1664,7 +1664,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# This CoW preserved a producer's own boundary
|
||||
# state in cow_block; hand it to the connector for
|
||||
# partial-tail offload once the copy has run.
|
||||
self._pending_partial_tail_offloads.append(
|
||||
self._pending_boundary_state_offloads.append(
|
||||
(
|
||||
request_id,
|
||||
self.kv_cache_group_id,
|
||||
@@ -1685,17 +1685,31 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
returned_blocks.extend(new_blocks)
|
||||
return returned_blocks
|
||||
|
||||
def _relocate_speculative_block(
|
||||
self, req_blocks: list[KVCacheBlock], block_idx: int
|
||||
) -> None:
|
||||
block = req_blocks[block_idx]
|
||||
assert self.block_pool.is_block_writable(block), (
|
||||
"Speculative Mamba blocks must be exclusively owned and unhashed "
|
||||
"before relocation"
|
||||
)
|
||||
req_blocks.append(block)
|
||||
req_blocks[block_idx] = self._null_block
|
||||
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
if self.mamba_cache_mode == "align":
|
||||
self._allocated_block_reqs.discard(request_id)
|
||||
self.last_state_block_idx.pop(request_id, None)
|
||||
self._num_checkpoint_blocks.pop(request_id, None)
|
||||
self._producer_partial_tail_reqs.pop(request_id, None)
|
||||
# A hand-off whose request died in this same scheduling pass must
|
||||
# not reach the connector: its unpin hook (free) has already run.
|
||||
self._pending_partial_tail_offloads = [
|
||||
# An offer is only guaranteed to hold committed bytes until the end
|
||||
# of the pass that made it. This request's blocks are going back to
|
||||
# the pool now, so drop its not-yet-offered hand-offs rather than
|
||||
# let a connector claim a block another request may already have
|
||||
# been handed.
|
||||
self._pending_boundary_state_offloads = [
|
||||
entry
|
||||
for entry in self._pending_partial_tail_offloads
|
||||
for entry in self._pending_boundary_state_offloads
|
||||
if entry[0] != request_id
|
||||
]
|
||||
return super().pop_blocks_for_free(request_id)
|
||||
@@ -1722,9 +1736,9 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
if partial_hash is not None:
|
||||
self.cached_blocks_this_step.add(partial_hash)
|
||||
if num_cached_blocks_after > num_cached_blocks_before:
|
||||
for block in self.req_to_blocks[request.request_id][
|
||||
num_cached_blocks_before:num_cached_blocks_after
|
||||
]:
|
||||
blocks = self.req_to_blocks[request.request_id]
|
||||
for idx in range(num_cached_blocks_before, num_cached_blocks_after):
|
||||
block = blocks[idx]
|
||||
# Skip null blocks (align-mode skipped states) and blocks that
|
||||
# were not cached this step — with sparse retention
|
||||
# (reachable_block_mask) the intermediate state snapshots carry
|
||||
@@ -1732,6 +1746,18 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
if block.is_null or block.block_hash is None:
|
||||
continue
|
||||
self.cached_blocks_this_step.add(block.block_hash)
|
||||
if self.mamba_cache_mode == "align":
|
||||
# Offer every retained boundary with its exact block.
|
||||
# The connector filters against its save window, which may
|
||||
# extend past the original prompt during resumed prefill.
|
||||
self._pending_boundary_state_offloads.append(
|
||||
(
|
||||
request.request_id,
|
||||
self.kv_cache_group_id,
|
||||
block,
|
||||
(idx + 1) * self.block_size,
|
||||
)
|
||||
)
|
||||
|
||||
def new_step_starts(self) -> None:
|
||||
self.cached_blocks_this_step.clear()
|
||||
|
||||
Reference in New Issue
Block a user