[Bugfix][Mooncake] Save exact Mamba boundary states (#51358)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Dao Le <daole@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5
OpenAI Codex
Dao Le
parent
16d6c376bc
commit
6b110badbb
@@ -69,6 +69,19 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
|
||||
manager.get_computed_blocks.assert_not_called()
|
||||
|
||||
|
||||
def drain_boundary_state_offloads(manager):
|
||||
"""Drain exact boundary-state block ids offered to a connector."""
|
||||
return manager.take_boundary_state_offloads()
|
||||
|
||||
|
||||
def _free_block_ids(manager):
|
||||
"""Block ids the pool would hand out to the next allocation."""
|
||||
return {
|
||||
block.block_id
|
||||
for block in manager.block_pool.free_block_queue.get_all_free_blocks()
|
||||
}
|
||||
|
||||
|
||||
def make_full_mamba_manager(
|
||||
*,
|
||||
dcp_world_size: int,
|
||||
@@ -618,10 +631,9 @@ def test_external_mamba_hit_same_block_uses_running_cow_on_continue():
|
||||
assert moved[0].block_id == cow_copy.dst_block_id
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_returns_cow_target():
|
||||
"""The connector offload hand-off exposes the mamba CoW *target* block Y
|
||||
(the durable boundary state), not the overwritten source X, and only at
|
||||
the CoW step."""
|
||||
def test_boundary_state_offloads_returns_cow_target():
|
||||
"""Boundary hand-offs expose aligned snapshots and the partial-tail CoW
|
||||
target, never the overwritten CoW source."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -660,8 +672,18 @@ def test_take_partial_tail_offloads_returns_cow_target():
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# Step A registered the partial tail but has not CoW'd yet: no offload.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
# Step A offers the materialized aligned checkpoint immediately.
|
||||
((group_id, block_id, boundary_tokens),) = drain_boundary_state_offloads(manager)[
|
||||
"0"
|
||||
]
|
||||
assert group_id == 1
|
||||
assert boundary_tokens == 4
|
||||
aligned_hash = req0.block_hashes[4 // hash_block_size - 1]
|
||||
aligned_block = manager.block_pool.get_cached_block(
|
||||
aligned_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert aligned_block is not None
|
||||
assert block_id == aligned_block[0].block_id
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
source_block = manager.block_pool.get_cached_block(
|
||||
@@ -675,34 +697,27 @@ def test_take_partial_tail_offloads_returns_cow_target():
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
assert list(offloads.keys()) == ["0"]
|
||||
assert len(offloads["0"]) == 1
|
||||
group_id, block_id, boundary_tokens = offloads["0"][0]
|
||||
assert group_id == 1 # the mamba group
|
||||
assert boundary_tokens == 6
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
cow_copy = next(c for c in copies if c.src_block_id == source_block_id)
|
||||
# The offload points at the durable CoW target Y, not the overwritten X.
|
||||
assert block_id == cow_copy.dst_block_id
|
||||
assert block_id != source_block_id
|
||||
# Draining clears it.
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
# The hand-off pinned Y (its CoW retention is released after this step,
|
||||
# and Y is off the request block table); freeing the request unpins it.
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
pinned_ref = cow_block.ref_cnt
|
||||
assert pinned_ref >= 1
|
||||
manager.block_pool.free_blocks(retained)
|
||||
manager.free(req0)
|
||||
assert cow_block.ref_cnt == pinned_ref - 1
|
||||
|
||||
|
||||
def test_partial_tail_pin_survives_released_cow_retention():
|
||||
"""If the CoW retention is released before the hand-off is drained
|
||||
(immediate-free mode), the drain must rescue the cow block from the free
|
||||
queue: a raw ref increment would leave a ref>0 block allocatable, and the
|
||||
next allocation would pop it and assert."""
|
||||
def test_block_pool_touch_pins_released_cow_target():
|
||||
"""The connector can rescue an offered CoW target after its step-scoped
|
||||
retention is released by using the bound BlockPool's touch method."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -746,13 +761,18 @@ def test_partial_tail_pin_survives_released_cow_retention():
|
||||
_copies, retained = manager.take_kv_cache_block_copies()
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
offloads = manager.take_partial_tail_offloads()
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((_group_id, block_id, boundary_tokens),) = offloads["0"]
|
||||
assert boundary_tokens == 6
|
||||
cow_block = manager.block_pool.blocks[block_id]
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert cow_block.ref_cnt == 0
|
||||
assert block_id in _free_block_ids(manager)
|
||||
|
||||
# The pinned block is out of the free queue: draining every free block
|
||||
manager.block_pool.touch([cow_block])
|
||||
assert cow_block.ref_cnt == 1
|
||||
assert block_id not in _free_block_ids(manager)
|
||||
|
||||
# The connector-pinned block is out of the free queue: draining every free block
|
||||
# neither trips the allocator's ref_cnt assert nor hands it out.
|
||||
new_blocks = manager.block_pool.get_new_blocks(
|
||||
manager.block_pool.get_num_free_blocks()
|
||||
@@ -760,7 +780,7 @@ def test_partial_tail_pin_survives_released_cow_retention():
|
||||
assert block_id not in {b.block_id for b in new_blocks}
|
||||
|
||||
|
||||
def test_partial_tail_offload_dropped_when_request_freed_before_drain():
|
||||
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
|
||||
"""A hand-off recorded in the same scheduling pass as the request's death
|
||||
must not be drained: its release hook has already run, so draining would
|
||||
leak a pinned block."""
|
||||
@@ -805,12 +825,12 @@ def test_partial_tail_offload_dropped_when_request_freed_before_drain():
|
||||
|
||||
# The request dies (preempt/abort) before the scheduler drains.
|
||||
manager.block_pool.free_blocks(manager.pop_blocks_for_free(req0))
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
|
||||
def test_take_partial_tail_offloads_empty_without_partial_tail():
|
||||
"""A prompt ending on a block boundary registers no partial tail, so there
|
||||
is nothing to offload."""
|
||||
def test_boundary_state_offloads_block_aligned_prompt():
|
||||
"""A prompt ending on a block boundary registers no CoW partial tail; its
|
||||
boundary state block is handed off as a snapshot instead (once)."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
@@ -848,12 +868,17 @@ def test_take_partial_tail_offloads_empty_without_partial_tail():
|
||||
req0 = make_request("0", [0, 0, 1, 1], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == block_size
|
||||
assert block_id == mamba_blocks[0].block_id
|
||||
|
||||
req0.num_computed_tokens = 4
|
||||
req0.append_output_token_ids([2])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
assert manager.take_partial_tail_offloads() == {}
|
||||
# The boundary was already handed off; decoding emits nothing new.
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
|
||||
def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
@@ -1786,3 +1811,213 @@ def test_dcp_partial_hit_with_eagle_rewinds_one_hash_unit():
|
||||
assert num_computed == 4
|
||||
assert [len(group) for group in computed_blocks.blocks] == [1, 1]
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
|
||||
|
||||
def _snapshot_offload_kv_cache_config(
|
||||
hash_block_size: int, block_size: int, num_blocks: int = 24
|
||||
):
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_retention_snapshots_handed_off_with_exact_block_ids():
|
||||
"""Under sparse retention, each retained mamba boundary state block is
|
||||
handed to the connector with its exact block id. Connectors must not resolve
|
||||
align-mode state blocks positionally because the block table is not
|
||||
append-only."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
manager.coordinator.retention_interval = 2 * block_size
|
||||
|
||||
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((group_id, block_id, boundary),) = offloads["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == 2 * block_size
|
||||
assert block_id == mamba_blocks[1].block_id
|
||||
|
||||
req0.num_computed_tokens = 8
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 8) is not None
|
||||
|
||||
offloads = drain_boundary_state_offloads(manager)
|
||||
((group_id, block_id2, boundary2),) = offloads["0"]
|
||||
assert group_id == 1
|
||||
assert boundary2 == 4 * block_size
|
||||
assert block_id2 == mamba_blocks[3].block_id
|
||||
# Interior align-mode positions stay null and are never handed off.
|
||||
assert mamba_blocks[0].is_null and mamba_blocks[2].is_null
|
||||
|
||||
manager.free(req0)
|
||||
|
||||
|
||||
def test_snapshot_handoff_dense_default_retention():
|
||||
"""With dense (default) retention, every materialized mamba boundary
|
||||
state block is handed off — regular mamba-align + prefix-match-unit
|
||||
deployments get store-able boundary snapshots without setting
|
||||
VLLM_PREFIX_CACHE_RETENTION_INTERVAL."""
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(hash_block_size, block_size),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
assert manager.coordinator.retention_interval is None
|
||||
|
||||
req0 = make_request("0", [0] * 16, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
|
||||
mamba_blocks = manager.coordinator.single_type_managers[1].req_to_blocks["0"]
|
||||
((group_id, block_id, boundary),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary == 2 * block_size
|
||||
assert block_id == mamba_blocks[1].block_id
|
||||
|
||||
req0.num_computed_tokens = 8
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 8) is not None
|
||||
((group_id, block_id2, boundary2),) = drain_boundary_state_offloads(manager)["0"]
|
||||
assert group_id == 1
|
||||
assert boundary2 == 4 * block_size
|
||||
assert block_id2 == mamba_blocks[3].block_id
|
||||
|
||||
|
||||
def _run_chunked_prefill(manager, req, chunk_size, num_chunks):
|
||||
"""Prefill ``req`` one ``chunk_size`` chunk per scheduler step, yielding the
|
||||
boundary-state hand-offs offered (and claimed) at each step."""
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
for step in range(num_chunks):
|
||||
manager.new_step_starts()
|
||||
req.num_computed_tokens = step * chunk_size
|
||||
if step == 0:
|
||||
allocated = manager.allocate_slots(
|
||||
req, chunk_size, num_computed, computed_blocks
|
||||
)
|
||||
else:
|
||||
allocated = manager.allocate_slots(req, chunk_size)
|
||||
assert allocated is not None
|
||||
yield drain_boundary_state_offloads(manager).get(req.request_id, [])
|
||||
|
||||
|
||||
def test_boundary_state_offer_includes_more_mamba_groups_than_two():
|
||||
"""One offer batch carries one exact block entry per mamba group."""
|
||||
block_size = 8
|
||||
num_mamba_groups = 3
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=KVCacheConfig(
|
||||
num_blocks=200,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
*(
|
||||
KVCacheGroupSpec(
|
||||
[f"mamba{i}"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
)
|
||||
for i in range(num_mamba_groups)
|
||||
),
|
||||
],
|
||||
),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0] * (2 * block_size), block_size, sha256)
|
||||
entries = next(_run_chunked_prefill(manager, req0, block_size, 1))
|
||||
|
||||
assert [group_id for group_id, _, _ in entries] == [1, 2, 3]
|
||||
assert {boundary for _, _, boundary in entries} == {block_size}
|
||||
|
||||
|
||||
def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
"""The core offers every committed boundary, including past
|
||||
``num_prompt_tokens``. A resumed request re-prefills its generated tokens
|
||||
and every group re-saves them, so filtering on the original prompt length
|
||||
would silently strip the mamba key for boundaries full attention still
|
||||
stores; only the connector knows where its save window ends."""
|
||||
block_size = 8
|
||||
prompt_len = block_size
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=_snapshot_offload_kv_cache_config(
|
||||
hash_block_size=block_size, block_size=block_size, num_blocks=200
|
||||
),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0] * prompt_len, block_size, sha256)
|
||||
assert [
|
||||
boundary
|
||||
for entries in _run_chunked_prefill(manager, req0, block_size, 1)
|
||||
for _, _, boundary in entries
|
||||
] == [prompt_len]
|
||||
|
||||
# Generate past the next mamba boundary, then replay it as a resumed
|
||||
# prefill: its committed boundary is offered even though it is past the
|
||||
# prompt, matching what full attention saves for the same range.
|
||||
for i in range(block_size):
|
||||
manager.new_step_starts()
|
||||
req0.num_computed_tokens = prompt_len + i
|
||||
req0.append_output_token_ids([1])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
drain_boundary_state_offloads(manager)
|
||||
assert req0.num_tokens == 2 * block_size
|
||||
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
req0.num_computed_tokens = 0
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, req0.num_tokens - num_computed, num_computed)
|
||||
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
|
||||
assert 2 * block_size in offered
|
||||
assert req0.num_prompt_tokens < 2 * block_size
|
||||
|
||||
@@ -15,12 +15,14 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
FullAttentionManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
@@ -69,6 +71,36 @@ def test_external_computed_blocks_do_not_corrupt_free_pool():
|
||||
assert len(manager.req_to_blocks[request_id]) == 3
|
||||
|
||||
|
||||
def test_mamba_speculative_block_relocation_requires_exclusive_ownership():
|
||||
spec = MambaSpec(
|
||||
block_size=4,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=1,
|
||||
)
|
||||
block_pool = BlockPool(num_gpu_blocks=4, enable_caching=True, hash_block_size=4)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=block_pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=4,
|
||||
)
|
||||
block = block_pool.get_new_blocks(1)[0]
|
||||
blocks = [block]
|
||||
|
||||
manager._relocate_speculative_block(blocks, 0)
|
||||
|
||||
assert blocks == [block_pool.null_block, block]
|
||||
assert block.ref_cnt == 1
|
||||
|
||||
pinned_block = block_pool.get_new_blocks(1)[0]
|
||||
block_pool.touch((pinned_block,))
|
||||
with pytest.raises(AssertionError, match="exclusively owned and unhashed"):
|
||||
manager._relocate_speculative_block([pinned_block], 0)
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -36,7 +36,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
is_store_reachable_swa_chunk,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock, KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import (
|
||||
KVConnectorBlockState,
|
||||
SchedulerOutput,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -107,11 +111,16 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11, 12]
|
||||
req_status.group_states[1].block_ids[:] = [0, 21]
|
||||
scheduler.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(partial_tail_offloads={"req": [(1, 99, 28)]})
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
@@ -155,6 +164,83 @@ def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
assert recurrent_event.parent_block_hash is None
|
||||
|
||||
|
||||
def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11, 12]
|
||||
req_status.group_states[1].block_ids[:] = [0, 21]
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(
|
||||
kv_connector_block_state=KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 98, 16), (1, 99, 28)]},
|
||||
)
|
||||
)
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 2
|
||||
src_spec = next(
|
||||
job.src_spec for job in jobs.values() if len(job.src_spec.block_ids) == 1
|
||||
)
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
assert src_spec.block_ids.tolist() == [98]
|
||||
assert src_spec.group_sizes == [0, 1]
|
||||
assert src_spec.block_indices == [0, 0]
|
||||
|
||||
|
||||
def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_connector_block_state = KVConnectorBlockState(
|
||||
block_ids={},
|
||||
boundary_state_offloads={"req": [(1, 99, 16)]},
|
||||
)
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
[job_id] = meta.store_jobs
|
||||
|
||||
output = SchedulerOutput.make_empty()
|
||||
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
|
||||
meta = scheduler.build_connector_meta(output)
|
||||
|
||||
assert meta.jobs_to_flush == {job_id}
|
||||
|
||||
|
||||
def test_normal_store_excludes_align_mode_mamba_sources():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
request.num_computed_tokens = 0
|
||||
request.status = RequestStatus.RUNNING
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11]
|
||||
req_status.group_states[1].block_ids[:] = [99]
|
||||
req_status.update_offload_keys()
|
||||
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(
|
||||
num_scheduled_tokens={"req": 16},
|
||||
finished_req_ids=set(),
|
||||
)
|
||||
jobs = scheduler._build_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
[job] = jobs.values()
|
||||
src_spec = job.src_spec
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
assert src_spec.block_ids.tolist() == [11]
|
||||
assert src_spec.group_sizes == [1, 0]
|
||||
|
||||
|
||||
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
|
||||
@@ -5,6 +5,9 @@ import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -31,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
@@ -62,6 +66,30 @@ def _make_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_requires_align_mode_for_mamba():
|
||||
vllm_config = _make_vllm_config()
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
),
|
||||
pytest.raises(AssertionError, match="requires mamba_cache_mode='align'"),
|
||||
):
|
||||
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
def _make_block_stored() -> BlockStored:
|
||||
return BlockStored(
|
||||
block_hashes=[b"hash"],
|
||||
@@ -97,6 +125,10 @@ def test_scheduler_role_initializes_store_scheduler_only():
|
||||
mock_worker.assert_not_called()
|
||||
assert connector.connector_scheduler is mock_scheduler.return_value
|
||||
assert connector.connector_worker is None
|
||||
block_pool = MagicMock()
|
||||
|
||||
connector.bind_gpu_block_pool(block_pool)
|
||||
mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool)
|
||||
|
||||
|
||||
def test_worker_methods_delegate_to_store_worker():
|
||||
@@ -241,9 +273,13 @@ def test_update_connector_output_and_take_events():
|
||||
|
||||
kv_events = mooncake_store_connector.MooncakeStoreKVEvents(num_workers=1)
|
||||
kv_events.add_events([event])
|
||||
connector.update_connector_output(KVConnectorOutput(kv_cache_events=kv_events))
|
||||
output = KVConnectorOutput(kv_cache_events=kv_events)
|
||||
connector.update_connector_output(output)
|
||||
|
||||
assert connector._kv_cache_events is kv_events
|
||||
connector.connector_scheduler.update_connector_output.assert_called_once_with(
|
||||
output
|
||||
)
|
||||
assert list(connector.take_events()) == [event]
|
||||
assert connector._kv_cache_events is None
|
||||
|
||||
|
||||
@@ -451,6 +451,27 @@ def test_store_mask_retention_prefix_stable_as_aligned_length_grows():
|
||||
assert longer[: len(shorter)] == shorter
|
||||
|
||||
|
||||
def test_store_mask_excludes_mamba_groups_lookup_unaffected():
|
||||
"""Align-mode mamba block tables are not append-only (interior states are
|
||||
nulled/freed; speculative blocks relocate), so the positional normal save
|
||||
must never cover mamba chunks — regardless of retention. Lookups still
|
||||
probe mamba boundaries: their keys come from the pinned snapshot/CoW
|
||||
hand-off path instead."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["L0"], _full(16)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(16)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks[0] is None # full attn stays dense
|
||||
assert masks[1] == [False] * 4
|
||||
|
||||
coord = _make_coord(groups, hash_block_size=16, retention_interval=32)
|
||||
assert coord.store_mask(64)[1] == [False] * 4
|
||||
|
||||
assert coord.lookup_mask(64)[1] is None
|
||||
|
||||
|
||||
# ----- Eagle / MTP interaction with load_mask -----
|
||||
|
||||
|
||||
|
||||
@@ -236,6 +236,7 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
)
|
||||
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
|
||||
# 1-based: block 0 is the reserved null block and the save skips it.
|
||||
save_req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
@@ -441,10 +442,10 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
send._maybe_offload_boundary_states(req)
|
||||
|
||||
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
|
||||
partial_hash = hs[2]
|
||||
@@ -512,8 +513,8 @@ def test_offload_syncs_event_before_put():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
store_job_id=1,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
req.current_event = event
|
||||
|
||||
@@ -589,10 +590,10 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, mamba_cow_block, 12)],
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_partial_tail(req)
|
||||
send._maybe_offload_boundary_states(req)
|
||||
|
||||
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
|
||||
# block-end hashes; mamba (block 16): the partial boundary block under
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.sched.output import KVConnectorBlockState
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
@@ -47,9 +48,20 @@ def _make_bare_scheduler(
|
||||
scheduler._num_workers = 1
|
||||
scheduler._next_store_job_id = 0
|
||||
scheduler._pinned_saves = {}
|
||||
scheduler._boundary_state_group_ids = frozenset({1})
|
||||
return scheduler
|
||||
|
||||
|
||||
def _make_connector_block_state(
|
||||
block_ids: tuple[list[int], ...] | None = None,
|
||||
offloads: list[tuple[int, int, int]] | None = None,
|
||||
) -> KVConnectorBlockState:
|
||||
return KVConnectorBlockState(
|
||||
block_ids={} if block_ids is None else {"req-0": block_ids},
|
||||
boundary_state_offloads=({} if offloads is None else {"req-0": offloads}),
|
||||
)
|
||||
|
||||
|
||||
def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
@@ -65,6 +77,7 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
|
||||
scheduled_spec_decode_tokens=(
|
||||
{"req-0": scheduled_spec_tokens} if scheduled_spec_tokens else {}
|
||||
),
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -84,6 +97,7 @@ def _make_decode_scheduler_output(
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -109,6 +123,7 @@ def _make_new_scheduler_output() -> SimpleNamespace:
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 32},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -491,6 +506,7 @@ def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamesp
|
||||
),
|
||||
num_scheduled_tokens={"req-0": num_scheduled_tokens},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(block_ids=([0, 1, 2],)),
|
||||
)
|
||||
|
||||
|
||||
@@ -915,7 +931,10 @@ def _add_pending_partial_tail_request(
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=block_ids,
|
||||
offloads=[(1, 7, 12)],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -940,15 +959,262 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.can_save is True
|
||||
assert req_meta.token_len_chunk == 0
|
||||
assert req_meta.partial_tail_offloads == [(1, 7, 12)]
|
||||
assert req_meta.boundary_state_offloads == [(1, 7, 12)]
|
||||
assert req_meta.num_prompt_tokens == 12
|
||||
assert req_meta.block_ids == ([0],)
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7]
|
||||
assert scheduler._gpu_block_pool.blocks[0].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
def test_decode_boundary_state_offload_dropped_unclaimed():
|
||||
# A hand-off past the prefill end can never complete a joint hybrid hit
|
||||
# (every other group stops saving there), so it is neither transferred nor
|
||||
# claimed — leaving the core free to release the block immediately.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(24)),
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4", b"h5"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([0],),
|
||||
num_saved_tokens=12,
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
# Boundary 16 is a decode boundary for a 12-token prefill.
|
||||
kv_connector_block_state=_make_connector_block_state(offloads=[(1, 7, 16)]),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert meta.requests == []
|
||||
assert scheduler._pinned_saves == {}
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._request_trackers["req-0"].has_pending_offload is False
|
||||
|
||||
|
||||
def _register_offload_request(scheduler, *, prefill_end_tokens, num_prompt_tokens):
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(64)),
|
||||
block_hashes=[bytes([i]) for i in range(16)],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=prefill_end_tokens,
|
||||
allocated_block_ids=([0],),
|
||||
num_saved_tokens=prefill_end_tokens,
|
||||
token_ids=list(range(prefill_end_tokens)),
|
||||
prefill_end_tokens=prefill_end_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _make_offload_only_output(entries, block_ids=([0],)):
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=block_ids,
|
||||
offloads=entries,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_resumed_prefill_claims_boundaries_past_prompt_length():
|
||||
# A resumed request re-prefills its previously generated tokens, so its
|
||||
# save window (`prefill_end_tokens`) extends past `num_prompt_tokens`.
|
||||
# Boundaries in that range must still be claimed, or the mamba key would be
|
||||
# missing for boundaries full attention does store and no joint hit could
|
||||
# complete there.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=20, num_prompt_tokens=12)
|
||||
out = _make_offload_only_output([(1, 7, 16), (1, 9, 20), (1, 11, 24)])
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
# 16 and 20 are inside the resumed prefill; 24 is past it.
|
||||
assert meta.requests[0].boundary_state_offloads == [(1, 7, 16), (1, 9, 20)]
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 9]
|
||||
|
||||
|
||||
def test_boundary_state_job_pins_exact_blocks_once():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
scheduler._request_trackers["req-0"].allocated_block_ids = ([7],)
|
||||
out = _make_offload_only_output(
|
||||
[(1, 7, 16), (1, 8, 32), (1, 9, 48)], block_ids=([7],)
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.boundary_state_offloads == [
|
||||
(1, 7, 16),
|
||||
(1, 8, 32),
|
||||
(1, 9, 48),
|
||||
]
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 8, 9]
|
||||
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert [scheduler._gpu_block_pool.blocks[i].ref_cnt for i in (7, 8, 9)] == [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
]
|
||||
|
||||
|
||||
def test_store_job_pins_current_non_null_non_mamba_blocks():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[bytes([i]) for i in range(12)],
|
||||
num_output_placeholders=0,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([7, 2, 0], [21, 0, 22]))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=44,
|
||||
allocated_block_ids=([7, 2, 0], [21, 0, 22]),
|
||||
num_saved_tokens=32,
|
||||
token_ids=list(range(44)),
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
stale_block_ids = ([7, 2, 0, 5], [21, 0, 22, 23])
|
||||
current_block_ids = ([7, 2, 0, 6], [24, 0, 25, 26])
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=["req-0"],
|
||||
new_block_ids=[([5], [23])],
|
||||
num_computed_tokens=[44],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={"req-0": 4},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(
|
||||
block_ids=current_block_ids,
|
||||
offloads=[(1, 8, 16)],
|
||||
),
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert scheduler._request_trackers["req-0"].allocated_block_ids == stale_block_ids
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.block_ids == current_block_ids
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [8, 7, 2, 6]
|
||||
assert pool.blocks[0].ref_cnt == 0
|
||||
assert pool.blocks[5].ref_cnt == 0
|
||||
assert [pool.blocks[i].ref_cnt for i in (21, 22, 23, 24, 25, 26)] == [
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
]
|
||||
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [1, 1, 1, 1]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
|
||||
assert [pool.blocks[i].ref_cnt for i in (8, 7, 2, 6)] == [0, 0, 0, 0]
|
||||
|
||||
|
||||
def test_boundary_state_release_is_per_store_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
|
||||
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
|
||||
first_job_id = first.requests[0].store_job_id
|
||||
second_job_id = second.requests[0].store_job_id
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
|
||||
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
|
||||
assert first_job_id not in scheduler._pinned_saves
|
||||
assert second_job_id in scheduler._pinned_saves
|
||||
|
||||
|
||||
def test_preemption_and_request_id_reuse_do_not_release_inflight_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
first = scheduler.build_connector_meta(_make_offload_only_output([(1, 7, 16)]))
|
||||
first_job_id = first.requests[0].store_job_id
|
||||
|
||||
scheduler.build_connector_meta(_make_preemption_scheduler_output())
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 1
|
||||
assert first_job_id in scheduler._pinned_saves
|
||||
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
second = scheduler.build_connector_meta(_make_offload_only_output([(1, 8, 32)]))
|
||||
second_job_id = second.requests[0].store_job_id
|
||||
assert second.requests[0].boundary_state_offloads == [(1, 8, 32)]
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({first_job_id: 1}))
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
assert scheduler._gpu_block_pool.blocks[8].ref_cnt == 1
|
||||
assert second_job_id in scheduler._pinned_saves
|
||||
|
||||
|
||||
def test_boundary_state_never_claimed_without_a_send_thread():
|
||||
# A kv_consumer has no store job that can acknowledge the block reference.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.kv_role = "kv_consumer"
|
||||
_register_offload_request(scheduler, prefill_end_tokens=64, num_prompt_tokens=64)
|
||||
out = _make_offload_only_output([(1, 7, 16)])
|
||||
|
||||
assert scheduler.build_connector_meta(out).requests == []
|
||||
assert scheduler._pinned_saves == {}
|
||||
assert scheduler._gpu_block_pool.blocks[7].ref_cnt == 0
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_exact_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
out = _add_pending_partial_tail_request(
|
||||
@@ -961,7 +1227,7 @@ def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].partial_tail_offloads == [(1, 7, 12)]
|
||||
assert meta.requests[0].boundary_state_offloads == [(1, 7, 12)]
|
||||
# Ordinary metadata retains the full resumed prefill range.
|
||||
assert meta.requests[0].num_prompt_tokens == 20
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
@@ -969,8 +1235,9 @@ def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
def test_resumed_partial_tail_attached_to_save_keeps_exact_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler._boundary_state_group_ids = frozenset({0})
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
@@ -987,13 +1254,13 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
out = _make_scheduler_output(scheduled_spec_tokens=None)
|
||||
out.partial_tail_offloads = {"req-0": [(0, 7, 36)]}
|
||||
out.kv_connector_block_state.boundary_state_offloads = {"req-0": [(0, 7, 36)]}
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
assert meta.requests[0].can_save is True
|
||||
assert meta.requests[0].partial_tail_offloads == [(0, 7, 36)]
|
||||
assert meta.requests[0].boundary_state_offloads == [(0, 7, 36)]
|
||||
assert meta.requests[0].num_prompt_tokens == 48
|
||||
# Ordinary saving still covers the full resumed prefill range.
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
@@ -1019,7 +1286,8 @@ def test_partial_tail_cow_block_is_referenced_for_the_job():
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
|
||||
# puts it last in eviction priority.
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7]
|
||||
assert pool.blocks[0].ref_cnt == 0
|
||||
assert pool.blocks[7].ref_cnt == 1
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
|
||||
@@ -181,6 +181,7 @@ def _run_store_req(thread, req_meta: ReqMeta) -> None:
|
||||
|
||||
|
||||
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
# 1-based: block 0 is the reserved null block and the save skips it.
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=32,
|
||||
@@ -651,6 +652,7 @@ def _make_partial_tail_send_thread(
|
||||
enable_partial_hash_hits=True,
|
||||
hash_block_size=4,
|
||||
lcm_block_size=16,
|
||||
mamba_group_ids={1},
|
||||
)
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0),
|
||||
@@ -659,10 +661,18 @@ def _make_partial_tail_send_thread(
|
||||
)
|
||||
db.set_kv_caches_base_addr([0x1000])
|
||||
db.set_block_len([256])
|
||||
# Group 1 models the mamba "align" group the hand-offs reference.
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([256])
|
||||
return _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db],
|
||||
token_databases=[db, db_mamba],
|
||||
replicate_config=replicate_config,
|
||||
enable_group_semantics=enable_group_semantics,
|
||||
supports_group_ids=supports_group_ids,
|
||||
@@ -673,20 +683,20 @@ def _make_partial_tail_req(block_ids: list[int]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=(block_ids,),
|
||||
block_ids=(block_ids, [1]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
|
||||
|
||||
def test_partial_tail_offload_skips_null_source_blocks():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([0, 2, 3]))
|
||||
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([0, 2, 3]))
|
||||
|
||||
keys, addrs, _sizes, _replicate_config = (
|
||||
store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -694,15 +704,21 @@ def test_partial_tail_offload_skips_null_source_blocks():
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
|
||||
]
|
||||
# FA reads blocks 2 and 3 (block 0 is null and skipped); the mamba
|
||||
# boundary block reads the core-provided CoW block 7, not block_ids.
|
||||
assert addrs == [
|
||||
[0x1000 + 2 * 256],
|
||||
[0x1000 + 3 * 256],
|
||||
[0x2000 + 7 * 256],
|
||||
]
|
||||
assert addrs == [[0x1000 + 2 * 256], [0x1000 + 3 * 256]]
|
||||
|
||||
|
||||
def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
@@ -711,16 +727,11 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
lambda keys, addrs, sizes, replicate_config: [256] * len(keys)
|
||||
)
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
sparse = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["mamba"], mamba),
|
||||
KVCacheGroupSpec(["sparse"], sparse),
|
||||
],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
@@ -732,17 +743,17 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
db_sparse = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=16,
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([512])
|
||||
db_sparse.set_kv_caches_base_addr([0x2000])
|
||||
db_sparse.set_block_len([512])
|
||||
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_mamba],
|
||||
token_databases=[db_full, db_sparse],
|
||||
)
|
||||
_run_store_req(
|
||||
thread,
|
||||
@@ -756,15 +767,86 @@ def test_store_sending_thread_skips_null_sparse_group_blocks():
|
||||
)
|
||||
|
||||
keys, addrs, _, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
mamba_keys = [key for key in keys if "@group:1" in key]
|
||||
assert [key.rsplit("@", 1)[-1] for key in mamba_keys] == [b"a1".hex()]
|
||||
sparse_keys = [key for key in keys if "@group:1" in key]
|
||||
assert [key.rsplit("@", 1)[-1] for key in sparse_keys] == [b"a1".hex()]
|
||||
assert all(addr[0] >= 0x2000 for key, addr in zip(keys, addrs) if "@group:1" in key)
|
||||
|
||||
|
||||
def test_partial_tail_offload_skips_cap_omitted_mamba_group():
|
||||
"""A Mamba group omitted by the handoff cap must not fall back to the
|
||||
connector's positional block table."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=4, num_kv_heads=8, head_size=64, dtype=torch.float32
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[
|
||||
KVCacheGroupSpec(["full"], full_spec),
|
||||
KVCacheGroupSpec(["mamba-1"], mamba_spec),
|
||||
KVCacheGroupSpec(["mamba-2"], mamba_spec),
|
||||
],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
)
|
||||
|
||||
def make_db(group_id: int, block_size: int, base_addr: int):
|
||||
db = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=group_id),
|
||||
block_size=block_size,
|
||||
hash_block_size=4,
|
||||
)
|
||||
db.set_kv_caches_base_addr([base_addr])
|
||||
db.set_block_len([256])
|
||||
return db
|
||||
|
||||
db_full = make_db(0, 4, 0x1000)
|
||||
db_mamba_1 = make_db(1, 16, 0x2000)
|
||||
db_mamba_2 = make_db(2, 16, 0x3000)
|
||||
thread = _make_store_sending_thread(
|
||||
store,
|
||||
coord=coord,
|
||||
token_databases=[db_full, db_mamba_1, db_mamba_2],
|
||||
)
|
||||
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5], [9]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
# The scheduler accepted group 1 and omitted group 2 at boundary 12.
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _replicate_config = (
|
||||
store.batch_put_from_multi_buffers.call_args.args
|
||||
)
|
||||
puts = list(zip(keys, addrs, strict=True))
|
||||
boundary_hash = BlockHash(b"a2")
|
||||
assert (db_mamba_1.key_for(boundary_hash), [0x2000 + 7 * 256]) in puts
|
||||
assert (db_mamba_2.key_for(boundary_hash), [0x3000 + 9 * 256]) not in puts
|
||||
|
||||
|
||||
def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [1, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
store.batch_is_exist.return_value = [1, 0, 0, 0]
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256, 256]
|
||||
replicate_config = SimpleNamespace(group_ids=["stale"])
|
||||
thread = _make_partial_tail_send_thread(
|
||||
store,
|
||||
@@ -773,17 +855,19 @@ def test_partial_tail_offload_replaces_stale_group_ids_after_filtering():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
assert thread._maybe_offload_partial_tail(_make_partial_tail_req([1, 2, 3]))
|
||||
assert thread._maybe_offload_boundary_states(_make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6132",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6132",
|
||||
]
|
||||
assert config is replicate_config
|
||||
assert config.group_ids == [
|
||||
"vllm-mooncake-store:test-model@6131",
|
||||
"vllm-mooncake-store:test-model@6132",
|
||||
"vllm-mooncake-store:test-model@6132",
|
||||
]
|
||||
|
||||
|
||||
@@ -815,6 +899,165 @@ def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
def test_normal_save_excludes_mamba_group_and_null_blocks():
|
||||
"""The positional normal save must never cover mamba chunks (align-mode
|
||||
block tables are not append-only) and must skip chunks whose block is the
|
||||
reserved null block, for any group."""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
|
||||
full_spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None
|
||||
)
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
[KVCacheGroupSpec(["L0"], full_spec), KVCacheGroupSpec(["L1"], mamba_spec)],
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
)
|
||||
db_full = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), block_size=16
|
||||
)
|
||||
db_full.set_kv_caches_base_addr([0x1000])
|
||||
db_full.set_block_len([256])
|
||||
db_mamba = ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), block_size=16
|
||||
)
|
||||
db_mamba.set_kv_caches_base_addr([0x2000])
|
||||
db_mamba.set_block_len([256])
|
||||
thread = _make_store_sending_thread(
|
||||
store, coord=coord, token_databases=[db_full, db_mamba]
|
||||
)
|
||||
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
# FA has a null at chunk 1; the mamba table is align-mode shaped
|
||||
# (interior nulls + one state block) and must not be read at all.
|
||||
block_ids=([1, 0, 3, 4], [0, 0, 0, 9]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
),
|
||||
)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert all("@group:0@" in k for k in keys)
|
||||
assert [k.rsplit("@", 1)[-1] for k in keys] == ["6130", "6132", "6133"]
|
||||
assert addrs == [[0x1000 + 256], [0x1000 + 3 * 256], [0x1000 + 4 * 256]]
|
||||
|
||||
|
||||
def test_block_aligned_snapshot_offload_uses_provided_block():
|
||||
"""A block-aligned hand-off (sparse-retention mamba checkpoint) uploads
|
||||
exactly the core-pinned block under the boundary-end hash key; the
|
||||
positional block table entry for that chunk must be ignored."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)] # 8 hash units = 32 tokens
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, 7, 32)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
# boundary 32 is block-aligned for the mamba group (block 16): one key,
|
||||
# keyed by hs[32 // 4 - 1], read from the handed-off block 7 — not from
|
||||
# block_ids[1] and with no FA gap coverage.
|
||||
assert keys == [thread.token_databases[1].key_for(BlockHash(hs[7]))]
|
||||
assert addrs == [[0x2000 + 7 * 256]]
|
||||
|
||||
|
||||
def test_mixed_snapshot_and_sub_block_offloads():
|
||||
"""A retention snapshot and the prompt-end sub-block CoW tail can arrive
|
||||
in one hand-off; the sub-block path covers FA gap blocks but reads the
|
||||
mamba boundary only from the CoW block, never positionally."""
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.side_effect = lambda keys, *a: [256] * len(keys)
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(11)] # 11 hash units = 44 tokens
|
||||
req = ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [0, 0, 0]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, 9, 32), (1, 7, 44)],
|
||||
)
|
||||
assert thread._maybe_offload_boundary_states(req)
|
||||
|
||||
keys, addrs, _sizes, _ = store.batch_put_from_multi_buffers.call_args.args
|
||||
db_full, db_mamba = thread.token_databases
|
||||
assert keys == [
|
||||
# Aligned snapshot: boundary 32 from the handed-off block 9.
|
||||
db_mamba.key_for(BlockHash(hs[7])),
|
||||
# Sub-block boundary 44: FA gap blocks ending at 4, 8, 12.
|
||||
db_full.key_for(BlockHash(hs[0])),
|
||||
db_full.key_for(BlockHash(hs[1])),
|
||||
db_full.key_for(BlockHash(hs[2])),
|
||||
# Mamba boundary block from the CoW hand-off (block 7).
|
||||
db_mamba.key_for(BlockHash(hs[10])),
|
||||
]
|
||||
assert addrs == [
|
||||
[0x2000 + 9 * 256],
|
||||
[0x1000 + 1 * 256],
|
||||
[0x1000 + 2 * 256],
|
||||
[0x1000 + 3 * 256],
|
||||
[0x2000 + 7 * 256],
|
||||
]
|
||||
|
||||
|
||||
def test_snapshot_offload_skips_null_handoff_block():
|
||||
"""A hand-off the core could not materialize (null block) carries no
|
||||
committed state; persisting it would poison the boundary key."""
|
||||
store = MagicMock()
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
assert thread._maybe_offload_boundary_states(
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1, 2, 3], [5]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
boundary_state_offloads=[(1, NULL_BLOCK_ID, 32)],
|
||||
)
|
||||
)
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
|
||||
|
||||
def test_worker_meta_is_absent_before_any_completed_job():
|
||||
send_thread = _make_partial_tail_send_thread(MagicMock())
|
||||
store_worker = object.__new__(mooncake_store_worker.MooncakeStoreWorker)
|
||||
store_worker.kv_send_thread = send_thread
|
||||
|
||||
assert store_worker.build_connector_worker_meta() is None
|
||||
|
||||
|
||||
def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
|
||||
@@ -212,12 +212,21 @@ def _accuracy_test(llm: LLM, subscriber: MockSubscriber | None):
|
||||
if subscriber is not None:
|
||||
subscriber.get_new_cpu_stored_events()
|
||||
|
||||
# Pad prompt so its token count is a multiple of cpu_block_size.
|
||||
# Use the tokenizer directly to avoid expensive llm.generate() calls.
|
||||
# Align the reusable prefix; the final token must be recomputed for logits.
|
||||
tokenizer = llm.get_tokenizer()
|
||||
prompt = "Let's count to 10. One, two, three, four,"
|
||||
while len(tokenizer.encode(prompt)) % cpu_block_size != 0:
|
||||
prompt = ". " + prompt
|
||||
prompt_token_ids = tokenizer.encode("Let's count to 10. One, two, three, four,")
|
||||
reusable_prefix = prompt_token_ids[:-1]
|
||||
final_token = prompt_token_ids[-1:]
|
||||
|
||||
[padding_token_id] = tokenizer.encode(".", add_special_tokens=False)
|
||||
padding = [padding_token_id] * (-len(reusable_prefix) % cpu_block_size)
|
||||
insert_at = int(reusable_prefix[0] == tokenizer.bos_token_id)
|
||||
reusable_prefix[insert_at:insert_at] = padding
|
||||
assert len(reusable_prefix) % cpu_block_size == 0
|
||||
|
||||
prompt = TokensPrompt(
|
||||
prompt_token_ids=reusable_prefix + final_token,
|
||||
)
|
||||
|
||||
# Seed the CPU cache with the prompt.
|
||||
llm.generate(prompt, sampling_params, use_tqdm=False)
|
||||
|
||||
@@ -22,8 +22,13 @@ from vllm.v1.request import Request
|
||||
|
||||
|
||||
class DummyConnectorMetadata(KVConnectorMetadata):
|
||||
def __init__(self, block_hashes_by_req: dict[str, list[BlockHash]]):
|
||||
def __init__(
|
||||
self,
|
||||
block_hashes_by_req: dict[str, list[BlockHash]],
|
||||
block_ids_by_req: dict[str, tuple[list[int], ...]],
|
||||
):
|
||||
self.block_hashes_by_req = block_hashes_by_req
|
||||
self.block_ids_by_req = block_ids_by_req
|
||||
|
||||
|
||||
class DummyKVConnector(KVConnectorBase_V1):
|
||||
@@ -47,8 +52,12 @@ class DummyKVConnector(KVConnectorBase_V1):
|
||||
assert block_hashes_by_req is not None, (
|
||||
"DummyKVConnector expected 'block_hashes_by_req' on scheduler_output"
|
||||
)
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None
|
||||
block_ids_by_req = block_state.block_ids
|
||||
return DummyConnectorMetadata(
|
||||
block_hashes_by_req=block_hashes_by_req,
|
||||
block_ids_by_req=block_ids_by_req,
|
||||
)
|
||||
|
||||
def start_load_kv(self, kv_caches, finished_req_ids):
|
||||
@@ -129,3 +138,9 @@ def test_connector_receives_block_hashes(_load_plugin):
|
||||
num_tokens // block_size
|
||||
)
|
||||
assert meta.block_hashes_by_req[req.request_id] == req.block_hashes
|
||||
|
||||
expected_block_ids = {
|
||||
req.req_id: req.block_ids for req in output.scheduled_new_reqs
|
||||
}
|
||||
assert meta.block_ids_by_req == expected_block_ids
|
||||
assert output.kv_connector_block_state is None
|
||||
|
||||
Reference in New Issue
Block a user