Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
get_block_hash,
|
||||
get_group_id,
|
||||
init_none_hash,
|
||||
resolve_cache_hit_alignment_tokens,
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -148,29 +149,84 @@ def make_full_mamba_manager(
|
||||
)
|
||||
|
||||
|
||||
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
|
||||
"""DCP must not discard a reusable Mamba state at hash alignment."""
|
||||
hash_block_size = 2
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_world_size,hash_block_size,block_size,prompt_tokens,checkpoint_blocks",
|
||||
[
|
||||
(4, 2, 2, 7, 0),
|
||||
(8, 16, 16, 512, 1),
|
||||
(8, 16, 16, 513, 1),
|
||||
*[(8, 128, 896, p, 1) for p in (127, 128, 129, 7040, 7168, 7296, 7449)],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_eagle", [False, True])
|
||||
@pytest.mark.parametrize("extension_tokens", [0, 32])
|
||||
def test_dcp_fine_hit_retention_uses_hash_alignment(
|
||||
dcp_world_size,
|
||||
hash_block_size,
|
||||
block_size,
|
||||
prompt_tokens,
|
||||
checkpoint_blocks,
|
||||
use_eagle,
|
||||
extension_tokens,
|
||||
):
|
||||
"""Resends and extensions reuse the retained hash-aligned checkpoint."""
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=4,
|
||||
dcp_world_size=dcp_world_size,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=hash_block_size,
|
||||
num_blocks=64,
|
||||
full_block_size=block_size,
|
||||
mamba_block_size=block_size,
|
||||
num_blocks=128,
|
||||
use_eagle=use_eagle,
|
||||
num_speculative_blocks=4 if use_eagle else 0,
|
||||
num_prefill_checkpoint_blocks=checkpoint_blocks,
|
||||
)
|
||||
manager.coordinator.retention_interval = 0
|
||||
|
||||
token_ids = list(range(7))
|
||||
producer = make_request("producer", token_ids, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
scheduler = SimpleNamespace(
|
||||
block_size=block_size * dcp_world_size,
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
hash_block_size=hash_block_size,
|
||||
max_num_scheduled_tokens=8192,
|
||||
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
|
||||
use_eagle_block_drop=use_eagle,
|
||||
mamba_partial_cache_hit=True,
|
||||
mamba_shared_prefix_checkpoint=False,
|
||||
mamba_has_prefill_checkpoint_blocks=bool(checkpoint_blocks),
|
||||
mamba_prefill_checkpoint_alignment=16,
|
||||
)
|
||||
producer = make_request(
|
||||
"producer", list(range(prompt_tokens)), hash_block_size, sha256
|
||||
)
|
||||
while producer.num_computed_tokens < prompt_tokens:
|
||||
scheduled = Scheduler._mamba_block_aligned_split(
|
||||
scheduler, producer, prompt_tokens - producer.num_computed_tokens
|
||||
)
|
||||
assert scheduled > 0
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(producer, scheduled) is not None
|
||||
producer.num_computed_tokens += scheduled
|
||||
manager.free(producer)
|
||||
manager.new_step_starts()
|
||||
|
||||
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(consumer)
|
||||
assert num_computed == 6
|
||||
replay = make_request(
|
||||
"replay", list(range(prompt_tokens + extension_tokens)), hash_block_size, sha256
|
||||
)
|
||||
_, hit, _ = manager.get_computed_blocks(replay)
|
||||
expected = (
|
||||
max((prompt_tokens // hash_block_size - 1) * hash_block_size, 0)
|
||||
if use_eagle
|
||||
else ((prompt_tokens - 1) // hash_block_size * hash_block_size)
|
||||
)
|
||||
assert hit == expected
|
||||
cached_states = {
|
||||
end
|
||||
for end in range(hash_block_size, prompt_tokens + 1, hash_block_size)
|
||||
if manager.block_pool.get_cached_block(
|
||||
producer.block_hashes[end // hash_block_size - 1], [1]
|
||||
)
|
||||
is not None
|
||||
}
|
||||
assert cached_states == ({expected} if expected else set())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 4])
|
||||
@@ -339,7 +395,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
|
||||
@@ -488,7 +544,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
@@ -538,7 +594,7 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
|
||||
owner = make_request("owner", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
owner = make_request("owner", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(owner)
|
||||
assert manager.allocate_slots(owner, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(owner)
|
||||
@@ -619,44 +675,70 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
|
||||
assert num_computed == 112
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"retention_interval,transient_published",
|
||||
[(None, True), (0, False)],
|
||||
)
|
||||
@pytest.mark.parametrize("retention_interval", [None, 0, 32, 64, 256])
|
||||
@pytest.mark.parametrize("hash_block_size", [16, 32])
|
||||
@pytest.mark.parametrize("use_eagle", [False, True])
|
||||
def test_internal_checkpoint_publication_respects_retention(
|
||||
retention_interval, transient_published
|
||||
retention_interval, hash_block_size, use_eagle
|
||||
):
|
||||
"""With retention_interval=0, a mid-prompt internal checkpoint is
|
||||
request-local and must not enter the prefix cache; the prompt-end
|
||||
checkpoint keeps the existing publication behavior. Dense retention
|
||||
(None) is unchanged.
|
||||
"""
|
||||
hash_block_size = 16
|
||||
"""Publish only retained block boundaries and the final prompt checkpoint."""
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=32,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
use_eagle=use_eagle,
|
||||
)
|
||||
manager.coordinator.retention_interval = retention_interval
|
||||
|
||||
request = make_request("producer", list(range(240)), hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
|
||||
# Mid-prompt chunk: exports a transient checkpoint at 112.
|
||||
# The EAGLE adjustment can make a chunk checkpoint block-aligned or partial.
|
||||
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
|
||||
transient_hash = request.block_hashes[112 // hash_block_size - 1]
|
||||
checkpoint = (128 if use_eagle else 127) // hash_block_size * hash_block_size - (
|
||||
hash_block_size if use_eagle else 0
|
||||
)
|
||||
retained_checkpoint = checkpoint % 32 == 0 and (
|
||||
retention_interval is None
|
||||
or (retention_interval > 0 and checkpoint % retention_interval == 0)
|
||||
)
|
||||
transient_hash = request.block_hashes[checkpoint // hash_block_size - 1]
|
||||
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
|
||||
assert (transient_hit is not None) == transient_published
|
||||
assert (transient_hit is not None) == retained_checkpoint
|
||||
offloads = drain_boundary_state_offloads(manager).get(request.request_id, [])
|
||||
assert any(position == checkpoint for _, _, position in offloads) == (
|
||||
retained_checkpoint
|
||||
)
|
||||
assert all(
|
||||
position % 32 == 0
|
||||
and (
|
||||
retention_interval is None
|
||||
or (retention_interval > 0 and position % retention_interval == 0)
|
||||
)
|
||||
for _, _, position in offloads
|
||||
)
|
||||
if retention_interval in (None, 32, 64):
|
||||
retained_hash = request.block_hashes[128 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(retained_hash, [1]) is not None
|
||||
|
||||
request.num_computed_tokens = 128
|
||||
manager.new_step_starts()
|
||||
|
||||
# Final chunk: the prompt-end checkpoint at 224 stays published.
|
||||
# The final checkpoint is reusable regardless of the retention interval.
|
||||
assert manager.allocate_slots(request, 112) is not None
|
||||
end_hash = request.block_hashes[224 // hash_block_size - 1]
|
||||
prompt_checkpoint = (
|
||||
240 if use_eagle else 239
|
||||
) // hash_block_size * hash_block_size - (hash_block_size if use_eagle else 0)
|
||||
end_hash = request.block_hashes[prompt_checkpoint // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
|
||||
assert any(
|
||||
position == prompt_checkpoint
|
||||
for _, _, position in drain_boundary_state_offloads(manager).get(
|
||||
request.request_id, []
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_transient_checkpoint_evicts_retained_boundary_hash():
|
||||
@@ -802,9 +884,9 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
request = make_request("producer", list(range(14)), hash_block_size, sha256)
|
||||
request = make_request("producer", list(range(15)), hash_block_size, sha256)
|
||||
|
||||
new_blocks = manager.allocate_slots(request, request.num_tokens)
|
||||
new_blocks = manager.allocate_slots(request, 14)
|
||||
|
||||
assert new_blocks is not None
|
||||
mamba_manager = manager.coordinator.single_type_managers[1]
|
||||
@@ -928,7 +1010,7 @@ def test_boundary_state_offloads_returns_cow_target():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -1004,7 +1086,7 @@ def test_finished_partial_tail_uses_table_source_once():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
|
||||
@@ -1024,7 +1106,7 @@ def test_finished_partial_tail_uses_table_source_once():
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
|
||||
assert (
|
||||
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
|
||||
@@ -1212,7 +1294,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
|
||||
manager.free(producer)
|
||||
@@ -1267,7 +1349,7 @@ def test_truncate_computed_blocks_allows_short_mamba_group_only():
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
blocks, num_computed, _ = manager.get_computed_blocks(producer)
|
||||
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
|
||||
manager.free(producer)
|
||||
@@ -1331,7 +1413,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
@@ -1411,7 +1493,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
@@ -1470,7 +1552,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
@@ -1544,7 +1626,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
@@ -1700,7 +1782,7 @@ def test_cow_retained_blocks_returned_for_release():
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
@@ -2133,6 +2215,72 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("with_sliding_window", [False, True])
|
||||
def test_resolved_cache_hit_alignment_matches_coordinator(with_sliding_window):
|
||||
"""Workers place Mamba checkpoints on the engine-resolved alignment, so it
|
||||
must be where the coordinator lands hits, including when a group without
|
||||
fine-grained lookups disables partial hits."""
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 4 * hash_block_size
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
]
|
||||
if with_sliding_window:
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(
|
||||
["swa"],
|
||||
SlidingWindowSpec(
|
||||
block_size=2 * hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=2 * hash_block_size,
|
||||
),
|
||||
)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
|
||||
)
|
||||
coordinator = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
).coordinator
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(enable_prefix_caching=True),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
)
|
||||
|
||||
assert coordinator.enable_partial_hash_hits != with_sliding_window
|
||||
assert (
|
||||
resolve_cache_hit_alignment_tokens(
|
||||
kv_cache_config,
|
||||
vllm_config,
|
||||
coordinator.scheduler_block_size,
|
||||
hash_block_size,
|
||||
)
|
||||
== coordinator._cache_hit_alignment_tokens
|
||||
)
|
||||
|
||||
|
||||
def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
@@ -2179,7 +2327,7 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
@@ -2240,7 +2388,7 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
shared_prefix = [10] * (12 * hash_block_size)
|
||||
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
|
||||
req0 = make_request("0", shared_prefix + [11], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert (
|
||||
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
|
||||
@@ -2266,7 +2414,7 @@ def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
|
||||
)
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
|
||||
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2, 3], 2, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
@@ -2318,7 +2466,7 @@ def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
|
||||
assert manager.coordinator.single_type_managers[1].block_size == block_size
|
||||
|
||||
prefix = list(range(12))
|
||||
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
|
||||
req0 = make_request("snapshot-owner", prefix + [12], block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
@@ -2355,7 +2503,7 @@ def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
|
||||
mamba_block_size=block_size,
|
||||
)
|
||||
prefix = list(range(12))
|
||||
req0 = make_request("joint-owner", prefix, block_size, sha256)
|
||||
req0 = make_request("joint-owner", prefix + [12], block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
|
||||
manager.new_step_starts()
|
||||
@@ -2702,6 +2850,7 @@ def test_checkpoint_reservation_matches_worker(
|
||||
mamba_block_size=mamba_block_size,
|
||||
checkpoint_alignment=16,
|
||||
drop_eagle_block=use_eagle,
|
||||
cache_hit_alignment_tokens=mamba_manager.cache_hit_alignment_tokens,
|
||||
)
|
||||
reserved = mamba_manager._checkpoints.get(request.request_id)
|
||||
if reserved is not None:
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_kv_cache_groups,
|
||||
get_hisparse_steady_state_concurrency,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
@@ -198,6 +199,48 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
def test_hisparse_steady_state_concurrency_excludes_spilled_resident_pages():
|
||||
"""Running requests pin only their resident tail; the newest still needs its
|
||||
full in-flight window to be admitted."""
|
||||
block_size = 16
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=10 * block_size),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
max_in_flight_tokens=10 * block_size,
|
||||
)
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=1, head_size=64, dtype=torch.float16
|
||||
)
|
||||
page_size = attn_spec.page_size_bytes
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=64,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["source"], attn_spec, host_resident=True),
|
||||
KVCacheGroupSpec(["indexer"], attn_spec),
|
||||
KVCacheGroupSpec(
|
||||
["resident"],
|
||||
HiSparseResidentSpec(block_size=block_size, page_size=page_size),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["hot"],
|
||||
HiSparseHotSpec(
|
||||
block_size=block_size, page_size=page_size, blocks_per_request=4
|
||||
),
|
||||
),
|
||||
],
|
||||
hisparse_host_num_blocks=100,
|
||||
)
|
||||
|
||||
# Admission takes indexer 10 + resident 10 + hot 4 = 24 blocks and steady
|
||||
# state 16, so 3 fit (2 x 16 + 24 <= 64) but not 4 (3 x 16 + 24 > 64).
|
||||
assert get_hisparse_steady_state_concurrency(config, kv_cache_config) == 3.5
|
||||
# The worst-case bound charges all 10 resident pages.
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
config, kv_cache_config
|
||||
) == pytest.approx(64 / 24)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
|
||||
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
|
||||
"""Every page needs a host block, so one max_model_len request must fit."""
|
||||
@@ -923,6 +966,24 @@ def test_free_kv_cache_block_queue_popleft_n():
|
||||
assert block.next_free_block is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_blocks", [1, 2, 16])
|
||||
def test_free_kv_cache_block_queue_popleft_n_drain_and_refill(num_blocks):
|
||||
"""Draining a batch must leave the sentinels ready for reuse."""
|
||||
blocks = [KVCacheBlock(block_id=i) for i in range(num_blocks)]
|
||||
queue = FreeKVCacheBlockQueue(blocks)
|
||||
|
||||
assert queue.popleft_n(num_blocks) == blocks
|
||||
assert queue.num_free_blocks == 0
|
||||
assert queue.fake_free_list_head.next_free_block is queue.fake_free_list_tail
|
||||
assert queue.fake_free_list_tail.prev_free_block is queue.fake_free_list_head
|
||||
assert all(b.prev_free_block is None and b.next_free_block is None for b in blocks)
|
||||
assert queue.popleft_n(0) == []
|
||||
|
||||
queue.append_n(blocks)
|
||||
assert queue.popleft_n(num_blocks) == blocks
|
||||
assert queue.num_free_blocks == 0
|
||||
|
||||
|
||||
def test_free_kv_cache_block_queue_get_all_free_blocks():
|
||||
# Create a list of KVCacheBlock objects
|
||||
blocks = [KVCacheBlock(block_id=i) for i in range(5)]
|
||||
@@ -1873,6 +1934,27 @@ def test_dcp_target_allocates_replicated_draft_independently(
|
||||
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
|
||||
|
||||
|
||||
def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
|
||||
"""A replicated kpool tail ring is per-request state, not a draft group."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.cache_config.kv_cache_layout = None
|
||||
tail = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
|
||||
assert layout == KVCacheLayout.LBHNC
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mla", [False, True])
|
||||
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
|
||||
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
failed_recving_kv_req_ids=set(),
|
||||
finished_recving_kv_req_ids={request.request_id},
|
||||
prefix_replay_tokens=0,
|
||||
prefix_replay_group_ids=(),
|
||||
)
|
||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||
scheduler._load_restores_replay_window = MethodType(
|
||||
Scheduler._load_restores_replay_window, scheduler
|
||||
)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||
@@ -5043,19 +5047,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
|
||||
|
||||
|
||||
def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
|
||||
"""An identical resend and a longer sibling resume at DIFFERENT positions.
|
||||
"""EAGLE resend and extension reuse the same retained checkpoint.
|
||||
|
||||
How far a lookup matches depends on who is asking. A resend of the same
|
||||
prompt caps its lookup at ``num_tokens - 1`` (the last token is recomputed
|
||||
for logits), while a sibling whose prompt merely starts with this one caps
|
||||
above the prompt. The two coincide unless the prompt length is an exact
|
||||
multiple of the alignment -- there they differ by one alignment unit, and
|
||||
under the EAGLE drop BOTH are reachable.
|
||||
|
||||
Retaining only the higher one leaves the resend with every retained state
|
||||
above every candidate its lookup can produce, and the reconciled hit
|
||||
collapses to 0 -- the same zero-hit failure sparse retention already fixes
|
||||
at unaligned prompt lengths.
|
||||
Attention may inspect the prompt's final block before dropping it, so an
|
||||
aligned resend must not lose another block to the P - 1 lookup limit.
|
||||
"""
|
||||
block_size = 32
|
||||
num_spec = 3
|
||||
@@ -5093,16 +5088,12 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# 128 tokens, an exact multiple of the 32-token alignment. A longer sibling
|
||||
# matches 128 and drops to 96; this prompt's own resend caps at 127, matches
|
||||
# 96 and drops to 64. Both states must survive retention.
|
||||
# Both requests prove 128 attention tokens, drop one block, and reuse 96.
|
||||
token_ids = [i for i in range(4) for _ in range(block_size)]
|
||||
req0 = make_request("0", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
# Prefill in block-aligned chunks the way the align-mode scheduler does: a
|
||||
# state only materializes as a chunk's running-state block, so a
|
||||
# single-shot prefill could not retain the lower one.
|
||||
# Materialize the checkpoint through normal block-aligned prefill chunks.
|
||||
for chunk_end in (32, 64, 96, 128):
|
||||
blocks = manager.allocate_slots(
|
||||
req0,
|
||||
@@ -5114,8 +5105,7 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
|
||||
assert blocks is not None
|
||||
req0.num_computed_tokens = chunk_end
|
||||
|
||||
# Block ``i`` ends at token ``(i + 1) * 32``, so positions 64 and 96 are
|
||||
# mamba blocks 1 and 2.
|
||||
# Both materialized replay checkpoints survive; resend and extension use 96.
|
||||
pool = manager.block_pool
|
||||
expected_mamba_cached = {1, 2}
|
||||
for i in range(4):
|
||||
@@ -5126,15 +5116,13 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
|
||||
assert cached is None, f"mamba hash {i} should not be cached"
|
||||
manager.free(req0)
|
||||
|
||||
# The identical resend: full attention matches blocks 0-2 (96 tokens, capped
|
||||
# by num_tokens - 1) and the EAGLE drop caps the candidate at 64. Without
|
||||
# the lower state retained the reconciled hit would be 0.
|
||||
# The identical resend must reuse the retained checkpoint, not miss to zero.
|
||||
req1 = make_request("1", token_ids, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed_tokens == 2 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 2]
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 3]
|
||||
|
||||
# The longer sibling resumes one alignment unit higher, off the same prompt.
|
||||
# The longer sibling reuses that same checkpoint.
|
||||
longer = make_request("2", token_ids + [9] * block_size, block_size, sha256)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(longer)
|
||||
assert num_computed_tokens == 3 * block_size
|
||||
|
||||
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
|
||||
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
|
||||
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
|
||||
"""A load of the request's own blocks (P/D) restores every transfer group,
|
||||
window included, so the hit is taken as is: no replay, no cut to whole
|
||||
blocks, and a hit no longer than the window is kept."""
|
||||
scheduler = _replay_scheduler(
|
||||
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
if is_async:
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.update_from_output(
|
||||
out, create_model_runner_output([], finished_recving={request.request_id})
|
||||
)
|
||||
out = scheduler.schedule()
|
||||
new_req = _new_req_data(out, request)
|
||||
assert new_req.num_computed_tokens == matched
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
|
||||
|
||||
|
||||
@pytest.mark.parametrize("via_connector", [False, True])
|
||||
def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
"""Such a hit would be recomputed in full anyway. It is not adopted (an
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -35,6 +35,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
@@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ECConnectorOutput,
|
||||
@@ -278,6 +280,24 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
|
||||
assert scheduler.running[i] == request
|
||||
|
||||
|
||||
def test_pooling_chunked_prefill_can_finish_at_max_model_len():
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=1, max_model_len=8, max_num_batched_tokens=4, runner="pooling"
|
||||
)
|
||||
assert scheduler.max_model_len == 8
|
||||
request = Request(
|
||||
request_id="pool",
|
||||
prompt_token_ids=[1] * 8,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.schedule()
|
||||
# The final chunk must reach max_model_len, not stop one token short.
|
||||
assert scheduler.schedule().num_scheduled_tokens == {"pool": 4}
|
||||
|
||||
|
||||
def test_scheduler_stats_route_to_existing_output_client():
|
||||
scheduler = create_scheduler()
|
||||
request = create_requests(num_requests=1)[0]
|
||||
@@ -688,6 +708,33 @@ def test_throttle_capacity_bound_guard_admits():
|
||||
assert "b" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_same_step_duplicate_encoder_input_stays_cached():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
max_model_len=2048,
|
||||
)
|
||||
request = create_requests(
|
||||
1,
|
||||
num_tokens=2000,
|
||||
req_ids=["repeated"],
|
||||
mm_hashes_list=[["image", "image", "image"]],
|
||||
mm_positions=[
|
||||
[PlaceholderRange(offset=offset, length=576) for offset in (0, 600, 1300)]
|
||||
],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
|
||||
_model_output(scheduler, output, [[]])
|
||||
|
||||
# The second occurrence is partially consumed; the third is not scheduled.
|
||||
cache = scheduler.encoder_cache_manager
|
||||
assert cache.get_cached_input_ids(request) == {1}
|
||||
assert "image" not in cache.freeable
|
||||
|
||||
|
||||
def test_no_mm_input_chunking():
|
||||
# Disable multimodal input chunking.
|
||||
scheduler = create_scheduler(
|
||||
@@ -2526,6 +2573,96 @@ def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
return initial_ecos
|
||||
|
||||
|
||||
def _runner_output(
|
||||
req_ids: list[str], finished_recving: set[str] | None = None
|
||||
) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
|
||||
sampled_token_ids=[[1] for _ in req_ids],
|
||||
kv_connector_output=KVConnectorOutput(finished_recving=finished_recving),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_done_before_pause", [False, True], ids=["load-in-flight", "load-done"]
|
||||
)
|
||||
def test_pause_new_drains_async_kv_loads(load_done_before_pause: bool):
|
||||
"""A PAUSED_NEW (pause mode="wait") drain must finish requests that hold KV
|
||||
blocks for an async load, keep blockless queued requests queued, and leave
|
||||
no blocks behind, so the post-drain cache reset succeeds."""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=2 * block_size, is_async=True),
|
||||
block_size=block_size,
|
||||
)
|
||||
loading, queued = create_requests(
|
||||
num_requests=2,
|
||||
num_tokens=3 * block_size,
|
||||
max_tokens=2,
|
||||
block_size=block_size,
|
||||
req_ids=["loading", "queued"],
|
||||
)
|
||||
scheduler.add_request(loading)
|
||||
output = scheduler.schedule()
|
||||
assert loading.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
pending_recv = {"loading"}
|
||||
if load_done_before_pause:
|
||||
scheduler.update_from_output(output, _runner_output([], pending_recv))
|
||||
pending_recv = None
|
||||
else:
|
||||
scheduler.update_from_output(output, _runner_output([]))
|
||||
|
||||
scheduler.set_pause_state(PauseState.PAUSED_NEW)
|
||||
scheduler.add_request(queued)
|
||||
|
||||
num_steps = 0
|
||||
while scheduler.has_requests():
|
||||
num_steps += 1
|
||||
assert num_steps <= 4, "pause drain did not finish"
|
||||
output = scheduler.schedule()
|
||||
assert set(output.num_scheduled_tokens) <= {"loading"}
|
||||
scheduler.update_from_output(
|
||||
output, _runner_output(list(output.num_scheduled_tokens), pending_recv)
|
||||
)
|
||||
pending_recv = None
|
||||
|
||||
# What pause runs once drained; it raises if the load still holds blocks.
|
||||
assert scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
assert loading.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
assert loading.request_id not in scheduler.requests
|
||||
assert list(scheduler.waiting) == [queued]
|
||||
assert queued.status == RequestStatus.WAITING
|
||||
|
||||
scheduler.set_pause_state(PauseState.UNPAUSED)
|
||||
scheduler.schedule()
|
||||
assert queued.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
|
||||
def test_pause_new_ignores_streaming_session_waiting_for_input():
|
||||
"""A session idle in WAITING_FOR_STREAMING_REQ cannot progress, so it must
|
||||
not keep a PAUSED_NEW drain from completing."""
|
||||
stop_token = 7
|
||||
scheduler = create_scheduler()
|
||||
(session,) = create_requests(
|
||||
num_requests=1, num_tokens=4, stop_token_ids=[stop_token], req_ids=["s"]
|
||||
)
|
||||
session.resumable = True
|
||||
scheduler.add_request(session)
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=["s"], req_id_to_index={"s": 0}, sampled_token_ids=[[stop_token]]
|
||||
),
|
||||
)
|
||||
assert session in scheduler.kv_holding_waiting
|
||||
|
||||
scheduler.set_pause_state(PauseState.PAUSED_NEW)
|
||||
assert scheduler.get_num_unfinished_requests() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("load_modes", "expected_has_sync_loads"),
|
||||
[
|
||||
@@ -2586,6 +2723,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
|
||||
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
|
||||
block_size = 16
|
||||
num_matched_tokens = 2 * block_size
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(
|
||||
matched_tokens=num_matched_tokens,
|
||||
is_async=is_async,
|
||||
),
|
||||
block_size=block_size,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=2 * num_matched_tokens,
|
||||
block_size=block_size,
|
||||
)[0]
|
||||
assert scheduler.connector is not None
|
||||
get_sources = Mock(
|
||||
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
|
||||
)
|
||||
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
|
||||
calls = Mock()
|
||||
calls.attach_mock(update_state, "allocated")
|
||||
calls.attach_mock(get_sources, "sources")
|
||||
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "get_external_cache_hit_sources", get_sources
|
||||
)
|
||||
|
||||
scheduler.add_request(request)
|
||||
# A failed allocation is not an admission: nothing is attributed.
|
||||
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
|
||||
scheduler.schedule()
|
||||
get_sources.assert_not_called()
|
||||
update_state.assert_not_called()
|
||||
|
||||
scheduler.schedule()
|
||||
# Attribution runs once, after the connector has built its load plan.
|
||||
get_sources.assert_called_once_with(request, num_matched_tokens)
|
||||
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits == num_matched_tokens
|
||||
assert connector_stats.hits_by_source == {
|
||||
CacheHitSource.P2P: block_size,
|
||||
CacheHitSource.HOST: block_size,
|
||||
}
|
||||
|
||||
if is_async:
|
||||
# Re-admission after the async load completes is not a new hit.
|
||||
scheduler.make_stats()
|
||||
_step_until_kv_transfer_finished(scheduler, [request.request_id])
|
||||
get_sources.assert_called_once()
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits_by_source == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
|
||||
@@ -6,10 +6,14 @@ import random
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
get_request_block_hasher,
|
||||
init_none_hash,
|
||||
make_block_hash_with_group_id,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
@@ -30,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -238,6 +243,71 @@ def test_mamba_checkpoint_admission_matches_allocation(
|
||||
assert admission_estimate == allocation_estimate == allocated
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_blocks", [1, 2, 3])
|
||||
@pytest.mark.parametrize("first_chunk_blocks", [2, 4])
|
||||
@pytest.mark.parametrize("retention_interval", [0, None])
|
||||
def test_mamba_checkpoint_hash_maps_only_to_checkpoint_block(
|
||||
num_speculative_blocks, first_chunk_blocks, retention_interval
|
||||
):
|
||||
"""A checkpoint step must not leave a never-written speculative block
|
||||
cached."""
|
||||
block_size = 128
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=16,
|
||||
)
|
||||
pool = BlockPool(num_gpu_blocks=64, enable_caching=True, hash_block_size=128)
|
||||
manager = MambaManager(
|
||||
spec,
|
||||
block_pool=pool,
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=block_size,
|
||||
)
|
||||
# MTP/EAGLE moves the checkpoint one block below the last full block.
|
||||
manager.drop_eagle_checkpoint_block = True
|
||||
init_none_hash(sha256)
|
||||
prompt_tokens = 6 * block_size + 50
|
||||
request = Request(
|
||||
request_id="r",
|
||||
prompt_token_ids=list(range(prompt_tokens)),
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
block_hasher=get_request_block_hasher(block_size, sha256),
|
||||
)
|
||||
|
||||
def step(start, end):
|
||||
estimate = manager.get_num_blocks_to_allocate("r", end, [], start, start, end)
|
||||
free_before = pool.get_num_free_blocks()
|
||||
manager.allocate_new_blocks("r", end, end)
|
||||
assert free_before - pool.get_num_free_blocks() == estimate
|
||||
manager.cache_blocks(
|
||||
request,
|
||||
end,
|
||||
retention_interval=retention_interval,
|
||||
replay_boundaries=[5 * block_size],
|
||||
)
|
||||
request.num_computed_tokens = end
|
||||
|
||||
step(0, first_chunk_blocks * block_size)
|
||||
speculative_blocks = manager.req_to_blocks["r"][-num_speculative_blocks:]
|
||||
manager.new_step_starts()
|
||||
step(first_chunk_blocks * block_size, prompt_tokens)
|
||||
|
||||
checkpoint_position, checkpoint_idx = manager._checkpoints["r"]
|
||||
checkpoint_block = manager.req_to_blocks["r"][checkpoint_idx]
|
||||
assert all(
|
||||
b.block_hash is None or b is checkpoint_block for b in speculative_blocks
|
||||
)
|
||||
block_hash = request.block_hashes[checkpoint_position // block_size - 1]
|
||||
assert pool.get_cached_block(block_hash, [0]) == [checkpoint_block]
|
||||
|
||||
|
||||
def get_sliding_window_manager(
|
||||
sliding_window_spec,
|
||||
block_pool,
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.config import (
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.config.scheduler import SchedulerPolicy
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -78,6 +79,7 @@ def create_scheduler(
|
||||
ec_role: str | None = None,
|
||||
use_v2_model_runner: bool | None = None,
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
runner: RunnerOption = "auto",
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
@@ -109,6 +111,7 @@ def create_scheduler(
|
||||
# The scheduler reads model_config.max_model_len, not the
|
||||
# SchedulerConfig one, so both must agree.
|
||||
max_model_len=max_model_len,
|
||||
runner=runner,
|
||||
)
|
||||
if mm_encoder_only:
|
||||
model_config.multimodal_config.mm_encoder_only = True
|
||||
|
||||
Reference in New Issue
Block a user