Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -22,6 +22,7 @@ from vllm.v1.core.kv_cache_utils import (
get_block_hash,
get_group_id,
init_none_hash,
resolve_cache_hit_alignment_tokens,
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
@@ -148,29 +149,84 @@ def make_full_mamba_manager(
)
def test_dcp_fine_hit_retention_uses_hash_alignment_without_eagle():
"""DCP must not discard a reusable Mamba state at hash alignment."""
hash_block_size = 2
@pytest.mark.parametrize(
"dcp_world_size,hash_block_size,block_size,prompt_tokens,checkpoint_blocks",
[
(4, 2, 2, 7, 0),
(8, 16, 16, 512, 1),
(8, 16, 16, 513, 1),
*[(8, 128, 896, p, 1) for p in (127, 128, 129, 7040, 7168, 7296, 7449)],
],
)
@pytest.mark.parametrize("use_eagle", [False, True])
@pytest.mark.parametrize("extension_tokens", [0, 32])
def test_dcp_fine_hit_retention_uses_hash_alignment(
dcp_world_size,
hash_block_size,
block_size,
prompt_tokens,
checkpoint_blocks,
use_eagle,
extension_tokens,
):
"""Resends and extensions reuse the retained hash-aligned checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=4,
dcp_world_size=dcp_world_size,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=hash_block_size,
num_blocks=64,
full_block_size=block_size,
mamba_block_size=block_size,
num_blocks=128,
use_eagle=use_eagle,
num_speculative_blocks=4 if use_eagle else 0,
num_prefill_checkpoint_blocks=checkpoint_blocks,
)
manager.coordinator.retention_interval = 0
token_ids = list(range(7))
producer = make_request("producer", token_ids, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert num_computed == 0
assert manager.allocate_slots(producer, 6, 0, computed_blocks) is not None
assert manager.coordinator.enable_partial_hash_hits
scheduler = SimpleNamespace(
block_size=block_size * dcp_world_size,
cache_config=SimpleNamespace(block_size=block_size),
hash_block_size=hash_block_size,
max_num_scheduled_tokens=8192,
scheduler_config=SimpleNamespace(long_prefill_token_threshold=0),
use_eagle_block_drop=use_eagle,
mamba_partial_cache_hit=True,
mamba_shared_prefix_checkpoint=False,
mamba_has_prefill_checkpoint_blocks=bool(checkpoint_blocks),
mamba_prefill_checkpoint_alignment=16,
)
producer = make_request(
"producer", list(range(prompt_tokens)), hash_block_size, sha256
)
while producer.num_computed_tokens < prompt_tokens:
scheduled = Scheduler._mamba_block_aligned_split(
scheduler, producer, prompt_tokens - producer.num_computed_tokens
)
assert scheduled > 0
manager.new_step_starts()
assert manager.allocate_slots(producer, scheduled) is not None
producer.num_computed_tokens += scheduled
manager.free(producer)
manager.new_step_starts()
consumer = make_request("consumer", token_ids, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(consumer)
assert num_computed == 6
replay = make_request(
"replay", list(range(prompt_tokens + extension_tokens)), hash_block_size, sha256
)
_, hit, _ = manager.get_computed_blocks(replay)
expected = (
max((prompt_tokens // hash_block_size - 1) * hash_block_size, 0)
if use_eagle
else ((prompt_tokens - 1) // hash_block_size * hash_block_size)
)
assert hit == expected
cached_states = {
end
for end in range(hash_block_size, prompt_tokens + 1, hash_block_size)
if manager.block_pool.get_cached_block(
producer.block_hashes[end // hash_block_size - 1], [1]
)
is not None
}
assert cached_states == ({expected} if expected else set())
@pytest.mark.parametrize("dcp_world_size", [1, 4])
@@ -339,7 +395,7 @@ def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
@@ -488,7 +544,7 @@ def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -538,7 +594,7 @@ def test_partial_hit_then_internal_checkpoint_uses_distinct_mamba_blocks():
num_prefill_checkpoint_blocks=1,
)
owner = make_request("owner", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
owner = make_request("owner", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(owner)
assert manager.allocate_slots(owner, 6, num_computed, computed_blocks) is not None
manager.free(owner)
@@ -619,44 +675,70 @@ def test_internal_checkpoint_uses_partial_hash_lifecycle():
assert num_computed == 112
@pytest.mark.parametrize(
"retention_interval,transient_published",
[(None, True), (0, False)],
)
@pytest.mark.parametrize("retention_interval", [None, 0, 32, 64, 256])
@pytest.mark.parametrize("hash_block_size", [16, 32])
@pytest.mark.parametrize("use_eagle", [False, True])
def test_internal_checkpoint_publication_respects_retention(
retention_interval, transient_published
retention_interval, hash_block_size, use_eagle
):
"""With retention_interval=0, a mid-prompt internal checkpoint is
request-local and must not enter the prefix cache; the prompt-end
checkpoint keeps the existing publication behavior. Dense retention
(None) is unchanged.
"""
hash_block_size = 16
"""Publish only retained block boundaries and the final prompt checkpoint."""
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
use_eagle=use_eagle,
)
manager.coordinator.retention_interval = retention_interval
request = make_request("producer", list(range(240)), hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
# Mid-prompt chunk: exports a transient checkpoint at 112.
# The EAGLE adjustment can make a chunk checkpoint block-aligned or partial.
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
transient_hash = request.block_hashes[112 // hash_block_size - 1]
checkpoint = (128 if use_eagle else 127) // hash_block_size * hash_block_size - (
hash_block_size if use_eagle else 0
)
retained_checkpoint = checkpoint % 32 == 0 and (
retention_interval is None
or (retention_interval > 0 and checkpoint % retention_interval == 0)
)
transient_hash = request.block_hashes[checkpoint // hash_block_size - 1]
transient_hit = manager.block_pool.get_cached_block(transient_hash, [1])
assert (transient_hit is not None) == transient_published
assert (transient_hit is not None) == retained_checkpoint
offloads = drain_boundary_state_offloads(manager).get(request.request_id, [])
assert any(position == checkpoint for _, _, position in offloads) == (
retained_checkpoint
)
assert all(
position % 32 == 0
and (
retention_interval is None
or (retention_interval > 0 and position % retention_interval == 0)
)
for _, _, position in offloads
)
if retention_interval in (None, 32, 64):
retained_hash = request.block_hashes[128 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(retained_hash, [1]) is not None
request.num_computed_tokens = 128
manager.new_step_starts()
# Final chunk: the prompt-end checkpoint at 224 stays published.
# The final checkpoint is reusable regardless of the retention interval.
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // hash_block_size - 1]
prompt_checkpoint = (
240 if use_eagle else 239
) // hash_block_size * hash_block_size - (hash_block_size if use_eagle else 0)
end_hash = request.block_hashes[prompt_checkpoint // hash_block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
assert any(
position == prompt_checkpoint
for _, _, position in drain_boundary_state_offloads(manager).get(
request.request_id, []
)
)
def test_transient_checkpoint_evicts_retained_boundary_hash():
@@ -802,9 +884,9 @@ def test_hash_aligned_query_end_uses_regular_partial_tail():
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
request = make_request("producer", list(range(14)), hash_block_size, sha256)
request = make_request("producer", list(range(15)), hash_block_size, sha256)
new_blocks = manager.allocate_slots(request, request.num_tokens)
new_blocks = manager.allocate_slots(request, 14)
assert new_blocks is not None
mamba_manager = manager.coordinator.single_type_managers[1]
@@ -928,7 +1010,7 @@ def test_boundary_state_offloads_returns_cow_target():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1004,7 +1086,7 @@ def test_finished_partial_tail_uses_table_source_once():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
@@ -1024,7 +1106,7 @@ def test_finished_partial_tail_uses_table_source_once():
enable_caching=True,
hash_block_size=hash_block_size,
)
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
assert (
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
@@ -1212,7 +1294,7 @@ def test_truncate_computed_blocks_preserves_sparse_prefix_positions():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1267,7 +1349,7 @@ def test_truncate_computed_blocks_allows_short_mamba_group_only():
enable_caching=True,
hash_block_size=hash_block_size,
)
producer = make_request("producer", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
producer = make_request("producer", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
blocks, num_computed, _ = manager.get_computed_blocks(producer)
assert manager.allocate_slots(producer, 6, num_computed, blocks) is not None
manager.free(producer)
@@ -1331,7 +1413,7 @@ def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1411,7 +1493,7 @@ def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1470,7 +1552,7 @@ def test_hybrid_full_attention_partial_hash_hit_uses_cow():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1544,7 +1626,7 @@ def test_hybrid_partial_hit_cow_target_starts_uncached():
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -1700,7 +1782,7 @@ def test_cow_retained_blocks_returned_for_release():
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
@@ -2133,6 +2215,72 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
@pytest.mark.parametrize("with_sliding_window", [False, True])
def test_resolved_cache_hit_alignment_matches_coordinator(with_sliding_window):
"""Workers place Mamba checkpoints on the engine-resolved alignment, so it
must be where the coordinator lands hits, including when a group without
fine-grained lookups disables partial hits."""
hash_block_size = 2
mamba_block_size = 4 * hash_block_size
kv_cache_groups = [
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
]
if with_sliding_window:
kv_cache_groups.append(
KVCacheGroupSpec(
["swa"],
SlidingWindowSpec(
block_size=2 * hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=2 * hash_block_size,
),
)
)
kv_cache_config = KVCacheConfig(
num_blocks=64, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups
)
coordinator = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
).coordinator
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(enable_prefix_caching=True),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
assert coordinator.enable_partial_hash_hits != with_sliding_window
assert (
resolve_cache_hit_alignment_tokens(
kv_cache_config,
vllm_config,
coordinator.scheduler_block_size,
hash_block_size,
)
== coordinator._cache_hit_alignment_tokens
)
def test_opted_out_scratch_group_keeps_partial_hash_hits():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
@@ -2179,7 +2327,7 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
req0 = make_request("0", [0, 0, 1, 1, 2, 2, 3], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2240,7 +2388,7 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
assert manager.coordinator.enable_partial_hash_hits
shared_prefix = [10] * (12 * hash_block_size)
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
req0 = make_request("0", shared_prefix + [11], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert (
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
@@ -2266,7 +2414,7 @@ def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
)
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2], 2, sha256)
req0 = make_request("dcp-owner", [0, 0, 1, 1, 2, 2, 3], 2, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2318,7 +2466,7 @@ def test_dcp_partial_hit_resumes_on_replicated_mamba_snapshot(
assert manager.coordinator.single_type_managers[1].block_size == block_size
prefix = list(range(12))
req0 = make_request("snapshot-owner", prefix, block_size, sha256)
req0 = make_request("snapshot-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 12, num_computed, computed_blocks) is not None
manager.free(req0)
@@ -2355,7 +2503,7 @@ def test_dcp_joint_hit_is_bounded_by_replicated_mamba_snapshots():
mamba_block_size=block_size,
)
prefix = list(range(12))
req0 = make_request("joint-owner", prefix, block_size, sha256)
req0 = make_request("joint-owner", prefix + [12], block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 8, num_computed, computed_blocks) is not None
manager.new_step_starts()
@@ -2702,6 +2850,7 @@ def test_checkpoint_reservation_matches_worker(
mamba_block_size=mamba_block_size,
checkpoint_alignment=16,
drop_eagle_block=use_eagle,
cache_hit_alignment_tokens=mamba_manager.cache_hit_alignment_tokens,
)
reserved = mamba_manager._checkpoints.get(request.request_id)
if reserved is not None:
+82
View File
@@ -60,6 +60,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_kv_cache_groups,
get_hisparse_steady_state_concurrency,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
@@ -198,6 +199,48 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
def test_hisparse_steady_state_concurrency_excludes_spilled_resident_pages():
"""Running requests pin only their resident tail; the newest still needs its
full in-flight window to be admitted."""
block_size = 16
config = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=10 * block_size),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
max_in_flight_tokens=10 * block_size,
)
attn_spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=64, dtype=torch.float16
)
page_size = attn_spec.page_size_bytes
kv_cache_config = KVCacheConfig(
num_blocks=64,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["source"], attn_spec, host_resident=True),
KVCacheGroupSpec(["indexer"], attn_spec),
KVCacheGroupSpec(
["resident"],
HiSparseResidentSpec(block_size=block_size, page_size=page_size),
),
KVCacheGroupSpec(
["hot"],
HiSparseHotSpec(
block_size=block_size, page_size=page_size, blocks_per_request=4
),
),
],
hisparse_host_num_blocks=100,
)
# Admission takes indexer 10 + resident 10 + hot 4 = 24 blocks and steady
# state 16, so 3 fit (2 x 16 + 24 <= 64) but not 4 (3 x 16 + 24 > 64).
assert get_hisparse_steady_state_concurrency(config, kv_cache_config) == 3.5
# The worst-case bound charges all 10 resident pages.
assert get_max_concurrency_for_kv_cache_config(
config, kv_cache_config
) == pytest.approx(64 / 24)
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""
@@ -923,6 +966,24 @@ def test_free_kv_cache_block_queue_popleft_n():
assert block.next_free_block is None
@pytest.mark.parametrize("num_blocks", [1, 2, 16])
def test_free_kv_cache_block_queue_popleft_n_drain_and_refill(num_blocks):
"""Draining a batch must leave the sentinels ready for reuse."""
blocks = [KVCacheBlock(block_id=i) for i in range(num_blocks)]
queue = FreeKVCacheBlockQueue(blocks)
assert queue.popleft_n(num_blocks) == blocks
assert queue.num_free_blocks == 0
assert queue.fake_free_list_head.next_free_block is queue.fake_free_list_tail
assert queue.fake_free_list_tail.prev_free_block is queue.fake_free_list_head
assert all(b.prev_free_block is None and b.next_free_block is None for b in blocks)
assert queue.popleft_n(0) == []
queue.append_n(blocks)
assert queue.popleft_n(num_blocks) == blocks
assert queue.num_free_blocks == 0
def test_free_kv_cache_block_queue_get_all_free_blocks():
# Create a list of KVCacheBlock objects
blocks = [KVCacheBlock(block_id=i) for i in range(5)]
@@ -1873,6 +1934,27 @@ def test_dcp_target_allocates_replicated_draft_independently(
assert [len(group) for group in cached_blocks.blocks] == [1, 4]
def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
"""A replicated kpool tail ring is per-request state, not a draft group."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.cache_config.kv_cache_layout = None
tail = KpoolTailSpec(
block_size=4,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
dcp_sharded=False,
)
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
assert layout == KVCacheLayout.LBHNC
@pytest.mark.parametrize("use_mla", [False, True])
def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
+14 -26
View File
@@ -594,8 +594,12 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
failed_recving_kv_req_ids=set(),
finished_recving_kv_req_ids={request.request_id},
prefix_replay_tokens=0,
prefix_replay_group_ids=(),
)
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
scheduler._load_restores_replay_window = MethodType(
Scheduler._load_restores_replay_window, scheduler
)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
@@ -5043,19 +5047,10 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary():
def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
"""An identical resend and a longer sibling resume at DIFFERENT positions.
"""EAGLE resend and extension reuse the same retained checkpoint.
How far a lookup matches depends on who is asking. A resend of the same
prompt caps its lookup at ``num_tokens - 1`` (the last token is recomputed
for logits), while a sibling whose prompt merely starts with this one caps
above the prompt. The two coincide unless the prompt length is an exact
multiple of the alignment -- there they differ by one alignment unit, and
under the EAGLE drop BOTH are reachable.
Retaining only the higher one leaves the resend with every retained state
above every candidate its lookup can produce, and the reconciled hit
collapses to 0 -- the same zero-hit failure sparse retention already fixes
at unaligned prompt lengths.
Attention may inspect the prompt's final block before dropping it, so an
aligned resend must not lose another block to the P - 1 lookup limit.
"""
block_size = 32
num_spec = 3
@@ -5093,16 +5088,12 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
use_eagle=True,
)
# 128 tokens, an exact multiple of the 32-token alignment. A longer sibling
# matches 128 and drops to 96; this prompt's own resend caps at 127, matches
# 96 and drops to 64. Both states must survive retention.
# Both requests prove 128 attention tokens, drop one block, and reuse 96.
token_ids = [i for i in range(4) for _ in range(block_size)]
req0 = make_request("0", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
# Prefill in block-aligned chunks the way the align-mode scheduler does: a
# state only materializes as a chunk's running-state block, so a
# single-shot prefill could not retain the lower one.
# Materialize the checkpoint through normal block-aligned prefill chunks.
for chunk_end in (32, 64, 96, 128):
blocks = manager.allocate_slots(
req0,
@@ -5114,8 +5105,7 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
assert blocks is not None
req0.num_computed_tokens = chunk_end
# Block ``i`` ends at token ``(i + 1) * 32``, so positions 64 and 96 are
# mamba blocks 1 and 2.
# Both materialized replay checkpoints survive; resend and extension use 96.
pool = manager.block_pool
expected_mamba_cached = {1, 2}
for i in range(4):
@@ -5126,15 +5116,13 @@ def test_hybrid_mamba_retention_mtp_resend_of_aligned_prompt():
assert cached is None, f"mamba hash {i} should not be cached"
manager.free(req0)
# The identical resend: full attention matches blocks 0-2 (96 tokens, capped
# by num_tokens - 1) and the EAGLE drop caps the candidate at 64. Without
# the lower state retained the reconciled hit would be 0.
# The identical resend must reuse the retained checkpoint, not miss to zero.
req1 = make_request("1", token_ids, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
assert num_computed_tokens == 2 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [2, 2]
assert num_computed_tokens == 3 * block_size
assert [len(blocks) for blocks in computed_blocks.blocks] == [3, 3]
# The longer sibling resumes one alignment unit higher, off the same prompt.
# The longer sibling reuses that same checkpoint.
longer = make_request("2", token_ids + [9] * block_size, block_size, sha256)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(longer)
assert num_computed_tokens == 3 * block_size
+27
View File
@@ -247,6 +247,33 @@ def test_remote_kv_hit_is_taken_in_whole_blocks():
assert swa_blocks[new_req.replay_start // BLOCK_SIZE] is not swa_manager._null_block
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize("matched", [WINDOW, 4 * BLOCK_SIZE - 1])
def test_kv_load_carrying_window_does_not_replay(is_async, matched):
"""A load of the request's own blocks (P/D) restores every transfer group,
window included, so the hit is taken as is: no replay, no cut to whole
blocks, and a hit no longer than the window is kept."""
scheduler = _replay_scheduler(
use_kv_connector=MockKVConfig(matched_tokens=matched, is_async=is_async)
)
request = create_requests(
num_requests=1, num_tokens=NUM_PROMPT_TOKENS, block_size=BLOCK_SIZE
)[0]
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
scheduler.add_request(request)
out = scheduler.schedule()
if is_async:
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.update_from_output(
out, create_model_runner_output([], finished_recving={request.request_id})
)
out = scheduler.schedule()
new_req = _new_req_data(out, request)
assert new_req.num_computed_tokens == matched
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS - matched
@pytest.mark.parametrize("via_connector", [False, True])
def test_hit_no_longer_than_window_is_ignored(via_connector):
"""Such a hit would be recomputed in full anyway. It is not adopted (an
+197 -1
View File
@@ -3,7 +3,7 @@
import dataclasses
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import Mock
from unittest.mock import Mock, patch
import numpy as np
import pytest
@@ -35,6 +35,7 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
@@ -60,6 +61,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
MambaSpec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import (
DraftTokenIds,
ECConnectorOutput,
@@ -278,6 +280,24 @@ def test_schedule(enable_prefix_caching: bool, prompt_logprobs: int | None):
assert scheduler.running[i] == request
def test_pooling_chunked_prefill_can_finish_at_max_model_len():
scheduler = create_scheduler(
max_num_seqs=1, max_model_len=8, max_num_batched_tokens=4, runner="pooling"
)
assert scheduler.max_model_len == 8
request = Request(
request_id="pool",
prompt_token_ids=[1] * 8,
sampling_params=None,
pooling_params=PoolingParams(task="embed"),
)
scheduler.add_request(request)
scheduler.schedule()
# The final chunk must reach max_model_len, not stop one token short.
assert scheduler.schedule().num_scheduled_tokens == {"pool": 4}
def test_scheduler_stats_route_to_existing_output_client():
scheduler = create_scheduler()
request = create_requests(num_requests=1)[0]
@@ -688,6 +708,33 @@ def test_throttle_capacity_bound_guard_admits():
assert "b" in output.num_scheduled_tokens
def test_same_step_duplicate_encoder_input_stays_cached():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
max_model_len=2048,
)
request = create_requests(
1,
num_tokens=2000,
req_ids=["repeated"],
mm_hashes_list=[["image", "image", "image"]],
mm_positions=[
[PlaceholderRange(offset=offset, length=576) for offset in (0, 600, 1300)]
],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
_model_output(scheduler, output, [[]])
# The second occurrence is partially consumed; the third is not scheduled.
cache = scheduler.encoder_cache_manager
assert cache.get_cached_input_ids(request) == {1}
assert "image" not in cache.freeable
def test_no_mm_input_chunking():
# Disable multimodal input chunking.
scheduler = create_scheduler(
@@ -2526,6 +2573,96 @@ def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
return initial_ecos
def _runner_output(
req_ids: list[str], finished_recving: set[str] | None = None
) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
sampled_token_ids=[[1] for _ in req_ids],
kv_connector_output=KVConnectorOutput(finished_recving=finished_recving),
)
@pytest.mark.parametrize(
"load_done_before_pause", [False, True], ids=["load-in-flight", "load-done"]
)
def test_pause_new_drains_async_kv_loads(load_done_before_pause: bool):
"""A PAUSED_NEW (pause mode="wait") drain must finish requests that hold KV
blocks for an async load, keep blockless queued requests queued, and leave
no blocks behind, so the post-drain cache reset succeeds."""
block_size = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(matched_tokens=2 * block_size, is_async=True),
block_size=block_size,
)
loading, queued = create_requests(
num_requests=2,
num_tokens=3 * block_size,
max_tokens=2,
block_size=block_size,
req_ids=["loading", "queued"],
)
scheduler.add_request(loading)
output = scheduler.schedule()
assert loading.status == RequestStatus.WAITING_FOR_REMOTE_KVS
pending_recv = {"loading"}
if load_done_before_pause:
scheduler.update_from_output(output, _runner_output([], pending_recv))
pending_recv = None
else:
scheduler.update_from_output(output, _runner_output([]))
scheduler.set_pause_state(PauseState.PAUSED_NEW)
scheduler.add_request(queued)
num_steps = 0
while scheduler.has_requests():
num_steps += 1
assert num_steps <= 4, "pause drain did not finish"
output = scheduler.schedule()
assert set(output.num_scheduled_tokens) <= {"loading"}
scheduler.update_from_output(
output, _runner_output(list(output.num_scheduled_tokens), pending_recv)
)
pending_recv = None
# What pause runs once drained; it raises if the load still holds blocks.
assert scheduler.reset_prefix_cache(reset_running_requests=True)
assert loading.status == RequestStatus.FINISHED_LENGTH_CAPPED
assert loading.request_id not in scheduler.requests
assert list(scheduler.waiting) == [queued]
assert queued.status == RequestStatus.WAITING
scheduler.set_pause_state(PauseState.UNPAUSED)
scheduler.schedule()
assert queued.status == RequestStatus.WAITING_FOR_REMOTE_KVS
def test_pause_new_ignores_streaming_session_waiting_for_input():
"""A session idle in WAITING_FOR_STREAMING_REQ cannot progress, so it must
not keep a PAUSED_NEW drain from completing."""
stop_token = 7
scheduler = create_scheduler()
(session,) = create_requests(
num_requests=1, num_tokens=4, stop_token_ids=[stop_token], req_ids=["s"]
)
session.resumable = True
scheduler.add_request(session)
output = scheduler.schedule()
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=["s"], req_id_to_index={"s": 0}, sampled_token_ids=[[stop_token]]
),
)
assert session in scheduler.kv_holding_waiting
scheduler.set_pause_state(PauseState.PAUSED_NEW)
assert scheduler.get_num_unfinished_requests() == 0
@pytest.mark.parametrize(
("load_modes", "expected_has_sync_loads"),
[
@@ -2586,6 +2723,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
block_size = 16
num_matched_tokens = 2 * block_size
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(
matched_tokens=num_matched_tokens,
is_async=is_async,
),
block_size=block_size,
)
request = create_requests(
num_requests=1,
num_tokens=2 * num_matched_tokens,
block_size=block_size,
)[0]
assert scheduler.connector is not None
get_sources = Mock(
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
)
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
calls = Mock()
calls.attach_mock(update_state, "allocated")
calls.attach_mock(get_sources, "sources")
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
monkeypatch.setattr(
scheduler.connector, "get_external_cache_hit_sources", get_sources
)
scheduler.add_request(request)
# A failed allocation is not an admission: nothing is attributed.
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
scheduler.schedule()
get_sources.assert_not_called()
update_state.assert_not_called()
scheduler.schedule()
# Attribution runs once, after the connector has built its load plan.
get_sources.assert_called_once_with(request, num_matched_tokens)
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits == num_matched_tokens
assert connector_stats.hits_by_source == {
CacheHitSource.P2P: block_size,
CacheHitSource.HOST: block_size,
}
if is_async:
# Re-admission after the async load completes is not a new hit.
scheduler.make_stats()
_step_until_kv_transfer_finished(scheduler, [request.request_id])
get_sources.assert_called_once()
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits_by_source == {}
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""Test whether Scheduler with KVConnector schedules tokens, allocates
@@ -6,10 +6,14 @@ import random
import pytest
import torch
from vllm.sampling_params import SamplingParams
from vllm.utils.hashing import sha256
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
get_request_block_hasher,
init_none_hash,
make_block_hash_with_group_id,
)
from vllm.v1.core.single_type_kv_cache_manager import (
@@ -30,6 +34,7 @@ from vllm.v1.kv_cache_interface import (
RSWASpec,
SlidingWindowSpec,
)
from vllm.v1.request import Request
pytestmark = pytest.mark.cpu_test
@@ -238,6 +243,71 @@ def test_mamba_checkpoint_admission_matches_allocation(
assert admission_estimate == allocation_estimate == allocated
@pytest.mark.parametrize("num_speculative_blocks", [1, 2, 3])
@pytest.mark.parametrize("first_chunk_blocks", [2, 4])
@pytest.mark.parametrize("retention_interval", [0, None])
def test_mamba_checkpoint_hash_maps_only_to_checkpoint_block(
num_speculative_blocks, first_chunk_blocks, retention_interval
):
"""A checkpoint step must not leave a never-written speculative block
cached."""
block_size = 128
spec = MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=16,
)
pool = BlockPool(num_gpu_blocks=64, enable_caching=True, hash_block_size=128)
manager = MambaManager(
spec,
block_pool=pool,
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=block_size,
)
# MTP/EAGLE moves the checkpoint one block below the last full block.
manager.drop_eagle_checkpoint_block = True
init_none_hash(sha256)
prompt_tokens = 6 * block_size + 50
request = Request(
request_id="r",
prompt_token_ids=list(range(prompt_tokens)),
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
block_hasher=get_request_block_hasher(block_size, sha256),
)
def step(start, end):
estimate = manager.get_num_blocks_to_allocate("r", end, [], start, start, end)
free_before = pool.get_num_free_blocks()
manager.allocate_new_blocks("r", end, end)
assert free_before - pool.get_num_free_blocks() == estimate
manager.cache_blocks(
request,
end,
retention_interval=retention_interval,
replay_boundaries=[5 * block_size],
)
request.num_computed_tokens = end
step(0, first_chunk_blocks * block_size)
speculative_blocks = manager.req_to_blocks["r"][-num_speculative_blocks:]
manager.new_step_starts()
step(first_chunk_blocks * block_size, prompt_tokens)
checkpoint_position, checkpoint_idx = manager._checkpoints["r"]
checkpoint_block = manager.req_to_blocks["r"][checkpoint_idx]
assert all(
b.block_hash is None or b is checkpoint_block for b in speculative_blocks
)
block_hash = request.block_hashes[checkpoint_position // block_size - 1]
assert pool.get_cached_block(block_hash, [0]) == [checkpoint_block]
def get_sliding_window_manager(
sliding_window_spec,
block_pool,
+3
View File
@@ -18,6 +18,7 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.config.model import RunnerOption
from vllm.config.scheduler import SchedulerPolicy
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -78,6 +79,7 @@ def create_scheduler(
ec_role: str | None = None,
use_v2_model_runner: bool | None = None,
kv_cache_spec: KVCacheSpec | None = None,
runner: RunnerOption = "auto",
per_request_spec_decode_metrics: str = "none",
scheduling_policy: SchedulerPolicy = "fcfs",
diffusion_canvas_length: int | None = None,
@@ -109,6 +111,7 @@ def create_scheduler(
# The scheduler reads model_config.max_model_len, not the
# SchedulerConfig one, so both must agree.
max_model_len=max_model_len,
runner=runner,
)
if mm_encoder_only:
model_config.multimodal_config.mm_encoder_only = True