Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -38,9 +38,7 @@ def _manager(
num_prefill_lookahead=0,
):
init_none_hash(sha256)
config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
)
config = _make_hybrid_kv_cache_config(block_size, num_blocks, ["full", "mamba"])
if eagle_group is not None:
groups = list(config.kv_cache_groups)
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=mamba_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=mamba_block_size,
),
),
],
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=kpool_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=kpool_block_size,
),
),
],
-1
View File
@@ -340,7 +340,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
total_num_scheduled_tokens=1,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
+44 -13
View File
@@ -18,13 +18,15 @@ import torch
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import (
from vllm.v1.core.kv_cache_planning import (
_get_kv_cache_bytes_per_block,
_get_packed_kv_cache_groups,
_pool_bytes_per_block,
generate_scheduler_kv_cache_config,
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_utils import (
generate_scheduler_kv_cache_config,
resolve_kv_cache_block_sizes,
)
from vllm.v1.hisparse.layout import _build_hisparse_kv_cache_tensors
@@ -88,7 +90,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
alignment = lcm(1152, hot.page_size_bytes)
stride = _pool_bytes_per_block(groups)
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
assert stride == alignment
layout = KVCacheLayout.BLHNC
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
@@ -254,7 +256,7 @@ class TestCSALinearGrouping:
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
pages = _pages(groups)
for group in groups:
@@ -276,7 +278,14 @@ class TestCSALinearGrouping:
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs().items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=BYTES_PER_BLOCK * 64
)
@@ -418,7 +427,7 @@ class TestCSALinearGrouping:
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
@@ -465,7 +474,7 @@ class TestSlidingWindowBucketCap:
pages = _pages(groups)
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
main_bytes = sum(pages[n] for n in main.layer_names)
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
per_group = main_bytes // pages["layers.0.swa"]
assert len(swa_groups) == -(-43 // per_group)
@@ -479,9 +488,9 @@ class TestSlidingWindowBucketCap:
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
groups
)
assert _get_kv_cache_bytes_per_block(
groups, KVCacheLayout.BLHNC
) == _expected_bytes_per_block(groups)
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
@@ -554,6 +563,20 @@ class TestDensePacking:
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
assert mla_tensor.block_stride == block_stride
def test_layer_outer_pads_only_unaligned_pages(self):
"""Layer-outer block strides are pages, so only an unaligned page pads."""
mla_page = _mla(512).page_size_bytes
idx = replace(_mla(128), block_stride_alignment=12288)
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
config = get_kv_cache_config_from_groups(
_mock_vllm_config("LBNHC"), groups, MEMORY
)
mla_tensor, idx_tensor = config.kv_cache_tensors
assert config.num_blocks == MEMORY // (mla_page + 12288)
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
assert idx_tensor.offset == mla_page * config.num_blocks
_bind(config, "LBNHC")
def test_overlaid_groups_alias_and_stay_isolated(self):
groups, g1, g2 = _mixed_page_groups()
# Overlay models resolve to a block-outer layout at backend selection (the
@@ -562,7 +585,9 @@ class TestDensePacking:
_mock_vllm_config("BLNHC"), groups, MEMORY
)
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(
_mock_vllm_config("BLNHC"), groups
) == _expected_bytes_per_block(groups)
views = _bind(config, "BLNHC")
assert set(views) == set(g1) | set(g2)
@@ -667,7 +692,10 @@ class TestCompressorRingGroup:
assert len(ring_groups) == 1
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
manager = KVCacheManager(
@@ -716,7 +744,10 @@ class TestSWABoundedReplayGrouping:
for g in groups
) == [(False, 128)] * 4 + [(True, 0)]
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([kv_cache_config]),
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+97 -87
View File
@@ -279,6 +279,12 @@ def _allocate_scheduled(
return blocks
def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]:
"""The GPU block HiSparse reads each resident page from; 0 if host-only."""
resident = get_hisparse_coordinator(manager).resident_managers[0]
return [block.block_id for block in resident.get_residency_row(request_id)]
def test_hisparse_does_not_write_back_reprefillable_tokens():
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
@@ -319,7 +325,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
def test_hisparse_async_speculation_mirrors_uncertain_position_range():
"""Unresolved drafts must not leave gaps in the eager host mirror."""
coordinator = MagicMock(host_group_id=0)
coordinator.take_block_table_updates.return_value = {}
coordinator.take_residency_updates.return_value = {}
coordinator.build_offload_command.return_value = None
coordinator.build_row_mirrors.return_value = ()
scheduler = HiSparseConnectorScheduler(
@@ -332,7 +338,6 @@ def test_hisparse_async_speculation_mirrors_uncertain_position_range():
connector.connector_scheduler = scheduler
connector.update_state_after_alloc(request, None, 0)
scheduler_output = SimpleNamespace(
block_table_updates=None,
kv_cache_block_copies=None,
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
@@ -370,11 +375,66 @@ def test_hisparse_reports_when_context_is_fully_resident():
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
_publish_hisparse_pages(manager)
coordinator.take_residency_updates([request.request_id])
core_row = manager.get_block_ids(request.request_id)[2]
pool = manager.block_pool
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert coordinator.take_block_table_updates().keys() == {request.request_id}
# The block-table row stays append-only; the worker learns of the lost
# pages from the residency update instead.
assert manager.get_block_ids(request.request_id)[2] == core_row
resident_ids = _resident_block_ids(manager, request.request_id)
lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0]
assert lost
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == lost
assert update[request.request_id].block_ids[0] == [0] * len(lost)
def test_hisparse_residency_update_sends_only_changed_pages():
"""Losing one early page must not resend the pages after it: per-step
residency updates would otherwise grow with the context length."""
manager = make_hisparse_kv_cache_manager(32, 16)
tokens = list(range(6 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator = get_hisparse_coordinator(manager)
for hot_manager in coordinator.hot_managers:
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator.take_residency_updates([request.request_id])
pool = manager.block_pool
num_never_used = pool.get_num_free_blocks()
# Pages unpin as their host copies complete, so page 0 is reused first.
_publish_hisparse_pages(manager)
pool.get_new_blocks(num_never_used + 1)
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0]
assert update[request.request_id].block_ids == ([0],)
def test_hisparse_readmitted_request_resends_its_full_residency():
"""A preempted request may resume in a worker state row holding another
request's residency, so its first update must cover every page again."""
manager = make_hisparse_kv_cache_manager(32, 16)
coordinator = get_hisparse_coordinator(manager)
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
assert request.request_id in coordinator.take_residency_updates(
[request.request_id]
)
manager.free(request)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0, 1]
assert update[request.request_id].block_ids[0] == _resident_block_ids(
manager, request.request_id
)
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
@@ -416,9 +476,7 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
assert len(source) == len(indexer) == len(resident) == 4
assert len(hot) == 2
assert not any(block.is_null for block in resident[:2])
assert not resident[2].is_null
assert not resident[3].is_null
assert 0 not in _resident_block_ids(manager, resumed.request_id)
coordinator = get_hisparse_coordinator(manager)
assert not coordinator.build_offload_command().page_transfers
coordinator.finish_host_import(resumed.request_id, failed=False)
@@ -666,9 +724,10 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
pool.free_blocks(held)
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
assert all(m.has_hot("first") for m in coordinator.hot_managers)
coordinator.take_residency_updates(["first"])
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert "first" in coordinator.take_block_table_updates()
assert "first" in coordinator.take_residency_updates(["first"])
def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
@@ -1204,11 +1263,7 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert [block.block_id for block in resident_blocks[:3]] == (
original_resident_ids[:3]
)
assert not any(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3]
assert get_hisparse_coordinator(manager).all_context_pages_resident(
((resumed.request_id, num_computed, len(tokens) - num_computed),)
)
@@ -1249,17 +1304,13 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
new_computed_blocks=computed,
)
resumed_blocks = manager.get_blocks("resumed").blocks
assert [block.block_id for block in resumed_blocks[2][:3]] == [
resident_ids = _resident_block_ids(manager, "resumed")
assert resident_ids[:3] == [
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
for page in range(3)
]
gpu_ids = [
block.block_id
for group in resumed_blocks[1:]
for block in group
if not block.is_null
]
_, indexer, _, hot = manager.get_block_ids("resumed")
gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id]
assert len(gpu_ids) == len(set(gpu_ids))
@@ -1488,8 +1539,7 @@ def test_hisparse_reused_copy_is_not_adopted():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert all(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0]
def test_hisparse_external_import_uses_hard_gpu_footprint():
@@ -1567,6 +1617,7 @@ def make_kv_cache_config_hybrid_model(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
return KVCacheConfig(
@@ -1602,6 +1653,7 @@ def make_kv_cache_config_three_types(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
elif third_spec_type == "sliding_window":
third_spec = SlidingWindowSpec(
@@ -1682,6 +1734,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
@@ -1707,10 +1760,13 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, len(req0.prompt_token_ids), 0, computed_blocks
)
assert blocks is not None
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the shared prefix.
for num_new_tokens in (len(common_token_ids), 7):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
@@ -2235,7 +2291,7 @@ def _make_hybrid_kv_cache_config(
- "full": FullAttentionSpec
- "sliding_window": SlidingWindowSpec with window=2*block_size
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
- "mamba": MambaSpec
- "mamba": MambaSpec in prefix-caching ("align") mode
"""
spec_map = {
@@ -2263,11 +2319,6 @@ def _make_hybrid_kv_cache_config(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
),
"mamba_align": lambda: MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
}
@@ -2377,14 +2428,16 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
assert not computed_blocks.blocks[0] # No cache hit initially
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, 55, len(computed_blocks.blocks[0]) * block_size, computed_blocks
)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
manager.new_step_starts()
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the common prefix.
for num_new_tokens in (len(common_token_ids), len(unique_token_ids)):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
manager.new_step_starts()
# Second request: should hit cached blocks for common prefix
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
@@ -2499,7 +2552,7 @@ def test_prefill_hybrid_model_mamba_align():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = make_kv_cache_manager(
kv_cache_config,
@@ -2535,7 +2588,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
"""
block_size = 16
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
@@ -2610,7 +2663,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = KVCacheManager(
kv_cache_config,
@@ -5846,49 +5899,6 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
assert retained(0, None) == {14}
def test_mamba_shared_prefix_survives_zero_retention():
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
token_ids = [i for i in range(16) for _ in range(block_size)]
def cached_mamba_blocks(shared_prefix_boundary):
# Fresh manager per scenario so cached blocks don't leak between runs.
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
)
req = make_request("r", token_ids, block_size, sha256)
req.shared_prefix_boundary = shared_prefix_boundary
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req, len(token_ids), num_computed, computed_blocks
)
assert blocks is not None
pool = manager.block_pool
return {
i
for i in range(16)
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
}
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
assert cached_mamba_blocks(0) == {14}
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
# later request sharing that prefix can hit the Mamba state.
assert cached_mamba_blocks(96) == {5, 14}
def test_mamba_shared_prefix_reuse_under_zero_retention():
"""Full cross-request Marconi flow: a partial shared prefix cached by the
detecting request must stay reusable by a later request under
@@ -5899,7 +5909,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
def last_req_hit(retention, pin):
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
+80 -12
View File
@@ -352,6 +352,12 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
_ = scheduler.schedule()
assert req.num_output_placeholders > 0
if scheduler.use_v2_model_runner:
# MRV2 staggers same-request decodes by pp_size steps to match the
# worker-side sampled-token broadcast cadence.
output = scheduler.schedule()
assert req.request_id not in output.num_scheduled_tokens
# before any update_from_output, we still expect the request can be
# scheduled again (multi-step in-flight).
output = scheduler.schedule()
@@ -735,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached():
assert "image" not in cache.freeable
def test_no_mm_input_chunking():
@pytest.mark.parametrize("warm_encoder_cache", [False, True])
def test_no_mm_input_chunking(warm_encoder_cache: bool):
# Disable multimodal input chunking.
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
@@ -747,6 +754,16 @@ def test_no_mm_input_chunking():
requests = create_requests(
num_requests=1, num_tokens=1200, mm_positions=mm_positions
)
if warm_encoder_cache:
cached_request = create_requests(
num_requests=1,
num_tokens=1200,
mm_positions=mm_positions,
req_ids=["cache-primer"],
)[0]
scheduler.encoder_cache_manager.allocate(cached_request, 0)
scheduler.encoder_cache_manager.free(cached_request)
for request in requests:
scheduler.add_request(request)
@@ -774,6 +791,8 @@ def test_no_mm_input_chunking():
assert output.scheduled_cached_reqs.num_reqs == 1
assert len(output.finished_req_ids) == 0
assert output.num_scheduled_tokens[requests[0].request_id] == 800
if warm_encoder_cache:
assert not output.scheduled_encoder_inputs
# Test that we fail if we disable chunked mm input and use too small
# of a max_num_batched_tokens for the mm input.
@@ -785,6 +804,66 @@ def test_no_mm_input_chunking():
)
def test_no_mm_input_chunking_for_same_step_duplicate():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
disable_chunked_mm_input=True,
max_model_len=2048,
)
request = create_requests(
num_requests=1,
num_tokens=1352,
mm_hashes_list=[["image", "image"]],
mm_positions=[
[
PlaceholderRange(offset=100, length=576),
PlaceholderRange(offset=776, length=576),
]
],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == 776
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
def test_no_mm_input_chunking_at_item_boundary():
"""A chunk starting exactly at a multimodal item must not split it.
Regression test for issue #52306: the rollback guard only fired while
``num_computed_tokens < start_pos``, so a chunk beginning exactly at the
item skipped the check and could cover only part of it - silently breaking
models with bidirectional attention over the item.
"""
mm_positions = [[PlaceholderRange(offset=400, length=800)]]
def ask(num_computed_tokens: int, num_new_tokens: int) -> int:
# A fresh scheduler per call: the encoder cache is stateful.
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
disable_chunked_mm_input=True,
max_model_len=2048,
)
request = create_requests(
num_requests=1, num_tokens=1200, mm_positions=mm_positions
)[0]
return scheduler._try_schedule_encoder_inputs(
request, num_computed_tokens, num_new_tokens, encoder_compute_budget=100_000
)[1]
# Starts before the item and cannot reach past it: stop just before it.
assert ask(0, 500) == 400
# Starts exactly at the item and cannot cover it: schedule nothing rather
# than half an item.
assert ask(400, 500) == 0
# Starts exactly at the item and covers it: schedule it.
assert ask(400, 800) == 800
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
"""Test scheduling behavior with concurrent partial requests.
@@ -929,7 +1008,6 @@ def test_update_from_output_routes_sampling_masks_by_request():
total_num_scheduled_tokens=3,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -973,7 +1051,6 @@ def test_update_from_output_routes_multi_position_sampling_masks():
total_num_scheduled_tokens=2,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1021,7 +1098,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [],
requests[1].request_id: [10],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1067,7 +1143,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [10, 42],
requests[1].request_id: [13],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1111,7 +1186,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [10, 11],
requests[1].request_id: [],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1149,7 +1223,6 @@ def test_stop_via_update_from_output():
total_num_scheduled_tokens=3,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={requests[0].request_id: [EOS_TOKEN_ID, 10]},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -4200,7 +4273,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
total_num_scheduled_tokens=1,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5675,7 +5747,6 @@ def test_delayed_kv_connector_free_keeps_scheduler_active():
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5757,7 +5828,6 @@ def test_scheduler_kv_connector_stats():
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5838,7 +5908,6 @@ def test_scheduler_ec_connector_stats():
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5870,7 +5939,6 @@ def test_ec_connector_update_connector_output_called():
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -399,22 +397,21 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
manager.cache_blocks(request_id, 1024, replay_boundaries=(0,))
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0
assert manager.req_to_blocks[request_id] == blocks
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
"""The kpool tail ring is never zeroed, transferred or not: a pool only
reads ring slots its own request wrote."""
spec = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
manager = CircularBufferManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
@@ -428,9 +425,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.take_new_block_ids() == []
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []
+75 -1
View File
@@ -36,6 +36,10 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVQuantMode,
MambaSpec,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -238,7 +242,7 @@ def create_scheduler(
structured_output_manager=StructuredOutputManager(vllm_config),
)
if use_v2_model_runner is None:
use_v2_model_runner = bool(envs.VLLM_USE_V2_MODEL_RUNNER)
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER is not False
scheduler.use_v2_model_runner = use_v2_model_runner
return scheduler
@@ -333,3 +337,73 @@ def create_requests(
)
requests.append(request)
return requests
def new_kv_cache_spec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float32,
page_size_padded=None,
sliding_window=None,
attention_chunk_size=None,
kv_quant_mode=KVQuantMode.NONE,
):
return FullAttentionSpec(
block_size=block_size,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
attention_chunk_size=attention_chunk_size,
kv_quant_mode=kv_quant_mode,
)
def new_sliding_window_spec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float32,
page_size_padded=None,
sliding_window=1,
):
return SlidingWindowSpec(
block_size=block_size,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
)
def new_mamba_spec(
block_size=16,
shapes=((2, 512), (3, 32, 32)),
dtypes=(torch.float32, torch.float32),
num_speculative_blocks=2,
mamba_cache_mode="none",
page_size_padded=None,
):
return MambaSpec(
block_size=block_size,
shapes=shapes,
dtypes=dtypes,
page_size_padded=page_size_padded,
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=num_speculative_blocks,
)
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
return MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=576,
dtype=torch.float32,
cache_dtype_str=cache_dtype_str,
)