Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -13,6 +13,9 @@ import torch
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
from vllm.distributed.kv_transfer.kv_connector.v1.base import SupportsHMA
from vllm.model_executor.layers.mamba.checkpoint import (
compute_mamba_prefill_checkpoints,
)
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import (
KVCacheBlockCopy,
@@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
MambaSpec,
SlidingWindowSpec,
get_mamba_prefill_checkpoint_position,
)
@@ -686,6 +690,81 @@ def test_transient_checkpoint_evicts_retained_boundary_hash():
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
@pytest.mark.parametrize("prompt_len", [120, 104, 96])
def test_prompt_end_checkpoint_survives_block_aligned_cache_count(prompt_len):
"""At the default match unit the hybrid coordinator hands ``cache_blocks``
a block-aligned token count, which is below the prompt length whenever the
prompt is not block-aligned. The prompt-end checkpoint is the replay
boundary and must stay published under retention_interval=0; only a
mid-prompt checkpoint is transient.
"""
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = 0
assert not manager.coordinator.enable_partial_hash_hits
request = make_request("producer", list(range(prompt_len)), hash_block_size, sha256)
assert manager.allocate_slots(request, request.num_tokens) is not None
checkpoint_position = get_mamba_prefill_checkpoint_position(
prompt_len, hash_block_size, drop_eagle_block=False
)
checkpoint_hash = request.block_hashes[checkpoint_position // hash_block_size - 1]
hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
assert hit is not None
assert hit[0].block_hash_num_tokens == checkpoint_position
manager.free(request)
replay = make_request("replay", list(range(prompt_len)), hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(replay)
assert num_computed == checkpoint_position
@pytest.mark.parametrize("retention_interval,reserved", [(None, True), (0, False)])
def test_intermediate_chunk_checkpoint_reserved_only_under_dense_retention(
retention_interval, reserved
):
"""A checkpoint inside a chunk that does not finish the prefill is only
publishable under dense retention. With retention_interval=0 it would be
evicted as transient, so the block is not reserved in the first place;
the prefill-end checkpoint is reserved either way.
"""
hash_block_size = 16
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=32,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = retention_interval
mamba_manager = manager.coordinator.single_type_managers[1]
request = make_request("producer", list(range(240)), hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
free_before = manager.block_pool.get_num_free_blocks()
# Mid-prompt chunk: the chunk-keyed position (112) is inside it. Blocks:
# the attention blocks, the running mamba block, plus the checkpoint.
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
assert (request.request_id in mamba_manager._checkpoints) == reserved
allocated = free_before - manager.block_pool.get_num_free_blocks()
assert allocated == 128 // hash_block_size + 1 + int(reserved)
request.num_computed_tokens = 128
manager.new_step_starts()
# Final chunk: the prefill-end checkpoint at 224 is reserved either way.
assert manager.allocate_slots(request, 112) is not None
assert mamba_manager._checkpoints[request.request_id][0] == 224
end_hash = request.block_hashes[224 // hash_block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
hash_block_size = mamba_block_size = 32
manager = make_full_mamba_manager(
@@ -2531,3 +2610,105 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
assert 2 * block_size in offered
assert req0.num_prompt_tokens < 2 * block_size
def test_prompt_end_checkpoint_survives_sparse_retention_with_coarse_hash():
"""With hash_block_size == mamba block size (no fine-grained matching), the
hybrid coordinator caches the final chunk only up to its last full block,
so the managers see ``num_tokens < num_prompt_tokens`` there too. The
prompt-end checkpoint must still be published under retention_interval=0,
otherwise the next request that extends this prompt misses the prefix.
"""
block_size = 32
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=block_size,
full_block_size=block_size,
mamba_block_size=block_size,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = 0
# 240 tokens: not block aligned, prompt-end checkpoint at 224.
request = make_request("producer", list(range(240)), block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
request.num_computed_tokens = 128
manager.new_step_starts()
assert manager.allocate_slots(request, 112) is not None
end_hash = request.block_hashes[224 // block_size - 1]
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
# A request extending the prompt resumes from that checkpoint.
follower = make_request("follower", list(range(240)) + [7] * 40, block_size, sha256)
_, num_hit, _ = manager.get_computed_blocks(follower)
assert num_hit == 224
@pytest.mark.parametrize(
"hash_block_size,mamba_block_size,use_eagle",
[(32, 32, False), (16, 32, False), (32, 32, True)],
)
@pytest.mark.parametrize("retention_interval", [0, None])
@pytest.mark.parametrize(
"num_prompt_tokens,num_output_tokens",
[(104, 0), (200, 0), (40, 24), (40, 25), (150, 43)],
)
@pytest.mark.parametrize("chunk_blocks", [2, 100])
def test_checkpoint_reservation_matches_worker(
hash_block_size,
mamba_block_size,
use_eagle,
retention_interval,
num_prompt_tokens,
num_output_tokens,
chunk_blocks,
):
"""The worker exports a checkpoint wherever `compute_mamba_prefill_checkpoints`
says, into whatever block sits at its column. Each chunk, the manager must
either reserve exactly that position and column, or leave the column null so
the export is masked; anything else publishes a hash over the wrong state.
Under sparse retention, only the prefill-end chunk may reserve one.
"""
manager = make_full_mamba_manager(
dcp_world_size=1,
hash_block_size=hash_block_size,
full_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
num_blocks=64,
use_eagle=use_eagle,
num_speculative_blocks=3 if use_eagle else 0,
num_prefill_checkpoint_blocks=1,
)
manager.coordinator.retention_interval = retention_interval
mamba_manager = manager.coordinator.single_type_managers[1]
request = make_request(
"req", list(range(num_prompt_tokens)), hash_block_size, sha256
)
request.append_output_token_ids(list(range(1000, 1000 + num_output_tokens)))
num_tokens = request.num_tokens
prefill_end = max(num_prompt_tokens, num_tokens - 1)
start = 0
while start < num_tokens:
end = min(start + chunk_blocks * mamba_block_size, num_tokens)
assert manager.allocate_slots(request, end - start) is not None
offsets, cols = compute_mamba_prefill_checkpoints(
[end],
[end - start],
hash_block_size=hash_block_size,
mamba_block_size=mamba_block_size,
checkpoint_alignment=16,
drop_eagle_block=use_eagle,
)
reserved = mamba_manager._checkpoints.get(request.request_id)
if reserved is not None:
assert reserved == (start + offsets[0], cols[0])
assert retention_interval is None or end >= prefill_end
elif offsets[0]:
assert mamba_manager.req_to_blocks[request.request_id][cols[0]].is_null
request.num_computed_tokens = end
manager.new_step_starts()
start = end
+337 -34
View File
@@ -52,12 +52,14 @@ from vllm.v1.core.kv_cache_utils import (
hash_block_tokens,
init_none_hash,
is_kv_cache_spec_uniform,
kv_cache_groups_tp_replicas,
make_block_hash_with_group_id,
tensor_data,
to_event_extra_keys,
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_gpu_memory_usage,
get_hisparse_kv_cache_groups,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
@@ -116,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
group = KVCacheGroupSpec(list(specs), group_spec)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
@@ -140,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
),
)
indexer_spec = specs["model.layers.0.self_attn.indexer"]
assert get_hisparse_gpu_memory_usage(config, [group]) == (
indexer_spec.max_memory_usage_bytes(config)
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, [group], available_memory=2**30
config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30
)
assert cache_config.num_blocks == 7
assert cache_config.hisparse_host_num_blocks is not None
@@ -204,6 +198,157 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
assert scheduler_block_size == hash_block_size == gpu_block_size
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
"""Every page needs a host block, so one max_model_len request must fit."""
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4
),
parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1),
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
)
host_page = specs["model.layers.0.self_attn"].page_size_bytes
host_budget = (4 + extra_blocks) * host_page
groups = get_hisparse_kv_cache_groups(config, specs)
if ok:
layout = create_hisparse_layout(config, groups, host_budget=host_budget)
assert layout.host_num_blocks == 4 + extra_blocks
else:
with pytest.raises(ValueError, match="increase host_pool_gib"):
create_hisparse_layout(config, groups, host_budget=host_budget)
@pytest.mark.parametrize(
"max_model_len,num_gpu_blocks,ok",
[
(8192, 160, False),
(8192, 1000, True),
(32768, 656, False),
(32768, 657, True),
(-1, 657, True),
],
)
def test_hisparse_pool_must_fit_max_model_len(
monkeypatch, max_model_len, num_gpu_blocks, ok
):
"""Each resident group takes its own blocks from HiSparse's shared GPU pool,
so a pool that fits only the indexer pages must be rejected at startup.
Resident pages are capped at the in-flight window: at 32768 tokens with 2048
in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks,
plus the null block. Auto-fit (-1) must pick a length that passes the same
check."""
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
specs: dict[str, KVCacheSpec] = {}
for i in range(4):
specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
)
specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
)
config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
config.scheduler_config.max_num_batched_tokens = 2048
# Async scheduling doubles the in-flight window, and with it the bound.
config.scheduler_config.async_scheduling = False
config.attention_config.hisparse_config = HiSparseConfig()
config.model_config.hf_config.index_topk = 128
config.cache_config.num_gpu_blocks_override = num_gpu_blocks
config.cache_config.kv_cache_layout = "BLHNC"
# Workers report the resident/hot caches HiSparse derives.
specs |= {
name: group.kv_cache_spec
for group in get_hisparse_kv_cache_groups(config, specs) or []
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
for name in group.layer_names
}
if ok:
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
else:
with pytest.raises(ValueError, match="max seq len"):
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch):
"""Workers report HiSparse's resident/hot caches, which lay out their own raw
backing; KV cache layout resolution must skip them."""
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
specs: dict[str, KVCacheSpec] = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=64,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.attention_config.hisparse_config = HiSparseConfig()
config.model_config.hf_config.index_topk = 128
reported = specs | {
name: group.kv_cache_spec
for group in get_hisparse_kv_cache_groups(config, specs) or []
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
for name in group.layer_names
}
assert len(reported) > len(specs)
layouts = []
for worker_specs in (specs, reported):
config.cache_config.kv_cache_layout = None
layouts.append(
resolve_kv_cache_layout(
config, [["BLHNC", "LBNHC"]], list(worker_specs.values())
)
)
assert layouts[0] == layouts[1]
def test_hisparse_rejects_deepseek_v4():
full_specs = {
"model.layers.0.attn": MLAAttentionSpec(
@@ -214,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4():
model_version="deepseek_v4",
)
}
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
assert full_uniform is not None
group = KVCacheGroupSpec(list(full_specs), full_uniform)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
@@ -224,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4():
)
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
create_hisparse_layout(
config,
[group],
host_budget=2**30,
)
get_hisparse_kv_cache_groups(config, full_specs)
@pytest.fixture(autouse=True)
@@ -822,12 +960,12 @@ def test_generate_block_hash_extra_keys():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (("hash1", 0),)
assert extra_keys == (("mm", "hash1", 0),)
assert next_mm_idx == 1
# Test with partial overlap
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
assert extra_keys == (("hash1", -3),)
assert extra_keys == (("mm", "hash1", -3),)
assert next_mm_idx == 1
# Test with no overlap
@@ -837,7 +975,7 @@ def test_generate_block_hash_extra_keys():
# Test with multiple extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
assert extra_keys == (("hash1", 0), ("hash2", 10))
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
assert next_mm_idx == 2
@@ -865,9 +1003,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
# salt is added for the first token
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
assert extra_keys == ("salt",)
assert extra_keys == (("cache_salt", "salt"),)
# no salt added for other tokens
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
@@ -888,7 +1026,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
# Test with no extra keys
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
assert extra_keys == (("hash1", 0), "salt")
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
assert next_mm_idx == 1
@@ -906,13 +1044,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
expected_embeds = prompt_embeds[0:5]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
# Test with prompt embeds for the second block
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
expected_embeds = prompt_embeds[5:10]
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
assert extra_keys == (expected_hash,)
assert extra_keys == (("prompt_embeds", expected_hash),)
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
@@ -976,13 +1114,80 @@ def test_generate_block_hash_extra_keys_lora():
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys == ("test_lora_adapter",)
assert extra_keys == (("lora", "test_lora_adapter", "/path/to/lora"),)
request.lora_request = None
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
assert extra_keys is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
lora_request = LoRARequest(
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
)
lora_req = Request(
request_id="lora",
prompt_token_ids=[0, 1, 2],
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
lora_request=lora_request,
block_hasher=get_request_block_hasher(3, hash_fn),
)
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_lora_path_change_changes_block_hashes(hash_fn):
"""Re-pointing a LoRA name at another adapter must not reuse its blocks."""
def make_lora_request(lora_path: str) -> Request:
return Request(
request_id=lora_path,
prompt_token_ids=[0, 1, 2],
sampling_params=SamplingParams(max_tokens=1),
pooling_params=None,
lora_request=LoRARequest(
lora_name="foo", lora_int_id=1, lora_path=lora_path
),
block_hasher=get_request_block_hasher(3, hash_fn),
)
first = make_lora_request("/path/to/lora_a")
second = make_lora_request("/path/to/lora_b")
assert first.block_hashes[0] != second.block_hashes[0]
def test_to_event_extra_keys_keeps_untagged_event_format():
"""KV events keep publishing the extra-key shapes consumers already parse."""
request = make_request(
"0",
list(range(10)),
mm_positions=[PlaceholderRange(offset=2, length=3)],
mm_hashes=["hash1"],
cache_salt="salt",
)
request.lora_request = LoRARequest(
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
)
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
assert extra_keys == (
("lora", "adapter", "/path/to/lora"),
("mm", "hash1", 2),
("cache_salt", "salt"),
)
event_keys = ("adapter", ("hash1", 2), "salt")
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
assert to_event_extra_keys([]) is None
assert to_event_extra_keys(None) is None
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
def test_hash_block_tokens(hash_fn):
parent_block_hash = BlockHash(b"123")
@@ -1013,9 +1218,11 @@ def test_request_block_hasher(hash_fn):
block_hashes = request.block_hashes
assert len(block_hashes) == 2
assert block_hashes[0] == hash_fn(
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
)
assert block_hashes[1] == hash_fn(
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
)
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
@@ -1043,7 +1250,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
)
expected_second_hash = hash_fn(
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
)
assert incremental.block_hashes[1] == expected_second_hash
assert incremental.block_hashes == fresh.block_hashes
@@ -3182,6 +3389,7 @@ def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
return MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=576,
dtype=torch.float32,
cache_dtype_str=cache_dtype_str,
@@ -3192,6 +3400,7 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
return SlidingWindowMLASpec(
block_size=16,
num_kv_heads=1,
max_tp_shards=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
@@ -3657,7 +3866,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(block1_embeds_hash,),
(("prompt_embeds", block1_embeds_hash),),
)
)
assert block_hashes[0] == expected_hash1
@@ -3669,7 +3878,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(block2_embeds_hash,),
(("prompt_embeds", block2_embeds_hash),),
)
)
assert block_hashes[1] == expected_hash2
@@ -3706,7 +3915,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
kv_cache_utils.NONE_HASH,
tuple(prompt_token_ids[:block_size]),
(("hash1", 0), block1_embeds_hash),
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
)
)
assert block_hashes[0] == expected_hash1
@@ -3718,7 +3927,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
(
block_hashes[0],
tuple(prompt_token_ids[block_size:num_tokens]),
(("hash2", 0), block2_embeds_hash),
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
)
)
assert block_hashes[1] == expected_hash2
@@ -4545,3 +4754,97 @@ def test_trailing_layer_fallback_requires_exact_partition():
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
assert not any(g.is_eagle_group for g in trimmed)
_GQA_SPEC = FullAttentionSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32
)
_GQA_SWA_SPEC = SlidingWindowSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32, sliding_window=128
)
@pytest.mark.parametrize(
"specs,expected",
[
pytest.param({"l.0": new_mla_spec(), "l.1": new_mla_spec()}, 4, id="mla"),
pytest.param(
{
"l.0": new_mla_spec(),
"l.1": new_mla_spec(),
"s.0": new_swa_mla_spec(),
"s.1": new_swa_mla_spec(),
},
4,
id="mla-and-swa-mla",
),
pytest.param({"l.0": _GQA_SPEC, "l.1": _GQA_SPEC}, 1, id="gqa"),
pytest.param(
{"l.0": new_mla_spec(), "l.1": _GQA_SPEC},
1,
id="mla-and-gqa-uniform-group",
),
pytest.param(
{
"l.0": new_mla_spec(),
"l.1": new_mla_spec(),
"s.0": _GQA_SWA_SPEC,
"s.1": _GQA_SWA_SPEC,
},
1,
id="mla-and-gqa-swa",
),
],
)
def test_kv_tp_replicas(monkeypatch, specs, expected):
"""Resolved per layer before scheduler flattening hides mixed groups."""
from vllm.config import ParallelConfig
from vllm.platforms import current_platform
monkeypatch.setattr(current_platform, "device_count", lambda: 4)
parallel_config = ParallelConfig(tensor_parallel_size=4)
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=16), parallel_config=parallel_config
)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
mem = sum(s.page_size_bytes for s in specs.values()) * 10
configs = get_kv_cache_configs(vllm_config, [specs], [mem])
assert configs[0].kv_tp_replicas == expected
scheduler = generate_scheduler_kv_cache_config(configs)
assert scheduler.kv_tp_replicas == expected
@pytest.mark.parametrize(
"specs,tp_size,dcp_size,expected",
[
pytest.param([new_mla_spec()], 8, 1, 8, id="mla"),
pytest.param([new_mla_spec()], 8, 2, 1, id="mla-dcp"),
pytest.param(
[replace(new_mla_spec(), max_tp_shards=None)], 8, 1, 1, id="mla-unset"
),
pytest.param([replace(_GQA_SPEC, max_tp_shards=2)], 8, 1, 4, id="gqa-partial"),
pytest.param([replace(_GQA_SPEC, max_tp_shards=8)], 4, 1, 1, id="gqa-sharded"),
pytest.param(
[new_mla_spec(), replace(_GQA_SWA_SPEC, max_tp_shards=4)],
8,
1,
2,
id="mla-and-gqa-partial",
),
pytest.param(
[
HiddenStateCacheSpec(
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32
)
],
8,
1,
1,
id="hidden-state",
),
],
)
def test_kv_cache_groups_tp_replicas(specs, tp_size, dcp_size, expected):
"""Replicas are the gcd of each layer's tp_size // max_tp_shards."""
groups = [KVCacheGroupSpec([f"l.{i}"], spec) for i, spec in enumerate(specs)]
assert kv_cache_groups_tp_replicas(groups, tp_size, dcp_size) == expected
+398 -86
View File
@@ -168,6 +168,7 @@ def make_hisparse_kv_cache_config(
host_num_blocks: int,
*,
transfer_device_cache: bool = False,
num_resident_groups: int = 1,
) -> KVCacheConfig:
source_spec = FullAttentionSpec(
block_size=HISPARSE_BLOCK_SIZE,
@@ -210,6 +211,13 @@ def make_hisparse_kv_cache_config(
enable_kv_transfer=False,
)
)
for group_idx in range(1, num_resident_groups):
groups.extend(
replace(
group, layer_names=[f"{name}_{group_idx}" for name in group.layer_names]
)
for group in groups[2:4]
)
return KVCacheConfig(
num_blocks=num_blocks,
hisparse_host_num_blocks=host_num_blocks,
@@ -225,6 +233,7 @@ def make_hisparse_kv_cache_manager(
max_model_len: int = 128,
max_in_flight_tokens: int | None = None,
enable_caching: bool = False,
num_prefill_lookahead: int = 0,
**config_kwargs,
) -> KVCacheManager:
config = make_hisparse_kv_cache_config(num_blocks, host_num_blocks, **config_kwargs)
@@ -234,9 +243,53 @@ def make_hisparse_kv_cache_manager(
max_in_flight_tokens=max_in_flight_tokens,
enable_caching=enable_caching,
hash_block_size=HISPARSE_BLOCK_SIZE,
num_prefill_lookahead=num_prefill_lookahead,
)
def _allocate_scheduled(
manager: KVCacheManager,
request: Request,
num_new_tokens: int,
*,
num_new_computed_tokens: int = 0,
num_external_computed_tokens: int = 0,
**kwargs,
) -> KVCacheBlocks | None:
"""``allocate_slots`` plus the per-step residency work the connector runs."""
blocks = manager.allocate_slots(
request,
num_new_tokens,
num_new_computed_tokens=num_new_computed_tokens,
num_external_computed_tokens=num_external_computed_tokens,
**kwargs,
)
if blocks is None or kwargs.get("delay_cache_blocks"):
return blocks
num_computed_tokens = min(
request.num_computed_tokens
+ num_new_computed_tokens
+ num_external_computed_tokens,
manager.max_model_len,
)
num_tokens = min(num_computed_tokens + num_new_tokens, request.num_tokens)
get_hisparse_coordinator(manager).advance_scheduled(
[(request.request_id, num_tokens)]
)
return blocks
def test_hisparse_does_not_write_back_reprefillable_tokens():
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
request = make_request(
"request", list(range(2 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
assert _allocate_scheduled(manager, request, 2 * HISPARSE_BLOCK_SIZE)
transfers = get_hisparse_coordinator(manager).build_offload_command().page_transfers
assert len(transfers) == 1
def test_hisparse_builds_dma_row_mirrors_across_pages():
manager = make_hisparse_kv_cache_manager(32, 16)
request = make_request(
@@ -245,7 +298,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
HISPARSE_BLOCK_SIZE,
sha256,
)
assert manager.allocate_slots(request, num_new_tokens=32) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None
coordinator = get_hisparse_coordinator(manager)
resident_blocks = coordinator.resident_managers[0].req_to_blocks[request.request_id]
host_blocks = coordinator.host_manager.req_to_blocks[request.request_id]
@@ -308,14 +361,14 @@ def test_hisparse_reports_when_context_is_fully_resident():
manager = make_hisparse_kv_cache_manager(32, 16)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
coordinator = get_hisparse_coordinator(manager)
scheduled = ((request.request_id, len(tokens) - 1, 1),)
assert coordinator.all_context_pages_resident(scheduled)
for hot_manager in coordinator.hot_managers:
hot_manager.require_hot(request.request_id)
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
_publish_hisparse_pages(manager)
pool = manager.block_pool
pool.get_new_blocks(pool.get_num_free_blocks())
@@ -333,7 +386,9 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers
spill_counts = {spill.transfer_id: 1 for spill in spills}
get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts)
@@ -348,7 +403,8 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
assert num_local == 2 * HISPARSE_BLOCK_SIZE
assert [len(group_blocks) for group_blocks in blocks.blocks] == [3, 2, 0, 0]
allocated = manager.allocate_slots(
allocated = _allocate_scheduled(
manager,
resumed,
num_new_tokens=1,
num_new_computed_tokens=num_local,
@@ -379,7 +435,9 @@ def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix():
)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers
spill_counts = {spill.transfer_id: 1 for spill in spills}
get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts)
@@ -410,7 +468,7 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(64))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=64) is not None
assert _allocate_scheduled(manager, original, num_new_tokens=64) is not None
_publish_hisparse_pages(manager)
host, indexer, _, _ = manager.get_blocks(original.request_id).blocks
missing_indexer = indexer[1].block_id
@@ -457,7 +515,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
assert asynchronous == (external > 0)
assert [len(group) for group in blocks.blocks] == [3, 1, 0, 0]
assert (
manager.allocate_slots(
_allocate_scheduled(
manager,
resumed,
int(external == 0),
delay_cache_blocks=True,
@@ -486,7 +545,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
def allocate_external_prefix(
manager: KVCacheManager, request: Request, num_tokens: int
) -> KVCacheBlocks | None:
return manager.allocate_slots(
return _allocate_scheduled(
manager,
request,
num_new_tokens=0,
num_external_computed_tokens=num_tokens,
@@ -538,7 +598,7 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert manager.allocate_slots(request, num_new_tokens=1) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)])
# A pending restore must not expose uninitialized GPU copies to prefix hits.
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
@@ -583,15 +643,15 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
32, 32, max_model_len=256, enable_caching=enable_caching
)
first = make_request("first", list(range(160)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(first, num_new_tokens=128) is not None
assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None
_publish_hisparse_pages(manager)
first.num_computed_tokens = 128
pool = manager.block_pool
held = pool.get_new_blocks(pool.get_num_free_blocks() - 3)
assert manager.allocate_slots(first, num_new_tokens=1) is not None
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256)
manager.allocate_slots(second, num_new_tokens=16)
_allocate_scheduled(manager, second, num_new_tokens=16)
coordinator = get_hisparse_coordinator(manager)
scheduled = (("first", 128, 1),)
assert coordinator.all_context_pages_resident(scheduled)
@@ -600,7 +660,7 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
# Once another allocation finishes, A can acquire its hot buffer and
# safely release its clean pages for other requests to reuse.
pool.free_blocks(held)
assert manager.allocate_slots(first, num_new_tokens=1) is not None
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
assert all(m.has_hot("first") for m in coordinator.hot_managers)
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
@@ -611,21 +671,24 @@ def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
"""Without room for a hot buffer, admission must defer instead of losing KV."""
manager = make_hisparse_kv_cache_manager(18, 18, max_model_len=160)
first = make_request("first", list(range(128)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(first, num_new_tokens=128) is not None
assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None
_publish_hisparse_pages(manager)
assert manager.block_pool.get_num_free_blocks() == 1
first.num_computed_tokens = 128
assert manager.allocate_slots(first, num_new_tokens=1) is None
assert _allocate_scheduled(manager, first, num_new_tokens=1) is None
assert get_hisparse_coordinator(manager).all_context_pages_resident(
(("first", 127, 1),)
)
manager.free(first)
second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(second, num_new_tokens=16) is not None
assert _allocate_scheduled(manager, second, num_new_tokens=16) is not None
def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
@pytest.mark.parametrize("num_resident_groups", [1, 3])
def test_hisparse_materializes_prefix_without_allocating_hot_blocks(
num_resident_groups,
):
"""A host prefix becomes visible only when every page is durable.
Completing a later page first must not expose a prefix with a hole.
@@ -634,10 +697,11 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
32,
16,
enable_caching=True,
num_resident_groups=num_resident_groups,
)
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
request = make_request("resident", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
blocks = manager.get_block_ids(request.request_id)
assert len(blocks[2]) == 2
assert blocks[3] == []
@@ -675,19 +739,21 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
assert blocks[3] == []
def test_hisparse_materialization_respects_per_step_spill_budget():
@pytest.mark.parametrize("num_resident_groups", [1, 3])
def test_hisparse_materialization_respects_per_step_spill_budget(num_resident_groups):
"""Prefix publication must not bypass the configured spill batch limit."""
manager = make_hisparse_kv_cache_manager(
32,
16,
enable_caching=True,
num_resident_groups=num_resident_groups,
)
coordinator = get_hisparse_coordinator(manager)
coordinator.max_spill_pages = 1
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
request = make_request("bounded", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
first = coordinator.build_offload_command().page_transfers
assert len(first) == 1
@@ -705,7 +771,9 @@ def test_hisparse_deferred_free_retains_host_blocks_until_fence():
host_free = host_pool.get_num_free_blocks()
device_free = manager.block_pool.get_num_free_blocks()
request = make_request("free", list(range(16)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 16, delay_cache_blocks=True) is not None
assert (
_allocate_scheduled(manager, request, 16, delay_cache_blocks=True) is not None
)
blocks = manager.pop_blocks_for_free(request)
assert host_pool.get_num_free_blocks() < host_free
assert blocks and any(host_pool.blocks[b.block_id] is b for b in blocks)
@@ -782,14 +850,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
assert required == 4
@pytest.mark.parametrize(
"full_sequence_must_fit,host_num_blocks,admitted",
[(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)],
)
def test_hisparse_async_admission_requires_only_import_destinations(
full_sequence_must_fit, host_num_blocks, admitted, tmp_path
@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)])
def test_hisparse_async_admission_requires_full_prompt_host_pages(
host_num_blocks, admitted, tmp_path
):
"""Imports need host destinations, but do not reserve future prefill pages."""
"""An async load is admitted only if its whole prompt's host pages fit.
The in-flight prefill's host page is not free, so the waiting request's
four prompt pages need six host blocks including the null block.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
@@ -804,7 +873,7 @@ def test_hisparse_async_admission_requires_only_import_destinations(
manager = make_hisparse_kv_cache_manager(32, host_num_blocks)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
scheduler.scheduler_reserve_full_isl = full_sequence_must_fit
assert scheduler.scheduler_reserve_full_isl
inflight = make_request(
"inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
@@ -822,6 +891,52 @@ def test_hisparse_async_admission_requires_only_import_destinations(
assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted
def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path):
"""Admitted async loads must not strand each other on host capacity.
Waiting requests keep their blocks and are never preempted, so two imports
that each hold a host page while together needing more than the pool has
left would never be scheduled again. HiSparse relies on full-ISL admission:
a load is admitted only if its whole prompt's host pages fit, so the newest
admitted load can always finish and free host blocks for the others.
"""
from .utils import create_scheduler, mock_kv
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
scheduler = create_scheduler(
model=str(tmp_path),
skip_tokenizer_init=True,
max_model_len=128,
use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True),
)
# One null block plus three usable host blocks.
manager = make_hisparse_kv_cache_manager(32, 4)
scheduler.kv_cache_manager = manager
scheduler.kv_cache_config = manager.kv_cache_config
assert scheduler.scheduler_reserve_full_isl
first = make_request(
"first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
)
second = make_request(
"second",
list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)),
HISPARSE_BLOCK_SIZE,
sha256,
)
for request in (first, second):
scheduler.add_request(request)
scheduler.schedule()
admitted = {r.request_id for r in scheduler._inflight_prefills}
scheduler.finished_recving_kv_req_ids.update(admitted)
num_scheduled_tokens = 0
for _ in range(4):
num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens
assert num_scheduled_tokens > 0
@pytest.mark.parametrize(
"evictable,local_tokens,free_blocks,admitted",
[
@@ -845,58 +960,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow(
assert host_pool.get_num_free_blocks() == free_blocks
assert (
coordinator.host_manager.get_num_blocks_to_allocate(
"waiting", 64, [hit], 32, local_tokens, 64
"waiting", 32, [hit], 32, local_tokens, 32
)
<= host_pool.get_num_free_blocks()
) == admitted
@pytest.mark.parametrize("enable_caching", [False, True])
def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching):
"""Pages without a host destination must survive GPU cache reclamation."""
manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching)
def test_hisparse_host_exhaustion_defers_allocation(enable_caching):
"""Pages are only allocated with host backing, so every one can be reclaimed."""
manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching)
donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, donor, 16) is not None
request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 64) is not None
free_gpu = manager.block_pool.get_num_free_blocks()
assert manager.allocate_slots(request, 64) is None
assert manager.block_pool.get_num_free_blocks() == free_gpu
# The donor's in-flight write-back holds its host block until acked.
manager.free(donor)
coordinator = get_hisparse_coordinator(manager)
assert coordinator.has_pending_reclamation()
assert manager.allocate_slots(request, 64) is None
_publish_hisparse_pages(manager)
assert not coordinator.has_pending_reclamation()
assert _allocate_scheduled(manager, request, 64) is not None
host = coordinator.host_manager
assert host is not None
assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [
False,
True,
True,
True,
]
assert not any(b.is_null for b in host.req_to_blocks[request.request_id])
_publish_hisparse_pages(manager)
for hot in coordinator.hot_managers:
hot.require_hot(request.request_id)
hot.allocate_new_blocks(request.request_id, 64, 64)
coordinator.update_residency(request.request_id)
assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1}
for resident in coordinator.resident_managers:
for block in resident.req_to_blocks[request.request_id][1:]:
assert not block.is_null and block.ref_cnt > 0
assert coordinator.request_states[request.request_id].valid_pages == {0}
pool = coordinator.get_host_block_pool()
assert pool is not None
manager.free(request)
assert pool.get_num_free_blocks() == 1
def test_hisparse_host_cow_takes_priority_over_new_pages():
"""The last host block must hold the private tail, not a fresh page."""
manager = make_hisparse_kv_cache_manager(32, 3)
coordinator = get_hisparse_coordinator(manager)
host = coordinator.host_manager
assert host is not None
source = host.block_pool.get_new_blocks(1)[0]
host.req_to_blocks["cow"] = [source]
host._partial_hit_reqs["cow"] = (0, source)
host.allocate_new_blocks("cow", 32, 32)
private, missing = host.req_to_blocks["cow"]
assert not private.is_null and private is not source
assert missing.is_null
assert host.take_host_cow_copies() == [(source, private)]
assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [
0,
0,
1,
1,
]
@pytest.mark.parametrize("failed", [False, True])
@@ -938,7 +1042,7 @@ def test_hisparse_recomputation_does_not_complete_failed_host_import():
# allocation reaches the original import boundary before the worker runs.
request.num_computed_tokens = 16
manager.cache_blocks(request, 16)
assert manager.allocate_slots(request, 16) is not None
assert _allocate_scheduled(manager, request, 16) is not None
state = get_hisparse_coordinator(manager).request_states.get(request.request_id)
assert state is None or 1 not in state.valid_pages
@@ -959,12 +1063,12 @@ def test_hisparse_resident_request_can_grow_without_hot_capacity():
"""Running resident history must not be mistaken for a new host-prefix hit."""
manager = make_hisparse_kv_cache_manager(8, 16)
request = make_request("resident", list(range(48)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_new_tokens=32) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None
request.num_computed_tokens = 32
# Three free blocks fit the next indexer/resident pages, but not a hot region.
assert manager.block_pool.get_num_free_blocks() == 3
assert manager.allocate_slots(request, num_new_tokens=1) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
assert manager.get_block_ids(request.request_id)[3] == []
@@ -1014,7 +1118,7 @@ def test_hisparse_events_report_host_and_device_placement():
enable_kv_cache_events=True,
)
request = make_request("request", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 64) is not None
assert _allocate_scheduled(manager, request, 64) is not None
_publish_hisparse_pages(manager)
stored = [
@@ -1077,7 +1181,9 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
_publish_hisparse_pages(manager)
original_resident_ids = [
block.block_id for block in manager.get_blocks("original").blocks[2]
@@ -1087,7 +1193,8 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
computed, num_computed, _ = manager.get_computed_blocks(resumed)
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
assert manager.allocate_slots(
assert _allocate_scheduled(
manager,
resumed,
num_new_tokens=len(tokens) - num_computed,
num_new_computed_tokens=num_computed,
@@ -1103,19 +1210,71 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
)
@pytest.mark.parametrize("free_blocks,adopted_pages", [(7, []), (10, [0, 1])])
def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
free_blocks, adopted_pages
):
"""A host prefix hit must not starve its own allocation of free blocks.
Admission counts free GPU copies as free, but adopting a copy pins it. When
adoption ran before the hit's allocation, it took blocks the allocation was
promised and the pool ran dry (``Cannot get N free blocks``). The allocation
now goes first, evicting a prefix's copies tail first; the copies it leaves
are adopted.
"""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
_publish_hisparse_pages(manager)
copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]]
manager.free(original)
pool = manager.block_pool
pool.get_new_blocks(pool.get_num_free_blocks() - free_blocks)
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
computed, num_computed, _ = manager.get_computed_blocks(resumed)
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
assert _allocate_scheduled(
manager,
resumed,
num_new_tokens=len(tokens) - num_computed,
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resumed_blocks = manager.get_blocks("resumed").blocks
assert [block.block_id for block in resumed_blocks[2][:3]] == [
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
for page in range(3)
]
gpu_ids = [
block.block_id
for group in resumed_blocks[1:]
for block in group
if not block.is_null
]
assert len(gpu_ids) == len(set(gpu_ids))
def test_hisparse_host_backed_request_accepts_local_prefix_hit():
"""Local group-completion hits need no external host import allocation."""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
_publish_hisparse_pages(manager)
manager.free(original)
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
computed, num_computed, _ = manager.get_computed_blocks(resumed)
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
assert manager.allocate_slots(
assert _allocate_scheduled(
manager,
resumed,
num_new_tokens=len(tokens) - num_computed,
num_new_computed_tokens=num_computed,
@@ -1129,7 +1288,9 @@ def test_hisparse_finished_request_leaves_free_copies():
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
_publish_hisparse_pages(manager)
manager.free(original)
@@ -1142,12 +1303,151 @@ def test_hisparse_finished_request_leaves_free_copies():
assert not coordinator.spills_to_send
@pytest.mark.parametrize(
"status,expected_cached_pages",
[
(RequestStatus.FINISHED_STOPPED, 2),
(RequestStatus.FINISHED_ABORTED, 2),
(RequestStatus.RUNNING, 0),
],
ids=["stopped", "aborted", "running"],
)
def test_hisparse_cleanup_publishes_only_finalized_terminal_pages(
status, expected_cached_pages
):
"""Only terminal cleanup may publish completed, finalized pages."""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
coordinator = get_hisparse_coordinator(manager)
request = make_request("terminal", list(range(33)), HISPARSE_BLOCK_SIZE, sha256)
request.spec_token_ids = list(range(33, 49))
assert (
_allocate_scheduled(manager, request, 49, num_lookahead_tokens=16) is not None
)
host_blocks = list(manager.get_blocks(request.request_id).blocks[0])
command = coordinator.build_offload_command()
counts = {transfer.transfer_id: 1 for transfer in command.page_transfers}
assert len(counts) == 2
request.status = status
manager.free(request)
coordinator.update_spills(counts, counts)
assert all(
block.block_hash is not None for block in host_blocks[:expected_cached_pages]
)
assert all(
block.block_hash is None for block in host_blocks[expected_cached_pages:]
)
assert all(block.ref_cnt == 0 for block in host_blocks)
assert not coordinator.has_pending_work()
@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)])
def test_hisparse_prefill_reads_host_once_it_fills_admission_window(
num_pages, reads_host
):
"""A prefill switches to host reads once it fills its admission window.
Admission caps each request's resident pages at the in-flight window,
assuming older pages move to host, but pinned pages were released only
once free blocks fell below a fixed watermark. A chunked prefill larger
than the watermark outgrew the pool first and was preempted.
"""
window_pages = 4
manager = make_hisparse_kv_cache_manager(
32,
16,
enable_caching=True,
max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE,
)
coordinator = get_hisparse_coordinator(manager)
tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE))
request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert (
_allocate_scheduled(manager, request, num_pages * HISPARSE_BLOCK_SIZE)
is not None
)
pool = coordinator.gpu_pool
assert pool is not None
assert pool.get_num_free_blocks() >= coordinator.transition_watermark
hot_manager = coordinator.hot_managers[0]
assert (request.request_id in hot_manager.hot_required) == reads_host
def test_hisparse_preempted_prefill_resumes_from_durable_prefix():
"""A prefill preempted mid-way must resume from the pages already on host.
Host writes trail a chunked prefill by a chunk, and host pages were only
published once the whole computed prefix was durable. Each chunk moved
that target past the finished writes, so nothing was published before the
prefill ended: a prefill preempted each time it outgrew the GPU pool
recomputed from the start forever.
"""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
coordinator = get_hisparse_coordinator(manager)
chunk = 2 * HISPARSE_BLOCK_SIZE
tokens = list(range(3 * chunk))
request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, chunk) is not None
request.num_computed_tokens = chunk
first_chunk_writes = coordinator.build_offload_command().page_transfers
assert _allocate_scheduled(manager, request, chunk) is not None
request.num_computed_tokens = 2 * chunk
acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes}
coordinator.update_spills(acks, acks)
manager.free(request)
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.get_computed_blocks(resumed)[1] == chunk
def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity():
"""Old completions retain their pages through pressure and request-ID reuse."""
manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True)
coordinator = get_hisparse_coordinator(manager)
host_pool = coordinator.get_host_block_pool()
tokens = list(range(2 * HISPARSE_BLOCK_SIZE + 1))
request = make_request("reused", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
old_blocks = list(manager.get_blocks(request.request_id).blocks[0])
command = coordinator.build_offload_command()
counts = {transfer.transfer_id: 2 for transfer in command.page_transfers}
partial = dict.fromkeys(counts, 1)
first_completions = partial.copy()
first_completions[next(iter(counts))] = 2
coordinator.update_spills(counts, first_completions)
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
manager.free(request)
replacement = make_request(
"reused", list(range(100, 117)), HISPARSE_BLOCK_SIZE, sha256
)
assert _allocate_scheduled(manager, replacement, 17) is not None
new_blocks = list(manager.get_blocks(replacement.request_id).blocks[0])
pressure = host_pool.get_new_blocks(host_pool.get_num_free_blocks())
assert not {block.block_id for block in old_blocks[:2]} & {
block.block_id for block in pressure + new_blocks
}
# Only the page every worker finished writing is published.
assert old_blocks[0].block_hash is not None
assert all(block.block_hash is None for block in old_blocks[1:])
coordinator.update_spills({}, partial)
repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE
assert all(block.block_hash is None for block in new_blocks)
assert all(block.ref_cnt == 0 for block in old_blocks[:2])
coordinator.update_spills({}, partial)
assert all(block.ref_cnt == 0 for block in old_blocks[:2])
host_pool.free_blocks(pressure)
manager.free(replacement)
_publish_hisparse_pages(manager)
assert not coordinator.has_pending_work()
def test_hisparse_reset_prefix_cache_drops_copies():
"""Reset must forget GPU copies along with the host hashes they mirror."""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
_publish_hisparse_pages(manager)
manager.free(request)
@@ -1162,7 +1462,9 @@ def test_hisparse_reused_copy_is_not_adopted():
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
assert (
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
)
_publish_hisparse_pages(manager)
indexer_blocks = list(manager.get_blocks("original").blocks[1])
manager.free(original)
@@ -1175,7 +1477,8 @@ def test_hisparse_reused_copy_is_not_adopted():
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
computed, num_computed, _ = manager.get_computed_blocks(resumed)
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
assert manager.allocate_slots(
assert _allocate_scheduled(
manager,
resumed,
num_new_tokens=len(tokens) - num_computed,
num_new_computed_tokens=num_computed,
@@ -2901,21 +3204,21 @@ def test_mm_prefix_caching():
(
kv_cache_utils.NONE_HASH,
tuple(all_token_ids[:block_size]),
(("aaa", 11),),
(("mm", "aaa", 11),),
)
)
assert block_hashes[1] == sha256(
(
block_hashes[0],
tuple(all_token_ids[block_size : block_size * 2]),
(("aaa", -5), ("bbb", 14)),
(("mm", "aaa", -5), ("mm", "bbb", 14)),
)
)
assert block_hashes[2] == sha256(
(
block_hashes[1],
tuple(all_token_ids[block_size * 2 : block_size * 3]),
(("bbb", -2),),
(("mm", "bbb", -2),),
)
)
@@ -2938,7 +3241,7 @@ def test_mm_prefix_caching():
(
block_hashes[2],
tuple(all_token_ids[3 * block_size :] + [8] * 5),
(("ccc", 0),),
(("mm", "ccc", 0),),
)
)
@@ -2984,7 +3287,11 @@ def test_cache_key_salting():
block_hashes = req0.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt1"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
@@ -3029,7 +3336,11 @@ def test_cache_key_salting():
block_hashes = req2.block_hashes
assert len(block_hashes) == 3
assert block_hashes[0] == sha256(
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
(
kv_cache_utils.NONE_HASH,
tuple(token_ids[:block_size]),
(("cache_salt", "salt2"),),
)
)
assert block_hashes[1] == sha256(
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
@@ -5067,7 +5378,8 @@ def test_can_fit_full_sequence_hisparse_caps_resident_pages():
)
assert (
manager.allocate_slots(
_allocate_scheduled(
manager,
request,
num_new_tokens=HISPARSE_BLOCK_SIZE,
full_sequence_must_fit=True,
@@ -5927,7 +6239,7 @@ def test_device_eviction_keeps_host_prefix():
"""Device eviction must not target a same-numbered host block."""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
request = make_request("producer", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, 64) is not None
assert _allocate_scheduled(manager, request, 64) is not None
_publish_hisparse_pages(manager)
host, indexer, _, _ = manager.get_blocks(request.request_id).blocks
device_block = indexer[0]
+176 -54
View File
@@ -39,9 +39,11 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.diffusion_scheduler import (
DiffusionAsyncScheduler,
DiffusionScheduler,
diffusion_canvas_width,
)
from vllm.v1.core.sched.interface import PauseState
@@ -436,6 +438,52 @@ def test_schedule_partial_requests():
assert requests[2].request_id not in output.num_scheduled_tokens
def test_encoder_only_prompt_longer_than_budget_is_chunked():
"""The engine switches chunked prefill off for an encoder-only instance,
but the token budget must still split its prompt across steps instead of
leaving the request waiting forever."""
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
max_model_len=4096,
mm_encoder_only=True,
)
# EngineCore turns this off after config validation whenever the instance
# holds no KV cache, which is every encoder-only instance.
scheduler.scheduler_config.enable_chunked_prefill = False
(request,) = create_requests(
num_requests=1,
num_tokens=2500,
mm_positions=[[PlaceholderRange(offset=100, length=600)]],
)
scheduler.add_request(request)
def advance(output):
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
first = scheduler.schedule()
assert first.num_scheduled_tokens[request.request_id] == 1024
assert request.request_id in first.scheduled_encoder_inputs
advance(first)
second = scheduler.schedule()
assert second.num_scheduled_tokens[request.request_id] == 1024
advance(second)
third = scheduler.schedule()
assert third.num_scheduled_tokens[request.request_id] == 452
@pytest.mark.parametrize("has_running", [True, False])
def test_schedule_prefills_gating(has_running: bool):
"""DP prefill-balancing gate: when `throttle_prefills` is True, a new
@@ -861,6 +909,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
assert all(out.new_sampling_mask.offsets is None for out in outputs)
def test_update_from_output_routes_multi_position_sampling_masks():
scheduler = create_scheduler()
scheduler.return_sampling_mask = True
requests = create_requests(num_requests=2, max_tokens=10)
for req in requests:
req.num_computed_tokens = req.num_tokens
scheduler.requests[req.request_id] = req
scheduler.running.append(req)
req.status = RequestStatus.RUNNING
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
num_scheduled_tokens={req.request_id: 1 for req in requests},
total_num_scheduled_tokens=2,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
model_output = ModelRunnerOutput(
req_ids=[req.request_id for req in requests],
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
sampled_token_ids=[[1, 2], [3, 4, 5]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
sampling_masks=SamplingMaskLists(
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
offsets=np.array([0, 2, 3, 6, 7, 9]),
cu_num_generated_tokens=[0, 2, 5],
),
)
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
[[1, 6], [2]],
[[3, 7, 8], [4], [5, 9]],
]
def test_stop_via_update_from_output():
"""Test stopping behavior through update_from_output."""
scheduler = create_scheduler(num_speculative_tokens=1)
@@ -1562,13 +1653,7 @@ def test_scheduler_reset_prefix_cache():
assert not scheduler.reset_prefix_cache()
scheduler.aux_output_connector.reset.assert_not_called()
with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"):
scheduler.reset_prefix_cache(reset_running_requests=True)
# pause(mode="keep") also waits for scheduled model outputs to drain.
scheduler.set_pause_state(PauseState.PAUSED_ALL)
with pytest.raises(RuntimeError, match="model output is in flight"):
scheduler.reset_prefix_cache(reset_running_requests=True)
# Pause completes pending model outputs before the caller resets the scheduler.
for request in requests:
request.num_in_flight_tokens = 0
@@ -6667,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
num_blocks: int = 100,
supports_divergent_hits: bool = True,
) -> Scheduler:
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
model_config = ModelConfig(
model="facebook/opt-125m",
trust_remote_code=True,
@@ -6688,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
cache_config=CacheConfig(
block_size=block_size,
enable_prefix_caching=True,
mamba_cache_mode="all",
mamba_cache_mode="align",
),
kv_transfer_config=KVTransferConfig(
kv_connector="MockKVConnector",
@@ -6720,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
mamba_cache_mode="align",
),
),
],
@@ -6736,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
)
def _seed_hybrid_prefix(
manager: KVCacheManager, num_blocks: int, block_size: int
) -> tuple[list[int], list[int]]:
"""Prefill a prefix one block per step so that align-mode Mamba caches
the state at every block boundary. Returns the FA and Mamba block ids."""
[fill] = create_requests(
num_requests=1,
num_tokens=num_blocks * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
assert num_computed == 0
mamba_ids = []
for i in range(num_blocks):
manager.allocate_slots(
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
)
fill.num_computed_tokens += block_size
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
mamba_ids.append(group_mamba_ids[i])
manager.free(fill)
return fa_ids, mamba_ids
@pytest.mark.parametrize(
"matched_tokens,expected_num_computed",
[
@@ -6761,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix so both groups cache all four boundaries
# (mamba cache mode "all" caches every block's state densely).
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
fa_ids = [b.block_id for b in blocks.blocks[0]]
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
# Seed a 4-block prefix so both groups cache all four boundaries.
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Evict the FA tail and the middle mamba states; block 0 (both groups)
# and the deep mamba state at block 3 survive.
@@ -6801,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
@pytest.mark.parametrize(
@@ -6837,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix in both groups.
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
@@ -6871,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
@@ -7115,14 +7199,15 @@ def diffusion_model_runner(monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
def _diffusion_scheduler(async_scheduling: bool = True, **kwargs) -> DiffusionScheduler:
scheduler_cls = DiffusionAsyncScheduler if async_scheduling else DiffusionScheduler
scheduler = create_scheduler(
async_scheduling=True,
async_scheduling=async_scheduling,
diffusion_canvas_length=8,
scheduler_cls=DiffusionAsyncScheduler,
scheduler_cls=scheduler_cls,
**kwargs,
)
assert isinstance(scheduler, DiffusionAsyncScheduler)
assert isinstance(scheduler, scheduler_cls)
return scheduler
@@ -7133,7 +7218,7 @@ def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
async_scheduling=async_scheduling, diffusion_canvas_length=8
).vllm_config.scheduler_config
assert config.get_scheduler_cls() is (
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
DiffusionAsyncScheduler if config.async_scheduling else DiffusionScheduler
)
@@ -7155,10 +7240,13 @@ def test_diffusion_scheduler_narrows_the_canvas_per_request():
@pytest.mark.parametrize("structured", [False, True])
@pytest.mark.parametrize("async_scheduling", [False, True])
@pytest.mark.usefixtures("diffusion_model_runner")
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
def test_diffusion_scheduler_trims_full_width_worker_drafts(
structured, async_scheduling
):
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
scheduler = _diffusion_scheduler()
scheduler = _diffusion_scheduler(async_scheduling=async_scheduling)
wide = _diffusion_request("wide", {})
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
for request in (wide, narrow):
@@ -7173,8 +7261,12 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
)
tokens = list(range(8)) if structured else [-1] * 8
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
output = scheduler.schedule()
scheduler.update_draft_token_ids_in_output(drafts, output)
if async_scheduling:
output = scheduler.schedule()
scheduler.update_draft_token_ids_in_output(drafts, output)
else:
scheduler.update_draft_token_ids(drafts)
output = scheduler.schedule()
assert output.scheduled_spec_decode_tokens == {
"wide": tokens,
@@ -7186,6 +7278,36 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
@pytest.mark.parametrize(
"sampled, expected_status",
[
pytest.param([], RequestStatus.RUNNING, id="denoising"),
pytest.param([1] * 4, RequestStatus.FINISHED_LENGTH_CAPPED, id="final"),
],
)
@pytest.mark.usefixtures("diffusion_model_runner")
def test_sync_diffusion_step_commits_only_emitted_tokens(sampled, expected_status):
"""An empty denoising step reuses the canvas; a completed read emits it."""
scheduler = _diffusion_scheduler(async_scheduling=False)
request = _diffusion_request(
"read", {"diffusion_canvas_length": 4, "diffusion_read_only": True}
)
request.max_tokens = request.sampling_params.max_tokens = 4
scheduler.add_request(request)
_model_output(scheduler, scheduler.schedule(), [[]])
scheduler.update_draft_token_ids(
DraftTokenIds([request.request_id], [list(range(8))])
)
output = scheduler.schedule()
assert output.num_scheduled_tokens == {request.request_id: 4}
_model_output(scheduler, output, [sampled])
assert request.num_computed_tokens == request.num_prompt_tokens + len(sampled)
assert list(request.output_token_ids) == sampled
assert request.status == expected_status
@pytest.mark.usefixtures("diffusion_model_runner")
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
scheduler = _diffusion_scheduler()
+3
View File
@@ -60,6 +60,7 @@ def create_scheduler(
long_prefill_token_threshold: int = 0,
long_prefill_token_threshold_adaptive: bool = False,
disable_chunked_mm_input: bool = False,
mm_encoder_only: bool = False,
use_kv_connector: None | bool | str | MockKVConfig = None,
kv_role: str = "kv_both",
num_blocks: int = 10000,
@@ -109,6 +110,8 @@ def create_scheduler(
# SchedulerConfig one, so both must agree.
max_model_len=max_model_len,
)
if mm_encoder_only:
model_config.multimodal_config.mm_encoder_only = True
if use_ec_connector and ec_role == "ec_producer":
model_config.multimodal_config = MultiModalConfig()
if max_model_len is None: