Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -13,6 +13,9 @@ import torch
|
||||
|
||||
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import SupportsHMA
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
compute_mamba_prefill_checkpoints,
|
||||
)
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
KVCacheBlockCopy,
|
||||
@@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
)
|
||||
|
||||
|
||||
@@ -686,6 +690,81 @@ def test_transient_checkpoint_evicts_retained_boundary_hash():
|
||||
assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_len", [120, 104, 96])
|
||||
def test_prompt_end_checkpoint_survives_block_aligned_cache_count(prompt_len):
|
||||
"""At the default match unit the hybrid coordinator hands ``cache_blocks``
|
||||
a block-aligned token count, which is below the prompt length whenever the
|
||||
prompt is not block-aligned. The prompt-end checkpoint is the replay
|
||||
boundary and must stay published under retention_interval=0; only a
|
||||
mid-prompt checkpoint is transient.
|
||||
"""
|
||||
hash_block_size = mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = 0
|
||||
assert not manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
request = make_request("producer", list(range(prompt_len)), hash_block_size, sha256)
|
||||
assert manager.allocate_slots(request, request.num_tokens) is not None
|
||||
checkpoint_position = get_mamba_prefill_checkpoint_position(
|
||||
prompt_len, hash_block_size, drop_eagle_block=False
|
||||
)
|
||||
checkpoint_hash = request.block_hashes[checkpoint_position // hash_block_size - 1]
|
||||
hit = manager.block_pool.get_cached_block(checkpoint_hash, [1])
|
||||
assert hit is not None
|
||||
assert hit[0].block_hash_num_tokens == checkpoint_position
|
||||
|
||||
manager.free(request)
|
||||
replay = make_request("replay", list(range(prompt_len)), hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(replay)
|
||||
assert num_computed == checkpoint_position
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retention_interval,reserved", [(None, True), (0, False)])
|
||||
def test_intermediate_chunk_checkpoint_reserved_only_under_dense_retention(
|
||||
retention_interval, reserved
|
||||
):
|
||||
"""A checkpoint inside a chunk that does not finish the prefill is only
|
||||
publishable under dense retention. With retention_interval=0 it would be
|
||||
evicted as transient, so the block is not reserved in the first place;
|
||||
the prefill-end checkpoint is reserved either way.
|
||||
"""
|
||||
hash_block_size = 16
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=32,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = retention_interval
|
||||
mamba_manager = manager.coordinator.single_type_managers[1]
|
||||
|
||||
request = make_request("producer", list(range(240)), hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
free_before = manager.block_pool.get_num_free_blocks()
|
||||
|
||||
# Mid-prompt chunk: the chunk-keyed position (112) is inside it. Blocks:
|
||||
# the attention blocks, the running mamba block, plus the checkpoint.
|
||||
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
|
||||
assert (request.request_id in mamba_manager._checkpoints) == reserved
|
||||
allocated = free_before - manager.block_pool.get_num_free_blocks()
|
||||
assert allocated == 128 // hash_block_size + 1 + int(reserved)
|
||||
|
||||
request.num_computed_tokens = 128
|
||||
manager.new_step_starts()
|
||||
# Final chunk: the prefill-end checkpoint at 224 is reserved either way.
|
||||
assert manager.allocate_slots(request, 112) is not None
|
||||
assert mamba_manager._checkpoints[request.request_id][0] == 224
|
||||
end_hash = request.block_hashes[224 // hash_block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
|
||||
|
||||
|
||||
def test_eagle_block_aligned_checkpoint_replaces_newer_hash():
|
||||
hash_block_size = mamba_block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
@@ -2531,3 +2610,105 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill():
|
||||
offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])]
|
||||
assert 2 * block_size in offered
|
||||
assert req0.num_prompt_tokens < 2 * block_size
|
||||
|
||||
|
||||
def test_prompt_end_checkpoint_survives_sparse_retention_with_coarse_hash():
|
||||
"""With hash_block_size == mamba block size (no fine-grained matching), the
|
||||
hybrid coordinator caches the final chunk only up to its last full block,
|
||||
so the managers see ``num_tokens < num_prompt_tokens`` there too. The
|
||||
prompt-end checkpoint must still be published under retention_interval=0,
|
||||
otherwise the next request that extends this prompt misses the prefix.
|
||||
"""
|
||||
block_size = 32
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=block_size,
|
||||
full_block_size=block_size,
|
||||
mamba_block_size=block_size,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = 0
|
||||
|
||||
# 240 tokens: not block aligned, prompt-end checkpoint at 224.
|
||||
request = make_request("producer", list(range(240)), block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(request)
|
||||
assert manager.allocate_slots(request, 128, num_computed, computed_blocks)
|
||||
request.num_computed_tokens = 128
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(request, 112) is not None
|
||||
|
||||
end_hash = request.block_hashes[224 // block_size - 1]
|
||||
assert manager.block_pool.get_cached_block(end_hash, [1]) is not None
|
||||
|
||||
# A request extending the prompt resumes from that checkpoint.
|
||||
follower = make_request("follower", list(range(240)) + [7] * 40, block_size, sha256)
|
||||
_, num_hit, _ = manager.get_computed_blocks(follower)
|
||||
assert num_hit == 224
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hash_block_size,mamba_block_size,use_eagle",
|
||||
[(32, 32, False), (16, 32, False), (32, 32, True)],
|
||||
)
|
||||
@pytest.mark.parametrize("retention_interval", [0, None])
|
||||
@pytest.mark.parametrize(
|
||||
"num_prompt_tokens,num_output_tokens",
|
||||
[(104, 0), (200, 0), (40, 24), (40, 25), (150, 43)],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_blocks", [2, 100])
|
||||
def test_checkpoint_reservation_matches_worker(
|
||||
hash_block_size,
|
||||
mamba_block_size,
|
||||
use_eagle,
|
||||
retention_interval,
|
||||
num_prompt_tokens,
|
||||
num_output_tokens,
|
||||
chunk_blocks,
|
||||
):
|
||||
"""The worker exports a checkpoint wherever `compute_mamba_prefill_checkpoints`
|
||||
says, into whatever block sits at its column. Each chunk, the manager must
|
||||
either reserve exactly that position and column, or leave the column null so
|
||||
the export is masked; anything else publishes a hash over the wrong state.
|
||||
Under sparse retention, only the prefill-end chunk may reserve one.
|
||||
"""
|
||||
manager = make_full_mamba_manager(
|
||||
dcp_world_size=1,
|
||||
hash_block_size=hash_block_size,
|
||||
full_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
num_blocks=64,
|
||||
use_eagle=use_eagle,
|
||||
num_speculative_blocks=3 if use_eagle else 0,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
)
|
||||
manager.coordinator.retention_interval = retention_interval
|
||||
mamba_manager = manager.coordinator.single_type_managers[1]
|
||||
|
||||
request = make_request(
|
||||
"req", list(range(num_prompt_tokens)), hash_block_size, sha256
|
||||
)
|
||||
request.append_output_token_ids(list(range(1000, 1000 + num_output_tokens)))
|
||||
num_tokens = request.num_tokens
|
||||
prefill_end = max(num_prompt_tokens, num_tokens - 1)
|
||||
|
||||
start = 0
|
||||
while start < num_tokens:
|
||||
end = min(start + chunk_blocks * mamba_block_size, num_tokens)
|
||||
assert manager.allocate_slots(request, end - start) is not None
|
||||
offsets, cols = compute_mamba_prefill_checkpoints(
|
||||
[end],
|
||||
[end - start],
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_block_size=mamba_block_size,
|
||||
checkpoint_alignment=16,
|
||||
drop_eagle_block=use_eagle,
|
||||
)
|
||||
reserved = mamba_manager._checkpoints.get(request.request_id)
|
||||
if reserved is not None:
|
||||
assert reserved == (start + offsets[0], cols[0])
|
||||
assert retention_interval is None or end >= prefill_end
|
||||
elif offsets[0]:
|
||||
assert mamba_manager.req_to_blocks[request.request_id][cols[0]].is_null
|
||||
request.num_computed_tokens = end
|
||||
manager.new_step_starts()
|
||||
start = end
|
||||
|
||||
@@ -52,12 +52,14 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
is_kv_cache_spec_uniform,
|
||||
kv_cache_groups_tp_replicas,
|
||||
make_block_hash_with_group_id,
|
||||
tensor_data,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import (
|
||||
create_hisparse_layout,
|
||||
get_hisparse_gpu_memory_usage,
|
||||
get_hisparse_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
@@ -116,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
|
||||
assert group_spec is not None
|
||||
group = KVCacheGroupSpec(list(specs), group_spec)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
@@ -140,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
|
||||
),
|
||||
)
|
||||
indexer_spec = specs["model.layers.0.self_attn.indexer"]
|
||||
assert get_hisparse_gpu_memory_usage(config, [group]) == (
|
||||
indexer_spec.max_memory_usage_bytes(config)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
config, [group], available_memory=2**30
|
||||
config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30
|
||||
)
|
||||
assert cache_config.num_blocks == 7
|
||||
assert cache_config.hisparse_host_num_blocks is not None
|
||||
@@ -204,6 +198,157 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
assert scheduler_block_size == hash_block_size == gpu_block_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)])
|
||||
def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok):
|
||||
"""Every page needs a host block, so one max_model_len request must fit."""
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4
|
||||
),
|
||||
parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1),
|
||||
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
|
||||
)
|
||||
host_page = specs["model.layers.0.self_attn"].page_size_bytes
|
||||
host_budget = (4 + extra_blocks) * host_page
|
||||
groups = get_hisparse_kv_cache_groups(config, specs)
|
||||
|
||||
if ok:
|
||||
layout = create_hisparse_layout(config, groups, host_budget=host_budget)
|
||||
assert layout.host_num_blocks == 4 + extra_blocks
|
||||
else:
|
||||
with pytest.raises(ValueError, match="increase host_pool_gib"):
|
||||
create_hisparse_layout(config, groups, host_budget=host_budget)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"max_model_len,num_gpu_blocks,ok",
|
||||
[
|
||||
(8192, 160, False),
|
||||
(8192, 1000, True),
|
||||
(32768, 656, False),
|
||||
(32768, 657, True),
|
||||
(-1, 657, True),
|
||||
],
|
||||
)
|
||||
def test_hisparse_pool_must_fit_max_model_len(
|
||||
monkeypatch, max_model_len, num_gpu_blocks, ok
|
||||
):
|
||||
"""Each resident group takes its own blocks from HiSparse's shared GPU pool,
|
||||
so a pool that fits only the indexer pages must be rejected at startup.
|
||||
Resident pages are capped at the in-flight window: at 32768 tokens with 2048
|
||||
in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks,
|
||||
plus the null block. Auto-fit (-1) must pick a length that passes the same
|
||||
check."""
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for i in range(4):
|
||||
specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
)
|
||||
specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
)
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
config.scheduler_config.max_num_batched_tokens = 2048
|
||||
# Async scheduling doubles the in-flight window, and with it the bound.
|
||||
config.scheduler_config.async_scheduling = False
|
||||
config.attention_config.hisparse_config = HiSparseConfig()
|
||||
config.model_config.hf_config.index_topk = 128
|
||||
config.cache_config.num_gpu_blocks_override = num_gpu_blocks
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
# Workers report the resident/hot caches HiSparse derives.
|
||||
specs |= {
|
||||
name: group.kv_cache_spec
|
||||
for group in get_hisparse_kv_cache_groups(config, specs) or []
|
||||
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
for name in group.layer_names
|
||||
}
|
||||
|
||||
if ok:
|
||||
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
|
||||
else:
|
||||
with pytest.raises(ValueError, match="max seq len"):
|
||||
kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34])
|
||||
|
||||
|
||||
def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch):
|
||||
"""Workers report HiSparse's resident/hot caches, which lay out their own raw
|
||||
backing; KV cache layout resolution must skip them."""
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False)
|
||||
specs: dict[str, KVCacheSpec] = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
is_index_group_leader=True,
|
||||
),
|
||||
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
|
||||
block_size=64,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
cache_role=SparseCacheRole.INDEXER,
|
||||
),
|
||||
}
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.attention_config.hisparse_config = HiSparseConfig()
|
||||
config.model_config.hf_config.index_topk = 128
|
||||
reported = specs | {
|
||||
name: group.kv_cache_spec
|
||||
for group in get_hisparse_kv_cache_groups(config, specs) or []
|
||||
if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec))
|
||||
for name in group.layer_names
|
||||
}
|
||||
assert len(reported) > len(specs)
|
||||
|
||||
layouts = []
|
||||
for worker_specs in (specs, reported):
|
||||
config.cache_config.kv_cache_layout = None
|
||||
layouts.append(
|
||||
resolve_kv_cache_layout(
|
||||
config, [["BLHNC", "LBNHC"]], list(worker_specs.values())
|
||||
)
|
||||
)
|
||||
assert layouts[0] == layouts[1]
|
||||
|
||||
|
||||
def test_hisparse_rejects_deepseek_v4():
|
||||
full_specs = {
|
||||
"model.layers.0.attn": MLAAttentionSpec(
|
||||
@@ -214,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4():
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
}
|
||||
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
|
||||
assert full_uniform is not None
|
||||
group = KVCacheGroupSpec(list(full_specs), full_uniform)
|
||||
config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
|
||||
@@ -224,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4():
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
|
||||
create_hisparse_layout(
|
||||
config,
|
||||
[group],
|
||||
host_budget=2**30,
|
||||
)
|
||||
get_hisparse_kv_cache_groups(config, full_specs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -822,12 +960,12 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0),)
|
||||
assert extra_keys == (("mm", "hash1", 0),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with partial overlap
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0)
|
||||
assert extra_keys == (("hash1", -3),)
|
||||
assert extra_keys == (("mm", "hash1", -3),)
|
||||
assert next_mm_idx == 1
|
||||
|
||||
# Test with no overlap
|
||||
@@ -837,7 +975,7 @@ def test_generate_block_hash_extra_keys():
|
||||
|
||||
# Test with multiple extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0)
|
||||
assert extra_keys == (("hash1", 0), ("hash2", 10))
|
||||
assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10))
|
||||
assert next_mm_idx == 2
|
||||
|
||||
|
||||
@@ -865,9 +1003,9 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# salt is added for the first token
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0)
|
||||
assert extra_keys == ("salt",)
|
||||
assert extra_keys == (("cache_salt", "salt"),)
|
||||
|
||||
# no salt added for other tokens
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0)
|
||||
@@ -888,7 +1026,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
|
||||
|
||||
# Test with no extra keys
|
||||
extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0)
|
||||
assert extra_keys == (("hash1", 0), "salt")
|
||||
assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt"))
|
||||
assert next_mm_idx == 1
|
||||
|
||||
|
||||
@@ -906,13 +1044,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds():
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
expected_embeds = prompt_embeds[0:5]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
# Test with prompt embeds for the second block
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0)
|
||||
expected_embeds = prompt_embeds[5:10]
|
||||
expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest()
|
||||
assert extra_keys == (expected_hash,)
|
||||
assert extra_keys == (("prompt_embeds", expected_hash),)
|
||||
|
||||
|
||||
def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch):
|
||||
@@ -976,13 +1114,80 @@ def test_generate_block_hash_extra_keys_lora():
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys == ("test_lora_adapter",)
|
||||
assert extra_keys == (("lora", "test_lora_adapter", "/path/to/lora"),)
|
||||
|
||||
request.lora_request = None
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0)
|
||||
assert extra_keys is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn):
|
||||
"""A cache_salt equal to a LoRA name must not share that LoRA's blocks."""
|
||||
lora_request = LoRARequest(
|
||||
lora_name="foo", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
lora_req = Request(
|
||||
request_id="lora",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
lora_request=lora_request,
|
||||
block_hasher=get_request_block_hasher(3, hash_fn),
|
||||
)
|
||||
salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo")
|
||||
|
||||
assert lora_req.block_hashes[0] != salted_req.block_hashes[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_lora_path_change_changes_block_hashes(hash_fn):
|
||||
"""Re-pointing a LoRA name at another adapter must not reuse its blocks."""
|
||||
|
||||
def make_lora_request(lora_path: str) -> Request:
|
||||
return Request(
|
||||
request_id=lora_path,
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=SamplingParams(max_tokens=1),
|
||||
pooling_params=None,
|
||||
lora_request=LoRARequest(
|
||||
lora_name="foo", lora_int_id=1, lora_path=lora_path
|
||||
),
|
||||
block_hasher=get_request_block_hasher(3, hash_fn),
|
||||
)
|
||||
|
||||
first = make_lora_request("/path/to/lora_a")
|
||||
second = make_lora_request("/path/to/lora_b")
|
||||
|
||||
assert first.block_hashes[0] != second.block_hashes[0]
|
||||
|
||||
|
||||
def test_to_event_extra_keys_keeps_untagged_event_format():
|
||||
"""KV events keep publishing the extra-key shapes consumers already parse."""
|
||||
request = make_request(
|
||||
"0",
|
||||
list(range(10)),
|
||||
mm_positions=[PlaceholderRange(offset=2, length=3)],
|
||||
mm_hashes=["hash1"],
|
||||
cache_salt="salt",
|
||||
)
|
||||
request.lora_request = LoRARequest(
|
||||
lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora"
|
||||
)
|
||||
|
||||
extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0)
|
||||
|
||||
assert extra_keys == (
|
||||
("lora", "adapter", "/path/to/lora"),
|
||||
("mm", "hash1", 2),
|
||||
("cache_salt", "salt"),
|
||||
)
|
||||
event_keys = ("adapter", ("hash1", 2), "salt")
|
||||
assert to_event_extra_keys([extra_keys, None]) == [event_keys, None]
|
||||
assert to_event_extra_keys([]) is None
|
||||
assert to_event_extra_keys(None) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
def test_hash_block_tokens(hash_fn):
|
||||
parent_block_hash = BlockHash(b"123")
|
||||
@@ -1013,9 +1218,11 @@ def test_request_block_hasher(hash_fn):
|
||||
block_hashes = request.block_hashes
|
||||
assert len(block_hashes) == 2
|
||||
assert block_hashes[0] == hash_fn(
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),))
|
||||
(kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn(
|
||||
(block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),))
|
||||
)
|
||||
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor])
|
||||
@@ -1043,7 +1250,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_
|
||||
)
|
||||
|
||||
expected_second_hash = hash_fn(
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2)))
|
||||
(incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2)))
|
||||
)
|
||||
assert incremental.block_hashes[1] == expected_second_hash
|
||||
assert incremental.block_hashes == fresh.block_hashes
|
||||
@@ -3182,6 +3389,7 @@ def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
return MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
cache_dtype_str=cache_dtype_str,
|
||||
@@ -3192,6 +3400,7 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None):
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=head_size,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
@@ -3657,7 +3866,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(block1_embeds_hash,),
|
||||
(("prompt_embeds", block1_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3669,7 +3878,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes]
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(block2_embeds_hash,),
|
||||
(("prompt_embeds", block2_embeds_hash),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
@@ -3706,7 +3915,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(prompt_token_ids[:block_size]),
|
||||
(("hash1", 0), block1_embeds_hash),
|
||||
(("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[0] == expected_hash1
|
||||
@@ -3718,7 +3927,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(prompt_token_ids[block_size:num_tokens]),
|
||||
(("hash2", 0), block2_embeds_hash),
|
||||
(("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == expected_hash2
|
||||
@@ -4545,3 +4754,97 @@ def test_trailing_layer_fallback_requires_exact_partition():
|
||||
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
|
||||
|
||||
assert not any(g.is_eagle_group for g in trimmed)
|
||||
|
||||
|
||||
_GQA_SPEC = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32
|
||||
)
|
||||
_GQA_SWA_SPEC = SlidingWindowSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32, sliding_window=128
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"specs,expected",
|
||||
[
|
||||
pytest.param({"l.0": new_mla_spec(), "l.1": new_mla_spec()}, 4, id="mla"),
|
||||
pytest.param(
|
||||
{
|
||||
"l.0": new_mla_spec(),
|
||||
"l.1": new_mla_spec(),
|
||||
"s.0": new_swa_mla_spec(),
|
||||
"s.1": new_swa_mla_spec(),
|
||||
},
|
||||
4,
|
||||
id="mla-and-swa-mla",
|
||||
),
|
||||
pytest.param({"l.0": _GQA_SPEC, "l.1": _GQA_SPEC}, 1, id="gqa"),
|
||||
pytest.param(
|
||||
{"l.0": new_mla_spec(), "l.1": _GQA_SPEC},
|
||||
1,
|
||||
id="mla-and-gqa-uniform-group",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"l.0": new_mla_spec(),
|
||||
"l.1": new_mla_spec(),
|
||||
"s.0": _GQA_SWA_SPEC,
|
||||
"s.1": _GQA_SWA_SPEC,
|
||||
},
|
||||
1,
|
||||
id="mla-and-gqa-swa",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_kv_tp_replicas(monkeypatch, specs, expected):
|
||||
"""Resolved per layer before scheduler flattening hides mixed groups."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 4)
|
||||
parallel_config = ParallelConfig(tensor_parallel_size=4)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=16), parallel_config=parallel_config
|
||||
)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
mem = sum(s.page_size_bytes for s in specs.values()) * 10
|
||||
configs = get_kv_cache_configs(vllm_config, [specs], [mem])
|
||||
assert configs[0].kv_tp_replicas == expected
|
||||
scheduler = generate_scheduler_kv_cache_config(configs)
|
||||
assert scheduler.kv_tp_replicas == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"specs,tp_size,dcp_size,expected",
|
||||
[
|
||||
pytest.param([new_mla_spec()], 8, 1, 8, id="mla"),
|
||||
pytest.param([new_mla_spec()], 8, 2, 1, id="mla-dcp"),
|
||||
pytest.param(
|
||||
[replace(new_mla_spec(), max_tp_shards=None)], 8, 1, 1, id="mla-unset"
|
||||
),
|
||||
pytest.param([replace(_GQA_SPEC, max_tp_shards=2)], 8, 1, 4, id="gqa-partial"),
|
||||
pytest.param([replace(_GQA_SPEC, max_tp_shards=8)], 4, 1, 1, id="gqa-sharded"),
|
||||
pytest.param(
|
||||
[new_mla_spec(), replace(_GQA_SWA_SPEC, max_tp_shards=4)],
|
||||
8,
|
||||
1,
|
||||
2,
|
||||
id="mla-and-gqa-partial",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
HiddenStateCacheSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32
|
||||
)
|
||||
],
|
||||
8,
|
||||
1,
|
||||
1,
|
||||
id="hidden-state",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_kv_cache_groups_tp_replicas(specs, tp_size, dcp_size, expected):
|
||||
"""Replicas are the gcd of each layer's tp_size // max_tp_shards."""
|
||||
groups = [KVCacheGroupSpec([f"l.{i}"], spec) for i, spec in enumerate(specs)]
|
||||
assert kv_cache_groups_tp_replicas(groups, tp_size, dcp_size) == expected
|
||||
|
||||
@@ -168,6 +168,7 @@ def make_hisparse_kv_cache_config(
|
||||
host_num_blocks: int,
|
||||
*,
|
||||
transfer_device_cache: bool = False,
|
||||
num_resident_groups: int = 1,
|
||||
) -> KVCacheConfig:
|
||||
source_spec = FullAttentionSpec(
|
||||
block_size=HISPARSE_BLOCK_SIZE,
|
||||
@@ -210,6 +211,13 @@ def make_hisparse_kv_cache_config(
|
||||
enable_kv_transfer=False,
|
||||
)
|
||||
)
|
||||
for group_idx in range(1, num_resident_groups):
|
||||
groups.extend(
|
||||
replace(
|
||||
group, layer_names=[f"{name}_{group_idx}" for name in group.layer_names]
|
||||
)
|
||||
for group in groups[2:4]
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
hisparse_host_num_blocks=host_num_blocks,
|
||||
@@ -225,6 +233,7 @@ def make_hisparse_kv_cache_manager(
|
||||
max_model_len: int = 128,
|
||||
max_in_flight_tokens: int | None = None,
|
||||
enable_caching: bool = False,
|
||||
num_prefill_lookahead: int = 0,
|
||||
**config_kwargs,
|
||||
) -> KVCacheManager:
|
||||
config = make_hisparse_kv_cache_config(num_blocks, host_num_blocks, **config_kwargs)
|
||||
@@ -234,9 +243,53 @@ def make_hisparse_kv_cache_manager(
|
||||
max_in_flight_tokens=max_in_flight_tokens,
|
||||
enable_caching=enable_caching,
|
||||
hash_block_size=HISPARSE_BLOCK_SIZE,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
|
||||
|
||||
def _allocate_scheduled(
|
||||
manager: KVCacheManager,
|
||||
request: Request,
|
||||
num_new_tokens: int,
|
||||
*,
|
||||
num_new_computed_tokens: int = 0,
|
||||
num_external_computed_tokens: int = 0,
|
||||
**kwargs,
|
||||
) -> KVCacheBlocks | None:
|
||||
"""``allocate_slots`` plus the per-step residency work the connector runs."""
|
||||
blocks = manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_computed_tokens=num_new_computed_tokens,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
if blocks is None or kwargs.get("delay_cache_blocks"):
|
||||
return blocks
|
||||
num_computed_tokens = min(
|
||||
request.num_computed_tokens
|
||||
+ num_new_computed_tokens
|
||||
+ num_external_computed_tokens,
|
||||
manager.max_model_len,
|
||||
)
|
||||
num_tokens = min(num_computed_tokens + num_new_tokens, request.num_tokens)
|
||||
get_hisparse_coordinator(manager).advance_scheduled(
|
||||
[(request.request_id, num_tokens)]
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def test_hisparse_does_not_write_back_reprefillable_tokens():
|
||||
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
|
||||
request = make_request(
|
||||
"request", list(range(2 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
assert _allocate_scheduled(manager, request, 2 * HISPARSE_BLOCK_SIZE)
|
||||
transfers = get_hisparse_coordinator(manager).build_offload_command().page_transfers
|
||||
assert len(transfers) == 1
|
||||
|
||||
|
||||
def test_hisparse_builds_dma_row_mirrors_across_pages():
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
request = make_request(
|
||||
@@ -245,7 +298,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
sha256,
|
||||
)
|
||||
assert manager.allocate_slots(request, num_new_tokens=32) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
resident_blocks = coordinator.resident_managers[0].req_to_blocks[request.request_id]
|
||||
host_blocks = coordinator.host_manager.req_to_blocks[request.request_id]
|
||||
@@ -308,14 +361,14 @@ def test_hisparse_reports_when_context_is_fully_resident():
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
scheduled = ((request.request_id, len(tokens) - 1, 1),)
|
||||
assert coordinator.all_context_pages_resident(scheduled)
|
||||
|
||||
for hot_manager in coordinator.hot_managers:
|
||||
hot_manager.require_hot(request.request_id)
|
||||
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
pool = manager.block_pool
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
@@ -333,7 +386,9 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers
|
||||
spill_counts = {spill.transfer_id: 1 for spill in spills}
|
||||
get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts)
|
||||
@@ -348,7 +403,8 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
assert num_local == 2 * HISPARSE_BLOCK_SIZE
|
||||
assert [len(group_blocks) for group_blocks in blocks.blocks] == [3, 2, 0, 0]
|
||||
|
||||
allocated = manager.allocate_slots(
|
||||
allocated = _allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
num_new_tokens=1,
|
||||
num_new_computed_tokens=num_local,
|
||||
@@ -379,7 +435,9 @@ def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix():
|
||||
)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers
|
||||
spill_counts = {spill.transfer_id: 1 for spill in spills}
|
||||
get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts)
|
||||
@@ -410,7 +468,7 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(64))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=64) is not None
|
||||
assert _allocate_scheduled(manager, original, num_new_tokens=64) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
host, indexer, _, _ = manager.get_blocks(original.request_id).blocks
|
||||
missing_indexer = indexer[1].block_id
|
||||
@@ -457,7 +515,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
|
||||
assert asynchronous == (external > 0)
|
||||
assert [len(group) for group in blocks.blocks] == [3, 1, 0, 0]
|
||||
assert (
|
||||
manager.allocate_slots(
|
||||
_allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
int(external == 0),
|
||||
delay_cache_blocks=True,
|
||||
@@ -486,7 +545,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
|
||||
def allocate_external_prefix(
|
||||
manager: KVCacheManager, request: Request, num_tokens: int
|
||||
) -> KVCacheBlocks | None:
|
||||
return manager.allocate_slots(
|
||||
return _allocate_scheduled(
|
||||
manager,
|
||||
request,
|
||||
num_new_tokens=0,
|
||||
num_external_computed_tokens=num_tokens,
|
||||
@@ -538,7 +598,7 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert manager.allocate_slots(request, num_new_tokens=1) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||
assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)])
|
||||
# A pending restore must not expose uninitialized GPU copies to prefix hits.
|
||||
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
|
||||
@@ -583,15 +643,15 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
|
||||
32, 32, max_model_len=256, enable_caching=enable_caching
|
||||
)
|
||||
first = make_request("first", list(range(160)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(first, num_new_tokens=128) is not None
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
first.num_computed_tokens = 128
|
||||
pool = manager.block_pool
|
||||
held = pool.get_new_blocks(pool.get_num_free_blocks() - 3)
|
||||
|
||||
assert manager.allocate_slots(first, num_new_tokens=1) is not None
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
|
||||
second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256)
|
||||
manager.allocate_slots(second, num_new_tokens=16)
|
||||
_allocate_scheduled(manager, second, num_new_tokens=16)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
scheduled = (("first", 128, 1),)
|
||||
assert coordinator.all_context_pages_resident(scheduled)
|
||||
@@ -600,7 +660,7 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
|
||||
# Once another allocation finishes, A can acquire its hot buffer and
|
||||
# safely release its clean pages for other requests to reuse.
|
||||
pool.free_blocks(held)
|
||||
assert manager.allocate_slots(first, num_new_tokens=1) is not None
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
|
||||
assert all(m.has_hot("first") for m in coordinator.hot_managers)
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
assert not coordinator.all_context_pages_resident(scheduled)
|
||||
@@ -611,21 +671,24 @@ def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
|
||||
"""Without room for a hot buffer, admission must defer instead of losing KV."""
|
||||
manager = make_hisparse_kv_cache_manager(18, 18, max_model_len=160)
|
||||
first = make_request("first", list(range(128)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(first, num_new_tokens=128) is not None
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
assert manager.block_pool.get_num_free_blocks() == 1
|
||||
first.num_computed_tokens = 128
|
||||
assert manager.allocate_slots(first, num_new_tokens=1) is None
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=1) is None
|
||||
assert get_hisparse_coordinator(manager).all_context_pages_resident(
|
||||
(("first", 127, 1),)
|
||||
)
|
||||
|
||||
manager.free(first)
|
||||
second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(second, num_new_tokens=16) is not None
|
||||
assert _allocate_scheduled(manager, second, num_new_tokens=16) is not None
|
||||
|
||||
|
||||
def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
|
||||
@pytest.mark.parametrize("num_resident_groups", [1, 3])
|
||||
def test_hisparse_materializes_prefix_without_allocating_hot_blocks(
|
||||
num_resident_groups,
|
||||
):
|
||||
"""A host prefix becomes visible only when every page is durable.
|
||||
|
||||
Completing a later page first must not expose a prefix with a hole.
|
||||
@@ -634,10 +697,11 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
|
||||
32,
|
||||
16,
|
||||
enable_caching=True,
|
||||
num_resident_groups=num_resident_groups,
|
||||
)
|
||||
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("resident", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
blocks = manager.get_block_ids(request.request_id)
|
||||
assert len(blocks[2]) == 2
|
||||
assert blocks[3] == []
|
||||
@@ -675,19 +739,21 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks():
|
||||
assert blocks[3] == []
|
||||
|
||||
|
||||
def test_hisparse_materialization_respects_per_step_spill_budget():
|
||||
@pytest.mark.parametrize("num_resident_groups", [1, 3])
|
||||
def test_hisparse_materialization_respects_per_step_spill_budget(num_resident_groups):
|
||||
"""Prefix publication must not bypass the configured spill batch limit."""
|
||||
manager = make_hisparse_kv_cache_manager(
|
||||
32,
|
||||
16,
|
||||
enable_caching=True,
|
||||
num_resident_groups=num_resident_groups,
|
||||
)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
coordinator.max_spill_pages = 1
|
||||
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("bounded", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
|
||||
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
first = coordinator.build_offload_command().page_transfers
|
||||
assert len(first) == 1
|
||||
|
||||
@@ -705,7 +771,9 @@ def test_hisparse_deferred_free_retains_host_blocks_until_fence():
|
||||
host_free = host_pool.get_num_free_blocks()
|
||||
device_free = manager.block_pool.get_num_free_blocks()
|
||||
request = make_request("free", list(range(16)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, 16, delay_cache_blocks=True) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, request, 16, delay_cache_blocks=True) is not None
|
||||
)
|
||||
blocks = manager.pop_blocks_for_free(request)
|
||||
assert host_pool.get_num_free_blocks() < host_free
|
||||
assert blocks and any(host_pool.blocks[b.block_id] is b for b in blocks)
|
||||
@@ -782,14 +850,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
|
||||
assert required == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_sequence_must_fit,host_num_blocks,admitted",
|
||||
[(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)],
|
||||
)
|
||||
def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
full_sequence_must_fit, host_num_blocks, admitted, tmp_path
|
||||
@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)])
|
||||
def test_hisparse_async_admission_requires_full_prompt_host_pages(
|
||||
host_num_blocks, admitted, tmp_path
|
||||
):
|
||||
"""Imports need host destinations, but do not reserve future prefill pages."""
|
||||
"""An async load is admitted only if its whole prompt's host pages fit.
|
||||
|
||||
The in-flight prefill's host page is not free, so the waiting request's
|
||||
four prompt pages need six host blocks including the null block.
|
||||
"""
|
||||
from .utils import create_scheduler, mock_kv
|
||||
|
||||
(tmp_path / "config.json").write_text(
|
||||
@@ -804,7 +873,7 @@ def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
manager = make_hisparse_kv_cache_manager(32, host_num_blocks)
|
||||
scheduler.kv_cache_manager = manager
|
||||
scheduler.kv_cache_config = manager.kv_cache_config
|
||||
scheduler.scheduler_reserve_full_isl = full_sequence_must_fit
|
||||
assert scheduler.scheduler_reserve_full_isl
|
||||
inflight = make_request(
|
||||
"inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
@@ -822,6 +891,52 @@ def test_hisparse_async_admission_requires_only_import_destinations(
|
||||
assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted
|
||||
|
||||
|
||||
def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path):
|
||||
"""Admitted async loads must not strand each other on host capacity.
|
||||
|
||||
Waiting requests keep their blocks and are never preempted, so two imports
|
||||
that each hold a host page while together needing more than the pool has
|
||||
left would never be scheduled again. HiSparse relies on full-ISL admission:
|
||||
a load is admitted only if its whole prompt's host pages fit, so the newest
|
||||
admitted load can always finish and free host blocks for the others.
|
||||
"""
|
||||
from .utils import create_scheduler, mock_kv
|
||||
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
skip_tokenizer_init=True,
|
||||
max_model_len=128,
|
||||
use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True),
|
||||
)
|
||||
# One null block plus three usable host blocks.
|
||||
manager = make_hisparse_kv_cache_manager(32, 4)
|
||||
scheduler.kv_cache_manager = manager
|
||||
scheduler.kv_cache_config = manager.kv_cache_config
|
||||
assert scheduler.scheduler_reserve_full_isl
|
||||
first = make_request(
|
||||
"first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
second = make_request(
|
||||
"second",
|
||||
list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)),
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
sha256,
|
||||
)
|
||||
for request in (first, second):
|
||||
scheduler.add_request(request)
|
||||
scheduler.schedule()
|
||||
admitted = {r.request_id for r in scheduler._inflight_prefills}
|
||||
|
||||
scheduler.finished_recving_kv_req_ids.update(admitted)
|
||||
num_scheduled_tokens = 0
|
||||
for _ in range(4):
|
||||
num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens
|
||||
assert num_scheduled_tokens > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"evictable,local_tokens,free_blocks,admitted",
|
||||
[
|
||||
@@ -845,58 +960,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow(
|
||||
assert host_pool.get_num_free_blocks() == free_blocks
|
||||
assert (
|
||||
coordinator.host_manager.get_num_blocks_to_allocate(
|
||||
"waiting", 64, [hit], 32, local_tokens, 64
|
||||
"waiting", 32, [hit], 32, local_tokens, 32
|
||||
)
|
||||
<= host_pool.get_num_free_blocks()
|
||||
) == admitted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_caching", [False, True])
|
||||
def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching):
|
||||
"""Pages without a host destination must survive GPU cache reclamation."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching)
|
||||
def test_hisparse_host_exhaustion_defers_allocation(enable_caching):
|
||||
"""Pages are only allocated with host backing, so every one can be reclaimed."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching)
|
||||
donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, donor, 16) is not None
|
||||
request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
free_gpu = manager.block_pool.get_num_free_blocks()
|
||||
assert manager.allocate_slots(request, 64) is None
|
||||
assert manager.block_pool.get_num_free_blocks() == free_gpu
|
||||
|
||||
# The donor's in-flight write-back holds its host block until acked.
|
||||
manager.free(donor)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert coordinator.has_pending_reclamation()
|
||||
assert manager.allocate_slots(request, 64) is None
|
||||
_publish_hisparse_pages(manager)
|
||||
assert not coordinator.has_pending_reclamation()
|
||||
assert _allocate_scheduled(manager, request, 64) is not None
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
]
|
||||
assert not any(b.is_null for b in host.req_to_blocks[request.request_id])
|
||||
_publish_hisparse_pages(manager)
|
||||
for hot in coordinator.hot_managers:
|
||||
hot.require_hot(request.request_id)
|
||||
hot.allocate_new_blocks(request.request_id, 64, 64)
|
||||
coordinator.update_residency(request.request_id)
|
||||
assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1}
|
||||
for resident in coordinator.resident_managers:
|
||||
for block in resident.req_to_blocks[request.request_id][1:]:
|
||||
assert not block.is_null and block.ref_cnt > 0
|
||||
assert coordinator.request_states[request.request_id].valid_pages == {0}
|
||||
pool = coordinator.get_host_block_pool()
|
||||
assert pool is not None
|
||||
manager.free(request)
|
||||
assert pool.get_num_free_blocks() == 1
|
||||
|
||||
|
||||
def test_hisparse_host_cow_takes_priority_over_new_pages():
|
||||
"""The last host block must hold the private tail, not a fresh page."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 3)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
host = coordinator.host_manager
|
||||
assert host is not None
|
||||
source = host.block_pool.get_new_blocks(1)[0]
|
||||
host.req_to_blocks["cow"] = [source]
|
||||
host._partial_hit_reqs["cow"] = (0, source)
|
||||
|
||||
host.allocate_new_blocks("cow", 32, 32)
|
||||
|
||||
private, missing = host.req_to_blocks["cow"]
|
||||
assert not private.is_null and private is not source
|
||||
assert missing.is_null
|
||||
assert host.take_host_cow_copies() == [(source, private)]
|
||||
assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
@@ -938,7 +1042,7 @@ def test_hisparse_recomputation_does_not_complete_failed_host_import():
|
||||
# allocation reaches the original import boundary before the worker runs.
|
||||
request.num_computed_tokens = 16
|
||||
manager.cache_blocks(request, 16)
|
||||
assert manager.allocate_slots(request, 16) is not None
|
||||
assert _allocate_scheduled(manager, request, 16) is not None
|
||||
state = get_hisparse_coordinator(manager).request_states.get(request.request_id)
|
||||
assert state is None or 1 not in state.valid_pages
|
||||
|
||||
@@ -959,12 +1063,12 @@ def test_hisparse_resident_request_can_grow_without_hot_capacity():
|
||||
"""Running resident history must not be mistaken for a new host-prefix hit."""
|
||||
manager = make_hisparse_kv_cache_manager(8, 16)
|
||||
request = make_request("resident", list(range(48)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, num_new_tokens=32) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None
|
||||
request.num_computed_tokens = 32
|
||||
|
||||
# Three free blocks fit the next indexer/resident pages, but not a hot region.
|
||||
assert manager.block_pool.get_num_free_blocks() == 3
|
||||
assert manager.allocate_slots(request, num_new_tokens=1) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None
|
||||
assert manager.get_block_ids(request.request_id)[3] == []
|
||||
|
||||
|
||||
@@ -1014,7 +1118,7 @@ def test_hisparse_events_report_host_and_device_placement():
|
||||
enable_kv_cache_events=True,
|
||||
)
|
||||
request = make_request("request", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
assert _allocate_scheduled(manager, request, 64) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
|
||||
stored = [
|
||||
@@ -1077,7 +1181,9 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
_publish_hisparse_pages(manager)
|
||||
original_resident_ids = [
|
||||
block.block_id for block in manager.get_blocks("original").blocks[2]
|
||||
@@ -1087,7 +1193,8 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
computed, num_computed, _ = manager.get_computed_blocks(resumed)
|
||||
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
|
||||
assert manager.allocate_slots(
|
||||
assert _allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
num_new_tokens=len(tokens) - num_computed,
|
||||
num_new_computed_tokens=num_computed,
|
||||
@@ -1103,19 +1210,71 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("free_blocks,adopted_pages", [(7, []), (10, [0, 1])])
|
||||
def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
|
||||
free_blocks, adopted_pages
|
||||
):
|
||||
"""A host prefix hit must not starve its own allocation of free blocks.
|
||||
|
||||
Admission counts free GPU copies as free, but adopting a copy pins it. When
|
||||
adoption ran before the hit's allocation, it took blocks the allocation was
|
||||
promised and the pool ran dry (``Cannot get N free blocks``). The allocation
|
||||
now goes first, evicting a prefix's copies tail first; the copies it leaves
|
||||
are adopted.
|
||||
"""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
_publish_hisparse_pages(manager)
|
||||
copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]]
|
||||
manager.free(original)
|
||||
pool = manager.block_pool
|
||||
pool.get_new_blocks(pool.get_num_free_blocks() - free_blocks)
|
||||
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
computed, num_computed, _ = manager.get_computed_blocks(resumed)
|
||||
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
|
||||
assert _allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
num_new_tokens=len(tokens) - num_computed,
|
||||
num_new_computed_tokens=num_computed,
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
|
||||
resumed_blocks = manager.get_blocks("resumed").blocks
|
||||
assert [block.block_id for block in resumed_blocks[2][:3]] == [
|
||||
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
|
||||
for page in range(3)
|
||||
]
|
||||
gpu_ids = [
|
||||
block.block_id
|
||||
for group in resumed_blocks[1:]
|
||||
for block in group
|
||||
if not block.is_null
|
||||
]
|
||||
assert len(gpu_ids) == len(set(gpu_ids))
|
||||
|
||||
|
||||
def test_hisparse_host_backed_request_accepts_local_prefix_hit():
|
||||
"""Local group-completion hits need no external host import allocation."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
_publish_hisparse_pages(manager)
|
||||
manager.free(original)
|
||||
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
computed, num_computed, _ = manager.get_computed_blocks(resumed)
|
||||
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
|
||||
assert manager.allocate_slots(
|
||||
assert _allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
num_new_tokens=len(tokens) - num_computed,
|
||||
num_new_computed_tokens=num_computed,
|
||||
@@ -1129,7 +1288,9 @@ def test_hisparse_finished_request_leaves_free_copies():
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
_publish_hisparse_pages(manager)
|
||||
manager.free(original)
|
||||
|
||||
@@ -1142,12 +1303,151 @@ def test_hisparse_finished_request_leaves_free_copies():
|
||||
assert not coordinator.spills_to_send
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status,expected_cached_pages",
|
||||
[
|
||||
(RequestStatus.FINISHED_STOPPED, 2),
|
||||
(RequestStatus.FINISHED_ABORTED, 2),
|
||||
(RequestStatus.RUNNING, 0),
|
||||
],
|
||||
ids=["stopped", "aborted", "running"],
|
||||
)
|
||||
def test_hisparse_cleanup_publishes_only_finalized_terminal_pages(
|
||||
status, expected_cached_pages
|
||||
):
|
||||
"""Only terminal cleanup may publish completed, finalized pages."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
request = make_request("terminal", list(range(33)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
request.spec_token_ids = list(range(33, 49))
|
||||
assert (
|
||||
_allocate_scheduled(manager, request, 49, num_lookahead_tokens=16) is not None
|
||||
)
|
||||
host_blocks = list(manager.get_blocks(request.request_id).blocks[0])
|
||||
command = coordinator.build_offload_command()
|
||||
counts = {transfer.transfer_id: 1 for transfer in command.page_transfers}
|
||||
assert len(counts) == 2
|
||||
request.status = status
|
||||
manager.free(request)
|
||||
coordinator.update_spills(counts, counts)
|
||||
assert all(
|
||||
block.block_hash is not None for block in host_blocks[:expected_cached_pages]
|
||||
)
|
||||
assert all(
|
||||
block.block_hash is None for block in host_blocks[expected_cached_pages:]
|
||||
)
|
||||
assert all(block.ref_cnt == 0 for block in host_blocks)
|
||||
assert not coordinator.has_pending_work()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)])
|
||||
def test_hisparse_prefill_reads_host_once_it_fills_admission_window(
|
||||
num_pages, reads_host
|
||||
):
|
||||
"""A prefill switches to host reads once it fills its admission window.
|
||||
|
||||
Admission caps each request's resident pages at the in-flight window,
|
||||
assuming older pages move to host, but pinned pages were released only
|
||||
once free blocks fell below a fixed watermark. A chunked prefill larger
|
||||
than the watermark outgrew the pool first and was preempted.
|
||||
"""
|
||||
window_pages = 4
|
||||
manager = make_hisparse_kv_cache_manager(
|
||||
32,
|
||||
16,
|
||||
enable_caching=True,
|
||||
max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE,
|
||||
)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert (
|
||||
_allocate_scheduled(manager, request, num_pages * HISPARSE_BLOCK_SIZE)
|
||||
is not None
|
||||
)
|
||||
pool = coordinator.gpu_pool
|
||||
assert pool is not None
|
||||
assert pool.get_num_free_blocks() >= coordinator.transition_watermark
|
||||
|
||||
hot_manager = coordinator.hot_managers[0]
|
||||
assert (request.request_id in hot_manager.hot_required) == reads_host
|
||||
|
||||
|
||||
def test_hisparse_preempted_prefill_resumes_from_durable_prefix():
|
||||
"""A prefill preempted mid-way must resume from the pages already on host.
|
||||
|
||||
Host writes trail a chunked prefill by a chunk, and host pages were only
|
||||
published once the whole computed prefix was durable. Each chunk moved
|
||||
that target past the finished writes, so nothing was published before the
|
||||
prefill ended: a prefill preempted each time it outgrew the GPU pool
|
||||
recomputed from the start forever.
|
||||
"""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
chunk = 2 * HISPARSE_BLOCK_SIZE
|
||||
tokens = list(range(3 * chunk))
|
||||
request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, chunk) is not None
|
||||
request.num_computed_tokens = chunk
|
||||
first_chunk_writes = coordinator.build_offload_command().page_transfers
|
||||
assert _allocate_scheduled(manager, request, chunk) is not None
|
||||
request.num_computed_tokens = 2 * chunk
|
||||
acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes}
|
||||
coordinator.update_spills(acks, acks)
|
||||
manager.free(request)
|
||||
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.get_computed_blocks(resumed)[1] == chunk
|
||||
|
||||
|
||||
def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity():
|
||||
"""Old completions retain their pages through pressure and request-ID reuse."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
host_pool = coordinator.get_host_block_pool()
|
||||
tokens = list(range(2 * HISPARSE_BLOCK_SIZE + 1))
|
||||
request = make_request("reused", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, len(tokens)) is not None
|
||||
old_blocks = list(manager.get_blocks(request.request_id).blocks[0])
|
||||
command = coordinator.build_offload_command()
|
||||
counts = {transfer.transfer_id: 2 for transfer in command.page_transfers}
|
||||
partial = dict.fromkeys(counts, 1)
|
||||
first_completions = partial.copy()
|
||||
first_completions[next(iter(counts))] = 2
|
||||
coordinator.update_spills(counts, first_completions)
|
||||
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
manager.free(request)
|
||||
replacement = make_request(
|
||||
"reused", list(range(100, 117)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
assert _allocate_scheduled(manager, replacement, 17) is not None
|
||||
new_blocks = list(manager.get_blocks(replacement.request_id).blocks[0])
|
||||
pressure = host_pool.get_new_blocks(host_pool.get_num_free_blocks())
|
||||
assert not {block.block_id for block in old_blocks[:2]} & {
|
||||
block.block_id for block in pressure + new_blocks
|
||||
}
|
||||
# Only the page every worker finished writing is published.
|
||||
assert old_blocks[0].block_hash is not None
|
||||
assert all(block.block_hash is None for block in old_blocks[1:])
|
||||
coordinator.update_spills({}, partial)
|
||||
repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE
|
||||
assert all(block.block_hash is None for block in new_blocks)
|
||||
assert all(block.ref_cnt == 0 for block in old_blocks[:2])
|
||||
coordinator.update_spills({}, partial)
|
||||
assert all(block.ref_cnt == 0 for block in old_blocks[:2])
|
||||
host_pool.free_blocks(pressure)
|
||||
manager.free(replacement)
|
||||
_publish_hisparse_pages(manager)
|
||||
assert not coordinator.has_pending_work()
|
||||
|
||||
|
||||
def test_hisparse_reset_prefix_cache_drops_copies():
|
||||
"""Reset must forget GPU copies along with the host hashes they mirror."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
manager.free(request)
|
||||
|
||||
@@ -1162,7 +1462,9 @@ def test_hisparse_reused_copy_is_not_adopted():
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
tokens = list(range(4 * HISPARSE_BLOCK_SIZE))
|
||||
original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None
|
||||
assert (
|
||||
_allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None
|
||||
)
|
||||
_publish_hisparse_pages(manager)
|
||||
indexer_blocks = list(manager.get_blocks("original").blocks[1])
|
||||
manager.free(original)
|
||||
@@ -1175,7 +1477,8 @@ def test_hisparse_reused_copy_is_not_adopted():
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
computed, num_computed, _ = manager.get_computed_blocks(resumed)
|
||||
assert num_computed == 3 * HISPARSE_BLOCK_SIZE
|
||||
assert manager.allocate_slots(
|
||||
assert _allocate_scheduled(
|
||||
manager,
|
||||
resumed,
|
||||
num_new_tokens=len(tokens) - num_computed,
|
||||
num_new_computed_tokens=num_computed,
|
||||
@@ -2901,21 +3204,21 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(all_token_ids[:block_size]),
|
||||
(("aaa", 11),),
|
||||
(("mm", "aaa", 11),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(
|
||||
block_hashes[0],
|
||||
tuple(all_token_ids[block_size : block_size * 2]),
|
||||
(("aaa", -5), ("bbb", 14)),
|
||||
(("mm", "aaa", -5), ("mm", "bbb", 14)),
|
||||
)
|
||||
)
|
||||
assert block_hashes[2] == sha256(
|
||||
(
|
||||
block_hashes[1],
|
||||
tuple(all_token_ids[block_size * 2 : block_size * 3]),
|
||||
(("bbb", -2),),
|
||||
(("mm", "bbb", -2),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2938,7 +3241,7 @@ def test_mm_prefix_caching():
|
||||
(
|
||||
block_hashes[2],
|
||||
tuple(all_token_ids[3 * block_size :] + [8] * 5),
|
||||
(("ccc", 0),),
|
||||
(("mm", "ccc", 0),),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2984,7 +3287,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req0.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt1"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
@@ -3029,7 +3336,11 @@ def test_cache_key_salting():
|
||||
block_hashes = req2.block_hashes
|
||||
assert len(block_hashes) == 3
|
||||
assert block_hashes[0] == sha256(
|
||||
(kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",))
|
||||
(
|
||||
kv_cache_utils.NONE_HASH,
|
||||
tuple(token_ids[:block_size]),
|
||||
(("cache_salt", "salt2"),),
|
||||
)
|
||||
)
|
||||
assert block_hashes[1] == sha256(
|
||||
(block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None)
|
||||
@@ -5067,7 +5378,8 @@ def test_can_fit_full_sequence_hisparse_caps_resident_pages():
|
||||
)
|
||||
|
||||
assert (
|
||||
manager.allocate_slots(
|
||||
_allocate_scheduled(
|
||||
manager,
|
||||
request,
|
||||
num_new_tokens=HISPARSE_BLOCK_SIZE,
|
||||
full_sequence_must_fit=True,
|
||||
@@ -5927,7 +6239,7 @@ def test_device_eviction_keeps_host_prefix():
|
||||
"""Device eviction must not target a same-numbered host block."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
request = make_request("producer", list(range(64)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, 64) is not None
|
||||
assert _allocate_scheduled(manager, request, 64) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
host, indexer, _, _ = manager.get_blocks(request.request_id).blocks
|
||||
device_block = indexer[0]
|
||||
|
||||
+176
-54
@@ -39,9 +39,11 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.diffusion_scheduler import (
|
||||
DiffusionAsyncScheduler,
|
||||
DiffusionScheduler,
|
||||
diffusion_canvas_width,
|
||||
)
|
||||
from vllm.v1.core.sched.interface import PauseState
|
||||
@@ -436,6 +438,52 @@ def test_schedule_partial_requests():
|
||||
assert requests[2].request_id not in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_encoder_only_prompt_longer_than_budget_is_chunked():
|
||||
"""The engine switches chunked prefill off for an encoder-only instance,
|
||||
but the token budget must still split its prompt across steps instead of
|
||||
leaving the request waiting forever."""
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
max_model_len=4096,
|
||||
mm_encoder_only=True,
|
||||
)
|
||||
# EngineCore turns this off after config validation whenever the instance
|
||||
# holds no KV cache, which is every encoder-only instance.
|
||||
scheduler.scheduler_config.enable_chunked_prefill = False
|
||||
(request,) = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=2500,
|
||||
mm_positions=[[PlaceholderRange(offset=100, length=600)]],
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
|
||||
def advance(output):
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
first = scheduler.schedule()
|
||||
assert first.num_scheduled_tokens[request.request_id] == 1024
|
||||
assert request.request_id in first.scheduled_encoder_inputs
|
||||
advance(first)
|
||||
|
||||
second = scheduler.schedule()
|
||||
assert second.num_scheduled_tokens[request.request_id] == 1024
|
||||
advance(second)
|
||||
|
||||
third = scheduler.schedule()
|
||||
assert third.num_scheduled_tokens[request.request_id] == 452
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_running", [True, False])
|
||||
def test_schedule_prefills_gating(has_running: bool):
|
||||
"""DP prefill-balancing gate: when `throttle_prefills` is True, a new
|
||||
@@ -861,6 +909,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
assert all(out.new_sampling_mask.offsets is None for out in outputs)
|
||||
|
||||
|
||||
def test_update_from_output_routes_multi_position_sampling_masks():
|
||||
scheduler = create_scheduler()
|
||||
scheduler.return_sampling_mask = True
|
||||
requests = create_requests(num_requests=2, max_tokens=10)
|
||||
for req in requests:
|
||||
req.num_computed_tokens = req.num_tokens
|
||||
scheduler.requests[req.request_id] = req
|
||||
scheduler.running.append(req)
|
||||
req.status = RequestStatus.RUNNING
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={req.request_id: 1 for req in requests},
|
||||
total_num_scheduled_tokens=2,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in requests],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
|
||||
sampled_token_ids=[[1, 2], [3, 4, 5]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
sampling_masks=SamplingMaskLists(
|
||||
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
|
||||
offsets=np.array([0, 2, 3, 6, 7, 9]),
|
||||
cu_num_generated_tokens=[0, 2, 5],
|
||||
),
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
|
||||
|
||||
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
|
||||
[[1, 6], [2]],
|
||||
[[3, 7, 8], [4], [5, 9]],
|
||||
]
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
@@ -1562,13 +1653,7 @@ def test_scheduler_reset_prefix_cache():
|
||||
assert not scheduler.reset_prefix_cache()
|
||||
scheduler.aux_output_connector.reset.assert_not_called()
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
|
||||
# pause(mode="keep") also waits for scheduled model outputs to drain.
|
||||
scheduler.set_pause_state(PauseState.PAUSED_ALL)
|
||||
with pytest.raises(RuntimeError, match="model output is in flight"):
|
||||
scheduler.reset_prefix_cache(reset_running_requests=True)
|
||||
# Pause completes pending model outputs before the caller resets the scheduler.
|
||||
for request in requests:
|
||||
request.num_in_flight_tokens = 0
|
||||
|
||||
@@ -6667,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
num_blocks: int = 100,
|
||||
supports_divergent_hits: bool = True,
|
||||
) -> Scheduler:
|
||||
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
|
||||
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
trust_remote_code=True,
|
||||
@@ -6688,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="MockKVConnector",
|
||||
@@ -6720,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -6736,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
)
|
||||
|
||||
|
||||
def _seed_hybrid_prefix(
|
||||
manager: KVCacheManager, num_blocks: int, block_size: int
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Prefill a prefix one block per step so that align-mode Mamba caches
|
||||
the state at every block boundary. Returns the FA and Mamba block ids."""
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=num_blocks * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
assert num_computed == 0
|
||||
mamba_ids = []
|
||||
for i in range(num_blocks):
|
||||
manager.allocate_slots(
|
||||
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
|
||||
)
|
||||
fill.num_computed_tokens += block_size
|
||||
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
|
||||
mamba_ids.append(group_mamba_ids[i])
|
||||
manager.free(fill)
|
||||
return fa_ids, mamba_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matched_tokens,expected_num_computed",
|
||||
[
|
||||
@@ -6761,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries
|
||||
# (mamba cache mode "all" caches every block's state densely).
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
fa_ids = [b.block_id for b in blocks.blocks[0]]
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries.
|
||||
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Evict the FA tail and the middle mamba states; block 0 (both groups)
|
||||
# and the deep mamba state at block 3 survive.
|
||||
@@ -6801,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -6837,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix in both groups.
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
|
||||
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
|
||||
@@ -6871,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
|
||||
@@ -7115,14 +7199,15 @@ def diffusion_model_runner(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
|
||||
|
||||
def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler:
|
||||
def _diffusion_scheduler(async_scheduling: bool = True, **kwargs) -> DiffusionScheduler:
|
||||
scheduler_cls = DiffusionAsyncScheduler if async_scheduling else DiffusionScheduler
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
async_scheduling=async_scheduling,
|
||||
diffusion_canvas_length=8,
|
||||
scheduler_cls=DiffusionAsyncScheduler,
|
||||
scheduler_cls=scheduler_cls,
|
||||
**kwargs,
|
||||
)
|
||||
assert isinstance(scheduler, DiffusionAsyncScheduler)
|
||||
assert isinstance(scheduler, scheduler_cls)
|
||||
return scheduler
|
||||
|
||||
|
||||
@@ -7133,7 +7218,7 @@ def test_diffusion_scheduler_is_selected_by_default(async_scheduling):
|
||||
async_scheduling=async_scheduling, diffusion_canvas_length=8
|
||||
).vllm_config.scheduler_config
|
||||
assert config.get_scheduler_cls() is (
|
||||
DiffusionAsyncScheduler if config.async_scheduling else Scheduler
|
||||
DiffusionAsyncScheduler if config.async_scheduling else DiffusionScheduler
|
||||
)
|
||||
|
||||
|
||||
@@ -7155,10 +7240,13 @@ def test_diffusion_scheduler_narrows_the_canvas_per_request():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("structured", [False, True])
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
@pytest.mark.usefixtures("diffusion_model_runner")
|
||||
def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
|
||||
def test_diffusion_scheduler_trims_full_width_worker_drafts(
|
||||
structured, async_scheduling
|
||||
):
|
||||
"""Padded worker drafts must be narrowed before scheduling or grammar validation."""
|
||||
scheduler = _diffusion_scheduler()
|
||||
scheduler = _diffusion_scheduler(async_scheduling=async_scheduling)
|
||||
wide = _diffusion_request("wide", {})
|
||||
narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4})
|
||||
for request in (wide, narrow):
|
||||
@@ -7173,8 +7261,12 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
|
||||
)
|
||||
tokens = list(range(8)) if structured else [-1] * 8
|
||||
drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()])
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_draft_token_ids_in_output(drafts, output)
|
||||
if async_scheduling:
|
||||
output = scheduler.schedule()
|
||||
scheduler.update_draft_token_ids_in_output(drafts, output)
|
||||
else:
|
||||
scheduler.update_draft_token_ids(drafts)
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.scheduled_spec_decode_tokens == {
|
||||
"wide": tokens,
|
||||
@@ -7186,6 +7278,36 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured):
|
||||
assert narrow.structured_output_request.grammar.seen == [tokens[:4]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sampled, expected_status",
|
||||
[
|
||||
pytest.param([], RequestStatus.RUNNING, id="denoising"),
|
||||
pytest.param([1] * 4, RequestStatus.FINISHED_LENGTH_CAPPED, id="final"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("diffusion_model_runner")
|
||||
def test_sync_diffusion_step_commits_only_emitted_tokens(sampled, expected_status):
|
||||
"""An empty denoising step reuses the canvas; a completed read emits it."""
|
||||
scheduler = _diffusion_scheduler(async_scheduling=False)
|
||||
request = _diffusion_request(
|
||||
"read", {"diffusion_canvas_length": 4, "diffusion_read_only": True}
|
||||
)
|
||||
request.max_tokens = request.sampling_params.max_tokens = 4
|
||||
scheduler.add_request(request)
|
||||
_model_output(scheduler, scheduler.schedule(), [[]])
|
||||
|
||||
scheduler.update_draft_token_ids(
|
||||
DraftTokenIds([request.request_id], [list(range(8))])
|
||||
)
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens == {request.request_id: 4}
|
||||
_model_output(scheduler, output, [sampled])
|
||||
|
||||
assert request.num_computed_tokens == request.num_prompt_tokens + len(sampled)
|
||||
assert list(request.output_token_ids) == sampled
|
||||
assert request.status == expected_status
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("diffusion_model_runner")
|
||||
def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight():
|
||||
scheduler = _diffusion_scheduler()
|
||||
|
||||
@@ -60,6 +60,7 @@ def create_scheduler(
|
||||
long_prefill_token_threshold: int = 0,
|
||||
long_prefill_token_threshold_adaptive: bool = False,
|
||||
disable_chunked_mm_input: bool = False,
|
||||
mm_encoder_only: bool = False,
|
||||
use_kv_connector: None | bool | str | MockKVConfig = None,
|
||||
kv_role: str = "kv_both",
|
||||
num_blocks: int = 10000,
|
||||
@@ -109,6 +110,8 @@ def create_scheduler(
|
||||
# SchedulerConfig one, so both must agree.
|
||||
max_model_len=max_model_len,
|
||||
)
|
||||
if mm_encoder_only:
|
||||
model_config.multimodal_config.mm_encoder_only = True
|
||||
if use_ec_connector and ec_role == "ec_producer":
|
||||
model_config.multimodal_config = MultiModalConfig()
|
||||
if max_model_len is None:
|
||||
|
||||
Reference in New Issue
Block a user