Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-14 14:36:05 +09:00
43 changed files with 1724 additions and 362 deletions
+63 -41
View File
@@ -1025,17 +1025,20 @@ def test_decode_index_topk_correctness(
"expected",
),
[
(0, 24, 1, 16, True, (2, False, False)),
(1, 1, 1, 16, True, (16, True, True)),
(512, 4, 1, 16, True, (16, True, True)),
(513, 24, 4, 16, True, (16, True, True)),
(2760, 5, 2, 16, True, (16, True, True)),
(8192, 64, 4, 16, True, (16, True, True)),
(8193, 4, 1, 16, True, (16, False, False)),
(8193, 16, 1, 16, True, (4, False, False)),
(65534, 64, 1, 16, True, (1, False, False)),
(513, 16, 1, 13, True, (4, False, False)),
(513, 16, 4, 16, False, (1, False, False)),
(0, 24, 1, 16, True, (2, 512, 8, 2, False, False)),
(1, 1, 1, 16, True, (1, 128, 2, 1, True, True)),
(74, 4, 1, 16, True, (1, 128, 2, 1, True, True)),
(128, 4, 1, 16, True, (1, 128, 2, 1, True, True)),
(129, 4, 1, 16, True, (16, 512, 4, 2, True, True)),
(512, 4, 1, 16, True, (16, 512, 4, 2, True, True)),
(513, 24, 4, 16, True, (16, 512, 4, 2, True, True)),
(2760, 5, 2, 16, True, (16, 512, 4, 2, True, True)),
(8192, 64, 4, 16, True, (16, 512, 4, 2, True, True)),
(8193, 4, 1, 16, True, (16, 512, 8, 2, False, False)),
(8193, 16, 1, 16, True, (4, 512, 8, 2, False, False)),
(65534, 64, 1, 16, True, (1, 512, 8, 2, False, False)),
(513, 16, 1, 13, True, (4, 512, 8, 2, False, False)),
(513, 16, 4, 16, False, (1, 512, 8, 2, False, False)),
],
)
def test_amd_decode_topk_launch_policy(
@@ -1044,9 +1047,9 @@ def test_amd_decode_topk_launch_policy(
num_idx_heads: int,
topk: int,
is_gfx950: bool,
expected: tuple[int, bool, bool],
expected: tuple[int, int, int, int, bool, bool],
):
"""All supported head and query layouts use adaptive selection."""
"""The short-context specialization has strict dispatch boundaries."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
_decode_topk_launch_policy,
)
@@ -1073,19 +1076,23 @@ def test_amd_decode_topk_launch_policy(
"single_tile_guaranteed",
"adaptive_final_merge",
"max_blocks",
"block_size_k",
"num_warps",
"num_stages",
"topk",
"num_idx_heads",
),
[
(1, False, False, 65534, 16, 1),
(2, False, False, 8193, 16, 1),
(4, False, False, 8193, 16, 1),
(8, False, False, 8193, 16, 1),
(16, False, False, 8193, 16, 1),
(16, False, False, 65534, 16, 1),
(2, True, False, 513, 13, 1),
(4, False, True, 8192, 16, 2),
(16, True, True, 8192, 16, 4),
(1, False, False, 65534, 512, 8, 2, 16, 1),
(2, False, False, 8193, 512, 8, 2, 16, 1),
(4, False, False, 8193, 512, 8, 2, 16, 1),
(8, False, False, 8193, 512, 8, 2, 16, 1),
(16, False, False, 8193, 512, 8, 2, 16, 1),
(16, False, False, 65534, 512, 8, 2, 16, 1),
(2, True, False, 513, 512, 8, 2, 13, 1),
(4, False, True, 8192, 512, 4, 2, 16, 2),
(16, True, True, 8192, 512, 4, 2, 16, 4),
(1, True, True, 74, 128, 2, 1, 16, 1),
],
)
def test_amd_decode_fused_topk_total_order_and_replay(
@@ -1093,10 +1100,13 @@ def test_amd_decode_fused_topk_total_order_and_replay(
single_tile_guaranteed: bool,
adaptive_final_merge: bool,
max_blocks: int,
block_size_k: int,
num_warps: int,
num_stages: int,
topk: int,
num_idx_heads: int,
):
"""Atomic multi-chunk selection is ordered and resets graph state."""
"""Selector specializations preserve ordering and graph state."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
SPARSE_BLOCK_SIZE,
_decode_topk_fused_kernel,
@@ -1104,18 +1114,25 @@ def test_amd_decode_fused_topk_total_order_and_replay(
set_random_seed(0)
block_size_t = 1 << (topk - 1).bit_length()
block_counts = tuple(
count
for count in (
15,
16,
17,
block_size_k // 2 - 1,
block_size_k // 2,
block_size_k // 2 + 1,
block_size_k - 1,
block_size_k,
block_size_k + 1,
)
if count <= max_blocks
)
long_seq_lens_list = [
0,
1,
15 * SPARSE_BLOCK_SIZE,
16 * SPARSE_BLOCK_SIZE,
17 * SPARSE_BLOCK_SIZE,
255 * SPARSE_BLOCK_SIZE,
256 * SPARSE_BLOCK_SIZE,
257 * SPARSE_BLOCK_SIZE,
511 * SPARSE_BLOCK_SIZE,
512 * SPARSE_BLOCK_SIZE,
513 * SPARSE_BLOCK_SIZE,
*(blocks * SPARSE_BLOCK_SIZE for blocks in block_counts),
max_blocks * SPARSE_BLOCK_SIZE,
]
short_seq_lens_list = [0, 1, 127, 128, 129, 0, 1, 128, 129, 0, 1, 129]
@@ -1126,12 +1143,13 @@ def test_amd_decode_fused_topk_total_order_and_replay(
boundary_limit = 4
if boundary_limit:
boundary_blocks = [
(active_chunks - 1) * 512 + 1
(active_chunks - 1) * block_size_k + 1
for active_chunks in range(3, boundary_limit + 1)
]
assert [(num_blocks + 511) // 512 for num_blocks in boundary_blocks] == list(
range(3, boundary_limit + 1)
)
assert [
(num_blocks + block_size_k - 1) // block_size_k
for num_blocks in boundary_blocks
] == list(range(3, boundary_limit + 1))
long_seq_lens_list.extend(
num_blocks * SPARSE_BLOCK_SIZE for num_blocks in boundary_blocks
)
@@ -1139,6 +1157,8 @@ def test_amd_decode_fused_topk_total_order_and_replay(
num_blocks * SPARSE_BLOCK_SIZE for num_blocks in reversed(boundary_blocks)
)
batch = len(long_seq_lens_list)
repeats = (batch + len(short_seq_lens_list) - 1) // len(short_seq_lens_list)
short_seq_lens_list = (short_seq_lens_list * repeats)[:batch]
score = torch.randn((num_idx_heads, batch, max_blocks), device="cuda")
score[:, :, 0] = float("nan")
score[:, :, 1] = float("inf")
@@ -1147,8 +1167,9 @@ def test_amd_decode_fused_topk_total_order_and_replay(
score[:, :, 4] = -0.0
score[:, :, 5] = 7.0
score[:, :, 6] = -1e30
score[:, :, 256] = 7.0
score[:, :, 512] = 7.0
for index in (block_size_k // 2, block_size_k):
if index < max_blocks:
score[:, :, index] = 7.0
seq_lens = torch.empty(batch, device="cuda", dtype=torch.int32)
partial = torch.full(
@@ -1198,13 +1219,13 @@ def test_amd_decode_fused_topk_total_order_and_replay(
pages_per_sparse_block=8,
block_page_stride=8,
NUM_TOPK_CHUNKS=num_topk_chunks,
BLOCK_SIZE_K=512,
BLOCK_SIZE_K=block_size_k,
BLOCK_SIZE_T=block_size_t,
EMIT_SPARSE_TABLE=False,
SINGLE_TILE_GUARANTEED=single_tile_guaranteed,
ADAPTIVE_FINAL_MERGE=adaptive_final_merge,
num_warps=4 if adaptive_final_merge else 8,
num_stages=2,
num_warps=num_warps,
num_stages=num_stages,
)
score_cpu = score.cpu()
@@ -1251,6 +1272,7 @@ def test_amd_decode_fused_topk_total_order_and_replay(
(16, False, False, 8193),
(4, False, True, 8192),
(16, True, True, 8192),
(1, True, True, 74),
):
graph = torch.cuda.CUDAGraph()
torch.accelerator.synchronize()
@@ -832,6 +832,7 @@ def test_models_fse_init(
runner_type="generate",
is_moe=True,
logits_processors=None,
rswa_window=None,
)
vllm_config.parallel_config.enable_expert_parallel = False
if model_type == "deepseek_v4":
+18
View File
@@ -99,6 +99,24 @@ def test_raw_text_parts_preserve_reference_separator():
assert messages == original
@pytest.mark.parametrize(
("role", "responses_type"),
[("user", "input_text"), ("assistant", "output_text")],
)
def test_responses_text_parts_match_chat_text_parts(role, responses_type):
responses_message = {
"role": role,
"content": [{"type": responses_type, "text": "hello"}],
}
chat_message = {
"role": role,
"content": [{"type": "text", "text": "hello"}],
}
assert render([responses_message], thinking=False) == render(
[chat_message], thinking=False
)
def test_mid_system_gets_its_own_marker_and_generation_header():
assert render(
[
+41 -1
View File
@@ -35,7 +35,7 @@ from vllm.multimodal.inputs import (
from vllm.sampling_params import SamplingParams
from vllm.utils.hashing import sha256, sha256_cbor, xxhash, xxhash_cbor
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.kv_cache_utils import (
BlockHash,
FreeKVCacheBlockQueue,
@@ -61,6 +61,7 @@ from vllm.v1.hisparse.layout import (
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
HiSparseHotSpec,
@@ -324,6 +325,45 @@ def test_kv_cache_config_selects_only_transferable_groups():
)
def test_kv_cache_config_selects_prefix_cacheable_groups():
"""Prefix stores exclude scratch state without changing transfer groups."""
full_group = KVCacheGroupSpec(["full"], new_kv_cache_spec())
qsa_group = KVCacheGroupSpec(
["qsa"],
CircularBufferSpec(
block_size=4,
num_kv_heads=1,
head_size=64,
head_size_v=0,
dtype=torch.float16,
),
)
disabled_group = KVCacheGroupSpec(
["disabled"], new_kv_cache_spec(), enable_kv_transfer=False
)
config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[full_group, qsa_group, disabled_group],
)
assert config.transfer_group_ids == (0, 1)
assert config.select_transfer_block_ids(([1], [2], [3])) == ([1], [2])
assert config.prefix_cacheable_group_ids == (0,)
assert config.prefix_cacheable_groups == (full_group,)
def test_kv_cache_blocks_selects_requested_groups():
blocks = KVCacheBlocks(
(
[KVCacheBlock(1)],
[KVCacheBlock(2)],
[KVCacheBlock(3)],
)
)
assert blocks.get_block_ids(group_ids=(0, 2)) == ([1], [3])
def new_sliding_window_spec(
block_size=16,
num_kv_heads=2,
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
MooncakeStoreConnectorStats,
)
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -91,6 +92,53 @@ def _make_kv_cache_config() -> KVCacheConfig:
)
def _make_qsa_hybrid_kv_cache_config(mamba_mode: str):
full_spec = FullAttentionSpec(
block_size=800, num_kv_heads=8, head_size=64, dtype=None
)
qsa_spec = CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=64,
head_size_v=0,
dtype=torch.float16,
)
mamba_spec = MambaSpec(
block_size=800,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode=mamba_mode,
)
return KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["full"], full_spec),
KVCacheGroupSpec(["qsa"], qsa_spec),
KVCacheGroupSpec(["mamba"], mamba_spec),
],
)
def test_validation_accepts_aligned_mamba_after_block_size_rewrite():
vllm_config = _make_vllm_config()
vllm_config.cache_config.block_size = 8
mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config(
vllm_config, _make_qsa_hybrid_kv_cache_config("align")
)
def test_validation_rejects_mamba_mode_directly():
vllm_config = _make_vllm_config()
vllm_config.cache_config.block_size = 800
with pytest.raises(ValueError, match="mamba_cache_mode=\x27none\x27"):
mooncake_store_connector.MooncakeStoreConnector._validate_kv_cache_config(
vllm_config, _make_qsa_hybrid_kv_cache_config("none")
)
def test_scheduler_requires_align_mode_for_mamba():
vllm_config = _make_vllm_config()
mamba_spec = MambaSpec(
@@ -759,8 +759,9 @@ def test_worker_setup_tolerates_finer_scratch_group():
worker.pp_size = 1
worker.num_kv_head = 8
assert worker.coord.enable_partial_hash_hits
# The scratch DB is keyed at its own block size and never probed.
assert worker.token_dbs[2].hash_block_size == 4
# Only the participating full-attention and Mamba groups are registered.
assert len(worker.token_dbs) == 2
assert all(db.hash_block_size == 8 for db in worker.token_dbs)
for g_idx, db in enumerate(worker.token_dbs):
db.set_kv_caches_base_addr([g_idx * 10_000])
@@ -772,11 +773,11 @@ def test_worker_setup_tolerates_finer_scratch_group():
block_size=worker.block_size,
coord=worker.coord,
tp_rank=0,
group_put_steps=[1, 1, 1],
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
group_participates=[True, True, False],
group_participates=[True, True],
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
@@ -784,7 +785,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
req = ReqMeta(
req_id="r0",
token_len_chunk=0,
block_ids=([1], [2], [3]),
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
@@ -797,8 +798,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
# A 13-token prompt sharing the prefix must hit the first hash unit.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
# The scratch group's namespace never enters the store.
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
assert not any(key.startswith(scratch_prefix) for key in store._data)
assert not any("@group:2" in key for key in store._data)
def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
@@ -838,7 +838,8 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
worker.tp_size = 1
worker.pp_size = 1
worker.num_kv_head = 8
assert worker.token_dbs[1].hash_block_size == 8
assert len(worker.token_dbs) == 1
assert worker.token_dbs[0].hash_block_size == 16
raw_full = torch.zeros(4 * full.page_size_bytes, dtype=torch.int8)
raw_ring = torch.zeros(4 * ring.page_size_bytes, dtype=torch.int8)
@@ -879,14 +880,14 @@ def test_ring_scratch_group_is_never_stored_and_does_not_block_hits():
kv_role=worker.kv_role,
ready_event=threading.Event(),
enable_kv_event=False,
group_participates=[True, False],
group_participates=[True],
)
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(4)]
save_req = ReqMeta(
req_id="r0",
token_len_chunk=64,
# The ring group holds one block for the request's lifetime.
block_ids=([1, 2, 3, 4], [1]),
# Scheduler metadata is projected to the participating store groups.
block_ids=([1, 2, 3, 4],),
block_hashes=hs,
can_save=True,
store_job_id=1,
@@ -2,12 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
MooncakeLookupResult,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
@@ -16,7 +19,16 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
MooncakeStoreScheduler,
)
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
from vllm.v1.core.sched.output import KVConnectorBlockState
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
)
def _make_bare_scheduler(
@@ -35,9 +47,9 @@ def _make_bare_scheduler(
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
)
scheduler.kv_cache_config = SimpleNamespace()
scheduler._store_group_ids = (0,)
scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 1: 1}
scheduler.load_specs = {}
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
@@ -228,14 +240,82 @@ def test_pending_load_for_non_chosen_connector_is_dropped():
assert "req-0" not in scheduler._request_trackers
def _make_qsa_hybrid_cache_config():
full = FullAttentionSpec(block_size=800, num_kv_heads=8, head_size=64, dtype=None)
circular = CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=64,
head_size_v=0,
dtype=torch.float16,
)
mamba = MambaSpec(
block_size=800,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
return KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(["full"], full),
KVCacheGroupSpec(["qsa"], circular),
KVCacheGroupSpec(["mamba"], mamba),
],
)
def test_scheduler_projects_nonprefix_groups_and_mamba_ids():
vllm_config = SimpleNamespace(
kv_transfer_config=SimpleNamespace(
kv_role="kv_both", kv_connector_extra_config={}
),
kv_events_config=None,
cache_config=SimpleNamespace(
block_size=800, enable_prefix_caching=True, prefix_match_unit=None
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1, world_size=1),
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
):
scheduler = MooncakeStoreScheduler(vllm_config, _make_qsa_hybrid_cache_config())
assert scheduler._store_group_ids == (0, 2)
assert scheduler._boundary_state_group_ids == frozenset({1})
def test_current_save_block_ids_use_store_group_projection():
scheduler = _make_bare_scheduler()
scheduler.kv_cache_config = _make_qsa_hybrid_cache_config()
scheduler._store_group_ids = (0, 2)
meta = MooncakeStoreConnectorMetadata(set(), set())
req_meta = ReqMeta(
req_id="req-0",
token_len_chunk=800,
block_ids=(),
block_hashes=[b"h0"],
can_save=True,
)
meta.add_request(req_meta)
output = SimpleNamespace(
kv_connector_block_state=_make_connector_block_state(([10], [80], [30]))
)
scheduler._apply_current_save_block_ids(meta, output)
assert req_meta.block_ids == ([10], [30])
def test_update_state_excludes_nontransfer_groups():
"""Store metadata must match the worker's registered cache groups."""
scheduler = _make_bare_scheduler()
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
)
scheduler._store_group_ids = (0,)
request = SimpleNamespace(request_id="req-1")
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
blocks = KVCacheBlocks(([KVCacheBlock(1), KVCacheBlock(2)], [KVCacheBlock(9)]))
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
@@ -1003,6 +1083,7 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
def test_finished_partial_tail_is_pre_pinned_as_store_job():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler._store_group_ids = (0, 1)
scheduler.client = SimpleNamespace(discard=lambda *_: None)
request = SimpleNamespace(
request_id="req-0",
@@ -1143,6 +1224,20 @@ def _make_offload_only_output(entries, block_ids=([0],)):
)
def test_boundary_state_group_ids_are_remapped_to_store_projection():
scheduler = _make_bare_scheduler(hash_block_size=800, enable_partial_hash_hits=True)
scheduler.kv_cache_config = _make_qsa_hybrid_cache_config()
scheduler._store_group_ids = (0, 2)
scheduler._store_group_id_by_kv_cache_group_id = {0: 0, 2: 1}
scheduler._boundary_state_group_ids = frozenset({1})
_register_offload_request(scheduler, prefill_end_tokens=800, num_prompt_tokens=800)
meta = MooncakeStoreConnectorMetadata(set(), set())
scheduler._handle_boundary_state_offloads({"req-0": [(2, 7, 800)]}, meta)
assert meta.requests[0].boundary_state_offloads == [(1, 7, 800)]
def test_resumed_prefill_claims_boundaries_past_prompt_length():
# A resumed request re-prefills its previously generated tokens, so its
# save window (`prefill_end_tokens`) extends past `num_prompt_tokens`.
@@ -1195,6 +1290,7 @@ def test_boundary_state_job_pins_exact_blocks_once():
def test_store_job_pins_current_non_null_non_mamba_blocks():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
scheduler._store_group_ids = (0, 1)
request = SimpleNamespace(
all_token_ids=list(range(48)),
block_hashes=[bytes([i]) for i in range(12)],
@@ -46,8 +46,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
RSWASpec,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
@@ -342,6 +345,43 @@ def _make_kv_cache_config(
)
def _make_qsa_hybrid_kv_cache_config():
return KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=800,
num_kv_heads=8,
head_size=64,
dtype=None,
),
),
KVCacheGroupSpec(
["qsa"],
CircularBufferSpec(
block_size=8,
num_kv_heads=1,
head_size=64,
head_size_v=0,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=800,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
def _write_mooncake_config(tmp_path, config: dict[str, object]) -> str:
config_path = tmp_path / "mooncake_config.json"
config_path.write_text(json.dumps(config), encoding="utf-8")
@@ -2107,6 +2147,31 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch)
monkeypatch.setattr(
worker.MooncakeStoreConfig,
"load_from_config",
staticmethod(lambda: _make_config()),
)
vllm_config = _make_vllm_config()
vllm_config.cache_config.block_size = 800
vllm_config.cache_config.enable_prefix_caching = True
vllm_config.cache_config.prefix_match_unit = None
store_worker = worker.MooncakeStoreWorker(
vllm_config, _make_qsa_hybrid_kv_cache_config()
)
assert [
group.kv_cache_spec.block_size for group in store_worker._kv_cache_groups
] == [800, 800]
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -4144,6 +4209,31 @@ def test_register_kv_caches_uses_transfer_group_memory_domain():
)
def test_register_kv_caches_excludes_nonprefix_cacheable_layers():
num_blocks = 2
worker = _make_bare_worker(num_gpu_blocks=num_blocks)
store_layout = MagicMock()
worker.token_dbs[0].store_layout = store_layout
full_cache = torch.zeros(num_blocks, 4, dtype=torch.float16)
qsa_cache = torch.zeros(num_blocks, 4, dtype=torch.float16)
_register_with_mocked_threads(
worker,
{"layer0": full_cache, "qsa": qsa_cache},
)
store_layout.register_kv_caches.assert_called_once()
registered_caches, registered_num_blocks = (
store_layout.register_kv_caches.call_args.args
)
assert registered_num_blocks == num_blocks
assert len(registered_caches) == 1
assert registered_caches[0] is full_cache
worker.store.register_buffer.assert_called_once_with(
full_cache.data_ptr(), full_cache.untyped_storage().nbytes()
)
def test_register_kv_caches_separate_head_groups():
# LHBNC gives each head group its own region; the K/V split doubles heads.
layout = KVCacheLayout.LHBNC