[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -313,3 +313,49 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
ref_logits = ref_logits.masked_fill(~mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
def test_deepgemm_paged_mqa_packed_manager_block_stride():
|
||||
torch.manual_seed(0)
|
||||
num_blocks, block_size, packed_rows = 4, 256, 320
|
||||
num_heads, head_dim, row_bytes = 64, 128, 132
|
||||
page_size, pages_per_block = 64, block_size // 64
|
||||
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
|
||||
compact_pages = kv_cache_cast_to_fp8(
|
||||
torch.randn(
|
||||
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
|
||||
)
|
||||
packed_backing = torch.zeros(
|
||||
num_blocks,
|
||||
packed_rows,
|
||||
row_bytes,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
packed_manager = packed_backing[:, :block_size]
|
||||
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
|
||||
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
|
||||
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
|
||||
tables = [
|
||||
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
|
||||
]
|
||||
|
||||
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
weights = torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
|
||||
schedule = get_paged_mqa_logits_metadata(context_lens, page_size, get_num_sms())
|
||||
logits = [
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(q, None), cache, weights, context_lens, table, schedule, block_size, False
|
||||
)
|
||||
for cache, table in zip((compact_pages, packed_pages), tables)
|
||||
]
|
||||
torch.testing.assert_close(*logits, rtol=0, atol=0)
|
||||
|
||||
@@ -463,7 +463,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
set_random_seed(412)
|
||||
num_tokens, num_heads, block_size = 2, 8, 16
|
||||
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2 * block_size, 1, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
|
||||
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
|
||||
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
@@ -504,7 +504,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
references = [
|
||||
_sink_reference(q[i], kv[:, 0][selected[i].long()], sinks, SM_SCALE)
|
||||
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
|
||||
for i in range(num_tokens)
|
||||
]
|
||||
expected, _, value_scale, score_error = (
|
||||
|
||||
@@ -553,12 +553,10 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
owner = ComputeSlotMappingKernel()
|
||||
compile_key = owner.CompileKey(
|
||||
kv_cache_block_size=16,
|
||||
blocks_per_kv_block=1,
|
||||
total_cp_world_size=2,
|
||||
total_cp_rank=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=128,
|
||||
block_size=16,
|
||||
)
|
||||
launches: list[tuple[Any, ...]] = []
|
||||
|
||||
@@ -574,7 +572,6 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
assert inputs["block_table_stride"] == 128
|
||||
assert kwargs == {
|
||||
"KV_CACHE_BLOCK_SIZE": 16,
|
||||
"BLOCKS_PER_KV_BLOCK": 1,
|
||||
"TOTAL_CP_WORLD_SIZE": 2,
|
||||
"TOTAL_CP_RANK": 1,
|
||||
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
|
||||
|
||||
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
|
||||
if block_size not in backend_cls.get_supported_kernel_block_sizes():
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
|
||||
@@ -29,7 +29,7 @@ HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
|
||||
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
meta.req_id_per_token,
|
||||
meta.block_table,
|
||||
impl.topk_indices_buffer,
|
||||
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
|
||||
)
|
||||
offset = 0
|
||||
for t in range(rows):
|
||||
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
|
||||
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
QUANTIZED_DS_MLA_CACHE_FORMATS,
|
||||
FlashMLASparseBackend,
|
||||
_fp8_kv_pages,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
allocate_kv_cache,
|
||||
prepare_kernel_block_sizes,
|
||||
select_common_block_size,
|
||||
)
|
||||
|
||||
SM90 = DeviceCapability(major=9, minor=0)
|
||||
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
|
||||
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
|
||||
|
||||
|
||||
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
cache_dtype_str="bfloat16",
|
||||
block_stride_alignment=1024,
|
||||
)
|
||||
num_blocks = 2
|
||||
layers = ["layer.0", "layer.1"]
|
||||
page_size = spec.page_size_bytes
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=num_blocks * len(layers) * page_size,
|
||||
layers=layers,
|
||||
layer_stride=page_size,
|
||||
block_stride=len(layers) * page_size,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
|
||||
)
|
||||
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
|
||||
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
|
||||
caches = allocate_kv_cache(
|
||||
config,
|
||||
torch.device("cpu"),
|
||||
KVCacheLayout.BLHNC,
|
||||
kernel_block_sizes,
|
||||
)
|
||||
|
||||
assert kernel_block_sizes == [1152]
|
||||
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm100", [False, True])
|
||||
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
|
||||
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
|
||||
(TMA) and keeps 64-token blocks."""
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family, device_id=0: sm100 and family == 100,
|
||||
)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
|
||||
state_content_bytes=656,
|
||||
)
|
||||
|
||||
kernel_block_size = 64 if sm100 else 1152
|
||||
assert (
|
||||
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
|
||||
== kernel_block_size
|
||||
)
|
||||
|
||||
|
||||
def test_fp8_kv_pages_address_packed_blocks():
|
||||
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
|
||||
64-token blocks are passed through."""
|
||||
row, block_size, stride_rows = 656, 128, 192
|
||||
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
|
||||
cache = backing.view(3, stride_rows, row)[:, :block_size]
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
|
||||
assert block_stride_rows == stride_rows
|
||||
assert pages.shape[1:] == (64, row)
|
||||
slot = 2 * block_stride_rows + 100
|
||||
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
|
||||
assert block_stride_rows is None
|
||||
assert torch.equal(pages, cache[:, :64])
|
||||
|
||||
|
||||
def test_quantized_ds_mla_formats_are_the_envelope_set():
|
||||
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import indexer
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rocm,sm120,expected",
|
||||
[
|
||||
(False, False, [KPOOL * 32, KPOOL * 64]),
|
||||
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
|
||||
(False, True, [KPOOL * 64]),
|
||||
# ROCm's AITER paged-MQA logits take the same pool pages.
|
||||
(True, False, [KPOOL * 32, KPOOL * 64]),
|
||||
],
|
||||
)
|
||||
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm120 and family == 120,
|
||||
)
|
||||
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
|
||||
config = SimpleNamespace(model_config=model_config)
|
||||
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
|
||||
|
||||
spec = SimpleNamespace(tokens_per_state=KPOOL)
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
|
||||
# Platform block-size selection queries without a spec.
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
|
||||
@@ -122,7 +122,6 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = 16
|
||||
builder.kernel_block_size = None
|
||||
builder.num_sms = num_sms
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_states=64)
|
||||
@@ -170,30 +169,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
|
||||
)
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
monkeypatch, block_size, kernel_block_size, compress_ratio
|
||||
):
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
|
||||
"""Without a prior build(), the refresh must not keep the last batch's state."""
|
||||
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
|
||||
device = torch.device("cuda")
|
||||
num_reqs, width = 3, 8
|
||||
factor = block_size // kernel_block_size
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = compress_ratio
|
||||
builder.kernel_block_size = kernel_block_size
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
|
||||
def stale(*shape: int) -> torch.Tensor:
|
||||
return torch.full(shape, -7, dtype=torch.int32, device=device)
|
||||
|
||||
decode = DeepSeekV32IndexerDecodeMetadata(
|
||||
block_table=stale(num_reqs, width // factor),
|
||||
block_table=stale(num_reqs, width),
|
||||
seq_lens=stale(num_reqs, 1),
|
||||
decode_lens=stale(num_reqs),
|
||||
requires_padding=False,
|
||||
@@ -219,8 +212,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected_block_table = request_block_table[:, ::factor] // factor
|
||||
torch.testing.assert_close(decode.block_table, expected_block_table)
|
||||
torch.testing.assert_close(decode.block_table, request_block_table)
|
||||
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
|
||||
assert torch.all(decode.per_req_decode_lens == 1)
|
||||
assert torch.all(decode.decode_lens == 1)
|
||||
@@ -471,7 +463,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
|
||||
builder.pcp_rank = 0
|
||||
builder.dcp_world_size = 1
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.kernel_block_size = None
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
|
||||
"""CPU tests for the kpool indexer's kernel block selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
|
||||
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_mod
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
KpoolTailBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
|
||||
# compressed states (hence storage_block_size = page * 4).
|
||||
# compressed states (hence a kernel block is page * 4 tokens).
|
||||
INDEX_KPOOL = 4
|
||||
|
||||
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
|
||||
@@ -59,14 +53,17 @@ def _prepare(
|
||||
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
|
||||
) -> list[int]:
|
||||
kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
|
||||
kv_cache_tensors=[],
|
||||
)
|
||||
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
|
||||
attn_groups = [
|
||||
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
|
||||
]
|
||||
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
|
||||
def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
def test_kpool_group_maps_pool_pages(
|
||||
monkeypatch: pytest.MonkeyPatch, manager_block: int
|
||||
):
|
||||
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
|
||||
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
"""
|
||||
_mock_rocm_platform(monkeypatch)
|
||||
spec = _kpool_storage_spec(manager_block)
|
||||
expected_storage = 256 if manager_block % 256 == 0 else 128
|
||||
assert spec.storage_block_size == expected_storage
|
||||
expected = 256 if manager_block % 256 == 0 else 128
|
||||
|
||||
selected = _prepare(
|
||||
spec,
|
||||
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
|
||||
)
|
||||
assert selected == [expected_storage]
|
||||
# The kpool group's AttentionGroup selects its kernel block this way.
|
||||
selected = [
|
||||
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
|
||||
]
|
||||
assert selected == [expected]
|
||||
assert selected != [manager_block]
|
||||
# The hybrid block-table split stays integral.
|
||||
assert manager_block % selected[0] == 0
|
||||
|
||||
|
||||
def test_prepare_falls_back_when_storage_block_is_not_supported():
|
||||
"""No storage block, or a storage block the backends reject (e.g. the
|
||||
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
|
||||
`select_common_block_size`'s answer, unchanged from no-fix behavior.
|
||||
"""
|
||||
def test_prepare_uses_the_backend_vote():
|
||||
"""Unpacked groups take `select_common_block_size`'s answer."""
|
||||
|
||||
class Fixed64Backend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [64]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FIXED64"
|
||||
|
||||
# Storage block the group's backend does not accept -> backend vote (64).
|
||||
spec = _kpool_storage_spec(640)
|
||||
assert spec.storage_block_size == 128
|
||||
assert _prepare(spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# No storage block at all (any non-kpool MLA model) -> backend vote.
|
||||
plain_spec = MLAAttentionSpec(
|
||||
block_size=640,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
assert plain_spec.storage_block_size is None
|
||||
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# ...and with backends that accept the manager block, the manager wins,
|
||||
# exactly as on main.
|
||||
class AcceptAllBackend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
spec = CircularBufferSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert layout is KVCacheLayout.BLHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
|
||||
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
)
|
||||
|
||||
assert spec.block_size == spec.sliding_window == ring
|
||||
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
|
||||
assert ring >= KPOOL + num_speculative_tokens
|
||||
assert 640 % ring == 0
|
||||
|
||||
|
||||
def test_tail_spec_opts_out_of_generic_slot_mapping():
|
||||
"""The tail row is one block wide (padded to the block-table alignment), so
|
||||
the generic kernel's ``pos // kpool`` column index runs off the end of the
|
||||
allocation for long prompts. The spec must opt out of it entirely."""
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
max_len = 1 << 20
|
||||
width = get_block_table_width(
|
||||
spec.max_num_blocks_per_req(None, max_len),
|
||||
spec.block_size,
|
||||
token_alignment=spec.block_table_token_alignment,
|
||||
)
|
||||
|
||||
assert width * KPOOL < max_len
|
||||
assert spec.uses_slot_mapping is False
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
request's single CircularBufferManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
tokens between the last request boundary and num_actual_tokens (mapped to
|
||||
the last request), untouched padding beyond num_actual, and PAD for
|
||||
requests on the null block."""
|
||||
ring_size = 2 * KPOOL
|
||||
positions, qsl, slot_mapping, _, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
got = circular_tail_slots(
|
||||
ref = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
|
||||
)
|
||||
got = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping.cuda(),
|
||||
bt.cuda(),
|
||||
qsl.cuda().to(torch.int32),
|
||||
positions.cuda(),
|
||||
num_actual,
|
||||
num_reqs,
|
||||
ring_size,
|
||||
)
|
||||
torch.testing.assert_close(got.cpu(), ref)
|
||||
for req, blk in enumerate(own_blocks):
|
||||
|
||||
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
|
||||
)
|
||||
layer.mm_prefix_clamp_sliding_window = True
|
||||
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
|
||||
builder.set_kernel_block_size(block_size)
|
||||
# Match production, where unwritten page tails are zero-initialized.
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_mla_supports_glm_packed_layout():
|
||||
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
|
||||
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
block_size=32,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 32, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
monkeypatch,
|
||||
block_stride_rows: int | None = None,
|
||||
):
|
||||
if qk_rope_head_dim == 0 and (
|
||||
backend_cls != FlashMLASparseBackend
|
||||
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
|
||||
if device_capability is None or device_capability.major != 10:
|
||||
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||
|
||||
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
||||
if block_size not in supported_block_sizes:
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
scale=kv_cache_scale,
|
||||
)
|
||||
if block_stride_rows is not None:
|
||||
pad = (0, 0, 0, block_stride_rows - block_size)
|
||||
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
|
||||
slot = int(common_attn_metadata.slot_mapping[-1])
|
||||
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
|
||||
|
||||
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||
# static_forward_context; register a mock layer carrying one.
|
||||
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
|
||||
out_buffer,
|
||||
)
|
||||
|
||||
if block_stride_rows is not None and kv_cache_dtype == "auto":
|
||||
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
|
||||
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
|
||||
elif block_stride_rows is not None:
|
||||
assert query_row.any()
|
||||
|
||||
assert backend_output.shape == sdpa_reference.shape
|
||||
assert backend_output.dtype == sdpa_reference.dtype
|
||||
assert torch.isfinite(backend_output).all()
|
||||
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
|
||||
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
|
||||
[
|
||||
pytest.param(
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
"auto",
|
||||
256,
|
||||
320,
|
||||
(128, 128, 64, 128),
|
||||
id="FlashInferTRTLLM",
|
||||
),
|
||||
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
|
||||
pytest.param(
|
||||
FlashMLASparseBackend,
|
||||
"fp8_ds_mla",
|
||||
128,
|
||||
192,
|
||||
(64, 256, 0, 256),
|
||||
id="FlashMLA-fp8_ds_mla",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_packed_stride(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
backend_cls,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
block_stride_rows,
|
||||
head_dims,
|
||||
):
|
||||
test_sparse_backend_decode_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
backend_cls,
|
||||
"mixed_small",
|
||||
kv_cache_dtype,
|
||||
1,
|
||||
block_size,
|
||||
workspace_init,
|
||||
1.0,
|
||||
1.0,
|
||||
*head_dims,
|
||||
monkeypatch,
|
||||
block_stride_rows=block_stride_rows,
|
||||
)
|
||||
|
||||
|
||||
def _triton_convert_reference_impl(
|
||||
req_ids: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
@@ -25,7 +26,7 @@ from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -48,10 +49,10 @@ pytestmark = pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"backend,cache_dtype,sm100,stride_alignment",
|
||||
[
|
||||
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
|
||||
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
|
||||
(FlashInferMLASparseTRTLLMBackend, "auto", True, 32 * 1152),
|
||||
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 32 * 576),
|
||||
(FlashMLASparseBackend, "auto", True, 1152),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", False, MultipleOf(64)),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
|
||||
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
|
||||
],
|
||||
@@ -79,7 +80,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
|
||||
spec = MLAAttention.get_kv_cache_spec(layer, config)
|
||||
spec = customize_attention_spec(
|
||||
backend, MLAAttention.get_kv_cache_spec(layer, config)
|
||||
)
|
||||
indexer = SimpleNamespace(
|
||||
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
|
||||
)
|
||||
@@ -102,10 +105,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
MLAAttention.bind_kv_cache(layer, views["mla"])
|
||||
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
|
||||
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
|
||||
if stride_alignment is not None:
|
||||
assert (
|
||||
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
|
||||
)
|
||||
alignment = spec.get_block_stride_alignment()
|
||||
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
|
||||
if not layer._uses_flat_kv_cache():
|
||||
register.assert_not_called()
|
||||
else:
|
||||
@@ -113,7 +114,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
register.assert_called_once_with(config, block_stride_rows=stride)
|
||||
layer.kv_cache[1, 0].fill_(3)
|
||||
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
|
||||
if stride_alignment is None:
|
||||
if alignment == 1:
|
||||
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
|
||||
item.page_size_bytes for item in specs.values()
|
||||
)
|
||||
|
||||
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=mamba_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=mamba_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=kpool_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=kpool_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -88,7 +88,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
|
||||
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
|
||||
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
|
||||
alignment = lcm(1152, hot.page_size_bytes)
|
||||
stride = _pool_bytes_per_block(groups)
|
||||
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
|
||||
assert stride == alignment
|
||||
layout = KVCacheLayout.BLHNC
|
||||
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
|
||||
@@ -254,7 +254,7 @@ class TestCSALinearGrouping:
|
||||
def test_every_group_fits_one_packed_block(self):
|
||||
config = _shared_layout_config()
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
|
||||
|
||||
pages = _pages(groups)
|
||||
for group in groups:
|
||||
@@ -418,7 +418,7 @@ class TestCSALinearGrouping:
|
||||
groups = _get_packed_kv_cache_groups(config, specs)
|
||||
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
|
||||
|
||||
assert _get_kv_cache_bytes_per_block(groups) == sum(
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
|
||||
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
|
||||
)
|
||||
# That block holds every GDN state at once, so the repeat pattern alone
|
||||
@@ -465,7 +465,7 @@ class TestSlidingWindowBucketCap:
|
||||
pages = _pages(groups)
|
||||
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
|
||||
main_bytes = sum(pages[n] for n in main.layer_names)
|
||||
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
|
||||
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
|
||||
per_group = main_bytes // pages["layers.0.swa"]
|
||||
assert len(swa_groups) == -(-43 // per_group)
|
||||
@@ -479,9 +479,9 @@ class TestSlidingWindowBucketCap:
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
|
||||
groups
|
||||
)
|
||||
assert _get_kv_cache_bytes_per_block(
|
||||
groups, KVCacheLayout.BLHNC
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("BLHNC"), groups, MEMORY
|
||||
@@ -554,6 +554,20 @@ class TestDensePacking:
|
||||
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
|
||||
assert mla_tensor.block_stride == block_stride
|
||||
|
||||
def test_layer_outer_pads_only_unaligned_pages(self):
|
||||
"""Layer-outer block strides are pages, so only an unaligned page pads."""
|
||||
mla_page = _mla(512).page_size_bytes
|
||||
idx = replace(_mla(128), block_stride_alignment=12288)
|
||||
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("LBNHC"), groups, MEMORY
|
||||
)
|
||||
mla_tensor, idx_tensor = config.kv_cache_tensors
|
||||
assert config.num_blocks == MEMORY // (mla_page + 12288)
|
||||
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
|
||||
assert idx_tensor.offset == mla_page * config.num_blocks
|
||||
_bind(config, "LBNHC")
|
||||
|
||||
def test_overlaid_groups_alias_and_stay_isolated(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
# Overlay models resolve to a block-outer layout at backend selection (the
|
||||
@@ -562,7 +576,9 @@ class TestDensePacking:
|
||||
_mock_vllm_config("BLNHC"), groups, MEMORY
|
||||
)
|
||||
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(
|
||||
_mock_vllm_config("BLNHC"), groups
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
views = _bind(config, "BLNHC")
|
||||
assert set(views) == set(g1) | set(g2)
|
||||
@@ -667,7 +683,10 @@ class TestCompressorRingGroup:
|
||||
assert len(ring_groups) == 1
|
||||
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
|
||||
manager = KVCacheManager(
|
||||
@@ -716,7 +735,10 @@ class TestSWABoundedReplayGrouping:
|
||||
for g in groups
|
||||
) == [(False, 128)] * 4 + [(True, 0)]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([kv_cache_config]),
|
||||
|
||||
@@ -10,7 +10,7 @@ from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -19,7 +19,6 @@ import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
@@ -69,7 +68,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
HiddenStateCacheSpec,
|
||||
HiSparseHotSpec,
|
||||
HiSparseResidentSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
@@ -1942,13 +1940,12 @@ def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
|
||||
config.parallel_config.decode_context_parallel_size = 4
|
||||
config.cache_config.kv_cache_layout = None
|
||||
tail = KpoolTailSpec(
|
||||
tail = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
dcp_sharded=False,
|
||||
)
|
||||
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
|
||||
@@ -2906,564 +2903,77 @@ def test_generate_scheduler_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> tuple[dict[str, KVCacheSpec], list[str]]:
|
||||
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
|
||||
def test_packed_groups_glm5_like_hybrid():
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=8192))
|
||||
vllm_config.cache_config.kv_cache_layout = "BLHNC"
|
||||
kpool, block_size = 4, 1024
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
mamba_layers = []
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
name = f"layers.{i}.linear_attn"
|
||||
kv_cache_spec[name] = mamba_spec_factory()
|
||||
mamba_layers.append(name)
|
||||
return kv_cache_spec, mamba_layers
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec_with_tail(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
|
||||
|
||||
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
|
||||
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
|
||||
is why the fixture drops the tokens_per_state=16 shape used above.
|
||||
"""
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
|
||||
for i in range(3, 45, 4):
|
||||
kpool = 4
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = replace(
|
||||
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
|
||||
tokens_per_state=kpool,
|
||||
for i in range(11):
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
block_stride_alignment=32 * 576 * 2,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=kpool,
|
||||
block_stride_alignment=64 * 132,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = CircularBufferSpec(
|
||||
block_size=kpool,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=kpool,
|
||||
)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
|
||||
return {
|
||||
layer_name: tensor
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
for layer_name in tensor.layers
|
||||
}
|
||||
|
||||
|
||||
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
|
||||
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
|
||||
|
||||
|
||||
def test_get_kv_cache_config_balanced_mamba_hybrid():
|
||||
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
|
||||
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
|
||||
# gets an MLA slot.
|
||||
assert len(groups) == 5
|
||||
assert len(uniform_groups) == 1
|
||||
assert len(uniform_groups[0].layer_names) == 22
|
||||
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 4].layer_names
|
||||
# Mamba pages are padded up to the MLA page.
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_padded == mla_page
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
# Every block id is charged the full per-block sum.
|
||||
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
available_memory = bytes_per_block * 100 + 1
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, available_memory
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
|
||||
# Every logical layer has a view into one backing allocation. Mamba views
|
||||
# alias their corresponding MLA slot by using the same byte offset.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
|
||||
for i, mla_name in enumerate(mla_layer_names):
|
||||
mla_tensor = tensors[mla_name]
|
||||
assert mla_tensor.block_stride == mla_page
|
||||
for group in mamba_groups:
|
||||
if i < len(group.layer_names):
|
||||
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
|
||||
for i in range(11):
|
||||
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
|
||||
|
||||
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
|
||||
assert total_allocated == bytes_per_block * 100
|
||||
assert 0 <= available_memory - total_allocated < bytes_per_block
|
||||
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config
|
||||
) == pytest.approx(100 / blocks_per_request)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
|
||||
assert tail_logical_page < idx_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
tail_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and all(
|
||||
isinstance(spec, KpoolTailSpec)
|
||||
for spec in group.kv_cache_spec.kv_cache_specs.values()
|
||||
for i in range(34):
|
||||
kv_cache_spec[f"layers.{i}.kda"] = MambaSpec(
|
||||
shapes=((3, 6144), (16, 128, 128)),
|
||||
dtypes=(torch.bfloat16, torch.float32),
|
||||
block_size=block_size,
|
||||
)
|
||||
)
|
||||
# The tail never prefix-caches; its page is padded up to the indexer page
|
||||
# so the runner's strided view rides the indexer storage.
|
||||
assert not tail_group.kv_cache_spec.prefix_cacheable
|
||||
tail_inner = cast(
|
||||
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
|
||||
)
|
||||
assert tail_inner.page_size_padded == idx_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Tail layers get logical views but no additional storage: each view aliases
|
||||
# its sibling indexer at the same offset in the shared allocation.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 67
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
for i in range(11):
|
||||
idx_name = f"layers.{4 * i + 3}.indexer"
|
||||
tail_name = f"layers.{4 * i + 3}.tail"
|
||||
assert tensors[idx_name].offset == tensors[tail_name].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
# The layout detector surfaces the sibling names in layer order for the
|
||||
# accounting and connector paths.
|
||||
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
|
||||
assert layout is not None
|
||||
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
|
||||
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
|
||||
|
||||
# Accounting charges exactly one extra shared block per request for the
|
||||
# tail, not a per-sequence allocation.
|
||||
attn_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and group is not tail_group
|
||||
)
|
||||
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def align_mamba():
|
||||
return new_mamba_spec(mamba_cache_mode="align")
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(
|
||||
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
hash_vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
)
|
||||
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
|
||||
# the lcm including the tail (1024 % 4 == 0, so it coincides).
|
||||
assert kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, hash_vllm_config
|
||||
) == (1024, 16)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
|
||||
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
|
||||
def big_mamba_spec():
|
||||
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert big_mamba_spec().page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(ValueError, match="does not fit the MLA page"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
"""Use the generic layout error when no kpool indexer is present."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
|
||||
shapes=((512, 1024),), dtypes=(torch.float32,)
|
||||
)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(NotImplementedError, match="page size"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def prepadded_mamba_spec():
|
||||
return new_mamba_spec(page_size_padded=294_912)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert prepadded_mamba_spec().page_size_bytes < mla_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
attn = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and isinstance(g.kv_cache_spec.first_spec, MLAAttentionSpec)
|
||||
]
|
||||
assert len(groups) == 5
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
# PP=2 split at transformer layer 22/23.
|
||||
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
|
||||
# layers, projected mamba groups [5, 5, 4, 4].
|
||||
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
|
||||
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, worker_spec
|
||||
)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
rings = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and isinstance(g.kv_cache_spec.first_spec, CircularBufferSpec)
|
||||
]
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 5 * mla_page + 5 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
# Every projected Mamba slot aliases an MLA owner.
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
|
||||
# The last slot only hosts the two groups with a 5th projected layer.
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=4),
|
||||
)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
|
||||
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 5].layer_names
|
||||
|
||||
for rank in range(4):
|
||||
start, end = get_pp_indices(45, rank, 4)
|
||||
worker_spec = {
|
||||
n: s
|
||||
for n, s in kv_cache_spec.items()
|
||||
if start <= int(n.split(".")[1]) < end
|
||||
}
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
groups, worker_spec
|
||||
mamba = [
|
||||
g
|
||||
for g in groups
|
||||
if isinstance(
|
||||
getattr(g.kv_cache_spec, "first_spec", g.kv_cache_spec), MambaSpec
|
||||
)
|
||||
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
|
||||
from vllm.config import ParallelConfig
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, kv_transfer_config=kv_transfer_config
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert all(
|
||||
tensor.block_stride in (mla_page, idx_page)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
|
||||
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 4
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 7 * mla_page
|
||||
assert len(attn) == 1 and len(attn[0].layer_names) == 22
|
||||
assert len(rings) == 1 and len(rings[0].layer_names) == 11
|
||||
# 11 * (1.18 MB MLA + 34 KB indexer) per block fits 12 fp32-SSM KDA states.
|
||||
assert len(mamba) == 3
|
||||
assert sum(len(g.layer_names) for g in mamba) == 34
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Each group has one tensor descriptor and all groups alias the same slots.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 4
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
mla_offset = _layer_offset(tensors[mla_name], mla_name)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
mamba_name = group.layer_names[index]
|
||||
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_capacity_after_scheduler_unwrap():
|
||||
"""max_concurrency must survive the scheduler-config unwrap.
|
||||
|
||||
Regression for the balanced-mamba hybrid layout:
|
||||
``generate_scheduler_kv_cache_config`` flattens the MLA
|
||||
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
|
||||
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
|
||||
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
|
||||
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
|
||||
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
# Confirm we are exercising the regression path: the MLA group is no longer
|
||||
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
|
||||
assert not any(
|
||||
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for g in scheduler_config.kv_cache_groups
|
||||
)
|
||||
|
||||
unwrapped_groups = scheduler_config.kv_cache_groups
|
||||
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
|
||||
vllm_config, unwrapped_groups
|
||||
)
|
||||
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
|
||||
expected_blocks_per_request = (
|
||||
expected_max_mem + expected_pool - 1
|
||||
) // expected_pool
|
||||
|
||||
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
|
||||
assert max_concurrency > 0
|
||||
assert max_concurrency == pytest.approx(
|
||||
scheduler_config.num_blocks / expected_blocks_per_request
|
||||
vllm_config, groups, available_memory=2 * 1024**3
|
||||
)
|
||||
strides = {t.block_stride for t in kv_cache_config.kv_cache_tensors}
|
||||
assert len(strides) == 1
|
||||
(block_stride,) = strides
|
||||
# Blocks sit a whole number of 32-row MLA pages and 64-state indexer pages
|
||||
# apart so both caches can be re-paged for their kernels.
|
||||
assert block_stride % (32 * 576 * 2) == 0
|
||||
assert block_stride % (64 * 132) == 0
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
@@ -4483,6 +3993,7 @@ def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
@@ -4528,6 +4039,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
block_size = 16
|
||||
max_model_len = 512 # needs 512 / 16 = 32 blocks
|
||||
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
spec = new_kv_cache_spec(block_size=block_size)
|
||||
|
||||
# 32 blocks -> only 31 usable after the null block: one short -> reject.
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
KpoolTailManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -406,15 +404,15 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
|
||||
@pytest.mark.parametrize("record_for_zeroing", [True, False])
|
||||
def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
"""A transferred ring at index zero skips zeroing even with a local hit."""
|
||||
spec = KpoolTailSpec(
|
||||
"""The kpool tail ring is never zeroed, transferred or not: a pool only
|
||||
reads ring slots its own request wrote."""
|
||||
spec = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
manager = KpoolTailManager(
|
||||
manager = CircularBufferManager(
|
||||
spec,
|
||||
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
|
||||
enable_caching=True,
|
||||
@@ -428,9 +426,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
)
|
||||
blocks = manager.req_to_blocks["request"]
|
||||
assert len(blocks) == 1
|
||||
assert manager.take_new_block_ids() == (
|
||||
[blocks[0].block_id] if record_for_zeroing else []
|
||||
)
|
||||
assert manager.take_new_block_ids() == []
|
||||
assert manager.allocate_new_blocks("request", 20, 20) == []
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
|
||||
@@ -23,5 +23,5 @@ def _mock_cpu_backend_block_sizes(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
CPUAttentionBackend,
|
||||
"get_supported_kernel_block_sizes",
|
||||
staticmethod(lambda: [MultipleOf(16)]),
|
||||
staticmethod(lambda kv_cache_spec=None: [MultipleOf(16)]),
|
||||
)
|
||||
|
||||
@@ -164,13 +164,12 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
gdn_spec = kv_cache_config.kv_cache_groups[1].kv_cache_spec
|
||||
fa_raw = torch.empty(num_blocks * fa_spec.page_size_bytes, dtype=torch.int8)
|
||||
gdn_raw = torch.empty(num_blocks * gdn_spec.page_size_bytes, dtype=torch.int8)
|
||||
gdn_block_stride = gdn_spec.page_size_bytes + 64
|
||||
gdn_raw = torch.empty(num_blocks, gdn_block_stride, dtype=torch.int8)
|
||||
(fa_cache,) = dense_kv_cache_views(
|
||||
fa_raw, fa_spec, num_blocks, 1, KVCacheLayout.LBHNC
|
||||
)
|
||||
(gdn_cache,) = dense_kv_cache_views(
|
||||
gdn_raw, gdn_spec, num_blocks, 1, KVCacheLayout.LBHNC
|
||||
)
|
||||
gdn_cache = gdn_raw[:, : gdn_spec.page_size_bytes]
|
||||
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
@@ -184,7 +183,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
"model.layers.0.self_attn",
|
||||
"model.layers.1.linear_attn",
|
||||
]
|
||||
assert worker.block_len_per_layer == [
|
||||
assert worker.kv_block_len_per_layer == [
|
||||
fa_spec.page_size_bytes,
|
||||
gdn_spec.page_size_bytes,
|
||||
]
|
||||
|
||||
@@ -31,7 +31,6 @@ from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -711,13 +710,12 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
# Scratch block 4 is not divisible by the hash unit 8 below.
|
||||
scratch = KpoolTailSpec(
|
||||
scratch = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
|
||||
@@ -641,164 +641,43 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("logical_block_size", [1152, 640])
|
||||
@pytest.mark.parametrize("tail_first", [False, True])
|
||||
def test_register_compressed_indexer_uses_virtual_transfer_pages(
|
||||
logical_block_size, tail_first
|
||||
):
|
||||
"""Compressed indexer rows must split into contiguous NIXL transfer pages."""
|
||||
from unittest.mock import MagicMock
|
||||
def test_nixl_keeps_packed_sparse_mla_manager_block():
|
||||
"""Like the worker, NIXL keeps packed sparse MLA + indexer manager blocks."""
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import UniformTypeKVCacheSpecs
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
base_worker as bw,
|
||||
specs = {
|
||||
"mla": MLAAttentionSpec(
|
||||
1152, num_kv_heads=1, head_size=512, dtype=torch.bfloat16
|
||||
),
|
||||
"indexer": MLAAttentionSpec(
|
||||
1152, num_kv_heads=1, head_size=128, dtype=torch.uint8, tokens_per_state=4
|
||||
),
|
||||
}
|
||||
backends = {"mla": FlashMLASparseBackend, "indexer": Glm5NextIndexerBackend}
|
||||
layers = {
|
||||
name: MagicMock(spec=AttentionLayerBase, **{"get_attn_backend.return_value": b})
|
||||
for name, b in backends.items()
|
||||
}
|
||||
worker = object.__new__(bw.NixlBaseConnectorWorker)
|
||||
worker.block_size, worker.num_blocks, worker.attn_backends = 1152, 2, []
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
compilation_config=SimpleNamespace(static_forward_context=layers),
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
worker.kv_cache_config = KVCacheConfig(
|
||||
2, [], [KVCacheGroupSpec(list(specs), UniformTypeKVCacheSpecs(1152, specs))]
|
||||
)
|
||||
|
||||
num_logical_blocks = 3
|
||||
transfer_block_size = 64
|
||||
kernel_block_size = 128
|
||||
tokens_per_state = 4
|
||||
state_content_bytes = 132
|
||||
worker._sync_block_size_with_kernel()
|
||||
|
||||
indexer_spec = MLAAttentionSpec(
|
||||
block_size=logical_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=state_content_bytes,
|
||||
tokens_per_state=tokens_per_state,
|
||||
)
|
||||
indexer_page_size = indexer_spec.page_size_bytes
|
||||
tail_spec = KpoolTailSpec(
|
||||
block_size=tokens_per_state,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
page_size_padded=indexer_page_size,
|
||||
sliding_window=tokens_per_state,
|
||||
)
|
||||
|
||||
allocation_size = num_logical_blocks * indexer_page_size
|
||||
indexer_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["indexer"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
tail_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["tail"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_logical_blocks,
|
||||
kv_cache_tensors=[indexer_tensor, tail_tensor],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["indexer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=logical_block_size,
|
||||
kv_cache_specs={"indexer": indexer_spec},
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=tokens_per_state,
|
||||
kv_cache_specs={"tail": tail_spec},
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
raw = torch.zeros(allocation_size, dtype=torch.int8)
|
||||
(indexer_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
indexer_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
indexer_tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
)
|
||||
(tail_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
tail_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
tail_tensor,
|
||||
)
|
||||
assert indexer_cache.data_ptr() == tail_cache.data_ptr() == raw.data_ptr()
|
||||
|
||||
vllm_config = create_vllm_config(block_size=logical_block_size)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [transfer_block_size]
|
||||
fake_backend.get_name.return_value = "DEEPSEEK_V32_INDEXER"
|
||||
fake_backend.full_cls_name.return_value = "fake.DEEPSEEK_V32_INDEXER"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
caches = [("indexer", indexer_cache), ("tail", tail_cache)]
|
||||
if tail_first:
|
||||
caches.reverse()
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
worker.use_mla = True
|
||||
worker.register_kv_caches(dict(caches))
|
||||
|
||||
transfer_page_size = transfer_block_size // tokens_per_state * state_content_bytes
|
||||
num_transfer_blocks = num_logical_blocks * (
|
||||
logical_block_size // transfer_block_size
|
||||
)
|
||||
expected_descs = np.asarray(
|
||||
[
|
||||
[
|
||||
raw.data_ptr() + block_idx * transfer_page_size,
|
||||
transfer_page_size,
|
||||
0,
|
||||
]
|
||||
for block_idx in range(num_transfer_blocks)
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
assert worker.block_size == transfer_block_size
|
||||
assert worker.num_regions == 1
|
||||
assert worker.block_len_per_layer == [transfer_page_size]
|
||||
assert worker.block_stride_per_layer == [transfer_page_size]
|
||||
assert worker._region_is_mla == [True]
|
||||
assert worker.kv_caches_base_addr[worker.engine_id][0] == [raw.data_ptr()]
|
||||
assert worker._registered_descs[0] == [(raw.data_ptr(), raw.nbytes, 0, "")]
|
||||
np.testing.assert_array_equal(worker.src_blocks_data, expected_descs)
|
||||
assert expected_descs[-1, 0] + expected_descs[-1, 1] == (
|
||||
raw.data_ptr() + raw.nbytes
|
||||
)
|
||||
assert worker.block_size == 1152
|
||||
assert worker._physical_blocks_per_logical_kv_block == 1
|
||||
|
||||
|
||||
def _make_remote_meta(
|
||||
|
||||
@@ -47,7 +47,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -145,13 +144,12 @@ def _make_scratch_kv_cache_config(
|
||||
register_all_kvcache_specs(vllm_config=None)
|
||||
fa_config = _make_kv_cache_config(num_blocks, num_groups=1)
|
||||
scratch_layers = ["layer_scratch"]
|
||||
scratch_spec = KpoolTailSpec(
|
||||
scratch_spec = CircularBufferSpec(
|
||||
block_size=scratch_block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=HEAD_SIZE,
|
||||
head_size_v=0,
|
||||
dtype=DTYPE,
|
||||
sliding_window=scratch_block_size,
|
||||
)
|
||||
assert not scratch_spec.prefix_cacheable
|
||||
scratch_bytes = scratch_spec.page_size_bytes * num_blocks
|
||||
|
||||
@@ -102,7 +102,7 @@ def _run_prepare(
|
||||
block_tables=SimpleNamespace(
|
||||
slot_mappings=query_slot_mapping.unsqueeze(0),
|
||||
input_block_tables=[block_table],
|
||||
kernel_block_sizes=[4],
|
||||
block_sizes=[4],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=cp_interleave,
|
||||
|
||||
@@ -963,6 +963,9 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
# Mock draft_attn_groups for attention metadata building.
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.build_metadata_for_drafting = (
|
||||
attn_metadata_builder.build_for_drafting
|
||||
)
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
@@ -1066,6 +1069,9 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
|
||||
)
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.build_metadata_for_drafting = (
|
||||
attn_metadata_builder.build_for_drafting
|
||||
)
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for SpecDecodeBaseProposer.initialize_attn_backend.
|
||||
|
||||
Block tables are stored at kernel-block granularity, so the proposer's
|
||||
``block_size`` (used for slot-mapping math) must be the kernel block size,
|
||||
not the KV cache manager's block size — the two differ when manager blocks
|
||||
are split for the attention kernel. The value must also be deterministic:
|
||||
Block tables are stored at manager-block granularity, so the proposer's
|
||||
``block_size`` (used for slot-mapping math) must be the KV cache manager's
|
||||
block size. Attention groups map to kernel blocks for their metadata builders.
|
||||
The value must also be deterministic:
|
||||
``_draft_attn_layer_names`` is a set, whose iteration order varies across
|
||||
processes, so anything derived from iteration order must not leak into
|
||||
``block_size``.
|
||||
@@ -68,9 +68,9 @@ def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
|
||||
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
"""The proposer's slot-mapping math runs against the kernel-granularity
|
||||
block table, so block_size must come from kernel_block_sizes."""
|
||||
def test_block_size_uses_manager_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
"""The proposer's slot-mapping math runs against the manager-granularity
|
||||
block table; attention groups map kernel blocks for metadata builders."""
|
||||
layer_names = {"draft.0.self_attn.attn"}
|
||||
proposer = _make_proposer(monkeypatch, layer_names)
|
||||
|
||||
@@ -79,12 +79,8 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
kernel_block_sizes=[KERNEL_BLOCK_SIZE],
|
||||
)
|
||||
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
|
||||
# The metadata builder keeps receiving the kernel block size as well.
|
||||
group = proposer.draft_attn_groups[0]
|
||||
assert isinstance(group, _FakeAttentionGroup)
|
||||
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
|
||||
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
|
||||
|
||||
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -112,4 +108,4 @@ def test_draft_layer_iteration_is_deterministic(monkeypatch: pytest.MonkeyPatch)
|
||||
)
|
||||
assert len(proposer.draft_attn_groups) == 1
|
||||
assert proposer.draft_attn_groups[0].layer_names == expected_order
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
|
||||
|
||||
@@ -38,7 +38,6 @@ def mock_model_runner_with_input_batch():
|
||||
device="cpu",
|
||||
vocab_size=32000,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
logitsprocs=None,
|
||||
is_pooling_model=False,
|
||||
|
||||
@@ -15,7 +15,11 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -42,6 +46,8 @@ from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
allocate_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
)
|
||||
|
||||
|
||||
@@ -79,7 +85,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
backend = SimpleNamespace(
|
||||
get_name=lambda name=name: name,
|
||||
customize_spec=AttentionBackend.customize_spec,
|
||||
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
|
||||
get_supported_kernel_block_sizes=lambda _=None, sizes=sizes: sizes,
|
||||
)
|
||||
layers[name] = SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
|
||||
@@ -99,7 +105,8 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
|
||||
resolved = attn_utils.get_kv_cache_spec(config)
|
||||
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
|
||||
assert resolved["dense"] is specs["dense"]
|
||||
dense_backend = layers["dense"].get_attn_backend()
|
||||
assert resolved["dense"] == customize_attention_spec(dense_backend, specs["dense"])
|
||||
assert all(spec.block_size == block_size for spec in specs.values())
|
||||
|
||||
|
||||
@@ -495,20 +502,17 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kernel_block_sizes",
|
||||
"storage_block_size",
|
||||
"expected_num_blocks",
|
||||
"expected_num_states",
|
||||
),
|
||||
[
|
||||
(None, None, 4, 64),
|
||||
([256], None, 4, 64),
|
||||
([64], None, 16, 16),
|
||||
([64], 256, 4, 64),
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
storage_block_size: int | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
@@ -518,7 +522,6 @@ def test_allocate_compressed_mla_cache(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
storage_block_size=storage_block_size,
|
||||
)
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
@@ -744,3 +747,92 @@ def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
|
||||
backing.untyped_storage().data_ptr()
|
||||
== caches["indexer"].untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
|
||||
class _TableBuilder:
|
||||
requires_block_table_width = False
|
||||
supports_update_block_table = False
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
return common_attn_metadata.block_table_tensor
|
||||
|
||||
|
||||
_VLLM_CONFIG = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
)
|
||||
|
||||
|
||||
def _packed_group(tokens_per_state, row_bytes, supported_sizes):
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=row_bytes,
|
||||
tokens_per_state=tokens_per_state,
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
get_builder_cls=lambda: _TableBuilder,
|
||||
get_supported_kernel_block_sizes=lambda _=None: supported_sizes,
|
||||
get_name=lambda: "FAKE",
|
||||
customize_spec=lambda spec: spec,
|
||||
)
|
||||
return AttentionGroup(backend, ["layer.0"], spec, 0)
|
||||
|
||||
|
||||
def test_packed_compressed_group_gets_kernel_block_table():
|
||||
"""A packed indexer cache whose kernel needs 64-state pages gets a table and
|
||||
a bound view in kernel blocks that address the same packed-block rows."""
|
||||
row, stride_rows = 132, 5 * 64
|
||||
group = _packed_group(4, row, [128, 256])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
|
||||
assert group.get_metadata_builder().kv_cache_spec.block_size == 256
|
||||
|
||||
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
|
||||
manager = backing.view(3, 1, stride_rows, row)[:, :, :256]
|
||||
pages = map_kv_caches_to_kernel_blocks({"layer.0": manager}, [group])["layer.0"]
|
||||
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
|
||||
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
|
||||
table = group.build_metadata(metadata)
|
||||
assert table.tolist() == [[0, 1, 2, 3, 10, 11, 12, 13]]
|
||||
assert torch.equal(pages[table[0, 4:], 0].flatten(0, 1), manager[2, 0])
|
||||
|
||||
|
||||
def test_split_group_maps_manager_block_table():
|
||||
"""Groups split in a dense layout keep manager block tables; builders get
|
||||
b * blocks_per_kv_block + j against the view allocated in kernel blocks."""
|
||||
group = _packed_group(4, 132, [128, 256])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=256)
|
||||
kernel_view = torch.zeros(12, 1, 64, 132)
|
||||
bound = map_kv_caches_to_kernel_blocks({"layer.0": kernel_view}, [group])
|
||||
assert bound["layer.0"] is kernel_view
|
||||
|
||||
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
|
||||
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
|
||||
table = group.build_metadata(metadata)
|
||||
assert table.tolist() == [[0, 1, 2, 3, 8, 9, 10, 11]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_size", "page_states"), [(1024, 64), (384, 32), (256, 0)]
|
||||
)
|
||||
def test_split_blocks_align_to_kernel_blocks(block_size, page_states):
|
||||
"""Backends taking only fixed kernel blocks align packed blocks to the kernel
|
||||
block the final block size is split into, and not at all when unsplit."""
|
||||
group = _packed_group(4, 132, [128, 256])
|
||||
spec = customize_attention_spec(group.backend, group.kv_cache_spec)
|
||||
spec = spec.copy_with_new_block_size(block_size)
|
||||
assert spec.get_block_stride_alignment() == max(page_states * 132, 1)
|
||||
|
||||
|
||||
def test_packed_token_cache_cannot_be_split():
|
||||
group = _packed_group(1, 1152, [64])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
|
||||
packed = torch.zeros(3, 1, 2048, 1152, dtype=torch.uint8)[:, :, :1024]
|
||||
with pytest.raises(ValueError, match="cannot be split"):
|
||||
map_kv_caches_to_kernel_blocks({"layer.0": packed}, [group])
|
||||
|
||||
@@ -22,7 +22,6 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8, 8, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16, 16, 8],
|
||||
)
|
||||
|
||||
def fail_if_apply_write_called():
|
||||
@@ -48,33 +47,31 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
block_tables.block_tables[0].gpu[0, :2],
|
||||
torch.tensor([1, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
|
||||
# kernel block IDs.
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :4],
|
||||
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[0, :2],
|
||||
torch.tensor([10, 11], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[1, :1],
|
||||
torch.tensor([3], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[1, :2],
|
||||
torch.tensor([24, 25], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[1, :1],
|
||||
torch.tensor([12], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[1, :2],
|
||||
torch.tensor([5, 6], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 2
|
||||
assert block_tables.num_blocks.np[1, 0] == 4
|
||||
assert block_tables.num_blocks.np[1, 0] == 2
|
||||
assert block_tables.num_blocks.np[2, 0] == 0
|
||||
assert block_tables.num_blocks.np[0, 1] == 1
|
||||
assert block_tables.num_blocks.np[1, 1] == 2
|
||||
assert block_tables.num_blocks.np[1, 1] == 1
|
||||
assert block_tables.num_blocks.np[2, 1] == 2
|
||||
assert torch.equal(
|
||||
block_tables.num_blocks.gpu[:, :2],
|
||||
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
|
||||
torch.tensor([[2, 1], [2, 1], [0, 2]], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
for block_table in block_tables.block_tables:
|
||||
@@ -96,15 +93,15 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :6],
|
||||
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[0, :3],
|
||||
torch.tensor([10, 11, 13], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[0, :1],
|
||||
torch.tensor([8], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 3
|
||||
assert block_tables.num_blocks.np[1, 0] == 6
|
||||
assert block_tables.num_blocks.np[1, 0] == 3
|
||||
assert block_tables.num_blocks.np[2, 0] == 1
|
||||
|
||||
|
||||
@@ -116,7 +113,6 @@ def test_block_tables_apply_staged_writes_single_group():
|
||||
max_num_batched_tokens=16,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
|
||||
block_tables.append_block_ids(
|
||||
@@ -141,7 +137,6 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
max_num_batched_tokens=4,
|
||||
max_num_blocks_per_group=[1, 1],
|
||||
device=device,
|
||||
kernel_block_sizes=[8, 262144],
|
||||
slot_mapping_enabled=[False, True],
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
@@ -170,7 +165,7 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
def test_dcp_slot_mapping_only_shards_sharded_groups(cp_rank: int):
|
||||
"""Only sharded groups use DCP interleave in logical-block coordinates."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
@@ -179,7 +174,6 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_blocks_per_group=[2, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[64, 64],
|
||||
dcp_sharded=[True, False],
|
||||
cp_size=4,
|
||||
cp_rank=cp_rank,
|
||||
@@ -230,7 +224,6 @@ def test_v1_block_table_move_row_clears_vacated_row():
|
||||
max_num_batched_tokens=64,
|
||||
pin_memory=False,
|
||||
device=torch.device("cuda"),
|
||||
kernel_block_size=16,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
block_table.add_row([7, 8, 9], row_idx=0)
|
||||
@@ -260,7 +253,6 @@ def test_get_dummy_block_tables_returns_zeroed_rows():
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
# Simulate a real step: stage a request's blocks and gather them into
|
||||
# the persistent input block tables.
|
||||
@@ -292,7 +284,6 @@ def test_dummy_request_slot_mapping_is_pad():
|
||||
max_num_batched_tokens=16,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[4],
|
||||
)
|
||||
block_tables.append_block_ids(req_index=0, new_block_ids=([7, 8],), overwrite=True)
|
||||
block_tables.apply_staged_writes()
|
||||
|
||||
@@ -233,7 +233,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
reqs: list[CachedRequestState] = []
|
||||
@@ -327,7 +326,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
ref_input_batch: InputBatch = InputBatch(
|
||||
@@ -337,7 +335,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -388,7 +385,6 @@ def test_condense_clears_stale_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -446,7 +442,6 @@ def test_swap_states_preserves_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -516,7 +511,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -552,7 +546,6 @@ def test_placeholder_spec_token_ids_written_verbatim():
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[1],
|
||||
)
|
||||
req = CachedRequestState(
|
||||
@@ -599,7 +592,6 @@ def test_pooling_metadata_token_id_buffers(
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -630,7 +622,6 @@ def _make_input_batch(is_pooling_model: bool = False) -> InputBatch:
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=is_pooling_model,
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
The kpool tail cache holds one circular block per request, so its block table
|
||||
row is orders of magnitude shorter than the sequence it serves.
|
||||
`_compute_slot_mappings_kernel` indexes block tables by
|
||||
`position // kernel_block_size` with an effectively unmasked load, so
|
||||
`position // block_size` with an effectively unmasked load, so
|
||||
this leads to a read of neighboring requests' rows (and for the later requests, outside
|
||||
the buffer entirely).
|
||||
"""
|
||||
@@ -15,7 +15,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
@@ -32,13 +32,12 @@ POISON_BLOCK_ID = 1 << 20
|
||||
|
||||
|
||||
def make_tail_spec():
|
||||
return KpoolTailSpec(
|
||||
return CircularBufferSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
|
||||
|
||||
@@ -67,7 +66,6 @@ def test_kpool_tail_group_never_position_indexes_its_block_table():
|
||||
max_num_batched_tokens=len(positions),
|
||||
max_num_blocks_per_group=[mla_width, tail_width],
|
||||
device=device,
|
||||
kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size],
|
||||
slot_mapping_enabled=[True, spec.uses_slot_mapping],
|
||||
)
|
||||
mla_blocks = list(range(1, mla_width + 1))
|
||||
|
||||
@@ -111,9 +111,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
@@ -303,7 +300,7 @@ def _make_mock_backend_for_kernel_block_size(
|
||||
return "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return supported_sizes
|
||||
|
||||
return _MockBackend()
|
||||
@@ -441,7 +438,7 @@ def _mock_backend(supported: list, *, exact: bool = False):
|
||||
return "MOCK_EXACT" if exact else "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return list(supported)
|
||||
|
||||
if exact:
|
||||
@@ -1625,7 +1622,6 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.is_pooling_model = False
|
||||
runner._init_block_sizes = [16]
|
||||
runner._init_kernel_block_sizes = [16]
|
||||
runner._init_max_num_blocks = [4]
|
||||
runner._init_slot_mapping_modes = [
|
||||
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
|
||||
@@ -1653,7 +1649,7 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
|
||||
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
|
||||
)
|
||||
|
||||
runner.may_reinitialize_input_batch(kv_cache_config, [16])
|
||||
runner.may_reinitialize_input_batch(kv_cache_config)
|
||||
|
||||
assert input_batch_cls.call_count == 1
|
||||
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
|
||||
@@ -1681,68 +1677,10 @@ def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
|
||||
assert runner.cp_interleave == 16
|
||||
|
||||
|
||||
def test_hybrid_block_table_initialization():
|
||||
"""Test hybrid block table with different kernel and kvcache_manager block
|
||||
sizes."""
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
|
||||
# Test configuration: kvcache_manager block size = 32,
|
||||
# kernel block size = 16
|
||||
block_size = 32
|
||||
kernel_block_sizes = [16]
|
||||
max_num_reqs = 10
|
||||
max_num_blocks_per_req = 20
|
||||
max_num_batched_tokens = 512
|
||||
cp_kv_cache_interleave_size = 8
|
||||
|
||||
block_table = BlockTable(
|
||||
block_size=block_size,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_blocks_per_req=max_num_blocks_per_req,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
pin_memory=False,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
kernel_block_size=kernel_block_sizes[0],
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
# Verify hybrid block configuration
|
||||
assert block_table.use_hybrid_blocks is True
|
||||
assert block_table.block_size == kernel_block_sizes[0]
|
||||
assert block_table.blocks_per_kv_block == (
|
||||
block_size // kernel_block_sizes[0]
|
||||
) # Changed to use first element
|
||||
|
||||
# Test block table conversion logic
|
||||
# One kvcache_manager block should map to multiple kernel blocks
|
||||
kvcache_manager_blocks = [0, 1, 2]
|
||||
|
||||
# Verify that kvcache_manager blocks can be converted to kernel blocks
|
||||
# and that block table operations work correctly.
|
||||
req_index = 0
|
||||
block_table.append_row(kvcache_manager_blocks, req_index)
|
||||
# Get expected kernel blocks from the implementation for verification.
|
||||
expected_kernel_blocks = block_table.map_to_kernel_blocks(
|
||||
np.array(kvcache_manager_blocks),
|
||||
block_table.blocks_per_kv_block,
|
||||
block_table._kernel_block_arange,
|
||||
)
|
||||
# Verify block table state
|
||||
assert block_table.num_blocks_per_row[req_index] == len(expected_kernel_blocks)
|
||||
assert np.array_equal(
|
||||
block_table.block_table.np[req_index, : len(expected_kernel_blocks)],
|
||||
expected_kernel_blocks,
|
||||
)
|
||||
|
||||
|
||||
def test_get_block_table_width_aligns_to_128_tokens():
|
||||
assert get_block_table_width(1875, 64) == 1876
|
||||
|
||||
|
||||
def test_get_block_table_width_splits_virtual_blocks():
|
||||
assert get_block_table_width(235, 256, 64) == 940
|
||||
|
||||
|
||||
def test_mamba_state_table_width_is_not_aligned():
|
||||
block_tables = MultiGroupBlockTable(
|
||||
max_num_reqs=1,
|
||||
@@ -1750,7 +1688,6 @@ def test_mamba_state_table_width_is_not_aligned():
|
||||
pin_memory=False,
|
||||
device=torch.device("cpu"),
|
||||
block_sizes=[39664],
|
||||
kernel_block_sizes=[39664],
|
||||
max_num_blocks=[1],
|
||||
slot_mapping_modes=[SlotMappingMode.NONE],
|
||||
)
|
||||
@@ -1758,42 +1695,6 @@ def test_mamba_state_table_width_is_not_aligned():
|
||||
assert block_tables[0].max_num_blocks_per_req == 1
|
||||
|
||||
|
||||
def test_input_batch_with_kernel_block_sizes():
|
||||
"""Test InputBatch initialization with kernel_block_sizes parameter."""
|
||||
max_num_reqs = 10
|
||||
max_model_len = 512
|
||||
max_num_batched_tokens = 512
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
vocab_size = 50272
|
||||
|
||||
# Test with different kernel block sizes
|
||||
block_sizes = [32, 64]
|
||||
kernel_block_sizes = [16, 32]
|
||||
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
device=device,
|
||||
vocab_size=vocab_size,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=[16, 8],
|
||||
)
|
||||
|
||||
# Verify that block tables were created with kernel block sizes
|
||||
assert len(input_batch.block_table.block_tables) == len(block_sizes)
|
||||
|
||||
for i, (kv_size, kernel_size) in enumerate(zip(block_sizes, kernel_block_sizes)):
|
||||
block_table = input_batch.block_table.block_tables[i]
|
||||
if kv_size != kernel_size:
|
||||
assert block_table.use_hybrid_blocks is True
|
||||
assert block_table.block_size == kernel_size
|
||||
else:
|
||||
assert block_table.use_hybrid_blocks is False
|
||||
assert block_table.block_size == kernel_size
|
||||
|
||||
|
||||
def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
"""Test hybrid cache architecture integration with GPUModelRunner."""
|
||||
# Create a new model runner with hybrid cache configuration
|
||||
@@ -1843,9 +1744,8 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
) # Use kernel block size
|
||||
)
|
||||
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
|
||||
@@ -119,9 +119,6 @@ def _initialize_kv_cache(runner: GPUModelRunner) -> None:
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
@@ -237,7 +237,6 @@ def test_append_block_ids_rejects_write_past_row_capacity():
|
||||
|
||||
block_tables = BlockTables.__new__(BlockTables)
|
||||
block_tables.num_kv_cache_groups = 1
|
||||
block_tables.blocks_per_kv_block = [1]
|
||||
block_tables.block_tables = [_BlockTable()]
|
||||
block_tables.num_blocks = SimpleNamespace(
|
||||
np=torch.tensor([[0, 3]], dtype=torch.int32)
|
||||
|
||||
@@ -47,8 +47,7 @@ def _make_capture_manager(block_table: torch.Tensor):
|
||||
block_tables = SimpleNamespace(
|
||||
input_block_tables=(block_table,),
|
||||
num_kv_cache_groups=1,
|
||||
kernel_block_sizes=(2,),
|
||||
blocks_per_kv_block=(1,),
|
||||
block_sizes=(2,),
|
||||
)
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.models.mistral_large_3_eagle import (
|
||||
from vllm.v1.attention.backends import flash_attn as flash_attn_module
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
|
||||
@@ -30,6 +31,7 @@ from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spe
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class _TestSpeculator(TargetDependentARSpeculator):
|
||||
@@ -102,6 +104,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
get_metadata_builder=lambda _: SimpleNamespace(
|
||||
build=build, supports_update_block_table=False
|
||||
),
|
||||
build_metadata=lambda cm, ubatch_idx=0, **_: build(0, cm),
|
||||
layer_names=["draft"],
|
||||
)
|
||||
]
|
||||
@@ -427,6 +430,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
query_start_loc=torch.arange(3),
|
||||
)
|
||||
speculator.idx_mapping = torch.arange(2)
|
||||
speculator.attn_groups = []
|
||||
speculator.block_tables = SimpleNamespace(input_block_tables=[])
|
||||
generate_draft = Mock()
|
||||
speculator._generate_draft = generate_draft
|
||||
run_fullgraph = Mock()
|
||||
@@ -450,6 +455,52 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
assert run_fullgraph.call_count == expected_graph_replays
|
||||
|
||||
|
||||
def test_fused_full_graph_maps_split_group_block_tables():
|
||||
"""The fused FULL replay skips build(), so split groups' tables are mapped first."""
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1024, num_kv_heads=1, head_size=128, dtype=torch.uint8
|
||||
)
|
||||
group = AttentionGroup(
|
||||
SimpleNamespace(),
|
||||
["layer.0"],
|
||||
spec,
|
||||
0,
|
||||
kernel_block_size=256,
|
||||
kernel_block_table=torch.zeros(1, 4, 32, dtype=torch.int32),
|
||||
kernel_block_offsets=torch.arange(4, dtype=torch.int32),
|
||||
)
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(2), query_start_loc=torch.arange(3)
|
||||
)
|
||||
speculator.idx_mapping = torch.arange(2)
|
||||
speculator.attn_groups = [[group]]
|
||||
speculator.block_tables = SimpleNamespace(
|
||||
input_block_tables=[torch.tensor([[1, 2], [0, 3]], dtype=torch.int32)]
|
||||
)
|
||||
replayed_tables = []
|
||||
speculator.decode_cudagraph_manager = SimpleNamespace(
|
||||
run_fullgraph=lambda _: replayed_tables.append(
|
||||
group.kernel_block_table[0, :2, :8].tolist()
|
||||
)
|
||||
)
|
||||
|
||||
speculator._fused_multi_step_decode(
|
||||
num_reqs=2,
|
||||
skip_attn=False,
|
||||
batch_desc=BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.FULL, num_tokens=2, num_reqs=2
|
||||
),
|
||||
num_tokens_across_dp=None,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
num_speculative_steps=4,
|
||||
)
|
||||
|
||||
assert replayed_tables == [
|
||||
[[4, 5, 6, 7, 8, 9, 10, 11], [0, 1, 2, 3, 12, 13, 14, 15]]
|
||||
]
|
||||
|
||||
|
||||
def test_update_draft_decode_metadata_updates_fa3_scheduler_metadata(
|
||||
monkeypatch,
|
||||
):
|
||||
|
||||
@@ -12,40 +12,24 @@ if not current_platform.is_cuda_alike():
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_block_size", "blocks_per_kv_block", "block_size", "block_size_rep"),
|
||||
[
|
||||
(256, 1, 256, 16),
|
||||
(256, 4, 64, 16),
|
||||
(64, 1, 64, 16),
|
||||
(8, 1, 8, 2),
|
||||
(4, 1, 4, 2),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_block_size", [256, 64, 8, 4])
|
||||
def test_compute_slot_mapping_warmup_matches_runtime_specializations(
|
||||
kv_cache_block_size: int,
|
||||
blocks_per_kv_block: int,
|
||||
block_size: int,
|
||||
block_size_rep: int,
|
||||
) -> None:
|
||||
kernel = ComputeSlotMappingKernel()
|
||||
kwargs = dict(
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
blocks_per_kv_block=blocks_per_kv_block,
|
||||
total_cp_world_size=1,
|
||||
total_cp_rank=0,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=32768,
|
||||
block_size=block_size,
|
||||
)
|
||||
expected = kernel.CompileKey(
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
blocks_per_kv_block=blocks_per_kv_block,
|
||||
total_cp_world_size=1,
|
||||
total_cp_rank=0,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=16,
|
||||
block_size=block_size_rep,
|
||||
)
|
||||
|
||||
assert kernel.dispatch(**kwargs) == expected
|
||||
|
||||
@@ -112,6 +112,8 @@ def test_mrv1_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
kv_caches=[],
|
||||
attn_groups=[],
|
||||
speculative_config=None,
|
||||
)
|
||||
result = gpu_model_runner.GPUModelRunner.initialize_kv_cache_tensors(
|
||||
runner,
|
||||
|
||||
@@ -659,7 +659,6 @@ def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[cfg.block_size],
|
||||
)
|
||||
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
|
||||
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
|
||||
assert (backing[9] == -1).all()
|
||||
|
||||
|
||||
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
|
||||
"""Block copies index scheduler blocks, so the runner keeps the unmapped
|
||||
views while a packed layer is bound with kernel blocks."""
|
||||
|
||||
class _Layer:
|
||||
def bind_kv_cache(self, kv_cache):
|
||||
self.kv_cache = kv_cache
|
||||
|
||||
num_blocks, row = 4, 64
|
||||
storage = torch.zeros(num_blocks * row)
|
||||
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
|
||||
# split into two 2-row kernel blocks.
|
||||
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
|
||||
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
|
||||
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
|
||||
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
|
||||
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
|
||||
ctx = {name: _Layer() for name in kv_caches}
|
||||
runner_kv_caches: list[torch.Tensor] = []
|
||||
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
ctx,
|
||||
runner_kv_caches,
|
||||
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
|
||||
)
|
||||
|
||||
assert ctx["layers.1.indexer"].kv_cache is mapped
|
||||
assert runner_kv_caches == [mla, indexer]
|
||||
copies = [KVCacheBlockCopy(1, 3)]
|
||||
with pytest.raises(AssertionError, match="not divisible"):
|
||||
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
|
||||
storage[row : 2 * row] = 7
|
||||
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
|
||||
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
|
||||
@pytest.mark.parametrize("has_copies", [False, True])
|
||||
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
|
||||
|
||||
Reference in New Issue
Block a user