[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
|
||||
if block_size not in backend_cls.get_supported_kernel_block_sizes():
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
|
||||
@@ -29,7 +29,7 @@ HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
|
||||
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
meta.req_id_per_token,
|
||||
meta.block_table,
|
||||
impl.topk_indices_buffer,
|
||||
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
|
||||
)
|
||||
offset = 0
|
||||
for t in range(rows):
|
||||
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
|
||||
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
QUANTIZED_DS_MLA_CACHE_FORMATS,
|
||||
FlashMLASparseBackend,
|
||||
_fp8_kv_pages,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
allocate_kv_cache,
|
||||
prepare_kernel_block_sizes,
|
||||
select_common_block_size,
|
||||
)
|
||||
|
||||
SM90 = DeviceCapability(major=9, minor=0)
|
||||
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
|
||||
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
|
||||
|
||||
|
||||
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
cache_dtype_str="bfloat16",
|
||||
block_stride_alignment=1024,
|
||||
)
|
||||
num_blocks = 2
|
||||
layers = ["layer.0", "layer.1"]
|
||||
page_size = spec.page_size_bytes
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=num_blocks * len(layers) * page_size,
|
||||
layers=layers,
|
||||
layer_stride=page_size,
|
||||
block_stride=len(layers) * page_size,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
|
||||
)
|
||||
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
|
||||
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
|
||||
caches = allocate_kv_cache(
|
||||
config,
|
||||
torch.device("cpu"),
|
||||
KVCacheLayout.BLHNC,
|
||||
kernel_block_sizes,
|
||||
)
|
||||
|
||||
assert kernel_block_sizes == [1152]
|
||||
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm100", [False, True])
|
||||
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
|
||||
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
|
||||
(TMA) and keeps 64-token blocks."""
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family, device_id=0: sm100 and family == 100,
|
||||
)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
|
||||
state_content_bytes=656,
|
||||
)
|
||||
|
||||
kernel_block_size = 64 if sm100 else 1152
|
||||
assert (
|
||||
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
|
||||
== kernel_block_size
|
||||
)
|
||||
|
||||
|
||||
def test_fp8_kv_pages_address_packed_blocks():
|
||||
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
|
||||
64-token blocks are passed through."""
|
||||
row, block_size, stride_rows = 656, 128, 192
|
||||
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
|
||||
cache = backing.view(3, stride_rows, row)[:, :block_size]
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
|
||||
assert block_stride_rows == stride_rows
|
||||
assert pages.shape[1:] == (64, row)
|
||||
slot = 2 * block_stride_rows + 100
|
||||
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
|
||||
assert block_stride_rows is None
|
||||
assert torch.equal(pages, cache[:, :64])
|
||||
|
||||
|
||||
def test_quantized_ds_mla_formats_are_the_envelope_set():
|
||||
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import indexer
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rocm,sm120,expected",
|
||||
[
|
||||
(False, False, [KPOOL * 32, KPOOL * 64]),
|
||||
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
|
||||
(False, True, [KPOOL * 64]),
|
||||
# ROCm's AITER paged-MQA logits take the same pool pages.
|
||||
(True, False, [KPOOL * 32, KPOOL * 64]),
|
||||
],
|
||||
)
|
||||
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm120 and family == 120,
|
||||
)
|
||||
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
|
||||
config = SimpleNamespace(model_config=model_config)
|
||||
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
|
||||
|
||||
spec = SimpleNamespace(tokens_per_state=KPOOL)
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
|
||||
# Platform block-size selection queries without a spec.
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
|
||||
@@ -122,7 +122,6 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = 16
|
||||
builder.kernel_block_size = None
|
||||
builder.num_sms = num_sms
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_states=64)
|
||||
@@ -170,30 +169,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
|
||||
)
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
monkeypatch, block_size, kernel_block_size, compress_ratio
|
||||
):
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
|
||||
"""Without a prior build(), the refresh must not keep the last batch's state."""
|
||||
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
|
||||
device = torch.device("cuda")
|
||||
num_reqs, width = 3, 8
|
||||
factor = block_size // kernel_block_size
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = compress_ratio
|
||||
builder.kernel_block_size = kernel_block_size
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
|
||||
def stale(*shape: int) -> torch.Tensor:
|
||||
return torch.full(shape, -7, dtype=torch.int32, device=device)
|
||||
|
||||
decode = DeepSeekV32IndexerDecodeMetadata(
|
||||
block_table=stale(num_reqs, width // factor),
|
||||
block_table=stale(num_reqs, width),
|
||||
seq_lens=stale(num_reqs, 1),
|
||||
decode_lens=stale(num_reqs),
|
||||
requires_padding=False,
|
||||
@@ -219,8 +212,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected_block_table = request_block_table[:, ::factor] // factor
|
||||
torch.testing.assert_close(decode.block_table, expected_block_table)
|
||||
torch.testing.assert_close(decode.block_table, request_block_table)
|
||||
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
|
||||
assert torch.all(decode.per_req_decode_lens == 1)
|
||||
assert torch.all(decode.decode_lens == 1)
|
||||
@@ -471,7 +463,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
|
||||
builder.pcp_rank = 0
|
||||
builder.dcp_world_size = 1
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.kernel_block_size = None
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
|
||||
"""CPU tests for the kpool indexer's kernel block selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
|
||||
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_mod
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
KpoolTailBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
|
||||
# compressed states (hence storage_block_size = page * 4).
|
||||
# compressed states (hence a kernel block is page * 4 tokens).
|
||||
INDEX_KPOOL = 4
|
||||
|
||||
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
|
||||
@@ -59,14 +53,17 @@ def _prepare(
|
||||
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
|
||||
) -> list[int]:
|
||||
kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
|
||||
kv_cache_tensors=[],
|
||||
)
|
||||
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
|
||||
attn_groups = [
|
||||
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
|
||||
]
|
||||
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
|
||||
def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
def test_kpool_group_maps_pool_pages(
|
||||
monkeypatch: pytest.MonkeyPatch, manager_block: int
|
||||
):
|
||||
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
|
||||
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
"""
|
||||
_mock_rocm_platform(monkeypatch)
|
||||
spec = _kpool_storage_spec(manager_block)
|
||||
expected_storage = 256 if manager_block % 256 == 0 else 128
|
||||
assert spec.storage_block_size == expected_storage
|
||||
expected = 256 if manager_block % 256 == 0 else 128
|
||||
|
||||
selected = _prepare(
|
||||
spec,
|
||||
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
|
||||
)
|
||||
assert selected == [expected_storage]
|
||||
# The kpool group's AttentionGroup selects its kernel block this way.
|
||||
selected = [
|
||||
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
|
||||
]
|
||||
assert selected == [expected]
|
||||
assert selected != [manager_block]
|
||||
# The hybrid block-table split stays integral.
|
||||
assert manager_block % selected[0] == 0
|
||||
|
||||
|
||||
def test_prepare_falls_back_when_storage_block_is_not_supported():
|
||||
"""No storage block, or a storage block the backends reject (e.g. the
|
||||
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
|
||||
`select_common_block_size`'s answer, unchanged from no-fix behavior.
|
||||
"""
|
||||
def test_prepare_uses_the_backend_vote():
|
||||
"""Unpacked groups take `select_common_block_size`'s answer."""
|
||||
|
||||
class Fixed64Backend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [64]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FIXED64"
|
||||
|
||||
# Storage block the group's backend does not accept -> backend vote (64).
|
||||
spec = _kpool_storage_spec(640)
|
||||
assert spec.storage_block_size == 128
|
||||
assert _prepare(spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# No storage block at all (any non-kpool MLA model) -> backend vote.
|
||||
plain_spec = MLAAttentionSpec(
|
||||
block_size=640,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
assert plain_spec.storage_block_size is None
|
||||
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# ...and with backends that accept the manager block, the manager wins,
|
||||
# exactly as on main.
|
||||
class AcceptAllBackend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
spec = CircularBufferSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert layout is KVCacheLayout.BLHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
|
||||
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
)
|
||||
|
||||
assert spec.block_size == spec.sliding_window == ring
|
||||
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
|
||||
assert ring >= KPOOL + num_speculative_tokens
|
||||
assert 640 % ring == 0
|
||||
|
||||
|
||||
def test_tail_spec_opts_out_of_generic_slot_mapping():
|
||||
"""The tail row is one block wide (padded to the block-table alignment), so
|
||||
the generic kernel's ``pos // kpool`` column index runs off the end of the
|
||||
allocation for long prompts. The spec must opt out of it entirely."""
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
max_len = 1 << 20
|
||||
width = get_block_table_width(
|
||||
spec.max_num_blocks_per_req(None, max_len),
|
||||
spec.block_size,
|
||||
token_alignment=spec.block_table_token_alignment,
|
||||
)
|
||||
|
||||
assert width * KPOOL < max_len
|
||||
assert spec.uses_slot_mapping is False
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
request's single CircularBufferManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
tokens between the last request boundary and num_actual_tokens (mapped to
|
||||
the last request), untouched padding beyond num_actual, and PAD for
|
||||
requests on the null block."""
|
||||
ring_size = 2 * KPOOL
|
||||
positions, qsl, slot_mapping, _, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
got = circular_tail_slots(
|
||||
ref = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
|
||||
)
|
||||
got = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping.cuda(),
|
||||
bt.cuda(),
|
||||
qsl.cuda().to(torch.int32),
|
||||
positions.cuda(),
|
||||
num_actual,
|
||||
num_reqs,
|
||||
ring_size,
|
||||
)
|
||||
torch.testing.assert_close(got.cpu(), ref)
|
||||
for req, blk in enumerate(own_blocks):
|
||||
|
||||
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
|
||||
)
|
||||
layer.mm_prefix_clamp_sliding_window = True
|
||||
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
|
||||
builder.set_kernel_block_size(block_size)
|
||||
# Match production, where unwritten page tails are zero-initialized.
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_mla_supports_glm_packed_layout():
|
||||
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
|
||||
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
block_size=32,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 32, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
monkeypatch,
|
||||
block_stride_rows: int | None = None,
|
||||
):
|
||||
if qk_rope_head_dim == 0 and (
|
||||
backend_cls != FlashMLASparseBackend
|
||||
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
|
||||
if device_capability is None or device_capability.major != 10:
|
||||
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||
|
||||
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
||||
if block_size not in supported_block_sizes:
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
scale=kv_cache_scale,
|
||||
)
|
||||
if block_stride_rows is not None:
|
||||
pad = (0, 0, 0, block_stride_rows - block_size)
|
||||
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
|
||||
slot = int(common_attn_metadata.slot_mapping[-1])
|
||||
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
|
||||
|
||||
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||
# static_forward_context; register a mock layer carrying one.
|
||||
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
|
||||
out_buffer,
|
||||
)
|
||||
|
||||
if block_stride_rows is not None and kv_cache_dtype == "auto":
|
||||
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
|
||||
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
|
||||
elif block_stride_rows is not None:
|
||||
assert query_row.any()
|
||||
|
||||
assert backend_output.shape == sdpa_reference.shape
|
||||
assert backend_output.dtype == sdpa_reference.dtype
|
||||
assert torch.isfinite(backend_output).all()
|
||||
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
|
||||
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
|
||||
[
|
||||
pytest.param(
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
"auto",
|
||||
256,
|
||||
320,
|
||||
(128, 128, 64, 128),
|
||||
id="FlashInferTRTLLM",
|
||||
),
|
||||
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
|
||||
pytest.param(
|
||||
FlashMLASparseBackend,
|
||||
"fp8_ds_mla",
|
||||
128,
|
||||
192,
|
||||
(64, 256, 0, 256),
|
||||
id="FlashMLA-fp8_ds_mla",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_packed_stride(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
backend_cls,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
block_stride_rows,
|
||||
head_dims,
|
||||
):
|
||||
test_sparse_backend_decode_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
backend_cls,
|
||||
"mixed_small",
|
||||
kv_cache_dtype,
|
||||
1,
|
||||
block_size,
|
||||
workspace_init,
|
||||
1.0,
|
||||
1.0,
|
||||
*head_dims,
|
||||
monkeypatch,
|
||||
block_stride_rows=block_stride_rows,
|
||||
)
|
||||
|
||||
|
||||
def _triton_convert_reference_impl(
|
||||
req_ids: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
@@ -25,7 +26,7 @@ from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -48,10 +49,10 @@ pytestmark = pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"backend,cache_dtype,sm100,stride_alignment",
|
||||
[
|
||||
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
|
||||
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
|
||||
(FlashInferMLASparseTRTLLMBackend, "auto", True, 32 * 1152),
|
||||
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 32 * 576),
|
||||
(FlashMLASparseBackend, "auto", True, 1152),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", False, MultipleOf(64)),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
|
||||
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
|
||||
],
|
||||
@@ -79,7 +80,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
|
||||
spec = MLAAttention.get_kv_cache_spec(layer, config)
|
||||
spec = customize_attention_spec(
|
||||
backend, MLAAttention.get_kv_cache_spec(layer, config)
|
||||
)
|
||||
indexer = SimpleNamespace(
|
||||
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
|
||||
)
|
||||
@@ -102,10 +105,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
MLAAttention.bind_kv_cache(layer, views["mla"])
|
||||
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
|
||||
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
|
||||
if stride_alignment is not None:
|
||||
assert (
|
||||
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
|
||||
)
|
||||
alignment = spec.get_block_stride_alignment()
|
||||
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
|
||||
if not layer._uses_flat_kv_cache():
|
||||
register.assert_not_called()
|
||||
else:
|
||||
@@ -113,7 +114,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
register.assert_called_once_with(config, block_stride_rows=stride)
|
||||
layer.kv_cache[1, 0].fill_(3)
|
||||
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
|
||||
if stride_alignment is None:
|
||||
if alignment == 1:
|
||||
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
|
||||
item.page_size_bytes for item in specs.values()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user