[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
and kv_cache_dtype != "fp8_ds_mla"
):
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
if block_size not in backend_cls.get_supported_kernel_block_sizes():
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -29,7 +29,7 @@ HEAD = 512
TOPK = 128 # triton convert requires width % 128 == 0
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
out = torch.full_like(token_indices, -1)
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
for t in range(token_indices.shape[0]):
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
if blk < 0:
continue
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
counts[t] = len(vals)
return out, counts
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
meta.state = state
q_nope = torch.randn(rows, impl.num_heads, HEAD)
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
cache = torch.zeros(
8 * BLOCK_SIZE,
impl.head_size,
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
)
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
out, lse = impl.forward_mqa(
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
# Reserved buffers carry this step's slots; lengths are NOT refreshed
# here (the builder plans them host-side before capture/replay).
ref_slots, ref_counts = ref_convert(
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
meta.req_id_per_token,
meta.block_table,
impl.topk_indices_buffer,
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
)
offset = 0
for t in range(rows):
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
assert state.wrapper.run_args is not None
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
assert kpe.shape[-1] == qk_rope
if impl.use_fp8_kv_cache:
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.flashmla_sparse import (
QUANTIZED_DS_MLA_CACHE_FORMATS,
FlashMLASparseBackend,
_fp8_kv_pages,
)
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
KVQuantMode,
MLAAttentionSpec,
)
from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
prepare_kernel_block_sizes,
select_common_block_size,
)
SM90 = DeviceCapability(major=9, minor=0)
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
cache_dtype_str="bfloat16",
block_stride_alignment=1024,
)
num_blocks = 2
layers = ["layer.0", "layer.1"]
page_size = spec.page_size_bytes
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=num_blocks * len(layers) * page_size,
layers=layers,
layer_stride=page_size,
block_stride=len(layers) * page_size,
)
],
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
)
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
caches = allocate_kv_cache(
config,
torch.device("cpu"),
KVCacheLayout.BLHNC,
kernel_block_sizes,
)
assert kernel_block_sizes == [1152]
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
@pytest.mark.parametrize("sm100", [False, True])
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
(TMA) and keeps 64-token blocks."""
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family, device_id=0: sm100 and family == 100,
)
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=576,
dtype=torch.float8_e4m3fn,
cache_dtype_str="fp8_ds_mla",
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
state_content_bytes=656,
)
kernel_block_size = 64 if sm100 else 1152
assert (
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
== kernel_block_size
)
def test_fp8_kv_pages_address_packed_blocks():
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
64-token blocks are passed through."""
row, block_size, stride_rows = 656, 128, 192
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
cache = backing.view(3, stride_rows, row)[:, :block_size]
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
assert block_stride_rows == stride_rows
assert pages.shape[1:] == (64, row)
slot = 2 * block_stride_rows + 100
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
assert block_stride_rows is None
assert torch.equal(pages, cache[:, :64])
def test_quantized_ds_mla_formats_are_the_envelope_set():
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
from types import SimpleNamespace
import pytest
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import indexer
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
KPOOL = 4
@pytest.mark.parametrize(
"rocm,sm120,expected",
[
(False, False, [KPOOL * 32, KPOOL * 64]),
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
(False, True, [KPOOL * 64]),
# ROCm's AITER paged-MQA logits take the same pool pages.
(True, False, [KPOOL * 32, KPOOL * 64]),
],
)
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family: sm120 and family == 120,
)
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
config = SimpleNamespace(model_config=model_config)
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
spec = SimpleNamespace(tokens_per_state=KPOOL)
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
# Platform block-size selection queries without a spec.
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
@@ -122,7 +122,6 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = 16
builder.kernel_block_size = None
builder.num_sms = num_sms
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(num_states=64)
@@ -170,30 +169,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
)
def test_indexer_draft_decode_refresh_follows_new_batch(
monkeypatch, block_size, kernel_block_size, compress_ratio
):
@pytest.mark.parametrize("compress_ratio", [1, 4])
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
"""Without a prior build(), the refresh must not keep the last batch's state."""
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
device = torch.device("cuda")
num_reqs, width = 3, 8
factor = block_size // kernel_block_size
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = compress_ratio
builder.kernel_block_size = kernel_block_size
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
def stale(*shape: int) -> torch.Tensor:
return torch.full(shape, -7, dtype=torch.int32, device=device)
decode = DeepSeekV32IndexerDecodeMetadata(
block_table=stale(num_reqs, width // factor),
block_table=stale(num_reqs, width),
seq_lens=stale(num_reqs, 1),
decode_lens=stale(num_reqs),
requires_padding=False,
@@ -219,8 +212,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
builder.update_draft_decode_metadata(metadata)
expected_block_table = request_block_table[:, ::factor] // factor
torch.testing.assert_close(decode.block_table, expected_block_table)
torch.testing.assert_close(decode.block_table, request_block_table)
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
assert torch.all(decode.per_req_decode_lens == 1)
assert torch.all(decode.decode_lens == 1)
@@ -471,7 +463,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
builder.pcp_rank = 0
builder.dcp_world_size = 1
builder.cp_kv_cache_interleave_size = 1
builder.kernel_block_size = None
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
"""CPU tests for the kpool indexer's kernel block selection."""
from types import SimpleNamespace
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
from vllm.v1.attention.backends.mla import indexer as indexer_mod
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerBackend,
KpoolTailBackend,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.utils import prepare_kernel_block_sizes
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
pytestmark = pytest.mark.cpu_test
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
# compressed states (hence storage_block_size = page * 4).
# compressed states (hence a kernel block is page * 4 tokens).
INDEX_KPOOL = 4
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
@@ -59,14 +53,17 @@ def _prepare(
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
) -> list[int]:
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
kv_cache_tensors=[],
)
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
attn_groups = [
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
]
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
def test_prepare_uses_storage_block_for_the_kpool_group(
def test_kpool_group_maps_pool_pages(
monkeypatch: pytest.MonkeyPatch, manager_block: int
):
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
"""
_mock_rocm_platform(monkeypatch)
spec = _kpool_storage_spec(manager_block)
expected_storage = 256 if manager_block % 256 == 0 else 128
assert spec.storage_block_size == expected_storage
expected = 256 if manager_block % 256 == 0 else 128
selected = _prepare(
spec,
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
)
assert selected == [expected_storage]
# The kpool group's AttentionGroup selects its kernel block this way.
selected = [
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
]
assert selected == [expected]
assert selected != [manager_block]
# The hybrid block-table split stays integral.
assert manager_block % selected[0] == 0
def test_prepare_falls_back_when_storage_block_is_not_supported():
"""No storage block, or a storage block the backends reject (e.g. the
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
`select_common_block_size`'s answer, unchanged from no-fix behavior.
"""
def test_prepare_uses_the_backend_vote():
"""Unpacked groups take `select_common_block_size`'s answer."""
class Fixed64Backend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [64]
@staticmethod
def get_name() -> str:
return "FIXED64"
# Storage block the group's backend does not accept -> backend vote (64).
spec = _kpool_storage_spec(640)
assert spec.storage_block_size == 128
assert _prepare(spec, [Fixed64Backend]) == [64]
# No storage block at all (any non-kpool MLA model) -> backend vote.
plain_spec = MLAAttentionSpec(
block_size=640,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
assert plain_spec.storage_block_size is None
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
# ...and with backends that accept the manager block, the manager wins,
# exactly as on main.
class AcceptAllBackend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [MultipleOf(1)]
@staticmethod
@@ -3,7 +3,7 @@
"""CPU tests for the kpool tail slot mapping (no GPU required).
The kpool tail cache is a 1-block-per-request circular ring addressed by
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
allocated per request, never grown, so only column 0 of its block table is
ever written; the rest stays zero-initialized).
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
KpoolTailMetadataBuilder,
compute_kpool_tail_slot_mapping,
)
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.block_table import get_block_table_width
KPOOL = 4
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
spec = KpoolTailSpec(
spec = CircularBufferSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
_, _, head_stride, state_stride, content_stride = strides
assert layout is KVCacheLayout.LBHNC
assert layout is KVCacheLayout.BLHNC
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
assert state_stride == 128 * torch.bfloat16.itemsize
assert content_stride == 1
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
)
assert spec.block_size == spec.sliding_window == ring
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
assert ring >= KPOOL + num_speculative_tokens
assert 640 % ring == 0
def test_tail_spec_opts_out_of_generic_slot_mapping():
"""The tail row is one block wide (padded to the block-table alignment), so
the generic kernel's ``pos // kpool`` column index runs off the end of the
allocation for long prompts. The spec must opt out of it entirely."""
spec = KpoolTailSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
max_len = 1 << 20
width = get_block_table_width(
spec.max_num_blocks_per_req(None, max_len),
spec.block_size,
token_alignment=spec.block_table_token_alignment,
)
assert width * KPOOL < max_len
assert spec.uses_slot_mapping is False
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
request's single CircularBufferManager block, the remaining columns are never
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
tokens between the last request boundary and num_actual_tokens (mapped to
the last request), untouched padding beyond num_actual, and PAD for
requests on the null block."""
ring_size = 2 * KPOOL
positions, qsl, slot_mapping, _, num_reqs = make_batch(
per_req, padded_len=padded_len
)
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
bt = make_tail_block_table(own_blocks)
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
got = circular_tail_slots(
ref = compute_kpool_tail_slot_mapping(
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
)
got = compute_kpool_tail_slot_mapping(
slot_mapping.cuda(),
bt.cuda(),
qsl.cuda().to(torch.int32),
positions.cuda(),
num_actual,
num_reqs,
ring_size,
)
torch.testing.assert_close(got.cpu(), ref)
for req, blk in enumerate(own_blocks):
-1
View File
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
)
layer.mm_prefix_clamp_sliding_window = True
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
builder.set_kernel_block_size(block_size)
# Match production, where unwritten page tails are zero-initialized.
cache = torch.zeros(
num_blocks,
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
_use_rocm_sparse_triton,
fit_kpool_indices_to_aiter,
)
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_validate_dsv4_sparse_dims,
_validate_sparse_dims,
)
from vllm.v1.kv_cache_interface import KVCacheLayout
@triton.jit
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
)
def test_rocm_sparse_mla_supports_glm_packed_layout():
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
@pytest.mark.parametrize(
(
"kv_cache_dtype",
+65 -4
View File
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
block_size=32,
)
recorded_scale = None
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 32, kv_lora_rank),
metadata,
SimpleNamespace(),
)
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
qk_rope_head_dim: int,
v_head_dim: int,
monkeypatch,
block_stride_rows: int | None = None,
):
if qk_rope_head_dim == 0 and (
backend_cls != FlashMLASparseBackend
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
if block_size not in supported_block_sizes:
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
kv_cache_dtype=kv_cache_dtype,
scale=kv_cache_scale,
)
if block_stride_rows is not None:
pad = (0, 0, 0, block_stride_rows - block_size)
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
slot = int(common_attn_metadata.slot_mapping[-1])
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
out_buffer,
)
if block_stride_rows is not None and kv_cache_dtype == "auto":
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
elif block_stride_rows is not None:
assert query_row.any()
assert backend_output.shape == sdpa_reference.shape
assert backend_output.dtype == sdpa_reference.dtype
assert torch.isfinite(backend_output).all()
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
@pytest.mark.parametrize(
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
[
pytest.param(
FlashInferMLASparseTRTLLMBackend,
"auto",
256,
320,
(128, 128, 64, 128),
id="FlashInferTRTLLM",
),
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
pytest.param(
FlashMLASparseBackend,
"fp8_ds_mla",
128,
192,
(64, 256, 0, 256),
id="FlashMLA-fp8_ds_mla",
),
],
)
def test_sparse_mla_packed_stride(
default_vllm_config,
dist_init,
workspace_init,
monkeypatch,
backend_cls,
kv_cache_dtype,
block_size,
block_stride_rows,
head_dims,
):
test_sparse_backend_decode_correctness(
default_vllm_config,
dist_init,
backend_cls,
"mixed_small",
kv_cache_dtype,
1,
block_size,
workspace_init,
1.0,
1.0,
*head_dims,
monkeypatch,
block_stride_rows=block_stride_rows,
)
def _triton_convert_reference_impl(
req_ids: torch.Tensor,
block_table: torch.Tensor,
@@ -12,6 +12,7 @@ from vllm.config import CacheConfig
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
@@ -25,7 +26,7 @@ from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.utils import allocate_kv_cache
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
pytestmark = pytest.mark.skip_global_cleanup
@@ -48,10 +49,10 @@ pytestmark = pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"backend,cache_dtype,sm100,stride_alignment",
[
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
(FlashInferMLASparseTRTLLMBackend, "auto", True, 32 * 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 32 * 576),
(FlashMLASparseBackend, "auto", True, 1152),
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
(FlashMLASparseBackend, "fp8_ds_mla", False, MultipleOf(64)),
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
],
@@ -79,7 +80,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
non_causal_multi_token_decode=False,
)
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
spec = MLAAttention.get_kv_cache_spec(layer, config)
spec = customize_attention_spec(
backend, MLAAttention.get_kv_cache_spec(layer, config)
)
indexer = SimpleNamespace(
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
)
@@ -102,10 +105,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
MLAAttention.bind_kv_cache(layer, views["mla"])
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
if stride_alignment is not None:
assert (
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
)
alignment = spec.get_block_stride_alignment()
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
if not layer._uses_flat_kv_cache():
register.assert_not_called()
else:
@@ -113,7 +114,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
register.assert_called_once_with(config, block_stride_rows=stride)
layer.kv_cache[1, 0].fill_(3)
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
if stride_alignment is None:
if alignment == 1:
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
item.page_size_bytes for item in specs.values()
)