[Bugfix][KV Cache][MLA] Align packed block strides for V3.2 sparse MLA (#55528)

Signed-off-by: lz <145014769+200lz@users.noreply.github.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
lz
2026-09-25 05:33:36 +00:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Robert Shaw mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3a58aeb1a1
commit 29468dde8b
6 changed files with 187 additions and 27 deletions
+3 -1
View File
@@ -9,7 +9,7 @@ Known Issues:
"""
import sys
from types import SimpleNamespace
from types import MethodType, SimpleNamespace
import pytest
import torch
@@ -218,12 +218,14 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
cache_dtype: str, expected_quant_mode: KVQuantMode
):
layer = SimpleNamespace(
attn_backend=flashmla_module.FlashMLABackend,
kv_cache_dtype=cache_dtype,
head_size=576,
indexer=None,
non_causal_multi_token_decode=False,
sliding_window=None,
)
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64), model_config=None
)
@@ -0,0 +1,119 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse MLA allocation and warmup must agree on physical addressing."""
from types import MethodType, SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config import CacheConfig
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
from vllm.v1.attention.backends.mla.sparse_utils import (
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
flat_kv_row_view,
)
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.utils import allocate_kv_cache
pytestmark = pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
),
],
)
@pytest.mark.parametrize(
"layout", [KVCacheLayout.BLHNC, KVCacheLayout.BLNHC, KVCacheLayout.LBNHC]
)
@pytest.mark.parametrize(
"backend,cache_dtype,sm100,stride_alignment",
[
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
(FlashMLASparseBackend, "auto", True, 1152),
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
],
)
def test_allocation_and_warmup_follow_addressing_mode(
monkeypatch, layout, backend, cache_dtype, sm100, stride_alignment, device
):
monkeypatch.setattr(
current_platform, "is_device_capability_family", lambda family: sm100
)
config = SimpleNamespace(
cache_config=CacheConfig(block_size=64),
model_config=SimpleNamespace(dtype=torch.bfloat16),
attention_config=SimpleNamespace(hisparse_config=None),
kernel_config=SimpleNamespace(enable_jit_warmup=True),
)
config.cache_config.kv_cache_layout = layout.name
layer = SimpleNamespace(
_vllm_config=config,
attn_backend=backend,
kv_cache_dtype=cache_dtype,
head_size=576,
sliding_window=None,
indexer=None,
non_causal_multi_token_decode=False,
)
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
spec = MLAAttention.get_kv_cache_spec(layer, config)
indexer = SimpleNamespace(
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
)
index_spec = DeepseekV32IndexerCache.get_kv_cache_spec(indexer, config)
assert spec.block_stride_alignment == stride_alignment
assert index_spec.block_stride_alignment is None
assert layout in get_supported_kv_cache_layouts(
[backend, DeepseekV32IndexerBackend]
)
specs = {"mla": spec, "indexer": index_spec}
group = KVCacheGroupSpec(
list(specs), UniformTypeKVCacheSpecs(block_size=64, kv_cache_specs=specs)
)
cache = get_kv_cache_config_from_groups(config, [group], 2**20)
views = allocate_kv_cache(cache, torch.device(device), layout)
register = MagicMock()
monkeypatch.setattr(
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL, "register_warmup", register
)
MLAAttention.bind_kv_cache(layer, views["mla"])
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
if stride_alignment is not None:
assert (
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
)
if not layer._uses_flat_kv_cache():
register.assert_not_called()
else:
rows, stride = flat_kv_row_view(layer.kv_cache, 64)
register.assert_called_once_with(config, block_stride_rows=stride)
layer.kv_cache[1, 0].fill_(3)
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
if stride_alignment is None:
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
item.page_size_bytes for item in specs.values()
)
@@ -10,6 +10,7 @@ block); the allocation is the same either way.
"""
from dataclasses import replace
from math import lcm
from unittest.mock import MagicMock
import pytest
@@ -26,9 +27,11 @@ from vllm.v1.core.kv_cache_utils import (
get_kv_cache_groups,
resolve_kv_cache_block_sizes,
)
from vllm.v1.hisparse.layout import _build_hisparse_kv_cache_tensors
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
HiSparseHotSpec,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheSpec,
@@ -79,6 +82,30 @@ def _mock_vllm_config(layout: str | None):
return config
def test_packed_alignment_preserves_hot_pages_and_generic_stride():
"""One shared stride must satisfy both constraints, regardless of rounding order."""
spec = replace(_mla(128), block_stride_alignment=1152)
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
alignment = lcm(1152, hot.page_size_bytes)
stride = _pool_bytes_per_block(groups)
assert stride == alignment
layout = KVCacheLayout.BLHNC
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
assert {tensor.block_stride for tensor in tensors} == {alignment}
assert {tensor.size for tensor in tensors} == {3 * alignment}
@pytest.mark.parametrize("spec", [_full(), _mla(128)])
def test_block_alignment_survives_merge_and_rejects_incompatible_specs(spec):
aligned = replace(spec, block_stride_alignment=512)
assert type(spec).merge([aligned, aligned]).block_stride_alignment == 512
with pytest.raises(AssertionError):
type(spec).merge([aligned, spec])
with pytest.raises(ValueError, match="must be positive"):
replace(spec, block_stride_alignment=0)
def _pages(groups) -> dict[str, int]:
return {
name: group.kv_cache_spec.kv_cache_specs[name].page_size_bytes