[Bugfix] Sparse MLA: enable fp8_ds_mla dense prefill (#48642)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-07-16 22:44:04 +00:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f17be06fbe
commit 67f9046e4a
13 changed files with 516 additions and 85 deletions
+63 -13
View File
@@ -27,8 +27,8 @@ def _build_test_case(seq_lens, block_size, seed=42):
seed: Random seed for reproducibility.
Returns:
Tuple of (cache, block_table, seq_lens_t, workspace_starts_t,
num_reqs, total_tokens, expected_output).
Tuple of (cache, block_table, workspace_starts_t, num_reqs,
total_tokens, expected_output).
"""
torch.manual_seed(seed)
@@ -112,7 +112,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
# Expected output: exact copy
expected[out_idx, NOPE_DIM:] = rope
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
workspace_starts_t = torch.tensor(
workspace_starts, dtype=torch.int32, device="cuda"
)
@@ -120,7 +119,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
return (
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -197,7 +195,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
flat_cache[:sl] = token_data[ws : ws + sl]
block_start += nb
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
workspace_starts_t = torch.tensor(
workspace_starts, dtype=torch.int32, device="cuda"
)
@@ -205,7 +202,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
return (
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -232,7 +228,6 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
(
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -244,7 +239,7 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
cache, dst, block_table, workspace_starts_t, num_reqs
)
# NoPE: fp8 dequant has rounding error, so we allow small tolerance.
@@ -277,8 +272,6 @@ def test_cp_gather_fp8_shuffled_blocks():
)
block_table = torch.tensor([[3, 1]], dtype=torch.int32, device="cuda")
workspace_starts = torch.tensor([0], dtype=torch.int32, device="cuda")
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
expected = torch.zeros(
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
)
@@ -314,7 +307,7 @@ def test_cp_gather_fp8_shuffled_blocks():
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts, len(seq_lens)
cache, dst, block_table, workspace_starts, len(seq_lens)
)
torch.testing.assert_close(
@@ -323,6 +316,64 @@ def test_cp_gather_fp8_shuffled_blocks():
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"gather_seq_lens,seq_starts",
[
([6, 5, 3], [3, 4, 5]),
([0, 5, 3], [12, 4, 5]),
],
)
def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
"""Gather request slices beginning at arbitrary cache positions."""
full_seq_lens = [12, 11, 9]
(
cache,
block_table,
_workspace_starts_t,
num_reqs,
_total_tokens,
full_expected,
) = _build_test_case(full_seq_lens, block_size=4)
workspace_starts = torch.tensor(
[0, gather_seq_lens[0], sum(gather_seq_lens[:2])],
dtype=torch.int32,
device="cuda",
)
seq_starts_t = torch.tensor(seq_starts, dtype=torch.int32, device="cuda")
dst = torch.empty(
sum(gather_seq_lens),
NOPE_DIM + ROPE_DIM,
dtype=torch.bfloat16,
device="cuda",
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache,
dst,
block_table,
workspace_starts,
num_reqs,
seq_starts_t,
)
full_workspace_starts = [0, full_seq_lens[0], sum(full_seq_lens[:2])]
expected = torch.cat(
[
full_expected[
full_workspace_starts[i] + seq_starts[i] : full_workspace_starts[i]
+ seq_starts[i]
+ gather_seq_lens[i]
]
for i in range(num_reqs)
]
)
torch.testing.assert_close(
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
)
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"seq_lens,block_size",
[
@@ -342,7 +393,6 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
(
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -354,7 +404,7 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
cache, dst, block_table, workspace_starts_t, num_reqs
)
torch.testing.assert_close(
+29 -1
View File
@@ -42,7 +42,10 @@ from vllm.v1.attention.backends.mla.prefill import (
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.kv_cache_interface import (
KVQuantMode,
MLAAttentionSpec,
)
BACKENDS_TO_TEST = [
AttentionBackendEnum.CUTLASS_MLA,
@@ -55,6 +58,31 @@ BACKENDS_TO_TEST = [
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("cache_dtype", "expected_quant_mode"),
[
("auto", KVQuantMode.NONE),
("fp8_ds_mla", KVQuantMode.FP8_PER_TENSOR),
],
)
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
cache_dtype: str, expected_quant_mode: KVQuantMode
):
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64), model_config=None
)
spec = MLAAttention.get_kv_cache_spec(layer, vllm_config)
assert isinstance(spec, MLAAttentionSpec)
assert spec.cache_dtype_str == cache_dtype
assert spec.kv_quant_mode == expected_quant_mode
if cache_dtype == "fp8_ds_mla":
assert spec.page_size_bytes == 64 * 656
# Remove sm100 backends from the list if not using sm100
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
+321 -1
View File
@@ -40,10 +40,16 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
FlashMLASparseBackend,
FlashMLASparseImpl,
FlashMLASparseMetadata,
FlashMLASparseMetadataBuilder,
triton_convert_req_index_to_global_index,
)
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
from vllm.v1.attention.backends.utils import split_prefill_chunks
from vllm.v1.attention.backends.utils import (
split_decodes_and_prefills,
split_prefill_chunks,
)
from vllm.v1.attention.ops import flashmla
SPARSE_BACKEND_BATCH_SPECS = {
@@ -721,6 +727,120 @@ def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_s
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_rejects_req_id_longer_than_token_indices():
"""Guard against the #47327 regression: the kernel grid is sized by
req_id but the output is allocated like token_indices, so a full-batch
req_id combined with an MQA-subset token_indices wrote past the end of
the output buffer. The wrapper must reject the length mismatch instead
of corrupting memory."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)
# Full batch: 2 decode tokens + 10 prefill tokens
req_id_full = torch.tensor(
[0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
)
num_mqa_tokens = 2
token_indices = torch.randint(
0,
block_size * 10,
(num_mqa_tokens, num_topk_tokens),
dtype=torch.int32,
device=device,
)
with pytest.raises(AssertionError, match="must cover the same tokens"):
triton_convert_req_index_to_global_index(
req_id_full,
block_table,
token_indices,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk_tokens,
)
# The sliced call is the intended usage and must match the reference.
result = triton_convert_req_index_to_global_index(
req_id_full[:num_mqa_tokens],
block_table,
token_indices,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk_tokens,
)
reference = _triton_convert_reference_impl(
req_id_full[:num_mqa_tokens],
block_table,
token_indices,
block_size,
num_topk_tokens,
)
torch.testing.assert_close(result, reference, rtol=0, atol=0)
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
"""Guard against the #47327 regression: when the dense-MHA prefill split
is active, forward_mqa only receives the leading decode tokens, but
_forward_bf16_kv passed the full-batch req_id_per_token to the index
conversion, making it write past the end of its output buffer. The call
site must slice req_id_per_token to the MQA tokens."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
num_batch_tokens = 12
num_mqa_tokens = 2
attn_metadata = SimpleNamespace(
req_id_per_token=torch.tensor(
[0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
),
block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
block_size=block_size,
)
assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens
q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
kv_cache = torch.zeros(40 * block_size, 576, dtype=torch.bfloat16, device=device)
topk_indices = torch.randint(
0,
block_size * 10,
(num_mqa_tokens, num_topk_tokens),
dtype=torch.int32,
device=device,
)
captured = {}
def _stub_kernel(q, kv, indices, lengths):
captured["indices"] = indices
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
out = FlashMLASparseImpl._forward_bf16_kv(
stub_impl, q, kv_cache, topk_indices, attn_metadata
)
assert out.shape[0] == num_mqa_tokens
assert captured["indices"].shape[0] == num_mqa_tokens
reference = _triton_convert_reference_impl(
attn_metadata.req_id_per_token[:num_mqa_tokens],
attn_metadata.block_table,
topk_indices,
block_size,
num_topk_tokens,
)
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
@pytest.mark.parametrize(
"seq_lens,max_buf,expected",
[
@@ -1124,3 +1244,203 @@ def test_triton_convert_returns_valid_counts():
)
assert isinstance(result_only, torch.Tensor)
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
def test_flashmla_cache_dtype_aliases_use_ds_layout():
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
# stores kv_cache_dtype, so they cannot bypass the gate.
for alias in ("fp8", "fp8_e4m3"):
assert (
_canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
== "fp8_ds_mla"
)
def test_flashmla_fp8_metadata_reuses_common_batch_split():
builder = SimpleNamespace(
device=torch.device(DEVICE_TYPE),
vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=8)),
)
common_metadata = SimpleNamespace(
num_actual_tokens=1,
seq_lens_cpu_upper_bound=torch.tensor([1]),
query_start_loc_cpu=torch.tensor([0, 1]),
block_table_tensor=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
)
metadata = FlashMLASparseMetadata(
num_reqs=1,
max_query_len=1,
max_seq_len=1,
num_actual_tokens=1,
query_start_loc=torch.tensor([0, 1], device=DEVICE_TYPE),
slot_mapping=torch.tensor([0], device=DEVICE_TYPE),
block_table=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
req_id_per_token=torch.zeros(1, dtype=torch.int32, device=DEVICE_TYPE),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
)
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
builder, common_metadata, metadata
)
assert fp8_metadata.num_decodes == 0
assert fp8_metadata.num_prefills == 1
assert fp8_metadata.num_decode_tokens == 0
assert fp8_metadata.num_prefill_tokens == 1
def test_flashmla_common_metadata_requires_uniform_decodes():
common_metadata = SimpleNamespace(
max_query_len=3,
num_reqs=3,
num_actual_tokens=6,
query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
is_prefilling=None,
)
split = split_decodes_and_prefills(
common_metadata,
decode_threshold=128,
require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
)
assert split == (1, 2, 1, 5)
def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
lambda: (object(), None),
)
builder = SimpleNamespace(
device=torch.device(DEVICE_TYPE),
dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
)
query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
common_metadata = SimpleNamespace(
num_actual_tokens=660,
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=torch.arange(7, device=DEVICE_TYPE),
)
metadata = FlashMLASparseMetadata(
num_reqs=7,
max_query_len=110,
max_seq_len=110,
num_actual_tokens=660,
query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
slot_mapping=torch.arange(660, device=DEVICE_TYPE),
block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
num_decodes=7,
num_prefills=0,
num_decode_tokens=660,
)
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
builder, common_metadata, metadata
)
assert fp8_metadata.num_decodes == 6
assert fp8_metadata.num_decode_tokens == 660
assert fp8_metadata.decode is not None
assert fp8_metadata.decode.decode_query_len == 110
torch.testing.assert_close(
fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
)
@pytest.mark.parametrize("use_mixed_batch", [False, True])
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
num_decode_tokens = 2
num_batch_tokens = 5
q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
kernel_q_shapes = []
def convert_indices(*args, **kwargs): # noqa: ARG001
assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
if not kwargs.get("return_valid_counts", False):
return topk_indices
valid_counts = torch.full(
(num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
)
return topk_indices, valid_counts
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse."
"triton_convert_req_index_to_global_index",
convert_indices,
)
def run_kernel(**kwargs):
kernel_q_shapes.append(kwargs["q"].shape)
return kwargs["q"][..., :1], None
if use_mixed_batch:
fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
scheduler_metadata=object(), # type: ignore[arg-type]
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
)
else:
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
fp8_metadata = FP8Meta(
num_decodes=1,
num_prefills=1,
num_decode_tokens=num_decode_tokens,
num_prefill_tokens=num_batch_tokens - num_decode_tokens,
decode=FP8Meta.Decode(
seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
kernel_metadata=object(), # type: ignore[arg-type]
decode_query_len=num_decode_tokens,
),
prefill=FP8Meta.Prefill(
request_ids=torch.empty(
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
),
workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
chunks=[],
),
)
metadata = SimpleNamespace(
fp8_extra_metadata=fp8_metadata,
fp8_use_mixed_batch=use_mixed_batch,
num_actual_tokens=num_batch_tokens,
req_id_per_token=torch.empty(
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
),
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
block_size=64,
)
impl = SimpleNamespace(
kv_cache_dtype="fp8_ds_mla",
topk_indices_buffer=topk_indices,
num_heads=2,
kv_lora_rank=1,
_fp8_flash_mla_kernel=run_kernel,
)
impl._forward_fp8_kv_mixed_batch = MethodType(
FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
)
impl._forward_fp8_kv_separate_prefill_decode = MethodType(
FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
)
output, lse = FlashMLASparseImpl.forward_mqa(
impl,
q,
torch.empty(0, device=DEVICE_TYPE),
metadata,
None,
)
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
assert output.shape == (num_decode_tokens, 2, 1)
assert lse is None