[Bugfix] Sparse MLA: enable fp8_ds_mla dense prefill (#48642)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-07-16 22:44:04 +00:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f17be06fbe
commit 67f9046e4a
13 changed files with 516 additions and 85 deletions
+3 -4
View File
@@ -69,12 +69,11 @@ def make_inputs(total_tokens, num_reqs, block_size):
# Output workspace
dst = torch.zeros(total_tokens, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
workspace_starts_t = torch.tensor(
workspace_starts, dtype=torch.int32, device="cuda"
)
return cache, dst, block_table, seq_lens_t, workspace_starts_t
return cache, dst, block_table, workspace_starts_t
def bench_scenario(label, num_reqs, total_tokens_list, save_path):
@@ -94,7 +93,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path):
)
)
def bench_fn(total_tokens, provider, num_reqs):
cache, dst, block_table, seq_lens_t, ws_starts = make_inputs(
cache, dst, block_table, ws_starts = make_inputs(
total_tokens, num_reqs, BLOCK_SIZE
)
@@ -102,7 +101,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path):
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
lambda: ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, ws_starts, num_reqs
cache, dst, block_table, ws_starts, num_reqs
),
quantiles=quantiles,
rep=500,
+1 -2
View File
@@ -67,9 +67,8 @@ void cp_gather_and_upconvert_fp8_kv_cache(
torch::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656]
torch::Tensor const& dst, // [TOT_TOKENS, 576]
torch::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
torch::Tensor const& seq_lens, // [BATCH]
torch::Tensor const& workspace_starts, // [BATCH]
int64_t batch_size);
int64_t batch_size, std::optional<torch::Tensor> seq_starts = std::nullopt);
// Indexer K quantization and cache function
void indexer_k_quant_and_cache(
+19 -9
View File
@@ -1174,7 +1174,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache(
const int32_t num_reqs, const int32_t block_size,
const int32_t total_tokens, const int64_t block_table_stride,
const int64_t cache_block_stride, const int64_t cache_entry_stride,
const int64_t dst_entry_stride) {
const int64_t dst_entry_stride,
const int32_t* __restrict__ seq_starts) { // Optional source offsets
const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5;
if (flat_warp_id >= total_tokens) return;
const int lane_id = threadIdx.x & 31;
@@ -1192,7 +1193,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache(
// Compute physical token address via block table
const int out_token_id = flat_warp_id;
const int token_offset = out_token_id - workspace_starts[req_id];
int token_offset = out_token_id - workspace_starts[req_id];
if (seq_starts != nullptr) token_offset += seq_starts[req_id];
const int cache_block_idx = token_offset / block_size;
const int offset_in_block = token_offset % block_size;
const int physical_block =
@@ -1383,9 +1385,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656]
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
torch::stable::Tensor const& seq_lens, // [BATCH]
torch::stable::Tensor const& workspace_starts, // [BATCH]
int64_t batch_size) {
int64_t batch_size,
std::optional<torch::stable::Tensor> seq_starts = std::nullopt) {
torch::stable::accelerator::DeviceGuard device_guard(
src_cache.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
@@ -1396,20 +1398,25 @@ void cp_gather_and_upconvert_fp8_kv_cache(
STD_TORCH_CHECK(
block_table.scalar_type() == torch::headeronly::ScalarType::Int,
"block_table must be int32");
STD_TORCH_CHECK(seq_lens.scalar_type() == torch::headeronly::ScalarType::Int,
"seq_lens must be int32");
STD_TORCH_CHECK(
workspace_starts.scalar_type() == torch::headeronly::ScalarType::Int,
"workspace_starts must be int32");
if (seq_starts.has_value()) {
STD_TORCH_CHECK(
seq_starts.value().scalar_type() == torch::headeronly::ScalarType::Int,
"seq_starts must be int32");
}
STD_TORCH_CHECK(src_cache.device() == dst.device(),
"src_cache and dst must be on the same device");
STD_TORCH_CHECK(src_cache.device() == block_table.device(),
"src_cache and block_table must be on the same device");
STD_TORCH_CHECK(src_cache.device() == seq_lens.device(),
"src_cache and seq_lens must be on the same device");
STD_TORCH_CHECK(src_cache.device() == workspace_starts.device(),
"src_cache and workspace_starts must be on the same device");
if (seq_starts.has_value()) {
STD_TORCH_CHECK(src_cache.device() == seq_starts.value().device(),
"src_cache and seq_starts must be on the same device");
}
auto dtype = src_cache.scalar_type();
STD_TORCH_CHECK(
dtype == torch::headeronly::ScalarType::Byte || // uint8
@@ -1438,6 +1445,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
constexpr int warps_per_block = 8;
const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block;
const int block_size_threads = warps_per_block * 32; // 256 threads
const int32_t* seq_starts_ptr =
seq_starts.has_value() ? seq_starts.value().const_data_ptr<int32_t>()
: nullptr;
vllm::cp_gather_and_upconvert_fp8_kv_cache<<<grid_size, block_size_threads, 0,
stream>>>(
@@ -1446,7 +1456,7 @@ void cp_gather_and_upconvert_fp8_kv_cache(
workspace_starts.const_data_ptr<int32_t>(),
static_cast<int32_t>(batch_size), block_size, total_tokens,
block_table_stride, cache_block_stride, cache_entry_stride,
dst_entry_stride);
dst_entry_stride, seq_starts_ptr);
}
// Macro to dispatch the kernel based on the data type.
+2 -2
View File
@@ -527,9 +527,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
// 656]
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
torch::stable::Tensor const& seq_lens, // [BATCH]
torch::stable::Tensor const& workspace_starts, // [BATCH]
int64_t batch_size);
int64_t batch_size,
std::optional<torch::stable::Tensor> seq_starts = std::nullopt);
// Indexer K quantization and cache function
void indexer_k_quant_and_cache(
+2 -2
View File
@@ -847,8 +847,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
ops.def(
"cp_gather_and_upconvert_fp8_kv_cache(Tensor src_cache, Tensor! dst, "
"Tensor block_table, Tensor seq_lens, Tensor workspace_starts, int "
"batch_size) -> ()");
"Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? "
"seq_starts) -> ()");
ops.def(
"indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor "
+63 -13
View File
@@ -27,8 +27,8 @@ def _build_test_case(seq_lens, block_size, seed=42):
seed: Random seed for reproducibility.
Returns:
Tuple of (cache, block_table, seq_lens_t, workspace_starts_t,
num_reqs, total_tokens, expected_output).
Tuple of (cache, block_table, workspace_starts_t, num_reqs,
total_tokens, expected_output).
"""
torch.manual_seed(seed)
@@ -112,7 +112,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
# Expected output: exact copy
expected[out_idx, NOPE_DIM:] = rope
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
workspace_starts_t = torch.tensor(
workspace_starts, dtype=torch.int32, device="cuda"
)
@@ -120,7 +119,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
return (
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -197,7 +195,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
flat_cache[:sl] = token_data[ws : ws + sl]
block_start += nb
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
workspace_starts_t = torch.tensor(
workspace_starts, dtype=torch.int32, device="cuda"
)
@@ -205,7 +202,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
return (
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -232,7 +228,6 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
(
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -244,7 +239,7 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
cache, dst, block_table, workspace_starts_t, num_reqs
)
# NoPE: fp8 dequant has rounding error, so we allow small tolerance.
@@ -277,8 +272,6 @@ def test_cp_gather_fp8_shuffled_blocks():
)
block_table = torch.tensor([[3, 1]], dtype=torch.int32, device="cuda")
workspace_starts = torch.tensor([0], dtype=torch.int32, device="cuda")
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
expected = torch.zeros(
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
)
@@ -314,7 +307,7 @@ def test_cp_gather_fp8_shuffled_blocks():
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts, len(seq_lens)
cache, dst, block_table, workspace_starts, len(seq_lens)
)
torch.testing.assert_close(
@@ -323,6 +316,64 @@ def test_cp_gather_fp8_shuffled_blocks():
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"gather_seq_lens,seq_starts",
[
([6, 5, 3], [3, 4, 5]),
([0, 5, 3], [12, 4, 5]),
],
)
def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
"""Gather request slices beginning at arbitrary cache positions."""
full_seq_lens = [12, 11, 9]
(
cache,
block_table,
_workspace_starts_t,
num_reqs,
_total_tokens,
full_expected,
) = _build_test_case(full_seq_lens, block_size=4)
workspace_starts = torch.tensor(
[0, gather_seq_lens[0], sum(gather_seq_lens[:2])],
dtype=torch.int32,
device="cuda",
)
seq_starts_t = torch.tensor(seq_starts, dtype=torch.int32, device="cuda")
dst = torch.empty(
sum(gather_seq_lens),
NOPE_DIM + ROPE_DIM,
dtype=torch.bfloat16,
device="cuda",
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache,
dst,
block_table,
workspace_starts,
num_reqs,
seq_starts_t,
)
full_workspace_starts = [0, full_seq_lens[0], sum(full_seq_lens[:2])]
expected = torch.cat(
[
full_expected[
full_workspace_starts[i] + seq_starts[i] : full_workspace_starts[i]
+ seq_starts[i]
+ gather_seq_lens[i]
]
for i in range(num_reqs)
]
)
torch.testing.assert_close(
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
)
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"seq_lens,block_size",
[
@@ -342,7 +393,6 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
(
cache,
block_table,
seq_lens_t,
workspace_starts_t,
num_reqs,
total_tokens,
@@ -354,7 +404,7 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
cache, dst, block_table, workspace_starts_t, num_reqs
)
torch.testing.assert_close(
+29 -1
View File
@@ -42,7 +42,10 @@ from vllm.v1.attention.backends.mla.prefill import (
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.kv_cache_interface import (
KVQuantMode,
MLAAttentionSpec,
)
BACKENDS_TO_TEST = [
AttentionBackendEnum.CUTLASS_MLA,
@@ -55,6 +58,31 @@ BACKENDS_TO_TEST = [
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("cache_dtype", "expected_quant_mode"),
[
("auto", KVQuantMode.NONE),
("fp8_ds_mla", KVQuantMode.FP8_PER_TENSOR),
],
)
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
cache_dtype: str, expected_quant_mode: KVQuantMode
):
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=64), model_config=None
)
spec = MLAAttention.get_kv_cache_spec(layer, vllm_config)
assert isinstance(spec, MLAAttentionSpec)
assert spec.cache_dtype_str == cache_dtype
assert spec.kv_quant_mode == expected_quant_mode
if cache_dtype == "fp8_ds_mla":
assert spec.page_size_bytes == 64 * 656
# Remove sm100 backends from the list if not using sm100
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
+321 -1
View File
@@ -40,10 +40,16 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
FlashMLASparseBackend,
FlashMLASparseImpl,
FlashMLASparseMetadata,
FlashMLASparseMetadataBuilder,
triton_convert_req_index_to_global_index,
)
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
from vllm.v1.attention.backends.utils import split_prefill_chunks
from vllm.v1.attention.backends.utils import (
split_decodes_and_prefills,
split_prefill_chunks,
)
from vllm.v1.attention.ops import flashmla
SPARSE_BACKEND_BATCH_SPECS = {
@@ -721,6 +727,120 @@ def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_s
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_triton_convert_rejects_req_id_longer_than_token_indices():
"""Guard against the #47327 regression: the kernel grid is sized by
req_id but the output is allocated like token_indices, so a full-batch
req_id combined with an MQA-subset token_indices wrote past the end of
the output buffer. The wrapper must reject the length mismatch instead
of corrupting memory."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)
# Full batch: 2 decode tokens + 10 prefill tokens
req_id_full = torch.tensor(
[0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
)
num_mqa_tokens = 2
token_indices = torch.randint(
0,
block_size * 10,
(num_mqa_tokens, num_topk_tokens),
dtype=torch.int32,
device=device,
)
with pytest.raises(AssertionError, match="must cover the same tokens"):
triton_convert_req_index_to_global_index(
req_id_full,
block_table,
token_indices,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk_tokens,
)
# The sliced call is the intended usage and must match the reference.
result = triton_convert_req_index_to_global_index(
req_id_full[:num_mqa_tokens],
block_table,
token_indices,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk_tokens,
)
reference = _triton_convert_reference_impl(
req_id_full[:num_mqa_tokens],
block_table,
token_indices,
block_size,
num_topk_tokens,
)
torch.testing.assert_close(result, reference, rtol=0, atol=0)
@pytest.mark.skipif(
torch.cuda.get_device_capability() < (9, 0),
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
)
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
"""Guard against the #47327 regression: when the dense-MHA prefill split
is active, forward_mqa only receives the leading decode tokens, but
_forward_bf16_kv passed the full-batch req_id_per_token to the index
conversion, making it write past the end of its output buffer. The call
site must slice req_id_per_token to the MQA tokens."""
device = torch.device(DEVICE_TYPE)
num_topk_tokens = 128
block_size = 64
num_batch_tokens = 12
num_mqa_tokens = 2
attn_metadata = SimpleNamespace(
req_id_per_token=torch.tensor(
[0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
),
block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
block_size=block_size,
)
assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens
q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
kv_cache = torch.zeros(40 * block_size, 576, dtype=torch.bfloat16, device=device)
topk_indices = torch.randint(
0,
block_size * 10,
(num_mqa_tokens, num_topk_tokens),
dtype=torch.int32,
device=device,
)
captured = {}
def _stub_kernel(q, kv, indices, lengths):
captured["indices"] = indices
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
out = FlashMLASparseImpl._forward_bf16_kv(
stub_impl, q, kv_cache, topk_indices, attn_metadata
)
assert out.shape[0] == num_mqa_tokens
assert captured["indices"].shape[0] == num_mqa_tokens
reference = _triton_convert_reference_impl(
attn_metadata.req_id_per_token[:num_mqa_tokens],
attn_metadata.block_table,
topk_indices,
block_size,
num_topk_tokens,
)
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
@pytest.mark.parametrize(
"seq_lens,max_buf,expected",
[
@@ -1124,3 +1244,203 @@ def test_triton_convert_returns_valid_counts():
)
assert isinstance(result_only, torch.Tensor)
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
def test_flashmla_cache_dtype_aliases_use_ds_layout():
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
# stores kv_cache_dtype, so they cannot bypass the gate.
for alias in ("fp8", "fp8_e4m3"):
assert (
_canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
== "fp8_ds_mla"
)
def test_flashmla_fp8_metadata_reuses_common_batch_split():
builder = SimpleNamespace(
device=torch.device(DEVICE_TYPE),
vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=8)),
)
common_metadata = SimpleNamespace(
num_actual_tokens=1,
seq_lens_cpu_upper_bound=torch.tensor([1]),
query_start_loc_cpu=torch.tensor([0, 1]),
block_table_tensor=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
)
metadata = FlashMLASparseMetadata(
num_reqs=1,
max_query_len=1,
max_seq_len=1,
num_actual_tokens=1,
query_start_loc=torch.tensor([0, 1], device=DEVICE_TYPE),
slot_mapping=torch.tensor([0], device=DEVICE_TYPE),
block_table=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
req_id_per_token=torch.zeros(1, dtype=torch.int32, device=DEVICE_TYPE),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
)
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
builder, common_metadata, metadata
)
assert fp8_metadata.num_decodes == 0
assert fp8_metadata.num_prefills == 1
assert fp8_metadata.num_decode_tokens == 0
assert fp8_metadata.num_prefill_tokens == 1
def test_flashmla_common_metadata_requires_uniform_decodes():
common_metadata = SimpleNamespace(
max_query_len=3,
num_reqs=3,
num_actual_tokens=6,
query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
is_prefilling=None,
)
split = split_decodes_and_prefills(
common_metadata,
decode_threshold=128,
require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
)
assert split == (1, 2, 1, 5)
def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
lambda: (object(), None),
)
builder = SimpleNamespace(
device=torch.device(DEVICE_TYPE),
dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
)
query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
common_metadata = SimpleNamespace(
num_actual_tokens=660,
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=torch.arange(7, device=DEVICE_TYPE),
)
metadata = FlashMLASparseMetadata(
num_reqs=7,
max_query_len=110,
max_seq_len=110,
num_actual_tokens=660,
query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
slot_mapping=torch.arange(660, device=DEVICE_TYPE),
block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
num_decodes=7,
num_prefills=0,
num_decode_tokens=660,
)
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
builder, common_metadata, metadata
)
assert fp8_metadata.num_decodes == 6
assert fp8_metadata.num_decode_tokens == 660
assert fp8_metadata.decode is not None
assert fp8_metadata.decode.decode_query_len == 110
torch.testing.assert_close(
fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
)
@pytest.mark.parametrize("use_mixed_batch", [False, True])
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
num_decode_tokens = 2
num_batch_tokens = 5
q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
kernel_q_shapes = []
def convert_indices(*args, **kwargs): # noqa: ARG001
assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
if not kwargs.get("return_valid_counts", False):
return topk_indices
valid_counts = torch.full(
(num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
)
return topk_indices, valid_counts
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.flashmla_sparse."
"triton_convert_req_index_to_global_index",
convert_indices,
)
def run_kernel(**kwargs):
kernel_q_shapes.append(kwargs["q"].shape)
return kwargs["q"][..., :1], None
if use_mixed_batch:
fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
scheduler_metadata=object(), # type: ignore[arg-type]
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
)
else:
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
fp8_metadata = FP8Meta(
num_decodes=1,
num_prefills=1,
num_decode_tokens=num_decode_tokens,
num_prefill_tokens=num_batch_tokens - num_decode_tokens,
decode=FP8Meta.Decode(
seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
kernel_metadata=object(), # type: ignore[arg-type]
decode_query_len=num_decode_tokens,
),
prefill=FP8Meta.Prefill(
request_ids=torch.empty(
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
),
workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
chunks=[],
),
)
metadata = SimpleNamespace(
fp8_extra_metadata=fp8_metadata,
fp8_use_mixed_batch=use_mixed_batch,
num_actual_tokens=num_batch_tokens,
req_id_per_token=torch.empty(
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
),
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
block_size=64,
)
impl = SimpleNamespace(
kv_cache_dtype="fp8_ds_mla",
topk_indices_buffer=topk_indices,
num_heads=2,
kv_lora_rank=1,
_fp8_flash_mla_kernel=run_kernel,
)
impl._forward_fp8_kv_mixed_batch = MethodType(
FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
)
impl._forward_fp8_kv_separate_prefill_decode = MethodType(
FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
)
output, lse = FlashMLASparseImpl.forward_mqa(
impl,
q,
torch.empty(0, device=DEVICE_TYPE),
metadata,
None,
)
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
assert output.shape == (num_decode_tokens, 2, 1)
assert lse is None
+3 -3
View File
@@ -2759,9 +2759,9 @@ def cp_gather_and_upconvert_fp8_kv_cache(
src_cache: torch.Tensor,
dst: torch.Tensor,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
workspace_starts: torch.Tensor,
batch_size: int,
seq_starts: torch.Tensor | None = None,
) -> None:
"""Gather and upconvert FP8 KV cache to BF16 workspace.
@@ -2769,12 +2769,12 @@ def cp_gather_and_upconvert_fp8_kv_cache(
src_cache: FP8 KV cache [num_blocks, block_size, 656]
dst: BF16 output workspace [total_tokens, 576]
block_table: Block indices [num_reqs, max_blocks]
seq_lens: Sequence lengths [num_reqs]
workspace_starts: Workspace start offsets [num_reqs]
batch_size: Number of requests
seq_starts: Optional source sequence offsets [num_reqs]
"""
torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache, dst, block_table, seq_lens, workspace_starts, batch_size
src_cache, dst, block_table, workspace_starts, batch_size, seq_starts
)
@@ -1749,6 +1749,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.determine_chunked_prefill_workspace_size(vllm_config)
)
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
if self.dcp_world_size > 1:
# Note(hc): The local kvcache is incomplete when DCP is triggered,
# an additional kvcache allgather across the DCP group is therefore
@@ -1761,7 +1762,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
+ self.chunked_prefill_workspace_size // self.dcp_world_size,
self.model_config.get_head_size(),
),
dtype=self.model_config.dtype,
dtype=torch.bfloat16
if use_packed_fp8_cache
else self.model_config.dtype,
device=device,
)
else:
@@ -1770,7 +1773,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
self.chunked_prefill_workspace_size,
self.model_config.get_head_size(),
),
dtype=self.q_data_type,
dtype=torch.bfloat16 if use_packed_fp8_cache else self.q_data_type,
device=device,
)
@@ -2115,7 +2118,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
for i in range(iters):
toks = prefill_metadata.chunked_context.seq_tot[i]
if not use_fp8_prefill:
if self.kv_cache_dtype == "fp8_ds_mla":
ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=prefill_metadata.block_table,
workspace_starts=prefill_metadata.chunked_context.cu_seq_lens[i],
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
elif not use_fp8_prefill:
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace,
@@ -2230,9 +2242,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
padded_local_cu_seq_lens = (
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
)
if is_quantized_kv_cache(self.kv_cache_dtype) and (
self.kv_cache_dtype != "fp8_ds_mla"
):
if self.kv_cache_dtype == "fp8_ds_mla":
ops.cp_gather_and_upconvert_fp8_kv_cache(
src_cache=kv_c_and_k_pe_cache,
dst=workspace[:toks],
block_table=prefill_metadata.block_table,
workspace_starts=padded_local_cu_seq_lens,
batch_size=attn_metadata.num_prefills,
seq_starts=prefill_metadata.chunked_context.starts[i],
)
elif is_quantized_kv_cache(self.kv_cache_dtype):
assert k_scale is not None
ops.gather_and_maybe_dequant_cache(
src_cache=kv_c_and_k_pe_cache,
@@ -3,7 +3,7 @@
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
from shutil import which
from typing import TYPE_CHECKING, Generic, TypeVar
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
import numpy as np
import torch
@@ -38,6 +38,7 @@ T = TypeVar("T", bound=AttentionMetadata)
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
metadata_cls: type[T]
require_uniform_decodes: ClassVar[bool] = False
def __init__(
self,
@@ -173,6 +174,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold or 1,
require_uniform=self.require_uniform_decodes,
)
(
prefill_query_start_loc,
@@ -32,7 +32,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
from vllm.v1.attention.backends.utils import (
reshape_attn_output_for_spec_decode,
reshape_query_for_spec_decode,
split_decodes_and_prefills,
split_prefill_chunks,
)
from vllm.v1.attention.ops.flashmla import (
@@ -57,8 +56,8 @@ logger = init_logger(__name__)
# the FP8 decode kernel for decode.
# Currently we use #1 when the number of heads per rank is low (i.e. TP) since the BF16
# prefill kernel requires padding the number of heads to 128 while the decode does not
# so when the per ranke head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
# batch mode (#2).
# so when the per-rank head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
# batch mode (#1).
MIN_HEADS_FOR_BF16_PREFILL = 32
"""
@@ -182,10 +181,6 @@ class FlashMLASparseMetadata(AttentionMetadata):
@dataclass
class Prefill:
# Sequence lengths (context + query) for prefill requests
# Shape: [num_prefill_reqs]
seq_lens: torch.Tensor
# Request ID for each token: -1 for decode tokens, request index
# (0, 1, 2, ...) for prefill tokens.
# Shape: [num_actual_tokens]
@@ -204,7 +199,6 @@ class FlashMLASparseMetadata(AttentionMetadata):
Prefill requests may be chunked to fit within the fixed workspace size.
"""
seq_lens: torch.Tensor
tokens_slice: slice
block_table: torch.Tensor
req_start_idx: int
@@ -239,6 +233,7 @@ class FlashMLASparseMetadataBuilder(
SparseMLACommonMetadataBuilder[FlashMLASparseMetadata]
):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
require_uniform_decodes: ClassVar[bool] = True
metadata_cls = FlashMLASparseMetadata
def __init__(
@@ -322,10 +317,11 @@ class FlashMLASparseMetadataBuilder(
self,
common_attn_metadata: CommonAttentionMetadata,
) -> "FlashMLASparseMetadata.FP8KernelMetadata":
"""Build FP8 metadata treating all tokens as one mixed batch.
"""Build FP8 metadata treating MQA tokens as one batch.
This matches main branch's approach and avoids the BF16 prefill kernel
which has head padding overhead when num_heads is small (high TP case).
The scheduler initializes lazily from the runtime query shape, which may
be the full batch or only decodes when prefills use dense MHA. This avoids
the BF16 prefill kernel's head-padding overhead at high TP.
"""
num_tokens = common_attn_metadata.num_actual_tokens
@@ -353,20 +349,29 @@ class FlashMLASparseMetadataBuilder(
def _build_fp8_separate_prefill_decode(
self,
common_attn_metadata: CommonAttentionMetadata,
metadata: FlashMLASparseMetadata,
) -> "FlashMLASparseMetadata.FP8SeparatePrefillDecode":
num_tokens = common_attn_metadata.num_actual_tokens
(num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold or 1,
require_uniform=True,
)
metadata.num_decodes,
metadata.num_prefills,
metadata.num_decode_tokens,
num_tokens - metadata.num_decode_tokens,
)
decode_query_len = 0
active_num_decodes = num_decodes
if num_decodes > 0:
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item()
assert decode_query_len > 0
active_num_decodes = num_decode_tokens // decode_query_len
assert active_num_decodes * decode_query_len == num_decode_tokens
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
fp8_metadata = FP8Meta(
num_decodes=num_decodes,
num_decodes=active_num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decode_tokens,
num_prefill_tokens=num_prefill_tokens,
@@ -374,7 +379,6 @@ class FlashMLASparseMetadataBuilder(
# Extract prefill sequence lengths (context + query, not just query)
# Decode requests come first in the batch, prefill requests follow
prefill_seq_lens = None
prefill_request_id = None
prefill_workspace_starts = None
prefill_chunks = None
@@ -386,11 +390,9 @@ class FlashMLASparseMetadataBuilder(
# slice below), so no D2H sync is needed.
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
seq_lens = common_attn_metadata.seq_lens
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
prefill_seq_lens = seq_lens[num_decodes:]
# Build prefill_request_id: -1 for decode, request index for
# prefill. This enables a single
@@ -438,7 +440,6 @@ class FlashMLASparseMetadataBuilder(
offset = prefill_workspace_starts_cpu[chunk_start].item()
prefill_workspace_starts_cpu[chunk_start:chunk_end] -= offset
chunk_seq_lens = prefill_seq_lens[chunk_start:chunk_end]
chunk_tot_seqlen = prefill_seq_lens_cpu[chunk_start:chunk_end].sum()
token_start = query_start_loc_cpu[num_decodes + chunk_start].item()
token_end = query_start_loc_cpu[num_decodes + chunk_end].item()
@@ -452,7 +453,6 @@ class FlashMLASparseMetadataBuilder(
prefill_chunks.append(
FP8Meta.Prefill.Chunk(
seq_lens=chunk_seq_lens,
tokens_slice=tokens_slice,
block_table=chunk_block_table,
req_start_idx=chunk_start,
@@ -466,27 +466,22 @@ class FlashMLASparseMetadataBuilder(
)
fp8_metadata.prefill = FP8Meta.Prefill(
seq_lens=prefill_seq_lens,
request_ids=prefill_request_id,
workspace_starts=prefill_workspace_starts,
chunks=prefill_chunks,
)
if num_decodes > 0:
# Compute decode_query_len for spec decode (uniform due to require_uniform)
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item()
# Use padded head count since that's what the kernel will see
scheduler_metadata, _ = get_mla_metadata()
kernel_meta = FlashMLASparseMetadata.FP8KernelMetadata(
scheduler_metadata=scheduler_metadata,
dummy_block_table=self.dummy_block_table[:num_decodes],
cache_lens=self.max_model_len_tensor[:num_decodes],
dummy_block_table=self.dummy_block_table[:active_num_decodes],
cache_lens=self.max_model_len_tensor[:active_num_decodes],
)
fp8_metadata.decode = FP8Meta.Decode(
seq_lens=common_attn_metadata.seq_lens[:num_decodes],
seq_lens=common_attn_metadata.seq_lens[:active_num_decodes],
kernel_metadata=kernel_meta,
decode_query_len=decode_query_len,
)
@@ -510,7 +505,7 @@ class FlashMLASparseMetadataBuilder(
)
else:
metadata.fp8_extra_metadata = self._build_fp8_separate_prefill_decode(
common_attn_metadata
common_attn_metadata, metadata
)
return metadata
@@ -597,9 +592,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
attn_metadata: FlashMLASparseMetadata,
) -> torch.Tensor:
# Convert per-request indices to global slots (decode) or workspace
# offsets (prefill).
# offsets (prefill). req_id_per_token covers the whole batch; slice it
# to the MQA tokens (q may exclude prefill tokens routed to dense MHA).
topk_indices, topk_length = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token,
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
@@ -624,11 +620,18 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
fp8_metadata = attn_metadata.fp8_extra_metadata
assert isinstance(fp8_metadata, FlashMLASparseMetadata.FP8SeparatePrefillDecode)
num_decodes = fp8_metadata.num_decodes
num_mqa_tokens = q.shape[0]
num_decode_tokens = fp8_metadata.num_decode_tokens
num_prefill_tokens = num_mqa_tokens - num_decode_tokens
assert num_prefill_tokens in (0, fp8_metadata.num_prefill_tokens), (
"FP8 sparse MLA expects either the decode subset or the full batch"
)
prefill_request_ids = None
prefill_workspace_starts = None
has_prefill_workspace = False
if fp8_metadata.prefill is not None:
if num_prefill_tokens > 0:
assert fp8_metadata.prefill is not None
prefill_request_ids = fp8_metadata.prefill.request_ids
prefill_workspace_starts = fp8_metadata.prefill.workspace_starts
has_prefill_workspace = True
@@ -640,7 +643,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# prefill_workspace_starts has been adjusted in-place per chunk so
# prefill indices automatically come out chunk-local
topk_indices, topk_length = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token,
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
@@ -676,9 +679,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# -> (num_decode_tokens, num_heads, head_dim_v)
return reshape_attn_output_for_spec_decode(attn_out)
num_decode_tokens = fp8_metadata.num_decode_tokens
num_prefill_tokens = fp8_metadata.num_prefill_tokens
# Pure decode: direct call without allocation
if num_decode_tokens > 0 and num_prefill_tokens == 0:
assert fp8_metadata.decode is not None
@@ -686,7 +686,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
else:
# Mixed or pure prefill: allocate output tensor
attn_out = q.new_empty(
(attn_metadata.num_actual_tokens, self.num_heads, self.kv_lora_rank),
(num_mqa_tokens, self.num_heads, self.kv_lora_rank),
dtype=q.dtype,
device=q.device,
)
@@ -704,7 +704,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
kv_c_and_k_pe_cache,
chunk_workspace,
chunk.block_table,
chunk.seq_lens,
chunk.workspace_starts,
len(chunk.block_table),
)
@@ -738,7 +737,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
# Convert per-request indices to global slots (decode) or workspace
# offsets (prefill).
topk_indices = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token,
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
@@ -154,6 +154,11 @@ def triton_convert_req_index_to_global_index(
assert req_id.dtype == torch.int32
assert block_table.dtype == torch.int32
assert token_indices.dtype == torch.int32
assert req_id.shape[0] == token_indices.shape[0], (
f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) "
"must cover the same tokens; the grid is sized by req_id but the output "
"is allocated like token_indices, so a longer req_id writes out of bounds"
)
assert token_indices.shape[1] == NUM_TOPK_TOKENS
assert NUM_TOPK_TOKENS % BLOCK_N == 0, (
f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible by BLOCK_N ({BLOCK_N})"