[Bugfix] Sparse MLA: enable fp8_ds_mla dense prefill (#48642)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f17be06fbe
commit
67f9046e4a
@@ -69,12 +69,11 @@ def make_inputs(total_tokens, num_reqs, block_size):
|
||||
# Output workspace
|
||||
dst = torch.zeros(total_tokens, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
workspace_starts_t = torch.tensor(
|
||||
workspace_starts, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
return cache, dst, block_table, seq_lens_t, workspace_starts_t
|
||||
return cache, dst, block_table, workspace_starts_t
|
||||
|
||||
|
||||
def bench_scenario(label, num_reqs, total_tokens_list, save_path):
|
||||
@@ -94,7 +93,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path):
|
||||
)
|
||||
)
|
||||
def bench_fn(total_tokens, provider, num_reqs):
|
||||
cache, dst, block_table, seq_lens_t, ws_starts = make_inputs(
|
||||
cache, dst, block_table, ws_starts = make_inputs(
|
||||
total_tokens, num_reqs, BLOCK_SIZE
|
||||
)
|
||||
|
||||
@@ -102,7 +101,7 @@ def bench_scenario(label, num_reqs, total_tokens_list, save_path):
|
||||
|
||||
ms, min_ms, max_ms = triton.testing.do_bench_cudagraph(
|
||||
lambda: ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, ws_starts, num_reqs
|
||||
cache, dst, block_table, ws_starts, num_reqs
|
||||
),
|
||||
quantiles=quantiles,
|
||||
rep=500,
|
||||
|
||||
+1
-2
@@ -67,9 +67,8 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
torch::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656]
|
||||
torch::Tensor const& dst, // [TOT_TOKENS, 576]
|
||||
torch::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
|
||||
torch::Tensor const& seq_lens, // [BATCH]
|
||||
torch::Tensor const& workspace_starts, // [BATCH]
|
||||
int64_t batch_size);
|
||||
int64_t batch_size, std::optional<torch::Tensor> seq_starts = std::nullopt);
|
||||
|
||||
// Indexer K quantization and cache function
|
||||
void indexer_k_quant_and_cache(
|
||||
|
||||
@@ -1174,7 +1174,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
const int32_t num_reqs, const int32_t block_size,
|
||||
const int32_t total_tokens, const int64_t block_table_stride,
|
||||
const int64_t cache_block_stride, const int64_t cache_entry_stride,
|
||||
const int64_t dst_entry_stride) {
|
||||
const int64_t dst_entry_stride,
|
||||
const int32_t* __restrict__ seq_starts) { // Optional source offsets
|
||||
const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) >> 5;
|
||||
if (flat_warp_id >= total_tokens) return;
|
||||
const int lane_id = threadIdx.x & 31;
|
||||
@@ -1192,7 +1193,8 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
|
||||
// Compute physical token address via block table
|
||||
const int out_token_id = flat_warp_id;
|
||||
const int token_offset = out_token_id - workspace_starts[req_id];
|
||||
int token_offset = out_token_id - workspace_starts[req_id];
|
||||
if (seq_starts != nullptr) token_offset += seq_starts[req_id];
|
||||
const int cache_block_idx = token_offset / block_size;
|
||||
const int offset_in_block = token_offset % block_size;
|
||||
const int physical_block =
|
||||
@@ -1383,9 +1385,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
torch::stable::Tensor const& src_cache, // [NUM_BLOCKS, BLOCK_SIZE, 656]
|
||||
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
|
||||
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
|
||||
torch::stable::Tensor const& seq_lens, // [BATCH]
|
||||
torch::stable::Tensor const& workspace_starts, // [BATCH]
|
||||
int64_t batch_size) {
|
||||
int64_t batch_size,
|
||||
std::optional<torch::stable::Tensor> seq_starts = std::nullopt) {
|
||||
torch::stable::accelerator::DeviceGuard device_guard(
|
||||
src_cache.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
@@ -1396,20 +1398,25 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
STD_TORCH_CHECK(
|
||||
block_table.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"block_table must be int32");
|
||||
STD_TORCH_CHECK(seq_lens.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"seq_lens must be int32");
|
||||
STD_TORCH_CHECK(
|
||||
workspace_starts.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"workspace_starts must be int32");
|
||||
if (seq_starts.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
seq_starts.value().scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"seq_starts must be int32");
|
||||
}
|
||||
|
||||
STD_TORCH_CHECK(src_cache.device() == dst.device(),
|
||||
"src_cache and dst must be on the same device");
|
||||
STD_TORCH_CHECK(src_cache.device() == block_table.device(),
|
||||
"src_cache and block_table must be on the same device");
|
||||
STD_TORCH_CHECK(src_cache.device() == seq_lens.device(),
|
||||
"src_cache and seq_lens must be on the same device");
|
||||
STD_TORCH_CHECK(src_cache.device() == workspace_starts.device(),
|
||||
"src_cache and workspace_starts must be on the same device");
|
||||
if (seq_starts.has_value()) {
|
||||
STD_TORCH_CHECK(src_cache.device() == seq_starts.value().device(),
|
||||
"src_cache and seq_starts must be on the same device");
|
||||
}
|
||||
auto dtype = src_cache.scalar_type();
|
||||
STD_TORCH_CHECK(
|
||||
dtype == torch::headeronly::ScalarType::Byte || // uint8
|
||||
@@ -1438,6 +1445,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
constexpr int warps_per_block = 8;
|
||||
const int grid_size = (total_tokens + warps_per_block - 1) / warps_per_block;
|
||||
const int block_size_threads = warps_per_block * 32; // 256 threads
|
||||
const int32_t* seq_starts_ptr =
|
||||
seq_starts.has_value() ? seq_starts.value().const_data_ptr<int32_t>()
|
||||
: nullptr;
|
||||
|
||||
vllm::cp_gather_and_upconvert_fp8_kv_cache<<<grid_size, block_size_threads, 0,
|
||||
stream>>>(
|
||||
@@ -1446,7 +1456,7 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
workspace_starts.const_data_ptr<int32_t>(),
|
||||
static_cast<int32_t>(batch_size), block_size, total_tokens,
|
||||
block_table_stride, cache_block_stride, cache_entry_stride,
|
||||
dst_entry_stride);
|
||||
dst_entry_stride, seq_starts_ptr);
|
||||
}
|
||||
|
||||
// Macro to dispatch the kernel based on the data type.
|
||||
|
||||
@@ -527,9 +527,9 @@ void cp_gather_and_upconvert_fp8_kv_cache(
|
||||
// 656]
|
||||
torch::stable::Tensor const& dst, // [TOT_TOKENS, 576]
|
||||
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
|
||||
torch::stable::Tensor const& seq_lens, // [BATCH]
|
||||
torch::stable::Tensor const& workspace_starts, // [BATCH]
|
||||
int64_t batch_size);
|
||||
int64_t batch_size,
|
||||
std::optional<torch::stable::Tensor> seq_starts = std::nullopt);
|
||||
|
||||
// Indexer K quantization and cache function
|
||||
void indexer_k_quant_and_cache(
|
||||
|
||||
@@ -847,8 +847,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
|
||||
ops.def(
|
||||
"cp_gather_and_upconvert_fp8_kv_cache(Tensor src_cache, Tensor! dst, "
|
||||
"Tensor block_table, Tensor seq_lens, Tensor workspace_starts, int "
|
||||
"batch_size) -> ()");
|
||||
"Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? "
|
||||
"seq_starts) -> ()");
|
||||
|
||||
ops.def(
|
||||
"indexer_k_quant_and_cache(Tensor k, Tensor! kv_cache, Tensor "
|
||||
|
||||
@@ -27,8 +27,8 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
seed: Random seed for reproducibility.
|
||||
|
||||
Returns:
|
||||
Tuple of (cache, block_table, seq_lens_t, workspace_starts_t,
|
||||
num_reqs, total_tokens, expected_output).
|
||||
Tuple of (cache, block_table, workspace_starts_t, num_reqs,
|
||||
total_tokens, expected_output).
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
@@ -112,7 +112,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
# Expected output: exact copy
|
||||
expected[out_idx, NOPE_DIM:] = rope
|
||||
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
workspace_starts_t = torch.tensor(
|
||||
workspace_starts, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
@@ -120,7 +119,6 @@ def _build_test_case(seq_lens, block_size, seed=42):
|
||||
return (
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -197,7 +195,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
|
||||
flat_cache[:sl] = token_data[ws : ws + sl]
|
||||
block_start += nb
|
||||
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
workspace_starts_t = torch.tensor(
|
||||
workspace_starts, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
@@ -205,7 +202,6 @@ def _build_test_case_fast(seq_lens, block_size, seed=42):
|
||||
return (
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -232,7 +228,6 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -244,7 +239,7 @@ def test_cp_gather_and_upconvert_fp8_kv_cache(seq_lens, block_size):
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
|
||||
cache, dst, block_table, workspace_starts_t, num_reqs
|
||||
)
|
||||
|
||||
# NoPE: fp8 dequant has rounding error, so we allow small tolerance.
|
||||
@@ -277,8 +272,6 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
)
|
||||
block_table = torch.tensor([[3, 1]], dtype=torch.int32, device="cuda")
|
||||
workspace_starts = torch.tensor([0], dtype=torch.int32, device="cuda")
|
||||
seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device="cuda")
|
||||
|
||||
expected = torch.zeros(
|
||||
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
@@ -314,7 +307,7 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts, len(seq_lens)
|
||||
cache, dst, block_table, workspace_starts, len(seq_lens)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
@@ -323,6 +316,64 @@ def test_cp_gather_fp8_shuffled_blocks():
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gather_seq_lens,seq_starts",
|
||||
[
|
||||
([6, 5, 3], [3, 4, 5]),
|
||||
([0, 5, 3], [12, 4, 5]),
|
||||
],
|
||||
)
|
||||
def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
|
||||
"""Gather request slices beginning at arbitrary cache positions."""
|
||||
full_seq_lens = [12, 11, 9]
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
_workspace_starts_t,
|
||||
num_reqs,
|
||||
_total_tokens,
|
||||
full_expected,
|
||||
) = _build_test_case(full_seq_lens, block_size=4)
|
||||
|
||||
workspace_starts = torch.tensor(
|
||||
[0, gather_seq_lens[0], sum(gather_seq_lens[:2])],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
seq_starts_t = torch.tensor(seq_starts, dtype=torch.int32, device="cuda")
|
||||
dst = torch.empty(
|
||||
sum(gather_seq_lens),
|
||||
NOPE_DIM + ROPE_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache,
|
||||
dst,
|
||||
block_table,
|
||||
workspace_starts,
|
||||
num_reqs,
|
||||
seq_starts_t,
|
||||
)
|
||||
|
||||
full_workspace_starts = [0, full_seq_lens[0], sum(full_seq_lens[:2])]
|
||||
expected = torch.cat(
|
||||
[
|
||||
full_expected[
|
||||
full_workspace_starts[i] + seq_starts[i] : full_workspace_starts[i]
|
||||
+ seq_starts[i]
|
||||
+ gather_seq_lens[i]
|
||||
]
|
||||
for i in range(num_reqs)
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
|
||||
)
|
||||
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,block_size",
|
||||
[
|
||||
@@ -342,7 +393,6 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
seq_lens_t,
|
||||
workspace_starts_t,
|
||||
num_reqs,
|
||||
total_tokens,
|
||||
@@ -354,7 +404,7 @@ def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
|
||||
)
|
||||
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
cache, dst, block_table, seq_lens_t, workspace_starts_t, num_reqs
|
||||
cache, dst, block_table, workspace_starts_t, num_reqs
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
|
||||
@@ -42,7 +42,10 @@ from vllm.v1.attention.backends.mla.prefill import (
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
BACKENDS_TO_TEST = [
|
||||
AttentionBackendEnum.CUTLASS_MLA,
|
||||
@@ -55,6 +58,31 @@ BACKENDS_TO_TEST = [
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_dtype", "expected_quant_mode"),
|
||||
[
|
||||
("auto", KVQuantMode.NONE),
|
||||
("fp8_ds_mla", KVQuantMode.FP8_PER_TENSOR),
|
||||
],
|
||||
)
|
||||
def test_mla_kv_cache_spec_uses_layer_cache_dtype(
|
||||
cache_dtype: str, expected_quant_mode: KVQuantMode
|
||||
):
|
||||
layer = SimpleNamespace(kv_cache_dtype=cache_dtype, head_size=576)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=64), model_config=None
|
||||
)
|
||||
|
||||
spec = MLAAttention.get_kv_cache_spec(layer, vllm_config)
|
||||
|
||||
assert isinstance(spec, MLAAttentionSpec)
|
||||
assert spec.cache_dtype_str == cache_dtype
|
||||
assert spec.kv_quant_mode == expected_quant_mode
|
||||
if cache_dtype == "fp8_ds_mla":
|
||||
assert spec.page_size_bytes == 64 * 656
|
||||
|
||||
|
||||
# Remove sm100 backends from the list if not using sm100
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_properties(0).major < 10:
|
||||
BACKENDS_TO_TEST.remove(AttentionBackendEnum.CUTLASS_MLA)
|
||||
|
||||
@@ -40,10 +40,16 @@ from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseBackend,
|
||||
FlashMLASparseImpl,
|
||||
FlashMLASparseMetadata,
|
||||
FlashMLASparseMetadataBuilder,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
|
||||
from vllm.v1.attention.backends.utils import split_prefill_chunks
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
split_decodes_and_prefills,
|
||||
split_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops import flashmla
|
||||
|
||||
SPARSE_BACKEND_BATCH_SPECS = {
|
||||
@@ -721,6 +727,120 @@ def test_triton_convert_req_index_to_global_index_with_prefill_workspace(block_s
|
||||
torch.testing.assert_close(result, reference_result, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
def test_triton_convert_rejects_req_id_longer_than_token_indices():
|
||||
"""Guard against the #47327 regression: the kernel grid is sized by
|
||||
req_id but the output is allocated like token_indices, so a full-batch
|
||||
req_id combined with an MQA-subset token_indices wrote past the end of
|
||||
the output buffer. The wrapper must reject the length mismatch instead
|
||||
of corrupting memory."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
block_table = torch.arange(40, dtype=torch.int32, device=device).view(4, 10)
|
||||
|
||||
# Full batch: 2 decode tokens + 10 prefill tokens
|
||||
req_id_full = torch.tensor(
|
||||
[0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], dtype=torch.int32, device=device
|
||||
)
|
||||
num_mqa_tokens = 2
|
||||
token_indices = torch.randint(
|
||||
0,
|
||||
block_size * 10,
|
||||
(num_mqa_tokens, num_topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
triton_convert_req_index_to_global_index(
|
||||
req_id_full,
|
||||
block_table,
|
||||
token_indices,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk_tokens,
|
||||
)
|
||||
|
||||
# The sliced call is the intended usage and must match the reference.
|
||||
result = triton_convert_req_index_to_global_index(
|
||||
req_id_full[:num_mqa_tokens],
|
||||
block_table,
|
||||
token_indices,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk_tokens,
|
||||
)
|
||||
reference = _triton_convert_reference_impl(
|
||||
req_id_full[:num_mqa_tokens],
|
||||
block_table,
|
||||
token_indices,
|
||||
block_size,
|
||||
num_topk_tokens,
|
||||
)
|
||||
torch.testing.assert_close(result, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.get_device_capability() < (9, 0),
|
||||
reason="FlashMLASparseBackend requires CUDA 9.0 or higher",
|
||||
)
|
||||
def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
||||
"""Guard against the #47327 regression: when the dense-MHA prefill split
|
||||
is active, forward_mqa only receives the leading decode tokens, but
|
||||
_forward_bf16_kv passed the full-batch req_id_per_token to the index
|
||||
conversion, making it write past the end of its output buffer. The call
|
||||
site must slice req_id_per_token to the MQA tokens."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_topk_tokens = 128
|
||||
block_size = 64
|
||||
num_batch_tokens = 12
|
||||
num_mqa_tokens = 2
|
||||
|
||||
attn_metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor(
|
||||
[0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32, device=device
|
||||
),
|
||||
block_table=torch.arange(40, dtype=torch.int32, device=device).view(4, 10),
|
||||
block_size=block_size,
|
||||
)
|
||||
assert attn_metadata.req_id_per_token.shape[0] == num_batch_tokens
|
||||
|
||||
q = torch.zeros(num_mqa_tokens, 4, 576, dtype=torch.bfloat16, device=device)
|
||||
kv_cache = torch.zeros(40 * block_size, 576, dtype=torch.bfloat16, device=device)
|
||||
topk_indices = torch.randint(
|
||||
0,
|
||||
block_size * 10,
|
||||
(num_mqa_tokens, num_topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
captured = {}
|
||||
|
||||
def _stub_kernel(q, kv, indices, lengths):
|
||||
captured["indices"] = indices
|
||||
return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device)
|
||||
|
||||
stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel)
|
||||
|
||||
out = FlashMLASparseImpl._forward_bf16_kv(
|
||||
stub_impl, q, kv_cache, topk_indices, attn_metadata
|
||||
)
|
||||
|
||||
assert out.shape[0] == num_mqa_tokens
|
||||
assert captured["indices"].shape[0] == num_mqa_tokens
|
||||
reference = _triton_convert_reference_impl(
|
||||
attn_metadata.req_id_per_token[:num_mqa_tokens],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
block_size,
|
||||
num_topk_tokens,
|
||||
)
|
||||
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,max_buf,expected",
|
||||
[
|
||||
@@ -1124,3 +1244,203 @@ def test_triton_convert_returns_valid_counts():
|
||||
)
|
||||
assert isinstance(result_only, torch.Tensor)
|
||||
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_flashmla_cache_dtype_aliases_use_ds_layout():
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
|
||||
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
|
||||
# stores kv_cache_dtype, so they cannot bypass the gate.
|
||||
for alias in ("fp8", "fp8_e4m3"):
|
||||
assert (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype(FlashMLASparseBackend, alias)
|
||||
== "fp8_ds_mla"
|
||||
)
|
||||
|
||||
|
||||
def test_flashmla_fp8_metadata_reuses_common_batch_split():
|
||||
builder = SimpleNamespace(
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=8)),
|
||||
)
|
||||
common_metadata = SimpleNamespace(
|
||||
num_actual_tokens=1,
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1]),
|
||||
query_start_loc_cpu=torch.tensor([0, 1]),
|
||||
block_table_tensor=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
)
|
||||
metadata = FlashMLASparseMetadata(
|
||||
num_reqs=1,
|
||||
max_query_len=1,
|
||||
max_seq_len=1,
|
||||
num_actual_tokens=1,
|
||||
query_start_loc=torch.tensor([0, 1], device=DEVICE_TYPE),
|
||||
slot_mapping=torch.tensor([0], device=DEVICE_TYPE),
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
)
|
||||
|
||||
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
|
||||
builder, common_metadata, metadata
|
||||
)
|
||||
|
||||
assert fp8_metadata.num_decodes == 0
|
||||
assert fp8_metadata.num_prefills == 1
|
||||
assert fp8_metadata.num_decode_tokens == 0
|
||||
assert fp8_metadata.num_prefill_tokens == 1
|
||||
|
||||
|
||||
def test_flashmla_common_metadata_requires_uniform_decodes():
|
||||
common_metadata = SimpleNamespace(
|
||||
max_query_len=3,
|
||||
num_reqs=3,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc_cpu=torch.tensor([0, 1, 3, 6]),
|
||||
is_prefilling=None,
|
||||
)
|
||||
|
||||
split = split_decodes_and_prefills(
|
||||
common_metadata,
|
||||
decode_threshold=128,
|
||||
require_uniform=FlashMLASparseMetadataBuilder.require_uniform_decodes,
|
||||
)
|
||||
|
||||
assert split == (1, 2, 1, 5)
|
||||
|
||||
|
||||
def test_flashmla_fp8_metadata_excludes_zero_token_decode_padding(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse.get_mla_metadata",
|
||||
lambda: (object(), None),
|
||||
)
|
||||
builder = SimpleNamespace(
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
dummy_block_table=torch.zeros(7, 1, device=DEVICE_TYPE),
|
||||
max_model_len_tensor=torch.zeros(7, device=DEVICE_TYPE),
|
||||
)
|
||||
query_start_loc_cpu = torch.tensor([0, 110, 220, 330, 440, 550, 660, 660])
|
||||
common_metadata = SimpleNamespace(
|
||||
num_actual_tokens=660,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=torch.arange(7, device=DEVICE_TYPE),
|
||||
)
|
||||
metadata = FlashMLASparseMetadata(
|
||||
num_reqs=7,
|
||||
max_query_len=110,
|
||||
max_seq_len=110,
|
||||
num_actual_tokens=660,
|
||||
query_start_loc=query_start_loc_cpu.to(DEVICE_TYPE),
|
||||
slot_mapping=torch.arange(660, device=DEVICE_TYPE),
|
||||
block_table=torch.zeros(7, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
req_id_per_token=torch.zeros(660, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_decodes=7,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=660,
|
||||
)
|
||||
|
||||
fp8_metadata = FlashMLASparseMetadataBuilder._build_fp8_separate_prefill_decode(
|
||||
builder, common_metadata, metadata
|
||||
)
|
||||
|
||||
assert fp8_metadata.num_decodes == 6
|
||||
assert fp8_metadata.num_decode_tokens == 660
|
||||
assert fp8_metadata.decode is not None
|
||||
assert fp8_metadata.decode.decode_query_len == 110
|
||||
torch.testing.assert_close(
|
||||
fp8_metadata.decode.seq_lens, torch.arange(6, device=DEVICE_TYPE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_mixed_batch", [False, True])
|
||||
def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: bool):
|
||||
num_decode_tokens = 2
|
||||
num_batch_tokens = 5
|
||||
q = torch.empty(num_decode_tokens, 2, 3, device=DEVICE_TYPE)
|
||||
topk_indices = torch.empty(num_decode_tokens, 4, device=DEVICE_TYPE)
|
||||
kernel_q_shapes = []
|
||||
|
||||
def convert_indices(*args, **kwargs): # noqa: ARG001
|
||||
assert not kwargs.get("HAS_PREFILL_WORKSPACE", False)
|
||||
if not kwargs.get("return_valid_counts", False):
|
||||
return topk_indices
|
||||
valid_counts = torch.full(
|
||||
(num_decode_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE
|
||||
)
|
||||
return topk_indices, valid_counts
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.flashmla_sparse."
|
||||
"triton_convert_req_index_to_global_index",
|
||||
convert_indices,
|
||||
)
|
||||
|
||||
def run_kernel(**kwargs):
|
||||
kernel_q_shapes.append(kwargs["q"].shape)
|
||||
return kwargs["q"][..., :1], None
|
||||
|
||||
if use_mixed_batch:
|
||||
fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata(
|
||||
scheduler_metadata=object(), # type: ignore[arg-type]
|
||||
dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
)
|
||||
else:
|
||||
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
||||
fp8_metadata = FP8Meta(
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefill_tokens=num_batch_tokens - num_decode_tokens,
|
||||
decode=FP8Meta.Decode(
|
||||
seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
kernel_metadata=object(), # type: ignore[arg-type]
|
||||
decode_query_len=num_decode_tokens,
|
||||
),
|
||||
prefill=FP8Meta.Prefill(
|
||||
request_ids=torch.empty(
|
||||
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
),
|
||||
workspace_starts=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
chunks=[],
|
||||
),
|
||||
)
|
||||
metadata = SimpleNamespace(
|
||||
fp8_extra_metadata=fp8_metadata,
|
||||
fp8_use_mixed_batch=use_mixed_batch,
|
||||
num_actual_tokens=num_batch_tokens,
|
||||
req_id_per_token=torch.empty(
|
||||
num_batch_tokens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
),
|
||||
block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
block_size=64,
|
||||
)
|
||||
impl = SimpleNamespace(
|
||||
kv_cache_dtype="fp8_ds_mla",
|
||||
topk_indices_buffer=topk_indices,
|
||||
num_heads=2,
|
||||
kv_lora_rank=1,
|
||||
_fp8_flash_mla_kernel=run_kernel,
|
||||
)
|
||||
impl._forward_fp8_kv_mixed_batch = MethodType(
|
||||
FlashMLASparseImpl._forward_fp8_kv_mixed_batch, impl
|
||||
)
|
||||
impl._forward_fp8_kv_separate_prefill_decode = MethodType(
|
||||
FlashMLASparseImpl._forward_fp8_kv_separate_prefill_decode, impl
|
||||
)
|
||||
|
||||
output, lse = FlashMLASparseImpl.forward_mqa(
|
||||
impl,
|
||||
q,
|
||||
torch.empty(0, device=DEVICE_TYPE),
|
||||
metadata,
|
||||
None,
|
||||
)
|
||||
|
||||
assert kernel_q_shapes == [(1, num_decode_tokens, 2, 3)]
|
||||
assert output.shape == (num_decode_tokens, 2, 1)
|
||||
assert lse is None
|
||||
|
||||
+3
-3
@@ -2759,9 +2759,9 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
workspace_starts: torch.Tensor,
|
||||
batch_size: int,
|
||||
seq_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Gather and upconvert FP8 KV cache to BF16 workspace.
|
||||
|
||||
@@ -2769,12 +2769,12 @@ def cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache: FP8 KV cache [num_blocks, block_size, 656]
|
||||
dst: BF16 output workspace [total_tokens, 576]
|
||||
block_table: Block indices [num_reqs, max_blocks]
|
||||
seq_lens: Sequence lengths [num_reqs]
|
||||
workspace_starts: Workspace start offsets [num_reqs]
|
||||
batch_size: Number of requests
|
||||
seq_starts: Optional source sequence offsets [num_reqs]
|
||||
"""
|
||||
torch.ops._C_cache_ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache, dst, block_table, seq_lens, workspace_starts, batch_size
|
||||
src_cache, dst, block_table, workspace_starts, batch_size, seq_starts
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1749,6 +1749,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.determine_chunked_prefill_workspace_size(vllm_config)
|
||||
)
|
||||
|
||||
use_packed_fp8_cache = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
if self.dcp_world_size > 1:
|
||||
# Note(hc): The local kvcache is incomplete when DCP is triggered,
|
||||
# an additional kvcache allgather across the DCP group is therefore
|
||||
@@ -1761,7 +1762,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
+ self.chunked_prefill_workspace_size // self.dcp_world_size,
|
||||
self.model_config.get_head_size(),
|
||||
),
|
||||
dtype=self.model_config.dtype,
|
||||
dtype=torch.bfloat16
|
||||
if use_packed_fp8_cache
|
||||
else self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
@@ -1770,7 +1773,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
self.chunked_prefill_workspace_size,
|
||||
self.model_config.get_head_size(),
|
||||
),
|
||||
dtype=self.q_data_type,
|
||||
dtype=torch.bfloat16 if use_packed_fp8_cache else self.q_data_type,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -2115,7 +2118,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
|
||||
for i in range(iters):
|
||||
toks = prefill_metadata.chunked_context.seq_tot[i]
|
||||
if not use_fp8_prefill:
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=prefill_metadata.block_table,
|
||||
workspace_starts=prefill_metadata.chunked_context.cu_seq_lens[i],
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
elif not use_fp8_prefill:
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace,
|
||||
@@ -2230,9 +2242,16 @@ class MLACommonBaseImpl(MLAAttentionImpl[A], Generic[A]):
|
||||
padded_local_cu_seq_lens = (
|
||||
prefill_metadata.chunked_context.padded_local_cu_seq_lens[i]
|
||||
)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype) and (
|
||||
self.kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
if self.kv_cache_dtype == "fp8_ds_mla":
|
||||
ops.cp_gather_and_upconvert_fp8_kv_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
dst=workspace[:toks],
|
||||
block_table=prefill_metadata.block_table,
|
||||
workspace_starts=padded_local_cu_seq_lens,
|
||||
batch_size=attn_metadata.num_prefills,
|
||||
seq_starts=prefill_metadata.chunked_context.starts[i],
|
||||
)
|
||||
elif is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
assert k_scale is not None
|
||||
ops.gather_and_maybe_dequant_cache(
|
||||
src_cache=kv_c_and_k_pe_cache,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Shared forward_mha implementation and metadata builder for sparse MLA backends."""
|
||||
|
||||
from shutil import which
|
||||
from typing import TYPE_CHECKING, Generic, TypeVar
|
||||
from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -38,6 +38,7 @@ T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
metadata_cls: type[T]
|
||||
require_uniform_decodes: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -173,6 +174,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
num_decodes, num_prefills, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
require_uniform=self.require_uniform_decodes,
|
||||
)
|
||||
(
|
||||
prefill_query_start_loc,
|
||||
|
||||
@@ -32,7 +32,6 @@ from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
reshape_attn_output_for_spec_decode,
|
||||
reshape_query_for_spec_decode,
|
||||
split_decodes_and_prefills,
|
||||
split_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import (
|
||||
@@ -57,8 +56,8 @@ logger = init_logger(__name__)
|
||||
# the FP8 decode kernel for decode.
|
||||
# Currently we use #1 when the number of heads per rank is low (i.e. TP) since the BF16
|
||||
# prefill kernel requires padding the number of heads to 128 while the decode does not
|
||||
# so when the per ranke head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
|
||||
# batch mode (#2).
|
||||
# so when the per-rank head count is below MIN_HEADS_FOR_BF16_PREFILL we use the mixed
|
||||
# batch mode (#1).
|
||||
MIN_HEADS_FOR_BF16_PREFILL = 32
|
||||
|
||||
"""
|
||||
@@ -182,10 +181,6 @@ class FlashMLASparseMetadata(AttentionMetadata):
|
||||
|
||||
@dataclass
|
||||
class Prefill:
|
||||
# Sequence lengths (context + query) for prefill requests
|
||||
# Shape: [num_prefill_reqs]
|
||||
seq_lens: torch.Tensor
|
||||
|
||||
# Request ID for each token: -1 for decode tokens, request index
|
||||
# (0, 1, 2, ...) for prefill tokens.
|
||||
# Shape: [num_actual_tokens]
|
||||
@@ -204,7 +199,6 @@ class FlashMLASparseMetadata(AttentionMetadata):
|
||||
Prefill requests may be chunked to fit within the fixed workspace size.
|
||||
"""
|
||||
|
||||
seq_lens: torch.Tensor
|
||||
tokens_slice: slice
|
||||
block_table: torch.Tensor
|
||||
req_start_idx: int
|
||||
@@ -239,6 +233,7 @@ class FlashMLASparseMetadataBuilder(
|
||||
SparseMLACommonMetadataBuilder[FlashMLASparseMetadata]
|
||||
):
|
||||
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
|
||||
require_uniform_decodes: ClassVar[bool] = True
|
||||
metadata_cls = FlashMLASparseMetadata
|
||||
|
||||
def __init__(
|
||||
@@ -322,10 +317,11 @@ class FlashMLASparseMetadataBuilder(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> "FlashMLASparseMetadata.FP8KernelMetadata":
|
||||
"""Build FP8 metadata treating all tokens as one mixed batch.
|
||||
"""Build FP8 metadata treating MQA tokens as one batch.
|
||||
|
||||
This matches main branch's approach and avoids the BF16 prefill kernel
|
||||
which has head padding overhead when num_heads is small (high TP case).
|
||||
The scheduler initializes lazily from the runtime query shape, which may
|
||||
be the full batch or only decodes when prefills use dense MHA. This avoids
|
||||
the BF16 prefill kernel's head-padding overhead at high TP.
|
||||
"""
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
|
||||
@@ -353,20 +349,29 @@ class FlashMLASparseMetadataBuilder(
|
||||
def _build_fp8_separate_prefill_decode(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
metadata: FlashMLASparseMetadata,
|
||||
) -> "FlashMLASparseMetadata.FP8SeparatePrefillDecode":
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
|
||||
(num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens) = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold or 1,
|
||||
require_uniform=True,
|
||||
)
|
||||
metadata.num_decodes,
|
||||
metadata.num_prefills,
|
||||
metadata.num_decode_tokens,
|
||||
num_tokens - metadata.num_decode_tokens,
|
||||
)
|
||||
|
||||
decode_query_len = 0
|
||||
active_num_decodes = num_decodes
|
||||
if num_decodes > 0:
|
||||
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item()
|
||||
assert decode_query_len > 0
|
||||
active_num_decodes = num_decode_tokens // decode_query_len
|
||||
assert active_num_decodes * decode_query_len == num_decode_tokens
|
||||
|
||||
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
||||
fp8_metadata = FP8Meta(
|
||||
num_decodes=num_decodes,
|
||||
num_decodes=active_num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
@@ -374,7 +379,6 @@ class FlashMLASparseMetadataBuilder(
|
||||
|
||||
# Extract prefill sequence lengths (context + query, not just query)
|
||||
# Decode requests come first in the batch, prefill requests follow
|
||||
prefill_seq_lens = None
|
||||
prefill_request_id = None
|
||||
prefill_workspace_starts = None
|
||||
prefill_chunks = None
|
||||
@@ -386,11 +390,9 @@ class FlashMLASparseMetadataBuilder(
|
||||
# slice below), so no D2H sync is needed.
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
|
||||
prefill_seq_lens_cpu = seq_lens_cpu[num_decodes:]
|
||||
prefill_seq_lens = seq_lens[num_decodes:]
|
||||
|
||||
# Build prefill_request_id: -1 for decode, request index for
|
||||
# prefill. This enables a single
|
||||
@@ -438,7 +440,6 @@ class FlashMLASparseMetadataBuilder(
|
||||
offset = prefill_workspace_starts_cpu[chunk_start].item()
|
||||
prefill_workspace_starts_cpu[chunk_start:chunk_end] -= offset
|
||||
|
||||
chunk_seq_lens = prefill_seq_lens[chunk_start:chunk_end]
|
||||
chunk_tot_seqlen = prefill_seq_lens_cpu[chunk_start:chunk_end].sum()
|
||||
token_start = query_start_loc_cpu[num_decodes + chunk_start].item()
|
||||
token_end = query_start_loc_cpu[num_decodes + chunk_end].item()
|
||||
@@ -452,7 +453,6 @@ class FlashMLASparseMetadataBuilder(
|
||||
|
||||
prefill_chunks.append(
|
||||
FP8Meta.Prefill.Chunk(
|
||||
seq_lens=chunk_seq_lens,
|
||||
tokens_slice=tokens_slice,
|
||||
block_table=chunk_block_table,
|
||||
req_start_idx=chunk_start,
|
||||
@@ -466,27 +466,22 @@ class FlashMLASparseMetadataBuilder(
|
||||
)
|
||||
|
||||
fp8_metadata.prefill = FP8Meta.Prefill(
|
||||
seq_lens=prefill_seq_lens,
|
||||
request_ids=prefill_request_id,
|
||||
workspace_starts=prefill_workspace_starts,
|
||||
chunks=prefill_chunks,
|
||||
)
|
||||
|
||||
if num_decodes > 0:
|
||||
# Compute decode_query_len for spec decode (uniform due to require_uniform)
|
||||
query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu
|
||||
decode_query_len = (query_start_loc_cpu[1] - query_start_loc_cpu[0]).item()
|
||||
|
||||
# Use padded head count since that's what the kernel will see
|
||||
scheduler_metadata, _ = get_mla_metadata()
|
||||
|
||||
kernel_meta = FlashMLASparseMetadata.FP8KernelMetadata(
|
||||
scheduler_metadata=scheduler_metadata,
|
||||
dummy_block_table=self.dummy_block_table[:num_decodes],
|
||||
cache_lens=self.max_model_len_tensor[:num_decodes],
|
||||
dummy_block_table=self.dummy_block_table[:active_num_decodes],
|
||||
cache_lens=self.max_model_len_tensor[:active_num_decodes],
|
||||
)
|
||||
fp8_metadata.decode = FP8Meta.Decode(
|
||||
seq_lens=common_attn_metadata.seq_lens[:num_decodes],
|
||||
seq_lens=common_attn_metadata.seq_lens[:active_num_decodes],
|
||||
kernel_metadata=kernel_meta,
|
||||
decode_query_len=decode_query_len,
|
||||
)
|
||||
@@ -510,7 +505,7 @@ class FlashMLASparseMetadataBuilder(
|
||||
)
|
||||
else:
|
||||
metadata.fp8_extra_metadata = self._build_fp8_separate_prefill_decode(
|
||||
common_attn_metadata
|
||||
common_attn_metadata, metadata
|
||||
)
|
||||
|
||||
return metadata
|
||||
@@ -597,9 +592,10 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
attn_metadata: FlashMLASparseMetadata,
|
||||
) -> torch.Tensor:
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill).
|
||||
# offsets (prefill). req_id_per_token covers the whole batch; slice it
|
||||
# to the MQA tokens (q may exclude prefill tokens routed to dense MHA).
|
||||
topk_indices, topk_length = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
@@ -624,11 +620,18 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
fp8_metadata = attn_metadata.fp8_extra_metadata
|
||||
assert isinstance(fp8_metadata, FlashMLASparseMetadata.FP8SeparatePrefillDecode)
|
||||
num_decodes = fp8_metadata.num_decodes
|
||||
num_mqa_tokens = q.shape[0]
|
||||
num_decode_tokens = fp8_metadata.num_decode_tokens
|
||||
num_prefill_tokens = num_mqa_tokens - num_decode_tokens
|
||||
assert num_prefill_tokens in (0, fp8_metadata.num_prefill_tokens), (
|
||||
"FP8 sparse MLA expects either the decode subset or the full batch"
|
||||
)
|
||||
|
||||
prefill_request_ids = None
|
||||
prefill_workspace_starts = None
|
||||
has_prefill_workspace = False
|
||||
if fp8_metadata.prefill is not None:
|
||||
if num_prefill_tokens > 0:
|
||||
assert fp8_metadata.prefill is not None
|
||||
prefill_request_ids = fp8_metadata.prefill.request_ids
|
||||
prefill_workspace_starts = fp8_metadata.prefill.workspace_starts
|
||||
has_prefill_workspace = True
|
||||
@@ -640,7 +643,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# prefill_workspace_starts has been adjusted in-place per chunk so
|
||||
# prefill indices automatically come out chunk-local
|
||||
topk_indices, topk_length = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
@@ -676,9 +679,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# -> (num_decode_tokens, num_heads, head_dim_v)
|
||||
return reshape_attn_output_for_spec_decode(attn_out)
|
||||
|
||||
num_decode_tokens = fp8_metadata.num_decode_tokens
|
||||
num_prefill_tokens = fp8_metadata.num_prefill_tokens
|
||||
|
||||
# Pure decode: direct call without allocation
|
||||
if num_decode_tokens > 0 and num_prefill_tokens == 0:
|
||||
assert fp8_metadata.decode is not None
|
||||
@@ -686,7 +686,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
else:
|
||||
# Mixed or pure prefill: allocate output tensor
|
||||
attn_out = q.new_empty(
|
||||
(attn_metadata.num_actual_tokens, self.num_heads, self.kv_lora_rank),
|
||||
(num_mqa_tokens, self.num_heads, self.kv_lora_rank),
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
@@ -704,7 +704,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
kv_c_and_k_pe_cache,
|
||||
chunk_workspace,
|
||||
chunk.block_table,
|
||||
chunk.seq_lens,
|
||||
chunk.workspace_starts,
|
||||
len(chunk.block_table),
|
||||
)
|
||||
@@ -738,7 +737,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
# Convert per-request indices to global slots (decode) or workspace
|
||||
# offsets (prefill).
|
||||
topk_indices = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token,
|
||||
attn_metadata.req_id_per_token[: topk_indices.shape[0]],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
|
||||
@@ -154,6 +154,11 @@ def triton_convert_req_index_to_global_index(
|
||||
assert req_id.dtype == torch.int32
|
||||
assert block_table.dtype == torch.int32
|
||||
assert token_indices.dtype == torch.int32
|
||||
assert req_id.shape[0] == token_indices.shape[0], (
|
||||
f"req_id ({req_id.shape[0]}) and token_indices ({token_indices.shape[0]}) "
|
||||
"must cover the same tokens; the grid is sized by req_id but the output "
|
||||
"is allocated like token_indices, so a longer req_id writes out of bounds"
|
||||
)
|
||||
assert token_indices.shape[1] == NUM_TOPK_TOKENS
|
||||
assert NUM_TOPK_TOKENS % BLOCK_N == 0, (
|
||||
f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible by BLOCK_N ({BLOCK_N})"
|
||||
|
||||
Reference in New Issue
Block a user