[Attention] Vectorize sparse MLA mask loads (#52217)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-08-18 20:32:41 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e575b5f1a9
commit 8e46accab2
3 changed files with 46 additions and 11 deletions
@@ -8,6 +8,7 @@ pytest.importorskip("cutlass")
from vllm.model_executor.layers.attention.sparse_mla_attention import (
_build_topk_mask,
_topk_mask_shape,
)
@@ -27,3 +28,24 @@ def test_build_topk_mask_single_request_matches_generic_path() -> None:
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
def test_topk_mask_rows_are_aligned_for_vectorized_loads() -> None:
assert _topk_mask_shape(2, 129, 129) == (2, 256, 8)
assert _topk_mask_shape(2, 129, 129, reserve_key_starts_word=True) == (
2,
256,
8,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_build_topk_mask_preserves_aligned_row_storage() -> None:
shape = _topk_mask_shape(1, 1, 129)
out = torch.zeros(shape, dtype=torch.int32, device="cuda")
topk = torch.tensor([[0, 128]], dtype=torch.int32, device="cuda")
mask = _build_topk_mask([topk], [1], 1, 129, out)
assert mask.shape == (1, 1, 8)
assert mask[0, 0].tolist() == [1, 0, 0, 0, 1, 0, 0, 0]
@@ -58,6 +58,7 @@ def _topk_mask_shape(
tile_m = 128 if max_query_len <= 128 else 256
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word)
num_words = triton.cdiv(num_words, 4) * 4
return batch_size, padded_q_len, num_words
@@ -432,12 +433,12 @@ def _build_topk_mask(
max_seq_len: int,
out: torch.Tensor,
) -> torch.Tensor:
"""Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``."""
"""Build a bit-packed top-k mask while preserving padded row storage."""
batch_size = len(q_lens)
num_words = (max_seq_len + 31) // 32
total_rows = batch_size * max_q_len
if total_rows == 0:
return out[:batch_size, :max_q_len, :num_words]
return out[:batch_size, :max_q_len]
total_q = sum(q_lens)
mask_row_stride = out.stride(-2)
@@ -457,7 +458,7 @@ def _build_topk_mask(
BLOCK_TOPK=triton.next_power_of_2(num_topk),
BLOCK_WORDS=block_words,
)
return out[:1, :max_q_len, :num_words]
return out[:1, :max_q_len]
topk_packed = torch.cat(topk_indices_per_req, dim=0)
num_topk = topk_packed.shape[1]
@@ -478,7 +479,7 @@ def _build_topk_mask(
BLOCK_TOPK=triton.next_power_of_2(num_topk),
BLOCK_WORDS=block_words,
)
return out[:batch_size, :max_q_len, :num_words]
return out[:batch_size, :max_q_len]
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
@@ -20,15 +20,27 @@ def dense_mask_mod(
batch_idx = utils.ssa_to_scalar(batch)
q_idx = utils.ssa_to_scalar(q_idx)
kv_idx = utils.ssa_to_scalar(kv_idx)
word_idx = kv_idx >> 5
bit_idx = cutlass.Uint32(kv_idx & 31)
word = dense_mask[batch_idx, q_idx, word_idx]
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
return result.load()
batch_stride, query_stride, _ = dense_mask.stride
aligned_mask = cute.make_tensor(
dense_mask.iterator,
cute.make_layout(
dense_mask.shape,
stride=(
cute.assume(batch_stride, divby=4),
cute.assume(query_stride, divby=4),
1,
),
),
)
mask_row = aligned_mask[batch_idx, q_idx, None]
mask_chunks = cute.flat_divide(mask_row, (4,))
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
loaded = cute.make_rmem_tensor_like(mask_chunk)
cute.autovec_copy(mask_chunk, loaded)
return cute.recast_tensor(loaded, cutlass.Uint32).load()
dense_mask_mod.__vec_size__ = 32
dense_mask_mod.__vec_size__ = 128
@cute.jit