[Attention] Vectorize sparse MLA mask loads (#52217)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e575b5f1a9
commit
8e46accab2
@@ -8,6 +8,7 @@ pytest.importorskip("cutlass")
|
||||
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
_build_topk_mask,
|
||||
_topk_mask_shape,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,3 +28,24 @@ def test_build_topk_mask_single_request_matches_generic_path() -> None:
|
||||
|
||||
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
|
||||
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
|
||||
|
||||
|
||||
def test_topk_mask_rows_are_aligned_for_vectorized_loads() -> None:
|
||||
assert _topk_mask_shape(2, 129, 129) == (2, 256, 8)
|
||||
assert _topk_mask_shape(2, 129, 129, reserve_key_starts_word=True) == (
|
||||
2,
|
||||
256,
|
||||
8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_build_topk_mask_preserves_aligned_row_storage() -> None:
|
||||
shape = _topk_mask_shape(1, 1, 129)
|
||||
out = torch.zeros(shape, dtype=torch.int32, device="cuda")
|
||||
topk = torch.tensor([[0, 128]], dtype=torch.int32, device="cuda")
|
||||
|
||||
mask = _build_topk_mask([topk], [1], 1, 129, out)
|
||||
|
||||
assert mask.shape == (1, 1, 8)
|
||||
assert mask[0, 0].tolist() == [1, 0, 0, 0, 1, 0, 0, 0]
|
||||
|
||||
@@ -58,6 +58,7 @@ def _topk_mask_shape(
|
||||
tile_m = 128 if max_query_len <= 128 else 256
|
||||
padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m
|
||||
num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word)
|
||||
num_words = triton.cdiv(num_words, 4) * 4
|
||||
return batch_size, padded_q_len, num_words
|
||||
|
||||
|
||||
@@ -432,12 +433,12 @@ def _build_topk_mask(
|
||||
max_seq_len: int,
|
||||
out: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``."""
|
||||
"""Build a bit-packed top-k mask while preserving padded row storage."""
|
||||
batch_size = len(q_lens)
|
||||
num_words = (max_seq_len + 31) // 32
|
||||
total_rows = batch_size * max_q_len
|
||||
if total_rows == 0:
|
||||
return out[:batch_size, :max_q_len, :num_words]
|
||||
return out[:batch_size, :max_q_len]
|
||||
|
||||
total_q = sum(q_lens)
|
||||
mask_row_stride = out.stride(-2)
|
||||
@@ -457,7 +458,7 @@ def _build_topk_mask(
|
||||
BLOCK_TOPK=triton.next_power_of_2(num_topk),
|
||||
BLOCK_WORDS=block_words,
|
||||
)
|
||||
return out[:1, :max_q_len, :num_words]
|
||||
return out[:1, :max_q_len]
|
||||
|
||||
topk_packed = torch.cat(topk_indices_per_req, dim=0)
|
||||
num_topk = topk_packed.shape[1]
|
||||
@@ -478,7 +479,7 @@ def _build_topk_mask(
|
||||
BLOCK_TOPK=triton.next_power_of_2(num_topk),
|
||||
BLOCK_WORDS=block_words,
|
||||
)
|
||||
return out[:batch_size, :max_q_len, :num_words]
|
||||
return out[:batch_size, :max_q_len]
|
||||
|
||||
|
||||
class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]):
|
||||
|
||||
@@ -20,15 +20,27 @@ def dense_mask_mod(
|
||||
batch_idx = utils.ssa_to_scalar(batch)
|
||||
q_idx = utils.ssa_to_scalar(q_idx)
|
||||
kv_idx = utils.ssa_to_scalar(kv_idx)
|
||||
word_idx = kv_idx >> 5
|
||||
bit_idx = cutlass.Uint32(kv_idx & 31)
|
||||
word = dense_mask[batch_idx, q_idx, word_idx]
|
||||
result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32)
|
||||
result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx)
|
||||
return result.load()
|
||||
batch_stride, query_stride, _ = dense_mask.stride
|
||||
aligned_mask = cute.make_tensor(
|
||||
dense_mask.iterator,
|
||||
cute.make_layout(
|
||||
dense_mask.shape,
|
||||
stride=(
|
||||
cute.assume(batch_stride, divby=4),
|
||||
cute.assume(query_stride, divby=4),
|
||||
1,
|
||||
),
|
||||
),
|
||||
)
|
||||
mask_row = aligned_mask[batch_idx, q_idx, None]
|
||||
mask_chunks = cute.flat_divide(mask_row, (4,))
|
||||
mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2]
|
||||
loaded = cute.make_rmem_tensor_like(mask_chunk)
|
||||
cute.autovec_copy(mask_chunk, loaded)
|
||||
return cute.recast_tensor(loaded, cutlass.Uint32).load()
|
||||
|
||||
|
||||
dense_mask_mod.__vec_size__ = 32
|
||||
dense_mask_mod.__vec_size__ = 128
|
||||
|
||||
|
||||
@cute.jit
|
||||
|
||||
Reference in New Issue
Block a user