[Attention] Vectorize sparse MLA mask loads (#52217)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e575b5f1a9
commit
8e46accab2
@@ -8,6 +8,7 @@ pytest.importorskip("cutlass")
|
||||
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
_build_topk_mask,
|
||||
_topk_mask_shape,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,3 +28,24 @@ def test_build_topk_mask_single_request_matches_generic_path() -> None:
|
||||
|
||||
torch.testing.assert_close(single_req[0, 0], generic[0, 0])
|
||||
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
|
||||
|
||||
|
||||
def test_topk_mask_rows_are_aligned_for_vectorized_loads() -> None:
|
||||
assert _topk_mask_shape(2, 129, 129) == (2, 256, 8)
|
||||
assert _topk_mask_shape(2, 129, 129, reserve_key_starts_word=True) == (
|
||||
2,
|
||||
256,
|
||||
8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_build_topk_mask_preserves_aligned_row_storage() -> None:
|
||||
shape = _topk_mask_shape(1, 1, 129)
|
||||
out = torch.zeros(shape, dtype=torch.int32, device="cuda")
|
||||
topk = torch.tensor([[0, 128]], dtype=torch.int32, device="cuda")
|
||||
|
||||
mask = _build_topk_mask([topk], [1], 1, 129, out)
|
||||
|
||||
assert mask.shape == (1, 1, 8)
|
||||
assert mask[0, 0].tolist() == [1, 0, 0, 0, 1, 0, 0, 0]
|
||||
|
||||
Reference in New Issue
Block a user