diff --git a/tests/v1/attention/test_sparse_mla_mask.py b/tests/v1/attention/test_sparse_mla_mask.py index 3ecc179be4..20c7631e1d 100644 --- a/tests/v1/attention/test_sparse_mla_mask.py +++ b/tests/v1/attention/test_sparse_mla_mask.py @@ -8,6 +8,7 @@ pytest.importorskip("cutlass") from vllm.model_executor.layers.attention.sparse_mla_attention import ( _build_topk_mask, + _topk_mask_shape, ) @@ -27,3 +28,24 @@ def test_build_topk_mask_single_request_matches_generic_path() -> None: torch.testing.assert_close(single_req[0, 0], generic[0, 0]) torch.testing.assert_close(single_req[0, 1], generic[1, 0]) + + +def test_topk_mask_rows_are_aligned_for_vectorized_loads() -> None: + assert _topk_mask_shape(2, 129, 129) == (2, 256, 8) + assert _topk_mask_shape(2, 129, 129, reserve_key_starts_word=True) == ( + 2, + 256, + 8, + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_build_topk_mask_preserves_aligned_row_storage() -> None: + shape = _topk_mask_shape(1, 1, 129) + out = torch.zeros(shape, dtype=torch.int32, device="cuda") + topk = torch.tensor([[0, 128]], dtype=torch.int32, device="cuda") + + mask = _build_topk_mask([topk], [1], 1, 129, out) + + assert mask.shape == (1, 1, 8) + assert mask[0, 0].tolist() == [1, 0, 0, 0, 1, 0, 0, 0] diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index a619b74035..85e09cd5d4 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -58,6 +58,7 @@ def _topk_mask_shape( tile_m = 128 if max_query_len <= 128 else 256 padded_q_len = triton.cdiv(max_query_len, tile_m) * tile_m num_words = triton.cdiv(max_key_len, 32) + int(reserve_key_starts_word) + num_words = triton.cdiv(num_words, 4) * 4 return batch_size, padded_q_len, num_words @@ -432,12 +433,12 @@ def _build_topk_mask( max_seq_len: int, out: torch.Tensor, ) -> torch.Tensor: - """Build a bit-packed top-k mask into ``out[:B, :max_Q, :num_words]``.""" + """Build a bit-packed top-k mask while preserving padded row storage.""" batch_size = len(q_lens) num_words = (max_seq_len + 31) // 32 total_rows = batch_size * max_q_len if total_rows == 0: - return out[:batch_size, :max_q_len, :num_words] + return out[:batch_size, :max_q_len] total_q = sum(q_lens) mask_row_stride = out.stride(-2) @@ -457,7 +458,7 @@ def _build_topk_mask( BLOCK_TOPK=triton.next_power_of_2(num_topk), BLOCK_WORDS=block_words, ) - return out[:1, :max_q_len, :num_words] + return out[:1, :max_q_len] topk_packed = torch.cat(topk_indices_per_req, dim=0) num_topk = topk_packed.shape[1] @@ -478,7 +479,7 @@ def _build_topk_mask( BLOCK_TOPK=triton.next_power_of_2(num_topk), BLOCK_WORDS=block_words, ) - return out[:batch_size, :max_q_len, :num_words] + return out[:batch_size, :max_q_len] class SparseMLACommonImpl(MLACommonBaseImpl[T], Generic[T]): diff --git a/vllm/model_executor/layers/attention/sparse_mla_mask.py b/vllm/model_executor/layers/attention/sparse_mla_mask.py index 1cd151034c..8659786bd1 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_mask.py +++ b/vllm/model_executor/layers/attention/sparse_mla_mask.py @@ -20,15 +20,27 @@ def dense_mask_mod( batch_idx = utils.ssa_to_scalar(batch) q_idx = utils.ssa_to_scalar(q_idx) kv_idx = utils.ssa_to_scalar(kv_idx) - word_idx = kv_idx >> 5 - bit_idx = cutlass.Uint32(kv_idx & 31) - word = dense_mask[batch_idx, q_idx, word_idx] - result = cute.make_rmem_tensor(1, dtype=cutlass.Uint32) - result[0] = utils.shr_u32(cutlass.Uint32(word), bit_idx) - return result.load() + batch_stride, query_stride, _ = dense_mask.stride + aligned_mask = cute.make_tensor( + dense_mask.iterator, + cute.make_layout( + dense_mask.shape, + stride=( + cute.assume(batch_stride, divby=4), + cute.assume(query_stride, divby=4), + 1, + ), + ), + ) + mask_row = aligned_mask[batch_idx, q_idx, None] + mask_chunks = cute.flat_divide(mask_row, (4,)) + mask_chunk = mask_chunks[None, (kv_idx >> 5) >> 2] + loaded = cute.make_rmem_tensor_like(mask_chunk) + cute.autovec_copy(mask_chunk, loaded) + return cute.recast_tensor(loaded, cutlass.Uint32).load() -dense_mask_mod.__vec_size__ = 32 +dense_mask_mod.__vec_size__ = 128 @cute.jit