[ROCm][MLA] Enable sparse MLA Gluon kernel from Aiter (#53492)

Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com>
Signed-off-by: Mehmet Cagri <mehmet.kaymak@amd.com>
This commit is contained in:
Mehmet Cagri
2026-09-29 09:08:18 +08:00
committed by GitHub
parent 869278cbb0
commit 70ae0b7435
7 changed files with 989 additions and 22 deletions
@@ -4306,3 +4306,99 @@ def rocm_sparse_attn_decode(
if direct_out is None:
output.copy_(attn_out.to(output.dtype))
return output.shape[0] if inv_rope_positions is not None else 0
@triton.jit(do_not_specialize=["token_offset"])
def _prefill_topk_global_slots_kernel(
out_ptr,
lens_ptr,
topk_ptr,
topk_stride,
token_to_req_ptr,
query_start_loc_ptr,
seq_lens_ptr,
is_valid_token_ptr,
block_table_ptr,
block_table_stride,
token_offset,
num_compressed,
TOPK: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK_W: tl.constexpr,
):
row = tl.program_id(0)
token = row + token_offset
req = tl.load(token_to_req_ptr + token)
query_start = tl.load(query_start_loc_ptr + req)
query_len = tl.load(query_start_loc_ptr + req + 1) - query_start
pos = tl.load(seq_lens_ptr + req) - query_len + token - query_start
# Only (pos + 1) // ratio compressed tokens exist yet; the rest of the row
# can hold stale indices.
row_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOPK)
row_len = tl.where(tl.load(is_valid_token_ptr + token), row_len, 0)
for start in range(0, TOPK, BLOCK_W):
cols = start + tl.arange(0, BLOCK_W)
live = cols < row_len
local = tl.load(topk_ptr + row * topk_stride + cols, mask=live, other=-1)
valid = live & (local >= 0) & (local < num_compressed)
local = tl.where(valid, local, 0)
block = tl.load(
block_table_ptr + req * block_table_stride + local // BLOCK_SIZE,
mask=valid,
other=0,
)
slot = tl.where(valid, block * BLOCK_SIZE + local % BLOCK_SIZE, -1)
tl.store(out_ptr + row * TOPK + cols, slot, mask=cols < TOPK)
tl.store(lens_ptr + row, row_len)
def build_prefill_topk_ragged_indices(
topk_indices: torch.Tensor,
token_to_req_indices: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
is_valid_token: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
compress_ratio: int,
num_compressed: int,
token_offset: int,
num_rows: int = -1,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Map prefill top-k rows to a ragged stream of compressed-cache slots.
``topk_indices`` holds local compressed positions for the prefill tokens,
which sit at ``token_offset`` in the batch; ``token_to_req_indices``,
``query_start_loc``, ``seq_lens`` and ``block_table`` are batch-wide.
``block_size`` is the compressed cache's, i.e. already divided by the ratio.
"""
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1)
num_tokens, width = topk_indices.shape
dense = torch.empty(
(num_tokens, width), dtype=torch.int32, device=topk_indices.device
)
lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
if num_tokens > 0 and width > 0:
_prefill_topk_global_slots_kernel[(num_tokens,)](
dense,
lens,
topk_indices,
topk_indices.stride(0),
token_to_req_indices,
query_start_loc,
seq_lens,
is_valid_token,
block_table,
block_table.stride(0),
token_offset,
num_compressed,
TOPK=width,
COMPRESS_RATIO=compress_ratio,
BLOCK_SIZE=block_size,
BLOCK_W=min(triton.next_power_of_2(width), 1024),
)
else:
lens.zero_()
return build_ragged_indices_from_dense(dense, lens, num_rows=num_rows)