[ROCm][Perf][GLM-5.3-Flash] Add AITER topk backend for decodes (#58167)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-10-02 11:47:41 +01:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 751fa96a49
commit 98b29aa99a
8 changed files with 705 additions and 240 deletions
+57 -10
View File
@@ -163,6 +163,14 @@ class SparseIndexerTopk(torch.nn.Module):
current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
self._is_rocm = current_platform.is_rocm()
self._aiter_enabled = False
self._aiter_capable = False
if self._is_rocm:
from vllm._aiter_ops import rocm_aiter_ops
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
def resolve_backend(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
@@ -193,6 +201,19 @@ class SparseIndexerTopk(torch.nn.Module):
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
f" stride={logits.stride()}, topk={topk_tokens}"
)
elif self._backend == "aiter":
if not self._is_rocm:
failures.append("requires a ROCm platform")
elif not self._aiter_enabled:
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
elif not self._aiter_capable:
failures.append(
"AITER has no tuned indexer top-k kernels for this GPU arch"
)
if topk_tokens not in (512, 1024, 2048, 4096):
failures.append(
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
)
elif self._backend == "flashinfer":
if not self._is_cuda:
failures.append("requires a CUDA platform")
@@ -216,11 +237,14 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The priority chain: cooperative -> persistent -> per_row.
deep_select/flashinfer/torch are opt-in only.
aiter/deep_select/flashinfer/torch are opt-in only.
cooperative_topk is preferred whenever it is applicable, i.e. within
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
persistent_topk.
aiter not auto-resolved as it does not outperform the per-row on every shape,
hence we delegate to the caller to decide when to use it.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
@@ -285,7 +309,8 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
@@ -344,15 +369,37 @@ class SparseIndexerTopk(torch.nn.Module):
) < row_ends.unsqueeze(1)
indices = torch.where(in_range, indices, -1)
topk_indices.copy_(indices)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
ops.top_k_per_row_decode(
elif backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
# AITER's decode kernel only implements the 1D/next_n == 1 form,
# which is equivalent once the ragged ends are expanded.
rocm_aiter_ops.indexer_top_k_decode(
logits,
next_n,
seq_lens,
1,
self._row_ends(seq_lens, next_n, logits.shape[0]),
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
@staticmethod
def _per_row(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
) -> None:
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)