[ROCm][Perf][GLM-5.3-Flash] Add AITER topk backend for decodes (#58167)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
751fa96a49
commit
98b29aa99a
@@ -163,6 +163,14 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
current_platform.has_device_capability(90)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
)
|
||||
self._is_rocm = current_platform.is_rocm()
|
||||
self._aiter_enabled = False
|
||||
self._aiter_capable = False
|
||||
if self._is_rocm:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
|
||||
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
|
||||
|
||||
def resolve_backend(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
@@ -193,6 +201,19 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
|
||||
f" stride={logits.stride()}, topk={topk_tokens}"
|
||||
)
|
||||
elif self._backend == "aiter":
|
||||
if not self._is_rocm:
|
||||
failures.append("requires a ROCm platform")
|
||||
elif not self._aiter_enabled:
|
||||
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
|
||||
elif not self._aiter_capable:
|
||||
failures.append(
|
||||
"AITER has no tuned indexer top-k kernels for this GPU arch"
|
||||
)
|
||||
if topk_tokens not in (512, 1024, 2048, 4096):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "flashinfer":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
@@ -216,11 +237,14 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The priority chain: cooperative -> persistent -> per_row.
|
||||
deep_select/flashinfer/torch are opt-in only.
|
||||
aiter/deep_select/flashinfer/torch are opt-in only.
|
||||
|
||||
cooperative_topk is preferred whenever it is applicable, i.e. within
|
||||
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
|
||||
persistent_topk.
|
||||
|
||||
aiter not auto-resolved as it does not outperform the per-row on every shape,
|
||||
hence we delegate to the caller to decide when to use it.
|
||||
"""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
@@ -285,7 +309,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
@@ -344,15 +369,37 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
) < row_ends.unsqueeze(1)
|
||||
indices = torch.where(in_range, indices, -1)
|
||||
topk_indices.copy_(indices)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
ops.top_k_per_row_decode(
|
||||
elif backend == "aiter":
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# AITER's decode kernel only implements the 1D/next_n == 1 form,
|
||||
# which is equivalent once the ragged ends are expanded.
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
1,
|
||||
self._row_ends(seq_lens, next_n, logits.shape[0]),
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
|
||||
|
||||
@staticmethod
|
||||
def _per_row(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user