[ROCm][Perf] Optimize DeepSeek V4 C4A top-k with AITER (#52882)

This commit is contained in:
Fangzhou Ai
2026-08-21 10:00:39 -07:00
committed by GitHub
parent 1baf372bfc
commit fe76112ff2
5 changed files with 611 additions and 30 deletions
+132 -11
View File
@@ -2,6 +2,8 @@
#include "dispatch_utils.h"
#include "torch_utils.h"
#include <cstring>
#ifndef USE_ROCM
#include <cub/cub.cuh>
#else
@@ -327,11 +329,14 @@ __device__ bool processHistogramStep(
}
// Follows half - 11 - 11 - 10 bit iterations
// Keep the adaptive kernel's device instantiation separate from legacy calls.
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
bool deviceLengthAware = false>
static __device__ void topKPerRowJob(const int* indices, const float* logits,
int rowStart, int rowEnd, int* outIndices,
float* outLogits, int stride1, int topK) {
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
// The number of slots for the final pass.
static constexpr int kNumFinalItems = 2048;
// The number of elements per thread for the final sort.
@@ -613,6 +618,57 @@ static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode(
indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK);
}
// Measured native-kernel policy for gfx950, FP32, and k=1024. This
// active-column split crossover is independent of the Python AITER/native
// backend crossover; the two policies can be retuned separately. At exactly
// 64K, an eager call may select AITER while a captured native replay uses two
// splits.
constexpr int kGfx950C4AShortRowNumBlocks = 3;
constexpr int kGfx950C4ALongRowNumBlocks = 2;
constexpr int kGfx950C4ATwoSplitMinRowLength = 64 * 1024;
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
static __global__
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
const float* logits, const int* seqLens, int* outIndices, int stride0,
int stride1, const int topK, int next_n, int seqLensIs2D = 0,
float* outLogits = nullptr, const int* indices = nullptr) {
static constexpr int kNumBins = 2048;
int rowIdx = blockIdx.x;
int batch_idx = rowIdx / next_n;
int next_n_idx = rowIdx % next_n;
int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx];
int rowEnd =
seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1);
const int activeBlocks = rowEnd >= kGfx950C4ATwoSplitMinRowLength
? kGfx950C4ALongRowNumBlocks
: kGfx950C4AShortRowNumBlocks;
int rowStart = 0;
if constexpr (!mergeBlocks) {
if (blockIdx.y >= activeBlocks) return;
const auto blockSize = rowEnd / activeBlocks;
rowStart = blockSize * blockIdx.y;
rowEnd = activeBlocks == blockIdx.y + 1 ? rowEnd : rowStart + blockSize;
outIndices +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
blockIdx.y * topK;
outLogits +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
blockIdx.y * topK;
} else {
rowEnd = activeBlocks * topK;
indices +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK;
outIndices += static_cast<int64_t>(rowIdx) * topK;
}
logits += static_cast<int64_t>(rowIdx) * stride0;
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, !mergeBlocks, mergeBlocks,
true>(indices, logits, rowStart, rowEnd, outIndices, outLogits,
stride1, topK);
}
} // namespace vllm
void apply_repetition_penalties_(
@@ -693,7 +749,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
static_cast<int>(next_n), seqLensIs2D);
} else {
// Long sequences are run in two steps
constexpr auto multipleBlocksPerRowConfig = 10;
#ifdef USE_ROCM
const bool useGfx950DeviceLengthAwareTopK1024 =
topK == 1024 && numRows > 64 && numRows <= 256 &&
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
if (useGfx950DeviceLengthAwareTopK1024) {
// Keep three blocks for short FULL-graph replays, but let each row use
// two blocks once its live device length reaches the long-context
// crossover. The grid and workspace shapes remain capture-stable.
constexpr int multipleBlocksPerRowConfig =
vllm::kGfx950C4AShortRowNumBlocks;
const auto outIndicesAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
const auto outLogitsAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
constexpr int kNumThreadsPerSplitBlock = 1024;
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerSplitBlock>
<<<dim3(numRows, multipleBlocksPerRowConfig),
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
constexpr int kNumThreadsPerBlockMerge = 1024;
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerBlockMerge, true>
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t),
stream>>>(
outLogitsAux.const_data_ptr<float>(),
seqLens.const_data_ptr<int>(), indices.mutable_data_ptr<int>(),
multipleBlocksPerRowConfig * topK, 1, static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D, nullptr,
outIndicesAux.const_data_ptr<int>());
return;
}
#endif
#ifdef USE_ROCM
const bool useGfx950TopK1024 =
topK == 1024 && numRows <= 64 &&
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
#else
constexpr bool useGfx950TopK1024 = false;
#endif
int multipleBlocksPerRowConfig = 10;
if (useGfx950TopK1024) {
if (numRows <= 32) {
multipleBlocksPerRowConfig = 8;
} else {
multipleBlocksPerRowConfig = 4;
}
}
const auto outIndicesAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
@@ -702,14 +811,26 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
if (useGfx950TopK1024) {
constexpr int kNumThreadsPerSplitBlock = 1024;
vllm::topKPerRowDecode<kNumThreadsPerSplitBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig),
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
} else {
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
}
constexpr int kNumThreadsPerBlockMerge = 1024;
vllm::topKPerRowDecode<kNumThreadsPerBlockMerge, true, false, true>
@@ -753,4 +874,4 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits,
static_cast<int>(stride0), static_cast<int>(stride1),
static_cast<int>(topK), kSortingAlgorithmThreshold);
}
}
}
+293
View File
@@ -374,6 +374,299 @@ def test_top_k_per_row_decode_large_vocab_size(clean_logits: bool) -> None:
)
@pytest.mark.parametrize(
"batch_size,num_speculative_tokens,min_seq_len,max_seq_len",
[
pytest.param(1, 2, 125_000, 250_000, id="eight-splits"),
pytest.param(7, 4, 125_000, 250_000, id="four-splits"),
pytest.param(16, 4, 2_500, 2_501, id="dynamic-short-rows"),
pytest.param(13, 4, 25_000, 25_001, id="dynamic-three-splits"),
pytest.param(16, 4, 65_535, 65_536, id="dynamic-boundary-mixed"),
pytest.param(16, 4, 125_000, 125_001, id="dynamic-two-splits"),
pytest.param(64, 3, 125_000, 125_001, id="dynamic-two-splits-max-rows"),
pytest.param(64, 4, 125_000, 125_001, id="baseline-fallback"),
],
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_top_k_per_row_decode_gfx950_long_c4a(
batch_size: int,
num_speculative_tokens: int,
min_seq_len: int,
max_seq_len: int,
) -> None:
properties = torch.cuda.get_device_properties(0)
if not properties.gcnArchName.startswith("gfx950"):
pytest.skip("This test exercises the gfx950 launch configuration")
query_tokens = num_speculative_tokens + 1
num_rows = batch_size * query_tokens
stride = 262_144
top_k = 1024
offsets = torch.arange(num_rows, dtype=torch.int32, device="cuda")
seq_lens = min_seq_len + offsets * (max_seq_len - min_seq_len) // max(
num_rows - 1, 1
)
seq_lens = seq_lens.reshape(batch_size, query_tokens)
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
torch.ops._C.top_k_per_row_decode(
logits,
query_tokens,
seq_lens,
indices,
num_rows,
logits.stride(0),
logits.stride(1),
top_k,
)
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
validate_topk_against_reference(
logits,
indices,
row_starts,
seq_lens.reshape(-1),
top_k,
"gfx950 long C4A top-k",
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_top_k_per_row_decode_gfx950_long_c4a_1d_seq_lens() -> None:
properties = torch.cuda.get_device_properties(0)
if not properties.gcnArchName.startswith("gfx950"):
pytest.skip("This test exercises the gfx950 launch configuration")
batch_size = 20
query_tokens = 4
num_rows = batch_size * query_tokens
stride = 262_144
top_k = 1024
seq_lens = torch.linspace(
125_003,
250_000,
batch_size,
dtype=torch.int32,
device="cuda",
)
row_offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
row_ends = (seq_lens[:, None] - query_tokens + row_offsets + 1).reshape(-1)
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
torch.ops._C.top_k_per_row_decode(
logits,
query_tokens,
seq_lens,
indices,
num_rows,
logits.stride(0),
logits.stride(1),
top_k,
)
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
validate_topk_against_reference(
logits,
indices,
row_starts,
row_ends,
top_k,
"gfx950 long C4A top-k with 1-D sequence lengths",
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_prefill,
)
row_starts = torch.tensor([1, 6, 20], dtype=torch.int32, device="cuda")
row_ends = torch.tensor([3, 18, 23], dtype=torch.int32, device="cuda")
logits = torch.arange(96, dtype=torch.float32, device="cuda").reshape(3, 32)
top_k = 4
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
is_prefill=True,
compress_ratio=4,
num_rows=logits.shape[0],
on_gfx950=True,
)
if aiter_topk_kernel is None:
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_prefill(
aiter_topk_kernel,
logits,
row_starts,
row_ends,
indices,
top_k,
)
is None
)
for row_idx, (row_start, row_end) in enumerate(
zip(row_starts.tolist(), row_ends.tolist())
):
num_valid = min(top_k, row_end - row_start)
expected = logits[row_idx, row_start:row_end].topk(num_valid).indices
actual = indices[row_idx, :num_valid]
assert set(actual.tolist()) == set(expected.tolist())
assert torch.all(indices[row_idx, num_valid:] == -1)
@pytest.mark.parametrize("num_speculative_tokens", [2, 3, 4])
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
num_speculative_tokens: int,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_decode,
)
query_tokens = num_speculative_tokens + 1
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
final_uncompressed_lens = torch.tensor([43, 59], dtype=torch.int32, device="cuda")
seq_lens = (final_uncompressed_lens[:, None] - query_tokens + offsets + 1) // 4
seq_lens[0, 0] = 0
num_rows = seq_lens.numel()
logits = torch.arange(num_rows * 16, dtype=torch.float32, device="cuda").reshape(
num_rows, 16
)
top_k = 4
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
is_prefill=False,
compress_ratio=4,
num_rows=num_rows,
max_valid_seq_len=seq_lens.max().item(),
on_gfx950=True,
)
if aiter_topk_kernel is None:
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_decode(
aiter_topk_kernel,
logits,
seq_lens,
indices,
top_k,
)
is None
)
for row_idx, row_end in enumerate(seq_lens.reshape(-1).tolist()):
num_valid = min(top_k, row_end)
expected = logits[row_idx, :row_end].topk(num_valid).indices
actual = indices[row_idx, :num_valid]
assert set(actual.tolist()) == set(expected.tolist())
assert torch.all(indices[row_idx, num_valid:] == -1)
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
def prefill_kernel(*args, **kwargs) -> None:
pass
def decode_kernel(*args, **kwargs) -> None:
pass
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
lambda: (prefill_kernel, decode_kernel),
)
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
eligible_cases = [
(
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
prefill_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_536,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
on_gfx950=True,
),
decode_kernel,
),
]
for kwargs, expected_kernel in eligible_cases:
assert get_kernel(**kwargs) is expected_kernel
def fail_if_imported() -> None:
pytest.fail("ineligible shapes must not import AITER top-k")
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
fail_if_imported,
)
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_537,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=256,
max_valid_seq_len=125_000,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=320,
max_valid_seq_len=2_500,
on_gfx950=False,
),
]
for kwargs in ineligible_cases:
assert get_kernel(**kwargs) is None
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
assert (
get_kernel(
is_prefill=True,
compress_ratio=4,
num_rows=1,
on_gfx950=True,
)
is None
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize(
"seq_len_range,test_id",
@@ -750,6 +750,7 @@ class SparseAttnIndexer(CustomOp):
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool = False,
use_fp4_cache: bool = False,
compress_ratio: int = 1,
):
super().__init__()
self.k_cache = k_cache
@@ -762,6 +763,7 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer = topk_indices_buffer
self.skip_k_cache_insert = skip_k_cache_insert
self.use_fp4_cache = use_fp4_cache
self.compress_ratio = compress_ratio
self.dense_mha_metadata_layer_name = ""
# DCP scalars are constant for the run; resolve them here (config is set
# during model construction) and pass them into the custom op, rather
@@ -873,6 +875,7 @@ class SparseAttnIndexer(CustomOp):
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=self.skip_k_cache_insert,
compress_ratio=self.compress_ratio,
)
raise RuntimeError(
"Sparse attention indexer ROCm path is only supported on AITER. "
+1
View File
@@ -854,6 +854,7 @@ class DeepseekV4Indexer(nn.Module):
self.topk_indices_buffer,
skip_k_cache_insert=True,
use_fp4_cache=self.use_fp4_kv,
compress_ratio=self.compress_ratio,
)
# None on ROCm — maybe_execute_in_parallel falls back to sequential.
+182 -19
View File
@@ -3,6 +3,7 @@
import functools
import importlib
import math
from collections.abc import Callable
from importlib.util import find_spec
import torch
@@ -10,6 +11,7 @@ import torch.nn.functional as F
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CUDAGraphMode
from vllm.forward_context import get_forward_context
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -25,6 +27,127 @@ else:
_ON_GFX950 = False
@functools.cache
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
try:
from aiter.ops.topk import (
top_k_per_row_decode,
top_k_per_row_prefill,
)
except ImportError:
return None
return top_k_per_row_prefill, top_k_per_row_decode
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
_GFX950_C4A_NATIVE_MAX_ROWS = 256
def _get_aiter_top_k_kernel(
*,
is_prefill: bool,
compress_ratio: int,
num_rows: int,
max_valid_seq_len: int | None = None,
on_gfx950: bool = _ON_GFX950,
) -> Callable[..., None] | None:
if compress_ratio <= 1 or not on_gfx950:
return None
if not is_prefill:
assert max_valid_seq_len is not None
# AITER v0.1.19 decode is one-block only. This measured gfx950
# FP32/k=1024 compressed-row boundary is independent of the native
# split-count boundary in sampler.cu.
if (
num_rows <= _GFX950_C4A_NATIVE_MAX_ROWS
and max_valid_seq_len > _GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN
):
return None
topk_ops = _get_aiter_topk_ops()
if topk_ops is None:
return None
return topk_ops[0] if is_prefill else topk_ops[1]
@triton.jit
def _localize_aiter_prefill_topk_kernel(
indices_ptr,
row_starts_ptr,
indices_stride,
num_topk,
BLOCK_SIZE: tl.constexpr,
):
row_idx = tl.program_id(0)
offsets = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_topk
indices = tl.load(indices_ptr + row_idx * indices_stride + offsets, mask=mask)
row_start = tl.load(row_starts_ptr + row_idx)
localized = tl.where(indices < 0, indices, indices - row_start)
tl.store(
indices_ptr + row_idx * indices_stride + offsets,
localized,
mask=mask,
)
def _localize_aiter_prefill_topk(
indices: torch.Tensor,
row_starts: torch.Tensor,
) -> None:
num_rows, num_topk = indices.shape
block_size = 256
_localize_aiter_prefill_topk_kernel[(num_rows, triton.cdiv(num_topk, block_size))](
indices,
row_starts,
indices.stride(0),
num_topk,
BLOCK_SIZE=block_size,
)
def _launch_aiter_top_k_per_row_prefill(
top_k_per_row_prefill: Callable[..., None],
logits: torch.Tensor,
row_starts: torch.Tensor,
row_ends: torch.Tensor,
indices: torch.Tensor,
topk_tokens: int,
) -> None:
top_k_per_row_prefill(
logits,
row_starts,
row_ends,
indices,
None,
logits.shape[0],
logits.stride(0),
logits.stride(1),
k=topk_tokens,
)
_localize_aiter_prefill_topk(indices, row_starts)
def _launch_aiter_top_k_per_row_decode(
top_k_per_row_decode: Callable[..., None],
logits: torch.Tensor,
seq_lens: torch.Tensor,
indices: torch.Tensor,
topk_tokens: int,
) -> None:
top_k_per_row_decode(
logits,
1,
seq_lens.reshape(-1),
indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
k=topk_tokens,
)
@triton.jit
def _indexer_k_quant_and_cache_kernel(
k_ptr, # [num_tokens, head_dim]
@@ -692,6 +815,7 @@ def rocm_aiter_sparse_attn_indexer_fake(
total_seq_lens: int,
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool = False,
compress_ratio: int = 1,
) -> torch.Tensor:
return topk_indices_buffer
@@ -712,9 +836,11 @@ def rocm_aiter_sparse_attn_indexer(
total_seq_lens: int,
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool = False,
compress_ratio: int = 1,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
fp8_dtype = current_platform.fp8_dtype()
from vllm.utils.torch_utils import _resolve_layer_name
@@ -773,6 +899,7 @@ def rocm_aiter_sparse_attn_indexer(
total_seq_lens,
topk_indices_buffer,
skip_k_cache_insert,
compress_ratio,
)
layer_attn_metadata = attn_metadata[k_cache_prefix]
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
@@ -833,16 +960,31 @@ def rocm_aiter_sparse_attn_indexer(
num_rows = logits.shape[0]
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
aiter_topk_kernel = _get_aiter_top_k_kernel(
is_prefill=True,
compress_ratio=compress_ratio,
num_rows=num_rows,
)
if aiter_topk_kernel is not None:
_launch_aiter_top_k_per_row_prefill(
aiter_topk_kernel,
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
topk_tokens,
)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
if has_decode:
decode_metadata = layer_attn_metadata.decode
@@ -882,16 +1024,37 @@ def rocm_aiter_sparse_attn_indexer(
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
num_rows = logits.shape[0]
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
decode_metadata.seq_lens,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
# FULL graphs are not keyed by context length, so use a replay-safe
# upper bound when choosing the captured kernel.
if forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL:
max_compressed_seq_len = max_model_len
else:
max_compressed_seq_len = layer_attn_metadata.max_seq_len // compress_ratio
aiter_topk_kernel = _get_aiter_top_k_kernel(
is_prefill=False,
compress_ratio=compress_ratio,
num_rows=num_rows,
max_valid_seq_len=max_compressed_seq_len,
)
if aiter_topk_kernel is not None:
_launch_aiter_top_k_per_row_decode(
aiter_topk_kernel,
logits,
decode_metadata.seq_lens,
topk_indices,
topk_tokens,
)
else:
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
decode_metadata.seq_lens,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
if decode_metadata.requires_padding:
# if padded, we need to unpack