From fe76112ff2981e9373765d76687d53f94880f38e Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:00:39 -0400 Subject: [PATCH] [ROCm][Perf] Optimize DeepSeek V4 C4A top-k with AITER (#52882) --- csrc/libtorch_stable/sampler.cu | 143 ++++++++- tests/kernels/test_top_k_per_row.py | 293 ++++++++++++++++++ .../layers/sparse_attn_indexer.py | 3 + vllm/models/deepseek_v4/attention.py | 1 + .../v1/attention/ops/rocm_aiter_mla_sparse.py | 201 ++++++++++-- 5 files changed, 611 insertions(+), 30 deletions(-) diff --git a/csrc/libtorch_stable/sampler.cu b/csrc/libtorch_stable/sampler.cu index 519e213281..ada1c4ba39 100644 --- a/csrc/libtorch_stable/sampler.cu +++ b/csrc/libtorch_stable/sampler.cu @@ -2,6 +2,8 @@ #include "dispatch_utils.h" #include "torch_utils.h" +#include + #ifndef USE_ROCM #include #else @@ -327,11 +329,14 @@ __device__ bool processHistogramStep( } // Follows half - 11 - 11 - 10 bit iterations +// Keep the adaptive kernel's device instantiation separate from legacy calls. template + bool multipleBlocksPerRow = false, bool mergeBlocks = false, + bool deviceLengthAware = false> static __device__ void topKPerRowJob(const int* indices, const float* logits, int rowStart, int rowEnd, int* outIndices, float* outLogits, int stride1, int topK) { + static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks); // The number of slots for the final pass. static constexpr int kNumFinalItems = 2048; // The number of elements per thread for the final sort. @@ -613,6 +618,57 @@ static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode( indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK); } +// Measured native-kernel policy for gfx950, FP32, and k=1024. This +// active-column split crossover is independent of the Python AITER/native +// backend crossover; the two policies can be retuned separately. At exactly +// 64K, an eager call may select AITER while a captured native replay uses two +// splits. +constexpr int kGfx950C4AShortRowNumBlocks = 3; +constexpr int kGfx950C4ALongRowNumBlocks = 2; +constexpr int kGfx950C4ATwoSplitMinRowLength = 64 * 1024; + +template +static __global__ +__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware( + const float* logits, const int* seqLens, int* outIndices, int stride0, + int stride1, const int topK, int next_n, int seqLensIs2D = 0, + float* outLogits = nullptr, const int* indices = nullptr) { + static constexpr int kNumBins = 2048; + int rowIdx = blockIdx.x; + int batch_idx = rowIdx / next_n; + int next_n_idx = rowIdx % next_n; + int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx]; + int rowEnd = + seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1); + const int activeBlocks = rowEnd >= kGfx950C4ATwoSplitMinRowLength + ? kGfx950C4ALongRowNumBlocks + : kGfx950C4AShortRowNumBlocks; + int rowStart = 0; + + if constexpr (!mergeBlocks) { + if (blockIdx.y >= activeBlocks) return; + const auto blockSize = rowEnd / activeBlocks; + rowStart = blockSize * blockIdx.y; + rowEnd = activeBlocks == blockIdx.y + 1 ? rowEnd : rowStart + blockSize; + outIndices += + static_cast(rowIdx) * kGfx950C4AShortRowNumBlocks * topK + + blockIdx.y * topK; + outLogits += + static_cast(rowIdx) * kGfx950C4AShortRowNumBlocks * topK + + blockIdx.y * topK; + } else { + rowEnd = activeBlocks * topK; + indices += + static_cast(rowIdx) * kGfx950C4AShortRowNumBlocks * topK; + outIndices += static_cast(rowIdx) * topK; + } + logits += static_cast(rowIdx) * stride0; + + topKPerRowJob(indices, logits, rowStart, rowEnd, outIndices, outLogits, + stride1, topK); +} + } // namespace vllm void apply_repetition_penalties_( @@ -693,7 +749,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n, static_cast(next_n), seqLensIs2D); } else { // Long sequences are run in two steps - constexpr auto multipleBlocksPerRowConfig = 10; +#ifdef USE_ROCM + const bool useGfx950DeviceLengthAwareTopK1024 = + topK == 1024 && numRows > 64 && numRows <= 256 && + std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0; + if (useGfx950DeviceLengthAwareTopK1024) { + // Keep three blocks for short FULL-graph replays, but let each row use + // two blocks once its live device length reaches the long-context + // crossover. The grid and workspace shapes remain capture-stable. + constexpr int multipleBlocksPerRowConfig = + vllm::kGfx950C4AShortRowNumBlocks; + const auto outIndicesAux = torch::stable::empty( + {numRows, multipleBlocksPerRowConfig, topK}, + torch::headeronly::ScalarType::Int, std::nullopt, logits.device()); + const auto outLogitsAux = torch::stable::empty( + {numRows, multipleBlocksPerRowConfig, topK}, + torch::headeronly::ScalarType::Float, std::nullopt, logits.device()); + + constexpr int kNumThreadsPerSplitBlock = 1024; + vllm::topKPerRowDecodeDeviceLengthAware + <<>>( + logits.const_data_ptr(), seqLens.const_data_ptr(), + outIndicesAux.mutable_data_ptr(), static_cast(stride0), + static_cast(stride1), static_cast(topK), + static_cast(next_n), seqLensIs2D, + outLogitsAux.mutable_data_ptr()); + constexpr int kNumThreadsPerBlockMerge = 1024; + vllm::topKPerRowDecodeDeviceLengthAware + <<>>( + outLogitsAux.const_data_ptr(), + seqLens.const_data_ptr(), indices.mutable_data_ptr(), + multipleBlocksPerRowConfig * topK, 1, static_cast(topK), + static_cast(next_n), seqLensIs2D, nullptr, + outIndicesAux.const_data_ptr()); + return; + } +#endif + +#ifdef USE_ROCM + const bool useGfx950TopK1024 = + topK == 1024 && numRows <= 64 && + std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0; +#else + constexpr bool useGfx950TopK1024 = false; +#endif + int multipleBlocksPerRowConfig = 10; + if (useGfx950TopK1024) { + if (numRows <= 32) { + multipleBlocksPerRowConfig = 8; + } else { + multipleBlocksPerRowConfig = 4; + } + } const auto outIndicesAux = torch::stable::empty( {numRows, multipleBlocksPerRowConfig, topK}, @@ -702,14 +811,26 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n, {numRows, multipleBlocksPerRowConfig, topK}, torch::headeronly::ScalarType::Float, std::nullopt, logits.device()); - vllm::topKPerRowDecode - <<>>( - logits.const_data_ptr(), seqLens.const_data_ptr(), - outIndicesAux.mutable_data_ptr(), static_cast(stride0), - static_cast(stride1), static_cast(topK), - static_cast(next_n), seqLensIs2D, - outLogitsAux.mutable_data_ptr()); + if (useGfx950TopK1024) { + constexpr int kNumThreadsPerSplitBlock = 1024; + vllm::topKPerRowDecode + <<>>( + logits.const_data_ptr(), seqLens.const_data_ptr(), + outIndicesAux.mutable_data_ptr(), static_cast(stride0), + static_cast(stride1), static_cast(topK), + static_cast(next_n), seqLensIs2D, + outLogitsAux.mutable_data_ptr()); + } else { + vllm::topKPerRowDecode + <<>>( + logits.const_data_ptr(), seqLens.const_data_ptr(), + outIndicesAux.mutable_data_ptr(), static_cast(stride0), + static_cast(stride1), static_cast(topK), + static_cast(next_n), seqLensIs2D, + outLogitsAux.mutable_data_ptr()); + } constexpr int kNumThreadsPerBlockMerge = 1024; vllm::topKPerRowDecode @@ -753,4 +874,4 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits, static_cast(stride0), static_cast(stride1), static_cast(topK), kSortingAlgorithmThreshold); } -} \ No newline at end of file +} diff --git a/tests/kernels/test_top_k_per_row.py b/tests/kernels/test_top_k_per_row.py index dba247bbdb..bb283e26a2 100644 --- a/tests/kernels/test_top_k_per_row.py +++ b/tests/kernels/test_top_k_per_row.py @@ -374,6 +374,299 @@ def test_top_k_per_row_decode_large_vocab_size(clean_logits: bool) -> None: ) +@pytest.mark.parametrize( + "batch_size,num_speculative_tokens,min_seq_len,max_seq_len", + [ + pytest.param(1, 2, 125_000, 250_000, id="eight-splits"), + pytest.param(7, 4, 125_000, 250_000, id="four-splits"), + pytest.param(16, 4, 2_500, 2_501, id="dynamic-short-rows"), + pytest.param(13, 4, 25_000, 25_001, id="dynamic-three-splits"), + pytest.param(16, 4, 65_535, 65_536, id="dynamic-boundary-mixed"), + pytest.param(16, 4, 125_000, 125_001, id="dynamic-two-splits"), + pytest.param(64, 3, 125_000, 125_001, id="dynamic-two-splits-max-rows"), + pytest.param(64, 4, 125_000, 125_001, id="baseline-fallback"), + ], +) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm") +@torch.inference_mode() +def test_top_k_per_row_decode_gfx950_long_c4a( + batch_size: int, + num_speculative_tokens: int, + min_seq_len: int, + max_seq_len: int, +) -> None: + properties = torch.cuda.get_device_properties(0) + if not properties.gcnArchName.startswith("gfx950"): + pytest.skip("This test exercises the gfx950 launch configuration") + + query_tokens = num_speculative_tokens + 1 + num_rows = batch_size * query_tokens + stride = 262_144 + top_k = 1024 + offsets = torch.arange(num_rows, dtype=torch.int32, device="cuda") + seq_lens = min_seq_len + offsets * (max_seq_len - min_seq_len) // max( + num_rows - 1, 1 + ) + seq_lens = seq_lens.reshape(batch_size, query_tokens) + logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda") + indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda") + + torch.ops._C.top_k_per_row_decode( + logits, + query_tokens, + seq_lens, + indices, + num_rows, + logits.stride(0), + logits.stride(1), + top_k, + ) + + row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda") + validate_topk_against_reference( + logits, + indices, + row_starts, + seq_lens.reshape(-1), + top_k, + "gfx950 long C4A top-k", + ) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm") +@torch.inference_mode() +def test_top_k_per_row_decode_gfx950_long_c4a_1d_seq_lens() -> None: + properties = torch.cuda.get_device_properties(0) + if not properties.gcnArchName.startswith("gfx950"): + pytest.skip("This test exercises the gfx950 launch configuration") + + batch_size = 20 + query_tokens = 4 + num_rows = batch_size * query_tokens + stride = 262_144 + top_k = 1024 + seq_lens = torch.linspace( + 125_003, + 250_000, + batch_size, + dtype=torch.int32, + device="cuda", + ) + row_offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda") + row_ends = (seq_lens[:, None] - query_tokens + row_offsets + 1).reshape(-1) + logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda") + indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda") + + torch.ops._C.top_k_per_row_decode( + logits, + query_tokens, + seq_lens, + indices, + num_rows, + logits.stride(0), + logits.stride(1), + top_k, + ) + + row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda") + validate_topk_against_reference( + logits, + indices, + row_starts, + row_ends, + top_k, + "gfx950 long C4A top-k with 1-D sequence lengths", + ) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm") +@torch.inference_mode() +def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _get_aiter_top_k_kernel, + _launch_aiter_top_k_per_row_prefill, + ) + + row_starts = torch.tensor([1, 6, 20], dtype=torch.int32, device="cuda") + row_ends = torch.tensor([3, 18, 23], dtype=torch.int32, device="cuda") + logits = torch.arange(96, dtype=torch.float32, device="cuda").reshape(3, 32) + top_k = 4 + indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda") + + aiter_topk_kernel = _get_aiter_top_k_kernel( + is_prefill=True, + compress_ratio=4, + num_rows=logits.shape[0], + on_gfx950=True, + ) + if aiter_topk_kernel is None: + pytest.skip("AITER top-k is unavailable") + assert ( + _launch_aiter_top_k_per_row_prefill( + aiter_topk_kernel, + logits, + row_starts, + row_ends, + indices, + top_k, + ) + is None + ) + + for row_idx, (row_start, row_end) in enumerate( + zip(row_starts.tolist(), row_ends.tolist()) + ): + num_valid = min(top_k, row_end - row_start) + expected = logits[row_idx, row_start:row_end].topk(num_valid).indices + actual = indices[row_idx, :num_valid] + assert set(actual.tolist()) == set(expected.tolist()) + assert torch.all(indices[row_idx, num_valid:] == -1) + + +@pytest.mark.parametrize("num_speculative_tokens", [2, 3, 4]) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm") +@torch.inference_mode() +def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths( + num_speculative_tokens: int, +) -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _get_aiter_top_k_kernel, + _launch_aiter_top_k_per_row_decode, + ) + + query_tokens = num_speculative_tokens + 1 + offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda") + final_uncompressed_lens = torch.tensor([43, 59], dtype=torch.int32, device="cuda") + seq_lens = (final_uncompressed_lens[:, None] - query_tokens + offsets + 1) // 4 + seq_lens[0, 0] = 0 + + num_rows = seq_lens.numel() + logits = torch.arange(num_rows * 16, dtype=torch.float32, device="cuda").reshape( + num_rows, 16 + ) + top_k = 4 + indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda") + + aiter_topk_kernel = _get_aiter_top_k_kernel( + is_prefill=False, + compress_ratio=4, + num_rows=num_rows, + max_valid_seq_len=seq_lens.max().item(), + on_gfx950=True, + ) + if aiter_topk_kernel is None: + pytest.skip("AITER top-k is unavailable") + assert ( + _launch_aiter_top_k_per_row_decode( + aiter_topk_kernel, + logits, + seq_lens, + indices, + top_k, + ) + is None + ) + + for row_idx, row_end in enumerate(seq_lens.reshape(-1).tolist()): + num_valid = min(top_k, row_end) + expected = logits[row_idx, :row_end].topk(num_valid).indices + actual = indices[row_idx, :num_valid] + assert set(actual.tolist()) == set(expected.tolist()) + assert torch.all(indices[row_idx, num_valid:] == -1) + + +def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None: + from vllm.v1.attention.ops import rocm_aiter_mla_sparse + + def prefill_kernel(*args, **kwargs) -> None: + pass + + def decode_kernel(*args, **kwargs) -> None: + pass + + monkeypatch.setattr( + rocm_aiter_mla_sparse, + "_get_aiter_topk_ops", + lambda: (prefill_kernel, decode_kernel), + ) + get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel + + eligible_cases = [ + ( + dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True), + prefill_kernel, + ), + ( + dict( + is_prefill=False, + compress_ratio=4, + num_rows=1, + max_valid_seq_len=65_536, + on_gfx950=True, + ), + decode_kernel, + ), + ( + dict( + is_prefill=False, + compress_ratio=4, + num_rows=257, + max_valid_seq_len=250_000, + on_gfx950=True, + ), + decode_kernel, + ), + ] + for kwargs, expected_kernel in eligible_cases: + assert get_kernel(**kwargs) is expected_kernel + + def fail_if_imported() -> None: + pytest.fail("ineligible shapes must not import AITER top-k") + + monkeypatch.setattr( + rocm_aiter_mla_sparse, + "_get_aiter_topk_ops", + fail_if_imported, + ) + ineligible_cases = [ + dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True), + dict( + is_prefill=False, + compress_ratio=4, + num_rows=1, + max_valid_seq_len=65_537, + on_gfx950=True, + ), + dict( + is_prefill=False, + compress_ratio=4, + num_rows=256, + max_valid_seq_len=125_000, + on_gfx950=True, + ), + dict( + is_prefill=False, + compress_ratio=4, + num_rows=320, + max_valid_seq_len=2_500, + on_gfx950=False, + ), + ] + for kwargs in ineligible_cases: + assert get_kernel(**kwargs) is None + + monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None) + assert ( + get_kernel( + is_prefill=True, + compress_ratio=4, + num_rows=1, + on_gfx950=True, + ) + is None + ) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") @pytest.mark.parametrize( "seq_len_range,test_id", diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 8c7332aac7..600094ad8c 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -750,6 +750,7 @@ class SparseAttnIndexer(CustomOp): topk_indices_buffer: torch.Tensor, skip_k_cache_insert: bool = False, use_fp4_cache: bool = False, + compress_ratio: int = 1, ): super().__init__() self.k_cache = k_cache @@ -762,6 +763,7 @@ class SparseAttnIndexer(CustomOp): self.topk_indices_buffer = topk_indices_buffer self.skip_k_cache_insert = skip_k_cache_insert self.use_fp4_cache = use_fp4_cache + self.compress_ratio = compress_ratio self.dense_mha_metadata_layer_name = "" # DCP scalars are constant for the run; resolve them here (config is set # during model construction) and pass them into the custom op, rather @@ -873,6 +875,7 @@ class SparseAttnIndexer(CustomOp): self.max_total_seq_len, self.topk_indices_buffer, skip_k_cache_insert=self.skip_k_cache_insert, + compress_ratio=self.compress_ratio, ) raise RuntimeError( "Sparse attention indexer ROCm path is only supported on AITER. " diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index ef6f9ea96f..c06185be93 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -854,6 +854,7 @@ class DeepseekV4Indexer(nn.Module): self.topk_indices_buffer, skip_k_cache_insert=True, use_fp4_cache=self.use_fp4_kv, + compress_ratio=self.compress_ratio, ) # None on ROCm — maybe_execute_in_parallel falls back to sequential. diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 0f55556c5d..7fef84fdc6 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -3,6 +3,7 @@ import functools import importlib import math +from collections.abc import Callable from importlib.util import find_spec import torch @@ -10,6 +11,7 @@ import torch.nn.functional as F import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import CUDAGraphMode from vllm.forward_context import get_forward_context from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -25,6 +27,127 @@ else: _ON_GFX950 = False +@functools.cache +def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None: + try: + from aiter.ops.topk import ( + top_k_per_row_decode, + top_k_per_row_prefill, + ) + except ImportError: + return None + return top_k_per_row_prefill, top_k_per_row_decode + + +_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024 +_GFX950_C4A_NATIVE_MAX_ROWS = 256 + + +def _get_aiter_top_k_kernel( + *, + is_prefill: bool, + compress_ratio: int, + num_rows: int, + max_valid_seq_len: int | None = None, + on_gfx950: bool = _ON_GFX950, +) -> Callable[..., None] | None: + if compress_ratio <= 1 or not on_gfx950: + return None + + if not is_prefill: + assert max_valid_seq_len is not None + # AITER v0.1.19 decode is one-block only. This measured gfx950 + # FP32/k=1024 compressed-row boundary is independent of the native + # split-count boundary in sampler.cu. + if ( + num_rows <= _GFX950_C4A_NATIVE_MAX_ROWS + and max_valid_seq_len > _GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN + ): + return None + + topk_ops = _get_aiter_topk_ops() + if topk_ops is None: + return None + return topk_ops[0] if is_prefill else topk_ops[1] + + +@triton.jit +def _localize_aiter_prefill_topk_kernel( + indices_ptr, + row_starts_ptr, + indices_stride, + num_topk, + BLOCK_SIZE: tl.constexpr, +): + row_idx = tl.program_id(0) + offsets = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offsets < num_topk + indices = tl.load(indices_ptr + row_idx * indices_stride + offsets, mask=mask) + row_start = tl.load(row_starts_ptr + row_idx) + localized = tl.where(indices < 0, indices, indices - row_start) + tl.store( + indices_ptr + row_idx * indices_stride + offsets, + localized, + mask=mask, + ) + + +def _localize_aiter_prefill_topk( + indices: torch.Tensor, + row_starts: torch.Tensor, +) -> None: + num_rows, num_topk = indices.shape + block_size = 256 + _localize_aiter_prefill_topk_kernel[(num_rows, triton.cdiv(num_topk, block_size))]( + indices, + row_starts, + indices.stride(0), + num_topk, + BLOCK_SIZE=block_size, + ) + + +def _launch_aiter_top_k_per_row_prefill( + top_k_per_row_prefill: Callable[..., None], + logits: torch.Tensor, + row_starts: torch.Tensor, + row_ends: torch.Tensor, + indices: torch.Tensor, + topk_tokens: int, +) -> None: + top_k_per_row_prefill( + logits, + row_starts, + row_ends, + indices, + None, + logits.shape[0], + logits.stride(0), + logits.stride(1), + k=topk_tokens, + ) + _localize_aiter_prefill_topk(indices, row_starts) + + +def _launch_aiter_top_k_per_row_decode( + top_k_per_row_decode: Callable[..., None], + logits: torch.Tensor, + seq_lens: torch.Tensor, + indices: torch.Tensor, + topk_tokens: int, +) -> None: + top_k_per_row_decode( + logits, + 1, + seq_lens.reshape(-1), + indices, + logits.shape[0], + logits.stride(0), + logits.stride(1), + k=topk_tokens, + ) + + @triton.jit def _indexer_k_quant_and_cache_kernel( k_ptr, # [num_tokens, head_dim] @@ -692,6 +815,7 @@ def rocm_aiter_sparse_attn_indexer_fake( total_seq_lens: int, topk_indices_buffer: torch.Tensor | None, skip_k_cache_insert: bool = False, + compress_ratio: int = 1, ) -> torch.Tensor: return topk_indices_buffer @@ -712,9 +836,11 @@ def rocm_aiter_sparse_attn_indexer( total_seq_lens: int, topk_indices_buffer: torch.Tensor | None, skip_k_cache_insert: bool = False, + compress_ratio: int = 1, ) -> torch.Tensor: # careful! this will be None in dummy run - attn_metadata = get_forward_context().attn_metadata + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata fp8_dtype = current_platform.fp8_dtype() from vllm.utils.torch_utils import _resolve_layer_name @@ -773,6 +899,7 @@ def rocm_aiter_sparse_attn_indexer( total_seq_lens, topk_indices_buffer, skip_k_cache_insert, + compress_ratio, ) layer_attn_metadata = attn_metadata[k_cache_prefix] assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata) @@ -833,16 +960,31 @@ def rocm_aiter_sparse_attn_indexer( num_rows = logits.shape[0] - torch.ops._C.top_k_per_row_prefill( - logits, - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, - topk_indices, - num_rows, - logits.stride(0), - logits.stride(1), - topk_tokens, + aiter_topk_kernel = _get_aiter_top_k_kernel( + is_prefill=True, + compress_ratio=compress_ratio, + num_rows=num_rows, ) + if aiter_topk_kernel is not None: + _launch_aiter_top_k_per_row_prefill( + aiter_topk_kernel, + logits, + chunk.cu_seqlen_ks, + chunk.cu_seqlen_ke, + topk_indices, + topk_tokens, + ) + else: + torch.ops._C.top_k_per_row_prefill( + logits, + chunk.cu_seqlen_ks, + chunk.cu_seqlen_ke, + topk_indices, + num_rows, + logits.stride(0), + logits.stride(1), + topk_tokens, + ) if has_decode: decode_metadata = layer_attn_metadata.decode @@ -882,16 +1024,37 @@ def rocm_aiter_sparse_attn_indexer( topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens] num_rows = logits.shape[0] - torch.ops._C.top_k_per_row_decode( - logits, - next_n, - decode_metadata.seq_lens, - topk_indices, - num_rows, - logits.stride(0), - logits.stride(1), - topk_tokens, + # FULL graphs are not keyed by context length, so use a replay-safe + # upper bound when choosing the captured kernel. + if forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL: + max_compressed_seq_len = max_model_len + else: + max_compressed_seq_len = layer_attn_metadata.max_seq_len // compress_ratio + aiter_topk_kernel = _get_aiter_top_k_kernel( + is_prefill=False, + compress_ratio=compress_ratio, + num_rows=num_rows, + max_valid_seq_len=max_compressed_seq_len, ) + if aiter_topk_kernel is not None: + _launch_aiter_top_k_per_row_decode( + aiter_topk_kernel, + logits, + decode_metadata.seq_lens, + topk_indices, + topk_tokens, + ) + else: + torch.ops._C.top_k_per_row_decode( + logits, + next_n, + decode_metadata.seq_lens, + topk_indices, + num_rows, + logits.stride(0), + logits.stride(1), + topk_tokens, + ) if decode_metadata.requires_padding: # if padded, we need to unpack