[ROCm][Perf] Optimize DeepSeek V4 C4A top-k with AITER (#52882)
This commit is contained in:
+132
-11
@@ -2,6 +2,8 @@
|
||||
#include "dispatch_utils.h"
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/cub.cuh>
|
||||
#else
|
||||
@@ -327,11 +329,14 @@ __device__ bool processHistogramStep(
|
||||
}
|
||||
|
||||
// Follows half - 11 - 11 - 10 bit iterations
|
||||
// Keep the adaptive kernel's device instantiation separate from legacy calls.
|
||||
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
|
||||
bool deviceLengthAware = false>
|
||||
static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
int rowStart, int rowEnd, int* outIndices,
|
||||
float* outLogits, int stride1, int topK) {
|
||||
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
|
||||
// The number of slots for the final pass.
|
||||
static constexpr int kNumFinalItems = 2048;
|
||||
// The number of elements per thread for the final sort.
|
||||
@@ -613,6 +618,57 @@ static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode(
|
||||
indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK);
|
||||
}
|
||||
|
||||
// Measured native-kernel policy for gfx950, FP32, and k=1024. This
|
||||
// active-column split crossover is independent of the Python AITER/native
|
||||
// backend crossover; the two policies can be retuned separately. At exactly
|
||||
// 64K, an eager call may select AITER while a captured native replay uses two
|
||||
// splits.
|
||||
constexpr int kGfx950C4AShortRowNumBlocks = 3;
|
||||
constexpr int kGfx950C4ALongRowNumBlocks = 2;
|
||||
constexpr int kGfx950C4ATwoSplitMinRowLength = 64 * 1024;
|
||||
|
||||
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
|
||||
static __global__
|
||||
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
|
||||
const float* logits, const int* seqLens, int* outIndices, int stride0,
|
||||
int stride1, const int topK, int next_n, int seqLensIs2D = 0,
|
||||
float* outLogits = nullptr, const int* indices = nullptr) {
|
||||
static constexpr int kNumBins = 2048;
|
||||
int rowIdx = blockIdx.x;
|
||||
int batch_idx = rowIdx / next_n;
|
||||
int next_n_idx = rowIdx % next_n;
|
||||
int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx];
|
||||
int rowEnd =
|
||||
seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1);
|
||||
const int activeBlocks = rowEnd >= kGfx950C4ATwoSplitMinRowLength
|
||||
? kGfx950C4ALongRowNumBlocks
|
||||
: kGfx950C4AShortRowNumBlocks;
|
||||
int rowStart = 0;
|
||||
|
||||
if constexpr (!mergeBlocks) {
|
||||
if (blockIdx.y >= activeBlocks) return;
|
||||
const auto blockSize = rowEnd / activeBlocks;
|
||||
rowStart = blockSize * blockIdx.y;
|
||||
rowEnd = activeBlocks == blockIdx.y + 1 ? rowEnd : rowStart + blockSize;
|
||||
outIndices +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
|
||||
blockIdx.y * topK;
|
||||
outLogits +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
|
||||
blockIdx.y * topK;
|
||||
} else {
|
||||
rowEnd = activeBlocks * topK;
|
||||
indices +=
|
||||
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK;
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
}
|
||||
logits += static_cast<int64_t>(rowIdx) * stride0;
|
||||
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, !mergeBlocks, mergeBlocks,
|
||||
true>(indices, logits, rowStart, rowEnd, outIndices, outLogits,
|
||||
stride1, topK);
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void apply_repetition_penalties_(
|
||||
@@ -693,7 +749,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
static_cast<int>(next_n), seqLensIs2D);
|
||||
} else {
|
||||
// Long sequences are run in two steps
|
||||
constexpr auto multipleBlocksPerRowConfig = 10;
|
||||
#ifdef USE_ROCM
|
||||
const bool useGfx950DeviceLengthAwareTopK1024 =
|
||||
topK == 1024 && numRows > 64 && numRows <= 256 &&
|
||||
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
|
||||
if (useGfx950DeviceLengthAwareTopK1024) {
|
||||
// Keep three blocks for short FULL-graph replays, but let each row use
|
||||
// two blocks once its live device length reaches the long-context
|
||||
// crossover. The grid and workspace shapes remain capture-stable.
|
||||
constexpr int multipleBlocksPerRowConfig =
|
||||
vllm::kGfx950C4AShortRowNumBlocks;
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
|
||||
const auto outLogitsAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
|
||||
constexpr int kNumThreadsPerSplitBlock = 1024;
|
||||
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerSplitBlock>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig),
|
||||
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerBlockMerge, true>
|
||||
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t),
|
||||
stream>>>(
|
||||
outLogitsAux.const_data_ptr<float>(),
|
||||
seqLens.const_data_ptr<int>(), indices.mutable_data_ptr<int>(),
|
||||
multipleBlocksPerRowConfig * topK, 1, static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D, nullptr,
|
||||
outIndicesAux.const_data_ptr<int>());
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef USE_ROCM
|
||||
const bool useGfx950TopK1024 =
|
||||
topK == 1024 && numRows <= 64 &&
|
||||
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
|
||||
#else
|
||||
constexpr bool useGfx950TopK1024 = false;
|
||||
#endif
|
||||
int multipleBlocksPerRowConfig = 10;
|
||||
if (useGfx950TopK1024) {
|
||||
if (numRows <= 32) {
|
||||
multipleBlocksPerRowConfig = 8;
|
||||
} else {
|
||||
multipleBlocksPerRowConfig = 4;
|
||||
}
|
||||
}
|
||||
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
@@ -702,14 +811,26 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
if (useGfx950TopK1024) {
|
||||
constexpr int kNumThreadsPerSplitBlock = 1024;
|
||||
vllm::topKPerRowDecode<kNumThreadsPerSplitBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig),
|
||||
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
} else {
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
}
|
||||
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlockMerge, true, false, true>
|
||||
@@ -753,4 +874,4 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits,
|
||||
static_cast<int>(stride0), static_cast<int>(stride1),
|
||||
static_cast<int>(topK), kSortingAlgorithmThreshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,6 +374,299 @@ def test_top_k_per_row_decode_large_vocab_size(clean_logits: bool) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size,num_speculative_tokens,min_seq_len,max_seq_len",
|
||||
[
|
||||
pytest.param(1, 2, 125_000, 250_000, id="eight-splits"),
|
||||
pytest.param(7, 4, 125_000, 250_000, id="four-splits"),
|
||||
pytest.param(16, 4, 2_500, 2_501, id="dynamic-short-rows"),
|
||||
pytest.param(13, 4, 25_000, 25_001, id="dynamic-three-splits"),
|
||||
pytest.param(16, 4, 65_535, 65_536, id="dynamic-boundary-mixed"),
|
||||
pytest.param(16, 4, 125_000, 125_001, id="dynamic-two-splits"),
|
||||
pytest.param(64, 3, 125_000, 125_001, id="dynamic-two-splits-max-rows"),
|
||||
pytest.param(64, 4, 125_000, 125_001, id="baseline-fallback"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_top_k_per_row_decode_gfx950_long_c4a(
|
||||
batch_size: int,
|
||||
num_speculative_tokens: int,
|
||||
min_seq_len: int,
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
properties = torch.cuda.get_device_properties(0)
|
||||
if not properties.gcnArchName.startswith("gfx950"):
|
||||
pytest.skip("This test exercises the gfx950 launch configuration")
|
||||
|
||||
query_tokens = num_speculative_tokens + 1
|
||||
num_rows = batch_size * query_tokens
|
||||
stride = 262_144
|
||||
top_k = 1024
|
||||
offsets = torch.arange(num_rows, dtype=torch.int32, device="cuda")
|
||||
seq_lens = min_seq_len + offsets * (max_seq_len - min_seq_len) // max(
|
||||
num_rows - 1, 1
|
||||
)
|
||||
seq_lens = seq_lens.reshape(batch_size, query_tokens)
|
||||
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
query_tokens,
|
||||
seq_lens,
|
||||
indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
top_k,
|
||||
)
|
||||
|
||||
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
|
||||
validate_topk_against_reference(
|
||||
logits,
|
||||
indices,
|
||||
row_starts,
|
||||
seq_lens.reshape(-1),
|
||||
top_k,
|
||||
"gfx950 long C4A top-k",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_top_k_per_row_decode_gfx950_long_c4a_1d_seq_lens() -> None:
|
||||
properties = torch.cuda.get_device_properties(0)
|
||||
if not properties.gcnArchName.startswith("gfx950"):
|
||||
pytest.skip("This test exercises the gfx950 launch configuration")
|
||||
|
||||
batch_size = 20
|
||||
query_tokens = 4
|
||||
num_rows = batch_size * query_tokens
|
||||
stride = 262_144
|
||||
top_k = 1024
|
||||
seq_lens = torch.linspace(
|
||||
125_003,
|
||||
250_000,
|
||||
batch_size,
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
row_offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
|
||||
row_ends = (seq_lens[:, None] - query_tokens + row_offsets + 1).reshape(-1)
|
||||
logits = torch.randn(num_rows, stride, dtype=torch.float32, device="cuda")
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
query_tokens,
|
||||
seq_lens,
|
||||
indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
top_k,
|
||||
)
|
||||
|
||||
row_starts = torch.zeros(num_rows, dtype=torch.int32, device="cuda")
|
||||
validate_topk_against_reference(
|
||||
logits,
|
||||
indices,
|
||||
row_starts,
|
||||
row_ends,
|
||||
top_k,
|
||||
"gfx950 long C4A top-k with 1-D sequence lengths",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_prefill,
|
||||
)
|
||||
|
||||
row_starts = torch.tensor([1, 6, 20], dtype=torch.int32, device="cuda")
|
||||
row_ends = torch.tensor([3, 18, 23], dtype=torch.int32, device="cuda")
|
||||
logits = torch.arange(96, dtype=torch.float32, device="cuda").reshape(3, 32)
|
||||
top_k = 4
|
||||
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=logits.shape[0],
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_prefill(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
row_starts,
|
||||
row_ends,
|
||||
indices,
|
||||
top_k,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
for row_idx, (row_start, row_end) in enumerate(
|
||||
zip(row_starts.tolist(), row_ends.tolist())
|
||||
):
|
||||
num_valid = min(top_k, row_end - row_start)
|
||||
expected = logits[row_idx, row_start:row_end].topk(num_valid).indices
|
||||
actual = indices[row_idx, :num_valid]
|
||||
assert set(actual.tolist()) == set(expected.tolist())
|
||||
assert torch.all(indices[row_idx, num_valid:] == -1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [2, 3, 4])
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
num_speculative_tokens: int,
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_decode,
|
||||
)
|
||||
|
||||
query_tokens = num_speculative_tokens + 1
|
||||
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
|
||||
final_uncompressed_lens = torch.tensor([43, 59], dtype=torch.int32, device="cuda")
|
||||
seq_lens = (final_uncompressed_lens[:, None] - query_tokens + offsets + 1) // 4
|
||||
seq_lens[0, 0] = 0
|
||||
|
||||
num_rows = seq_lens.numel()
|
||||
logits = torch.arange(num_rows * 16, dtype=torch.float32, device="cuda").reshape(
|
||||
num_rows, 16
|
||||
)
|
||||
top_k = 4
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=num_rows,
|
||||
max_valid_seq_len=seq_lens.max().item(),
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_decode(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
seq_lens,
|
||||
indices,
|
||||
top_k,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
for row_idx, row_end in enumerate(seq_lens.reshape(-1).tolist()):
|
||||
num_valid = min(top_k, row_end)
|
||||
expected = logits[row_idx, :row_end].topk(num_valid).indices
|
||||
actual = indices[row_idx, :num_valid]
|
||||
assert set(actual.tolist()) == set(expected.tolist())
|
||||
assert torch.all(indices[row_idx, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
|
||||
|
||||
def prefill_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
lambda: (prefill_kernel, decode_kernel),
|
||||
)
|
||||
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
|
||||
|
||||
eligible_cases = [
|
||||
(
|
||||
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
|
||||
prefill_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_536,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
]
|
||||
for kwargs, expected_kernel in eligible_cases:
|
||||
assert get_kernel(**kwargs) is expected_kernel
|
||||
|
||||
def fail_if_imported() -> None:
|
||||
pytest.fail("ineligible shapes must not import AITER top-k")
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
fail_if_imported,
|
||||
)
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_537,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=256,
|
||||
max_valid_seq_len=125_000,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=320,
|
||||
max_valid_seq_len=2_500,
|
||||
on_gfx950=False,
|
||||
),
|
||||
]
|
||||
for kwargs in ineligible_cases:
|
||||
assert get_kernel(**kwargs) is None
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
|
||||
assert (
|
||||
get_kernel(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
on_gfx950=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"seq_len_range,test_id",
|
||||
|
||||
@@ -750,6 +750,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool = False,
|
||||
use_fp4_cache: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
):
|
||||
super().__init__()
|
||||
self.k_cache = k_cache
|
||||
@@ -762,6 +763,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
self.compress_ratio = compress_ratio
|
||||
self.dense_mha_metadata_layer_name = ""
|
||||
# DCP scalars are constant for the run; resolve them here (config is set
|
||||
# during model construction) and pass them into the custom op, rather
|
||||
@@ -873,6 +875,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
raise RuntimeError(
|
||||
"Sparse attention indexer ROCm path is only supported on AITER. "
|
||||
|
||||
@@ -854,6 +854,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_fp4_cache=self.use_fp4_kv,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
|
||||
# None on ROCm — maybe_execute_in_parallel falls back to sequential.
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import functools
|
||||
import importlib
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from importlib.util import find_spec
|
||||
|
||||
import torch
|
||||
@@ -10,6 +11,7 @@ import torch.nn.functional as F
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -25,6 +27,127 @@ else:
|
||||
_ON_GFX950 = False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _get_aiter_topk_ops() -> tuple[Callable[..., None], Callable[..., None]] | None:
|
||||
try:
|
||||
from aiter.ops.topk import (
|
||||
top_k_per_row_decode,
|
||||
top_k_per_row_prefill,
|
||||
)
|
||||
except ImportError:
|
||||
return None
|
||||
return top_k_per_row_prefill, top_k_per_row_decode
|
||||
|
||||
|
||||
_GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN = 64 * 1024
|
||||
_GFX950_C4A_NATIVE_MAX_ROWS = 256
|
||||
|
||||
|
||||
def _get_aiter_top_k_kernel(
|
||||
*,
|
||||
is_prefill: bool,
|
||||
compress_ratio: int,
|
||||
num_rows: int,
|
||||
max_valid_seq_len: int | None = None,
|
||||
on_gfx950: bool = _ON_GFX950,
|
||||
) -> Callable[..., None] | None:
|
||||
if compress_ratio <= 1 or not on_gfx950:
|
||||
return None
|
||||
|
||||
if not is_prefill:
|
||||
assert max_valid_seq_len is not None
|
||||
# AITER v0.1.19 decode is one-block only. This measured gfx950
|
||||
# FP32/k=1024 compressed-row boundary is independent of the native
|
||||
# split-count boundary in sampler.cu.
|
||||
if (
|
||||
num_rows <= _GFX950_C4A_NATIVE_MAX_ROWS
|
||||
and max_valid_seq_len > _GFX950_C4A_AITER_MAX_COMPRESSED_SEQ_LEN
|
||||
):
|
||||
return None
|
||||
|
||||
topk_ops = _get_aiter_topk_ops()
|
||||
if topk_ops is None:
|
||||
return None
|
||||
return topk_ops[0] if is_prefill else topk_ops[1]
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _localize_aiter_prefill_topk_kernel(
|
||||
indices_ptr,
|
||||
row_starts_ptr,
|
||||
indices_stride,
|
||||
num_topk,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0)
|
||||
offsets = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_topk
|
||||
indices = tl.load(indices_ptr + row_idx * indices_stride + offsets, mask=mask)
|
||||
row_start = tl.load(row_starts_ptr + row_idx)
|
||||
localized = tl.where(indices < 0, indices, indices - row_start)
|
||||
tl.store(
|
||||
indices_ptr + row_idx * indices_stride + offsets,
|
||||
localized,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def _localize_aiter_prefill_topk(
|
||||
indices: torch.Tensor,
|
||||
row_starts: torch.Tensor,
|
||||
) -> None:
|
||||
num_rows, num_topk = indices.shape
|
||||
block_size = 256
|
||||
_localize_aiter_prefill_topk_kernel[(num_rows, triton.cdiv(num_topk, block_size))](
|
||||
indices,
|
||||
row_starts,
|
||||
indices.stride(0),
|
||||
num_topk,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
|
||||
def _launch_aiter_top_k_per_row_prefill(
|
||||
top_k_per_row_prefill: Callable[..., None],
|
||||
logits: torch.Tensor,
|
||||
row_starts: torch.Tensor,
|
||||
row_ends: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
top_k_per_row_prefill(
|
||||
logits,
|
||||
row_starts,
|
||||
row_ends,
|
||||
indices,
|
||||
None,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
k=topk_tokens,
|
||||
)
|
||||
_localize_aiter_prefill_topk(indices, row_starts)
|
||||
|
||||
|
||||
def _launch_aiter_top_k_per_row_decode(
|
||||
top_k_per_row_decode: Callable[..., None],
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
top_k_per_row_decode(
|
||||
logits,
|
||||
1,
|
||||
seq_lens.reshape(-1),
|
||||
indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
k=topk_tokens,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _indexer_k_quant_and_cache_kernel(
|
||||
k_ptr, # [num_tokens, head_dim]
|
||||
@@ -692,6 +815,7 @@ def rocm_aiter_sparse_attn_indexer_fake(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
|
||||
@@ -712,9 +836,11 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
total_seq_lens: int,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool = False,
|
||||
compress_ratio: int = 1,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
from vllm.utils.torch_utils import _resolve_layer_name
|
||||
|
||||
@@ -773,6 +899,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
total_seq_lens,
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
compress_ratio,
|
||||
)
|
||||
layer_attn_metadata = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
|
||||
@@ -833,16 +960,31 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
is_prefill=True,
|
||||
compress_ratio=compress_ratio,
|
||||
num_rows=num_rows,
|
||||
)
|
||||
if aiter_topk_kernel is not None:
|
||||
_launch_aiter_top_k_per_row_prefill(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_prefill(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = layer_attn_metadata.decode
|
||||
@@ -882,16 +1024,37 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
decode_metadata.seq_lens,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
# FULL graphs are not keyed by context length, so use a replay-safe
|
||||
# upper bound when choosing the captured kernel.
|
||||
if forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
max_compressed_seq_len = max_model_len
|
||||
else:
|
||||
max_compressed_seq_len = layer_attn_metadata.max_seq_len // compress_ratio
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
is_prefill=False,
|
||||
compress_ratio=compress_ratio,
|
||||
num_rows=num_rows,
|
||||
max_valid_seq_len=max_compressed_seq_len,
|
||||
)
|
||||
if aiter_topk_kernel is not None:
|
||||
_launch_aiter_top_k_per_row_decode(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
decode_metadata.seq_lens,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
decode_metadata.seq_lens,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
if decode_metadata.requires_padding:
|
||||
# if padded, we need to unpack
|
||||
|
||||
Reference in New Issue
Block a user