[ROCm][Perf] Optimize DeepSeek V4 C4A top-k with AITER (#52882)

This commit is contained in:
Fangzhou Ai
2026-08-21 10:00:39 -07:00
committed by GitHub
parent 1baf372bfc
commit fe76112ff2
5 changed files with 611 additions and 30 deletions
+132 -11
View File
@@ -2,6 +2,8 @@
#include "dispatch_utils.h"
#include "torch_utils.h"
#include <cstring>
#ifndef USE_ROCM
#include <cub/cub.cuh>
#else
@@ -327,11 +329,14 @@ __device__ bool processHistogramStep(
}
// Follows half - 11 - 11 - 10 bit iterations
// Keep the adaptive kernel's device instantiation separate from legacy calls.
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
bool deviceLengthAware = false>
static __device__ void topKPerRowJob(const int* indices, const float* logits,
int rowStart, int rowEnd, int* outIndices,
float* outLogits, int stride1, int topK) {
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
// The number of slots for the final pass.
static constexpr int kNumFinalItems = 2048;
// The number of elements per thread for the final sort.
@@ -613,6 +618,57 @@ static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode(
indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK);
}
// Measured native-kernel policy for gfx950, FP32, and k=1024. This
// active-column split crossover is independent of the Python AITER/native
// backend crossover; the two policies can be retuned separately. At exactly
// 64K, an eager call may select AITER while a captured native replay uses two
// splits.
constexpr int kGfx950C4AShortRowNumBlocks = 3;
constexpr int kGfx950C4ALongRowNumBlocks = 2;
constexpr int kGfx950C4ATwoSplitMinRowLength = 64 * 1024;
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
static __global__
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
const float* logits, const int* seqLens, int* outIndices, int stride0,
int stride1, const int topK, int next_n, int seqLensIs2D = 0,
float* outLogits = nullptr, const int* indices = nullptr) {
static constexpr int kNumBins = 2048;
int rowIdx = blockIdx.x;
int batch_idx = rowIdx / next_n;
int next_n_idx = rowIdx % next_n;
int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx];
int rowEnd =
seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1);
const int activeBlocks = rowEnd >= kGfx950C4ATwoSplitMinRowLength
? kGfx950C4ALongRowNumBlocks
: kGfx950C4AShortRowNumBlocks;
int rowStart = 0;
if constexpr (!mergeBlocks) {
if (blockIdx.y >= activeBlocks) return;
const auto blockSize = rowEnd / activeBlocks;
rowStart = blockSize * blockIdx.y;
rowEnd = activeBlocks == blockIdx.y + 1 ? rowEnd : rowStart + blockSize;
outIndices +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
blockIdx.y * topK;
outLogits +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK +
blockIdx.y * topK;
} else {
rowEnd = activeBlocks * topK;
indices +=
static_cast<int64_t>(rowIdx) * kGfx950C4AShortRowNumBlocks * topK;
outIndices += static_cast<int64_t>(rowIdx) * topK;
}
logits += static_cast<int64_t>(rowIdx) * stride0;
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, !mergeBlocks, mergeBlocks,
true>(indices, logits, rowStart, rowEnd, outIndices, outLogits,
stride1, topK);
}
} // namespace vllm
void apply_repetition_penalties_(
@@ -693,7 +749,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
static_cast<int>(next_n), seqLensIs2D);
} else {
// Long sequences are run in two steps
constexpr auto multipleBlocksPerRowConfig = 10;
#ifdef USE_ROCM
const bool useGfx950DeviceLengthAwareTopK1024 =
topK == 1024 && numRows > 64 && numRows <= 256 &&
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
if (useGfx950DeviceLengthAwareTopK1024) {
// Keep three blocks for short FULL-graph replays, but let each row use
// two blocks once its live device length reaches the long-context
// crossover. The grid and workspace shapes remain capture-stable.
constexpr int multipleBlocksPerRowConfig =
vllm::kGfx950C4AShortRowNumBlocks;
const auto outIndicesAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
const auto outLogitsAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
constexpr int kNumThreadsPerSplitBlock = 1024;
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerSplitBlock>
<<<dim3(numRows, multipleBlocksPerRowConfig),
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
constexpr int kNumThreadsPerBlockMerge = 1024;
vllm::topKPerRowDecodeDeviceLengthAware<kNumThreadsPerBlockMerge, true>
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t),
stream>>>(
outLogitsAux.const_data_ptr<float>(),
seqLens.const_data_ptr<int>(), indices.mutable_data_ptr<int>(),
multipleBlocksPerRowConfig * topK, 1, static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D, nullptr,
outIndicesAux.const_data_ptr<int>());
return;
}
#endif
#ifdef USE_ROCM
const bool useGfx950TopK1024 =
topK == 1024 && numRows <= 64 &&
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
#else
constexpr bool useGfx950TopK1024 = false;
#endif
int multipleBlocksPerRowConfig = 10;
if (useGfx950TopK1024) {
if (numRows <= 32) {
multipleBlocksPerRowConfig = 8;
} else {
multipleBlocksPerRowConfig = 4;
}
}
const auto outIndicesAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
@@ -702,14 +811,26 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
if (useGfx950TopK1024) {
constexpr int kNumThreadsPerSplitBlock = 1024;
vllm::topKPerRowDecode<kNumThreadsPerSplitBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig),
kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
} else {
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
static_cast<int>(stride1), static_cast<int>(topK),
static_cast<int>(next_n), seqLensIs2D,
outLogitsAux.mutable_data_ptr<float>());
}
constexpr int kNumThreadsPerBlockMerge = 1024;
vllm::topKPerRowDecode<kNumThreadsPerBlockMerge, true, false, true>
@@ -753,4 +874,4 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits,
static_cast<int>(stride0), static_cast<int>(stride1),
static_cast<int>(topK), kSortingAlgorithmThreshold);
}
}
}