[ROCm][Perf] Optimize DSV4.1 K=512 decode top-k on gfx950 (#56743)

Signed-off-by: fai <fangzhouai@gmail.com>
This commit is contained in:
Fangzhou Ai
2026-09-15 18:34:24 +00:00
committed by GitHub
parent 2f46a1d9c6
commit dffbb714e4
3 changed files with 406 additions and 58 deletions
+205 -58
View File
@@ -154,7 +154,8 @@ __device__ void vectorized_process(size_t thread_rank, size_t num_threads,
}
template <int step, int kNumThreadsPerBlock, int kNumBins, int kNumFinalItems,
bool multipleBlocksPerRow, bool mergeBlocks, typename SmemFinalType,
bool multipleBlocksPerRow, bool mergeBlocks,
bool useTopK512Optimization, typename SmemFinalType,
typename SmemOutputType>
__device__ bool processHistogramStep(
const int* indices, const float* logits, int rowEnd, uint32_t& logitPattern,
@@ -180,10 +181,15 @@ __device__ bool processHistogramStep(
<< patternShift;
}
int negativeInfinityCount = 0;
auto distributeToBins = [&](float logit, int /* idx */ = 0) {
if (isPartialMatch<patternShift>(logit, logitPattern)) {
uint32_t binIdx = extractBinIdx<step>(logit);
atomicAdd(&smemFinal.histo.data[binIdx], 1);
if (useTopK512Optimization && __float_as_uint(logit) == 0xff800000u) {
++negativeInfinityCount;
} else {
atomicAdd(&smemFinal.histo.data[binIdx], 1);
}
}
};
@@ -198,6 +204,12 @@ __device__ bool processHistogramStep(
distributeToBins(logit, idx);
}
}
if constexpr (useTopK512Optimization) {
if (negativeInfinityCount > 0) {
const auto bin = extractBinIdx<step>(-INFINITY);
atomicAdd(&smemFinal.histo.data[bin], negativeInfinityCount);
}
}
// Make sure the histogram is ready.
__syncthreads();
@@ -254,6 +266,12 @@ __device__ bool processHistogramStep(
// The threshold bin.
thresholdBinIdx = smemThresholdBinIdx[0];
// Resolve a -inf cutoff with exact tie emission, keeping sort padding out.
const bool finalBinFits =
smemFinalBinSize[0] <= kNumFinalItems &&
!(useTopK512Optimization && step < 3 &&
isPartialMatch<patternShift>(-INFINITY, logitPattern) &&
thresholdBinIdx == extractBinIdx<step>(-INFINITY));
auto processBins = [&](float logit, int idx) {
if (isPartialMatch<patternShift>(logit, logitPattern)) {
@@ -262,8 +280,7 @@ __device__ bool processHistogramStep(
// 1. This is step 0 and the threshold bin is small enough (no step 1)
// 2. This is step >= 1 (where pattern matching filters correctly)
// This prevents duplicates when step 0 and step 1 both run.
bool shouldWriteDirectly =
(step == 0 && smemFinalBinSize[0] <= kNumFinalItems) || (step >= 1);
bool shouldWriteDirectly = (step == 0 && finalBinFits) || (step >= 1);
if (binIdx < thresholdBinIdx && shouldWriteDirectly) {
// The element is part of the top-k selection
int dstIdx = atomicAdd(&smemFoundTopKValues[0], 1);
@@ -279,8 +296,7 @@ __device__ bool processHistogramStep(
}
if constexpr (step < 3) {
// Only fill the final items for sorting if the threshold bin fits
if (binIdx == thresholdBinIdx &&
smemFinalBinSize[0] <= kNumFinalItems) {
if (binIdx == thresholdBinIdx && finalBinFits) {
int dstIdx = atomicAdd(&smemFinalDstIdx[0], 1);
smemFinal.items.logits[dstIdx] = logit;
if constexpr (mergeBlocks) {
@@ -325,20 +341,21 @@ __device__ bool processHistogramStep(
__syncthreads();
// Check if we should continue to next step
return smemFinalBinSize[0] > kNumFinalItems;
return !finalBinFits;
}
// Follows half - 11 - 11 - 10 bit iterations
// Keep the adaptive kernel's device instantiation separate from legacy calls.
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
bool multipleBlocksPerRow = false, bool mergeBlocks = false,
bool deviceLengthAware = false>
bool deviceLengthAware = false, bool useTopK512Optimization = false>
static __device__ void topKPerRowJob(const int* indices, const float* logits,
int rowStart, int rowEnd, int* outIndices,
float* outLogits, int stride1, int topK) {
static_assert(!deviceLengthAware || multipleBlocksPerRow != mergeBlocks);
static_assert(!useTopK512Optimization || kNumThreadsPerBlock == 1024);
// The number of slots for the final pass.
static constexpr int kNumFinalItems = 2048;
static constexpr int kNumFinalItems = useTopK512Optimization ? 1024 : 2048;
// The number of elements per thread for the final sort.
static constexpr int kNumFinalItemsPerThread =
kNumFinalItems / kNumThreadsPerBlock;
@@ -404,7 +421,7 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
rowIt += kNumThreadsPerBlock) {
outIndices[rowIt] = -1;
if constexpr (multipleBlocksPerRow) {
outLogits[rowIt] = -FLT_MAX;
outLogits[rowIt] = useTopK512Optimization ? -INFINITY : -FLT_MAX;
}
}
@@ -422,7 +439,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
// Step 0: Process first 11 bits of half representation
bool continueToNextStep =
processHistogramStep<0, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
multipleBlocksPerRow, mergeBlocks>(
multipleBlocksPerRow, mergeBlocks,
useTopK512Optimization>(
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
@@ -431,7 +449,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
// Step 1: Process next 11 bits
continueToNextStep =
processHistogramStep<1, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
multipleBlocksPerRow, mergeBlocks>(
multipleBlocksPerRow, mergeBlocks,
useTopK512Optimization>(
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
@@ -441,7 +460,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
// Step 2: Process next 11 bits
continueToNextStep =
processHistogramStep<2, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
multipleBlocksPerRow, mergeBlocks>(
multipleBlocksPerRow, mergeBlocks,
useTopK512Optimization>(
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
@@ -450,7 +470,8 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
if (continueToNextStep) {
// Step 3: Process last 10 bits
processHistogramStep<3, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
multipleBlocksPerRow, mergeBlocks>(
multipleBlocksPerRow, mergeBlocks,
useTopK512Optimization>(
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
@@ -460,50 +481,7 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
// The histogram did not proceed to the final 10 bits, therefore we need to
// sort the final items The logits of the elements to be sorted in the final
// pass.
if constexpr (useRadixSort) {
// Sorting with radix sort
float finalLogits[kNumFinalItemsPerThread];
// The indices of the elements to be sorted in the final pass.
int finalIndices[kNumFinalItemsPerThread];
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
finalLogits[ii] = -FLT_MAX;
}
// Read the elements from SMEM.
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
if (srcIdx < smemFinalDstIdx[0]) {
finalLogits[ii] = smemFinal.items.logits[srcIdx];
finalIndices[ii] = smemFinal.items.indices[srcIdx];
}
}
// Make sure the shared memory has been read.
__syncthreads();
// Sort the elements.
FinalSort(smemFinal.finalSort)
.SortDescendingBlockedToStriped(finalLogits, finalIndices);
// Copy the data back to the shared memory storage.
int baseIdx = smemFoundTopKValues[0];
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
int dstIdx = baseIdx + srcIdx;
if (dstIdx < topK) {
smemOutput[dstIdx] = finalIndices[ii];
if constexpr (multipleBlocksPerRow) {
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] =
finalLogits[ii];
}
}
}
} else {
auto insertionSort = [&]() {
// Sorting with insertion sort
auto baseIdx = smemFoundTopKValues[0];
for (int i = threadIdx.x; i < smemFinalDstIdx[0];
@@ -525,6 +503,57 @@ static __device__ void topKPerRowJob(const int* indices, const float* logits,
}
}
}
};
if constexpr (useRadixSort) {
if (useTopK512Optimization && smemFinalDstIdx[0] <= 128) {
insertionSort();
} else {
// Sorting with radix sort
float finalLogits[kNumFinalItemsPerThread];
// The indices of the elements to be sorted in the final pass.
int finalIndices[kNumFinalItemsPerThread];
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
finalLogits[ii] = useTopK512Optimization ? -INFINITY : -FLT_MAX;
if constexpr (useTopK512Optimization) finalIndices[ii] = -1;
}
// Read the elements from SMEM.
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
if (srcIdx < smemFinalDstIdx[0]) {
finalLogits[ii] = smemFinal.items.logits[srcIdx];
finalIndices[ii] = smemFinal.items.indices[srcIdx];
}
}
// Make sure the shared memory has been read.
__syncthreads();
// Sort the elements.
FinalSort(smemFinal.finalSort)
.SortDescendingBlockedToStriped(finalLogits, finalIndices);
// Copy the data back to the shared memory storage.
int baseIdx = smemFoundTopKValues[0];
#pragma unroll
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
int dstIdx = baseIdx + srcIdx;
if (dstIdx < topK) {
smemOutput[dstIdx] = finalIndices[ii];
if constexpr (multipleBlocksPerRow) {
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] =
finalLogits[ii];
}
}
}
}
} else {
insertionSort();
}
__syncthreads();
}
@@ -669,6 +698,70 @@ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecodeDeviceLengthAware(
stride1, topK);
}
#ifdef USE_ROCM
// The grid and workspace stay fixed while graph replays change device lengths.
__device__ int gfx950TopK512ActiveBlocks(int rowLength, int numRows,
int maxBlocks) {
int activeBlocks = 1;
if (numRows <= 32 && rowLength > 16 * 1024) {
activeBlocks = rowLength <= 128 * 1024 ? 8
: rowLength <= 256 * 1024 ? 10
: 16;
} else if (numRows <= 128 && rowLength > 64 * 1024) {
activeBlocks = rowLength <= 512 * 1024 ? 4 : 10;
} else if (numRows > 128 && numRows <= 256 && rowLength > 256 * 1024) {
activeBlocks = numRows <= 160 ? 3 : 2;
}
// Every block in a merged row must contain at least k real indices.
return min(min(activeBlocks, maxBlocks), max(1, rowLength / 512));
}
template <int kNumThreadsPerBlock, bool mergeBlocks = false>
static __global__
__launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode512DeviceLengthAware(
const float* logits, const int* seqLens, int* outIndices,
int* outIndicesAux, float* outLogitsAux, int stride0, int next_n,
int seqLensIs2D, int maxBlocks) {
#if defined(__gfx950__)
constexpr int topK = 512;
constexpr int kNumBins = 2048;
const int rowIdx = blockIdx.x;
const int rowEnd =
seqLensIs2D
? max(0, seqLens[rowIdx])
: max(0, seqLens[rowIdx / next_n] - next_n + rowIdx % next_n + 1);
const int activeBlocks =
gfx950TopK512ActiveBlocks(rowEnd, gridDim.x, maxBlocks);
outIndices += static_cast<int64_t>(rowIdx) * topK;
if constexpr (mergeBlocks) {
if (activeBlocks == 1) return;
const int64_t offset = static_cast<int64_t>(rowIdx) * maxBlocks * topK;
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, false, true, false,
true>(outIndicesAux + offset, outLogitsAux + offset, 0,
activeBlocks * topK, outIndices, nullptr, 1, topK);
} else {
if (blockIdx.y >= activeBlocks) return;
logits += static_cast<int64_t>(rowIdx) * stride0;
if (activeBlocks == 1) {
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, false, false, false,
true>(nullptr, logits, 0, rowEnd, outIndices, nullptr, 1,
topK);
return;
}
const int blockSize = rowEnd / activeBlocks;
const int rowStart = blockSize * blockIdx.y;
const int blockRowEnd =
blockIdx.y + 1 == activeBlocks ? rowEnd : rowStart + blockSize;
const int64_t offset =
(static_cast<int64_t>(rowIdx) * maxBlocks + blockIdx.y) * topK;
topKPerRowJob<kNumThreadsPerBlock, kNumBins, true, true, false, false,
true>(nullptr, logits, rowStart, blockRowEnd,
outIndicesAux + offset, outLogitsAux + offset, 1, topK);
}
#endif
}
#endif
} // namespace vllm
void apply_repetition_penalties_(
@@ -731,6 +824,60 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
// the same seq_len and the kernel computes the per-row offset itself.
int seqLensIs2D = seqLens.dim() == 2 ? 1 : 0;
#ifdef USE_ROCM
const bool useGfx950DeviceLengthAwareTopK512 =
topK == 512 && stride1 == 1 && numRows > 0 && numRows <= 384 &&
numColumns <= 1024 * 1024 &&
std::strncmp(get_device_prop()->gcnArchName, "gfx950", 6) == 0;
if (useGfx950DeviceLengthAwareTopK512) {
int multipleBlocksPerRowConfig = 1;
if (numRows <= 32 && numColumns > 16 * 1024) {
multipleBlocksPerRowConfig = 16;
} else if (numRows <= 128 && numColumns > 64 * 1024) {
multipleBlocksPerRowConfig = 10;
} else if (numRows <= 160 && numColumns > 256 * 1024) {
multipleBlocksPerRowConfig = 3;
} else if (numRows <= 256 && numColumns > 256 * 1024) {
multipleBlocksPerRowConfig = 2;
}
constexpr int kNumThreadsPerSplitBlock = 1024;
if (multipleBlocksPerRowConfig == 1) {
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerSplitBlock>
<<<numRows, kNumThreadsPerSplitBlock, 2 * topK * sizeof(int32_t),
stream>>>(logits.const_data_ptr<float>(),
seqLens.const_data_ptr<int>(),
indices.mutable_data_ptr<int>(), nullptr, nullptr,
static_cast<int>(stride0), static_cast<int>(next_n),
seqLensIs2D, multipleBlocksPerRowConfig);
return;
}
const auto outIndicesAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
const auto outLogitsAux = torch::stable::empty(
{numRows, multipleBlocksPerRowConfig, topK},
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerSplitBlock>
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerSplitBlock,
2 * topK * sizeof(int32_t), stream>>>(
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
indices.mutable_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(),
outLogitsAux.mutable_data_ptr<float>(), static_cast<int>(stride0),
static_cast<int>(next_n), seqLensIs2D, multipleBlocksPerRowConfig);
constexpr int kNumThreadsPerBlockMerge = 1024;
vllm::topKPerRowDecode512DeviceLengthAware<kNumThreadsPerBlockMerge, true>
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t), stream>>>(
nullptr, seqLens.const_data_ptr<int>(),
indices.mutable_data_ptr<int>(),
outIndicesAux.mutable_data_ptr<int>(),
outLogitsAux.mutable_data_ptr<float>(), 0, static_cast<int>(next_n),
seqLensIs2D, multipleBlocksPerRowConfig);
return;
}
#endif
if (numColumns < kSortingAlgorithmThreshold) {
// Use insertion sort
vllm::topKPerRowDecode<kNumThreadsPerBlock, false>